diff --git a/README.md b/README.md index 67c53a2..3a2655a 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,14 @@ npx @agentgram/ax-score mcp --sweep --limit 25 --output report.json The sweep fetches the latest version of each server, audits them concurrently, prints a markdown leaderboard, and (with `--output`) writes the full JSON ranking report. +### Write the bounded strategic MCP report + +```bash +npx @agentgram/ax-score mcp-report --json-output reports/mcp-report.json --markdown-output reports/mcp-report.md +``` + +`mcp-report` audits a small curated list of MCP servers (no registry sweep or automatic discovery) and writes both a machine-readable JSON report and a markdown leaderboard for sharing. + ### MCP CLI Options ``` @@ -130,6 +138,17 @@ The sweep fetches the latest version of each server, audits them concurrently, p -o, --output Write the JSON report to a file (sweep mode) ``` +### MCP report CLI Options + +``` +-t, --timeout Request timeout in milliseconds (default: "30000") + --registry MCP Registry base URL + --concurrency Maximum concurrent server audits (default: 5) + --json-output Path for the JSON report (default: mcp-report.json) + --markdown-output + Path for the markdown report (default: mcp-report.md) +``` + **Strongly recommended for sweeps:** set `GITHUB_TOKEN` (any classic or fine-grained token, no scopes needed) to raise the GitHub API rate limit from 60 to 5,000 requests/hour. Without it, unauthenticated sweeps exhaust the quota after ~30 servers; when that happens ax-score shares the rate-limit state across the whole sweep — it either waits for an imminent quota reset (< 2 minutes) or marks every subsequent server's repository evidence as indeterminate and stamps the affected entries with `rateLimited: true`, so scores stay position-independent and comparable. ### MCP Categories diff --git a/src/bin/ax-score.ts b/src/bin/ax-score.ts index 8623b32..a4e3284 100644 --- a/src/bin/ax-score.ts +++ b/src/bin/ax-score.ts @@ -4,10 +4,11 @@ import { writeFileSync } from 'node:fs'; import { Command, InvalidArgumentError } from 'commander'; import ora from 'ora'; import { runRepeatedAudit } from '../runner.js'; -import { runMcpAudit, runMcpSweep } from '../mcp-runner.js'; +import { runMcpAudit, runMcpStaticReport, runMcpSweep } from '../mcp-runner.js'; import { renderReport } from '../reporter/cli.js'; import { renderJSON } from '../reporter/json.js'; import { renderMcpReport, renderMcpLeaderboard } from '../reporter/mcp.js'; +import { writeMcpReportFiles } from '../reporter/mcp-files.js'; import { uploadReport } from '../upload.js'; import { VERSION } from '../config/default.js'; import { @@ -35,6 +36,14 @@ interface McpCliOptions { output?: string; } +interface McpReportCliOptions { + timeout: string; + registry: string; + concurrency: number; + jsonOutput: string; + markdownOutput: string; +} + const DEFAULT_API_URL = 'https://agentgram.co/api/v1/ax-score/scan'; const DEFAULT_SWEEP_LIMIT = 50; @@ -182,11 +191,13 @@ program process.exit(1); } - const spinner = ora(`Auditing MCP server ${server}...`).start(); + const serverName = server as string; + + const spinner = ora(`Auditing MCP server ${serverName}...`).start(); try { const report = await runMcpAudit({ - server, + server: serverName, registryUrl: options.registry, timeout, }); @@ -201,7 +212,53 @@ program process.exit(report.score >= 50 ? 0 : 1); } catch (error) { - spinner.fail(`Failed to audit ${server}`); + spinner.fail(`Failed to audit ${serverName}`); + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } + }); + +program + .command('mcp-report') + .description('Audit the curated MCP server set and write JSON plus markdown report files') + .option('-t, --timeout ', 'Request timeout in milliseconds', '30000') + .option('--registry ', 'MCP Registry base URL', DEFAULT_MCP_REGISTRY_URL) + .option( + '--concurrency ', + 'Maximum concurrent server audits', + parsePositiveInteger, + DEFAULT_MCP_SWEEP_CONCURRENCY + ) + .option('--json-output ', 'Path for the JSON report', 'mcp-report.json') + .option('--markdown-output ', 'Path for the markdown report', 'mcp-report.md') + .action(async (options: McpReportCliOptions) => { + const timeout = parseInt(options.timeout, 10); + const spinner = ora('Auditing curated MCP server set...').start(); + + try { + const report = await runMcpStaticReport( + { + registryUrl: options.registry, + timeout, + concurrency: options.concurrency, + }, + (progress) => { + spinner.text = `Scoring curated MCP servers... ${progress.completed}/${progress.total}`; + } + ); + + spinner.stop(); + writeMcpReportFiles(report, { + json: options.jsonOutput, + markdown: options.markdownOutput, + }); + + console.log(renderMcpLeaderboard(report)); + console.error(`JSON report written to ${options.jsonOutput}`); + console.error(`Markdown report written to ${options.markdownOutput}`); + process.exit(report.scored > 0 ? 0 : 1); + } catch (error) { + spinner.fail('Curated MCP report failed'); console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } diff --git a/src/config/mcp.ts b/src/config/mcp.ts index e474d4e..5e78790 100644 --- a/src/config/mcp.ts +++ b/src/config/mcp.ts @@ -6,6 +6,20 @@ export const DEFAULT_MCP_REGISTRY_URL = 'https://registry.modelcontextprotocol.i /** Default number of concurrent server audits during a sweep. */ export const DEFAULT_MCP_SWEEP_CONCURRENCY = 5; +/** + * Bounded, curated MCP server set used by the static report CLI. + * + * This intentionally avoids crawling the Registry API. The list is small enough + * for a quick strategic report while still covering package, repository, and + * remote-endpoint evidence when the server records declare it. + */ +export const DEFAULT_MCP_REPORT_SERVERS = [ + 'io.github.github/github-mcp-server', + 'io.github.domdomegg/airtable-mcp-server', + 'ac.tandem/docs-mcp', + 'ai.agenttrust/mcp-server', +] as const; + /** * Category definitions for MCP server scoring. * diff --git a/src/index.ts b/src/index.ts index 59f42f1..9f6dddc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,12 @@ // Main API export { runAudit, runRepeatedAudit } from './runner.js'; -export { runMcpAudit, runMcpSweep } from './mcp-runner.js'; -export type { McpSweepConfig, McpSweepProgress, McpAuditContext } from './mcp-runner.js'; +export { runMcpAudit, runMcpStaticReport, runMcpSweep } from './mcp-runner.js'; +export type { + McpStaticReportConfig, + McpSweepConfig, + McpSweepProgress, + McpAuditContext, +} from './mcp-runner.js'; // Types export type { @@ -59,10 +64,17 @@ export { McpRemoteGatherer, isPrivateHost } from './gatherers/mcp-remote.js'; export type { McpRemoteGatherResult, RemoteProbe } from './gatherers/mcp-remote.js'; // MCP config -export { getMcpCategories, MCP_CATEGORIES, DEFAULT_MCP_REGISTRY_URL } from './config/mcp.js'; +export { + getMcpCategories, + MCP_CATEGORIES, + DEFAULT_MCP_REGISTRY_URL, + DEFAULT_MCP_REPORT_SERVERS, +} from './config/mcp.js'; // Reporters export { renderMcpReport, renderMcpLeaderboard } from './reporter/mcp.js'; +export { writeMcpReportFiles } from './reporter/mcp-files.js'; +export type { McpReportFilePaths } from './reporter/mcp-files.js'; // Upload export { uploadReport } from './upload.js'; diff --git a/src/mcp-runner.ts b/src/mcp-runner.ts index 1238d20..2c73b10 100644 --- a/src/mcp-runner.ts +++ b/src/mcp-runner.ts @@ -11,6 +11,7 @@ import type { CategoryConfig } from './config/default.js'; import { VERSION } from './config/default.js'; import { DEFAULT_MCP_REGISTRY_URL, + DEFAULT_MCP_REPORT_SERVERS, DEFAULT_MCP_SWEEP_CONCURRENCY, getMcpCategories, } from './config/mcp.js'; @@ -202,6 +203,13 @@ export interface McpSweepConfig { concurrency?: number; } +export interface McpStaticReportConfig { + registryUrl?: string; + timeout?: number; + /** Maximum concurrent server audits. Defaults to 5. */ + concurrency?: number; +} + export interface McpSweepProgress { completed: number; total: number; @@ -231,6 +239,53 @@ async function mapWithConcurrency( return results; } +function entryFromMcpReport(report: McpReport): McpSweepEntry { + // A fully excluded category (weight 0) is reported as null — "we could + // not evaluate this" — never as a genuine score of 0. + const categoryScores: Record = {}; + for (const category of report.categories) { + categoryScores[category.id] = category.weight === 0 ? null : category.score; + } + let notApplicableAudits = 0; + let indeterminateAudits = 0; + for (const audit of Object.values(report.audits)) { + if (audit.applicability === 'not-applicable') notApplicableAudits += 1; + if (audit.applicability === 'indeterminate') indeterminateAudits += 1; + } + return { + server: report.server, + serverVersion: report.serverVersion, + score: report.score, + categoryScores, + notApplicableAudits, + indeterminateAudits, + rateLimited: report.rateLimited, + }; +} + +function sortMcpSweepEntries(entries: McpSweepEntry[]): McpSweepEntry[] { + return [...entries].sort((a, b) => (b.score ?? -1) - (a.score ?? -1)); +} + +function buildMcpSweepReport(args: { + registryUrl: string; + requested: number; + entries: McpSweepEntry[]; +}): McpSweepReport { + const ranked = sortMcpSweepEntries(args.entries); + const failed = ranked.filter((e) => e.score === null).length; + + return { + registryUrl: args.registryUrl, + timestamp: new Date().toISOString(), + version: VERSION, + requested: args.requested, + scored: ranked.length - failed, + failed, + entries: ranked, + }; +} + /** * Fetch up to `limit` servers from the registry, audit them concurrently, * and return a ranking report (highest score first, failures last). @@ -268,27 +323,7 @@ export async function runMcpSweep( }, { githubLimiter } ); - // A fully excluded category (weight 0) is reported as null — "we could - // not evaluate this" — never as a genuine score of 0. - const categoryScores: Record = {}; - for (const category of report.categories) { - categoryScores[category.id] = category.weight === 0 ? null : category.score; - } - let notApplicableAudits = 0; - let indeterminateAudits = 0; - for (const audit of Object.values(report.audits)) { - if (audit.applicability === 'not-applicable') notApplicableAudits += 1; - if (audit.applicability === 'indeterminate') indeterminateAudits += 1; - } - entry = { - server: report.server, - serverVersion: report.serverVersion, - score: report.score, - categoryScores, - notApplicableAudits, - indeterminateAudits, - rateLimited: report.rateLimited, - }; + entry = entryFromMcpReport(report); } catch (err) { entry = { server: serverName, @@ -307,16 +342,52 @@ export async function runMcpSweep( return entry; }); - const ranked = [...entries].sort((a, b) => (b.score ?? -1) - (a.score ?? -1)); - const failed = ranked.filter((e) => e.score === null).length; + return buildMcpSweepReport({ registryUrl, requested: config.limit, entries }); +} - return { - registryUrl, - timestamp: new Date().toISOString(), - version: VERSION, - requested: config.limit, - scored: ranked.length - failed, - failed, - entries: ranked, - }; +/** + * Audit the curated MCP report list without auto-fetching a registry sweep. + */ +export async function runMcpStaticReport( + config: McpStaticReportConfig = {}, + onProgress?: (progress: McpSweepProgress) => void +): Promise { + const registryUrl = (config.registryUrl ?? DEFAULT_MCP_REGISTRY_URL).replace(/\/+$/, ''); + const concurrency = config.concurrency ?? DEFAULT_MCP_SWEEP_CONCURRENCY; + const servers = [...DEFAULT_MCP_REPORT_SERVERS]; + const githubLimiter = new GithubRateLimiter(); + + let completed = 0; + const entries = await mapWithConcurrency(servers, concurrency, async (server) => { + let entry: McpSweepEntry; + + try { + const report = await runMcpAudit( + { + server, + registryUrl, + timeout: config.timeout, + }, + { githubLimiter } + ); + entry = entryFromMcpReport(report); + } catch (err) { + entry = { + server, + serverVersion: null, + score: null, + categoryScores: {}, + notApplicableAudits: 0, + indeterminateAudits: 0, + rateLimited: githubLimiter.isExhausted, + error: err instanceof Error ? err.message : 'Unknown error', + }; + } + + completed += 1; + onProgress?.({ completed, total: servers.length, server }); + return entry; + }); + + return buildMcpSweepReport({ registryUrl, requested: servers.length, entries }); } diff --git a/src/mcp-static-report.test.ts b/src/mcp-static-report.test.ts new file mode 100644 index 0000000..af83e2e --- /dev/null +++ b/src/mcp-static-report.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { DEFAULT_MCP_REPORT_SERVERS } from './config/mcp.js'; +import { runMcpStaticReport } from './mcp-runner.js'; + +function jsonResponse(body: unknown, status = 200): Promise { + return Promise.resolve({ + ok: status >= 200 && status < 300, + status, + body: null, + headers: { get: () => null }, + json: () => Promise.resolve(body), + } as unknown as Response); +} + +function daysAgoIso(days: number): string { + return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(); +} + +function registryRecord(server: string): unknown { + return { + server: { + name: server, + title: `Static report fixture for ${server}`, + description: + 'A documented MCP server used by the bounded AX Score report fixture for agent tooling evaluation.', + version: '1.0.0', + }, + _meta: { + 'io.modelcontextprotocol.registry/official': { + status: 'active', + publishedAt: daysAgoIso(20), + updatedAt: daysAgoIso(3), + isLatest: true, + }, + }, + }; +} + +describe('runMcpStaticReport', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('audits the curated hardcoded server set without using registry sweep pagination', async () => { + const requestedUrls: string[] = []; + vi.spyOn(globalThis, 'fetch').mockImplementation((input) => { + const url = String(input); + requestedUrls.push(url); + + if (url.includes('/v0/servers?')) { + return jsonResponse({ error: 'sweep endpoint should not be used' }, 500); + } + + const match = url.match(/\/v0\/servers\/(.+)\/versions\/latest$/); + if (match?.[1]) { + return jsonResponse(registryRecord(decodeURIComponent(match[1]))); + } + + return jsonResponse({}, 404); + }); + + const progress: string[] = []; + const report = await runMcpStaticReport({ concurrency: 2 }, (p) => progress.push(p.server)); + + expect(report.requested).toBe(DEFAULT_MCP_REPORT_SERVERS.length); + expect(report.scored).toBe(DEFAULT_MCP_REPORT_SERVERS.length); + expect(report.failed).toBe(0); + expect(report.entries.map((entry) => entry.server).sort()).toEqual( + [...DEFAULT_MCP_REPORT_SERVERS].sort() + ); + expect(progress).toHaveLength(DEFAULT_MCP_REPORT_SERVERS.length); + expect(requestedUrls.some((url) => url.includes('/v0/servers?'))).toBe(false); + }); +}); diff --git a/src/reporter/mcp-files.test.ts b/src/reporter/mcp-files.test.ts new file mode 100644 index 0000000..6eae23d --- /dev/null +++ b/src/reporter/mcp-files.test.ts @@ -0,0 +1,56 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { writeMcpReportFiles } from './mcp-files.js'; +import type { McpSweepReport } from '../types.js'; + +function makeReport(): McpSweepReport { + return { + registryUrl: 'https://registry.modelcontextprotocol.io', + timestamp: '2026-07-13T00:00:00.000Z', + version: '0.3.0', + requested: 1, + scored: 1, + failed: 0, + entries: [ + { + server: 'io.github.acme/todo-server', + serverVersion: '1.2.3', + score: 93, + categoryScores: { + 'mcp-metadata': 95, + 'mcp-distribution': 90, + 'mcp-provenance': 100, + 'mcp-operational': 88, + 'mcp-documentation': 92, + }, + notApplicableAudits: 0, + indeterminateAudits: 0, + rateLimited: false, + }, + ], + }; +} + +describe('writeMcpReportFiles', () => { + it('writes JSON and markdown reports to nested paths', () => { + const dir = mkdtempSync(join(tmpdir(), 'ax-score-mcp-report-')); + + try { + const paths = writeMcpReportFiles(makeReport(), { + json: join(dir, 'nested', 'report.json'), + markdown: join(dir, 'nested', 'report.md'), + }); + + const json = JSON.parse(readFileSync(paths.json, 'utf8')) as McpSweepReport; + const markdown = readFileSync(paths.markdown, 'utf8'); + + expect(json.entries[0]?.server).toBe('io.github.acme/todo-server'); + expect(markdown).toContain('# MCP Server Leaderboard'); + expect(markdown).toContain('| 1 | io.github.acme/todo-server | **93** |'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/reporter/mcp-files.ts b/src/reporter/mcp-files.ts new file mode 100644 index 0000000..a8a9885 --- /dev/null +++ b/src/reporter/mcp-files.ts @@ -0,0 +1,25 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import type { McpSweepReport } from '../types.js'; +import { renderJSON } from './json.js'; +import { renderMcpLeaderboard } from './mcp.js'; + +export interface McpReportFilePaths { + json: string; + markdown: string; +} + +function writeTextFile(path: string, content: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); +} + +/** Write the bounded MCP report as both machine-readable JSON and markdown. */ +export function writeMcpReportFiles( + report: McpSweepReport, + paths: McpReportFilePaths +): McpReportFilePaths { + writeTextFile(paths.json, renderJSON(report)); + writeTextFile(paths.markdown, renderMcpLeaderboard(report)); + return paths; +}