conformance/src/index.ts

707 lines
23 KiB
JavaScript

#!/usr/bin/env node
import { Command } from 'commander';
import { ZodError } from 'zod';
import {
runConformanceTest,
printClientResults,
runServerConformanceTest,
printServerResults,
printServerSummary,
runInteractiveMode
} from './runner';
import {
printAuthorizationServerResults,
printAuthorizationServerSummary,
runAuthorizationServerConformanceTest
} from './runner/authorization-server';
import {
listScenarios,
listClientScenarios,
listActiveClientScenarios,
listPendingClientScenarios,
listAuthScenarios,
listMetadataScenarios,
listCoreScenarios,
listExtensionScenarios,
listBackcompatScenarios,
listDraftScenarios,
listDraftClientScenarios,
listScenariosForSpec,
listClientScenariosForSpec,
getScenarioSpecVersions,
listClientScenariosForAuthorizationServer,
listClientScenariosForAuthorizationServerForSpec,
resolveSpecVersion
} from './scenarios';
import type { SpecVersion } from './scenarios';
import { ConformanceCheck } from './types';
import {
AuthorizationServerOptionsSchema,
ClientOptionsSchema,
ServerOptionsSchema
} from './schemas';
import {
loadExpectedFailures,
evaluateBaseline,
printBaselineResults
} from './expected-failures';
import { createTierCheckCommand } from './tier-check';
import { createNewSepCommand } from './new-sep';
import { createSdkCommand } from './sdk-runner';
import { createTraceabilityCommand } from './traceability';
import packageJson from '../package.json';
// Note on naming: `command` refers to which CLI command is calling this.
// The `client` command tests Scenario objects (which test clients),
// and the `server` command tests ClientScenario objects (which test servers).
// This matches the inverted naming in scenarios/index.ts.
function filterScenariosBySpecVersion(
allScenarios: string[],
version: SpecVersion,
command: 'client' | 'server' | 'authorization'
): string[] {
let versionScenarios: string[];
if (command === 'client') {
versionScenarios = listScenariosForSpec(version);
} else if (command === 'server') {
versionScenarios = listClientScenariosForSpec(version);
} else if (command === 'authorization') {
versionScenarios =
listClientScenariosForAuthorizationServerForSpec(version);
} else {
versionScenarios = [];
}
const allowed = new Set(versionScenarios);
return allScenarios.filter((s) => allowed.has(s));
}
const program = new Command();
program
.name('conformance')
.description('MCP Conformance Test Suite')
.version(packageJson.version);
// Client command - tests a client implementation against scenarios
program
.command('client')
.description(
'Run conformance tests against a client implementation or start interactive mode'
)
.option('--command <command>', 'Command to run the client')
.option('--scenario <scenario>', 'Scenario to test')
.option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
.option(
'--expected-failures <path>',
'Path to YAML file listing expected failures (baseline)'
)
.option('-o, --output-dir <path>', 'Save results to this directory')
.option(
'--spec-version <version>',
'Filter scenarios by spec version (cumulative for date versions)'
)
.option(
'--force',
'Run a scenario even if it is not applicable at the requested --spec-version'
)
.option('--verbose', 'Show verbose output')
.action(async (options) => {
try {
const timeout = parseInt(options.timeout, 10);
const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
const specVersionFilter = options.specVersion
? resolveSpecVersion(options.specVersion)
: undefined;
// Handle suite mode
if (options.suite) {
if (!options.command) {
console.error('--command is required when using --suite');
process.exit(1);
}
const suites: Record<string, () => string[]> = {
all: listScenarios,
core: listCoreScenarios,
extensions: listExtensionScenarios,
backcompat: listBackcompatScenarios,
auth: listAuthScenarios,
metadata: listMetadataScenarios,
draft: listDraftScenarios,
'sep-835': () =>
listAuthScenarios().filter((name) => name.startsWith('auth/scope-'))
};
const suiteName = options.suite.toLowerCase();
if (!suites[suiteName]) {
console.error(`Unknown suite: ${suiteName}`);
console.error(`Available suites: ${Object.keys(suites).join(', ')}`);
process.exit(1);
}
let scenarios = suites[suiteName]();
if (specVersionFilter) {
scenarios = filterScenariosBySpecVersion(
scenarios,
specVersionFilter,
'client'
);
}
console.log(
`Running ${suiteName} suite (${scenarios.length} scenarios) in parallel...\n`
);
const results = await Promise.all(
scenarios.map(async (scenarioName) => {
try {
const result = await runConformanceTest(
options.command,
scenarioName,
timeout,
outputDir,
specVersionFilter,
options.force ?? false
);
return {
scenario: scenarioName,
checks: result.checks,
skipped: result.skipped,
error: null
};
} catch (error) {
return {
scenario: scenarioName,
checks: [
{
id: scenarioName,
name: scenarioName,
description: 'Failed to run scenario',
status: 'FAILURE' as const,
timestamp: new Date().toISOString(),
errorMessage:
error instanceof Error ? error.message : String(error)
}
],
skipped: undefined,
error
};
}
})
);
console.log('\n=== SUITE SUMMARY ===\n');
let totalPassed = 0;
let totalFailed = 0;
let totalWarnings = 0;
let totalSkipped = 0;
for (const result of results) {
// Inapplicable scenario/spec-version combination (already logged by
// the runner). Not a failure: report distinctly.
if (result.skipped) {
totalSkipped++;
console.log(`- ${result.scenario}: skipped`);
continue;
}
const passed = result.checks.filter(
(c) => c.status === 'SUCCESS'
).length;
const failed = result.checks.filter(
(c) => c.status === 'FAILURE'
).length;
const warnings = result.checks.filter(
(c) => c.status === 'WARNING'
).length;
totalPassed += passed;
totalFailed += failed;
totalWarnings += warnings;
const status = failed === 0 && warnings === 0 ? '✓' : '✗';
const warningStr = warnings > 0 ? `, ${warnings} warnings` : '';
console.log(
`${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}`
);
if (verbose && failed > 0) {
result.checks
.filter((c) => c.status === 'FAILURE')
.forEach((c) => {
console.log(
` - ${c.name}: ${c.errorMessage || c.description}`
);
});
}
}
const skippedStr = totalSkipped > 0 ? `, ${totalSkipped} skipped` : '';
console.log(
`\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings${skippedStr}`
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.client ?? [];
const baselineResult = evaluateBaseline(results, baselineScenarios);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0);
}
// Require either --scenario or --suite
if (!options.scenario) {
console.error('Either --scenario or --suite is required');
console.error('\nAvailable client scenarios:');
listScenarios().forEach((s) => console.error(` - ${s}`));
console.error(
'\nAvailable suites: all, core, extensions, backcompat, auth, metadata, draft, sep-835'
);
process.exit(1);
}
// Validate options with Zod for single scenario mode
const validated = ClientOptionsSchema.parse(options);
// If no command provided, run in interactive mode
if (!validated.command) {
await runInteractiveMode(
validated.scenario,
verbose,
outputDir,
specVersionFilter,
options.force ?? false
);
process.exit(0);
}
// Otherwise run conformance test
const result = await runConformanceTest(
validated.command,
validated.scenario,
timeout,
outputDir,
specVersionFilter,
options.force ?? false
);
// Inapplicable scenario/spec-version combination (already logged by
// the runner). Not a failure: exit 0.
if (result.skipped) {
process.exit(0);
}
const { overallFailure } = printClientResults(
result.checks,
verbose,
result.clientOutput,
result.allowClientError
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.client ?? [];
const baselineResult = evaluateBaseline(
[{ scenario: validated.scenario, checks: result.checks }],
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(overallFailure ? 1 : 0);
} catch (error) {
if (error instanceof ZodError) {
console.error('Validation error:');
error.issues.forEach((err) => {
console.error(` ${err.path.join('.')}: ${err.message}`);
});
console.error('\nAvailable client scenarios:');
listScenarios().forEach((s) => console.error(` - ${s}`));
process.exit(1);
}
console.error('Client test error:', error);
process.exit(1);
}
});
// Server command - tests a server implementation
program
.command('server')
.description('Run conformance tests against a server implementation')
.requiredOption('--url <url>', 'URL of the server to test')
.option(
'--scenario <scenario>',
'Scenario to test (defaults to active suite if not specified)'
)
.option(
'--suite <suite>',
'Suite to run: "active" (default, excludes pending and draft), "all", "draft", or "pending"',
'active'
)
.option(
'--expected-failures <path>',
'Path to YAML file listing expected failures (baseline)'
)
.option('-o, --output-dir <path>', 'Save results to this directory')
.option(
'--spec-version <version>',
'Filter scenarios by spec version (cumulative for date versions)'
)
.option(
'--force',
'Run a scenario even if it is not applicable at the requested --spec-version'
)
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options) => {
try {
// Validate options with Zod
const validated = ServerOptionsSchema.parse(options);
const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
const specVersionFilter = options.specVersion
? resolveSpecVersion(options.specVersion)
: undefined;
// If a single scenario is specified, run just that one
if (validated.scenario) {
const result = await runServerConformanceTest(
validated.url,
validated.scenario,
outputDir,
specVersionFilter,
options.force ?? false
);
// Inapplicable scenario/spec-version combination (already logged by
// the runner). Not a failure: exit 0.
if (result.skipped) {
process.exit(0);
}
const { failed } = printServerResults(
result.checks,
result.scenarioDescription,
verbose
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.server ?? [];
const baselineResult = evaluateBaseline(
[{ scenario: validated.scenario!, checks: result.checks }],
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(failed > 0 ? 1 : 0);
} else {
// Run scenarios based on suite
const suite = options.suite?.toLowerCase() || 'active';
let scenarios: string[];
if (suite === 'all') {
scenarios = listClientScenarios();
} else if (suite === 'active' || suite === 'core') {
// 'core' is an alias for 'active' - tier 1 requirements
scenarios = listActiveClientScenarios();
} else if (suite === 'pending') {
scenarios = listPendingClientScenarios();
} else if (suite === 'draft') {
// Scenarios targeting the in-progress draft spec; excluded from
// 'active' until the draft is published as a dated release.
scenarios = listDraftClientScenarios();
} else {
console.error(`Unknown suite: ${suite}`);
console.error('Available suites: active, all, core, draft, pending');
process.exit(1);
}
if (specVersionFilter) {
scenarios = filterScenariosBySpecVersion(
scenarios,
specVersionFilter,
'server'
);
}
console.log(
`Running ${suite} suite (${scenarios.length} scenarios) against ${validated.url}\n`
);
const allResults: { scenario: string; checks: ConformanceCheck[] }[] =
[];
for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
try {
const result = await runServerConformanceTest(
validated.url,
scenarioName,
outputDir,
specVersionFilter
);
allResults.push({ scenario: scenarioName, checks: result.checks });
} catch (error) {
console.error(`Failed to run scenario ${scenarioName}:`, error);
allResults.push({
scenario: scenarioName,
checks: [
{
id: scenarioName,
name: scenarioName,
description: 'Failed to run scenario',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
error instanceof Error ? error.message : String(error)
}
]
});
}
}
const { totalFailed } = printServerSummary(allResults);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.server ?? [];
const baselineResult = evaluateBaseline(
allResults,
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(totalFailed > 0 ? 1 : 0);
}
} catch (error) {
if (error instanceof ZodError) {
console.error('Validation error:');
error.issues.forEach((err) => {
console.error(` ${err.path.join('.')}: ${err.message}`);
});
console.error('\nAvailable server scenarios:');
listClientScenarios().forEach((s) => console.error(` - ${s}`));
process.exit(1);
}
console.error('Server test error:', error);
process.exit(1);
}
});
// Authorization command - tests an authorization server implementation
program
.command('authorization')
.description(
'Run conformance tests against an authorization server implementation'
)
.requiredOption('--url <url>', 'URL of the authorization server issuer')
.option('--scenario <scenario>', 'Test scenario to run')
.option('-o, --output-dir <path>', 'Save results to this directory')
.option(
'--spec-version <version>',
'Filter scenarios by spec version (cumulative for date versions)'
)
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options) => {
try {
// Validate options with Zod
const validated = AuthorizationServerOptionsSchema.parse(options);
const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
const specVersionFilter = options.specVersion
? resolveSpecVersion(options.specVersion)
: undefined;
// If a single scenario is specified, run just that one
if (validated.scenario) {
const result = await runAuthorizationServerConformanceTest(
validated.url,
validated.scenario,
outputDir
);
const { failed } = printAuthorizationServerResults(
result.checks,
result.scenarioDescription,
verbose
);
process.exit(failed > 0 ? 1 : 0);
}
let scenarios: string[];
scenarios = listClientScenariosForAuthorizationServer();
if (specVersionFilter) {
scenarios = filterScenariosBySpecVersion(
scenarios,
specVersionFilter,
'authorization'
);
}
console.log(
`Running test (${scenarios.length} scenarios) against ${validated.url}\n`
);
const allResults: { scenario: string; checks: ConformanceCheck[] }[] = [];
for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
try {
const result = await runAuthorizationServerConformanceTest(
validated.url,
scenarioName,
outputDir
);
allResults.push({ scenario: scenarioName, checks: result.checks });
} catch (error) {
console.error(`Failed to run scenario ${scenarioName}:`, error);
allResults.push({
scenario: scenarioName,
checks: [
{
id: scenarioName,
name: scenarioName,
description: 'Failed to run scenario',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
error instanceof Error ? error.message : String(error)
}
]
});
}
}
const { totalFailed } = printAuthorizationServerSummary(allResults);
process.exit(totalFailed > 0 ? 1 : 0);
} catch (error) {
if (error instanceof ZodError) {
console.error('Validation error:');
error.issues.forEach((err) => {
console.error(` ${err.path.join('.')}: ${err.message}`);
});
console.error('\nAvailable authorization server scenarios:');
listClientScenariosForAuthorizationServer().forEach((s) =>
console.error(` - ${s}`)
);
process.exit(1);
}
console.error('Authorization server test error:', error);
process.exit(1);
}
});
// Tier check command
program.addCommand(createTierCheckCommand());
// New SEP scaffolding command
program.addCommand(createNewSepCommand());
// SDK command - run local conformance against an SDK at a specific ref
program.addCommand(createSdkCommand());
// SEP traceability manifest command
program.addCommand(createTraceabilityCommand());
// List scenarios command
program
.command('list')
.description('List available test scenarios')
.option('--client', 'List client scenarios')
.option('--server', 'List server scenarios')
.option('--authorization', 'List authorization server scenarios')
.option(
'--spec-version <version>',
'Filter scenarios by spec version (cumulative for date versions)'
)
.action((options) => {
const specVersionFilter = options.specVersion
? resolveSpecVersion(options.specVersion)
: undefined;
if (
options.server ||
(!options.client && !options.server && !options.authorization)
) {
console.log('Server scenarios (test against a server):');
let serverScenarios = listClientScenarios();
if (specVersionFilter) {
serverScenarios = filterScenariosBySpecVersion(
serverScenarios,
specVersionFilter,
'server'
);
}
serverScenarios.forEach((s) => {
const v = getScenarioSpecVersions(s);
console.log(` - ${s}${v ? ` [${v}]` : ''}`);
});
}
if (
options.client ||
(!options.client && !options.server && !options.authorization)
) {
if (options.server || (!options.client && !options.server)) {
console.log('');
}
console.log('Client scenarios (test against a client):');
let clientScenarioNames = listScenarios();
if (specVersionFilter) {
clientScenarioNames = filterScenariosBySpecVersion(
clientScenarioNames,
specVersionFilter,
'client'
);
}
clientScenarioNames.forEach((s) => {
const v = getScenarioSpecVersions(s);
console.log(` - ${s}${v ? ` [${v}]` : ''}`);
});
}
if (
options.authorization ||
(!options.authorization && !options.server && !options.client)
) {
if (!(options.authorization && !options.server && !options.client)) {
console.log('');
}
console.log(
'Authorization server scenarios (test against an authorization server):'
);
let authorizationServerScenarios =
listClientScenariosForAuthorizationServer();
if (specVersionFilter) {
authorizationServerScenarios = filterScenariosBySpecVersion(
authorizationServerScenarios,
specVersionFilter,
'authorization'
);
}
authorizationServerScenarios.forEach((s) => {
const v = getScenarioSpecVersions(s);
console.log(` - ${s}${v ? ` [${v}]` : ''}`);
});
}
});
program.parse();