modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1 from modelcontextprotocol/pcarleton-runner1

Browse source 
Adjust test and allow running in interactive mode
This commit is contained in:
Paul Carleton 2025-10-31 17:54:48 +00:00 • committed by GitHub
commit 3d7343abd8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 58 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/checks.ts
View file

@ -31,29 +31,17 @@ export function createClientInitializationCheck(initializeRequest: any, expected
};
}
export function createServerInfoCheck(serverInfo: { name?: string; version?: string } | null): ConformanceCheck {
const hasName = !!serverInfo?.name;
const hasVersion = !!serverInfo?.version;
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
const errors: string[] = [];
if (!hasName) errors.push('Server name missing');
if (!hasVersion) errors.push('Server version missing');
export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck {
return {
id: 'server-info',
name: 'ServerInfo',
description: 'Test server info returned to client',
status,
status: 'INFO',
timestamp: new Date().toISOString(),
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
details: {
serverName: serverInfo?.name,
serverVersion: serverInfo?.version,
hasName,
hasVersion
},
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
logs: errors.length > 0 ? errors : undefined
serverName: serverInfo.name,
serverVersion: serverInfo.version
}
};
}

58
src/runner/index.ts
View file

@ -122,6 +122,41 @@ export async function runConformanceTest(
}
}
async function runInteractiveMode(scenarioName: string): Promise<void> {
await ensureResultsDir();
const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true });
const scenario = getScenario(scenarioName);
if (!scenario) {
throw new Error(`Unknown scenario: ${scenarioName}`);
}
console.log(`Starting scenario: ${scenarioName}`);
const urls = await scenario.start();
console.log(`Server URL: ${urls.serverUrl}`);
console.log('Press Ctrl+C to stop and save checks...');
const handleShutdown = async () => {
console.log('\nShutting down...');
const checks = scenario.getChecks();
await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
console.log(`\nChecks saved to ${resultDir}/checks.json`);
await scenario.stop();
process.exit(0);
};
process.on('SIGINT', handleShutdown);
process.on('SIGTERM', handleShutdown);
await new Promise(() => {});
}
async function main(): Promise<void> {
const args = process.argv.slice(2);
let command: string | null = null;
@ -137,21 +172,34 @@ async function main(): Promise<void> {
}
}
if (!command || !scenario) {
console.error('Usage: runner --command "<command>" --scenario <scenario>');
console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
if (!scenario) {
console.error('Usage: runner --scenario <scenario> [--command "<command>"]');
console.error('Example: runner --scenario initialize --command "tsx examples/clients/typescript/test1.ts"');
console.error('Or run without --command for interactive mode');
process.exit(1);
}
if (!command) {
try {
await runInteractiveMode(scenario);
} catch (error) {
console.error('Interactive mode error:', error);
process.exit(1);
}
return;
}
try {
const result = await runConformanceTest(command, scenario);
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${result.checks.length}`);
console.log(`Failed: ${failed}/${result.checks.length}`);
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
if (failed > 0) {
console.log('\nFailed Checks:');