modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[compliance tests] initial server scenarios (#4)

Browse source 
* spec for servers

* initial version of everything server and tests - tets are very simple

* some clean up

* lint etc.

* fix package lock agian

---------

Co-authored-by: Paul Carleton <paulc@anthropic.com>
This commit is contained in:
Inna Harper 2025-11-04 15:08:20 +00:00 • committed by GitHub
commit a692dd7bba
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5678 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

104
src/server-runner.ts
View file

@ -48,6 +48,7 @@ async function main(): Promise<void> {
const args = process.argv.slice(2);
let serverUrl: string | null = null;
let scenario: string | null = null;
let runAll = false;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--server-url' && i + 1 < args.length) {
@ -56,40 +57,99 @@ async function main(): Promise<void> {
} else if (args[i] === '--scenario' && i + 1 < args.length) {
scenario = args[i + 1];
i++;
} else if (args[i] === '--all') {
runAll = true;
}
}
if (!serverUrl || !scenario) {
console.error('Usage: server-runner --server-url <url> --scenario <scenario>');
console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize');
if (!serverUrl) {
console.error('Usage: server-runner --server-url <url> [--scenario <scenario> | --all]');
console.error('Example: server-runner --server-url http://localhost:3000/mcp --scenario server-initialize');
console.error('Example: server-runner --server-url http://localhost:3000/mcp --all');
process.exit(1);
}
if (!scenario && !runAll) {
console.error('Must specify either --scenario or --all');
process.exit(1);
}
try {
const result = await runServerConformanceTest(serverUrl, scenario);
const allResults: { scenario: string; checks: ConformanceCheck[] }[] = [];
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
if (runAll) {
// Get all server scenarios
const { listClientScenarios } = await import('./scenarios/index.js');
const scenarios = listClientScenarios();
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
console.log(`Running ${scenarios.length} scenarios against ${serverUrl}\n`);
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
try {
const result = await runServerConformanceTest(serverUrl, scenarioName);
allResults.push({ scenario: scenarioName, checks: result.checks });
} catch (error) {
console.error(`Failed to run scenario ${scenarioName}:`, error);
allResults.push({
scenario: scenarioName,
checks: [
{
id: scenarioName,
name: scenarioName,
description: `Failed to run scenario`,
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: error instanceof Error ? error.message : String(error)
}
]
});
}
}
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
// Summary
console.log('\n\n=== SUMMARY ===');
let totalPassed = 0;
let totalFailed = 0;
for (const result of allResults) {
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
totalPassed += passed;
totalFailed += failed;
const status = failed === 0 ? '✓' : '✗';
console.log(`${status} ${result.scenario}: ${passed} passed, ${failed} failed`);
}
console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`);
process.exit(totalFailed > 0 ? 1 : 0);
} else if (scenario) {
const result = await runServerConformanceTest(serverUrl, scenario);
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
}
process.exit(failed > 0 ? 1 : 0);
}
process.exit(failed > 0 ? 1 : 0);
} catch (error) {
console.error('Server test runner error:', error);
process.exit(1);