modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

apply linting

Browse source
This commit is contained in:
Paul Carleton 2025-10-30 13:09:16 +00:00
commit f32b01c57d
No known key found for this signature in database
11 changed files with 510 additions and 527 deletions
Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -12,6 +12,7 @@ npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario
## Overview ## Overview
The conformance test framework validates MCP client implementations by: The conformance test framework validates MCP client implementations by:
1. Starting a test server for the specified scenario 1. Starting a test server for the specified scenario
2. Running the client implementation with the test server URL 2. Running the client implementation with the test server URL
3. Capturing MCP protocol interactions 3. Capturing MCP protocol interactions
@ -32,6 +33,7 @@ The framework appends the server URL as the final argument to your command.
## Test Results ## Test Results
Results are saved to `results/<scenario>-<timestamp>/`: Results are saved to `results/<scenario>-<timestamp>/`:
- `checks.json` - Array of conformance check results with pass/fail status - `checks.json` - Array of conformance check results with pass/fail status
- `stdout.txt` - Client stdout output - `stdout.txt` - Client stdout output
- `stderr.txt` - Client stderr output - `stderr.txt` - Client stderr output
@ -44,9 +46,9 @@ Results are saved to `results/<scenario>-<timestamp>/`:
## Available Scenarios ## Available Scenarios
- **initialize** - Tests MCP client initialization handshake - **initialize** - Tests MCP client initialization handshake
- Validates protocol version - Validates protocol version
- Validates clientInfo (name and version) - Validates clientInfo (name and version)
- Validates server response handling - Validates server response handling
## Architecture ## Architecture

82
examples/clients/typescript/test-broken.ts
View file

@ -3,57 +3,55 @@
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
async function main(): Promise<void> { async function main(): Promise<void> {
const serverUrl = process.argv[2]; const serverUrl = process.argv[2];
if (!serverUrl) { if (!serverUrl) {
console.error('Usage: test-broken-client <server-url>'); console.error('Usage: test-broken-client <server-url>');
process.exit(1); process.exit(1);
} }
console.log(`Connecting to MCP server at: ${serverUrl}`); console.log(`Connecting to MCP server at: ${serverUrl}`);
try { try {
const transport = new StreamableHTTPClientTransport( const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
new URL(serverUrl)
);
await transport.start(); await transport.start();
const initRequest = { const initRequest = {
jsonrpc: '2.0', jsonrpc: '2.0',
id: 1, id: 1,
method: 'initialize', method: 'initialize',
params: { params: {
protocolVersion: '2025-06-18', protocolVersion: '2025-06-18',
clientInfo: { clientInfo: {
version: '1.0.0' version: '1.0.0'
}, },
capabilities: {} capabilities: {}
} }
}; };
const response = await fetch(serverUrl, { const response = await fetch(serverUrl, {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify(initRequest) body: JSON.stringify(initRequest)
}); });
const result = await response.json(); const result = await response.json();
console.log('✅ Received response:', JSON.stringify(result, null, 2)); console.log('✅ Received response:', JSON.stringify(result, null, 2));
await transport.close(); await transport.close();
console.log('✅ Connection closed'); console.log('✅ Connection closed');
process.exit(0); process.exit(0);
} catch (error) { } catch (error) {
console.error('❌ Error:', error); console.error('❌ Error:', error);
process.exit(1); process.exit(1);
} }
} }
main().catch((error) => { main().catch(error => {
console.error('Unhandled error:', error); console.error('Unhandled error:', error);
process.exit(1); process.exit(1);
}); });

61
examples/clients/typescript/test1.ts
View file

@ -4,44 +4,45 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
async function main(): Promise<void> { async function main(): Promise<void> {
const serverUrl = process.argv[2]; const serverUrl = process.argv[2];
if (!serverUrl) { if (!serverUrl) {
console.error('Usage: test-client <server-url>'); console.error('Usage: test-client <server-url>');
process.exit(1); process.exit(1);
} }
console.log(`Connecting to MCP server at: ${serverUrl}`); console.log(`Connecting to MCP server at: ${serverUrl}`);
try { try {
const client = new Client({ const client = new Client(
name: 'test-client', {
version: '1.0.0' name: 'test-client',
}, { version: '1.0.0'
capabilities: {} },
}); {
capabilities: {}
}
);
const transport = new StreamableHTTPClientTransport( const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
new URL(serverUrl)
);
await client.connect(transport); await client.connect(transport);
console.log('✅ Successfully connected to MCP server'); console.log('✅ Successfully connected to MCP server');
await client.listTools(); await client.listTools();
console.log('✅ Successfully listed tools'); console.log('✅ Successfully listed tools');
await transport.close(); await transport.close();
console.log('✅ Connection closed successfully'); console.log('✅ Connection closed successfully');
process.exit(0); process.exit(0);
} catch (error) { } catch (error) {
console.error('❌ Failed to connect to MCP server:', error); console.error('❌ Failed to connect to MCP server:', error);
process.exit(1); process.exit(1);
} }
} }
main().catch((error) => { main().catch(error => {
console.error('Unhandled error:', error); console.error('Unhandled error:', error);
process.exit(1); process.exit(1);
}); });

12
jest.config.js
View file

@ -1,8 +1,8 @@
export default { export default {
preset: 'ts-jest/presets/default-esm', preset: 'ts-jest/presets/default-esm',
extensionsToTreatAsEsm: ['.ts'], extensionsToTreatAsEsm: ['.ts'],
testEnvironment: 'node', testEnvironment: 'node',
moduleNameMapper: { moduleNameMapper: {
'^(\\.{1,2}/.*)\\.js$': '$1', '^(\\.{1,2}/.*)\\.js$': '$1'
}, }
}; };

138
src/checks.test.ts
View file

@ -1,84 +1,84 @@
import { createClientInitializationCheck } from './checks'; import { createClientInitializationCheck } from './checks';
describe('createClientInitializationCheck', () => { describe('createClientInitializationCheck', () => {
it('should return SUCCESS for a valid initialize request', () => { it('should return SUCCESS for a valid initialize request', () => {
const validRequest = { const validRequest = {
protocolVersion: '2025-06-18', protocolVersion: '2025-06-18',
clientInfo: { clientInfo: {
name: 'TestClient', name: 'TestClient',
version: '1.0.0' version: '1.0.0'
} }
}; };
const check = createClientInitializationCheck(validRequest); const check = createClientInitializationCheck(validRequest);
expect(check.status).toBe('SUCCESS'); expect(check.status).toBe('SUCCESS');
expect(check.errorMessage).toBeUndefined(); expect(check.errorMessage).toBeUndefined();
}); });
it('should return FAILURE when protocol version is missing', () => { it('should return FAILURE when protocol version is missing', () => {
const invalidRequest = { const invalidRequest = {
clientInfo: { clientInfo: {
name: 'TestClient', name: 'TestClient',
version: '1.0.0' version: '1.0.0'
} }
}; };
const check = createClientInitializationCheck(invalidRequest); const check = createClientInitializationCheck(invalidRequest);
expect(check.status).toBe('FAILURE'); expect(check.status).toBe('FAILURE');
expect(check.errorMessage).toContain('Protocol version not provided'); expect(check.errorMessage).toContain('Protocol version not provided');
}); });
it('should return FAILURE when protocol version does not match', () => { it('should return FAILURE when protocol version does not match', () => {
const invalidRequest = { const invalidRequest = {
protocolVersion: '2024-11-05', protocolVersion: '2024-11-05',
clientInfo: { clientInfo: {
name: 'TestClient', name: 'TestClient',
version: '1.0.0' version: '1.0.0'
} }
}; };
const check = createClientInitializationCheck(invalidRequest); const check = createClientInitializationCheck(invalidRequest);
expect(check.status).toBe('FAILURE'); expect(check.status).toBe('FAILURE');
expect(check.errorMessage).toContain('Version mismatch'); expect(check.errorMessage).toContain('Version mismatch');
}); });
it('should return FAILURE when client name is missing', () => { it('should return FAILURE when client name is missing', () => {
const invalidRequest = { const invalidRequest = {
protocolVersion: '2025-06-18', protocolVersion: '2025-06-18',
clientInfo: { clientInfo: {
version: '1.0.0' version: '1.0.0'
} }
}; };
const check = createClientInitializationCheck(invalidRequest); const check = createClientInitializationCheck(invalidRequest);
expect(check.status).toBe('FAILURE'); expect(check.status).toBe('FAILURE');
expect(check.errorMessage).toContain('Client name missing'); expect(check.errorMessage).toContain('Client name missing');
}); });
it('should return FAILURE when client version is missing', () => { it('should return FAILURE when client version is missing', () => {
const invalidRequest = { const invalidRequest = {
protocolVersion: '2025-06-18', protocolVersion: '2025-06-18',
clientInfo: { clientInfo: {
name: 'TestClient' name: 'TestClient'
} }
}; };
const check = createClientInitializationCheck(invalidRequest); const check = createClientInitializationCheck(invalidRequest);
expect(check.status).toBe('FAILURE'); expect(check.status).toBe('FAILURE');
expect(check.errorMessage).toContain('Client version missing'); expect(check.errorMessage).toContain('Client version missing');
}); });
it('should support custom expected spec version', () => { it('should support custom expected spec version', () => {
const request = { const request = {
protocolVersion: '2024-11-05', protocolVersion: '2024-11-05',
clientInfo: { clientInfo: {
name: 'TestClient', name: 'TestClient',
version: '1.0.0' version: '1.0.0'
} }
}; };
const check = createClientInitializationCheck(request, '2024-11-05'); const check = createClientInitializationCheck(request, '2024-11-05');
expect(check.status).toBe('SUCCESS'); expect(check.status).toBe('SUCCESS');
expect(check.errorMessage).toBeUndefined(); expect(check.errorMessage).toBeUndefined();
}); });
}); });

107
src/checks.ts
View file

@ -1,68 +1,59 @@
import { ConformanceCheck, CheckStatus } from './types.js'; import { ConformanceCheck, CheckStatus } from './types.js';
export function createClientInitializationCheck( export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
initializeRequest: any, const protocolVersionSent = initializeRequest?.protocolVersion;
expectedSpecVersion: string = '2025-06-18' const versionMatch = protocolVersionSent === expectedSpecVersion;
): ConformanceCheck {
const protocolVersionSent = initializeRequest?.protocolVersion;
const versionMatch = protocolVersionSent === expectedSpecVersion;
const errors: string[] = []; const errors: string[] = [];
if (!protocolVersionSent) errors.push('Protocol version not provided'); if (!protocolVersionSent) errors.push('Protocol version not provided');
if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`); if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`);
if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing');
if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing'); if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing');
const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
return { return {
id: 'mcp-client-initialization', id: 'mcp-client-initialization',
name: 'MCPClientInitialization', name: 'MCPClientInitialization',
description: 'Validates that MCP client properly initializes with server', description: 'Validates that MCP client properly initializes with server',
status, status,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
specReferences: [ specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' } details: {
], protocolVersionSent,
details: { expectedSpecVersion,
protocolVersionSent, versionMatch,
expectedSpecVersion, clientName: initializeRequest?.clientInfo?.name,
versionMatch, clientVersion: initializeRequest?.clientInfo?.version
clientName: initializeRequest?.clientInfo?.name, },
clientVersion: initializeRequest?.clientInfo?.version errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
}, logs: errors.length > 0 ? errors : undefined
errorMessage: errors.length > 0 ? errors.join('; ') : undefined, };
logs: errors.length > 0 ? errors : undefined
};
} }
export function createServerInfoCheck( export function createServerInfoCheck(serverInfo: { name?: string; version?: string } | null): ConformanceCheck {
serverInfo: { name?: string; version?: string } | null const hasName = !!serverInfo?.name;
): ConformanceCheck { const hasVersion = !!serverInfo?.version;
const hasName = !!serverInfo?.name; const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
const hasVersion = !!serverInfo?.version;
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
const errors: string[] = []; const errors: string[] = [];
if (!hasName) errors.push('Server name missing'); if (!hasName) errors.push('Server name missing');
if (!hasVersion) errors.push('Server version missing'); if (!hasVersion) errors.push('Server version missing');
return { return {
id: 'server-info', id: 'server-info',
name: 'ServerInfo', name: 'ServerInfo',
description: 'Test server info returned to client', description: 'Test server info returned to client',
status, status,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
specReferences: [ specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' } details: {
], serverName: serverInfo?.name,
details: { serverVersion: serverInfo?.version,
serverName: serverInfo?.name, hasName,
serverVersion: serverInfo?.version, hasVersion
hasName, },
hasVersion errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
}, logs: errors.length > 0 ? errors : undefined
errorMessage: errors.length > 0 ? errors.join('; ') : undefined, };
logs: errors.length > 0 ? errors : undefined
};
} }

2
src/runner/DESIGN.md
View file

@ -47,6 +47,7 @@ A scenario represents a specific test case that validates one or more aspects of
- Validates both client behavior and server responses - Validates both client behavior and server responses
Example scenarios: Example scenarios:
- `initialize` - Tests client initialization handshake - `initialize` - Tests client initialization handshake
- `list-tools` - Tests tool discovery - `list-tools` - Tests tool discovery
- `call-tool` - Tests tool invocation - `call-tool` - Tests tool invocation
@ -57,6 +58,7 @@ Example scenarios:
Results are written to: `results/<scenario>-<timestamp>/` Results are written to: `results/<scenario>-<timestamp>/`
Files: Files:
- `checks.json` - Array of ConformanceCheck objects with validation results - `checks.json` - Array of ConformanceCheck objects with validation results
- `stdout.txt` - Complete stdout from the client process - `stdout.txt` - Complete stdout from the client process
- `stderr.txt` - Complete stderr from the client process - `stderr.txt` - Complete stderr from the client process

279
src/runner/index.ts
View file

@ -5,184 +5,171 @@ import { ConformanceCheck } from '../types.js';
import { getScenario } from '../scenarios/index.js'; import { getScenario } from '../scenarios/index.js';
export interface ClientExecutionResult { export interface ClientExecutionResult {
exitCode: number; exitCode: number;
stdout: string; stdout: string;
stderr: string; stderr: string;
timedOut: boolean; timedOut: boolean;
} }
async function ensureResultsDir(): Promise<string> { async function ensureResultsDir(): Promise<string> {
const resultsDir = path.join(process.cwd(), 'results'); const resultsDir = path.join(process.cwd(), 'results');
await fs.mkdir(resultsDir, { recursive: true }); await fs.mkdir(resultsDir, { recursive: true });
return resultsDir; return resultsDir;
} }
function createResultDir(scenario: string): string { function createResultDir(scenario: string): string {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
return path.join('results', `${scenario}-${timestamp}`); return path.join('results', `${scenario}-${timestamp}`);
} }
async function executeClient( async function executeClient(command: string, serverUrl: string, timeout: number = 30000): Promise<ClientExecutionResult> {
command: string, const commandParts = command.split(' ');
serverUrl: string, const executable = commandParts[0];
timeout: number = 30000 const args = [...commandParts.slice(1), serverUrl];
): Promise<ClientExecutionResult> {
const commandParts = command.split(' ');
const executable = commandParts[0];
const args = [...commandParts.slice(1), serverUrl];
let stdout = ''; let stdout = '';
let stderr = ''; let stderr = '';
let timedOut = false; let timedOut = false;
return new Promise((resolve) => { return new Promise(resolve => {
const process = spawn(executable, args, { const process = spawn(executable, args, {
shell: true, shell: true,
stdio: 'pipe' stdio: 'pipe'
});
const timeoutHandle = setTimeout(() => {
timedOut = true;
process.kill();
}, timeout);
if (process.stdout) {
process.stdout.on('data', data => {
stdout += data.toString();
});
}
if (process.stderr) {
process.stderr.on('data', data => {
stderr += data.toString();
});
}
process.on('close', code => {
clearTimeout(timeoutHandle);
resolve({
exitCode: code || 0,
stdout,
stderr,
timedOut
});
});
process.on('error', error => {
clearTimeout(timeoutHandle);
resolve({
exitCode: -1,
stdout,
stderr: stderr + `\nProcess error: ${error.message}`,
timedOut
});
});
}); });
const timeoutHandle = setTimeout(() => {
timedOut = true;
process.kill();
}, timeout);
if (process.stdout) {
process.stdout.on('data', (data) => {
stdout += data.toString();
});
}
if (process.stderr) {
process.stderr.on('data', (data) => {
stderr += data.toString();
});
}
process.on('close', (code) => {
clearTimeout(timeoutHandle);
resolve({
exitCode: code || 0,
stdout,
stderr,
timedOut
});
});
process.on('error', (error) => {
clearTimeout(timeoutHandle);
resolve({
exitCode: -1,
stdout,
stderr: stderr + `\nProcess error: ${error.message}`,
timedOut
});
});
});
} }
export async function runConformanceTest( export async function runConformanceTest(
clientCommand: string, clientCommand: string,
scenarioName: string, scenarioName: string,
timeout: number = 30000 timeout: number = 30000
): Promise<{ ): Promise<{
checks: ConformanceCheck[]; checks: ConformanceCheck[];
clientOutput: ClientExecutionResult; clientOutput: ClientExecutionResult;
resultDir: string; resultDir: string;
}> { }> {
await ensureResultsDir(); await ensureResultsDir();
const resultDir = createResultDir(scenarioName); const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true }); await fs.mkdir(resultDir, { recursive: true });
const scenario = getScenario(scenarioName); const scenario = getScenario(scenarioName);
if (!scenario) { if (!scenario) {
throw new Error(`Unknown scenario: ${scenarioName}`); throw new Error(`Unknown scenario: ${scenarioName}`);
} }
console.log(`Starting scenario: ${scenarioName}`); console.log(`Starting scenario: ${scenarioName}`);
const urls = await scenario.start(); const urls = await scenario.start();
console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`); console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
try { try {
const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout); const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout);
const checks = scenario.getChecks(); const checks = scenario.getChecks();
await fs.writeFile( await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
await fs.writeFile( await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
path.join(resultDir, 'stdout.txt'),
clientOutput.stdout
);
await fs.writeFile( await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
path.join(resultDir, 'stderr.txt'),
clientOutput.stderr
);
console.log(`Results saved to ${resultDir}`); console.log(`Results saved to ${resultDir}`);
return { return {
checks, checks,
clientOutput, clientOutput,
resultDir resultDir
}; };
} finally { } finally {
await scenario.stop(); await scenario.stop();
} }
} }
async function main(): Promise<void> { async function main(): Promise<void> {
const args = process.argv.slice(2); const args = process.argv.slice(2);
let command: string | null = null; let command: string | null = null;
let scenario: string | null = null; let scenario: string | null = null;
for (let i = 0; i < args.length; i++) { for (let i = 0; i < args.length; i++) {
if (args[i] === '--command' && i + 1 < args.length) { if (args[i] === '--command' && i + 1 < args.length) {
command = args[i + 1]; command = args[i + 1];
i++; i++;
} else if (args[i] === '--scenario' && i + 1 < args.length) { } else if (args[i] === '--scenario' && i + 1 < args.length) {
scenario = args[i + 1]; scenario = args[i + 1];
i++; i++;
} }
}
if (!command || !scenario) {
console.error('Usage: runner --command "<command>" --scenario <scenario>');
console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
process.exit(1);
}
try {
const result = await runConformanceTest(command, scenario);
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${result.checks.length}`);
console.log(`Failed: ${failed}/${result.checks.length}`);
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
} }
process.exit(failed > 0 ? 1 : 0); if (!command || !scenario) {
} catch (error) { console.error('Usage: runner --command "<command>" --scenario <scenario>');
console.error('Test runner error:', error); console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
process.exit(1); process.exit(1);
} }
try {
const result = await runConformanceTest(command, scenario);
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${result.checks.length}`);
console.log(`Failed: ${failed}/${result.checks.length}`);
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
}
process.exit(failed > 0 ? 1 : 0);
} catch (error) {
console.error('Test runner error:', error);
process.exit(1);
}
} }
main(); main();

10
src/scenarios/index.ts
View file

@ -1,18 +1,16 @@
import { Scenario } from '../types.js'; import { Scenario } from '../types.js';
import { InitializeScenario } from './initialize.js'; import { InitializeScenario } from './initialize.js';
export const scenarios = new Map<string, Scenario>([ export const scenarios = new Map<string, Scenario>([['initialize', new InitializeScenario()]]);
['initialize', new InitializeScenario()],
]);
export function registerScenario(name: string, scenario: Scenario): void { export function registerScenario(name: string, scenario: Scenario): void {
scenarios.set(name, scenario); scenarios.set(name, scenario);
} }
export function getScenario(name: string): Scenario | undefined { export function getScenario(name: string): Scenario | undefined {
return scenarios.get(name); return scenarios.get(name);
} }
export function listScenarios(): string[] { export function listScenarios(): string[] {
return Array.from(scenarios.keys()); return Array.from(scenarios.keys());
} }

226
src/scenarios/initialize.ts
View file

@ -3,130 +3,134 @@ import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js';
import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js'; import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js';
export class InitializeScenario implements Scenario { export class InitializeScenario implements Scenario {
name = 'initialize'; name = 'initialize';
description = 'Tests MCP client initialization handshake'; description = 'Tests MCP client initialization handshake';
private server: http.Server | null = null; private server: http.Server | null = null;
private checks: ConformanceCheck[] = []; private checks: ConformanceCheck[] = [];
private port: number = 0; private port: number = 0;
async start(): Promise<ScenarioUrls> { async start(): Promise<ScenarioUrls> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
this.server = http.createServer((req, res) => { this.server = http.createServer((req, res) => {
this.handleRequest(req, res); this.handleRequest(req, res);
}); });
this.server.on('error', reject); this.server.on('error', reject);
this.server.listen(0, () => { this.server.listen(0, () => {
const address = this.server!.address(); const address = this.server!.address();
if (address && typeof address === 'object') { if (address && typeof address === 'object') {
this.port = address.port; this.port = address.port;
resolve({ resolve({
serverUrl: `http://localhost:${this.port}` serverUrl: `http://localhost:${this.port}`
}); });
} else { } else {
reject(new Error('Failed to get server address')); reject(new Error('Failed to get server address'));
} }
}); });
});
}
async stop(): Promise<void> {
return new Promise((resolve, reject) => {
if (this.server) {
this.server.close((err) => {
if (err) {
reject(err);
} else {
this.server = null;
resolve();
}
}); });
} else { }
resolve();
}
});
}
getChecks(): ConformanceCheck[] { async stop(): Promise<void> {
return this.checks; return new Promise((resolve, reject) => {
} if (this.server) {
this.server.close(err => {
if (err) {
reject(err);
} else {
this.server = null;
resolve();
}
});
} else {
resolve();
}
});
}
private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void { getChecks(): ConformanceCheck[] {
let body = ''; return this.checks;
}
req.on('data', (chunk) => { private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
body += chunk.toString(); let body = '';
});
req.on('end', () => { req.on('data', chunk => {
try { body += chunk.toString();
const request = JSON.parse(body); });
if (request.method === 'initialize') { req.on('end', () => {
this.handleInitialize(request, res); try {
} else if (request.method === 'tools/list') { const request = JSON.parse(body);
this.handleToolsList(request, res);
} else { if (request.method === 'initialize') {
res.writeHead(200, { 'Content-Type': 'application/json' }); this.handleInitialize(request, res);
res.end(JSON.stringify({ } else if (request.method === 'tools/list') {
this.handleToolsList(request, res);
} else {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(
JSON.stringify({
jsonrpc: '2.0',
id: request.id,
result: {}
})
);
}
} catch (error) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(
JSON.stringify({
jsonrpc: '2.0',
error: {
code: -32700,
message: `Parse error ${error}`
}
})
);
}
});
}
private handleInitialize(request: any, res: http.ServerResponse): void {
const initializeRequest = request.params;
const check = createClientInitializationCheck(initializeRequest);
this.checks.push(check);
const serverInfo = {
name: 'test-server',
version: '1.0.0'
};
const serverInfoCheck = createServerInfoCheck(serverInfo);
this.checks.push(serverInfoCheck);
const response = {
jsonrpc: '2.0', jsonrpc: '2.0',
id: request.id, id: request.id,
result: {} result: {
})); protocolVersion: '2025-06-18',
} serverInfo,
} catch (error) { capabilities: {}
res.writeHead(400, { 'Content-Type': 'application/json' }); }
res.end(JSON.stringify({ };
jsonrpc: '2.0',
error: {
code: -32700,
message: 'Parse error'
}
}));
}
});
}
private handleInitialize(request: any, res: http.ServerResponse): void { res.writeHead(200, { 'Content-Type': 'application/json' });
const initializeRequest = request.params; res.end(JSON.stringify(response));
}
const check = createClientInitializationCheck(initializeRequest); private handleToolsList(request: any, res: http.ServerResponse): void {
this.checks.push(check); const response = {
jsonrpc: '2.0',
id: request.id,
result: {
tools: []
}
};
const serverInfo = { res.writeHead(200, { 'Content-Type': 'application/json' });
name: 'test-server', res.end(JSON.stringify(response));
version: '1.0.0' }
};
const serverInfoCheck = createServerInfoCheck(serverInfo);
this.checks.push(serverInfoCheck);
const response = {
jsonrpc: '2.0',
id: request.id,
result: {
protocolVersion: '2025-06-18',
serverInfo,
capabilities: {}
}
};
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(response));
}
private handleToolsList(request: any, res: http.ServerResponse): void {
const response = {
jsonrpc: '2.0',
id: request.id,
result: {
tools: []
}
};
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(response));
}
} }

38
src/types.ts
View file

@ -1,32 +1,32 @@
export type CheckStatus = 'SUCCESS' | 'FAILURE' | 'WARNING' | 'SKIPPED' | 'INFO'; export type CheckStatus = 'SUCCESS' | 'FAILURE' | 'WARNING' | 'SKIPPED' | 'INFO';
export interface SpecReference { export interface SpecReference {
id: string; id: string;
url?: string; url?: string;
} }
export interface ConformanceCheck { export interface ConformanceCheck {
id: string; id: string;
name: string; name: string;
description: string; description: string;
status: CheckStatus; status: CheckStatus;
timestamp: string; timestamp: string;
specReferences?: SpecReference[]; specReferences?: SpecReference[];
details?: Record<string, unknown>; details?: Record<string, unknown>;
metadata?: Record<string, unknown>; metadata?: Record<string, unknown>;
errorMessage?: string; errorMessage?: string;
logs?: string[]; logs?: string[];
} }
export interface ScenarioUrls { export interface ScenarioUrls {
serverUrl: string; serverUrl: string;
authUrl?: string; authUrl?: string;
} }
export interface Scenario { export interface Scenario {
name: string; name: string;
description: string; description: string;
start(): Promise<ScenarioUrls>; start(): Promise<ScenarioUrls>;
stop(): Promise<void>; stop(): Promise<void>;
getChecks(): ConformanceCheck[]; getChecks(): ConformanceCheck[];
} }