diff --git a/README.md b/README.md index 0ac72b6..26451b0 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario ## Overview The conformance test framework validates MCP client implementations by: + 1. Starting a test server for the specified scenario 2. Running the client implementation with the test server URL 3. Capturing MCP protocol interactions @@ -32,6 +33,7 @@ The framework appends the server URL as the final argument to your command. ## Test Results Results are saved to `results/-/`: + - `checks.json` - Array of conformance check results with pass/fail status - `stdout.txt` - Client stdout output - `stderr.txt` - Client stderr output @@ -44,9 +46,9 @@ Results are saved to `results/-/`: ## Available Scenarios - **initialize** - Tests MCP client initialization handshake - - Validates protocol version - - Validates clientInfo (name and version) - - Validates server response handling + - Validates protocol version + - Validates clientInfo (name and version) + - Validates server response handling ## Architecture diff --git a/examples/clients/typescript/test-broken.ts b/examples/clients/typescript/test-broken.ts index 4a9825e..2b4f477 100644 --- a/examples/clients/typescript/test-broken.ts +++ b/examples/clients/typescript/test-broken.ts @@ -3,57 +3,55 @@ import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; async function main(): Promise { - const serverUrl = process.argv[2]; + const serverUrl = process.argv[2]; - if (!serverUrl) { - console.error('Usage: test-broken-client '); - process.exit(1); - } + if (!serverUrl) { + console.error('Usage: test-broken-client '); + process.exit(1); + } - console.log(`Connecting to MCP server at: ${serverUrl}`); + console.log(`Connecting to MCP server at: ${serverUrl}`); - try { - const transport = new StreamableHTTPClientTransport( - new URL(serverUrl) - ); + try { + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); - await transport.start(); + await transport.start(); - const initRequest = { - jsonrpc: '2.0', - id: 1, - method: 'initialize', - params: { - protocolVersion: '2025-06-18', - clientInfo: { - version: '1.0.0' - }, - capabilities: {} - } - }; + const initRequest = { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { + version: '1.0.0' + }, + capabilities: {} + } + }; - const response = await fetch(serverUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(initRequest) - }); + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(initRequest) + }); - const result = await response.json(); - console.log('✅ Received response:', JSON.stringify(result, null, 2)); + const result = await response.json(); + console.log('✅ Received response:', JSON.stringify(result, null, 2)); - await transport.close(); - console.log('✅ Connection closed'); + await transport.close(); + console.log('✅ Connection closed'); - process.exit(0); - } catch (error) { - console.error('❌ Error:', error); - process.exit(1); - } + process.exit(0); + } catch (error) { + console.error('❌ Error:', error); + process.exit(1); + } } -main().catch((error) => { - console.error('Unhandled error:', error); - process.exit(1); +main().catch(error => { + console.error('Unhandled error:', error); + process.exit(1); }); diff --git a/examples/clients/typescript/test1.ts b/examples/clients/typescript/test1.ts index 60dd85a..2b08db1 100644 --- a/examples/clients/typescript/test1.ts +++ b/examples/clients/typescript/test1.ts @@ -4,44 +4,45 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; async function main(): Promise { - const serverUrl = process.argv[2]; + const serverUrl = process.argv[2]; - if (!serverUrl) { - console.error('Usage: test-client '); - process.exit(1); - } + if (!serverUrl) { + console.error('Usage: test-client '); + process.exit(1); + } - console.log(`Connecting to MCP server at: ${serverUrl}`); + console.log(`Connecting to MCP server at: ${serverUrl}`); - try { - const client = new Client({ - name: 'test-client', - version: '1.0.0' - }, { - capabilities: {} - }); + try { + const client = new Client( + { + name: 'test-client', + version: '1.0.0' + }, + { + capabilities: {} + } + ); - const transport = new StreamableHTTPClientTransport( - new URL(serverUrl) - ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); - await client.connect(transport); - console.log('✅ Successfully connected to MCP server'); + await client.connect(transport); + console.log('✅ Successfully connected to MCP server'); - await client.listTools(); - console.log('✅ Successfully listed tools'); + await client.listTools(); + console.log('✅ Successfully listed tools'); - await transport.close(); - console.log('✅ Connection closed successfully'); + await transport.close(); + console.log('✅ Connection closed successfully'); - process.exit(0); - } catch (error) { - console.error('❌ Failed to connect to MCP server:', error); - process.exit(1); - } + process.exit(0); + } catch (error) { + console.error('❌ Failed to connect to MCP server:', error); + process.exit(1); + } } -main().catch((error) => { - console.error('Unhandled error:', error); - process.exit(1); +main().catch(error => { + console.error('Unhandled error:', error); + process.exit(1); }); diff --git a/jest.config.js b/jest.config.js index 21358df..03f6967 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,8 +1,8 @@ export default { - preset: 'ts-jest/presets/default-esm', - extensionsToTreatAsEsm: ['.ts'], - testEnvironment: 'node', - moduleNameMapper: { - '^(\\.{1,2}/.*)\\.js$': '$1', - }, + preset: 'ts-jest/presets/default-esm', + extensionsToTreatAsEsm: ['.ts'], + testEnvironment: 'node', + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1' + } }; diff --git a/src/checks.test.ts b/src/checks.test.ts index 1fb5cd4..b7c19eb 100644 --- a/src/checks.test.ts +++ b/src/checks.test.ts @@ -1,84 +1,84 @@ import { createClientInitializationCheck } from './checks'; describe('createClientInitializationCheck', () => { - it('should return SUCCESS for a valid initialize request', () => { - const validRequest = { - protocolVersion: '2025-06-18', - clientInfo: { - name: 'TestClient', - version: '1.0.0' - } - }; + it('should return SUCCESS for a valid initialize request', () => { + const validRequest = { + protocolVersion: '2025-06-18', + clientInfo: { + name: 'TestClient', + version: '1.0.0' + } + }; - const check = createClientInitializationCheck(validRequest); - expect(check.status).toBe('SUCCESS'); - expect(check.errorMessage).toBeUndefined(); - }); + const check = createClientInitializationCheck(validRequest); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + }); - it('should return FAILURE when protocol version is missing', () => { - const invalidRequest = { - clientInfo: { - name: 'TestClient', - version: '1.0.0' - } - }; + it('should return FAILURE when protocol version is missing', () => { + const invalidRequest = { + clientInfo: { + name: 'TestClient', + version: '1.0.0' + } + }; - const check = createClientInitializationCheck(invalidRequest); - expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Protocol version not provided'); - }); + const check = createClientInitializationCheck(invalidRequest); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Protocol version not provided'); + }); - it('should return FAILURE when protocol version does not match', () => { - const invalidRequest = { - protocolVersion: '2024-11-05', - clientInfo: { - name: 'TestClient', - version: '1.0.0' - } - }; + it('should return FAILURE when protocol version does not match', () => { + const invalidRequest = { + protocolVersion: '2024-11-05', + clientInfo: { + name: 'TestClient', + version: '1.0.0' + } + }; - const check = createClientInitializationCheck(invalidRequest); - expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Version mismatch'); - }); + const check = createClientInitializationCheck(invalidRequest); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Version mismatch'); + }); - it('should return FAILURE when client name is missing', () => { - const invalidRequest = { - protocolVersion: '2025-06-18', - clientInfo: { - version: '1.0.0' - } - }; + it('should return FAILURE when client name is missing', () => { + const invalidRequest = { + protocolVersion: '2025-06-18', + clientInfo: { + version: '1.0.0' + } + }; - const check = createClientInitializationCheck(invalidRequest); - expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Client name missing'); - }); + const check = createClientInitializationCheck(invalidRequest); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Client name missing'); + }); - it('should return FAILURE when client version is missing', () => { - const invalidRequest = { - protocolVersion: '2025-06-18', - clientInfo: { - name: 'TestClient' - } - }; + it('should return FAILURE when client version is missing', () => { + const invalidRequest = { + protocolVersion: '2025-06-18', + clientInfo: { + name: 'TestClient' + } + }; - const check = createClientInitializationCheck(invalidRequest); - expect(check.status).toBe('FAILURE'); - expect(check.errorMessage).toContain('Client version missing'); - }); + const check = createClientInitializationCheck(invalidRequest); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Client version missing'); + }); - it('should support custom expected spec version', () => { - const request = { - protocolVersion: '2024-11-05', - clientInfo: { - name: 'TestClient', - version: '1.0.0' - } - }; + it('should support custom expected spec version', () => { + const request = { + protocolVersion: '2024-11-05', + clientInfo: { + name: 'TestClient', + version: '1.0.0' + } + }; - const check = createClientInitializationCheck(request, '2024-11-05'); - expect(check.status).toBe('SUCCESS'); - expect(check.errorMessage).toBeUndefined(); - }); + const check = createClientInitializationCheck(request, '2024-11-05'); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + }); }); diff --git a/src/checks.ts b/src/checks.ts index 72338c1..6ff4e35 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -1,68 +1,59 @@ import { ConformanceCheck, CheckStatus } from './types.js'; -export function createClientInitializationCheck( - initializeRequest: any, - expectedSpecVersion: string = '2025-06-18' -): ConformanceCheck { - const protocolVersionSent = initializeRequest?.protocolVersion; - const versionMatch = protocolVersionSent === expectedSpecVersion; +export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { + const protocolVersionSent = initializeRequest?.protocolVersion; + const versionMatch = protocolVersionSent === expectedSpecVersion; - const errors: string[] = []; - if (!protocolVersionSent) errors.push('Protocol version not provided'); - if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`); - if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); - if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing'); + const errors: string[] = []; + if (!protocolVersionSent) errors.push('Protocol version not provided'); + if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`); + if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); + if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing'); - const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; + const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; - return { - id: 'mcp-client-initialization', - name: 'MCPClientInitialization', - description: 'Validates that MCP client properly initializes with server', - status, - timestamp: new Date().toISOString(), - specReferences: [ - { id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' } - ], - details: { - protocolVersionSent, - expectedSpecVersion, - versionMatch, - clientName: initializeRequest?.clientInfo?.name, - clientVersion: initializeRequest?.clientInfo?.version - }, - errorMessage: errors.length > 0 ? errors.join('; ') : undefined, - logs: errors.length > 0 ? errors : undefined - }; + return { + id: 'mcp-client-initialization', + name: 'MCPClientInitialization', + description: 'Validates that MCP client properly initializes with server', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + protocolVersionSent, + expectedSpecVersion, + versionMatch, + clientName: initializeRequest?.clientInfo?.name, + clientVersion: initializeRequest?.clientInfo?.version + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; } -export function createServerInfoCheck( - serverInfo: { name?: string; version?: string } | null -): ConformanceCheck { - const hasName = !!serverInfo?.name; - const hasVersion = !!serverInfo?.version; - const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE'; +export function createServerInfoCheck(serverInfo: { name?: string; version?: string } | null): ConformanceCheck { + const hasName = !!serverInfo?.name; + const hasVersion = !!serverInfo?.version; + const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE'; - const errors: string[] = []; - if (!hasName) errors.push('Server name missing'); - if (!hasVersion) errors.push('Server version missing'); + const errors: string[] = []; + if (!hasName) errors.push('Server name missing'); + if (!hasVersion) errors.push('Server version missing'); - return { - id: 'server-info', - name: 'ServerInfo', - description: 'Test server info returned to client', - status, - timestamp: new Date().toISOString(), - specReferences: [ - { id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' } - ], - details: { - serverName: serverInfo?.name, - serverVersion: serverInfo?.version, - hasName, - hasVersion - }, - errorMessage: errors.length > 0 ? errors.join('; ') : undefined, - logs: errors.length > 0 ? errors : undefined - }; + return { + id: 'server-info', + name: 'ServerInfo', + description: 'Test server info returned to client', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + serverName: serverInfo?.name, + serverVersion: serverInfo?.version, + hasName, + hasVersion + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; } diff --git a/src/runner/DESIGN.md b/src/runner/DESIGN.md index 638adc7..e5bec10 100644 --- a/src/runner/DESIGN.md +++ b/src/runner/DESIGN.md @@ -47,6 +47,7 @@ A scenario represents a specific test case that validates one or more aspects of - Validates both client behavior and server responses Example scenarios: + - `initialize` - Tests client initialization handshake - `list-tools` - Tests tool discovery - `call-tool` - Tests tool invocation @@ -57,6 +58,7 @@ Example scenarios: Results are written to: `results/-/` Files: + - `checks.json` - Array of ConformanceCheck objects with validation results - `stdout.txt` - Complete stdout from the client process - `stderr.txt` - Complete stderr from the client process diff --git a/src/runner/index.ts b/src/runner/index.ts index baa633b..582072f 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -5,184 +5,171 @@ import { ConformanceCheck } from '../types.js'; import { getScenario } from '../scenarios/index.js'; export interface ClientExecutionResult { - exitCode: number; - stdout: string; - stderr: string; - timedOut: boolean; + exitCode: number; + stdout: string; + stderr: string; + timedOut: boolean; } async function ensureResultsDir(): Promise { - const resultsDir = path.join(process.cwd(), 'results'); - await fs.mkdir(resultsDir, { recursive: true }); - return resultsDir; + const resultsDir = path.join(process.cwd(), 'results'); + await fs.mkdir(resultsDir, { recursive: true }); + return resultsDir; } function createResultDir(scenario: string): string { - const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - return path.join('results', `${scenario}-${timestamp}`); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + return path.join('results', `${scenario}-${timestamp}`); } -async function executeClient( - command: string, - serverUrl: string, - timeout: number = 30000 -): Promise { - const commandParts = command.split(' '); - const executable = commandParts[0]; - const args = [...commandParts.slice(1), serverUrl]; +async function executeClient(command: string, serverUrl: string, timeout: number = 30000): Promise { + const commandParts = command.split(' '); + const executable = commandParts[0]; + const args = [...commandParts.slice(1), serverUrl]; - let stdout = ''; - let stderr = ''; - let timedOut = false; + let stdout = ''; + let stderr = ''; + let timedOut = false; - return new Promise((resolve) => { - const process = spawn(executable, args, { - shell: true, - stdio: 'pipe' + return new Promise(resolve => { + const process = spawn(executable, args, { + shell: true, + stdio: 'pipe' + }); + + const timeoutHandle = setTimeout(() => { + timedOut = true; + process.kill(); + }, timeout); + + if (process.stdout) { + process.stdout.on('data', data => { + stdout += data.toString(); + }); + } + + if (process.stderr) { + process.stderr.on('data', data => { + stderr += data.toString(); + }); + } + + process.on('close', code => { + clearTimeout(timeoutHandle); + resolve({ + exitCode: code || 0, + stdout, + stderr, + timedOut + }); + }); + + process.on('error', error => { + clearTimeout(timeoutHandle); + resolve({ + exitCode: -1, + stdout, + stderr: stderr + `\nProcess error: ${error.message}`, + timedOut + }); + }); }); - - const timeoutHandle = setTimeout(() => { - timedOut = true; - process.kill(); - }, timeout); - - if (process.stdout) { - process.stdout.on('data', (data) => { - stdout += data.toString(); - }); - } - - if (process.stderr) { - process.stderr.on('data', (data) => { - stderr += data.toString(); - }); - } - - process.on('close', (code) => { - clearTimeout(timeoutHandle); - resolve({ - exitCode: code || 0, - stdout, - stderr, - timedOut - }); - }); - - process.on('error', (error) => { - clearTimeout(timeoutHandle); - resolve({ - exitCode: -1, - stdout, - stderr: stderr + `\nProcess error: ${error.message}`, - timedOut - }); - }); - }); } export async function runConformanceTest( - clientCommand: string, - scenarioName: string, - timeout: number = 30000 + clientCommand: string, + scenarioName: string, + timeout: number = 30000 ): Promise<{ - checks: ConformanceCheck[]; - clientOutput: ClientExecutionResult; - resultDir: string; + checks: ConformanceCheck[]; + clientOutput: ClientExecutionResult; + resultDir: string; }> { - await ensureResultsDir(); - const resultDir = createResultDir(scenarioName); - await fs.mkdir(resultDir, { recursive: true }); + await ensureResultsDir(); + const resultDir = createResultDir(scenarioName); + await fs.mkdir(resultDir, { recursive: true }); - const scenario = getScenario(scenarioName); - if (!scenario) { - throw new Error(`Unknown scenario: ${scenarioName}`); - } + const scenario = getScenario(scenarioName); + if (!scenario) { + throw new Error(`Unknown scenario: ${scenarioName}`); + } - console.log(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + console.log(`Starting scenario: ${scenarioName}`); + const urls = await scenario.start(); - console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`); + console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`); - try { - const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout); + try { + const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout); - const checks = scenario.getChecks(); + const checks = scenario.getChecks(); - await fs.writeFile( - path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) - ); + await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2)); - await fs.writeFile( - path.join(resultDir, 'stdout.txt'), - clientOutput.stdout - ); + await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout); - await fs.writeFile( - path.join(resultDir, 'stderr.txt'), - clientOutput.stderr - ); + await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr); - console.log(`Results saved to ${resultDir}`); + console.log(`Results saved to ${resultDir}`); - return { - checks, - clientOutput, - resultDir - }; - } finally { - await scenario.stop(); - } + return { + checks, + clientOutput, + resultDir + }; + } finally { + await scenario.stop(); + } } async function main(): Promise { - const args = process.argv.slice(2); - let command: string | null = null; - let scenario: string | null = null; + const args = process.argv.slice(2); + let command: string | null = null; + let scenario: string | null = null; - for (let i = 0; i < args.length; i++) { - if (args[i] === '--command' && i + 1 < args.length) { - command = args[i + 1]; - i++; - } else if (args[i] === '--scenario' && i + 1 < args.length) { - scenario = args[i + 1]; - i++; - } - } - - if (!command || !scenario) { - console.error('Usage: runner --command "" --scenario '); - console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize'); - process.exit(1); - } - - try { - const result = await runConformanceTest(command, scenario); - - const passed = result.checks.filter(c => c.status === 'SUCCESS').length; - const failed = result.checks.filter(c => c.status === 'FAILURE').length; - - console.log(`\nTest Results:`); - console.log(`Passed: ${passed}/${result.checks.length}`); - console.log(`Failed: ${failed}/${result.checks.length}`); - - if (failed > 0) { - console.log('\nFailed Checks:'); - result.checks - .filter(c => c.status === 'FAILURE') - .forEach(c => { - console.log(` - ${c.name}: ${c.description}`); - if (c.errorMessage) { - console.log(` Error: ${c.errorMessage}`); - } - }); + for (let i = 0; i < args.length; i++) { + if (args[i] === '--command' && i + 1 < args.length) { + command = args[i + 1]; + i++; + } else if (args[i] === '--scenario' && i + 1 < args.length) { + scenario = args[i + 1]; + i++; + } } - process.exit(failed > 0 ? 1 : 0); - } catch (error) { - console.error('Test runner error:', error); - process.exit(1); - } + if (!command || !scenario) { + console.error('Usage: runner --command "" --scenario '); + console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize'); + process.exit(1); + } + + try { + const result = await runConformanceTest(command, scenario); + + const passed = result.checks.filter(c => c.status === 'SUCCESS').length; + const failed = result.checks.filter(c => c.status === 'FAILURE').length; + + console.log(`\nTest Results:`); + console.log(`Passed: ${passed}/${result.checks.length}`); + console.log(`Failed: ${failed}/${result.checks.length}`); + + if (failed > 0) { + console.log('\nFailed Checks:'); + result.checks + .filter(c => c.status === 'FAILURE') + .forEach(c => { + console.log(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.log(` Error: ${c.errorMessage}`); + } + }); + } + + process.exit(failed > 0 ? 1 : 0); + } catch (error) { + console.error('Test runner error:', error); + process.exit(1); + } } main(); diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index b3f83bb..f105ae4 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,18 +1,16 @@ import { Scenario } from '../types.js'; import { InitializeScenario } from './initialize.js'; -export const scenarios = new Map([ - ['initialize', new InitializeScenario()], -]); +export const scenarios = new Map([['initialize', new InitializeScenario()]]); export function registerScenario(name: string, scenario: Scenario): void { - scenarios.set(name, scenario); + scenarios.set(name, scenario); } export function getScenario(name: string): Scenario | undefined { - return scenarios.get(name); + return scenarios.get(name); } export function listScenarios(): string[] { - return Array.from(scenarios.keys()); + return Array.from(scenarios.keys()); } diff --git a/src/scenarios/initialize.ts b/src/scenarios/initialize.ts index 7f1a93e..9919b4d 100644 --- a/src/scenarios/initialize.ts +++ b/src/scenarios/initialize.ts @@ -3,130 +3,134 @@ import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js'; import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js'; export class InitializeScenario implements Scenario { - name = 'initialize'; - description = 'Tests MCP client initialization handshake'; + name = 'initialize'; + description = 'Tests MCP client initialization handshake'; - private server: http.Server | null = null; - private checks: ConformanceCheck[] = []; - private port: number = 0; + private server: http.Server | null = null; + private checks: ConformanceCheck[] = []; + private port: number = 0; - async start(): Promise { - return new Promise((resolve, reject) => { - this.server = http.createServer((req, res) => { - this.handleRequest(req, res); - }); + async start(): Promise { + return new Promise((resolve, reject) => { + this.server = http.createServer((req, res) => { + this.handleRequest(req, res); + }); - this.server.on('error', reject); + this.server.on('error', reject); - this.server.listen(0, () => { - const address = this.server!.address(); - if (address && typeof address === 'object') { - this.port = address.port; - resolve({ - serverUrl: `http://localhost:${this.port}` - }); - } else { - reject(new Error('Failed to get server address')); - } - }); - }); - } - - async stop(): Promise { - return new Promise((resolve, reject) => { - if (this.server) { - this.server.close((err) => { - if (err) { - reject(err); - } else { - this.server = null; - resolve(); - } + this.server.listen(0, () => { + const address = this.server!.address(); + if (address && typeof address === 'object') { + this.port = address.port; + resolve({ + serverUrl: `http://localhost:${this.port}` + }); + } else { + reject(new Error('Failed to get server address')); + } + }); }); - } else { - resolve(); - } - }); - } + } - getChecks(): ConformanceCheck[] { - return this.checks; - } + async stop(): Promise { + return new Promise((resolve, reject) => { + if (this.server) { + this.server.close(err => { + if (err) { + reject(err); + } else { + this.server = null; + resolve(); + } + }); + } else { + resolve(); + } + }); + } - private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void { - let body = ''; + getChecks(): ConformanceCheck[] { + return this.checks; + } - req.on('data', (chunk) => { - body += chunk.toString(); - }); + private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void { + let body = ''; - req.on('end', () => { - try { - const request = JSON.parse(body); + req.on('data', chunk => { + body += chunk.toString(); + }); - if (request.method === 'initialize') { - this.handleInitialize(request, res); - } else if (request.method === 'tools/list') { - this.handleToolsList(request, res); - } else { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ + req.on('end', () => { + try { + const request = JSON.parse(body); + + if (request.method === 'initialize') { + this.handleInitialize(request, res); + } else if (request.method === 'tools/list') { + this.handleToolsList(request, res); + } else { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + result: {} + }) + ); + } + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + error: { + code: -32700, + message: `Parse error ${error}` + } + }) + ); + } + }); + } + + private handleInitialize(request: any, res: http.ServerResponse): void { + const initializeRequest = request.params; + + const check = createClientInitializationCheck(initializeRequest); + this.checks.push(check); + + const serverInfo = { + name: 'test-server', + version: '1.0.0' + }; + + const serverInfoCheck = createServerInfoCheck(serverInfo); + this.checks.push(serverInfoCheck); + + const response = { jsonrpc: '2.0', id: request.id, - result: {} - })); - } - } catch (error) { - res.writeHead(400, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - jsonrpc: '2.0', - error: { - code: -32700, - message: 'Parse error' - } - })); - } - }); - } + result: { + protocolVersion: '2025-06-18', + serverInfo, + capabilities: {} + } + }; - private handleInitialize(request: any, res: http.ServerResponse): void { - const initializeRequest = request.params; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + } - const check = createClientInitializationCheck(initializeRequest); - this.checks.push(check); + private handleToolsList(request: any, res: http.ServerResponse): void { + const response = { + jsonrpc: '2.0', + id: request.id, + result: { + tools: [] + } + }; - const serverInfo = { - name: 'test-server', - version: '1.0.0' - }; - - const serverInfoCheck = createServerInfoCheck(serverInfo); - this.checks.push(serverInfoCheck); - - const response = { - jsonrpc: '2.0', - id: request.id, - result: { - protocolVersion: '2025-06-18', - serverInfo, - capabilities: {} - } - }; - - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(response)); - } - - private handleToolsList(request: any, res: http.ServerResponse): void { - const response = { - jsonrpc: '2.0', - id: request.id, - result: { - tools: [] - } - }; - - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(response)); - } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + } } diff --git a/src/types.ts b/src/types.ts index 869b920..1bd7733 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,32 +1,32 @@ export type CheckStatus = 'SUCCESS' | 'FAILURE' | 'WARNING' | 'SKIPPED' | 'INFO'; export interface SpecReference { - id: string; - url?: string; + id: string; + url?: string; } export interface ConformanceCheck { - id: string; - name: string; - description: string; - status: CheckStatus; - timestamp: string; - specReferences?: SpecReference[]; - details?: Record; - metadata?: Record; - errorMessage?: string; - logs?: string[]; + id: string; + name: string; + description: string; + status: CheckStatus; + timestamp: string; + specReferences?: SpecReference[]; + details?: Record; + metadata?: Record; + errorMessage?: string; + logs?: string[]; } export interface ScenarioUrls { - serverUrl: string; - authUrl?: string; + serverUrl: string; + authUrl?: string; } export interface Scenario { - name: string; - description: string; - start(): Promise; - stop(): Promise; - getChecks(): ConformanceCheck[]; + name: string; + description: string; + start(): Promise; + stop(): Promise; + getChecks(): ConformanceCheck[]; }