diff --git a/src/checks.ts b/src/checks.ts index 43a1628..72338c1 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -50,7 +50,7 @@ export function createServerInfoCheck( return { id: 'server-info', name: 'ServerInfo', - description: 'Validates server implementation includes required name and version', + description: 'Test server info returned to client', status, timestamp: new Date().toISOString(), specReferences: [ diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 6311e6d..1ff7b4c 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,6 +1,9 @@ import { Scenario } from '../types.js'; +import { initializeScenario } from './initialize/index.js'; -export const scenarios = new Map(); +export const scenarios = new Map([ + ['initialize', initializeScenario], +]); export function registerScenario(name: string, scenario: Scenario): void { scenarios.set(name, scenario); diff --git a/src/scenarios/initialize/TODO.md b/src/scenarios/initialize/TODO.md deleted file mode 100644 index ae0ac1d..0000000 --- a/src/scenarios/initialize/TODO.md +++ /dev/null @@ -1,203 +0,0 @@ -# Initialize Scenario Implementation TODO - -## Overview - -The `initialize` scenario tests the MCP client initialization handshake according to the MCP specification lifecycle requirements. - -## Architecture - -### Scenario Interface - -Each scenario provides server setup/teardown and result retrieval: - -```typescript -interface Scenario { - name: string; - description: string; - - // Start test server(s), return URLs - start(): Promise; - - // Stop test server(s) and cleanup - stop(): Promise; - - // Get conformance check results from server - getChecks(): ConformanceCheck[]; -} - -interface ScenarioUrls { - serverUrl: string; // MCP server URL (required) - authUrl?: string; // Auth server URL (optional, for auth scenarios) -} -``` - -### Flow Control (Runner's Responsibility) - -The runner orchestrates: -1. **Start Scenario**: Call `scenario.start()` to get server URL(s) -2. **Execute Client**: Spawn client process with server URL appended -3. **Capture Output**: Collect stdout/stderr from client -4. **Wait for Completion**: Wait for client to exit (or timeout) -5. **Get Results**: Call `scenario.getChecks()` to retrieve conformance checks -6. **Stop Scenario**: Call `scenario.stop()` to shutdown server(s) -7. **Write Results**: Save checks, stdout, stderr to `results/-/` - -### Server Responsibilities - -The test server: -- Captures MCP protocol interactions -- **Runs conformance checks internally** as interactions occur -- Stores checks for retrieval via `getChecks()` -- Supports clean shutdown - -## Implementation Tasks - -### 1. Create Test Server (`server.ts`) - -- [ ] Implement `InitializeTestServer` class with: - - `constructor()` - set up server - - `start(): Promise` - start listening on random port, return port number - - `stop(): Promise` - close server and connections - - `getChecks(): ConformanceCheck[]` - return conformance check results - -- [ ] Server behavior: - - Accept HTTP connections (dummy for now, MCP later) - - Capture the `initialize` request from client - - **Run conformance checks immediately** using existing check functions: - - `createClientInitializationCheck(initializeRequest)` - - Store checks in internal array - - Respond with valid `initialize` response: - - `protocolVersion: "2025-06-18"` - - `serverInfo: { name: "test-server", version: "1.0.0" }` - - `capabilities: {}` - - After response, optionally run server info check: - - `createServerInfoCheck(serverInfo)` (validates we provided proper info) - -### 2. Create Scenario Class (`index.ts`) - -- [ ] Implement `InitializeScenario` class: - ```typescript - export class InitializeScenario implements Scenario { - name = 'initialize'; - description = 'Tests MCP client initialization handshake'; - - private server: InitializeTestServer | null = null; - - async start(): Promise { - this.server = new InitializeTestServer(); - const port = await this.server.start(); - return { - serverUrl: `http://localhost:${port}` - }; - } - - async stop(): Promise { - if (this.server) { - await this.server.stop(); - this.server = null; - } - } - - getChecks(): ConformanceCheck[] { - if (!this.server) { - return []; - } - return this.server.getChecks(); - } - } - ``` - -- [ ] Export singleton instance: `export const initializeScenario = new InitializeScenario();` - -### 3. Scenario Registry (`src/scenarios/index.ts`) - -- [ ] Create scenario registry: - ```typescript - import { initializeScenario } from './initialize/index.js'; - - export const scenarios = new Map([ - ['initialize', initializeScenario], - // future scenarios here - ]); - - export function getScenario(name: string): Scenario | undefined { - return scenarios.get(name); - } - ``` - -## File Structure - -``` -src/scenarios/ -├── index.ts (scenario registry/map) -└── initialize/ - ├── TODO.md (this file - delete when complete) - ├── index.ts (InitializeScenario class) - └── server.ts (InitializeTestServer class) -``` - -## Example Usage - -From runner: - -```typescript -import { getScenario } from './scenarios/index.js'; - -const scenario = getScenario('initialize'); -if (!scenario) { - throw new Error('Unknown scenario: initialize'); -} - -// Start server -const urls = await scenario.start(); - -try { - // Execute client (runner's responsibility) - const clientOutput = await executeClient(clientCommand, urls.serverUrl); - - // Get checks from scenario - const checks = scenario.getChecks(); - - // Write results (runner's responsibility) - await writeResults(outputDir, { checks, ...clientOutput }); -} finally { - // Stop server - await scenario.stop(); -} -``` - -## Conformance Checks - -The server runs these checks (from `src/checks.ts`): -1. `createClientInitializationCheck(initializeRequest)` - validates client's initialize request -2. `createServerInfoCheck(serverInfo)` - validates server info was provided (optional) - -Additional checks to consider: -- Client exit code validation (runner handles this) -- Connection established successfully -- No protocol errors - -## Dependencies - -- `http` for HTTP server (dummy for prototyping) -- Existing check functions from `src/checks.ts` - -Note: The runner handles `child_process.spawn` and `fs/promises` for file writing. - -## Testing - -After implementation: -- [ ] Test with example client: `tsx examples/clients/typescript/test1.ts` -- [ ] Verify checks array is populated correctly -- [ ] Verify server starts and stops cleanly -- [ ] Test with broken client to ensure checks catch issues -- [ ] Use runner to execute full test flow - -## Notes - -- **Runner executes the client**, not the scenario -- **Scenario only manages server lifecycle** (start/stop) and provides results (getChecks) -- Server does checks internally and stores them -- Clean shutdown is critical - runner should use try/finally for scenario.stop() -- Start with dummy HTTP server, upgrade to MCP later -- For auth scenarios, return both `serverUrl` and `authUrl` from `start()` diff --git a/src/scenarios/initialize/index.ts b/src/scenarios/initialize/index.ts new file mode 100644 index 0000000..ed36249 --- /dev/null +++ b/src/scenarios/initialize/index.ts @@ -0,0 +1,33 @@ +import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; +import { InitializeTestServer } from './server.js'; + +export class InitializeScenario implements Scenario { + name = 'initialize'; + description = 'Tests MCP client initialization handshake'; + + private server: InitializeTestServer | null = null; + + async start(): Promise { + this.server = new InitializeTestServer(); + const port = await this.server.start(); + return { + serverUrl: `http://localhost:${port}` + }; + } + + async stop(): Promise { + if (this.server) { + await this.server.stop(); + this.server = null; + } + } + + getChecks(): ConformanceCheck[] { + if (!this.server) { + return []; + } + return this.server.getChecks(); + } +} + +export const initializeScenario = new InitializeScenario(); diff --git a/src/scenarios/initialize/server.ts b/src/scenarios/initialize/server.ts new file mode 100644 index 0000000..fca2a46 --- /dev/null +++ b/src/scenarios/initialize/server.ts @@ -0,0 +1,127 @@ +import http from 'http'; +import { ConformanceCheck } from '../../types.js'; +import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js'; + +export class InitializeTestServer { + private server: http.Server | null = null; + private checks: ConformanceCheck[] = []; + private port: number = 0; + + async start(): Promise { + return new Promise((resolve, reject) => { + this.server = http.createServer((req, res) => { + this.handleRequest(req, res); + }); + + this.server.on('error', reject); + + this.server.listen(0, () => { + const address = this.server!.address(); + if (address && typeof address === 'object') { + this.port = address.port; + resolve(this.port); + } else { + reject(new Error('Failed to get server address')); + } + }); + }); + } + + async stop(): Promise { + return new Promise((resolve, reject) => { + if (this.server) { + this.server.close((err) => { + if (err) { + reject(err); + } else { + this.server = null; + resolve(); + } + }); + } else { + resolve(); + } + }); + } + + getChecks(): ConformanceCheck[] { + return this.checks; + } + + private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void { + let body = ''; + + req.on('data', (chunk) => { + body += chunk.toString(); + }); + + req.on('end', () => { + try { + const request = JSON.parse(body); + + if (request.method === 'initialize') { + this.handleInitialize(request, res); + } else if (request.method === 'tools/list') { + this.handleToolsList(request, res); + } else { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + result: {} + })); + } + } catch (error) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + jsonrpc: '2.0', + error: { + code: -32700, + message: 'Parse error' + } + })); + } + }); + } + + private handleInitialize(request: any, res: http.ServerResponse): void { + const initializeRequest = request.params; + + const check = createClientInitializationCheck(initializeRequest); + this.checks.push(check); + + const serverInfo = { + name: 'test-server', + version: '1.0.0' + }; + + const serverInfoCheck = createServerInfoCheck(serverInfo); + this.checks.push(serverInfoCheck); + + const response = { + jsonrpc: '2.0', + id: request.id, + result: { + protocolVersion: '2025-06-18', + serverInfo, + capabilities: {} + } + }; + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + } + + private handleToolsList(request: any, res: http.ServerResponse): void { + const response = { + jsonrpc: '2.0', + id: request.id, + result: { + tools: [] + } + }; + + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(response)); + } +}