From 98bf5aaf87d26aa6767d6760b61fe1fcd3b48ff5 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Wed, 29 Oct 2025 14:38:11 +0000 Subject: [PATCH] wip first crack at scenario and runner --- .gitignore | 1 + package-lock.json | 4 +- package.json | 4 +- src/runner.ts | 21 +--- src/runner/DESIGN.md | 91 ++++++++++++++ src/runner/index.ts | 188 ++++++++++++++++++++++++++++ src/scenarios/index.ts | 15 +++ src/scenarios/initialize/TODO.md | 203 +++++++++++++++++++++++++++++++ src/types.ts | 13 ++ 9 files changed, 520 insertions(+), 20 deletions(-) create mode 100644 src/runner/DESIGN.md create mode 100644 src/runner/index.ts create mode 100644 src/scenarios/index.ts create mode 100644 src/scenarios/initialize/TODO.md diff --git a/.gitignore b/.gitignore index 3c3629e..2be9418 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +results/ diff --git a/package-lock.json b/package-lock.json index dbd2aed..94d1e94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,9 @@ "name": "conformance", "version": "1.0.0", "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.1" + "@modelcontextprotocol/sdk": "^1.20.1", + "express": "^5.1.0", + "zod": "^3.25.76" }, "devDependencies": { "@jest/globals": "^29.7.0", diff --git a/package.json b/package.json index da65495..e9b9c84 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "typescript": "^5.3.0" }, "dependencies": { - "@modelcontextprotocol/sdk": "^1.20.1" + "@modelcontextprotocol/sdk": "^1.20.1", + "express": "^5.1.0", + "zod": "^3.25.76" } } diff --git a/src/runner.ts b/src/runner.ts index 5c3be9c..0ee592e 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,19 +1,4 @@ -import { createClientInitializationCheck, createServerInfoCheck } from './checks.ts'; +import { runConformanceTest } from './runner/index.js'; -const initializeRequest = { - protocolVersion: '2025-06-18', - clientInfo: { - name: 'ExampleClient', - version: '1.0.0' - } -}; - -const checks = [ - createClientInitializationCheck(initializeRequest), - createServerInfoCheck({ - name: 'ExampleMCPServer', - version: '1.0.0' - }) -]; - -console.log(JSON.stringify(checks, null, 2)); +export { runConformanceTest }; +export type { ScenarioConfig } from './runner/index.js'; diff --git a/src/runner/DESIGN.md b/src/runner/DESIGN.md new file mode 100644 index 0000000..638adc7 --- /dev/null +++ b/src/runner/DESIGN.md @@ -0,0 +1,91 @@ +# Conformance Test Runner Design + +## Overview + +The conformance test runner is a framework for testing MCP client implementations against the MCP specification. It executes client implementations in controlled scenarios and validates their behavior through a series of conformance checks. + +## Architecture + +### Components + +1. **Runner** - Orchestrates test execution +2. **Test Server** - MCP server (and optionally auth server for auth scenarios) +3. **Client Process** - The MCP client implementation under test +4. **Scenario** - A specific test case with expected behaviors +5. **Checks** - Validation functions that produce ConformanceCheck results + +### Execution Flow + +``` +1. Runner starts test server(s) on available port(s) +2. Runner spawns client process with server URL as final argument +3. Server captures MCP interactions +4. Runner captures client stdout/stderr +5. Scenario-specific checks are executed +6. Results are written to results/-/ +``` + +## CLI Interface + +```bash +npm run start -- --command "tsx examples/clients/typescript/test1.ts --verbose" --scenario initialize +``` + +### Arguments + +- `--command` - The command to run the client (can include existing flags) +- `--scenario` - The scenario to test (e.g., "initialize", "list-tools", "call-tool") + +The runner will append the server URL as the final argument to the command. + +## Scenarios + +A scenario represents a specific test case that validates one or more aspects of MCP client behavior. Each scenario: + +- Configures the test server with expected behavior +- May run multiple conformance checks +- Validates both client behavior and server responses + +Example scenarios: +- `initialize` - Tests client initialization handshake +- `list-tools` - Tests tool discovery +- `call-tool` - Tests tool invocation +- `auth-flow` - Tests OAuth/authorization flows (requires auth server) + +## Output Structure + +Results are written to: `results/-/` + +Files: +- `checks.json` - Array of ConformanceCheck objects with validation results +- `stdout.txt` - Complete stdout from the client process +- `stderr.txt` - Complete stderr from the client process + +### checks.json Format + +```json +[ + { + "id": "mcp-client-initialization", + "name": "MCPClientInitialization", + "description": "Validates that MCP client properly initializes with server", + "status": "SUCCESS", + "timestamp": "2024-10-29T14:30:00.000Z", + "specReferences": [ + { + "id": "MCP-Lifecycle", + "url": "https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle" + } + ], + "details": { ... } + } +] +``` + +## Future Enhancements + +- **Test Suites** - Group multiple scenarios for convenience +- **Configurable Timeouts** - Per-scenario timeout configuration +- **Parallel Execution** - Run multiple scenarios concurrently +- **Custom Servers** - Allow custom MCP server implementations for advanced scenarios +- **Report Formats** - HTML, Markdown, or other human-readable formats diff --git a/src/runner/index.ts b/src/runner/index.ts new file mode 100644 index 0000000..baa633b --- /dev/null +++ b/src/runner/index.ts @@ -0,0 +1,188 @@ +import { spawn } from 'child_process'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { ConformanceCheck } from '../types.js'; +import { getScenario } from '../scenarios/index.js'; + +export interface ClientExecutionResult { + exitCode: number; + stdout: string; + stderr: string; + timedOut: boolean; +} + +async function ensureResultsDir(): Promise { + const resultsDir = path.join(process.cwd(), 'results'); + await fs.mkdir(resultsDir, { recursive: true }); + return resultsDir; +} + +function createResultDir(scenario: string): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + return path.join('results', `${scenario}-${timestamp}`); +} + +async function executeClient( + command: string, + serverUrl: string, + timeout: number = 30000 +): Promise { + const commandParts = command.split(' '); + const executable = commandParts[0]; + const args = [...commandParts.slice(1), serverUrl]; + + let stdout = ''; + let stderr = ''; + let timedOut = false; + + return new Promise((resolve) => { + const process = spawn(executable, args, { + shell: true, + stdio: 'pipe' + }); + + const timeoutHandle = setTimeout(() => { + timedOut = true; + process.kill(); + }, timeout); + + if (process.stdout) { + process.stdout.on('data', (data) => { + stdout += data.toString(); + }); + } + + if (process.stderr) { + process.stderr.on('data', (data) => { + stderr += data.toString(); + }); + } + + process.on('close', (code) => { + clearTimeout(timeoutHandle); + resolve({ + exitCode: code || 0, + stdout, + stderr, + timedOut + }); + }); + + process.on('error', (error) => { + clearTimeout(timeoutHandle); + resolve({ + exitCode: -1, + stdout, + stderr: stderr + `\nProcess error: ${error.message}`, + timedOut + }); + }); + }); +} + +export async function runConformanceTest( + clientCommand: string, + scenarioName: string, + timeout: number = 30000 +): Promise<{ + checks: ConformanceCheck[]; + clientOutput: ClientExecutionResult; + resultDir: string; +}> { + await ensureResultsDir(); + const resultDir = createResultDir(scenarioName); + await fs.mkdir(resultDir, { recursive: true }); + + const scenario = getScenario(scenarioName); + if (!scenario) { + throw new Error(`Unknown scenario: ${scenarioName}`); + } + + console.log(`Starting scenario: ${scenarioName}`); + const urls = await scenario.start(); + + console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`); + + try { + const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout); + + const checks = scenario.getChecks(); + + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); + + await fs.writeFile( + path.join(resultDir, 'stdout.txt'), + clientOutput.stdout + ); + + await fs.writeFile( + path.join(resultDir, 'stderr.txt'), + clientOutput.stderr + ); + + console.log(`Results saved to ${resultDir}`); + + return { + checks, + clientOutput, + resultDir + }; + } finally { + await scenario.stop(); + } +} + +async function main(): Promise { + const args = process.argv.slice(2); + let command: string | null = null; + let scenario: string | null = null; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--command' && i + 1 < args.length) { + command = args[i + 1]; + i++; + } else if (args[i] === '--scenario' && i + 1 < args.length) { + scenario = args[i + 1]; + i++; + } + } + + if (!command || !scenario) { + console.error('Usage: runner --command "" --scenario '); + console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize'); + process.exit(1); + } + + try { + const result = await runConformanceTest(command, scenario); + + const passed = result.checks.filter(c => c.status === 'SUCCESS').length; + const failed = result.checks.filter(c => c.status === 'FAILURE').length; + + console.log(`\nTest Results:`); + console.log(`Passed: ${passed}/${result.checks.length}`); + console.log(`Failed: ${failed}/${result.checks.length}`); + + if (failed > 0) { + console.log('\nFailed Checks:'); + result.checks + .filter(c => c.status === 'FAILURE') + .forEach(c => { + console.log(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.log(` Error: ${c.errorMessage}`); + } + }); + } + + process.exit(failed > 0 ? 1 : 0); + } catch (error) { + console.error('Test runner error:', error); + process.exit(1); + } +} + +main(); diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts new file mode 100644 index 0000000..6311e6d --- /dev/null +++ b/src/scenarios/index.ts @@ -0,0 +1,15 @@ +import { Scenario } from '../types.js'; + +export const scenarios = new Map(); + +export function registerScenario(name: string, scenario: Scenario): void { + scenarios.set(name, scenario); +} + +export function getScenario(name: string): Scenario | undefined { + return scenarios.get(name); +} + +export function listScenarios(): string[] { + return Array.from(scenarios.keys()); +} diff --git a/src/scenarios/initialize/TODO.md b/src/scenarios/initialize/TODO.md new file mode 100644 index 0000000..ae0ac1d --- /dev/null +++ b/src/scenarios/initialize/TODO.md @@ -0,0 +1,203 @@ +# Initialize Scenario Implementation TODO + +## Overview + +The `initialize` scenario tests the MCP client initialization handshake according to the MCP specification lifecycle requirements. + +## Architecture + +### Scenario Interface + +Each scenario provides server setup/teardown and result retrieval: + +```typescript +interface Scenario { + name: string; + description: string; + + // Start test server(s), return URLs + start(): Promise; + + // Stop test server(s) and cleanup + stop(): Promise; + + // Get conformance check results from server + getChecks(): ConformanceCheck[]; +} + +interface ScenarioUrls { + serverUrl: string; // MCP server URL (required) + authUrl?: string; // Auth server URL (optional, for auth scenarios) +} +``` + +### Flow Control (Runner's Responsibility) + +The runner orchestrates: +1. **Start Scenario**: Call `scenario.start()` to get server URL(s) +2. **Execute Client**: Spawn client process with server URL appended +3. **Capture Output**: Collect stdout/stderr from client +4. **Wait for Completion**: Wait for client to exit (or timeout) +5. **Get Results**: Call `scenario.getChecks()` to retrieve conformance checks +6. **Stop Scenario**: Call `scenario.stop()` to shutdown server(s) +7. **Write Results**: Save checks, stdout, stderr to `results/-/` + +### Server Responsibilities + +The test server: +- Captures MCP protocol interactions +- **Runs conformance checks internally** as interactions occur +- Stores checks for retrieval via `getChecks()` +- Supports clean shutdown + +## Implementation Tasks + +### 1. Create Test Server (`server.ts`) + +- [ ] Implement `InitializeTestServer` class with: + - `constructor()` - set up server + - `start(): Promise` - start listening on random port, return port number + - `stop(): Promise` - close server and connections + - `getChecks(): ConformanceCheck[]` - return conformance check results + +- [ ] Server behavior: + - Accept HTTP connections (dummy for now, MCP later) + - Capture the `initialize` request from client + - **Run conformance checks immediately** using existing check functions: + - `createClientInitializationCheck(initializeRequest)` + - Store checks in internal array + - Respond with valid `initialize` response: + - `protocolVersion: "2025-06-18"` + - `serverInfo: { name: "test-server", version: "1.0.0" }` + - `capabilities: {}` + - After response, optionally run server info check: + - `createServerInfoCheck(serverInfo)` (validates we provided proper info) + +### 2. Create Scenario Class (`index.ts`) + +- [ ] Implement `InitializeScenario` class: + ```typescript + export class InitializeScenario implements Scenario { + name = 'initialize'; + description = 'Tests MCP client initialization handshake'; + + private server: InitializeTestServer | null = null; + + async start(): Promise { + this.server = new InitializeTestServer(); + const port = await this.server.start(); + return { + serverUrl: `http://localhost:${port}` + }; + } + + async stop(): Promise { + if (this.server) { + await this.server.stop(); + this.server = null; + } + } + + getChecks(): ConformanceCheck[] { + if (!this.server) { + return []; + } + return this.server.getChecks(); + } + } + ``` + +- [ ] Export singleton instance: `export const initializeScenario = new InitializeScenario();` + +### 3. Scenario Registry (`src/scenarios/index.ts`) + +- [ ] Create scenario registry: + ```typescript + import { initializeScenario } from './initialize/index.js'; + + export const scenarios = new Map([ + ['initialize', initializeScenario], + // future scenarios here + ]); + + export function getScenario(name: string): Scenario | undefined { + return scenarios.get(name); + } + ``` + +## File Structure + +``` +src/scenarios/ +├── index.ts (scenario registry/map) +└── initialize/ + ├── TODO.md (this file - delete when complete) + ├── index.ts (InitializeScenario class) + └── server.ts (InitializeTestServer class) +``` + +## Example Usage + +From runner: + +```typescript +import { getScenario } from './scenarios/index.js'; + +const scenario = getScenario('initialize'); +if (!scenario) { + throw new Error('Unknown scenario: initialize'); +} + +// Start server +const urls = await scenario.start(); + +try { + // Execute client (runner's responsibility) + const clientOutput = await executeClient(clientCommand, urls.serverUrl); + + // Get checks from scenario + const checks = scenario.getChecks(); + + // Write results (runner's responsibility) + await writeResults(outputDir, { checks, ...clientOutput }); +} finally { + // Stop server + await scenario.stop(); +} +``` + +## Conformance Checks + +The server runs these checks (from `src/checks.ts`): +1. `createClientInitializationCheck(initializeRequest)` - validates client's initialize request +2. `createServerInfoCheck(serverInfo)` - validates server info was provided (optional) + +Additional checks to consider: +- Client exit code validation (runner handles this) +- Connection established successfully +- No protocol errors + +## Dependencies + +- `http` for HTTP server (dummy for prototyping) +- Existing check functions from `src/checks.ts` + +Note: The runner handles `child_process.spawn` and `fs/promises` for file writing. + +## Testing + +After implementation: +- [ ] Test with example client: `tsx examples/clients/typescript/test1.ts` +- [ ] Verify checks array is populated correctly +- [ ] Verify server starts and stops cleanly +- [ ] Test with broken client to ensure checks catch issues +- [ ] Use runner to execute full test flow + +## Notes + +- **Runner executes the client**, not the scenario +- **Scenario only manages server lifecycle** (start/stop) and provides results (getChecks) +- Server does checks internally and stores them +- Clean shutdown is critical - runner should use try/finally for scenario.stop() +- Start with dummy HTTP server, upgrade to MCP later +- For auth scenarios, return both `serverUrl` and `authUrl` from `start()` diff --git a/src/types.ts b/src/types.ts index b4b764f..869b920 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,3 +17,16 @@ export interface ConformanceCheck { errorMessage?: string; logs?: string[]; } + +export interface ScenarioUrls { + serverUrl: string; + authUrl?: string; +} + +export interface Scenario { + name: string; + description: string; + start(): Promise; + stop(): Promise; + getChecks(): ConformanceCheck[]; +}