wip first crack at scenario and runner

.gitignore

@@ -1 +1,2 @@
 node_modules
+results/

package-lock.json

@@ -8,7 +8,9 @@
       "name": "conformance",
       "version": "1.0.0",
       "dependencies": {
-        "@modelcontextprotocol/sdk": "^1.20.1"
+        "@modelcontextprotocol/sdk": "^1.20.1",
+        "express": "^5.1.0",
+        "zod": "^3.25.76"
       },
       "devDependencies": {
         "@jest/globals": "^29.7.0",

package.json

@@ -15,6 +15,8 @@
     "typescript": "^5.3.0"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.20.1"
+    "@modelcontextprotocol/sdk": "^1.20.1",
+    "express": "^5.1.0",
+    "zod": "^3.25.76"
   }
 }

src/runner.ts

@@ -1,19 +1,4 @@
-import { createClientInitializationCheck, createServerInfoCheck } from './checks.ts';
+import { runConformanceTest } from './runner/index.js';
 
-const initializeRequest = {
+export { runConformanceTest };
-  protocolVersion: '2025-06-18',
+export type { ScenarioConfig } from './runner/index.js';
-  clientInfo: {
-    name: 'ExampleClient',
-    version: '1.0.0'
-  }
-};
-
-const checks = [
-  createClientInitializationCheck(initializeRequest),
-  createServerInfoCheck({
-    name: 'ExampleMCPServer',
-    version: '1.0.0'
-  })
-];
-
-console.log(JSON.stringify(checks, null, 2));

src/runner/DESIGN.md

@@ -0,0 +1,91 @@
+# Conformance Test Runner Design
+
+## Overview
+
+The conformance test runner is a framework for testing MCP client implementations against the MCP specification. It executes client implementations in controlled scenarios and validates their behavior through a series of conformance checks.
+
+## Architecture
+
+### Components
+
+1. **Runner** - Orchestrates test execution
+2. **Test Server** - MCP server (and optionally auth server for auth scenarios)
+3. **Client Process** - The MCP client implementation under test
+4. **Scenario** - A specific test case with expected behaviors
+5. **Checks** - Validation functions that produce ConformanceCheck results
+
+### Execution Flow
+
+```
+1. Runner starts test server(s) on available port(s)
+2. Runner spawns client process with server URL as final argument
+3. Server captures MCP interactions
+4. Runner captures client stdout/stderr
+5. Scenario-specific checks are executed
+6. Results are written to results/<scenario>-<timestamp>/
+```
+
+## CLI Interface
+
+```bash
+npm run start -- --command "tsx examples/clients/typescript/test1.ts --verbose" --scenario initialize
+```
+
+### Arguments
+
+- `--command` - The command to run the client (can include existing flags)
+- `--scenario` - The scenario to test (e.g., "initialize", "list-tools", "call-tool")
+
+The runner will append the server URL as the final argument to the command.
+
+## Scenarios
+
+A scenario represents a specific test case that validates one or more aspects of MCP client behavior. Each scenario:
+
+- Configures the test server with expected behavior
+- May run multiple conformance checks
+- Validates both client behavior and server responses
+
+Example scenarios:
+- `initialize` - Tests client initialization handshake
+- `list-tools` - Tests tool discovery
+- `call-tool` - Tests tool invocation
+- `auth-flow` - Tests OAuth/authorization flows (requires auth server)
+
+## Output Structure
+
+Results are written to: `results/<scenario>-<timestamp>/`
+
+Files:
+- `checks.json` - Array of ConformanceCheck objects with validation results
+- `stdout.txt` - Complete stdout from the client process
+- `stderr.txt` - Complete stderr from the client process
+
+### checks.json Format
+
+```json
+[
+  {
+    "id": "mcp-client-initialization",
+    "name": "MCPClientInitialization",
+    "description": "Validates that MCP client properly initializes with server",
+    "status": "SUCCESS",
+    "timestamp": "2024-10-29T14:30:00.000Z",
+    "specReferences": [
+      {
+        "id": "MCP-Lifecycle",
+        "url": "https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle"
+      }
+    ],
+    "details": { ... }
+  }
+]
+```
+
+## Future Enhancements
+
+- **Test Suites** - Group multiple scenarios for convenience
+- **Configurable Timeouts** - Per-scenario timeout configuration
+- **Parallel Execution** - Run multiple scenarios concurrently
+- **Custom Servers** - Allow custom MCP server implementations for advanced scenarios
+- **Report Formats** - HTML, Markdown, or other human-readable formats

src/runner/index.ts

@@ -0,0 +1,188 @@
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+import path from 'path';
+import { ConformanceCheck } from '../types.js';
+import { getScenario } from '../scenarios/index.js';
+
+export interface ClientExecutionResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  timedOut: boolean;
+}
+
+async function ensureResultsDir(): Promise<string> {
+  const resultsDir = path.join(process.cwd(), 'results');
+  await fs.mkdir(resultsDir, { recursive: true });
+  return resultsDir;
+}
+
+function createResultDir(scenario: string): string {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  return path.join('results', `${scenario}-${timestamp}`);
+}
+
+async function executeClient(
+  command: string,
+  serverUrl: string,
+  timeout: number = 30000
+): Promise<ClientExecutionResult> {
+  const commandParts = command.split(' ');
+  const executable = commandParts[0];
+  const args = [...commandParts.slice(1), serverUrl];
+
+  let stdout = '';
+  let stderr = '';
+  let timedOut = false;
+
+  return new Promise((resolve) => {
+    const process = spawn(executable, args, {
+      shell: true,
+      stdio: 'pipe'
+    });
+
+    const timeoutHandle = setTimeout(() => {
+      timedOut = true;
+      process.kill();
+    }, timeout);
+
+    if (process.stdout) {
+      process.stdout.on('data', (data) => {
+        stdout += data.toString();
+      });
+    }
+
+    if (process.stderr) {
+      process.stderr.on('data', (data) => {
+        stderr += data.toString();
+      });
+    }
+
+    process.on('close', (code) => {
+      clearTimeout(timeoutHandle);
+      resolve({
+        exitCode: code || 0,
+        stdout,
+        stderr,
+        timedOut
+      });
+    });
+
+    process.on('error', (error) => {
+      clearTimeout(timeoutHandle);
+      resolve({
+        exitCode: -1,
+        stdout,
+        stderr: stderr + `\nProcess error: ${error.message}`,
+        timedOut
+      });
+    });
+  });
+}
+
+export async function runConformanceTest(
+  clientCommand: string,
+  scenarioName: string,
+  timeout: number = 30000
+): Promise<{
+  checks: ConformanceCheck[];
+  clientOutput: ClientExecutionResult;
+  resultDir: string;
+}> {
+  await ensureResultsDir();
+  const resultDir = createResultDir(scenarioName);
+  await fs.mkdir(resultDir, { recursive: true });
+
+  const scenario = getScenario(scenarioName);
+  if (!scenario) {
+    throw new Error(`Unknown scenario: ${scenarioName}`);
+  }
+
+  console.log(`Starting scenario: ${scenarioName}`);
+  const urls = await scenario.start();
+
+  console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
+
+  try {
+    const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout);
+
+    const checks = scenario.getChecks();
+
+    await fs.writeFile(
+      path.join(resultDir, 'checks.json'),
+      JSON.stringify(checks, null, 2)
+    );
+
+    await fs.writeFile(
+      path.join(resultDir, 'stdout.txt'),
+      clientOutput.stdout
+    );
+
+    await fs.writeFile(
+      path.join(resultDir, 'stderr.txt'),
+      clientOutput.stderr
+    );
+
+    console.log(`Results saved to ${resultDir}`);
+
+    return {
+      checks,
+      clientOutput,
+      resultDir
+    };
+  } finally {
+    await scenario.stop();
+  }
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+  let command: string | null = null;
+  let scenario: string | null = null;
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--command' && i + 1 < args.length) {
+      command = args[i + 1];
+      i++;
+    } else if (args[i] === '--scenario' && i + 1 < args.length) {
+      scenario = args[i + 1];
+      i++;
+    }
+  }
+
+  if (!command || !scenario) {
+    console.error('Usage: runner --command "<command>" --scenario <scenario>');
+    console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
+    process.exit(1);
+  }
+
+  try {
+    const result = await runConformanceTest(command, scenario);
+
+    const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
+    const failed = result.checks.filter(c => c.status === 'FAILURE').length;
+
+    console.log(`\nTest Results:`);
+    console.log(`Passed: ${passed}/${result.checks.length}`);
+    console.log(`Failed: ${failed}/${result.checks.length}`);
+
+    if (failed > 0) {
+      console.log('\nFailed Checks:');
+      result.checks
+        .filter(c => c.status === 'FAILURE')
+        .forEach(c => {
+          console.log(`  - ${c.name}: ${c.description}`);
+          if (c.errorMessage) {
+            console.log(`    Error: ${c.errorMessage}`);
+          }
+        });
+    }
+
+    process.exit(failed > 0 ? 1 : 0);
+  } catch (error) {
+    console.error('Test runner error:', error);
+    process.exit(1);
+  }
+}
+
+main();

src/scenarios/index.ts

@@ -0,0 +1,15 @@
+import { Scenario } from '../types.js';
+
+export const scenarios = new Map<string, Scenario>();
+
+export function registerScenario(name: string, scenario: Scenario): void {
+  scenarios.set(name, scenario);
+}
+
+export function getScenario(name: string): Scenario | undefined {
+  return scenarios.get(name);
+}
+
+export function listScenarios(): string[] {
+  return Array.from(scenarios.keys());
+}

src/scenarios/initialize/TODO.md

@@ -0,0 +1,203 @@
+# Initialize Scenario Implementation TODO
+
+## Overview
+
+The `initialize` scenario tests the MCP client initialization handshake according to the MCP specification lifecycle requirements.
+
+## Architecture
+
+### Scenario Interface
+
+Each scenario provides server setup/teardown and result retrieval:
+
+```typescript
+interface Scenario {
+  name: string;
+  description: string;
+  
+  // Start test server(s), return URLs
+  start(): Promise<ScenarioUrls>;
+  
+  // Stop test server(s) and cleanup
+  stop(): Promise<void>;
+  
+  // Get conformance check results from server
+  getChecks(): ConformanceCheck[];
+}
+
+interface ScenarioUrls {
+  serverUrl: string;      // MCP server URL (required)
+  authUrl?: string;       // Auth server URL (optional, for auth scenarios)
+}
+```
+
+### Flow Control (Runner's Responsibility)
+
+The runner orchestrates:
+1. **Start Scenario**: Call `scenario.start()` to get server URL(s)
+2. **Execute Client**: Spawn client process with server URL appended
+3. **Capture Output**: Collect stdout/stderr from client
+4. **Wait for Completion**: Wait for client to exit (or timeout)
+5. **Get Results**: Call `scenario.getChecks()` to retrieve conformance checks
+6. **Stop Scenario**: Call `scenario.stop()` to shutdown server(s)
+7. **Write Results**: Save checks, stdout, stderr to `results/<scenario>-<timestamp>/`
+
+### Server Responsibilities
+
+The test server:
+- Captures MCP protocol interactions
+- **Runs conformance checks internally** as interactions occur
+- Stores checks for retrieval via `getChecks()`
+- Supports clean shutdown
+
+## Implementation Tasks
+
+### 1. Create Test Server (`server.ts`)
+
+- [ ] Implement `InitializeTestServer` class with:
+  - `constructor()` - set up server
+  - `start(): Promise<number>` - start listening on random port, return port number
+  - `stop(): Promise<void>` - close server and connections
+  - `getChecks(): ConformanceCheck[]` - return conformance check results
+  
+- [ ] Server behavior:
+  - Accept HTTP connections (dummy for now, MCP later)
+  - Capture the `initialize` request from client
+  - **Run conformance checks immediately** using existing check functions:
+    - `createClientInitializationCheck(initializeRequest)`
+    - Store checks in internal array
+  - Respond with valid `initialize` response:
+    - `protocolVersion: "2025-06-18"`
+    - `serverInfo: { name: "test-server", version: "1.0.0" }`
+    - `capabilities: {}`
+  - After response, optionally run server info check:
+    - `createServerInfoCheck(serverInfo)` (validates we provided proper info)
+
+### 2. Create Scenario Class (`index.ts`)
+
+- [ ] Implement `InitializeScenario` class:
+  ```typescript
+  export class InitializeScenario implements Scenario {
+    name = 'initialize';
+    description = 'Tests MCP client initialization handshake';
+    
+    private server: InitializeTestServer | null = null;
+    
+    async start(): Promise<ScenarioUrls> {
+      this.server = new InitializeTestServer();
+      const port = await this.server.start();
+      return {
+        serverUrl: `http://localhost:${port}`
+      };
+    }
+    
+    async stop(): Promise<void> {
+      if (this.server) {
+        await this.server.stop();
+        this.server = null;
+      }
+    }
+    
+    getChecks(): ConformanceCheck[] {
+      if (!this.server) {
+        return [];
+      }
+      return this.server.getChecks();
+    }
+  }
+  ```
+
+- [ ] Export singleton instance: `export const initializeScenario = new InitializeScenario();`
+
+### 3. Scenario Registry (`src/scenarios/index.ts`)
+
+- [ ] Create scenario registry:
+  ```typescript
+  import { initializeScenario } from './initialize/index.js';
+  
+  export const scenarios = new Map<string, Scenario>([
+    ['initialize', initializeScenario],
+    // future scenarios here
+  ]);
+  
+  export function getScenario(name: string): Scenario | undefined {
+    return scenarios.get(name);
+  }
+  ```
+
+## File Structure
+
+```
+src/scenarios/
+├── index.ts (scenario registry/map)
+└── initialize/
+    ├── TODO.md (this file - delete when complete)
+    ├── index.ts (InitializeScenario class)
+    └── server.ts (InitializeTestServer class)
+```
+
+## Example Usage
+
+From runner:
+
+```typescript
+import { getScenario } from './scenarios/index.js';
+
+const scenario = getScenario('initialize');
+if (!scenario) {
+  throw new Error('Unknown scenario: initialize');
+}
+
+// Start server
+const urls = await scenario.start();
+
+try {
+  // Execute client (runner's responsibility)
+  const clientOutput = await executeClient(clientCommand, urls.serverUrl);
+  
+  // Get checks from scenario
+  const checks = scenario.getChecks();
+  
+  // Write results (runner's responsibility)
+  await writeResults(outputDir, { checks, ...clientOutput });
+} finally {
+  // Stop server
+  await scenario.stop();
+}
+```
+
+## Conformance Checks
+
+The server runs these checks (from `src/checks.ts`):
+1. `createClientInitializationCheck(initializeRequest)` - validates client's initialize request
+2. `createServerInfoCheck(serverInfo)` - validates server info was provided (optional)
+
+Additional checks to consider:
+- Client exit code validation (runner handles this)
+- Connection established successfully
+- No protocol errors
+
+## Dependencies
+
+- `http` for HTTP server (dummy for prototyping)
+- Existing check functions from `src/checks.ts`
+
+Note: The runner handles `child_process.spawn` and `fs/promises` for file writing.
+
+## Testing
+
+After implementation:
+- [ ] Test with example client: `tsx examples/clients/typescript/test1.ts`
+- [ ] Verify checks array is populated correctly
+- [ ] Verify server starts and stops cleanly
+- [ ] Test with broken client to ensure checks catch issues
+- [ ] Use runner to execute full test flow
+
+## Notes
+
+- **Runner executes the client**, not the scenario
+- **Scenario only manages server lifecycle** (start/stop) and provides results (getChecks)
+- Server does checks internally and stores them
+- Clean shutdown is critical - runner should use try/finally for scenario.stop()
+- Start with dummy HTTP server, upgrade to MCP later
+- For auth scenarios, return both `serverUrl` and `authUrl` from `start()`

src/types.ts

@@ -17,3 +17,16 @@ export interface ConformanceCheck {
   errorMessage?: string;
   logs?: string[];
 }
+
+export interface ScenarioUrls {
+  serverUrl: string;
+  authUrl?: string;
+}
+
+export interface Scenario {
+  name: string;
+  description: string;
+  start(): Promise<ScenarioUrls>;
+  stop(): Promise<void>;
+  getChecks(): ConformanceCheck[];
+}