readme

README.md

@@ -0,0 +1,68 @@
+# MCP Conformance Test Framework
+
+A framework for testing MCP (Model Context Protocol) client implementations against the specification.
+
+## Quick Start
+
+```bash
+npm install
+npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario initialize
+```
+
+## Overview
+
+The conformance test framework validates MCP client implementations by:
+1. Starting a test server for the specified scenario
+2. Running the client implementation with the test server URL
+3. Capturing MCP protocol interactions
+4. Running conformance checks against the specification
+5. Generating detailed test results
+
+## Usage
+
+```bash
+npm run start -- --command "<client-command>" --scenario <scenario-name>
+```
+
+- `--command` - The command to run your MCP client (can include flags)
+- `--scenario` - The test scenario to run (e.g., "initialize")
+
+The framework appends the server URL as the final argument to your command.
+
+## Test Results
+
+Results are saved to `results/<scenario>-<timestamp>/`:
+- `checks.json` - Array of conformance check results with pass/fail status
+- `stdout.txt` - Client stdout output
+- `stderr.txt` - Client stderr output
+
+## Example Clients
+
+- `examples/clients/typescript/test1.ts` - Valid MCP client (passes all checks)
+- `examples/clients/typescript/test-broken.ts` - Invalid client missing required fields (fails checks)
+
+## Available Scenarios
+
+- **initialize** - Tests MCP client initialization handshake
+  - Validates protocol version
+  - Validates clientInfo (name and version)
+  - Validates server response handling
+
+## Architecture
+
+See `src/runner/DESIGN.md` for detailed architecture documentation.
+
+### Key Components
+
+- **Runner** (`src/runner/`) - Orchestrates test execution and result generation
+- **Scenarios** (`src/scenarios/`) - Test scenarios with custom server implementations
+- **Checks** (`src/checks.ts`) - Conformance validation functions
+- **Types** (`src/types.ts`) - Shared type definitions
+
+## Adding New Scenarios
+
+1. Create a new directory in `src/scenarios/<scenario-name>/`
+2. Implement the `Scenario` interface with `start()`, `stop()`, and `getChecks()`
+3. Register the scenario in `src/scenarios/index.ts`
+
+See `src/scenarios/initialize/` for a reference implementation.

examples/clients/typescript/test-broken.ts

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+
+async function main(): Promise<void> {
+  const serverUrl = process.argv[2];
+
+  if (!serverUrl) {
+    console.error('Usage: test-broken-client <server-url>');
+    process.exit(1);
+  }
+
+  console.log(`Connecting to MCP server at: ${serverUrl}`);
+
+  try {
+    const transport = new StreamableHTTPClientTransport(
+      new URL(serverUrl)
+    );
+
+    await transport.start();
+
+    const initRequest = {
+      jsonrpc: '2.0',
+      id: 1,
+      method: 'initialize',
+      params: {
+        protocolVersion: '2025-06-18',
+        clientInfo: {
+          version: '1.0.0'
+        },
+        capabilities: {}
+      }
+    };
+
+    const response = await fetch(serverUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(initRequest)
+    });
+
+    const result = await response.json();
+    console.log('✅ Received response:', JSON.stringify(result, null, 2));
+
+    await transport.close();
+    console.log('✅ Connection closed');
+
+    process.exit(0);
+  } catch (error) {
+    console.error('❌ Error:', error);
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error('Unhandled error:', error);
+  process.exit(1);
+});

src/scenarios/index.ts

@@ -1,8 +1,8 @@
 import { Scenario } from '../types.js';
-import { initializeScenario } from './initialize/index.js';
+import { InitializeScenario } from './initialize.js';
 
 export const scenarios = new Map<string, Scenario>([
-  ['initialize', initializeScenario],
+  ['initialize', new InitializeScenario()],
 ]);
 
 export function registerScenario(name: string, scenario: Scenario): void {

src/scenarios/initialize.ts

@@ -0,0 +1,132 @@
+import http from 'http';
+import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js';
+import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js';
+
+export class InitializeScenario implements Scenario {
+  name = 'initialize';
+  description = 'Tests MCP client initialization handshake';
+
+  private server: http.Server | null = null;
+  private checks: ConformanceCheck[] = [];
+  private port: number = 0;
+
+  async start(): Promise<ScenarioUrls> {
+    return new Promise((resolve, reject) => {
+      this.server = http.createServer((req, res) => {
+        this.handleRequest(req, res);
+      });
+
+      this.server.on('error', reject);
+
+      this.server.listen(0, () => {
+        const address = this.server!.address();
+        if (address && typeof address === 'object') {
+          this.port = address.port;
+          resolve({
+            serverUrl: `http://localhost:${this.port}`
+          });
+        } else {
+          reject(new Error('Failed to get server address'));
+        }
+      });
+    });
+  }
+
+  async stop(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      if (this.server) {
+        this.server.close((err) => {
+          if (err) {
+            reject(err);
+          } else {
+            this.server = null;
+            resolve();
+          }
+        });
+      } else {
+        resolve();
+      }
+    });
+  }
+
+  getChecks(): ConformanceCheck[] {
+    return this.checks;
+  }
+
+  private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
+    let body = '';
+
+    req.on('data', (chunk) => {
+      body += chunk.toString();
+    });
+
+    req.on('end', () => {
+      try {
+        const request = JSON.parse(body);
+
+        if (request.method === 'initialize') {
+          this.handleInitialize(request, res);
+        } else if (request.method === 'tools/list') {
+          this.handleToolsList(request, res);
+        } else {
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({
+            jsonrpc: '2.0',
+            id: request.id,
+            result: {}
+          }));
+        }
+      } catch (error) {
+        res.writeHead(400, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+          jsonrpc: '2.0',
+          error: {
+            code: -32700,
+            message: 'Parse error'
+          }
+        }));
+      }
+    });
+  }
+
+  private handleInitialize(request: any, res: http.ServerResponse): void {
+    const initializeRequest = request.params;
+
+    const check = createClientInitializationCheck(initializeRequest);
+    this.checks.push(check);
+
+    const serverInfo = {
+      name: 'test-server',
+      version: '1.0.0'
+    };
+
+    const serverInfoCheck = createServerInfoCheck(serverInfo);
+    this.checks.push(serverInfoCheck);
+
+    const response = {
+      jsonrpc: '2.0',
+      id: request.id,
+      result: {
+        protocolVersion: '2025-06-18',
+        serverInfo,
+        capabilities: {}
+      }
+    };
+
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(response));
+  }
+
+  private handleToolsList(request: any, res: http.ServerResponse): void {
+    const response = {
+      jsonrpc: '2.0',
+      id: request.id,
+      result: {
+        tools: []
+      }
+    };
+
+    res.writeHead(200, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(response));
+  }
+}

src/scenarios/initialize/index.ts

@@ -1,33 +0,0 @@
-import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js';
-import { InitializeTestServer } from './server.js';
-
-export class InitializeScenario implements Scenario {
-  name = 'initialize';
-  description = 'Tests MCP client initialization handshake';
-
-  private server: InitializeTestServer | null = null;
-
-  async start(): Promise<ScenarioUrls> {
-    this.server = new InitializeTestServer();
-    const port = await this.server.start();
-    return {
-      serverUrl: `http://localhost:${port}`
-    };
-  }
-
-  async stop(): Promise<void> {
-    if (this.server) {
-      await this.server.stop();
-      this.server = null;
-    }
-  }
-
-  getChecks(): ConformanceCheck[] {
-    if (!this.server) {
-      return [];
-    }
-    return this.server.getChecks();
-  }
-}
-
-export const initializeScenario = new InitializeScenario();