diff --git a/README.md b/README.md new file mode 100644 index 0000000..0ac72b6 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# MCP Conformance Test Framework + +A framework for testing MCP (Model Context Protocol) client implementations against the specification. + +## Quick Start + +```bash +npm install +npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario initialize +``` + +## Overview + +The conformance test framework validates MCP client implementations by: +1. Starting a test server for the specified scenario +2. Running the client implementation with the test server URL +3. Capturing MCP protocol interactions +4. Running conformance checks against the specification +5. Generating detailed test results + +## Usage + +```bash +npm run start -- --command "" --scenario +``` + +- `--command` - The command to run your MCP client (can include flags) +- `--scenario` - The test scenario to run (e.g., "initialize") + +The framework appends the server URL as the final argument to your command. + +## Test Results + +Results are saved to `results/-/`: +- `checks.json` - Array of conformance check results with pass/fail status +- `stdout.txt` - Client stdout output +- `stderr.txt` - Client stderr output + +## Example Clients + +- `examples/clients/typescript/test1.ts` - Valid MCP client (passes all checks) +- `examples/clients/typescript/test-broken.ts` - Invalid client missing required fields (fails checks) + +## Available Scenarios + +- **initialize** - Tests MCP client initialization handshake + - Validates protocol version + - Validates clientInfo (name and version) + - Validates server response handling + +## Architecture + +See `src/runner/DESIGN.md` for detailed architecture documentation. + +### Key Components + +- **Runner** (`src/runner/`) - Orchestrates test execution and result generation +- **Scenarios** (`src/scenarios/`) - Test scenarios with custom server implementations +- **Checks** (`src/checks.ts`) - Conformance validation functions +- **Types** (`src/types.ts`) - Shared type definitions + +## Adding New Scenarios + +1. Create a new directory in `src/scenarios//` +2. Implement the `Scenario` interface with `start()`, `stop()`, and `getChecks()` +3. Register the scenario in `src/scenarios/index.ts` + +See `src/scenarios/initialize/` for a reference implementation. diff --git a/examples/clients/typescript/test-broken.ts b/examples/clients/typescript/test-broken.ts new file mode 100644 index 0000000..4a9825e --- /dev/null +++ b/examples/clients/typescript/test-broken.ts @@ -0,0 +1,59 @@ +#!/usr/bin/env node + +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; + +async function main(): Promise { + const serverUrl = process.argv[2]; + + if (!serverUrl) { + console.error('Usage: test-broken-client '); + process.exit(1); + } + + console.log(`Connecting to MCP server at: ${serverUrl}`); + + try { + const transport = new StreamableHTTPClientTransport( + new URL(serverUrl) + ); + + await transport.start(); + + const initRequest = { + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + clientInfo: { + version: '1.0.0' + }, + capabilities: {} + } + }; + + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(initRequest) + }); + + const result = await response.json(); + console.log('✅ Received response:', JSON.stringify(result, null, 2)); + + await transport.close(); + console.log('✅ Connection closed'); + + process.exit(0); + } catch (error) { + console.error('❌ Error:', error); + process.exit(1); + } +} + +main().catch((error) => { + console.error('Unhandled error:', error); + process.exit(1); +}); diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 1ff7b4c..b3f83bb 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,8 +1,8 @@ import { Scenario } from '../types.js'; -import { initializeScenario } from './initialize/index.js'; +import { InitializeScenario } from './initialize.js'; export const scenarios = new Map([ - ['initialize', initializeScenario], + ['initialize', new InitializeScenario()], ]); export function registerScenario(name: string, scenario: Scenario): void { diff --git a/src/scenarios/initialize/server.ts b/src/scenarios/initialize.ts similarity index 89% rename from src/scenarios/initialize/server.ts rename to src/scenarios/initialize.ts index fca2a46..7f1a93e 100644 --- a/src/scenarios/initialize/server.ts +++ b/src/scenarios/initialize.ts @@ -1,13 +1,16 @@ import http from 'http'; -import { ConformanceCheck } from '../../types.js'; -import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js'; +import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js'; +import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js'; + +export class InitializeScenario implements Scenario { + name = 'initialize'; + description = 'Tests MCP client initialization handshake'; -export class InitializeTestServer { private server: http.Server | null = null; private checks: ConformanceCheck[] = []; private port: number = 0; - async start(): Promise { + async start(): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); @@ -19,7 +22,9 @@ export class InitializeTestServer { const address = this.server!.address(); if (address && typeof address === 'object') { this.port = address.port; - resolve(this.port); + resolve({ + serverUrl: `http://localhost:${this.port}` + }); } else { reject(new Error('Failed to get server address')); } diff --git a/src/scenarios/initialize/index.ts b/src/scenarios/initialize/index.ts deleted file mode 100644 index ed36249..0000000 --- a/src/scenarios/initialize/index.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; -import { InitializeTestServer } from './server.js'; - -export class InitializeScenario implements Scenario { - name = 'initialize'; - description = 'Tests MCP client initialization handshake'; - - private server: InitializeTestServer | null = null; - - async start(): Promise { - this.server = new InitializeTestServer(); - const port = await this.server.start(); - return { - serverUrl: `http://localhost:${port}` - }; - } - - async stop(): Promise { - if (this.server) { - await this.server.stop(); - this.server = null; - } - } - - getChecks(): ConformanceCheck[] { - if (!this.server) { - return []; - } - return this.server.getChecks(); - } -} - -export const initializeScenario = new InitializeScenario();