initial runner for checking server conformance (#3)

* create some checks

* basic initialize working

* rename

src/checks/checks.test.ts

@@ -0,0 +1,84 @@
+import { createClientInitializationCheck } from './client';
+
+describe('createClientInitializationCheck', () => {
+    it('should return SUCCESS for a valid initialize request', () => {
+        const validRequest = {
+            protocolVersion: '2025-06-18',
+            clientInfo: {
+                name: 'TestClient',
+                version: '1.0.0'
+            }
+        };
+
+        const check = createClientInitializationCheck(validRequest);
+        expect(check.status).toBe('SUCCESS');
+        expect(check.errorMessage).toBeUndefined();
+    });
+
+    it('should return FAILURE when protocol version is missing', () => {
+        const invalidRequest = {
+            clientInfo: {
+                name: 'TestClient',
+                version: '1.0.0'
+            }
+        };
+
+        const check = createClientInitializationCheck(invalidRequest);
+        expect(check.status).toBe('FAILURE');
+        expect(check.errorMessage).toContain('Protocol version not provided');
+    });
+
+    it('should return FAILURE when protocol version does not match', () => {
+        const invalidRequest = {
+            protocolVersion: '2024-11-05',
+            clientInfo: {
+                name: 'TestClient',
+                version: '1.0.0'
+            }
+        };
+
+        const check = createClientInitializationCheck(invalidRequest);
+        expect(check.status).toBe('FAILURE');
+        expect(check.errorMessage).toContain('Version mismatch');
+    });
+
+    it('should return FAILURE when client name is missing', () => {
+        const invalidRequest = {
+            protocolVersion: '2025-06-18',
+            clientInfo: {
+                version: '1.0.0'
+            }
+        };
+
+        const check = createClientInitializationCheck(invalidRequest);
+        expect(check.status).toBe('FAILURE');
+        expect(check.errorMessage).toContain('Client name missing');
+    });
+
+    it('should return FAILURE when client version is missing', () => {
+        const invalidRequest = {
+            protocolVersion: '2025-06-18',
+            clientInfo: {
+                name: 'TestClient'
+            }
+        };
+
+        const check = createClientInitializationCheck(invalidRequest);
+        expect(check.status).toBe('FAILURE');
+        expect(check.errorMessage).toContain('Client version missing');
+    });
+
+    it('should support custom expected spec version', () => {
+        const request = {
+            protocolVersion: '2024-11-05',
+            clientInfo: {
+                name: 'TestClient',
+                version: '1.0.0'
+            }
+        };
+
+        const check = createClientInitializationCheck(request, '2024-11-05');
+        expect(check.status).toBe('SUCCESS');
+        expect(check.errorMessage).toBeUndefined();
+    });
+});

src/checks/client.ts

@@ -0,0 +1,47 @@
+import { ConformanceCheck, CheckStatus } from '../types.js';
+
+export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck {
+    return {
+        id: 'server-info',
+        name: 'ServerInfo',
+        description: 'Test server info returned to client',
+        status: 'INFO',
+        timestamp: new Date().toISOString(),
+        specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
+        details: {
+            serverName: serverInfo.name,
+            serverVersion: serverInfo.version
+        }
+    };
+}
+
+export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
+    const protocolVersionSent = initializeRequest?.protocolVersion;
+    const versionMatch = protocolVersionSent === expectedSpecVersion;
+
+    const errors: string[] = [];
+    if (!protocolVersionSent) errors.push('Protocol version not provided');
+    if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`);
+    if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing');
+    if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing');
+
+    const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
+
+    return {
+        id: 'mcp-client-initialization',
+        name: 'MCPClientInitialization',
+        description: 'Validates that MCP client properly initializes with server',
+        status,
+        timestamp: new Date().toISOString(),
+        specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
+        details: {
+            protocolVersionSent,
+            expectedSpecVersion,
+            versionMatch,
+            clientName: initializeRequest?.clientInfo?.name,
+            clientVersion: initializeRequest?.clientInfo?.version
+        },
+        errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
+        logs: errors.length > 0 ? errors : undefined
+    };
+}

src/checks/index.ts

@@ -0,0 +1,7 @@
+// Namespaced exports for client and server checks
+import * as client from './client.js';
+import * as server from './server.js';
+
+
+export const clientChecks = client;
+export const serverChecks = server;

src/checks/server.ts

@@ -0,0 +1,36 @@
+import { ConformanceCheck, CheckStatus } from '../types.js';
+
+export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
+    const result = initializeResponse?.result;
+    const protocolVersion = result?.protocolVersion;
+    const serverInfo = result?.serverInfo;
+    const capabilities = result?.capabilities;
+    
+    const errors: string[] = [];
+    if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field');
+    if (!initializeResponse?.id) errors.push('Missing id field');
+    if (!result) errors.push('Missing result field');
+    if (!protocolVersion) errors.push('Missing protocolVersion in result');
+    if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`);
+    if (!serverInfo) errors.push('Missing serverInfo in result');
+    if (!serverInfo?.name) errors.push('Missing server name in serverInfo');
+    if (!serverInfo?.version) errors.push('Missing server version in serverInfo');
+    if (capabilities === undefined) errors.push('Missing capabilities in result');
+
+    const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
+
+    return {
+        id: 'mcp-server-initialization',
+        name: 'MCPServerInitialization',
+        description: 'Validates that MCP server properly responds to initialize request',
+        status,
+        timestamp: new Date().toISOString(),
+        specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
+        details: {
+            expectedSpecVersion,
+            response: initializeResponse
+        },
+        errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
+        logs: errors.length > 0 ? errors : undefined
+    };
+}

src/scenarios/client/initialize.ts

@@ -1,6 +1,6 @@
 import http from 'http';
 import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js';
-import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js';
+import { clientChecks } from '../../checks/index.js';
 
 export class InitializeScenario implements Scenario {
     name = 'initialize';
@@ -96,7 +96,7 @@ export class InitializeScenario implements Scenario {
     private handleInitialize(request: any, res: http.ServerResponse): void {
         const initializeRequest = request.params;
 
-        const check = createClientInitializationCheck(initializeRequest);
+        const check = clientChecks.createClientInitializationCheck(initializeRequest);
         this.checks.push(check);
 
         const serverInfo = {
@@ -104,8 +104,7 @@ export class InitializeScenario implements Scenario {
             version: '1.0.0'
         };
 
-        const serverInfoCheck = createServerInfoCheck(serverInfo);
-        this.checks.push(serverInfoCheck);
+        this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
 
         const response = {
             jsonrpc: '2.0',

src/scenarios/index.ts

@@ -1,12 +1,17 @@
-import { Scenario } from '../types.js';
+import { Scenario, ClientScenario } from '../types.js';
 import { InitializeScenario } from './client/initialize.js';
 import { ToolsCallScenario } from './client/tools_call.js';
+import { ServerInitializeClientScenario } from './server/server_initialize.js';
 
 export const scenarios = new Map<string, Scenario>([
     ['initialize', new InitializeScenario()],
     ['tools-call', new ToolsCallScenario()]
 ]);
 
+export const clientScenarios = new Map<string, ClientScenario>([
+    ['initialize', new ServerInitializeClientScenario()]
+]);
+
 export function registerScenario(name: string, scenario: Scenario): void {
     scenarios.set(name, scenario);
 }
@@ -15,6 +20,14 @@ export function getScenario(name: string): Scenario | undefined {
     return scenarios.get(name);
 }
 
+export function getClientScenario(name: string): ClientScenario | undefined {
+    return clientScenarios.get(name);
+}
+
 export function listScenarios(): string[] {
     return Array.from(scenarios.keys());
 }
+
+export function listClientScenarios(): string[] {
+    return Array.from(clientScenarios.keys());
+}

src/scenarios/server/server_initialize.ts

@@ -0,0 +1,82 @@
+import { ClientScenario, ConformanceCheck } from '../../types.js';
+import { serverChecks } from '../../checks/index.js';
+
+export class ServerInitializeClientScenario implements ClientScenario {
+    name = 'server-initialize';
+    description = 'Acts as MCP client to test external server initialization';
+
+    async run(serverUrl: string): Promise<ConformanceCheck[]> {
+        const checks: ConformanceCheck[] = [];
+
+        try {
+            const response = await fetch(serverUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Accept': 'application/json, text/event-stream',
+                },
+                body: JSON.stringify({
+                    jsonrpc: '2.0',
+                    id: 1,
+                    method: 'initialize',
+                    params: {
+                        protocolVersion: '2025-06-18',
+                        capabilities: {},
+                        clientInfo: {
+                            name: 'conformance-test-client',
+                            version: '1.0.0'
+                        }
+                    }
+                })
+            });
+
+            if (!response.ok) {
+                const responseBody = await response.text();
+                throw new Error(`HTTP ${response.status}: ${response.statusText}. Response body: ${responseBody}`);
+            }
+
+            const responseText = await response.text();
+            
+            // Handle SSE format
+            let result;
+            if (responseText.startsWith('event:') || responseText.includes('\ndata:')) {
+                // Parse SSE format - extract JSON from data: lines
+                const lines = responseText.split('\n');
+                const dataLines = lines.filter(line => line.startsWith('data: '));
+                if (dataLines.length > 0) {
+                    const jsonData = dataLines[0].substring(6); // Remove 'data: ' prefix
+                    result = JSON.parse(jsonData);
+                } else {
+                    throw new Error(`SSE response without data line: ${responseText}`);
+                }
+            } else {
+                // Regular JSON response
+                result = JSON.parse(responseText);
+            }
+            
+            const check = serverChecks.createServerInitializationCheck(result);
+            checks.push(check);
+        } catch (error) {
+            checks.push({
+                id: 'server-initialize-request',
+                name: 'ServerInitializeRequest', 
+                description: 'Tests server response to initialize request',
+                status: 'FAILURE',
+                timestamp: new Date().toISOString(),
+                errorMessage: `Failed to send initialize request: ${error instanceof Error ? error.message : String(error)}`,
+                details: {
+                    error: error instanceof Error ? error.message : String(error),
+                    serverUrl
+                },
+                specReferences: [
+                    {
+                        id: 'MCP-Initialize',
+                        url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
+                    }
+                ]
+            });
+        }
+
+        return checks;
+    }
+}

src/server-runner.ts

@@ -0,0 +1,101 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import { ConformanceCheck } from './types.js';
+import { getClientScenario } from './scenarios/index.js';
+
+async function ensureResultsDir(): Promise<string> {
+    const resultsDir = path.join(process.cwd(), 'results');
+    await fs.mkdir(resultsDir, { recursive: true });
+    return resultsDir;
+}
+
+function createResultDir(scenario: string): string {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    return path.join('results', `server-${scenario}-${timestamp}`);
+}
+
+export async function runServerConformanceTest(
+    serverUrl: string,
+    scenarioName: string
+): Promise<{
+    checks: ConformanceCheck[];
+    resultDir: string;
+}> {
+    await ensureResultsDir();
+    const resultDir = createResultDir(scenarioName);
+    await fs.mkdir(resultDir, { recursive: true });
+
+    const scenario = getClientScenario(scenarioName);
+    if (!scenario) {
+        throw new Error(`Unknown client scenario: ${scenarioName}`);
+    }
+
+    console.log(`Running client scenario '${scenarioName}' against server: ${serverUrl}`);
+
+    const checks = await scenario.run(serverUrl);
+
+    await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
+
+    console.log(`Results saved to ${resultDir}`);
+
+    return {
+        checks,
+        resultDir
+    };
+}
+
+async function main(): Promise<void> {
+    const args = process.argv.slice(2);
+    let serverUrl: string | null = null;
+    let scenario: string | null = null;
+
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--server-url' && i + 1 < args.length) {
+            serverUrl = args[i + 1];
+            i++;
+        } else if (args[i] === '--scenario' && i + 1 < args.length) {
+            scenario = args[i + 1];
+            i++;
+        }
+    }
+
+    if (!serverUrl || !scenario) {
+        console.error('Usage: server-runner --server-url <url> --scenario <scenario>');
+        console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize');
+        process.exit(1);
+    }
+
+    try {
+        const result = await runServerConformanceTest(serverUrl, scenario);
+
+        const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
+        const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
+        const failed = result.checks.filter(c => c.status === 'FAILURE').length;
+
+        console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
+
+        console.log(`\nTest Results:`);
+        console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
+
+        if (failed > 0) {
+            console.log('\nFailed Checks:');
+            result.checks
+                .filter(c => c.status === 'FAILURE')
+                .forEach(c => {
+                    console.log(`  - ${c.name}: ${c.description}`);
+                    if (c.errorMessage) {
+                        console.log(`    Error: ${c.errorMessage}`);
+                    }
+                });
+        }
+
+        process.exit(failed > 0 ? 1 : 0);
+    } catch (error) {
+        console.error('Server test runner error:', error);
+        process.exit(1);
+    }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+    main();
+}

src/types.ts

@@ -30,3 +30,9 @@ export interface Scenario {
     stop(): Promise<void>;
     getChecks(): ConformanceCheck[];
 }
+
+export interface ClientScenario {
+    name: string;
+    description: string;
+    run(serverUrl: string): Promise<ConformanceCheck[]>;
+}