apply linting
This commit is contained in:
Paul Carleton
parent
ae9451f5e0
commit
f32b01c57d
11 changed files with 510 additions and 527 deletions
|
|
@ -12,6 +12,7 @@ npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario
|
|||
## Overview
|
||||
|
||||
The conformance test framework validates MCP client implementations by:
|
||||
|
||||
1. Starting a test server for the specified scenario
|
||||
2. Running the client implementation with the test server URL
|
||||
3. Capturing MCP protocol interactions
|
||||
|
|
@ -32,6 +33,7 @@ The framework appends the server URL as the final argument to your command.
|
|||
## Test Results
|
||||
|
||||
Results are saved to `results/<scenario>-<timestamp>/`:
|
||||
|
||||
- `checks.json` - Array of conformance check results with pass/fail status
|
||||
- `stdout.txt` - Client stdout output
|
||||
- `stderr.txt` - Client stderr output
|
||||
|
|
@ -44,9 +46,9 @@ Results are saved to `results/<scenario>-<timestamp>/`:
|
|||
## Available Scenarios
|
||||
|
||||
- **initialize** - Tests MCP client initialization handshake
|
||||
- Validates protocol version
|
||||
- Validates clientInfo (name and version)
|
||||
- Validates server response handling
|
||||
- Validates protocol version
|
||||
- Validates clientInfo (name and version)
|
||||
- Validates server response handling
|
||||
|
||||
## Architecture
|
||||
|
||||
|
|
|
|||
|
|
@ -3,57 +3,55 @@
|
|||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const serverUrl = process.argv[2];
|
||||
const serverUrl = process.argv[2];
|
||||
|
||||
if (!serverUrl) {
|
||||
console.error('Usage: test-broken-client <server-url>');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!serverUrl) {
|
||||
console.error('Usage: test-broken-client <server-url>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||
|
||||
try {
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(serverUrl)
|
||||
);
|
||||
try {
|
||||
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||
|
||||
await transport.start();
|
||||
await transport.start();
|
||||
|
||||
const initRequest = {
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'initialize',
|
||||
params: {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
version: '1.0.0'
|
||||
},
|
||||
capabilities: {}
|
||||
}
|
||||
};
|
||||
const initRequest = {
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'initialize',
|
||||
params: {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
version: '1.0.0'
|
||||
},
|
||||
capabilities: {}
|
||||
}
|
||||
};
|
||||
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(initRequest)
|
||||
});
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(initRequest)
|
||||
});
|
||||
|
||||
const result = await response.json();
|
||||
console.log('✅ Received response:', JSON.stringify(result, null, 2));
|
||||
const result = await response.json();
|
||||
console.log('✅ Received response:', JSON.stringify(result, null, 2));
|
||||
|
||||
await transport.close();
|
||||
console.log('✅ Connection closed');
|
||||
await transport.close();
|
||||
console.log('✅ Connection closed');
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('❌ Error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('❌ Error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Unhandled error:', error);
|
||||
process.exit(1);
|
||||
main().catch(error => {
|
||||
console.error('Unhandled error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,44 +4,45 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
|||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const serverUrl = process.argv[2];
|
||||
const serverUrl = process.argv[2];
|
||||
|
||||
if (!serverUrl) {
|
||||
console.error('Usage: test-client <server-url>');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!serverUrl) {
|
||||
console.error('Usage: test-client <server-url>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||
|
||||
try {
|
||||
const client = new Client({
|
||||
name: 'test-client',
|
||||
version: '1.0.0'
|
||||
}, {
|
||||
capabilities: {}
|
||||
});
|
||||
try {
|
||||
const client = new Client(
|
||||
{
|
||||
name: 'test-client',
|
||||
version: '1.0.0'
|
||||
},
|
||||
{
|
||||
capabilities: {}
|
||||
}
|
||||
);
|
||||
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(serverUrl)
|
||||
);
|
||||
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||
|
||||
await client.connect(transport);
|
||||
console.log('✅ Successfully connected to MCP server');
|
||||
await client.connect(transport);
|
||||
console.log('✅ Successfully connected to MCP server');
|
||||
|
||||
await client.listTools();
|
||||
console.log('✅ Successfully listed tools');
|
||||
await client.listTools();
|
||||
console.log('✅ Successfully listed tools');
|
||||
|
||||
await transport.close();
|
||||
console.log('✅ Connection closed successfully');
|
||||
await transport.close();
|
||||
console.log('✅ Connection closed successfully');
|
||||
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to MCP server:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to connect to MCP server:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Unhandled error:', error);
|
||||
process.exit(1);
|
||||
main().catch(error => {
|
||||
console.error('Unhandled error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
export default {
|
||||
preset: 'ts-jest/presets/default-esm',
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
testEnvironment: 'node',
|
||||
moduleNameMapper: {
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
preset: 'ts-jest/presets/default-esm',
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
testEnvironment: 'node',
|
||||
moduleNameMapper: {
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1'
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,84 +1,84 @@
|
|||
import { createClientInitializationCheck } from './checks';
|
||||
|
||||
describe('createClientInitializationCheck', () => {
|
||||
it('should return SUCCESS for a valid initialize request', () => {
|
||||
const validRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
it('should return SUCCESS for a valid initialize request', () => {
|
||||
const validRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(validRequest);
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
});
|
||||
const check = createClientInitializationCheck(validRequest);
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return FAILURE when protocol version is missing', () => {
|
||||
const invalidRequest = {
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
it('should return FAILURE when protocol version is missing', () => {
|
||||
const invalidRequest = {
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Protocol version not provided');
|
||||
});
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Protocol version not provided');
|
||||
});
|
||||
|
||||
it('should return FAILURE when protocol version does not match', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2024-11-05',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
it('should return FAILURE when protocol version does not match', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2024-11-05',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Version mismatch');
|
||||
});
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Version mismatch');
|
||||
});
|
||||
|
||||
it('should return FAILURE when client name is missing', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
it('should return FAILURE when client name is missing', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Client name missing');
|
||||
});
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Client name missing');
|
||||
});
|
||||
|
||||
it('should return FAILURE when client version is missing', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
name: 'TestClient'
|
||||
}
|
||||
};
|
||||
it('should return FAILURE when client version is missing', () => {
|
||||
const invalidRequest = {
|
||||
protocolVersion: '2025-06-18',
|
||||
clientInfo: {
|
||||
name: 'TestClient'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Client version missing');
|
||||
});
|
||||
const check = createClientInitializationCheck(invalidRequest);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Client version missing');
|
||||
});
|
||||
|
||||
it('should support custom expected spec version', () => {
|
||||
const request = {
|
||||
protocolVersion: '2024-11-05',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
it('should support custom expected spec version', () => {
|
||||
const request = {
|
||||
protocolVersion: '2024-11-05',
|
||||
clientInfo: {
|
||||
name: 'TestClient',
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(request, '2024-11-05');
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
});
|
||||
const check = createClientInitializationCheck(request, '2024-11-05');
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
107
src/checks.ts
107
src/checks.ts
|
|
@ -1,68 +1,59 @@
|
|||
import { ConformanceCheck, CheckStatus } from './types.js';
|
||||
|
||||
export function createClientInitializationCheck(
|
||||
initializeRequest: any,
|
||||
expectedSpecVersion: string = '2025-06-18'
|
||||
): ConformanceCheck {
|
||||
const protocolVersionSent = initializeRequest?.protocolVersion;
|
||||
const versionMatch = protocolVersionSent === expectedSpecVersion;
|
||||
export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
|
||||
const protocolVersionSent = initializeRequest?.protocolVersion;
|
||||
const versionMatch = protocolVersionSent === expectedSpecVersion;
|
||||
|
||||
const errors: string[] = [];
|
||||
if (!protocolVersionSent) errors.push('Protocol version not provided');
|
||||
if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`);
|
||||
if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing');
|
||||
if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing');
|
||||
const errors: string[] = [];
|
||||
if (!protocolVersionSent) errors.push('Protocol version not provided');
|
||||
if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`);
|
||||
if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing');
|
||||
if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing');
|
||||
|
||||
const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
|
||||
const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
|
||||
|
||||
return {
|
||||
id: 'mcp-client-initialization',
|
||||
name: 'MCPClientInitialization',
|
||||
description: 'Validates that MCP client properly initializes with server',
|
||||
status,
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }
|
||||
],
|
||||
details: {
|
||||
protocolVersionSent,
|
||||
expectedSpecVersion,
|
||||
versionMatch,
|
||||
clientName: initializeRequest?.clientInfo?.name,
|
||||
clientVersion: initializeRequest?.clientInfo?.version
|
||||
},
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
|
||||
logs: errors.length > 0 ? errors : undefined
|
||||
};
|
||||
return {
|
||||
id: 'mcp-client-initialization',
|
||||
name: 'MCPClientInitialization',
|
||||
description: 'Validates that MCP client properly initializes with server',
|
||||
status,
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
|
||||
details: {
|
||||
protocolVersionSent,
|
||||
expectedSpecVersion,
|
||||
versionMatch,
|
||||
clientName: initializeRequest?.clientInfo?.name,
|
||||
clientVersion: initializeRequest?.clientInfo?.version
|
||||
},
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
|
||||
logs: errors.length > 0 ? errors : undefined
|
||||
};
|
||||
}
|
||||
|
||||
export function createServerInfoCheck(
|
||||
serverInfo: { name?: string; version?: string } | null
|
||||
): ConformanceCheck {
|
||||
const hasName = !!serverInfo?.name;
|
||||
const hasVersion = !!serverInfo?.version;
|
||||
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
|
||||
export function createServerInfoCheck(serverInfo: { name?: string; version?: string } | null): ConformanceCheck {
|
||||
const hasName = !!serverInfo?.name;
|
||||
const hasVersion = !!serverInfo?.version;
|
||||
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
|
||||
|
||||
const errors: string[] = [];
|
||||
if (!hasName) errors.push('Server name missing');
|
||||
if (!hasVersion) errors.push('Server version missing');
|
||||
const errors: string[] = [];
|
||||
if (!hasName) errors.push('Server name missing');
|
||||
if (!hasVersion) errors.push('Server version missing');
|
||||
|
||||
return {
|
||||
id: 'server-info',
|
||||
name: 'ServerInfo',
|
||||
description: 'Test server info returned to client',
|
||||
status,
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }
|
||||
],
|
||||
details: {
|
||||
serverName: serverInfo?.name,
|
||||
serverVersion: serverInfo?.version,
|
||||
hasName,
|
||||
hasVersion
|
||||
},
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
|
||||
logs: errors.length > 0 ? errors : undefined
|
||||
};
|
||||
return {
|
||||
id: 'server-info',
|
||||
name: 'ServerInfo',
|
||||
description: 'Test server info returned to client',
|
||||
status,
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
|
||||
details: {
|
||||
serverName: serverInfo?.name,
|
||||
serverVersion: serverInfo?.version,
|
||||
hasName,
|
||||
hasVersion
|
||||
},
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
|
||||
logs: errors.length > 0 ? errors : undefined
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ A scenario represents a specific test case that validates one or more aspects of
|
|||
- Validates both client behavior and server responses
|
||||
|
||||
Example scenarios:
|
||||
|
||||
- `initialize` - Tests client initialization handshake
|
||||
- `list-tools` - Tests tool discovery
|
||||
- `call-tool` - Tests tool invocation
|
||||
|
|
@ -57,6 +58,7 @@ Example scenarios:
|
|||
Results are written to: `results/<scenario>-<timestamp>/`
|
||||
|
||||
Files:
|
||||
|
||||
- `checks.json` - Array of ConformanceCheck objects with validation results
|
||||
- `stdout.txt` - Complete stdout from the client process
|
||||
- `stderr.txt` - Complete stderr from the client process
|
||||
|
|
|
|||
|
|
@ -5,184 +5,171 @@ import { ConformanceCheck } from '../types.js';
|
|||
import { getScenario } from '../scenarios/index.js';
|
||||
|
||||
export interface ClientExecutionResult {
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
async function ensureResultsDir(): Promise<string> {
|
||||
const resultsDir = path.join(process.cwd(), 'results');
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
return resultsDir;
|
||||
const resultsDir = path.join(process.cwd(), 'results');
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
return resultsDir;
|
||||
}
|
||||
|
||||
function createResultDir(scenario: string): string {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
return path.join('results', `${scenario}-${timestamp}`);
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
return path.join('results', `${scenario}-${timestamp}`);
|
||||
}
|
||||
|
||||
async function executeClient(
|
||||
command: string,
|
||||
serverUrl: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ClientExecutionResult> {
|
||||
const commandParts = command.split(' ');
|
||||
const executable = commandParts[0];
|
||||
const args = [...commandParts.slice(1), serverUrl];
|
||||
async function executeClient(command: string, serverUrl: string, timeout: number = 30000): Promise<ClientExecutionResult> {
|
||||
const commandParts = command.split(' ');
|
||||
const executable = commandParts[0];
|
||||
const args = [...commandParts.slice(1), serverUrl];
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let timedOut = false;
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let timedOut = false;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const process = spawn(executable, args, {
|
||||
shell: true,
|
||||
stdio: 'pipe'
|
||||
return new Promise(resolve => {
|
||||
const process = spawn(executable, args, {
|
||||
shell: true,
|
||||
stdio: 'pipe'
|
||||
});
|
||||
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
timedOut = true;
|
||||
process.kill();
|
||||
}, timeout);
|
||||
|
||||
if (process.stdout) {
|
||||
process.stdout.on('data', data => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
if (process.stderr) {
|
||||
process.stderr.on('data', data => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
process.on('close', code => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: code || 0,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
|
||||
process.on('error', error => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
stdout,
|
||||
stderr: stderr + `\nProcess error: ${error.message}`,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
timedOut = true;
|
||||
process.kill();
|
||||
}, timeout);
|
||||
|
||||
if (process.stdout) {
|
||||
process.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
if (process.stderr) {
|
||||
process.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
process.on('close', (code) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: code || 0,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
|
||||
process.on('error', (error) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
stdout,
|
||||
stderr: stderr + `\nProcess error: ${error.message}`,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runConformanceTest(
|
||||
clientCommand: string,
|
||||
scenarioName: string,
|
||||
timeout: number = 30000
|
||||
clientCommand: string,
|
||||
scenarioName: string,
|
||||
timeout: number = 30000
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
clientOutput: ClientExecutionResult;
|
||||
resultDir: string;
|
||||
checks: ConformanceCheck[];
|
||||
clientOutput: ClientExecutionResult;
|
||||
resultDir: string;
|
||||
}> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
const scenario = getScenario(scenarioName);
|
||||
if (!scenario) {
|
||||
throw new Error(`Unknown scenario: ${scenarioName}`);
|
||||
}
|
||||
const scenario = getScenario(scenarioName);
|
||||
if (!scenario) {
|
||||
throw new Error(`Unknown scenario: ${scenarioName}`);
|
||||
}
|
||||
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
|
||||
console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
|
||||
console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
|
||||
|
||||
try {
|
||||
const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout);
|
||||
try {
|
||||
const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout);
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
const checks = scenario.getChecks();
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'stdout.txt'),
|
||||
clientOutput.stdout
|
||||
);
|
||||
await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'stderr.txt'),
|
||||
clientOutput.stderr
|
||||
);
|
||||
await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
|
||||
|
||||
console.log(`Results saved to ${resultDir}`);
|
||||
console.log(`Results saved to ${resultDir}`);
|
||||
|
||||
return {
|
||||
checks,
|
||||
clientOutput,
|
||||
resultDir
|
||||
};
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
return {
|
||||
checks,
|
||||
clientOutput,
|
||||
resultDir
|
||||
};
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
let command: string | null = null;
|
||||
let scenario: string | null = null;
|
||||
const args = process.argv.slice(2);
|
||||
let command: string | null = null;
|
||||
let scenario: string | null = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--command' && i + 1 < args.length) {
|
||||
command = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--scenario' && i + 1 < args.length) {
|
||||
scenario = args[i + 1];
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (!command || !scenario) {
|
||||
console.error('Usage: runner --command "<command>" --scenario <scenario>');
|
||||
console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runConformanceTest(command, scenario);
|
||||
|
||||
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
|
||||
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
|
||||
|
||||
console.log(`\nTest Results:`);
|
||||
console.log(`Passed: ${passed}/${result.checks.length}`);
|
||||
console.log(`Failed: ${failed}/${result.checks.length}`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
result.checks
|
||||
.filter(c => c.status === 'FAILURE')
|
||||
.forEach(c => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
});
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--command' && i + 1 < args.length) {
|
||||
command = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--scenario' && i + 1 < args.length) {
|
||||
scenario = args[i + 1];
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
} catch (error) {
|
||||
console.error('Test runner error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!command || !scenario) {
|
||||
console.error('Usage: runner --command "<command>" --scenario <scenario>');
|
||||
console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runConformanceTest(command, scenario);
|
||||
|
||||
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
|
||||
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
|
||||
|
||||
console.log(`\nTest Results:`);
|
||||
console.log(`Passed: ${passed}/${result.checks.length}`);
|
||||
console.log(`Failed: ${failed}/${result.checks.length}`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
result.checks
|
||||
.filter(c => c.status === 'FAILURE')
|
||||
.forEach(c => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
} catch (error) {
|
||||
console.error('Test runner error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
|
|||
|
|
@ -1,18 +1,16 @@
|
|||
import { Scenario } from '../types.js';
|
||||
import { InitializeScenario } from './initialize.js';
|
||||
|
||||
export const scenarios = new Map<string, Scenario>([
|
||||
['initialize', new InitializeScenario()],
|
||||
]);
|
||||
export const scenarios = new Map<string, Scenario>([['initialize', new InitializeScenario()]]);
|
||||
|
||||
export function registerScenario(name: string, scenario: Scenario): void {
|
||||
scenarios.set(name, scenario);
|
||||
scenarios.set(name, scenario);
|
||||
}
|
||||
|
||||
export function getScenario(name: string): Scenario | undefined {
|
||||
return scenarios.get(name);
|
||||
return scenarios.get(name);
|
||||
}
|
||||
|
||||
export function listScenarios(): string[] {
|
||||
return Array.from(scenarios.keys());
|
||||
return Array.from(scenarios.keys());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,130 +3,134 @@ import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js';
|
|||
import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js';
|
||||
|
||||
export class InitializeScenario implements Scenario {
|
||||
name = 'initialize';
|
||||
description = 'Tests MCP client initialization handshake';
|
||||
name = 'initialize';
|
||||
description = 'Tests MCP client initialization handshake';
|
||||
|
||||
private server: http.Server | null = null;
|
||||
private checks: ConformanceCheck[] = [];
|
||||
private port: number = 0;
|
||||
private server: http.Server | null = null;
|
||||
private checks: ConformanceCheck[] = [];
|
||||
private port: number = 0;
|
||||
|
||||
async start(): Promise<ScenarioUrls> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.server = http.createServer((req, res) => {
|
||||
this.handleRequest(req, res);
|
||||
});
|
||||
async start(): Promise<ScenarioUrls> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.server = http.createServer((req, res) => {
|
||||
this.handleRequest(req, res);
|
||||
});
|
||||
|
||||
this.server.on('error', reject);
|
||||
this.server.on('error', reject);
|
||||
|
||||
this.server.listen(0, () => {
|
||||
const address = this.server!.address();
|
||||
if (address && typeof address === 'object') {
|
||||
this.port = address.port;
|
||||
resolve({
|
||||
serverUrl: `http://localhost:${this.port}`
|
||||
});
|
||||
} else {
|
||||
reject(new Error('Failed to get server address'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (this.server) {
|
||||
this.server.close((err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
this.server = null;
|
||||
resolve();
|
||||
}
|
||||
this.server.listen(0, () => {
|
||||
const address = this.server!.address();
|
||||
if (address && typeof address === 'object') {
|
||||
this.port = address.port;
|
||||
resolve({
|
||||
serverUrl: `http://localhost:${this.port}`
|
||||
});
|
||||
} else {
|
||||
reject(new Error('Failed to get server address'));
|
||||
}
|
||||
});
|
||||
});
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
getChecks(): ConformanceCheck[] {
|
||||
return this.checks;
|
||||
}
|
||||
async stop(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (this.server) {
|
||||
this.server.close(err => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
this.server = null;
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
|
||||
let body = '';
|
||||
getChecks(): ConformanceCheck[] {
|
||||
return this.checks;
|
||||
}
|
||||
|
||||
req.on('data', (chunk) => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
|
||||
let body = '';
|
||||
|
||||
req.on('end', () => {
|
||||
try {
|
||||
const request = JSON.parse(body);
|
||||
req.on('data', chunk => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
|
||||
if (request.method === 'initialize') {
|
||||
this.handleInitialize(request, res);
|
||||
} else if (request.method === 'tools/list') {
|
||||
this.handleToolsList(request, res);
|
||||
} else {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
req.on('end', () => {
|
||||
try {
|
||||
const request = JSON.parse(body);
|
||||
|
||||
if (request.method === 'initialize') {
|
||||
this.handleInitialize(request, res);
|
||||
} else if (request.method === 'tools/list') {
|
||||
this.handleToolsList(request, res);
|
||||
} else {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {}
|
||||
})
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
error: {
|
||||
code: -32700,
|
||||
message: `Parse error ${error}`
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private handleInitialize(request: any, res: http.ServerResponse): void {
|
||||
const initializeRequest = request.params;
|
||||
|
||||
const check = createClientInitializationCheck(initializeRequest);
|
||||
this.checks.push(check);
|
||||
|
||||
const serverInfo = {
|
||||
name: 'test-server',
|
||||
version: '1.0.0'
|
||||
};
|
||||
|
||||
const serverInfoCheck = createServerInfoCheck(serverInfo);
|
||||
this.checks.push(serverInfoCheck);
|
||||
|
||||
const response = {
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {}
|
||||
}));
|
||||
}
|
||||
} catch (error) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
error: {
|
||||
code: -32700,
|
||||
message: 'Parse error'
|
||||
}
|
||||
}));
|
||||
}
|
||||
});
|
||||
}
|
||||
result: {
|
||||
protocolVersion: '2025-06-18',
|
||||
serverInfo,
|
||||
capabilities: {}
|
||||
}
|
||||
};
|
||||
|
||||
private handleInitialize(request: any, res: http.ServerResponse): void {
|
||||
const initializeRequest = request.params;
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(response));
|
||||
}
|
||||
|
||||
const check = createClientInitializationCheck(initializeRequest);
|
||||
this.checks.push(check);
|
||||
private handleToolsList(request: any, res: http.ServerResponse): void {
|
||||
const response = {
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {
|
||||
tools: []
|
||||
}
|
||||
};
|
||||
|
||||
const serverInfo = {
|
||||
name: 'test-server',
|
||||
version: '1.0.0'
|
||||
};
|
||||
|
||||
const serverInfoCheck = createServerInfoCheck(serverInfo);
|
||||
this.checks.push(serverInfoCheck);
|
||||
|
||||
const response = {
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {
|
||||
protocolVersion: '2025-06-18',
|
||||
serverInfo,
|
||||
capabilities: {}
|
||||
}
|
||||
};
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(response));
|
||||
}
|
||||
|
||||
private handleToolsList(request: any, res: http.ServerResponse): void {
|
||||
const response = {
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {
|
||||
tools: []
|
||||
}
|
||||
};
|
||||
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(response));
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify(response));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
38
src/types.ts
38
src/types.ts
|
|
@ -1,32 +1,32 @@
|
|||
export type CheckStatus = 'SUCCESS' | 'FAILURE' | 'WARNING' | 'SKIPPED' | 'INFO';
|
||||
|
||||
export interface SpecReference {
|
||||
id: string;
|
||||
url?: string;
|
||||
id: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface ConformanceCheck {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
status: CheckStatus;
|
||||
timestamp: string;
|
||||
specReferences?: SpecReference[];
|
||||
details?: Record<string, unknown>;
|
||||
metadata?: Record<string, unknown>;
|
||||
errorMessage?: string;
|
||||
logs?: string[];
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
status: CheckStatus;
|
||||
timestamp: string;
|
||||
specReferences?: SpecReference[];
|
||||
details?: Record<string, unknown>;
|
||||
metadata?: Record<string, unknown>;
|
||||
errorMessage?: string;
|
||||
logs?: string[];
|
||||
}
|
||||
|
||||
export interface ScenarioUrls {
|
||||
serverUrl: string;
|
||||
authUrl?: string;
|
||||
serverUrl: string;
|
||||
authUrl?: string;
|
||||
}
|
||||
|
||||
export interface Scenario {
|
||||
name: string;
|
||||
description: string;
|
||||
start(): Promise<ScenarioUrls>;
|
||||
stop(): Promise<void>;
|
||||
getChecks(): ConformanceCheck[];
|
||||
name: string;
|
||||
description: string;
|
||||
start(): Promise<ScenarioUrls>;
|
||||
stop(): Promise<void>;
|
||||
getChecks(): ConformanceCheck[];
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue