apply linting
This commit is contained in:
parent
ae9451f5e0
commit
f32b01c57d
11 changed files with 510 additions and 527 deletions
|
|
@ -12,6 +12,7 @@ npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
The conformance test framework validates MCP client implementations by:
|
The conformance test framework validates MCP client implementations by:
|
||||||
|
|
||||||
1. Starting a test server for the specified scenario
|
1. Starting a test server for the specified scenario
|
||||||
2. Running the client implementation with the test server URL
|
2. Running the client implementation with the test server URL
|
||||||
3. Capturing MCP protocol interactions
|
3. Capturing MCP protocol interactions
|
||||||
|
|
@ -32,6 +33,7 @@ The framework appends the server URL as the final argument to your command.
|
||||||
## Test Results
|
## Test Results
|
||||||
|
|
||||||
Results are saved to `results/<scenario>-<timestamp>/`:
|
Results are saved to `results/<scenario>-<timestamp>/`:
|
||||||
|
|
||||||
- `checks.json` - Array of conformance check results with pass/fail status
|
- `checks.json` - Array of conformance check results with pass/fail status
|
||||||
- `stdout.txt` - Client stdout output
|
- `stdout.txt` - Client stdout output
|
||||||
- `stderr.txt` - Client stderr output
|
- `stderr.txt` - Client stderr output
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,7 @@ async function main(): Promise<void> {
|
||||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const transport = new StreamableHTTPClientTransport(
|
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||||
new URL(serverUrl)
|
|
||||||
);
|
|
||||||
|
|
||||||
await transport.start();
|
await transport.start();
|
||||||
|
|
||||||
|
|
@ -53,7 +51,7 @@ async function main(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch((error) => {
|
main().catch(error => {
|
||||||
console.error('Unhandled error:', error);
|
console.error('Unhandled error:', error);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -14,17 +14,18 @@ async function main(): Promise<void> {
|
||||||
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
console.log(`Connecting to MCP server at: ${serverUrl}`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const client = new Client({
|
const client = new Client(
|
||||||
|
{
|
||||||
name: 'test-client',
|
name: 'test-client',
|
||||||
version: '1.0.0'
|
version: '1.0.0'
|
||||||
}, {
|
},
|
||||||
|
{
|
||||||
capabilities: {}
|
capabilities: {}
|
||||||
});
|
}
|
||||||
|
|
||||||
const transport = new StreamableHTTPClientTransport(
|
|
||||||
new URL(serverUrl)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||||
|
|
||||||
await client.connect(transport);
|
await client.connect(transport);
|
||||||
console.log('✅ Successfully connected to MCP server');
|
console.log('✅ Successfully connected to MCP server');
|
||||||
|
|
||||||
|
|
@ -41,7 +42,7 @@ async function main(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch((error) => {
|
main().catch(error => {
|
||||||
console.error('Unhandled error:', error);
|
console.error('Unhandled error:', error);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,6 @@ export default {
|
||||||
extensionsToTreatAsEsm: ['.ts'],
|
extensionsToTreatAsEsm: ['.ts'],
|
||||||
testEnvironment: 'node',
|
testEnvironment: 'node',
|
||||||
moduleNameMapper: {
|
moduleNameMapper: {
|
||||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
'^(\\.{1,2}/.*)\\.js$': '$1'
|
||||||
},
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,6 @@
|
||||||
import { ConformanceCheck, CheckStatus } from './types.js';
|
import { ConformanceCheck, CheckStatus } from './types.js';
|
||||||
|
|
||||||
export function createClientInitializationCheck(
|
export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
|
||||||
initializeRequest: any,
|
|
||||||
expectedSpecVersion: string = '2025-06-18'
|
|
||||||
): ConformanceCheck {
|
|
||||||
const protocolVersionSent = initializeRequest?.protocolVersion;
|
const protocolVersionSent = initializeRequest?.protocolVersion;
|
||||||
const versionMatch = protocolVersionSent === expectedSpecVersion;
|
const versionMatch = protocolVersionSent === expectedSpecVersion;
|
||||||
|
|
||||||
|
|
@ -21,9 +18,7 @@ export function createClientInitializationCheck(
|
||||||
description: 'Validates that MCP client properly initializes with server',
|
description: 'Validates that MCP client properly initializes with server',
|
||||||
status,
|
status,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
specReferences: [
|
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
|
||||||
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }
|
|
||||||
],
|
|
||||||
details: {
|
details: {
|
||||||
protocolVersionSent,
|
protocolVersionSent,
|
||||||
expectedSpecVersion,
|
expectedSpecVersion,
|
||||||
|
|
@ -36,9 +31,7 @@ export function createClientInitializationCheck(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createServerInfoCheck(
|
export function createServerInfoCheck(serverInfo: { name?: string; version?: string } | null): ConformanceCheck {
|
||||||
serverInfo: { name?: string; version?: string } | null
|
|
||||||
): ConformanceCheck {
|
|
||||||
const hasName = !!serverInfo?.name;
|
const hasName = !!serverInfo?.name;
|
||||||
const hasVersion = !!serverInfo?.version;
|
const hasVersion = !!serverInfo?.version;
|
||||||
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
|
const status: CheckStatus = hasName && hasVersion ? 'SUCCESS' : 'FAILURE';
|
||||||
|
|
@ -53,9 +46,7 @@ export function createServerInfoCheck(
|
||||||
description: 'Test server info returned to client',
|
description: 'Test server info returned to client',
|
||||||
status,
|
status,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
specReferences: [
|
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
|
||||||
{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }
|
|
||||||
],
|
|
||||||
details: {
|
details: {
|
||||||
serverName: serverInfo?.name,
|
serverName: serverInfo?.name,
|
||||||
serverVersion: serverInfo?.version,
|
serverVersion: serverInfo?.version,
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,7 @@ A scenario represents a specific test case that validates one or more aspects of
|
||||||
- Validates both client behavior and server responses
|
- Validates both client behavior and server responses
|
||||||
|
|
||||||
Example scenarios:
|
Example scenarios:
|
||||||
|
|
||||||
- `initialize` - Tests client initialization handshake
|
- `initialize` - Tests client initialization handshake
|
||||||
- `list-tools` - Tests tool discovery
|
- `list-tools` - Tests tool discovery
|
||||||
- `call-tool` - Tests tool invocation
|
- `call-tool` - Tests tool invocation
|
||||||
|
|
@ -57,6 +58,7 @@ Example scenarios:
|
||||||
Results are written to: `results/<scenario>-<timestamp>/`
|
Results are written to: `results/<scenario>-<timestamp>/`
|
||||||
|
|
||||||
Files:
|
Files:
|
||||||
|
|
||||||
- `checks.json` - Array of ConformanceCheck objects with validation results
|
- `checks.json` - Array of ConformanceCheck objects with validation results
|
||||||
- `stdout.txt` - Complete stdout from the client process
|
- `stdout.txt` - Complete stdout from the client process
|
||||||
- `stderr.txt` - Complete stderr from the client process
|
- `stderr.txt` - Complete stderr from the client process
|
||||||
|
|
|
||||||
|
|
@ -22,11 +22,7 @@ function createResultDir(scenario: string): string {
|
||||||
return path.join('results', `${scenario}-${timestamp}`);
|
return path.join('results', `${scenario}-${timestamp}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function executeClient(
|
async function executeClient(command: string, serverUrl: string, timeout: number = 30000): Promise<ClientExecutionResult> {
|
||||||
command: string,
|
|
||||||
serverUrl: string,
|
|
||||||
timeout: number = 30000
|
|
||||||
): Promise<ClientExecutionResult> {
|
|
||||||
const commandParts = command.split(' ');
|
const commandParts = command.split(' ');
|
||||||
const executable = commandParts[0];
|
const executable = commandParts[0];
|
||||||
const args = [...commandParts.slice(1), serverUrl];
|
const args = [...commandParts.slice(1), serverUrl];
|
||||||
|
|
@ -35,7 +31,7 @@ async function executeClient(
|
||||||
let stderr = '';
|
let stderr = '';
|
||||||
let timedOut = false;
|
let timedOut = false;
|
||||||
|
|
||||||
return new Promise((resolve) => {
|
return new Promise(resolve => {
|
||||||
const process = spawn(executable, args, {
|
const process = spawn(executable, args, {
|
||||||
shell: true,
|
shell: true,
|
||||||
stdio: 'pipe'
|
stdio: 'pipe'
|
||||||
|
|
@ -47,18 +43,18 @@ async function executeClient(
|
||||||
}, timeout);
|
}, timeout);
|
||||||
|
|
||||||
if (process.stdout) {
|
if (process.stdout) {
|
||||||
process.stdout.on('data', (data) => {
|
process.stdout.on('data', data => {
|
||||||
stdout += data.toString();
|
stdout += data.toString();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (process.stderr) {
|
if (process.stderr) {
|
||||||
process.stderr.on('data', (data) => {
|
process.stderr.on('data', data => {
|
||||||
stderr += data.toString();
|
stderr += data.toString();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
process.on('close', (code) => {
|
process.on('close', code => {
|
||||||
clearTimeout(timeoutHandle);
|
clearTimeout(timeoutHandle);
|
||||||
resolve({
|
resolve({
|
||||||
exitCode: code || 0,
|
exitCode: code || 0,
|
||||||
|
|
@ -68,7 +64,7 @@ async function executeClient(
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on('error', (error) => {
|
process.on('error', error => {
|
||||||
clearTimeout(timeoutHandle);
|
clearTimeout(timeoutHandle);
|
||||||
resolve({
|
resolve({
|
||||||
exitCode: -1,
|
exitCode: -1,
|
||||||
|
|
@ -108,20 +104,11 @@ export async function runConformanceTest(
|
||||||
|
|
||||||
const checks = scenario.getChecks();
|
const checks = scenario.getChecks();
|
||||||
|
|
||||||
await fs.writeFile(
|
await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
|
||||||
path.join(resultDir, 'checks.json'),
|
|
||||||
JSON.stringify(checks, null, 2)
|
|
||||||
);
|
|
||||||
|
|
||||||
await fs.writeFile(
|
await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
|
||||||
path.join(resultDir, 'stdout.txt'),
|
|
||||||
clientOutput.stdout
|
|
||||||
);
|
|
||||||
|
|
||||||
await fs.writeFile(
|
await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
|
||||||
path.join(resultDir, 'stderr.txt'),
|
|
||||||
clientOutput.stderr
|
|
||||||
);
|
|
||||||
|
|
||||||
console.log(`Results saved to ${resultDir}`);
|
console.log(`Results saved to ${resultDir}`);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
import { Scenario } from '../types.js';
|
import { Scenario } from '../types.js';
|
||||||
import { InitializeScenario } from './initialize.js';
|
import { InitializeScenario } from './initialize.js';
|
||||||
|
|
||||||
export const scenarios = new Map<string, Scenario>([
|
export const scenarios = new Map<string, Scenario>([['initialize', new InitializeScenario()]]);
|
||||||
['initialize', new InitializeScenario()],
|
|
||||||
]);
|
|
||||||
|
|
||||||
export function registerScenario(name: string, scenario: Scenario): void {
|
export function registerScenario(name: string, scenario: Scenario): void {
|
||||||
scenarios.set(name, scenario);
|
scenarios.set(name, scenario);
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ export class InitializeScenario implements Scenario {
|
||||||
async stop(): Promise<void> {
|
async stop(): Promise<void> {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
if (this.server) {
|
if (this.server) {
|
||||||
this.server.close((err) => {
|
this.server.close(err => {
|
||||||
if (err) {
|
if (err) {
|
||||||
reject(err);
|
reject(err);
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -56,7 +56,7 @@ export class InitializeScenario implements Scenario {
|
||||||
private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
|
private handleRequest(req: http.IncomingMessage, res: http.ServerResponse): void {
|
||||||
let body = '';
|
let body = '';
|
||||||
|
|
||||||
req.on('data', (chunk) => {
|
req.on('data', chunk => {
|
||||||
body += chunk.toString();
|
body += chunk.toString();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -70,21 +70,25 @@ export class InitializeScenario implements Scenario {
|
||||||
this.handleToolsList(request, res);
|
this.handleToolsList(request, res);
|
||||||
} else {
|
} else {
|
||||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
res.end(JSON.stringify({
|
res.end(
|
||||||
|
JSON.stringify({
|
||||||
jsonrpc: '2.0',
|
jsonrpc: '2.0',
|
||||||
id: request.id,
|
id: request.id,
|
||||||
result: {}
|
result: {}
|
||||||
}));
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||||
res.end(JSON.stringify({
|
res.end(
|
||||||
|
JSON.stringify({
|
||||||
jsonrpc: '2.0',
|
jsonrpc: '2.0',
|
||||||
error: {
|
error: {
|
||||||
code: -32700,
|
code: -32700,
|
||||||
message: 'Parse error'
|
message: `Parse error ${error}`
|
||||||
}
|
}
|
||||||
}));
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue