tsdown + cli with commander (#17)
* feat: tsdown * fix: formatting * automate version updates in cli * fix: docs * scenarios * fix: commander deps
This commit is contained in:
Matt
GitHub
parent
18f48e5dea
commit
7f1515c475
5 changed files with 1600 additions and 539 deletions
|
|
@ -2,20 +2,45 @@
|
|||
|
||||
## Overview
|
||||
|
||||
The conformance test runner is a framework for testing MCP client implementations against the MCP specification. It executes client implementations in controlled scenarios and validates their behavior through a series of conformance checks.
|
||||
The conformance test runner is a framework for testing both MCP client and server implementations against the MCP specification. It provides two testing modes:
|
||||
|
||||
1. **Client Testing** - Executes client implementations in controlled scenarios and validates their behavior
|
||||
2. **Server Testing** - Tests server implementations by acting as an MCP client and validating responses
|
||||
|
||||
## Architecture
|
||||
|
||||
### File Structure
|
||||
|
||||
```
|
||||
src/runner/
|
||||
├── index.ts # Exports all public functions
|
||||
├── client.ts # Client testing implementation
|
||||
├── server.ts # Server testing implementation
|
||||
└── utils.ts # Shared utilities (formatting, file I/O, ANSI colors)
|
||||
```
|
||||
|
||||
### Components
|
||||
|
||||
**Client Testing:**
|
||||
|
||||
1. **Runner** - Orchestrates test execution
|
||||
2. **Test Server** - MCP server (and optionally auth server for auth scenarios)
|
||||
3. **Client Process** - The MCP client implementation under test
|
||||
4. **Scenario** - A specific test case with expected behaviors
|
||||
5. **Checks** - Validation functions that produce ConformanceCheck results
|
||||
|
||||
**Server Testing:**
|
||||
|
||||
1. **Runner** - Orchestrates test execution
|
||||
2. **Test Client** - MCP SDK client that connects to the server under test
|
||||
3. **Server Process** - The MCP server implementation under test (external)
|
||||
4. **Scenario** - A specific test case with expected server behaviors
|
||||
5. **Checks** - Validation functions that produce ConformanceCheck results
|
||||
|
||||
### Execution Flow
|
||||
|
||||
**Client Testing:**
|
||||
|
||||
```
|
||||
1. Runner starts test server(s) on available port(s)
|
||||
2. Runner spawns client process with server URL as final argument
|
||||
|
|
@ -25,44 +50,124 @@ The conformance test runner is a framework for testing MCP client implementation
|
|||
6. Results are written to results/<scenario>-<timestamp>/
|
||||
```
|
||||
|
||||
## CLI Interface
|
||||
**Server Testing:**
|
||||
|
||||
```bash
|
||||
npm run start -- --command "tsx examples/clients/typescript/test1.ts --verbose" --scenario initialize
|
||||
```
|
||||
1. User starts their server implementation
|
||||
2. Runner connects to server URL as an MCP client
|
||||
3. Runner sends requests and captures responses
|
||||
4. Scenario-specific checks validate server behavior
|
||||
5. Results are written to results/server-<scenario>-<timestamp>/
|
||||
```
|
||||
|
||||
### Arguments
|
||||
## CLI Interface
|
||||
|
||||
The conformance suite provides a unified CLI with two main commands:
|
||||
|
||||
### Client Testing
|
||||
|
||||
```bash
|
||||
# Run a single client scenario
|
||||
npm run start -- client --command "tsx examples/clients/typescript/test1.ts" --scenario initialize
|
||||
|
||||
# With verbose output
|
||||
npm run start -- client --command "tsx examples/clients/typescript/test1.ts" --scenario initialize --verbose
|
||||
|
||||
# With custom timeout
|
||||
npm run start -- client --command "tsx examples/clients/typescript/test1.ts" --scenario initialize --timeout 60000
|
||||
```
|
||||
|
||||
**Arguments:**
|
||||
|
||||
- `--command` - The command to run the client (can include existing flags)
|
||||
- `--scenario` - The scenario to test (e.g., "initialize", "list-tools", "call-tool")
|
||||
- `--scenario` - The scenario to test (e.g., "initialize", "tools-call")
|
||||
- `--timeout` - Timeout in milliseconds (default: 30000)
|
||||
- `--verbose` - Show verbose output (JSON format)
|
||||
|
||||
The runner will append the server URL as the final argument to the command.
|
||||
|
||||
### Server Testing
|
||||
|
||||
```bash
|
||||
# Run a single server scenario
|
||||
npm run start -- server --url http://localhost:3000/mcp --scenario server-initialize
|
||||
|
||||
# Run all server scenarios (default when no --scenario specified)
|
||||
npm run start -- server --url http://localhost:3000/mcp
|
||||
```
|
||||
|
||||
**Arguments:**
|
||||
|
||||
- `--url` - URL of the server to test
|
||||
- `--scenario <scenario>` - Scenario to test (optional, defaults to all scenarios if not specified)
|
||||
|
||||
### List Available Scenarios
|
||||
|
||||
```bash
|
||||
# List all scenarios
|
||||
npm run start -- list
|
||||
|
||||
# List only client scenarios
|
||||
npm run start -- list --client
|
||||
|
||||
# List only server scenarios
|
||||
npm run start -- list --server
|
||||
```
|
||||
|
||||
## Validation
|
||||
|
||||
All CLI arguments are validated using Zod schemas (`src/schemas.ts`) before being passed to runner functions:
|
||||
|
||||
**Client Validation:**
|
||||
|
||||
- Command is non-empty
|
||||
- Scenario exists in available scenarios
|
||||
- Timeout is a positive integer
|
||||
|
||||
**Server Validation:**
|
||||
|
||||
- URL is a valid HTTP/HTTPS URL
|
||||
- All specified scenarios exist (if provided)
|
||||
- Defaults to all scenarios when no `--scenario` is specified
|
||||
|
||||
## Scenarios
|
||||
|
||||
A scenario represents a specific test case that validates one or more aspects of MCP client behavior. Each scenario:
|
||||
A scenario represents a specific test case that validates one or more aspects of MCP behavior. Each scenario:
|
||||
|
||||
- Configures the test server with expected behavior
|
||||
- Configures the test environment with expected behavior
|
||||
- May run multiple conformance checks
|
||||
- Validates both client behavior and server responses
|
||||
- Validates both request/response patterns and protocol compliance
|
||||
|
||||
Example scenarios:
|
||||
### Client Scenario Examples
|
||||
|
||||
- `initialize` - Tests client initialization handshake
|
||||
- `list-tools` - Tests tool discovery
|
||||
- `call-tool` - Tests tool invocation
|
||||
- `auth-flow` - Tests OAuth/authorization flows (requires auth server)
|
||||
- `tools-call` - Tests tool invocation
|
||||
- `auth/basic-dcr` - Tests OAuth Dynamic Client Registration flow
|
||||
- `auth/basic-metadata-var1` - Tests OAuth with authorization metadata variation 1
|
||||
|
||||
### Server Scenario Examples
|
||||
|
||||
- `server-initialize` - Tests server initialization and capabilities
|
||||
- `tools-list` - Tests tool listing endpoint
|
||||
- `tools-call-simple-text` - Tests tool invocation with text response
|
||||
- `resources-list` - Tests resource listing
|
||||
- `prompts-list` - Tests prompt listing
|
||||
- `logging-set-level` - Tests logging level configuration
|
||||
|
||||
## Output Structure
|
||||
|
||||
Results are written to: `results/<scenario>-<timestamp>/`
|
||||
Results are written to: `results/<scenario>-<timestamp>/` or `results/server-<scenario>-<timestamp>/`
|
||||
|
||||
Files:
|
||||
**Client Testing Files:**
|
||||
|
||||
- `checks.json` - Array of ConformanceCheck objects with validation results
|
||||
- `stdout.txt` - Complete stdout from the client process
|
||||
- `stderr.txt` - Complete stderr from the client process
|
||||
|
||||
**Server Testing Files:**
|
||||
|
||||
- `checks.json` - Array of ConformanceCheck objects with validation results
|
||||
|
||||
### checks.json Format
|
||||
|
||||
```json
|
||||
|
|
@ -84,6 +189,30 @@ Files:
|
|||
]
|
||||
```
|
||||
|
||||
## Programmatic Usage
|
||||
|
||||
The runner can also be used programmatically:
|
||||
|
||||
```typescript
|
||||
import { runConformanceTest, printClientResults, runServerConformanceTest, printServerResults } from './runner/index.js';
|
||||
|
||||
// Test a client
|
||||
const clientResult = await runConformanceTest('tsx my-client.ts', 'initialize', 30000);
|
||||
const { failed } = printClientResults(clientResult.checks);
|
||||
|
||||
// Test a server
|
||||
const serverResult = await runServerConformanceTest('http://localhost:3000/mcp', 'server-initialize');
|
||||
const { failed } = printServerResults(serverResult.checks);
|
||||
```
|
||||
|
||||
## Shared Utilities
|
||||
|
||||
The `utils.ts` module provides shared functionality:
|
||||
|
||||
- **File Operations:** `ensureResultsDir()`, `createResultDir()`
|
||||
- **Formatting:** `formatPrettyChecks()` for colored console output
|
||||
- **Styling:** ANSI color constants and helpers
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
- **Test Suites** - Group multiple scenarios for convenience
|
||||
|
|
@ -91,3 +220,5 @@ Files:
|
|||
- **Parallel Execution** - Run multiple scenarios concurrently
|
||||
- **Custom Servers** - Allow custom MCP server implementations for advanced scenarios
|
||||
- **Report Formats** - HTML, Markdown, or other human-readable formats
|
||||
- **Watch Mode** - Automatically re-run tests on file changes
|
||||
- **Coverage Tracking** - Track which parts of the spec are tested
|
||||
|
|
|
|||
207
src/runner/client.ts
Normal file
207
src/runner/client.ts
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { ConformanceCheck } from '../types';
|
||||
import { getScenario } from '../scenarios';
|
||||
import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils';
|
||||
|
||||
export interface ClientExecutionResult {
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
async function executeClient(
|
||||
command: string,
|
||||
serverUrl: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ClientExecutionResult> {
|
||||
const commandParts = command.split(' ');
|
||||
const executable = commandParts[0];
|
||||
const args = [...commandParts.slice(1), serverUrl];
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let timedOut = false;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const process = spawn(executable, args, {
|
||||
shell: true,
|
||||
stdio: 'pipe'
|
||||
});
|
||||
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
timedOut = true;
|
||||
process.kill();
|
||||
}, timeout);
|
||||
|
||||
if (process.stdout) {
|
||||
process.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
if (process.stderr) {
|
||||
process.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
process.on('close', (code) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: code || 0,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
|
||||
process.on('error', (error) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
stdout,
|
||||
stderr: stderr + `\nProcess error: ${error.message}`,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runConformanceTest(
|
||||
clientCommand: string,
|
||||
scenarioName: string,
|
||||
timeout: number = 30000
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
clientOutput: ClientExecutionResult;
|
||||
resultDir: string;
|
||||
}> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
// Scenario is guaranteed to exist by CLI validation
|
||||
const scenario = getScenario(scenarioName)!;
|
||||
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
|
||||
console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
|
||||
|
||||
try {
|
||||
const clientOutput = await executeClient(
|
||||
clientCommand,
|
||||
urls.serverUrl,
|
||||
timeout
|
||||
);
|
||||
|
||||
// Print stdout/stderr if client exited with nonzero code
|
||||
if (clientOutput.exitCode !== 0) {
|
||||
console.error(`\nClient exited with code ${clientOutput.exitCode}`);
|
||||
if (clientOutput.stdout) {
|
||||
console.error(`\nStdout:\n${clientOutput.stdout}`);
|
||||
}
|
||||
if (clientOutput.stderr) {
|
||||
console.error(`\nStderr:\n${clientOutput.stderr}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (clientOutput.timedOut) {
|
||||
console.error(`\nClient timed out after ${timeout}ms`);
|
||||
}
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
|
||||
await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
|
||||
|
||||
await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
|
||||
|
||||
console.log(`Results saved to ${resultDir}`);
|
||||
|
||||
return {
|
||||
checks,
|
||||
clientOutput,
|
||||
resultDir
|
||||
};
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export function printClientResults(
|
||||
checks: ConformanceCheck[],
|
||||
verbose: boolean = false
|
||||
): { passed: number; failed: number; denominator: number } {
|
||||
const denominator = checks.filter(
|
||||
(c) => c.status === 'SUCCESS' || c.status === 'FAILURE'
|
||||
).length;
|
||||
const passed = checks.filter((c) => c.status === 'SUCCESS').length;
|
||||
const failed = checks.filter((c) => c.status === 'FAILURE').length;
|
||||
|
||||
if (verbose) {
|
||||
console.log(`Checks:\n${JSON.stringify(checks, null, 2)}`);
|
||||
} else {
|
||||
console.log(`Checks:\n${formatPrettyChecks(checks)}`);
|
||||
}
|
||||
|
||||
console.log(`\nTest Results:`);
|
||||
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
checks
|
||||
.filter((c) => c.status === 'FAILURE')
|
||||
.forEach((c) => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { passed, failed, denominator };
|
||||
}
|
||||
|
||||
export async function runInteractiveMode(scenarioName: string): Promise<void> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
// Scenario is guaranteed to exist by CLI validation
|
||||
const scenario = getScenario(scenarioName)!;
|
||||
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
|
||||
console.log(`Server URL: ${urls.serverUrl}`);
|
||||
console.log('Press Ctrl+C to stop and save checks...');
|
||||
|
||||
const handleShutdown = async () => {
|
||||
console.log('\nShutting down...');
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
|
||||
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
|
||||
console.log(`\nChecks saved to ${resultDir}/checks.json`);
|
||||
|
||||
await scenario.stop();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGINT', handleShutdown);
|
||||
process.on('SIGTERM', handleShutdown);
|
||||
|
||||
await new Promise(() => {});
|
||||
}
|
||||
|
|
@ -1,312 +1,23 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { ConformanceCheck } from '../types.js';
|
||||
import { getScenario } from '../scenarios/index.js';
|
||||
// Export client functions
|
||||
export {
|
||||
runConformanceTest,
|
||||
printClientResults,
|
||||
runInteractiveMode,
|
||||
type ClientExecutionResult
|
||||
} from './client';
|
||||
|
||||
// ANSI color codes
|
||||
const COLORS = {
|
||||
RESET: '\x1b[0m',
|
||||
GRAY: '\x1b[90m',
|
||||
GREEN: '\x1b[32m',
|
||||
YELLOW: '\x1b[33m',
|
||||
RED: '\x1b[31m',
|
||||
BLUE: '\x1b[36m'
|
||||
};
|
||||
// Export server functions
|
||||
export {
|
||||
runServerConformanceTest,
|
||||
printServerResults,
|
||||
printServerSummary
|
||||
} from './server';
|
||||
|
||||
function getStatusColor(status: string): string {
|
||||
switch (status) {
|
||||
case 'SUCCESS':
|
||||
return COLORS.GREEN;
|
||||
case 'FAILURE':
|
||||
return COLORS.RED;
|
||||
case 'INFO':
|
||||
return COLORS.BLUE;
|
||||
default:
|
||||
return COLORS.RESET;
|
||||
}
|
||||
}
|
||||
|
||||
function formatPrettyChecks(checks: ConformanceCheck[]): string {
|
||||
// Find the longest id and status for column alignment
|
||||
const maxIdLength = Math.max(...checks.map((c) => c.id.length));
|
||||
const maxStatusLength = Math.max(...checks.map((c) => c.status.length));
|
||||
|
||||
return checks
|
||||
.map((check) => {
|
||||
const timestamp = `${COLORS.GRAY}${check.timestamp}${COLORS.RESET}`;
|
||||
const id = check.id.padEnd(maxIdLength);
|
||||
const statusColor = getStatusColor(check.status);
|
||||
const status = `${statusColor}${check.status.padEnd(maxStatusLength)}${COLORS.RESET}`;
|
||||
const description = check.description;
|
||||
const line = `${timestamp} [${id}] ${status} ${description}`;
|
||||
// Add newline after outgoing responses for better visual separation
|
||||
return (
|
||||
line +
|
||||
(check.id.includes('outgoing') && check.id.includes('response')
|
||||
? '\n'
|
||||
: '')
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export interface ClientExecutionResult {
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
}
|
||||
|
||||
async function ensureResultsDir(): Promise<string> {
|
||||
const resultsDir = path.join(process.cwd(), 'results');
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
return resultsDir;
|
||||
}
|
||||
|
||||
function createResultDir(scenario: string): string {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
return path.join('results', `${scenario}-${timestamp}`);
|
||||
}
|
||||
|
||||
async function executeClient(
|
||||
command: string,
|
||||
serverUrl: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ClientExecutionResult> {
|
||||
const commandParts = command.split(' ');
|
||||
const executable = commandParts[0];
|
||||
const args = [...commandParts.slice(1), serverUrl];
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let timedOut = false;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const process = spawn(executable, args, {
|
||||
shell: true,
|
||||
stdio: 'pipe'
|
||||
});
|
||||
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
timedOut = true;
|
||||
process.kill();
|
||||
}, timeout);
|
||||
|
||||
if (process.stdout) {
|
||||
process.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
if (process.stderr) {
|
||||
process.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
}
|
||||
|
||||
process.on('close', (code) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: code || 0,
|
||||
stdout,
|
||||
stderr,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
|
||||
process.on('error', (error) => {
|
||||
clearTimeout(timeoutHandle);
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
stdout,
|
||||
stderr: stderr + `\nProcess error: ${error.message}`,
|
||||
timedOut
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runConformanceTest(
|
||||
clientCommand: string,
|
||||
scenarioName: string,
|
||||
timeout: number = 30000
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
clientOutput: ClientExecutionResult;
|
||||
resultDir: string;
|
||||
}> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
const scenario = getScenario(scenarioName);
|
||||
if (!scenario) {
|
||||
throw new Error(`Unknown scenario: ${scenarioName}`);
|
||||
}
|
||||
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
|
||||
console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`);
|
||||
|
||||
try {
|
||||
const clientOutput = await executeClient(
|
||||
clientCommand,
|
||||
urls.serverUrl,
|
||||
timeout
|
||||
);
|
||||
|
||||
// Print stdout/stderr if client exited with nonzero code
|
||||
if (clientOutput.exitCode !== 0) {
|
||||
console.error(`\nClient exited with code ${clientOutput.exitCode}`);
|
||||
if (clientOutput.stdout) {
|
||||
console.error(`\nStdout:\n${clientOutput.stdout}`);
|
||||
}
|
||||
if (clientOutput.stderr) {
|
||||
console.error(`\nStderr:\n${clientOutput.stderr}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (clientOutput.timedOut) {
|
||||
console.error(`\nClient timed out after ${timeout}ms`);
|
||||
}
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
|
||||
await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
|
||||
|
||||
await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
|
||||
|
||||
console.log(`Results saved to ${resultDir}`);
|
||||
|
||||
return {
|
||||
checks,
|
||||
clientOutput,
|
||||
resultDir
|
||||
};
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function runInteractiveMode(scenarioName: string): Promise<void> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName);
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
const scenario = getScenario(scenarioName);
|
||||
if (!scenario) {
|
||||
throw new Error(`Unknown scenario: ${scenarioName}`);
|
||||
}
|
||||
|
||||
console.log(`Starting scenario: ${scenarioName}`);
|
||||
const urls = await scenario.start();
|
||||
|
||||
console.log(`Server URL: ${urls.serverUrl}`);
|
||||
console.log('Press Ctrl+C to stop and save checks...');
|
||||
|
||||
const handleShutdown = async () => {
|
||||
console.log('\nShutting down...');
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
|
||||
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
|
||||
console.log(`\nChecks saved to ${resultDir}/checks.json`);
|
||||
|
||||
await scenario.stop();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGINT', handleShutdown);
|
||||
process.on('SIGTERM', handleShutdown);
|
||||
|
||||
await new Promise(() => {});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
let command: string | null = null;
|
||||
let scenario: string | null = null;
|
||||
let verbose = false;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--command' && i + 1 < args.length) {
|
||||
command = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--scenario' && i + 1 < args.length) {
|
||||
scenario = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] === '--verbose') {
|
||||
verbose = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!scenario) {
|
||||
console.error(
|
||||
'Usage: runner --scenario <scenario> [--command "<command>"] [--verbose]'
|
||||
);
|
||||
console.error(
|
||||
'Example: runner --scenario initialize --command "tsx examples/clients/typescript/test1.ts"'
|
||||
);
|
||||
console.error('Or run without --command for interactive mode');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!command) {
|
||||
try {
|
||||
await runInteractiveMode(scenario);
|
||||
} catch (error) {
|
||||
console.error('Interactive mode error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await runConformanceTest(command, scenario);
|
||||
|
||||
const denominator = result.checks.filter(
|
||||
(c) => c.status === 'SUCCESS' || c.status == 'FAILURE'
|
||||
).length;
|
||||
const passed = result.checks.filter((c) => c.status === 'SUCCESS').length;
|
||||
const failed = result.checks.filter((c) => c.status === 'FAILURE').length;
|
||||
|
||||
if (verbose) {
|
||||
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
|
||||
} else {
|
||||
console.log(`Checks:\n${formatPrettyChecks(result.checks)}`);
|
||||
}
|
||||
|
||||
console.log(`\nTest Results:`);
|
||||
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
result.checks
|
||||
.filter((c) => c.status === 'FAILURE')
|
||||
.forEach((c) => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
} catch (error) {
|
||||
console.error('Test runner error:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
// Export utilities
|
||||
export {
|
||||
ensureResultsDir,
|
||||
createResultDir,
|
||||
formatPrettyChecks,
|
||||
getStatusColor,
|
||||
COLORS
|
||||
} from './utils';
|
||||
|
|
|
|||
93
src/runner/server.ts
Normal file
93
src/runner/server.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { ConformanceCheck } from '../types';
|
||||
import { getClientScenario } from '../scenarios';
|
||||
import { ensureResultsDir, createResultDir } from './utils';
|
||||
|
||||
export async function runServerConformanceTest(
|
||||
serverUrl: string,
|
||||
scenarioName: string
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
resultDir: string;
|
||||
}> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName, 'server');
|
||||
await fs.mkdir(resultDir, { recursive: true });
|
||||
|
||||
// Scenario is guaranteed to exist by CLI validation
|
||||
const scenario = getClientScenario(scenarioName)!;
|
||||
|
||||
console.log(
|
||||
`Running client scenario '${scenarioName}' against server: ${serverUrl}`
|
||||
);
|
||||
|
||||
const checks = await scenario.run(serverUrl);
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(resultDir, 'checks.json'),
|
||||
JSON.stringify(checks, null, 2)
|
||||
);
|
||||
|
||||
console.log(`Results saved to ${resultDir}`);
|
||||
|
||||
return {
|
||||
checks,
|
||||
resultDir
|
||||
};
|
||||
}
|
||||
|
||||
export function printServerResults(checks: ConformanceCheck[]): {
|
||||
passed: number;
|
||||
failed: number;
|
||||
denominator: number;
|
||||
} {
|
||||
const denominator = checks.filter(
|
||||
(c) => c.status === 'SUCCESS' || c.status === 'FAILURE'
|
||||
).length;
|
||||
const passed = checks.filter((c) => c.status === 'SUCCESS').length;
|
||||
const failed = checks.filter((c) => c.status === 'FAILURE').length;
|
||||
|
||||
console.log(`Checks:\n${JSON.stringify(checks, null, 2)}`);
|
||||
|
||||
console.log(`\nTest Results:`);
|
||||
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
checks
|
||||
.filter((c) => c.status === 'FAILURE')
|
||||
.forEach((c) => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { passed, failed, denominator };
|
||||
}
|
||||
|
||||
export function printServerSummary(
|
||||
allResults: { scenario: string; checks: ConformanceCheck[] }[]
|
||||
): { totalPassed: number; totalFailed: number } {
|
||||
console.log('\n\n=== SUMMARY ===');
|
||||
let totalPassed = 0;
|
||||
let totalFailed = 0;
|
||||
|
||||
for (const result of allResults) {
|
||||
const passed = result.checks.filter((c) => c.status === 'SUCCESS').length;
|
||||
const failed = result.checks.filter((c) => c.status === 'FAILURE').length;
|
||||
totalPassed += passed;
|
||||
totalFailed += failed;
|
||||
|
||||
const status = failed === 0 ? '✓' : '✗';
|
||||
console.log(
|
||||
`${status} ${result.scenario}: ${passed} passed, ${failed} failed`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`);
|
||||
|
||||
return { totalPassed, totalFailed };
|
||||
}
|
||||
62
src/runner/utils.ts
Normal file
62
src/runner/utils.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { ConformanceCheck } from '../types';
|
||||
|
||||
// ANSI color codes
|
||||
export const COLORS = {
|
||||
RESET: '\x1b[0m',
|
||||
GRAY: '\x1b[90m',
|
||||
GREEN: '\x1b[32m',
|
||||
YELLOW: '\x1b[33m',
|
||||
RED: '\x1b[31m',
|
||||
BLUE: '\x1b[36m'
|
||||
};
|
||||
|
||||
export function getStatusColor(status: string): string {
|
||||
switch (status) {
|
||||
case 'SUCCESS':
|
||||
return COLORS.GREEN;
|
||||
case 'FAILURE':
|
||||
return COLORS.RED;
|
||||
case 'INFO':
|
||||
return COLORS.BLUE;
|
||||
default:
|
||||
return COLORS.RESET;
|
||||
}
|
||||
}
|
||||
|
||||
export function formatPrettyChecks(checks: ConformanceCheck[]): string {
|
||||
// Find the longest id and status for column alignment
|
||||
const maxIdLength = Math.max(...checks.map((c) => c.id.length));
|
||||
const maxStatusLength = Math.max(...checks.map((c) => c.status.length));
|
||||
|
||||
return checks
|
||||
.map((check) => {
|
||||
const timestamp = `${COLORS.GRAY}${check.timestamp}${COLORS.RESET}`;
|
||||
const id = check.id.padEnd(maxIdLength);
|
||||
const statusColor = getStatusColor(check.status);
|
||||
const status = `${statusColor}${check.status.padEnd(maxStatusLength)}${COLORS.RESET}`;
|
||||
const description = check.description;
|
||||
const line = `${timestamp} [${id}] ${status} ${description}`;
|
||||
// Add newline after outgoing responses for better visual separation
|
||||
return (
|
||||
line +
|
||||
(check.id.includes('outgoing') && check.id.includes('response')
|
||||
? '\n'
|
||||
: '')
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export async function ensureResultsDir(): Promise<string> {
|
||||
const resultsDir = path.join(process.cwd(), 'results');
|
||||
await fs.mkdir(resultsDir, { recursive: true });
|
||||
return resultsDir;
|
||||
}
|
||||
|
||||
export function createResultDir(scenario: string, prefix = ''): string {
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const dirName = prefix ? `${prefix}-${scenario}` : scenario;
|
||||
return path.join('results', `${dirName}-${timestamp}`);
|
||||
}
|
||||
Loading…
Reference in a new issue