modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: make results directory output optional (#107)

Browse source 
Add -o, --output-dir option to both client and server commands.
By default, no results directory is created - only console output.
When -o is specified, results are saved to that directory with
timestamped subdirectories.

This eliminates the need to add 'results/' to .gitignore in every
repo where conformance tests are run.

Claude-Generated-By: Claude Code (cli/claude-opus-4-5=100%)
Claude-Steers: 0
Claude-Permission-Prompts: 2
Claude-Escapes: 0
This commit is contained in:
Paul Carleton 2026-01-14 20:16:28 +00:00 • committed by GitHub
commit 62875097f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 81 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/index.ts
View file

@ -39,11 +39,13 @@ program
.option('--scenario <scenario>', 'Scenario to test')
.option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
.option('-o, --output-dir <path>', 'Save results to this directory')
.option('--verbose', 'Show verbose output')
.action(async (options) => {
try {
const timeout = parseInt(options.timeout, 10);
const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
// Handle suite mode
if (options.suite) {
@ -78,7 +80,8 @@ program
const result = await runConformanceTest(
options.command,
scenarioName,
timeout
timeout,
outputDir
);
return {
scenario: scenarioName,
@ -163,7 +166,7 @@ program
// If no command provided, run in interactive mode
if (!validated.command) {
await runInteractiveMode(validated.scenario, verbose);
await runInteractiveMode(validated.scenario, verbose, outputDir);
process.exit(0);
}
@ -171,7 +174,8 @@ program
const result = await runConformanceTest(
validated.command,
validated.scenario,
timeout
timeout,
outputDir
);
const { overallFailure } = printClientResults(
@ -209,6 +213,7 @@ program
'Suite to run: "active" (default, excludes pending), "all", or "pending"',
'active'
)
.option('-o, --output-dir <path>', 'Save results to this directory')
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options) => {
try {
@ -216,12 +221,14 @@ program
const validated = ServerOptionsSchema.parse(options);
const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
// If a single scenario is specified, run just that one
if (validated.scenario) {
const result = await runServerConformanceTest(
validated.url,
validated.scenario
validated.scenario,
outputDir
);
const { failed } = printServerResults(
@ -259,7 +266,8 @@ program
try {
const result = await runServerConformanceTest(
validated.url,
scenarioName
scenarioName,
outputDir
);
allResults.push({ scenario: scenarioName, checks: result.checks });
} catch (error) {

68
src/runner/client.ts
View file

@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
import path from 'path';
import { ConformanceCheck } from '../types';
import { getScenario } from '../scenarios';
import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils';
import { createResultDir, formatPrettyChecks } from './utils';
export interface ClientExecutionResult {
exitCode: number;
@ -91,15 +91,19 @@ async function executeClient(
export async function runConformanceTest(
clientCommand: string,
scenarioName: string,
timeout: number = 30000
timeout: number = 30000,
outputDir?: string
): Promise<{
checks: ConformanceCheck[];
clientOutput: ClientExecutionResult;
resultDir: string;
resultDir?: string;
}> {
await ensureResultsDir();
const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true });
let resultDir: string | undefined;
if (outputDir) {
resultDir = createResultDir(outputDir, scenarioName);
await fs.mkdir(resultDir, { recursive: true });
}
// Scenario is guaranteed to exist by CLI validation
const scenario = getScenario(scenarioName)!;
@ -138,16 +142,24 @@ export async function runConformanceTest(
const checks = scenario.getChecks();
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
if (resultDir) {
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
await fs.writeFile(
path.join(resultDir, 'stdout.txt'),
clientOutput.stdout
);
await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
await fs.writeFile(
path.join(resultDir, 'stderr.txt'),
clientOutput.stderr
);
console.error(`Results saved to ${resultDir}`);
console.error(`Results saved to ${resultDir}`);
}
return {
checks,
@ -244,11 +256,15 @@ export function printClientResults(
export async function runInteractiveMode(
scenarioName: string,
verbose: boolean = false
verbose: boolean = false,
outputDir?: string
): Promise<void> {
await ensureResultsDir();
const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true });
let resultDir: string | undefined;
if (outputDir) {
resultDir = createResultDir(outputDir, scenarioName);
await fs.mkdir(resultDir, { recursive: true });
}
// Scenario is guaranteed to exist by CLI validation
const scenario = getScenario(scenarioName)!;
@ -257,23 +273,29 @@ export async function runInteractiveMode(
const urls = await scenario.start();
console.log(`Server URL: ${urls.serverUrl}`);
console.log('Press Ctrl+C to stop and save checks...');
console.log('Press Ctrl+C to stop...');
const handleShutdown = async () => {
console.log('\nShutting down...');
const checks = scenario.getChecks();
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
if (resultDir) {
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
}
if (verbose) {
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
} else {
console.log(`\nChecks:\n${formatPrettyChecks(checks)}`);
}
console.log(`\nChecks saved to ${resultDir}/checks.json`);
if (resultDir) {
console.log(`\nChecks saved to ${resultDir}/checks.json`);
}
await scenario.stop();
process.exit(0);

1
src/runner/index.ts
View file

@ -15,7 +15,6 @@ export {
// Export utilities
export {
ensureResultsDir,
createResultDir,
formatPrettyChecks,
getStatusColor,

28
src/runner/server.ts
View file

@ -2,7 +2,7 @@ import { promises as fs } from 'fs';
import path from 'path';
import { ConformanceCheck } from '../types';
import { getClientScenario } from '../scenarios';
import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils';
import { createResultDir, formatPrettyChecks } from './utils';
/**
* Format markdown-style text for terminal output using ANSI codes
@ -19,15 +19,19 @@ function formatMarkdown(text: string): string {
export async function runServerConformanceTest(
serverUrl: string,
scenarioName: string
scenarioName: string,
outputDir?: string
): Promise<{
checks: ConformanceCheck[];
resultDir: string;
resultDir?: string;
scenarioDescription: string;
}> {
await ensureResultsDir();
const resultDir = createResultDir(scenarioName, 'server');
await fs.mkdir(resultDir, { recursive: true });
let resultDir: string | undefined;
if (outputDir) {
resultDir = createResultDir(outputDir, scenarioName, 'server');
await fs.mkdir(resultDir, { recursive: true });
}
// Scenario is guaranteed to exist by CLI validation
const scenario = getClientScenario(scenarioName)!;
@ -38,12 +42,14 @@ export async function runServerConformanceTest(
const checks = await scenario.run(serverUrl);
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
if (resultDir) {
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
console.log(`Results saved to ${resultDir}`);
console.log(`Results saved to ${resultDir}`);
}
return {
checks,

15
src/runner/utils.ts
View file

@ -1,4 +1,3 @@
import { promises as fs } from 'fs';
import path from 'path';
import { ConformanceCheck } from '../types';
@ -51,14 +50,12 @@ export function formatPrettyChecks(checks: ConformanceCheck[]): string {
.join('\n');
}
export async function ensureResultsDir(): Promise<string> {
const resultsDir = path.join(process.cwd(), 'results');
await fs.mkdir(resultsDir, { recursive: true });
return resultsDir;
}
export function createResultDir(scenario: string, prefix = ''): string {
export function createResultDir(
baseDir: string,
scenario: string,
prefix = ''
): string {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const dirName = prefix ? `${prefix}-${scenario}` : scenario;
return path.join('results', `${dirName}-${timestamp}`);
return path.join(baseDir, `${dirName}-${timestamp}`);
}