feat: add --expected-failures baseline and composite GitHub Action (#113)

Add support for SDKs that don't yet pass all conformance tests to still
run conformance in CI without failing, by specifying a YAML baseline file
of known/expected failures.

CLI changes:
- Add --expected-failures <path> flag to both server and client commands
- Exit 0 if only expected failures occur
- Exit 1 if unexpected failures occur (regression)
- Exit 1 if an expected failure now passes (stale baseline, must update)

GitHub Action (action.yml):
- Composite action that builds from source and runs conformance tests
- SDKs reference it as: uses: modelcontextprotocol/conformance@<ref>
- Inputs: mode, url, command, expected-failures, suite, scenario, etc.

New files:
- src/expected-failures.ts: YAML loading, baseline evaluation, reporting
- src/expected-failures.test.ts: 16 unit tests
- action.yml: composite GitHub Action

Closes #99
This commit is contained in:
Max Isbey 2026-01-22 10:41:04 +00:00 • committed by GitHub
commit 83ff334e0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -20,6 +20,11 @@ import {
} from './scenarios';
import { ConformanceCheck } from './types';
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
import {
loadExpectedFailures,
evaluateBaseline,
printBaselineResults
} from './expected-failures';
import packageJson from '../package.json';
const program = new Command();
@ -39,6 +44,10 @@ program
.option('--scenario <scenario>', 'Scenario to test')
.option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
.option(
'--expected-failures <path>',
'Path to YAML file listing expected failures (baseline)'
)
.option('-o, --output-dir <path>', 'Save results to this directory')
.option('--verbose', 'Show verbose output')
.action(async (options) => {
@ -149,6 +158,17 @@ program
console.log(
`\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings`
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.client ?? [];
const baselineResult = evaluateBaseline(results, baselineScenarios);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0);
}
@ -183,6 +203,20 @@ program
verbose,
result.clientOutput
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.client ?? [];
const baselineResult = evaluateBaseline(
[{ scenario: validated.scenario, checks: result.checks }],
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(overallFailure ? 1 : 0);
} catch (error) {
if (error instanceof ZodError) {
@ -213,6 +247,10 @@ program
'Suite to run: "active" (default, excludes pending), "all", or "pending"',
'active'
)
.option(
'--expected-failures <path>',
'Path to YAML file listing expected failures (baseline)'
)
.option('-o, --output-dir <path>', 'Save results to this directory')
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options) => {
@ -236,6 +274,20 @@ program
result.scenarioDescription,
verbose
);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.server ?? [];
const baselineResult = evaluateBaseline(
[{ scenario: validated.scenario!, checks: result.checks }],
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(failed > 0 ? 1 : 0);
} else {
// Run scenarios based on suite
@ -290,6 +342,20 @@ program
}
const { totalFailed } = printServerSummary(allResults);
if (options.expectedFailures) {
const expectedFailuresConfig = await loadExpectedFailures(
options.expectedFailures
);
const baselineScenarios = expectedFailuresConfig.server ?? [];
const baselineResult = evaluateBaseline(
allResults,
baselineScenarios
);
printBaselineResults(baselineResult);
process.exit(baselineResult.exitCode);
}
process.exit(totalFailed > 0 ? 1 : 0);
}
} catch (error) {