feat: add --expected-failures baseline and composite GitHub Action (#113)
Add support for SDKs that don't yet pass all conformance tests to still run conformance in CI without failing, by specifying a YAML baseline file of known/expected failures. CLI changes: - Add --expected-failures <path> flag to both server and client commands - Exit 0 if only expected failures occur - Exit 1 if unexpected failures occur (regression) - Exit 1 if an expected failure now passes (stale baseline, must update) GitHub Action (action.yml): - Composite action that builds from source and runs conformance tests - SDKs reference it as: uses: modelcontextprotocol/conformance@<ref> - Inputs: mode, url, command, expected-failures, suite, scenario, etc. New files: - src/expected-failures.ts: YAML loading, baseline evaluation, reporting - src/expected-failures.test.ts: 16 unit tests - action.yml: composite GitHub Action Closes #99
This commit is contained in:
parent
cc784d96e2
commit
83ff334e0e
1 changed files with 697 additions and 1 deletions
66
src/index.ts
66
src/index.ts
|
|
@ -20,6 +20,11 @@ import {
|
|||
} from './scenarios';
|
||||
import { ConformanceCheck } from './types';
|
||||
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
|
||||
import {
|
||||
loadExpectedFailures,
|
||||
evaluateBaseline,
|
||||
printBaselineResults
|
||||
} from './expected-failures';
|
||||
import packageJson from '../package.json';
|
||||
|
||||
const program = new Command();
|
||||
|
|
@ -39,6 +44,10 @@ program
|
|||
.option('--scenario <scenario>', 'Scenario to test')
|
||||
.option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
|
||||
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
|
||||
.option(
|
||||
'--expected-failures <path>',
|
||||
'Path to YAML file listing expected failures (baseline)'
|
||||
)
|
||||
.option('-o, --output-dir <path>', 'Save results to this directory')
|
||||
.option('--verbose', 'Show verbose output')
|
||||
.action(async (options) => {
|
||||
|
|
@ -149,6 +158,17 @@ program
|
|||
console.log(
|
||||
`\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings`
|
||||
);
|
||||
|
||||
if (options.expectedFailures) {
|
||||
const expectedFailuresConfig = await loadExpectedFailures(
|
||||
options.expectedFailures
|
||||
);
|
||||
const baselineScenarios = expectedFailuresConfig.client ?? [];
|
||||
const baselineResult = evaluateBaseline(results, baselineScenarios);
|
||||
printBaselineResults(baselineResult);
|
||||
process.exit(baselineResult.exitCode);
|
||||
}
|
||||
|
||||
process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
|
|
@ -183,6 +203,20 @@ program
|
|||
verbose,
|
||||
result.clientOutput
|
||||
);
|
||||
|
||||
if (options.expectedFailures) {
|
||||
const expectedFailuresConfig = await loadExpectedFailures(
|
||||
options.expectedFailures
|
||||
);
|
||||
const baselineScenarios = expectedFailuresConfig.client ?? [];
|
||||
const baselineResult = evaluateBaseline(
|
||||
[{ scenario: validated.scenario, checks: result.checks }],
|
||||
baselineScenarios
|
||||
);
|
||||
printBaselineResults(baselineResult);
|
||||
process.exit(baselineResult.exitCode);
|
||||
}
|
||||
|
||||
process.exit(overallFailure ? 1 : 0);
|
||||
} catch (error) {
|
||||
if (error instanceof ZodError) {
|
||||
|
|
@ -213,6 +247,10 @@ program
|
|||
'Suite to run: "active" (default, excludes pending), "all", or "pending"',
|
||||
'active'
|
||||
)
|
||||
.option(
|
||||
'--expected-failures <path>',
|
||||
'Path to YAML file listing expected failures (baseline)'
|
||||
)
|
||||
.option('-o, --output-dir <path>', 'Save results to this directory')
|
||||
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
|
||||
.action(async (options) => {
|
||||
|
|
@ -236,6 +274,20 @@ program
|
|||
result.scenarioDescription,
|
||||
verbose
|
||||
);
|
||||
|
||||
if (options.expectedFailures) {
|
||||
const expectedFailuresConfig = await loadExpectedFailures(
|
||||
options.expectedFailures
|
||||
);
|
||||
const baselineScenarios = expectedFailuresConfig.server ?? [];
|
||||
const baselineResult = evaluateBaseline(
|
||||
[{ scenario: validated.scenario!, checks: result.checks }],
|
||||
baselineScenarios
|
||||
);
|
||||
printBaselineResults(baselineResult);
|
||||
process.exit(baselineResult.exitCode);
|
||||
}
|
||||
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
} else {
|
||||
// Run scenarios based on suite
|
||||
|
|
@ -290,6 +342,20 @@ program
|
|||
}
|
||||
|
||||
const { totalFailed } = printServerSummary(allResults);
|
||||
|
||||
if (options.expectedFailures) {
|
||||
const expectedFailuresConfig = await loadExpectedFailures(
|
||||
options.expectedFailures
|
||||
);
|
||||
const baselineScenarios = expectedFailuresConfig.server ?? [];
|
||||
const baselineResult = evaluateBaseline(
|
||||
allResults,
|
||||
baselineScenarios
|
||||
);
|
||||
printBaselineResults(baselineResult);
|
||||
process.exit(baselineResult.exitCode);
|
||||
}
|
||||
|
||||
process.exit(totalFailed > 0 ? 1 : 0);
|
||||
}
|
||||
} catch (error) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue