conformance/src/sdk-runner/known-sdks.ts
Felix Weinberger 232a901445
Freeze conformance requirements per spec revision, run each at its wire (#447)
There is no way to answer "which scenarios must my implementation pass
to conform to spec revision X". The suite accumulates scenarios
continuously, so what an implementation is measured against keeps
growing after the revision shipped, and it can drop below 100% while
standing still. --spec-version does not answer it: it filters on which
revision a scenario targets, not on when the scenario became a
requirement.

Add requirements/<revision>.yaml naming the scenarios a revision
requires, and --requirements <revision> on the client, server, list and
sdk commands to run exactly that set. The revision also fixes the wire
version its scenarios speak: the dated revisions through 2025-11-25 use
the stateful initialize handshake and 2026-07-28 is stateless with
per-request _meta, so a scenario belonging to two revisions runs once
under each and one run does not cover the other. Passing no flag keeps
today's behaviour of running everything.

tier-check gains --sdk <name[@ref]> / --sdk-path <dir>: for each claimed
revision it resolves that revision's build, server invocation and client
command from the sdk-runner's KNOWN_SDKS specOverrides, manages the
server, runs both legs with cwd = the SDK checkout, and merges. Every
claimed revision must pass for Tier 1. This is what makes multi-revision
tiering correct for SDKs whose wire era is fixed per server process
(go-sdk only speaks >= 2026-07-28 when constructed stateless) or per
endpoint (csharp-sdk serves it at /stateless): one
--conformance-server-url cannot describe them, and their config already
does. The single-URL mode remains for SDKs not in the config.

Ship two sets. 2026-07-28 is a snapshot of 0.2.0-alpha.10, published the
day before that revision shipped. 2025-11-25 is a reconstruction from
the same anchor and its header says so: the release current on its ship
date predates spec-version awareness entirely. not_scored entries run
and are reported with a reason (extension per SEP-1730, or
added-after-release) but never affect a pass rate or the exit code, and
--expected-failures judges scored scenarios only. Reports label every
entry with its revision, and JSON passed/failed/total describe the
scored set so they always agree with pass_rate.

tier-check's exit code now gates on the machine-checkable half: nonzero
when a scored scenario fails or a leg could not be measured. A child run
that cannot happen at all (a set naming a renamed scenario, an
unreachable server) reports as not measured rather than as 0% against
the implementation. Conformance runs to completion before the GitHub
checks, so an API failure can no longer interrupt a leg and orphan a
managed server, and the legs spawn their child asynchronously: the
previous synchronous exec blocked the event loop, the parent stopped
draining the managed server's log pipes, and a chatty server (rust-sdk's
per-request tracing) froze on a full pipe and was scored as failing.

Driving every tier-1 SDK through the new path also fixed three configs:
csharp-sdk's server now launches from its build-output directory so
ASP.NET finds appsettings.json (its AllowedHosts filter, which
dns-rebinding-protection tests, silently never loaded from the repo
root); rust-sdk's STATELESS=1 override is removed because its CI runs
one default server for both revisions and the forced mode fails its
SEP-2575 scenarios; python-sdk's build mirrors its CI's two targeted
syncs because --all-packages is unbuildable on main. withManagedServer
now waits for its child to exit before returning, so consecutive runs on
one port cannot race a dying predecessor.
2026-08-07 14:55:10 +01:00

177 lines
7.8 KiB
TypeScript

import type { SdkConfig } from './config';
/**
* Built-in conformance configs for official SDKs, keyed by repo name.
*
* These live here (not in the SDK repos) so adding an SDK to the matrix
* doesn't require a coordinated cross-repo PR. Any field can be overridden
* per-invocation via the CLI flags (--build-cmd / --client-cmd / etc.).
*/
export const KNOWN_SDKS: Record<string, SdkConfig> = {
// v2 — the monorepo on `main` (pnpm). Default ref is `main`.
'typescript-sdk': {
build: 'pnpm install && pnpm run build:all',
client: {
command: 'npx tsx test/conformance/src/everythingClient.ts'
},
server: {
command: 'npx tsx test/conformance/src/everythingServer.ts',
url: 'http://localhost:3000/mcp'
},
expectedFailures: 'test/conformance/expected-failures.yaml'
},
// v1.x — the published npm line. Same fixtures as v2; differs only in the
// build (npm, not pnpm) and the baseline filename. Clones the typescript-sdk
// repo, defaulting to the `v1.x` branch. Targets the latest dated spec, so
// draft-only scenarios and checks are excluded by default.
'typescript-sdk-v1': {
repo: 'typescript-sdk',
defaultRef: 'v1.x',
specVersion: '2025-11-25',
build: 'npm ci && npm run build',
client: {
command: 'npx tsx test/conformance/src/everythingClient.ts'
},
server: {
command: 'npx tsx test/conformance/src/everythingServer.ts',
url: 'http://localhost:3000/mcp'
},
expectedFailures: 'test/conformance/conformance-baseline.yml'
},
// Fixtures live under conformance/ (everything-client + everything-server,
// mirroring scripts/{client,server}-conformance.sh). The server's -stateless
// flag defaults to true; the dated-spec (`active` suite) scenarios need the
// stateful transport, so the base command pins -stateless=false and the
// 2026-07-28 override drops it for the SEP-2575 stateless lifecycle.
'go-sdk': {
build:
'go build -o ./.conformance-server ./conformance/everything-server && go build -o ./.conformance-client ./conformance/everything-client',
client: {
command: './.conformance-client'
},
server: {
command: './.conformance-server -http=localhost:3000 -stateless=false',
url: 'http://localhost:3000'
},
expectedFailures: 'conformance/baseline.yml',
specOverrides: {
'2026-07-28': {
server: { command: './.conformance-server -http=localhost:3000' }
}
}
},
// main — targets the 2026-07-28 revision. Same uv workspace layout as v1.x
// (client fixture in .github/actions/conformance/, mcp-everything-server
// workspace package). Two baselines exist upstream, one per spec target;
// the 2026-07-28 override picks the matching one.
'python-sdk': {
// Mirrors the SDK's own CI, which syncs the two conformance packages
// individually (--inexact so the second sync doesn't prune the first).
// --all-packages is unbuildable on main: an example package's declared
// README is missing, and uv builds every workspace member.
build:
'uv sync --frozen --all-extras --package mcp-everything-server && uv sync --frozen --all-extras --package mcp --inexact',
client: {
command: 'uv run --frozen python .github/actions/conformance/client.py'
},
server: {
command: 'uv run --frozen mcp-everything-server --port 3000',
url: 'http://localhost:3000/mcp'
},
expectedFailures: '.github/actions/conformance/expected-failures.yml',
specOverrides: {
'2026-07-28': {
expectedFailures:
'.github/actions/conformance/expected-failures.2026-07-28.yml'
}
}
},
// v1.x — the stable, published line of the python-sdk, analogous to
// typescript-sdk-v1. Clones the python-sdk repo, defaulting to the `v1.x`
// branch, and targets the latest
// dated spec so draft-only scenarios/checks are excluded by default. uv
// workspace: the `mcp` (client) and `mcp-everything-server` (server) packages
// are both members, so one `uv sync --all-packages` covers both modes.
// Fixtures live in the python-sdk repo (.github/actions/conformance/ and
// examples/servers/everything-server). `--port 3000` matches the url and the
// 3000 convention used above; the server's own default is 3001.
'python-sdk-v1': {
repo: 'python-sdk',
defaultRef: 'v1.x',
specVersion: '2025-11-25',
build: 'uv sync --frozen --all-extras --all-packages',
client: {
command: 'uv run --frozen python .github/actions/conformance/client.py'
},
server: {
command: 'uv run --frozen mcp-everything-server --port 3000',
url: 'http://localhost:3000/mcp'
},
expectedFailures: '.github/actions/conformance/expected-failures.yml'
},
// Fixtures live in conformance/ (the mcp-conformance package's
// conformance-client + conformance-server bins; the package is excluded from
// the workspace default-members, so build it explicitly). The server reads
// PORT and STATELESS from the environment: the stateful (dated-spec)
// lifecycle is the default, and the 2026-07-28 override sets STATELESS=1
// for the SEP-2575 stateless lifecycle.
// One default server serves every revision — rust-sdk's own CI starts a
// single un-flagged conformance-server and runs both the 2025-11-25 and
// 2026-07-28 legs against it. (A STATELESS=1 override here used to force the
// 2026 leg onto a mode the SDK's CI never exercises, failing its SEP-2575
// input-required scenarios.)
'rust-sdk': {
build: 'cargo build -p mcp-conformance',
client: {
command: './target/debug/conformance-client'
},
server: {
command: 'PORT=3000 ./target/debug/conformance-server',
url: 'http://localhost:3000/mcp'
}
},
// Fixtures live in tests/ModelContextProtocol.ConformanceClient and
// tests/ModelContextProtocol.ConformanceServer (requires the .NET 10 SDK,
// per global.json); build output goes to the repo-level artifacts/ tree
// (UseArtifactsOutput), not per-project bin/. The client binary takes the scenario as its first
// argument rather than reading MCP_CONFORMANCE_SCENARIO, so the command
// bridges the env var into argv ($1 is the server URL the harness appends).
// The server serves the stateful lifecycle at / and the SEP-2575 stateless
// lifecycle at /stateless from the same port; the 2026-07-28 override
// points runs at the stateless endpoint.
'csharp-sdk': {
build:
'dotnet build tests/ModelContextProtocol.ConformanceClient -c Release -f net10.0 && dotnet build tests/ModelContextProtocol.ConformanceServer -c Release -f net10.0',
client: {
command: `bash -c 'exec dotnet artifacts/bin/ModelContextProtocol.ConformanceClient/Release/net10.0/ModelContextProtocol.ConformanceClient.dll "$MCP_CONFORMANCE_SCENARIO" "$1"' conformance-client`
},
server: {
// Launch from the build-output directory: ASP.NET resolves
// appsettings.json from the content root (the cwd), and that file
// carries the AllowedHosts filter dns-rebinding-protection tests.
// Launched from the repo root it silently never loads.
command:
"bash -c 'cd artifacts/bin/ModelContextProtocol.ConformanceServer/Release/net10.0 && exec dotnet ModelContextProtocol.ConformanceServer.dll --urls http://localhost:3000'",
url: 'http://localhost:3000'
},
specOverrides: {
'2026-07-28': {
server: { url: 'http://localhost:3000/stateless' }
}
}
}
};
/**
* Look up a built-in config by SDK name. Accepts bare names (typescript-sdk),
* owner/repo (modelcontextprotocol/typescript-sdk), or a checkout path
* basename — only the final path segment is used as the key.
*/
export function lookupBuiltinConfig(name: string): SdkConfig | null {
const key = name.split('/').pop() ?? name;
return KNOWN_SDKS[key] ?? null;
}
export function knownSdkNames(): string[] {
return Object.keys(KNOWN_SDKS);
}