232a901445There is no way to answer "which scenarios must my implementation pass to conform to spec revision X". The suite accumulates scenarios continuously, so what an implementation is measured against keeps growing after the revision shipped, and it can drop below 100% while standing still. --spec-version does not answer it: it filters on which revision a scenario targets, not on when the scenario became a requirement. Add requirements/<revision>.yaml naming the scenarios a revision requires, and --requirements <revision> on the client, server, list and sdk commands to run exactly that set. The revision also fixes the wire version its scenarios speak: the dated revisions through 2025-11-25 use the stateful initialize handshake and 2026-07-28 is stateless with per-request _meta, so a scenario belonging to two revisions runs once under each and one run does not cover the other. Passing no flag keeps today's behaviour of running everything. tier-check gains --sdk <name[@ref]> / --sdk-path <dir>: for each claimed revision it resolves that revision's build, server invocation and client command from the sdk-runner's KNOWN_SDKS specOverrides, manages the server, runs both legs with cwd = the SDK checkout, and merges. Every claimed revision must pass for Tier 1. This is what makes multi-revision tiering correct for SDKs whose wire era is fixed per server process (go-sdk only speaks >= 2026-07-28 when constructed stateless) or per endpoint (csharp-sdk serves it at /stateless): one --conformance-server-url cannot describe them, and their config already does. The single-URL mode remains for SDKs not in the config. Ship two sets. 2026-07-28 is a snapshot of 0.2.0-alpha.10, published the day before that revision shipped. 2025-11-25 is a reconstruction from the same anchor and its header says so: the release current on its ship date predates spec-version awareness entirely. not_scored entries run and are reported with a reason (extension per SEP-1730, or added-after-release) but never affect a pass rate or the exit code, and --expected-failures judges scored scenarios only. Reports label every entry with its revision, and JSON passed/failed/total describe the scored set so they always agree with pass_rate. tier-check's exit code now gates on the machine-checkable half: nonzero when a scored scenario fails or a leg could not be measured. A child run that cannot happen at all (a set naming a renamed scenario, an unreachable server) reports as not measured rather than as 0% against the implementation. Conformance runs to completion before the GitHub checks, so an API failure can no longer interrupt a leg and orphan a managed server, and the legs spawn their child asynchronously: the previous synchronous exec blocked the event loop, the parent stopped draining the managed server's log pipes, and a chatty server (rust-sdk's per-request tracing) froze on a full pipe and was scored as failing. Driving every tier-1 SDK through the new path also fixed three configs: csharp-sdk's server now launches from its build-output directory so ASP.NET finds appsettings.json (its AllowedHosts filter, which dns-rebinding-protection tests, silently never loaded from the repo root); rust-sdk's STATELESS=1 override is removed because its CI runs one default server for both revisions and the forced mode fails its SEP-2575 scenarios; python-sdk's build mirrors its CI's two targeted syncs because --all-packages is unbuildable on main. withManagedServer now waits for its child to exit before returning, so consecutive runs on one port cannot race a dying predecessor.
65 lines
1.9 KiB
JSON
65 lines
1.9 KiB
JSON
{
|
|
"name": "@modelcontextprotocol/conformance",
|
|
"version": "0.2.0-alpha.10",
|
|
"type": "module",
|
|
"license": "MIT",
|
|
"author": "Anthropic, PBC (https://anthropic.com)",
|
|
"homepage": "https://modelcontextprotocol.io",
|
|
"bugs": "https://github.com/modelcontextprotocol/conformance/issues",
|
|
"repository": {
|
|
"type": "git",
|
|
"url": "git+https://github.com/modelcontextprotocol/conformance.git"
|
|
},
|
|
"scripts": {
|
|
"start": "tsx src/index.ts",
|
|
"test": "vitest run",
|
|
"test:watch": "vitest",
|
|
"build": "tsdown src/index.ts --minify --clean --target node20 --no-fixed-extension",
|
|
"lint": "eslint src/ examples/ && prettier --check .",
|
|
"lint:fix": "eslint src/ examples/ --fix && prettier --write .",
|
|
"lint:fix_check": "npm run lint:fix && git diff --exit-code --quiet",
|
|
"tier-check": "node dist/index.js tier-check",
|
|
"traceability": "tsx src/index.ts traceability",
|
|
"sync-schema": "tsx scripts/sync-schema.ts",
|
|
"check": "npm run typecheck && npm run lint",
|
|
"typecheck": "tsgo --noEmit",
|
|
"prepack": "npm run build",
|
|
"prepare": "npm run build"
|
|
},
|
|
"files": [
|
|
"dist",
|
|
"requirements"
|
|
],
|
|
"bin": {
|
|
"conformance": "dist/index.js"
|
|
},
|
|
"devDependencies": {
|
|
"@eslint/js": "^9.39.4",
|
|
"@types/express": "^5.0.3",
|
|
"@types/node": "^25.6.0",
|
|
"@typescript/native-preview": "^7.0.0-dev.20260424.1",
|
|
"cors": "^2.8.5",
|
|
"eslint": "^9.39.4",
|
|
"eslint-config-prettier": "^10.1.8",
|
|
"lefthook": "^2.1.6",
|
|
"prettier": "^3.8.3",
|
|
"tsdown": "^0.21.10",
|
|
"tsx": "^4.7.0",
|
|
"typescript": "^5.5.4",
|
|
"typescript-eslint": "^8.59.0",
|
|
"vitest": "^4.1.5"
|
|
},
|
|
"dependencies": {
|
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
"@octokit/rest": "^22.0.0",
|
|
"ajv": "^8.20.0",
|
|
"ajv-formats": "^3.0.1",
|
|
"commander": "^14.0.2",
|
|
"eventsource-parser": "^3.0.8",
|
|
"express": "^5.1.0",
|
|
"jose": "^6.1.2",
|
|
"undici": "^7.25.0",
|
|
"yaml": "^2.8.2",
|
|
"zod": "^4.3.6"
|
|
}
|
|
}
|