test(conformance): pin conformance 0.2.0-alpha.1 and baseline the draft-spec suites (#2227)
This commit is contained in:
Felix Weinberger
GitHub
parent
c54abdb172
commit
71dcc70411
6 changed files with 122 additions and 29 deletions
3
.github/workflows/conformance.yml
vendored
3
.github/workflows/conformance.yml
vendored
|
|
@ -16,7 +16,6 @@ permissions:
|
|||
jobs:
|
||||
client-conformance:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install pnpm
|
||||
|
|
@ -34,7 +33,6 @@ jobs:
|
|||
|
||||
server-conformance:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install pnpm
|
||||
|
|
@ -49,3 +47,4 @@ jobs:
|
|||
- run: pnpm install
|
||||
- run: pnpm run build:all
|
||||
- run: pnpm run test:conformance:server
|
||||
- run: pnpm run test:conformance:server:draft
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@
|
|||
"test:conformance:client:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:all",
|
||||
"test:conformance:client:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:client:run",
|
||||
"test:conformance:server": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server",
|
||||
"test:conformance:server:draft": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:draft",
|
||||
"test:conformance:server:all": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:all",
|
||||
"test:conformance:server:run": "pnpm --filter @modelcontextprotocol/test-conformance run test:conformance:server:run",
|
||||
"test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
|
||||
|
|
|
|||
51
pnpm-lock.yaml
generated
51
pnpm-lock.yaml
generated
|
|
@ -1099,8 +1099,8 @@ importers:
|
|||
specifier: workspace:^
|
||||
version: link:../../packages/client
|
||||
'@modelcontextprotocol/conformance':
|
||||
specifier: 0.1.15
|
||||
version: 0.1.15(@cfworker/json-schema@4.1.1)
|
||||
specifier: 0.2.0-alpha.1
|
||||
version: 0.2.0-alpha.1(@cfworker/json-schema@4.1.1)
|
||||
'@modelcontextprotocol/core':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/core
|
||||
|
|
@ -2108,12 +2108,12 @@ packages:
|
|||
'@manypkg/get-packages@1.1.3':
|
||||
resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==}
|
||||
|
||||
'@modelcontextprotocol/conformance@0.1.15':
|
||||
resolution: {integrity: sha512-B1eNYpv5kas9YFC40su7SWhKHR2DwYzJRpiX6dfWzDWMcDr71myKVj//PwyqUHC7oucs3EJqcqnwvSuUwvenoQ==}
|
||||
'@modelcontextprotocol/conformance@0.2.0-alpha.1':
|
||||
resolution: {integrity: sha512-w5YpjJ7uNXGIWs2r+gMwowNgREykYvDlrFS1Ckl47lVbkZJn2CAytmuV/zCrsCuZkeE7djK1tJkuZ0oa72BhGA==}
|
||||
hasBin: true
|
||||
|
||||
'@modelcontextprotocol/sdk@1.28.0':
|
||||
resolution: {integrity: sha512-gmloF+i+flI8ouQK7MWW4mOwuMh4RePBuPFAEPC6+pdqyWOUMDOixb6qZ69owLJpz6XmyllCouc4t8YWO+E2Nw==}
|
||||
'@modelcontextprotocol/sdk@1.29.0':
|
||||
resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==}
|
||||
engines: {node: '>=18'}
|
||||
peerDependencies:
|
||||
'@cfworker/json-schema': ^4.1.1
|
||||
|
|
@ -3639,6 +3639,10 @@ packages:
|
|||
resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
eventsource-parser@3.0.8:
|
||||
resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
eventsource@3.0.7:
|
||||
resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -5087,8 +5091,8 @@ packages:
|
|||
resolution: {integrity: sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
undici@7.24.6:
|
||||
resolution: {integrity: sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA==}
|
||||
undici@7.25.0:
|
||||
resolution: {integrity: sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
unenv@2.0.0-rc.24:
|
||||
|
|
@ -5315,9 +5319,6 @@ packages:
|
|||
peerDependencies:
|
||||
zod: ^3.25.28 || ^4
|
||||
|
||||
zod@3.25.76:
|
||||
resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==}
|
||||
|
||||
zod@4.3.6:
|
||||
resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==}
|
||||
|
||||
|
|
@ -5997,22 +5998,22 @@ snapshots:
|
|||
globby: 11.1.0
|
||||
read-yaml-file: 1.1.0
|
||||
|
||||
'@modelcontextprotocol/conformance@0.1.15(@cfworker/json-schema@4.1.1)':
|
||||
'@modelcontextprotocol/conformance@0.2.0-alpha.1(@cfworker/json-schema@4.1.1)':
|
||||
dependencies:
|
||||
'@modelcontextprotocol/sdk': 1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76)
|
||||
'@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)
|
||||
'@octokit/rest': 22.0.1
|
||||
commander: 14.0.3
|
||||
eventsource-parser: 3.0.6
|
||||
eventsource-parser: 3.0.8
|
||||
express: 5.2.1
|
||||
jose: 6.2.2
|
||||
undici: 7.24.6
|
||||
undici: 7.25.0
|
||||
yaml: 2.8.3
|
||||
zod: 3.25.76
|
||||
zod: 4.3.6
|
||||
transitivePeerDependencies:
|
||||
- '@cfworker/json-schema'
|
||||
- supports-color
|
||||
|
||||
'@modelcontextprotocol/sdk@1.28.0(@cfworker/json-schema@4.1.1)(zod@3.25.76)':
|
||||
'@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)':
|
||||
dependencies:
|
||||
'@hono/node-server': 1.19.11(hono@4.12.9)
|
||||
ajv: 8.18.0
|
||||
|
|
@ -6021,7 +6022,7 @@ snapshots:
|
|||
cors: 2.8.6
|
||||
cross-spawn: 7.0.6
|
||||
eventsource: 3.0.7
|
||||
eventsource-parser: 3.0.6
|
||||
eventsource-parser: 3.0.8
|
||||
express: 5.2.1
|
||||
express-rate-limit: 8.3.1(express@5.2.1)
|
||||
hono: 4.12.9
|
||||
|
|
@ -6029,8 +6030,8 @@ snapshots:
|
|||
json-schema-typed: 8.0.2
|
||||
pkce-challenge: 5.0.1
|
||||
raw-body: 3.0.2
|
||||
zod: 3.25.76
|
||||
zod-to-json-schema: 3.25.2(zod@3.25.76)
|
||||
zod: 4.3.6
|
||||
zod-to-json-schema: 3.25.2(zod@4.3.6)
|
||||
optionalDependencies:
|
||||
'@cfworker/json-schema': 4.1.1
|
||||
transitivePeerDependencies:
|
||||
|
|
@ -7518,6 +7519,8 @@ snapshots:
|
|||
|
||||
eventsource-parser@3.0.6: {}
|
||||
|
||||
eventsource-parser@3.0.8: {}
|
||||
|
||||
eventsource@3.0.7:
|
||||
dependencies:
|
||||
eventsource-parser: 3.0.6
|
||||
|
|
@ -9091,7 +9094,7 @@ snapshots:
|
|||
|
||||
undici@7.24.4: {}
|
||||
|
||||
undici@7.24.6: {}
|
||||
undici@7.25.0: {}
|
||||
|
||||
unenv@2.0.0-rc.24:
|
||||
dependencies:
|
||||
|
|
@ -9353,10 +9356,8 @@ snapshots:
|
|||
cookie: 1.1.1
|
||||
youch-core: 0.3.3
|
||||
|
||||
zod-to-json-schema@3.25.2(zod@3.25.76):
|
||||
zod-to-json-schema@3.25.2(zod@4.3.6):
|
||||
dependencies:
|
||||
zod: 3.25.76
|
||||
|
||||
zod@3.25.76: {}
|
||||
zod: 4.3.6
|
||||
|
||||
zod@4.3.6: {}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
This directory contains conformance test implementations for the TypeScript MCP SDK.
|
||||
|
||||
> `@modelcontextprotocol/conformance` is pinned to an exact version (no `^` range) in `package.json`. New conformance releases are adopted by deliberately bumping the pin and updating `expected-failures.yaml` for any new scenarios/checks in the same change.
|
||||
|
||||
## Client Conformance Tests
|
||||
|
||||
Tests the SDK's client implementation against a conformance test server.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,93 @@
|
|||
# Conformance scenarios not yet implemented in the v2 TypeScript SDK.
|
||||
# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
|
||||
#
|
||||
# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1,
|
||||
# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549,
|
||||
# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type
|
||||
# during DCR).
|
||||
#
|
||||
# Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the
|
||||
# corresponding scenarios start passing and MUST be removed from this list (the
|
||||
# runner fails on stale entries), so the baseline burns down per milestone.
|
||||
|
||||
client: []
|
||||
client:
|
||||
# --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) ---
|
||||
# SEP-2575 (request metadata / _meta envelope): client does not populate the
|
||||
# _meta envelope or the MCP-Protocol-Version header semantics yet.
|
||||
- request-metadata
|
||||
# SEP-2322 (multi-round-trip requests): client does not echo requestState /
|
||||
# handle IncompleteResult yet.
|
||||
- sep-2322-client-request-state
|
||||
# SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
|
||||
- http-custom-headers
|
||||
- http-invalid-tool-headers
|
||||
# SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs.
|
||||
- json-schema-ref-no-deref
|
||||
# SEP-2468 (authorization response iss parameter): not implemented in the client.
|
||||
- auth/iss-supported
|
||||
- auth/iss-not-advertised
|
||||
- auth/iss-supported-missing
|
||||
- auth/iss-wrong-issuer
|
||||
- auth/iss-unexpected
|
||||
- auth/iss-normalized
|
||||
- auth/metadata-issuer-mismatch
|
||||
# SEP-2352 (authorization server migration): client does not re-register when
|
||||
# PRM authorization_servers changes.
|
||||
- auth/authorization-server-migration
|
||||
# SEP-2207 (offline_access scope) scenario, currently failing only on the new
|
||||
# SEP-837 application_type check (see the SEP-837 group below).
|
||||
- auth/offline-access-not-supported
|
||||
|
||||
# --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 ---
|
||||
# SEP-837: client MUST send application_type during Dynamic Client Registration.
|
||||
# Single new check; everything else in these scenarios passes.
|
||||
- auth/metadata-default
|
||||
- auth/metadata-var1
|
||||
- auth/metadata-var2
|
||||
- auth/metadata-var3
|
||||
- auth/scope-from-www-authenticate
|
||||
- auth/scope-from-scopes-supported
|
||||
- auth/scope-omitted-when-undefined
|
||||
- auth/scope-step-up
|
||||
- auth/scope-retry-limit
|
||||
- auth/token-endpoint-auth-basic
|
||||
- auth/token-endpoint-auth-post
|
||||
- auth/token-endpoint-auth-none
|
||||
- auth/2025-03-26-oauth-metadata-backcompat
|
||||
# SEP-990 (enterprise-managed authorization extension): no fixture handler /
|
||||
# client support for the token-exchange + JWT bearer flow.
|
||||
- auth/enterprise-managed-authorization
|
||||
|
||||
server:
|
||||
# --- Draft-spec scenarios (in `--suite draft`; the default `active` suite is green) ---
|
||||
# SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode,
|
||||
# _meta-derived capabilities, error-code mappings, or server/discover yet.
|
||||
- server-stateless
|
||||
# SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented;
|
||||
# most scenarios currently fail early with "Session ID required" because the
|
||||
# fixture only runs in stateful mode.
|
||||
- input-required-result-basic-elicitation
|
||||
- input-required-result-basic-sampling
|
||||
- input-required-result-basic-list-roots
|
||||
- input-required-result-request-state
|
||||
- input-required-result-multiple-input-requests
|
||||
- input-required-result-multi-round
|
||||
- input-required-result-non-tool-request
|
||||
- input-required-result-result-type
|
||||
- input-required-result-tampered-state
|
||||
- input-required-result-capability-check
|
||||
# SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the
|
||||
# stateful-mode "Session ID required" error.
|
||||
- caching
|
||||
# SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and
|
||||
# case-insensitive/whitespace-trimmed header validation not implemented.
|
||||
- http-header-validation
|
||||
- http-custom-header-server-validation
|
||||
# WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level
|
||||
# WARNINGs, but the expected-failures evaluator counts WARNINGs as failures.
|
||||
# SEP-2164: server returns -32002 without the requested URI in error.data.
|
||||
- sep-2164-resource-not-found
|
||||
# SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore
|
||||
# unrecognized inputResponses keys).
|
||||
- input-required-result-missing-input-response
|
||||
- input-required-result-ignore-extra-params
|
||||
|
|
|
|||
|
|
@ -32,12 +32,13 @@
|
|||
"test:conformance:client:all": "conformance client --command 'npx tsx ./src/everythingClient.ts' --suite all --expected-failures ./expected-failures.yaml",
|
||||
"test:conformance:client:run": "npx tsx ./src/everythingClient.ts",
|
||||
"test:conformance:server": "scripts/run-server-conformance.sh --expected-failures ./expected-failures.yaml",
|
||||
"test:conformance:server:draft": "scripts/run-server-conformance.sh --suite draft --expected-failures ./expected-failures.yaml",
|
||||
"test:conformance:server:all": "scripts/run-server-conformance.sh --suite all --expected-failures ./expected-failures.yaml",
|
||||
"test:conformance:server:run": "npx tsx ./src/everythingServer.ts",
|
||||
"test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@modelcontextprotocol/conformance": "0.1.15",
|
||||
"@modelcontextprotocol/conformance": "0.2.0-alpha.1",
|
||||
"@modelcontextprotocol/client": "workspace:^",
|
||||
"@modelcontextprotocol/server": "workspace:^",
|
||||
"@modelcontextprotocol/core": "workspace:^",
|
||||
|
|
|
|||
Loading…
Reference in a new issue