From 82c1195f8a77fff516e43afb47c1b4faafea7707 Mon Sep 17 00:00:00 2001 From: Felix Weinberger <3823880+felixweinberger@users.noreply.github.com> Date: Wed, 3 Jun 2026 15:09:36 +0100 Subject: [PATCH] feat: version-aware MockServer abstraction for client scenarios (#321) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: vendor spec schema types per-version Copies schema/{version}/schema.ts from the modelcontextprotocol spec repo into src/spec-types/{version}.ts so the conformance suite can type against draft spec versions before any SDK ships them. npm run sync-schema -- refreshes the copies and records the spec commit in src/spec-types/SOURCE. * feat: add Connection abstraction and RunContext Connection encapsulates how the conformance suite talks to a server-under-test for a given spec version: - connectStateful: 2025-x lifecycle. Thin adapter over the SDK Client (initialize handshake, session id, SSE handled by the SDK). - connectStateless: 2026-x lifecycle (SEP-2575). Raw fetch with per-request _meta + MCP-Protocol-Version header. Decoupled from the SDK so the suite can test draft spec versions before the SDK implements them. connectFor(specVersion) picks the implementation. RunContext bundles serverUrl, specVersion and a bound connect() for the runner to hand to each scenario. Nothing uses this yet; wiring follows in the next commit. * refactor: thread RunContext through ClientScenario.run ClientScenario.run(serverUrl) becomes run(ctx: RunContext). The runner builds the context from --spec-version and the server URL; scenarios destructure ctx.serverUrl and otherwise behave identically. No scenario uses ctx.connect() yet, so behaviour is unchanged: 214/214 tests pass, all-scenarios.test.ts still drives the everything-server fixture exactly as before. Test files use a testContext(url) helper to construct a RunContext. The authorization-server scenario list is retyped to ClientScenarioForAuthorizationServer since those scenarios test an OAuth server, not an MCP server, and keep run(serverUrl). * refactor: migrate server scenarios to ctx.connect() + conn.request() 22 carry-forward and lifecycle scenarios now go through the Connection abstraction instead of connectToServer + SDK Client: - tools.ts (8): list, call x6, with-progress - prompts.ts (5) - resources.ts (7): list, read x3, subscribe x2, not-found - utils.ts (3): completion, ping, set-level - json-schema-2020-12.ts, caching.ts, http-standard-headers.ts Result types come from spec-types/{introducedIn}. Same scenario code now passes under both --spec-version 2025-11-25 (SDK-backed stateful) and --spec-version draft (raw stateless). ToolsCallSampling/Elicitation/WithLogging and elicitation-* keep connectToServer (need setRequestHandler/setLoggingLevel SDK surface); they are tagged removedIn: DRAFT in the next commit. stateless.ts and input-required-result.ts keep their sendRpc helper; migrating those is deferred to the DRAFT-scenario coherence pass. * fix: normalize Connection error to JsonRpcError; clean up RunContext import connectStateful now catches the SDK's McpError and rethrows as JsonRpcError so scenarios always see the same error class regardless of which Connection impl ran. ResourcesNotFoundError uses instanceof JsonRpcError instead of duck-typing. types.ts uses a normal top-level import for RunContext instead of an inline import() type. * fix(dns-rebinding): use version-appropriate probe body The 'valid Host accepted' check was sending an initialize body, which a 2026 server returns 404 for. Probe body is now picked from ctx.specVersion: initialize for the stateful lifecycle, server/discover with _meta for the stateless lifecycle. The Host/Origin rejection check is unchanged since rejection happens before body parsing. * feat(everything-server): route stateless carry-forward methods to McpServer Adds an in-memory dispatch client connected to the same McpServer the stateful path uses. Stateless requests for tools/call, resources/*, prompts/get and completion/complete that fall through the MRTR-specific handlers are routed to it, so the fixture serves the carry-forward scenarios under --spec-version draft without duplicating ~500 lines of tool/resource/prompt registrations. tools/list now merges the McpServer's tool list with the MRTR-only stubs so json-schema-2020-12 finds its tool. draft suite against the fixture: 36/39 (was 13/39). Remaining 3 are fixture-side SEP gaps (no SSE forwarding for progress, no SEP-2549 ttlMs, no SEP-2243 Mcp-Method validation in the stateless path). * refactor(connection): drop unused RequestOptions; move sdk-client; add tests Addresses self-review findings on the new connection module: - RequestOptions.handlers/.meta and ServerRequestHandler removed: zero callers (the scenarios that motivated them are deferred). They can be reintroduced when something actually uses them. - scenarios/server/client-helper.ts moved to connection/sdk-client.ts so the connection module no longer depends on the scenarios module. - connectStateless: handle CRLF SSE event separators; throw a useful error for non-JSON/non-SSE responses instead of a JSON parse error. - JSONRPCNotification consistently imported from spec-types/2025-11-25. - New connection.test.ts (8 tests) covering connectFor selection, _meta injection, error mapping, SSE LF/CRLF parsing, and the server-request-on-stream rejection. * fix: address bughunt findings (response.ok check; targetVersion naming) - connectStateless: throw on non-2xx responses that lack a JSON-RPC error envelope (e.g. gateway/framework error JSON), matching the stateful path's behavior. Previously such a response would return undefined as the result. - json-schema-2020-12: rename negotiatedVersion to targetVersion and reword the skip message and details field. The value is ctx.specVersion (the run's --spec-version), not a negotiated version; the old wording was misleading. Drop dead 'unknown' fallback (specVersion is required) and the corresponding undefined test cases. * fix(sse-multiple-streams): keep scenario in draft; version-aware request scaffolding Concurrent POSTs each answered with JSON or their own SSE stream are core transport behavior in the draft spec too, so removedIn was wrong for this scenario. Only the request scaffolding is version-specific: stateful runs keep the initialize/session-id setup, draft runs send _meta + the MCP-Protocol-Version header on each request instead. The two checks are unchanged. * feat: add MockServer abstraction and ScenarioContext MockServer encapsulates the lifecycle scaffold a client-conformance scenario presents to the client-under-test: - createServerStateful: 2025-x lifecycle. SDK Server + StreamableHTTPServerTransport (sessionless mode); the SDK handles the initialize handshake. - createServerStateless: 2026-x lifecycle (SEP-2575). Raw express app that validates _meta + MCP-Protocol-Version on every request, serves server/discover, routes other methods to the supplied handlers. createServerFor(specVersion) picks the implementation. ScenarioContext bundles specVersion and a bound createServer() for the runner to hand to each scenario. This is the client-conformance mirror of src/connection (PR #318). Nothing uses it yet; wiring follows in the next commit. * refactor: thread ScenarioContext through Scenario.start() Scenario.start() becomes start(ctx: ScenarioContext). The runner builds the context from --spec-version (defaulting to LATEST_SPEC_VERSION) and passes it through; scenarios receive it as _ctx and otherwise behave identically. No scenario uses ctx.createServer() yet, so behaviour is unchanged: 231/231 tests pass. Test files use a testScenarioContext() helper. The runner already threads MCP_CONFORMANCE_PROTOCOL_VERSION to the spawned client process, so the fixture-side env wiring is unchanged. * refactor: migrate tools_call to ctx.createServer; tag 2025-only client scenarios ToolsCallScenario now goes through ctx.createServer() instead of an inline express + SDK Server build. Same handlers, same checks; the assertion now reads from srv.recorded so it works regardless of which lifecycle scaffold the runner picked. initialize, sse-retry, and elicitation-defaults are tagged removedIn: DRAFT (initialize/GET-SSE/SSE-embedded-elicitation are gone in the 2026 lifecycle; the MRTR sibling for elicitation-defaults is a follow-up). spec-version.test.ts: the 'draft is a superset of latest' invariant no longer holds once removedIn: DRAFT exists; the test now asserts that any scenario in latest-but-not-draft is explicitly removedIn. * feat(auth): make createServer helper version-aware via ScenarioContext The auth helper now takes ctx: ScenarioContext as its first argument and branches on ctx.specVersion inside the /mcp route: the stateful path (SDK Server + StreamableHTTPServerTransport) is unchanged; under the draft version a raw stateless handler validates _meta + the MCP-Protocol-Version header, serves server/discover, and routes the same tools/list and tools/call responses. The PRM endpoint, bearer-auth middleware, and request logger sit above the branch and are version-independent. All 25 call sites across the 12 auth scenario files pass ctx through; ServerLifecycle and the express.Application return type are unchanged so stop()/getChecks() are untouched. Deviation from the MockServer wrapper approach: keeping the helper's return type as express.Application avoids restructuring 25 call sites' ServerLifecycle handling in this PR. Folding the auth seam onto ctx.createServer() fully is a follow-up once the lifecycle ownership moves into MockServer. * feat(everything-client): pick stateless requester by MCP_CONFORMANCE_PROTOCOL_VERSION Adds a statelessRequest(serverUrl, method, params) helper that POSTs with _meta + MCP-Protocol-Version (the SEP-2575 lifecycle), shimming around the SDK Client not yet supporting stateless mode. The runRequestMetadataClient handler's meta constants are extracted to share with the helper. runBasicClient (initialize, tools_call, json-schema-ref-no-deref) now branches on MCP_CONFORMANCE_PROTOCOL_VERSION: for the draft version it uses statelessRequest to call tools/list then tools/call; for dated versions it keeps the SDK Client path. The runner already passes MCP_CONFORMANCE_PROTOCOL_VERSION to the spawned client, so no runner change is needed. * fix: address review findings on MockServer (dead opts, shared validator, capability derivation, recorded parity, specVersion threading) - MockServerOptions removed (capabilities/configure had zero callers); opts param dropped from createServerStateful/Stateless/For and ScenarioContext. - validateStatelessRequest extracted from mock-server/stateless and exported; both the stateless MockServer and auth/helpers/createServer.ts call it so _meta/header/version validation cannot drift. - isStatefulVersion exported from connection/select; mock-server/select uses it instead of duplicating the version set. - runner/client.ts: env MCP_CONFORMANCE_PROTOCOL_VERSION set unconditionally to the resolved version; runInteractiveMode now takes specVersion and the CLI passes it. - createServerStateful: capabilities derived from handler method prefixes; newServer() moved inside the try so a capability mismatch surfaces as JSON-RPC -32603 instead of an HTML 500. Recording moved to the express layer so unregistered methods are captured (parity with stateless). - readFinalSseMessage return type now declares error.data. - Tests added for the capability derivation and unregistered-method recording. * fix(mock-server): record stateless requests before validation; document preamble exclusions The stateless impl pushed to recorded[] only after validateStatelessRequest passed, so requests rejected for a missing header or _meta never appeared, diverging from the stateful impl which records at the HTTP layer before SDK dispatch. Record before validation, excluding only the server/discover lifecycle preamble (mirroring the stateful initialize/notifications-initialized exclusion), and fix the MockServer.recorded docstring which wrongly claimed stateless excludes nothing. * refactor(mock-server): tri-state result for validateStatelessRequest (reject/handled/route) A successful server/discover was previously modeled as {ok: false}, which read as a validation failure but actually meant "already answered, do not route". Replace the boolean discriminant with an explicit kind: 'reject' (validation failed), 'handled' (valid and already answered), or 'route' (dispatch to handlers). Both consumers (the stateless /mcp route and the auth helper's handleStateless) still write status/body for reject and handled and route only on 'route', so responses are unchanged. Add direct unit tests covering each branch of the new union. * feat(runner): single-source the version→lifecycle mapping; export MCP_CONFORMANCE_LIFECYCLE The stateless mock and the example client both hardcoded the draft protocol version, so dating the draft (or adding a second stateless version) would silently break them: the mock would keep rejecting everything but the old constant, and the client would fall back to the stateful path. - Derive STATELESS_SPEC_VERSIONS and a lifecycleFor() helper from isStatefulVersion() in connection/select.ts, making it the single source of truth for the version→lifecycle mapping. - Thread the resolved spec version into the stateless mock: createServerFor() binds it into createServerStateless(), and validateStatelessRequest() now takes the supported-versions list as a parameter. A mock bound to a version accepts exactly that version and echoes it in -32004 rejections and server/discover; unbound mocks accept every known stateless version. - Export MCP_CONFORMANCE_LIFECYCLE (stateful|stateless) to client processes alongside MCP_CONFORMANCE_PROTOCOL_VERSION, in both the runner and the in-process test helper, so clients in any language can pick the lifecycle without keeping their own version map. - everything-client: import DRAFT_PROTOCOL_VERSION from src/types instead of duplicating the string, branch on MCP_CONFORMANCE_LIFECYCLE (falling back to the version comparison for older runners), and send the runner-resolved version on stateless requests. - Document MCP_CONFORMANCE_LIFECYCLE in the README and ignore the local .claude/ workspace dir in .prettierignore so prettier --check . is not tripped by untracked worktree checkouts. * fix(everything-client): list-only flow for json-schema-ref-no-deref runBasicClient now follows tools/list with a tools/call on the first listed tool, but the json-schema-ref-no-deref scenario mock only serves tools/list, so the call got -32601 and the client exited non-zero. Register the scenario with a dedicated handler that stops after listing tools, keeping the existing lifecycle branch (raw stateless request vs SDK client). Listing is the whole flow for SEP-2106: the check is that the client never fetches the network $ref canary while processing the advertised schema. * fix(tools_call): make getChecks() idempotent getChecks() pushed into an instance array on every call, so calling it more than once accumulated duplicate check entries. The runner is allowed to call getChecks() repeatedly, and other scenarios already build their result fresh per call for this reason. Return a freshly built array instead and drop the now-unused instance field. Add a unit test covering the not-called FAILURE case and asserting repeated getChecks() calls do not grow the result. * fix: ensure dispatch and transport cleanup runs when handlers throw The stateless tools/list and prompts/list handlers in the everything server awaited dispatch.close() after the in-memory request, so a throw from dispatch.client.request() leaked the Client+Server+Transport pair. Wrap both in try/finally, matching the tools/call and catch-all handlers in the same file. In the stateful mock server, the res 'close' listener that tears down the per-request SDK transport and server was registered after the handleRequest await; a throw there skipped registration and leaked the pair. Register the listener before handing the request to the transport. * fix(everything-client): send Accept header on stateless requests The draft transport spec requires POST requests to accept both application/json and text/event-stream. Servers built on the SDK's StreamableHTTPServerTransport already enforce this and reject requests without it. Does not yet make json-schema-ref-no-deref pass under --spec-version draft: the SDK transport also rejects the draft protocol version itself, which is the known fixture gap until the 2026-native scenarios migrate onto MockServer. * chore: exclude .claude/ from vitest globs Local agent worktree checkouts under .claude/worktrees/ carry their own copies of the test suite, which the bare **/*.test.ts include was adding to every run (and to the pre-push hook). Mirrors the existing .prettierignore entry. * fix(request-metadata): use spec error code -32004 for unsupported protocol version The draft spec defines -32004 (UnsupportedProtocolVersionError) with data { supported, requested } and HTTP 400 for protocol version rejections; -32001 (HeaderMismatch) is reserved for HTTP header validation failures. - request-metadata scenario: return -32004 with spec-shaped data for the simulated version-negotiation rejection (was -32001) - everything-client: trigger negotiation on -32004 (was -32001) - stateless mock server: include the requested version in the -32004 error data alongside the supported list - update tests to match and assert the new requested field * fix(auth): register transport cleanup before handleRequest in createServer The res 'close' listener that tears down the transport/server pair was registered after awaiting handleRequest, so a throw mid-request skipped cleanup registration entirely. Register it first, matching the stateful mock server. * fix(everything-server): return JSON-RPC errors from stateless list dispatch on failure The stateless tools/list and prompts/list dispatch handlers had try/finally but no catch, so a dispatch failure produced an unhandled rejection instead of a response. Add the same catch-and-respond block the carry-forward dispatch handler already uses. * refactor(runner): drop MCP_CONFORMANCE_LIFECYCLE; derive lifecycle from protocol version The protocol version fully determines the lifecycle, and deriving it is part of being a conformant client. The in-repo example client imports the stateless version set from src/, so there is no drift risk, and a single env var cannot disagree with itself. README documents the version→lifecycle mapping for clients in other languages. * fix(everything-server): return JSON-RPC errors from stateless resource list dispatch on failure resources/list and resources/templates/list arrived from main with try/finally but no catch, unlike the sibling tools/list and prompts/list handlers. A throwing dispatch would escape to Express's HTML 500 instead of a JSON-RPC error. Mirrors the existing pattern. * fix(everything-server): include requested field in -32004 error data The spec's UnsupportedProtocolVersionError requires data to carry both supported and requested. The mock-server and request-metadata -32004 producers already include it; this was the remaining producer. * feat(runner): client-side spec-version inference, skip, and --force Mirrors the server runner's policy from #318: an explicitly requested --spec-version outside a scenario's applicability window skips it (exit 0) unless --force; when --spec-version is omitted the version is inferred from the scenario's applicability instead of defaulting to the latest dated version. * fix(auth): gate SEP-837 application_type check to spec versions that include it The sep-837-application-type-present check fired on every Dynamic Client Registration regardless of the targeted spec version, but SEP-837 only exists in the draft spec. Clients conforming to 2025-11-25 or earlier were failed on a requirement that is not part of the version under test. Thread the ScenarioContext into createAuthServer (mirroring createServer) and only emit the check when the resolved spec version includes SEP-837, i.e. the stateless draft lifecycle. At dated versions the check is not emitted at all. Audited the other auth helper checks (PKCE, metadata discovery, token verification); all of them enforce requirements that are already in the dated specs, so SEP-837 was the only draft-only check. Unit tests cover both sides: the check fails an omitting client at the draft version and is absent at 2025-11-25. * feat(sdk-runner): --spec-version passthrough with per-SDK default The sdk command forwarded --scenario/--suite/--timeout to the underlying client/server invocation but had no way to set the spec version, so SDK runs always used the suite default. Add a --spec-version flag that is passed through to the inner run. Also add an optional specVersion field to the per-SDK config, used as the default when the flag isn't given; an explicit --spec-version overrides it. Set typescript-sdk-v1 to 2025-11-25: the v1 line targets the latest dated spec, which keeps draft-only scenarios out of its runs by default. --------- Co-authored-by: Paul Carleton --- README.md | 10 +- .../clients/typescript/everything-client.ts | 156 ++++++- .../servers/typescript/everything-server.ts | 29 +- src/connection/connection.test.ts | 19 +- src/connection/select.ts | 28 +- src/index.ts | 38 +- src/mock-server/index.ts | 60 +++ src/mock-server/mock-server.test.ts | 395 ++++++++++++++++++ src/mock-server/select.ts | 13 + src/mock-server/stateful.ts | 145 +++++++ src/mock-server/stateless.ts | 194 +++++++++ src/mock-server/testing.ts | 17 + src/runner/client.test.ts | 67 +++ src/runner/client.ts | 106 ++++- .../auth/authorization-server-migration.ts | 8 +- src/scenarios/client/auth/basic-cimd.ts | 6 +- .../client/auth/client-credentials.ts | 11 +- .../client/auth/discovery-metadata.ts | 7 +- .../auth/enterprise-managed-authorization.ts | 6 +- .../client/auth/helpers/createAuthServer.ts | 39 +- .../client/auth/helpers/createServer.ts | 47 ++- src/scenarios/client/auth/index.test.ts | 15 +- src/scenarios/client/auth/issuer-parameter.ts | 36 +- .../client/auth/march-spec-backcompat.ts | 9 +- src/scenarios/client/auth/offline-access.ts | 11 +- src/scenarios/client/auth/pre-registration.ts | 6 +- .../client/auth/resource-mismatch.ts | 6 +- src/scenarios/client/auth/scope-handling.ts | 26 +- .../client/auth/test_helpers/testClient.ts | 19 +- .../client/auth/token-endpoint-auth.ts | 6 +- src/scenarios/client/elicitation-defaults.ts | 10 +- src/scenarios/client/http-base.ts | 3 +- .../client/http-custom-headers.test.ts | 11 +- src/scenarios/client/http-custom-headers.ts | 5 +- .../client/http-standard-headers.test.ts | 7 +- src/scenarios/client/initialize.ts | 11 +- .../client/json-schema-ref-deref.test.ts | 3 +- src/scenarios/client/json-schema-ref-deref.ts | 3 +- src/scenarios/client/mrtr-client.ts | 3 +- src/scenarios/client/request-metadata.test.ts | 6 +- src/scenarios/client/request-metadata.ts | 6 +- src/scenarios/client/sse-retry.ts | 15 +- src/scenarios/client/tools_call.test.ts | 48 +++ src/scenarios/client/tools_call.ts | 213 +++------- src/scenarios/spec-version.test.ts | 11 +- src/sdk-runner/config.ts | 6 +- src/sdk-runner/index.ts | 15 +- src/sdk-runner/known-sdks.ts | 4 +- src/sdk-runner/sdk-runner.test.ts | 17 +- src/types.ts | 3 +- 50 files changed, 1638 insertions(+), 297 deletions(-) create mode 100644 src/mock-server/index.ts create mode 100644 src/mock-server/mock-server.test.ts create mode 100644 src/mock-server/select.ts create mode 100644 src/mock-server/stateful.ts create mode 100644 src/mock-server/stateless.ts create mode 100644 src/mock-server/testing.ts create mode 100644 src/runner/client.test.ts create mode 100644 src/scenarios/client/tools_call.test.ts diff --git a/README.md b/README.md index 71a8cb2..41f5e58 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,13 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") - `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), or `sep-835` -- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios +- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios. When omitted, the version is inferred from the scenario's spec applicability (draft-only scenarios run at the draft version, everything else at the latest dated release); an explicitly requested version outside a scenario's applicability window skips the scenario (exit 0) unless `--force` is passed +- `--force` - Run a scenario even if it is not applicable at the requested `--spec-version` - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. +The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. Clients under test must derive the lifecycle from the protocol version they are asked to run: dated versions through `2025-11-25` use the stateful lifecycle (initialize handshake), while the 2026 draft (`DRAFT-2026-v1`) uses the stateless lifecycle (per-request `_meta`). ### Server Testing @@ -237,6 +238,11 @@ npm start -- sdk --path ../typescript-sdk --skip-build --mode client # Narrow to one scenario / suite npm start -- sdk --path ../typescript-sdk --mode server --scenario server-initialize npm start -- sdk typescript-sdk --mode client --suite auth + +# Target a specific spec version (passed through to the underlying run). +# When omitted, the SDK's `specVersion` from KNOWN_SDKS is used, if set — +# e.g. typescript-sdk-v1 defaults to 2025-11-25. +npm start -- sdk typescript-sdk --mode client --spec-version draft ``` Build/run commands for each official SDK are looked up by name from [`src/sdk-runner/known-sdks.ts`](src/sdk-runner/known-sdks.ts) — no config file is required in the SDK repo. Resolution order is **CLI flag > built-in entry**, so any field can be overridden on the command line for refs that diverge from the built-in. diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 63ca051..ef272e4 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -21,6 +21,8 @@ import { } from '@modelcontextprotocol/sdk/client/auth-extensions.js'; import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import { ClientConformanceContextSchema } from '../../../src/schemas/context.js'; +import { DRAFT_PROTOCOL_VERSION } from '../../../src/types.js'; +import { STATELESS_SPEC_VERSIONS } from '../../../src/connection/select.js'; import { auth, extractWWWAuthenticateParams @@ -70,10 +72,135 @@ export function getHandler(scenarioName: string): ScenarioHandler | undefined { } // ============================================================================ -// Basic scenarios (initialize, tools-call) +// Stateless requester (SEP-2575 / 2026-x lifecycle) +// +// Shim for the fact that the SDK Client doesn't support stateless mode yet. +// Carry-forward handlers below pick this when the runner says the resolved +// spec version is stateless, so the same handler exercises both lifecycles. +// ============================================================================ + +const PROTOCOL_VERSION = process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; + +// Lifecycle decision: derived from the runner-provided protocol version. +// The version→lifecycle mapping is spec knowledge a client must own; this +// in-repo client imports the stateless version set from src/ so it cannot +// drift from the runner's mapping. +const USE_STATELESS_LIFECYCLE = PROTOCOL_VERSION + ? (STATELESS_SPEC_VERSIONS as readonly string[]).includes(PROTOCOL_VERSION) + : false; + +// Wire protocolVersion for stateless requests: the runner-resolved version +// when available (so a dated stateless release is exercised under its own +// identifier), the current draft otherwise. +const STATELESS_PROTOCOL_VERSION = PROTOCOL_VERSION ?? DRAFT_PROTOCOL_VERSION; + +const STATELESS_META_BASE = { + 'io.modelcontextprotocol/clientInfo': { + name: 'conformance-test-client', + version: '1.0.0' + }, + 'io.modelcontextprotocol/clientCapabilities': { + tools: {}, + roots: {}, + sampling: {}, + elicitation: {} + } +}; + +let _nextStatelessId = 1; +async function statelessRequest( + serverUrl: string, + method: string, + params: Record = {} +): Promise { + const _meta = { + 'io.modelcontextprotocol/protocolVersion': STATELESS_PROTOCOL_VERSION, + ...STATELESS_META_BASE, + ...((params._meta as object | undefined) ?? {}) + }; + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + // Servers built on the SDK's StreamableHTTPServerTransport reject + // requests that don't accept both JSON and SSE responses. + Accept: 'application/json, text/event-stream', + 'MCP-Protocol-Version': STATELESS_PROTOCOL_VERSION + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: _nextStatelessId++, + method, + params: { ...params, _meta } + }) + }); + const body = await response.json(); + if (body.error) { + throw new Error( + `${method} failed: ${body.error.code} ${body.error.message}` + ); + } + return body.result; +} + +// ============================================================================ +// Basic scenarios (initialize, tools_call) // ============================================================================ async function runBasicClient(serverUrl: string): Promise { + if (USE_STATELESS_LIFECYCLE) { + logger.debug('Stateless lifecycle: calling tools/list + tools/call'); + const list = await statelessRequest(serverUrl, 'tools/list'); + logger.debug('Successfully listed tools:', JSON.stringify(list)); + const tool = list?.tools?.[0]; + if (tool) { + const result = await statelessRequest(serverUrl, 'tools/call', { + name: tool.name, + arguments: { a: 2, b: 3 } + }); + logger.debug('Successfully called tool:', JSON.stringify(result)); + } + return; + } + + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + const list = await client.listTools(); + logger.debug('Successfully listed tools'); + + const tool = list.tools[0]; + if (tool) { + await client.callTool({ name: tool.name, arguments: { a: 2, b: 3 } }); + logger.debug('Successfully called tool'); + } + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenarios(['initialize', 'tools_call', 'tools-call'], runBasicClient); + +// SEP-2106: json-schema-ref-no-deref advertises a tool whose inputSchema +// contains a network-URI $ref. A conformant client lists tools normally and +// simply never fetches that URI. The scenario's mock only serves tools/list, +// so this handler stops after listing instead of reusing runBasicClient +// (whose tools/call would get -32601 and fail the run). +async function runListToolsOnlyClient(serverUrl: string): Promise { + if (USE_STATELESS_LIFECYCLE) { + logger.debug('Stateless lifecycle: calling tools/list'); + const list = await statelessRequest(serverUrl, 'tools/list'); + logger.debug('Successfully listed tools:', JSON.stringify(list)); + return; + } + const client = new Client( { name: 'test-client', version: '1.0.0' }, { capabilities: {} } @@ -91,13 +218,7 @@ async function runBasicClient(serverUrl: string): Promise { logger.debug('Connection closed successfully'); } -registerScenarios(['initialize', 'tools-call'], runBasicClient); - -// SEP-2106: json-schema-ref-no-deref advertises a tool whose inputSchema -// contains a network-URI $ref. A conformant client lists tools normally and -// simply never fetches that URI, so the basic connect+listTools flow is the -// correct behavior here. -registerScenario('json-schema-ref-no-deref', runBasicClient); +registerScenario('json-schema-ref-no-deref', runListToolsOnlyClient); // ============================================================================ // request-metadata scenario (SEP-2575) @@ -106,20 +227,9 @@ registerScenario('json-schema-ref-no-deref', runBasicClient); async function runRequestMetadataClient(serverUrl: string): Promise { logger.debug('Starting request-metadata client flow...'); - const meta = { - 'io.modelcontextprotocol/clientInfo': { - name: 'conformance-test-client', - version: '1.0.0' - }, - 'io.modelcontextprotocol/clientCapabilities': { - tools: {}, - roots: {}, - sampling: {}, - elicitation: {} - } - }; + const meta = STATELESS_META_BASE; - let activeVersion = 'DRAFT-2026-v1'; + let activeVersion = STATELESS_PROTOCOL_VERSION; const sendRequestWithNegotiation = async ( method: string, @@ -166,7 +276,9 @@ async function runRequestMetadataClient(serverUrl: string): Promise { ); const serverSupported: string[] = errorResult.error.data?.supported || []; - const clientSupported = ['DRAFT-2026-v1']; + const clientSupported = [ + ...new Set([STATELESS_PROTOCOL_VERSION, DRAFT_PROTOCOL_VERSION]) + ]; const mutuallySupported = clientSupported.filter((v) => serverSupported.includes(v) ); diff --git a/examples/servers/typescript/everything-server.ts b/examples/servers/typescript/everything-server.ts index 7a587ff..ce43945 100644 --- a/examples/servers/typescript/everything-server.ts +++ b/examples/servers/typescript/everything-server.ts @@ -1283,7 +1283,10 @@ app.post('/mcp', async (req, res) => { error: { code: -32004, message: 'UnsupportedProtocolVersionError', - data: { supported: ['DRAFT-2026-v1'] } + data: { + supported: ['DRAFT-2026-v1'], + requested: String(metaVersion) + } } }); } @@ -1436,6 +1439,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1465,6 +1474,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1549,6 +1564,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1570,6 +1591,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } diff --git a/src/connection/connection.test.ts b/src/connection/connection.test.ts index b308284..be09341 100644 --- a/src/connection/connection.test.ts +++ b/src/connection/connection.test.ts @@ -1,8 +1,13 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import { connectFor } from './select'; +import { + connectFor, + isStatefulVersion, + STATELESS_SPEC_VERSIONS +} from './select'; import { connectStateful } from './stateful'; import { connectStateless } from './stateless'; import { JsonRpcError } from './index'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; describe('connectFor', () => { it('returns stateful for dated 2025-x versions', () => { @@ -20,6 +25,18 @@ describe('connectFor', () => { }); }); +describe('STATELESS_SPEC_VERSIONS', () => { + it('contains exactly the versions isStatefulVersion rejects', () => { + expect(STATELESS_SPEC_VERSIONS.length).toBeGreaterThan(0); + for (const v of STATELESS_SPEC_VERSIONS) { + expect(isStatefulVersion(v)).toBe(false); + } + }); + it('currently contains only the draft version', () => { + expect(STATELESS_SPEC_VERSIONS).toEqual([DRAFT_PROTOCOL_VERSION]); + }); +}); + describe('connectStateless', () => { const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); diff --git a/src/connection/select.ts b/src/connection/select.ts index ca33b4e..aa6fe17 100644 --- a/src/connection/select.ts +++ b/src/connection/select.ts @@ -1,4 +1,8 @@ -import type { SpecVersion } from '../types'; +import { + DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION, + type SpecVersion +} from '../types'; import type { Connection } from './index'; import { connectStateful } from './stateful'; import { connectStateless } from './stateless'; @@ -14,10 +18,30 @@ const STATEFUL_VERSIONS: ReadonlySet = new Set([ '2025-11-25' ]); +/** Every spec version the suite can target, in timeline order. */ +const ALL_SPEC_VERSIONS: readonly SpecVersion[] = [ + ...DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION +]; + +export function isStatefulVersion(v: SpecVersion): boolean { + return STATEFUL_VERSIONS.has(v); +} + +/** + * Spec versions that use the stateless lifecycle, derived from + * {@link isStatefulVersion} so there is a single source of truth for the + * version→lifecycle mapping. The list grows automatically when the draft is + * dated (added to `DATED_SPEC_VERSIONS` without joining `STATEFUL_VERSIONS`) + * or a second stateless version appears. + */ +export const STATELESS_SPEC_VERSIONS: readonly SpecVersion[] = + ALL_SPEC_VERSIONS.filter((v) => !isStatefulVersion(v)); + export function connectFor( specVersion: SpecVersion ): (serverUrl: string) => Promise { - return STATEFUL_VERSIONS.has(specVersion) + return isStatefulVersion(specVersion) ? connectStateful : // Pass the version through so stateless requests declare the spec // version the run was invoked with (matters under --force). diff --git a/src/index.ts b/src/index.ts index 637a357..f122ccb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -101,6 +101,10 @@ program '--spec-version ', 'Filter scenarios by spec version (cumulative for date versions)' ) + .option( + '--force', + 'Run a scenario even if it is not applicable at the requested --spec-version' + ) .option('--verbose', 'Show verbose output') .action(async (options) => { try { @@ -157,11 +161,13 @@ program scenarioName, timeout, outputDir, - specVersionFilter + specVersionFilter, + options.force ?? false ); return { scenario: scenarioName, checks: result.checks, + skipped: result.skipped, error: null }; } catch (error) { @@ -178,6 +184,7 @@ program error instanceof Error ? error.message : String(error) } ], + skipped: undefined, error }; } @@ -189,8 +196,17 @@ program let totalPassed = 0; let totalFailed = 0; let totalWarnings = 0; + let totalSkipped = 0; for (const result of results) { + // Inapplicable scenario/spec-version combination (already logged by + // the runner). Not a failure: report distinctly. + if (result.skipped) { + totalSkipped++; + console.log(`- ${result.scenario}: skipped`); + continue; + } + const passed = result.checks.filter( (c) => c.status === 'SUCCESS' ).length; @@ -222,8 +238,9 @@ program } } + const skippedStr = totalSkipped > 0 ? `, ${totalSkipped} skipped` : ''; console.log( - `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings` + `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings${skippedStr}` ); if (options.expectedFailures) { @@ -255,7 +272,13 @@ program // If no command provided, run in interactive mode if (!validated.command) { - await runInteractiveMode(validated.scenario, verbose, outputDir); + await runInteractiveMode( + validated.scenario, + verbose, + outputDir, + specVersionFilter, + options.force ?? false + ); process.exit(0); } @@ -265,9 +288,16 @@ program validated.scenario, timeout, outputDir, - specVersionFilter + specVersionFilter, + options.force ?? false ); + // Inapplicable scenario/spec-version combination (already logged by + // the runner). Not a failure: exit 0. + if (result.skipped) { + process.exit(0); + } + const { overallFailure } = printClientResults( result.checks, verbose, diff --git a/src/mock-server/index.ts b/src/mock-server/index.ts new file mode 100644 index 0000000..7303cb8 --- /dev/null +++ b/src/mock-server/index.ts @@ -0,0 +1,60 @@ +/** + * Version-aware mock-server abstraction for client-conformance scenarios. + * + * A `MockServer` is the HTTP server a client-under-test connects to. The + * lifecycle scaffold (initialize handshake vs per-request `_meta` validation) + * is supplied by the runner based on `--spec-version`; the scenario only + * provides per-method handlers and asserts on the recorded requests. + * + * This is the client-conformance mirror of `Connection` in `../connection`. + */ + +import type { SpecVersion } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; + +/** + * Per-method response handlers. Called with the request `params` object; + * return value becomes the JSON-RPC `result`. Throw to produce an error + * response. + */ +export type RequestHandlers = Record< + string, + ( + params: Record, + request: JSONRPCRequest + ) => unknown | Promise +>; + +export interface MockServer { + /** Full URL of the `/mcp` endpoint. */ + url: string; + /** Base URL (no `/mcp` suffix), for scenarios that serve sibling routes. */ + baseUrl: string; + /** + * Every JSON-RPC request the client sent, in arrival order, excluding the + * lifecycle preamble (`initialize` / `notifications/initialized` under the + * stateful impl; `server/discover` under stateless). Recording happens + * before validation, so requests the server rejects (e.g. missing header + * or `_meta`) still appear here. + */ + readonly recorded: JSONRPCRequest[]; + close(): Promise; +} + +/** + * Per-run context handed to `Scenario.start()`. The runner constructs this + * from the resolved `--spec-version`. + */ +export interface ScenarioContext { + specVersion: SpecVersion; + /** + * Create a version-appropriate mock server. Scenarios that test the + * lifecycle itself (initialize, SSE-retry) bypass this and build a raw + * `http.createServer`. + */ + createServer(handlers: RequestHandlers): Promise; +} + +export { createServerStateful } from './stateful'; +export { createServerStateless, validateStatelessRequest } from './stateless'; +export { createServerFor } from './select'; diff --git a/src/mock-server/mock-server.test.ts b/src/mock-server/mock-server.test.ts new file mode 100644 index 0000000..c0247f8 --- /dev/null +++ b/src/mock-server/mock-server.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect } from 'vitest'; +import { createServerFor } from './select'; +import { createServerStateful } from './stateful'; +import { createServerStateless, validateStatelessRequest } from './stateless'; +import { STATELESS_SPEC_VERSIONS } from '../connection/select'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; + +const meta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientInfo': { name: 't', version: '1' }, + 'io.modelcontextprotocol/clientCapabilities': {} +}; +const headers = { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION }; + +async function post(url: string, body: object, headers: object = {}) { + const r = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + return { status: r.status, body: await r.json() }; +} + +describe('validateStatelessRequest', () => { + it('returns reject for invalid requests', () => { + const v = validateStatelessRequest( + { + headers: {}, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'reject', status: 400 }); + }); + + it('returns handled for server/discover', () => { + const v = validateStatelessRequest( + { + headers, + body: { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'handled', status: 200 }); + }); + + it('returns route for valid non-discover requests', () => { + const v = validateStatelessRequest( + { + headers, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'route', id: 1, method: 'tools/list' }); + }); + + it('rejects versions outside the supported list with -32004 and echoes it', () => { + const v = validateStatelessRequest( + { + headers: { 'mcp-protocol-version': '2099-01-01' }, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { + _meta: { + ...meta, + 'io.modelcontextprotocol/protocolVersion': '2099-01-01' + } + } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ + kind: 'reject', + status: 400, + body: { + error: { + code: -32004, + data: { supported: [DRAFT_PROTOCOL_VERSION], requested: '2099-01-01' } + } + } + }); + }); +}); + +describe('createServerFor', () => { + it('returns stateful for dated 2025-x versions', () => { + expect(createServerFor('2025-06-18')).toBe(createServerStateful); + expect(createServerFor('2025-11-25')).toBe(createServerStateful); + }); + it('returns a stateless factory bound to the requested version', async () => { + const srv = await createServerFor(DRAFT_PROTOCOL_VERSION)({}); + try { + const { status, body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + headers + ); + expect(status).toBe(200); + expect(body.result.supportedVersions).toEqual([DRAFT_PROTOCOL_VERSION]); + } finally { + await srv.close(); + } + }); +}); + +describe('createServerStateless', () => { + it('rejects requests missing the version header', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post(srv.url, { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }); + expect(status).toBe(400); + expect(body.error.code).toBe(-32001); + } finally { + await srv.close(); + } + }); + + it('rejects requests missing required _meta keys', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(400); + expect(body.error.code).toBe(-32602); + } finally { + await srv.close(); + } + }); + + it('serves server/discover, defaulting to every known stateless version', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(200); + expect(body.result.supportedVersions).toEqual(STATELESS_SPEC_VERSIONS); + expect(body.result.serverInfo.name).toBe('conformance-mock-server'); + } finally { + await srv.close(); + } + }); + + it('accepts the version it was created for and rejects others with -32004', async () => { + const srv = await createServerStateless( + { 'tools/list': () => ({ tools: [] }) }, + DRAFT_PROTOCOL_VERSION + ); + try { + const accepted = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }, + headers + ); + expect(accepted.status).toBe(200); + + const rejected = await post( + srv.url, + { + jsonrpc: '2.0', + id: 2, + method: 'tools/list', + params: { + _meta: { + ...meta, + 'io.modelcontextprotocol/protocolVersion': '2099-01-01' + } + } + }, + { 'mcp-protocol-version': '2099-01-01' } + ); + expect(rejected.status).toBe(400); + expect(rejected.body.error.code).toBe(-32004); + expect(rejected.body.error.data.supported).toEqual([ + DRAFT_PROTOCOL_VERSION + ]); + expect(rejected.body.error.data.requested).toBe('2099-01-01'); + } finally { + await srv.close(); + } + }); + + it('routes to handlers and records requests', async () => { + const srv = await createServerStateless({ + 'tools/list': () => ({ tools: [{ name: 'x' }] }) + }); + try { + const { body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(body.result.tools[0].name).toBe('x'); + expect(srv.recorded).toHaveLength(1); + expect(srv.recorded[0].method).toBe('tools/list'); + } finally { + await srv.close(); + } + }); + + it('records requests rejected by validation (missing _meta)', async () => { + const srv = await createServerStateless({}); + try { + const { status } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(400); + expect(srv.recorded.map((r) => r.method)).toEqual(['tools/list']); + } finally { + await srv.close(); + } + }); + + it('does not record the server/discover preamble', async () => { + const srv = await createServerStateless({}); + try { + const { status } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(200); + expect(srv.recorded).toHaveLength(0); + } finally { + await srv.close(); + } + }); + + it('returns -32601 for unknown methods', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'nope', params: { _meta: meta } }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(404); + expect(body.error.code).toBe(-32601); + } finally { + await srv.close(); + } + }); +}); + +describe('createServerStateful', () => { + async function postInit(url: string) { + const r = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-11-25', + capabilities: {}, + clientInfo: { name: 't', version: '1' } + } + }) + }); + return { + status: r.status, + contentType: r.headers.get('content-type') ?? '' + }; + } + + it('accepts initialize and routes to handlers, recording non-preamble', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + // SDK transport in sessionless mode handles initialize internally; we + // can drive it via the SDK Client. + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client.listTools(); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toEqual(['tools/list']); + } finally { + await srv.close(); + } + }); + + it('derives capabilities from handler keys; non-tools handler does not 500 initialize', async () => { + const srv = await createServerStateful({ + 'prompts/list': () => ({ prompts: [] }) + }); + try { + const { status, contentType } = await postInit(srv.url); + expect(status).toBe(200); + expect(contentType).not.toContain('text/html'); + } finally { + await srv.close(); + } + }); + + it('records requests for unregistered methods (parity with stateless)', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const { ResultSchema } = + await import('@modelcontextprotocol/sdk/types.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client + .request( + { method: 'tools/call', params: { name: 'nope' } }, + ResultSchema + ) + .catch(() => {}); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toContain('tools/call'); + } finally { + await srv.close(); + } + }); +}); diff --git a/src/mock-server/select.ts b/src/mock-server/select.ts new file mode 100644 index 0000000..3458647 --- /dev/null +++ b/src/mock-server/select.ts @@ -0,0 +1,13 @@ +import type { SpecVersion } from '../types'; +import type { MockServer, RequestHandlers } from './index'; +import { isStatefulVersion } from '../connection/select'; +import { createServerStateful } from './stateful'; +import { createServerStateless } from './stateless'; + +export function createServerFor( + specVersion: SpecVersion +): (handlers: RequestHandlers) => Promise { + return isStatefulVersion(specVersion) + ? createServerStateful + : (handlers) => createServerStateless(handlers, specVersion); +} diff --git a/src/mock-server/stateful.ts b/src/mock-server/stateful.ts new file mode 100644 index 0000000..8dab541 --- /dev/null +++ b/src/mock-server/stateful.ts @@ -0,0 +1,145 @@ +/** + * Stateful mock server: 2025-x lifecycle (initialize handshake). + * + * Backed by the SDK's `Server` + `StreamableHTTPServerTransport` so we don't + * reimplement the handshake or SSE response framing. The SDK is the scaffold + * here, not the system-under-test; the client-under-test connecting to this + * mock is what's being verified. + */ + +import express from 'express'; +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; +import { z } from 'zod'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; + +const CAPABILITY_BY_PREFIX: Record = { + tools: 'tools', + prompts: 'prompts', + resources: 'resources', + completion: 'completions', + logging: 'logging' +}; + +/** + * Derive the server `capabilities` object from the registered handler method + * names so the SDK's `assertRequestHandlerCapability` gate is always satisfied. + * Shared with the stateless impl for `server/discover`. + */ +export function capabilitiesFromHandlers( + handlers: RequestHandlers +): Record { + const out: Record = {}; + for (const method of Object.keys(handlers)) { + const cap = CAPABILITY_BY_PREFIX[method.split('/')[0]]; + if (cap) out[cap] = {}; + } + return out; +} + +export async function createServerStateful( + handlers: RequestHandlers +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + + // Fresh SDK Server per HTTP request (the SDK transport is single-shot in + // sessionless mode after GHSA-345p-7cg4-v4c7). + function newServer(): Server { + const server = new Server( + { name: 'conformance-mock-server', version: '1.0.0' }, + { capabilities } + ); + for (const [method, handler] of Object.entries(handlers)) { + // The SDK's setRequestHandler matches by parsing against the schema's + // method literal; build a minimal schema so any method string works. + const schema = z.object({ + method: z.literal(method), + params: z.unknown().optional() + }); + server.setRequestHandler(schema, async (request) => { + try { + return (await handler( + (request.params ?? {}) as Record, + request as JSONRPCRequest + )) as Record; + } catch (e) { + if (e instanceof McpError) throw e; + throw new McpError( + ErrorCode.InternalError, + e instanceof Error ? e.message : String(e) + ); + } + }); + } + return server; + } + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + // Record every JSON-RPC request the client sends (excluding the lifecycle + // preamble) at the HTTP layer so unregistered methods are captured too, + // matching the stateless impl and the MockServer.recorded contract. + const body = req.body; + if ( + body?.method && + body.method !== 'initialize' && + body.method !== 'notifications/initialized' + ) { + recorded.push(body as JSONRPCRequest); + } + try { + const server = newServer(); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined + }); + await server.connect(transport); + // Register cleanup before handing the request to the transport so the + // pair is torn down even when handleRequest throws. + res.on('close', () => { + transport.close(); + server.close(); + }); + await transport.handleRequest(req, res, req.body); + } catch (e) { + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + id: req.body?.id ?? null, + error: { code: -32603, message: String(e) } + }); + } + } + }); + + return listen(app, recorded); +} + +function listen( + app: express.Application, + recorded: JSONRPCRequest[] +): Promise { + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((res) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => res()); + }) + }); + }); + }); +} diff --git a/src/mock-server/stateless.ts b/src/mock-server/stateless.ts new file mode 100644 index 0000000..78fd638 --- /dev/null +++ b/src/mock-server/stateless.ts @@ -0,0 +1,194 @@ +/** + * Stateless mock server: 2026-x lifecycle (SEP-2575). + * + * No initialize handshake. Validates `_meta` (protocolVersion / clientInfo / + * clientCapabilities) and the `MCP-Protocol-Version` header on every request, + * serves `server/discover`, and routes other methods to the supplied handlers. + * Implemented with raw express so it can front-run SDK support. + */ + +import express from 'express'; +import type { SpecVersion } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; +import { STATELESS_SPEC_VERSIONS } from '../connection/select'; +import { capabilitiesFromHandlers } from './stateful'; + +const META_KEYS = [ + 'io.modelcontextprotocol/protocolVersion', + 'io.modelcontextprotocol/clientInfo', + 'io.modelcontextprotocol/clientCapabilities' +] as const; + +type IncomingHeaders = Record; + +export type StatelessValidation = + | { kind: 'reject'; status: number; body: object } + | { kind: 'handled'; status: number; body: object } + | { + kind: 'route'; + id: string | number | null; + method: string; + params: Record; + }; + +/** + * Shared SEP-2575 request validation: header presence, `_meta` 3-key check, + * header/`_meta` version match, version-supported check, and `server/discover` + * handling. Returns `reject` when validation failed, `handled` when the + * request was valid and already answered (`server/discover`), and `route` + * when the caller should dispatch to its own handlers. Consumers write + * `res.status(v.status).json(v.body)` for `reject` and `handled` alike and + * route only on `route`. + * + * Exported so any mock server that needs a stateless `/mcp` route (e.g. + * `auth/helpers/createServer.ts`) uses the same validation as this module. + * + * `supportedVersions` is the list of wire protocolVersion strings this + * endpoint accepts; anything else is rejected with -32004 carrying + * `{ supported, requested }` in the error data, and the list is echoed in + * the `server/discover` result. + */ +export function validateStatelessRequest( + req: { headers: IncomingHeaders; body: unknown }, + capabilities: Record, + supportedVersions: readonly string[] +): StatelessValidation { + const body = (req.body ?? {}) as Record; + const id = (body.id ?? null) as string | number | null; + const method = body.method as string; + const params = (body.params ?? {}) as Record; + const meta = params._meta as Record | undefined; + + const reject = (status: number, code: number, message: string) => + ({ + kind: 'reject', + status, + body: { jsonrpc: '2.0', id, error: { code, message } } + }) as const; + + const headerVersion = req.headers['mcp-protocol-version']; + if (!headerVersion) { + return reject(400, -32001, 'Missing MCP-Protocol-Version header'); + } + const missing = META_KEYS.filter((k) => meta?.[k] === undefined); + if (missing.length > 0) { + return reject( + 400, + -32602, + `Invalid params: missing _meta keys: ${missing.join(', ')}` + ); + } + if (meta?.[META_KEYS[0]] !== headerVersion) { + return reject( + 400, + -32001, + 'MCP-Protocol-Version header does not match _meta.protocolVersion' + ); + } + if ( + typeof headerVersion !== 'string' || + !supportedVersions.includes(headerVersion) + ) { + return { + kind: 'reject', + status: 400, + body: { + jsonrpc: '2.0', + id, + error: { + code: -32004, + message: 'Unsupported protocol version', + data: { + supported: supportedVersions, + requested: String(headerVersion) + } + } + } + }; + } + if (method === 'server/discover') { + return { + kind: 'handled', + status: 200, + body: { + jsonrpc: '2.0', + id, + result: { + supportedVersions, + capabilities, + serverInfo: { name: 'conformance-mock-server', version: '1.0.0' } + } + } + }; + } + return { kind: 'route', id, method, params }; +} + +/** + * When `specVersion` is given (the runner's resolved `--spec-version`), the + * server accepts exactly that version. Without it, every known stateless + * version is accepted. + */ +export async function createServerStateless( + handlers: RequestHandlers, + specVersion?: SpecVersion +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + const supportedVersions: readonly string[] = specVersion + ? [specVersion] + : STATELESS_SPEC_VERSIONS; + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + // Record every JSON-RPC request the client sends (excluding the + // `server/discover` lifecycle preamble) before validation so rejected + // requests are captured too, matching the stateful impl and the + // MockServer.recorded contract. + const body = req.body as Record | undefined; + if (body?.method && body.method !== 'server/discover') { + recorded.push(req.body as JSONRPCRequest); + } + const v = validateStatelessRequest(req, capabilities, supportedVersions); + if (v.kind !== 'route') { + return res.status(v.status).json(v.body); + } + const { id, method, params } = v; + const error = (status: number, code: number, message: string) => + res.status(status).json({ jsonrpc: '2.0', id, error: { code, message } }); + + const handler = handlers[method]; + if (!handler) { + return error(404, -32601, `Method not found: ${method}`); + } + try { + const result = await handler(params, req.body as JSONRPCRequest); + return res.json({ jsonrpc: '2.0', id, result }); + } catch (e) { + return error(500, -32603, e instanceof Error ? e.message : String(e)); + } + }); + + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((r) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => r()); + }) + }); + }); + }); +} diff --git a/src/mock-server/testing.ts b/src/mock-server/testing.ts new file mode 100644 index 0000000..1e307b8 --- /dev/null +++ b/src/mock-server/testing.ts @@ -0,0 +1,17 @@ +import { LATEST_SPEC_VERSION, type SpecVersion } from '../types'; +import { createServerFor } from './select'; +import type { ScenarioContext } from './index'; + +/** + * Build a ScenarioContext for unit tests that drive a Scenario directly. + * Defaults to the latest dated spec version (stateful lifecycle) so existing + * tests keep their pre-ScenarioContext behaviour. + */ +export function testScenarioContext( + specVersion: SpecVersion = LATEST_SPEC_VERSION +): ScenarioContext { + return { + specVersion, + createServer: (handlers) => createServerFor(specVersion)(handlers) + }; +} diff --git a/src/runner/client.test.ts b/src/runner/client.test.ts new file mode 100644 index 0000000..8ce464a --- /dev/null +++ b/src/runner/client.test.ts @@ -0,0 +1,67 @@ +/** + * Tests for the client-conformance runner: spec-version applicability + * skipping (an explicitly-requested version outside a scenario's window + * skips rather than silently testing something else; --force overrides) + * and version inference when --spec-version is omitted. + */ +import { describe, test, expect } from 'vitest'; +import { runConformanceTest } from './client'; +import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../types'; + +// A "client" that just prints the protocol version handed to it, so tests +// can observe which version the runner resolved. +const PRINT_VERSION_COMMAND = + 'node -e "console.log(process.env.MCP_CONFORMANCE_PROTOCOL_VERSION)"'; + +describe('runConformanceTest spec-version applicability', () => { + test('skips a draft-only scenario at an explicit dated spec version', async () => { + // http-custom-headers is introducedIn DRAFT, so a dated version + // contradicts it. The skip happens before the mock server starts and + // before the client command is spawned. + const result = await runConformanceTest( + PRINT_VERSION_COMMAND, + 'http-custom-headers', + 5000, + undefined, + LATEST_SPEC_VERSION + ); + expect(result.skipped).toBe(true); + expect(result.checks).toEqual([]); + expect(result.clientOutput).toBeUndefined(); + }); + + test('--force runs an inapplicable scenario at the requested version', async () => { + const result = await runConformanceTest( + PRINT_VERSION_COMMAND, + 'http-custom-headers', + 10000, + undefined, + LATEST_SPEC_VERSION, + true + ); + expect(result.skipped).toBeUndefined(); + expect(result.clientOutput?.stdout).toContain(LATEST_SPEC_VERSION); + }, 30000); + + test('infers the draft version for a draft-only scenario when --spec-version is omitted', async () => { + const result = await runConformanceTest( + PRINT_VERSION_COMMAND, + 'http-custom-headers', + 10000 + ); + expect(result.skipped).toBeUndefined(); + expect(result.clientOutput?.stdout).toContain(DRAFT_PROTOCOL_VERSION); + }, 30000); + + test('infers the latest dated version for a dual-version scenario when --spec-version is omitted', async () => { + // tools_call is introducedIn 2025-06-18 and still applicable at draft; + // omitting --spec-version must keep the latest dated default. + const result = await runConformanceTest( + PRINT_VERSION_COMMAND, + 'tools_call', + 10000 + ); + expect(result.skipped).toBeUndefined(); + expect(result.clientOutput?.stdout).toContain(LATEST_SPEC_VERSION); + }, 30000); +}); diff --git a/src/runner/client.ts b/src/runner/client.ts index 1bf8c9f..2f9898c 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,8 +1,15 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck, SpecVersion } from '../types'; -import { getScenario } from '../scenarios'; +import { + ConformanceCheck, + ScenarioSource, + SpecVersion, + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +} from '../types'; +import { getScenario, isScenarioApplicableAt } from '../scenarios'; +import { createServerFor, type ScenarioContext } from '../mock-server'; import { createResultDir, formatPrettyChecks } from './utils'; export interface ClientExecutionResult { @@ -35,9 +42,7 @@ async function executeClient( // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; env.MCP_CONFORMANCE_SCENARIO = scenarioName; - if (specVersion) { - env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion; - } + env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion ?? LATEST_SPEC_VERSION; if (context) { // Include scenario name in context for discriminated union parsing env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ @@ -92,17 +97,64 @@ async function executeClient( }); } +// An explicitly-requested spec version outside the scenario's applicability +// window is a contradiction: running anyway would test something other than +// what the flag claims. Skip (exit 0) unless --force. +function shouldSkipForSpecVersion( + source: ScenarioSource, + scenarioName: string, + specVersion: SpecVersion | undefined, + force: boolean +): boolean { + if ( + specVersion === undefined || + force || + isScenarioApplicableAt(source, specVersion) + ) { + return false; + } + const introduced = + 'introducedIn' in source + ? `introduced in ${source.introducedIn}` + + (source.removedIn !== undefined + ? `, removed in ${source.removedIn}` + : '') + : 'extension scenario, not on the spec timeline'; + console.log( + `SKIPPED: scenario '${scenarioName}' is not applicable at spec version ` + + `${specVersion} (${introduced}). Use --force to run it anyway.` + ); + return true; +} + +// When --spec-version is omitted, infer the version from the scenario's +// declared source so draft-only scenarios get the draft (stateless) mock +// server rather than the stateful latest-spec default. +function resolveScenarioSpecVersion( + source: ScenarioSource, + specVersion: SpecVersion | undefined +): SpecVersion { + return ( + specVersion ?? + ('introducedIn' in source && source.introducedIn === DRAFT_PROTOCOL_VERSION + ? DRAFT_PROTOCOL_VERSION + : LATEST_SPEC_VERSION) + ); +} + export async function runConformanceTest( clientCommand: string, scenarioName: string, timeout: number = 30000, outputDir?: string, - specVersion?: SpecVersion + specVersion?: SpecVersion, + force = false ): Promise<{ checks: ConformanceCheck[]; - clientOutput: ClientExecutionResult; + clientOutput?: ClientExecutionResult; resultDir?: string; allowClientError?: boolean; + skipped?: boolean; }> { let resultDir: string | undefined; @@ -114,8 +166,23 @@ export async function runConformanceTest( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + if ( + shouldSkipForSpecVersion(scenario.source, scenarioName, specVersion, force) + ) { + return { checks: [], resultDir, skipped: true }; + } + + const resolvedVersion = resolveScenarioSpecVersion( + scenario.source, + specVersion + ); + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.error(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.error(`Executing client: ${clientCommand} ${urls.serverUrl}`); if (urls.context) { @@ -129,7 +196,7 @@ export async function runConformanceTest( urls.serverUrl, timeout, urls.context, - specVersion + resolvedVersion ); // Print stdout/stderr if client exited with nonzero code @@ -269,7 +336,9 @@ export function printClientResults( export async function runInteractiveMode( scenarioName: string, verbose: boolean = false, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion, + force = false ): Promise { let resultDir: string | undefined; @@ -281,8 +350,23 @@ export async function runInteractiveMode( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + if ( + shouldSkipForSpecVersion(scenario.source, scenarioName, specVersion, force) + ) { + return; + } + + const resolvedVersion = resolveScenarioSpecVersion( + scenario.source, + specVersion + ); + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.log(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.log(`Server URL: ${urls.serverUrl}`); console.log('Press Ctrl+C to stop...'); diff --git a/src/scenarios/client/auth/authorization-server-migration.ts b/src/scenarios/client/auth/authorization-server-migration.ts index 858bddb..6c7c989 100644 --- a/src/scenarios/client/auth/authorization-server-migration.ts +++ b/src/scenarios/client/auth/authorization-server-migration.ts @@ -6,6 +6,7 @@ * to AS₂. On the next 401 the client re-discovers PRM, sees a new issuer, and * MUST re-register with AS₂ rather than reuse AS₁'s client credentials. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Request, Response, NextFunction } from 'express'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; @@ -25,7 +26,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, ['mcp:basic']); @@ -36,7 +37,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { let as2SawAs1ClientIdAtToken = false; // ── AS₁ ──────────────────────────────────────────────────────────────── - const as1App = createAuthServer(this.checks, this.as1.getUrl, { + const as1App = createAuthServer(ctx, this.checks, this.as1.getUrl, { tokenVerifier, onRegistrationRequest: () => ({ clientId: AS1_CLIENT_ID, @@ -46,7 +47,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { await this.as1.start(as1App); // ── AS₂ ──────────────────────────────────────────────────────────────── - const as2App = createAuthServer(this.checks, this.as2.getUrl, { + const as2App = createAuthServer(ctx, this.checks, this.as2.getUrl, { tokenVerifier, onRegistrationRequest: () => { as2SawRegister = true; @@ -125,6 +126,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { }; const app = createServer( + ctx, this.checks, this.server.getUrl, currentAuthServerUrl, diff --git a/src/scenarios/client/auth/basic-cimd.ts b/src/scenarios/client/auth/basic-cimd.ts index a99b4e3..1e4b37c 100644 --- a/src/scenarios/client/auth/basic-cimd.ts +++ b/src/scenarios/client/auth/basic-cimd.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -29,10 +30,10 @@ export class AuthBasicCIMDScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { clientIdMetadataDocumentSupported: true, onAuthorizationRequest: (data) => { // Check if client used URL-based client ID @@ -60,6 +61,7 @@ export class AuthBasicCIMDScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/client-credentials.ts b/src/scenarios/client/auth/client-credentials.ts index eaebcb0..d633f7a 100644 --- a/src/scenarios/client/auth/client-credentials.ts +++ b/src/scenarios/client/auth/client-credentials.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; @@ -42,13 +43,13 @@ export class ClientCredentialsJwtScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate a fresh keypair for this test run const { publicKey, privateKeyPem } = await generateTestKeypair(); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { grantTypesSupported: ['client_credentials'], tokenEndpointAuthMethodsSupported: ['private_key_jwt'], tokenEndpointAuthSigningAlgValuesSupported: ['ES256'], @@ -201,6 +202,7 @@ export class ClientCredentialsJwtScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl @@ -263,10 +265,10 @@ export class ClientCredentialsBasicScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { grantTypesSupported: ['client_credentials'], tokenEndpointAuthMethodsSupported: ['client_secret_basic'], onTokenRequest: async ({ @@ -364,6 +366,7 @@ export class ClientCredentialsBasicScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/discovery-metadata.ts b/src/scenarios/client/auth/discovery-metadata.ts index 7387ead..3c53e08 100644 --- a/src/scenarios/client/auth/discovery-metadata.ts +++ b/src/scenarios/client/auth/discovery-metadata.ts @@ -6,6 +6,7 @@ * generated from them. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -94,10 +95,10 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { **OAuth metadata:** ${config.oauthMetadataLocation} `, - async start(): Promise { + async start(ctx: ScenarioContext): Promise { checks = []; - const authApp = createAuthServer(checks, authServer.getUrl, { + const authApp = createAuthServer(ctx, checks, authServer.getUrl, { metadataPath: config.oauthMetadataLocation, isOpenIdConfiguration, ...(routePrefix && { routePrefix }) @@ -131,7 +132,7 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { ? () => `${authServer.getUrl()}${routePrefix}` : authServer.getUrl; - const app = createServer(checks, server.getUrl, getAuthServerUrl, { + const app = createServer(ctx, checks, server.getUrl, getAuthServerUrl, { prmPath: config.prmLocation, includePrmInWwwAuth: config.inWwwAuth }); diff --git a/src/scenarios/client/auth/enterprise-managed-authorization.ts b/src/scenarios/client/auth/enterprise-managed-authorization.ts index ddadfbf..2310011 100644 --- a/src/scenarios/client/auth/enterprise-managed-authorization.ts +++ b/src/scenarios/client/auth/enterprise-managed-authorization.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import express, { type Request, type Response } from 'express'; @@ -69,7 +70,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { private idpPrivateKey?: CryptoKey; private grantKeypairs: Map = new Map(); - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate IDP keypair @@ -86,7 +87,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { // Start auth server with JWT bearer grant support only // Token exchange is handled by IdP - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { grantTypesSupported: ['urn:ietf:params:oauth:grant-type:jwt-bearer'], tokenEndpointAuthMethodsSupported: ['client_secret_basic'], tokenVerifier, @@ -120,6 +121,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { // Start MCP server with shared token verifier const mcpApp = createServer( + ctx, this.checks, this.mcpServer.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/helpers/createAuthServer.ts b/src/scenarios/client/auth/helpers/createAuthServer.ts index d4acb7f..d7d1fc4 100644 --- a/src/scenarios/client/auth/helpers/createAuthServer.ts +++ b/src/scenarios/client/auth/helpers/createAuthServer.ts @@ -1,6 +1,8 @@ import express, { Request, Response } from 'express'; import { createHash } from 'crypto'; import type { ConformanceCheck } from '../../../../types'; +import type { ScenarioContext } from '../../../../mock-server'; +import { isStatefulVersion } from '../../../../connection/select'; import { createRequestLogger } from '../../../request-logger'; import { SpecReferences } from '../spec-references'; import { MockTokenVerifier } from './mockTokenVerifier'; @@ -86,6 +88,7 @@ export interface AuthServerOptions { } export function createAuthServer( + ctx: ScenarioContext, checks: ConformanceCheck[], getAuthBaseUrl: () => string, options: AuthServerOptions = {} @@ -423,21 +426,27 @@ export function createAuthServer( // SEP-837: clients MUST specify an appropriate application_type during DCR. // The harness can't know the client's real class (native vs web), so this // checks presence + that the value is one of the two OIDC-defined values. - const appType = req.body.application_type; - const validAppType = appType === 'native' || appType === 'web'; - checks.push({ - id: 'sep-837-application-type-present', - name: 'DCR application_type specified', - description: validAppType - ? `Client specified application_type "${appType}" during Dynamic Client Registration` - : appType === undefined - ? 'Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); field was omitted' - : `Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); got "${appType}", expected "native" or "web"`, - status: validAppType ? 'SUCCESS' : 'FAILURE', - timestamp: new Date().toISOString(), - specReferences: [SpecReferences.MCP_DCR], - details: { application_type: appType ?? '(omitted)' } - }); + // SEP-837 first appears in the draft spec (the same revision that + // introduces the stateless lifecycle), so the check only exists for runs + // targeting a version that includes it; at dated versions it is not + // emitted at all. + if (!isStatefulVersion(ctx.specVersion)) { + const appType = req.body.application_type; + const validAppType = appType === 'native' || appType === 'web'; + checks.push({ + id: 'sep-837-application-type-present', + name: 'DCR application_type specified', + description: validAppType + ? `Client specified application_type "${appType}" during Dynamic Client Registration` + : appType === undefined + ? 'Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); field was omitted' + : `Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); got "${appType}", expected "native" or "web"`, + status: validAppType ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_DCR], + details: { application_type: appType ?? '(omitted)' } + }); + } res.status(201).json({ client_id: clientId, diff --git a/src/scenarios/client/auth/helpers/createServer.ts b/src/scenarios/client/auth/helpers/createServer.ts index c836630..8cf04d9 100644 --- a/src/scenarios/client/auth/helpers/createServer.ts +++ b/src/scenarios/client/auth/helpers/createServer.ts @@ -10,6 +10,11 @@ import { import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js'; import express, { Request, Response, NextFunction } from 'express'; import type { ConformanceCheck } from '../../../../types'; +import { + validateStatelessRequest, + type ScenarioContext +} from '../../../../mock-server'; +import { isStatefulVersion } from '../../../../connection/select'; import { createRequestLogger } from '../../../request-logger'; import { MockTokenVerifier } from './mockTokenVerifier'; import { SpecReferences } from '../spec-references'; @@ -27,6 +32,7 @@ export interface ServerOptions { } export function createServer( + ctx: ScenarioContext, checks: ConformanceCheck[], getBaseUrl: () => string, getAuthServerUrl: () => string, @@ -157,6 +163,9 @@ export function createServer( authMiddleware(req, res, async (err?: any) => { if (err) return next(err); + if (!isStatefulVersion(ctx.specVersion)) { + return handleStateless(req, res); + } const server = createMcpServer(); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined @@ -164,12 +173,13 @@ export function createServer( try { await server.connect(transport); - - await transport.handleRequest(req, res, req.body); + // Register cleanup before handing the request to the transport so the + // pair is torn down even when handleRequest throws. res.on('close', () => { transport.close(); server.close(); }); + await transport.handleRequest(req, res, req.body); } catch (error) { console.error('Error handling MCP request:', error); if (!res.headersSent) { @@ -186,5 +196,38 @@ export function createServer( }); }); + // Stateless lifecycle for the /mcp route: shared SEP-2575 validation + + // server/discover from mock-server/stateless, then the same tools handlers + // as createMcpServer. Bearer-auth middleware and PRM route above are + // version-independent. + function handleStateless(req: Request, res: Response) { + const v = validateStatelessRequest(req, { tools: {} }, [ctx.specVersion]); + if (v.kind !== 'route') { + return res.status(v.status).json(v.body); + } + const { id, method } = v; + if (method === 'tools/list') { + return res.json({ + jsonrpc: '2.0', + id, + result: { + tools: [{ name: 'test-tool', inputSchema: { type: 'object' } }] + } + }); + } + if (method === 'tools/call') { + return res.json({ + jsonrpc: '2.0', + id, + result: { content: [{ type: 'text', text: 'test' }] } + }); + } + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: 'Method not found' } + }); + } + return app; } diff --git a/src/scenarios/client/auth/index.test.ts b/src/scenarios/client/auth/index.test.ts index 6089333..3d2d6ff 100644 --- a/src/scenarios/client/auth/index.test.ts +++ b/src/scenarios/client/auth/index.test.ts @@ -21,6 +21,7 @@ import { runClient as issNormalizeClient } from '../../../../examples/clients/ty import { runClient as echoScopeClient } from '../../../../examples/clients/typescript/auth-test-echo-scope'; import { getHandler } from '../../../../examples/clients/typescript/everything-client'; import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; beforeAll(() => { setLogLevel('error'); @@ -176,13 +177,25 @@ describe('Negative tests', () => { ); }); - test('client omits application_type during DCR (SEP-837)', async () => { + test('client omits application_type during DCR (SEP-837) at draft', async () => { + // SEP-837 is a draft-spec requirement, so the check is only enforced when + // the run targets the draft version. const runner = new InlineClientRunner(noAppTypeClient); await runClientAgainstScenario(runner, 'auth/metadata-default', { + specVersion: DRAFT_PROTOCOL_VERSION, expectedFailureSlugs: ['sep-837-application-type-present'] }); }); + test('client omits application_type during DCR (SEP-837) at 2025-11-25', async () => { + // At dated spec versions that predate SEP-837 the check is not emitted at + // all, so a client omitting application_type passes cleanly. + const runner = new InlineClientRunner(noAppTypeClient); + await runClientAgainstScenario(runner, 'auth/metadata-default', { + specVersion: '2025-11-25' + }); + }); + test('client does not use PKCE', async () => { const runner = new InlineClientRunner(noPkceClient); await runClientAgainstScenario(runner, 'auth/metadata-default', { diff --git a/src/scenarios/client/auth/issuer-parameter.ts b/src/scenarios/client/auth/issuer-parameter.ts index ecb2bc2..b55e974 100644 --- a/src/scenarios/client/auth/issuer-parameter.ts +++ b/src/scenarios/client/auth/issuer-parameter.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -30,13 +31,13 @@ export class IssParameterSupportedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: true, issInRedirect: 'correct', @@ -48,6 +49,7 @@ export class IssParameterSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -99,13 +101,13 @@ export class IssParameterNotAdvertisedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: null, issInRedirect: 'omit', @@ -117,6 +119,7 @@ export class IssParameterNotAdvertisedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -170,14 +173,14 @@ export class IssParameterSupportedMissingScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: true, issInRedirect: 'omit', // advertise support but don't send iss @@ -192,6 +195,7 @@ export class IssParameterSupportedMissingScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -256,14 +260,14 @@ export class IssParameterWrongIssuerScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: true, issInRedirect: 'wrong', // send iss that doesn't match metadata issuer @@ -278,6 +282,7 @@ export class IssParameterWrongIssuerScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -343,14 +348,14 @@ export class IssParameterUnexpectedScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: null, issInRedirect: 'wrong', // send mismatched iss without advertising support @@ -365,6 +370,7 @@ export class IssParameterUnexpectedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -435,14 +441,14 @@ export class IssParameterNormalizedVariantScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, issParameterSupported: true, issInRedirect: 'normalized', // correct issuer + trailing slash @@ -457,6 +463,7 @@ export class IssParameterNormalizedVariantScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -522,13 +529,13 @@ export class MetadataIssuerMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private metadataEndpointsUsed = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.metadataEndpointsUsed = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, // The well-known URL is constructed from the AS URL advertised in PRM // (the bare origin), so the document's issuer must equal that origin. @@ -555,6 +562,7 @@ export class MetadataIssuerMismatchScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/march-spec-backcompat.ts b/src/scenarios/client/auth/march-spec-backcompat.ts index b250209..3e54388 100644 --- a/src/scenarios/client/auth/march-spec-backcompat.ts +++ b/src/scenarios/client/auth/march-spec-backcompat.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -17,17 +18,18 @@ export class Auth20250326OAuthMetadataBackcompatScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Legacy server, so we create the auth server endpoints on the // same URL as the main server (rather than separating AS / RS). - const authApp = createAuthServer(this.checks, this.server.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.server.getUrl, { // Disable logging since the main server will already have logging enabled loggingEnabled: false, // Add a prefix to auth endpoints to avoid being caught by auth fallbacks routePrefix: '/oauth' }); const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, @@ -81,10 +83,11 @@ export class Auth20250326OEndpointFallbackScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, diff --git a/src/scenarios/client/auth/offline-access.ts b/src/scenarios/client/auth/offline-access.ts index afcd593..77eccca 100644 --- a/src/scenarios/client/auth/offline-access.ts +++ b/src/scenarios/client/auth/offline-access.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -33,14 +34,14 @@ export class OfflineAccessScopeScenario implements Scenario { private grantTypesChecked = false; private capturedCimdUrl: string | undefined; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.grantTypesChecked = false; this.capturedCimdUrl = undefined; const tokenVerifier = new MockTokenVerifier(this.checks, ['mcp:basic']); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, scopesSupported: ['mcp:basic', 'offline_access'], clientIdMetadataDocumentSupported: true, @@ -101,6 +102,7 @@ export class OfflineAccessScopeScenario implements Scenario { // PRM does NOT include offline_access (per SEP-2207 server guidance: // servers SHOULD NOT include offline_access in PRM scopes_supported) const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -235,7 +237,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, [ @@ -243,7 +245,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { 'mcp:read' ]); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, scopesSupported: ['mcp:basic', 'mcp:read'], onAuthorizationRequest: (data) => { @@ -268,6 +270,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/pre-registration.ts b/src/scenarios/client/auth/pre-registration.ts index b482e4f..11f8269 100644 --- a/src/scenarios/client/auth/pre-registration.ts +++ b/src/scenarios/client/auth/pre-registration.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; @@ -27,11 +28,11 @@ export class PreRegistrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, disableDynamicRegistration: true, tokenEndpointAuthMethodsSupported: ['client_secret_basic'], @@ -105,6 +106,7 @@ export class PreRegistrationScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts index 9c15d4d..2e8b02f 100644 --- a/src/scenarios/client/auth/resource-mismatch.ts +++ b/src/scenarios/client/auth/resource-mismatch.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -37,13 +38,13 @@ export class ResourceMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private authorizationRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationRequestMade = false; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, tokenEndpointAuthMethodsSupported: ['none'], onAuthorizationRequest: () => { @@ -60,6 +61,7 @@ export class ResourceMismatchScenario implements Scenario { // Create server that returns a mismatched resource in PRM const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index c24d51d..2134678 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -22,13 +23,13 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const expectedScope = 'mcp:basic'; const tokenVerifier = new MockTokenVerifier(this.checks, [expectedScope]); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, onAuthorizationRequest: (data) => { // Check if client used the scope from WWW-Authenticate header @@ -53,6 +54,7 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -108,13 +110,13 @@ export class ScopeFromScopesSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const scopesSupported = ['mcp:basic', 'mcp:read', 'mcp:write']; const tokenVerifier = new MockTokenVerifier(this.checks, scopesSupported); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, onAuthorizationRequest: (data) => { // Check if client requested all scopes from scopes_supported @@ -148,6 +150,7 @@ export class ScopeFromScopesSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -204,12 +207,12 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, onAuthorizationRequest: (data) => { // Check if client omitted scope parameter @@ -232,6 +235,7 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -291,7 +295,7 @@ export class ScopeStepUpAuthScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const initialScope = 'mcp:basic'; @@ -303,7 +307,7 @@ export class ScopeStepUpAuthScenario implements Scenario { const tokenVerifier = new MockTokenVerifier(this.checks, escalatedScopes); let authRequestCount = 0; - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, onAuthorizationRequest: (data) => { authRequestCount++; @@ -437,6 +441,7 @@ export class ScopeStepUpAuthScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -528,14 +533,14 @@ export class ScopeRetryLimitScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const requiredScope = 'mcp:admin'; const tokenVerifier = new MockTokenVerifier(this.checks, []); let authRequestCount = 0; - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, onAuthorizationRequest: (data) => { authRequestCount++; @@ -613,6 +618,7 @@ export class ScopeRetryLimitScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/test_helpers/testClient.ts b/src/scenarios/client/auth/test_helpers/testClient.ts index 0dbc8d8..8790a7b 100644 --- a/src/scenarios/client/auth/test_helpers/testClient.ts +++ b/src/scenarios/client/auth/test_helpers/testClient.ts @@ -1,4 +1,6 @@ import { getScenario } from '../../../index'; +import { testScenarioContext } from '../../../../mock-server/testing'; +import type { SpecVersion } from '../../../../types'; import { spawn } from 'child_process'; const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete @@ -86,6 +88,12 @@ export class InlineClientRunner implements ClientRunner { export interface RunClientOptions { expectedFailureSlugs?: string[]; allowClientError?: boolean; + /** + * Spec version to run the scenario at. Defaults to the latest dated spec + * version (see {@link testScenarioContext}). Pass the draft version to + * exercise checks gated to draft-only requirements. + */ + specVersion?: SpecVersion; } export async function runClientAgainstScenario( @@ -93,7 +101,11 @@ export async function runClientAgainstScenario( scenarioName: string, options: RunClientOptions = {} ): Promise { - const { expectedFailureSlugs = [], allowClientError = false } = options; + const { + expectedFailureSlugs = [], + allowClientError = false, + specVersion + } = options; const runner = clientRunner; @@ -103,13 +115,15 @@ export async function runClientAgainstScenario( } // Start the scenario server - const urls = await scenario.start(); + const ctx = testScenarioContext(specVersion); + const urls = await scenario.start(ctx); const serverUrl = urls.serverUrl; try { // Set environment variables for inline clients // These mirror what src/runner/client.ts does for spawned processes process.env.MCP_CONFORMANCE_SCENARIO = scenarioName; + process.env.MCP_CONFORMANCE_PROTOCOL_VERSION = ctx.specVersion; if (urls.context) { process.env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ name: scenarioName, @@ -179,6 +193,7 @@ export async function runClientAgainstScenario( // Clean up environment variables delete process.env.MCP_CONFORMANCE_SCENARIO; delete process.env.MCP_CONFORMANCE_CONTEXT; + delete process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; // Stop the scenario server await scenario.stop(); diff --git a/src/scenarios/client/auth/token-endpoint-auth.ts b/src/scenarios/client/auth/token-endpoint-auth.ts index 79d6296..53643fd 100644 --- a/src/scenarios/client/auth/token-endpoint-auth.ts +++ b/src/scenarios/client/auth/token-endpoint-auth.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -62,13 +63,13 @@ class TokenEndpointAuthScenario implements Scenario { this.description = `Tests that client uses ${AUTH_METHOD_NAMES[expectedAuthMethod]} when server only supports ${expectedAuthMethod}`; } - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationResource = undefined; this.tokenResource = undefined; const tokenVerifier = new MockTokenVerifier(this.checks, []); - const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + const authApp = createAuthServer(ctx, this.checks, this.authServer.getUrl, { tokenVerifier, tokenEndpointAuthMethodsSupported: [this.expectedAuthMethod], onAuthorizationRequest: ({ resource }) => { @@ -137,6 +138,7 @@ class TokenEndpointAuthScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/elicitation-defaults.ts b/src/scenarios/client/elicitation-defaults.ts index c78f349..d417029 100644 --- a/src/scenarios/client/elicitation-defaults.ts +++ b/src/scenarios/client/elicitation-defaults.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-1034: Elicitation defaults test * Validates that clients properly apply default values for omitted fields @@ -13,7 +14,7 @@ import { } from '@modelcontextprotocol/sdk/types.js'; import type { Scenario, ConformanceCheck } from '../../types'; import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; +import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; import { createRequestLogger } from '../request-logger'; import { randomUUID } from 'crypto'; @@ -474,7 +475,10 @@ function createServer(checks: ConformanceCheck[]): { export class ElicitationClientDefaultsScenario implements Scenario { name = 'elicitation-sep1034-client-defaults'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests client applies default values for omitted elicitation fields (SEP-1034)'; private app: express.Application | null = null; @@ -482,7 +486,7 @@ export class ElicitationClientDefaultsScenario implements Scenario { private checks: ConformanceCheck[] = []; private cleanup: (() => void) | null = null; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; const { app, cleanup } = createServer(this.checks); this.app = app; diff --git a/src/scenarios/client/http-base.ts b/src/scenarios/client/http-base.ts index 06c2afa..9f6187d 100644 --- a/src/scenarios/client/http-base.ts +++ b/src/scenarios/client/http-base.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * Shared HTTP test-server scaffold for client-under-test SEP-2243 scenarios. * @@ -27,7 +28,7 @@ export abstract class BaseHttpScenario implements Scenario { protected port: number = 0; protected sessionId: string = `session-${Date.now()}`; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/http-custom-headers.test.ts b/src/scenarios/client/http-custom-headers.test.ts index 88714ee..8a68f22 100644 --- a/src/scenarios/client/http-custom-headers.test.ts +++ b/src/scenarios/client/http-custom-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpCustomHeadersScenario, @@ -43,7 +44,7 @@ function statusesFor( describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('emits exactly the declared requirement IDs as FAILURE when the client never connects', async () => { const scenario = new HttpCustomHeadersScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); expect(idsOf(checks)).toEqual(new Set(CUSTOM_HEADERS_DECLARED_CHECK_IDS)); @@ -57,7 +58,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('maps each parameter kind to its requirement ID on a conforming tool call', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { const nonAscii = 'Hello, 世界'; const nonAsciiB64 = Buffer.from(nonAscii, 'utf-8').toString('base64'); @@ -123,7 +124,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('FAILs client-mirrors-designated-params when an annotated header is missing', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post( serverUrl, @@ -154,7 +155,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('emits every x-mcp-header constraint ID, SUCCESS when only valid_tool is called', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { @@ -176,7 +177,7 @@ describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('FAILs the violated constraint ID when the client calls an invalid tool', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { diff --git a/src/scenarios/client/http-custom-headers.ts b/src/scenarios/client/http-custom-headers.ts index e91cf2c..25e3661 100644 --- a/src/scenarios/client/http-custom-headers.ts +++ b/src/scenarios/client/http-custom-headers.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * HTTP Custom Headers conformance test scenario for MCP clients (SEP-2243) * @@ -186,8 +187,8 @@ export class HttpCustomHeadersScenario extends BaseHttpScenario { private toolCallReceived: boolean = false; private nullToolCallReceived: boolean = false; - async start(): Promise { - const urls = await super.start(); + async start(_ctx: ScenarioContext): Promise { + const urls = await super.start(_ctx); // Pass test values via context for encoding edge cases. // The conformance client should use these values when calling test_custom_headers. urls.context = { diff --git a/src/scenarios/client/http-standard-headers.test.ts b/src/scenarios/client/http-standard-headers.test.ts index 8c5ff6a..3478e06 100644 --- a/src/scenarios/client/http-standard-headers.test.ts +++ b/src/scenarios/client/http-standard-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpStandardHeadersScenario } from './http-standard-headers'; @@ -38,7 +39,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('FAILs the initialize Mcp-Method emission when Mcp-Method is missing', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, {}); // no Mcp-Method header const checks = scenario.getChecks(); @@ -53,7 +54,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('SUCCEEDs the initialize Mcp-Method emission when Mcp-Method matches', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const checks = scenario.getChecks(); @@ -68,7 +69,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('getChecks() is idempotent', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const first = scenario.getChecks(); diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index b5e2aef..be8d708 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,23 +1,28 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, ScenarioUrls, ConformanceCheck, LATEST_SPEC_VERSION, - NEGOTIABLE_PROTOCOL_VERSIONS + NEGOTIABLE_PROTOCOL_VERSIONS, + DRAFT_PROTOCOL_VERSION } from '../../types'; import { clientChecks } from '../../checks/index'; export class InitializeScenario implements Scenario { name = 'initialize'; - readonly source = { introducedIn: '2025-06-18' } as const; + readonly source = { + introducedIn: '2025-06-18', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests MCP client initialization handshake'; private server: http.Server | null = null; private checks: ConformanceCheck[] = []; private port: number = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/json-schema-ref-deref.test.ts b/src/scenarios/client/json-schema-ref-deref.test.ts index b11d2d2..a309193 100644 --- a/src/scenarios/client/json-schema-ref-deref.test.ts +++ b/src/scenarios/client/json-schema-ref-deref.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; @@ -92,7 +93,7 @@ describe('json-schema-ref-no-deref (SEP-2106)', () => { test('client that never lists tools fails: requirement cannot be evaluated', async () => { const scenario = new JsonSchemaRefDerefScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const check = checks.find( diff --git a/src/scenarios/client/json-schema-ref-deref.ts b/src/scenarios/client/json-schema-ref-deref.ts index 91ccc9e..a7b315e 100644 --- a/src/scenarios/client/json-schema-ref-deref.ts +++ b/src/scenarios/client/json-schema-ref-deref.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; @@ -82,7 +83,7 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at private canaryRequests: Array<{ method: string; userAgent?: string }> = []; private toolsListed = false; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.canaryRequests = []; this.toolsListed = false; diff --git a/src/scenarios/client/mrtr-client.ts b/src/scenarios/client/mrtr-client.ts index 431fafe..4521ca0 100644 --- a/src/scenarios/client/mrtr-client.ts +++ b/src/scenarios/client/mrtr-client.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-2322: MRTR Client Conformance Tests * @@ -440,7 +441,7 @@ export class MRTRClientScenario implements Scenario { private httpServer: ReturnType | null = null; private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; this.app = createMRTRServer(this.checks); this.httpServer = this.app.listen(0); diff --git a/src/scenarios/client/request-metadata.test.ts b/src/scenarios/client/request-metadata.test.ts index b2f9b63..d1ef675 100644 --- a/src/scenarios/client/request-metadata.test.ts +++ b/src/scenarios/client/request-metadata.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { runClientAgainstScenario, @@ -104,10 +105,9 @@ async function incompatibleVersionClient(serverUrl: string) { if (response.status === 400) { const body = await response.json(); - if (body.error?.code === -32004 || body.error?.code === -32001) { + if (body.error?.code === -32004) { return body; // Abort cleanly } - return body; } return response.json(); } @@ -207,7 +207,7 @@ describe('request-metadata client scenario — client never connects', () => { throw new Error('Scenario not found'); } - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const byId = new Map(checks.map((c) => [c.id, c])); diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts index 9d9fd69..38576b5 100644 --- a/src/scenarios/client/request-metadata.ts +++ b/src/scenarios/client/request-metadata.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, @@ -46,7 +47,7 @@ export class RequestMetadataScenario implements Scenario { private hasSimulatedRejection = false; private requestsObserved = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.hasSimulatedRejection = false; this.checks = []; this.requestsObserved = 0; @@ -281,7 +282,8 @@ export class RequestMetadataScenario implements Scenario { code: -32004, message: 'Unsupported protocol version', data: { - supported: [DRAFT_PROTOCOL_VERSION] + supported: [DRAFT_PROTOCOL_VERSION], + requested: String(headerVersion ?? metaVersion ?? '') } } }) diff --git a/src/scenarios/client/sse-retry.ts b/src/scenarios/client/sse-retry.ts index b90bf40..e89b23e 100644 --- a/src/scenarios/client/sse-retry.ts +++ b/src/scenarios/client/sse-retry.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SSE Retry conformance test scenarios for MCP clients (SEP-1699) * @@ -8,11 +9,19 @@ */ import http from 'http'; -import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; +import { + Scenario, + ScenarioUrls, + ConformanceCheck, + DRAFT_PROTOCOL_VERSION +} from '../../types.js'; export class SSERetryScenario implements Scenario { name = 'sse-retry'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests that client respects SSE retry field timing and reconnects properly (SEP-1699)'; @@ -38,7 +47,7 @@ export class SSERetryScenario implements Scenario { private readonly LATE_TOLERANCE = 200; // Allow 200ms late for network/event loop private readonly VERY_LATE_MULTIPLIER = 2; // If >2x retry value, client is likely ignoring it - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/tools_call.test.ts b/src/scenarios/client/tools_call.test.ts new file mode 100644 index 0000000..b9fc42d --- /dev/null +++ b/src/scenarios/client/tools_call.test.ts @@ -0,0 +1,48 @@ +import { testScenarioContext } from '../../mock-server/testing'; +import { describe, it, expect } from 'vitest'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { ToolsCallScenario } from './tools_call'; + +describe('tools_call scenario', () => { + it('emits a single FAILURE check when the tool was never called', async () => { + const scenario = new ToolsCallScenario(); + await scenario.start(testScenarioContext()); + try { + const checks = scenario.getChecks(); + expect(checks).toHaveLength(1); + expect(checks[0]).toMatchObject({ + id: 'tool-add-numbers', + status: 'FAILURE' + }); + } finally { + await scenario.stop(); + } + }); + + it('emits SUCCESS after a valid tools/call and getChecks() is idempotent', async () => { + const scenario = new ToolsCallScenario(); + const { serverUrl } = await scenario.start(testScenarioContext()); + try { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + await client.connect(transport); + await client.callTool({ name: 'add_numbers', arguments: { a: 2, b: 3 } }); + await transport.close(); + + const first = scenario.getChecks(); + expect(first).toHaveLength(1); + expect(first[0].status).toBe('SUCCESS'); + + // Repeated calls must not accumulate duplicate checks. + const second = scenario.getChecks(); + expect(second).toHaveLength(1); + expect(second[0].status).toBe('SUCCESS'); + } finally { + await scenario.stop(); + } + }); +}); diff --git a/src/scenarios/client/tools_call.ts b/src/scenarios/client/tools_call.ts index 59470f3..b3dcb74 100644 --- a/src/scenarios/client/tools_call.ts +++ b/src/scenarios/client/tools_call.ts @@ -1,164 +1,83 @@ -import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; -import { - CallToolRequestSchema, - ListToolsRequestSchema -} from '@modelcontextprotocol/sdk/types.js'; -import type { Scenario, ConformanceCheck } from '../../types'; -import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; -import { createRequestLogger } from '../request-logger'; +import type { ScenarioContext, MockServer } from '../../mock-server'; +import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../types'; +import type { CallToolRequest } from '../../spec-types/2025-06-18'; -function createMcpServer(checks: ConformanceCheck[]): Server { - const server = new Server( - { - name: 'add-numbers-server', - version: '1.0.0' - }, - { - capabilities: { - tools: {} - } - } - ); - - server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [ - { - name: 'add_numbers', - description: 'Add two numbers together', - inputSchema: { - type: 'object', - properties: { - a: { - type: 'number', - description: 'First number' - }, - b: { - type: 'number', - description: 'Second number' - } - }, - required: ['a', 'b'] - } - } - ] - }; - }); - - server.setRequestHandler(CallToolRequestSchema, async (request) => { - if (request.params.name === 'add_numbers') { - const { a, b } = request.params.arguments as { a: number; b: number }; - const result = a + b; - - checks.push({ - id: 'tool-add-numbers', - name: 'ToolAddNumbers', - description: 'Validates that the add_numbers tool works correctly', - status: 'SUCCESS', - timestamp: new Date().toISOString(), - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' - } - ], - details: { - a, - b, - result - } - }); - - return { - content: [ - { - type: 'text', - text: `The sum of ${a} and ${b} is ${result}` - } - ] - }; - } - - throw new Error(`Unknown tool: ${request.params.name}`); - }); - - return server; -} - -function createServerApp(checks: ConformanceCheck[]): express.Application { - const app = express(); - app.use(express.json()); - - app.use( - createRequestLogger(checks, { - incomingId: 'incoming-request', - outgoingId: 'outgoing-response', - mcpRoute: '/mcp' - }) - ); - - app.post('/mcp', async (req: Request, res: Response) => { - // Stateless: create a fresh server and transport per request - const server = createMcpServer(checks); - const transport = new StreamableHTTPServerTransport({ - sessionIdGenerator: undefined - }); - await server.connect(transport); - - await transport.handleRequest(req, res, req.body); - }); - - return app; -} +const SPEC_REF = { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' +}; export class ToolsCallScenario implements Scenario { name = 'tools_call'; readonly source = { introducedIn: '2025-06-18' } as const; description = 'Tests calling tools with various parameter types'; - private app: express.Application | null = null; - private httpServer: any = null; - private checks: ConformanceCheck[] = []; + private srv: MockServer | null = null; - async start(): Promise { - this.checks = []; - this.app = createServerApp(this.checks); - this.httpServer = this.app.listen(0); - const port = this.httpServer.address().port; - return { serverUrl: `http://localhost:${port}/mcp` }; + async start(ctx: ScenarioContext): Promise { + this.srv = await ctx.createServer({ + 'tools/list': () => ({ + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers together', + inputSchema: { + type: 'object', + properties: { + a: { type: 'number', description: 'First number' }, + b: { type: 'number', description: 'Second number' } + }, + required: ['a', 'b'] + } + } + ] + }), + 'tools/call': (params) => { + const p = params as CallToolRequest['params']; + if (p.name !== 'add_numbers') { + throw new Error(`Unknown tool: ${p.name}`); + } + const { a, b } = p.arguments as { a: number; b: number }; + return { + content: [ + { type: 'text', text: `The sum of ${a} and ${b} is ${a + b}` } + ] + }; + } + }); + return { serverUrl: this.srv.url }; } async stop() { - if (this.httpServer) { - await new Promise((resolve) => this.httpServer.close(resolve)); - this.httpServer = null; - } - this.app = null; + await this.srv?.close(); + this.srv = null; } getChecks(): ConformanceCheck[] { - const expectedSlugs = ['tool-add-numbers']; - // add a failure if not in there already - for (const slug of expectedSlugs) { - if (!this.checks.find((c) => c.id === slug)) { - // TODO: this is duplicated from above, refactor - this.checks.push({ - id: slug, - name: `ToolAddNumbers`, - description: `Validates that the add_numbers tool works correctly`, - status: 'FAILURE', - timestamp: new Date().toISOString(), - details: { message: 'Tool was not called by client' }, - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' + // Built fresh on every call so getChecks() is idempotent — the runner may + // call it more than once and we must not accumulate duplicates. + const call = this.srv?.recorded.find((r) => r.method === 'tools/call'); + const args = (call?.params as CallToolRequest['params'] | undefined) + ?.arguments as { a?: unknown; b?: unknown } | undefined; + const ok = + call !== undefined && + typeof args?.a === 'number' && + typeof args?.b === 'number'; + return [ + { + id: 'tool-add-numbers', + name: 'ToolAddNumbers', + description: 'Validates that the add_numbers tool works correctly', + status: ok ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REF], + details: ok + ? { + a: args!.a, + b: args!.b, + result: (args!.a as number) + (args!.b as number) } - ] - }); + : { message: 'Tool was not called by client' } } - } - return this.checks; + ]; } } diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index 93b4662..dc84b0e 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -6,6 +6,7 @@ import { listDraftClientScenarios, listActiveClientScenarios, listExtensionScenarios, + getScenario, getScenarioSpecVersions, resolveSpecVersion, ALL_SPEC_VERSIONS, @@ -57,11 +58,17 @@ describe('specVersions helpers', () => { expect(current.length).toBeGreaterThan(overlap.length); }); - it('the draft spec version is a superset of the latest dated release', () => { + it('every scenario in latest but not in draft is explicitly removedIn: DRAFT', () => { const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION)); for (const name of latest) { - expect(draft.has(name)).toBe(true); + if (!draft.has(name)) { + const s = getScenario(name)!; + expect( + 'removedIn' in s.source && s.source.removedIn, + `"${name}" is in ${LATEST_SPEC_VERSION} but not in draft without removedIn` + ).toBe(DRAFT_PROTOCOL_VERSION); + } } for (const name of listDraftScenarios()) { expect(draft.has(name)).toBe(true); diff --git a/src/sdk-runner/config.ts b/src/sdk-runner/config.ts index 0945b99..1f8a9e5 100644 --- a/src/sdk-runner/config.ts +++ b/src/sdk-runner/config.ts @@ -19,7 +19,11 @@ export const SdkConfigSchema = z.object({ readyTimeoutMs: z.number().int().positive().optional() }) .optional(), - expectedFailures: z.string().optional() + expectedFailures: z.string().optional(), + // Spec version this SDK targets, used as the default --spec-version when + // the flag isn't given (e.g. a v1 SDK pinned to the latest dated spec). + // An explicit --spec-version on the sdk command always wins. + specVersion: z.string().optional() }); export type SdkConfig = z.infer; diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts index 932c51e..0797163 100644 --- a/src/sdk-runner/index.ts +++ b/src/sdk-runner/index.ts @@ -119,6 +119,7 @@ function passThrough(options: { timeout?: string; verbose?: boolean; output?: string; + specVersion?: string; }): string[] { const args: string[] = []; if (options.scenario) args.push('--scenario', options.scenario); @@ -126,6 +127,7 @@ function passThrough(options: { if (options.timeout) args.push('--timeout', options.timeout); if (options.verbose) args.push('--verbose'); if (options.output) args.push('-o', options.output); + if (options.specVersion) args.push('--spec-version', options.specVersion); return args; } @@ -166,6 +168,10 @@ export function createSdkCommand(): Command { ) .option('--timeout ', 'Per-scenario client timeout (passed through)') .option('-o, --output ', 'Output directory (passed through)') + .option( + '--spec-version ', + 'Spec version to target (passed through; defaults to the SDK config)' + ) .option('--verbose', 'Verbose output (passed through)') .action(async (sdkArg: string | undefined, options) => { try { @@ -217,6 +223,9 @@ export function createSdkCommand(): Command { const output = options.output ? path.resolve(options.output) : undefined; + // Explicit flag wins over the per-SDK default. + const specVersion: string | undefined = + options.specVersion ?? builtinConfig.specVersion; if (buildCmd && !options.skipBuild) { console.error(`[sdk] Building: ${buildCmd}`); @@ -244,7 +253,8 @@ export function createSdkCommand(): Command { suite: options.suite ?? 'all', timeout: options.timeout, verbose: options.verbose, - output + output, + specVersion }) ]; if (expectedFailures) @@ -265,7 +275,8 @@ export function createSdkCommand(): Command { scenario: options.scenario, suite: options.suite, verbose: options.verbose, - output + output, + specVersion }) ]; if (expectedFailures) diff --git a/src/sdk-runner/known-sdks.ts b/src/sdk-runner/known-sdks.ts index b6550df..528bfd8 100644 --- a/src/sdk-runner/known-sdks.ts +++ b/src/sdk-runner/known-sdks.ts @@ -22,10 +22,12 @@ export const KNOWN_SDKS: Record = { }, // v1.x — the published npm line. Same fixtures as v2; differs only in the // build (npm, not pnpm) and the baseline filename. Clones the typescript-sdk - // repo, defaulting to the `v1.x` branch. + // repo, defaulting to the `v1.x` branch. Targets the latest dated spec, so + // draft-only scenarios and checks are excluded by default. 'typescript-sdk-v1': { repo: 'typescript-sdk', defaultRef: 'v1.x', + specVersion: '2025-11-25', build: 'npm ci && npm run build', client: { command: 'npx tsx test/conformance/src/everythingClient.ts' diff --git a/src/sdk-runner/sdk-runner.test.ts b/src/sdk-runner/sdk-runner.test.ts index c58e993..fbfa7fb 100644 --- a/src/sdk-runner/sdk-runner.test.ts +++ b/src/sdk-runner/sdk-runner.test.ts @@ -44,6 +44,14 @@ describe('SdkConfigSchema', () => { expect(cfg.server).toBeUndefined(); }); + it('accepts an optional specVersion default', () => { + const cfg = SdkConfigSchema.parse({ + client: { command: 'tsx fixture.ts' }, + specVersion: '2025-11-25' + }); + expect(cfg.specVersion).toBe('2025-11-25'); + }); + it('rejects server config without a url', () => { expect(() => SdkConfigSchema.parse({ server: { command: 'tsx server.ts' } }) @@ -75,8 +83,15 @@ describe('lookupBuiltinConfig', () => { expect(v1?.defaultRef).toBe('v1.x'); }); - it('bare typescript-sdk (v2) has no defaultRef', () => { + it('typescript-sdk-v1 defaults to the latest dated spec version', () => { + expect(lookupBuiltinConfig('typescript-sdk-v1')?.specVersion).toBe( + '2025-11-25' + ); + }); + + it('bare typescript-sdk (v2) has no defaultRef or specVersion default', () => { expect(lookupBuiltinConfig('typescript-sdk')?.defaultRef).toBeUndefined(); + expect(lookupBuiltinConfig('typescript-sdk')?.specVersion).toBeUndefined(); }); it('every built-in entry validates against SdkConfigSchema', () => { diff --git a/src/types.ts b/src/types.ts index 6c6376e..2e9dd22 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,5 @@ import type { RunContext } from './connection'; +import type { ScenarioContext } from './mock-server'; export type CheckStatus = | 'SUCCESS' @@ -102,7 +103,7 @@ export interface Scenario { * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). */ allowClientError?: boolean; - start(): Promise; + start(ctx: ScenarioContext): Promise; stop(): Promise; getChecks(): ConformanceCheck[]; }