diff --git a/src/scenarios/client/request-metadata.test.ts b/src/scenarios/client/request-metadata.test.ts index 41ba4c4..6938464 100644 --- a/src/scenarios/client/request-metadata.test.ts +++ b/src/scenarios/client/request-metadata.test.ts @@ -5,6 +5,7 @@ import { } from './auth/test_helpers/testClient'; import { getHandler } from '../../../examples/clients/typescript/everything-client'; import { getScenario } from '../index'; +import { DECLARED_CHECK_IDS } from './request-metadata'; // A bad client that does not send _meta async function badClient(serverUrl: string) { @@ -182,6 +183,65 @@ describe('request-metadata client scenario — positive test', () => { checks.find((c) => c.id === 'sep-2575-client-retry-supported-version') ?.status ).toBe('SUCCESS'); + + // Declared ↔ emitted completeness, both directions: every declared check + // is genuinely observed, and every emitted check is declared. + for (const check of checks) { + expect(DECLARED_CHECK_IDS).toContain(check.id); + expect(check.errorMessage ?? '').not.toContain('not observed'); + } + expect(new Set(checks.map((c) => c.id))).toEqual( + new Set(DECLARED_CHECK_IDS) + ); + }); +}); + +describe('request-metadata client scenario — client never connects', () => { + // A client with no handler for this scenario exits without sending a + // request. Every declared check must still be emitted (as FAILURE) instead + // of reporting "0 passed, 0 failed". + test('emits every declared check as FAILURE when no request is received', async () => { + const scenario = getScenario('request-metadata'); + if (!scenario) { + throw new Error('Scenario not found'); + } + + await scenario.start(); + try { + const checks = scenario.getChecks(); + const byId = new Map(checks.map((c) => [c.id, c])); + + for (const id of DECLARED_CHECK_IDS) { + const check = byId.get(id); + expect(check, `expected check ${id} to be emitted`).toBeDefined(); + expect(check?.status, `expected ${id} to be FAILURE`).toBe('FAILURE'); + expect(check?.errorMessage).toContain('never sent a request'); + } + expect(checks).toHaveLength(DECLARED_CHECK_IDS.length); + } finally { + await scenario.stop(); + } + }); + + test('does not overwrite checks recorded from a real request', async () => { + const runner = new InlineClientRunner(badClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: [ + 'sep-2575-client-populates-meta', + 'sep-2575-http-client-sends-version-header' + ] + }); + + const scenario = getScenario('request-metadata'); + const checks = scenario!.getChecks(); + // badClient connects, so its failures must carry the observed request's + // details, not the "never sent a request" backfill. + const populatesMeta = checks.find( + (c) => c.id === 'sep-2575-client-populates-meta' + ); + expect(populatesMeta?.errorMessage ?? '').not.toContain( + 'never sent a request' + ); }); }); diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts index af1fde0..47d4f00 100644 --- a/src/scenarios/client/request-metadata.ts +++ b/src/scenarios/client/request-metadata.ts @@ -6,6 +6,21 @@ import { DRAFT_PROTOCOL_VERSION } from '../../types'; +/** + * Every check ID this scenario can emit. Declared-but-unemitted checks are + * backfilled as FAILURE by getChecks(), so the emitted ID set is the same for + * every client. The positive-path test asserts this list is exact. + */ +export const DECLARED_CHECK_IDS = [ + 'sep-2575-http-client-sends-version-header', + 'sep-2575-client-populates-meta', + 'sep-2575-http-version-header-matches-meta', + 'sep-2575-client-declares-roots-capability', + 'sep-2575-client-declares-sampling-capability', + 'sep-2575-client-declares-elicitation-capability', + 'sep-2575-client-retry-supported-version' +] as const; + export class RequestMetadataScenario implements Scenario { name = 'request-metadata'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; @@ -15,10 +30,12 @@ export class RequestMetadataScenario implements Scenario { private server: http.Server | null = null; private checks: ConformanceCheck[] = []; private hasSimulatedRejection = false; + private requestsObserved = 0; async start(): Promise { this.hasSimulatedRejection = false; this.checks = []; + this.requestsObserved = 0; return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); @@ -46,6 +63,30 @@ export class RequestMetadataScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Declared but never emitted -> FAILURE. A check that is legitimately not + // applicable must be emitted as SKIPPED explicitly to avoid this. + for (const id of DECLARED_CHECK_IDS) { + if (!this.checks.some((c) => c.id === id)) { + this.checks.push({ + id, + name: 'NotObserved', + description: `Declared check ${id} was never emitted`, + status: 'FAILURE', + errorMessage: + this.requestsObserved === 0 + ? 'Check was not observed: the client never sent a request to the scenario server (no handler registered for this scenario?)' + : 'Check was not observed: no request exercised it', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/index#meta' + } + ], + details: { observed: false, requestsObserved: this.requestsObserved } + }); + } + } return this.checks; } @@ -68,6 +109,7 @@ export class RequestMetadataScenario implements Scenario { }); req.on('end', () => { const request = JSON.parse(body); + this.requestsObserved++; // Extract version and headers const meta = request.params?._meta; @@ -117,25 +159,30 @@ export class RequestMetadataScenario implements Scenario { }); // 3. "The header value MUST match the io.modelcontextprotocol/protocolVersion - // field carried in the request body's _meta." Only meaningful when both - // are present; absence is already covered by the two checks above. - if (headerVersion !== undefined && metaVersion !== undefined) { - this.addOrUpdateCheck({ - id: 'sep-2575-http-version-header-matches-meta', - name: 'ClientVersionHeaderMatchesMeta', - description: - 'MCP-Protocol-Version header matches _meta.protocolVersion', - status: headerVersion === metaVersion ? 'SUCCESS' : 'FAILURE', - timestamp: new Date().toISOString(), - specReferences: [ - { - id: 'SEP-2575', - url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' - } - ], - details: { headerVersion, metaVersion } - }); - } + // field carried in the request body's _meta." Only comparable when both + // are present; absence is already covered by the two checks above, so + // emit SKIPPED rather than falling through to the declared-check failure. + const bothVersionsPresent = + headerVersion !== undefined && metaVersion !== undefined; + this.addOrUpdateCheck({ + id: 'sep-2575-http-version-header-matches-meta', + name: 'ClientVersionHeaderMatchesMeta', + description: + 'MCP-Protocol-Version header matches _meta.protocolVersion', + status: !bothVersionsPresent + ? 'SKIPPED' + : headerVersion === metaVersion + ? 'SUCCESS' + : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' + } + ], + details: { headerVersion, metaVersion } + }); // 4. Optional client capabilities conditional verification const capabilities = meta?.['io.modelcontextprotocol/clientCapabilities']; diff --git a/src/seps/traceability.json b/src/seps/traceability.json index 8bc933d..9c4ab45 100644 --- a/src/seps/traceability.json +++ b/src/seps/traceability.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "docs": "https://github.com/modelcontextprotocol/conformance/blob/main/AGENTS.md#traceability-manifest", - "source": "typescript-sdk@4e153aef0538", + "source": "typescript-sdk@22595b96", "seps": { "837": { "yaml": "src/seps/sep-837.yaml", @@ -381,7 +381,7 @@ "requirements": [ { "check": "sep-2575-client-populates-meta", - "status": "untested", + "status": "tested", "text": "Every client request MUST include the following io.modelcontextprotocol/* fields in _meta: protocolVersion, clientInfo, clientCapabilities.", "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" }, @@ -399,20 +399,10 @@ }, { "check": "sep-2575-server-tags-subscription-id", - "status": "untested", + "status": "tested", "text": "On notifications delivered via a subscriptions/listen stream, the server MUST include io.modelcontextprotocol/subscriptionId in _meta so the client can correlate the notification with the originating subscription request.", "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" }, - { - "check": "sep-2575-server-stateless-no-prior-context", - "status": "untested", - "text": "A server MUST NOT treat connection or process identity as a proxy for conversation or session continuity. / Servers MUST NOT rely on prior requests over the same connection to establish context (e.g., capabilities, protocol version, client identity)." - }, - { - "check": "sep-2575-server-stateless-no-connection-reuse-required", - "status": "untested", - "text": "Servers MUST NOT require that a client reuse the same connection to perform related operations." - }, { "check": "sep-2575-server-unsupported-version-error", "status": "tested", @@ -421,7 +411,7 @@ }, { "check": "sep-2575-client-retry-supported-version", - "status": "untested", + "status": "tested", "text": "The client SHOULD select a mutually supported version from the supported list and retry the request, or surface an error to the user if no compatible version exists.", "url": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation" }, @@ -433,31 +423,19 @@ }, { "check": "sep-2575-http-server-no-independent-requests-on-stream", - "status": "untested", + "status": "tested", "text": "The server MUST NOT send independent JSON-RPC requests on this stream. Server-to-client interactions are embedded as input requests inside an IncompleteResult.", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#receiving-messages-1" }, - { - "check": "sep-2575-http-server-disconnect-is-cancel", - "status": "untested", - "text": "Closing the SSE response stream MUST be treated by the server as cancellation of that request.", - "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1" - }, - { - "check": "sep-2575-http-server-stops-on-cancel", - "status": "untested", - "text": "The server SHOULD stop work on the cancelled request as soon as practical and MUST NOT send any further messages for it [HTTP].", - "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1" - }, { "check": "sep-2575-http-client-sends-version-header", - "status": "untested", + "status": "tested", "text": "Every POST request to the MCP endpoint MUST include an MCP-Protocol-Version header.", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" }, { "check": "sep-2575-http-version-header-matches-meta", - "status": "untested", + "status": "tested", "text": "The header value MUST match the io.modelcontextprotocol/protocolVersion field carried in the request body _meta.", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" }, @@ -481,31 +459,31 @@ }, { "check": "sep-2575-server-honors-notification-filter", - "status": "untested", + "status": "tested", "text": "The server MUST NOT send notification types the client has not explicitly requested.", "url": "https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#opening-a-stream" }, { "check": "sep-2575-server-sends-subscription-ack", - "status": "untested", + "status": "tested", "text": "The server MUST send notifications/subscriptions/acknowledged as the first message on the stream.", "url": "https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#acknowledgment" }, { "check": "sep-2575-client-declares-elicitation-capability", - "status": "untested", + "status": "tested", "text": "Clients that support elicitation MUST declare the elicitation capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", "url": "https://modelcontextprotocol.io/specification/draft/client/elicitation#capabilities" }, { "check": "sep-2575-client-declares-roots-capability", - "status": "untested", + "status": "tested", "text": "Clients that support roots MUST declare the roots capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", "url": "https://modelcontextprotocol.io/specification/draft/client/roots#capabilities" }, { "check": "sep-2575-client-declares-sampling-capability", - "status": "untested", + "status": "tested", "text": "Clients that support sampling MUST declare the sampling capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", "url": "https://modelcontextprotocol.io/specification/draft/client/sampling#capabilities" }, @@ -517,24 +495,40 @@ }, { "check": "sep-2575-server-sends-prompts-list-changed-on-subscription", - "status": "untested", + "status": "tested", "text": "[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with promptsListChanged: true.", "url": "https://modelcontextprotocol.io/specification/draft/server/prompts#list-changed-notification" }, { "check": "sep-2575-server-sends-tools-list-changed-on-subscription", - "status": "untested", + "status": "tested", "text": "[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with toolsListChanged: true.", "url": "https://modelcontextprotocol.io/specification/draft/server/tools#list-changed-notification" }, { "check": "sep-2575-server-no-log-without-loglevel", - "status": "untested", + "status": "tested", "text": "The server MUST NOT emit notifications/message for a request that does not include [io.modelcontextprotocol/logLevel in _meta].", "url": "https://modelcontextprotocol.io/specification/draft/server/utilities/logging#per-request-log-level" } ], "excluded": [ + { + "text": "A server MUST NOT treat connection or process identity as a proxy for conversation or session continuity. / Servers MUST NOT rely on prior requests over the same connection to establish context (e.g., capabilities, protocol version, client identity).", + "reason": "internal server state, not directly wire-observable; the observable consequence (rejecting requests with incomplete _meta rather than falling back to remembered state) is covered by sep-2575-request-meta-invalid-* — see https://github.com/modelcontextprotocol/conformance/issues/296" + }, + { + "text": "Servers MUST NOT require that a client reuse the same connection to perform related operations.", + "reason": "not observable from a black-box harness; every harness request already arrives on an independent connection — see https://github.com/modelcontextprotocol/conformance/issues/296" + }, + { + "text": "Closing the SSE response stream MUST be treated by the server as cancellation of that request.", + "reason": "\"treated as cancellation\" is internal server state; once the stream is closed there is no channel left on which to observe the effect — see https://github.com/modelcontextprotocol/conformance/issues/296" + }, + { + "text": "The server SHOULD stop work on the cancelled request as soon as practical and MUST NOT send any further messages for it [HTTP].", + "reason": "\"stop work as soon as practical\" is unobservable from a black-box harness, and \"no further messages\" cannot be verified once the response stream is closed — see https://github.com/modelcontextprotocol/conformance/issues/296" + }, { "text": "State that needs to span multiple requests (e.g., long-running tasks, application-level handles) MUST be referenced by an explicit identifier the client passes on each request.", "reason": "architectural guidance, observable only via subscriptionId/task-id rows already listed" @@ -587,9 +581,9 @@ "sep-2575-request-meta-invalid-missing-protocol-version" ], "summary": { - "tested": 8, - "untested": 18, - "excluded": 9, + "tested": 22, + "untested": 0, + "excluded": 13, "untracked": 11, "unkeyed": 0 }