feat: add client conformance tests for SEP-2575 (#270)
* feat: merge latest request-metadata scenario stacked branch * test: implement optional capability conditional checks and simulated negotiation retry
This commit is contained in:
Anubhav Dhawan
GitHub
parent
fee585cf45
commit
5c1b815cd1
3 changed files with 730 additions and 0 deletions
236
src/scenarios/client/request-metadata.test.ts
Normal file
236
src/scenarios/client/request-metadata.test.ts
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
import { describe, test, expect } from 'vitest';
|
||||
import {
|
||||
runClientAgainstScenario,
|
||||
InlineClientRunner
|
||||
} from './auth/test_helpers/testClient';
|
||||
import { getHandler } from '../../../examples/clients/typescript/everything-client';
|
||||
import { getScenario } from '../index';
|
||||
|
||||
// A bad client that does not send _meta
|
||||
async function badClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: {} // Missing _meta
|
||||
})
|
||||
});
|
||||
return response.json();
|
||||
}
|
||||
|
||||
const goodMeta = {
|
||||
'io.modelcontextprotocol/protocolVersion': 'DRAFT-2026-v1',
|
||||
'io.modelcontextprotocol/clientInfo': { name: 'test', version: '1.0' },
|
||||
'io.modelcontextprotocol/clientCapabilities': {}
|
||||
};
|
||||
|
||||
// A client that misses the HTTP header
|
||||
async function missingHeaderClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' }, // Missing MCP-Protocol-Version header
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: { _meta: goodMeta }
|
||||
})
|
||||
});
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// A client whose header disagrees with _meta.protocolVersion
|
||||
async function mismatchedHeaderClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'MCP-Protocol-Version': '2025-11-25' // != _meta.protocolVersion
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: { _meta: goodMeta }
|
||||
})
|
||||
});
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// A client that fails to negotiate/retry on a 400 response
|
||||
async function nonRetryingClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'MCP-Protocol-Version': 'DRAFT-2026-v1'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: { _meta: goodMeta }
|
||||
})
|
||||
});
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// A client that has empty version intersection and terminates
|
||||
async function incompatibleVersionClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'MCP-Protocol-Version': 'UNSUPPORTED-VERSION'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: {
|
||||
_meta: {
|
||||
...goodMeta,
|
||||
'io.modelcontextprotocol/protocolVersion': 'UNSUPPORTED-VERSION'
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
if (response.status === 400) {
|
||||
const body = await response.json();
|
||||
if (body.error?.code === -32001) {
|
||||
return body; // Abort cleanly
|
||||
}
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// A client that sends invalid (non-object) capabilities
|
||||
async function malformedCapabilitiesClient(serverUrl: string) {
|
||||
const response = await fetch(serverUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'MCP-Protocol-Version': 'DRAFT-2026-v1'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: 1,
|
||||
method: 'tools/list',
|
||||
params: {
|
||||
_meta: {
|
||||
...goodMeta,
|
||||
'io.modelcontextprotocol/clientCapabilities': {
|
||||
roots: 'malformed-string',
|
||||
sampling: {},
|
||||
elicitation: true
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
return response.json();
|
||||
}
|
||||
|
||||
describe('request-metadata client scenario — positive test', () => {
|
||||
test('everything-client passes request-metadata scenario with success status for optional capabilities', async () => {
|
||||
const clientFn = getHandler('request-metadata');
|
||||
if (!clientFn) {
|
||||
throw new Error('No handler registered for scenario: request-metadata');
|
||||
}
|
||||
|
||||
const scenario = getScenario('request-metadata');
|
||||
if (!scenario) {
|
||||
throw new Error('Scenario not found');
|
||||
}
|
||||
|
||||
const runner = new InlineClientRunner(clientFn);
|
||||
await runClientAgainstScenario(runner, 'request-metadata');
|
||||
|
||||
// Extract checks directly from the scenario instance
|
||||
const checks = scenario.getChecks();
|
||||
|
||||
// 4-Line Bulk Assertion Loop
|
||||
for (const check of checks) {
|
||||
expect(check.status).not.toBe('FAILURE');
|
||||
expect(check.status).not.toBe('WARNING');
|
||||
}
|
||||
|
||||
// Strategic Targeted Optional Assertions
|
||||
expect(
|
||||
checks.find((c) => c.id === 'sep-2575-client-declares-roots-capability')
|
||||
?.status
|
||||
).toBe('SUCCESS');
|
||||
expect(
|
||||
checks.find(
|
||||
(c) => c.id === 'sep-2575-client-declares-sampling-capability'
|
||||
)?.status
|
||||
).toBe('SUCCESS');
|
||||
expect(
|
||||
checks.find(
|
||||
(c) => c.id === 'sep-2575-client-declares-elicitation-capability'
|
||||
)?.status
|
||||
).toBe('SUCCESS');
|
||||
|
||||
// Assert version negotiation retry succeeded
|
||||
expect(
|
||||
checks.find((c) => c.id === 'sep-2575-client-retry-supported-version')
|
||||
?.status
|
||||
).toBe('SUCCESS');
|
||||
});
|
||||
});
|
||||
|
||||
describe('request-metadata client scenario — negative tests', () => {
|
||||
test('client fails when omitting _meta', async () => {
|
||||
const runner = new InlineClientRunner(badClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: [
|
||||
'sep-2575-client-populates-meta',
|
||||
'sep-2575-http-client-sends-version-header'
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
test('client fails when missing version header', async () => {
|
||||
const runner = new InlineClientRunner(missingHeaderClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: ['sep-2575-http-client-sends-version-header']
|
||||
});
|
||||
});
|
||||
|
||||
test('client fails when header disagrees with _meta', async () => {
|
||||
const runner = new InlineClientRunner(mismatchedHeaderClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: ['sep-2575-http-version-header-matches-meta']
|
||||
});
|
||||
});
|
||||
|
||||
test('client fails retry check when it does not handle 400 rejection', async () => {
|
||||
const runner = new InlineClientRunner(nonRetryingClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: ['sep-2575-client-retry-supported-version']
|
||||
});
|
||||
});
|
||||
|
||||
test('client aborts cleanly without hanging when negotiation has empty version intersection', async () => {
|
||||
const runner = new InlineClientRunner(incompatibleVersionClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: ['sep-2575-client-retry-supported-version']
|
||||
});
|
||||
});
|
||||
|
||||
test('client triggers failures for malformed capabilities', async () => {
|
||||
const runner = new InlineClientRunner(malformedCapabilitiesClient);
|
||||
await runClientAgainstScenario(runner, 'request-metadata', {
|
||||
expectedFailureSlugs: [
|
||||
'sep-2575-client-declares-roots-capability',
|
||||
'sep-2575-client-declares-elicitation-capability'
|
||||
]
|
||||
});
|
||||
});
|
||||
});
|
||||
272
src/scenarios/client/request-metadata.ts
Normal file
272
src/scenarios/client/request-metadata.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
import http from 'http';
|
||||
import {
|
||||
Scenario,
|
||||
ScenarioUrls,
|
||||
ConformanceCheck,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
} from '../../types';
|
||||
|
||||
export class RequestMetadataScenario implements Scenario {
|
||||
name = 'request-metadata';
|
||||
readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const;
|
||||
description =
|
||||
'Per-request _meta and MCP-Protocol-Version header obligations (SEP-2575)';
|
||||
|
||||
private server: http.Server | null = null;
|
||||
private checks: ConformanceCheck[] = [];
|
||||
private hasSimulatedRejection = false;
|
||||
|
||||
async start(): Promise<ScenarioUrls> {
|
||||
this.hasSimulatedRejection = false;
|
||||
this.checks = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
this.server = http.createServer((req, res) => {
|
||||
this.handleRequest(req, res);
|
||||
});
|
||||
this.server.on('error', reject);
|
||||
this.server.listen(0, () => {
|
||||
const address = this.server!.address();
|
||||
if (address && typeof address === 'object') {
|
||||
resolve({ serverUrl: `http://localhost:${address.port}` });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (this.server) {
|
||||
this.server.close(() => {
|
||||
resolve();
|
||||
});
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
getChecks(): ConformanceCheck[] {
|
||||
return this.checks;
|
||||
}
|
||||
|
||||
private addOrUpdateCheck(check: ConformanceCheck): void {
|
||||
const index = this.checks.findIndex((c) => c.id === check.id);
|
||||
if (index !== -1) {
|
||||
this.checks[index] = check;
|
||||
} else {
|
||||
this.checks.push(check);
|
||||
}
|
||||
}
|
||||
|
||||
private handleRequest(
|
||||
req: http.IncomingMessage,
|
||||
res: http.ServerResponse
|
||||
): void {
|
||||
let body = '';
|
||||
req.on('data', (chunk) => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
req.on('end', () => {
|
||||
const request = JSON.parse(body);
|
||||
|
||||
// Extract version and headers
|
||||
const meta = request.params?._meta;
|
||||
const metaVersion = meta?.['io.modelcontextprotocol/protocolVersion'];
|
||||
const headerVersion = req.headers['mcp-protocol-version'];
|
||||
|
||||
// 1. "Every POST request to the MCP endpoint MUST include an
|
||||
// MCP-Protocol-Version header." — unconditional, so this fires for
|
||||
// server/discover too.
|
||||
this.addOrUpdateCheck({
|
||||
id: 'sep-2575-http-client-sends-version-header',
|
||||
name: 'ClientSendsVersionHeader',
|
||||
description: 'Client sends MCP-Protocol-Version header on every POST',
|
||||
status: headerVersion !== undefined ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-2575',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header'
|
||||
}
|
||||
],
|
||||
details: { method: request.method, headerVersion }
|
||||
});
|
||||
|
||||
// 2. "Every client request MUST include the following
|
||||
// io.modelcontextprotocol/* fields in _meta: protocolVersion,
|
||||
// clientInfo, clientCapabilities."
|
||||
const hasClientInfo = meta?.['io.modelcontextprotocol/clientInfo'];
|
||||
const hasCapabilities =
|
||||
meta?.['io.modelcontextprotocol/clientCapabilities'];
|
||||
const metaIsValid = metaVersion && hasClientInfo && hasCapabilities;
|
||||
|
||||
this.addOrUpdateCheck({
|
||||
id: 'sep-2575-client-populates-meta',
|
||||
name: 'ClientPopulatesMeta',
|
||||
description:
|
||||
'Client populates _meta on every request with all three required fields',
|
||||
status: metaIsValid ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-2575',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/basic/index#meta'
|
||||
}
|
||||
],
|
||||
details: { method: request.method, meta }
|
||||
});
|
||||
|
||||
// 3. "The header value MUST match the io.modelcontextprotocol/protocolVersion
|
||||
// field carried in the request body's _meta." Only meaningful when both
|
||||
// are present; absence is already covered by the two checks above.
|
||||
if (headerVersion !== undefined && metaVersion !== undefined) {
|
||||
this.addOrUpdateCheck({
|
||||
id: 'sep-2575-http-version-header-matches-meta',
|
||||
name: 'ClientVersionHeaderMatchesMeta',
|
||||
description:
|
||||
'MCP-Protocol-Version header matches _meta.protocolVersion',
|
||||
status: headerVersion === metaVersion ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-2575',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header'
|
||||
}
|
||||
],
|
||||
details: { headerVersion, metaVersion }
|
||||
});
|
||||
}
|
||||
|
||||
// 4. Optional client capabilities conditional verification
|
||||
const capabilities = meta?.['io.modelcontextprotocol/clientCapabilities'];
|
||||
const checkOptionalCapability = (
|
||||
capabilityName: string,
|
||||
checkId: string,
|
||||
checkName: string
|
||||
) => {
|
||||
let status: 'SUCCESS' | 'FAILURE' | 'SKIPPED' = 'SKIPPED';
|
||||
if (capabilities && capabilityName in capabilities) {
|
||||
const val = capabilities[capabilityName];
|
||||
const isValidObject =
|
||||
typeof val === 'object' && val !== null && !Array.isArray(val);
|
||||
status = isValidObject ? 'SUCCESS' : 'FAILURE';
|
||||
}
|
||||
this.addOrUpdateCheck({
|
||||
id: checkId,
|
||||
name: checkName,
|
||||
description: `Client declares valid ${capabilityName} capability if present`,
|
||||
status,
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-2575',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/basic/index#capabilities'
|
||||
}
|
||||
],
|
||||
details: { capabilityValue: capabilities?.[capabilityName] }
|
||||
});
|
||||
};
|
||||
|
||||
checkOptionalCapability(
|
||||
'roots',
|
||||
'sep-2575-client-declares-roots-capability',
|
||||
'ClientDeclaresRootsCapability'
|
||||
);
|
||||
checkOptionalCapability(
|
||||
'sampling',
|
||||
'sep-2575-client-declares-sampling-capability',
|
||||
'ClientDeclaresSamplingCapability'
|
||||
);
|
||||
checkOptionalCapability(
|
||||
'elicitation',
|
||||
'sep-2575-client-declares-elicitation-capability',
|
||||
'ClientDeclaresElicitationCapability'
|
||||
);
|
||||
|
||||
// 5. Simulated Version Negotiation Retry Check
|
||||
if (!this.hasSimulatedRejection) {
|
||||
this.hasSimulatedRejection = true;
|
||||
|
||||
this.addOrUpdateCheck({
|
||||
id: 'sep-2575-client-retry-supported-version',
|
||||
name: 'ClientRetrySupportedVersion',
|
||||
description:
|
||||
'Client retries with a supported version when first choice is rejected',
|
||||
status: 'WARNING',
|
||||
timestamp: new Date().toISOString(),
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-2575',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header'
|
||||
}
|
||||
],
|
||||
details: { headerVersion }
|
||||
});
|
||||
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: request.id ?? null,
|
||||
error: {
|
||||
code: -32001,
|
||||
message: 'Unsupported protocol version',
|
||||
data: {
|
||||
supported: [DRAFT_PROTOCOL_VERSION]
|
||||
}
|
||||
}
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const retryCheck = this.checks.find(
|
||||
(c) => c.id === 'sep-2575-client-retry-supported-version'
|
||||
);
|
||||
if (retryCheck) {
|
||||
if (
|
||||
headerVersion === DRAFT_PROTOCOL_VERSION &&
|
||||
metaVersion === DRAFT_PROTOCOL_VERSION
|
||||
) {
|
||||
retryCheck.status = 'SUCCESS';
|
||||
} else {
|
||||
retryCheck.status = 'WARNING';
|
||||
}
|
||||
retryCheck.details = {
|
||||
...retryCheck.details,
|
||||
retryHeaderVersion: headerVersion,
|
||||
retryMetaVersion: metaVersion
|
||||
};
|
||||
}
|
||||
|
||||
// server/discover is optional for clients (spec: "Clients MAY call it"),
|
||||
// so no check is emitted; we still respond so a client that does call it
|
||||
// proceeds normally and exercises the per-request _meta/header checks above.
|
||||
if (request.method === 'server/discover') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: request.id,
|
||||
result: {
|
||||
supportedVersions: [DRAFT_PROTOCOL_VERSION],
|
||||
capabilities: {},
|
||||
serverInfo: { name: 'test', version: '1.0' }
|
||||
}
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Return generic response to unblock client
|
||||
let result: object = {};
|
||||
if (request.method === 'tools/list') {
|
||||
result = { tools: [] };
|
||||
} else if (request.method === 'tools/call') {
|
||||
result = { content: [] };
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ jsonrpc: '2.0', id: request.id, result }));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import { InitializeScenario } from './client/initialize';
|
|||
import { ToolsCallScenario } from './client/tools_call';
|
||||
import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults';
|
||||
import { SSERetryScenario } from './client/sse-retry';
|
||||
import { RequestMetadataScenario } from './client/request-metadata';
|
||||
|
||||
// Import all new server test scenarios
|
||||
import { ServerInitializeScenario } from './server/lifecycle';
|
||||
|
|
@ -201,6 +202,7 @@ const scenariosList: Scenario[] = [
|
|||
new ToolsCallScenario(),
|
||||
new ElicitationClientDefaultsScenario(),
|
||||
new SSERetryScenario(),
|
||||
new RequestMetadataScenario(),
|
||||
...authScenariosList,
|
||||
...backcompatScenariosList,
|
||||
...draftScenariosList,
|
||||
|
|
|
|||
Loading…
Reference in a new issue