modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add conformance Tests for SEP-2549: TTL for List Results (#275)

Browse source
This commit is contained in:
Caitie McCaffrey 2026-05-22 12:57:49 -07:00 • committed by GitHub
commit 7f1b8712ab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 649 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

45
examples/servers/typescript/everything-server.ts
View file

@ -22,6 +22,9 @@ import { createMcpExpressApp } from '@modelcontextprotocol/sdk/server/express.js
import {
ElicitResultSchema,
ListToolsRequestSchema,
ListPromptsRequestSchema,
ListResourcesRequestSchema,
ListResourceTemplatesRequestSchema,
type ListToolsResult,
type Tool
} from '@modelcontextprotocol/sdk/types.js';
@ -128,6 +131,46 @@ function createMcpServer() {
}
);
// SEP-2549: Wrap setRequestHandler so the SDK's own list handlers
// automatically get caching hints appended to their responses.
const originalSetRequestHandler = mcpServer.server.setRequestHandler.bind(
mcpServer.server
);
const listSchemasForCaching = new Set([
ListToolsRequestSchema,
ListPromptsRequestSchema,
ListResourcesRequestSchema,
ListResourceTemplatesRequestSchema
]);
mcpServer.server.setRequestHandler = ((schema: any, handler: any) => {
if (listSchemasForCaching.has(schema)) {
return originalSetRequestHandler(schema, async (...args: any[]) => {
const result = await handler(...args);
return { ...result, ttlMs: 300000, cacheScope: 'public' as const };
});
}
return originalSetRequestHandler(schema, handler);
}) as typeof mcpServer.server.setRequestHandler;
const registerResourceWithCacheHints =
mcpServer.registerResource.bind(mcpServer);
mcpServer.registerResource = ((
name: string,
uriOrTemplate: string | ResourceTemplate,
config: any,
readCallback: any
) =>
registerResourceWithCacheHints(
name,
uriOrTemplate as any,
config,
async (...args: any[]) => ({
...(await readCallback(...args)),
ttlMs: 300000,
cacheScope: 'private' as const
})
)) as typeof mcpServer.registerResource;
// Helper to send log messages using the underlying server
function sendLog(
level:
@ -1024,6 +1067,8 @@ function createMcpServer() {
_meta: tool._meta
};
})
// Note: SEP-2549 caching hints are added automatically by the
// setRequestHandler wrapper above
};
}
);

159
examples/servers/typescript/sep-2549-no-caching-hints.ts Normal file
View file

@ -0,0 +1,159 @@
#!/usr/bin/env node
/**
* SEP-2549 Negative Test Server
*
* Returns list and read results WITHOUT ttlMs and cacheScope fields,
* violating the SEP-2549 MUST. The caching scenario should emit FAILURE
* for presence checks against this server.
*/
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
import {
ListToolsRequestSchema,
ListPromptsRequestSchema,
ListResourcesRequestSchema,
ListResourceTemplatesRequestSchema,
ReadResourceRequestSchema
} from '@modelcontextprotocol/sdk/types.js';
import express from 'express';
import { randomUUID } from 'crypto';
const transports: Record<string, StreamableHTTPServerTransport> = {};
function isInitializeRequest(body: any): boolean {
return body?.method === 'initialize';
}
function createServer() {
const server = new Server(
{ name: 'sep-2549-no-caching-hints', version: '1.0.0' },
{
capabilities: {
tools: {},
resources: {},
prompts: {}
}
}
);
// Deliberately omit ttlMs and cacheScope from all responses
server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: [
{
name: 'test_tool',
description: 'A test tool',
inputSchema: { type: 'object' as const }
}
]
}));
server.setRequestHandler(ListPromptsRequestSchema, async () => ({
prompts: [
{
name: 'test_prompt',
description: 'A test prompt'
}
]
}));
server.setRequestHandler(ListResourcesRequestSchema, async () => ({
resources: [
{
uri: 'test://static-text',
name: 'Static Text',
description: 'A static text resource'
}
]
}));
server.setRequestHandler(ListResourceTemplatesRequestSchema, async () => ({
resourceTemplates: []
}));
server.setRequestHandler(ReadResourceRequestSchema, async () => ({
contents: [
{
uri: 'test://static-text',
mimeType: 'text/plain',
text: 'Static text content.'
}
]
}));
return server;
}
const app = express();
app.use(express.json());
app.post('/mcp', async (req, res) => {
const sessionId = req.headers['mcp-session-id'] as string | undefined;
try {
if (sessionId && transports[sessionId]) {
await transports[sessionId].handleRequest(req, res, req.body);
return;
}
if (!sessionId && isInitializeRequest(req.body)) {
const server = createServer();
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => randomUUID(),
onsessioninitialized: (newSessionId) => {
transports[newSessionId] = transport;
}
});
transport.onclose = () => {
const sid = transport.sessionId;
if (sid) delete transports[sid];
};
await server.connect(transport);
await transport.handleRequest(req, res, req.body);
return;
}
res.status(400).json({
jsonrpc: '2.0',
error: { code: -32000, message: 'Invalid or missing session ID' },
id: null
});
} catch (error) {
if (!res.headersSent) {
res.status(500).json({
jsonrpc: '2.0',
error: {
code: -32603,
message: `Internal error: ${error instanceof Error ? error.message : String(error)}`
},
id: null
});
}
}
});
app.get('/mcp', async (req, res) => {
const sessionId = req.headers['mcp-session-id'] as string | undefined;
if (sessionId && transports[sessionId]) {
await transports[sessionId].handleRequest(req, res);
} else {
res.status(400).json({ error: 'Invalid or missing session ID' });
}
});
app.delete('/mcp', async (req, res) => {
const sessionId = req.headers['mcp-session-id'] as string | undefined;
if (sessionId && transports[sessionId]) {
await transports[sessionId].handleRequest(req, res);
} else {
res.status(400).json({ error: 'Invalid or missing session ID' });
}
});
const PORT = parseInt(process.env.PORT || '3006', 10);
app.listen(PORT, () => {
console.log(
`SEP-2549 negative test server running on http://localhost:${PORT}/mcp`
);
});

3
src/scenarios/index.ts
View file

@ -65,6 +65,7 @@ import {
} from './server/prompts';
import { DNSRebindingProtectionScenario } from './server/dns-rebinding';
import { CachingScenario } from './server/caching';
import {
HttpHeaderValidationScenario,
@ -162,6 +163,8 @@ const allClientScenariosList: ClientScenario[] = [
// Security scenarios
new DNSRebindingProtectionScenario(),
// Caching scenarios (SEP-2549)
new CachingScenario(),
// HTTP Standardization scenarios (SEP-2243)
new HttpHeaderValidationScenario(),
new HttpCustomHeaderServerValidationScenario()

356
src/scenarios/server/caching.ts Normal file
View file

@ -0,0 +1,356 @@
/**
* Caching (SEP-2549) test scenario for MCP servers
*
* Tests that servers include ttlMs and cacheScope on cacheable results:
* tools/list, prompts/list, resources/list, resources/templates/list, resources/read
*/
import {
ClientScenario,
ConformanceCheck,
DRAFT_PROTOCOL_VERSION
} from '../../types';
import { connectToServer } from './client-helper';
import {
ListToolsResultSchema,
ListPromptsResultSchema,
ListResourcesResultSchema,
ListResourceTemplatesResultSchema,
ReadResourceResultSchema
} from '@modelcontextprotocol/sdk/types.js';
const SPEC_REFS = [
{
id: 'MCP-Caching',
url: 'https://modelcontextprotocol.io/specification/draft/server/utilities/caching'
},
{
id: 'SEP-2549',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/pull/2549'
}
];
interface CachingFields {
ttlMs: unknown;
cacheScope: unknown;
hasTtlMs: boolean;
hasCacheScope: boolean;
}
function extractCachingFields(result: Record<string, unknown>): CachingFields {
const hasTtlMs = 'ttlMs' in result;
const hasCacheScope = 'cacheScope' in result;
return {
ttlMs: hasTtlMs ? result.ttlMs : undefined,
cacheScope: hasCacheScope ? result.cacheScope : undefined,
hasTtlMs,
hasCacheScope
};
}
function buildPresenceCheck(
id: string,
name: string,
endpoint: string,
fields: CachingFields
): ConformanceCheck {
const errors: string[] = [];
if (!fields.hasTtlMs) {
errors.push(`${endpoint} response missing ttlMs`);
}
if (!fields.hasCacheScope) {
errors.push(`${endpoint} response missing cacheScope`);
}
return {
id,
name,
description: `${endpoint} response includes ttlMs and cacheScope caching hints`,
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: SPEC_REFS,
details: {
ttlMs: fields.ttlMs,
cacheScope: fields.cacheScope,
hasTtlMs: fields.hasTtlMs,
hasCacheScope: fields.hasCacheScope
}
};
}
export class CachingScenario implements ClientScenario {
name = 'caching';
readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const;
description = `Test that servers include caching hints (ttlMs and cacheScope) on cacheable results (SEP-2549).
**Server Implementation Requirements:**
Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "private") on results from:
- \`tools/list\`
- \`prompts/list\`
- \`resources/list\`
- \`resources/templates/list\`
- \`resources/read\``;
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
const allFields: Array<{ endpoint: string; fields: CachingFields }> = [];
try {
const connection = await connectToServer(serverUrl);
// 1. tools/list
try {
const toolsResult = await connection.client.request(
{ method: 'tools/list', params: {} },
ListToolsResultSchema
);
const fields = extractCachingFields(
toolsResult as Record<string, unknown>
);
allFields.push({ endpoint: 'tools/list', fields });
checks.push(
buildPresenceCheck(
'sep-2549-tools-list-caching-hints',
'ToolsListCachingHints',
'tools/list',
fields
)
);
} catch (error) {
checks.push({
id: 'sep-2549-tools-list-caching-hints',
name: 'ToolsListCachingHints',
description:
'tools/list response includes ttlMs and cacheScope caching hints',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `tools/list request failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
// 2. prompts/list
try {
const promptsResult = await connection.client.request(
{ method: 'prompts/list', params: {} },
ListPromptsResultSchema
);
const fields = extractCachingFields(
promptsResult as Record<string, unknown>
);
allFields.push({ endpoint: 'prompts/list', fields });
checks.push(
buildPresenceCheck(
'sep-2549-prompts-list-caching-hints',
'PromptsListCachingHints',
'prompts/list',
fields
)
);
} catch (error) {
checks.push({
id: 'sep-2549-prompts-list-caching-hints',
name: 'PromptsListCachingHints',
description:
'prompts/list response includes ttlMs and cacheScope caching hints',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `prompts/list request failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
// 3. resources/list
let firstResourceUri: string | undefined;
try {
const resourcesResult = await connection.client.request(
{ method: 'resources/list', params: {} },
ListResourcesResultSchema
);
const fields = extractCachingFields(
resourcesResult as Record<string, unknown>
);
allFields.push({ endpoint: 'resources/list', fields });
checks.push(
buildPresenceCheck(
'sep-2549-resources-list-caching-hints',
'ResourcesListCachingHints',
'resources/list',
fields
)
);
// Capture the first resource URI for the resources/read check
if (resourcesResult.resources && resourcesResult.resources.length > 0) {
firstResourceUri = resourcesResult.resources[0].uri;
}
} catch (error) {
checks.push({
id: 'sep-2549-resources-list-caching-hints',
name: 'ResourcesListCachingHints',
description:
'resources/list response includes ttlMs and cacheScope caching hints',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `resources/list request failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
// 4. resources/templates/list
try {
const templatesResult = await connection.client.request(
{ method: 'resources/templates/list', params: {} },
ListResourceTemplatesResultSchema
);
const fields = extractCachingFields(
templatesResult as Record<string, unknown>
);
allFields.push({ endpoint: 'resources/templates/list', fields });
checks.push(
buildPresenceCheck(
'sep-2549-resources-templates-list-caching-hints',
'ResourcesTemplatesListCachingHints',
'resources/templates/list',
fields
)
);
} catch (error) {
checks.push({
id: 'sep-2549-resources-templates-list-caching-hints',
name: 'ResourcesTemplatesListCachingHints',
description:
'resources/templates/list response includes ttlMs and cacheScope caching hints',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `resources/templates/list request failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
// 5. resources/read — use first resource from resources/list
if (firstResourceUri) {
try {
const readResult = await connection.client.request(
{
method: 'resources/read',
params: { uri: firstResourceUri }
},
ReadResourceResultSchema
);
const fields = extractCachingFields(
readResult as Record<string, unknown>
);
allFields.push({ endpoint: 'resources/read', fields });
checks.push(
buildPresenceCheck(
'sep-2549-resources-read-caching-hints',
'ResourcesReadCachingHints',
'resources/read',
fields
)
);
} catch (error) {
checks.push({
id: 'sep-2549-resources-read-caching-hints',
name: 'ResourcesReadCachingHints',
description:
'resources/read response includes ttlMs and cacheScope caching hints',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `resources/read request failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
}
// 6. Aggregate: ttlMs must be a non-negative integer
const ttlErrors: string[] = [];
const endpointsWithTtl = allFields.filter((f) => f.fields.hasTtlMs);
if (endpointsWithTtl.length === 0) {
ttlErrors.push('no endpoints returned ttlMs');
} else {
for (const { endpoint, fields } of endpointsWithTtl) {
const val = fields.ttlMs;
if (typeof val !== 'number') {
ttlErrors.push(
`${endpoint}: ttlMs is ${typeof val}, expected number`
);
} else if (!Number.isInteger(val)) {
ttlErrors.push(`${endpoint}: ttlMs is ${val}, expected integer`);
} else if (val < 0) {
ttlErrors.push(`${endpoint}: ttlMs is ${val}, must be >= 0`);
}
}
}
checks.push({
id: 'sep-2549-ttl-non-negative',
name: 'TtlNonNegative',
description: 'All ttlMs values are non-negative integers',
status: ttlErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: ttlErrors.length > 0 ? ttlErrors.join('; ') : undefined,
specReferences: SPEC_REFS,
details: {
endpoints: allFields.map((f) => ({
endpoint: f.endpoint,
ttlMs: f.fields.ttlMs
}))
}
});
// 7. Aggregate: cacheScope must be "public" or "private"
const scopeErrors: string[] = [];
const endpointsWithScope = allFields.filter(
(f) => f.fields.hasCacheScope
);
if (endpointsWithScope.length === 0) {
scopeErrors.push('no endpoints returned cacheScope');
} else {
for (const { endpoint, fields } of endpointsWithScope) {
const val = fields.cacheScope;
if (val !== 'public' && val !== 'private') {
scopeErrors.push(
`${endpoint}: cacheScope is ${JSON.stringify(val)}, expected "public" or "private"`
);
}
}
}
checks.push({
id: 'sep-2549-cache-scope-valid',
name: 'CacheScopeValid',
description: 'All cacheScope values are "public" or "private"',
status: scopeErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
scopeErrors.length > 0 ? scopeErrors.join('; ') : undefined,
specReferences: SPEC_REFS,
details: {
endpoints: allFields.map((f) => ({
endpoint: f.endpoint,
cacheScope: f.fields.cacheScope
}))
}
});
await connection.close();
} catch (error) {
// Connection-level failure — push a single failure check
checks.push({
id: 'sep-2549-caching-connection',
name: 'CachingConnection',
description: 'Caching hints scenario failed to connect',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: SPEC_REFS
});
}
return checks;
}
}

42
src/scenarios/server/negative.test.ts
View file

@ -2,6 +2,7 @@ import { spawn, ChildProcess } from 'child_process';
import path from 'path';
import { DNSRebindingProtectionScenario } from './dns-rebinding';
import { ResourcesNotFoundErrorScenario } from './resources';
import { CachingScenario } from './caching';
function startServer(scriptPath: string, port: number): Promise<ChildProcess> {
return new Promise((resolve, reject) => {
@ -106,4 +107,45 @@ describe('Server scenario negative tests', () => {
expect(errorCode?.status).toBe('WARNING');
}, 10000);
});
describe('sep-2549-caching-hints', () => {
let serverProcess: ChildProcess | null = null;
const PORT = 3006;
beforeAll(async () => {
serverProcess = await startServer(
path.join(
process.cwd(),
'examples/servers/typescript/sep-2549-no-caching-hints.ts'
),
PORT
);
}, 35000);
afterAll(async () => {
await stopServer(serverProcess);
});
it('emits FAILURE for presence checks against a server without caching hints', async () => {
const scenario = new CachingScenario();
const checks = await scenario.run(`http://localhost:${PORT}/mcp`);
// Should have at least 7 checks (5 presence + 2 aggregate)
expect(checks.length).toBeGreaterThanOrEqual(7);
const presenceCheckIds = [
'sep-2549-tools-list-caching-hints',
'sep-2549-prompts-list-caching-hints',
'sep-2549-resources-list-caching-hints',
'sep-2549-resources-templates-list-caching-hints',
'sep-2549-resources-read-caching-hints'
];
for (const checkId of presenceCheckIds) {
const check = checks.find((c) => c.id === checkId);
expect(check).toBeDefined();
expect(check?.status).toBe('FAILURE');
}
}, 15000);
});
});

44
src/seps/sep-2549.yaml Normal file
View file

@ -0,0 +1,44 @@
sep: 2549
spec_url: https://modelcontextprotocol.io/specification/draft/server/utilities/caching
requirements:
- check: sep-2549-tools-list-caching-hints
text: 'Servers MUST include caching hints on results returned by tools/list'
- check: sep-2549-prompts-list-caching-hints
text: 'Servers MUST include caching hints on results returned by prompts/list'
- check: sep-2549-resources-list-caching-hints
text: 'Servers MUST include caching hints on results returned by resources/list'
- check: sep-2549-resources-templates-list-caching-hints
text: 'Servers MUST include caching hints on results returned by resources/templates/list'
- check: sep-2549-resources-read-caching-hints
text: 'Servers MUST include caching hints on results returned by resources/read'
- check: sep-2549-ttl-non-negative
text: 'Servers MUST provide a ttlMs value that is >= 0'
- check: sep-2549-cache-scope-valid
text: 'cacheScope indicates the intended scope of the cached response, either "public" or "private"'
- text: 'If ttlMs is 0, the response SHOULD be considered immediately stale.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'If ttlMs is positive, the client SHOULD consider the result fresh for that many milliseconds after receiving the response.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'If ttlMs is absent, clients SHOULD assume a default of 0 (immediately stale) and rely on their own caching heuristics or notifications.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'If ttlMs is negative, clients SHOULD ignore it and treat it as 0.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'Once the TTL expires, the response is stale and the client SHOULD re-fetch on next access.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'Clients SHOULD NOT treat TTL as a polling interval that triggers automatic background refetches.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'Implementations that do choose to poll MUST apply jitter and backoff.'
excluded: 'Client-side polling behavior; not observable at the protocol level'
- text: 'Cached responses MAY be reused for the same authorization context. Caches MUST NOT be shared across authorization contexts (e.g. a different access token requires a different cache).'
excluded: 'Client/cache-side behavior; not observable at the protocol level'
- text: 'When a cached page expires, the client SHOULD re-fetch that page using its cursor.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'Clients that require a consistent snapshot of the full list SHOULD re-fetch from the beginning (without a cursor).'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'If a cursor becomes invalid (e.g., the server returns an error for a previously valid cursor), the client SHOULD discard all cached pages and re-fetch from the beginning.'
excluded: 'Client-side caching behavior; not observable at the protocol level'
- text: 'Servers MUST be aware that responses with a "public" cacheScope may be shared between callers even if the Result is coming from an authenticated endpoint.'
excluded: 'Server-side awareness requirement; implementation guidance not testable via protocol messages'
- text: 'Server implementors MUST apply appropriate per-primitive access controls, and MUST NOT rely on cacheScope alone to prevent unauthorized access to primitives.'
excluded: 'Server-side access control; implementation guidance not testable via protocol messages'