feat: add conformance Tests for SEP-2549: TTL for List Results (#275)
This commit is contained in:
Caitie McCaffrey
GitHub
parent
583d42d2d2
commit
7f1b8712ab
2 changed files with 649 additions and 0 deletions
356
src/scenarios/server/caching.ts
Normal file
356
src/scenarios/server/caching.ts
Normal file
|
|
@ -0,0 +1,356 @@
|
|||
/**
|
||||
* Caching (SEP-2549) test scenario for MCP servers
|
||||
*
|
||||
* Tests that servers include ttlMs and cacheScope on cacheable results:
|
||||
* tools/list, prompts/list, resources/list, resources/templates/list, resources/read
|
||||
*/
|
||||
|
||||
import {
|
||||
ClientScenario,
|
||||
ConformanceCheck,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
} from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
import {
|
||||
ListToolsResultSchema,
|
||||
ListPromptsResultSchema,
|
||||
ListResourcesResultSchema,
|
||||
ListResourceTemplatesResultSchema,
|
||||
ReadResourceResultSchema
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
|
||||
const SPEC_REFS = [
|
||||
{
|
||||
id: 'MCP-Caching',
|
||||
url: 'https://modelcontextprotocol.io/specification/draft/server/utilities/caching'
|
||||
},
|
||||
{
|
||||
id: 'SEP-2549',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/pull/2549'
|
||||
}
|
||||
];
|
||||
|
||||
interface CachingFields {
|
||||
ttlMs: unknown;
|
||||
cacheScope: unknown;
|
||||
hasTtlMs: boolean;
|
||||
hasCacheScope: boolean;
|
||||
}
|
||||
|
||||
function extractCachingFields(result: Record<string, unknown>): CachingFields {
|
||||
const hasTtlMs = 'ttlMs' in result;
|
||||
const hasCacheScope = 'cacheScope' in result;
|
||||
return {
|
||||
ttlMs: hasTtlMs ? result.ttlMs : undefined,
|
||||
cacheScope: hasCacheScope ? result.cacheScope : undefined,
|
||||
hasTtlMs,
|
||||
hasCacheScope
|
||||
};
|
||||
}
|
||||
|
||||
function buildPresenceCheck(
|
||||
id: string,
|
||||
name: string,
|
||||
endpoint: string,
|
||||
fields: CachingFields
|
||||
): ConformanceCheck {
|
||||
const errors: string[] = [];
|
||||
|
||||
if (!fields.hasTtlMs) {
|
||||
errors.push(`${endpoint} response missing ttlMs`);
|
||||
}
|
||||
if (!fields.hasCacheScope) {
|
||||
errors.push(`${endpoint} response missing cacheScope`);
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
description: `${endpoint} response includes ttlMs and cacheScope caching hints`,
|
||||
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
|
||||
specReferences: SPEC_REFS,
|
||||
details: {
|
||||
ttlMs: fields.ttlMs,
|
||||
cacheScope: fields.cacheScope,
|
||||
hasTtlMs: fields.hasTtlMs,
|
||||
hasCacheScope: fields.hasCacheScope
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export class CachingScenario implements ClientScenario {
|
||||
name = 'caching';
|
||||
readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const;
|
||||
description = `Test that servers include caching hints (ttlMs and cacheScope) on cacheable results (SEP-2549).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "private") on results from:
|
||||
- \`tools/list\`
|
||||
- \`prompts/list\`
|
||||
- \`resources/list\`
|
||||
- \`resources/templates/list\`
|
||||
- \`resources/read\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
const allFields: Array<{ endpoint: string; fields: CachingFields }> = [];
|
||||
|
||||
try {
|
||||
const connection = await connectToServer(serverUrl);
|
||||
|
||||
// 1. tools/list
|
||||
try {
|
||||
const toolsResult = await connection.client.request(
|
||||
{ method: 'tools/list', params: {} },
|
||||
ListToolsResultSchema
|
||||
);
|
||||
const fields = extractCachingFields(
|
||||
toolsResult as Record<string, unknown>
|
||||
);
|
||||
allFields.push({ endpoint: 'tools/list', fields });
|
||||
checks.push(
|
||||
buildPresenceCheck(
|
||||
'sep-2549-tools-list-caching-hints',
|
||||
'ToolsListCachingHints',
|
||||
'tools/list',
|
||||
fields
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'sep-2549-tools-list-caching-hints',
|
||||
name: 'ToolsListCachingHints',
|
||||
description:
|
||||
'tools/list response includes ttlMs and cacheScope caching hints',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `tools/list request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
|
||||
// 2. prompts/list
|
||||
try {
|
||||
const promptsResult = await connection.client.request(
|
||||
{ method: 'prompts/list', params: {} },
|
||||
ListPromptsResultSchema
|
||||
);
|
||||
const fields = extractCachingFields(
|
||||
promptsResult as Record<string, unknown>
|
||||
);
|
||||
allFields.push({ endpoint: 'prompts/list', fields });
|
||||
checks.push(
|
||||
buildPresenceCheck(
|
||||
'sep-2549-prompts-list-caching-hints',
|
||||
'PromptsListCachingHints',
|
||||
'prompts/list',
|
||||
fields
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'sep-2549-prompts-list-caching-hints',
|
||||
name: 'PromptsListCachingHints',
|
||||
description:
|
||||
'prompts/list response includes ttlMs and cacheScope caching hints',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `prompts/list request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
|
||||
// 3. resources/list
|
||||
let firstResourceUri: string | undefined;
|
||||
try {
|
||||
const resourcesResult = await connection.client.request(
|
||||
{ method: 'resources/list', params: {} },
|
||||
ListResourcesResultSchema
|
||||
);
|
||||
const fields = extractCachingFields(
|
||||
resourcesResult as Record<string, unknown>
|
||||
);
|
||||
allFields.push({ endpoint: 'resources/list', fields });
|
||||
checks.push(
|
||||
buildPresenceCheck(
|
||||
'sep-2549-resources-list-caching-hints',
|
||||
'ResourcesListCachingHints',
|
||||
'resources/list',
|
||||
fields
|
||||
)
|
||||
);
|
||||
// Capture the first resource URI for the resources/read check
|
||||
if (resourcesResult.resources && resourcesResult.resources.length > 0) {
|
||||
firstResourceUri = resourcesResult.resources[0].uri;
|
||||
}
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'sep-2549-resources-list-caching-hints',
|
||||
name: 'ResourcesListCachingHints',
|
||||
description:
|
||||
'resources/list response includes ttlMs and cacheScope caching hints',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `resources/list request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
|
||||
// 4. resources/templates/list
|
||||
try {
|
||||
const templatesResult = await connection.client.request(
|
||||
{ method: 'resources/templates/list', params: {} },
|
||||
ListResourceTemplatesResultSchema
|
||||
);
|
||||
const fields = extractCachingFields(
|
||||
templatesResult as Record<string, unknown>
|
||||
);
|
||||
allFields.push({ endpoint: 'resources/templates/list', fields });
|
||||
checks.push(
|
||||
buildPresenceCheck(
|
||||
'sep-2549-resources-templates-list-caching-hints',
|
||||
'ResourcesTemplatesListCachingHints',
|
||||
'resources/templates/list',
|
||||
fields
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'sep-2549-resources-templates-list-caching-hints',
|
||||
name: 'ResourcesTemplatesListCachingHints',
|
||||
description:
|
||||
'resources/templates/list response includes ttlMs and cacheScope caching hints',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `resources/templates/list request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
|
||||
// 5. resources/read — use first resource from resources/list
|
||||
if (firstResourceUri) {
|
||||
try {
|
||||
const readResult = await connection.client.request(
|
||||
{
|
||||
method: 'resources/read',
|
||||
params: { uri: firstResourceUri }
|
||||
},
|
||||
ReadResourceResultSchema
|
||||
);
|
||||
const fields = extractCachingFields(
|
||||
readResult as Record<string, unknown>
|
||||
);
|
||||
allFields.push({ endpoint: 'resources/read', fields });
|
||||
checks.push(
|
||||
buildPresenceCheck(
|
||||
'sep-2549-resources-read-caching-hints',
|
||||
'ResourcesReadCachingHints',
|
||||
'resources/read',
|
||||
fields
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'sep-2549-resources-read-caching-hints',
|
||||
name: 'ResourcesReadCachingHints',
|
||||
description:
|
||||
'resources/read response includes ttlMs and cacheScope caching hints',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `resources/read request failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Aggregate: ttlMs must be a non-negative integer
|
||||
const ttlErrors: string[] = [];
|
||||
const endpointsWithTtl = allFields.filter((f) => f.fields.hasTtlMs);
|
||||
if (endpointsWithTtl.length === 0) {
|
||||
ttlErrors.push('no endpoints returned ttlMs');
|
||||
} else {
|
||||
for (const { endpoint, fields } of endpointsWithTtl) {
|
||||
const val = fields.ttlMs;
|
||||
if (typeof val !== 'number') {
|
||||
ttlErrors.push(
|
||||
`${endpoint}: ttlMs is ${typeof val}, expected number`
|
||||
);
|
||||
} else if (!Number.isInteger(val)) {
|
||||
ttlErrors.push(`${endpoint}: ttlMs is ${val}, expected integer`);
|
||||
} else if (val < 0) {
|
||||
ttlErrors.push(`${endpoint}: ttlMs is ${val}, must be >= 0`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'sep-2549-ttl-non-negative',
|
||||
name: 'TtlNonNegative',
|
||||
description: 'All ttlMs values are non-negative integers',
|
||||
status: ttlErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: ttlErrors.length > 0 ? ttlErrors.join('; ') : undefined,
|
||||
specReferences: SPEC_REFS,
|
||||
details: {
|
||||
endpoints: allFields.map((f) => ({
|
||||
endpoint: f.endpoint,
|
||||
ttlMs: f.fields.ttlMs
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
// 7. Aggregate: cacheScope must be "public" or "private"
|
||||
const scopeErrors: string[] = [];
|
||||
const endpointsWithScope = allFields.filter(
|
||||
(f) => f.fields.hasCacheScope
|
||||
);
|
||||
if (endpointsWithScope.length === 0) {
|
||||
scopeErrors.push('no endpoints returned cacheScope');
|
||||
} else {
|
||||
for (const { endpoint, fields } of endpointsWithScope) {
|
||||
const val = fields.cacheScope;
|
||||
if (val !== 'public' && val !== 'private') {
|
||||
scopeErrors.push(
|
||||
`${endpoint}: cacheScope is ${JSON.stringify(val)}, expected "public" or "private"`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'sep-2549-cache-scope-valid',
|
||||
name: 'CacheScopeValid',
|
||||
description: 'All cacheScope values are "public" or "private"',
|
||||
status: scopeErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
scopeErrors.length > 0 ? scopeErrors.join('; ') : undefined,
|
||||
specReferences: SPEC_REFS,
|
||||
details: {
|
||||
endpoints: allFields.map((f) => ({
|
||||
endpoint: f.endpoint,
|
||||
cacheScope: f.fields.cacheScope
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
await connection.close();
|
||||
} catch (error) {
|
||||
// Connection-level failure — push a single failure check
|
||||
checks.push({
|
||||
id: 'sep-2549-caching-connection',
|
||||
name: 'CachingConnection',
|
||||
description: 'Caching hints scenario failed to connect',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: SPEC_REFS
|
||||
});
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ import { spawn, ChildProcess } from 'child_process';
|
|||
import path from 'path';
|
||||
import { DNSRebindingProtectionScenario } from './dns-rebinding';
|
||||
import { ResourcesNotFoundErrorScenario } from './resources';
|
||||
import { CachingScenario } from './caching';
|
||||
|
||||
function startServer(scriptPath: string, port: number): Promise<ChildProcess> {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
|
@ -106,4 +107,45 @@ describe('Server scenario negative tests', () => {
|
|||
expect(errorCode?.status).toBe('WARNING');
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
describe('sep-2549-caching-hints', () => {
|
||||
let serverProcess: ChildProcess | null = null;
|
||||
const PORT = 3006;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverProcess = await startServer(
|
||||
path.join(
|
||||
process.cwd(),
|
||||
'examples/servers/typescript/sep-2549-no-caching-hints.ts'
|
||||
),
|
||||
PORT
|
||||
);
|
||||
}, 35000);
|
||||
|
||||
afterAll(async () => {
|
||||
await stopServer(serverProcess);
|
||||
});
|
||||
|
||||
it('emits FAILURE for presence checks against a server without caching hints', async () => {
|
||||
const scenario = new CachingScenario();
|
||||
const checks = await scenario.run(`http://localhost:${PORT}/mcp`);
|
||||
|
||||
// Should have at least 7 checks (5 presence + 2 aggregate)
|
||||
expect(checks.length).toBeGreaterThanOrEqual(7);
|
||||
|
||||
const presenceCheckIds = [
|
||||
'sep-2549-tools-list-caching-hints',
|
||||
'sep-2549-prompts-list-caching-hints',
|
||||
'sep-2549-resources-list-caching-hints',
|
||||
'sep-2549-resources-templates-list-caching-hints',
|
||||
'sep-2549-resources-read-caching-hints'
|
||||
];
|
||||
|
||||
for (const checkId of presenceCheckIds) {
|
||||
const check = checks.find((c) => c.id === checkId);
|
||||
expect(check).toBeDefined();
|
||||
expect(check?.status).toBe('FAILURE');
|
||||
}
|
||||
}, 15000);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue