feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire (#492)
* feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire The 2026-07-28 revision adds to server/tools.mdx: "Servers SHOULD return tools in a deterministic order (i.e., the same ordering across requests when the underlying set of tools has not changed)." Nothing in the suite exercised it. Add a tools-list-deterministic-order check to the existing tools-list scenario. From the 2026-07-28 wire on, the scenario takes two more consecutive tools/list snapshots and compares the three name sequences. Same multiset of names, different order is WARNING (SHOULD). A set that changed between probes is reported as untestable rather than as a violation, because the spec scopes the SHOULD to an unchanged set. Fewer than two tools on every probe is INFO. The check carries source.introducedIn '2026-07-28', written as a literal since that revision is published and dated, so runners drop it on earlier wires. Prove it fails: examples/servers/typescript/tools-list-rotated-order.ts advertises the same four tools and rotates the list by one position per call; a negative.test.ts case asserts WARNING while tools-list itself still passes. Unit tests cover the helper that builds the check. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BjeC3sCvfHRpdEPz2x41B2 * tools-list: inline the deterministic-order check id and introducedIn literal Check ids and introducedIn revisions are written inline everywhere else in src/scenarios (tests pin the literal slug on purpose, since SDK baselines reference it), so drop the two exported constants. Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com> Co-authored-by: Paul Carleton <paulc@anthropic.com>
This commit is contained in:
AKS
GitHub
parent
0d824aebd8
commit
542381b7dc
4 changed files with 416 additions and 2 deletions
76
examples/servers/typescript/tools-list-rotated-order.ts
Normal file
76
examples/servers/typescript/tools-list-rotated-order.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* tools/list ordering negative test server.
|
||||
*
|
||||
* Speaks the sessionless 2026-07-28 wire (SEP-2575) and advertises the same
|
||||
* four tools on every tools/list request, but rotates the list by one
|
||||
* position each time (it keeps a call counter for that, nothing else). The set never changes, only the order, which violates the
|
||||
* 2026-07-28 SHOULD "Servers SHOULD return tools in a deterministic order".
|
||||
* The tools-list scenario should emit WARNING for
|
||||
* tools-list-deterministic-order against this server while tools-list itself
|
||||
* still passes, since every response is structurally valid.
|
||||
*/
|
||||
|
||||
import express from 'express';
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
|
||||
const TOOLS = ['alpha', 'bravo', 'charlie', 'delta'].map((name) => ({
|
||||
name,
|
||||
description: `Fixture tool ${name}`,
|
||||
inputSchema: { type: 'object', properties: {} }
|
||||
}));
|
||||
|
||||
let listCalls = 0;
|
||||
|
||||
app.post('/mcp', (req, res) => {
|
||||
const body = req.body || {};
|
||||
const id = body.id ?? null;
|
||||
const method = body.method;
|
||||
|
||||
switch (method) {
|
||||
case 'server/discover':
|
||||
return res.json({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
result: {
|
||||
resultType: 'complete',
|
||||
ttlMs: 0,
|
||||
cacheScope: 'private',
|
||||
supportedVersions: ['2026-07-28'],
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: 'tools-list-rotated-order', version: '1.0.0' }
|
||||
}
|
||||
});
|
||||
case 'tools/list': {
|
||||
// Rotate by one position per call: the same set, never the same order.
|
||||
const offset = listCalls++ % TOOLS.length;
|
||||
const tools = [...TOOLS.slice(offset), ...TOOLS.slice(0, offset)];
|
||||
return res.json({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
result: {
|
||||
resultType: 'complete',
|
||||
ttlMs: 0,
|
||||
cacheScope: 'private',
|
||||
tools
|
||||
}
|
||||
});
|
||||
}
|
||||
default:
|
||||
return res.status(404).json({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
error: { code: -32601, message: 'Method not found' }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '3008', 10);
|
||||
app.listen(PORT, '127.0.0.1', () => {
|
||||
console.log(
|
||||
`tools/list rotated-order negative test server running on http://localhost:${PORT}/mcp`
|
||||
);
|
||||
});
|
||||
|
|
@ -4,11 +4,11 @@ import path from 'path';
|
|||
import { DNSRebindingProtectionScenario } from './dns-rebinding';
|
||||
import { ResourcesNotFoundErrorScenario } from './resources';
|
||||
import { CachingScenario } from './caching';
|
||||
import { ToolsListScenario } from './tools';
|
||||
import {
|
||||
JsonSchema2020_12Scenario,
|
||||
sep2106KeywordCheckStatus
|
||||
} from './json-schema-2020-12';
|
||||
import { ToolsListScenario } from './tools';
|
||||
import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types';
|
||||
import { takeWireViolations } from '../../validation/wire-schema';
|
||||
|
||||
|
|
@ -217,6 +217,43 @@ describe('Server scenario negative tests', () => {
|
|||
}, 10000);
|
||||
});
|
||||
|
||||
describe('tools-list-deterministic-order', () => {
|
||||
let serverProcess: ChildProcess | null = null;
|
||||
const PORT = 3008;
|
||||
|
||||
beforeAll(async () => {
|
||||
serverProcess = await startServer(
|
||||
path.join(
|
||||
process.cwd(),
|
||||
'examples/servers/typescript/tools-list-rotated-order.ts'
|
||||
),
|
||||
PORT
|
||||
);
|
||||
}, 35000);
|
||||
|
||||
afterAll(async () => {
|
||||
await stopServer(serverProcess);
|
||||
});
|
||||
|
||||
it('emits WARNING for deterministic-order while tools-list still passes against a server that rotates its tool list', async () => {
|
||||
const scenario = new ToolsListScenario();
|
||||
const checks = await scenario.run(
|
||||
testContext(`http://localhost:${PORT}/mcp`, DRAFT_PROTOCOL_VERSION)
|
||||
);
|
||||
|
||||
const list = checks.find((c) => c.id === 'tools-list');
|
||||
expect(list?.status).toBe('SUCCESS');
|
||||
|
||||
const order = checks.find(
|
||||
(c) => c.id === 'tools-list-deterministic-order'
|
||||
);
|
||||
expect(order?.status).toBe('WARNING');
|
||||
expect(order?.errorMessage).toMatch(/different order/);
|
||||
expect(order?.details).toMatchObject({ toolCount: 4, probes: 3 });
|
||||
expect(order?.details?.untestable).toBeUndefined();
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
describe('tools-name-format', () => {
|
||||
let serverProcess: ChildProcess | null = null;
|
||||
const PORT = 3009;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
buildToolsListDeterministicOrderCheck,
|
||||
buildToolsNameFormatCheck,
|
||||
toolNameFormatCheckApplies,
|
||||
ToolsListScenario,
|
||||
|
|
@ -193,3 +194,143 @@ describe('ToolsListScenario version gate', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildToolsListDeterministicOrderCheck', () => {
|
||||
const names = (...ns: string[]) => ns.map((name) => ({ name }));
|
||||
|
||||
it('returns SUCCESS when every probe lists the same tools in the same order', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b', 'c'),
|
||||
names('a', 'b', 'c'),
|
||||
names('a', 'b', 'c')
|
||||
]);
|
||||
expect(check.id).toBe('tools-list-deterministic-order');
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
expect(check.details).toEqual({
|
||||
toolCount: 3,
|
||||
probes: 3,
|
||||
orders: [
|
||||
['a', 'b', 'c'],
|
||||
['a', 'b', 'c'],
|
||||
['a', 'b', 'c']
|
||||
]
|
||||
});
|
||||
expect(check.specReferences?.[0]?.url).toContain('2026-07-28/server/tools');
|
||||
});
|
||||
|
||||
it('returns WARNING when the same tools come back in a different order', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b', 'c'),
|
||||
names('b', 'c', 'a'),
|
||||
names('c', 'a', 'b')
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/different order/);
|
||||
expect(check.errorMessage).toMatch(/index 0/);
|
||||
expect(check.details).toMatchObject({ toolCount: 3, probes: 3 });
|
||||
expect(check.details?.untestable).toBeUndefined();
|
||||
expect(check.details?.orders).toEqual([
|
||||
['a', 'b', 'c'],
|
||||
['b', 'c', 'a'],
|
||||
['c', 'a', 'b']
|
||||
]);
|
||||
});
|
||||
|
||||
it('flags a divergence that only appears on the last probe', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b', 'c'),
|
||||
names('a', 'b', 'c'),
|
||||
names('a', 'c', 'b')
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/probe 3/);
|
||||
expect(check.errorMessage).toMatch(/index 1/);
|
||||
});
|
||||
|
||||
it('reports untestable (WARNING) when the set of tools changed between probes', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b', 'c'),
|
||||
names('a', 'b', 'd')
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/^Not testable: /);
|
||||
expect(check.errorMessage).toMatch(/added: d/);
|
||||
expect(check.errorMessage).toMatch(/removed: c/);
|
||||
expect(check.details).toMatchObject({ untestable: true });
|
||||
});
|
||||
|
||||
it('reports untestable (WARNING) when a probe returned no tools array', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b'),
|
||||
undefined
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/^Not testable: /);
|
||||
expect(check.details).toMatchObject({ untestable: true });
|
||||
});
|
||||
|
||||
it('reports untestable (WARNING) with fewer than two probes', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([names('a', 'b')]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/^Not testable: /);
|
||||
});
|
||||
|
||||
it('returns INFO when no probe saw two tools to order', () => {
|
||||
const one = buildToolsListDeterministicOrderCheck([names('a'), names('a')]);
|
||||
expect(one.status).toBe('INFO');
|
||||
expect(one.errorMessage).toMatch(/nothing to compare/);
|
||||
expect(one.details).toEqual({
|
||||
toolCount: 1,
|
||||
probes: 2,
|
||||
orders: [['a'], ['a']]
|
||||
});
|
||||
const none = buildToolsListDeterministicOrderCheck([[], []]);
|
||||
expect(none.status).toBe('INFO');
|
||||
});
|
||||
|
||||
it('reports untestable when a probe grows from one tool to two', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a'),
|
||||
names('a', 'b')
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/^Not testable: /);
|
||||
expect(check.errorMessage).toMatch(/added: b/);
|
||||
});
|
||||
|
||||
it('treats a tool without a string name as one placeholder entry', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
[{ name: 'a' }, { name: 42 }],
|
||||
[{ name: 42 }, { name: 'a' }]
|
||||
]);
|
||||
expect(check.status).toBe('WARNING');
|
||||
expect(check.errorMessage).toMatch(/different order/);
|
||||
expect(check.details?.orders).toEqual([
|
||||
['a', '<tool missing name>'],
|
||||
['<tool missing name>', 'a']
|
||||
]);
|
||||
});
|
||||
|
||||
it('treats duplicate names as a set change only when their multiplicity changes', () => {
|
||||
const stable = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'a', 'b'),
|
||||
names('a', 'a', 'b')
|
||||
]);
|
||||
expect(stable.status).toBe('SUCCESS');
|
||||
const changed = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'a', 'b'),
|
||||
names('a', 'b', 'b')
|
||||
]);
|
||||
expect(changed.status).toBe('WARNING');
|
||||
expect(changed.errorMessage).toMatch(/^Not testable: /);
|
||||
});
|
||||
|
||||
it('gates itself to the 2026-07-28 wire', () => {
|
||||
const check = buildToolsListDeterministicOrderCheck([
|
||||
names('a', 'b'),
|
||||
names('a', 'b')
|
||||
]);
|
||||
expect(check.source).toEqual({ introducedIn: '2026-07-28' });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
type SpecVersion
|
||||
} from '../../types';
|
||||
import type { RunContext } from '../../connection';
|
||||
import { notTestable, untestableCheck } from '../untestable';
|
||||
import type {
|
||||
ListToolsResult,
|
||||
CallToolResult
|
||||
|
|
@ -120,6 +121,131 @@ export function buildToolsNameFormatCheck(
|
|||
};
|
||||
}
|
||||
|
||||
/** How many consecutive tools/list snapshots the ordering check compares. */
|
||||
const TOOLS_LIST_ORDER_PROBES = 3;
|
||||
|
||||
const TOOLS_LIST_ORDER_SPEC_REFS = [
|
||||
{
|
||||
id: 'MCP-Tools-Deterministic-Order',
|
||||
url: 'https://modelcontextprotocol.io/specification/2026-07-28/server/tools#capabilities'
|
||||
}
|
||||
];
|
||||
|
||||
/**
|
||||
* Build the tools-list-deterministic-order check from consecutive tools/list
|
||||
* snapshots.
|
||||
*
|
||||
* 2026-07-28 server/tools.mdx: "Servers SHOULD return tools in a deterministic
|
||||
* order (i.e., the same ordering across requests when the underlying set of
|
||||
* tools has not changed)." SHOULD, so a violation is WARNING.
|
||||
*
|
||||
* The spec scopes the SHOULD to an unchanged set, so a set that differs
|
||||
* between probes is reported as untestable (issue #248) rather than as a
|
||||
* violation: from the outside, a sample cannot tell a nondeterministic server
|
||||
* from one whose tools legitimately changed between two requests.
|
||||
*/
|
||||
export function buildToolsListDeterministicOrderCheck(
|
||||
snapshots: ReadonlyArray<ReadonlyArray<{ name?: unknown }> | undefined>
|
||||
): ConformanceCheck {
|
||||
const timestamp = new Date().toISOString();
|
||||
const baseCheck = {
|
||||
id: 'tools-list-deterministic-order',
|
||||
name: 'ToolsListDeterministicOrder',
|
||||
description:
|
||||
'Consecutive tools/list requests return the same tools in the same order',
|
||||
specReferences: TOOLS_LIST_ORDER_SPEC_REFS,
|
||||
source: { introducedIn: '2026-07-28' as const },
|
||||
timestamp
|
||||
};
|
||||
const untestable = (reason: string): ConformanceCheck => ({
|
||||
...baseCheck,
|
||||
status: 'WARNING',
|
||||
errorMessage: notTestable(reason),
|
||||
details: { untestable: true, reason, probes: snapshots.length }
|
||||
});
|
||||
|
||||
if (snapshots.length < 2) {
|
||||
return untestable(
|
||||
`needs at least two tools/list snapshots to compare, got ${snapshots.length}`
|
||||
);
|
||||
}
|
||||
const missing = snapshots.findIndex((tools) => !Array.isArray(tools));
|
||||
if (missing !== -1) {
|
||||
return untestable(
|
||||
`tools/list probe ${missing + 1} did not return a tools array`
|
||||
);
|
||||
}
|
||||
|
||||
// A position-independent placeholder, so that a nameless tool moving
|
||||
// around reads as an order change rather than as a set change.
|
||||
const orders = snapshots.map((tools) =>
|
||||
(tools as ReadonlyArray<{ name?: unknown }>).map((tool) =>
|
||||
typeof tool.name === 'string' ? tool.name : '<tool missing name>'
|
||||
)
|
||||
);
|
||||
|
||||
// Nothing to order when no probe saw two tools.
|
||||
if (orders.every((order) => order.length < 2)) {
|
||||
return {
|
||||
...baseCheck,
|
||||
status: 'INFO',
|
||||
errorMessage: `${orders[0].length} tool(s) advertised; nothing to compare`,
|
||||
details: { toolCount: orders[0].length, probes: orders.length, orders }
|
||||
};
|
||||
}
|
||||
|
||||
// The SHOULD only binds while the set is unchanged: compare multisets first.
|
||||
const countNames = (names: string[]): Map<string, number> => {
|
||||
const counts = new Map<string, number>();
|
||||
for (const name of names) counts.set(name, (counts.get(name) ?? 0) + 1);
|
||||
return counts;
|
||||
};
|
||||
const baseline = countNames(orders[0]);
|
||||
for (let probe = 1; probe < orders.length; probe++) {
|
||||
const current = countNames(orders[probe]);
|
||||
const added: string[] = [];
|
||||
const removed: string[] = [];
|
||||
for (const [name, count] of current) {
|
||||
const before = baseline.get(name) ?? 0;
|
||||
if (count > before) added.push(name);
|
||||
}
|
||||
for (const [name, count] of baseline) {
|
||||
const now = current.get(name) ?? 0;
|
||||
if (count > now) removed.push(name);
|
||||
}
|
||||
if (added.length > 0 || removed.length > 0) {
|
||||
return untestable(
|
||||
`the set of tools changed between tools/list probe 1 and probe ${probe + 1} ` +
|
||||
`(added: ${added.join(', ') || 'none'}; removed: ${removed.join(', ') || 'none'}), ` +
|
||||
'so the deterministic-order SHOULD does not apply to this sample'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const toolCount = orders[0].length;
|
||||
|
||||
for (let probe = 1; probe < orders.length; probe++) {
|
||||
const index = orders[probe].findIndex((name, i) => name !== orders[0][i]);
|
||||
if (index !== -1) {
|
||||
return {
|
||||
...baseCheck,
|
||||
status: 'WARNING',
|
||||
errorMessage:
|
||||
`tools/list returned the same ${toolCount} tools in a different order across ` +
|
||||
`consecutive requests: probe ${probe + 1} diverges from probe 1 at index ${index} ` +
|
||||
`(${orders[0][index]} vs ${orders[probe][index]})`,
|
||||
details: { toolCount, probes: orders.length, orders }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...baseCheck,
|
||||
status: 'SUCCESS',
|
||||
details: { toolCount, probes: orders.length, orders }
|
||||
};
|
||||
}
|
||||
|
||||
export class ToolsListScenario implements ClientScenario {
|
||||
name = 'tools-list';
|
||||
readonly source = { introducedIn: '2025-06-18' } as const;
|
||||
|
|
@ -135,7 +261,9 @@ export class ToolsListScenario implements ClientScenario {
|
|||
- \`name\` (string)
|
||||
- \`description\` (string)
|
||||
- \`inputSchema\` (valid JSON Schema object)
|
||||
- From 2025-11-25 onward, advertised \`name\` values SHOULD follow the spec Tool Names rules (1–128 chars, \`[A-Za-z0-9_.-]\` only) — see \`tools-name-format\` check`;
|
||||
- From 2025-11-25 onward, advertised \`name\` values SHOULD follow the spec Tool Names rules (1–128 chars, \`[A-Za-z0-9_.-]\` only) — see \`tools-name-format\` check
|
||||
- From 2026-07-28: return tools in a deterministic order across requests
|
||||
when the set of tools has not changed (SHOULD)`;
|
||||
|
||||
async run(ctx: RunContext): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -186,6 +314,38 @@ export class ToolsListScenario implements ClientScenario {
|
|||
checks.push(buildToolsNameFormatCheck(result.tools));
|
||||
}
|
||||
|
||||
// 2026-07-28: tools SHOULD come back in a deterministic order across
|
||||
// requests. Take two more consecutive tools/list snapshots and compare.
|
||||
if (specVersionAtLeast(ctx.specVersion, '2026-07-28')) {
|
||||
const snapshots: Array<ListToolsResult['tools'] | undefined> = [
|
||||
result.tools
|
||||
];
|
||||
let probeError: unknown;
|
||||
try {
|
||||
while (snapshots.length < TOOLS_LIST_ORDER_PROBES) {
|
||||
const again = await conn.request<ListToolsResult>('tools/list');
|
||||
snapshots.push(again.tools);
|
||||
}
|
||||
} catch (error) {
|
||||
probeError = error;
|
||||
}
|
||||
checks.push(
|
||||
probeError === undefined
|
||||
? buildToolsListDeterministicOrderCheck(snapshots)
|
||||
: {
|
||||
...untestableCheck(
|
||||
'tools-list-deterministic-order',
|
||||
'ToolsListDeterministicOrder',
|
||||
'Consecutive tools/list requests return the same tools in the same order',
|
||||
`repeated tools/list request failed: ${probeError instanceof Error ? probeError.message : String(probeError)}`,
|
||||
TOOLS_LIST_ORDER_SPEC_REFS,
|
||||
'WARNING'
|
||||
),
|
||||
source: { introducedIn: '2026-07-28' }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
await conn.close();
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
|
|
|
|||
Loading…
Reference in a new issue