feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire (#492)

* feat(tools-list): check deterministic tools/list ordering on the 2026-07-28 wire

The 2026-07-28 revision adds to server/tools.mdx: "Servers SHOULD return
tools in a deterministic order (i.e., the same ordering across requests
when the underlying set of tools has not changed)." Nothing in the suite
exercised it.

Add a tools-list-deterministic-order check to the existing tools-list
scenario. From the 2026-07-28 wire on, the scenario takes two more
consecutive tools/list snapshots and compares the three name sequences.
Same multiset of names, different order is WARNING (SHOULD). A set that
changed between probes is reported as untestable rather than as a
violation, because the spec scopes the SHOULD to an unchanged set. Fewer
than two tools on every probe is INFO. The check carries
source.introducedIn '2026-07-28', written as a literal since that revision
is published and dated, so runners drop it on earlier wires.

Prove it fails: examples/servers/typescript/tools-list-rotated-order.ts
advertises the same four tools and rotates the list by one position per
call; a negative.test.ts case asserts WARNING while tools-list itself
still passes. Unit tests cover the helper that builds the check.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BjeC3sCvfHRpdEPz2x41B2

* tools-list: inline the deterministic-order check id and introducedIn literal

Check ids and introducedIn revisions are written inline everywhere else in
src/scenarios (tests pin the literal slug on purpose, since SDK baselines
reference it), so drop the two exported constants.

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>
Co-authored-by: Paul Carleton <paulc@anthropic.com>

examples/servers/typescript/tools-list-rotated-order.ts

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+
+/**
+ * tools/list ordering negative test server.
+ *
+ * Speaks the sessionless 2026-07-28 wire (SEP-2575) and advertises the same
+ * four tools on every tools/list request, but rotates the list by one
+ * position each time (it keeps a call counter for that, nothing else). The set never changes, only the order, which violates the
+ * 2026-07-28 SHOULD "Servers SHOULD return tools in a deterministic order".
+ * The tools-list scenario should emit WARNING for
+ * tools-list-deterministic-order against this server while tools-list itself
+ * still passes, since every response is structurally valid.
+ */
+
+import express from 'express';
+
+const app = express();
+app.use(express.json());
+
+const TOOLS = ['alpha', 'bravo', 'charlie', 'delta'].map((name) => ({
+  name,
+  description: `Fixture tool ${name}`,
+  inputSchema: { type: 'object', properties: {} }
+}));
+
+let listCalls = 0;
+
+app.post('/mcp', (req, res) => {
+  const body = req.body || {};
+  const id = body.id ?? null;
+  const method = body.method;
+
+  switch (method) {
+    case 'server/discover':
+      return res.json({
+        jsonrpc: '2.0',
+        id,
+        result: {
+          resultType: 'complete',
+          ttlMs: 0,
+          cacheScope: 'private',
+          supportedVersions: ['2026-07-28'],
+          capabilities: { tools: {} },
+          serverInfo: { name: 'tools-list-rotated-order', version: '1.0.0' }
+        }
+      });
+    case 'tools/list': {
+      // Rotate by one position per call: the same set, never the same order.
+      const offset = listCalls++ % TOOLS.length;
+      const tools = [...TOOLS.slice(offset), ...TOOLS.slice(0, offset)];
+      return res.json({
+        jsonrpc: '2.0',
+        id,
+        result: {
+          resultType: 'complete',
+          ttlMs: 0,
+          cacheScope: 'private',
+          tools
+        }
+      });
+    }
+    default:
+      return res.status(404).json({
+        jsonrpc: '2.0',
+        id,
+        error: { code: -32601, message: 'Method not found' }
+      });
+  }
+});
+
+const PORT = parseInt(process.env.PORT || '3008', 10);
+app.listen(PORT, '127.0.0.1', () => {
+  console.log(
+    `tools/list rotated-order negative test server running on http://localhost:${PORT}/mcp`
+  );
+});

src/scenarios/server/negative.test.ts

@@ -4,11 +4,11 @@ import path from 'path';
 import { DNSRebindingProtectionScenario } from './dns-rebinding';
 import { ResourcesNotFoundErrorScenario } from './resources';
 import { CachingScenario } from './caching';
+import { ToolsListScenario } from './tools';
 import {
   JsonSchema2020_12Scenario,
   sep2106KeywordCheckStatus
 } from './json-schema-2020-12';
-import { ToolsListScenario } from './tools';
 import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types';
 import { takeWireViolations } from '../../validation/wire-schema';
 
@@ -217,6 +217,43 @@ describe('Server scenario negative tests', () => {
     }, 10000);
   });
 
+  describe('tools-list-deterministic-order', () => {
+    let serverProcess: ChildProcess | null = null;
+    const PORT = 3008;
+
+    beforeAll(async () => {
+      serverProcess = await startServer(
+        path.join(
+          process.cwd(),
+          'examples/servers/typescript/tools-list-rotated-order.ts'
+        ),
+        PORT
+      );
+    }, 35000);
+
+    afterAll(async () => {
+      await stopServer(serverProcess);
+    });
+
+    it('emits WARNING for deterministic-order while tools-list still passes against a server that rotates its tool list', async () => {
+      const scenario = new ToolsListScenario();
+      const checks = await scenario.run(
+        testContext(`http://localhost:${PORT}/mcp`, DRAFT_PROTOCOL_VERSION)
+      );
+
+      const list = checks.find((c) => c.id === 'tools-list');
+      expect(list?.status).toBe('SUCCESS');
+
+      const order = checks.find(
+        (c) => c.id === 'tools-list-deterministic-order'
+      );
+      expect(order?.status).toBe('WARNING');
+      expect(order?.errorMessage).toMatch(/different order/);
+      expect(order?.details).toMatchObject({ toolCount: 4, probes: 3 });
+      expect(order?.details?.untestable).toBeUndefined();
+    }, 10000);
+  });
+
   describe('tools-name-format', () => {
     let serverProcess: ChildProcess | null = null;
     const PORT = 3009;

src/scenarios/server/tools.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from 'vitest';
 import {
+  buildToolsListDeterministicOrderCheck,
   buildToolsNameFormatCheck,
   toolNameFormatCheckApplies,
   ToolsListScenario,
@@ -193,3 +194,143 @@ describe('ToolsListScenario version gate', () => {
     );
   });
 });
+
+describe('buildToolsListDeterministicOrderCheck', () => {
+  const names = (...ns: string[]) => ns.map((name) => ({ name }));
+
+  it('returns SUCCESS when every probe lists the same tools in the same order', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b', 'c'),
+      names('a', 'b', 'c'),
+      names('a', 'b', 'c')
+    ]);
+    expect(check.id).toBe('tools-list-deterministic-order');
+    expect(check.status).toBe('SUCCESS');
+    expect(check.errorMessage).toBeUndefined();
+    expect(check.details).toEqual({
+      toolCount: 3,
+      probes: 3,
+      orders: [
+        ['a', 'b', 'c'],
+        ['a', 'b', 'c'],
+        ['a', 'b', 'c']
+      ]
+    });
+    expect(check.specReferences?.[0]?.url).toContain('2026-07-28/server/tools');
+  });
+
+  it('returns WARNING when the same tools come back in a different order', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b', 'c'),
+      names('b', 'c', 'a'),
+      names('c', 'a', 'b')
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/different order/);
+    expect(check.errorMessage).toMatch(/index 0/);
+    expect(check.details).toMatchObject({ toolCount: 3, probes: 3 });
+    expect(check.details?.untestable).toBeUndefined();
+    expect(check.details?.orders).toEqual([
+      ['a', 'b', 'c'],
+      ['b', 'c', 'a'],
+      ['c', 'a', 'b']
+    ]);
+  });
+
+  it('flags a divergence that only appears on the last probe', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b', 'c'),
+      names('a', 'b', 'c'),
+      names('a', 'c', 'b')
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/probe 3/);
+    expect(check.errorMessage).toMatch(/index 1/);
+  });
+
+  it('reports untestable (WARNING) when the set of tools changed between probes', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b', 'c'),
+      names('a', 'b', 'd')
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/^Not testable: /);
+    expect(check.errorMessage).toMatch(/added: d/);
+    expect(check.errorMessage).toMatch(/removed: c/);
+    expect(check.details).toMatchObject({ untestable: true });
+  });
+
+  it('reports untestable (WARNING) when a probe returned no tools array', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b'),
+      undefined
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/^Not testable: /);
+    expect(check.details).toMatchObject({ untestable: true });
+  });
+
+  it('reports untestable (WARNING) with fewer than two probes', () => {
+    const check = buildToolsListDeterministicOrderCheck([names('a', 'b')]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/^Not testable: /);
+  });
+
+  it('returns INFO when no probe saw two tools to order', () => {
+    const one = buildToolsListDeterministicOrderCheck([names('a'), names('a')]);
+    expect(one.status).toBe('INFO');
+    expect(one.errorMessage).toMatch(/nothing to compare/);
+    expect(one.details).toEqual({
+      toolCount: 1,
+      probes: 2,
+      orders: [['a'], ['a']]
+    });
+    const none = buildToolsListDeterministicOrderCheck([[], []]);
+    expect(none.status).toBe('INFO');
+  });
+
+  it('reports untestable when a probe grows from one tool to two', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a'),
+      names('a', 'b')
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/^Not testable: /);
+    expect(check.errorMessage).toMatch(/added: b/);
+  });
+
+  it('treats a tool without a string name as one placeholder entry', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      [{ name: 'a' }, { name: 42 }],
+      [{ name: 42 }, { name: 'a' }]
+    ]);
+    expect(check.status).toBe('WARNING');
+    expect(check.errorMessage).toMatch(/different order/);
+    expect(check.details?.orders).toEqual([
+      ['a', '<tool missing name>'],
+      ['<tool missing name>', 'a']
+    ]);
+  });
+
+  it('treats duplicate names as a set change only when their multiplicity changes', () => {
+    const stable = buildToolsListDeterministicOrderCheck([
+      names('a', 'a', 'b'),
+      names('a', 'a', 'b')
+    ]);
+    expect(stable.status).toBe('SUCCESS');
+    const changed = buildToolsListDeterministicOrderCheck([
+      names('a', 'a', 'b'),
+      names('a', 'b', 'b')
+    ]);
+    expect(changed.status).toBe('WARNING');
+    expect(changed.errorMessage).toMatch(/^Not testable: /);
+  });
+
+  it('gates itself to the 2026-07-28 wire', () => {
+    const check = buildToolsListDeterministicOrderCheck([
+      names('a', 'b'),
+      names('a', 'b')
+    ]);
+    expect(check.source).toEqual({ introducedIn: '2026-07-28' });
+  });
+});

src/scenarios/server/tools.ts

@@ -10,6 +10,7 @@ import {
   type SpecVersion
 } from '../../types';
 import type { RunContext } from '../../connection';
+import { notTestable, untestableCheck } from '../untestable';
 import type {
   ListToolsResult,
   CallToolResult
@@ -120,6 +121,131 @@ export function buildToolsNameFormatCheck(
   };
 }
 
+/** How many consecutive tools/list snapshots the ordering check compares. */
+const TOOLS_LIST_ORDER_PROBES = 3;
+
+const TOOLS_LIST_ORDER_SPEC_REFS = [
+  {
+    id: 'MCP-Tools-Deterministic-Order',
+    url: 'https://modelcontextprotocol.io/specification/2026-07-28/server/tools#capabilities'
+  }
+];
+
+/**
+ * Build the tools-list-deterministic-order check from consecutive tools/list
+ * snapshots.
+ *
+ * 2026-07-28 server/tools.mdx: "Servers SHOULD return tools in a deterministic
+ * order (i.e., the same ordering across requests when the underlying set of
+ * tools has not changed)." SHOULD, so a violation is WARNING.
+ *
+ * The spec scopes the SHOULD to an unchanged set, so a set that differs
+ * between probes is reported as untestable (issue #248) rather than as a
+ * violation: from the outside, a sample cannot tell a nondeterministic server
+ * from one whose tools legitimately changed between two requests.
+ */
+export function buildToolsListDeterministicOrderCheck(
+  snapshots: ReadonlyArray<ReadonlyArray<{ name?: unknown }> | undefined>
+): ConformanceCheck {
+  const timestamp = new Date().toISOString();
+  const baseCheck = {
+    id: 'tools-list-deterministic-order',
+    name: 'ToolsListDeterministicOrder',
+    description:
+      'Consecutive tools/list requests return the same tools in the same order',
+    specReferences: TOOLS_LIST_ORDER_SPEC_REFS,
+    source: { introducedIn: '2026-07-28' as const },
+    timestamp
+  };
+  const untestable = (reason: string): ConformanceCheck => ({
+    ...baseCheck,
+    status: 'WARNING',
+    errorMessage: notTestable(reason),
+    details: { untestable: true, reason, probes: snapshots.length }
+  });
+
+  if (snapshots.length < 2) {
+    return untestable(
+      `needs at least two tools/list snapshots to compare, got ${snapshots.length}`
+    );
+  }
+  const missing = snapshots.findIndex((tools) => !Array.isArray(tools));
+  if (missing !== -1) {
+    return untestable(
+      `tools/list probe ${missing + 1} did not return a tools array`
+    );
+  }
+
+  // A position-independent placeholder, so that a nameless tool moving
+  // around reads as an order change rather than as a set change.
+  const orders = snapshots.map((tools) =>
+    (tools as ReadonlyArray<{ name?: unknown }>).map((tool) =>
+      typeof tool.name === 'string' ? tool.name : '<tool missing name>'
+    )
+  );
+
+  // Nothing to order when no probe saw two tools.
+  if (orders.every((order) => order.length < 2)) {
+    return {
+      ...baseCheck,
+      status: 'INFO',
+      errorMessage: `${orders[0].length} tool(s) advertised; nothing to compare`,
+      details: { toolCount: orders[0].length, probes: orders.length, orders }
+    };
+  }
+
+  // The SHOULD only binds while the set is unchanged: compare multisets first.
+  const countNames = (names: string[]): Map<string, number> => {
+    const counts = new Map<string, number>();
+    for (const name of names) counts.set(name, (counts.get(name) ?? 0) + 1);
+    return counts;
+  };
+  const baseline = countNames(orders[0]);
+  for (let probe = 1; probe < orders.length; probe++) {
+    const current = countNames(orders[probe]);
+    const added: string[] = [];
+    const removed: string[] = [];
+    for (const [name, count] of current) {
+      const before = baseline.get(name) ?? 0;
+      if (count > before) added.push(name);
+    }
+    for (const [name, count] of baseline) {
+      const now = current.get(name) ?? 0;
+      if (count > now) removed.push(name);
+    }
+    if (added.length > 0 || removed.length > 0) {
+      return untestable(
+        `the set of tools changed between tools/list probe 1 and probe ${probe + 1} ` +
+          `(added: ${added.join(', ') || 'none'}; removed: ${removed.join(', ') || 'none'}), ` +
+          'so the deterministic-order SHOULD does not apply to this sample'
+      );
+    }
+  }
+
+  const toolCount = orders[0].length;
+
+  for (let probe = 1; probe < orders.length; probe++) {
+    const index = orders[probe].findIndex((name, i) => name !== orders[0][i]);
+    if (index !== -1) {
+      return {
+        ...baseCheck,
+        status: 'WARNING',
+        errorMessage:
+          `tools/list returned the same ${toolCount} tools in a different order across ` +
+          `consecutive requests: probe ${probe + 1} diverges from probe 1 at index ${index} ` +
+          `(${orders[0][index]} vs ${orders[probe][index]})`,
+        details: { toolCount, probes: orders.length, orders }
+      };
+    }
+  }
+
+  return {
+    ...baseCheck,
+    status: 'SUCCESS',
+    details: { toolCount, probes: orders.length, orders }
+  };
+}
+
 export class ToolsListScenario implements ClientScenario {
   name = 'tools-list';
   readonly source = { introducedIn: '2025-06-18' } as const;
@@ -135,7 +261,9 @@ export class ToolsListScenario implements ClientScenario {
   - \`name\` (string)
   - \`description\` (string)
   - \`inputSchema\` (valid JSON Schema object)
-- From 2025-11-25 onward, advertised \`name\` values SHOULD follow the spec Tool Names rules (1–128 chars, \`[A-Za-z0-9_.-]\` only) — see \`tools-name-format\` check`;
+- From 2025-11-25 onward, advertised \`name\` values SHOULD follow the spec Tool Names rules (1–128 chars, \`[A-Za-z0-9_.-]\` only) — see \`tools-name-format\` check
+- From 2026-07-28: return tools in a deterministic order across requests
+  when the set of tools has not changed (SHOULD)`;
 
   async run(ctx: RunContext): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
@@ -186,6 +314,38 @@ export class ToolsListScenario implements ClientScenario {
         checks.push(buildToolsNameFormatCheck(result.tools));
       }
 
+      // 2026-07-28: tools SHOULD come back in a deterministic order across
+      // requests. Take two more consecutive tools/list snapshots and compare.
+      if (specVersionAtLeast(ctx.specVersion, '2026-07-28')) {
+        const snapshots: Array<ListToolsResult['tools'] | undefined> = [
+          result.tools
+        ];
+        let probeError: unknown;
+        try {
+          while (snapshots.length < TOOLS_LIST_ORDER_PROBES) {
+            const again = await conn.request<ListToolsResult>('tools/list');
+            snapshots.push(again.tools);
+          }
+        } catch (error) {
+          probeError = error;
+        }
+        checks.push(
+          probeError === undefined
+            ? buildToolsListDeterministicOrderCheck(snapshots)
+            : {
+                ...untestableCheck(
+                  'tools-list-deterministic-order',
+                  'ToolsListDeterministicOrder',
+                  'Consecutive tools/list requests return the same tools in the same order',
+                  `repeated tools/list request failed: ${probeError instanceof Error ? probeError.message : String(probeError)}`,
+                  TOOLS_LIST_ORDER_SPEC_REFS,
+                  'WARNING'
+                ),
+                source: { introducedIn: '2026-07-28' }
+              }
+        );
+      }
+
       await conn.close();
     } catch (error) {
       checks.push({