Conformance Tests for SEP-2322 MRTR (#188)

This commit is contained in:
Caitie McCaffrey 2026-05-22 14:26:11 -07:00 • committed by GitHub
commit 43fbf60d3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 3507 additions and 3 deletions

View file

@ -0,0 +1,41 @@
/**
* Integration test for MRTR client conformance scenario (SEP-2322).
*
* Runs the everything-client's MRTR handler in-process against the scenario server
* and verifies all checks pass.
*/
import { describe, test, expect } from 'vitest';
import {
runClientAgainstScenario,
InlineClientRunner
} from './auth/test_helpers/testClient';
import { getHandler } from '../../../examples/clients/typescript/everything-client';
import { getScenario } from '../index';
describe('MRTR client scenario (SEP-2322)', () => {
test('everything-client passes sep-2322-client-request-state scenario', async () => {
const clientFn = getHandler('sep-2322-client-request-state');
if (!clientFn) {
throw new Error(
'No handler registered for scenario: sep-2322-client-request-state'
);
}
const scenario = getScenario('sep-2322-client-request-state');
if (!scenario) {
throw new Error('Scenario not found: sep-2322-client-request-state');
}
const runner = new InlineClientRunner(clientFn);
await runClientAgainstScenario(runner, 'sep-2322-client-request-state');
const checks = scenario.getChecks();
for (const check of checks) {
expect(
check.status,
`Check "${check.id}" failed: ${check.errorMessage ?? ''}`
).toBe('SUCCESS');
}
});
});

View file

@ -0,0 +1,486 @@
/**
* SEP-2322: MRTR Client Conformance Tests
*
* Tests that clients correctly handle the MRTR (Multi-Round Tool Resolution) flow:
* - Echo requestState back unchanged when retrying
* - Don't include requestState when server didn't send one
* - Use a different JSON-RPC id on retry
*
* The server exposes two tools. The client calls each tool, gets InputRequiredResult,
* fulfills the elicitation, and retries. The server verifies correct client behavior.
*/
import type { Scenario, ConformanceCheck } from '../../types';
import { DRAFT_PROTOCOL_VERSION, ScenarioUrls } from '../../types';
import express, { Request, Response } from 'express';
import { randomUUID } from 'crypto';
const MRTR_SPEC_REFERENCES = [
{
id: 'SEP-2322-MRTR',
url: 'https://modelcontextprotocol.io/specification/draft/basic/utilities/mrtr'
}
];
const TOOLS = [
{
name: 'test_mrtr_echo_state',
description:
'Test tool: triggers MRTR flow with requestState. Client must echo state back unchanged.',
inputSchema: {
type: 'object' as const,
properties: {},
required: [] as string[]
}
},
{
name: 'test_mrtr_no_state',
description:
'Test tool: triggers MRTR flow WITHOUT requestState. Client must NOT include requestState in retry.',
inputSchema: {
type: 'object' as const,
properties: {},
required: [] as string[]
}
},
{
name: 'test_mrtr_unrelated',
description:
'Test tool: simple tool called between MRTR rounds. Must NOT carry inputResponses or requestState from another tool.',
inputSchema: {
type: 'object' as const,
properties: {},
required: [] as string[]
}
},
{
name: 'test_mrtr_no_result_type',
description:
'Test tool: returns a result without resultType. Client must treat it as complete (default).',
inputSchema: {
type: 'object' as const,
properties: {},
required: [] as string[]
}
}
];
interface JsonRpcRequest {
jsonrpc: '2.0';
id: string | number;
method: string;
params?: Record<string, unknown>;
}
function createMRTRServer(checks: ConformanceCheck[]): express.Application {
const app = express();
app.use(express.json());
// Track original JSON-RPC ids per tool to verify they change on retry
const originalIds = new Map<string, string | number>();
// Track the exact requestState string sent, to verify byte-exact echo on retry
const sentStates = new Map<string, string>();
app.post('/mcp', (req: Request, res: Response) => {
const body = req.body as JsonRpcRequest;
const { id, method, params } = body;
switch (method) {
case 'notifications/initialized': {
res.status(204).end();
return;
}
case 'tools/list': {
res.json({
jsonrpc: '2.0',
id,
result: { tools: TOOLS }
});
return;
}
case 'tools/call': {
const toolName = (params as Record<string, unknown>)?.name as string;
const inputResponses = (params as Record<string, unknown>)
?.inputResponses as Record<string, unknown> | undefined;
const requestState = (params as Record<string, unknown>)
?.requestState as string | undefined;
if (toolName === 'test_mrtr_echo_state') {
handleEchoState(id, inputResponses, requestState, checks, res);
return;
}
if (toolName === 'test_mrtr_no_state') {
handleNoState(id, inputResponses, requestState, checks, res);
return;
}
if (toolName === 'test_mrtr_unrelated') {
handleUnrelated(inputResponses, requestState, checks, res, id);
return;
}
if (toolName === 'test_mrtr_no_result_type') {
handleNoResultType(id, inputResponses, checks, res);
return;
}
res.json({
jsonrpc: '2.0',
id,
error: { code: -32601, message: `Unknown tool: ${toolName}` }
});
return;
}
case 'elicitation/create': {
// Client is fulfilling our ElicitRequest — accept it
res.json({
jsonrpc: '2.0',
id,
result: { action: 'accept', content: { confirmed: true } }
});
return;
}
default: {
res.json({
jsonrpc: '2.0',
id,
error: { code: -32601, message: `Method not found: ${method}` }
});
return;
}
}
});
function handleEchoState(
id: string | number,
inputResponses: Record<string, unknown> | undefined,
requestState: string | undefined,
checks: ConformanceCheck[],
res: Response
) {
if (!inputResponses) {
// Initial call — store original id, return InputRequiredResult with requestState
originalIds.set('echo_state', id);
const state = JSON.stringify({
nonce: randomUUID(),
originalId: id
});
sentStates.set('echo_state', state);
res.json({
jsonrpc: '2.0',
id,
result: {
resultType: 'input_required',
inputRequests: {
confirm: {
method: 'elicitation/create',
params: {
message: 'Please confirm to continue',
requestedSchema: {
type: 'object',
properties: {
confirmed: { type: 'boolean', description: 'Confirm?' }
}
}
}
}
},
requestState: state
}
});
return;
}
// Retry — verify requestState was echoed back correctly
const originalId = originalIds.get('echo_state');
const sentState = sentStates.get('echo_state');
// Check 1: requestState must be present and byte-for-byte identical to
// what the server sent. The spec requires the client to echo back the
// *exact value* without inspecting, parsing, or modifying it — so a
// semantically-equal but re-serialized string is still a failure.
const stateErrors: string[] = [];
if (!requestState) {
stateErrors.push('Client did not include requestState in retry');
} else if (requestState !== sentState) {
stateErrors.push(
'requestState was not echoed back exactly — clients MUST NOT inspect, parse, or modify it'
);
}
checks.push({
id: 'sep-2322-client-request-state-echoed',
name: 'MRTRClientRequestStateEchoed',
description:
'Client MUST echo back the exact value of requestState when retrying',
status: stateErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: stateErrors.length > 0 ? stateErrors.join('; ') : undefined,
specReferences: MRTR_SPEC_REFERENCES,
details: {
requestStateSent: sentState,
requestStateReceived: requestState,
originalId
}
});
// Check 2: JSON-RPC id must differ from original
const idErrors: string[] = [];
if (id === originalId) {
idErrors.push(
`JSON-RPC id is the same on retry (${id}) — MUST be different`
);
}
checks.push({
id: 'sep-2322-client-jsonrpc-id-different',
name: 'MRTRClientJsonRpcIdDifferent',
description:
'The JSON-RPC id MUST be different between the initial request and the retry',
status: idErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: idErrors.length > 0 ? idErrors.join('; ') : undefined,
specReferences: MRTR_SPEC_REFERENCES,
details: {
originalId,
retryId: id
}
});
// Return complete result
res.json({
jsonrpc: '2.0',
id,
result: {
content: [{ type: 'text', text: 'echo-state-ok' }]
}
});
}
function handleNoState(
id: string | number,
inputResponses: Record<string, unknown> | undefined,
requestState: string | undefined,
checks: ConformanceCheck[],
res: Response
) {
if (!inputResponses) {
// Initial call — return InputRequiredResult WITHOUT requestState
res.json({
jsonrpc: '2.0',
id,
result: {
resultType: 'input_required',
inputRequests: {
confirm: {
method: 'elicitation/create',
params: {
message: 'Please confirm to continue (no state test)',
requestedSchema: {
type: 'object',
properties: {
confirmed: { type: 'boolean', description: 'Confirm?' }
}
}
}
}
}
// No requestState field!
}
});
return;
}
// Retry — verify client did NOT include requestState
const errors: string[] = [];
if (requestState !== undefined) {
errors.push(
`Client included requestState ("${requestState}") but server did not send one — MUST NOT include it`
);
}
checks.push({
id: 'sep-2322-client-no-state-omitted',
name: 'MRTRClientNoStateOmitted',
description:
'If InputRequiredResult does not contain requestState, client MUST NOT include one in the retry',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: MRTR_SPEC_REFERENCES,
details: {
requestStateReceived: requestState
}
});
// Return complete result
res.json({
jsonrpc: '2.0',
id,
result: {
content: [{ type: 'text', text: 'no-state-ok' }]
}
});
}
function handleUnrelated(
inputResponses: Record<string, unknown> | undefined,
requestState: string | undefined,
checks: ConformanceCheck[],
res: Response,
id: string | number
) {
// This tool should NEVER receive inputResponses or requestState —
// those belong to a different tool's MRTR flow
const errors: string[] = [];
if (inputResponses !== undefined) {
errors.push(
`Unrelated tool call included inputResponses from another tool's MRTR flow`
);
}
if (requestState !== undefined) {
errors.push(
`Unrelated tool call included requestState from another tool's MRTR flow`
);
}
checks.push({
id: 'sep-2322-client-parallel-isolation',
name: 'MRTRClientParallelIsolation',
description:
'inputRequests and requestState MUST NOT be used for any other request the client may be sending',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: MRTR_SPEC_REFERENCES,
details: {
inputResponsesReceived: inputResponses,
requestStateReceived: requestState
}
});
// Return a normal complete result
res.json({
jsonrpc: '2.0',
id,
result: {
content: [{ type: 'text', text: 'unrelated-ok' }]
}
});
}
function handleNoResultType(
id: string | number,
inputResponses: Record<string, unknown> | undefined,
checks: ConformanceCheck[],
res: Response
) {
const checkId = 'sep-2322-default-result-type-complete';
// If the client retries this tool, it means it did NOT treat the result as complete
if (inputResponses) {
// Remove any prior SUCCESS for this check (emitted when we first sent the response)
const existingIdx = checks.findIndex((c) => c.id === checkId);
if (existingIdx !== -1) checks.splice(existingIdx, 1);
checks.push({
id: checkId,
name: 'DefaultResultTypeComplete',
description:
'Client MUST assume resultType "complete" when not specified',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
'Client retried with inputResponses even though the result had no resultType (should default to complete)',
specReferences: MRTR_SPEC_REFERENCES
});
res.json({
jsonrpc: '2.0',
id,
result: { content: [{ type: 'text', text: 'unexpected-retry' }] }
});
return;
}
// Return a result WITHOUT resultType — client should treat as complete
checks.push({
id: checkId,
name: 'DefaultResultTypeComplete',
description:
'Client MUST assume resultType "complete" when not specified',
status: 'SUCCESS',
timestamp: new Date().toISOString(),
specReferences: MRTR_SPEC_REFERENCES
});
res.json({
jsonrpc: '2.0',
id,
result: {
// Deliberately NO resultType field
content: [{ type: 'text', text: 'no-result-type-test-ok' }]
}
});
}
return app;
}
export class MRTRClientScenario implements Scenario {
name = 'sep-2322-client-request-state';
readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const;
description =
'Tests client MRTR behavior: requestState echo, no-state omission, and JSON-RPC id uniqueness (SEP-2322)';
private app: express.Application | null = null;
private httpServer: ReturnType<express.Application['listen']> | null = null;
private checks: ConformanceCheck[] = [];
async start(): Promise<ScenarioUrls> {
this.checks = [];
this.app = createMRTRServer(this.checks);
this.httpServer = this.app.listen(0);
const addr = this.httpServer.address();
const port = typeof addr === 'object' && addr ? addr.port : 0;
return { serverUrl: `http://localhost:${port}/mcp` };
}
async stop() {
if (this.httpServer) {
await new Promise((resolve) => this.httpServer!.close(resolve));
this.httpServer = null;
}
this.app = null;
}
getChecks(): ConformanceCheck[] {
const expectedSlugs = [
'sep-2322-client-request-state-echoed',
'sep-2322-client-jsonrpc-id-different',
'sep-2322-client-no-state-omitted',
'sep-2322-client-parallel-isolation',
'sep-2322-default-result-type-complete'
];
for (const slug of expectedSlugs) {
if (!this.checks.find((c) => c.id === slug)) {
this.checks.push({
id: slug,
name: slug,
description: `MRTR client check: ${slug}`,
status: 'FAILURE',
timestamp: new Date().toISOString(),
details: {
message: 'Tool was not called by client or MRTR flow not completed'
},
specReferences: MRTR_SPEC_REFERENCES
});
}
}
return this.checks;
}
}

View file

@ -14,6 +14,7 @@ import { ToolsCallScenario } from './client/tools_call';
import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults';
import { SSERetryScenario } from './client/sse-retry';
import { RequestMetadataScenario } from './client/request-metadata';
import { MRTRClientScenario } from './client/mrtr-client';
// Import all new server test scenarios
import { ServerInitializeScenario } from './server/lifecycle';
@ -67,6 +68,24 @@ import {
import { DNSRebindingProtectionScenario } from './server/dns-rebinding';
import { CachingScenario } from './server/caching';
// InputRequiredResult scenarios from (SEP-2322)
import {
InputRequiredResultBasicElicitationScenario,
InputRequiredResultBasicSamplingScenario,
InputRequiredResultBasicListRootsScenario,
InputRequiredResultRequestStateScenario,
InputRequiredResultMultipleInputRequestsScenario,
InputRequiredResultMultiRoundScenario,
InputRequiredResultMissingInputResponseScenario,
InputRequiredResultNonToolRequestScenario,
InputRequiredResultResultTypeScenario,
InputRequiredResultUnsupportedMethodsScenario,
InputRequiredResultTamperedStateScenario,
InputRequiredResultCapabilityCheckScenario,
InputRequiredResultIgnoreExtraParamsScenario,
InputRequiredResultValidateInputScenario
} from './server/input-required-result';
import {
HttpHeaderValidationScenario,
HttpCustomHeaderServerValidationScenario
@ -167,7 +186,23 @@ const allClientScenariosList: ClientScenario[] = [
new CachingScenario(),
// HTTP Standardization scenarios (SEP-2243)
new HttpHeaderValidationScenario(),
new HttpCustomHeaderServerValidationScenario()
new HttpCustomHeaderServerValidationScenario(),
// InputRequiredResult scenarios (SEP-2322)
new InputRequiredResultBasicElicitationScenario(),
new InputRequiredResultBasicSamplingScenario(),
new InputRequiredResultBasicListRootsScenario(),
new InputRequiredResultRequestStateScenario(),
new InputRequiredResultMultipleInputRequestsScenario(),
new InputRequiredResultMultiRoundScenario(),
new InputRequiredResultMissingInputResponseScenario(),
new InputRequiredResultNonToolRequestScenario(),
new InputRequiredResultResultTypeScenario(),
new InputRequiredResultUnsupportedMethodsScenario(),
new InputRequiredResultTamperedStateScenario(),
new InputRequiredResultCapabilityCheckScenario(),
new InputRequiredResultIgnoreExtraParamsScenario(),
new InputRequiredResultValidateInputScenario()
];
// Active client scenarios (excludes pending)
@ -213,6 +248,9 @@ const scenariosList: Scenario[] = [
...draftScenariosList,
...extensionScenariosList,
// MRTR client conformance (SEP-2322)
new MRTRClientScenario(),
// HTTP Standardization scenarios (SEP-2243)
new HttpStandardHeadersScenario(),
new HttpCustomHeadersScenario(),

View file

@ -0,0 +1,166 @@
/**
* Helpers for SEP-2322 conformance tests.
*
* Provides InputRequiredResult-specific type guards, mock response builders,
* and a stateless JSON-RPC transport helper.
*/
import { DRAFT_PROTOCOL_VERSION } from '../../types';
// ─── JSON-RPC Types ──────────────────────────────────────────────────────────
export interface JsonRpcResponse {
jsonrpc: '2.0';
id: number;
result?: Record<string, unknown>;
error?: { code: number; message: string; data?: unknown };
}
// ─── Stateless RPC Helper ────────────────────────────────────────────────────
let nextId = 1;
/**
* Send a stateless JSON-RPC request (SEP-2575 pattern).
* Automatically injects _meta with protocolVersion, clientInfo, clientCapabilities.
*/
export async function sendRpc(
serverUrl: string,
method: string,
params?: Record<string, unknown>
): Promise<JsonRpcResponse> {
const id = nextId++;
const enrichedParams = {
...params,
_meta: {
'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION,
'io.modelcontextprotocol/clientInfo': {
name: 'conformance-test-client',
version: '1.0.0'
},
'io.modelcontextprotocol/clientCapabilities': {
sampling: {},
elicitation: {},
roots: { listChanged: true }
},
...(params?._meta as Record<string, unknown> | undefined)
}
};
const response = await fetch(serverUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
'MCP-Protocol-Version': DRAFT_PROTOCOL_VERSION
},
body: JSON.stringify({ jsonrpc: '2.0', id, method, params: enrichedParams })
});
return (await response.json()) as JsonRpcResponse;
}
// ─── InputRequiredResult Types ───────────────────────────────────────────────
export interface InputRequiredResultData {
resultType?: 'input_required';
inputRequests?: Record<string, InputRequestObject>;
requestState?: string;
_meta?: Record<string, unknown>;
[key: string]: unknown;
}
export interface InputRequestObject {
method: string;
params?: Record<string, unknown>;
}
// ─── Type Guards ─────────────────────────────────────────────────────────────
/**
* Check if a JSON-RPC result is an InputRequiredResult.
*/
export function isInputRequiredResult(
result: Record<string, unknown> | undefined
): result is InputRequiredResultData {
if (!result) return false;
if (result.resultType === 'input_required') return true;
return false;
}
/**
* Check if a JSON-RPC result is a complete result (not input_required).
* complete is the default so if resultType is missing we assume it's complete.
*/
export function isCompleteResult(
result: Record<string, unknown> | undefined
): boolean {
if (!result) return false;
if (result.resultType === 'input_required') return false;
return true;
}
/**
* Extract inputRequests from an InputRequiredResult.
*/
export function getInputRequests(
result: InputRequiredResultData
): Record<string, InputRequestObject> | undefined {
return result.inputRequests;
}
// ─── Mock Response Builders ──────────────────────────────────────────────────
/**
* Build a mock elicitation response (ElicitResult).
*/
export function mockElicitResponse(
content: Record<string, unknown>
): Record<string, unknown> {
return {
action: 'accept',
content
};
}
/**
* Build a mock sampling response (CreateMessageResult).
*/
export function mockSamplingResponse(text: string): Record<string, unknown> {
return {
role: 'assistant',
content: {
type: 'text',
text
},
model: 'test-model',
stopReason: 'endTurn'
};
}
/**
* Build a mock list roots response (ListRootsResult).
*/
export function mockListRootsResponse(): Record<string, unknown> {
return {
roots: [
{
uri: 'file:///test/root',
name: 'Test Root'
}
]
};
}
// ─── Spec References ─────────────────────────────────────────────────────────
/**
* SEP reference for InputRequiredResult / MRTR tests.
*/
export const MRTR_SPEC_REFERENCES = [
{
id: 'SEP-2322',
url: 'https://modelcontextprotocol.io/specification/draft/basic/utilities/mrtr'
}
];

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,125 @@
/**
* SEP-2322 MRTR negative tests.
*
* Positive tests run via all-scenarios.test.ts against the everything-server
* (which implements MRTR in its stateless path). These negative tests run
* against a deliberately broken server to verify checks emit FAILURE.
*/
import { spawn, ChildProcess } from 'child_process';
import { createServer } from 'net';
import path from 'path';
import {
InputRequiredResultResultTypeScenario,
InputRequiredResultUnsupportedMethodsScenario,
InputRequiredResultTamperedStateScenario
} from './input-required-result';
function getFreePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.listen(0, () => {
const port = (server.address() as { port: number }).port;
server.close(() => resolve(port));
});
server.on('error', reject);
});
}
function startServer(scriptPath: string, port: number): Promise<ChildProcess> {
return new Promise((resolve, reject) => {
const isWindows = process.platform === 'win32';
const proc = spawn('npx', ['tsx', scriptPath], {
env: { ...process.env, PORT: port.toString() },
stdio: ['ignore', 'pipe', 'pipe'],
shell: isWindows
});
let stderr = '';
proc.stderr?.on('data', (d) => (stderr += d.toString()));
const timeout = setTimeout(() => {
proc.kill('SIGKILL');
reject(
new Error(`Server ${scriptPath} failed to start within 30s: ${stderr}`)
);
}, 30000);
proc.stdout?.on('data', (data) => {
if (data.toString().includes('running on')) {
clearTimeout(timeout);
resolve(proc);
}
});
proc.on('error', (err) => {
clearTimeout(timeout);
reject(err);
});
});
}
function stopServer(proc: ChildProcess | null): Promise<void> {
return new Promise((resolve) => {
if (!proc || proc.killed) return resolve();
const t = setTimeout(() => {
proc.kill('SIGKILL');
resolve();
}, 5000);
proc.once('exit', () => {
clearTimeout(t);
resolve();
});
proc.kill('SIGTERM');
});
}
describe('SEP-2322 MRTR negative tests', () => {
let serverProcess: ChildProcess | null = null;
let SERVER_URL: string;
beforeAll(async () => {
const port = await getFreePort();
SERVER_URL = `http://localhost:${port}/mcp`;
serverProcess = await startServer(
path.join(
process.cwd(),
'examples/servers/typescript/sep-2322-mrtr-broken-server.ts'
),
port
);
}, 35000);
afterAll(async () => {
await stopServer(serverProcess);
});
it('emits FAILURE for sep-2322-result-type-included against server that omits resultType', async () => {
const scenario = new InputRequiredResultResultTypeScenario();
const checks = await scenario.run(SERVER_URL);
const resultTypeCheck = checks.find(
(c) => c.id === 'sep-2322-result-type-included'
);
expect(resultTypeCheck).toBeDefined();
expect(resultTypeCheck?.status).toBe('FAILURE');
}, 10000);
it('emits FAILURE for sep-2322-not-on-unsupported-requests against server returning InputRequiredResult on tools/list', async () => {
const scenario = new InputRequiredResultUnsupportedMethodsScenario();
const checks = await scenario.run(SERVER_URL);
const unsupportedCheck = checks.find(
(c) => c.id === 'sep-2322-not-on-unsupported-requests'
);
expect(unsupportedCheck).toBeDefined();
expect(unsupportedCheck?.status).toBe('FAILURE');
}, 10000);
it('emits FAILURE for sep-2322-reject-tampered-state against server that accepts tampered state', async () => {
const scenario = new InputRequiredResultTamperedStateScenario();
const checks = await scenario.run(SERVER_URL);
const tamperedCheck = checks.find(
(c) => c.id === 'sep-2322-reject-tampered-state'
);
expect(tamperedCheck).toBeDefined();
expect(tamperedCheck?.status).toBe('FAILURE');
}, 10000);
});