feat: add client-side JSON Schema 2020-12 preservation scenario (#335)
* feat: add client-side JSON Schema 2020-12 preservation scenario The mock server, using the version-aware `MockServer` abstraction from #321, first advertises a focal tool with a detailed inputSchema and also offers a permissive `json_schema_echo` tool. The client under test retrieves the inputSchema via `tools/list`, then sends the same inputSchema back to the server using the echo tool, completing the round-trip. The scenario diffs from the echoed schema against the canonical fixture. It emits eight per-keyword conformance checks. SEP-2106 vocabulary checks are soft-gated using the existing sep2106KeywordCheckStatus helper. They only flip to FAILURE during DRAFT-2026-v1 runs. Signed-off-by: Satoshi Ito <satoshi.ito.tf@hitachi.com> * Drive draft-target preservation tests through the stateless lifecycle The stateful test helper passed createServerStateful bare, which no longer compiles now that it takes a second specVersion argument, and forcing the stateful mock at the draft version trips wire-schema validation (draft results require the SEP-2549 resultType/ttlMs/cacheScope members). Use testScenarioContext(DRAFT_PROTOCOL_VERSION), which selects the stateless mock server, and drive it with sendStatelessRequest, matching the pattern in json-schema-ref-deref.test.ts. * Reword client keyword-preservation rows as issue-derived expectations SEP-2106 §1 contains no normative preservation sentence, so the six new sep-2106.yaml rows should not attribute a spec MUST to it. Reword the remaining rows as expectations derived from issue #101. Also drop the three rows whose check IDs are not sep-prefixed: the traceability manifest only collects IDs matching /^sep-\d+-/, so those rows could never be marked tested. * Replace stale DRAFT-2026-v1 references with the 2026-07-28 draft name The draft protocol identifier was renamed in #331. Interpolate DRAFT_PROTOCOL_VERSION into the SKIPPED errorMessage suffix (matching the server-side scenario) and update the header comment. --------- Signed-off-by: Satoshi Ito <satoshi.ito.tf@hitachi.com> Co-authored-by: Paul Carleton <paulc@anthropic.com>
This commit is contained in:
Satoshi Ito
GitHub
parent
3531a64803
commit
81eb1c3eda
6 changed files with 812 additions and 38 deletions
|
|
@ -228,6 +228,69 @@ async function runListToolsOnlyClient(serverUrl: string): Promise<void> {
|
|||
|
||||
registerScenario('json-schema-ref-no-deref', runListToolsOnlyClient);
|
||||
|
||||
// ============================================================================
|
||||
// json-schema-2020-12-preservation (SEP-1613, SEP-2106, Issue #101)
|
||||
//
|
||||
// Scenario contract:
|
||||
// 1. tools/list — observe `json_schema_2020_12_tool` and its inputSchema
|
||||
// 2. tools/call json_schema_echo with `{ schema: <observed inputSchema> }`
|
||||
// The scenario diffs the echoed schema against its fixture to detect
|
||||
// client-side keyword stripping; this handler just round-trips the schema
|
||||
// verbatim, which is the compliant behavior.
|
||||
// ============================================================================
|
||||
|
||||
const FOCAL_TOOL_NAME = 'json_schema_2020_12_tool';
|
||||
const ECHO_TOOL_NAME = 'json_schema_echo';
|
||||
|
||||
async function runJsonSchema2020_12PreservationClient(
|
||||
serverUrl: string
|
||||
): Promise<void> {
|
||||
if (USE_STATELESS_LIFECYCLE) {
|
||||
logger.debug(
|
||||
'Stateless lifecycle: listing tools and echoing observed schema'
|
||||
);
|
||||
const list = await statelessRequest(serverUrl, 'tools/list');
|
||||
const focal = list?.tools?.find(
|
||||
(t: { name: string }) => t.name === FOCAL_TOOL_NAME
|
||||
);
|
||||
if (!focal) {
|
||||
throw new Error(`Focal tool '${FOCAL_TOOL_NAME}' not advertised`);
|
||||
}
|
||||
await statelessRequest(serverUrl, 'tools/call', {
|
||||
name: ECHO_TOOL_NAME,
|
||||
arguments: { schema: focal.inputSchema }
|
||||
});
|
||||
logger.debug('Successfully echoed observed inputSchema');
|
||||
return;
|
||||
}
|
||||
|
||||
const client = new Client(
|
||||
{ name: 'test-client', version: '1.0.0' },
|
||||
{ capabilities: {} }
|
||||
);
|
||||
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||
await client.connect(transport);
|
||||
try {
|
||||
const list = await client.listTools();
|
||||
const focal = list.tools.find((t) => t.name === FOCAL_TOOL_NAME);
|
||||
if (!focal) {
|
||||
throw new Error(`Focal tool '${FOCAL_TOOL_NAME}' not advertised`);
|
||||
}
|
||||
await client.callTool({
|
||||
name: ECHO_TOOL_NAME,
|
||||
arguments: { schema: focal.inputSchema as Record<string, unknown> }
|
||||
});
|
||||
logger.debug('Successfully echoed observed inputSchema');
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
|
||||
registerScenario(
|
||||
'json-schema-2020-12-preservation',
|
||||
runJsonSchema2020_12PreservationClient
|
||||
);
|
||||
|
||||
// ============================================================================
|
||||
// request-metadata scenario (SEP-2575)
|
||||
// ============================================================================
|
||||
|
|
|
|||
271
src/scenarios/client/json-schema-2020-12-preservation.test.ts
Normal file
271
src/scenarios/client/json-schema-2020-12-preservation.test.ts
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
import { testScenarioContext } from '../../mock-server/testing';
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
||||
import { JsonSchema2020_12PreservationScenario } from './json-schema-2020-12-preservation';
|
||||
import { sendStatelessRequest } from '../../connection/stateless';
|
||||
import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types';
|
||||
import { JSON_SCHEMA_2020_12_FIXTURE } from '../server/json-schema-2020-12';
|
||||
|
||||
const FOCAL_TOOL = 'json_schema_2020_12_tool';
|
||||
const ECHO_TOOL = 'json_schema_echo';
|
||||
|
||||
/**
|
||||
* Connect a real SDK client to the scenario's mock server, list tools, and
|
||||
* echo back the focal tool's inputSchema (verbatim or after the caller-
|
||||
* supplied transform). Returns nothing — the scenario observes the echo
|
||||
* internally.
|
||||
*/
|
||||
async function runEchoClient(
|
||||
serverUrl: string,
|
||||
transform: (schema: Record<string, unknown>) => Record<string, unknown> = (
|
||||
s
|
||||
) => s
|
||||
): Promise<void> {
|
||||
const client = new Client(
|
||||
{ name: 'preservation-test-client', version: '1.0.0' },
|
||||
{ capabilities: {} }
|
||||
);
|
||||
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
|
||||
await client.connect(transport);
|
||||
try {
|
||||
const listed = await client.listTools();
|
||||
const focal = listed.tools.find((t) => t.name === FOCAL_TOOL);
|
||||
if (!focal) {
|
||||
throw new Error(`Focal tool ${FOCAL_TOOL} not advertised`);
|
||||
}
|
||||
const observed = focal.inputSchema as Record<string, unknown>;
|
||||
await client.callTool({
|
||||
name: ECHO_TOOL,
|
||||
arguments: { schema: transform(observed) }
|
||||
});
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive the scenario's mock server through the SEP-2575 stateless lifecycle
|
||||
* that the draft protocol version uses: list tools, then echo the focal
|
||||
* tool's inputSchema back (verbatim or after the caller-supplied transform).
|
||||
* The SDK `Client` does not support the stateless lifecycle, so these
|
||||
* requests go through the raw stateless helper instead.
|
||||
*/
|
||||
async function runStatelessEchoClient(
|
||||
serverUrl: string,
|
||||
transform: (schema: Record<string, unknown>) => Record<string, unknown> = (
|
||||
s
|
||||
) => s
|
||||
): Promise<void> {
|
||||
const listed = await sendStatelessRequest(serverUrl, 'tools/list');
|
||||
const tools = (
|
||||
listed.body?.result as
|
||||
| { tools?: { name: string; inputSchema: Record<string, unknown> }[] }
|
||||
| undefined
|
||||
)?.tools;
|
||||
const focal = tools?.find((t) => t.name === FOCAL_TOOL);
|
||||
if (!focal) {
|
||||
throw new Error(`Focal tool ${FOCAL_TOOL} not advertised`);
|
||||
}
|
||||
await sendStatelessRequest(serverUrl, 'tools/call', {
|
||||
name: ECHO_TOOL,
|
||||
arguments: { schema: transform(focal.inputSchema) }
|
||||
});
|
||||
}
|
||||
|
||||
describe('json-schema-2020-12-preservation scenario', () => {
|
||||
it('emits FAILURE preconditions and SKIPPED preservation checks when no client connects', async () => {
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
await scenario.start(testScenarioContext());
|
||||
try {
|
||||
const checks = scenario.getChecks();
|
||||
expect(checks).toHaveLength(8);
|
||||
expect(
|
||||
checks.find((c) => c.id === 'json-schema-2020-12-client-tool-found')
|
||||
?.status
|
||||
).toBe('FAILURE');
|
||||
expect(
|
||||
checks.find((c) => c.id === 'json-schema-2020-12-client-echo-completed')
|
||||
?.status
|
||||
).toBe('FAILURE');
|
||||
const preservationIds = [
|
||||
'json-schema-2020-12-client-$schema-preserved',
|
||||
'json-schema-2020-12-client-$defs-preserved',
|
||||
'json-schema-2020-12-client-additionalProperties-preserved',
|
||||
'sep-2106-client-composition-keywords-preserved',
|
||||
'sep-2106-client-conditional-keywords-preserved',
|
||||
'sep-2106-client-anchor-keyword-preserved'
|
||||
];
|
||||
for (const id of preservationIds) {
|
||||
expect(checks.find((c) => c.id === id)?.status).toBe('SKIPPED');
|
||||
}
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('emits SUCCESS for SEP-1613 keywords and SKIPPED for SEP-2106 when a compliant client echoes back on a dated target', async () => {
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
const { serverUrl } = await scenario.start(
|
||||
testScenarioContext(LATEST_SPEC_VERSION)
|
||||
);
|
||||
try {
|
||||
await runEchoClient(serverUrl);
|
||||
|
||||
const first = scenario.getChecks();
|
||||
expect(first).toHaveLength(8);
|
||||
const byId = (id: string) =>
|
||||
first.find((c) => c.id === id) ?? expect.fail(`missing check ${id}`);
|
||||
expect(byId('json-schema-2020-12-client-tool-found').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-echo-completed').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-$schema-preserved').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-$defs-preserved').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(
|
||||
byId('json-schema-2020-12-client-additionalProperties-preserved').status
|
||||
).toBe('SUCCESS');
|
||||
// SEP-2106 keywords were preserved (compliant client), so the soft gate
|
||||
// returns SUCCESS regardless of target version.
|
||||
expect(
|
||||
byId('sep-2106-client-composition-keywords-preserved').status
|
||||
).toBe('SUCCESS');
|
||||
expect(
|
||||
byId('sep-2106-client-conditional-keywords-preserved').status
|
||||
).toBe('SUCCESS');
|
||||
expect(byId('sep-2106-client-anchor-keyword-preserved').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
|
||||
// getChecks() must be idempotent — duplicate IDs would break the
|
||||
// traceability manifest.
|
||||
expect(scenario.getChecks()).toHaveLength(8);
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('emits all SUCCESS when a compliant client echoes back on the draft target', async () => {
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
const { serverUrl } = await scenario.start(
|
||||
testScenarioContext(DRAFT_PROTOCOL_VERSION)
|
||||
);
|
||||
try {
|
||||
await runStatelessEchoClient(serverUrl);
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
for (const c of checks) {
|
||||
expect(c.status, `check ${c.id}`).toBe('SUCCESS');
|
||||
}
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('flags SEP-1613 FAILURE when a client strips $schema and $defs before echoing', async () => {
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
const { serverUrl } = await scenario.start(
|
||||
testScenarioContext(LATEST_SPEC_VERSION)
|
||||
);
|
||||
try {
|
||||
await runEchoClient(serverUrl, (schema) => {
|
||||
const stripped = { ...schema };
|
||||
delete stripped['$schema'];
|
||||
delete stripped['$defs'];
|
||||
return stripped;
|
||||
});
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
const byId = (id: string) =>
|
||||
checks.find((c) => c.id === id) ?? expect.fail(`missing check ${id}`);
|
||||
|
||||
expect(byId('json-schema-2020-12-client-tool-found').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-echo-completed').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-$schema-preserved').status).toBe(
|
||||
'FAILURE'
|
||||
);
|
||||
expect(byId('json-schema-2020-12-client-$defs-preserved').status).toBe(
|
||||
'FAILURE'
|
||||
);
|
||||
// additionalProperties was left intact, so it still passes.
|
||||
expect(
|
||||
byId('json-schema-2020-12-client-additionalProperties-preserved').status
|
||||
).toBe('SUCCESS');
|
||||
// $anchor lives inside the stripped $defs, so it's also missing — but
|
||||
// the soft gate is SKIPPED on the dated target rather than FAILURE.
|
||||
expect(byId('sep-2106-client-anchor-keyword-preserved').status).toBe(
|
||||
'SKIPPED'
|
||||
);
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('flags SEP-2106 FAILURE on the draft target when composition keywords are stripped', async () => {
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
const { serverUrl } = await scenario.start(
|
||||
testScenarioContext(DRAFT_PROTOCOL_VERSION)
|
||||
);
|
||||
try {
|
||||
await runStatelessEchoClient(serverUrl, (schema) => {
|
||||
const stripped = { ...schema };
|
||||
delete stripped['allOf'];
|
||||
delete stripped['anyOf'];
|
||||
delete stripped['if'];
|
||||
delete stripped['then'];
|
||||
delete stripped['else'];
|
||||
return stripped;
|
||||
});
|
||||
|
||||
const checks = scenario.getChecks();
|
||||
const byId = (id: string) =>
|
||||
checks.find((c) => c.id === id) ?? expect.fail(`missing check ${id}`);
|
||||
|
||||
expect(byId('json-schema-2020-12-client-$schema-preserved').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
expect(
|
||||
byId('sep-2106-client-composition-keywords-preserved').status
|
||||
).toBe('FAILURE');
|
||||
expect(
|
||||
byId('sep-2106-client-conditional-keywords-preserved').status
|
||||
).toBe('FAILURE');
|
||||
// $anchor is preserved inside $defs.address, which was left intact.
|
||||
expect(byId('sep-2106-client-anchor-keyword-preserved').status).toBe(
|
||||
'SUCCESS'
|
||||
);
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves the focal fixture verbatim through the echo when the client does not transform it', async () => {
|
||||
// Sanity check that the round-trip itself does not introduce drift; if
|
||||
// this fails, all the other tests are unreliable.
|
||||
const scenario = new JsonSchema2020_12PreservationScenario();
|
||||
const { serverUrl } = await scenario.start(testScenarioContext());
|
||||
try {
|
||||
await runEchoClient(serverUrl);
|
||||
const checks = scenario.getChecks();
|
||||
const preserved = checks.find(
|
||||
(c) => c.id === 'json-schema-2020-12-client-$defs-preserved'
|
||||
);
|
||||
expect(preserved?.details).toMatchObject({
|
||||
hasDefs: true,
|
||||
defsKeys: Object.keys(JSON_SCHEMA_2020_12_FIXTURE['$defs'])
|
||||
});
|
||||
} finally {
|
||||
await scenario.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
403
src/scenarios/client/json-schema-2020-12-preservation.ts
Normal file
403
src/scenarios/client/json-schema-2020-12-preservation.ts
Normal file
|
|
@ -0,0 +1,403 @@
|
|||
/**
|
||||
* Client-side JSON Schema 2020-12 keyword preservation (SEP-1613, SEP-2106)
|
||||
*
|
||||
* Validates that a client receiving a tool definition via `tools/list`
|
||||
* preserves the full JSON Schema 2020-12 vocabulary in its internal
|
||||
* representation, rather than silently stripping `$schema`, `$defs`,
|
||||
* `additionalProperties`, `$anchor`, composition (`allOf`/`anyOf`), or
|
||||
* conditional (`if`/`then`/`else`) keywords during parsing.
|
||||
*
|
||||
* This is the client-side counterpart of the server-side
|
||||
* `json-schema-2020-12` scenario.
|
||||
* This scenario exercises:
|
||||
*
|
||||
* 1. The mock server advertises a focal tool (`json_schema_2020_12_tool`)
|
||||
* whose inputSchema contains the full SEP-1613/SEP-2106 vocabulary, plus
|
||||
* a permissive echo tool (`json_schema_echo`).
|
||||
* 2. The client under test calls `tools/list`, observes the focal tool's
|
||||
* inputSchema, then round-trips it back via `tools/call json_schema_echo`
|
||||
* with `{ schema: <observed inputSchema> }`.
|
||||
* 3. The scenario diffs the echoed schema against the original fixture and
|
||||
* flags any missing or altered keywords.
|
||||
*
|
||||
* SEP-2106 vocabulary checks are soft-gated via `sep2106KeywordCheckStatus`:
|
||||
* stripping is FAILURE only when the run targets the draft version
|
||||
* (`2026-07-28`); on earlier dated versions the check reports SKIPPED.
|
||||
*/
|
||||
|
||||
import type { ScenarioContext, MockServer } from '../../mock-server';
|
||||
import type {
|
||||
CallToolRequest,
|
||||
ListToolsResult
|
||||
} from '../../spec-types/2025-11-25';
|
||||
import {
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
LATEST_SPEC_VERSION,
|
||||
type ConformanceCheck,
|
||||
type Scenario,
|
||||
type ScenarioUrls,
|
||||
type SpecVersion
|
||||
} from '../../types';
|
||||
import {
|
||||
EXPECTED_SCHEMA_DIALECT,
|
||||
EXPECTED_TOOL_NAME,
|
||||
JSON_SCHEMA_2020_12_FIXTURE,
|
||||
sep2106KeywordCheckStatus
|
||||
} from '../server/json-schema-2020-12';
|
||||
|
||||
const ECHO_TOOL_NAME = 'json_schema_echo';
|
||||
|
||||
const SEP_1613_REF = {
|
||||
id: 'SEP-1613',
|
||||
url: 'https://github.com/modelcontextprotocol/specification/pull/655'
|
||||
};
|
||||
const SEP_2106_REF = {
|
||||
id: 'SEP-2106',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/pull/2106'
|
||||
};
|
||||
|
||||
export class JsonSchema2020_12PreservationScenario implements Scenario {
|
||||
name = 'json-schema-2020-12-preservation';
|
||||
readonly source = { introducedIn: '2025-11-25' } as const;
|
||||
description = `Validates client-side preservation of JSON Schema 2020-12 keywords (SEP-1613, SEP-2106).
|
||||
|
||||
**Client Implementation Requirements:**
|
||||
|
||||
1. Call \`tools/list\`. The mock server advertises two tools: \`${EXPECTED_TOOL_NAME}\` (focal, carries the rich inputSchema) and \`${ECHO_TOOL_NAME}\` (permissive echo).
|
||||
2. Round-trip the focal tool's \`inputSchema\` back via \`tools/call\` on \`${ECHO_TOOL_NAME}\`, passing it verbatim as the \`schema\` argument:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"name": "${ECHO_TOOL_NAME}",
|
||||
"arguments": {
|
||||
"schema": "<the inputSchema the client received for ${EXPECTED_TOOL_NAME}>"
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
The scenario compares the echoed schema against the original fixture and flags any keyword that was stripped or altered during the client's internal parsing.
|
||||
|
||||
**Verification**: \`$schema\`, \`$defs\`, and \`additionalProperties\` must be preserved (SEP-1613). The SEP-2106 vocabulary (\`$anchor\` inside \`$defs\`, composition \`allOf\`/\`anyOf\`, conditional \`if\`/\`then\`/\`else\`) must also survive; for dated protocol versions these checks are SKIPPED rather than FAILURE since SEP-2106 applies from the draft version.`;
|
||||
|
||||
private srv: MockServer | null = null;
|
||||
private specVersion: SpecVersion | null = null;
|
||||
private observedSchema: Record<string, unknown> | null = null;
|
||||
private toolsListed = false;
|
||||
|
||||
async start(ctx: ScenarioContext): Promise<ScenarioUrls> {
|
||||
this.specVersion = ctx.specVersion;
|
||||
this.observedSchema = null;
|
||||
this.toolsListed = false;
|
||||
|
||||
this.srv = await ctx.createServer({
|
||||
'tools/list': (): ListToolsResult => {
|
||||
this.toolsListed = true;
|
||||
return {
|
||||
tools: [
|
||||
{
|
||||
name: EXPECTED_TOOL_NAME,
|
||||
description: 'Tool with JSON Schema 2020-12 features',
|
||||
inputSchema: JSON_SCHEMA_2020_12_FIXTURE
|
||||
},
|
||||
{
|
||||
name: ECHO_TOOL_NAME,
|
||||
description:
|
||||
'Echoes back an arbitrary JSON Schema object so the harness can observe what the client preserved.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
schema: {
|
||||
type: 'object',
|
||||
description: `The inputSchema the client observed for ${EXPECTED_TOOL_NAME}, passed back verbatim.`
|
||||
}
|
||||
},
|
||||
required: ['schema']
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
},
|
||||
'tools/call': (params) => {
|
||||
const p = params as unknown as CallToolRequest['params'];
|
||||
if (p.name === ECHO_TOOL_NAME) {
|
||||
const args = p.arguments as { schema?: unknown } | undefined;
|
||||
if (args && typeof args.schema === 'object' && args.schema !== null) {
|
||||
this.observedSchema = args.schema as Record<string, unknown>;
|
||||
}
|
||||
return { content: [{ type: 'text', text: 'echoed' }] };
|
||||
}
|
||||
if (p.name === EXPECTED_TOOL_NAME) {
|
||||
return { content: [{ type: 'text', text: 'ok' }] };
|
||||
}
|
||||
throw new Error(`Unknown tool: ${p.name}`);
|
||||
}
|
||||
});
|
||||
return { serverUrl: this.srv.url };
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.srv?.close();
|
||||
this.srv = null;
|
||||
}
|
||||
|
||||
getChecks(): ConformanceCheck[] {
|
||||
// Built fresh on every call so getChecks() is idempotent — the runner may
|
||||
// call it more than once and we must not accumulate duplicates.
|
||||
const timestamp = new Date().toISOString();
|
||||
const targetVersion = this.specVersion ?? LATEST_SPEC_VERSION;
|
||||
const echo = this.observedSchema;
|
||||
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-client-tool-found',
|
||||
name: 'JsonSchema2020_12ClientToolFound',
|
||||
description: `Client called tools/list and the mock server advertised '${EXPECTED_TOOL_NAME}'`,
|
||||
status: this.toolsListed ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: this.toolsListed
|
||||
? undefined
|
||||
: 'Client never called tools/list; the scenario cannot observe schema preservation',
|
||||
details: { toolsListCalled: this.toolsListed }
|
||||
});
|
||||
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-client-echo-completed',
|
||||
name: 'JsonSchema2020_12ClientEchoCompleted',
|
||||
description: `Client called tools/call '${ECHO_TOOL_NAME}' with a 'schema' object argument`,
|
||||
status: echo !== null ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage:
|
||||
echo !== null
|
||||
? undefined
|
||||
: 'Client did not echo back the inputSchema; downstream preservation checks are SKIPPED',
|
||||
details: { echoCompleted: echo !== null }
|
||||
});
|
||||
|
||||
// If the client never echoed, the downstream preservation checks cannot
|
||||
// be evaluated. Emit SKIPPED for each so the SEP traceability manifest
|
||||
// still records all known check IDs from this scenario.
|
||||
if (echo === null) {
|
||||
const skipReason =
|
||||
'Skipped because the client did not complete the schema echo';
|
||||
checks.push(
|
||||
{
|
||||
id: 'json-schema-2020-12-client-$schema-preserved',
|
||||
name: 'JsonSchema2020_12Client$SchemaPreserved',
|
||||
description:
|
||||
'Client preserved the $schema field of the focal tool inputSchema',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: skipReason
|
||||
},
|
||||
{
|
||||
id: 'json-schema-2020-12-client-$defs-preserved',
|
||||
name: 'JsonSchema2020_12Client$DefsPreserved',
|
||||
description:
|
||||
'Client preserved the $defs field of the focal tool inputSchema',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: skipReason
|
||||
},
|
||||
{
|
||||
id: 'json-schema-2020-12-client-additionalProperties-preserved',
|
||||
name: 'JsonSchema2020_12ClientAdditionalPropertiesPreserved',
|
||||
description:
|
||||
'Client preserved the additionalProperties field of the focal tool inputSchema',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: skipReason
|
||||
},
|
||||
{
|
||||
id: 'sep-2106-client-composition-keywords-preserved',
|
||||
name: 'JsonSchema2020_12ClientCompositionKeywordsPreserved',
|
||||
description:
|
||||
'Client preserved composition keywords (allOf/anyOf) on the focal tool inputSchema',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: skipReason
|
||||
},
|
||||
{
|
||||
id: 'sep-2106-client-conditional-keywords-preserved',
|
||||
name: 'JsonSchema2020_12ClientConditionalKeywordsPreserved',
|
||||
description:
|
||||
'Client preserved conditional keywords (if/then/else) on the focal tool inputSchema',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: skipReason
|
||||
},
|
||||
{
|
||||
id: 'sep-2106-client-anchor-keyword-preserved',
|
||||
name: 'JsonSchema2020_12ClientAnchorKeywordPreserved',
|
||||
description:
|
||||
'Client preserved the $anchor keyword inside $defs.address',
|
||||
status: 'SKIPPED',
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: skipReason
|
||||
}
|
||||
);
|
||||
return checks;
|
||||
}
|
||||
|
||||
// SEP-1613: $schema preserved
|
||||
const hasSchema = '$schema' in echo;
|
||||
const schemaCorrect = echo['$schema'] === EXPECTED_SCHEMA_DIALECT;
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-client-$schema-preserved',
|
||||
name: 'JsonSchema2020_12Client$SchemaPreserved',
|
||||
description: `Client preserved $schema = '${EXPECTED_SCHEMA_DIALECT}'`,
|
||||
status: hasSchema && schemaCorrect ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: !hasSchema
|
||||
? '$schema missing from echoed inputSchema — client likely stripped it during parsing'
|
||||
: !schemaCorrect
|
||||
? `$schema has unexpected value: ${JSON.stringify(echo['$schema'])}`
|
||||
: undefined,
|
||||
details: { hasSchema, schemaValue: echo['$schema'] }
|
||||
});
|
||||
|
||||
// SEP-1613: $defs preserved
|
||||
const defs = echo['$defs'] as Record<string, unknown> | undefined;
|
||||
const hasDefs = '$defs' in echo;
|
||||
const defsHasAddress = hasDefs && defs !== undefined && 'address' in defs;
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-client-$defs-preserved',
|
||||
name: 'JsonSchema2020_12Client$DefsPreserved',
|
||||
description:
|
||||
'Client preserved $defs with the expected nested definitions',
|
||||
status: hasDefs && defsHasAddress ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: !hasDefs
|
||||
? '$defs missing from echoed inputSchema — client likely stripped it during parsing'
|
||||
: !defsHasAddress
|
||||
? '$defs exists but missing expected "address" definition'
|
||||
: undefined,
|
||||
details: {
|
||||
hasDefs,
|
||||
defsKeys: defs !== undefined ? Object.keys(defs) : []
|
||||
}
|
||||
});
|
||||
|
||||
// SEP-1613: additionalProperties preserved
|
||||
const hasAdditionalProps = 'additionalProperties' in echo;
|
||||
const additionalPropsCorrect = echo['additionalProperties'] === false;
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-client-additionalProperties-preserved',
|
||||
name: 'JsonSchema2020_12ClientAdditionalPropertiesPreserved',
|
||||
description:
|
||||
'Client preserved additionalProperties = false on the focal inputSchema',
|
||||
status:
|
||||
hasAdditionalProps && additionalPropsCorrect ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp,
|
||||
specReferences: [SEP_1613_REF],
|
||||
errorMessage: !hasAdditionalProps
|
||||
? 'additionalProperties missing from echoed inputSchema — client likely stripped it during parsing'
|
||||
: !additionalPropsCorrect
|
||||
? `additionalProperties has unexpected value: ${JSON.stringify(echo['additionalProperties'])}`
|
||||
: undefined,
|
||||
details: { hasAdditionalProps, value: echo['additionalProperties'] }
|
||||
});
|
||||
|
||||
// SEP-2106 vocabulary — soft-gated via sep2106KeywordCheckStatus.
|
||||
const skippedSuffix = ` (run targets protocol version ${targetVersion}; SEP-2106 applies from ${DRAFT_PROTOCOL_VERSION})`;
|
||||
|
||||
// SEP-2106: composition (allOf/anyOf) preserved
|
||||
const allOf = echo['allOf'];
|
||||
const hasAllOf = Array.isArray(allOf) && allOf.length > 0;
|
||||
const hasNestedAnyOf =
|
||||
Array.isArray(allOf) &&
|
||||
allOf.some(
|
||||
(sub) =>
|
||||
typeof sub === 'object' &&
|
||||
sub !== null &&
|
||||
Array.isArray((sub as Record<string, unknown>)['anyOf'])
|
||||
);
|
||||
const compositionPreserved = hasAllOf && hasNestedAnyOf;
|
||||
const compositionStatus = sep2106KeywordCheckStatus(
|
||||
compositionPreserved,
|
||||
targetVersion
|
||||
);
|
||||
checks.push({
|
||||
id: 'sep-2106-client-composition-keywords-preserved',
|
||||
name: 'JsonSchema2020_12ClientCompositionKeywordsPreserved',
|
||||
description:
|
||||
'Client preserved composition keywords (allOf/anyOf) on the focal inputSchema',
|
||||
status: compositionStatus,
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: compositionPreserved
|
||||
? undefined
|
||||
: (!hasAllOf
|
||||
? 'allOf missing from echoed inputSchema — composition keyword was likely stripped'
|
||||
: 'nested anyOf missing from echoed allOf — composition keyword was likely stripped') +
|
||||
(compositionStatus === 'SKIPPED' ? skippedSuffix : ''),
|
||||
details: {
|
||||
hasAllOf,
|
||||
hasNestedAnyOf,
|
||||
targetProtocolVersion: targetVersion
|
||||
}
|
||||
});
|
||||
|
||||
// SEP-2106: conditional (if/then/else) preserved
|
||||
const hasIf = 'if' in echo;
|
||||
const hasThen = 'then' in echo;
|
||||
const hasElse = 'else' in echo;
|
||||
const conditionalPreserved = hasIf && hasThen && hasElse;
|
||||
const conditionalStatus = sep2106KeywordCheckStatus(
|
||||
conditionalPreserved,
|
||||
targetVersion
|
||||
);
|
||||
checks.push({
|
||||
id: 'sep-2106-client-conditional-keywords-preserved',
|
||||
name: 'JsonSchema2020_12ClientConditionalKeywordsPreserved',
|
||||
description:
|
||||
'Client preserved conditional keywords (if/then/else) on the focal inputSchema',
|
||||
status: conditionalStatus,
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: conditionalPreserved
|
||||
? undefined
|
||||
: `Conditional keywords missing (if=${hasIf}, then=${hasThen}, else=${hasElse}) — likely stripped` +
|
||||
(conditionalStatus === 'SKIPPED' ? skippedSuffix : ''),
|
||||
details: {
|
||||
hasIf,
|
||||
hasThen,
|
||||
hasElse,
|
||||
targetProtocolVersion: targetVersion
|
||||
}
|
||||
});
|
||||
|
||||
// SEP-2106: $anchor inside $defs.address preserved
|
||||
const address = defs?.['address'] as Record<string, unknown> | undefined;
|
||||
const hasAnchor = !!address && '$anchor' in address;
|
||||
const anchorStatus = sep2106KeywordCheckStatus(hasAnchor, targetVersion);
|
||||
checks.push({
|
||||
id: 'sep-2106-client-anchor-keyword-preserved',
|
||||
name: 'JsonSchema2020_12ClientAnchorKeywordPreserved',
|
||||
description: 'Client preserved $anchor inside $defs.address',
|
||||
status: anchorStatus,
|
||||
timestamp,
|
||||
specReferences: [SEP_2106_REF],
|
||||
errorMessage: hasAnchor
|
||||
? undefined
|
||||
: '$anchor missing from echoed $defs.address — reference keyword was likely stripped' +
|
||||
(anchorStatus === 'SKIPPED' ? skippedSuffix : ''),
|
||||
details: {
|
||||
hasAnchor,
|
||||
anchorValue: address?.['$anchor']
|
||||
}
|
||||
});
|
||||
|
||||
return checks;
|
||||
}
|
||||
}
|
||||
|
|
@ -119,6 +119,7 @@ import {
|
|||
HttpInvalidToolHeadersScenario
|
||||
} from './client/http-custom-headers';
|
||||
import { JsonSchemaRefDerefScenario } from './client/json-schema-ref-deref';
|
||||
import { JsonSchema2020_12PreservationScenario } from './client/json-schema-2020-12-preservation';
|
||||
|
||||
// Pending client scenarios (not yet fully tested/implemented)
|
||||
const pendingClientScenariosList: ClientScenario[] = [
|
||||
|
|
@ -313,7 +314,10 @@ const scenariosList: Scenario[] = [
|
|||
new HttpInvalidToolHeadersScenario(),
|
||||
|
||||
// JSON Schema network $ref dereferencing (SEP-2106)
|
||||
new JsonSchemaRefDerefScenario()
|
||||
new JsonSchemaRefDerefScenario(),
|
||||
|
||||
// JSON Schema 2020-12 client-side keyword preservation (SEP-1613, SEP-2106)
|
||||
new JsonSchema2020_12PreservationScenario()
|
||||
];
|
||||
|
||||
// Core scenarios (tier 1 requirements)
|
||||
|
|
|
|||
|
|
@ -20,8 +20,48 @@ import {
|
|||
import type { RunContext } from '../../connection';
|
||||
import type { ListToolsResult } from '../../spec-types/2025-11-25';
|
||||
|
||||
const EXPECTED_TOOL_NAME = 'json_schema_2020_12_tool';
|
||||
const EXPECTED_SCHEMA_DIALECT = 'https://json-schema.org/draft/2020-12/schema';
|
||||
export const EXPECTED_TOOL_NAME = 'json_schema_2020_12_tool';
|
||||
export const EXPECTED_SCHEMA_DIALECT =
|
||||
'https://json-schema.org/draft/2020-12/schema';
|
||||
|
||||
/**
|
||||
* Canonical focal `inputSchema` used by both the server-side
|
||||
* `json-schema-2020-12` scenario (as the schema servers under test are
|
||||
* expected to emit) and the client-side `json-schema-2020-12-preservation`
|
||||
* scenario (as the schema the mock server actually emits on the wire).
|
||||
*
|
||||
* Keeping a single source of truth ensures the documented requirements and
|
||||
* the runtime test fixture stay in sync.
|
||||
*/
|
||||
export const JSON_SCHEMA_2020_12_FIXTURE = {
|
||||
$schema: EXPECTED_SCHEMA_DIALECT,
|
||||
type: 'object',
|
||||
$defs: {
|
||||
address: {
|
||||
$anchor: 'addressDef',
|
||||
type: 'object',
|
||||
properties: {
|
||||
street: { type: 'string' },
|
||||
city: { type: 'string' }
|
||||
}
|
||||
}
|
||||
},
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
address: { $ref: '#/$defs/address' },
|
||||
contactMethod: { type: 'string', enum: ['phone', 'email'] },
|
||||
phone: { type: 'string' },
|
||||
email: { type: 'string' }
|
||||
},
|
||||
allOf: [{ anyOf: [{ required: ['phone'] }, { required: ['email'] }] }],
|
||||
if: {
|
||||
properties: { contactMethod: { const: 'phone' } },
|
||||
required: ['contactMethod']
|
||||
},
|
||||
then: { required: ['phone'] },
|
||||
else: { required: ['email'] },
|
||||
additionalProperties: false
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Soft version gate for the SEP-2106 keyword-preservation checks.
|
||||
|
|
@ -55,41 +95,15 @@ export class JsonSchema2020_12Scenario implements ClientScenario {
|
|||
Implement tool \`${EXPECTED_TOOL_NAME}\` with inputSchema containing JSON Schema 2020-12 features, including the broader vocabulary permitted by SEP-2106 (an \`$anchor\` inside \`$defs\`, composition keywords \`allOf\`/\`anyOf\`, and conditional keywords \`if\`/\`then\`/\`else\`):
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"name": "${EXPECTED_TOOL_NAME}",
|
||||
"description": "Tool with JSON Schema 2020-12 features",
|
||||
"inputSchema": {
|
||||
"$schema": "${EXPECTED_SCHEMA_DIALECT}",
|
||||
"type": "object",
|
||||
"$defs": {
|
||||
"address": {
|
||||
"$anchor": "addressDef",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"street": { "type": "string" },
|
||||
"city": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"address": { "$ref": "#/$defs/address" },
|
||||
"contactMethod": { "type": "string", "enum": ["phone", "email"] },
|
||||
"phone": { "type": "string" },
|
||||
"email": { "type": "string" }
|
||||
},
|
||||
"allOf": [
|
||||
{ "anyOf": [{ "required": ["phone"] }, { "required": ["email"] }] }
|
||||
],
|
||||
"if": {
|
||||
"properties": { "contactMethod": { "const": "phone" } },
|
||||
"required": ["contactMethod"]
|
||||
},
|
||||
"then": { "required": ["phone"] },
|
||||
"else": { "required": ["email"] },
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
${JSON.stringify(
|
||||
{
|
||||
name: EXPECTED_TOOL_NAME,
|
||||
description: 'Tool with JSON Schema 2020-12 features',
|
||||
inputSchema: JSON_SCHEMA_2020_12_FIXTURE
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}
|
||||
\`\`\`
|
||||
|
||||
**Verification**: The test verifies that \`$schema\`, \`$defs\`, and \`additionalProperties\` are preserved (SEP-1613), and that the composition (\`allOf\`/\`anyOf\`), conditional (\`if\`/\`then\`/\`else\`), and \`$anchor\` keywords are preserved (SEP-2106), in the tool listing response.`;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,25 @@ requirements:
|
|||
- check: sep-2106-no-network-ref-deref
|
||||
text: 'Implementations MUST NOT automatically dereference `$ref` values that resolve to a network URI (i.e. anything that is not a same-document JSON Pointer such as `#/$defs/Foo` or an internal `$anchor`); an opt-in mode that fetches non-local `$ref`s MUST be disabled by default / Schemas that fail to validate due to an unresolved external `$ref` SHOULD be rejected rather than silently treated as permissive'
|
||||
|
||||
# Client-side keyword-preservation checks, derived from issue #101 rather
|
||||
# than a normative sentence in the SEP: SEP-2106 §1 "Loosen inputSchema"
|
||||
# redefines `inputSchema` to permit any additional JSON Schema 2020-12
|
||||
# keyword, and a client that silently strips such a keyword while parsing
|
||||
# a server's tool definition breaks that contract, since downstream
|
||||
# consumers can no longer recover the intended semantics from what the
|
||||
# client exposes. The scenario's SEP-1613-scoped checks ($schema, $defs,
|
||||
# additionalProperties) are not declared here: their IDs are not
|
||||
# sep-2106-prefixed, so the traceability manifest never collects them.
|
||||
- check: sep-2106-client-composition-keywords-preserved
|
||||
text: 'The `inputSchema` field "now accepts any additional JSON Schema properties. This enables: Composition keywords: anyOf, oneOf, allOf, not". Derived expectation (issue #101): a client parsing a tool definition received from a server is expected to preserve these composition keywords.'
|
||||
url: https://modelcontextprotocol.io/seps/2106-json-schema-2020-12#1-loosen-inputschema
|
||||
- check: sep-2106-client-conditional-keywords-preserved
|
||||
text: 'The `inputSchema` field "now accepts any additional JSON Schema properties. This enables: ... Conditional schemas: if/then/else". Derived expectation (issue #101): a client parsing a tool definition received from a server is expected to preserve these conditional keywords.'
|
||||
url: https://modelcontextprotocol.io/seps/2106-json-schema-2020-12#1-loosen-inputschema
|
||||
- check: sep-2106-client-anchor-keyword-preserved
|
||||
text: 'The `inputSchema` field "now accepts any additional JSON Schema properties. This enables: ... Reference schemas: $ref, $defs". Derived expectation (issue #101): a client parsing a tool definition received from a server is expected to preserve `$anchor` (a JSON Schema 2020-12 reference-anchor keyword that lives inside `$defs`).'
|
||||
url: https://modelcontextprotocol.io/seps/2106-json-schema-2020-12#1-loosen-inputschema
|
||||
|
||||
- text: "SDK maintainers SHOULD: Document the migration in SDK release notes; Where ergonomic, provide typed helpers (e.g. generics over a tool's `outputSchema`) so consumers do not need to write narrowing guards by hand"
|
||||
excluded: 'Migration/deprecation guidance for SDK maintainers; about SDK source code, not protocol-observable wire behavior'
|
||||
url: https://modelcontextprotocol.io/seps/2106-json-schema-2020-12#backward-compatibility
|
||||
|
|
|
|||
Loading…
Reference in a new issue