Add JSON Schema 2020-12 conformance test (SEP-1613) (#45)
Add conformance test to validate that MCP servers correctly preserve JSON Schema 2020-12 keywords ($schema, $defs, additionalProperties) in tool definitions. Changes: - Add JsonSchema2020_12Scenario in src/scenarios/server/json-schema-2020-12.ts - Add json_schema_2020_12_tool to everything-server with tools/list override to return raw JSON Schema instead of Zod-converted schema - Register scenario in pendingClientScenariosList (requires SDK PR #1135) The test is added to pending because the client SDK currently strips the JSON Schema 2020-12 keywords. Once PR #1135 is merged into the SDK, this scenario can be moved to the active list.
This commit is contained in:
Felix Weinberger
GitHub
parent
8ef618bfe5
commit
f087e44bdb
3 changed files with 329 additions and 2 deletions
|
|
@ -13,7 +13,13 @@ import {
|
|||
ResourceTemplate
|
||||
} from '@modelcontextprotocol/sdk/server/mcp.js';
|
||||
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
|
||||
import { ElicitResultSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
import {
|
||||
ElicitResultSchema,
|
||||
ListToolsRequestSchema,
|
||||
type ListToolsResult,
|
||||
type Tool
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
import { zodToJsonSchema } from 'zod-to-json-schema';
|
||||
import { z } from 'zod';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
|
|
@ -35,6 +41,28 @@ const TEST_IMAGE_BASE64 =
|
|||
const TEST_AUDIO_BASE64 =
|
||||
'UklGRiYAAABXQVZFZm10IBAAAAABAAEAQB8AAAB9AAACABAAZGF0YQIAAAA=';
|
||||
|
||||
// SEP-1613: Raw JSON Schema 2020-12 definition for conformance testing
|
||||
// This schema includes $schema, $defs, and additionalProperties to test
|
||||
// that SDKs correctly preserve these fields
|
||||
const JSON_SCHEMA_2020_12_INPUT_SCHEMA = {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
type: 'object' as const,
|
||||
$defs: {
|
||||
address: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
street: { type: 'string' },
|
||||
city: { type: 'string' }
|
||||
}
|
||||
}
|
||||
},
|
||||
properties: {
|
||||
name: { type: 'string' },
|
||||
address: { $ref: '#/$defs/address' }
|
||||
},
|
||||
additionalProperties: false
|
||||
};
|
||||
|
||||
// Function to create a new MCP server instance (one per session)
|
||||
function createMcpServer() {
|
||||
const mcpServer = new McpServer(
|
||||
|
|
@ -566,6 +594,40 @@ function createMcpServer() {
|
|||
}
|
||||
);
|
||||
|
||||
// SEP-1613: JSON Schema 2020-12 conformance test tool
|
||||
// This tool is registered with a Zod schema for tools/call validation,
|
||||
// but the tools/list handler (below) returns the raw JSON Schema 2020-12
|
||||
// definition to test that SDKs preserve $schema, $defs, additionalProperties
|
||||
mcpServer.registerTool(
|
||||
'json_schema_2020_12_tool',
|
||||
{
|
||||
description:
|
||||
'Tool with JSON Schema 2020-12 features for conformance testing (SEP-1613)',
|
||||
inputSchema: {
|
||||
name: z.string().optional(),
|
||||
address: z
|
||||
.object({
|
||||
street: z.string().optional(),
|
||||
city: z.string().optional()
|
||||
})
|
||||
.optional()
|
||||
}
|
||||
},
|
||||
async (args: {
|
||||
name?: string;
|
||||
address?: { street?: string; city?: string };
|
||||
}) => {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `JSON Schema 2020-12 tool called with: ${JSON.stringify(args)}`
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// Dynamic tool (registered later via timer)
|
||||
|
||||
// ===== RESOURCES =====
|
||||
|
|
@ -829,6 +891,80 @@ function createMcpServer() {
|
|||
}
|
||||
);
|
||||
|
||||
// ===== SEP-1613: Override tools/list to return raw JSON Schema 2020-12 =====
|
||||
// This override is necessary because registerTool converts Zod schemas to
|
||||
// JSON Schema without preserving $schema, $defs, and additionalProperties.
|
||||
// We need to return the raw JSON Schema for our test tool while using the
|
||||
// SDK's conversion for other tools.
|
||||
mcpServer.server.setRequestHandler(
|
||||
ListToolsRequestSchema,
|
||||
(): ListToolsResult => {
|
||||
// Access internal registered tools (this is internal SDK API but stable)
|
||||
const registeredTools = (mcpServer as any)._registeredTools as Record<
|
||||
string,
|
||||
{
|
||||
enabled: boolean;
|
||||
title?: string;
|
||||
description?: string;
|
||||
inputSchema?: any;
|
||||
outputSchema?: any;
|
||||
annotations?: any;
|
||||
_meta?: any;
|
||||
}
|
||||
>;
|
||||
|
||||
return {
|
||||
tools: Object.entries(registeredTools)
|
||||
.filter(([, tool]) => tool.enabled)
|
||||
.map(([name, tool]): Tool => {
|
||||
// For our SEP-1613 test tool, return raw JSON Schema 2020-12
|
||||
if (name === 'json_schema_2020_12_tool') {
|
||||
return {
|
||||
name,
|
||||
description: tool.description,
|
||||
inputSchema: JSON_SCHEMA_2020_12_INPUT_SCHEMA
|
||||
};
|
||||
}
|
||||
|
||||
// For other tools, convert Zod to JSON Schema
|
||||
// Handle different inputSchema formats:
|
||||
// - undefined/null: use empty object schema
|
||||
// - Zod schema (has _def): convert directly
|
||||
// - Raw shape (object with Zod values): wrap in z.object first
|
||||
let inputSchema: Tool['inputSchema'];
|
||||
if (!tool.inputSchema) {
|
||||
inputSchema = { type: 'object' as const, properties: {} };
|
||||
} else if ('_def' in tool.inputSchema) {
|
||||
// Already a Zod schema
|
||||
inputSchema = zodToJsonSchema(tool.inputSchema, {
|
||||
strictUnions: true
|
||||
}) as Tool['inputSchema'];
|
||||
} else if (
|
||||
typeof tool.inputSchema === 'object' &&
|
||||
Object.keys(tool.inputSchema).length > 0
|
||||
) {
|
||||
// Raw shape with Zod values
|
||||
inputSchema = zodToJsonSchema(z.object(tool.inputSchema), {
|
||||
strictUnions: true
|
||||
}) as Tool['inputSchema'];
|
||||
} else {
|
||||
// Empty object or unknown format
|
||||
inputSchema = { type: 'object' as const, properties: {} };
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
title: tool.title,
|
||||
description: tool.description,
|
||||
inputSchema,
|
||||
annotations: tool.annotations,
|
||||
_meta: tool._meta
|
||||
};
|
||||
})
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
return mcpServer;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ import {
|
|||
ToolsCallEmbeddedResourceScenario
|
||||
} from './server/tools.js';
|
||||
|
||||
import { JsonSchema2020_12Scenario } from './server/json-schema-2020-12.js';
|
||||
|
||||
import { ElicitationDefaultsScenario } from './server/elicitation-defaults.js';
|
||||
import { ElicitationEnumsScenario } from './server/elicitation-enums.js';
|
||||
|
||||
|
|
@ -51,7 +53,12 @@ import { listMetadataScenarios } from './client/auth/discovery-metadata.js';
|
|||
// Pending client scenarios (not yet fully tested/implemented)
|
||||
const pendingClientScenariosList: ClientScenario[] = [
|
||||
// Elicitation scenarios (SEP-1330)
|
||||
new ElicitationEnumsScenario()
|
||||
new ElicitationEnumsScenario(),
|
||||
|
||||
// JSON Schema 2020-12 (SEP-1613)
|
||||
// This test is pending until the SDK includes PR #1135 which preserves
|
||||
// $schema, $defs, and additionalProperties fields in tool schemas.
|
||||
new JsonSchema2020_12Scenario()
|
||||
];
|
||||
|
||||
// All client scenarios
|
||||
|
|
@ -76,6 +83,9 @@ const allClientScenariosList: ClientScenario[] = [
|
|||
new ToolsCallSamplingScenario(),
|
||||
new ToolsCallElicitationScenario(),
|
||||
|
||||
// JSON Schema 2020-12 support (SEP-1613)
|
||||
new JsonSchema2020_12Scenario(),
|
||||
|
||||
// Elicitation scenarios (SEP-1034)
|
||||
new ElicitationDefaultsScenario(),
|
||||
|
||||
|
|
|
|||
181
src/scenarios/server/json-schema-2020-12.ts
Normal file
181
src/scenarios/server/json-schema-2020-12.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
/**
|
||||
* JSON Schema 2020-12 conformance test scenario (SEP-1613)
|
||||
*
|
||||
* Validates that MCP servers correctly preserve JSON Schema 2020-12 keywords
|
||||
* in tool definitions, ensuring implementations don't strip $schema, $defs,
|
||||
* or additionalProperties fields.
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types.js';
|
||||
import { connectToServer } from './client-helper.js';
|
||||
|
||||
const EXPECTED_TOOL_NAME = 'json_schema_2020_12_tool';
|
||||
const EXPECTED_SCHEMA_DIALECT = 'https://json-schema.org/draft/2020-12/schema';
|
||||
|
||||
export class JsonSchema2020_12Scenario implements ClientScenario {
|
||||
name = 'json-schema-2020-12';
|
||||
description = `Validates JSON Schema 2020-12 keyword preservation (SEP-1613).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`${EXPECTED_TOOL_NAME}\` with inputSchema containing JSON Schema 2020-12 features:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"name": "${EXPECTED_TOOL_NAME}",
|
||||
"description": "Tool with JSON Schema 2020-12 features",
|
||||
"inputSchema": {
|
||||
"$schema": "${EXPECTED_SCHEMA_DIALECT}",
|
||||
"type": "object",
|
||||
"$defs": {
|
||||
"address": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"street": { "type": "string" },
|
||||
"city": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"name": { "type": "string" },
|
||||
"address": { "$ref": "#/$defs/address" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Verification**: The test verifies that \`$schema\`, \`$defs\`, and \`additionalProperties\` are preserved in the tool listing response.`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
const specReferences = [
|
||||
{
|
||||
id: 'SEP-1613',
|
||||
url: 'https://github.com/modelcontextprotocol/specification/pull/655'
|
||||
}
|
||||
];
|
||||
|
||||
try {
|
||||
const connection = await connectToServer(serverUrl);
|
||||
const result = await connection.client.listTools();
|
||||
|
||||
// Find the test tool
|
||||
const tool = result.tools?.find((t) => t.name === EXPECTED_TOOL_NAME);
|
||||
|
||||
// Check 1: Tool exists
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-tool-found',
|
||||
name: 'JsonSchema2020_12ToolFound',
|
||||
description: `Server advertises tool '${EXPECTED_TOOL_NAME}'`,
|
||||
status: tool ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: tool
|
||||
? undefined
|
||||
: `Tool '${EXPECTED_TOOL_NAME}' not found. Available tools: ${result.tools?.map((t) => t.name).join(', ') || 'none'}`,
|
||||
specReferences,
|
||||
details: {
|
||||
toolFound: !!tool,
|
||||
availableTools: result.tools?.map((t) => t.name) || []
|
||||
}
|
||||
});
|
||||
|
||||
if (!tool) {
|
||||
await connection.close();
|
||||
return checks;
|
||||
}
|
||||
|
||||
const inputSchema = tool.inputSchema as Record<string, unknown>;
|
||||
|
||||
// Check 2: $schema field preserved
|
||||
const hasSchema = '$schema' in inputSchema;
|
||||
const schemaValue = inputSchema['$schema'];
|
||||
const schemaCorrect = schemaValue === EXPECTED_SCHEMA_DIALECT;
|
||||
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-$schema',
|
||||
name: 'JsonSchema2020_12$Schema',
|
||||
description: `inputSchema.$schema field preserved with value '${EXPECTED_SCHEMA_DIALECT}'`,
|
||||
status: hasSchema && schemaCorrect ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: !hasSchema
|
||||
? '$schema field missing from inputSchema - field was likely stripped'
|
||||
: !schemaCorrect
|
||||
? `$schema has unexpected value: ${JSON.stringify(schemaValue)}`
|
||||
: undefined,
|
||||
specReferences,
|
||||
details: {
|
||||
hasSchema,
|
||||
schemaValue,
|
||||
expected: EXPECTED_SCHEMA_DIALECT
|
||||
}
|
||||
});
|
||||
|
||||
// Check 3: $defs field preserved
|
||||
const hasDefs = '$defs' in inputSchema;
|
||||
const defsValue = inputSchema['$defs'] as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const defsHasAddress = defsValue && 'address' in defsValue;
|
||||
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-$defs',
|
||||
name: 'JsonSchema2020_12$Defs',
|
||||
description:
|
||||
'inputSchema.$defs field preserved with expected structure',
|
||||
status: hasDefs && defsHasAddress ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: !hasDefs
|
||||
? '$defs field missing from inputSchema - field was likely stripped'
|
||||
: !defsHasAddress
|
||||
? '$defs exists but missing expected "address" definition'
|
||||
: undefined,
|
||||
specReferences,
|
||||
details: {
|
||||
hasDefs,
|
||||
defsKeys: defsValue ? Object.keys(defsValue) : [],
|
||||
defsValue
|
||||
}
|
||||
});
|
||||
|
||||
// Check 4: additionalProperties field preserved
|
||||
const hasAdditionalProps = 'additionalProperties' in inputSchema;
|
||||
const additionalPropsValue = inputSchema['additionalProperties'];
|
||||
const additionalPropsCorrect = additionalPropsValue === false;
|
||||
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-additionalProperties',
|
||||
name: 'JsonSchema2020_12AdditionalProperties',
|
||||
description: 'inputSchema.additionalProperties field preserved',
|
||||
status:
|
||||
hasAdditionalProps && additionalPropsCorrect ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: !hasAdditionalProps
|
||||
? 'additionalProperties field missing from inputSchema - field was likely stripped'
|
||||
: !additionalPropsCorrect
|
||||
? `additionalProperties has unexpected value: ${JSON.stringify(additionalPropsValue)}, expected: false`
|
||||
: undefined,
|
||||
specReferences,
|
||||
details: {
|
||||
hasAdditionalProps,
|
||||
additionalPropsValue,
|
||||
expected: false
|
||||
}
|
||||
});
|
||||
|
||||
await connection.close();
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'json-schema-2020-12-error',
|
||||
name: 'JsonSchema2020_12Error',
|
||||
description: 'JSON Schema 2020-12 conformance test',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences
|
||||
});
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue