feat: add specVersion classification to conformance scenarios (#147)
* feat: add specVersions classification to conformance scenarios
Each scenario declares which spec versions it applies to as a list.
Scenarios that carry forward (e.g. initialize) list all applicable versions
['2025-06-18', '2025-11-25']. Scenarios removed from newer specs (e.g.
backcompat auth) only list their original version ['2025-03-26'].
- specVersions list on Scenario and ClientScenario interfaces
- --spec-version CLI filter uses simple .includes()
- Tier-check conformance matrix (Server / Client: Core / Client: Auth)
with per-version columns and unique All* count
- 7 unit tests for specVersions helpers
- Updated tier-audit skill docs with matrix format
* fix: fix console output template for tier audit skill
The template had an orphan table header (Check | Value | T2 | T1)
with no rows above the conformance matrix, causing an empty table
to render. The scorecard rows below the matrix also lacked their
own header.
Fix: two self-contained tables with clear labels — 'Conformance:'
for the per-version matrix, 'Scorecard:' for the check rows.
* fix: align skill console template with tier-check script output
Add asterisk footnote ('unique scenarios — a scenario may apply to
multiple spec versions') and rename 'Scorecard' to 'Repository Health'
to match the labels used by the tier-check CLI output.
* fix: add specVersions to CrossAppAccessCompleteFlowScenario
Added in 83c446d on main after this branch diverged.
This commit is contained in:
parent
dd14862ee4
commit
3f9048a019
11 changed files with 521 additions and 59 deletions
|
|
@ -5,7 +5,7 @@
|
|||
* to prevent DNS rebinding attacks. See GHSA-w48q-cv73-mx4w for details.
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { request } from 'undici';
|
||||
|
||||
const SPEC_REFERENCES = [
|
||||
|
|
@ -85,6 +85,7 @@ async function sendRequestWithHostAndOrigin(
|
|||
|
||||
export class DNSRebindingProtectionScenario implements ClientScenario {
|
||||
name = 'dns-rebinding-protection';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description = `Test DNS rebinding protection for localhost servers.
|
||||
|
||||
**Scope:** This test applies to localhost MCP servers running without HTTPS and without
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@
|
|||
* SEP-1034: Elicitation default values test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
|
||||
export class ElicitationDefaultsScenario implements ClientScenario {
|
||||
name = 'elicitation-sep1034-defaults';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description = `Test elicitation with default values for all primitive types (SEP-1034).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@
|
|||
* SEP-1330: Elicitation enum schema improvements test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
|
||||
export class ElicitationEnumsScenario implements ClientScenario {
|
||||
name = 'elicitation-sep1330-enums';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description = `Test elicitation with enum schema improvements (SEP-1330).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* or additionalProperties fields.
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types.js';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types.js';
|
||||
import { connectToServer } from './client-helper.js';
|
||||
|
||||
const EXPECTED_TOOL_NAME = 'json_schema_2020_12_tool';
|
||||
|
|
@ -14,6 +14,7 @@ const EXPECTED_SCHEMA_DIALECT = 'https://json-schema.org/draft/2020-12/schema';
|
|||
|
||||
export class JsonSchema2020_12Scenario implements ClientScenario {
|
||||
name = 'json-schema-2020-12';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description = `Validates JSON Schema 2020-12 keyword preservation (SEP-1613).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
* Lifecycle test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
|
||||
export class ServerInitializeScenario implements ClientScenario {
|
||||
name = 'server-initialize';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test basic server initialization handshake.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
* Prompts test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
|
||||
export class PromptsListScenario implements ClientScenario {
|
||||
name = 'prompts-list';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test listing available prompts.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -87,6 +88,7 @@ export class PromptsListScenario implements ClientScenario {
|
|||
|
||||
export class PromptsGetSimpleScenario implements ClientScenario {
|
||||
name = 'prompts-get-simple';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test getting a simple prompt without arguments.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -171,6 +173,7 @@ Implement a prompt named \`test_simple_prompt\` with no arguments that returns:
|
|||
|
||||
export class PromptsGetWithArgsScenario implements ClientScenario {
|
||||
name = 'prompts-get-with-args';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test parameterized prompt.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -266,6 +269,7 @@ Returns (with args \`{arg1: "hello", arg2: "world"}\`):
|
|||
|
||||
export class PromptsGetEmbeddedResourceScenario implements ClientScenario {
|
||||
name = 'prompts-get-embedded-resource';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test prompt with embedded resource content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -371,6 +375,7 @@ Returns:
|
|||
|
||||
export class PromptsGetWithImageScenario implements ClientScenario {
|
||||
name = 'prompts-get-with-image';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test prompt with image content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Resources test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
import {
|
||||
TextResourceContents,
|
||||
|
|
@ -11,6 +11,7 @@ import {
|
|||
|
||||
export class ResourcesListScenario implements ClientScenario {
|
||||
name = 'resources-list';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test listing available resources.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -91,6 +92,7 @@ export class ResourcesListScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesReadTextScenario implements ClientScenario {
|
||||
name = 'resources-read-text';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test reading text resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -177,6 +179,7 @@ Implement resource \`test://static-text\` that returns:
|
|||
|
||||
export class ResourcesReadBinaryScenario implements ClientScenario {
|
||||
name = 'resources-read-binary';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test reading binary resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -261,6 +264,7 @@ Implement resource \`test://static-binary\` that returns:
|
|||
|
||||
export class ResourcesTemplateReadScenario implements ClientScenario {
|
||||
name = 'resources-templates-read';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test reading resource from template.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -362,6 +366,7 @@ Returns (for \`uri: "test://template/123/data"\`):
|
|||
|
||||
export class ResourcesSubscribeScenario implements ClientScenario {
|
||||
name = 'resources-subscribe';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test subscribing to resource updates.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -432,6 +437,7 @@ Example request:
|
|||
|
||||
export class ResourcesUnsubscribeScenario implements ClientScenario {
|
||||
name = 'resources-unsubscribe';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test unsubscribing from resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -9,13 +9,14 @@
|
|||
* Multiple concurrent streams are achieved via POST requests, each getting their own stream.
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types.js';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types.js';
|
||||
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
||||
|
||||
export class ServerSSEMultipleStreamsScenario implements ClientScenario {
|
||||
name = 'server-sse-multiple-streams';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description =
|
||||
'Test server supports multiple concurrent POST SSE streams (SEP-1699)';
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* - Replaying events when client reconnects with Last-Event-ID
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types.js';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types.js';
|
||||
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
||||
|
|
@ -67,6 +67,7 @@ function createLoggingFetch(checks: ConformanceCheck[]) {
|
|||
|
||||
export class ServerSSEPollingScenario implements ClientScenario {
|
||||
name = 'server-sse-polling';
|
||||
specVersions: SpecVersion[] = ['2025-11-25'];
|
||||
description =
|
||||
'Test server SSE polling via test_reconnection tool that closes stream mid-call (SEP-1699)';
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* Tools test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer, NotificationCollector } from './client-helper';
|
||||
import {
|
||||
CallToolResultSchema,
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
|
||||
export class ToolsListScenario implements ClientScenario {
|
||||
name = 'tools-list';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test listing available tools.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -95,6 +96,7 @@ export class ToolsListScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallSimpleTextScenario implements ClientScenario {
|
||||
name = 'tools-call-simple-text';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns simple text.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -179,6 +181,7 @@ Implement tool \`test_simple_text\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallImageScenario implements ClientScenario {
|
||||
name = 'tools-call-image';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns image content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -266,6 +269,7 @@ Implement tool \`test_image_content\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallMultipleContentTypesScenario implements ClientScenario {
|
||||
name = 'tools-call-mixed-content';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool returning multiple content types.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -366,6 +370,7 @@ Implement tool \`test_multiple_content_types\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallWithLoggingScenario implements ClientScenario {
|
||||
name = 'tools-call-with-logging';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that sends log messages during execution.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -454,6 +459,7 @@ Implement tool \`test_tool_with_logging\` with no arguments.
|
|||
|
||||
export class ToolsCallErrorScenario implements ClientScenario {
|
||||
name = 'tools-call-error';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool error reporting.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -538,6 +544,7 @@ Implement tool \`test_error_handling\` with no arguments.
|
|||
|
||||
export class ToolsCallWithProgressScenario implements ClientScenario {
|
||||
name = 'tools-call-with-progress';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that reports progress notifications.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -657,6 +664,7 @@ If no progress token provided, just execute with delays.
|
|||
|
||||
export class ToolsCallSamplingScenario implements ClientScenario {
|
||||
name = 'tools-call-sampling';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that requests LLM sampling from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -784,6 +792,7 @@ Implement tool \`test_sampling\` with argument:
|
|||
|
||||
export class ToolsCallElicitationScenario implements ClientScenario {
|
||||
name = 'tools-call-elicitation';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that requests user input (elicitation) from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -914,6 +923,7 @@ Implement tool \`test_elicitation\` with argument:
|
|||
|
||||
export class ToolsCallAudioScenario implements ClientScenario {
|
||||
name = 'tools-call-audio';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns audio content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -1008,6 +1018,7 @@ Implement tool \`test_audio_content\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallEmbeddedResourceScenario implements ClientScenario {
|
||||
name = 'tools-call-embedded-resource';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns embedded resource content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@
|
|||
* Utilities test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
|
||||
export class LoggingSetLevelScenario implements ClientScenario {
|
||||
name = 'logging-set-level';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test setting logging level.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -85,6 +86,7 @@ export class LoggingSetLevelScenario implements ClientScenario {
|
|||
|
||||
export class PingScenario implements ClientScenario {
|
||||
name = 'ping';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test ping utility for connection health check.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -174,6 +176,7 @@ export class PingScenario implements ClientScenario {
|
|||
|
||||
export class CompletionCompleteScenario implements ClientScenario {
|
||||
name = 'completion-complete';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test completion endpoint.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
Loading…
Reference in a new issue