feat: add specVersion classification to conformance scenarios (#147)
* feat: add specVersions classification to conformance scenarios
Each scenario declares which spec versions it applies to as a list.
Scenarios that carry forward (e.g. initialize) list all applicable versions
['2025-06-18', '2025-11-25']. Scenarios removed from newer specs (e.g.
backcompat auth) only list their original version ['2025-03-26'].
- specVersions list on Scenario and ClientScenario interfaces
- --spec-version CLI filter uses simple .includes()
- Tier-check conformance matrix (Server / Client: Core / Client: Auth)
with per-version columns and unique All* count
- 7 unit tests for specVersions helpers
- Updated tier-audit skill docs with matrix format
* fix: fix console output template for tier audit skill
The template had an orphan table header (Check | Value | T2 | T1)
with no rows above the conformance matrix, causing an empty table
to render. The scorecard rows below the matrix also lacked their
own header.
Fix: two self-contained tables with clear labels — 'Conformance:'
for the per-version matrix, 'Scorecard:' for the check rows.
* fix: align skill console template with tier-check script output
Add asterisk footnote ('unique scenarios — a scenario may apply to
multiple spec versions') and rename 'Scorecard' to 'Repository Health'
to match the labels used by the tier-check CLI output.
* fix: add specVersions to CrossAppAccessCompleteFlowScenario
Added in 83c446d on main after this branch diverged.
This commit is contained in:
parent
dd14862ee4
commit
3f9048a019
1 changed files with 521 additions and 59 deletions
|
|
@ -2,7 +2,7 @@
|
|||
* Tools test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types';
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import { connectToServer, NotificationCollector } from './client-helper';
|
||||
import {
|
||||
CallToolResultSchema,
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
|
||||
export class ToolsListScenario implements ClientScenario {
|
||||
name = 'tools-list';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test listing available tools.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -95,6 +96,7 @@ export class ToolsListScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallSimpleTextScenario implements ClientScenario {
|
||||
name = 'tools-call-simple-text';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns simple text.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -179,6 +181,7 @@ Implement tool \`test_simple_text\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallImageScenario implements ClientScenario {
|
||||
name = 'tools-call-image';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns image content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -266,6 +269,7 @@ Implement tool \`test_image_content\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallMultipleContentTypesScenario implements ClientScenario {
|
||||
name = 'tools-call-mixed-content';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool returning multiple content types.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -366,6 +370,7 @@ Implement tool \`test_multiple_content_types\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallWithLoggingScenario implements ClientScenario {
|
||||
name = 'tools-call-with-logging';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that sends log messages during execution.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -454,6 +459,7 @@ Implement tool \`test_tool_with_logging\` with no arguments.
|
|||
|
||||
export class ToolsCallErrorScenario implements ClientScenario {
|
||||
name = 'tools-call-error';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool error reporting.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -538,6 +544,7 @@ Implement tool \`test_error_handling\` with no arguments.
|
|||
|
||||
export class ToolsCallWithProgressScenario implements ClientScenario {
|
||||
name = 'tools-call-with-progress';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that reports progress notifications.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -657,6 +664,7 @@ If no progress token provided, just execute with delays.
|
|||
|
||||
export class ToolsCallSamplingScenario implements ClientScenario {
|
||||
name = 'tools-call-sampling';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that requests LLM sampling from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -784,6 +792,7 @@ Implement tool \`test_sampling\` with argument:
|
|||
|
||||
export class ToolsCallElicitationScenario implements ClientScenario {
|
||||
name = 'tools-call-elicitation';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test tool that requests user input (elicitation) from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -914,6 +923,7 @@ Implement tool \`test_elicitation\` with argument:
|
|||
|
||||
export class ToolsCallAudioScenario implements ClientScenario {
|
||||
name = 'tools-call-audio';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns audio content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
@ -1008,6 +1018,7 @@ Implement tool \`test_audio_content\` with no arguments that returns:
|
|||
|
||||
export class ToolsCallEmbeddedResourceScenario implements ClientScenario {
|
||||
name = 'tools-call-embedded-resource';
|
||||
specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
|
||||
description = `Test calling a tool that returns embedded resource content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
Loading…
Reference in a new issue