conformance/src/scenarios/index.ts
Paul Carleton 17f1f93cc0
feat: support "draft" as a first-class spec version target (#255)
* feat: support "draft" as a first-class spec version target

- Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next
  spec release is a one-line change in types.ts.
- Accept "draft" as a valid protocolVersion in the initialize check and
  mock-server response.
- --spec-version draft now selects latest-dated scenarios plus draft-tagged
  ones, so SEP authors can run the full suite against an SDK tracking the
  in-progress spec without retagging core scenarios.
- Forward --spec-version to the client process via
  MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching
  protocolVersion.

Closes #253

* review: collapse SpecVersion to wire strings; dedup NEGOTIABLE_PROTOCOL_VERSIONS; split out 'extension'

Addresses review feedback from @felixweinberger and @mikekistler:

- SpecVersion is now always a wire protocolVersion string
  (DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION). The separate
  'draft' tag literal is gone from the type system; 'draft' survives
  only as a CLI input alias in resolveSpecVersion. Removes the
  tag-vs-wire confusion and the specVersionToProtocolVersion mapping.
- NEGOTIABLE_PROTOCOL_VERSIONS in types.ts is the single source for
  what the mock server accepts on initialize (was duplicated in
  checks/client.ts and scenarios/client/initialize.ts).
- 'extension' moved out of SpecVersion into a separate ScenarioSpecTag
  type. --spec-version extension is no longer valid; extension scenarios
  remain reachable via --suite extensions.
- Draft-tagged scenarios now use the DRAFT_PROTOCOL_VERSION constant
  so a draft revision bump is a one-line change in types.ts.
2026-04-30 11:37:32 +01:00

328 lines
9.7 KiB
TypeScript

import {
Scenario,
ClientScenario,
ClientScenarioForAuthorizationServer,
SpecVersion,
ScenarioSpecTag,
DATED_SPEC_VERSIONS,
DRAFT_PROTOCOL_VERSION,
LATEST_SPEC_VERSION
} from '../types';
import { InitializeScenario } from './client/initialize';
import { ToolsCallScenario } from './client/tools_call';
import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults';
import { SSERetryScenario } from './client/sse-retry';
// Import all new server test scenarios
import { ServerInitializeScenario } from './server/lifecycle';
import {
PingScenario,
LoggingSetLevelScenario,
CompletionCompleteScenario
} from './server/utils';
import {
ToolsListScenario,
ToolsCallSimpleTextScenario,
ToolsCallImageScenario,
ToolsCallMultipleContentTypesScenario,
ToolsCallWithLoggingScenario,
ToolsCallErrorScenario,
ToolsCallWithProgressScenario,
ToolsCallSamplingScenario,
ToolsCallElicitationScenario,
ToolsCallAudioScenario,
ToolsCallEmbeddedResourceScenario
} from './server/tools';
import { JsonSchema2020_12Scenario } from './server/json-schema-2020-12';
import { ElicitationDefaultsScenario } from './server/elicitation-defaults';
import { ElicitationEnumsScenario } from './server/elicitation-enums';
import { ServerSSEPollingScenario } from './server/sse-polling';
import { ServerSSEMultipleStreamsScenario } from './server/sse-multiple-streams';
import {
ResourcesListScenario,
ResourcesReadTextScenario,
ResourcesReadBinaryScenario,
ResourcesTemplateReadScenario,
ResourcesSubscribeScenario,
ResourcesUnsubscribeScenario,
ResourcesNotFoundErrorScenario
} from './server/resources';
import {
PromptsListScenario,
PromptsGetSimpleScenario,
PromptsGetWithArgsScenario,
PromptsGetEmbeddedResourceScenario,
PromptsGetWithImageScenario
} from './server/prompts';
import { DNSRebindingProtectionScenario } from './server/dns-rebinding';
import {
authScenariosList,
backcompatScenariosList,
draftScenariosList,
extensionScenariosList
} from './client/auth/index';
import { listMetadataScenarios } from './client/auth/discovery-metadata';
import { AuthorizationServerMetadataEndpointScenario } from './authorization-server/authorization-server-metadata';
// Pending client scenarios (not yet fully tested/implemented)
const pendingClientScenariosList: ClientScenario[] = [
// JSON Schema 2020-12 (SEP-1613)
// This test is pending until the SDK includes PR #1135 which preserves
// $schema, $defs, and additionalProperties fields in tool schemas.
new JsonSchema2020_12Scenario(),
// On hold until server-side SSE improvements are made
// https://github.com/modelcontextprotocol/typescript-sdk/pull/1129
new ServerSSEPollingScenario()
];
// All client scenarios
const allClientScenariosList: ClientScenario[] = [
// Lifecycle scenarios
new ServerInitializeScenario(),
// Utilities scenarios
new LoggingSetLevelScenario(),
new PingScenario(),
new CompletionCompleteScenario(),
// Tools scenarios
new ToolsListScenario(),
new ToolsCallSimpleTextScenario(),
new ToolsCallImageScenario(),
new ToolsCallAudioScenario(),
new ToolsCallEmbeddedResourceScenario(),
new ToolsCallMultipleContentTypesScenario(),
new ToolsCallWithLoggingScenario(),
new ToolsCallErrorScenario(),
new ToolsCallWithProgressScenario(),
new ToolsCallSamplingScenario(),
new ToolsCallElicitationScenario(),
// JSON Schema 2020-12 support (SEP-1613)
new JsonSchema2020_12Scenario(),
// Elicitation scenarios (SEP-1034)
new ElicitationDefaultsScenario(),
// SSE Polling scenarios (SEP-1699)
new ServerSSEPollingScenario(),
new ServerSSEMultipleStreamsScenario(),
// Elicitation scenarios (SEP-1330) - pending
new ElicitationEnumsScenario(),
// Resources scenarios
new ResourcesListScenario(),
new ResourcesReadTextScenario(),
new ResourcesReadBinaryScenario(),
new ResourcesTemplateReadScenario(),
new ResourcesSubscribeScenario(),
new ResourcesUnsubscribeScenario(),
// Resources error handling (SEP-2164)
new ResourcesNotFoundErrorScenario(),
// Prompts scenarios
new PromptsListScenario(),
new PromptsGetSimpleScenario(),
new PromptsGetWithArgsScenario(),
new PromptsGetEmbeddedResourceScenario(),
new PromptsGetWithImageScenario(),
// Security scenarios
new DNSRebindingProtectionScenario()
];
// Active client scenarios (excludes pending)
const activeClientScenariosList: ClientScenario[] =
allClientScenariosList.filter(
(scenario) =>
!pendingClientScenariosList.some(
(pending) => pending.name === scenario.name
)
);
// Client scenarios map - built from list
export const clientScenarios = new Map<string, ClientScenario>(
allClientScenariosList.map((scenario) => [scenario.name, scenario])
);
// All client scenarios for authorization server
const allClientScenariosListForAuthorizationServer: ClientScenario[] = [
// Authorization server scenarios
new AuthorizationServerMetadataEndpointScenario()
];
// Client scenarios map for authorization server - built from list
export const clientScenariosForAuthorizationServer = new Map<
string,
ClientScenario
>(
allClientScenariosListForAuthorizationServer.map((scenario) => [
scenario.name,
scenario
])
);
// All client test scenarios (core + backcompat + extensions)
const scenariosList: Scenario[] = [
new InitializeScenario(),
new ToolsCallScenario(),
new ElicitationClientDefaultsScenario(),
new SSERetryScenario(),
...authScenariosList,
...backcompatScenariosList,
...draftScenariosList,
...extensionScenariosList
];
// Core scenarios (tier 1 requirements)
const coreScenariosList: Scenario[] = [
new InitializeScenario(),
new ToolsCallScenario(),
new ElicitationClientDefaultsScenario(),
new SSERetryScenario(),
...authScenariosList
];
// Scenarios map - built from list
export const scenarios = new Map<string, Scenario>(
scenariosList.map((scenario) => [scenario.name, scenario])
);
export function registerScenario(name: string, scenario: Scenario): void {
scenarios.set(name, scenario);
}
export function getScenario(name: string): Scenario | undefined {
return scenarios.get(name);
}
export function getClientScenario(name: string): ClientScenario | undefined {
return clientScenarios.get(name);
}
export function getClientScenarioForAuthorizationServer(
name: string
): ClientScenarioForAuthorizationServer | undefined {
return clientScenariosForAuthorizationServer.get(name);
}
export function listScenarios(): string[] {
return Array.from(scenarios.keys());
}
export function listClientScenarios(): string[] {
return Array.from(clientScenarios.keys());
}
export function listActiveClientScenarios(): string[] {
return activeClientScenariosList.map((scenario) => scenario.name);
}
export function listPendingClientScenarios(): string[] {
return pendingClientScenariosList.map((scenario) => scenario.name);
}
export function listAuthScenarios(): string[] {
return authScenariosList.map((scenario) => scenario.name);
}
export function listCoreScenarios(): string[] {
return coreScenariosList.map((scenario) => scenario.name);
}
export function listExtensionScenarios(): string[] {
return extensionScenariosList.map((scenario) => scenario.name);
}
export function listBackcompatScenarios(): string[] {
return backcompatScenariosList.map((scenario) => scenario.name);
}
export function listClientScenariosForAuthorizationServer(): string[] {
return Array.from(clientScenariosForAuthorizationServer.keys());
}
export function listDraftScenarios(): string[] {
return draftScenariosList.map((scenario) => scenario.name);
}
export { listMetadataScenarios };
// All valid spec versions, used by the CLI to validate --spec-version input.
// 'extension' is intentionally excluded — extension scenarios are off-timeline
// and selected via `--suite extensions`, not `--spec-version`.
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
...DATED_SPEC_VERSIONS,
DRAFT_PROTOCOL_VERSION
];
export function resolveSpecVersion(value: string): SpecVersion {
if (value === 'draft') return DRAFT_PROTOCOL_VERSION;
if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
return value as SpecVersion;
}
console.error(`Unknown spec version: ${value}`);
console.error(
`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})`
);
process.exit(1);
}
// The draft version selects everything in the latest dated release plus
// scenarios tagged draft-only, so SEP authors can run the full suite against an
// SDK tracking the in-progress spec without retagging core scenarios.
function matchesSpecVersion(
scenario: { specVersions: ScenarioSpecTag[] },
version: SpecVersion
): boolean {
if (version === DRAFT_PROTOCOL_VERSION) {
return (
scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) ||
scenario.specVersions.includes(LATEST_SPEC_VERSION)
);
}
return scenario.specVersions.includes(version);
}
export function listScenariosForSpec(version: SpecVersion): string[] {
return scenariosList
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}
export function listClientScenariosForSpec(version: SpecVersion): string[] {
return allClientScenariosList
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}
export function listClientScenariosForAuthorizationServerForSpec(
version: SpecVersion
): string[] {
return allClientScenariosListForAuthorizationServer
.filter((s) => matchesSpecVersion(s, version))
.map((s) => s.name);
}
export function getScenarioSpecVersions(
name: string
): ScenarioSpecTag[] | undefined {
return (
scenarios.get(name)?.specVersions ??
clientScenarios.get(name)?.specVersions ??
clientScenariosForAuthorizationServer.get(name)?.specVersions
);
}
export type { SpecVersion, ScenarioSpecTag };