feat: support "draft" as a first-class spec version target (#255)
* feat: support "draft" as a first-class spec version target - Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253 * review: collapse SpecVersion to wire strings; dedup NEGOTIABLE_PROTOCOL_VERSIONS; split out 'extension' Addresses review feedback from @felixweinberger and @mikekistler: - SpecVersion is now always a wire protocolVersion string (DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION). The separate 'draft' tag literal is gone from the type system; 'draft' survives only as a CLI input alias in resolveSpecVersion. Removes the tag-vs-wire confusion and the specVersionToProtocolVersion mapping. - NEGOTIABLE_PROTOCOL_VERSIONS in types.ts is the single source for what the mock server accepts on initialize (was duplicated in checks/client.ts and scenarios/client/initialize.ts). - 'extension' moved out of SpecVersion into a separate ScenarioSpecTag type. --spec-version extension is no longer valid; extension scenarios remain reachable via --suite extensions. - Draft-tagged scenarios now use the DRAFT_PROTOCOL_VERSION constant so a draft revision bump is a one-line change in types.ts.
This commit is contained in:
parent
d944122003
commit
17f1f93cc0
17 changed files with 221 additions and 78 deletions
|
|
@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
|
||||||
- `--command` - The command to run your MCP client (can include flags)
|
- `--command` - The command to run your MCP client (can include flags)
|
||||||
- `--scenario` - The test scenario to run (e.g., "initialize")
|
- `--scenario` - The test scenario to run (e.g., "initialize")
|
||||||
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
|
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
|
||||||
|
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios
|
||||||
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
|
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
|
||||||
- `--timeout` - Timeout in milliseconds (default: 30000)
|
- `--timeout` - Timeout in milliseconds (default: 30000)
|
||||||
- `--verbose` - Show verbose output
|
- `--verbose` - Show verbose output
|
||||||
|
|
||||||
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
|
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it.
|
||||||
|
|
||||||
### Server Testing
|
### Server Testing
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { createClientInitializationCheck } from './client';
|
import { createClientInitializationCheck } from './client';
|
||||||
|
import { DRAFT_PROTOCOL_VERSION } from '../types';
|
||||||
|
|
||||||
describe('createClientInitializationCheck', () => {
|
describe('createClientInitializationCheck', () => {
|
||||||
it('should return SUCCESS for a valid initialize request', () => {
|
it('should return SUCCESS for a valid initialize request', () => {
|
||||||
|
|
@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => {
|
||||||
expect(check.errorMessage).toContain('Client version missing');
|
expect(check.errorMessage).toContain('Client version missing');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('should accept the current draft protocol version', () => {
|
||||||
|
const request = {
|
||||||
|
protocolVersion: DRAFT_PROTOCOL_VERSION,
|
||||||
|
clientInfo: { name: 'TestClient', version: '1.0.0' }
|
||||||
|
};
|
||||||
|
|
||||||
|
const check = createClientInitializationCheck(request);
|
||||||
|
expect(check.status).toBe('SUCCESS');
|
||||||
|
expect(check.errorMessage).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(['DRAFT-2025-v1', 'draft'])(
|
||||||
|
'should reject stale or non-canonical draft version %s',
|
||||||
|
(protocolVersion) => {
|
||||||
|
const request = {
|
||||||
|
protocolVersion,
|
||||||
|
clientInfo: { name: 'TestClient', version: '1.0.0' }
|
||||||
|
};
|
||||||
|
|
||||||
|
const check = createClientInitializationCheck(request);
|
||||||
|
expect(check.status).toBe('FAILURE');
|
||||||
|
expect(check.errorMessage).toContain('Version mismatch');
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
it('should support custom expected spec version', () => {
|
it('should support custom expected spec version', () => {
|
||||||
const request = {
|
const request = {
|
||||||
protocolVersion: '2024-11-05',
|
protocolVersion: '2024-11-05',
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,9 @@
|
||||||
import { ConformanceCheck, CheckStatus } from '../types';
|
import {
|
||||||
|
ConformanceCheck,
|
||||||
|
CheckStatus,
|
||||||
|
LATEST_SPEC_VERSION,
|
||||||
|
NEGOTIABLE_PROTOCOL_VERSIONS
|
||||||
|
} from '../types';
|
||||||
|
|
||||||
export function createServerInfoCheck(serverInfo: {
|
export function createServerInfoCheck(serverInfo: {
|
||||||
name: string;
|
name: string;
|
||||||
|
|
@ -23,19 +28,18 @@ export function createServerInfoCheck(serverInfo: {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Valid MCP protocol versions
|
|
||||||
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];
|
|
||||||
|
|
||||||
export function createClientInitializationCheck(
|
export function createClientInitializationCheck(
|
||||||
initializeRequest: any,
|
initializeRequest: any,
|
||||||
expectedSpecVersion: string = '2025-11-25'
|
expectedSpecVersion: string = LATEST_SPEC_VERSION
|
||||||
): ConformanceCheck {
|
): ConformanceCheck {
|
||||||
const protocolVersionSent = initializeRequest?.protocolVersion;
|
const protocolVersionSent = initializeRequest?.protocolVersion;
|
||||||
|
|
||||||
// Accept known valid versions OR custom expected version (for backward compatibility)
|
// Accept known valid versions OR custom expected version (for backward compatibility)
|
||||||
const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion)
|
const validVersions = NEGOTIABLE_PROTOCOL_VERSIONS.includes(
|
||||||
? VALID_PROTOCOL_VERSIONS
|
expectedSpecVersion
|
||||||
: [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion];
|
)
|
||||||
|
? NEGOTIABLE_PROTOCOL_VERSIONS
|
||||||
|
: [...NEGOTIABLE_PROTOCOL_VERSIONS, expectedSpecVersion];
|
||||||
const versionMatch = validVersions.includes(protocolVersionSent);
|
const versionMatch = validVersions.includes(protocolVersionSent);
|
||||||
|
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,8 @@ program
|
||||||
options.command,
|
options.command,
|
||||||
scenarioName,
|
scenarioName,
|
||||||
timeout,
|
timeout,
|
||||||
outputDir
|
outputDir,
|
||||||
|
specVersionFilter
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
scenario: scenarioName,
|
scenario: scenarioName,
|
||||||
|
|
@ -259,7 +260,8 @@ program
|
||||||
validated.command,
|
validated.command,
|
||||||
validated.scenario,
|
validated.scenario,
|
||||||
timeout,
|
timeout,
|
||||||
outputDir
|
outputDir,
|
||||||
|
specVersionFilter
|
||||||
);
|
);
|
||||||
|
|
||||||
const { overallFailure } = printClientResults(
|
const { overallFailure } = printClientResults(
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { spawn } from 'child_process';
|
import { spawn } from 'child_process';
|
||||||
import { promises as fs } from 'fs';
|
import { promises as fs } from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { ConformanceCheck } from '../types';
|
import { ConformanceCheck, SpecVersion } from '../types';
|
||||||
import { getScenario } from '../scenarios';
|
import { getScenario } from '../scenarios';
|
||||||
import { createResultDir, formatPrettyChecks } from './utils';
|
import { createResultDir, formatPrettyChecks } from './utils';
|
||||||
|
|
||||||
|
|
@ -17,7 +17,8 @@ async function executeClient(
|
||||||
scenarioName: string,
|
scenarioName: string,
|
||||||
serverUrl: string,
|
serverUrl: string,
|
||||||
timeout: number = 30000,
|
timeout: number = 30000,
|
||||||
context?: Record<string, unknown>
|
context?: Record<string, unknown>,
|
||||||
|
specVersion?: SpecVersion
|
||||||
): Promise<ClientExecutionResult> {
|
): Promise<ClientExecutionResult> {
|
||||||
const commandParts = command.split(' ');
|
const commandParts = command.split(' ');
|
||||||
const executable = commandParts[0];
|
const executable = commandParts[0];
|
||||||
|
|
@ -34,6 +35,9 @@ async function executeClient(
|
||||||
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
|
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
|
||||||
const env = { ...process.env };
|
const env = { ...process.env };
|
||||||
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
|
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
|
||||||
|
if (specVersion) {
|
||||||
|
env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion;
|
||||||
|
}
|
||||||
if (context) {
|
if (context) {
|
||||||
// Include scenario name in context for discriminated union parsing
|
// Include scenario name in context for discriminated union parsing
|
||||||
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
|
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
|
||||||
|
|
@ -92,7 +96,8 @@ export async function runConformanceTest(
|
||||||
clientCommand: string,
|
clientCommand: string,
|
||||||
scenarioName: string,
|
scenarioName: string,
|
||||||
timeout: number = 30000,
|
timeout: number = 30000,
|
||||||
outputDir?: string
|
outputDir?: string,
|
||||||
|
specVersion?: SpecVersion
|
||||||
): Promise<{
|
): Promise<{
|
||||||
checks: ConformanceCheck[];
|
checks: ConformanceCheck[];
|
||||||
clientOutput: ClientExecutionResult;
|
clientOutput: ClientExecutionResult;
|
||||||
|
|
@ -123,7 +128,8 @@ export async function runConformanceTest(
|
||||||
scenarioName,
|
scenarioName,
|
||||||
urls.serverUrl,
|
urls.serverUrl,
|
||||||
timeout,
|
timeout,
|
||||||
urls.context
|
urls.context,
|
||||||
|
specVersion
|
||||||
);
|
);
|
||||||
|
|
||||||
// Print stdout/stderr if client exited with nonzero code
|
// Print stdout/stderr if client exited with nonzero code
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import type {
|
||||||
Scenario,
|
Scenario,
|
||||||
ConformanceCheck,
|
ConformanceCheck,
|
||||||
ScenarioUrls,
|
ScenarioUrls,
|
||||||
SpecVersion
|
ScenarioSpecTag
|
||||||
} from '../../../types';
|
} from '../../../types';
|
||||||
import { createAuthServer } from './helpers/createAuthServer';
|
import { createAuthServer } from './helpers/createAuthServer';
|
||||||
import { createServer } from './helpers/createServer';
|
import { createServer } from './helpers/createServer';
|
||||||
|
|
@ -37,7 +37,7 @@ async function generateTestKeypair(): Promise<{
|
||||||
*/
|
*/
|
||||||
export class ClientCredentialsJwtScenario implements Scenario {
|
export class ClientCredentialsJwtScenario implements Scenario {
|
||||||
name = 'auth/client-credentials-jwt';
|
name = 'auth/client-credentials-jwt';
|
||||||
specVersions: SpecVersion[] = ['extension'];
|
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||||
description =
|
description =
|
||||||
'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)';
|
'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)';
|
||||||
|
|
||||||
|
|
@ -256,7 +256,7 @@ export class ClientCredentialsJwtScenario implements Scenario {
|
||||||
*/
|
*/
|
||||||
export class ClientCredentialsBasicScenario implements Scenario {
|
export class ClientCredentialsBasicScenario implements Scenario {
|
||||||
name = 'auth/client-credentials-basic';
|
name = 'auth/client-credentials-basic';
|
||||||
specVersions: SpecVersion[] = ['extension'];
|
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||||
description =
|
description =
|
||||||
'Tests OAuth client_credentials flow with client_secret_basic authentication';
|
'Tests OAuth client_credentials flow with client_secret_basic authentication';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import type {
|
||||||
Scenario,
|
Scenario,
|
||||||
ConformanceCheck,
|
ConformanceCheck,
|
||||||
ScenarioUrls,
|
ScenarioUrls,
|
||||||
SpecVersion
|
ScenarioSpecTag
|
||||||
} from '../../../types';
|
} from '../../../types';
|
||||||
import { createAuthServer } from './helpers/createAuthServer';
|
import { createAuthServer } from './helpers/createAuthServer';
|
||||||
import { createServer } from './helpers/createServer';
|
import { createServer } from './helpers/createServer';
|
||||||
|
|
@ -60,7 +60,7 @@ async function createIdpIdToken(
|
||||||
*/
|
*/
|
||||||
export class CrossAppAccessCompleteFlowScenario implements Scenario {
|
export class CrossAppAccessCompleteFlowScenario implements Scenario {
|
||||||
name = 'auth/cross-app-access-complete-flow';
|
name = 'auth/cross-app-access-complete-flow';
|
||||||
specVersions: SpecVersion[] = ['extension'];
|
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||||
description =
|
description =
|
||||||
'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)';
|
'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,9 @@
|
||||||
import type { Scenario, ConformanceCheck } from '../../../types';
|
import type { Scenario, ConformanceCheck } from '../../../types';
|
||||||
import { ScenarioUrls, SpecVersion } from '../../../types';
|
import {
|
||||||
|
ScenarioUrls,
|
||||||
|
SpecVersion,
|
||||||
|
DRAFT_PROTOCOL_VERSION
|
||||||
|
} from '../../../types';
|
||||||
import { createAuthServer } from './helpers/createAuthServer';
|
import { createAuthServer } from './helpers/createAuthServer';
|
||||||
import { createServer } from './helpers/createServer';
|
import { createServer } from './helpers/createServer';
|
||||||
import { ServerLifecycle } from './helpers/serverLifecycle';
|
import { ServerLifecycle } from './helpers/serverLifecycle';
|
||||||
|
|
@ -23,7 +27,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier';
|
||||||
*/
|
*/
|
||||||
export class OfflineAccessScopeScenario implements Scenario {
|
export class OfflineAccessScopeScenario implements Scenario {
|
||||||
name = 'auth/offline-access-scope';
|
name = 'auth/offline-access-scope';
|
||||||
specVersions: SpecVersion[] = ['draft'];
|
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||||
description =
|
description =
|
||||||
'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)';
|
'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)';
|
||||||
|
|
||||||
|
|
@ -227,7 +231,7 @@ export class OfflineAccessScopeScenario implements Scenario {
|
||||||
*/
|
*/
|
||||||
export class OfflineAccessNotSupportedScenario implements Scenario {
|
export class OfflineAccessNotSupportedScenario implements Scenario {
|
||||||
name = 'auth/offline-access-not-supported';
|
name = 'auth/offline-access-not-supported';
|
||||||
specVersions: SpecVersion[] = ['draft'];
|
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||||
description =
|
description =
|
||||||
'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)';
|
'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,9 @@
|
||||||
import type { Scenario, ConformanceCheck } from '../../../types.js';
|
import type { Scenario, ConformanceCheck } from '../../../types.js';
|
||||||
import { ScenarioUrls, SpecVersion } from '../../../types.js';
|
import {
|
||||||
|
ScenarioUrls,
|
||||||
|
SpecVersion,
|
||||||
|
DRAFT_PROTOCOL_VERSION
|
||||||
|
} from '../../../types.js';
|
||||||
import { createAuthServer } from './helpers/createAuthServer.js';
|
import { createAuthServer } from './helpers/createAuthServer.js';
|
||||||
import { createServer } from './helpers/createServer.js';
|
import { createServer } from './helpers/createServer.js';
|
||||||
import { ServerLifecycle } from './helpers/serverLifecycle.js';
|
import { ServerLifecycle } from './helpers/serverLifecycle.js';
|
||||||
|
|
@ -27,7 +31,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier.js';
|
||||||
*/
|
*/
|
||||||
export class ResourceMismatchScenario implements Scenario {
|
export class ResourceMismatchScenario implements Scenario {
|
||||||
name = 'auth/resource-mismatch';
|
name = 'auth/resource-mismatch';
|
||||||
specVersions: SpecVersion[] = ['draft'];
|
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||||
description =
|
description =
|
||||||
'Tests that client rejects when PRM resource does not match server URL';
|
'Tests that client rejects when PRM resource does not match server URL';
|
||||||
allowClientError = true;
|
allowClientError = true;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,9 @@ import {
|
||||||
Scenario,
|
Scenario,
|
||||||
ScenarioUrls,
|
ScenarioUrls,
|
||||||
ConformanceCheck,
|
ConformanceCheck,
|
||||||
SpecVersion
|
SpecVersion,
|
||||||
|
LATEST_SPEC_VERSION,
|
||||||
|
NEGOTIABLE_PROTOCOL_VERSIONS
|
||||||
} from '../../types';
|
} from '../../types';
|
||||||
import { clientChecks } from '../../checks/index';
|
import { clientChecks } from '../../checks/index';
|
||||||
|
|
||||||
|
|
@ -117,11 +119,10 @@ export class InitializeScenario implements Scenario {
|
||||||
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
|
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
|
||||||
|
|
||||||
// Echo back client's version if valid, otherwise use latest
|
// Echo back client's version if valid, otherwise use latest
|
||||||
const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
|
|
||||||
const clientVersion = initializeRequest?.protocolVersion;
|
const clientVersion = initializeRequest?.protocolVersion;
|
||||||
const responseVersion = VALID_VERSIONS.includes(clientVersion)
|
const responseVersion = NEGOTIABLE_PROTOCOL_VERSIONS.includes(clientVersion)
|
||||||
? clientVersion
|
? clientVersion
|
||||||
: '2025-11-25';
|
: LATEST_SPEC_VERSION;
|
||||||
|
|
||||||
const response = {
|
const response = {
|
||||||
jsonrpc: '2.0',
|
jsonrpc: '2.0',
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,11 @@ import {
|
||||||
Scenario,
|
Scenario,
|
||||||
ClientScenario,
|
ClientScenario,
|
||||||
ClientScenarioForAuthorizationServer,
|
ClientScenarioForAuthorizationServer,
|
||||||
SpecVersion
|
SpecVersion,
|
||||||
|
ScenarioSpecTag,
|
||||||
|
DATED_SPEC_VERSIONS,
|
||||||
|
DRAFT_PROTOCOL_VERSION,
|
||||||
|
LATEST_SPEC_VERSION
|
||||||
} from '../types';
|
} from '../types';
|
||||||
import { InitializeScenario } from './client/initialize';
|
import { InitializeScenario } from './client/initialize';
|
||||||
import { ToolsCallScenario } from './client/tools_call';
|
import { ToolsCallScenario } from './client/tools_call';
|
||||||
|
|
@ -256,32 +260,50 @@ export function listDraftScenarios(): string[] {
|
||||||
export { listMetadataScenarios };
|
export { listMetadataScenarios };
|
||||||
|
|
||||||
// All valid spec versions, used by the CLI to validate --spec-version input.
|
// All valid spec versions, used by the CLI to validate --spec-version input.
|
||||||
|
// 'extension' is intentionally excluded — extension scenarios are off-timeline
|
||||||
|
// and selected via `--suite extensions`, not `--spec-version`.
|
||||||
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
|
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
|
||||||
'2025-03-26',
|
...DATED_SPEC_VERSIONS,
|
||||||
'2025-06-18',
|
DRAFT_PROTOCOL_VERSION
|
||||||
'2025-11-25',
|
|
||||||
'draft',
|
|
||||||
'extension'
|
|
||||||
];
|
];
|
||||||
|
|
||||||
export function resolveSpecVersion(value: string): SpecVersion {
|
export function resolveSpecVersion(value: string): SpecVersion {
|
||||||
|
if (value === 'draft') return DRAFT_PROTOCOL_VERSION;
|
||||||
if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
|
if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
|
||||||
return value as SpecVersion;
|
return value as SpecVersion;
|
||||||
}
|
}
|
||||||
console.error(`Unknown spec version: ${value}`);
|
console.error(`Unknown spec version: ${value}`);
|
||||||
console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
|
console.error(
|
||||||
|
`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})`
|
||||||
|
);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The draft version selects everything in the latest dated release plus
|
||||||
|
// scenarios tagged draft-only, so SEP authors can run the full suite against an
|
||||||
|
// SDK tracking the in-progress spec without retagging core scenarios.
|
||||||
|
function matchesSpecVersion(
|
||||||
|
scenario: { specVersions: ScenarioSpecTag[] },
|
||||||
|
version: SpecVersion
|
||||||
|
): boolean {
|
||||||
|
if (version === DRAFT_PROTOCOL_VERSION) {
|
||||||
|
return (
|
||||||
|
scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) ||
|
||||||
|
scenario.specVersions.includes(LATEST_SPEC_VERSION)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return scenario.specVersions.includes(version);
|
||||||
|
}
|
||||||
|
|
||||||
export function listScenariosForSpec(version: SpecVersion): string[] {
|
export function listScenariosForSpec(version: SpecVersion): string[] {
|
||||||
return scenariosList
|
return scenariosList
|
||||||
.filter((s) => s.specVersions.includes(version))
|
.filter((s) => matchesSpecVersion(s, version))
|
||||||
.map((s) => s.name);
|
.map((s) => s.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listClientScenariosForSpec(version: SpecVersion): string[] {
|
export function listClientScenariosForSpec(version: SpecVersion): string[] {
|
||||||
return allClientScenariosList
|
return allClientScenariosList
|
||||||
.filter((s) => s.specVersions.includes(version))
|
.filter((s) => matchesSpecVersion(s, version))
|
||||||
.map((s) => s.name);
|
.map((s) => s.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -289,13 +311,13 @@ export function listClientScenariosForAuthorizationServerForSpec(
|
||||||
version: SpecVersion
|
version: SpecVersion
|
||||||
): string[] {
|
): string[] {
|
||||||
return allClientScenariosListForAuthorizationServer
|
return allClientScenariosListForAuthorizationServer
|
||||||
.filter((s) => s.specVersions.includes(version))
|
.filter((s) => matchesSpecVersion(s, version))
|
||||||
.map((s) => s.name);
|
.map((s) => s.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getScenarioSpecVersions(
|
export function getScenarioSpecVersions(
|
||||||
name: string
|
name: string
|
||||||
): SpecVersion[] | undefined {
|
): ScenarioSpecTag[] | undefined {
|
||||||
return (
|
return (
|
||||||
scenarios.get(name)?.specVersions ??
|
scenarios.get(name)?.specVersions ??
|
||||||
clientScenarios.get(name)?.specVersions ??
|
clientScenarios.get(name)?.specVersions ??
|
||||||
|
|
@ -303,4 +325,4 @@ export function getScenarioSpecVersions(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export type { SpecVersion };
|
export type { SpecVersion, ScenarioSpecTag };
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,12 @@
|
||||||
* Resources test scenarios for MCP servers
|
* Resources test scenarios for MCP servers
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
import {
|
||||||
|
ClientScenario,
|
||||||
|
ConformanceCheck,
|
||||||
|
SpecVersion,
|
||||||
|
DRAFT_PROTOCOL_VERSION
|
||||||
|
} from '../../types';
|
||||||
import { connectToServer } from './client-helper';
|
import { connectToServer } from './client-helper';
|
||||||
import {
|
import {
|
||||||
TextResourceContents,
|
TextResourceContents,
|
||||||
|
|
@ -438,7 +443,7 @@ Example request:
|
||||||
|
|
||||||
export class ResourcesNotFoundErrorScenario implements ClientScenario {
|
export class ResourcesNotFoundErrorScenario implements ClientScenario {
|
||||||
name = 'sep-2164-resource-not-found';
|
name = 'sep-2164-resource-not-found';
|
||||||
specVersions: SpecVersion[] = ['draft'];
|
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||||
description = `Test error handling for non-existent resources (SEP-2164).
|
description = `Test error handling for non-existent resources (SEP-2164).
|
||||||
|
|
||||||
**Server Implementation Requirements:**
|
**Server Implementation Requirements:**
|
||||||
|
|
|
||||||
|
|
@ -3,9 +3,23 @@ import {
|
||||||
listScenarios,
|
listScenarios,
|
||||||
listClientScenarios,
|
listClientScenarios,
|
||||||
listScenariosForSpec,
|
listScenariosForSpec,
|
||||||
|
listDraftScenarios,
|
||||||
|
listExtensionScenarios,
|
||||||
getScenarioSpecVersions,
|
getScenarioSpecVersions,
|
||||||
|
resolveSpecVersion,
|
||||||
ALL_SPEC_VERSIONS
|
ALL_SPEC_VERSIONS
|
||||||
} from './index';
|
} from './index';
|
||||||
|
import {
|
||||||
|
DATED_SPEC_VERSIONS,
|
||||||
|
DRAFT_PROTOCOL_VERSION,
|
||||||
|
LATEST_SPEC_VERSION,
|
||||||
|
ScenarioSpecTag
|
||||||
|
} from '../types';
|
||||||
|
|
||||||
|
const ALL_SCENARIO_SPEC_TAGS: ScenarioSpecTag[] = [
|
||||||
|
...ALL_SPEC_VERSIONS,
|
||||||
|
'extension'
|
||||||
|
];
|
||||||
|
|
||||||
describe('specVersions helpers', () => {
|
describe('specVersions helpers', () => {
|
||||||
it('every Scenario has specVersions', () => {
|
it('every Scenario has specVersions', () => {
|
||||||
|
|
@ -17,7 +31,7 @@ describe('specVersions helpers', () => {
|
||||||
).toBeDefined();
|
).toBeDefined();
|
||||||
expect(versions!.length).toBeGreaterThan(0);
|
expect(versions!.length).toBeGreaterThan(0);
|
||||||
for (const v of versions!) {
|
for (const v of versions!) {
|
||||||
expect(ALL_SPEC_VERSIONS).toContain(v);
|
expect(ALL_SCENARIO_SPEC_TAGS).toContain(v);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -31,7 +45,7 @@ describe('specVersions helpers', () => {
|
||||||
).toBeDefined();
|
).toBeDefined();
|
||||||
expect(versions!.length).toBeGreaterThan(0);
|
expect(versions!.length).toBeGreaterThan(0);
|
||||||
for (const v of versions!) {
|
for (const v of versions!) {
|
||||||
expect(ALL_SPEC_VERSIONS).toContain(v);
|
expect(ALL_SCENARIO_SPEC_TAGS).toContain(v);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -69,26 +83,43 @@ describe('specVersions helpers', () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it('draft and extension scenarios are isolated', () => {
|
it('the draft spec version is a superset of the latest dated release', () => {
|
||||||
const draft = listScenariosForSpec('draft');
|
const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION));
|
||||||
for (const name of draft) {
|
const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION));
|
||||||
expect(getScenarioSpecVersions(name)).toContain('draft');
|
for (const name of latest) {
|
||||||
|
expect(draft.has(name)).toBe(true);
|
||||||
}
|
}
|
||||||
const ext = listScenariosForSpec('extension');
|
for (const name of listDraftScenarios()) {
|
||||||
for (const name of ext) {
|
expect(draft.has(name)).toBe(true);
|
||||||
expect(getScenarioSpecVersions(name)).toContain('extension');
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it('draft scenarios are not in dated versions', () => {
|
it('draft-tagged scenarios are not also tagged with a dated version', () => {
|
||||||
const draft = listScenariosForSpec('draft');
|
for (const name of listDraftScenarios()) {
|
||||||
const dated = new Set([
|
const versions = getScenarioSpecVersions(name)!;
|
||||||
...listScenariosForSpec('2025-03-26'),
|
for (const dated of DATED_SPEC_VERSIONS) {
|
||||||
...listScenariosForSpec('2025-06-18'),
|
expect(
|
||||||
...listScenariosForSpec('2025-11-25')
|
versions,
|
||||||
]);
|
`scenario "${name}" is tagged with both DRAFT_PROTOCOL_VERSION and '${dated}'`
|
||||||
for (const name of draft) {
|
).not.toContain(dated);
|
||||||
expect(dated.has(name)).toBe(false);
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolveSpecVersion accepts 'draft' as an alias", () => {
|
||||||
|
expect(resolveSpecVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION);
|
||||||
|
expect(resolveSpecVersion(LATEST_SPEC_VERSION)).toBe(LATEST_SPEC_VERSION);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extension-tagged scenarios are not selected by any --spec-version', () => {
|
||||||
|
for (const version of ALL_SPEC_VERSIONS) {
|
||||||
|
const selected = new Set(listScenariosForSpec(version));
|
||||||
|
for (const name of listExtensionScenarios()) {
|
||||||
|
expect(
|
||||||
|
selected.has(name),
|
||||||
|
`extension scenario "${name}" was selected by --spec-version ${version}`
|
||||||
|
).toBe(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -10,14 +10,22 @@ import {
|
||||||
listClientScenariosForSpec,
|
listClientScenariosForSpec,
|
||||||
getScenarioSpecVersions
|
getScenarioSpecVersions
|
||||||
} from '../../scenarios';
|
} from '../../scenarios';
|
||||||
import { ConformanceCheck, SpecVersion } from '../../types';
|
import {
|
||||||
|
ConformanceCheck,
|
||||||
|
DRAFT_PROTOCOL_VERSION,
|
||||||
|
ScenarioSpecTag,
|
||||||
|
SpecVersion
|
||||||
|
} from '../../types';
|
||||||
|
|
||||||
const NON_SCORING_VERSIONS: SpecVersion[] = ['draft', 'extension'];
|
const NON_SCORING_TAGS: ScenarioSpecTag[] = [
|
||||||
|
DRAFT_PROTOCOL_VERSION,
|
||||||
|
'extension'
|
||||||
|
];
|
||||||
|
|
||||||
/** Whether a scenario counts toward tier scoring (has at least one date-versioned spec). */
|
/** Whether a scenario counts toward tier scoring (has at least one date-versioned spec). */
|
||||||
function isTierScoring(specVersions?: SpecVersion[]): boolean {
|
function isTierScoring(specVersions?: ScenarioSpecTag[]): boolean {
|
||||||
if (!specVersions || specVersions.length === 0) return true; // unknown = count it
|
if (!specVersions || specVersions.length === 0) return true; // unknown = count it
|
||||||
return specVersions.some((v) => !NON_SCORING_VERSIONS.includes(v));
|
return specVersions.some((v) => !NON_SCORING_TAGS.includes(v));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { TierScorecard, CheckStatus, ConformanceResult } from './types';
|
import { TierScorecard, CheckStatus, ConformanceResult } from './types';
|
||||||
|
import { DATED_SPEC_VERSIONS, DRAFT_PROTOCOL_VERSION } from '../types';
|
||||||
|
|
||||||
const COLORS = {
|
const COLORS = {
|
||||||
RESET: '\x1b[0m',
|
RESET: '\x1b[0m',
|
||||||
|
|
@ -23,9 +24,9 @@ function statusIcon(status: CheckStatus): string {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const TIER_SPEC_VERSIONS = ['2025-03-26', '2025-06-18', '2025-11-25'] as const;
|
const TIER_SPEC_VERSIONS = DATED_SPEC_VERSIONS;
|
||||||
|
|
||||||
const INFO_SPEC_VERSIONS = ['draft', 'extension'] as const;
|
const INFO_SPEC_VERSIONS = [DRAFT_PROTOCOL_VERSION, 'extension'] as const;
|
||||||
|
|
||||||
type Cell = { passed: number; total: number };
|
type Cell = { passed: number; total: number };
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { SpecVersion } from '../types';
|
import type { ScenarioSpecTag } from '../types';
|
||||||
|
|
||||||
export type CheckStatus = 'pass' | 'fail' | 'partial' | 'skipped';
|
export type CheckStatus = 'pass' | 'fail' | 'partial' | 'skipped';
|
||||||
|
|
||||||
|
|
@ -17,7 +17,7 @@ export interface ConformanceResult extends CheckResult {
|
||||||
passed: boolean;
|
passed: boolean;
|
||||||
checks_passed: number;
|
checks_passed: number;
|
||||||
checks_failed: number;
|
checks_failed: number;
|
||||||
specVersions?: SpecVersion[];
|
specVersions?: ScenarioSpecTag[];
|
||||||
}>;
|
}>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
46
src/types.ts
46
src/types.ts
|
|
@ -23,12 +23,40 @@ export interface ConformanceCheck {
|
||||||
logs?: string[];
|
logs?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SpecVersion =
|
export const DATED_SPEC_VERSIONS = [
|
||||||
| '2025-03-26'
|
'2025-03-26',
|
||||||
| '2025-06-18'
|
'2025-06-18',
|
||||||
| '2025-11-25'
|
'2025-11-25'
|
||||||
| 'draft'
|
] as const;
|
||||||
| 'extension';
|
|
||||||
|
export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number];
|
||||||
|
|
||||||
|
export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wire `protocolVersion` for the in-progress spec. Mirrors
|
||||||
|
* `LATEST_PROTOCOL_VERSION` in the spec repo's `schema/draft/schema.ts`;
|
||||||
|
* bump when that constant changes.
|
||||||
|
*/
|
||||||
|
export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1';
|
||||||
|
|
||||||
|
// Wire protocolVersion strings the mock server will negotiate on initialize.
|
||||||
|
export const NEGOTIABLE_PROTOCOL_VERSIONS: readonly string[] = [
|
||||||
|
'2025-06-18',
|
||||||
|
LATEST_SPEC_VERSION,
|
||||||
|
DRAFT_PROTOCOL_VERSION
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A spec revision the conformance suite can target via `--spec-version`.
|
||||||
|
* Always a wire `protocolVersion` string. The CLI also accepts `'draft'` as
|
||||||
|
* an alias for {@link DRAFT_PROTOCOL_VERSION}.
|
||||||
|
*/
|
||||||
|
export type SpecVersion = DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION;
|
||||||
|
|
||||||
|
// Scenarios may also be tagged 'extension' to mark them as off-timeline
|
||||||
|
// (selectable via --suite extensions, never via --spec-version). See #256.
|
||||||
|
export type ScenarioSpecTag = SpecVersion | 'extension';
|
||||||
|
|
||||||
export interface ScenarioUrls {
|
export interface ScenarioUrls {
|
||||||
serverUrl: string;
|
serverUrl: string;
|
||||||
|
|
@ -43,7 +71,7 @@ export interface ScenarioUrls {
|
||||||
export interface Scenario {
|
export interface Scenario {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
specVersions: SpecVersion[];
|
specVersions: ScenarioSpecTag[];
|
||||||
/**
|
/**
|
||||||
* If true, a non-zero client exit code is expected and will not cause the test to fail.
|
* If true, a non-zero client exit code is expected and will not cause the test to fail.
|
||||||
* Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth).
|
* Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth).
|
||||||
|
|
@ -57,13 +85,13 @@ export interface Scenario {
|
||||||
export interface ClientScenario {
|
export interface ClientScenario {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
specVersions: SpecVersion[];
|
specVersions: ScenarioSpecTag[];
|
||||||
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClientScenarioForAuthorizationServer {
|
export interface ClientScenarioForAuthorizationServer {
|
||||||
name: string;
|
name: string;
|
||||||
description: string;
|
description: string;
|
||||||
specVersions: SpecVersion[];
|
specVersions: ScenarioSpecTag[];
|
||||||
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue