feat: support "draft" as a first-class spec version target (#255)
* feat: support "draft" as a first-class spec version target - Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253 * review: collapse SpecVersion to wire strings; dedup NEGOTIABLE_PROTOCOL_VERSIONS; split out 'extension' Addresses review feedback from @felixweinberger and @mikekistler: - SpecVersion is now always a wire protocolVersion string (DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION). The separate 'draft' tag literal is gone from the type system; 'draft' survives only as a CLI input alias in resolveSpecVersion. Removes the tag-vs-wire confusion and the specVersionToProtocolVersion mapping. - NEGOTIABLE_PROTOCOL_VERSIONS in types.ts is the single source for what the mock server accepts on initialize (was duplicated in checks/client.ts and scenarios/client/initialize.ts). - 'extension' moved out of SpecVersion into a separate ScenarioSpecTag type. --spec-version extension is no longer valid; extension scenarios remain reachable via --suite extensions. - Draft-tagged scenarios now use the DRAFT_PROTOCOL_VERSION constant so a draft revision bump is a one-line change in types.ts.
This commit is contained in:
parent
d944122003
commit
17f1f93cc0
17 changed files with 221 additions and 78 deletions
|
|
@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
|
|||
- `--command` - The command to run your MCP client (can include flags)
|
||||
- `--scenario` - The test scenario to run (e.g., "initialize")
|
||||
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
|
||||
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios
|
||||
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
|
||||
- `--timeout` - Timeout in milliseconds (default: 30000)
|
||||
- `--verbose` - Show verbose output
|
||||
|
||||
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
|
||||
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it.
|
||||
|
||||
### Server Testing
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { createClientInitializationCheck } from './client';
|
||||
import { DRAFT_PROTOCOL_VERSION } from '../types';
|
||||
|
||||
describe('createClientInitializationCheck', () => {
|
||||
it('should return SUCCESS for a valid initialize request', () => {
|
||||
|
|
@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => {
|
|||
expect(check.errorMessage).toContain('Client version missing');
|
||||
});
|
||||
|
||||
it('should accept the current draft protocol version', () => {
|
||||
const request = {
|
||||
protocolVersion: DRAFT_PROTOCOL_VERSION,
|
||||
clientInfo: { name: 'TestClient', version: '1.0.0' }
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(request);
|
||||
expect(check.status).toBe('SUCCESS');
|
||||
expect(check.errorMessage).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(['DRAFT-2025-v1', 'draft'])(
|
||||
'should reject stale or non-canonical draft version %s',
|
||||
(protocolVersion) => {
|
||||
const request = {
|
||||
protocolVersion,
|
||||
clientInfo: { name: 'TestClient', version: '1.0.0' }
|
||||
};
|
||||
|
||||
const check = createClientInitializationCheck(request);
|
||||
expect(check.status).toBe('FAILURE');
|
||||
expect(check.errorMessage).toContain('Version mismatch');
|
||||
}
|
||||
);
|
||||
|
||||
it('should support custom expected spec version', () => {
|
||||
const request = {
|
||||
protocolVersion: '2024-11-05',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import { ConformanceCheck, CheckStatus } from '../types';
|
||||
import {
|
||||
ConformanceCheck,
|
||||
CheckStatus,
|
||||
LATEST_SPEC_VERSION,
|
||||
NEGOTIABLE_PROTOCOL_VERSIONS
|
||||
} from '../types';
|
||||
|
||||
export function createServerInfoCheck(serverInfo: {
|
||||
name: string;
|
||||
|
|
@ -23,19 +28,18 @@ export function createServerInfoCheck(serverInfo: {
|
|||
};
|
||||
}
|
||||
|
||||
// Valid MCP protocol versions
|
||||
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];
|
||||
|
||||
export function createClientInitializationCheck(
|
||||
initializeRequest: any,
|
||||
expectedSpecVersion: string = '2025-11-25'
|
||||
expectedSpecVersion: string = LATEST_SPEC_VERSION
|
||||
): ConformanceCheck {
|
||||
const protocolVersionSent = initializeRequest?.protocolVersion;
|
||||
|
||||
// Accept known valid versions OR custom expected version (for backward compatibility)
|
||||
const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion)
|
||||
? VALID_PROTOCOL_VERSIONS
|
||||
: [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion];
|
||||
const validVersions = NEGOTIABLE_PROTOCOL_VERSIONS.includes(
|
||||
expectedSpecVersion
|
||||
)
|
||||
? NEGOTIABLE_PROTOCOL_VERSIONS
|
||||
: [...NEGOTIABLE_PROTOCOL_VERSIONS, expectedSpecVersion];
|
||||
const versionMatch = validVersions.includes(protocolVersionSent);
|
||||
|
||||
const errors: string[] = [];
|
||||
|
|
|
|||
|
|
@ -152,7 +152,8 @@ program
|
|||
options.command,
|
||||
scenarioName,
|
||||
timeout,
|
||||
outputDir
|
||||
outputDir,
|
||||
specVersionFilter
|
||||
);
|
||||
return {
|
||||
scenario: scenarioName,
|
||||
|
|
@ -259,7 +260,8 @@ program
|
|||
validated.command,
|
||||
validated.scenario,
|
||||
timeout,
|
||||
outputDir
|
||||
outputDir,
|
||||
specVersionFilter
|
||||
);
|
||||
|
||||
const { overallFailure } = printClientResults(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { spawn } from 'child_process';
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { ConformanceCheck } from '../types';
|
||||
import { ConformanceCheck, SpecVersion } from '../types';
|
||||
import { getScenario } from '../scenarios';
|
||||
import { createResultDir, formatPrettyChecks } from './utils';
|
||||
|
||||
|
|
@ -17,7 +17,8 @@ async function executeClient(
|
|||
scenarioName: string,
|
||||
serverUrl: string,
|
||||
timeout: number = 30000,
|
||||
context?: Record<string, unknown>
|
||||
context?: Record<string, unknown>,
|
||||
specVersion?: SpecVersion
|
||||
): Promise<ClientExecutionResult> {
|
||||
const commandParts = command.split(' ');
|
||||
const executable = commandParts[0];
|
||||
|
|
@ -34,6 +35,9 @@ async function executeClient(
|
|||
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
|
||||
const env = { ...process.env };
|
||||
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
|
||||
if (specVersion) {
|
||||
env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion;
|
||||
}
|
||||
if (context) {
|
||||
// Include scenario name in context for discriminated union parsing
|
||||
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
|
||||
|
|
@ -92,7 +96,8 @@ export async function runConformanceTest(
|
|||
clientCommand: string,
|
||||
scenarioName: string,
|
||||
timeout: number = 30000,
|
||||
outputDir?: string
|
||||
outputDir?: string,
|
||||
specVersion?: SpecVersion
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
clientOutput: ClientExecutionResult;
|
||||
|
|
@ -123,7 +128,8 @@ export async function runConformanceTest(
|
|||
scenarioName,
|
||||
urls.serverUrl,
|
||||
timeout,
|
||||
urls.context
|
||||
urls.context,
|
||||
specVersion
|
||||
);
|
||||
|
||||
// Print stdout/stderr if client exited with nonzero code
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import type {
|
|||
Scenario,
|
||||
ConformanceCheck,
|
||||
ScenarioUrls,
|
||||
SpecVersion
|
||||
ScenarioSpecTag
|
||||
} from '../../../types';
|
||||
import { createAuthServer } from './helpers/createAuthServer';
|
||||
import { createServer } from './helpers/createServer';
|
||||
|
|
@ -37,7 +37,7 @@ async function generateTestKeypair(): Promise<{
|
|||
*/
|
||||
export class ClientCredentialsJwtScenario implements Scenario {
|
||||
name = 'auth/client-credentials-jwt';
|
||||
specVersions: SpecVersion[] = ['extension'];
|
||||
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||
description =
|
||||
'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)';
|
||||
|
||||
|
|
@ -256,7 +256,7 @@ export class ClientCredentialsJwtScenario implements Scenario {
|
|||
*/
|
||||
export class ClientCredentialsBasicScenario implements Scenario {
|
||||
name = 'auth/client-credentials-basic';
|
||||
specVersions: SpecVersion[] = ['extension'];
|
||||
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||
description =
|
||||
'Tests OAuth client_credentials flow with client_secret_basic authentication';
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type {
|
|||
Scenario,
|
||||
ConformanceCheck,
|
||||
ScenarioUrls,
|
||||
SpecVersion
|
||||
ScenarioSpecTag
|
||||
} from '../../../types';
|
||||
import { createAuthServer } from './helpers/createAuthServer';
|
||||
import { createServer } from './helpers/createServer';
|
||||
|
|
@ -60,7 +60,7 @@ async function createIdpIdToken(
|
|||
*/
|
||||
export class CrossAppAccessCompleteFlowScenario implements Scenario {
|
||||
name = 'auth/cross-app-access-complete-flow';
|
||||
specVersions: SpecVersion[] = ['extension'];
|
||||
specVersions: ScenarioSpecTag[] = ['extension'];
|
||||
description =
|
||||
'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import type { Scenario, ConformanceCheck } from '../../../types';
|
||||
import { ScenarioUrls, SpecVersion } from '../../../types';
|
||||
import {
|
||||
ScenarioUrls,
|
||||
SpecVersion,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
} from '../../../types';
|
||||
import { createAuthServer } from './helpers/createAuthServer';
|
||||
import { createServer } from './helpers/createServer';
|
||||
import { ServerLifecycle } from './helpers/serverLifecycle';
|
||||
|
|
@ -23,7 +27,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier';
|
|||
*/
|
||||
export class OfflineAccessScopeScenario implements Scenario {
|
||||
name = 'auth/offline-access-scope';
|
||||
specVersions: SpecVersion[] = ['draft'];
|
||||
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||
description =
|
||||
'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)';
|
||||
|
||||
|
|
@ -227,7 +231,7 @@ export class OfflineAccessScopeScenario implements Scenario {
|
|||
*/
|
||||
export class OfflineAccessNotSupportedScenario implements Scenario {
|
||||
name = 'auth/offline-access-not-supported';
|
||||
specVersions: SpecVersion[] = ['draft'];
|
||||
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||
description =
|
||||
'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import type { Scenario, ConformanceCheck } from '../../../types.js';
|
||||
import { ScenarioUrls, SpecVersion } from '../../../types.js';
|
||||
import {
|
||||
ScenarioUrls,
|
||||
SpecVersion,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
} from '../../../types.js';
|
||||
import { createAuthServer } from './helpers/createAuthServer.js';
|
||||
import { createServer } from './helpers/createServer.js';
|
||||
import { ServerLifecycle } from './helpers/serverLifecycle.js';
|
||||
|
|
@ -27,7 +31,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier.js';
|
|||
*/
|
||||
export class ResourceMismatchScenario implements Scenario {
|
||||
name = 'auth/resource-mismatch';
|
||||
specVersions: SpecVersion[] = ['draft'];
|
||||
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||
description =
|
||||
'Tests that client rejects when PRM resource does not match server URL';
|
||||
allowClientError = true;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import {
|
|||
Scenario,
|
||||
ScenarioUrls,
|
||||
ConformanceCheck,
|
||||
SpecVersion
|
||||
SpecVersion,
|
||||
LATEST_SPEC_VERSION,
|
||||
NEGOTIABLE_PROTOCOL_VERSIONS
|
||||
} from '../../types';
|
||||
import { clientChecks } from '../../checks/index';
|
||||
|
||||
|
|
@ -117,11 +119,10 @@ export class InitializeScenario implements Scenario {
|
|||
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
|
||||
|
||||
// Echo back client's version if valid, otherwise use latest
|
||||
const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
|
||||
const clientVersion = initializeRequest?.protocolVersion;
|
||||
const responseVersion = VALID_VERSIONS.includes(clientVersion)
|
||||
const responseVersion = NEGOTIABLE_PROTOCOL_VERSIONS.includes(clientVersion)
|
||||
? clientVersion
|
||||
: '2025-11-25';
|
||||
: LATEST_SPEC_VERSION;
|
||||
|
||||
const response = {
|
||||
jsonrpc: '2.0',
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ import {
|
|||
Scenario,
|
||||
ClientScenario,
|
||||
ClientScenarioForAuthorizationServer,
|
||||
SpecVersion
|
||||
SpecVersion,
|
||||
ScenarioSpecTag,
|
||||
DATED_SPEC_VERSIONS,
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
LATEST_SPEC_VERSION
|
||||
} from '../types';
|
||||
import { InitializeScenario } from './client/initialize';
|
||||
import { ToolsCallScenario } from './client/tools_call';
|
||||
|
|
@ -256,32 +260,50 @@ export function listDraftScenarios(): string[] {
|
|||
export { listMetadataScenarios };
|
||||
|
||||
// All valid spec versions, used by the CLI to validate --spec-version input.
|
||||
// 'extension' is intentionally excluded — extension scenarios are off-timeline
|
||||
// and selected via `--suite extensions`, not `--spec-version`.
|
||||
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
|
||||
'2025-03-26',
|
||||
'2025-06-18',
|
||||
'2025-11-25',
|
||||
'draft',
|
||||
'extension'
|
||||
...DATED_SPEC_VERSIONS,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
];
|
||||
|
||||
export function resolveSpecVersion(value: string): SpecVersion {
|
||||
if (value === 'draft') return DRAFT_PROTOCOL_VERSION;
|
||||
if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
|
||||
return value as SpecVersion;
|
||||
}
|
||||
console.error(`Unknown spec version: ${value}`);
|
||||
console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
|
||||
console.error(
|
||||
`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// The draft version selects everything in the latest dated release plus
|
||||
// scenarios tagged draft-only, so SEP authors can run the full suite against an
|
||||
// SDK tracking the in-progress spec without retagging core scenarios.
|
||||
function matchesSpecVersion(
|
||||
scenario: { specVersions: ScenarioSpecTag[] },
|
||||
version: SpecVersion
|
||||
): boolean {
|
||||
if (version === DRAFT_PROTOCOL_VERSION) {
|
||||
return (
|
||||
scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) ||
|
||||
scenario.specVersions.includes(LATEST_SPEC_VERSION)
|
||||
);
|
||||
}
|
||||
return scenario.specVersions.includes(version);
|
||||
}
|
||||
|
||||
export function listScenariosForSpec(version: SpecVersion): string[] {
|
||||
return scenariosList
|
||||
.filter((s) => s.specVersions.includes(version))
|
||||
.filter((s) => matchesSpecVersion(s, version))
|
||||
.map((s) => s.name);
|
||||
}
|
||||
|
||||
export function listClientScenariosForSpec(version: SpecVersion): string[] {
|
||||
return allClientScenariosList
|
||||
.filter((s) => s.specVersions.includes(version))
|
||||
.filter((s) => matchesSpecVersion(s, version))
|
||||
.map((s) => s.name);
|
||||
}
|
||||
|
||||
|
|
@ -289,13 +311,13 @@ export function listClientScenariosForAuthorizationServerForSpec(
|
|||
version: SpecVersion
|
||||
): string[] {
|
||||
return allClientScenariosListForAuthorizationServer
|
||||
.filter((s) => s.specVersions.includes(version))
|
||||
.filter((s) => matchesSpecVersion(s, version))
|
||||
.map((s) => s.name);
|
||||
}
|
||||
|
||||
export function getScenarioSpecVersions(
|
||||
name: string
|
||||
): SpecVersion[] | undefined {
|
||||
): ScenarioSpecTag[] | undefined {
|
||||
return (
|
||||
scenarios.get(name)?.specVersions ??
|
||||
clientScenarios.get(name)?.specVersions ??
|
||||
|
|
@ -303,4 +325,4 @@ export function getScenarioSpecVersions(
|
|||
);
|
||||
}
|
||||
|
||||
export type { SpecVersion };
|
||||
export type { SpecVersion, ScenarioSpecTag };
|
||||
|
|
|
|||
|
|
@ -2,7 +2,12 @@
|
|||
* Resources test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
|
||||
import {
|
||||
ClientScenario,
|
||||
ConformanceCheck,
|
||||
SpecVersion,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
} from '../../types';
|
||||
import { connectToServer } from './client-helper';
|
||||
import {
|
||||
TextResourceContents,
|
||||
|
|
@ -438,7 +443,7 @@ Example request:
|
|||
|
||||
export class ResourcesNotFoundErrorScenario implements ClientScenario {
|
||||
name = 'sep-2164-resource-not-found';
|
||||
specVersions: SpecVersion[] = ['draft'];
|
||||
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
|
||||
description = `Test error handling for non-existent resources (SEP-2164).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
|
|
|||
|
|
@ -3,9 +3,23 @@ import {
|
|||
listScenarios,
|
||||
listClientScenarios,
|
||||
listScenariosForSpec,
|
||||
listDraftScenarios,
|
||||
listExtensionScenarios,
|
||||
getScenarioSpecVersions,
|
||||
resolveSpecVersion,
|
||||
ALL_SPEC_VERSIONS
|
||||
} from './index';
|
||||
import {
|
||||
DATED_SPEC_VERSIONS,
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
LATEST_SPEC_VERSION,
|
||||
ScenarioSpecTag
|
||||
} from '../types';
|
||||
|
||||
const ALL_SCENARIO_SPEC_TAGS: ScenarioSpecTag[] = [
|
||||
...ALL_SPEC_VERSIONS,
|
||||
'extension'
|
||||
];
|
||||
|
||||
describe('specVersions helpers', () => {
|
||||
it('every Scenario has specVersions', () => {
|
||||
|
|
@ -17,7 +31,7 @@ describe('specVersions helpers', () => {
|
|||
).toBeDefined();
|
||||
expect(versions!.length).toBeGreaterThan(0);
|
||||
for (const v of versions!) {
|
||||
expect(ALL_SPEC_VERSIONS).toContain(v);
|
||||
expect(ALL_SCENARIO_SPEC_TAGS).toContain(v);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -31,7 +45,7 @@ describe('specVersions helpers', () => {
|
|||
).toBeDefined();
|
||||
expect(versions!.length).toBeGreaterThan(0);
|
||||
for (const v of versions!) {
|
||||
expect(ALL_SPEC_VERSIONS).toContain(v);
|
||||
expect(ALL_SCENARIO_SPEC_TAGS).toContain(v);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -69,26 +83,43 @@ describe('specVersions helpers', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('draft and extension scenarios are isolated', () => {
|
||||
const draft = listScenariosForSpec('draft');
|
||||
for (const name of draft) {
|
||||
expect(getScenarioSpecVersions(name)).toContain('draft');
|
||||
it('the draft spec version is a superset of the latest dated release', () => {
|
||||
const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION));
|
||||
const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION));
|
||||
for (const name of latest) {
|
||||
expect(draft.has(name)).toBe(true);
|
||||
}
|
||||
const ext = listScenariosForSpec('extension');
|
||||
for (const name of ext) {
|
||||
expect(getScenarioSpecVersions(name)).toContain('extension');
|
||||
for (const name of listDraftScenarios()) {
|
||||
expect(draft.has(name)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('draft scenarios are not in dated versions', () => {
|
||||
const draft = listScenariosForSpec('draft');
|
||||
const dated = new Set([
|
||||
...listScenariosForSpec('2025-03-26'),
|
||||
...listScenariosForSpec('2025-06-18'),
|
||||
...listScenariosForSpec('2025-11-25')
|
||||
]);
|
||||
for (const name of draft) {
|
||||
expect(dated.has(name)).toBe(false);
|
||||
it('draft-tagged scenarios are not also tagged with a dated version', () => {
|
||||
for (const name of listDraftScenarios()) {
|
||||
const versions = getScenarioSpecVersions(name)!;
|
||||
for (const dated of DATED_SPEC_VERSIONS) {
|
||||
expect(
|
||||
versions,
|
||||
`scenario "${name}" is tagged with both DRAFT_PROTOCOL_VERSION and '${dated}'`
|
||||
).not.toContain(dated);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("resolveSpecVersion accepts 'draft' as an alias", () => {
|
||||
expect(resolveSpecVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION);
|
||||
expect(resolveSpecVersion(LATEST_SPEC_VERSION)).toBe(LATEST_SPEC_VERSION);
|
||||
});
|
||||
|
||||
it('extension-tagged scenarios are not selected by any --spec-version', () => {
|
||||
for (const version of ALL_SPEC_VERSIONS) {
|
||||
const selected = new Set(listScenariosForSpec(version));
|
||||
for (const name of listExtensionScenarios()) {
|
||||
expect(
|
||||
selected.has(name),
|
||||
`extension scenario "${name}" was selected by --spec-version ${version}`
|
||||
).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,14 +10,22 @@ import {
|
|||
listClientScenariosForSpec,
|
||||
getScenarioSpecVersions
|
||||
} from '../../scenarios';
|
||||
import { ConformanceCheck, SpecVersion } from '../../types';
|
||||
import {
|
||||
ConformanceCheck,
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
ScenarioSpecTag,
|
||||
SpecVersion
|
||||
} from '../../types';
|
||||
|
||||
const NON_SCORING_VERSIONS: SpecVersion[] = ['draft', 'extension'];
|
||||
const NON_SCORING_TAGS: ScenarioSpecTag[] = [
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
'extension'
|
||||
];
|
||||
|
||||
/** Whether a scenario counts toward tier scoring (has at least one date-versioned spec). */
|
||||
function isTierScoring(specVersions?: SpecVersion[]): boolean {
|
||||
function isTierScoring(specVersions?: ScenarioSpecTag[]): boolean {
|
||||
if (!specVersions || specVersions.length === 0) return true; // unknown = count it
|
||||
return specVersions.some((v) => !NON_SCORING_VERSIONS.includes(v));
|
||||
return specVersions.some((v) => !NON_SCORING_TAGS.includes(v));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { TierScorecard, CheckStatus, ConformanceResult } from './types';
|
||||
import { DATED_SPEC_VERSIONS, DRAFT_PROTOCOL_VERSION } from '../types';
|
||||
|
||||
const COLORS = {
|
||||
RESET: '\x1b[0m',
|
||||
|
|
@ -23,9 +24,9 @@ function statusIcon(status: CheckStatus): string {
|
|||
}
|
||||
}
|
||||
|
||||
const TIER_SPEC_VERSIONS = ['2025-03-26', '2025-06-18', '2025-11-25'] as const;
|
||||
const TIER_SPEC_VERSIONS = DATED_SPEC_VERSIONS;
|
||||
|
||||
const INFO_SPEC_VERSIONS = ['draft', 'extension'] as const;
|
||||
const INFO_SPEC_VERSIONS = [DRAFT_PROTOCOL_VERSION, 'extension'] as const;
|
||||
|
||||
type Cell = { passed: number; total: number };
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { SpecVersion } from '../types';
|
||||
import type { ScenarioSpecTag } from '../types';
|
||||
|
||||
export type CheckStatus = 'pass' | 'fail' | 'partial' | 'skipped';
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ export interface ConformanceResult extends CheckResult {
|
|||
passed: boolean;
|
||||
checks_passed: number;
|
||||
checks_failed: number;
|
||||
specVersions?: SpecVersion[];
|
||||
specVersions?: ScenarioSpecTag[];
|
||||
}>;
|
||||
}
|
||||
|
||||
|
|
|
|||
46
src/types.ts
46
src/types.ts
|
|
@ -23,12 +23,40 @@ export interface ConformanceCheck {
|
|||
logs?: string[];
|
||||
}
|
||||
|
||||
export type SpecVersion =
|
||||
| '2025-03-26'
|
||||
| '2025-06-18'
|
||||
| '2025-11-25'
|
||||
| 'draft'
|
||||
| 'extension';
|
||||
export const DATED_SPEC_VERSIONS = [
|
||||
'2025-03-26',
|
||||
'2025-06-18',
|
||||
'2025-11-25'
|
||||
] as const;
|
||||
|
||||
export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number];
|
||||
|
||||
export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25';
|
||||
|
||||
/**
|
||||
* Wire `protocolVersion` for the in-progress spec. Mirrors
|
||||
* `LATEST_PROTOCOL_VERSION` in the spec repo's `schema/draft/schema.ts`;
|
||||
* bump when that constant changes.
|
||||
*/
|
||||
export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1';
|
||||
|
||||
// Wire protocolVersion strings the mock server will negotiate on initialize.
|
||||
export const NEGOTIABLE_PROTOCOL_VERSIONS: readonly string[] = [
|
||||
'2025-06-18',
|
||||
LATEST_SPEC_VERSION,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
];
|
||||
|
||||
/**
|
||||
* A spec revision the conformance suite can target via `--spec-version`.
|
||||
* Always a wire `protocolVersion` string. The CLI also accepts `'draft'` as
|
||||
* an alias for {@link DRAFT_PROTOCOL_VERSION}.
|
||||
*/
|
||||
export type SpecVersion = DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION;
|
||||
|
||||
// Scenarios may also be tagged 'extension' to mark them as off-timeline
|
||||
// (selectable via --suite extensions, never via --spec-version). See #256.
|
||||
export type ScenarioSpecTag = SpecVersion | 'extension';
|
||||
|
||||
export interface ScenarioUrls {
|
||||
serverUrl: string;
|
||||
|
|
@ -43,7 +71,7 @@ export interface ScenarioUrls {
|
|||
export interface Scenario {
|
||||
name: string;
|
||||
description: string;
|
||||
specVersions: SpecVersion[];
|
||||
specVersions: ScenarioSpecTag[];
|
||||
/**
|
||||
* If true, a non-zero client exit code is expected and will not cause the test to fail.
|
||||
* Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth).
|
||||
|
|
@ -57,13 +85,13 @@ export interface Scenario {
|
|||
export interface ClientScenario {
|
||||
name: string;
|
||||
description: string;
|
||||
specVersions: SpecVersion[];
|
||||
specVersions: ScenarioSpecTag[];
|
||||
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
||||
}
|
||||
|
||||
export interface ClientScenarioForAuthorizationServer {
|
||||
name: string;
|
||||
description: string;
|
||||
specVersions: SpecVersion[];
|
||||
specVersions: ScenarioSpecTag[];
|
||||
run(serverUrl: string): Promise<ConformanceCheck[]>;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue