feat: add core and extensions suites for SDK tiering (#117)
* feat: add core and extensions suites for SDK tiering Add new suites to support SDK tiering requirements: Client suites: - core (19 scenarios): Tier 1 requirements (base + auth) - extensions (2 scenarios): Optional protocol extensions (client credentials) - all (21 scenarios): Everything (core + extensions) - auth (15 scenarios): Auth scenarios only (excludes extensions) Server suites: - core: Alias for 'active' (tier 1 requirements) This allows SDK developers to run --suite core for tier 1 certification while keeping extension tests available but not blocking. * feat: promote elicitation tests from pending to active Move 3 elicitation scenarios to active suite: - elicitation-sep1330-enums - tools-call-elicitation - elicitation-sep1034-defaults These tests are now passing in the Python SDK and ready for SDK tiering requirements. Remaining pending tests (2): - json-schema-2020-12 (blocked on SDK PR #1135) - server-sse-polling (blocked on SDK PR #1129)
This commit is contained in:
parent
d0b2b4914d
commit
5dca74be9a
3 changed files with 37 additions and 15 deletions
15
src/index.ts
15
src/index.ts
|
|
@ -16,7 +16,9 @@ import {
|
||||||
listActiveClientScenarios,
|
listActiveClientScenarios,
|
||||||
listPendingClientScenarios,
|
listPendingClientScenarios,
|
||||||
listAuthScenarios,
|
listAuthScenarios,
|
||||||
listMetadataScenarios
|
listMetadataScenarios,
|
||||||
|
listCoreScenarios,
|
||||||
|
listExtensionScenarios
|
||||||
} from './scenarios';
|
} from './scenarios';
|
||||||
import { ConformanceCheck } from './types';
|
import { ConformanceCheck } from './types';
|
||||||
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
|
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
|
||||||
|
|
@ -65,6 +67,8 @@ program
|
||||||
|
|
||||||
const suites: Record<string, () => string[]> = {
|
const suites: Record<string, () => string[]> = {
|
||||||
all: listScenarios,
|
all: listScenarios,
|
||||||
|
core: listCoreScenarios,
|
||||||
|
extensions: listExtensionScenarios,
|
||||||
auth: listAuthScenarios,
|
auth: listAuthScenarios,
|
||||||
metadata: listMetadataScenarios,
|
metadata: listMetadataScenarios,
|
||||||
'sep-835': () =>
|
'sep-835': () =>
|
||||||
|
|
@ -177,7 +181,9 @@ program
|
||||||
console.error('Either --scenario or --suite is required');
|
console.error('Either --scenario or --suite is required');
|
||||||
console.error('\nAvailable client scenarios:');
|
console.error('\nAvailable client scenarios:');
|
||||||
listScenarios().forEach((s) => console.error(` - ${s}`));
|
listScenarios().forEach((s) => console.error(` - ${s}`));
|
||||||
console.error('\nAvailable suites: all, auth, metadata, sep-835');
|
console.error(
|
||||||
|
'\nAvailable suites: all, core, extensions, auth, metadata, sep-835'
|
||||||
|
);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -296,13 +302,14 @@ program
|
||||||
|
|
||||||
if (suite === 'all') {
|
if (suite === 'all') {
|
||||||
scenarios = listClientScenarios();
|
scenarios = listClientScenarios();
|
||||||
} else if (suite === 'active') {
|
} else if (suite === 'active' || suite === 'core') {
|
||||||
|
// 'core' is an alias for 'active' - tier 1 requirements
|
||||||
scenarios = listActiveClientScenarios();
|
scenarios = listActiveClientScenarios();
|
||||||
} else if (suite === 'pending') {
|
} else if (suite === 'pending') {
|
||||||
scenarios = listPendingClientScenarios();
|
scenarios = listPendingClientScenarios();
|
||||||
} else {
|
} else {
|
||||||
console.error(`Unknown suite: ${suite}`);
|
console.error(`Unknown suite: ${suite}`);
|
||||||
console.error('Available suites: active, all, pending');
|
console.error('Available suites: active, all, core, pending');
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ import {
|
||||||
ClientCredentialsBasicScenario
|
ClientCredentialsBasicScenario
|
||||||
} from './client-credentials';
|
} from './client-credentials';
|
||||||
|
|
||||||
|
// Auth scenarios (required for tier 1)
|
||||||
export const authScenariosList: Scenario[] = [
|
export const authScenariosList: Scenario[] = [
|
||||||
...metadataScenarios,
|
...metadataScenarios,
|
||||||
new AuthBasicCIMDScenario(),
|
new AuthBasicCIMDScenario(),
|
||||||
|
|
@ -34,7 +35,11 @@ export const authScenariosList: Scenario[] = [
|
||||||
new ScopeRetryLimitScenario(),
|
new ScopeRetryLimitScenario(),
|
||||||
new ClientSecretBasicAuthScenario(),
|
new ClientSecretBasicAuthScenario(),
|
||||||
new ClientSecretPostAuthScenario(),
|
new ClientSecretPostAuthScenario(),
|
||||||
new PublicClientAuthScenario(),
|
new PublicClientAuthScenario()
|
||||||
|
];
|
||||||
|
|
||||||
|
// Extension scenarios (optional for tier 1 - protocol extensions)
|
||||||
|
export const extensionScenariosList: Scenario[] = [
|
||||||
new ClientCredentialsJwtScenario(),
|
new ClientCredentialsJwtScenario(),
|
||||||
new ClientCredentialsBasicScenario()
|
new ClientCredentialsBasicScenario()
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -53,24 +53,16 @@ import {
|
||||||
|
|
||||||
import { DNSRebindingProtectionScenario } from './server/dns-rebinding';
|
import { DNSRebindingProtectionScenario } from './server/dns-rebinding';
|
||||||
|
|
||||||
import { authScenariosList } from './client/auth/index';
|
import { authScenariosList, extensionScenariosList } from './client/auth/index';
|
||||||
import { listMetadataScenarios } from './client/auth/discovery-metadata';
|
import { listMetadataScenarios } from './client/auth/discovery-metadata';
|
||||||
|
|
||||||
// Pending client scenarios (not yet fully tested/implemented)
|
// Pending client scenarios (not yet fully tested/implemented)
|
||||||
const pendingClientScenariosList: ClientScenario[] = [
|
const pendingClientScenariosList: ClientScenario[] = [
|
||||||
// Elicitation scenarios (SEP-1330)
|
|
||||||
new ElicitationEnumsScenario(),
|
|
||||||
|
|
||||||
// JSON Schema 2020-12 (SEP-1613)
|
// JSON Schema 2020-12 (SEP-1613)
|
||||||
// This test is pending until the SDK includes PR #1135 which preserves
|
// This test is pending until the SDK includes PR #1135 which preserves
|
||||||
// $schema, $defs, and additionalProperties fields in tool schemas.
|
// $schema, $defs, and additionalProperties fields in tool schemas.
|
||||||
new JsonSchema2020_12Scenario(),
|
new JsonSchema2020_12Scenario(),
|
||||||
|
|
||||||
// On hold until elicitation schema types are fixed
|
|
||||||
// https://github.com/modelcontextprotocol/modelcontextprotocol/pull/1863
|
|
||||||
new ToolsCallElicitationScenario(),
|
|
||||||
new ElicitationDefaultsScenario(),
|
|
||||||
|
|
||||||
// On hold until server-side SSE improvements are made
|
// On hold until server-side SSE improvements are made
|
||||||
// https://github.com/modelcontextprotocol/typescript-sdk/pull/1129
|
// https://github.com/modelcontextprotocol/typescript-sdk/pull/1129
|
||||||
new ServerSSEPollingScenario()
|
new ServerSSEPollingScenario()
|
||||||
|
|
@ -145,8 +137,18 @@ export const clientScenarios = new Map<string, ClientScenario>(
|
||||||
allClientScenariosList.map((scenario) => [scenario.name, scenario])
|
allClientScenariosList.map((scenario) => [scenario.name, scenario])
|
||||||
);
|
);
|
||||||
|
|
||||||
// Scenario scenarios
|
// All client test scenarios (core + extensions)
|
||||||
const scenariosList: Scenario[] = [
|
const scenariosList: Scenario[] = [
|
||||||
|
new InitializeScenario(),
|
||||||
|
new ToolsCallScenario(),
|
||||||
|
new ElicitationClientDefaultsScenario(),
|
||||||
|
new SSERetryScenario(),
|
||||||
|
...authScenariosList,
|
||||||
|
...extensionScenariosList
|
||||||
|
];
|
||||||
|
|
||||||
|
// Core scenarios (tier 1 requirements)
|
||||||
|
const coreScenariosList: Scenario[] = [
|
||||||
new InitializeScenario(),
|
new InitializeScenario(),
|
||||||
new ToolsCallScenario(),
|
new ToolsCallScenario(),
|
||||||
new ElicitationClientDefaultsScenario(),
|
new ElicitationClientDefaultsScenario(),
|
||||||
|
|
@ -191,4 +193,12 @@ export function listAuthScenarios(): string[] {
|
||||||
return authScenariosList.map((scenario) => scenario.name);
|
return authScenariosList.map((scenario) => scenario.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function listCoreScenarios(): string[] {
|
||||||
|
return coreScenariosList.map((scenario) => scenario.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function listExtensionScenarios(): string[] {
|
||||||
|
return extensionScenariosList.map((scenario) => scenario.name);
|
||||||
|
}
|
||||||
|
|
||||||
export { listMetadataScenarios };
|
export { listMetadataScenarios };
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue