Add conformance tests for SEP-1330 elicitation enums (#22)

* Add SEP-1330 conformance test for elicitation enum schemas

Adds server-side conformance test that validates servers properly request
elicitation with SEP-1330 enum schema improvements:
- Untitled single-select enum (type: string, enum: [...])
- Titled single-select enum (oneOf with const/title)
- Legacy titled enum (enumNames for backward compatibility)
- Untitled multi-select enum (type: array, items.enum)
- Titled multi-select enum (items.anyOf with const/title)

Test expects server to implement `test_elicitation_sep1330_enums` tool.

* Add reference server implementation for SEP-1330 conformance test

Adds test_elicitation_sep1330_enums tool to everything-server that
requests elicitation with all 5 enum schema variants. Also updates
SERVER_REQUIREMENTS.md with full specification.

* Fix prettier formatting

* only test active tests (#24)

* refactor maps to pull names from the test

* bring names back

* separate active from pending

* only test active

---------

Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>
This commit is contained in:
Felix Weinberger 2025-11-11 19:34:47 +00:00 • committed by GitHub
commit e19918d968
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 647 additions and 46 deletions

View file

@ -10,7 +10,11 @@ import {
printServerSummary,
runInteractiveMode
} from './runner';
import { listScenarios, listClientScenarios } from './scenarios';
import {
listScenarios,
listClientScenarios,
listActiveClientScenarios
} from './scenarios';
import { ConformanceCheck } from './types';
import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
import packageJson from '../package.json';
@ -97,8 +101,8 @@ program
const { failed } = printServerResults(result.checks);
process.exit(failed > 0 ? 1 : 0);
} else {
// Run all scenarios
const scenarios = listClientScenarios();
// Run all active scenarios
const scenarios = listActiveClientScenarios();
console.log(
`Running ${scenarios.length} scenarios against ${validated.url}\n`
);

View file

@ -6,7 +6,7 @@ import { ServerLifecycle } from './helpers/serverLifecycle.js';
import { Request, Response } from 'express';
export class AuthBasicDCRScenario implements Scenario {
name = 'auth-basic-dcr';
name = 'auth/basic-dcr';
description =
'Tests Basic OAuth flow with DCR, PRM at path-based location, OAuth metadata at root location, and no scopes required';
private authServer = new ServerLifecycle(() => this.authBaseUrl);

View file

@ -6,7 +6,7 @@ import { ServerLifecycle } from './helpers/serverLifecycle.js';
export class AuthBasicMetadataVar1Scenario implements Scenario {
// TODO: name should match what we put in the scenario map
name = 'auth-basic-metadata-var1';
name = 'auth/basic-metadata-var1';
description =
'Tests Basic OAuth flow with DCR, PRM at root location, OAuth metadata at OpenID discovery path, and no scopes required';
private authServer = new ServerLifecycle(() => this.authBaseUrl);

View file

@ -28,6 +28,7 @@ import {
} from './server/tools.js';
import { ElicitationDefaultsScenario } from './server/elicitation-defaults.js';
import { ElicitationEnumsScenario } from './server/elicitation-enums.js';
import {
ResourcesListScenario,
@ -46,56 +47,83 @@ import {
PromptsGetWithImageScenario
} from './server/prompts.js';
export const scenarios = new Map<string, Scenario>([
['initialize', new InitializeScenario()],
['tools-call', new ToolsCallScenario()],
['auth/basic-dcr', new AuthBasicDCRScenario()],
['auth/basic-metadata-var1', new AuthBasicMetadataVar1Scenario()],
[
'elicitation-sep1034-client-defaults',
new ElicitationClientDefaultsScenario()
]
]);
// Pending client scenarios (not yet fully tested/implemented)
const pendingClientScenariosList: ClientScenario[] = [
// Elicitation scenarios (SEP-1330)
new ElicitationEnumsScenario()
];
export const clientScenarios = new Map<string, ClientScenario>([
// All client scenarios
const allClientScenariosList: ClientScenario[] = [
// Lifecycle scenarios
['server-initialize', new ServerInitializeScenario()],
new ServerInitializeScenario(),
// Utilities scenarios
['logging-set-level', new LoggingSetLevelScenario()],
['completion-complete', new CompletionCompleteScenario()],
new LoggingSetLevelScenario(),
new CompletionCompleteScenario(),
// Tools scenarios
['tools-list', new ToolsListScenario()],
['tools-call-simple-text', new ToolsCallSimpleTextScenario()],
['tools-call-image', new ToolsCallImageScenario()],
['tools-call-audio', new ToolsCallAudioScenario()],
['tools-call-embedded-resource', new ToolsCallEmbeddedResourceScenario()],
['tools-call-mixed-content', new ToolsCallMultipleContentTypesScenario()],
['tools-call-with-logging', new ToolsCallWithLoggingScenario()],
['tools-call-error', new ToolsCallErrorScenario()],
['tools-call-with-progress', new ToolsCallWithProgressScenario()],
['tools-call-sampling', new ToolsCallSamplingScenario()],
['tools-call-elicitation', new ToolsCallElicitationScenario()],
new ToolsListScenario(),
new ToolsCallSimpleTextScenario(),
new ToolsCallImageScenario(),
new ToolsCallAudioScenario(),
new ToolsCallEmbeddedResourceScenario(),
new ToolsCallMultipleContentTypesScenario(),
new ToolsCallWithLoggingScenario(),
new ToolsCallErrorScenario(),
new ToolsCallWithProgressScenario(),
new ToolsCallSamplingScenario(),
new ToolsCallElicitationScenario(),
// Elicitation scenarios (SEP-1034)
['elicitation-sep1034-defaults', new ElicitationDefaultsScenario()],
new ElicitationDefaultsScenario(),
// Elicitation scenarios (SEP-1330) - pending
...pendingClientScenariosList,
// Resources scenarios
['resources-list', new ResourcesListScenario()],
['resources-read-text', new ResourcesReadTextScenario()],
['resources-read-binary', new ResourcesReadBinaryScenario()],
['resources-templates-read', new ResourcesTemplateReadScenario()],
['resources-subscribe', new ResourcesSubscribeScenario()],
['resources-unsubscribe', new ResourcesUnsubscribeScenario()],
new ResourcesListScenario(),
new ResourcesReadTextScenario(),
new ResourcesReadBinaryScenario(),
new ResourcesTemplateReadScenario(),
new ResourcesSubscribeScenario(),
new ResourcesUnsubscribeScenario(),
// Prompts scenarios
['prompts-list', new PromptsListScenario()],
['prompts-get-simple', new PromptsGetSimpleScenario()],
['prompts-get-with-args', new PromptsGetWithArgsScenario()],
['prompts-get-embedded-resource', new PromptsGetEmbeddedResourceScenario()],
['prompts-get-with-image', new PromptsGetWithImageScenario()]
]);
new PromptsListScenario(),
new PromptsGetSimpleScenario(),
new PromptsGetWithArgsScenario(),
new PromptsGetEmbeddedResourceScenario(),
new PromptsGetWithImageScenario()
];
// Active client scenarios (excludes pending)
const activeClientScenariosList: ClientScenario[] =
allClientScenariosList.filter(
(scenario) =>
!pendingClientScenariosList.some(
(pending) => pending.name === scenario.name
)
);
// Client scenarios map - built from list
export const clientScenarios = new Map<string, ClientScenario>(
allClientScenariosList.map((scenario) => [scenario.name, scenario])
);
// Scenario scenarios
const scenariosList: Scenario[] = [
new InitializeScenario(),
new ToolsCallScenario(),
new AuthBasicDCRScenario(),
new AuthBasicMetadataVar1Scenario(),
new ElicitationClientDefaultsScenario()
];
// Scenarios map - built from list
export const scenarios = new Map<string, Scenario>(
scenariosList.map((scenario) => [scenario.name, scenario])
);
export function registerScenario(name: string, scenario: Scenario): void {
scenarios.set(name, scenario);
@ -116,3 +144,7 @@ export function listScenarios(): string[] {
export function listClientScenarios(): string[] {
return Array.from(clientScenarios.keys());
}
export function listActiveClientScenarios(): string[] {
return activeClientScenariosList.map((scenario) => scenario.name);
}

View file

@ -1,5 +1,5 @@
import { spawn, ChildProcess } from 'child_process';
import { listClientScenarios, getClientScenario } from '../index.js';
import { getClientScenario, listActiveClientScenarios } from '../index.js';
import path from 'path';
describe('Server Scenarios', () => {
@ -76,7 +76,7 @@ describe('Server Scenarios', () => {
});
// Generate individual test for each scenario
const scenarios = listClientScenarios();
const scenarios = listActiveClientScenarios();
for (const scenarioName of scenarios) {
it(`${scenarioName}`, async () => {

View file

@ -0,0 +1,375 @@
/**
* SEP-1330: Elicitation enum schema improvements test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer } from './client-helper.js';
import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
export class ElicitationEnumsScenario implements ClientScenario {
name = 'elicitation-sep1330-enums';
description = 'Test elicitation with enum schema improvements (SEP-1330)';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
let capturedRequest: any = null;
connection.client.setRequestHandler(
ElicitRequestSchema,
async (request) => {
capturedRequest = request;
// Return mock data for all enum types
return {
action: 'accept',
content: {
untitledSingle: 'option1',
titledSingle: 'value1',
legacyEnum: 'opt1',
untitledMulti: ['option1', 'option2'],
titledMulti: ['value1', 'value2']
}
};
}
);
await connection.client.callTool({
name: 'test_elicitation_sep1330_enums',
arguments: {}
});
// Validate that elicitation was requested
if (!capturedRequest) {
checks.push({
id: 'elicitation-sep1330-general',
name: 'ElicitationSEP1330General',
description: 'Server requests elicitation with enum schemas',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: 'Server did not request elicitation from client',
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
]
});
await connection.close();
return checks;
}
const schema = capturedRequest.params?.requestedSchema;
const properties = schema?.properties;
// Validate untitled single-select enum
const untitledSingleErrors: string[] = [];
if (!properties?.untitledSingle) {
untitledSingleErrors.push(
'Missing untitled single-select enum field "untitledSingle"'
);
} else {
if (properties.untitledSingle.type !== 'string') {
untitledSingleErrors.push(
`Expected type "string", got "${properties.untitledSingle.type}"`
);
}
if (
!properties.untitledSingle.enum ||
!Array.isArray(properties.untitledSingle.enum)
) {
untitledSingleErrors.push('Missing or invalid enum array');
}
if (properties.untitledSingle.oneOf) {
untitledSingleErrors.push(
'Untitled enum should not have oneOf property'
);
}
if (properties.untitledSingle.enumNames) {
untitledSingleErrors.push(
'Untitled enum should not have enumNames property'
);
}
}
checks.push({
id: 'elicitation-sep1330-untitled-single',
name: 'ElicitationSEP1330UntitledSingle',
description: 'Untitled single-select enum schema uses enum array',
status: untitledSingleErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
untitledSingleErrors.length > 0
? untitledSingleErrors.join('; ')
: undefined,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
],
details: {
field: 'untitledSingle',
schema: properties?.untitledSingle
}
});
// Validate titled single-select enum (using oneOf with const/title)
const titledSingleErrors: string[] = [];
if (!properties?.titledSingle) {
titledSingleErrors.push(
'Missing titled single-select enum field "titledSingle"'
);
} else {
if (properties.titledSingle.type !== 'string') {
titledSingleErrors.push(
`Expected type "string", got "${properties.titledSingle.type}"`
);
}
if (
!properties.titledSingle.oneOf ||
!Array.isArray(properties.titledSingle.oneOf)
) {
titledSingleErrors.push(
'Missing or invalid oneOf array for titled enum'
);
} else {
// Validate oneOf structure has const/title pairs
const invalidItems = properties.titledSingle.oneOf.filter(
(item: any) =>
typeof item.const !== 'string' || typeof item.title !== 'string'
);
if (invalidItems.length > 0) {
titledSingleErrors.push(
`oneOf items must have "const" (string) and "title" (string) properties`
);
}
}
if (properties.titledSingle.enum) {
titledSingleErrors.push(
'Titled enum should use oneOf instead of enum array'
);
}
}
checks.push({
id: 'elicitation-sep1330-titled-single',
name: 'ElicitationSEP1330TitledSingle',
description:
'Titled single-select enum schema uses oneOf with const/title',
status: titledSingleErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
titledSingleErrors.length > 0
? titledSingleErrors.join('; ')
: undefined,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
],
details: {
field: 'titledSingle',
schema: properties?.titledSingle
}
});
// Validate legacy titled enum (using enumNames - deprecated)
const legacyEnumErrors: string[] = [];
if (!properties?.legacyEnum) {
legacyEnumErrors.push('Missing legacy titled enum field "legacyEnum"');
} else {
if (properties.legacyEnum.type !== 'string') {
legacyEnumErrors.push(
`Expected type "string", got "${properties.legacyEnum.type}"`
);
}
if (
!properties.legacyEnum.enum ||
!Array.isArray(properties.legacyEnum.enum)
) {
legacyEnumErrors.push('Missing or invalid enum array');
}
if (
!properties.legacyEnum.enumNames ||
!Array.isArray(properties.legacyEnum.enumNames)
) {
legacyEnumErrors.push(
'Missing or invalid enumNames array for legacy titled enum'
);
} else if (
properties.legacyEnum.enum &&
properties.legacyEnum.enumNames.length !==
properties.legacyEnum.enum.length
) {
legacyEnumErrors.push(
`enumNames length (${properties.legacyEnum.enumNames.length}) must match enum length (${properties.legacyEnum.enum.length})`
);
}
}
checks.push({
id: 'elicitation-sep1330-legacy-enumnames',
name: 'ElicitationSEP1330LegacyEnumNames',
description: 'Legacy titled enum schema uses enumNames (deprecated)',
status: legacyEnumErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
legacyEnumErrors.length > 0 ? legacyEnumErrors.join('; ') : undefined,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
],
details: {
field: 'legacyEnum',
schema: properties?.legacyEnum
}
});
// Validate untitled multi-select enum
const untitledMultiErrors: string[] = [];
if (!properties?.untitledMulti) {
untitledMultiErrors.push(
'Missing untitled multi-select enum field "untitledMulti"'
);
} else {
if (properties.untitledMulti.type !== 'array') {
untitledMultiErrors.push(
`Expected type "array", got "${properties.untitledMulti.type}"`
);
}
if (!properties.untitledMulti.items) {
untitledMultiErrors.push('Missing items property for array type');
} else {
if (properties.untitledMulti.items.type !== 'string') {
untitledMultiErrors.push(
`Expected items.type "string", got "${properties.untitledMulti.items.type}"`
);
}
if (
!properties.untitledMulti.items.enum ||
!Array.isArray(properties.untitledMulti.items.enum)
) {
untitledMultiErrors.push('Missing or invalid items.enum array');
}
if (properties.untitledMulti.items.anyOf) {
untitledMultiErrors.push(
'Untitled multi-select should use items.enum, not items.anyOf'
);
}
}
}
checks.push({
id: 'elicitation-sep1330-untitled-multi',
name: 'ElicitationSEP1330UntitledMulti',
description:
'Untitled multi-select enum schema uses array with items.enum',
status: untitledMultiErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
untitledMultiErrors.length > 0
? untitledMultiErrors.join('; ')
: undefined,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
],
details: {
field: 'untitledMulti',
schema: properties?.untitledMulti
}
});
// Validate titled multi-select enum (using items.anyOf with const/title)
const titledMultiErrors: string[] = [];
if (!properties?.titledMulti) {
titledMultiErrors.push(
'Missing titled multi-select enum field "titledMulti"'
);
} else {
if (properties.titledMulti.type !== 'array') {
titledMultiErrors.push(
`Expected type "array", got "${properties.titledMulti.type}"`
);
}
if (!properties.titledMulti.items) {
titledMultiErrors.push('Missing items property for array type');
} else {
if (
!properties.titledMulti.items.anyOf ||
!Array.isArray(properties.titledMulti.items.anyOf)
) {
titledMultiErrors.push(
'Missing or invalid items.anyOf array for titled multi-select'
);
} else {
// Validate anyOf structure has const/title pairs
const invalidItems = properties.titledMulti.items.anyOf.filter(
(item: any) =>
typeof item.const !== 'string' || typeof item.title !== 'string'
);
if (invalidItems.length > 0) {
titledMultiErrors.push(
`items.anyOf entries must have "const" (string) and "title" (string) properties`
);
}
}
if (properties.titledMulti.items.enum) {
titledMultiErrors.push(
'Titled multi-select should use items.anyOf, not items.enum'
);
}
}
}
checks.push({
id: 'elicitation-sep1330-titled-multi',
name: 'ElicitationSEP1330TitledMulti',
description:
'Titled multi-select enum schema uses array with items.anyOf',
status: titledMultiErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage:
titledMultiErrors.length > 0
? titledMultiErrors.join('; ')
: undefined,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
],
details: {
field: 'titledMulti',
schema: properties?.titledMulti
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'elicitation-sep1330-general',
name: 'ElicitationSEP1330General',
description: 'Server requests elicitation with enum schemas',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'SEP-1330',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
}
]
});
}
return checks;
}
}