Add conformance tests for SEP-1330 elicitation enums (#22)
* Add SEP-1330 conformance test for elicitation enum schemas Adds server-side conformance test that validates servers properly request elicitation with SEP-1330 enum schema improvements: - Untitled single-select enum (type: string, enum: [...]) - Titled single-select enum (oneOf with const/title) - Legacy titled enum (enumNames for backward compatibility) - Untitled multi-select enum (type: array, items.enum) - Titled multi-select enum (items.anyOf with const/title) Test expects server to implement `test_elicitation_sep1330_enums` tool. * Add reference server implementation for SEP-1330 conformance test Adds test_elicitation_sep1330_enums tool to everything-server that requests elicitation with all 5 enum schema variants. Also updates SERVER_REQUIREMENTS.md with full specification. * Fix prettier formatting * only test active tests (#24) * refactor maps to pull names from the test * bring names back * separate active from pending * only test active --------- Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>
This commit is contained in:
Felix Weinberger
GitHub
parent
73af777bc2
commit
e19918d968
2 changed files with 647 additions and 46 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import { spawn, ChildProcess } from 'child_process';
|
||||
import { listClientScenarios, getClientScenario } from '../index.js';
|
||||
import { getClientScenario, listActiveClientScenarios } from '../index.js';
|
||||
import path from 'path';
|
||||
|
||||
describe('Server Scenarios', () => {
|
||||
|
|
@ -76,7 +76,7 @@ describe('Server Scenarios', () => {
|
|||
});
|
||||
|
||||
// Generate individual test for each scenario
|
||||
const scenarios = listClientScenarios();
|
||||
const scenarios = listActiveClientScenarios();
|
||||
|
||||
for (const scenarioName of scenarios) {
|
||||
it(`${scenarioName}`, async () => {
|
||||
|
|
|
|||
375
src/scenarios/server/elicitation-enums.ts
Normal file
375
src/scenarios/server/elicitation-enums.ts
Normal file
|
|
@ -0,0 +1,375 @@
|
|||
/**
|
||||
* SEP-1330: Elicitation enum schema improvements test scenarios for MCP servers
|
||||
*/
|
||||
|
||||
import { ClientScenario, ConformanceCheck } from '../../types.js';
|
||||
import { connectToServer } from './client-helper.js';
|
||||
import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
|
||||
export class ElicitationEnumsScenario implements ClientScenario {
|
||||
name = 'elicitation-sep1330-enums';
|
||||
description = 'Test elicitation with enum schema improvements (SEP-1330)';
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
||||
try {
|
||||
const connection = await connectToServer(serverUrl);
|
||||
|
||||
let capturedRequest: any = null;
|
||||
connection.client.setRequestHandler(
|
||||
ElicitRequestSchema,
|
||||
async (request) => {
|
||||
capturedRequest = request;
|
||||
// Return mock data for all enum types
|
||||
return {
|
||||
action: 'accept',
|
||||
content: {
|
||||
untitledSingle: 'option1',
|
||||
titledSingle: 'value1',
|
||||
legacyEnum: 'opt1',
|
||||
untitledMulti: ['option1', 'option2'],
|
||||
titledMulti: ['value1', 'value2']
|
||||
}
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
await connection.client.callTool({
|
||||
name: 'test_elicitation_sep1330_enums',
|
||||
arguments: {}
|
||||
});
|
||||
|
||||
// Validate that elicitation was requested
|
||||
if (!capturedRequest) {
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-general',
|
||||
name: 'ElicitationSEP1330General',
|
||||
description: 'Server requests elicitation with enum schemas',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: 'Server did not request elicitation from client',
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
]
|
||||
});
|
||||
await connection.close();
|
||||
return checks;
|
||||
}
|
||||
|
||||
const schema = capturedRequest.params?.requestedSchema;
|
||||
const properties = schema?.properties;
|
||||
|
||||
// Validate untitled single-select enum
|
||||
const untitledSingleErrors: string[] = [];
|
||||
if (!properties?.untitledSingle) {
|
||||
untitledSingleErrors.push(
|
||||
'Missing untitled single-select enum field "untitledSingle"'
|
||||
);
|
||||
} else {
|
||||
if (properties.untitledSingle.type !== 'string') {
|
||||
untitledSingleErrors.push(
|
||||
`Expected type "string", got "${properties.untitledSingle.type}"`
|
||||
);
|
||||
}
|
||||
if (
|
||||
!properties.untitledSingle.enum ||
|
||||
!Array.isArray(properties.untitledSingle.enum)
|
||||
) {
|
||||
untitledSingleErrors.push('Missing or invalid enum array');
|
||||
}
|
||||
if (properties.untitledSingle.oneOf) {
|
||||
untitledSingleErrors.push(
|
||||
'Untitled enum should not have oneOf property'
|
||||
);
|
||||
}
|
||||
if (properties.untitledSingle.enumNames) {
|
||||
untitledSingleErrors.push(
|
||||
'Untitled enum should not have enumNames property'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-untitled-single',
|
||||
name: 'ElicitationSEP1330UntitledSingle',
|
||||
description: 'Untitled single-select enum schema uses enum array',
|
||||
status: untitledSingleErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
untitledSingleErrors.length > 0
|
||||
? untitledSingleErrors.join('; ')
|
||||
: undefined,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
],
|
||||
details: {
|
||||
field: 'untitledSingle',
|
||||
schema: properties?.untitledSingle
|
||||
}
|
||||
});
|
||||
|
||||
// Validate titled single-select enum (using oneOf with const/title)
|
||||
const titledSingleErrors: string[] = [];
|
||||
if (!properties?.titledSingle) {
|
||||
titledSingleErrors.push(
|
||||
'Missing titled single-select enum field "titledSingle"'
|
||||
);
|
||||
} else {
|
||||
if (properties.titledSingle.type !== 'string') {
|
||||
titledSingleErrors.push(
|
||||
`Expected type "string", got "${properties.titledSingle.type}"`
|
||||
);
|
||||
}
|
||||
if (
|
||||
!properties.titledSingle.oneOf ||
|
||||
!Array.isArray(properties.titledSingle.oneOf)
|
||||
) {
|
||||
titledSingleErrors.push(
|
||||
'Missing or invalid oneOf array for titled enum'
|
||||
);
|
||||
} else {
|
||||
// Validate oneOf structure has const/title pairs
|
||||
const invalidItems = properties.titledSingle.oneOf.filter(
|
||||
(item: any) =>
|
||||
typeof item.const !== 'string' || typeof item.title !== 'string'
|
||||
);
|
||||
if (invalidItems.length > 0) {
|
||||
titledSingleErrors.push(
|
||||
`oneOf items must have "const" (string) and "title" (string) properties`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (properties.titledSingle.enum) {
|
||||
titledSingleErrors.push(
|
||||
'Titled enum should use oneOf instead of enum array'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-titled-single',
|
||||
name: 'ElicitationSEP1330TitledSingle',
|
||||
description:
|
||||
'Titled single-select enum schema uses oneOf with const/title',
|
||||
status: titledSingleErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
titledSingleErrors.length > 0
|
||||
? titledSingleErrors.join('; ')
|
||||
: undefined,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
],
|
||||
details: {
|
||||
field: 'titledSingle',
|
||||
schema: properties?.titledSingle
|
||||
}
|
||||
});
|
||||
|
||||
// Validate legacy titled enum (using enumNames - deprecated)
|
||||
const legacyEnumErrors: string[] = [];
|
||||
if (!properties?.legacyEnum) {
|
||||
legacyEnumErrors.push('Missing legacy titled enum field "legacyEnum"');
|
||||
} else {
|
||||
if (properties.legacyEnum.type !== 'string') {
|
||||
legacyEnumErrors.push(
|
||||
`Expected type "string", got "${properties.legacyEnum.type}"`
|
||||
);
|
||||
}
|
||||
if (
|
||||
!properties.legacyEnum.enum ||
|
||||
!Array.isArray(properties.legacyEnum.enum)
|
||||
) {
|
||||
legacyEnumErrors.push('Missing or invalid enum array');
|
||||
}
|
||||
if (
|
||||
!properties.legacyEnum.enumNames ||
|
||||
!Array.isArray(properties.legacyEnum.enumNames)
|
||||
) {
|
||||
legacyEnumErrors.push(
|
||||
'Missing or invalid enumNames array for legacy titled enum'
|
||||
);
|
||||
} else if (
|
||||
properties.legacyEnum.enum &&
|
||||
properties.legacyEnum.enumNames.length !==
|
||||
properties.legacyEnum.enum.length
|
||||
) {
|
||||
legacyEnumErrors.push(
|
||||
`enumNames length (${properties.legacyEnum.enumNames.length}) must match enum length (${properties.legacyEnum.enum.length})`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-legacy-enumnames',
|
||||
name: 'ElicitationSEP1330LegacyEnumNames',
|
||||
description: 'Legacy titled enum schema uses enumNames (deprecated)',
|
||||
status: legacyEnumErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
legacyEnumErrors.length > 0 ? legacyEnumErrors.join('; ') : undefined,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
],
|
||||
details: {
|
||||
field: 'legacyEnum',
|
||||
schema: properties?.legacyEnum
|
||||
}
|
||||
});
|
||||
|
||||
// Validate untitled multi-select enum
|
||||
const untitledMultiErrors: string[] = [];
|
||||
if (!properties?.untitledMulti) {
|
||||
untitledMultiErrors.push(
|
||||
'Missing untitled multi-select enum field "untitledMulti"'
|
||||
);
|
||||
} else {
|
||||
if (properties.untitledMulti.type !== 'array') {
|
||||
untitledMultiErrors.push(
|
||||
`Expected type "array", got "${properties.untitledMulti.type}"`
|
||||
);
|
||||
}
|
||||
if (!properties.untitledMulti.items) {
|
||||
untitledMultiErrors.push('Missing items property for array type');
|
||||
} else {
|
||||
if (properties.untitledMulti.items.type !== 'string') {
|
||||
untitledMultiErrors.push(
|
||||
`Expected items.type "string", got "${properties.untitledMulti.items.type}"`
|
||||
);
|
||||
}
|
||||
if (
|
||||
!properties.untitledMulti.items.enum ||
|
||||
!Array.isArray(properties.untitledMulti.items.enum)
|
||||
) {
|
||||
untitledMultiErrors.push('Missing or invalid items.enum array');
|
||||
}
|
||||
if (properties.untitledMulti.items.anyOf) {
|
||||
untitledMultiErrors.push(
|
||||
'Untitled multi-select should use items.enum, not items.anyOf'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-untitled-multi',
|
||||
name: 'ElicitationSEP1330UntitledMulti',
|
||||
description:
|
||||
'Untitled multi-select enum schema uses array with items.enum',
|
||||
status: untitledMultiErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
untitledMultiErrors.length > 0
|
||||
? untitledMultiErrors.join('; ')
|
||||
: undefined,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
],
|
||||
details: {
|
||||
field: 'untitledMulti',
|
||||
schema: properties?.untitledMulti
|
||||
}
|
||||
});
|
||||
|
||||
// Validate titled multi-select enum (using items.anyOf with const/title)
|
||||
const titledMultiErrors: string[] = [];
|
||||
if (!properties?.titledMulti) {
|
||||
titledMultiErrors.push(
|
||||
'Missing titled multi-select enum field "titledMulti"'
|
||||
);
|
||||
} else {
|
||||
if (properties.titledMulti.type !== 'array') {
|
||||
titledMultiErrors.push(
|
||||
`Expected type "array", got "${properties.titledMulti.type}"`
|
||||
);
|
||||
}
|
||||
if (!properties.titledMulti.items) {
|
||||
titledMultiErrors.push('Missing items property for array type');
|
||||
} else {
|
||||
if (
|
||||
!properties.titledMulti.items.anyOf ||
|
||||
!Array.isArray(properties.titledMulti.items.anyOf)
|
||||
) {
|
||||
titledMultiErrors.push(
|
||||
'Missing or invalid items.anyOf array for titled multi-select'
|
||||
);
|
||||
} else {
|
||||
// Validate anyOf structure has const/title pairs
|
||||
const invalidItems = properties.titledMulti.items.anyOf.filter(
|
||||
(item: any) =>
|
||||
typeof item.const !== 'string' || typeof item.title !== 'string'
|
||||
);
|
||||
if (invalidItems.length > 0) {
|
||||
titledMultiErrors.push(
|
||||
`items.anyOf entries must have "const" (string) and "title" (string) properties`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (properties.titledMulti.items.enum) {
|
||||
titledMultiErrors.push(
|
||||
'Titled multi-select should use items.anyOf, not items.enum'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-titled-multi',
|
||||
name: 'ElicitationSEP1330TitledMulti',
|
||||
description:
|
||||
'Titled multi-select enum schema uses array with items.anyOf',
|
||||
status: titledMultiErrors.length === 0 ? 'SUCCESS' : 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage:
|
||||
titledMultiErrors.length > 0
|
||||
? titledMultiErrors.join('; ')
|
||||
: undefined,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
],
|
||||
details: {
|
||||
field: 'titledMulti',
|
||||
schema: properties?.titledMulti
|
||||
}
|
||||
});
|
||||
|
||||
await connection.close();
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
id: 'elicitation-sep1330-general',
|
||||
name: 'ElicitationSEP1330General',
|
||||
description: 'Server requests elicitation with enum schemas',
|
||||
status: 'FAILURE',
|
||||
timestamp: new Date().toISOString(),
|
||||
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
specReferences: [
|
||||
{
|
||||
id: 'SEP-1330',
|
||||
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1330'
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue