remove server requirements, move requirements to schenario description (#30)
This commit is contained in:
Inna Harper
GitHub
parent
ee60717a06
commit
38443d1a43
11 changed files with 633 additions and 1165 deletions
File diff suppressed because it is too large
Load diff
11
package-lock.json
generated
11
package-lock.json
generated
|
|
@ -1678,7 +1678,6 @@
|
|||
"integrity": "sha512-BnOroVl1SgrPLywqxyqdJ4l3S2MsKVLDVxZvjI1Eoe8ev2r3kGDo+PcMihNmDE+6/KjkTubSJnmqGZZjQSBq/g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@typescript-eslint/scope-manager": "8.46.2",
|
||||
"@typescript-eslint/types": "8.46.2",
|
||||
|
|
@ -1897,7 +1896,6 @@
|
|||
"integrity": "sha512-LKDTZxo2D2U75EZ+8xw0AJmG3VEqUVjlWu43IJLdl+Hjyhh2eFmfufhXw3mwPRAF6hEy+E3ldUtglmmliN/CiA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsgo": "bin/tsgo.js"
|
||||
},
|
||||
|
|
@ -2139,7 +2137,6 @@
|
|||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
|
|
@ -2727,7 +2724,6 @@
|
|||
"resolved": "https://registry.npmjs.org/eslint/-/eslint-9.38.0.tgz",
|
||||
"integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==",
|
||||
"dev": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.8.0",
|
||||
"@eslint-community/regexpp": "^4.12.1",
|
||||
|
|
@ -2946,7 +2942,6 @@
|
|||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-5.1.0.tgz",
|
||||
"integrity": "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA==",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"accepts": "^2.0.0",
|
||||
"body-parser": "^2.2.0",
|
||||
|
|
@ -4047,7 +4042,6 @@
|
|||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
|
|
@ -4283,7 +4277,6 @@
|
|||
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-beta.45.tgz",
|
||||
"integrity": "sha512-iMmuD72XXLf26Tqrv1cryNYLX6NNPLhZ3AmNkSf8+xda0H+yijjGJ+wVT9UdBUHOpKzq9RjKtQKRCWoEKQQBZQ==",
|
||||
"dev": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@oxc-project/types": "=0.95.0",
|
||||
"@rolldown/pluginutils": "1.0.0-beta.45"
|
||||
|
|
@ -4836,7 +4829,6 @@
|
|||
"integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "~0.25.0",
|
||||
"get-tsconfig": "^4.7.5"
|
||||
|
|
@ -4883,7 +4875,6 @@
|
|||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
|
|
@ -4970,7 +4961,6 @@
|
|||
"resolved": "https://registry.npmjs.org/vite/-/vite-7.1.12.tgz",
|
||||
"integrity": "sha512-ZWyE8YXEXqJrrSLvYgrRP7p62OziLW7xI5HYGWFzOvupfAlrLvURSzv/FyGyy0eidogEM3ujU+kUG1zuHgb6Ug==",
|
||||
"dev": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"esbuild": "^0.25.0",
|
||||
"fdir": "^6.5.0",
|
||||
|
|
@ -5191,7 +5181,6 @@
|
|||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
|
||||
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,7 +98,10 @@ program
|
|||
validated.scenario
|
||||
);
|
||||
|
||||
const { failed } = printServerResults(result.checks);
|
||||
const { failed } = printServerResults(
|
||||
result.checks,
|
||||
result.scenarioDescription
|
||||
);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
} else {
|
||||
// Run all active scenarios
|
||||
|
|
|
|||
|
|
@ -4,12 +4,26 @@ import { ConformanceCheck } from '../types';
|
|||
import { getClientScenario } from '../scenarios';
|
||||
import { ensureResultsDir, createResultDir } from './utils';
|
||||
|
||||
/**
|
||||
* Format markdown-style text for terminal output using ANSI codes
|
||||
*/
|
||||
function formatMarkdown(text: string): string {
|
||||
return (
|
||||
text
|
||||
// Bold text: **text** -> bold
|
||||
.replace(/\*\*([^*]+)\*\*/g, '\x1b[1m$1\x1b[0m')
|
||||
// Inline code: `code` -> dim/gray
|
||||
.replace(/`([^`]+)`/g, '\x1b[2m$1\x1b[0m')
|
||||
);
|
||||
}
|
||||
|
||||
export async function runServerConformanceTest(
|
||||
serverUrl: string,
|
||||
scenarioName: string
|
||||
): Promise<{
|
||||
checks: ConformanceCheck[];
|
||||
resultDir: string;
|
||||
scenarioDescription: string;
|
||||
}> {
|
||||
await ensureResultsDir();
|
||||
const resultDir = createResultDir(scenarioName, 'server');
|
||||
|
|
@ -33,11 +47,15 @@ export async function runServerConformanceTest(
|
|||
|
||||
return {
|
||||
checks,
|
||||
resultDir
|
||||
resultDir,
|
||||
scenarioDescription: scenario.description
|
||||
};
|
||||
}
|
||||
|
||||
export function printServerResults(checks: ConformanceCheck[]): {
|
||||
export function printServerResults(
|
||||
checks: ConformanceCheck[],
|
||||
scenarioDescription: string
|
||||
): {
|
||||
passed: number;
|
||||
failed: number;
|
||||
denominator: number;
|
||||
|
|
@ -54,14 +72,15 @@ export function printServerResults(checks: ConformanceCheck[]): {
|
|||
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
|
||||
|
||||
if (failed > 0) {
|
||||
console.log('\nFailed Checks:');
|
||||
console.log('\n=== Failed Checks ===');
|
||||
checks
|
||||
.filter((c) => c.status === 'FAILURE')
|
||||
.forEach((c) => {
|
||||
console.log(` - ${c.name}: ${c.description}`);
|
||||
console.log(`\n - ${c.name}: ${c.description}`);
|
||||
if (c.errorMessage) {
|
||||
console.log(` Error: ${c.errorMessage}`);
|
||||
}
|
||||
console.log(`\n${formatMarkdown(scenarioDescription)}`);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,29 @@ import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
|||
|
||||
export class ElicitationDefaultsScenario implements ClientScenario {
|
||||
name = 'elicitation-sep1034-defaults';
|
||||
description =
|
||||
'Test elicitation with default values for all primitive types (SEP-1034)';
|
||||
description = `Test elicitation with default values for all primitive types (SEP-1034).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a tool named \`test_elicitation_sep1034_defaults\` (no arguments) that requests \`elicitation/create\` from the client with a schema containing default values for all primitive types:
|
||||
- \`name\` (string): default "John Doe"
|
||||
- \`age\` (integer): default 30
|
||||
- \`score\` (number): default 95.5
|
||||
- \`status\` (string enum: ["active", "inactive", "pending"]): default "active"
|
||||
- \`verified\` (boolean): default true
|
||||
|
||||
**Returns**: Text content with the elicitation result
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Elicitation completed: action=<accept/decline/cancel>, content={...}"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
|
|
@ -8,7 +8,30 @@ import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
|||
|
||||
export class ElicitationEnumsScenario implements ClientScenario {
|
||||
name = 'elicitation-sep1330-enums';
|
||||
description = 'Test elicitation with enum schema improvements (SEP-1330)';
|
||||
description = `Test elicitation with enum schema improvements (SEP-1330).
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a tool named \`test_elicitation_sep1330_enums\` (no arguments) that requests \`elicitation/create\` from the client with a schema containing all 5 enum variants:
|
||||
|
||||
1. **Untitled single-select**: \`{ type: "string", enum: ["option1", "option2", "option3"] }\`
|
||||
2. **Titled single-select**: \`{ type: "string", oneOf: [{ const: "value1", title: "First Option" }, ...] }\`
|
||||
3. **Legacy titled (deprecated)**: \`{ type: "string", enum: ["opt1", "opt2", "opt3"], enumNames: ["Option One", "Option Two", "Option Three"] }\`
|
||||
4. **Untitled multi-select**: \`{ type: "array", items: { type: "string", enum: ["option1", "option2", "option3"] } }\`
|
||||
5. **Titled multi-select**: \`{ type: "array", items: { anyOf: [{ const: "value1", title: "First Choice" }, ...] } }\`
|
||||
|
||||
**Returns**: Text content with the elicitation result
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Elicitation completed: action=<accept/decline/cancel>, content={...}"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
|
|
@ -7,7 +7,18 @@ import { connectToServer } from './client-helper.js';
|
|||
|
||||
export class ServerInitializeScenario implements ClientScenario {
|
||||
name = 'server-initialize';
|
||||
description = 'Test basic server initialization handshake';
|
||||
description = `Test basic server initialization handshake.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`initialize\`
|
||||
|
||||
**Requirements**:
|
||||
- Accept \`initialize\` request with client info and capabilities
|
||||
- Return valid initialize response with server info, protocol version, and capabilities
|
||||
- Accept \`initialized\` notification from client after handshake
|
||||
|
||||
This test verifies the server can complete the two-phase initialization handshake successfully.`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
|
|
@ -7,7 +7,18 @@ import { connectToServer } from './client-helper.js';
|
|||
|
||||
export class PromptsListScenario implements ClientScenario {
|
||||
name = 'prompts-list';
|
||||
description = 'Test listing available prompts';
|
||||
description = `Test listing available prompts.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`prompts/list\`
|
||||
|
||||
**Requirements**:
|
||||
- Return array of all available prompts
|
||||
- Each prompt MUST have:
|
||||
- \`name\` (string)
|
||||
- \`description\` (string)
|
||||
- \`arguments\` (array, optional) - list of required arguments`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -76,7 +87,25 @@ export class PromptsListScenario implements ClientScenario {
|
|||
|
||||
export class PromptsGetSimpleScenario implements ClientScenario {
|
||||
name = 'prompts-get-simple';
|
||||
description = 'Test getting a simple prompt without arguments';
|
||||
description = `Test getting a simple prompt without arguments.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a prompt named \`test_simple_prompt\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "text",
|
||||
"text": "This is a simple prompt for testing."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -142,7 +171,29 @@ export class PromptsGetSimpleScenario implements ClientScenario {
|
|||
|
||||
export class PromptsGetWithArgsScenario implements ClientScenario {
|
||||
name = 'prompts-get-with-args';
|
||||
description = 'Test parameterized prompt';
|
||||
description = `Test parameterized prompt.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a prompt named \`test_prompt_with_arguments\` with arguments:
|
||||
- \`arg1\` (string, required) - First test argument
|
||||
- \`arg2\` (string, required) - Second test argument
|
||||
|
||||
Returns (with args \`{arg1: "hello", arg2: "world"}\`):
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "text",
|
||||
"text": "Prompt with arguments: arg1='hello', arg2='world'"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -215,7 +266,39 @@ export class PromptsGetWithArgsScenario implements ClientScenario {
|
|||
|
||||
export class PromptsGetEmbeddedResourceScenario implements ClientScenario {
|
||||
name = 'prompts-get-embedded-resource';
|
||||
description = 'Test prompt with embedded resource content';
|
||||
description = `Test prompt with embedded resource content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a prompt named \`test_prompt_with_embedded_resource\` with argument:
|
||||
- \`resourceUri\` (string, required) - URI of the resource to embed
|
||||
|
||||
Returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "resource",
|
||||
"resource": {
|
||||
"uri": "<resourceUri from arguments>",
|
||||
"mimeType": "text/plain",
|
||||
"text": "Embedded resource content for testing."
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "text",
|
||||
"text": "Please process the embedded resource above."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -288,7 +371,33 @@ export class PromptsGetEmbeddedResourceScenario implements ClientScenario {
|
|||
|
||||
export class PromptsGetWithImageScenario implements ClientScenario {
|
||||
name = 'prompts-get-with-image';
|
||||
description = 'Test prompt with image content';
|
||||
description = `Test prompt with image content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement a prompt named \`test_prompt_with_image\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "image",
|
||||
"data": "<base64-encoded-png>",
|
||||
"mimeType": "image/png"
|
||||
}
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "text",
|
||||
"text": "Please analyze the image above."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
|
|
@ -7,7 +7,19 @@ import { connectToServer } from './client-helper.js';
|
|||
|
||||
export class ResourcesListScenario implements ClientScenario {
|
||||
name = 'resources-list';
|
||||
description = 'Test listing available resources';
|
||||
description = `Test listing available resources.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`resources/list\`
|
||||
|
||||
**Requirements**:
|
||||
- Return array of all available **direct resources** (not templates)
|
||||
- Each resource MUST have:
|
||||
- \`uri\` (string)
|
||||
- \`name\` (string)
|
||||
- \`description\` (string)
|
||||
- \`mimeType\` (string, optional)`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -75,7 +87,23 @@ export class ResourcesListScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesReadTextScenario implements ClientScenario {
|
||||
name = 'resources-read-text';
|
||||
description = 'Test reading text resource';
|
||||
description = `Test reading text resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement resource \`test://static-text\` that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"contents": [
|
||||
{
|
||||
"uri": "test://static-text",
|
||||
"mimeType": "text/plain",
|
||||
"text": "This is the content of the static text resource."
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -145,7 +173,23 @@ export class ResourcesReadTextScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesReadBinaryScenario implements ClientScenario {
|
||||
name = 'resources-read-binary';
|
||||
description = 'Test reading binary resource';
|
||||
description = `Test reading binary resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement resource \`test://static-binary\` that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"contents": [
|
||||
{
|
||||
"uri": "test://static-binary",
|
||||
"mimeType": "image/png",
|
||||
"blob": "<base64-encoded-png>"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -213,7 +257,27 @@ export class ResourcesReadBinaryScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesTemplateReadScenario implements ClientScenario {
|
||||
name = 'resources-templates-read';
|
||||
description = 'Test reading resource from template';
|
||||
description = `Test reading resource from template.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement resource template \`test://template/{id}/data\` that substitutes parameters.
|
||||
|
||||
**Behavior**: When client requests \`test://template/123/data\`, substitute \`{id}\` with \`123\`
|
||||
|
||||
Returns (for \`uri: "test://template/123/data"\`):
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"contents": [
|
||||
{
|
||||
"uri": "test://template/123/data",
|
||||
"mimeType": "application/json",
|
||||
"text": "{"id":"123","templateTest":true,"data":"Data for ID: 123"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -286,7 +350,27 @@ export class ResourcesTemplateReadScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesSubscribeScenario implements ClientScenario {
|
||||
name = 'resources-subscribe';
|
||||
description = 'Test subscribing to resource updates';
|
||||
description = `Test subscribing to resource updates.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`resources/subscribe\`
|
||||
|
||||
**Requirements**:
|
||||
- Accept subscription request with URI
|
||||
- Track subscribed URIs
|
||||
- Return empty object \`{}\`
|
||||
|
||||
Example request:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"method": "resources/subscribe",
|
||||
"params": {
|
||||
"uri": "test://watched-resource"
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -336,7 +420,17 @@ export class ResourcesSubscribeScenario implements ClientScenario {
|
|||
|
||||
export class ResourcesUnsubscribeScenario implements ClientScenario {
|
||||
name = 'resources-unsubscribe';
|
||||
description = 'Test unsubscribing from resource';
|
||||
description = `Test unsubscribing from resource.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`resources/unsubscribe\`
|
||||
|
||||
**Requirements**:
|
||||
- Accept unsubscribe request with URI
|
||||
- Remove URI from subscriptions
|
||||
- Stop sending update notifications for that URI
|
||||
- Return empty object \`{}\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -363,7 +457,7 @@ export class ResourcesUnsubscribeScenario implements ClientScenario {
|
|||
specReferences: [
|
||||
{
|
||||
id: 'MCP-Resources-Subscribe',
|
||||
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-subscriptions'
|
||||
url: 'https://modelcontextprotocol.io/specification/2025-06-18/schema#unsubscriberequest'
|
||||
}
|
||||
]
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,7 +13,18 @@ import {
|
|||
|
||||
export class ToolsListScenario implements ClientScenario {
|
||||
name = 'tools-list';
|
||||
description = 'Test listing available tools';
|
||||
description = `Test listing available tools.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`tools/list\`
|
||||
|
||||
**Requirements**:
|
||||
- Return array of all available tools
|
||||
- Each tool MUST have:
|
||||
- \`name\` (string)
|
||||
- \`description\` (string)
|
||||
- \`inputSchema\` (valid JSON Schema object)`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -84,7 +95,22 @@ export class ToolsListScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallSimpleTextScenario implements ClientScenario {
|
||||
name = 'tools-call-simple-text';
|
||||
description = 'Test calling a tool that returns simple text';
|
||||
description = `Test calling a tool that returns simple text.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_simple_text\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "This is a simple text response for testing."
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -153,7 +179,25 @@ export class ToolsCallSimpleTextScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallImageScenario implements ClientScenario {
|
||||
name = 'tools-call-image';
|
||||
description = 'Test calling a tool that returns image content';
|
||||
description = `Test calling a tool that returns image content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_image_content\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"data": "<base64-encoded-png>",
|
||||
"mimeType": "image/png"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Implementation Note**: Use a minimal test image (e.g., 1x1 red pixel PNG)`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -222,7 +266,35 @@ export class ToolsCallImageScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallMultipleContentTypesScenario implements ClientScenario {
|
||||
name = 'tools-call-mixed-content';
|
||||
description = 'Test tool returning multiple content types';
|
||||
description = `Test tool returning multiple content types.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_multiple_content_types\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Multiple content types test:"
|
||||
},
|
||||
{
|
||||
"type": "image",
|
||||
"data": "<base64>",
|
||||
"mimeType": "image/png"
|
||||
},
|
||||
{
|
||||
"type": "resource",
|
||||
"resource": {
|
||||
"uri": "test://mixed-content-resource",
|
||||
"mimeType": "application/json",
|
||||
"text": "{"test":"data","value":123}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -294,7 +366,20 @@ export class ToolsCallMultipleContentTypesScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallWithLoggingScenario implements ClientScenario {
|
||||
name = 'tools-call-with-logging';
|
||||
description = 'Test tool that sends log messages during execution';
|
||||
description = `Test tool that sends log messages during execution.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_tool_with_logging\` with no arguments.
|
||||
|
||||
**Behavior**: During execution, send 3 log notifications at info level:
|
||||
1. "Tool execution started"
|
||||
2. "Tool processing data" (after ~50ms delay)
|
||||
3. "Tool execution completed" (after another ~50ms delay)
|
||||
|
||||
**Returns**: Text content confirming execution
|
||||
|
||||
**Implementation Note**: The delays are important to test that clients can receive multiple log notifications during tool execution`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -366,7 +451,27 @@ export class ToolsCallWithLoggingScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallErrorScenario implements ClientScenario {
|
||||
name = 'tools-call-error';
|
||||
description = 'Test tool error reporting';
|
||||
description = `Test tool error reporting.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_error_handling\` with no arguments.
|
||||
|
||||
**Behavior**: Always throw an error
|
||||
|
||||
**Returns**: JSON-RPC response with \`isError: true\`
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"isError": true,
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "This tool intentionally returns an error for testing"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -430,7 +535,35 @@ export class ToolsCallErrorScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallWithProgressScenario implements ClientScenario {
|
||||
name = 'tools-call-with-progress';
|
||||
description = 'Test tool that reports progress notifications';
|
||||
description = `Test tool that reports progress notifications.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_tool_with_progress\` with no arguments.
|
||||
|
||||
**Behavior**: If \`_meta.progressToken\` is provided in request:
|
||||
- Send progress notification: \`0/100\`
|
||||
- Wait ~50ms
|
||||
- Send progress notification: \`50/100\`
|
||||
- Wait ~50ms
|
||||
- Send progress notification: \`100/100\`
|
||||
|
||||
If no progress token provided, just execute with delays.
|
||||
|
||||
**Returns**: Text content confirming execution
|
||||
|
||||
**Progress Notification Format**:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"method": "notifications/progress",
|
||||
"params": {
|
||||
"progressToken": "<from request._meta.progressToken>",
|
||||
"progress": 50,
|
||||
"total": 100
|
||||
}
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -521,7 +654,49 @@ export class ToolsCallWithProgressScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallSamplingScenario implements ClientScenario {
|
||||
name = 'tools-call-sampling';
|
||||
description = 'Test tool that requests LLM sampling from client';
|
||||
description = `Test tool that requests LLM sampling from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_sampling\` with argument:
|
||||
- \`prompt\` (string, required) - The prompt to send to the LLM
|
||||
|
||||
**Behavior**: Request LLM sampling from the client using \`sampling/createMessage\`
|
||||
|
||||
**Sampling Request**:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"method": "sampling/createMessage",
|
||||
"params": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": {
|
||||
"type": "text",
|
||||
"text": "<prompt from arguments>"
|
||||
}
|
||||
}
|
||||
],
|
||||
"maxTokens": 100
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Returns**: Text content with the LLM's response
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "LLM response: <response from sampling>"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Implementation Note**: If the client doesn't support sampling (no \`sampling\` capability), return an error.`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -606,7 +781,54 @@ export class ToolsCallSamplingScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallElicitationScenario implements ClientScenario {
|
||||
name = 'tools-call-elicitation';
|
||||
description = 'Test tool that requests user input (elicitation) from client';
|
||||
description = `Test tool that requests user input (elicitation) from client.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_elicitation\` with argument:
|
||||
- \`message\` (string, required) - The message to show the user
|
||||
|
||||
**Behavior**: Request user input from the client using \`elicitation/create\`
|
||||
|
||||
**Elicitation Request**:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"method": "elicitation/create",
|
||||
"params": {
|
||||
"message": "<message from arguments>",
|
||||
"requestedSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"username": {
|
||||
"type": "string",
|
||||
"description": "User's response"
|
||||
},
|
||||
"email": {
|
||||
"type": "string",
|
||||
"description": "User's email address"
|
||||
}
|
||||
},
|
||||
"required": ["username", "email"]
|
||||
}
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Returns**: Text content with the user's response
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "User response: <action: accept/decline/cancel, content: {...}>"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Implementation Note**: If the client doesn't support elicitation (no \`elicitation\` capability), return an error.`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -689,7 +911,25 @@ export class ToolsCallElicitationScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallAudioScenario implements ClientScenario {
|
||||
name = 'tools-call-audio';
|
||||
description = 'Test calling a tool that returns audio content';
|
||||
description = `Test calling a tool that returns audio content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_audio_content\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "audio",
|
||||
"data": "<base64-encoded-wav>",
|
||||
"mimeType": "audio/wav"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Implementation Note**: Use a minimal test audio file`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -765,7 +1005,26 @@ export class ToolsCallAudioScenario implements ClientScenario {
|
|||
|
||||
export class ToolsCallEmbeddedResourceScenario implements ClientScenario {
|
||||
name = 'tools-call-embedded-resource';
|
||||
description = 'Test calling a tool that returns embedded resource content';
|
||||
description = `Test calling a tool that returns embedded resource content.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
Implement tool \`test_embedded_resource\` with no arguments that returns:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "resource",
|
||||
"resource": {
|
||||
"uri": "test://embedded-resource",
|
||||
"mimeType": "text/plain",
|
||||
"text": "This is an embedded resource content."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
|
|
@ -7,7 +7,26 @@ import { connectToServer } from './client-helper.js';
|
|||
|
||||
export class LoggingSetLevelScenario implements ClientScenario {
|
||||
name = 'logging-set-level';
|
||||
description = 'Test setting logging level';
|
||||
description = `Test setting logging level.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`logging/setLevel\`
|
||||
|
||||
**Requirements**:
|
||||
- Accept log level setting
|
||||
- Filter subsequent log notifications based on level
|
||||
- Return empty object \`{}\`
|
||||
|
||||
**Log Levels** (in order of severity):
|
||||
- \`debug\`
|
||||
- \`info\`
|
||||
- \`notice\`
|
||||
- \`warning\`
|
||||
- \`error\`
|
||||
- \`critical\`
|
||||
- \`alert\`
|
||||
- \`emergency\``;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
@ -66,7 +85,48 @@ export class LoggingSetLevelScenario implements ClientScenario {
|
|||
|
||||
export class CompletionCompleteScenario implements ClientScenario {
|
||||
name = 'completion-complete';
|
||||
description = 'Test completion endpoint';
|
||||
description = `Test completion endpoint.
|
||||
|
||||
**Server Implementation Requirements:**
|
||||
|
||||
**Endpoint**: \`completion/complete\`
|
||||
|
||||
**Requirements**:
|
||||
- Accept completion requests for prompt or resource template arguments
|
||||
- Provide contextual suggestions based on partial input
|
||||
- Return array of completion values ranked by relevance
|
||||
|
||||
**Request Format**:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"method": "completion/complete",
|
||||
"params": {
|
||||
"ref": {
|
||||
"type": "ref/prompt",
|
||||
"name": "test_prompt_with_arguments"
|
||||
},
|
||||
"argument": {
|
||||
"name": "arg1",
|
||||
"value": "par"
|
||||
}
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Response Format**:
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"completion": {
|
||||
"values": ["paris", "park", "party"],
|
||||
"total": 150,
|
||||
"hasMore": false
|
||||
}
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
**Implementation Note**: For conformance testing, completion support can be minimal or return empty arrays. The capability just needs to be declared and the endpoint must respond correctly.`;
|
||||
|
||||
async run(serverUrl: string): Promise<ConformanceCheck[]> {
|
||||
const checks: ConformanceCheck[] = [];
|
||||
|
|
|
|||
Loading…
Reference in a new issue