modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[compliance tests] initial server scenarios (#4)

Browse source 
* spec for servers

* initial version of everything server and tests - tets are very simple

* some clean up

* lint etc.

* fix package lock agian

---------

Co-authored-by: Paul Carleton <paulc@anthropic.com>
This commit is contained in:
Inna Harper 2025-11-04 15:08:20 +00:00 • committed by GitHub
commit a692dd7bba
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 5678 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/checks/index.ts
View file

@ -1,6 +1,4 @@
// Namespaced exports for client and server checks
// Namespaced exports for client checks
import * as client from './client.js';
import * as server from './server.js';
export const clientChecks = client;
export const serverChecks = server;

37
src/checks/server.ts
View file

@ -1,37 +0,0 @@
import { ConformanceCheck, CheckStatus } from '../types.js';
export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
const result = initializeResponse?.result;
const protocolVersion = result?.protocolVersion;
const serverInfo = result?.serverInfo;
const capabilities = result?.capabilities;
const errors: string[] = [];
if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field');
if (!initializeResponse?.id) errors.push('Missing id field');
if (!result) errors.push('Missing result field');
if (!protocolVersion) errors.push('Missing protocolVersion in result');
if (protocolVersion !== expectedSpecVersion)
errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`);
if (!serverInfo) errors.push('Missing serverInfo in result');
if (!serverInfo?.name) errors.push('Missing server name in serverInfo');
if (!serverInfo?.version) errors.push('Missing server version in serverInfo');
if (capabilities === undefined) errors.push('Missing capabilities in result');
const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
return {
id: 'mcp-server-initialization',
name: 'MCPServerInitialization',
description: 'Validates that MCP server properly responds to initialize request',
status,
timestamp: new Date().toISOString(),
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
details: {
expectedSpecVersion,
response: initializeResponse
},
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
logs: errors.length > 0 ? errors : undefined
};
}

1
src/runner.ts
View file

@ -1,4 +1,3 @@
import { runConformanceTest } from './runner/index.js';
export { runConformanceTest };
export type { ScenarioConfig } from './runner/index.js';

12
src/scenarios/client/tools_call.ts
View file

@ -2,7 +2,8 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import type { Scenario, ConformanceCheck } from '../../types.js';
import express from 'express';
// @ts-expect-error - express has incomplete type definitions
import express, { Request, Response, NextFunction } from 'express';
import { ScenarioUrls } from '../../types.js';
function createServer(checks: ConformanceCheck[]): express.Application {
@ -83,7 +84,7 @@ function createServer(checks: ConformanceCheck[]): express.Application {
const app = express();
app.use(express.json());
app.use((req, res, next) => {
app.use((req: Request, res: Response, next: NextFunction) => {
// Log incoming requests for debugging
// console.log(`Incoming request: ${req.method} ${req.url}`);
checks.push({
@ -105,13 +106,13 @@ function createServer(checks: ConformanceCheck[]): express.Application {
status: 'INFO',
timestamp: new Date().toISOString(),
details: {
// TODO: this isn't working
body: JSON.stringify(res.body)
// TODO: Response body not available in express middleware
statusCode: res.statusCode
}
});
});
app.post('/mcp', async (req, res) => {
app.post('/mcp', async (req: Request, res: Response) => {
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: undefined
});
@ -125,6 +126,7 @@ function createServer(checks: ConformanceCheck[]): express.Application {
export class ToolsCallScenario implements Scenario {
name = 'tools_call';
description = 'Tests calling tools with various parameter types';
private app: express.Application | null = null;
private httpServer: any = null;
private checks: ConformanceCheck[] = [];

74
src/scenarios/index.ts
View file

@ -1,14 +1,84 @@
import { Scenario, ClientScenario } from '../types.js';
import { InitializeScenario } from './client/initialize.js';
import { ToolsCallScenario } from './client/tools_call.js';
import { ServerInitializeClientScenario } from './server/server_initialize.js';
// Import all new server test scenarios
import { ServerInitializeScenario } from './server/lifecycle.js';
import { LoggingSetLevelScenario, CompletionCompleteScenario } from './server/utils.js';
import {
ToolsListScenario,
ToolsCallSimpleTextScenario,
ToolsCallImageScenario,
ToolsCallMultipleContentTypesScenario,
ToolsCallWithLoggingScenario,
ToolsCallErrorScenario,
ToolsCallWithProgressScenario,
ToolsCallSamplingScenario,
ToolsCallElicitationScenario,
ToolsCallAudioScenario,
ToolsCallEmbeddedResourceScenario
} from './server/tools.js';
import {
ResourcesListScenario,
ResourcesReadTextScenario,
ResourcesReadBinaryScenario,
ResourcesTemplateReadScenario,
ResourcesSubscribeScenario,
ResourcesUnsubscribeScenario
} from './server/resources.js';
import {
PromptsListScenario,
PromptsGetSimpleScenario,
PromptsGetWithArgsScenario,
PromptsGetEmbeddedResourceScenario,
PromptsGetWithImageScenario
} from './server/prompts.js';
export const scenarios = new Map<string, Scenario>([
['initialize', new InitializeScenario()],
['tools-call', new ToolsCallScenario()]
]);
export const clientScenarios = new Map<string, ClientScenario>([['initialize', new ServerInitializeClientScenario()]]);
export const clientScenarios = new Map<string, ClientScenario>([
// Lifecycle scenarios
['server-initialize', new ServerInitializeScenario()],
// Utilities scenarios
['logging-set-level', new LoggingSetLevelScenario()],
['completion-complete', new CompletionCompleteScenario()],
// Tools scenarios
['tools-list', new ToolsListScenario()],
['tools-call-simple-text', new ToolsCallSimpleTextScenario()],
['tools-call-image', new ToolsCallImageScenario()],
['tools-call-audio', new ToolsCallAudioScenario()],
['tools-call-embedded-resource', new ToolsCallEmbeddedResourceScenario()],
['tools-call-mixed-content', new ToolsCallMultipleContentTypesScenario()],
['tools-call-with-logging', new ToolsCallWithLoggingScenario()],
['tools-call-error', new ToolsCallErrorScenario()],
['tools-call-with-progress', new ToolsCallWithProgressScenario()],
['tools-call-sampling', new ToolsCallSamplingScenario()],
['tools-call-elicitation', new ToolsCallElicitationScenario()],
// Resources scenarios
['resources-list', new ResourcesListScenario()],
['resources-read-text', new ResourcesReadTextScenario()],
['resources-read-binary', new ResourcesReadBinaryScenario()],
['resources-templates-read', new ResourcesTemplateReadScenario()],
['resources-subscribe', new ResourcesSubscribeScenario()],
['resources-unsubscribe', new ResourcesUnsubscribeScenario()],
// Prompts scenarios
['prompts-list', new PromptsListScenario()],
['prompts-get-simple', new PromptsGetSimpleScenario()],
['prompts-get-with-args', new PromptsGetWithArgsScenario()],
['prompts-get-embedded-resource', new PromptsGetEmbeddedResourceScenario()],
['prompts-get-with-image', new PromptsGetWithImageScenario()]
]);
export function registerScenario(name: string, scenario: Scenario): void {
scenarios.set(name, scenario);

83
src/scenarios/server/client-helper.ts Normal file
View file

@ -0,0 +1,83 @@
/**
* Helper utilities for creating MCP clients to test servers
*/
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { LoggingMessageNotificationSchema, ProgressNotificationSchema } from '@modelcontextprotocol/sdk/types.js';
export interface MCPClientConnection {
client: Client;
close: () => Promise<void>;
}
/**
* Create and connect an MCP client to a server
*/
export async function connectToServer(serverUrl: string): Promise<MCPClientConnection> {
const client = new Client(
{
name: 'conformance-test-client',
version: '1.0.0'
},
{
capabilities: {
// Client capabilities
sampling: {},
elicitation: {}
}
}
);
const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
await client.connect(transport);
return {
client,
close: async () => {
await client.close();
}
};
}
/**
* Helper to collect notifications (logging and progress)
*/
export class NotificationCollector {
private loggingNotifications: any[] = [];
private progressNotifications: any[] = [];
constructor(client: Client) {
// Set up notification handler for logging messages
client.setNotificationHandler(LoggingMessageNotificationSchema, notification => {
this.loggingNotifications.push(notification);
});
// Set up notification handler for progress notifications
client.setNotificationHandler(ProgressNotificationSchema, notification => {
this.progressNotifications.push(notification);
});
}
/**
* Get all collected logging notifications
*/
getLoggingNotifications(): any[] {
return this.loggingNotifications;
}
/**
* Get all collected progress notifications
*/
getProgressNotifications(): any[] {
return this.progressNotifications;
}
/**
* Get all notifications (for backward compatibility)
*/
getNotifications(): any[] {
return this.loggingNotifications;
}
}

58
src/scenarios/server/lifecycle.ts Normal file
View file

@ -0,0 +1,58 @@
/**
* Lifecycle test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer } from './client-helper.js';
export class ServerInitializeScenario implements ClientScenario {
name = 'server-initialize';
description = 'Test basic server initialization handshake';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
// The connection process already does initialization
// Check that we have a connected client
checks.push({
id: 'server-initialize',
name: 'ServerInitialize',
description: 'Server responds to initialize request with valid structure',
status: 'SUCCESS',
timestamp: new Date().toISOString(),
specReferences: [
{
id: 'MCP-Initialize',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
}
],
details: {
serverUrl,
connected: true
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'server-initialize',
name: 'ServerInitialize',
description: 'Server responds to initialize request with valid structure',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed to initialize: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Initialize',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
}
]
});
}
return checks;
}
}

350
src/scenarios/server/prompts.ts Normal file
View file

@ -0,0 +1,350 @@
/**
* Prompts test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer } from './client-helper.js';
export class PromptsListScenario implements ClientScenario {
name = 'prompts-list';
description = 'Test listing available prompts';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.listPrompts();
// Validate response structure
const errors: string[] = [];
if (!result.prompts) {
errors.push('Missing prompts array');
} else {
if (!Array.isArray(result.prompts)) {
errors.push('prompts is not an array');
}
result.prompts.forEach((prompt, index) => {
if (!prompt.name) errors.push(`Prompt ${index}: missing name`);
if (!prompt.description) errors.push(`Prompt ${index}: missing description`);
});
}
checks.push({
id: 'prompts-list',
name: 'PromptsList',
description: 'Server lists available prompts with valid structure',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Prompts-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#listing-prompts'
}
],
details: {
promptCount: result.prompts?.length || 0,
prompts: result.prompts?.map(p => p.name)
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'prompts-list',
name: 'PromptsList',
description: 'Server lists available prompts with valid structure',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Prompts-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#listing-prompts'
}
]
});
}
return checks;
}
}
export class PromptsGetSimpleScenario implements ClientScenario {
name = 'prompts-get-simple';
description = 'Test getting a simple prompt without arguments';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.getPrompt({
name: 'test_simple_prompt'
});
// Validate response
const errors: string[] = [];
if (!result.messages) errors.push('Missing messages array');
if (!Array.isArray(result.messages)) errors.push('messages is not an array');
if (result.messages.length === 0) errors.push('messages array is empty');
result.messages.forEach((message: any, index: number) => {
if (!message.role) errors.push(`Message ${index}: missing role`);
if (!message.content) errors.push(`Message ${index}: missing content`);
});
checks.push({
id: 'prompts-get-simple',
name: 'PromptsGetSimple',
description: 'Get simple prompt successfully',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Prompts-Get',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#getting-prompts'
}
],
details: {
messageCount: result.messages?.length || 0
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'prompts-get-simple',
name: 'PromptsGetSimple',
description: 'Get simple prompt successfully',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Prompts-Get',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#getting-prompts'
}
]
});
}
return checks;
}
}
export class PromptsGetWithArgsScenario implements ClientScenario {
name = 'prompts-get-with-args';
description = 'Test parameterized prompt';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.getPrompt({
name: 'test_prompt_with_arguments',
arguments: {
arg1: 'testValue1',
arg2: 'testValue2'
}
});
// Validate response
const errors: string[] = [];
if (!result.messages) errors.push('Missing messages array');
if (result.messages.length === 0) errors.push('messages array is empty');
// Check if arguments were substituted
const messageText = JSON.stringify(result.messages);
if (!messageText.includes('testValue1')) {
errors.push('arg1 not substituted in prompt');
}
if (!messageText.includes('testValue2')) {
errors.push('arg2 not substituted in prompt');
}
checks.push({
id: 'prompts-get-with-args',
name: 'PromptsGetWithArgs',
description: 'Get parameterized prompt with argument substitution',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Prompts-Get',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#getting-prompts'
}
],
details: {
messageCount: result.messages?.length || 0,
messages: result.messages
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'prompts-get-with-args',
name: 'PromptsGetWithArgs',
description: 'Get parameterized prompt with argument substitution',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Prompts-Get',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#getting-prompts'
}
]
});
}
return checks;
}
}
export class PromptsGetEmbeddedResourceScenario implements ClientScenario {
name = 'prompts-get-embedded-resource';
description = 'Test prompt with embedded resource content';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.getPrompt({
name: 'test_prompt_with_embedded_resource',
arguments: {
resourceUri: 'test://example-resource'
}
});
// Validate response
const errors: string[] = [];
if (!result.messages) errors.push('Missing messages array');
// Look for resource content
const hasResourceContent = result.messages.some(
(msg: any) => msg.content?.type === 'resource' || msg.content?.resource !== undefined
);
if (!hasResourceContent) {
errors.push('No embedded resource found in prompt');
}
checks.push({
id: 'prompts-get-embedded-resource',
name: 'PromptsGetEmbeddedResource',
description: 'Get prompt with embedded resource',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Prompts-Embedded-Resources',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#embedded-resources'
}
],
details: {
messageCount: result.messages?.length || 0,
messages: result.messages
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'prompts-get-embedded-resource',
name: 'PromptsGetEmbeddedResource',
description: 'Get prompt with embedded resource',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Prompts-Embedded-Resources',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#embedded-resources'
}
]
});
}
return checks;
}
}
export class PromptsGetWithImageScenario implements ClientScenario {
name = 'prompts-get-with-image';
description = 'Test prompt with image content';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.getPrompt({
name: 'test_prompt_with_image'
});
// Validate response
const errors: string[] = [];
if (!result.messages) errors.push('Missing messages array');
// Look for image content
const hasImageContent = result.messages.some(
(msg: any) => msg.content?.type === 'image' && msg.content?.data && msg.content?.mimeType
);
if (!hasImageContent) {
errors.push('No image content found in prompt');
}
checks.push({
id: 'prompts-get-with-image',
name: 'PromptsGetWithImage',
description: 'Get prompt with image content',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Prompts-Image',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#image-content'
}
],
details: {
messageCount: result.messages?.length || 0
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'prompts-get-with-image',
name: 'PromptsGetWithImage',
description: 'Get prompt with image content',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Prompts-Image',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/prompts#image-content'
}
]
});
}
return checks;
}
}

389
src/scenarios/server/resources.ts Normal file
View file

@ -0,0 +1,389 @@
/**
* Resources test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer } from './client-helper.js';
export class ResourcesListScenario implements ClientScenario {
name = 'resources-list';
description = 'Test listing available resources';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.listResources();
// Validate response structure
const errors: string[] = [];
if (!result.resources) {
errors.push('Missing resources array');
} else {
if (!Array.isArray(result.resources)) {
errors.push('resources is not an array');
}
result.resources.forEach((resource, index) => {
if (!resource.uri) errors.push(`Resource ${index}: missing uri`);
if (!resource.name) errors.push(`Resource ${index}: missing name`);
});
}
checks.push({
id: 'resources-list',
name: 'ResourcesList',
description: 'Server lists available resources with valid structure',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Resources-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#listing-resources'
}
],
details: {
resourceCount: result.resources?.length || 0,
resources: result.resources?.map(r => r.uri)
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-list',
name: 'ResourcesList',
description: 'Server lists available resources with valid structure',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#listing-resources'
}
]
});
}
return checks;
}
}
export class ResourcesReadTextScenario implements ClientScenario {
name = 'resources-read-text';
description = 'Test reading text resource';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.readResource({
uri: 'test://static-text'
});
// Validate response
const errors: string[] = [];
if (!result.contents) errors.push('Missing contents array');
if (!Array.isArray(result.contents)) errors.push('contents is not an array');
if (result.contents.length === 0) errors.push('contents array is empty');
const content = result.contents[0];
if (content) {
if (!content.uri) errors.push('Content missing uri');
if (!content.mimeType) errors.push('Content missing mimeType');
if (!content.text) errors.push('Content missing text field');
}
checks.push({
id: 'resources-read-text',
name: 'ResourcesReadText',
description: 'Read text resource successfully',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Resources-Read',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#reading-resources'
}
],
details: {
uri: content?.uri,
mimeType: content?.mimeType,
hasText: !!content?.text
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-read-text',
name: 'ResourcesReadText',
description: 'Read text resource successfully',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-Read',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#reading-resources'
}
]
});
}
return checks;
}
}
export class ResourcesReadBinaryScenario implements ClientScenario {
name = 'resources-read-binary';
description = 'Test reading binary resource';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.readResource({
uri: 'test://static-binary'
});
// Validate response
const errors: string[] = [];
if (!result.contents) errors.push('Missing contents array');
if (result.contents.length === 0) errors.push('contents array is empty');
const content = result.contents[0];
if (content) {
if (!content.uri) errors.push('Content missing uri');
if (!content.mimeType) errors.push('Content missing mimeType');
if (!content.blob) errors.push('Content missing blob field');
}
checks.push({
id: 'resources-read-binary',
name: 'ResourcesReadBinary',
description: 'Read binary resource successfully',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Resources-Read',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#reading-resources'
}
],
details: {
uri: content?.uri,
mimeType: content?.mimeType,
hasBlob: !!content?.blob
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-read-binary',
name: 'ResourcesReadBinary',
description: 'Read binary resource successfully',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-Read',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#reading-resources'
}
]
});
}
return checks;
}
}
export class ResourcesTemplateReadScenario implements ClientScenario {
name = 'resources-templates-read';
description = 'Test reading resource from template';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.readResource({
uri: 'test://template/123/data'
});
// Validate response
const errors: string[] = [];
if (!result.contents) errors.push('Missing contents array');
if (result.contents.length === 0) errors.push('contents array is empty');
const content = result.contents[0];
if (content) {
if (!content.uri) errors.push('Content missing uri');
if (!content.text && !content.blob) errors.push('Content missing text or blob');
// Check if parameter was substituted
const text = content.text || (content.blob ? '[binary]' : '');
if (typeof text === 'string' && !text.includes('123')) {
errors.push('Parameter substitution not reflected in content');
}
}
checks.push({
id: 'resources-templates-read',
name: 'ResourcesTemplateRead',
description: 'Read resource from template with parameter substitution',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Resources-Templates',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-templates'
}
],
details: {
uri: content?.uri,
content: content?.text || content?.blob
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-templates-read',
name: 'ResourcesTemplateRead',
description: 'Read resource from template with parameter substitution',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-Templates',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-templates'
}
]
});
}
return checks;
}
}
export class ResourcesSubscribeScenario implements ClientScenario {
name = 'resources-subscribe';
description = 'Test subscribing to resource updates';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
await connection.client.subscribeResource({
uri: 'test://watched-resource'
});
checks.push({
id: 'resources-subscribe',
name: 'ResourcesSubscribe',
description: 'Subscribe to resource successfully',
status: 'SUCCESS',
timestamp: new Date().toISOString(),
specReferences: [
{
id: 'MCP-Resources-Subscribe',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-subscriptions'
}
]
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-subscribe',
name: 'ResourcesSubscribe',
description: 'Subscribe to resource successfully',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-Subscribe',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-subscriptions'
}
]
});
}
return checks;
}
}
export class ResourcesUnsubscribeScenario implements ClientScenario {
name = 'resources-unsubscribe';
description = 'Test unsubscribing from resource';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
// First subscribe
await connection.client.subscribeResource({
uri: 'test://watched-resource'
});
// Then unsubscribe
await connection.client.unsubscribeResource({
uri: 'test://watched-resource'
});
checks.push({
id: 'resources-unsubscribe',
name: 'ResourcesUnsubscribe',
description: 'Unsubscribe from resource successfully',
status: 'SUCCESS',
timestamp: new Date().toISOString(),
specReferences: [
{
id: 'MCP-Resources-Subscribe',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-subscriptions'
}
]
});
await connection.close();
} catch (error) {
checks.push({
id: 'resources-unsubscribe',
name: 'ResourcesUnsubscribe',
description: 'Unsubscribe from resource successfully',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Resources-Subscribe',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/resources#resource-subscriptions'
}
]
});
}
return checks;
}
}

807
src/scenarios/server/tools.ts Normal file
View file

@ -0,0 +1,807 @@
/**
* Tools test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer, NotificationCollector } from './client-helper.js';
import { CallToolResultSchema, CreateMessageRequestSchema, ElicitRequestSchema, Progress } from '@modelcontextprotocol/sdk/types.js';
export class ToolsListScenario implements ClientScenario {
name = 'tools-list';
description = 'Test listing available tools';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.listTools();
// Validate response structure
const errors: string[] = [];
if (!result.tools) {
errors.push('Missing tools array');
} else {
if (!Array.isArray(result.tools)) {
errors.push('tools is not an array');
}
result.tools.forEach((tool, index) => {
if (!tool.name) errors.push(`Tool ${index}: missing name`);
if (!tool.description) errors.push(`Tool ${index}: missing description`);
if (!tool.inputSchema) errors.push(`Tool ${index}: missing inputSchema`);
});
}
checks.push({
id: 'tools-list',
name: 'ToolsList',
description: 'Server lists available tools with valid structure',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#listing-tools'
}
],
details: {
toolCount: result.tools?.length || 0,
tools: result.tools?.map(t => t.name)
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-list',
name: 'ToolsList',
description: 'Server lists available tools with valid structure',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-List',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#listing-tools'
}
]
});
}
return checks;
}
}
export class ToolsCallSimpleTextScenario implements ClientScenario {
name = 'tools-call-simple-text';
description = 'Test calling a tool that returns simple text';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.callTool({
name: 'test_simple_text',
arguments: {}
});
// Validate response
const errors: string[] = [];
const content = (result as any).content;
if (!content) errors.push('Missing content array');
if (!Array.isArray(content)) errors.push('content is not an array');
if (content && content.length === 0) errors.push('content array is empty');
const textContent = content && content.find((c: any) => c.type === 'text');
if (!textContent) errors.push('No text content found');
if (textContent && !textContent.text) errors.push('Text content missing text field');
checks.push({
id: 'tools-call-simple-text',
name: 'ToolsCallSimpleText',
description: 'Tool returns simple text content',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
],
details: {
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-simple-text',
name: 'ToolsCallSimpleText',
description: 'Tool returns simple text content',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
]
});
}
return checks;
}
}
export class ToolsCallImageScenario implements ClientScenario {
name = 'tools-call-image';
description = 'Test calling a tool that returns image content';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.callTool({
name: 'test_image_content',
arguments: {}
});
// Validate response
const errors: string[] = [];
const content = (result as any).content;
if (!content) errors.push('Missing content array');
const imageContent = content && content.find((c: any) => c.type === 'image');
if (!imageContent) errors.push('No image content found');
if (imageContent && !imageContent.data) errors.push('Image content missing data field');
if (imageContent && !imageContent.mimeType) errors.push('Image content missing mimeType');
checks.push({
id: 'tools-call-image',
name: 'ToolsCallImage',
description: 'Tool returns image content',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
],
details: {
mimeType: imageContent?.mimeType,
hasData: !!imageContent?.data
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-image',
name: 'ToolsCallImage',
description: 'Tool returns image content',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
]
});
}
return checks;
}
}
export class ToolsCallMultipleContentTypesScenario implements ClientScenario {
name = 'tools-call-mixed-content';
description = 'Test tool returning multiple content types';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.callTool({
name: 'test_multiple_content_types',
arguments: {}
});
// Validate response
const errors: string[] = [];
const content = (result as any).content;
if (!content) errors.push('Missing content array');
if (content && content.length < 2) errors.push('Expected multiple content items');
const hasText = content && content.some((c: any) => c.type === 'text');
const hasImage = content && content.some((c: any) => c.type === 'image');
const hasResource = content && content.some((c: any) => c.type === 'resource');
if (!hasText) errors.push('Missing text content');
if (!hasImage) errors.push('Missing image content');
if (!hasResource) errors.push('Missing resource content');
checks.push({
id: 'tools-call-mixed-content',
name: 'ToolsCallMixedContent',
description: 'Tool returns multiple content types',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
],
details: {
contentCount: content ? content.length : 0,
contentTypes: content ? content.map((c: any) => c.type) : []
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-mixed-content',
name: 'ToolsCallMixedContent',
description: 'Tool returns multiple content types',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
]
});
}
return checks;
}
}
export class ToolsCallWithLoggingScenario implements ClientScenario {
name = 'tools-call-with-logging';
description = 'Test tool that sends log messages during execution';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const notifications = new NotificationCollector(connection.client);
await connection.client.callTool({
name: 'test_tool_with_logging',
arguments: {}
});
// Wait a bit for notifications to arrive
await new Promise(resolve => setTimeout(resolve, 200));
// Check for log notifications from the NotificationCollector
const logNotifications = notifications.getNotifications();
const errors: string[] = [];
if (logNotifications.length === 0) {
errors.push('No log notifications received');
} else if (logNotifications.length < 3) {
errors.push(`Expected at least 3 log messages, got ${logNotifications.length}`);
}
checks.push({
id: 'tools-call-with-logging',
name: 'ToolsCallWithLogging',
description: 'Tool sends log messages during execution',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Logging',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/logging'
}
],
details: {
logCount: logNotifications.length,
logs: logNotifications.map((n: any) => n.params)
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-with-logging',
name: 'ToolsCallWithLogging',
description: 'Tool sends log messages during execution',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Logging',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/logging'
}
]
});
}
return checks;
}
}
export class ToolsCallErrorScenario implements ClientScenario {
name = 'tools-call-error';
description = 'Test tool error reporting';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result: any = await connection.client.callTool({
name: 'test_error_handling',
arguments: {}
});
// Check if result has isError flag
const hasIsError = result.isError === true;
const hasContent = result.content && result.content.length > 0;
const hasErrorMessage = hasContent && result.content[0].text;
checks.push({
id: 'tools-call-error',
name: 'ToolsCallError',
description: 'Tool returns error correctly',
status: hasIsError && hasErrorMessage ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: !hasIsError
? 'Tool did not return isError: true'
: !hasErrorMessage
? 'Error result missing error message'
: undefined,
specReferences: [
{
id: 'MCP-Error-Handling',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle'
}
],
details: {
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-error',
name: 'ToolsCallError',
description: 'Tool returns error correctly',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Error-Handling',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle'
}
]
});
}
return checks;
}
}
export class ToolsCallWithProgressScenario implements ClientScenario {
name = 'tools-call-with-progress';
description = 'Test tool that reports progress notifications';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const progressUpdates: Array<Progress> = [];
// TODO: investigate why await connection.client.callTool didn't work for progress.
const result = await connection.client.request(
{
method: 'tools/call',
params: {
name: 'test_tool_with_progress',
arguments: {},
_meta: {
progressToken: 'progress-test-1'
}
}
},
CallToolResultSchema,
{
onprogress: progress => {
progressUpdates.push(progress);
}
}
);
const errors: string[] = [];
if (progressUpdates.length === 0) {
errors.push('No progress notifications received');
} else if (progressUpdates.length < 3) {
errors.push(`Expected at least 3 progress notifications, got ${progressUpdates.length}`);
}
if (progressUpdates.length >= 3) {
const progress0 = progressUpdates[0].progress;
const progress1 = progressUpdates[1].progress;
const progress2 = progressUpdates[2].progress;
if (!(progress0 <= progress1 && progress1 <= progress2)) {
errors.push('Progress values should be increasing');
}
}
checks.push({
id: 'tools-call-with-progress',
name: 'ToolsCallWithProgress',
description: 'Tool reports progress notifications',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Progress',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/progress'
}
],
details: {
progressCount: progressUpdates.length,
progressNotifications: progressUpdates.map((n: Progress) => n),
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-with-progress',
name: 'ToolsCallWithProgress',
description: 'Tool reports progress notifications',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Progress',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/progress'
}
]
});
}
return checks;
}
}
export class ToolsCallSamplingScenario implements ClientScenario {
name = 'tools-call-sampling';
description = 'Test tool that requests LLM sampling from client';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
let samplingRequested = false;
connection.client.setRequestHandler(CreateMessageRequestSchema, async _request => {
samplingRequested = true;
return {
role: 'assistant',
content: {
type: 'text',
text: 'This is a test response from the client'
},
model: 'test-model',
stopReason: 'endTurn'
};
});
const result = await connection.client.callTool({
name: 'test_sampling',
arguments: {
prompt: 'Test prompt for sampling'
}
});
const errors: string[] = [];
if (!samplingRequested) {
errors.push('Server did not request sampling from client');
}
const content = (result as any).content;
if (!content || content.length === 0) {
errors.push('Tool did not return content');
}
checks.push({
id: 'tools-call-sampling',
name: 'ToolsCallSampling',
description: 'Tool requests LLM sampling from client',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Sampling',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/sampling'
}
],
details: {
samplingRequested,
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-sampling',
name: 'ToolsCallSampling',
description: 'Tool requests LLM sampling from client',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Sampling',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/sampling'
}
]
});
}
return checks;
}
}
export class ToolsCallElicitationScenario implements ClientScenario {
name = 'tools-call-elicitation';
description = 'Test tool that requests user input (elicitation) from client';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
let elicitationRequested = false;
connection.client.setRequestHandler(ElicitRequestSchema, async _request => {
elicitationRequested = true;
return {
action: 'accept',
content: {
username: 'testuser',
email: 'test@example.com'
}
};
});
const result = await connection.client.callTool({
name: 'test_elicitation',
arguments: {
message: 'Please provide your information'
}
});
const errors: string[] = [];
if (!elicitationRequested) {
errors.push('Server did not request elicitation from client');
}
const content = (result as any).content;
if (!content || content.length === 0) {
errors.push('Tool did not return content');
}
checks.push({
id: 'tools-call-elicitation',
name: 'ToolsCallElicitation',
description: 'Tool requests user input from client',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Elicitation',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/elicitation'
}
],
details: {
elicitationRequested,
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-elicitation',
name: 'ToolsCallElicitation',
description: 'Tool requests user input from client',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Elicitation',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/elicitation'
}
]
});
}
return checks;
}
}
export class ToolsCallAudioScenario implements ClientScenario {
name = 'tools-call-audio';
description = 'Test calling a tool that returns audio content';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.callTool({
name: 'test_audio_content',
arguments: {}
});
// Validate response
const errors: string[] = [];
const content = (result as any).content;
if (!content) errors.push('Missing content array');
if (!Array.isArray(content)) errors.push('content is not an array');
if (content && content.length === 0) errors.push('content array is empty');
const audioContent = content && content.find((c: any) => c.type === 'audio');
if (!audioContent) errors.push('No audio content found');
if (audioContent && !audioContent.data) errors.push('Audio content missing data field');
if (audioContent && !audioContent.mimeType) errors.push('Audio content missing mimeType field');
if (audioContent && audioContent.mimeType !== 'audio/wav')
errors.push(`Expected mimeType 'audio/wav', got '${audioContent.mimeType}'`);
checks.push({
id: 'tools-call-audio',
name: 'ToolsCallAudio',
description: 'Tool returns audio content',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
],
details: {
hasAudioContent: !!audioContent,
audioDataLength: audioContent?.data?.length || 0
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-audio',
name: 'ToolsCallAudio',
description: 'Tool returns audio content',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
]
});
}
return checks;
}
}
export class ToolsCallEmbeddedResourceScenario implements ClientScenario {
name = 'tools-call-embedded-resource';
description = 'Test calling a tool that returns embedded resource content';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
const result = await connection.client.callTool({
name: 'test_embedded_resource',
arguments: {}
});
// Validate response
const errors: string[] = [];
const content = (result as any).content;
if (!content) errors.push('Missing content array');
if (!Array.isArray(content)) errors.push('content is not an array');
if (content && content.length === 0) errors.push('content array is empty');
const resourceContent = content && content.find((c: any) => c.type === 'resource');
if (!resourceContent) errors.push('No resource content found');
if (resourceContent && !resourceContent.resource) errors.push('Resource content missing resource field');
if (resourceContent?.resource) {
if (!resourceContent.resource.uri) errors.push('Resource missing uri field');
if (!resourceContent.resource.mimeType) errors.push('Resource missing mimeType field');
if (!resourceContent.resource.text && !resourceContent.resource.blob) {
errors.push('Resource missing both text and blob fields');
}
}
checks.push({
id: 'tools-call-embedded-resource',
name: 'ToolsCallEmbeddedResource',
description: 'Tool returns embedded resource content',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
],
details: {
hasResourceContent: !!resourceContent,
resourceUri: resourceContent?.resource?.uri
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'tools-call-embedded-resource',
name: 'ToolsCallEmbeddedResource',
description: 'Tool returns embedded resource content',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Tools-Call',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools'
}
]
});
}
return checks;
}
}

136
src/scenarios/server/utils.ts Normal file
View file

@ -0,0 +1,136 @@
/**
* Utilities test scenarios for MCP servers
*/
import { ClientScenario, ConformanceCheck } from '../../types.js';
import { connectToServer } from './client-helper.js';
export class LoggingSetLevelScenario implements ClientScenario {
name = 'logging-set-level';
description = 'Test setting logging level';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
// Send logging/setLevel request
const result = await connection.client.setLoggingLevel('info');
// Validate response (should return empty object {})
const errors: string[] = [];
if (result && Object.keys(result).length > 0) {
errors.push('Expected empty object {} response');
}
checks.push({
id: 'logging-set-level',
name: 'LoggingSetLevel',
description: 'Server accepts logging level setting',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Logging',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/logging'
}
],
details: {
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'logging-set-level',
name: 'LoggingSetLevel',
description: 'Server accepts logging level setting',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Logging',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/logging'
}
]
});
}
return checks;
}
}
export class CompletionCompleteScenario implements ClientScenario {
name = 'completion-complete';
description = 'Test completion endpoint';
async run(serverUrl: string): Promise<ConformanceCheck[]> {
const checks: ConformanceCheck[] = [];
try {
const connection = await connectToServer(serverUrl);
// Send completion/complete request
const result = await connection.client.complete({
ref: {
type: 'ref/prompt',
name: 'test_prompt_with_arguments'
},
argument: {
name: 'arg1',
value: 'test'
}
});
// Validate response structure
const errors: string[] = [];
if (!result.completion) {
errors.push('Missing completion field');
} else {
if (!result.completion.values) errors.push('Missing values array in completion');
if (!Array.isArray(result.completion.values)) errors.push('completion.values is not an array');
}
checks.push({
id: 'completion-complete',
name: 'CompletionComplete',
description: 'Server responds to completion requests',
status: errors.length === 0 ? 'SUCCESS' : 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
specReferences: [
{
id: 'MCP-Completion',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/completion'
}
],
details: {
result
}
});
await connection.close();
} catch (error) {
checks.push({
id: 'completion-complete',
name: 'CompletionComplete',
description: 'Server responds to completion requests',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: [
{
id: 'MCP-Completion',
url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/completion'
}
]
});
}
return checks;
}
}

104
src/server-runner.ts
View file

@ -48,6 +48,7 @@ async function main(): Promise<void> {
const args = process.argv.slice(2);
let serverUrl: string | null = null;
let scenario: string | null = null;
let runAll = false;
for (let i = 0; i < args.length; i++) {
if (args[i] === '--server-url' && i + 1 < args.length) {
@ -56,40 +57,99 @@ async function main(): Promise<void> {
} else if (args[i] === '--scenario' && i + 1 < args.length) {
scenario = args[i + 1];
i++;
} else if (args[i] === '--all') {
runAll = true;
}
}
if (!serverUrl || !scenario) {
console.error('Usage: server-runner --server-url <url> --scenario <scenario>');
console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize');
if (!serverUrl) {
console.error('Usage: server-runner --server-url <url> [--scenario <scenario> | --all]');
console.error('Example: server-runner --server-url http://localhost:3000/mcp --scenario server-initialize');
console.error('Example: server-runner --server-url http://localhost:3000/mcp --all');
process.exit(1);
}
if (!scenario && !runAll) {
console.error('Must specify either --scenario or --all');
process.exit(1);
}
try {
const result = await runServerConformanceTest(serverUrl, scenario);
const allResults: { scenario: string; checks: ConformanceCheck[] }[] = [];
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
if (runAll) {
// Get all server scenarios
const { listClientScenarios } = await import('./scenarios/index.js');
const scenarios = listClientScenarios();
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
console.log(`Running ${scenarios.length} scenarios against ${serverUrl}\n`);
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
try {
const result = await runServerConformanceTest(serverUrl, scenarioName);
allResults.push({ scenario: scenarioName, checks: result.checks });
} catch (error) {
console.error(`Failed to run scenario ${scenarioName}:`, error);
allResults.push({
scenario: scenarioName,
checks: [
{
id: scenarioName,
name: scenarioName,
description: `Failed to run scenario`,
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: error instanceof Error ? error.message : String(error)
}
]
});
}
}
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
// Summary
console.log('\n\n=== SUMMARY ===');
let totalPassed = 0;
let totalFailed = 0;
for (const result of allResults) {
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
totalPassed += passed;
totalFailed += failed;
const status = failed === 0 ? '✓' : '✗';
console.log(`${status} ${result.scenario}: ${passed} passed, ${failed} failed`);
}
console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`);
process.exit(totalFailed > 0 ? 1 : 0);
} else if (scenario) {
const result = await runServerConformanceTest(serverUrl, scenario);
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
console.log(`\nTest Results:`);
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
if (failed > 0) {
console.log('\nFailed Checks:');
result.checks
.filter(c => c.status === 'FAILURE')
.forEach(c => {
console.log(` - ${c.name}: ${c.description}`);
if (c.errorMessage) {
console.log(` Error: ${c.errorMessage}`);
}
});
}
process.exit(failed > 0 ? 1 : 0);
}
process.exit(failed > 0 ? 1 : 0);
} catch (error) {
console.error('Server test runner error:', error);
process.exit(1);