Converted CLI tests to use vitest (#1012)
* Pinned server-everything, fixed all tests to work with current pinned version, fixed problem with undetected failures (isError: true payloads). * First working vitest implementation * Refactoring some single-use configs fixtures and into the refeencing tests * No tests refere to server-everything (or any other server from a registry), all tests actually validate what they say they test. * Removed server-everything dep from CI, minor cleanup * Addressed Claude PR review comments: Added tsx dev dependency, beefed up process termination (possible leak on Windows), beefed up http server cleanup (close all connections), removed unused hasValidJsonOutput, reduced CLI timeout to give it breathing room with vitest timeout. * Made both stdio and http test servers fully composable, cleaned up types, terminology, and usage.
This commit is contained in:
Bob Dickinson
GitHub
parent
94ba4c3b56
commit
44d0e580f6
4 changed files with 4206 additions and 2490 deletions
|
|
@ -1,252 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Integration tests for header functionality
|
||||
* Tests the CLI header parsing end-to-end
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const CLI_PATH = resolve(__dirname, "..", "build", "index.js");
|
||||
|
||||
// ANSI colors for output
|
||||
const colors = {
|
||||
GREEN: "\x1b[32m",
|
||||
RED: "\x1b[31m",
|
||||
YELLOW: "\x1b[33m",
|
||||
BLUE: "\x1b[34m",
|
||||
NC: "\x1b[0m", // No Color
|
||||
};
|
||||
|
||||
let testsPassed = 0;
|
||||
let testsFailed = 0;
|
||||
|
||||
/**
|
||||
* Run a CLI test with given arguments and check for expected behavior
|
||||
*/
|
||||
function runHeaderTest(
|
||||
testName,
|
||||
args,
|
||||
expectSuccess = false,
|
||||
expectedInOutput = null,
|
||||
) {
|
||||
return new Promise((resolve) => {
|
||||
console.log(`\n${colors.BLUE}Testing: ${testName}${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${CLI_PATH} ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
const child = spawn("node", [CLI_PATH, ...args], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout.on("data", (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on("data", (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
const output = stdout + stderr;
|
||||
let passed = true;
|
||||
let reason = "";
|
||||
|
||||
// Check exit code expectation
|
||||
if (expectSuccess && code !== 0) {
|
||||
passed = false;
|
||||
reason = `Expected success (exit code 0) but got ${code}`;
|
||||
} else if (!expectSuccess && code === 0) {
|
||||
passed = false;
|
||||
reason = `Expected failure (non-zero exit code) but got success`;
|
||||
}
|
||||
|
||||
// Check expected output
|
||||
if (passed && expectedInOutput && !output.includes(expectedInOutput)) {
|
||||
passed = false;
|
||||
reason = `Expected output to contain "${expectedInOutput}"`;
|
||||
}
|
||||
|
||||
if (passed) {
|
||||
console.log(`${colors.GREEN}PASS: ${testName}${colors.NC}`);
|
||||
testsPassed++;
|
||||
} else {
|
||||
console.log(`${colors.RED}FAIL: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.RED}Reason: ${reason}${colors.NC}`);
|
||||
console.log(`${colors.RED}Exit code: ${code}${colors.NC}`);
|
||||
console.log(`${colors.RED}Output: ${output}${colors.NC}`);
|
||||
testsFailed++;
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
console.log(
|
||||
`${colors.RED}ERROR: ${testName} - ${error.message}${colors.NC}`,
|
||||
);
|
||||
testsFailed++;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runHeaderIntegrationTests() {
|
||||
console.log(
|
||||
`${colors.YELLOW}=== MCP Inspector CLI Header Integration Tests ===${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}Testing header parsing and validation${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 1: Valid header format should parse successfully (connection will fail)
|
||||
await runHeaderTest(
|
||||
"Valid single header",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
"Authorization: Bearer token123",
|
||||
],
|
||||
false,
|
||||
);
|
||||
|
||||
// Test 2: Multiple headers should parse successfully
|
||||
await runHeaderTest(
|
||||
"Multiple headers",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
"Authorization: Bearer token123",
|
||||
"--header",
|
||||
"X-API-Key: secret123",
|
||||
],
|
||||
false,
|
||||
);
|
||||
|
||||
// Test 3: Invalid header format - no colon
|
||||
await runHeaderTest(
|
||||
"Invalid header format - no colon",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
"InvalidHeader",
|
||||
],
|
||||
false,
|
||||
"Invalid header format",
|
||||
);
|
||||
|
||||
// Test 4: Invalid header format - empty name
|
||||
await runHeaderTest(
|
||||
"Invalid header format - empty name",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
": value",
|
||||
],
|
||||
false,
|
||||
"Invalid header format",
|
||||
);
|
||||
|
||||
// Test 5: Invalid header format - empty value
|
||||
await runHeaderTest(
|
||||
"Invalid header format - empty value",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
"Header:",
|
||||
],
|
||||
false,
|
||||
"Invalid header format",
|
||||
);
|
||||
|
||||
// Test 6: Header with colons in value
|
||||
await runHeaderTest(
|
||||
"Header with colons in value",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
"X-Time: 2023:12:25:10:30:45",
|
||||
],
|
||||
false,
|
||||
);
|
||||
|
||||
// Test 7: Whitespace handling
|
||||
await runHeaderTest(
|
||||
"Whitespace handling in headers",
|
||||
[
|
||||
"https://example.com",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--transport",
|
||||
"http",
|
||||
"--header",
|
||||
" X-Header : value with spaces ",
|
||||
],
|
||||
false,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}=== Test Results ===${colors.NC}`);
|
||||
console.log(`${colors.GREEN}Tests passed: ${testsPassed}${colors.NC}`);
|
||||
console.log(`${colors.RED}Tests failed: ${testsFailed}${colors.NC}`);
|
||||
|
||||
if (testsFailed === 0) {
|
||||
console.log(
|
||||
`${colors.GREEN}All header integration tests passed!${colors.NC}`,
|
||||
);
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.RED}Some header integration tests failed.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on("SIGINT", () => {
|
||||
console.log(`\n${colors.YELLOW}Test interrupted by user${colors.NC}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
console.log(`\n${colors.YELLOW}Test terminated${colors.NC}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Run the tests
|
||||
runHeaderIntegrationTests().catch((error) => {
|
||||
console.error(`${colors.RED}Test runner error: ${error.message}${colors.NC}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,676 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// Colors for output
|
||||
const colors = {
|
||||
GREEN: "\x1b[32m",
|
||||
YELLOW: "\x1b[33m",
|
||||
RED: "\x1b[31m",
|
||||
BLUE: "\x1b[34m",
|
||||
ORANGE: "\x1b[33m",
|
||||
NC: "\x1b[0m", // No Color
|
||||
};
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { spawn } from "child_process";
|
||||
import os from "os";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
// Get directory paths with ESM compatibility
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Track test results
|
||||
let PASSED_TESTS = 0;
|
||||
let FAILED_TESTS = 0;
|
||||
let SKIPPED_TESTS = 0;
|
||||
let TOTAL_TESTS = 0;
|
||||
|
||||
console.log(
|
||||
`${colors.YELLOW}=== MCP Inspector CLI Metadata Tests ===${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}This script tests the MCP Inspector CLI's metadata functionality:${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- General metadata with --metadata option${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Tool-specific metadata with --tool-metadata option${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Metadata parsing with various data types${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Metadata merging (tool-specific overrides general)${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Metadata evaluation in different MCP methods${colors.NC}`,
|
||||
);
|
||||
console.log(`\n`);
|
||||
|
||||
// Get directory paths
|
||||
const SCRIPTS_DIR = __dirname;
|
||||
const PROJECT_ROOT = path.join(SCRIPTS_DIR, "../../");
|
||||
const BUILD_DIR = path.resolve(SCRIPTS_DIR, "../build");
|
||||
|
||||
// Define the test server command using npx
|
||||
const TEST_CMD = "npx";
|
||||
const TEST_ARGS = ["@modelcontextprotocol/server-everything"];
|
||||
|
||||
// Create output directory for test results
|
||||
const OUTPUT_DIR = path.join(SCRIPTS_DIR, "metadata-test-output");
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Create a temporary directory for test files
|
||||
const TEMP_DIR = path.join(os.tmpdir(), "mcp-inspector-metadata-tests");
|
||||
fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
|
||||
// Track servers for cleanup
|
||||
let runningServers = [];
|
||||
|
||||
process.on("exit", () => {
|
||||
try {
|
||||
fs.rmSync(TEMP_DIR, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`${colors.RED}Failed to remove temp directory: ${err.message}${colors.NC}`,
|
||||
);
|
||||
}
|
||||
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Function to run a basic test
|
||||
async function runBasicTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(`${colors.YELLOW}Test timed out: ${testName}${colors.NC}`);
|
||||
child.kill();
|
||||
}, 15000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
// Check if we got valid JSON output (indicating success) even if process didn't exit cleanly
|
||||
const hasValidJsonOutput =
|
||||
output.includes('"tools"') ||
|
||||
output.includes('"resources"') ||
|
||||
output.includes('"prompts"') ||
|
||||
output.includes('"content"') ||
|
||||
output.includes('"messages"') ||
|
||||
output.includes('"contents"');
|
||||
|
||||
if (code === 0 || hasValidJsonOutput) {
|
||||
console.log(`${colors.GREEN}✓ Test passed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.BLUE}First few lines of output:${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(`${colors.RED}✗ Test failed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.RED}Error output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to run an error test (expected to fail)
|
||||
async function runErrorTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing error case: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(
|
||||
`${colors.YELLOW}Error test timed out: ${testName}${colors.NC}`,
|
||||
);
|
||||
child.kill();
|
||||
}, 15000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
// For error tests, we expect a non-zero exit code
|
||||
if (code !== 0) {
|
||||
console.log(
|
||||
`${colors.GREEN}✓ Error test passed: ${testName}${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}Error output (expected):${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.RED}✗ Error test failed: ${testName} (expected error but got success)${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.RED}Output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
async function runTests() {
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running General Metadata Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 1: General metadata with tools/list
|
||||
await runBasicTest(
|
||||
"metadata_tools_list",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
// Test 2: General metadata with resources/list
|
||||
await runBasicTest(
|
||||
"metadata_resources_list",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"resources/list",
|
||||
"--metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
// Test 3: General metadata with prompts/list
|
||||
await runBasicTest(
|
||||
"metadata_prompts_list",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/list",
|
||||
"--metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
// Test 4: General metadata with resources/read
|
||||
await runBasicTest(
|
||||
"metadata_resources_read",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"resources/read",
|
||||
"--uri",
|
||||
"test://static/resource/1",
|
||||
"--metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
// Test 5: General metadata with prompts/get
|
||||
await runBasicTest(
|
||||
"metadata_prompts_get",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"simple_prompt",
|
||||
"--metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Tool-Specific Metadata Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 6: Tool-specific metadata with tools/call
|
||||
await runBasicTest(
|
||||
"metadata_tools_call_tool_meta",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=hello world",
|
||||
"--tool-metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
// Test 7: Tool-specific metadata with complex tool
|
||||
await runBasicTest(
|
||||
"metadata_tools_call_complex_tool_meta",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"add",
|
||||
"--tool-arg",
|
||||
"a=10",
|
||||
"b=20",
|
||||
"--tool-metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Merging Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 8: General metadata + tool-specific metadata (tool-specific should override)
|
||||
await runBasicTest(
|
||||
"metadata_merging_general_and_tool",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=hello world",
|
||||
"--metadata",
|
||||
"client=general-client",
|
||||
"--tool-metadata",
|
||||
"client=test-client",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Parsing Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 10: Metadata with numeric values (should be converted to strings)
|
||||
await runBasicTest(
|
||||
"metadata_parsing_numbers",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"integer_value=42",
|
||||
"decimal_value=3.14159",
|
||||
"negative_value=-10",
|
||||
);
|
||||
|
||||
// Test 11: Metadata with JSON values (should be converted to strings)
|
||||
await runBasicTest(
|
||||
"metadata_parsing_json",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
'json_object="{\\"key\\":\\"value\\"}"',
|
||||
'json_array="[1,2,3]"',
|
||||
'json_string="\\"quoted\\""',
|
||||
);
|
||||
|
||||
// Test 12: Metadata with special characters
|
||||
await runBasicTest(
|
||||
"metadata_parsing_special_chars",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"unicode=🚀🎉✨",
|
||||
"special_chars=!@#$%^&*()",
|
||||
"spaces=hello world with spaces",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Edge Cases ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 13: Single metadata entry
|
||||
await runBasicTest(
|
||||
"metadata_single_entry",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"single_key=single_value",
|
||||
);
|
||||
|
||||
// Test 14: Many metadata entries
|
||||
await runBasicTest(
|
||||
"metadata_many_entries",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"key1=value1",
|
||||
"key2=value2",
|
||||
"key3=value3",
|
||||
"key4=value4",
|
||||
"key5=value5",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Error Cases ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 15: Invalid metadata format (missing equals)
|
||||
await runErrorTest(
|
||||
"metadata_error_invalid_format",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"invalid_format_no_equals",
|
||||
);
|
||||
|
||||
// Test 16: Invalid tool-meta format (missing equals)
|
||||
await runErrorTest(
|
||||
"metadata_error_invalid_tool_meta_format",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=test",
|
||||
"--tool-metadata",
|
||||
"invalid_format_no_equals",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Impact Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 17: Test tool-specific metadata vs general metadata precedence
|
||||
await runBasicTest(
|
||||
"metadata_precedence_tool_overrides_general",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=precedence test",
|
||||
"--metadata",
|
||||
"client=general-client",
|
||||
"--tool-metadata",
|
||||
"client=tool-specific-client",
|
||||
);
|
||||
|
||||
// Test 18: Test metadata with resources methods
|
||||
await runBasicTest(
|
||||
"metadata_resources_methods",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"resources/list",
|
||||
"--metadata",
|
||||
"resource_client=test-resource-client",
|
||||
);
|
||||
|
||||
// Test 19: Test metadata with prompts methods
|
||||
await runBasicTest(
|
||||
"metadata_prompts_methods",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"simple_prompt",
|
||||
"--metadata",
|
||||
"prompt_client=test-prompt-client",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Validation Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 20: Test metadata with special characters in keys
|
||||
await runBasicTest(
|
||||
"metadata_special_key_characters",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=special keys test",
|
||||
"--metadata",
|
||||
"key-with-dashes=value1",
|
||||
"key_with_underscores=value2",
|
||||
"key.with.dots=value3",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Metadata Integration Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 21: Metadata with all MCP methods
|
||||
await runBasicTest(
|
||||
"metadata_integration_all_methods",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--metadata",
|
||||
"integration_test=true",
|
||||
"test_phase=all_methods",
|
||||
);
|
||||
|
||||
// Test 22: Complex metadata scenario
|
||||
await runBasicTest(
|
||||
"metadata_complex_scenario",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=complex test",
|
||||
"--metadata",
|
||||
"session_id=12345",
|
||||
"user_id=67890",
|
||||
"timestamp=2024-01-01T00:00:00Z",
|
||||
"request_id=req-abc-123",
|
||||
"--tool-metadata",
|
||||
"tool_session=session-xyz-789",
|
||||
"execution_context=test",
|
||||
"priority=high",
|
||||
);
|
||||
|
||||
// Test 23: Metadata parsing validation test
|
||||
await runBasicTest(
|
||||
"metadata_parsing_validation",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=parsing validation test",
|
||||
"--metadata",
|
||||
"valid_key=valid_value",
|
||||
"numeric_key=123",
|
||||
"boolean_key=true",
|
||||
'json_key=\'{"test":"value"}\'',
|
||||
"special_key=!@#$%^&*()",
|
||||
"unicode_key=🚀🎉✨",
|
||||
);
|
||||
|
||||
// Print test summary
|
||||
console.log(`\n${colors.YELLOW}=== Test Summary ===${colors.NC}`);
|
||||
console.log(`${colors.GREEN}Passed: ${PASSED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.RED}Failed: ${FAILED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.ORANGE}Skipped: ${SKIPPED_TESTS}${colors.NC}`);
|
||||
console.log(`Total: ${TOTAL_TESTS}`);
|
||||
console.log(
|
||||
`${colors.BLUE}Detailed logs saved to: ${OUTPUT_DIR}${colors.NC}`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.GREEN}All metadata tests completed!${colors.NC}`);
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
runTests().catch((error) => {
|
||||
console.error(
|
||||
`${colors.RED}Tests failed with error: ${error.message}${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,935 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// Colors for output
|
||||
const colors = {
|
||||
GREEN: "\x1b[32m",
|
||||
YELLOW: "\x1b[33m",
|
||||
RED: "\x1b[31m",
|
||||
BLUE: "\x1b[34m",
|
||||
ORANGE: "\x1b[33m",
|
||||
NC: "\x1b[0m", // No Color
|
||||
};
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { spawn } from "child_process";
|
||||
import os from "os";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
// Get directory paths with ESM compatibility
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Track test results
|
||||
let PASSED_TESTS = 0;
|
||||
let FAILED_TESTS = 0;
|
||||
let SKIPPED_TESTS = 0;
|
||||
let TOTAL_TESTS = 0;
|
||||
|
||||
console.log(
|
||||
`${colors.YELLOW}=== MCP Inspector CLI Test Script ===${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}This script tests the MCP Inspector CLI's ability to handle various command line options:${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}- Basic CLI mode${colors.NC}`);
|
||||
console.log(`${colors.BLUE}- Environment variables (-e)${colors.NC}`);
|
||||
console.log(`${colors.BLUE}- Config file (--config)${colors.NC}`);
|
||||
console.log(`${colors.BLUE}- Server selection (--server)${colors.NC}`);
|
||||
console.log(`${colors.BLUE}- Method selection (--method)${colors.NC}`);
|
||||
console.log(`${colors.BLUE}- Resource-related options (--uri)${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}- Prompt-related options (--prompt-name, --prompt-args)${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}- Logging options (--log-level)${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}- Transport types (--transport http/sse/stdio)${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Transport inference from URL suffixes (/mcp, /sse)${colors.NC}`,
|
||||
);
|
||||
console.log(`\n`);
|
||||
|
||||
// Get directory paths
|
||||
const SCRIPTS_DIR = __dirname;
|
||||
const PROJECT_ROOT = path.join(SCRIPTS_DIR, "../../");
|
||||
const BUILD_DIR = path.resolve(SCRIPTS_DIR, "../build");
|
||||
|
||||
// Define the test server command using npx
|
||||
const TEST_CMD = "npx";
|
||||
const TEST_ARGS = ["@modelcontextprotocol/server-everything"];
|
||||
|
||||
// Create output directory for test results
|
||||
const OUTPUT_DIR = path.join(SCRIPTS_DIR, "test-output");
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Create a temporary directory for test files
|
||||
const TEMP_DIR = path.join(os.tmpdir(), "mcp-inspector-tests");
|
||||
fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
|
||||
// Track servers for cleanup
|
||||
let runningServers = [];
|
||||
|
||||
process.on("exit", () => {
|
||||
try {
|
||||
fs.rmSync(TEMP_DIR, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`${colors.RED}Failed to remove temp directory: ${err.message}${colors.NC}`,
|
||||
);
|
||||
}
|
||||
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Use the existing sample config file
|
||||
console.log(
|
||||
`${colors.BLUE}Using existing sample config file: ${PROJECT_ROOT}/sample-config.json${colors.NC}`,
|
||||
);
|
||||
try {
|
||||
const sampleConfig = fs.readFileSync(
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"utf8",
|
||||
);
|
||||
console.log(sampleConfig);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error reading sample config: ${error.message}${colors.NC}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Create an invalid config file for testing
|
||||
const invalidConfigPath = path.join(TEMP_DIR, "invalid-config.json");
|
||||
fs.writeFileSync(invalidConfigPath, '{\n "mcpServers": {\n "invalid": {');
|
||||
|
||||
// Create config files with different transport types for testing
|
||||
const sseConfigPath = path.join(TEMP_DIR, "sse-config.json");
|
||||
fs.writeFileSync(
|
||||
sseConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"test-sse": {
|
||||
type: "sse",
|
||||
url: "http://localhost:3000/sse",
|
||||
note: "Test SSE server",
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const httpConfigPath = path.join(TEMP_DIR, "http-config.json");
|
||||
fs.writeFileSync(
|
||||
httpConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"test-http": {
|
||||
type: "streamable-http",
|
||||
url: "http://localhost:3000/mcp",
|
||||
note: "Test HTTP server",
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const stdioConfigPath = path.join(TEMP_DIR, "stdio-config.json");
|
||||
fs.writeFileSync(
|
||||
stdioConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"test-stdio": {
|
||||
type: "stdio",
|
||||
command: "npx",
|
||||
args: ["@modelcontextprotocol/server-everything"],
|
||||
env: {
|
||||
TEST_ENV: "test-value",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
// Config without type field (backward compatibility)
|
||||
const legacyConfigPath = path.join(TEMP_DIR, "legacy-config.json");
|
||||
fs.writeFileSync(
|
||||
legacyConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"test-legacy": {
|
||||
command: "npx",
|
||||
args: ["@modelcontextprotocol/server-everything"],
|
||||
env: {
|
||||
LEGACY_ENV: "legacy-value",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
// Function to run a basic test
|
||||
async function runBasicTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(`${colors.YELLOW}Test timed out: ${testName}${colors.NC}`);
|
||||
child.kill();
|
||||
}, 10000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
if (code === 0) {
|
||||
console.log(`${colors.GREEN}✓ Test passed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.BLUE}First few lines of output:${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(`${colors.RED}✗ Test failed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.RED}Error output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to run an error test (expected to fail)
|
||||
async function runErrorTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing error case: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(
|
||||
`${colors.YELLOW}Error test timed out: ${testName}${colors.NC}`,
|
||||
);
|
||||
child.kill();
|
||||
}, 10000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
// For error tests, we expect a non-zero exit code
|
||||
if (code !== 0) {
|
||||
console.log(
|
||||
`${colors.GREEN}✓ Error test passed: ${testName}${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}Error output (expected):${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.RED}✗ Error test failed: ${testName} (expected error but got success)${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.RED}Output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
async function runTests() {
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Basic CLI Mode Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 1: Basic CLI mode with method
|
||||
await runBasicTest(
|
||||
"basic_cli_mode",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 2: CLI mode with non-existent method (should fail)
|
||||
await runErrorTest(
|
||||
"nonexistent_method",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"nonexistent/method",
|
||||
);
|
||||
|
||||
// Test 3: CLI mode without method (should fail)
|
||||
await runErrorTest("missing_method", TEST_CMD, ...TEST_ARGS, "--cli");
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Environment Variable Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 4: CLI mode with environment variables
|
||||
await runBasicTest(
|
||||
"env_variables",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"-e",
|
||||
"KEY1=value1",
|
||||
"-e",
|
||||
"KEY2=value2",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 5: CLI mode with invalid environment variable format (should fail)
|
||||
await runErrorTest(
|
||||
"invalid_env_format",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"-e",
|
||||
"INVALID_FORMAT",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 5b: CLI mode with environment variable containing equals sign in value
|
||||
await runBasicTest(
|
||||
"env_variable_with_equals",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"-e",
|
||||
"API_KEY=abc123=xyz789==",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 5c: CLI mode with environment variable containing base64-encoded value
|
||||
await runBasicTest(
|
||||
"env_variable_with_base64",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"-e",
|
||||
"JWT_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0=",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Config File Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 6: Using config file with CLI mode
|
||||
await runBasicTest(
|
||||
"config_file",
|
||||
"--config",
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"--server",
|
||||
"everything",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 7: Using config file without server name (should fail)
|
||||
await runErrorTest(
|
||||
"config_without_server",
|
||||
"--config",
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 8: Using server name without config file (should fail)
|
||||
await runErrorTest(
|
||||
"server_without_config",
|
||||
"--server",
|
||||
"everything",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 9: Using non-existent config file (should fail)
|
||||
await runErrorTest(
|
||||
"nonexistent_config",
|
||||
"--config",
|
||||
"./nonexistent-config.json",
|
||||
"--server",
|
||||
"everything",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 10: Using invalid config file format (should fail)
|
||||
await runErrorTest(
|
||||
"invalid_config",
|
||||
"--config",
|
||||
invalidConfigPath,
|
||||
"--server",
|
||||
"everything",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 11: Using config file with non-existent server (should fail)
|
||||
await runErrorTest(
|
||||
"nonexistent_server",
|
||||
"--config",
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"--server",
|
||||
"nonexistent",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Resource-Related Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 16: CLI mode with resource read
|
||||
await runBasicTest(
|
||||
"resource_read",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"resources/read",
|
||||
"--uri",
|
||||
"test://static/resource/1",
|
||||
);
|
||||
|
||||
// Test 17: CLI mode with resource read but missing URI (should fail)
|
||||
await runErrorTest(
|
||||
"missing_uri",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"resources/read",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Prompt-Related Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 18: CLI mode with prompt get
|
||||
await runBasicTest(
|
||||
"prompt_get",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"simple_prompt",
|
||||
);
|
||||
|
||||
// Test 19: CLI mode with prompt get and args
|
||||
await runBasicTest(
|
||||
"prompt_get_with_args",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"complex_prompt",
|
||||
"--prompt-args",
|
||||
"temperature=0.7",
|
||||
"style=concise",
|
||||
);
|
||||
|
||||
// Test 20: CLI mode with prompt get but missing prompt name (should fail)
|
||||
await runErrorTest(
|
||||
"missing_prompt_name",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}=== Running Logging Tests ===${colors.NC}`);
|
||||
|
||||
// Test 21: CLI mode with log level
|
||||
await runBasicTest(
|
||||
"log_level",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"logging/setLevel",
|
||||
"--log-level",
|
||||
"debug",
|
||||
);
|
||||
|
||||
// Test 22: CLI mode with invalid log level (should fail)
|
||||
await runErrorTest(
|
||||
"invalid_log_level",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"logging/setLevel",
|
||||
"--log-level",
|
||||
"invalid",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Combined Option Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Note about the combined options issue
|
||||
console.log(
|
||||
`${colors.BLUE}Testing combined options with environment variables and config file.${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 23: CLI mode with config file, environment variables, and tool call
|
||||
await runBasicTest(
|
||||
"combined_options",
|
||||
"--config",
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"--server",
|
||||
"everything",
|
||||
"-e",
|
||||
"CLI_ENV_VAR=cli_value",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 24: CLI mode with all possible options (that make sense together)
|
||||
await runBasicTest(
|
||||
"all_options",
|
||||
"--config",
|
||||
path.join(PROJECT_ROOT, "sample-config.json"),
|
||||
"--server",
|
||||
"everything",
|
||||
"-e",
|
||||
"CLI_ENV_VAR=cli_value",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=Hello",
|
||||
"--log-level",
|
||||
"debug",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Config Transport Type Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 25: Config with stdio transport type
|
||||
await runBasicTest(
|
||||
"config_stdio_type",
|
||||
"--config",
|
||||
stdioConfigPath,
|
||||
"--server",
|
||||
"test-stdio",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 26: Config with SSE transport type (CLI mode) - expects connection error
|
||||
await runErrorTest(
|
||||
"config_sse_type_cli",
|
||||
"--config",
|
||||
sseConfigPath,
|
||||
"--server",
|
||||
"test-sse",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 27: Config with streamable-http transport type (CLI mode) - expects connection error
|
||||
await runErrorTest(
|
||||
"config_http_type_cli",
|
||||
"--config",
|
||||
httpConfigPath,
|
||||
"--server",
|
||||
"test-http",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 28: Legacy config without type field (backward compatibility)
|
||||
await runBasicTest(
|
||||
"config_legacy_no_type",
|
||||
"--config",
|
||||
legacyConfigPath,
|
||||
"--server",
|
||||
"test-legacy",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Default Server Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Create config with single server for auto-selection
|
||||
const singleServerConfigPath = path.join(
|
||||
TEMP_DIR,
|
||||
"single-server-config.json",
|
||||
);
|
||||
fs.writeFileSync(
|
||||
singleServerConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"only-server": {
|
||||
command: "npx",
|
||||
args: ["@modelcontextprotocol/server-everything"],
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
// Create config with default-server
|
||||
const defaultServerConfigPath = path.join(
|
||||
TEMP_DIR,
|
||||
"default-server-config.json",
|
||||
);
|
||||
fs.writeFileSync(
|
||||
defaultServerConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
"default-server": {
|
||||
command: "npx",
|
||||
args: ["@modelcontextprotocol/server-everything"],
|
||||
},
|
||||
"other-server": {
|
||||
command: "node",
|
||||
args: ["other.js"],
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
// Create config with multiple servers (no default)
|
||||
const multiServerConfigPath = path.join(TEMP_DIR, "multi-server-config.json");
|
||||
fs.writeFileSync(
|
||||
multiServerConfigPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
server1: {
|
||||
command: "npx",
|
||||
args: ["@modelcontextprotocol/server-everything"],
|
||||
},
|
||||
server2: {
|
||||
command: "node",
|
||||
args: ["other.js"],
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
// Test 29: Config with single server auto-selection
|
||||
await runBasicTest(
|
||||
"single_server_auto_select",
|
||||
"--config",
|
||||
singleServerConfigPath,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 30: Config with default-server should now require explicit selection (multiple servers)
|
||||
await runErrorTest(
|
||||
"default_server_requires_explicit_selection",
|
||||
"--config",
|
||||
defaultServerConfigPath,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 31: Config with multiple servers and no default (should fail)
|
||||
await runErrorTest(
|
||||
"multi_server_no_default",
|
||||
"--config",
|
||||
multiServerConfigPath,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running HTTP Transport Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
console.log(
|
||||
`${colors.BLUE}Starting server-everything in streamableHttp mode.${colors.NC}`,
|
||||
);
|
||||
const httpServer = spawn(
|
||||
"npx",
|
||||
["@modelcontextprotocol/server-everything", "streamableHttp"],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
);
|
||||
runningServers.push(httpServer);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000));
|
||||
|
||||
// Test 32: HTTP transport inferred from URL ending with /mcp
|
||||
await runBasicTest(
|
||||
"http_transport_inferred",
|
||||
"http://127.0.0.1:3001/mcp",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 33: HTTP transport with explicit --transport http flag
|
||||
await runBasicTest(
|
||||
"http_transport_with_explicit_flag",
|
||||
"http://127.0.0.1:3001/mcp",
|
||||
"--transport",
|
||||
"http",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 34: HTTP transport with suffix and --transport http flag
|
||||
await runBasicTest(
|
||||
"http_transport_with_explicit_flag_and_suffix",
|
||||
"http://127.0.0.1:3001/mcp",
|
||||
"--transport",
|
||||
"http",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 35: SSE transport given to HTTP server (should fail)
|
||||
await runErrorTest(
|
||||
"sse_transport_given_to_http_server",
|
||||
"http://127.0.0.1:3001",
|
||||
"--transport",
|
||||
"sse",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 36: HTTP transport without URL (should fail)
|
||||
await runErrorTest(
|
||||
"http_transport_without_url",
|
||||
"--transport",
|
||||
"http",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Test 37: SSE transport without URL (should fail)
|
||||
await runErrorTest(
|
||||
"sse_transport_without_url",
|
||||
"--transport",
|
||||
"sse",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
// Kill HTTP server
|
||||
try {
|
||||
process.kill(-httpServer.pid);
|
||||
console.log(
|
||||
`${colors.BLUE}HTTP server killed, waiting for port to be released...${colors.NC}`,
|
||||
);
|
||||
} catch (e) {
|
||||
console.log(
|
||||
`${colors.RED}Error killing HTTP server: ${e.message}${colors.NC}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Print test summary
|
||||
console.log(`\n${colors.YELLOW}=== Test Summary ===${colors.NC}`);
|
||||
console.log(`${colors.GREEN}Passed: ${PASSED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.RED}Failed: ${FAILED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.ORANGE}Skipped: ${SKIPPED_TESTS}${colors.NC}`);
|
||||
console.log(`Total: ${TOTAL_TESTS}`);
|
||||
console.log(
|
||||
`${colors.BLUE}Detailed logs saved to: ${OUTPUT_DIR}${colors.NC}`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.GREEN}All tests completed!${colors.NC}`);
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
runTests().catch((error) => {
|
||||
console.error(
|
||||
`${colors.RED}Tests failed with error: ${error.message}${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,614 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
// Colors for output
|
||||
const colors = {
|
||||
GREEN: "\x1b[32m",
|
||||
YELLOW: "\x1b[33m",
|
||||
RED: "\x1b[31m",
|
||||
BLUE: "\x1b[34m",
|
||||
ORANGE: "\x1b[33m",
|
||||
NC: "\x1b[0m", // No Color
|
||||
};
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { spawn } from "child_process";
|
||||
import os from "os";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
// Get directory paths with ESM compatibility
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Track test results
|
||||
let PASSED_TESTS = 0;
|
||||
let FAILED_TESTS = 0;
|
||||
let SKIPPED_TESTS = 0;
|
||||
let TOTAL_TESTS = 0;
|
||||
|
||||
console.log(`${colors.YELLOW}=== MCP Inspector CLI Tool Tests ===${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}This script tests the MCP Inspector CLI's tool-related functionality:${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}- Tool discovery and listing${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}- JSON argument parsing (strings, numbers, booleans, objects, arrays)${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}- Tool schema validation${colors.NC}`);
|
||||
console.log(
|
||||
`${colors.BLUE}- Tool execution with various argument types${colors.NC}`,
|
||||
);
|
||||
console.log(
|
||||
`${colors.BLUE}- Error handling for invalid tools and arguments${colors.NC}`,
|
||||
);
|
||||
console.log(`\n`);
|
||||
|
||||
// Get directory paths
|
||||
const SCRIPTS_DIR = __dirname;
|
||||
const PROJECT_ROOT = path.join(SCRIPTS_DIR, "../../");
|
||||
const BUILD_DIR = path.resolve(SCRIPTS_DIR, "../build");
|
||||
|
||||
// Define the test server command using npx
|
||||
const TEST_CMD = "npx";
|
||||
const TEST_ARGS = ["@modelcontextprotocol/server-everything"];
|
||||
|
||||
// Create output directory for test results
|
||||
const OUTPUT_DIR = path.join(SCRIPTS_DIR, "tool-test-output");
|
||||
if (!fs.existsSync(OUTPUT_DIR)) {
|
||||
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Create a temporary directory for test files
|
||||
const TEMP_DIR = path.join(os.tmpdir(), "mcp-inspector-tool-tests");
|
||||
fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
|
||||
// Track servers for cleanup
|
||||
let runningServers = [];
|
||||
|
||||
process.on("exit", () => {
|
||||
try {
|
||||
fs.rmSync(TEMP_DIR, { recursive: true, force: true });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`${colors.RED}Failed to remove temp directory: ${err.message}${colors.NC}`,
|
||||
);
|
||||
}
|
||||
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGINT", () => {
|
||||
runningServers.forEach((server) => {
|
||||
try {
|
||||
process.kill(-server.pid);
|
||||
} catch (e) {}
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Function to run a basic test
|
||||
async function runBasicTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(`${colors.YELLOW}Test timed out: ${testName}${colors.NC}`);
|
||||
child.kill();
|
||||
}, 10000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
if (code === 0) {
|
||||
console.log(`${colors.GREEN}✓ Test passed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.BLUE}First few lines of output:${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(`${colors.RED}✗ Test failed: ${testName}${colors.NC}`);
|
||||
console.log(`${colors.RED}Error output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to run an error test (expected to fail)
|
||||
async function runErrorTest(testName, ...args) {
|
||||
const outputFile = path.join(
|
||||
OUTPUT_DIR,
|
||||
`${testName.replace(/\//g, "_")}.log`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.YELLOW}Testing error case: ${testName}${colors.NC}`);
|
||||
TOTAL_TESTS++;
|
||||
|
||||
// Run the command and capture output
|
||||
console.log(
|
||||
`${colors.BLUE}Command: node ${BUILD_DIR}/cli.js ${args.join(" ")}${colors.NC}`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Create a write stream for the output file
|
||||
const outputStream = fs.createWriteStream(outputFile);
|
||||
|
||||
// Spawn the process
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("node", [path.join(BUILD_DIR, "cli.js"), ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
console.log(
|
||||
`${colors.YELLOW}Error test timed out: ${testName}${colors.NC}`,
|
||||
);
|
||||
child.kill();
|
||||
}, 10000);
|
||||
|
||||
// Pipe stdout and stderr to the output file
|
||||
child.stdout.pipe(outputStream);
|
||||
child.stderr.pipe(outputStream);
|
||||
|
||||
// Also capture output for display
|
||||
let output = "";
|
||||
child.stdout.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timeout);
|
||||
outputStream.end();
|
||||
|
||||
// For error tests, we expect a non-zero exit code
|
||||
if (code !== 0) {
|
||||
console.log(
|
||||
`${colors.GREEN}✓ Error test passed: ${testName}${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.BLUE}Error output (expected):${colors.NC}`);
|
||||
const firstFewLines = output
|
||||
.split("\n")
|
||||
.slice(0, 5)
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n");
|
||||
console.log(firstFewLines);
|
||||
PASSED_TESTS++;
|
||||
resolve(true);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.RED}✗ Error test failed: ${testName} (expected error but got success)${colors.NC}`,
|
||||
);
|
||||
console.log(`${colors.RED}Output:${colors.NC}`);
|
||||
console.log(
|
||||
output
|
||||
.split("\n")
|
||||
.map((line) => ` ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
|
||||
// Stop after any error is encountered
|
||||
console.log(
|
||||
`${colors.YELLOW}Stopping tests due to error. Please validate and fix before continuing.${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`${colors.RED}Error running test: ${error.message}${colors.NC}`,
|
||||
);
|
||||
FAILED_TESTS++;
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
async function runTests() {
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Tool Discovery Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 1: List available tools
|
||||
await runBasicTest(
|
||||
"tool_discovery_list",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running JSON Argument Parsing Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 2: String arguments (backward compatibility)
|
||||
await runBasicTest(
|
||||
"json_args_string",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=hello world",
|
||||
);
|
||||
|
||||
// Test 3: Number arguments
|
||||
await runBasicTest(
|
||||
"json_args_number_integer",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"add",
|
||||
"--tool-arg",
|
||||
"a=42",
|
||||
"b=58",
|
||||
);
|
||||
|
||||
// Test 4: Number arguments with decimals (using add tool with decimal numbers)
|
||||
await runBasicTest(
|
||||
"json_args_number_decimal",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"add",
|
||||
"--tool-arg",
|
||||
"a=19.99",
|
||||
"b=20.01",
|
||||
);
|
||||
|
||||
// Test 5: Boolean arguments - true
|
||||
await runBasicTest(
|
||||
"json_args_boolean_true",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"annotatedMessage",
|
||||
"--tool-arg",
|
||||
"messageType=success",
|
||||
"includeImage=true",
|
||||
);
|
||||
|
||||
// Test 6: Boolean arguments - false
|
||||
await runBasicTest(
|
||||
"json_args_boolean_false",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"annotatedMessage",
|
||||
"--tool-arg",
|
||||
"messageType=error",
|
||||
"includeImage=false",
|
||||
);
|
||||
|
||||
// Test 7: Null arguments (using echo with string "null")
|
||||
await runBasicTest(
|
||||
"json_args_null",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
'message="null"',
|
||||
);
|
||||
|
||||
// Test 14: Multiple arguments with mixed types (using add tool)
|
||||
await runBasicTest(
|
||||
"json_args_multiple_mixed",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"add",
|
||||
"--tool-arg",
|
||||
"a=42.5",
|
||||
"b=57.5",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running JSON Parsing Edge Cases ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 15: Invalid JSON should fall back to string
|
||||
await runBasicTest(
|
||||
"json_args_invalid_fallback",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message={invalid json}",
|
||||
);
|
||||
|
||||
// Test 16: Empty string value
|
||||
await runBasicTest(
|
||||
"json_args_empty_value",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
'message=""',
|
||||
);
|
||||
|
||||
// Test 17: Special characters in strings
|
||||
await runBasicTest(
|
||||
"json_args_special_chars",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
'message="C:\\\\Users\\\\test"',
|
||||
);
|
||||
|
||||
// Test 18: Unicode characters
|
||||
await runBasicTest(
|
||||
"json_args_unicode",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
'message="🚀🎉✨"',
|
||||
);
|
||||
|
||||
// Test 19: Arguments with equals signs in values
|
||||
await runBasicTest(
|
||||
"json_args_equals_in_value",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=2+2=4",
|
||||
);
|
||||
|
||||
// Test 20: Base64-like strings
|
||||
await runBasicTest(
|
||||
"json_args_base64_like",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0=",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Tool Error Handling Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 21: Non-existent tool
|
||||
await runErrorTest(
|
||||
"tool_error_nonexistent",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"nonexistent_tool",
|
||||
"--tool-arg",
|
||||
"message=test",
|
||||
);
|
||||
|
||||
// Test 22: Missing tool name
|
||||
await runErrorTest(
|
||||
"tool_error_missing_name",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-arg",
|
||||
"message=test",
|
||||
);
|
||||
|
||||
// Test 23: Invalid tool argument format
|
||||
await runErrorTest(
|
||||
"tool_error_invalid_arg_format",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"invalid_format_no_equals",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Prompt JSON Argument Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 24: Prompt with JSON arguments
|
||||
await runBasicTest(
|
||||
"prompt_json_args_mixed",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"complex_prompt",
|
||||
"--prompt-args",
|
||||
"temperature=0.7",
|
||||
'style="concise"',
|
||||
'options={"format":"json","max_tokens":100}',
|
||||
);
|
||||
|
||||
// Test 25: Prompt with simple arguments
|
||||
await runBasicTest(
|
||||
"prompt_json_args_simple",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"prompts/get",
|
||||
"--prompt-name",
|
||||
"simple_prompt",
|
||||
"--prompt-args",
|
||||
"name=test",
|
||||
"count=5",
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\n${colors.YELLOW}=== Running Backward Compatibility Tests ===${colors.NC}`,
|
||||
);
|
||||
|
||||
// Test 26: Ensure existing string-only usage still works
|
||||
await runBasicTest(
|
||||
"backward_compatibility_strings",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"echo",
|
||||
"--tool-arg",
|
||||
"message=hello",
|
||||
);
|
||||
|
||||
// Test 27: Multiple string arguments (existing pattern) - using add tool
|
||||
await runBasicTest(
|
||||
"backward_compatibility_multiple_strings",
|
||||
TEST_CMD,
|
||||
...TEST_ARGS,
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/call",
|
||||
"--tool-name",
|
||||
"add",
|
||||
"--tool-arg",
|
||||
"a=10",
|
||||
"b=20",
|
||||
);
|
||||
|
||||
// Print test summary
|
||||
console.log(`\n${colors.YELLOW}=== Test Summary ===${colors.NC}`);
|
||||
console.log(`${colors.GREEN}Passed: ${PASSED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.RED}Failed: ${FAILED_TESTS}${colors.NC}`);
|
||||
console.log(`${colors.ORANGE}Skipped: ${SKIPPED_TESTS}${colors.NC}`);
|
||||
console.log(`Total: ${TOTAL_TESTS}`);
|
||||
console.log(
|
||||
`${colors.BLUE}Detailed logs saved to: ${OUTPUT_DIR}${colors.NC}`,
|
||||
);
|
||||
|
||||
console.log(`\n${colors.GREEN}All tool tests completed!${colors.NC}`);
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
runTests().catch((error) => {
|
||||
console.error(
|
||||
`${colors.RED}Tests failed with error: ${error.message}${colors.NC}`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Reference in a new issue