feat: flag unportable tool schemas in all three clients

Closes #1005

A tool schema can be perfectly legal JSON Schema and still be rejected by
the client the server is meant to run against — Go's `jsonschema` emits a
bare `true` for `interface{}`, which the SDK accepts and Claude Code
refuses with an opaque "Invalid input". The Inspector passed it through
silently.

Adds `core/json/schemaLint.ts`, a portability lint over a tool's
inputSchema/outputSchema, and surfaces one shared verdict everywhere:
the CLI's `--strict` report (exit 6 on an error-severity finding), the
TUI's tool list marker + detail pane, and the web Tools sidebar flag +
detail section.

Deliberately not a JSON Schema validator: a census of 617 public servers
found zero that fail the SDK's own parse, so a conformance check would
report nothing on real servers. Each rule is instead a construct that is
legal JSON Schema and known to be refused or mishandled by a shipping
client — a bare boolean in a schema position, an array-form `type`, a
remote `$ref`, a schema with no validation keyword, a non-object root.

`test-servers/configs/unportable-schemas-http.json` exercises every rule,
via a new `rawToolSchemas` override — the Zod-built presets cannot emit
any of these shapes, which is the same reason a real server only hits
this when its schemas come from another generator.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UL9retfvAXRgvi6EWk4SY4
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-08-25 01:18:53 -04:00
commit 2f9e0901c1

View file

@ -0,0 +1,251 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { emitResult, runCli } from "../src/cli.js";
import { CliExitCodeError, EXIT_CODES } from "../src/error-handler.js";
import {
lintListResult,
toolsFromResult,
writeSchemaLintReport,
} from "../src/handlers/schema-lint-report.js";
/** A `tools/list` result with one bare-`true` property — the issue's case. */
const DIRTY_LIST = {
tools: [
{
name: "info",
inputSchema: { type: "object", properties: {} },
outputSchema: { type: "object", properties: { data: true } },
},
],
};
/** Same shape, but every schema portable. */
const CLEAN_LIST = {
tools: [{ name: "ok", inputSchema: { type: "object", properties: {} } }],
};
describe("toolsFromResult", () => {
it("returns the tools array", () => {
expect(toolsFromResult(DIRTY_LIST).map((t) => t.name)).toEqual(["info"]);
});
it.each([
["a missing tools key", {}],
["a non-array tools value", { tools: "nope" }],
])("returns nothing for %s", (_label, result) => {
expect(toolsFromResult(result)).toEqual([]);
});
it("skips entries that are not tool-shaped", () => {
const result = {
tools: [null, 3, {}, { name: 7 }, { name: "real", inputSchema: {} }],
};
expect(toolsFromResult(result).map((t) => t.name)).toEqual(["real"]);
});
});
describe("lintListResult", () => {
it("reports the offending tool", () => {
const results = lintListResult(DIRTY_LIST);
expect(results.map((r) => r.toolName)).toEqual(["info"]);
});
it("reports nothing for a clean list", () => {
expect(lintListResult(CLEAN_LIST)).toEqual([]);
});
});
describe("writeSchemaLintReport", () => {
let stderr: string;
let originalWrite: typeof process.stderr.write;
beforeEach(() => {
stderr = "";
originalWrite = process.stderr.write;
process.stderr.write = ((chunk: unknown): boolean => {
stderr += typeof chunk === "string" ? chunk : String(chunk);
return true;
}) as typeof process.stderr.write;
});
afterEach(() => {
process.stderr.write = originalWrite;
});
it("writes nothing at all when there are no findings", () => {
writeSchemaLintReport([], true);
expect(stderr).toBe("");
});
it("writes a one-line hint without --strict", () => {
writeSchemaLintReport(lintListResult(DIRTY_LIST), false);
expect(stderr.trimEnd().split("\n")).toHaveLength(1);
expect(stderr).toContain("Re-run with --strict");
});
it("writes the full report under --strict", () => {
writeSchemaLintReport(lintListResult(DIRTY_LIST), true);
expect(stderr).toContain('Error: tool "info"');
expect(stderr).toContain("Path: outputSchema.properties.data");
expect(stderr).toContain("Suggestion: ");
expect(stderr).not.toContain("Re-run with --strict");
});
});
describe("emitResult — schema lint wiring (#1005)", () => {
let stdout: string;
let stderr: string;
let originalOut: typeof process.stdout.write;
let originalErr: typeof process.stderr.write;
beforeEach(() => {
stdout = "";
stderr = "";
originalOut = process.stdout.write;
originalErr = process.stderr.write;
process.stdout.write = ((chunk: unknown, ...rest: unknown[]): boolean => {
stdout += typeof chunk === "string" ? chunk : String(chunk);
(
rest.find((r) => typeof r === "function") as (() => void) | undefined
)?.();
return true;
}) as typeof process.stdout.write;
process.stderr.write = ((chunk: unknown): boolean => {
stderr += typeof chunk === "string" ? chunk : String(chunk);
return true;
}) as typeof process.stderr.write;
});
afterEach(() => {
process.stdout.write = originalOut;
process.stderr.write = originalErr;
});
it("exits 6 under --strict when a finding is error-severity", async () => {
const promise = emitResult(DIRTY_LIST, undefined, {
method: "tools/list",
strict: true,
format: "text",
});
await expect(promise).rejects.toBeInstanceOf(CliExitCodeError);
await promise.catch((e: CliExitCodeError) => {
expect(e.exitCode).toBe(EXIT_CODES.SCHEMA_INVALID);
expect(e.envelope?.code).toBe("schema_invalid");
});
// The result still goes to stdout; only the report is on stderr.
expect(JSON.parse(stdout)).toEqual(DIRTY_LIST);
expect(stderr).toContain("Path: outputSchema.properties.data");
});
it("succeeds under --strict when only warnings were found", async () => {
const warnOnly = {
tools: [
{
name: "w",
inputSchema: {
type: "object",
properties: { a: { type: ["null", "boolean"] } },
},
},
],
};
await expect(
emitResult(warnOnly, undefined, {
method: "tools/list",
strict: true,
format: "text",
}),
).resolves.toBeUndefined();
expect(stderr).toContain('Warning: tool "w"');
});
it("prints only the hint, and does not fail, without --strict", async () => {
await expect(
emitResult(DIRTY_LIST, undefined, {
method: "tools/list",
format: "text",
}),
).resolves.toBeUndefined();
expect(stderr).toContain("Re-run with --strict");
expect(stderr).not.toContain("Suggestion:");
});
it("folds findings into the --format json envelope under --strict", async () => {
await emitResult(DIRTY_LIST, undefined, {
method: "tools/list",
strict: true,
format: "json",
}).catch(() => {
// exit 6 is expected here; the envelope is what this test asserts.
});
const envelope = JSON.parse(stdout) as {
result: unknown;
schemaFindings?: { toolName: string }[];
};
expect(envelope.result).toEqual(DIRTY_LIST);
expect(envelope.schemaFindings?.[0]?.toolName).toBe("info");
});
it("leaves the json envelope alone when nothing was found", async () => {
await emitResult(CLEAN_LIST, undefined, {
method: "tools/list",
strict: true,
format: "json",
});
expect(JSON.parse(stdout)).toEqual({ result: CLEAN_LIST });
expect(stderr).toBe("");
});
it("does not lint a method other than tools/list", async () => {
// The same payload under `tools/call` must produce no report at all —
// `--strict` is a tools/list flag and the CLI rejects it elsewhere.
await emitResult(DIRTY_LIST, undefined, {
method: "tools/call",
strict: true,
format: "text",
});
expect(stderr).toBe("");
});
it("does not lint an --app-info run", async () => {
await emitResult(
DIRTY_LIST,
{ hasApp: true, toolName: "info" },
{ method: "tools/list", strict: true, appInfo: true, format: "text" },
);
expect(stderr).toBe("");
});
it("reports isError before the schema verdict", async () => {
// Both would throw; the tool-level failure is the more specific one and
// must win, so a caller branching on exit 5 is not shadowed by exit 6.
const promise = emitResult({ ...DIRTY_LIST, isError: true }, undefined, {
method: "tools/list",
strict: true,
toolName: "info",
format: "text",
});
await promise.catch((e: CliExitCodeError) => {
expect(e.exitCode).toBe(EXIT_CODES.TOOL_ERROR);
});
await expect(promise).rejects.toBeInstanceOf(CliExitCodeError);
});
});
describe("--strict argument validation", () => {
it("is rejected with a method other than tools/list", async () => {
await expect(
runCli([
"node",
"cli",
"--cli",
"--method",
"tools/call",
"--tool-name",
"x",
"--strict",
"--server-url",
"http://127.0.0.1:1/mcp",
]),
).rejects.toThrow("--strict requires --method tools/list.");
});
});