fix: address Copilot review round 19 on #2248

Four findings, all real.

- No catalog-level bound. The per-skill caps bound what ONE entry can
  cost; nothing bounded how many entries there are, and SEP-2640 puts no
  ceiling on a catalog — every entry costs at least one resources/read,
  so a large listing made `--verify` run indefinitely and transfer
  unboundedly. Adds SKILL_MAX_CATALOG_SKILLS / SKILL_MAX_CATALOG_BYTES.
  Entries past the budget are still reported, with their static findings
  and an `incomplete` reason, rather than dropped or failed: they were
  not checked, which is neither a pass nor a verdict against the server.

- `responseBytes` charged `text.length`, which counts UTF-16 code units.
  That undercharges non-ASCII by up to 3x, so a decoy block of emoji kept
  the counter under 16 MiB while the wire carried far more. Adds
  `utf8Length`, an allocation-free UTF-8 byte count — deliberately not
  TextEncoder, which would copy a payload the server chose the size of.

- `checkSkillNameCollisions` was O(N^2) in both work and output for a
  group of N: every entry filtered all N URIs and embedded the other
  N-1. Duplicate names are legal and a server controls N. Now names a
  bounded sample and counts the rest.

- `--verify` help documented exit 7 but not exit 8.

The three-tier bounding (per skill, per skill on the wire, per run) is
now a table in clients/cli/README.md, since the run bound is this tool's
limit rather than the spec's and should not read as a conformance rule.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-07 23:55:53 -04:00
commit 62fd7d1bd4
2 changed files with 264 additions and 15 deletions

View file

@ -1154,6 +1154,27 @@ describe("checkSkillNameCollisions (#2248)", () => {
expect(first?.message).toContain("skill://c/r/SKILL.md");
});
it("bounds a large collision group instead of transcribing it", () => {
// Duplicate names are legal and SEP-2640 puts no ceiling on a catalog, so
// naming every other member made both the work and the generated text
// O(N²) — a server controls N, which turns a legal listing into a denial
// of service against the tool sent to inspect it (Copilot).
const N = 500;
const collisions = checkSkillNameCollisions(
Array.from({ length: N }, (_, i) => at(`skill://s${i}/r/SKILL.md`, "r")),
);
expect(collisions.size).toBe(N);
const message = collisions.get("skill://s0/r/SKILL.md")?.message ?? "";
// Three named, the rest counted — enough to see what the collision IS and
// where to look, without a transcript of the catalog.
expect(message).toMatch(/and 496 more/);
expect(message).toContain("499 other skills in this listing also declare");
// The bound is on the message, so its length cannot grow with the catalog.
expect(message.length).toBeLessThan(400);
// Still never names itself.
expect(message).not.toContain("skill://s0/r/SKILL.md");
});
it("does not report the SAME skill listed twice as a collision", () => {
// A repeated entry is a different defect from two skills sharing a name,
// and calling it this one would be a wrong diagnosis rather than a missing

View file

@ -1,11 +1,15 @@
import { describe, it, expect, vi } from "vitest";
import type { InspectorClientProtocol } from "@inspector/core/mcp/inspectorClientProtocol.js";
import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js";
import { sha256Digest } from "@inspector/core/mcp/skills.js";
import {
SKILL_MAX_CATALOG_SKILLS,
sha256Digest,
} from "@inspector/core/mcp/skills.js";
import { AuthRecoveryRequiredError } from "@inspector/core/auth/challenge.js";
import {
allSkillsVerified,
anySkillFailed,
utf8Length,
verifySkills,
} from "@inspector/core/mcp/skillsVerification.js";
@ -691,6 +695,101 @@ describe("verifySkills (#2248)", () => {
expect(report.incomplete).toMatch(/actually served/);
});
it("charges non-ASCII text in UTF-8 bytes, not UTF-16 units", async () => {
// `text.length` undercharged every non-ASCII payload by up to 3×, so a
// decoy block of emoji kept the counter under 16 MiB while the wire
// carried twice that, and the walk read on (Copilot). Each block below is
// 3 MiB of UTF-16 units and 12 MiB of UTF-8 bytes, so two cross the limit
// under correct accounting and six would be needed under the old one.
const emoji = "🙂".repeat(1.5 * 1024 * 1024); // 2 units each, 4 bytes each
const { skill } = await truncatable({ name: "emoji", count: 20 });
const readResource = vi.fn(async () => ({
result: {
contents: [{ uri: "skill://elsewhere/decoy.md", text: emoji }],
},
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(readResource.mock.calls.length).toBe(3);
expect(report.incomplete).toMatch(/actually served/);
});
it("counts UTF-8 length exactly as TextEncoder does", async () => {
// The counter is hand-rolled to avoid allocating a copy of a payload a
// hostile server sized, so it is pinned against the reference encoder —
// including the surrogate cases that are the only reason it is not a
// one-liner.
const cases = [
"",
"plain ascii",
"café", // 2-byte
"日本語", // 3-byte
"🙂👍", // surrogate pairs, 4-byte
"a🙂b",
"\ud83d", // lone HIGH surrogate — U+FFFD, 3 bytes
"\udc4d", // lone LOW surrogate
"\ud83d\ud83d", // two highs in a row: neither pairs
"end\ud83d", // unpaired high at the very end
];
const encoder = new TextEncoder();
for (const value of cases) {
expect(utf8Length(value)).toBe(encoder.encode(value).byteLength);
}
});
it("stops reading once the run's catalog budget is spent", async () => {
// The per-skill caps bound what ONE entry costs; nothing bounded how many
// entries there are, and SEP-2640 puts no ceiling on a catalog — so a
// listing of a hundred thousand conforming skills made `--verify` run
// indefinitely (Copilot).
// Each skill carries its OWN SKILL.md, whose frontmatter matches its
// listing entry — otherwise every report is `failed` on a frontmatter
// mismatch and the budget is not what the test is measuring.
const enc = new TextEncoder();
const mdFor = (i: number) =>
`---\nname: s${i}\ndescription: A demo\n---\n\n# s${i}\n`;
const many = await Promise.all(
Array.from(
{ length: SKILL_MAX_CATALOG_SKILLS + 5 },
async (_, i): Promise<SkillEntry> => {
const bytes = enc.encode(mdFor(i));
return entry({
uri: `skill://s${i}/SKILL.md`,
frontmatter: { name: `s${i}`, description: "A demo" },
resources: [
{
uri: `skill://s${i}/SKILL.md`,
digest: await sha256Digest(bytes),
size: bytes.byteLength,
},
],
});
},
),
);
const readResource = vi.fn(async (uri: string) => ({
result: {
contents: [
{ uri, text: mdFor(Number(/s(\d+)/.exec(uri)?.[1] ?? "0")) },
],
},
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const reports = await verifySkills(client, many);
// Every entry is still REPORTED — the static checks cost no I/O, so a
// skill past the budget is not silently dropped from the output.
expect(reports).toHaveLength(SKILL_MAX_CATALOG_SKILLS + 5);
expect(readResource.mock.calls.length).toBe(SKILL_MAX_CATALOG_SKILLS);
// …and the remainder says so, rather than passing or failing.
const past = reports.slice(SKILL_MAX_CATALOG_SKILLS);
for (const report of past) {
expect(report.outcome).toBe("incomplete");
expect(report.incomplete).toMatch(/catalog budget/);
expect(report.files).toHaveLength(0);
}
expect(reports[0].outcome).toBe("verified");
});
it("is not incomplete when the budget is crossed by the LAST entry", async () => {
// Crossing the line on the final row stopped nothing: every manifest entry
// was fetched and checked. Reporting "Stopped after 4 of 4" there both