fix: address Copilot review round 2 on #2248

Four findings, two of them silent holes in the verification itself.

**A blob-served SKILL.md skipped the frontmatter check entirely.** The entry's
own file was captured as `contents.text`, so a server returning the markdown
as a base64 `blob` — a legal `resources/read` shape this module already
decodes for the digest — never reached the comparison, and the report still
said `ok`. A MUST that quietly did not run. The file is now held as bytes and
the text derived from them, which also guarantees the digest and the
frontmatter describe one snapshot.

**A dynamic skill whose SKILL.md could not be read reported `ok: true`.** It
has no manifest rows, so `files` stayed empty, and its only static finding is
a warning — a verification that could not be performed was reported as one
that passed. The fallback read now records a `read-error` for all three ways
it can fail. It is gated on `manifestListsSelf` (by normalized identity) so a
self-entry the manifest listed and failed to read is not read, or reported,
twice.

**A served `.nan` compared equal to a listed `null`.** YAML expresses
non-finite numbers, JSON does not, and `JSON.stringify` turns all of them
into `null` — so the canonical comparison reported a real mismatch as
agreement. Verified against the parser before fixing. They now canonicalize
to a form no JSON scalar can equal, and the value is named in the finding.

Also: the self-entry match is by normalized identity rather than raw string,
which stops a second read and stops this function disagreeing with
`checkSkillConformance`; and `SkillsTab.tsx` gains the file header AGENTS.md
requires.

The four `err instanceof Error` ternaries are one `reasonOf` helper now —
extracted because the coverage gate found every one of their non-Error arms
uncovered, and one honestly-tested branch beats four ignores.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-07 19:24:30 -04:00
commit 033600e4ca
2 changed files with 288 additions and 13 deletions

View file

@ -975,6 +975,43 @@ describe("checkSkillFrontmatterMatch (#2248)", () => {
).toHaveLength(1);
});
it("does not let a YAML non-finite number match a listing's null", () => {
// `.nan` / `.inf` are YAML values JSON cannot express, and
// `JSON.stringify` turns every one of them into `null` — so a naive
// canonical comparison reported a served `.nan` as EQUAL to a listed
// `null`: a mismatch silently presented as agreement (Copilot).
const issues = checkSkillFrontmatterMatch(
entry({ threshold: null }),
file("threshold: .nan"),
);
expect(issues).toHaveLength(1);
expect(issues[0].code).toBe("frontmatter-mismatch");
// The value is named in the finding rather than hidden behind `null`.
expect(issues[0].message).toContain("NaN");
});
it("distinguishes the three non-finite values from one another", () => {
expect(
checkSkillFrontmatterMatch(entry({ x: null }), file("x: .inf")),
).toHaveLength(1);
// Infinity vs -Infinity: both stringify to `null`, so they would have
// compared equal to each other as well.
const both = checkSkillFrontmatterMatch(
entry({ a: 1, b: 2 }),
file("a: .inf\nb: -.inf"),
);
expect(both).toHaveLength(2);
expect(both[0].message).toContain("Infinity");
expect(both[1].message).toContain("-Infinity");
});
it("still matches a null the served file also writes as null", () => {
// The fix must not turn a genuine agreement into a finding.
expect(
checkSkillFrontmatterMatch(entry({ x: null }), file("x: null")),
).toEqual([]);
});
it("reports nothing for two empty frontmatters", () => {
expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]);
});

View file

@ -282,6 +282,150 @@ describe("verifySkills (#2248)", () => {
reason: "expired",
} as never);
it("runs the frontmatter check when the SKILL.md arrives as a blob", async () => {
// A base64 `blob` is a legal `resources/read` shape, and this module
// already decodes it for the digest. Reading `contents.text` skipped the
// MANDATORY frontmatter comparison for such a server while still reporting
// `ok` (Copilot).
const skillMd = "---\nname: demo\ndescription: Served\n---\n\n# D\n";
const bytes = new TextEncoder().encode(skillMd);
const skill: SkillEntry = {
uri: "skill://demo/SKILL.md",
frontmatter: { name: "demo", description: "Listed" },
resources: [
{
uri: "skill://demo/SKILL.md",
digest: await sha256Digest(bytes),
size: bytes.byteLength,
},
],
};
const readResource = vi.fn(async () => ({
result: {
contents: [
{
uri: "skill://demo/SKILL.md",
blob: Buffer.from(skillMd, "utf8").toString("base64"),
},
],
},
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(report.files[0].status).toBe("verified");
expect(report.frontmatter).toHaveLength(1);
expect(report.ok).toBe(false);
});
it("does not re-read a self-entry written in an equivalent URI form", async () => {
// `checkSkillConformance` accepts a normalized-equivalent self-entry, so a
// raw string comparison here would disagree with it and read the file twice.
const skillMd = "---\nname: demo\ndescription: A demo\n---\n\n# D\n";
const bytes = new TextEncoder().encode(skillMd);
const skill: SkillEntry = {
uri: "skill://demo/SKILL.md",
frontmatter: { name: "demo", description: "A demo" },
resources: [
{
uri: "skill://demo/x/../SKILL.md",
digest: await sha256Digest(bytes),
size: bytes.byteLength,
},
],
};
const readResource = vi.fn(async (uri: string) => ({
result: { contents: [{ uri, text: skillMd }] },
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(readResource).toHaveBeenCalledTimes(1);
expect(report.frontmatter).toEqual([]);
expect(report.ok).toBe(true);
});
it("fails a dynamic skill whose SKILL.md cannot be read", async () => {
// A dynamic skill has no manifest rows, so `files` stayed empty and its
// only static finding is a warning — an unreadable SKILL.md therefore
// reported `ok: true` for a skill whose mandatory frontmatter check never
// ran (Copilot).
const skill: SkillEntry = {
uri: "skill://gen/SKILL.md",
frontmatter: { name: "gen", description: "Generated" },
resources: "dynamic",
};
const readResource = vi.fn(async () => {
throw new Error("gone");
});
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(report.files).toEqual([
expect.objectContaining({
uri: "skill://gen/SKILL.md",
status: "read-error",
reason: "gone",
}),
]);
expect(report.ok).toBe(false);
});
it("fails a dynamic skill whose SKILL.md answers with no matching block", async () => {
const skill: SkillEntry = {
uri: "skill://gen/SKILL.md",
frontmatter: { name: "gen", description: "Generated" },
resources: "dynamic",
};
const readResource = vi.fn(async () => ({
result: { contents: [{ uri: "skill://gen/other.md", text: "x" }] },
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(report.files[0].status).toBe("read-error");
expect(report.ok).toBe(false);
});
it("does not read a failed manifest self-entry a second time", async () => {
// Its failure is already recorded by the manifest loop; the fallback exists
// for a skill whose manifest never listed the file at all.
const skill = await entry();
const readResource = vi.fn(async () => {
throw new Error("boom");
});
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
// Two manifest entries, two reads — no third.
expect(readResource).toHaveBeenCalledTimes(2);
expect(report.files).toHaveLength(2);
});
it("stringifies a non-Error rejection rather than reading .message off it", async () => {
// A `throw "string"` anywhere in a transport reaches here; reading
// `.message` off one would put `undefined` where the diagnosis belongs.
const skill = await entry();
const readResource = vi.fn(() => {
// A non-Error rejection is the point of the test.
throw "plainstring";
});
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(report.files[0]).toMatchObject({
status: "read-error",
reason: "plainstring",
});
});
it("treats a result whose contents is not an array as no content", async () => {
// A server can return anything; `contents: "nope"` is not a block list, and
// hashing nothing against a digest would be a confident wrong answer.
const skill = await entry();
const readResource = vi.fn(async () => ({
result: { contents: "nope" },
}));
const client = { readResource } as unknown as InspectorClientProtocol;
const [report] = await verifySkills(client, [skill]);
expect(report.files.every((f) => f.status === "read-error")).toBe(true);
expect(report.ok).toBe(false);
});
it("re-throws an auth-recovery error instead of recording it per file", async () => {
// Not a property of the file in flight: the session's authorization
// expired, so every remaining read fails the same way. Absorbing it would