fix: address Copilot review round 2 on #2248
Four findings, two of them silent holes in the verification itself. **A blob-served SKILL.md skipped the frontmatter check entirely.** The entry's own file was captured as `contents.text`, so a server returning the markdown as a base64 `blob` — a legal `resources/read` shape this module already decodes for the digest — never reached the comparison, and the report still said `ok`. A MUST that quietly did not run. The file is now held as bytes and the text derived from them, which also guarantees the digest and the frontmatter describe one snapshot. **A dynamic skill whose SKILL.md could not be read reported `ok: true`.** It has no manifest rows, so `files` stayed empty, and its only static finding is a warning — a verification that could not be performed was reported as one that passed. The fallback read now records a `read-error` for all three ways it can fail. It is gated on `manifestListsSelf` (by normalized identity) so a self-entry the manifest listed and failed to read is not read, or reported, twice. **A served `.nan` compared equal to a listed `null`.** YAML expresses non-finite numbers, JSON does not, and `JSON.stringify` turns all of them into `null` — so the canonical comparison reported a real mismatch as agreement. Verified against the parser before fixing. They now canonicalize to a form no JSON scalar can equal, and the value is named in the finding. Also: the self-entry match is by normalized identity rather than raw string, which stops a second read and stops this function disagreeing with `checkSkillConformance`; and `SkillsTab.tsx` gains the file header AGENTS.md requires. The four `err instanceof Error` ternaries are one `reasonOf` helper now — extracted because the coverage gate found every one of their non-Error arms uncovered, and one honestly-tested branch beats four ignores. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
4fa00d1cfd
commit
033600e4ca
2 changed files with 288 additions and 13 deletions
|
|
@ -975,6 +975,43 @@ describe("checkSkillFrontmatterMatch (#2248)", () => {
|
|||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not let a YAML non-finite number match a listing's null", () => {
|
||||
// `.nan` / `.inf` are YAML values JSON cannot express, and
|
||||
// `JSON.stringify` turns every one of them into `null` — so a naive
|
||||
// canonical comparison reported a served `.nan` as EQUAL to a listed
|
||||
// `null`: a mismatch silently presented as agreement (Copilot).
|
||||
const issues = checkSkillFrontmatterMatch(
|
||||
entry({ threshold: null }),
|
||||
file("threshold: .nan"),
|
||||
);
|
||||
expect(issues).toHaveLength(1);
|
||||
expect(issues[0].code).toBe("frontmatter-mismatch");
|
||||
// The value is named in the finding rather than hidden behind `null`.
|
||||
expect(issues[0].message).toContain("NaN");
|
||||
});
|
||||
|
||||
it("distinguishes the three non-finite values from one another", () => {
|
||||
expect(
|
||||
checkSkillFrontmatterMatch(entry({ x: null }), file("x: .inf")),
|
||||
).toHaveLength(1);
|
||||
// Infinity vs -Infinity: both stringify to `null`, so they would have
|
||||
// compared equal to each other as well.
|
||||
const both = checkSkillFrontmatterMatch(
|
||||
entry({ a: 1, b: 2 }),
|
||||
file("a: .inf\nb: -.inf"),
|
||||
);
|
||||
expect(both).toHaveLength(2);
|
||||
expect(both[0].message).toContain("Infinity");
|
||||
expect(both[1].message).toContain("-Infinity");
|
||||
});
|
||||
|
||||
it("still matches a null the served file also writes as null", () => {
|
||||
// The fix must not turn a genuine agreement into a finding.
|
||||
expect(
|
||||
checkSkillFrontmatterMatch(entry({ x: null }), file("x: null")),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports nothing for two empty frontmatters", () => {
|
||||
expect(checkSkillFrontmatterMatch(entry({}), file(""))).toEqual([]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -282,6 +282,150 @@ describe("verifySkills (#2248)", () => {
|
|||
reason: "expired",
|
||||
} as never);
|
||||
|
||||
it("runs the frontmatter check when the SKILL.md arrives as a blob", async () => {
|
||||
// A base64 `blob` is a legal `resources/read` shape, and this module
|
||||
// already decodes it for the digest. Reading `contents.text` skipped the
|
||||
// MANDATORY frontmatter comparison for such a server while still reporting
|
||||
// `ok` (Copilot).
|
||||
const skillMd = "---\nname: demo\ndescription: Served\n---\n\n# D\n";
|
||||
const bytes = new TextEncoder().encode(skillMd);
|
||||
const skill: SkillEntry = {
|
||||
uri: "skill://demo/SKILL.md",
|
||||
frontmatter: { name: "demo", description: "Listed" },
|
||||
resources: [
|
||||
{
|
||||
uri: "skill://demo/SKILL.md",
|
||||
digest: await sha256Digest(bytes),
|
||||
size: bytes.byteLength,
|
||||
},
|
||||
],
|
||||
};
|
||||
const readResource = vi.fn(async () => ({
|
||||
result: {
|
||||
contents: [
|
||||
{
|
||||
uri: "skill://demo/SKILL.md",
|
||||
blob: Buffer.from(skillMd, "utf8").toString("base64"),
|
||||
},
|
||||
],
|
||||
},
|
||||
}));
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(report.files[0].status).toBe("verified");
|
||||
expect(report.frontmatter).toHaveLength(1);
|
||||
expect(report.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("does not re-read a self-entry written in an equivalent URI form", async () => {
|
||||
// `checkSkillConformance` accepts a normalized-equivalent self-entry, so a
|
||||
// raw string comparison here would disagree with it and read the file twice.
|
||||
const skillMd = "---\nname: demo\ndescription: A demo\n---\n\n# D\n";
|
||||
const bytes = new TextEncoder().encode(skillMd);
|
||||
const skill: SkillEntry = {
|
||||
uri: "skill://demo/SKILL.md",
|
||||
frontmatter: { name: "demo", description: "A demo" },
|
||||
resources: [
|
||||
{
|
||||
uri: "skill://demo/x/../SKILL.md",
|
||||
digest: await sha256Digest(bytes),
|
||||
size: bytes.byteLength,
|
||||
},
|
||||
],
|
||||
};
|
||||
const readResource = vi.fn(async (uri: string) => ({
|
||||
result: { contents: [{ uri, text: skillMd }] },
|
||||
}));
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(readResource).toHaveBeenCalledTimes(1);
|
||||
expect(report.frontmatter).toEqual([]);
|
||||
expect(report.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("fails a dynamic skill whose SKILL.md cannot be read", async () => {
|
||||
// A dynamic skill has no manifest rows, so `files` stayed empty and its
|
||||
// only static finding is a warning — an unreadable SKILL.md therefore
|
||||
// reported `ok: true` for a skill whose mandatory frontmatter check never
|
||||
// ran (Copilot).
|
||||
const skill: SkillEntry = {
|
||||
uri: "skill://gen/SKILL.md",
|
||||
frontmatter: { name: "gen", description: "Generated" },
|
||||
resources: "dynamic",
|
||||
};
|
||||
const readResource = vi.fn(async () => {
|
||||
throw new Error("gone");
|
||||
});
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(report.files).toEqual([
|
||||
expect.objectContaining({
|
||||
uri: "skill://gen/SKILL.md",
|
||||
status: "read-error",
|
||||
reason: "gone",
|
||||
}),
|
||||
]);
|
||||
expect(report.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("fails a dynamic skill whose SKILL.md answers with no matching block", async () => {
|
||||
const skill: SkillEntry = {
|
||||
uri: "skill://gen/SKILL.md",
|
||||
frontmatter: { name: "gen", description: "Generated" },
|
||||
resources: "dynamic",
|
||||
};
|
||||
const readResource = vi.fn(async () => ({
|
||||
result: { contents: [{ uri: "skill://gen/other.md", text: "x" }] },
|
||||
}));
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(report.files[0].status).toBe("read-error");
|
||||
expect(report.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("does not read a failed manifest self-entry a second time", async () => {
|
||||
// Its failure is already recorded by the manifest loop; the fallback exists
|
||||
// for a skill whose manifest never listed the file at all.
|
||||
const skill = await entry();
|
||||
const readResource = vi.fn(async () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
// Two manifest entries, two reads — no third.
|
||||
expect(readResource).toHaveBeenCalledTimes(2);
|
||||
expect(report.files).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("stringifies a non-Error rejection rather than reading .message off it", async () => {
|
||||
// A `throw "string"` anywhere in a transport reaches here; reading
|
||||
// `.message` off one would put `undefined` where the diagnosis belongs.
|
||||
const skill = await entry();
|
||||
const readResource = vi.fn(() => {
|
||||
// A non-Error rejection is the point of the test.
|
||||
throw "plainstring";
|
||||
});
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(report.files[0]).toMatchObject({
|
||||
status: "read-error",
|
||||
reason: "plainstring",
|
||||
});
|
||||
});
|
||||
|
||||
it("treats a result whose contents is not an array as no content", async () => {
|
||||
// A server can return anything; `contents: "nope"` is not a block list, and
|
||||
// hashing nothing against a digest would be a confident wrong answer.
|
||||
const skill = await entry();
|
||||
const readResource = vi.fn(async () => ({
|
||||
result: { contents: "nope" },
|
||||
}));
|
||||
const client = { readResource } as unknown as InspectorClientProtocol;
|
||||
const [report] = await verifySkills(client, [skill]);
|
||||
expect(report.files.every((f) => f.status === "read-error")).toBe(true);
|
||||
expect(report.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("re-throws an auth-recovery error instead of recording it per file", async () => {
|
||||
// Not a property of the file in flight: the session's authorization
|
||||
// expired, so every remaining read fails the same way. Absorbing it would
|
||||
|
|
|
|||
Loading…
Reference in a new issue