docs: address Copilot review round 1 on #2203
Fixes an unquoted `#` in `board-ops`'s description. YAML reads ` #28` onward as a comment, so the parser — and Claude Code, which parses the same way — only ever saw the description up to `Covers board`. Both board numbers and the option-deletion hazard were absent from the listing that decides whether the skill fires, and every check stayed green because they look for an *empty* description and a truncated one is still non-empty. Quoting restores all 271 characters, which is the whole of the 3,144 vs 3,234 discrepancy the review flagged. `parseSkill` now compares the raw scalar against what YAML kept and errors on any loss, so the next one fails the gate rather than shipping. Tests cover the truncated case and a legitimately quoted `#`. Re-aims the `project-structure` Node-backend case, which `AGENTS.md` answers via its tree and which had oscillated 80 -> 100% across runs — the partial-overlap signature this PR documents. The replacement names no internals and probed 4/4. Makes the documented probe reproduce the harness: the same `--disallowedTools` flags as `runPrompt` rather than `--allowedTools`, `printf` instead of zsh's `print -r`, and `jq` over `tool_use` blocks instead of a `grep` that also matched MCP server names in the init event. The first-tool-call table is relabelled as an unrestricted probe, since the eval cannot reach for Bash at all. RUNS=5 skills:eval: 63/63, 62 at 100%; restoring those 90 characters displaced nothing. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
906e970897
commit
50f3360d65
2 changed files with 104 additions and 8 deletions
|
|
@ -150,6 +150,38 @@ export function parseSkill(dirName, text) {
|
|||
errors.push("`paths` must be a list of glob strings");
|
||||
}
|
||||
}
|
||||
// An unquoted `#` is a YAML comment, so a description containing one is
|
||||
// truncated at that point — silently, and *not* to empty, which is why the
|
||||
// "malformed YAML" checks sail past it: what survives still parses as a
|
||||
// non-empty string. `board-ops` shipped for a while with its two board numbers
|
||||
// and the option-deletion hazard cut off this way. Compare the raw scalar
|
||||
// against what YAML actually kept and make any loss an error.
|
||||
for (const [field, parsed] of [
|
||||
["description", description],
|
||||
["when_to_use", whenToUse],
|
||||
]) {
|
||||
if (typeof parsed !== "string") continue;
|
||||
const raw = new RegExp("^" + field + ":[ \\t]*(.*)$", "m").exec(
|
||||
frontmatter,
|
||||
)?.[1];
|
||||
if (raw === undefined) continue;
|
||||
const trimmed = raw.trim();
|
||||
// Only an unquoted scalar can lose text to a comment; a quoted one is safe,
|
||||
// and a block scalar (`|`/`>`) puts nothing on this line to compare.
|
||||
if (/^["'|>]/.test(trimmed)) continue;
|
||||
if (trimmed.length > parsed.trim().length) {
|
||||
errors.push(
|
||||
"`" +
|
||||
field +
|
||||
"` is truncated by an unquoted `#` — YAML kept " +
|
||||
parsed.trim().length +
|
||||
" of " +
|
||||
trimmed.length +
|
||||
" characters; quote the value",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// A skill nothing can reach is dead weight that still costs a directory.
|
||||
if (!modelInvoked && meta["user-invocable"] === false) {
|
||||
errors.push(
|
||||
|
|
|
|||
|
|
@ -356,3 +356,46 @@ test("an eval file may only expect its own skill", () => {
|
|||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test("an unquoted `#` truncates a description, and that is an error", () => {
|
||||
// The dangerous part is that YAML leaves a *non-empty* string behind, so the
|
||||
// "description is required and non-empty" check passes and the skill keeps
|
||||
// auto-firing — just with its most distinctive words missing from the
|
||||
// listing. `board-ops` shipped this way, losing both board numbers.
|
||||
const withHash = [
|
||||
"---",
|
||||
"name: board-ops",
|
||||
"description: Recipes. Covers board #28 (v2) and board #11 (v1).",
|
||||
"disable-model-invocation: false",
|
||||
"---",
|
||||
"body",
|
||||
].join("\n");
|
||||
const parsed = parseSkill("board-ops", withHash);
|
||||
assert.equal(parsed.description, "Recipes. Covers board");
|
||||
assert.match(parsed.errors.join(" "), /truncated by an unquoted `#`/);
|
||||
|
||||
// Quoting is the fix, and it must come back clean.
|
||||
const quoted = withHash.replace(
|
||||
/^description: (.*)$/m,
|
||||
(_m, d) => `description: "${d}"`,
|
||||
);
|
||||
const ok = parseSkill("board-ops", quoted);
|
||||
assert.deepEqual(ok.errors, []);
|
||||
assert.equal(
|
||||
ok.description,
|
||||
"Recipes. Covers board #28 (v2) and board #11 (v1).",
|
||||
);
|
||||
});
|
||||
|
||||
test("a `#` inside a quoted description is not treated as truncation", () => {
|
||||
// Guard against the check firing on every legitimately quoted description.
|
||||
const quoted = [
|
||||
"---",
|
||||
"name: alpha",
|
||||
'description: "counts # and : safely"',
|
||||
"disable-model-invocation: false",
|
||||
"---",
|
||||
"body",
|
||||
].join("\n");
|
||||
assert.deepEqual(parseSkill("alpha", quoted).errors, []);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue