docs: address Copilot review round 1 on #2203

Fixes an unquoted `#` in `board-ops`'s description. YAML reads ` #28`
onward as a comment, so the parser — and Claude Code, which parses the
same way — only ever saw the description up to `Covers board`. Both
board numbers and the option-deletion hazard were absent from the
listing that decides whether the skill fires, and every check stayed
green because they look for an *empty* description and a truncated one
is still non-empty. Quoting restores all 271 characters, which is the
whole of the 3,144 vs 3,234 discrepancy the review flagged.

`parseSkill` now compares the raw scalar against what YAML kept and
errors on any loss, so the next one fails the gate rather than shipping.
Tests cover the truncated case and a legitimately quoted `#`.

Re-aims the `project-structure` Node-backend case, which `AGENTS.md`
answers via its tree and which had oscillated 80 -> 100% across runs —
the partial-overlap signature this PR documents. The replacement names
no internals and probed 4/4.

Makes the documented probe reproduce the harness: the same
`--disallowedTools` flags as `runPrompt` rather than `--allowedTools`,
`printf` instead of zsh's `print -r`, and `jq` over `tool_use` blocks
instead of a `grep` that also matched MCP server names in the init
event. The first-tool-call table is relabelled as an unrestricted probe,
since the eval cannot reach for Bash at all.

RUNS=5 skills:eval: 63/63, 62 at 100%; restoring those 90 characters
displaced nothing.

Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-01 10:09:52 -04:00
commit 50f3360d65
2 changed files with 104 additions and 8 deletions

View file

@ -150,6 +150,38 @@ export function parseSkill(dirName, text) {
errors.push("`paths` must be a list of glob strings");
}
}
// An unquoted `#` is a YAML comment, so a description containing one is
// truncated at that point — silently, and *not* to empty, which is why the
// "malformed YAML" checks sail past it: what survives still parses as a
// non-empty string. `board-ops` shipped for a while with its two board numbers
// and the option-deletion hazard cut off this way. Compare the raw scalar
// against what YAML actually kept and make any loss an error.
for (const [field, parsed] of [
["description", description],
["when_to_use", whenToUse],
]) {
if (typeof parsed !== "string") continue;
const raw = new RegExp("^" + field + ":[ \\t]*(.*)$", "m").exec(
frontmatter,
)?.[1];
if (raw === undefined) continue;
const trimmed = raw.trim();
// Only an unquoted scalar can lose text to a comment; a quoted one is safe,
// and a block scalar (`|`/`>`) puts nothing on this line to compare.
if (/^["'|>]/.test(trimmed)) continue;
if (trimmed.length > parsed.trim().length) {
errors.push(
"`" +
field +
"` is truncated by an unquoted `#` — YAML kept " +
parsed.trim().length +
" of " +
trimmed.length +
" characters; quote the value",
);
}
}
// A skill nothing can reach is dead weight that still costs a directory.
if (!modelInvoked && meta["user-invocable"] === false) {
errors.push(

View file

@ -356,3 +356,46 @@ test("an eval file may only expect its own skill", () => {
[],
);
});
test("an unquoted `#` truncates a description, and that is an error", () => {
// The dangerous part is that YAML leaves a *non-empty* string behind, so the
// "description is required and non-empty" check passes and the skill keeps
// auto-firing — just with its most distinctive words missing from the
// listing. `board-ops` shipped this way, losing both board numbers.
const withHash = [
"---",
"name: board-ops",
"description: Recipes. Covers board #28 (v2) and board #11 (v1).",
"disable-model-invocation: false",
"---",
"body",
].join("\n");
const parsed = parseSkill("board-ops", withHash);
assert.equal(parsed.description, "Recipes. Covers board");
assert.match(parsed.errors.join(" "), /truncated by an unquoted `#`/);
// Quoting is the fix, and it must come back clean.
const quoted = withHash.replace(
/^description: (.*)$/m,
(_m, d) => `description: "${d}"`,
);
const ok = parseSkill("board-ops", quoted);
assert.deepEqual(ok.errors, []);
assert.equal(
ok.description,
"Recipes. Covers board #28 (v2) and board #11 (v1).",
);
});
test("a `#` inside a quoted description is not treated as truncation", () => {
// Guard against the check firing on every legitimately quoted description.
const quoted = [
"---",
"name: alpha",
'description: "counts # and : safely"',
"disable-model-invocation: false",
"---",
"body",
].join("\n");
assert.deepEqual(parseSkill("alpha", quoted).errors, []);
});