docs: address Copilot review round 7 on #2203

Strip YAML properties (tags, anchors, in either order) before deciding
whether a description scalar is quoted — testing the first character alone
read `&copy "Counts # safely"` as unquoted and rejected a `#` the quotes
make harmless.

Count the positive-case floor by distinct prompt rather than by entry: the
floor exists for breadth, so five copies of one prompt cleared it while
exercising exactly the single trigger it is meant to look past. Compare on
collapsed, case-folded text so trivial edits do not read as a new situation.

Re-aim the two positive prompts that measured the wrong thing — the
project-structure case now poses a concrete "which piece handles what"
question, and the test-servers case names running a real MCP server.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SyBPs7caoMcyvnp9ArMBhV
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-01 12:21:09 -04:00
commit 0febaa0f6f

View file

@ -426,7 +426,7 @@ test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () =>
assert.match(
validateEvalCases("alpha", withPositives(1)).join(" "),
/only 1 positive case\(s\)/,
/only 1 distinct positive prompt\(s\)/,
);
assert.match(
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES - 1)).join(" "),
@ -533,3 +533,71 @@ test("the truncation guard ignores anchors and tags, which are not `#`", () => {
/`description` is truncated/,
);
});
test("a quoted scalar stays safe behind a tag or an anchor", () => {
// Properties may precede the value, in either order. Testing the first
// character alone read these as unquoted and then rejected the `#` that the
// quotes make harmless.
const mk = (value) =>
[
"---",
"name: alpha",
`description: ${value}`,
"disable-model-invocation: false",
"---",
"body",
].join("\n");
for (const value of [
'&copy "Counts # safely"',
'!!str "Counts # safely"',
'!!str &a "Counts # safely"',
]) {
const parsed = parseSkill("alpha", mk(value));
assert.deepEqual(parsed.errors, [], `rejected: ${value}`);
assert.equal(parsed.description, "Counts # safely");
}
});
test("the positive floor counts distinct prompts, not entries", () => {
// The floor exists for breadth, so repeating one prompt five times clears an
// entry count while exercising exactly the single trigger it looks past.
const duplicated = [
...Array.from({ length: MIN_POSITIVE_CASES }, () => ({
prompt: "the same prompt",
expect: "alpha",
})),
{ prompt: "n", expect: null },
];
const errors = validateEvalCases("alpha", duplicated).join(" ");
assert.match(
errors,
/only 1 distinct positive prompt\(s\) across 5 case\(s\)/,
);
// Whitespace and casing are not a new situation either.
const nearDuplicates = [
{ prompt: "How do I do the thing?", expect: "alpha" },
{ prompt: "how do i do the thing?", expect: "alpha" },
{ prompt: "How do I do the thing? ", expect: "alpha" },
{ prompt: "b", expect: "alpha" },
{ prompt: "c", expect: "alpha" },
{ prompt: "n", expect: null },
];
assert.match(
validateEvalCases("alpha", nearDuplicates).join(" "),
/only 3 distinct positive prompt\(s\)/,
);
// Five genuinely different prompts pass.
assert.deepEqual(
validateEvalCases("alpha", [
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
prompt: `p${i}`,
expect: "alpha",
})),
{ prompt: "n", expect: null },
]),
[],
);
});