docs: address Copilot review round 7 on #2203
Strip YAML properties (tags, anchors, in either order) before deciding whether a description scalar is quoted — testing the first character alone read `© "Counts # safely"` as unquoted and rejected a `#` the quotes make harmless. Count the positive-case floor by distinct prompt rather than by entry: the floor exists for breadth, so five copies of one prompt cleared it while exercising exactly the single trigger it is meant to look past. Compare on collapsed, case-folded text so trivial edits do not read as a new situation. Re-aim the two positive prompts that measured the wrong thing — the project-structure case now poses a concrete "which piece handles what" question, and the test-servers case names running a real MCP server. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SyBPs7caoMcyvnp9ArMBhV Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
5d3571597a
commit
0febaa0f6f
1 changed files with 97 additions and 8 deletions
|
|
@ -426,7 +426,7 @@ test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () =>
|
|||
|
||||
assert.match(
|
||||
validateEvalCases("alpha", withPositives(1)).join(" "),
|
||||
/only 1 positive case\(s\)/,
|
||||
/only 1 distinct positive prompt\(s\)/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES - 1)).join(" "),
|
||||
|
|
@ -533,3 +533,71 @@ test("the truncation guard ignores anchors and tags, which are not `#`", () => {
|
|||
/`description` is truncated/,
|
||||
);
|
||||
});
|
||||
|
||||
test("a quoted scalar stays safe behind a tag or an anchor", () => {
|
||||
// Properties may precede the value, in either order. Testing the first
|
||||
// character alone read these as unquoted and then rejected the `#` that the
|
||||
// quotes make harmless.
|
||||
const mk = (value) =>
|
||||
[
|
||||
"---",
|
||||
"name: alpha",
|
||||
`description: ${value}`,
|
||||
"disable-model-invocation: false",
|
||||
"---",
|
||||
"body",
|
||||
].join("\n");
|
||||
|
||||
for (const value of [
|
||||
'© "Counts # safely"',
|
||||
'!!str "Counts # safely"',
|
||||
'!!str &a "Counts # safely"',
|
||||
]) {
|
||||
const parsed = parseSkill("alpha", mk(value));
|
||||
assert.deepEqual(parsed.errors, [], `rejected: ${value}`);
|
||||
assert.equal(parsed.description, "Counts # safely");
|
||||
}
|
||||
});
|
||||
|
||||
test("the positive floor counts distinct prompts, not entries", () => {
|
||||
// The floor exists for breadth, so repeating one prompt five times clears an
|
||||
// entry count while exercising exactly the single trigger it looks past.
|
||||
const duplicated = [
|
||||
...Array.from({ length: MIN_POSITIVE_CASES }, () => ({
|
||||
prompt: "the same prompt",
|
||||
expect: "alpha",
|
||||
})),
|
||||
{ prompt: "n", expect: null },
|
||||
];
|
||||
const errors = validateEvalCases("alpha", duplicated).join(" ");
|
||||
assert.match(
|
||||
errors,
|
||||
/only 1 distinct positive prompt\(s\) across 5 case\(s\)/,
|
||||
);
|
||||
|
||||
// Whitespace and casing are not a new situation either.
|
||||
const nearDuplicates = [
|
||||
{ prompt: "How do I do the thing?", expect: "alpha" },
|
||||
{ prompt: "how do i do the thing?", expect: "alpha" },
|
||||
{ prompt: "How do I do the thing? ", expect: "alpha" },
|
||||
{ prompt: "b", expect: "alpha" },
|
||||
{ prompt: "c", expect: "alpha" },
|
||||
{ prompt: "n", expect: null },
|
||||
];
|
||||
assert.match(
|
||||
validateEvalCases("alpha", nearDuplicates).join(" "),
|
||||
/only 3 distinct positive prompt\(s\)/,
|
||||
);
|
||||
|
||||
// Five genuinely different prompts pass.
|
||||
assert.deepEqual(
|
||||
validateEvalCases("alpha", [
|
||||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||||
prompt: `p${i}`,
|
||||
expect: "alpha",
|
||||
})),
|
||||
{ prompt: "n", expect: null },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue