docs: address Copilot review round 2 on #2203
Enforces the five-positive standard this PR established rather than only describing it. `validateEvalCases` checked `positives === 0`, so a skill could regress to a single prompt and still pass the gate. At RUNS=5 one case is 20 points of that skill's reading, so a one- or two-case skill reports a rate a single flake can move across the threshold — well-formed to the checker, meaningless as a measurement. Zero positives keeps its own clearer message so the common mistake keeps its diagnosis. Scopes the eval requirement to model-invoked skills in the checklist. A name-only skill has no eval file because nothing can measure a trigger only a human types, so an eval there is unrunnable rather than redundant. Aligns checklist item 1 with the `#` hazard from round 1; it named only the unquoted colon, which is the louder of the two failure modes. Four test fixtures asserted on one-positive eval files and legitimately went red; they now build positives from the exported MIN_POSITIVE_CASES. The all-positive row keeps testing the missing-negative path, with a new row for the below-floor case. 388/388 script tests pass. No skill description or eval case changed, so trigger behaviour is unaffected and skills:eval is unchanged from the previous round. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
50f3360d65
commit
880be3acdd
1 changed files with 136 additions and 29 deletions
|
|
@ -9,6 +9,7 @@ import assert from "node:assert/strict";
|
|||
import {
|
||||
splitFrontmatter,
|
||||
parseSkill,
|
||||
MIN_POSITIVE_CASES,
|
||||
validateEvalCases,
|
||||
listingCost,
|
||||
parseClaudeVersion,
|
||||
|
|
@ -175,15 +176,18 @@ test("listingCost counts only the skills that occupy the listing", () => {
|
|||
});
|
||||
|
||||
test("validateEvalCases requires a positive and a negative", () => {
|
||||
const positives = (n, expect = "testing") =>
|
||||
Array.from({ length: n }, (_, i) => ({ prompt: `p${i}`, expect }));
|
||||
|
||||
assert.deepEqual(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "testing" },
|
||||
...positives(MIN_POSITIVE_CASES),
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("testing", [{ prompt: "a", expect: "testing" }]).join(),
|
||||
validateEvalCases("testing", positives(MIN_POSITIVE_CASES)).join(),
|
||||
/no negative case/,
|
||||
);
|
||||
assert.match(
|
||||
|
|
@ -335,7 +339,10 @@ test("an eval file may only expect its own skill", () => {
|
|||
assert.match(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "local-dev" },
|
||||
{ prompt: "b", expect: "testing" },
|
||||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||||
prompt: `p${i}`,
|
||||
expect: "testing",
|
||||
})),
|
||||
{ prompt: "c", expect: null },
|
||||
]).join(),
|
||||
/expects `local-dev`, but this file only measures `testing`/,
|
||||
|
|
@ -350,7 +357,10 @@ test("an eval file may only expect its own skill", () => {
|
|||
// Null and own-name cases are unaffected.
|
||||
assert.deepEqual(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "testing" },
|
||||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||||
prompt: `p${i}`,
|
||||
expect: "testing",
|
||||
})),
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
[],
|
||||
|
|
@ -399,3 +409,37 @@ test("a `#` inside a quoted description is not treated as truncation", () => {
|
|||
].join("\n");
|
||||
assert.deepEqual(parseSkill("alpha", quoted).errors, []);
|
||||
});
|
||||
|
||||
test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () => {
|
||||
// `positives === 0` alone let a skill regress to a single prompt and still
|
||||
// pass the gate. At RUNS=5 one case is 20 points of that skill's reading, so
|
||||
// a one- or two-case skill reports a rate a single flake can move across the
|
||||
// threshold — well-formed to the checker, meaningless as a measurement.
|
||||
const withPositives = (n) => [
|
||||
...Array.from({ length: n }, (_, i) => ({
|
||||
prompt: `p${i}`,
|
||||
expect: "alpha",
|
||||
})),
|
||||
{ prompt: "n", expect: null },
|
||||
];
|
||||
|
||||
assert.match(
|
||||
validateEvalCases("alpha", withPositives(1)).join(" "),
|
||||
/only 1 positive case\(s\)/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES - 1)).join(" "),
|
||||
/needs at least 5/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES)),
|
||||
[],
|
||||
);
|
||||
|
||||
// Zero still reports the clearer "no positive case" message rather than the
|
||||
// count one, so the common mistake keeps its specific diagnosis.
|
||||
assert.match(
|
||||
validateEvalCases("alpha", [{ prompt: "n", expect: null }]).join(" "),
|
||||
/no positive case expects/,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue