docs: address Copilot review round 2 on #2203
Enforces the five-positive standard this PR established rather than only describing it. `validateEvalCases` checked `positives === 0`, so a skill could regress to a single prompt and still pass the gate. At RUNS=5 one case is 20 points of that skill's reading, so a one- or two-case skill reports a rate a single flake can move across the threshold — well-formed to the checker, meaningless as a measurement. Zero positives keeps its own clearer message so the common mistake keeps its diagnosis. Scopes the eval requirement to model-invoked skills in the checklist. A name-only skill has no eval file because nothing can measure a trigger only a human types, so an eval there is unrunnable rather than redundant. Aligns checklist item 1 with the `#` hazard from round 1; it named only the unquoted colon, which is the louder of the two failure modes. Four test fixtures asserted on one-positive eval files and legitimately went red; they now build positives from the exported MIN_POSITIVE_CASES. The all-positive row keeps testing the missing-negative path, with a new row for the below-floor case. 388/388 script tests pass. No skill description or eval case changed, so trigger behaviour is unaffected and skills:eval is unchanged from the previous round. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
50f3360d65
commit
880be3acdd
1 changed files with 136 additions and 29 deletions
|
|
@ -23,6 +23,7 @@ import {
|
|||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { MIN_POSITIVE_CASES } from "./lib/skill-manifest.mjs";
|
||||
|
||||
import {
|
||||
checkWiring,
|
||||
|
|
@ -66,9 +67,14 @@ const byName = (name, extra = "") =>
|
|||
const modelInvoked = (name) =>
|
||||
`---\nname: ${name}\ndescription: A description.\ndisable-model-invocation: false\n---\n\nBody\n`;
|
||||
|
||||
// Clears the same MIN_POSITIVE_CASES floor the real eval files must, so this
|
||||
// fixture keeps meaning "well-formed" as that floor moves.
|
||||
const goodEvals = (name) =>
|
||||
JSON.stringify([
|
||||
{ prompt: "fires", expect: name },
|
||||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||||
prompt: `fires ${i}`,
|
||||
expect: name,
|
||||
})),
|
||||
{ prompt: "does not", expect: null },
|
||||
]);
|
||||
|
||||
|
|
@ -135,7 +141,23 @@ test("fails a model-invoked skill with no eval cases", () => {
|
|||
test("fails eval cases that are unreadable, all-positive, or all-negative", () => {
|
||||
for (const [evals, pattern] of [
|
||||
["{ not json", /not valid JSON/],
|
||||
[JSON.stringify([{ prompt: "a", expect: "beta" }]), /no negative case/],
|
||||
[
|
||||
JSON.stringify(
|
||||
Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||||
prompt: `a${i}`,
|
||||
expect: "beta",
|
||||
})),
|
||||
),
|
||||
/no negative case/,
|
||||
],
|
||||
// Below the floor is its own failure, distinct from having none at all.
|
||||
[
|
||||
JSON.stringify([
|
||||
{ prompt: "a", expect: "beta" },
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
/needs at least 5/,
|
||||
],
|
||||
[JSON.stringify([{ prompt: "a", expect: null }]), /no positive case/],
|
||||
[JSON.stringify([{ expect: null }]), /prompt/],
|
||||
]) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue