diff --git a/.claude/skills/local-dev/evals/evals.json b/.claude/skills/local-dev/evals/evals.json index d07254b1..62c70f25 100644 --- a/.claude/skills/local-dev/evals/evals.json +++ b/.claude/skills/local-dev/evals/evals.json @@ -1,6 +1,6 @@ [ { - "prompt": "How do I get the web client running locally?", + "prompt": "What is the difference between the launcher-driven run scripts and starting a client directly, and when should I use each?", "expect": "local-dev" }, { diff --git a/.claude/skills/project-structure/evals/evals.json b/.claude/skills/project-structure/evals/evals.json index f20592cc..e7df322d 100644 --- a/.claude/skills/project-structure/evals/evals.json +++ b/.claude/skills/project-structure/evals/evals.json @@ -1,6 +1,6 @@ [ { - "prompt": "What lives in clients/web versus @inspector/core?", + "prompt": "This repo has three clients over a shared core. How should I orient myself in it before changing anything?", "expect": "project-structure" }, { diff --git a/AGENTS.md b/AGENTS.md index e2cf38f9..ca6acc1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -223,9 +223,10 @@ that from happening: `evals/evals.json` — positives **and negatives**. Seeing a skill fire once tells you Claude found it, not that it finds it reliably, and a skill that fires on everything is a context regression that nobody notices by hand. `verify:skills` - requires both kinds — and at least **five** positives, since at `RUNS=5` a - skill measured by one or two prompts reports a rate a single flake can move - across the threshold; `npm run skills:eval` actually runs them (it needs the + requires both kinds — and at least **five** positives, for breadth rather than + variance: each prompt is scored on its own `passes / RUNS`, so more prompts + steady nothing, they cover more of the ways someone might reach the skill and + expose a description that only fires on one narrow phrasing; `npm run skills:eval` actually runs them (it needs the `claude` CLI and real model calls, so it is deliberately **not** in the gate — run it when adding a skill or editing a model-invoked description). Two things learned writing the first set, both of which make a case measure diff --git a/docs/skill-authoring.md b/docs/skill-authoring.md index 3984b4b0..4e641e3a 100644 --- a/docs/skill-authoring.md +++ b/docs/skill-authoring.md @@ -243,9 +243,11 @@ break. (`disable-model-invocation: true`, as `release` is) has no eval file at all: nothing can measure a trigger that only a human types, so an eval there would be unrunnable rather than merely redundant. `verify:skills` enforces the - floor — fewer than five positives fails the gate, because at `RUNS=5` one - case is 20 points of that skill's reading and a two-case skill reports a rate - a single flake can move across the threshold. + floor — fewer than five positives fails the gate. The reason is **breadth, + not variance**: each prompt is scored on its own `passes / RUNS`, so extra + prompts do not steady any rate (`RUNS` is the knob for that). Five prompts + cover five ways someone might arrive at the skill, which is what catches a + description that fires on one narrow phrasing and nothing else. 5. `npm run verify:skills` passes and the listing is under budget. 6. `RUNS=5 npm run skills:eval` — the **whole** suite — is ≥80% on every case, including the skills you did not touch. diff --git a/scripts/lib/skill-manifest.mjs b/scripts/lib/skill-manifest.mjs index 65107b23..7e31c5a9 100644 --- a/scripts/lib/skill-manifest.mjs +++ b/scripts/lib/skill-manifest.mjs @@ -224,6 +224,18 @@ export function listingCost(skills) { ); } +/** + * The number of positive cases a model-invoked skill's eval file must carry. + * + * This is a **breadth** floor, not a statistical one. The evaluator scores each + * prompt independently — `passes / RUNS` for that prompt alone — so adding + * prompts does not steady any other prompt's rate; `RUNS` is the only knob that + * does. What five prompts buy is coverage of five different ways someone might + * arrive at the skill, which is what exposes a description that fires on one + * narrow phrasing and nothing else (Copilot). + */ +export const MIN_POSITIVE_CASES = 5; + /** * Validate an `evals/evals.json` payload for a model-invoked skill. * @@ -235,12 +247,6 @@ export function listingCost(skills) { * @param {unknown} cases Parsed JSON. * @returns {string[]} errors */ -/** - * The number of positive cases a model-invoked skill's eval file must carry. - * Raised from an implicit 3 to 5 so a single flake cannot cross the threshold. - */ -export const MIN_POSITIVE_CASES = 5; - export function validateEvalCases(skillName, cases) { if (!Array.isArray(cases) || cases.length === 0) { return ["evals.json must be a non-empty array of cases"]; @@ -275,13 +281,13 @@ export function validateEvalCases(skillName, cases) { if (positives === 0) { errors.push(`no positive case expects \`${skillName}\``); } else if (positives < MIN_POSITIVE_CASES) { - // A non-zero floor is not enough. At RUNS=5 one case is worth 20 points of - // that skill's reading, so a skill measured by one or two prompts reports a - // rate that a single flake can move across the threshold — and the gate, - // seeing a positive, would call that well-formed. Five is the count every - // skill here was raised to for exactly that reason (Copilot). + // A non-zero floor is not enough: one prompt measures one phrasing. Since + // each prompt is scored on its own `passes / RUNS`, extra prompts do not + // make any single rate steadier — they cover more of the ways someone might + // reach the skill, so a description that only fires on one narrow shape is + // visible instead of passing on its single lucky case (Copilot). errors.push( - `only ${positives} positive case(s); a model-invoked skill needs at least ${MIN_POSITIVE_CASES} for its rate to mean anything`, + `only ${positives} positive case(s); a model-invoked skill needs at least ${MIN_POSITIVE_CASES} to cover the range of situations it should fire on`, ); } if (negatives === 0) { diff --git a/scripts/lib/skill-manifest.test.mjs b/scripts/lib/skill-manifest.test.mjs index fc439e22..6b9ec301 100644 --- a/scripts/lib/skill-manifest.test.mjs +++ b/scripts/lib/skill-manifest.test.mjs @@ -412,9 +412,10 @@ test("a `#` inside a quoted description is not treated as truncation", () => { test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () => { // `positives === 0` alone let a skill regress to a single prompt and still - // pass the gate. At RUNS=5 one case is 20 points of that skill's reading, so - // a one- or two-case skill reports a rate a single flake can move across the - // threshold — well-formed to the checker, meaningless as a measurement. + // pass the gate. The floor is about breadth, not variance: each prompt is + // scored on its own `passes / RUNS`, so extra prompts steady nothing — they + // cover more of the ways someone might reach the skill, so a description that + // fires on one narrow phrasing cannot pass on that single case alone. const withPositives = (n) => [ ...Array.from({ length: n }, (_, i) => ({ prompt: `p${i}`, @@ -429,7 +430,7 @@ test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () => ); assert.match( validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES - 1)).join(" "), - /needs at least 5/, + /needs at least 5 to cover the range/, ); assert.deepEqual( validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES)), diff --git a/scripts/verify-skills.main.test.mjs b/scripts/verify-skills.main.test.mjs index 0dd9d3cc..4371d81b 100644 --- a/scripts/verify-skills.main.test.mjs +++ b/scripts/verify-skills.main.test.mjs @@ -156,7 +156,7 @@ test("fails eval cases that are unreadable, all-positive, or all-negative", () = { prompt: "a", expect: "beta" }, { prompt: "b", expect: null }, ]), - /needs at least 5/, + /needs at least 5 to cover the range/, ], [JSON.stringify([{ prompt: "a", expect: null }]), /no positive case/], [JSON.stringify([{ expect: null }]), /prompt/],