chore: address Copilot review round 13

- skill-manifest: an eval file may only expect its own skill. A foreign
  `expect: "other-skill"` passed whenever that other skill fired, so the file
  reported a measurement of something it does not describe — and it satisfied
  the positive-case requirement while doing so, masking the absence of a real
  positive. Both halves covered by a regression case.
- local-dev: stop restating the dependency-placement rules. They are normative
  and belong only in AGENTS.md; the skill now points there and carries what a
  rule cannot — what each one is defending against, what it looked like when it
  was violated (#1970, #2082, #2067, #1952, #1896, #2058), and how to tell you
  have hit it. Including the two traps that make this class hard to see: the
  ordinary test tiers cannot catch a bundling miss, and probing one by hand with
  `node -e` falsely succeeds.

Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-08-29 03:57:25 -04:00
commit f13d415eae
2 changed files with 136 additions and 92 deletions

View file

@ -221,6 +221,15 @@ export function validateEvalCases(skillName, cases) {
(c.expect !== null && typeof c.expect !== "string")
) {
errors.push(`case ${i}: \`expect\` must be a skill name or null`);
} else if (c.expect !== null && c.expect !== skillName) {
// A case living in this skill's evals may only expect THIS skill. A
// foreign name passes the eval whenever that other skill fires, so the
// file reports a measurement of something it does not describe — and it
// still satisfies the positive-case requirement, hiding the absence of a
// real one (Copilot).
errors.push(
`case ${i}: expects \`${c.expect}\`, but this file only measures \`${skillName}\``,
);
}
});
const positives = cases.filter((c) => c && c.expect === skillName).length;

View file

@ -301,3 +301,32 @@ test("CRLF does not weaken the first-line rule", () => {
/never closed/,
);
});
test("an eval file may only expect its own skill", () => {
// A foreign name passes whenever that OTHER skill fires, so the file reports
// a measurement of something it does not describe — and it satisfies the
// positive-case requirement while doing it, hiding the absence of a real one.
assert.match(
validateEvalCases("testing", [
{ prompt: "a", expect: "local-dev" },
{ prompt: "b", expect: "testing" },
{ prompt: "c", expect: null },
]).join(),
/expects `local-dev`, but this file only measures `testing`/,
);
// The masking case: a foreign positive is no longer enough on its own.
const masked = validateEvalCases("testing", [
{ prompt: "a", expect: "local-dev" },
{ prompt: "b", expect: null },
]);
assert.match(masked.join(), /only measures/);
assert.match(masked.join(), /no positive case/);
// Null and own-name cases are unaffected.
assert.deepEqual(
validateEvalCases("testing", [
{ prompt: "a", expect: "testing" },
{ prompt: "b", expect: null },
]),
[],
);
});