chore: address Copilot review round 13
- skill-manifest: an eval file may only expect its own skill. A foreign `expect: "other-skill"` passed whenever that other skill fired, so the file reported a measurement of something it does not describe — and it satisfied the positive-case requirement while doing so, masking the absence of a real positive. Both halves covered by a regression case. - local-dev: stop restating the dependency-placement rules. They are normative and belong only in AGENTS.md; the skill now points there and carries what a rule cannot — what each one is defending against, what it looked like when it was violated (#1970, #2082, #2067, #1952, #1896, #2058), and how to tell you have hit it. Including the two traps that make this class hard to see: the ordinary test tiers cannot catch a bundling miss, and probing one by hand with `node -e` falsely succeeds. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
7d828bded0
commit
f13d415eae
2 changed files with 136 additions and 92 deletions
|
|
@ -221,6 +221,15 @@ export function validateEvalCases(skillName, cases) {
|
|||
(c.expect !== null && typeof c.expect !== "string")
|
||||
) {
|
||||
errors.push(`case ${i}: \`expect\` must be a skill name or null`);
|
||||
} else if (c.expect !== null && c.expect !== skillName) {
|
||||
// A case living in this skill's evals may only expect THIS skill. A
|
||||
// foreign name passes the eval whenever that other skill fires, so the
|
||||
// file reports a measurement of something it does not describe — and it
|
||||
// still satisfies the positive-case requirement, hiding the absence of a
|
||||
// real one (Copilot).
|
||||
errors.push(
|
||||
`case ${i}: expects \`${c.expect}\`, but this file only measures \`${skillName}\``,
|
||||
);
|
||||
}
|
||||
});
|
||||
const positives = cases.filter((c) => c && c.expect === skillName).length;
|
||||
|
|
|
|||
|
|
@ -301,3 +301,32 @@ test("CRLF does not weaken the first-line rule", () => {
|
|||
/never closed/,
|
||||
);
|
||||
});
|
||||
|
||||
test("an eval file may only expect its own skill", () => {
|
||||
// A foreign name passes whenever that OTHER skill fires, so the file reports
|
||||
// a measurement of something it does not describe — and it satisfies the
|
||||
// positive-case requirement while doing it, hiding the absence of a real one.
|
||||
assert.match(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "local-dev" },
|
||||
{ prompt: "b", expect: "testing" },
|
||||
{ prompt: "c", expect: null },
|
||||
]).join(),
|
||||
/expects `local-dev`, but this file only measures `testing`/,
|
||||
);
|
||||
// The masking case: a foreign positive is no longer enough on its own.
|
||||
const masked = validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "local-dev" },
|
||||
{ prompt: "b", expect: null },
|
||||
]);
|
||||
assert.match(masked.join(), /only measures/);
|
||||
assert.match(masked.join(), /no positive case/);
|
||||
// Null and own-name cases are unaffected.
|
||||
assert.deepEqual(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "testing" },
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue