chore: address Copilot review round 2
- verify-skills: version-gate the optional `claude plugin validate`. Probing
only for the executable failed the MANDATORY gate for anyone on a CLI older
than 2.1.233, which has no such subcommand and exits nonzero on it. Skips
below the floor; a failure now means a supported validator rejected the skills.
- skill-eval: match the invoked skill structurally. A substring test counted
`{"skill":"not-testing"}` as a hit for `testing` and inflated the rate.
- skill-eval: `runOnce` → exported `runPrompt` with an injectable `spawnFn`, so
the nonzero-exit branch is actually pinned. Reverting it now fails a test
(verified by mutation) rather than leaving the suite green.
- board-ops: filter the item lookups by repository. Issue numbers are repo-local
and these are org boards — #11 really does carry a `servers` card — so an
unfiltered `select` can move or delete the wrong card, or pass two ids.
- issue-triage: the audit's `own()` filter dropped draft cards, which carry no
`.content.repository`, so the "non-Issue on a board" check reported 0 while
the invariant was being violated. Verified against a fixture.
- pr-flow: cut the branch from the base it will target. The blanket "cut from
v2/main" was wrong for v1, whose history is unrelated.
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
4520e6aa2c
commit
ca6828916f
1 changed files with 221 additions and 17 deletions
|
|
@ -10,7 +10,13 @@
|
|||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { collectSkillInvocations, sampleHit } from "./skill-eval.mjs";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
collectSkillInvocations,
|
||||
invokedSkillNames,
|
||||
runPrompt,
|
||||
sampleHit,
|
||||
} from "./skill-eval.mjs";
|
||||
|
||||
const assistant = (...blocks) =>
|
||||
JSON.stringify({ type: "assistant", message: { content: blocks } });
|
||||
|
|
@ -67,3 +73,78 @@ test("sampleHit scores positive and negative cases", () => {
|
|||
assert.equal(sampleHit(null, fired), false);
|
||||
assert.equal(sampleHit("testing", none), false);
|
||||
});
|
||||
|
||||
test("invokedSkillNames matches structurally, not by substring", () => {
|
||||
// `{"skill":"not-testing"}` contains "testing" and must NOT count — a
|
||||
// substring match inflates the measured hit rate with invocations of a
|
||||
// different skill.
|
||||
assert.deepEqual(invokedSkillNames('{"skill":"not-testing"}'), [
|
||||
"not-testing",
|
||||
]);
|
||||
assert.equal(
|
||||
sampleHit("testing", new Set(['{"skill":"not-testing"}'])),
|
||||
false,
|
||||
);
|
||||
assert.equal(sampleHit("testing", new Set(['{"skill":"testing"}'])), true);
|
||||
// The field name is not assumed, so any string value is a candidate.
|
||||
assert.equal(
|
||||
sampleHit("testing", new Set(['{"name":"testing","args":""}'])),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("invokedSkillNames tolerates payloads it cannot read", () => {
|
||||
for (const payload of ["not json", "[1,2]", "null", '"testing"']) {
|
||||
assert.deepEqual(invokedSkillNames(payload), []);
|
||||
}
|
||||
});
|
||||
|
||||
/** Minimal stand-in for a ChildProcess: emits stdout chunks, then closes. */
|
||||
function fakeSpawn({ chunks = [], code = 0, error = null }) {
|
||||
return () => {
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
queueMicrotask(() => {
|
||||
if (error) {
|
||||
child.emit("error", error);
|
||||
return;
|
||||
}
|
||||
for (const c of chunks) child.stdout.emit("data", Buffer.from(c));
|
||||
child.emit("close", code);
|
||||
});
|
||||
return child;
|
||||
};
|
||||
}
|
||||
|
||||
test("runPrompt collects invocations across chunk boundaries", async () => {
|
||||
const whole =
|
||||
JSON.stringify({
|
||||
type: "assistant",
|
||||
message: {
|
||||
content: [
|
||||
{ type: "tool_use", name: "Skill", input: { skill: "testing" } },
|
||||
],
|
||||
},
|
||||
}) + "\n";
|
||||
const invoked = await runPrompt("p", {
|
||||
spawnFn: fakeSpawn({ chunks: [whole.slice(0, 30), whole.slice(30)] }),
|
||||
});
|
||||
assert.equal(sampleHit("testing", invoked), true);
|
||||
});
|
||||
|
||||
test("runPrompt rejects a nonzero exit rather than reporting an empty observation", async () => {
|
||||
// The regression that matters: resolving here would pass every negative case
|
||||
// and read as a trigger miss on every positive one, so a rate-limited run
|
||||
// would report a plausible hit rate.
|
||||
await assert.rejects(
|
||||
runPrompt("p", { spawnFn: fakeSpawn({ code: 1 }) }),
|
||||
/exited 1 for prompt: p/,
|
||||
);
|
||||
});
|
||||
|
||||
test("runPrompt propagates a spawn error", async () => {
|
||||
await assert.rejects(
|
||||
runPrompt("p", { spawnFn: fakeSpawn({ error: new Error("ENOENT") }) }),
|
||||
/ENOENT/,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue