chore: address Copilot review round 2

- verify-skills: version-gate the optional `claude plugin validate`. Probing
  only for the executable failed the MANDATORY gate for anyone on a CLI older
  than 2.1.233, which has no such subcommand and exits nonzero on it. Skips
  below the floor; a failure now means a supported validator rejected the skills.
- skill-eval: match the invoked skill structurally. A substring test counted
  `{"skill":"not-testing"}` as a hit for `testing` and inflated the rate.
- skill-eval: `runOnce` → exported `runPrompt` with an injectable `spawnFn`, so
  the nonzero-exit branch is actually pinned. Reverting it now fails a test
  (verified by mutation) rather than leaving the suite green.
- board-ops: filter the item lookups by repository. Issue numbers are repo-local
  and these are org boards — #11 really does carry a `servers` card — so an
  unfiltered `select` can move or delete the wrong card, or pass two ids.
- issue-triage: the audit's `own()` filter dropped draft cards, which carry no
  `.content.repository`, so the "non-Issue on a board" check reported 0 while
  the invariant was being violated. Verified against a fixture.
- pr-flow: cut the branch from the base it will target. The blanket "cut from
  v2/main" was wrong for v1, whose history is unrelated.

Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-08-29 01:07:41 -04:00
commit ca6828916f

View file

@ -10,7 +10,13 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { collectSkillInvocations, sampleHit } from "./skill-eval.mjs";
import { EventEmitter } from "node:events";
import {
collectSkillInvocations,
invokedSkillNames,
runPrompt,
sampleHit,
} from "./skill-eval.mjs";
const assistant = (...blocks) =>
JSON.stringify({ type: "assistant", message: { content: blocks } });
@ -67,3 +73,78 @@ test("sampleHit scores positive and negative cases", () => {
assert.equal(sampleHit(null, fired), false);
assert.equal(sampleHit("testing", none), false);
});
test("invokedSkillNames matches structurally, not by substring", () => {
// `{"skill":"not-testing"}` contains "testing" and must NOT count — a
// substring match inflates the measured hit rate with invocations of a
// different skill.
assert.deepEqual(invokedSkillNames('{"skill":"not-testing"}'), [
"not-testing",
]);
assert.equal(
sampleHit("testing", new Set(['{"skill":"not-testing"}'])),
false,
);
assert.equal(sampleHit("testing", new Set(['{"skill":"testing"}'])), true);
// The field name is not assumed, so any string value is a candidate.
assert.equal(
sampleHit("testing", new Set(['{"name":"testing","args":""}'])),
true,
);
});
test("invokedSkillNames tolerates payloads it cannot read", () => {
for (const payload of ["not json", "[1,2]", "null", '"testing"']) {
assert.deepEqual(invokedSkillNames(payload), []);
}
});
/** Minimal stand-in for a ChildProcess: emits stdout chunks, then closes. */
function fakeSpawn({ chunks = [], code = 0, error = null }) {
return () => {
const child = new EventEmitter();
child.stdout = new EventEmitter();
queueMicrotask(() => {
if (error) {
child.emit("error", error);
return;
}
for (const c of chunks) child.stdout.emit("data", Buffer.from(c));
child.emit("close", code);
});
return child;
};
}
test("runPrompt collects invocations across chunk boundaries", async () => {
const whole =
JSON.stringify({
type: "assistant",
message: {
content: [
{ type: "tool_use", name: "Skill", input: { skill: "testing" } },
],
},
}) + "\n";
const invoked = await runPrompt("p", {
spawnFn: fakeSpawn({ chunks: [whole.slice(0, 30), whole.slice(30)] }),
});
assert.equal(sampleHit("testing", invoked), true);
});
test("runPrompt rejects a nonzero exit rather than reporting an empty observation", async () => {
// The regression that matters: resolving here would pass every negative case
// and read as a trigger miss on every positive one, so a rate-limited run
// would report a plausible hit rate.
await assert.rejects(
runPrompt("p", { spawnFn: fakeSpawn({ code: 1 }) }),
/exited 1 for prompt: p/,
);
});
test("runPrompt propagates a spawn error", async () => {
await assert.rejects(
runPrompt("p", { spawnFn: fakeSpawn({ error: new Error("ENOENT") }) }),
/ENOENT/,
);
});