chore: address Copilot review round 2

- verify-skills: version-gate the optional `claude plugin validate`. Probing
  only for the executable failed the MANDATORY gate for anyone on a CLI older
  than 2.1.233, which has no such subcommand and exits nonzero on it. Skips
  below the floor; a failure now means a supported validator rejected the skills.
- skill-eval: match the invoked skill structurally. A substring test counted
  `{"skill":"not-testing"}` as a hit for `testing` and inflated the rate.
- skill-eval: `runOnce` → exported `runPrompt` with an injectable `spawnFn`, so
  the nonzero-exit branch is actually pinned. Reverting it now fails a test
  (verified by mutation) rather than leaving the suite green.
- board-ops: filter the item lookups by repository. Issue numbers are repo-local
  and these are org boards — #11 really does carry a `servers` card — so an
  unfiltered `select` can move or delete the wrong card, or pass two ids.
- issue-triage: the audit's `own()` filter dropped draft cards, which carry no
  `.content.repository`, so the "non-Issue on a board" check reported 0 while
  the invariant was being violated. Verified against a fixture.
- pr-flow: cut the branch from the base it will target. The blanket "cut from
  v2/main" was wrong for v1, whose history is unrelated.

Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-08-29 01:07:41 -04:00
commit ca6828916f

View file

@ -99,6 +99,29 @@ export function collectSkillInvocations(text) {
return { invoked, rest };
}
/**
* The skill names a `Skill` tool_use payload actually asked for.
*
* Structural rather than substring: matching `"testing"` against the raw JSON
* counts `{"skill":"not-testing"}` as a hit and inflates the measured rate
* (Copilot). The field name is still not assumed — every string value in the
* payload is a candidate, compared by equality.
*
* @param {string} payload JSON text as recorded by `collectSkillInvocations`.
* @returns {string[]}
*/
export function invokedSkillNames(payload) {
let parsed;
try {
parsed = JSON.parse(payload);
} catch {
return [];
}
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
return [];
return Object.values(parsed).filter((v) => typeof v === "string");
}
/**
* Whether one sample satisfies a case.
*
@ -107,13 +130,27 @@ export function collectSkillInvocations(text) {
*/
export function sampleHit(expect, invoked) {
if (expect === null) return invoked.size === 0;
return [...invoked].some((payload) => payload.includes(expect));
return [...invoked].some((payload) =>
invokedSkillNames(payload).includes(expect),
);
}
/** Returns the payloads the Skill tool was called with in one fresh session. */
function runOnce(prompt) {
/**
* Drive one fresh session and return the payloads the `Skill` tool was called
* with.
*
* `spawnFn` is injectable so the stream handling and — more importantly — the
* exit handling are testable. Without a seam here, changing the nonzero branch
* back to resolving an empty set would leave every test green while the eval
* silently reported a plausible hit rate for runs that never happened (Copilot).
*
* @param {string} prompt
* @param {{ spawnFn?: typeof spawn, cwd?: string }} [opts]
* @returns {Promise<Set<string>>}
*/
export function runPrompt(prompt, { spawnFn = spawn, cwd = ROOT } = {}) {
return new Promise((resolve, reject) => {
const p = spawn(
const p = spawnFn(
"claude",
[
"-p",
@ -128,7 +165,7 @@ function runOnce(prompt) {
"--disallowedTools",
"Bash,Write,Edit,NotebookEdit",
],
{ cwd: ROOT, stdio: ["ignore", "pipe", "inherit"] },
{ cwd, stdio: ["ignore", "pipe", "inherit"] },
);
let buf = "";
@ -143,7 +180,7 @@ function runOnce(prompt) {
// A CLI that failed to run observed NOTHING. Resolving it as an empty set
// would silently pass every negative case and read as a trigger miss on
// every positive one, so an auth error or a rate limit would come back as
// a plausible-looking hit rate (Copilot).
// a plausible-looking hit rate.
if (code !== 0) {
reject(new Error(`\`claude -p\` exited ${code} for prompt: ${prompt}`));
return;
@ -187,7 +224,7 @@ async function main() {
const jobs = cases.flatMap((c) => Array.from({ length: RUNS }, () => c));
const results = await pool(jobs, CONCURRENCY, async (c) => ({
c,
invoked: await runOnce(c.prompt),
invoked: await runPrompt(c.prompt),
}));
let failed = 0;