chore: address Copilot review round 4

- verify:skills:cli — the authoritative validator is now a GUARANTEED step in
  `local:gate` and CI, not a best-effort hand-off. It resolves the CLI itself
  (an installed one at 2.1.233+, else a pinned `npx -y`), so both tiers run the
  same script and cannot drift. `verify:skills` keeps skipping when the CLI is
  absent, which is right for `validate`: fast, offline, no Claude Code required.
- skill-manifest: the 1,536-char listing-entry cap covers `description` +
  `when_to_use` together; checking the description alone passed an over-cap
  entry. `when_to_use` is now parsed, validated and counted, with boundary tests
  at 1,536/1,537.
- verify-skills: report the budget as *this repo's contribution*. The real
  listing budget is shared with bundled skills and the contributor's own
  ~/.claude/skills, none of which is visible from here, so the number was
  claiming more than it measured.
- skill-eval: a negative case now means "no skill OF THIS REPO'S fired". It
  failed on any Skill invocation, so a contributor with an unrelated global
  skill got a false failure about their environment rather than these skills.
- docs/architecture + local-dev: the Node backend runner gets its
  `@inspector/core` alias from `clients/web/tsup.runner.config.ts`, not from
  Vite. The old text sent runner work to the wrong config.
- local-dev: the tarball does not ship "only each client's build/" — the
  allowlist also carries `clients/web/dist`, `clients/web/static` and the
  install helper. States the invariant that actually matters instead.
- issue-triage: the test for needing triage is "no board card". Adding "and no
  milestone" would skip the approved-but-unboarded case the same section then
  handles. Also corrected the `$M` note: the lookup is `--state all`, so closed
  issues DO have entries — `isopen` is about the invariants, not the lookup.
- issue-create: replaced "all five" with a per-version table. Steps 3 and 6
  (milestone, Priority) do not exist for v1, so the old list opened with a rule
  the same skill later tells you to violate.

Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-08-29 01:47:43 -04:00
commit d05e76d470

View file

@ -45,6 +45,7 @@ const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
/** Collect the committed cases for every model-invoked skill (optionally one). */
function collectCases(only) {
const cases = [];
const ours = new Set();
for (const dir of readdirSync(SKILLS_DIR).sort()) {
if (only && dir !== only) continue;
const skillFile = path.join(SKILLS_DIR, dir, "SKILL.md");
@ -64,6 +65,7 @@ function collectCases(only) {
);
}
if (!skill.modelInvoked) continue;
ours.add(dir);
const evalsFile = path.join(SKILLS_DIR, dir, "evals", "evals.json");
if (!existsSync(evalsFile)) {
throw new Error(
@ -84,7 +86,7 @@ function collectCases(only) {
}
for (const c of parsed) cases.push({ ...c, from: dir });
}
return cases;
return { cases, ours };
}
/**
@ -184,14 +186,22 @@ export function invokedSkillNames(payload) {
/**
* Whether one sample satisfies a case.
*
* A negative case asserts that no skill **of this repo's** fired. Asserting
* that nothing fired at all would fail for a contributor who happens to have an
* unrelated skill in `~/.claude/skills`, or when a bundled skill matches — a
* false failure about someone else's environment rather than about these
* skills (Copilot).
*
* @param {string | null} expect Skill name, or null for a negative case.
* @param {Set<string>} invoked
* @param {Set<string> | null} [ours] Repo skill names. Null counts any skill.
*/
export function sampleHit(expect, invoked) {
if (expect === null) return invoked.size === 0;
return [...invoked].some((payload) =>
invokedSkillNames(payload).includes(expect),
);
export function sampleHit(expect, invoked, ours = null) {
const names = [...invoked].flatMap(invokedSkillNames);
if (expect === null) {
return ours === null ? names.length === 0 : !names.some((n) => ours.has(n));
}
return names.includes(expect);
}
/**
@ -273,7 +283,7 @@ async function main() {
process.exit(1);
}
const cases = collectCases(process.argv[2]);
const { cases, ours } = collectCases(process.argv[2]);
if (cases.length === 0) {
console.error("skills:eval — no cases found.");
process.exit(1);
@ -288,7 +298,9 @@ async function main() {
let failed = 0;
for (const c of cases) {
const mine = results.filter((r) => r.c === c);
const passes = mine.filter((r) => sampleHit(c.expect, r.invoked)).length;
const passes = mine.filter((r) =>
sampleHit(c.expect, r.invoked, ours),
).length;
const rate = passes / mine.length;
const ok = rate >= THRESHOLD;
if (!ok) failed++;