chore: address Copilot review round 4

- verify:skills:cli — the authoritative validator is now a GUARANTEED step in
  `local:gate` and CI, not a best-effort hand-off. It resolves the CLI itself
  (an installed one at 2.1.233+, else a pinned `npx -y`), so both tiers run the
  same script and cannot drift. `verify:skills` keeps skipping when the CLI is
  absent, which is right for `validate`: fast, offline, no Claude Code required.
- skill-manifest: the 1,536-char listing-entry cap covers `description` +
  `when_to_use` together; checking the description alone passed an over-cap
  entry. `when_to_use` is now parsed, validated and counted, with boundary tests
  at 1,536/1,537.
- verify-skills: report the budget as *this repo's contribution*. The real
  listing budget is shared with bundled skills and the contributor's own
  ~/.claude/skills, none of which is visible from here, so the number was
  claiming more than it measured.
- skill-eval: a negative case now means "no skill OF THIS REPO'S fired". It
  failed on any Skill invocation, so a contributor with an unrelated global
  skill got a false failure about their environment rather than these skills.
- docs/architecture + local-dev: the Node backend runner gets its
  `@inspector/core` alias from `clients/web/tsup.runner.config.ts`, not from
  Vite. The old text sent runner work to the wrong config.
- local-dev: the tarball does not ship "only each client's build/" — the
  allowlist also carries `clients/web/dist`, `clients/web/static` and the
  install helper. States the invariant that actually matters instead.
- issue-triage: the test for needing triage is "no board card". Adding "and no
  milestone" would skip the approved-but-unboarded case the same section then
  handles. Also corrected the `$M` note: the lookup is `--state all`, so closed
  issues DO have entries — `isopen` is about the invariants, not the lookup.
- issue-create: replaced "all five" with a per-version table. Steps 3 and 6
  (milestone, Priority) do not exist for v1, so the old list opened with a rule
  the same skill later tells you to violate.

Signed-off-by: cliffhall <cliff@futurescale.com>

scripts/lib/skill-manifest.mjs

@@ -13,7 +13,11 @@
 
 import { parse as parseYaml } from "yaml";
 
-/** Combined per-entry cap Claude Code applies to a skill listing entry. */
+/**
+ * Per-entry cap Claude Code applies to a skill listing entry. It covers
+ * `description` **and** `when_to_use` together, so checking `description` alone
+ * would pass an over-cap entry (Copilot).
+ */
 export const DESCRIPTION_CAP = 1536;
 
 /**
@@ -101,9 +105,18 @@ export function parseSkill(dirName, text) {
   const description = meta.description;
   if (typeof description !== "string" || description.trim().length === 0) {
     errors.push("`description` is required and must be a non-empty string");
-  } else if (description.length > DESCRIPTION_CAP) {
+  }
+
+  const whenToUse = meta["when_to_use"];
+  if ("when_to_use" in meta && typeof whenToUse !== "string") {
+    errors.push("`when_to_use` must be a string");
+  }
+  const entryLength =
+    (typeof description === "string" ? description.length : 0) +
+    (typeof whenToUse === "string" ? whenToUse.length : 0);
+  if (entryLength > DESCRIPTION_CAP) {
     errors.push(
-      `\`description\` is ${description.length} chars; Claude Code caps a listing entry at ${DESCRIPTION_CAP}`,
+      `\`description\` + \`when_to_use\` is ${entryLength} chars; Claude Code caps a listing entry at ${DESCRIPTION_CAP}`,
     );
   }
 
@@ -141,6 +154,7 @@ export function parseSkill(dirName, text) {
   return {
     name: typeof name === "string" ? name : undefined,
     description: typeof description === "string" ? description : undefined,
+    whenToUse: typeof whenToUse === "string" ? whenToUse : undefined,
     modelInvoked,
     userInvocable: meta["user-invocable"] !== false,
     paths: Array.isArray(meta.paths) ? meta.paths : undefined,
@@ -152,13 +166,22 @@ export function parseSkill(dirName, text) {
  * Cost of the skill listing Claude Code loads into context, counting only the
  * skills that appear in it.
  *
- * @param {Array<{ name?: string, description?: string, modelInvoked?: boolean }>} skills
+ * The number is this repo's CONTRIBUTION to the listing, not the whole listing:
+ * bundled skills and a contributor's own `~/.claude/skills` share the same
+ * budget, and none of those is visible from here (Copilot). Keeping the repo's
+ * share well under the cap is what leaves room for them.
+ *
+ * @param {Array<{ name?: string, description?: string, whenToUse?: string, modelInvoked?: boolean }>} skills
  */
 export function listingCost(skills) {
   return skills
     .filter((s) => s.modelInvoked)
     .reduce(
-      (n, s) => n + (s.name?.length ?? 0) + (s.description?.length ?? 0),
+      (n, s) =>
+        n +
+        (s.name?.length ?? 0) +
+        (s.description?.length ?? 0) +
+        (s.whenToUse?.length ?? 0),
       0,
     );
 }
@@ -207,6 +230,13 @@ export function validateEvalCases(skillName, cases) {
   return errors;
 }
 
+/**
+ * Claude Code version the authoritative validator is pinned to when it has to
+ * be fetched. Pinned rather than @latest: a validator that moves on its own can
+ * start failing a PR that changed nothing.
+ */
+export const PINNED_CLI_VERSION = "2.1.250";
+
 /** First Claude Code release with `claude plugin validate`. */
 export const PLUGIN_VALIDATE_MIN_VERSION = [2, 1, 233];
 

scripts/lib/skill-manifest.test.mjs

@@ -219,3 +219,36 @@ test("compareVersions orders the plugin-validate floor correctly", () => {
   assert.ok(compareVersions([3], PLUGIN_VALIDATE_MIN_VERSION) > 0);
   assert.ok(compareVersions([1, 9, 999], PLUGIN_VALIDATE_MIN_VERSION) < 0);
 });
+
+test("the listing-entry cap covers description AND when_to_use together", () => {
+  // The cap is over the combined entry, so checking `description` alone would
+  // pass an over-cap listing entry.
+  const under = fm(
+    `name: x\ndescription: ${"a".repeat(1000)}\nwhen_to_use: ${"b".repeat(536)}\ndisable-model-invocation: true`,
+  );
+  assert.deepEqual(parseSkill("x", under).errors, []);
+
+  const over = fm(
+    `name: x\ndescription: ${"a".repeat(1000)}\nwhen_to_use: ${"b".repeat(537)}\ndisable-model-invocation: true`,
+  );
+  assert.match(parseSkill("x", over).errors.join(), /caps a listing entry/);
+
+  assert.match(
+    parseSkill(
+      "x",
+      fm(
+        "name: x\ndescription: d\nwhen_to_use: 7\ndisable-model-invocation: true",
+      ),
+    ).errors.join(),
+    /when_to_use. must be a string/,
+  );
+});
+
+test("listingCost counts when_to_use too", () => {
+  assert.equal(
+    listingCost([
+      { name: "aa", description: "bbb", whenToUse: "cc", modelInvoked: true },
+    ]),
+    7,
+  );
+});

scripts/skill-eval.mjs

@@ -45,6 +45,7 @@ const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
 /** Collect the committed cases for every model-invoked skill (optionally one). */
 function collectCases(only) {
   const cases = [];
+  const ours = new Set();
   for (const dir of readdirSync(SKILLS_DIR).sort()) {
     if (only && dir !== only) continue;
     const skillFile = path.join(SKILLS_DIR, dir, "SKILL.md");
@@ -64,6 +65,7 @@ function collectCases(only) {
       );
     }
     if (!skill.modelInvoked) continue;
+    ours.add(dir);
     const evalsFile = path.join(SKILLS_DIR, dir, "evals", "evals.json");
     if (!existsSync(evalsFile)) {
       throw new Error(
@@ -84,7 +86,7 @@ function collectCases(only) {
     }
     for (const c of parsed) cases.push({ ...c, from: dir });
   }
-  return cases;
+  return { cases, ours };
 }
 
 /**
@@ -184,14 +186,22 @@ export function invokedSkillNames(payload) {
 /**
  * Whether one sample satisfies a case.
  *
+ * A negative case asserts that no skill **of this repo's** fired. Asserting
+ * that nothing fired at all would fail for a contributor who happens to have an
+ * unrelated skill in `~/.claude/skills`, or when a bundled skill matches — a
+ * false failure about someone else's environment rather than about these
+ * skills (Copilot).
+ *
  * @param {string | null} expect Skill name, or null for a negative case.
  * @param {Set<string>} invoked
+ * @param {Set<string> | null} [ours] Repo skill names. Null counts any skill.
  */
-export function sampleHit(expect, invoked) {
-  if (expect === null) return invoked.size === 0;
-  return [...invoked].some((payload) =>
-    invokedSkillNames(payload).includes(expect),
-  );
+export function sampleHit(expect, invoked, ours = null) {
+  const names = [...invoked].flatMap(invokedSkillNames);
+  if (expect === null) {
+    return ours === null ? names.length === 0 : !names.some((n) => ours.has(n));
+  }
+  return names.includes(expect);
 }
 
 /**
@@ -273,7 +283,7 @@ async function main() {
     process.exit(1);
   }
 
-  const cases = collectCases(process.argv[2]);
+  const { cases, ours } = collectCases(process.argv[2]);
   if (cases.length === 0) {
     console.error("skills:eval — no cases found.");
     process.exit(1);
@@ -288,7 +298,9 @@ async function main() {
   let failed = 0;
   for (const c of cases) {
     const mine = results.filter((r) => r.c === c);
-    const passes = mine.filter((r) => sampleHit(c.expect, r.invoked)).length;
+    const passes = mine.filter((r) =>
+      sampleHit(c.expect, r.invoked, ours),
+    ).length;
     const rate = passes / mine.length;
     const ok = rate >= THRESHOLD;
     if (!ok) failed++;

scripts/skill-eval.test.mjs

@@ -197,3 +197,21 @@ test("runPrompt propagates a spawn error", async () => {
     /ENOENT/,
   );
 });
+
+test("a negative case ignores skills that are not this repo's", () => {
+  // A contributor's own `~/.claude/skills` entry, or a bundled skill, firing on
+  // a negative prompt says nothing about these skills — failing on it would be
+  // a false failure about someone else's environment.
+  const ours = new Set(["testing", "local-dev"]);
+  const foreign = new Set(['{"skill":"my-personal-notes"}']);
+  const mine = new Set(['{"skill":"testing"}']);
+
+  assert.equal(sampleHit(null, foreign, ours), true);
+  assert.equal(sampleHit(null, mine, ours), false);
+  assert.equal(sampleHit(null, new Set(), ours), true);
+  // A positive case is unaffected: it names the skill it wants.
+  assert.equal(sampleHit("testing", mine, ours), true);
+  assert.equal(sampleHit("testing", foreign, ours), false);
+  // With no repo set, any invocation still fails a negative case.
+  assert.equal(sampleHit(null, foreign), false);
+});

scripts/verify-skills-cli.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+// Runs `claude plugin validate` — the AUTHORITATIVE skill schema — as a
+// guaranteed step, resolving the CLI rather than hoping it is installed (#2163).
+//
+// `verify:skills` (inside `validate`) reimplements the parse and treats a
+// missing CLI as a skip, deliberately: `npm run validate` is the fast inner
+// loop, it must work offline, and a Node contributor should not need Claude Code
+// installed to run it. But "skips when absent" plus "usually absent" adds up to
+// "never runs", which made the acceptance criterion aspirational (Copilot).
+//
+// So the authoritative check gets its own step, in `local:gate` and in CI, and
+// it does NOT skip: an already-installed CLI new enough to have the subcommand
+// is used as-is, and otherwise a PINNED one is fetched with `npx -y`. Pinned
+// rather than @latest because a validator that moves on its own can start
+// failing a PR that changed nothing, which is how a gate loses its credibility.
+//
+// It needs no authentication — verified against a scrubbed environment and a
+// clean HOME.
+
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  compareVersions,
+  parseClaudeVersion,
+  PINNED_CLI_VERSION,
+  PLUGIN_VALIDATE_MIN_VERSION,
+} from "./lib/skill-manifest.mjs";
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const SKILLS_DIR = ".claude/skills";
+
+/**
+ * The argv that runs a usable validator: the local CLI when it is new enough,
+ * otherwise the pinned one through `npx`.
+ *
+ * @param {{ version: number[] | null }} local
+ * @returns {{ command: string, args: string[], via: string }}
+ */
+export function validatorCommand({ version }) {
+  if (
+    version !== null &&
+    compareVersions(version, PLUGIN_VALIDATE_MIN_VERSION) >= 0
+  ) {
+    return {
+      command: "claude",
+      args: ["plugin", "validate", SKILLS_DIR],
+      via: `local claude ${version.join(".")}`,
+    };
+  }
+  return {
+    command: "npx",
+    args: [
+      "-y",
+      `@anthropic-ai/claude-code@${PINNED_CLI_VERSION}`,
+      "plugin",
+      "validate",
+      SKILLS_DIR,
+    ],
+    via:
+      version === null
+        ? `npx @anthropic-ai/claude-code@${PINNED_CLI_VERSION} (no local CLI)`
+        : `npx @anthropic-ai/claude-code@${PINNED_CLI_VERSION} (local ${version.join(".")} predates the subcommand)`,
+  };
+}
+
+function main() {
+  const probe = spawnSync("claude", ["--version"], { encoding: "utf8" });
+  const version =
+    probe.error || probe.status !== 0
+      ? null
+      : parseClaudeVersion(probe.stdout ?? "");
+
+  const { command, args, via } = validatorCommand({ version });
+  console.log(`verify:skills:cli — validating ${SKILLS_DIR} via ${via}…`);
+
+  const res = spawnSync(command, args, {
+    cwd: ROOT,
+    stdio: "inherit",
+    shell: process.platform === "win32",
+  });
+  if (res.error) {
+    console.error(
+      `\nverify:skills:cli — could not run \`${command}\`: ${res.error.message}`,
+    );
+    process.exit(1);
+  }
+  if (res.status !== 0) {
+    console.error(
+      "\nverify:skills:cli — `claude plugin validate` rejected the skills.",
+    );
+    process.exit(1);
+  }
+}
+
+if (
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  main();
+}

scripts/verify-skills-cli.test.mjs

@@ -0,0 +1,52 @@
+// Tests for the authoritative-validator step's CLI resolution (#2163, Copilot).
+//
+// The point of the step is that it does NOT skip: `verify:skills` treats a
+// missing CLI as a skip, and "skips when absent" plus "usually absent" adds up
+// to "never runs", which is what made the acceptance criterion aspirational.
+// So the branch that matters is the one that falls back to a pinned `npx`
+// rather than giving up — and on a machine with a current CLI installed, that
+// branch is never taken by an actual run.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { validatorCommand } from "./verify-skills-cli.mjs";
+import {
+  PINNED_CLI_VERSION,
+  PLUGIN_VALIDATE_MIN_VERSION,
+} from "./lib/skill-manifest.mjs";
+
+test("uses an installed CLI that has the subcommand", () => {
+  const cmd = validatorCommand({ version: [2, 1, 250] });
+  assert.equal(cmd.command, "claude");
+  assert.deepEqual(cmd.args, ["plugin", "validate", ".claude/skills"]);
+  assert.match(cmd.via, /local claude 2\.1\.250/);
+});
+
+test("uses the installed CLI exactly at the floor", () => {
+  assert.equal(
+    validatorCommand({ version: PLUGIN_VALIDATE_MIN_VERSION }).command,
+    "claude",
+  );
+});
+
+test("falls back to a pinned npx when no CLI is present", () => {
+  const cmd = validatorCommand({ version: null });
+  assert.equal(cmd.command, "npx");
+  assert.deepEqual(cmd.args, [
+    "-y",
+    `@anthropic-ai/claude-code@${PINNED_CLI_VERSION}`,
+    "plugin",
+    "validate",
+    ".claude/skills",
+  ]);
+  assert.match(cmd.via, /no local CLI/);
+});
+
+test("falls back when the installed CLI predates the subcommand", () => {
+  // An older binary answers `--version` fine and then exits nonzero on
+  // `plugin validate` as an unknown command, so running it would fail the gate
+  // for someone whose only sin is not having upgraded.
+  const cmd = validatorCommand({ version: [2, 1, 232] });
+  assert.equal(cmd.command, "npx");
+  assert.match(cmd.via, /predates the subcommand/);
+});

scripts/verify-skills.main.test.mjs

@@ -147,7 +147,7 @@ test("fails when the skill listing exceeds its budget", () => {
   const over = fixture(many);
   const { code, out } = run(over);
   assert.equal(code, 1);
-  assert.match(out, /over the recorded budget/);
+  assert.match(out, /over its recorded budget/);
   rmSync(over, { recursive: true, force: true });
 
   const quiet = {};

scripts/verify-skills.mjs

@@ -111,8 +111,11 @@ function main(argv = process.argv.slice(2)) {
   const cost = listingCost(parsed);
   if (cost > LISTING_BUDGET) {
     failures.push(
-      `the skill listing costs ${cost} chars, over the recorded budget of ${LISTING_BUDGET}. ` +
-        "Tighten a description, or raise the budget deliberately in scripts/lib/skill-manifest.mjs.",
+      `this repo contributes ${cost} chars to the skill listing, over its recorded ` +
+        `budget of ${LISTING_BUDGET}. Tighten a description, or raise the budget ` +
+        "deliberately in scripts/lib/skill-manifest.mjs — but note the real listing " +
+        "budget is SHARED with bundled skills and the contributor's own " +
+        "~/.claude/skills, none of which is visible from here.",
     );
   }
 
@@ -126,7 +129,7 @@ function main(argv = process.argv.slice(2)) {
   const modelInvoked = parsed.filter((s) => s.modelInvoked).map((s) => s.name);
   console.log(
     `verify:skills — OK: ${parsed.length} skills, ${modelInvoked.length} model-invoked ` +
-      `(${modelInvoked.join(", ")}); listing ${cost}/${LISTING_BUDGET} chars.`,
+      `(${modelInvoked.join(", ")}); this repo's listing share ${cost}/${LISTING_BUDGET} chars.`,
   );
 
   if (override) return;