docs: accept several skill names in skills:eval and re-aim overlapping cases

Adds a multi-name filter to `npm run skills:eval` so a set of related
skills can be iterated together, and makes an unknown name a hard error
rather than an empty run — a typo would otherwise report a green 0/0
that reads exactly like a clean pass of the skill that was meant.

Re-aims two `local-dev` cases and one `test-servers` case that partially
overlapped `AGENTS.md`. Partial overlap oscillates across runs rather
than sitting at zero, so it reads as eval noise instead of a defect in
the case; `docs/skill-authoring.md` now records that failure mode and
the observed scores.

Widens the `test-servers` description to the "Use when …" shape and
points the `testing` skill at it, since anything needing a real server
to run against is a procedure `testing` does not carry. That growth
moves the skill listing from 2,921 to 3,144 of 4,000 characters, so the
figure recorded in AGENTS.md is updated to match.

Signed-off-by: cliffhall <cliff@futurescale.com>

scripts/skill-eval.mjs

@@ -25,8 +25,9 @@
 // Closing that would need a deterministic trigger oracle, which does not exist.
 //
 // Usage:
-//   npm run skills:eval                  # every model-invoked skill's cases
-//   npm run skills:eval -- testing       # one skill's cases
+//   npm run skills:eval                        # every model-invoked skill's cases
+//   npm run skills:eval -- testing             # one skill's cases
+//   npm run skills:eval -- testing test-servers  # several skills' cases
 //   RUNS=5 THRESHOLD=0.8 npm run skills:eval
 
 import { spawn } from "node:child_process";
@@ -58,11 +59,19 @@ const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
  * (Copilot). Every model-invoked skill is inspected; `only` is applied when
  * enqueueing.
  *
- * @param {string | undefined} only
+ * A name that matches no model-invoked skill is a hard error rather than an
+ * empty run: a typo would otherwise enqueue nothing and the eval would report a
+ * green 0/0, which reads exactly like a clean pass of the skill you meant.
+ *
+ * @param {string | string[] | undefined} only One or more skill names.
  * @param {string} [skillsDir]
  * @returns {{ cases: object[], ours: Set<string> }}
  */
 export function collectCases(only, skillsDir = SKILLS_DIR) {
+  const wanted =
+    only === undefined || only === null
+      ? null
+      : new Set(Array.isArray(only) ? only : [only]);
   const cases = [];
   const ours = new Set();
   for (const dir of readdirSync(skillsDir).sort()) {
@@ -103,9 +112,18 @@ export function collectCases(only, skillsDir = SKILLS_DIR) {
     if (invalid.length > 0) {
       throw new Error(`${dir}/evals/evals.json: ${invalid.join("; ")}`);
     }
-    if (only && dir !== only) continue;
+    if (wanted && !wanted.has(dir)) continue;
     for (const c of parsed) cases.push({ ...c, from: dir });
   }
+  if (wanted) {
+    const unknown = [...wanted].filter((n) => !ours.has(n));
+    if (unknown.length > 0) {
+      throw new Error(
+        `no model-invoked skill named ${unknown.map((n) => `\`${n}\``).join(", ")} — ` +
+          `known: ${[...ours].sort().join(", ")}`,
+      );
+    }
+  }
   return { cases, ours };
 }
 
@@ -310,7 +328,9 @@ async function main() {
     process.exit(1);
   }
 
-  const { cases, ours } = collectCases(process.argv[2]);
+  const { cases, ours } = collectCases(
+    process.argv.length > 2 ? process.argv.slice(2) : undefined,
+  );
   if (cases.length === 0) {
     console.error("skills:eval — no cases found.");
     process.exit(1);

scripts/skill-eval.test.mjs

@@ -322,6 +322,42 @@ test("focused mode narrows the cases but not the repo's own skill set", () => {
   rmSync(root, { recursive: true, force: true });
 });
 
+test("focused mode accepts several skill names", () => {
+  const root = skillsFixture();
+
+  const focused = collectCases(["alpha", "beta"], root);
+  assert.deepEqual(focused.cases.map((c) => c.prompt).sort(), [
+    "a+",
+    "a-",
+    "b+",
+    "b-",
+  ]);
+  // Narrowing to every skill is still not the same as no filter — `ours` is
+  // unchanged either way, which is what keeps negative scoring honest.
+  assert.deepEqual([...focused.ours].sort(), ["alpha", "beta"]);
+
+  // A single name keeps working, so the one-argument form is unaffected.
+  assert.deepEqual(
+    collectCases("beta", root).cases.map((c) => c.prompt),
+    ["b+", "b-"],
+  );
+
+  rmSync(root, { recursive: true, force: true });
+});
+
+test("an unknown skill name is an error, not an empty run", () => {
+  // A typo would otherwise enqueue zero cases and report a green 0/0, which
+  // reads exactly like a clean pass of the skill that was meant.
+  const root = skillsFixture();
+  assert.throws(
+    () => collectCases(["alpha", "aplha"], root),
+    /no model-invoked skill named `aplha`/,
+  );
+  // A name-only skill is not model-invoked, so asking for it is the same error.
+  assert.throws(() => collectCases(["gamma"], root), /no model-invoked skill/);
+  rmSync(root, { recursive: true, force: true });
+});
+
 test("a name-only skill is not part of the repo's model-invoked set", () => {
   const root = skillsFixture();
   const { ours } = collectCases(undefined, root);