chore: extract AGENTS.md recipes into agent skills
Closes #2163 `AGENTS.md` was one 1,223-line contract loaded in full on every turn, whether the task was a one-line CSS fix or a release cut. This splits it: rules stay resident, procedures move to `.claude/skills/`. - Ten skills under `.claude/skills/`, seven `disable-model-invocation: true` (invoked by name only, so they cost no listing context and have no trigger question) and three model-invoked with committed eval cases. - `AGENTS.md` 1,223 → 380 lines: rules, a skills index, and a project tree reduced to one line per entry. The removed recipes are not duplicated. - `README.md` 811 → 105 lines, split into `docs/{architecture,quality-gate, test-servers,publishing,docker}.md`, and dropped from `CLAUDE.md`'s auto-load. - `.github/copilot-instructions.md` and its mirroring rule removed — Copilot reads `AGENTS.md`. - `npm run verify:skills` (in `validate`, so in `local:gate` and CI) parses each SKILL.md the way Claude Code does, requires an explicit invocation mode and eval cases for model-invoked skills, and budgets the listing. - `npm run skills:eval` runs the committed trigger cases headless. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall
parent
8c91ae8820
commit
73447c72d1
4 changed files with 3291 additions and 1927 deletions
205
scripts/lib/skill-manifest.mjs
Normal file
205
scripts/lib/skill-manifest.mjs
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
// Pure parsers for the `.claude/skills/**/SKILL.md` contract, extracted so
|
||||
// `verify-skills.mjs` can be unit-tested without a filesystem (#2163).
|
||||
//
|
||||
// The failure this guards is silent by construction: Claude Code reads a
|
||||
// SKILL.md's frontmatter only when the opening `---` is the file's FIRST line
|
||||
// and the YAML between the fences parses. Otherwise the body loads with EMPTY
|
||||
// metadata — `/skill-name` still works, so a manual spot check passes, but
|
||||
// there is no `description` left for the model to match against and the skill
|
||||
// never auto-fires again. A stray unquoted colon in a description is enough.
|
||||
//
|
||||
// So every check here is deny-by-default: a file we cannot read the way Claude
|
||||
// Code reads it is an error, not a skip.
|
||||
|
||||
import { parse as parseYaml } from "yaml";
|
||||
|
||||
/** Combined per-entry cap Claude Code applies to a skill listing entry. */
|
||||
export const DESCRIPTION_CAP = 1536;
|
||||
|
||||
/**
|
||||
* Character budget for the skill LISTING — the sum of name + description over
|
||||
* the skills that occupy it. Only model-invoked skills do: a skill with
|
||||
* `disable-model-invocation: true` is reachable solely by name, so its
|
||||
* description never enters context.
|
||||
*
|
||||
* The listing is truncated when it overflows the model's budget, and the
|
||||
* entries dropped first are the ones invoked least — i.e. exactly the
|
||||
* model-invoked skills that must fire on their own. Recording the number here
|
||||
* is what makes a regression visible when a skill is added later.
|
||||
*/
|
||||
export const LISTING_BUDGET = 4000;
|
||||
|
||||
const FRONTMATTER_FENCE = "---";
|
||||
|
||||
/**
|
||||
* Split a SKILL.md into its raw frontmatter and body the way Claude Code does.
|
||||
* Returns `{ error }` rather than throwing, so the caller can report every
|
||||
* offending file in one pass instead of dying on the first.
|
||||
*
|
||||
* @param {string} text Full file contents.
|
||||
* @returns {{ frontmatter?: string, body?: string, error?: string }}
|
||||
*/
|
||||
export function splitFrontmatter(text) {
|
||||
// A BOM or a leading blank line means the fence is not the first line, and
|
||||
// Claude Code then treats the WHOLE file — `---` markers included — as body.
|
||||
if (!text.startsWith(FRONTMATTER_FENCE + "\n")) {
|
||||
return {
|
||||
error:
|
||||
"frontmatter must open with `---` on the very first line (no BOM, no leading blank line)",
|
||||
};
|
||||
}
|
||||
const rest = text.slice(FRONTMATTER_FENCE.length + 1);
|
||||
const end = rest.indexOf("\n" + FRONTMATTER_FENCE);
|
||||
if (end === -1)
|
||||
return { error: "frontmatter is never closed by a `---` line" };
|
||||
return {
|
||||
frontmatter: rest.slice(0, end),
|
||||
body: rest.slice(end + FRONTMATTER_FENCE.length + 2),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and validate one skill's frontmatter.
|
||||
*
|
||||
* @param {string} dirName Directory the SKILL.md lives in — the skill's identity.
|
||||
* @param {string} text Full file contents.
|
||||
* @returns {{ name?: string, description?: string, modelInvoked?: boolean,
|
||||
* userInvocable?: boolean, paths?: string[], errors: string[] }}
|
||||
*/
|
||||
export function parseSkill(dirName, text) {
|
||||
const errors = [];
|
||||
const { frontmatter, error } = splitFrontmatter(text);
|
||||
if (error) return { errors: [error] };
|
||||
|
||||
let meta;
|
||||
try {
|
||||
meta = parseYaml(frontmatter);
|
||||
} catch (e) {
|
||||
// The whole point of the guard: this is what strips the description.
|
||||
return {
|
||||
errors: [`frontmatter is not valid YAML — ${e.message.split("\n")[0]}`],
|
||||
};
|
||||
}
|
||||
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
||||
return { errors: ["frontmatter must parse to a mapping"] };
|
||||
}
|
||||
|
||||
const name = meta.name;
|
||||
if (typeof name !== "string" || name.length === 0) {
|
||||
errors.push("`name` is required and must be a string");
|
||||
} else if (name !== dirName) {
|
||||
errors.push(
|
||||
`\`name: ${name}\` does not match its directory \`${dirName}\``,
|
||||
);
|
||||
} else if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) {
|
||||
errors.push(`\`name: ${name}\` must be kebab-case`);
|
||||
}
|
||||
|
||||
const description = meta.description;
|
||||
if (typeof description !== "string" || description.trim().length === 0) {
|
||||
errors.push("`description` is required and must be a non-empty string");
|
||||
} else if (description.length > DESCRIPTION_CAP) {
|
||||
errors.push(
|
||||
`\`description\` is ${description.length} chars; Claude Code caps a listing entry at ${DESCRIPTION_CAP}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Every skill states its invocation mode rather than inheriting a default,
|
||||
// so the eval surface is knowable by reading the files.
|
||||
if (!("disable-model-invocation" in meta)) {
|
||||
errors.push(
|
||||
"`disable-model-invocation` must be declared explicitly (true = invoked by name only, false = the model may fire it)",
|
||||
);
|
||||
}
|
||||
const dmi = meta["disable-model-invocation"];
|
||||
if ("disable-model-invocation" in meta && typeof dmi !== "boolean") {
|
||||
errors.push("`disable-model-invocation` must be a boolean");
|
||||
}
|
||||
const modelInvoked = dmi === false;
|
||||
|
||||
if ("user-invocable" in meta && typeof meta["user-invocable"] !== "boolean") {
|
||||
errors.push("`user-invocable` must be a boolean");
|
||||
}
|
||||
if ("paths" in meta) {
|
||||
if (
|
||||
!Array.isArray(meta.paths) ||
|
||||
meta.paths.some((p) => typeof p !== "string")
|
||||
) {
|
||||
errors.push("`paths` must be a list of glob strings");
|
||||
}
|
||||
}
|
||||
// A skill nothing can reach is dead weight that still costs a directory.
|
||||
if (!modelInvoked && meta["user-invocable"] === false) {
|
||||
errors.push(
|
||||
"skill is unreachable: `disable-model-invocation: true` with `user-invocable: false`",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
name: typeof name === "string" ? name : undefined,
|
||||
description: typeof description === "string" ? description : undefined,
|
||||
modelInvoked,
|
||||
userInvocable: meta["user-invocable"] !== false,
|
||||
paths: Array.isArray(meta.paths) ? meta.paths : undefined,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cost of the skill listing Claude Code loads into context, counting only the
|
||||
* skills that appear in it.
|
||||
*
|
||||
* @param {Array<{ name?: string, description?: string, modelInvoked?: boolean }>} skills
|
||||
*/
|
||||
export function listingCost(skills) {
|
||||
return skills
|
||||
.filter((s) => s.modelInvoked)
|
||||
.reduce(
|
||||
(n, s) => n + (s.name?.length ?? 0) + (s.description?.length ?? 0),
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an `evals/evals.json` payload for a model-invoked skill.
|
||||
*
|
||||
* A skill that fires on everything is a context regression, not a win, and it
|
||||
* is the failure nobody notices by hand — so negatives are required, not
|
||||
* optional.
|
||||
*
|
||||
* @param {string} skillName
|
||||
* @param {unknown} cases Parsed JSON.
|
||||
* @returns {string[]} errors
|
||||
*/
|
||||
export function validateEvalCases(skillName, cases) {
|
||||
if (!Array.isArray(cases) || cases.length === 0) {
|
||||
return ["evals.json must be a non-empty array of cases"];
|
||||
}
|
||||
const errors = [];
|
||||
cases.forEach((c, i) => {
|
||||
if (c === null || typeof c !== "object" || Array.isArray(c)) {
|
||||
errors.push(`case ${i}: must be an object`);
|
||||
return;
|
||||
}
|
||||
if (typeof c.prompt !== "string" || c.prompt.trim() === "") {
|
||||
errors.push(`case ${i}: \`prompt\` must be a non-empty string`);
|
||||
}
|
||||
if (
|
||||
!("expect" in c) ||
|
||||
(c.expect !== null && typeof c.expect !== "string")
|
||||
) {
|
||||
errors.push(`case ${i}: \`expect\` must be a skill name or null`);
|
||||
}
|
||||
});
|
||||
const positives = cases.filter((c) => c && c.expect === skillName).length;
|
||||
const negatives = cases.filter((c) => c && c.expect === null).length;
|
||||
if (positives === 0) {
|
||||
errors.push(`no positive case expects \`${skillName}\``);
|
||||
}
|
||||
if (negatives === 0) {
|
||||
errors.push(
|
||||
"no negative case (`expect: null`) — a skill that fires on everything is a regression",
|
||||
);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
181
scripts/lib/skill-manifest.test.mjs
Normal file
181
scripts/lib/skill-manifest.test.mjs
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
// Table-driven tests for the `.claude/skills` frontmatter contract (#2163),
|
||||
// one case per rule the parsers encode. The guard that reads them
|
||||
// (`verify-skills.mjs`) walks the real skills directory, so a rule it enforces
|
||||
// is only observable there when a skill happens to violate it — which, by
|
||||
// construction, none does once the guard is green.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
splitFrontmatter,
|
||||
parseSkill,
|
||||
validateEvalCases,
|
||||
listingCost,
|
||||
} from "./skill-manifest.mjs";
|
||||
|
||||
const fm = (body) => `---\n${body}\n---\n\n# Body\n`;
|
||||
|
||||
test("splitFrontmatter requires the fence on the very first line", () => {
|
||||
// This is the whole failure class: anything before the fence and Claude Code
|
||||
// treats the WHOLE file, `---` markers included, as body content.
|
||||
for (const text of [
|
||||
"\n---\nname: x\n---\n",
|
||||
"---\nname: x\n---\n",
|
||||
"# Title\n---\nname: x\n---\n",
|
||||
]) {
|
||||
assert.match(splitFrontmatter(text).error ?? "", /first line/);
|
||||
}
|
||||
assert.equal(splitFrontmatter(fm("name: x")).error, undefined);
|
||||
});
|
||||
|
||||
test("splitFrontmatter reports an unterminated block", () => {
|
||||
assert.match(splitFrontmatter("---\nname: x\n").error ?? "", /never closed/);
|
||||
});
|
||||
|
||||
test("parseSkill accepts a well-formed skill", () => {
|
||||
const r = parseSkill(
|
||||
"local-dev",
|
||||
fm(
|
||||
"name: local-dev\ndescription: Does a thing.\ndisable-model-invocation: false",
|
||||
),
|
||||
);
|
||||
assert.deepEqual(r.errors, []);
|
||||
assert.equal(r.modelInvoked, true);
|
||||
assert.equal(r.userInvocable, true);
|
||||
});
|
||||
|
||||
test("parseSkill rejects frontmatter that is not valid YAML", () => {
|
||||
// An unquoted colon in a description is enough, and this is exactly what
|
||||
// silently strips the description Claude matches against.
|
||||
const r = parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: Use this: always\ndisable-model-invocation: true",
|
||||
),
|
||||
);
|
||||
assert.equal(r.errors.length, 1);
|
||||
assert.match(r.errors[0], /not valid YAML/);
|
||||
});
|
||||
|
||||
test("parseSkill requires name to match the directory, kebab-case", () => {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"board-ops",
|
||||
fm("name: boardOps\ndescription: d\ndisable-model-invocation: true"),
|
||||
).errors.join(),
|
||||
/does not match its directory/,
|
||||
);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"Board_Ops",
|
||||
fm("name: Board_Ops\ndescription: d\ndisable-model-invocation: true"),
|
||||
).errors.join(),
|
||||
/kebab-case/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill requires a non-empty description", () => {
|
||||
for (const d of ["", " "]) {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(`name: x\ndescription: "${d}"\ndisable-model-invocation: true`),
|
||||
).errors.join(),
|
||||
/description. is required/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("parseSkill caps the description at the listing-entry limit", () => {
|
||||
const long = "a".repeat(1537);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(`name: x\ndescription: ${long}\ndisable-model-invocation: true`),
|
||||
).errors.join(),
|
||||
/caps a listing entry/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill requires the invocation mode to be declared", () => {
|
||||
assert.match(
|
||||
parseSkill("x", fm("name: x\ndescription: d")).errors.join(),
|
||||
/must be declared explicitly/,
|
||||
);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm("name: x\ndescription: d\ndisable-model-invocation: yes-please"),
|
||||
).errors.join(),
|
||||
/must be a boolean/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill rejects a skill nothing can reach", () => {
|
||||
const r = parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: true\nuser-invocable: false",
|
||||
),
|
||||
);
|
||||
assert.match(r.errors.join(), /unreachable/);
|
||||
});
|
||||
|
||||
test("parseSkill validates the paths list", () => {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths: '**/*.ts'",
|
||||
),
|
||||
).errors.join(),
|
||||
/list of glob strings/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths:\n - '**/*.ts'",
|
||||
),
|
||||
).paths,
|
||||
["**/*.ts"],
|
||||
);
|
||||
});
|
||||
|
||||
test("listingCost counts only the skills that occupy the listing", () => {
|
||||
const skills = [
|
||||
{ name: "aa", description: "bbb", modelInvoked: true },
|
||||
{ name: "cccc", description: "ddddd", modelInvoked: false },
|
||||
];
|
||||
assert.equal(listingCost(skills), 5);
|
||||
});
|
||||
|
||||
test("validateEvalCases requires a positive and a negative", () => {
|
||||
assert.deepEqual(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "testing" },
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("testing", [{ prompt: "a", expect: "testing" }]).join(),
|
||||
/no negative case/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("testing", [{ prompt: "a", expect: null }]).join(),
|
||||
/no positive case/,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateEvalCases rejects malformed cases", () => {
|
||||
assert.match(validateEvalCases("x", []).join(), /non-empty array/);
|
||||
assert.match(
|
||||
validateEvalCases("x", [{ expect: null }]).join(),
|
||||
/prompt.*non-empty string/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("x", [{ prompt: "a", expect: 7 }]).join(),
|
||||
/expect. must be a skill name or null/,
|
||||
);
|
||||
});
|
||||
166
scripts/skill-eval.mjs
Executable file
166
scripts/skill-eval.mjs
Executable file
|
|
@ -0,0 +1,166 @@
|
|||
#!/usr/bin/env node
|
||||
// Trigger eval for the model-invoked skills in `.claude/skills` (#2163).
|
||||
//
|
||||
// `verify:skills` asserts a skill is well-formed and that its cases exist. This
|
||||
// runs them: each prompt is executed headless in a FRESH session and we assert
|
||||
// on whether the Skill tool fired with the expected skill.
|
||||
//
|
||||
// Seeing a skill fire once tells us Claude found it, not that it finds it
|
||||
// reliably — the measurement is a hit rate over repeated fresh sessions. It
|
||||
// needs negative cases too: a skill that fires on everything is a context
|
||||
// regression, and it is the failure nobody notices by hand. `verify:skills`
|
||||
// requires both.
|
||||
//
|
||||
// NOT part of `validate`, `local:gate`, or CI: it needs the `claude` CLI and
|
||||
// real model calls, so it is metered and non-deterministic. Run it when adding
|
||||
// a skill or editing a model-invoked skill's description.
|
||||
//
|
||||
// Usage:
|
||||
// npm run skills:eval # every model-invoked skill's cases
|
||||
// npm run skills:eval -- testing # one skill's cases
|
||||
// RUNS=5 THRESHOLD=0.8 npm run skills:eval
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseSkill } from "./lib/skill-manifest.mjs";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const SKILLS_DIR = path.join(ROOT, ".claude", "skills");
|
||||
|
||||
const THRESHOLD = Number(process.env.THRESHOLD ?? 0.8);
|
||||
const RUNS = Number(process.env.RUNS ?? 3);
|
||||
const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
|
||||
|
||||
/** Collect the committed cases for every model-invoked skill (optionally one). */
|
||||
function collectCases(only) {
|
||||
const cases = [];
|
||||
for (const dir of readdirSync(SKILLS_DIR).sort()) {
|
||||
if (only && dir !== only) continue;
|
||||
const skillFile = path.join(SKILLS_DIR, dir, "SKILL.md");
|
||||
if (
|
||||
!statSync(path.join(SKILLS_DIR, dir)).isDirectory() ||
|
||||
!existsSync(skillFile)
|
||||
)
|
||||
continue;
|
||||
const skill = parseSkill(dir, readFileSync(skillFile, "utf8"));
|
||||
if (skill.errors.length > 0 || !skill.modelInvoked) continue;
|
||||
const evalsFile = path.join(SKILLS_DIR, dir, "evals", "evals.json");
|
||||
if (!existsSync(evalsFile)) continue;
|
||||
for (const c of JSON.parse(readFileSync(evalsFile, "utf8")))
|
||||
cases.push({ ...c, from: dir });
|
||||
}
|
||||
return cases;
|
||||
}
|
||||
|
||||
/** Returns the payloads the Skill tool was called with in one fresh session. */
|
||||
function runOnce(prompt) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const p = spawn(
|
||||
"claude",
|
||||
[
|
||||
"-p",
|
||||
prompt,
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"--verbose",
|
||||
"--max-turns",
|
||||
"1",
|
||||
// Keep the run read-only. A skill may inject `!`-prefixed shell commands
|
||||
// on load, and those run BEFORE its content reaches the model.
|
||||
"--disallowedTools",
|
||||
"Bash,Write,Edit,NotebookEdit",
|
||||
],
|
||||
{ cwd: ROOT, stdio: ["ignore", "pipe", "ignore"] },
|
||||
);
|
||||
|
||||
let buf = "";
|
||||
const invoked = new Set();
|
||||
p.stdout.on("data", (chunk) => {
|
||||
buf += chunk.toString();
|
||||
const lines = buf.split("\n");
|
||||
buf = lines.pop() ?? "";
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
let evt;
|
||||
try {
|
||||
evt = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (evt?.type !== "assistant") continue;
|
||||
for (const block of evt.message?.content ?? []) {
|
||||
if (block?.type !== "tool_use" || block.name !== "Skill") continue;
|
||||
// Don't assume the input field's name — match on the whole payload.
|
||||
invoked.add(JSON.stringify(block.input ?? {}));
|
||||
}
|
||||
}
|
||||
});
|
||||
p.on("error", reject);
|
||||
p.on("close", () => resolve(invoked));
|
||||
});
|
||||
}
|
||||
|
||||
const hit = (invoked, skill) =>
|
||||
[...invoked].some((payload) => payload.includes(skill));
|
||||
|
||||
async function pool(items, n, fn) {
|
||||
const out = new Array(items.length);
|
||||
let i = 0;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(n, items.length) }, async () => {
|
||||
while (i < items.length) {
|
||||
const idx = i++;
|
||||
out[idx] = await fn(items[idx]);
|
||||
}
|
||||
}),
|
||||
);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (spawnSync("claude", ["--version"], { stdio: "ignore" }).error) {
|
||||
console.error(
|
||||
"skills:eval — the `claude` CLI is not on PATH. This eval needs it.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cases = collectCases(process.argv[2]);
|
||||
if (cases.length === 0) {
|
||||
console.error("skills:eval — no cases found.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const jobs = cases.flatMap((c) => Array.from({ length: RUNS }, () => c));
|
||||
const results = await pool(jobs, CONCURRENCY, async (c) => ({
|
||||
c,
|
||||
invoked: await runOnce(c.prompt),
|
||||
}));
|
||||
|
||||
let failed = 0;
|
||||
for (const c of cases) {
|
||||
const mine = results.filter((r) => r.c === c);
|
||||
const passes = mine.filter((r) =>
|
||||
c.expect === null ? r.invoked.size === 0 : hit(r.invoked, c.expect),
|
||||
).length;
|
||||
const rate = passes / mine.length;
|
||||
const ok = rate >= THRESHOLD;
|
||||
if (!ok) failed++;
|
||||
const label = c.expect ?? "(no skill)";
|
||||
console.log(
|
||||
`${ok ? "PASS" : "FAIL"} ${(rate * 100).toFixed(0).padStart(3)}% ${label.padEnd(20)} ${c.prompt}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`\n${cases.length - failed}/${cases.length} cases at or above ${THRESHOLD * 100}%.`,
|
||||
);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
147
scripts/verify-skills.mjs
Executable file
147
scripts/verify-skills.mjs
Executable file
|
|
@ -0,0 +1,147 @@
|
|||
#!/usr/bin/env node
|
||||
// Durable guard for `.claude/skills/**/SKILL.md` (#2163).
|
||||
//
|
||||
// `AGENTS.md` is loaded in full on every turn, so its rules are unconditionally
|
||||
// present. A skill is conditional — its body loads only when it is invoked — so
|
||||
// if a skill stops being reachable we lose behavior SILENTLY, and nothing in the
|
||||
// normal workflow surfaces that. This is the check that does.
|
||||
//
|
||||
// What it asserts, cheapest first:
|
||||
// 1. Every SKILL.md's frontmatter parses the way Claude Code parses it. This
|
||||
// is the whole-class failure: malformed YAML (or a fence that is not the
|
||||
// first line) loads the body with EMPTY metadata, so `/name` still works —
|
||||
// a manual spot check passes — while the description the model matches
|
||||
// against is gone and the skill never auto-fires again.
|
||||
// 2. Every skill declares its invocation mode explicitly, so the set that has
|
||||
// to fire on its own is knowable by reading the files rather than inferred.
|
||||
// 3. Every model-invoked skill has committed eval cases, including negatives.
|
||||
// Cases are run by `npm run skills:eval`, which needs the Claude CLI and is
|
||||
// therefore NOT in the gate; the cases existing is what this checks.
|
||||
// 4. The skill LISTING stays inside its recorded character budget. Claude Code
|
||||
// truncates the listing when it overflows, dropping the least-invoked
|
||||
// entries first — which are exactly the model-invoked skills.
|
||||
//
|
||||
// It additionally runs `claude plugin validate` when that CLI is on PATH. That
|
||||
// is the authoritative parse, but it cannot be a requirement: the CLI is absent
|
||||
// in CI and on a contributor's machine, and a guard that skips itself wherever
|
||||
// it actually runs is worse than one that reimplements the check.
|
||||
|
||||
import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
parseSkill,
|
||||
validateEvalCases,
|
||||
listingCost,
|
||||
LISTING_BUDGET,
|
||||
} from "./lib/skill-manifest.mjs";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const SKILLS_DIR = path.join(ROOT, ".claude", "skills");
|
||||
|
||||
/** Directory names under `.claude/skills` that contain a SKILL.md. */
|
||||
export function skillDirs(
|
||||
dir,
|
||||
{ readdir = readdirSync, stat = statSync } = {},
|
||||
) {
|
||||
return readdir(dir)
|
||||
.filter((n) => !n.startsWith("."))
|
||||
.filter((n) => stat(path.join(dir, n)).isDirectory())
|
||||
.sort();
|
||||
}
|
||||
|
||||
function main() {
|
||||
const failures = [];
|
||||
|
||||
if (!existsSync(SKILLS_DIR)) {
|
||||
console.error(
|
||||
`verify:skills — ${path.relative(ROOT, SKILLS_DIR)} does not exist.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const dirs = skillDirs(SKILLS_DIR);
|
||||
if (dirs.length === 0) {
|
||||
console.error("verify:skills — no skills found; the skill set is gone.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const parsed = [];
|
||||
for (const dir of dirs) {
|
||||
const file = path.join(SKILLS_DIR, dir, "SKILL.md");
|
||||
if (!existsSync(file)) {
|
||||
failures.push(`${dir}: no SKILL.md`);
|
||||
continue;
|
||||
}
|
||||
const skill = parseSkill(dir, readFileSync(file, "utf8"));
|
||||
for (const e of skill.errors) failures.push(`${dir}/SKILL.md: ${e}`);
|
||||
if (skill.errors.length > 0) continue;
|
||||
parsed.push(skill);
|
||||
|
||||
if (skill.modelInvoked) {
|
||||
const evalsFile = path.join(SKILLS_DIR, dir, "evals", "evals.json");
|
||||
if (!existsSync(evalsFile)) {
|
||||
failures.push(
|
||||
`${dir}: model-invoked skills need committed eval cases at evals/evals.json`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let cases;
|
||||
try {
|
||||
cases = JSON.parse(readFileSync(evalsFile, "utf8"));
|
||||
} catch (e) {
|
||||
failures.push(`${dir}/evals/evals.json: not valid JSON — ${e.message}`);
|
||||
continue;
|
||||
}
|
||||
for (const e of validateEvalCases(dir, cases)) {
|
||||
failures.push(`${dir}/evals/evals.json: ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const cost = listingCost(parsed);
|
||||
if (cost > LISTING_BUDGET) {
|
||||
failures.push(
|
||||
`the skill listing costs ${cost} chars, over the recorded budget of ${LISTING_BUDGET}. ` +
|
||||
"Tighten a description, or raise the budget deliberately in scripts/lib/skill-manifest.mjs.",
|
||||
);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`verify:skills — ${failures.length} problem(s):\n`);
|
||||
for (const f of failures) console.error(" " + f);
|
||||
console.error("\nSee AGENTS.md → Maintaining the skills.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const modelInvoked = parsed.filter((s) => s.modelInvoked).map((s) => s.name);
|
||||
console.log(
|
||||
`verify:skills — OK: ${parsed.length} skills, ${modelInvoked.length} model-invoked ` +
|
||||
`(${modelInvoked.join(", ")}); listing ${cost}/${LISTING_BUDGET} chars.`,
|
||||
);
|
||||
|
||||
// The authoritative parse, when it is available.
|
||||
const probe = spawnSync("claude", ["--version"], { stdio: "ignore" });
|
||||
if (probe.error) {
|
||||
console.log(
|
||||
"verify:skills — `claude` CLI not on PATH; skipped `claude plugin validate`.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const res = spawnSync("claude", ["plugin", "validate", ".claude/skills"], {
|
||||
cwd: ROOT,
|
||||
stdio: "inherit",
|
||||
});
|
||||
if (res.status !== 0) {
|
||||
console.error("\nverify:skills — `claude plugin validate` failed.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
main();
|
||||
}
|
||||
Loading…
Reference in a new issue