chore: extract AGENTS.md recipes into agent skills
Closes #2163 `AGENTS.md` was one 1,223-line contract loaded in full on every turn, whether the task was a one-line CSS fix or a release cut. This splits it: rules stay resident, procedures move to `.claude/skills/`. - Ten skills under `.claude/skills/`, seven `disable-model-invocation: true` (invoked by name only, so they cost no listing context and have no trigger question) and three model-invoked with committed eval cases. - `AGENTS.md` 1,223 → 380 lines: rules, a skills index, and a project tree reduced to one line per entry. The removed recipes are not duplicated. - `README.md` 811 → 105 lines, split into `docs/{architecture,quality-gate, test-servers,publishing,docker}.md`, and dropped from `CLAUDE.md`'s auto-load. - `.github/copilot-instructions.md` and its mirroring rule removed — Copilot reads `AGENTS.md`. - `npm run verify:skills` (in `validate`, so in `local:gate` and CI) parses each SKILL.md the way Claude Code does, requires an explicit invocation mode and eval cases for model-invoked skills, and budgets the listing. - `npm run skills:eval` runs the committed trigger cases headless. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall
parent
8c91ae8820
commit
73447c72d1
2 changed files with 3291 additions and 1927 deletions
205
scripts/lib/skill-manifest.mjs
Normal file
205
scripts/lib/skill-manifest.mjs
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
// Pure parsers for the `.claude/skills/**/SKILL.md` contract, extracted so
|
||||
// `verify-skills.mjs` can be unit-tested without a filesystem (#2163).
|
||||
//
|
||||
// The failure this guards is silent by construction: Claude Code reads a
|
||||
// SKILL.md's frontmatter only when the opening `---` is the file's FIRST line
|
||||
// and the YAML between the fences parses. Otherwise the body loads with EMPTY
|
||||
// metadata — `/skill-name` still works, so a manual spot check passes, but
|
||||
// there is no `description` left for the model to match against and the skill
|
||||
// never auto-fires again. A stray unquoted colon in a description is enough.
|
||||
//
|
||||
// So every check here is deny-by-default: a file we cannot read the way Claude
|
||||
// Code reads it is an error, not a skip.
|
||||
|
||||
import { parse as parseYaml } from "yaml";
|
||||
|
||||
/** Combined per-entry cap Claude Code applies to a skill listing entry. */
|
||||
export const DESCRIPTION_CAP = 1536;
|
||||
|
||||
/**
|
||||
* Character budget for the skill LISTING — the sum of name + description over
|
||||
* the skills that occupy it. Only model-invoked skills do: a skill with
|
||||
* `disable-model-invocation: true` is reachable solely by name, so its
|
||||
* description never enters context.
|
||||
*
|
||||
* The listing is truncated when it overflows the model's budget, and the
|
||||
* entries dropped first are the ones invoked least — i.e. exactly the
|
||||
* model-invoked skills that must fire on their own. Recording the number here
|
||||
* is what makes a regression visible when a skill is added later.
|
||||
*/
|
||||
export const LISTING_BUDGET = 4000;
|
||||
|
||||
const FRONTMATTER_FENCE = "---";
|
||||
|
||||
/**
|
||||
* Split a SKILL.md into its raw frontmatter and body the way Claude Code does.
|
||||
* Returns `{ error }` rather than throwing, so the caller can report every
|
||||
* offending file in one pass instead of dying on the first.
|
||||
*
|
||||
* @param {string} text Full file contents.
|
||||
* @returns {{ frontmatter?: string, body?: string, error?: string }}
|
||||
*/
|
||||
export function splitFrontmatter(text) {
|
||||
// A BOM or a leading blank line means the fence is not the first line, and
|
||||
// Claude Code then treats the WHOLE file — `---` markers included — as body.
|
||||
if (!text.startsWith(FRONTMATTER_FENCE + "\n")) {
|
||||
return {
|
||||
error:
|
||||
"frontmatter must open with `---` on the very first line (no BOM, no leading blank line)",
|
||||
};
|
||||
}
|
||||
const rest = text.slice(FRONTMATTER_FENCE.length + 1);
|
||||
const end = rest.indexOf("\n" + FRONTMATTER_FENCE);
|
||||
if (end === -1)
|
||||
return { error: "frontmatter is never closed by a `---` line" };
|
||||
return {
|
||||
frontmatter: rest.slice(0, end),
|
||||
body: rest.slice(end + FRONTMATTER_FENCE.length + 2),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and validate one skill's frontmatter.
|
||||
*
|
||||
* @param {string} dirName Directory the SKILL.md lives in — the skill's identity.
|
||||
* @param {string} text Full file contents.
|
||||
* @returns {{ name?: string, description?: string, modelInvoked?: boolean,
|
||||
* userInvocable?: boolean, paths?: string[], errors: string[] }}
|
||||
*/
|
||||
export function parseSkill(dirName, text) {
|
||||
const errors = [];
|
||||
const { frontmatter, error } = splitFrontmatter(text);
|
||||
if (error) return { errors: [error] };
|
||||
|
||||
let meta;
|
||||
try {
|
||||
meta = parseYaml(frontmatter);
|
||||
} catch (e) {
|
||||
// The whole point of the guard: this is what strips the description.
|
||||
return {
|
||||
errors: [`frontmatter is not valid YAML — ${e.message.split("\n")[0]}`],
|
||||
};
|
||||
}
|
||||
if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
|
||||
return { errors: ["frontmatter must parse to a mapping"] };
|
||||
}
|
||||
|
||||
const name = meta.name;
|
||||
if (typeof name !== "string" || name.length === 0) {
|
||||
errors.push("`name` is required and must be a string");
|
||||
} else if (name !== dirName) {
|
||||
errors.push(
|
||||
`\`name: ${name}\` does not match its directory \`${dirName}\``,
|
||||
);
|
||||
} else if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) {
|
||||
errors.push(`\`name: ${name}\` must be kebab-case`);
|
||||
}
|
||||
|
||||
const description = meta.description;
|
||||
if (typeof description !== "string" || description.trim().length === 0) {
|
||||
errors.push("`description` is required and must be a non-empty string");
|
||||
} else if (description.length > DESCRIPTION_CAP) {
|
||||
errors.push(
|
||||
`\`description\` is ${description.length} chars; Claude Code caps a listing entry at ${DESCRIPTION_CAP}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Every skill states its invocation mode rather than inheriting a default,
|
||||
// so the eval surface is knowable by reading the files.
|
||||
if (!("disable-model-invocation" in meta)) {
|
||||
errors.push(
|
||||
"`disable-model-invocation` must be declared explicitly (true = invoked by name only, false = the model may fire it)",
|
||||
);
|
||||
}
|
||||
const dmi = meta["disable-model-invocation"];
|
||||
if ("disable-model-invocation" in meta && typeof dmi !== "boolean") {
|
||||
errors.push("`disable-model-invocation` must be a boolean");
|
||||
}
|
||||
const modelInvoked = dmi === false;
|
||||
|
||||
if ("user-invocable" in meta && typeof meta["user-invocable"] !== "boolean") {
|
||||
errors.push("`user-invocable` must be a boolean");
|
||||
}
|
||||
if ("paths" in meta) {
|
||||
if (
|
||||
!Array.isArray(meta.paths) ||
|
||||
meta.paths.some((p) => typeof p !== "string")
|
||||
) {
|
||||
errors.push("`paths` must be a list of glob strings");
|
||||
}
|
||||
}
|
||||
// A skill nothing can reach is dead weight that still costs a directory.
|
||||
if (!modelInvoked && meta["user-invocable"] === false) {
|
||||
errors.push(
|
||||
"skill is unreachable: `disable-model-invocation: true` with `user-invocable: false`",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
name: typeof name === "string" ? name : undefined,
|
||||
description: typeof description === "string" ? description : undefined,
|
||||
modelInvoked,
|
||||
userInvocable: meta["user-invocable"] !== false,
|
||||
paths: Array.isArray(meta.paths) ? meta.paths : undefined,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cost of the skill listing Claude Code loads into context, counting only the
|
||||
* skills that appear in it.
|
||||
*
|
||||
* @param {Array<{ name?: string, description?: string, modelInvoked?: boolean }>} skills
|
||||
*/
|
||||
export function listingCost(skills) {
|
||||
return skills
|
||||
.filter((s) => s.modelInvoked)
|
||||
.reduce(
|
||||
(n, s) => n + (s.name?.length ?? 0) + (s.description?.length ?? 0),
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an `evals/evals.json` payload for a model-invoked skill.
|
||||
*
|
||||
* A skill that fires on everything is a context regression, not a win, and it
|
||||
* is the failure nobody notices by hand — so negatives are required, not
|
||||
* optional.
|
||||
*
|
||||
* @param {string} skillName
|
||||
* @param {unknown} cases Parsed JSON.
|
||||
* @returns {string[]} errors
|
||||
*/
|
||||
export function validateEvalCases(skillName, cases) {
|
||||
if (!Array.isArray(cases) || cases.length === 0) {
|
||||
return ["evals.json must be a non-empty array of cases"];
|
||||
}
|
||||
const errors = [];
|
||||
cases.forEach((c, i) => {
|
||||
if (c === null || typeof c !== "object" || Array.isArray(c)) {
|
||||
errors.push(`case ${i}: must be an object`);
|
||||
return;
|
||||
}
|
||||
if (typeof c.prompt !== "string" || c.prompt.trim() === "") {
|
||||
errors.push(`case ${i}: \`prompt\` must be a non-empty string`);
|
||||
}
|
||||
if (
|
||||
!("expect" in c) ||
|
||||
(c.expect !== null && typeof c.expect !== "string")
|
||||
) {
|
||||
errors.push(`case ${i}: \`expect\` must be a skill name or null`);
|
||||
}
|
||||
});
|
||||
const positives = cases.filter((c) => c && c.expect === skillName).length;
|
||||
const negatives = cases.filter((c) => c && c.expect === null).length;
|
||||
if (positives === 0) {
|
||||
errors.push(`no positive case expects \`${skillName}\``);
|
||||
}
|
||||
if (negatives === 0) {
|
||||
errors.push(
|
||||
"no negative case (`expect: null`) — a skill that fires on everything is a regression",
|
||||
);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
181
scripts/lib/skill-manifest.test.mjs
Normal file
181
scripts/lib/skill-manifest.test.mjs
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
// Table-driven tests for the `.claude/skills` frontmatter contract (#2163),
|
||||
// one case per rule the parsers encode. The guard that reads them
|
||||
// (`verify-skills.mjs`) walks the real skills directory, so a rule it enforces
|
||||
// is only observable there when a skill happens to violate it — which, by
|
||||
// construction, none does once the guard is green.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
splitFrontmatter,
|
||||
parseSkill,
|
||||
validateEvalCases,
|
||||
listingCost,
|
||||
} from "./skill-manifest.mjs";
|
||||
|
||||
const fm = (body) => `---\n${body}\n---\n\n# Body\n`;
|
||||
|
||||
test("splitFrontmatter requires the fence on the very first line", () => {
|
||||
// This is the whole failure class: anything before the fence and Claude Code
|
||||
// treats the WHOLE file, `---` markers included, as body content.
|
||||
for (const text of [
|
||||
"\n---\nname: x\n---\n",
|
||||
"---\nname: x\n---\n",
|
||||
"# Title\n---\nname: x\n---\n",
|
||||
]) {
|
||||
assert.match(splitFrontmatter(text).error ?? "", /first line/);
|
||||
}
|
||||
assert.equal(splitFrontmatter(fm("name: x")).error, undefined);
|
||||
});
|
||||
|
||||
test("splitFrontmatter reports an unterminated block", () => {
|
||||
assert.match(splitFrontmatter("---\nname: x\n").error ?? "", /never closed/);
|
||||
});
|
||||
|
||||
test("parseSkill accepts a well-formed skill", () => {
|
||||
const r = parseSkill(
|
||||
"local-dev",
|
||||
fm(
|
||||
"name: local-dev\ndescription: Does a thing.\ndisable-model-invocation: false",
|
||||
),
|
||||
);
|
||||
assert.deepEqual(r.errors, []);
|
||||
assert.equal(r.modelInvoked, true);
|
||||
assert.equal(r.userInvocable, true);
|
||||
});
|
||||
|
||||
test("parseSkill rejects frontmatter that is not valid YAML", () => {
|
||||
// An unquoted colon in a description is enough, and this is exactly what
|
||||
// silently strips the description Claude matches against.
|
||||
const r = parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: Use this: always\ndisable-model-invocation: true",
|
||||
),
|
||||
);
|
||||
assert.equal(r.errors.length, 1);
|
||||
assert.match(r.errors[0], /not valid YAML/);
|
||||
});
|
||||
|
||||
test("parseSkill requires name to match the directory, kebab-case", () => {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"board-ops",
|
||||
fm("name: boardOps\ndescription: d\ndisable-model-invocation: true"),
|
||||
).errors.join(),
|
||||
/does not match its directory/,
|
||||
);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"Board_Ops",
|
||||
fm("name: Board_Ops\ndescription: d\ndisable-model-invocation: true"),
|
||||
).errors.join(),
|
||||
/kebab-case/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill requires a non-empty description", () => {
|
||||
for (const d of ["", " "]) {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(`name: x\ndescription: "${d}"\ndisable-model-invocation: true`),
|
||||
).errors.join(),
|
||||
/description. is required/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("parseSkill caps the description at the listing-entry limit", () => {
|
||||
const long = "a".repeat(1537);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(`name: x\ndescription: ${long}\ndisable-model-invocation: true`),
|
||||
).errors.join(),
|
||||
/caps a listing entry/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill requires the invocation mode to be declared", () => {
|
||||
assert.match(
|
||||
parseSkill("x", fm("name: x\ndescription: d")).errors.join(),
|
||||
/must be declared explicitly/,
|
||||
);
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm("name: x\ndescription: d\ndisable-model-invocation: yes-please"),
|
||||
).errors.join(),
|
||||
/must be a boolean/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseSkill rejects a skill nothing can reach", () => {
|
||||
const r = parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: true\nuser-invocable: false",
|
||||
),
|
||||
);
|
||||
assert.match(r.errors.join(), /unreachable/);
|
||||
});
|
||||
|
||||
test("parseSkill validates the paths list", () => {
|
||||
assert.match(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths: '**/*.ts'",
|
||||
),
|
||||
).errors.join(),
|
||||
/list of glob strings/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseSkill(
|
||||
"x",
|
||||
fm(
|
||||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths:\n - '**/*.ts'",
|
||||
),
|
||||
).paths,
|
||||
["**/*.ts"],
|
||||
);
|
||||
});
|
||||
|
||||
test("listingCost counts only the skills that occupy the listing", () => {
|
||||
const skills = [
|
||||
{ name: "aa", description: "bbb", modelInvoked: true },
|
||||
{ name: "cccc", description: "ddddd", modelInvoked: false },
|
||||
];
|
||||
assert.equal(listingCost(skills), 5);
|
||||
});
|
||||
|
||||
test("validateEvalCases requires a positive and a negative", () => {
|
||||
assert.deepEqual(
|
||||
validateEvalCases("testing", [
|
||||
{ prompt: "a", expect: "testing" },
|
||||
{ prompt: "b", expect: null },
|
||||
]),
|
||||
[],
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("testing", [{ prompt: "a", expect: "testing" }]).join(),
|
||||
/no negative case/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("testing", [{ prompt: "a", expect: null }]).join(),
|
||||
/no positive case/,
|
||||
);
|
||||
});
|
||||
|
||||
test("validateEvalCases rejects malformed cases", () => {
|
||||
assert.match(validateEvalCases("x", []).join(), /non-empty array/);
|
||||
assert.match(
|
||||
validateEvalCases("x", [{ expect: null }]).join(),
|
||||
/prompt.*non-empty string/,
|
||||
);
|
||||
assert.match(
|
||||
validateEvalCases("x", [{ prompt: "a", expect: 7 }]).join(),
|
||||
/expect. must be a skill name or null/,
|
||||
);
|
||||
});
|
||||
Loading…
Reference in a new issue