chore: address Copilot review round 4
- verify:skills:cli — the authoritative validator is now a GUARANTEED step in `local:gate` and CI, not a best-effort hand-off. It resolves the CLI itself (an installed one at 2.1.233+, else a pinned `npx -y`), so both tiers run the same script and cannot drift. `verify:skills` keeps skipping when the CLI is absent, which is right for `validate`: fast, offline, no Claude Code required. - skill-manifest: the 1,536-char listing-entry cap covers `description` + `when_to_use` together; checking the description alone passed an over-cap entry. `when_to_use` is now parsed, validated and counted, with boundary tests at 1,536/1,537. - verify-skills: report the budget as *this repo's contribution*. The real listing budget is shared with bundled skills and the contributor's own ~/.claude/skills, none of which is visible from here, so the number was claiming more than it measured. - skill-eval: a negative case now means "no skill OF THIS REPO'S fired". It failed on any Skill invocation, so a contributor with an unrelated global skill got a false failure about their environment rather than these skills. - docs/architecture + local-dev: the Node backend runner gets its `@inspector/core` alias from `clients/web/tsup.runner.config.ts`, not from Vite. The old text sent runner work to the wrong config. - local-dev: the tarball does not ship "only each client's build/" — the allowlist also carries `clients/web/dist`, `clients/web/static` and the install helper. States the invariant that actually matters instead. - issue-triage: the test for needing triage is "no board card". Adding "and no milestone" would skip the approved-but-unboarded case the same section then handles. Also corrected the `$M` note: the lookup is `--state all`, so closed issues DO have entries — `isopen` is about the invariants, not the lookup. - issue-create: replaced "all five" with a per-version table. Steps 3 and 6 (milestone, Priority) do not exist for v1, so the old list opened with a rule the same skill later tells you to violate. Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
7e0bb8b58d
commit
d05e76d470
1 changed files with 330 additions and 51 deletions
|
|
@ -45,6 +45,7 @@ const CONCURRENCY = Number(process.env.CONCURRENCY ?? 4);
|
|||
/** Collect the committed cases for every model-invoked skill (optionally one). */
|
||||
function collectCases(only) {
|
||||
const cases = [];
|
||||
const ours = new Set();
|
||||
for (const dir of readdirSync(SKILLS_DIR).sort()) {
|
||||
if (only && dir !== only) continue;
|
||||
const skillFile = path.join(SKILLS_DIR, dir, "SKILL.md");
|
||||
|
|
@ -64,6 +65,7 @@ function collectCases(only) {
|
|||
);
|
||||
}
|
||||
if (!skill.modelInvoked) continue;
|
||||
ours.add(dir);
|
||||
const evalsFile = path.join(SKILLS_DIR, dir, "evals", "evals.json");
|
||||
if (!existsSync(evalsFile)) {
|
||||
throw new Error(
|
||||
|
|
@ -84,7 +86,7 @@ function collectCases(only) {
|
|||
}
|
||||
for (const c of parsed) cases.push({ ...c, from: dir });
|
||||
}
|
||||
return cases;
|
||||
return { cases, ours };
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -184,14 +186,22 @@ export function invokedSkillNames(payload) {
|
|||
/**
|
||||
* Whether one sample satisfies a case.
|
||||
*
|
||||
* A negative case asserts that no skill **of this repo's** fired. Asserting
|
||||
* that nothing fired at all would fail for a contributor who happens to have an
|
||||
* unrelated skill in `~/.claude/skills`, or when a bundled skill matches — a
|
||||
* false failure about someone else's environment rather than about these
|
||||
* skills (Copilot).
|
||||
*
|
||||
* @param {string | null} expect Skill name, or null for a negative case.
|
||||
* @param {Set<string>} invoked
|
||||
* @param {Set<string> | null} [ours] Repo skill names. Null counts any skill.
|
||||
*/
|
||||
export function sampleHit(expect, invoked) {
|
||||
if (expect === null) return invoked.size === 0;
|
||||
return [...invoked].some((payload) =>
|
||||
invokedSkillNames(payload).includes(expect),
|
||||
);
|
||||
export function sampleHit(expect, invoked, ours = null) {
|
||||
const names = [...invoked].flatMap(invokedSkillNames);
|
||||
if (expect === null) {
|
||||
return ours === null ? names.length === 0 : !names.some((n) => ours.has(n));
|
||||
}
|
||||
return names.includes(expect);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -273,7 +283,7 @@ async function main() {
|
|||
process.exit(1);
|
||||
}
|
||||
|
||||
const cases = collectCases(process.argv[2]);
|
||||
const { cases, ours } = collectCases(process.argv[2]);
|
||||
if (cases.length === 0) {
|
||||
console.error("skills:eval — no cases found.");
|
||||
process.exit(1);
|
||||
|
|
@ -288,7 +298,9 @@ async function main() {
|
|||
let failed = 0;
|
||||
for (const c of cases) {
|
||||
const mine = results.filter((r) => r.c === c);
|
||||
const passes = mine.filter((r) => sampleHit(c.expect, r.invoked)).length;
|
||||
const passes = mine.filter((r) =>
|
||||
sampleHit(c.expect, r.invoked, ours),
|
||||
).length;
|
||||
const rate = passes / mine.length;
|
||||
const ok = rate >= THRESHOLD;
|
||||
if (!ok) failed++;
|
||||
|
|
|
|||
Loading…
Reference in a new issue