`skills:eval` ran every case with `--max-turns 1`, so it measured one
thing: whether a skill is the model's FIRST tool call. That is right for a
skill a user reaches directly, and it leaves a whole class unmeasured — a
skill reached from inside another skill's body. `testing` tells the model
that picking a fixture is `/test-servers` and that it has to load it, and
`test-servers` scores 5/5 while every one of those five cases asks for it
by name. A skill only ever reachable through a hand-off would score a
clean 100% with the hand-off silently never firing.
An eval case is now one of two shapes, and exactly one:
{ "prompt": "…", "expect": "test-servers" } first move, 1 turn
{ "prompt": "…", "chain": ["testing", "test-servers"] } hand-off, 14 turns
A chain must END with the skill whose file it lives in. The case exists to
measure whether THIS skill is reachable, so the file that goes red is the
one belonging to the skill that stopped being reached; anchoring on the
first link would file a `test-servers` measurement under `testing`, where
a `test-servers` description edit would never be seen. Links are checked
against the repo's model-invoked set, which is why both `verify:skills`
and `collectCases` now validate in a second pass — `test-servers` sorts
before `testing`, so a per-directory check would reject a live chain as
unknown purely because of where the alphabet put it.
Scoring is an ordered SUBSEQUENCE, not a prefix and not a contiguous run:
the model may load something before the chain starts and something
unrelated in between, and neither changes the claim that A led to B.
`collectSkillInvocations` therefore returns an ordered array rather than a
Set — a B, A, B run has to stay distinguishable from one that never
reached B from A.
The two rates are reported in separate columns and never summed. A
hand-off is a second-hop load over many turns; a first-move rate is the
model's opening move. `CHAIN_MAX_TURNS` (14) and `CHAIN_THRESHOLD` are
their own knobs for the same reason, and `CHAIN_THRESHOLD` defaults to
**0.5** rather than inheriting 0.8: see the measurement below.
Read-only containment no longer leans on `--max-turns 1`, which was doing
much of it by itself. The deny list gains the agentic and network tools —
`Task` in particular, whose subagent the flag does not reach.
Measured, `skills:eval -- test-servers`:
First move (1 turn) 7/7 at 100%, unchanged
Hand-off (14 turns) 67% and 33% at RUNS=3
So the hand-off is real and unreliable — which is the fact nothing could
observe before this change, and the reason 0.5 rather than 0.8 is the
default bar: at 0.8 both committed cases are red no matter how strongly
the first skill points at the second, and the column stops carrying
signal. A `["pr-flow", "test-servers"]` probe measured 0% and was dropped
rather than kept: `pr-flow` says nothing about fixtures, so the case had
no lever short of broadening a description onto another skill's ground.
Both findings are written up in `docs/skill-authoring.md`.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ABdTnqyBodw6mHVVkGehqe
Signed-off-by: cliffhall <cliff@futurescale.com>
817 lines
26 KiB
JavaScript
817 lines
26 KiB
JavaScript
// Table-driven tests for the `.claude/skills` frontmatter contract (#2163),
|
||
// one case per rule the parsers encode. The guard that reads them
|
||
// (`verify-skills.mjs`) walks the real skills directory, so a rule it enforces
|
||
// is only observable there when a skill happens to violate it — which, by
|
||
// construction, none does once the guard is green.
|
||
|
||
import { test } from "node:test";
|
||
import assert from "node:assert/strict";
|
||
import {
|
||
splitFrontmatter,
|
||
parseSkill,
|
||
isChainCase,
|
||
MIN_CHAIN_LENGTH,
|
||
MIN_POSITIVE_CASES,
|
||
validateEvalCases,
|
||
listingCost,
|
||
parseClaudeVersion,
|
||
compareVersions,
|
||
PINNED_CLI_VERSION,
|
||
isPinnedVersion,
|
||
formatClaudeVersion,
|
||
} from "./skill-manifest.mjs";
|
||
|
||
const fm = (body) => `---\n${body}\n---\n\n# Body\n`;
|
||
|
||
test("splitFrontmatter requires the fence on the very first line", () => {
|
||
// This is the whole failure class: anything before the fence and Claude Code
|
||
// treats the WHOLE file, `---` markers included, as body content.
|
||
for (const text of [
|
||
"\n---\nname: x\n---\n",
|
||
"---\nname: x\n---\n",
|
||
"# Title\n---\nname: x\n---\n",
|
||
]) {
|
||
assert.match(splitFrontmatter(text).error ?? "", /first line/);
|
||
}
|
||
assert.equal(splitFrontmatter(fm("name: x")).error, undefined);
|
||
});
|
||
|
||
test("splitFrontmatter requires the closing fence to be a whole line", () => {
|
||
// `---oops` is not a terminator. Accepting it would silently truncate the
|
||
// frontmatter to whatever preceded it and report the file as valid.
|
||
assert.match(
|
||
splitFrontmatter("---\nname: x\n---oops\nbody\n").error ?? "",
|
||
/never closed/,
|
||
);
|
||
// Trailing whitespace on the fence is still a fence, and so is EOF.
|
||
assert.equal(
|
||
splitFrontmatter("---\nname: x\n--- \nbody\n").frontmatter,
|
||
"name: x",
|
||
);
|
||
assert.equal(splitFrontmatter("---\nname: x\n---").frontmatter, "name: x");
|
||
assert.equal(
|
||
splitFrontmatter("---\nname: x\n---\r\nbody").frontmatter,
|
||
"name: x",
|
||
);
|
||
});
|
||
|
||
test("splitFrontmatter reports an unterminated block", () => {
|
||
assert.match(splitFrontmatter("---\nname: x\n").error ?? "", /never closed/);
|
||
});
|
||
|
||
test("parseSkill accepts a well-formed skill", () => {
|
||
const r = parseSkill(
|
||
"local-dev",
|
||
fm(
|
||
"name: local-dev\ndescription: Does a thing.\ndisable-model-invocation: false",
|
||
),
|
||
);
|
||
assert.deepEqual(r.errors, []);
|
||
assert.equal(r.modelInvoked, true);
|
||
assert.equal(r.userInvocable, true);
|
||
});
|
||
|
||
test("parseSkill rejects frontmatter that is not valid YAML", () => {
|
||
// An unquoted colon in a description is enough, and this is exactly what
|
||
// silently strips the description Claude matches against.
|
||
const r = parseSkill(
|
||
"x",
|
||
fm(
|
||
"name: x\ndescription: Use this: always\ndisable-model-invocation: true",
|
||
),
|
||
);
|
||
assert.equal(r.errors.length, 1);
|
||
assert.match(r.errors[0], /not valid YAML/);
|
||
});
|
||
|
||
test("parseSkill requires name to match the directory, kebab-case", () => {
|
||
assert.match(
|
||
parseSkill(
|
||
"board-ops",
|
||
fm("name: boardOps\ndescription: d\ndisable-model-invocation: true"),
|
||
).errors.join(),
|
||
/does not match its directory/,
|
||
);
|
||
assert.match(
|
||
parseSkill(
|
||
"Board_Ops",
|
||
fm("name: Board_Ops\ndescription: d\ndisable-model-invocation: true"),
|
||
).errors.join(),
|
||
/kebab-case/,
|
||
);
|
||
});
|
||
|
||
test("parseSkill requires a non-empty description", () => {
|
||
for (const d of ["", " "]) {
|
||
assert.match(
|
||
parseSkill(
|
||
"x",
|
||
fm(`name: x\ndescription: "${d}"\ndisable-model-invocation: true`),
|
||
).errors.join(),
|
||
/description. is required/,
|
||
);
|
||
}
|
||
});
|
||
|
||
test("parseSkill caps the description at the listing-entry limit", () => {
|
||
const long = "a".repeat(1537);
|
||
assert.match(
|
||
parseSkill(
|
||
"x",
|
||
fm(`name: x\ndescription: ${long}\ndisable-model-invocation: true`),
|
||
).errors.join(),
|
||
/caps a listing entry/,
|
||
);
|
||
});
|
||
|
||
test("parseSkill requires the invocation mode to be declared", () => {
|
||
assert.match(
|
||
parseSkill("x", fm("name: x\ndescription: d")).errors.join(),
|
||
/must be declared explicitly/,
|
||
);
|
||
assert.match(
|
||
parseSkill(
|
||
"x",
|
||
fm("name: x\ndescription: d\ndisable-model-invocation: yes-please"),
|
||
).errors.join(),
|
||
/must be a boolean/,
|
||
);
|
||
});
|
||
|
||
test("parseSkill rejects a skill nothing can reach", () => {
|
||
const r = parseSkill(
|
||
"x",
|
||
fm(
|
||
"name: x\ndescription: d\ndisable-model-invocation: true\nuser-invocable: false",
|
||
),
|
||
);
|
||
assert.match(r.errors.join(), /unreachable/);
|
||
});
|
||
|
||
test("parseSkill validates the paths list", () => {
|
||
assert.match(
|
||
parseSkill(
|
||
"x",
|
||
fm(
|
||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths: '**/*.ts'",
|
||
),
|
||
).errors.join(),
|
||
/list of glob strings/,
|
||
);
|
||
assert.deepEqual(
|
||
parseSkill(
|
||
"x",
|
||
fm(
|
||
"name: x\ndescription: d\ndisable-model-invocation: false\npaths:\n - '**/*.ts'",
|
||
),
|
||
).paths,
|
||
["**/*.ts"],
|
||
);
|
||
});
|
||
|
||
test("listingCost counts only the skills that occupy the listing", () => {
|
||
const skills = [
|
||
{ name: "aa", description: "bbb", modelInvoked: true },
|
||
{ name: "cccc", description: "ddddd", modelInvoked: false },
|
||
];
|
||
assert.equal(listingCost(skills), 5);
|
||
});
|
||
|
||
test("validateEvalCases requires a positive and a negative", () => {
|
||
const positives = (n, expect = "testing") =>
|
||
Array.from({ length: n }, (_, i) => ({ prompt: `p${i}`, expect }));
|
||
|
||
assert.deepEqual(
|
||
validateEvalCases("testing", [
|
||
...positives(MIN_POSITIVE_CASES),
|
||
{ prompt: "b", expect: null },
|
||
]),
|
||
[],
|
||
);
|
||
assert.match(
|
||
validateEvalCases("testing", positives(MIN_POSITIVE_CASES)).join(),
|
||
/no negative case/,
|
||
);
|
||
assert.match(
|
||
validateEvalCases("testing", [{ prompt: "a", expect: null }]).join(),
|
||
/no positive case/,
|
||
);
|
||
});
|
||
|
||
test("validateEvalCases rejects malformed cases", () => {
|
||
assert.match(validateEvalCases("x", []).join(), /non-empty array/);
|
||
assert.match(
|
||
validateEvalCases("x", [{ expect: null }]).join(),
|
||
/prompt.*non-empty string/,
|
||
);
|
||
assert.match(
|
||
validateEvalCases("x", [{ prompt: "a", expect: 7 }]).join(),
|
||
/needs .expect./,
|
||
);
|
||
assert.match(
|
||
validateEvalCases("x", [{ prompt: "a" }]).join(),
|
||
/needs .expect. .* or .chain./,
|
||
);
|
||
});
|
||
|
||
test("a hand-off case names an ordered chain ending in this skill", () => {
|
||
const base = [
|
||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||
prompt: `p${i}`,
|
||
expect: "test-servers",
|
||
})),
|
||
{ prompt: "n", expect: null },
|
||
];
|
||
const withChain = (chain) => [...base, { prompt: "c", chain }];
|
||
const known = new Set(["testing", "test-servers", "pr-flow"]);
|
||
|
||
assert.deepEqual(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["testing", "test-servers"]),
|
||
known,
|
||
),
|
||
[],
|
||
);
|
||
// The case belongs to the skill that must be REACHED, so a chain anchored on
|
||
// its first link would file a `test-servers` measurement under `testing` and
|
||
// leave a `test-servers` description edit unmeasured by its own file.
|
||
assert.match(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["test-servers", "testing"]),
|
||
known,
|
||
).join(),
|
||
/ends with .testing., but this file measures .* .test-servers./,
|
||
);
|
||
assert.match(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["test-servers"]),
|
||
known,
|
||
).join(),
|
||
new RegExp(`at least ${MIN_CHAIN_LENGTH} skill names`),
|
||
);
|
||
assert.match(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["testing", "testing", "test-servers"]),
|
||
known,
|
||
).join(),
|
||
/repeats a skill name/,
|
||
);
|
||
assert.match(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["nope", "test-servers"]),
|
||
known,
|
||
).join(),
|
||
/.nope., which is not a model-invoked skill/,
|
||
);
|
||
assert.match(
|
||
validateEvalCases(
|
||
"test-servers",
|
||
withChain(["", "test-servers"]),
|
||
known,
|
||
).join(),
|
||
/non-empty string/,
|
||
);
|
||
// Without the known set the shape is still checked; only the link names go
|
||
// unverified, so a caller that has not parsed the directory can still run.
|
||
assert.deepEqual(
|
||
validateEvalCases("test-servers", withChain(["nope", "test-servers"])),
|
||
[],
|
||
);
|
||
});
|
||
|
||
test("a hand-off case satisfies neither floor and never doubles as a first move", () => {
|
||
// A hand-off is a second-hop load over many turns and is scored in its own
|
||
// column. Letting one stand in for a first-move positive would let a skill
|
||
// ship with no measurement of the way users actually reach it.
|
||
const chained = Array.from({ length: MIN_POSITIVE_CASES + 1 }, (_, i) => ({
|
||
prompt: `c${i}`,
|
||
chain: ["testing", "test-servers"],
|
||
}));
|
||
const errors = validateEvalCases("test-servers", chained).join(" ");
|
||
assert.match(errors, /no positive case/);
|
||
assert.match(errors, /no negative case/);
|
||
});
|
||
|
||
test("a case carrying both shapes is a typo, not a half-scored measurement", () => {
|
||
assert.match(
|
||
validateEvalCases("test-servers", [
|
||
{
|
||
prompt: "c",
|
||
expect: "test-servers",
|
||
chain: ["testing", "test-servers"],
|
||
},
|
||
]).join(),
|
||
/carries both .expect. and .chain./,
|
||
);
|
||
});
|
||
|
||
test("isChainCase tells the two shapes apart", () => {
|
||
assert.equal(isChainCase({ chain: ["a", "b"] }), true);
|
||
assert.equal(isChainCase({ expect: "a" }), false);
|
||
assert.equal(isChainCase({ expect: null }), false);
|
||
assert.equal(isChainCase(null), false);
|
||
assert.equal(isChainCase("chain"), false);
|
||
});
|
||
|
||
test("parseClaudeVersion reads the CLI's version banner", () => {
|
||
assert.deepEqual(parseClaudeVersion("2.1.250 (Claude Code)"), {
|
||
parts: [2, 1, 250],
|
||
prerelease: null,
|
||
});
|
||
assert.deepEqual(parseClaudeVersion("claude 10.0.4\n"), {
|
||
parts: [10, 0, 4],
|
||
prerelease: null,
|
||
});
|
||
assert.equal(parseClaudeVersion("no version here"), null);
|
||
});
|
||
|
||
test("parseClaudeVersion keeps a prerelease suffix rather than dropping it", () => {
|
||
// Dropping it made `2.1.250-beta.1` compare EQUAL to the stable pin — a
|
||
// different validator schema running while the log claimed an exact match.
|
||
assert.deepEqual(parseClaudeVersion("2.1.250-beta.1 (Claude Code)"), {
|
||
parts: [2, 1, 250],
|
||
prerelease: "beta.1",
|
||
});
|
||
assert.equal(
|
||
formatClaudeVersion(parseClaudeVersion("2.1.250-beta.1")),
|
||
"2.1.250-beta.1",
|
||
);
|
||
assert.equal(formatClaudeVersion(null), "(unreadable version)");
|
||
});
|
||
|
||
test("isPinnedVersion accepts only the exact stable pin", () => {
|
||
const pin = PINNED_CLI_VERSION;
|
||
assert.equal(isPinnedVersion(parseClaudeVersion(pin), pin), true);
|
||
// Build metadata is ignored for precedence by SemVer, so it still matches.
|
||
assert.equal(
|
||
isPinnedVersion(parseClaudeVersion(`${pin}+build.7`), pin),
|
||
true,
|
||
);
|
||
// A prerelease of the same triple is NOT the pin.
|
||
assert.equal(
|
||
isPinnedVersion(parseClaudeVersion(`${pin}-beta.1`), pin),
|
||
false,
|
||
);
|
||
assert.equal(isPinnedVersion(parseClaudeVersion("2.1.249"), pin), false);
|
||
assert.equal(isPinnedVersion(parseClaudeVersion("99.0.0"), pin), false);
|
||
assert.equal(isPinnedVersion(null, pin), false);
|
||
});
|
||
|
||
test("the listing-entry cap covers description AND when_to_use together", () => {
|
||
// The cap is over the combined entry, so checking `description` alone would
|
||
// pass an over-cap listing entry.
|
||
const under = fm(
|
||
`name: x\ndescription: ${"a".repeat(1000)}\nwhen_to_use: ${"b".repeat(536)}\ndisable-model-invocation: true`,
|
||
);
|
||
assert.deepEqual(parseSkill("x", under).errors, []);
|
||
|
||
const over = fm(
|
||
`name: x\ndescription: ${"a".repeat(1000)}\nwhen_to_use: ${"b".repeat(537)}\ndisable-model-invocation: true`,
|
||
);
|
||
assert.match(parseSkill("x", over).errors.join(), /caps a listing entry/);
|
||
|
||
assert.match(
|
||
parseSkill(
|
||
"x",
|
||
fm(
|
||
"name: x\ndescription: d\nwhen_to_use: 7\ndisable-model-invocation: true",
|
||
),
|
||
).errors.join(),
|
||
/when_to_use. must be a string/,
|
||
);
|
||
});
|
||
|
||
test("listingCost counts when_to_use too", () => {
|
||
assert.equal(
|
||
listingCost([
|
||
{ name: "aa", description: "bbb", whenToUse: "cc", modelInvoked: true },
|
||
]),
|
||
7,
|
||
);
|
||
});
|
||
|
||
test("a CRLF manifest is valid — the fence is still the first line", () => {
|
||
// A Windows checkout with `core.autocrlf` produces these. Rejecting one would
|
||
// fail `npm run validate` on a manifest the authoritative validator accepts,
|
||
// and would fail it *before* that validator ever ran.
|
||
const crlf =
|
||
"---\r\nname: x\r\ndescription: d\r\ndisable-model-invocation: true\r\n---\r\n\r\nBody\r\n";
|
||
assert.equal(
|
||
splitFrontmatter(crlf).frontmatter,
|
||
"name: x\r\ndescription: d\r\ndisable-model-invocation: true",
|
||
);
|
||
assert.deepEqual(parseSkill("x", crlf).errors, []);
|
||
|
||
// Mixed endings are still fine, and so is a CRLF file with no trailing body.
|
||
assert.deepEqual(
|
||
parseSkill(
|
||
"x",
|
||
"---\r\nname: x\ndescription: d\r\ndisable-model-invocation: true\n---\r\n",
|
||
).errors,
|
||
[],
|
||
);
|
||
assert.deepEqual(
|
||
parseSkill(
|
||
"x",
|
||
"---\r\nname: x\r\ndescription: d\r\ndisable-model-invocation: true\r\n---",
|
||
).errors,
|
||
[],
|
||
);
|
||
});
|
||
|
||
test("CRLF does not weaken the first-line rule", () => {
|
||
// The point of the opening check is that anything before the fence turns the
|
||
// whole file into body — accepting CRLF must not start accepting those.
|
||
for (const text of [
|
||
"\r\n---\r\nname: x\r\n---\r\n",
|
||
"---\r\nname: x\r\n---\r\n",
|
||
"# Title\r\n---\r\nname: x\r\n---\r\n",
|
||
"--- \r\nname: x\r\n---\r\n".replace("--- ", "---x"),
|
||
]) {
|
||
assert.match(splitFrontmatter(text).error ?? "", /first line|never closed/);
|
||
}
|
||
// And `---oops` is still not a terminator with CRLF endings either.
|
||
assert.match(
|
||
splitFrontmatter("---\r\nname: x\r\n---oops\r\nbody\r\n").error ?? "",
|
||
/never closed/,
|
||
);
|
||
});
|
||
|
||
test("an eval file may only expect its own skill", () => {
|
||
// A foreign name passes whenever that OTHER skill fires, so the file reports
|
||
// a measurement of something it does not describe — and it satisfies the
|
||
// positive-case requirement while doing it, hiding the absence of a real one.
|
||
assert.match(
|
||
validateEvalCases("testing", [
|
||
{ prompt: "a", expect: "local-dev" },
|
||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||
prompt: `p${i}`,
|
||
expect: "testing",
|
||
})),
|
||
{ prompt: "c", expect: null },
|
||
]).join(),
|
||
/expects `local-dev`, but this file only measures `testing`/,
|
||
);
|
||
// The masking case: a foreign positive is no longer enough on its own.
|
||
const masked = validateEvalCases("testing", [
|
||
{ prompt: "a", expect: "local-dev" },
|
||
{ prompt: "b", expect: null },
|
||
]);
|
||
assert.match(masked.join(), /only measures/);
|
||
assert.match(masked.join(), /no positive case/);
|
||
// Null and own-name cases are unaffected.
|
||
assert.deepEqual(
|
||
validateEvalCases("testing", [
|
||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||
prompt: `p${i}`,
|
||
expect: "testing",
|
||
})),
|
||
{ prompt: "b", expect: null },
|
||
]),
|
||
[],
|
||
);
|
||
});
|
||
|
||
test("an unquoted `#` truncates a description, and that is an error", () => {
|
||
// The dangerous part is that YAML leaves a *non-empty* string behind, so the
|
||
// "description is required and non-empty" check passes and the skill keeps
|
||
// auto-firing — just with its most distinctive words missing from the
|
||
// listing. `board-ops` shipped this way, losing both board numbers.
|
||
const withHash = [
|
||
"---",
|
||
"name: board-ops",
|
||
"description: Recipes. Covers board #28 (v2) and board #11 (v1).",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
const parsed = parseSkill("board-ops", withHash);
|
||
assert.equal(parsed.description, "Recipes. Covers board");
|
||
assert.match(parsed.errors.join(" "), /truncated by an unquoted `#`/);
|
||
|
||
// Quoting is the fix, and it must come back clean.
|
||
const quoted = withHash.replace(
|
||
/^description: (.*)$/m,
|
||
(_m, d) => `description: "${d}"`,
|
||
);
|
||
const ok = parseSkill("board-ops", quoted);
|
||
assert.deepEqual(ok.errors, []);
|
||
assert.equal(
|
||
ok.description,
|
||
"Recipes. Covers board #28 (v2) and board #11 (v1).",
|
||
);
|
||
});
|
||
|
||
test("a `#` inside a quoted description is not treated as truncation", () => {
|
||
// Guard against the check firing on every legitimately quoted description.
|
||
const quoted = [
|
||
"---",
|
||
"name: alpha",
|
||
'description: "counts # and : safely"',
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
assert.deepEqual(parseSkill("alpha", quoted).errors, []);
|
||
});
|
||
|
||
test("a model-invoked skill needs at least MIN_POSITIVE_CASES positives", () => {
|
||
// `positives === 0` alone let a skill regress to a single prompt and still
|
||
// pass the gate. The floor is about breadth, not variance: each prompt is
|
||
// scored on its own `passes / RUNS`, so extra prompts steady nothing — they
|
||
// cover more of the ways someone might reach the skill, so a description that
|
||
// fires on one narrow phrasing cannot pass on that single case alone.
|
||
const withPositives = (n) => [
|
||
...Array.from({ length: n }, (_, i) => ({
|
||
prompt: `p${i}`,
|
||
expect: "alpha",
|
||
})),
|
||
{ prompt: "n", expect: null },
|
||
];
|
||
|
||
assert.match(
|
||
validateEvalCases("alpha", withPositives(1)).join(" "),
|
||
/only 1 distinct positive prompt\(s\)/,
|
||
);
|
||
assert.match(
|
||
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES - 1)).join(" "),
|
||
/needs at least 5 to cover the range/,
|
||
);
|
||
assert.deepEqual(
|
||
validateEvalCases("alpha", withPositives(MIN_POSITIVE_CASES)),
|
||
[],
|
||
);
|
||
|
||
// Zero still reports the clearer "no positive case" message rather than the
|
||
// count one, so the common mistake keeps its specific diagnosis.
|
||
assert.match(
|
||
validateEvalCases("alpha", [{ prompt: "n", expect: null }]).join(" "),
|
||
/no positive case expects/,
|
||
);
|
||
});
|
||
|
||
test("the truncation guard covers `when_to_use`, not just `description`", () => {
|
||
// The guard loops over both fields, so a regression that dropped `when_to_use`
|
||
// from that list would be invisible to the description-only tests above.
|
||
const withHash = [
|
||
"---",
|
||
"name: alpha",
|
||
'description: "safe because quoted"',
|
||
"when_to_use: Use when filing against board #28 or board #11.",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
const parsed = parseSkill("alpha", withHash);
|
||
assert.equal(parsed.whenToUse, "Use when filing against board");
|
||
assert.match(parsed.errors.join(" "), /`when_to_use` is truncated/);
|
||
// The description is quoted, so it must not be implicated.
|
||
assert.doesNotMatch(parsed.errors.join(" "), /`description` is truncated/);
|
||
|
||
const quoted = withHash.replace(
|
||
/^when_to_use: (.*)$/m,
|
||
(_m, w) => `when_to_use: "${w}"`,
|
||
);
|
||
const ok = parseSkill("alpha", quoted);
|
||
assert.deepEqual(ok.errors, []);
|
||
assert.equal(ok.whenToUse, "Use when filing against board #28 or board #11.");
|
||
});
|
||
|
||
test("the truncation guard spans a multiline plain scalar", () => {
|
||
// A plain scalar continues onto more-indented lines, so a first-line-only
|
||
// comparison sees `Covers boards` on both sides and reports no loss — passing
|
||
// the exact truncation this guard exists to reject.
|
||
const multiline = [
|
||
"---",
|
||
"name: alpha",
|
||
"description: Covers boards",
|
||
" #28 and #11 and the hazard.",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
const parsed = parseSkill("alpha", multiline);
|
||
assert.equal(parsed.description, "Covers boards");
|
||
assert.match(parsed.errors.join(" "), /`description` is truncated/);
|
||
|
||
// A multiline plain scalar with no `#` folds to one line and must stay clean,
|
||
// so the fix cannot be "reject anything multiline".
|
||
const folded = [
|
||
"---",
|
||
"name: alpha",
|
||
"description: Covers boards",
|
||
" and the option-deletion hazard.",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
const ok = parseSkill("alpha", folded);
|
||
assert.deepEqual(ok.errors, []);
|
||
assert.equal(ok.description, "Covers boards and the option-deletion hazard.");
|
||
});
|
||
|
||
test("the truncation guard ignores anchors and tags, which are not `#`", () => {
|
||
// YAML strips presentation syntax from the parsed value, so `&anchor` and
|
||
// `!!tag` make the raw scalar legitimately longer than what it parses to.
|
||
// A bare length comparison called those truncation and blamed a `#` that was
|
||
// nowhere in the text.
|
||
const mk = (value) =>
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
`description: ${value}`,
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
|
||
const anchored = parseSkill("alpha", mk("© Some valid text."));
|
||
assert.deepEqual(anchored.errors, []);
|
||
assert.equal(anchored.description, "Some valid text.");
|
||
|
||
const tagged = parseSkill("alpha", mk("!!str Some tagged text."));
|
||
assert.deepEqual(tagged.errors, []);
|
||
|
||
// The real case must still fail, so the gate is not simply disarmed.
|
||
assert.match(
|
||
parseSkill("alpha", mk("Covers board #28 (v2).")).errors.join(" "),
|
||
/`description` is truncated/,
|
||
);
|
||
});
|
||
|
||
test("a quoted scalar stays safe behind a tag or an anchor", () => {
|
||
// Properties may precede the value, in either order. Testing the first
|
||
// character alone read these as unquoted and then rejected the `#` that the
|
||
// quotes make harmless.
|
||
const mk = (value) =>
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
`description: ${value}`,
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
|
||
for (const value of [
|
||
'© "Counts # safely"',
|
||
'!!str "Counts # safely"',
|
||
'!!str &a "Counts # safely"',
|
||
]) {
|
||
const parsed = parseSkill("alpha", mk(value));
|
||
assert.deepEqual(parsed.errors, [], `rejected: ${value}`);
|
||
assert.equal(parsed.description, "Counts # safely");
|
||
}
|
||
});
|
||
|
||
test("the positive floor counts distinct prompts, not entries", () => {
|
||
// The floor exists for breadth, so repeating one prompt five times clears an
|
||
// entry count while exercising exactly the single trigger it looks past.
|
||
const duplicated = [
|
||
...Array.from({ length: MIN_POSITIVE_CASES }, () => ({
|
||
prompt: "the same prompt",
|
||
expect: "alpha",
|
||
})),
|
||
{ prompt: "n", expect: null },
|
||
];
|
||
const errors = validateEvalCases("alpha", duplicated).join(" ");
|
||
assert.match(
|
||
errors,
|
||
/only 1 distinct positive prompt\(s\) across 5 case\(s\)/,
|
||
);
|
||
|
||
// Whitespace and casing are not a new situation either.
|
||
const nearDuplicates = [
|
||
{ prompt: "How do I do the thing?", expect: "alpha" },
|
||
{ prompt: "how do i do the thing?", expect: "alpha" },
|
||
{ prompt: "How do I do the thing? ", expect: "alpha" },
|
||
{ prompt: "b", expect: "alpha" },
|
||
{ prompt: "c", expect: "alpha" },
|
||
{ prompt: "n", expect: null },
|
||
];
|
||
assert.match(
|
||
validateEvalCases("alpha", nearDuplicates).join(" "),
|
||
/only 3 distinct positive prompt\(s\)/,
|
||
);
|
||
|
||
// Five genuinely different prompts pass.
|
||
assert.deepEqual(
|
||
validateEvalCases("alpha", [
|
||
...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
|
||
prompt: `p${i}`,
|
||
expect: "alpha",
|
||
})),
|
||
{ prompt: "n", expect: null },
|
||
]),
|
||
[],
|
||
);
|
||
});
|
||
|
||
test("a quoted or block scalar may carry a trailing comment", () => {
|
||
// `yaml` attaches a legitimate trailing comment to these nodes too, so keying
|
||
// on `.comment` alone rejected valid frontmatter and told the author to quote
|
||
// a value that was already quoted. Only a PLAIN scalar can lose text to one.
|
||
for (const value of ['"Complete value" # note', "'Complete value' # note"]) {
|
||
const parsed = parseSkill(
|
||
"alpha",
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
`description: ${value}`,
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n"),
|
||
);
|
||
assert.deepEqual(parsed.errors, [], value);
|
||
assert.equal(parsed.description, "Complete value");
|
||
}
|
||
|
||
const block = parseSkill(
|
||
"alpha",
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
"description: | # note",
|
||
" Complete value",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n"),
|
||
);
|
||
assert.deepEqual(block.errors, []);
|
||
});
|
||
|
||
test("the truncation guard covers noncanonical key forms", () => {
|
||
// `"description":` and `description :` both populate the mapping, so a guard
|
||
// that located the field by matching a `^description:` LINE skipped them
|
||
// entirely and left `verify:skills` green on a truncated listing (Copilot).
|
||
const mk = (key) =>
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
`${key} Covers board #28 and the hazard.`,
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n");
|
||
|
||
for (const key of ['"description":', "'description':", "description :"]) {
|
||
const parsed = parseSkill("alpha", mk(key));
|
||
assert.equal(parsed.description, "Covers board", `key form: ${key}`);
|
||
assert.match(
|
||
parsed.errors.join(" "),
|
||
/`description` is truncated/,
|
||
`key form: ${key}`,
|
||
);
|
||
}
|
||
});
|
||
|
||
test("the truncation guard spans a blank line inside a plain scalar", () => {
|
||
// A plain scalar continues across empty lines too, so a scan that stopped at
|
||
// the first blank one inspected only the opening paragraph and reported no
|
||
// loss (Copilot).
|
||
const parsed = parseSkill(
|
||
"alpha",
|
||
[
|
||
"---",
|
||
"name: alpha",
|
||
"description: Covers boards",
|
||
"",
|
||
" #28 and #11 and the hazard.",
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n"),
|
||
);
|
||
assert.equal(parsed.description, "Covers boards");
|
||
assert.match(parsed.errors.join(" "), /`description` is truncated/);
|
||
});
|
||
|
||
test("a standalone comment line is not read as truncation", () => {
|
||
// The guard keys on the comment the parser hung on the SCALAR. A comment on
|
||
// its own line belongs to the document, not to the description, and flagging
|
||
// it would fail a frontmatter that loses nothing.
|
||
for (const frontmatter of [
|
||
["# leading", "name: alpha", "description: Covers boards"],
|
||
["name: alpha", "description: Covers boards", "# standalone"],
|
||
["name: alpha", "description: Issue#28 needs no quoting"],
|
||
]) {
|
||
const parsed = parseSkill(
|
||
"alpha",
|
||
[
|
||
"---",
|
||
...frontmatter,
|
||
"disable-model-invocation: false",
|
||
"---",
|
||
"body",
|
||
].join("\n"),
|
||
);
|
||
assert.deepEqual(parsed.errors, [], frontmatter.join(" | "));
|
||
}
|
||
});
|