a51e62dfa1
- skill-manifest: the closing `---` fence must occupy a whole line. `---oops` was accepted as a terminator, silently truncating the frontmatter and passing validation — the exact failure the guard exists to catch. Regression cases added (prefix, trailing whitespace, EOF, CRLF). - skill-eval: reject a nonzero `claude -p` exit instead of resolving it as an empty observation. An auth error or rate limit was scoring as "no skill invoked", which passes every negative case and reads as a trigger miss on every positive one. - skill-eval: extract `collectSkillInvocations` / `sampleHit` behind a `main()` guard and cover them in `skill-eval.test.mjs` — the stream parsing and the process-outcome handling are unreachable from an eval run's happy path. - docs/test-servers: annotate the era on every catalogue row. Six legacy rows carried no annotation under a "unless noted, Modern" default, so the doc sent readers to the wrong era for the feature they were trying to see. - docs/quality-gate: `validate` runs four guards, not three; add the `verify:skills` row and the two new `test:scripts` suites; fix the `#publishing` fragment left dangling by the split. - issue-create / pr-flow: delegate the card steps to `/board-ops` rather than restating its option IDs, which are explicitly unstable. - issue-triage: the audit counted presence, not cardinality — an issue carrying both `v1` and `v2` passed a check that claims "exactly one". Same for the five type labels. Also scope the two milestone checks to board #28. - AGENTS.md: codify the v1 milestone exception. Every milestone is a v2 release bucket, so a v1 issue has none to take and the Incoming-milestone invariant cannot apply to board #11. - clients/launcher/README: its pointer promised an explanation of why the cli/tui manifests ship, which the extracted guide does not make. Signed-off-by: cliffhall <cliff@futurescale.com>
69 lines
2.8 KiB
JavaScript
69 lines
2.8 KiB
JavaScript
// Tests for the pure halves of the skill trigger eval (#2163).
|
|
//
|
|
// Everything else in `skill-eval.mjs` spawns a real `claude` CLI, so its stream
|
|
// parsing and its process-outcome handling are unreachable from an eval run's
|
|
// happy path — they would only ever be exercised by the very measurement they
|
|
// are supposed to make trustworthy. A nonzero exit read as "no skill invoked"
|
|
// (the bug these cases pin) passes every negative case and reads as a trigger
|
|
// miss on every positive one, so a rate-limited run reports a plausible number
|
|
// instead of failing.
|
|
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { collectSkillInvocations, sampleHit } from "./skill-eval.mjs";
|
|
|
|
const assistant = (...blocks) =>
|
|
JSON.stringify({ type: "assistant", message: { content: blocks } });
|
|
const skillUse = (name) => ({
|
|
type: "tool_use",
|
|
name: "Skill",
|
|
input: { skill: name },
|
|
});
|
|
|
|
test("collectSkillInvocations finds Skill tool_use payloads", () => {
|
|
const { invoked } = collectSkillInvocations(
|
|
assistant(skillUse("testing")) + "\n",
|
|
);
|
|
assert.equal(invoked.size, 1);
|
|
assert.ok([...invoked][0].includes("testing"));
|
|
});
|
|
|
|
test("collectSkillInvocations ignores other tools and other event types", () => {
|
|
const text =
|
|
assistant({ type: "tool_use", name: "Read", input: { file_path: "a" } }) +
|
|
"\n" +
|
|
JSON.stringify({ type: "result", result: "Skill" }) +
|
|
"\n";
|
|
assert.equal(collectSkillInvocations(text).invoked.size, 0);
|
|
});
|
|
|
|
test("collectSkillInvocations survives malformed and blank lines", () => {
|
|
const text = "not json\n\n" + assistant(skillUse("local-dev")) + "\n";
|
|
const { invoked } = collectSkillInvocations(text);
|
|
assert.equal(invoked.size, 1);
|
|
});
|
|
|
|
test("collectSkillInvocations holds back a trailing partial line", () => {
|
|
const whole = assistant(skillUse("local-dev"));
|
|
const first = collectSkillInvocations(whole.slice(0, 20));
|
|
assert.equal(first.invoked.size, 0);
|
|
assert.equal(first.rest, whole.slice(0, 20));
|
|
// Feeding the remainder back with the held-over prefix completes the event.
|
|
const second = collectSkillInvocations(first.rest + whole.slice(20) + "\n");
|
|
assert.equal(second.invoked.size, 1);
|
|
});
|
|
|
|
test("collectSkillInvocations tolerates a tool_use with no input", () => {
|
|
const text = assistant({ type: "tool_use", name: "Skill" }) + "\n";
|
|
assert.deepEqual([...collectSkillInvocations(text).invoked], ["{}"]);
|
|
});
|
|
|
|
test("sampleHit scores positive and negative cases", () => {
|
|
const fired = new Set(['{"skill":"testing"}']);
|
|
const none = new Set();
|
|
assert.equal(sampleHit("testing", fired), true);
|
|
assert.equal(sampleHit("local-dev", fired), false);
|
|
assert.equal(sampleHit(null, none), true);
|
|
assert.equal(sampleHit(null, fired), false);
|
|
assert.equal(sampleHit("testing", none), false);
|
|
});
|