75ab1b069f
- verify-skills-cli: use a local CLI only when it matches the pin EXACTLY. A
floor let a maintainer on a newer CLI validate against a different schema than
CI's, so the same `local:gate` could disagree across machines — the failure a
pin exists to prevent.
- verify-skills-cli: extract `runValidator({probe, spawn, log, error})` and
cover the orchestration — validator accepts, validator rejects, spawn fails,
and the three unusable-probe shapes. On a machine with a working CLI an actual
run only ever walks the happy path.
- pr-flow: stop putting the `gh` token in curl's argv, where any local process
can read it off the process table. Feed the header through `curl --config -`.
- docs/quality-gate: three places had drifted — the CI tier row, the
`local:gate` chain, and the `test:scripts` inventory (which now names all four
suites this PR adds, not two).
Signed-off-by: cliffhall <cliff@futurescale.com>
111 lines
4 KiB
JavaScript
111 lines
4 KiB
JavaScript
// Tests for the guaranteed authoritative-validator step (#2163, Copilot).
|
|
//
|
|
// Two things are covered, and neither is reachable from an ordinary run on a
|
|
// machine that has a working, pinned CLI:
|
|
//
|
|
// 1. Which validator gets run. A local CLI is used only when it matches the
|
|
// pin EXACTLY — accepting anything newer would mean a maintainer validates
|
|
// against a different schema than CI, so the same `local:gate` could
|
|
// disagree across machines, which is the failure a pin exists to prevent.
|
|
// 2. The orchestration around it. A regression in the probe, the spawn-error
|
|
// branch, or the propagation of a rejected validator would otherwise leave
|
|
// `test:scripts` green while this gate stopped gating.
|
|
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { validatorCommand, runValidator } from "./verify-skills-cli.mjs";
|
|
import { PINNED_CLI_VERSION } from "./lib/skill-manifest.mjs";
|
|
|
|
const pinned = PINNED_CLI_VERSION.split(".").map(Number);
|
|
const npxArgs = [
|
|
"-y",
|
|
`@anthropic-ai/claude-code@${PINNED_CLI_VERSION}`,
|
|
"plugin",
|
|
"validate",
|
|
".claude/skills",
|
|
];
|
|
|
|
test("uses a local CLI only when it matches the pin exactly", () => {
|
|
const cmd = validatorCommand({ version: pinned });
|
|
assert.equal(cmd.command, "claude");
|
|
assert.deepEqual(cmd.args, ["plugin", "validate", ".claude/skills"]);
|
|
assert.match(cmd.via, /matches the pin/);
|
|
});
|
|
|
|
test("falls back to the pinned package for any other local version", () => {
|
|
// Newer as well as older: a newer CLI is a DIFFERENT schema from CI's, which
|
|
// is exactly the cross-machine disagreement the pin exists to prevent.
|
|
for (const version of [
|
|
[pinned[0], pinned[1], pinned[2] + 1],
|
|
[pinned[0], pinned[1], pinned[2] - 1],
|
|
[pinned[0] + 1, 0, 0],
|
|
[2, 1, 232],
|
|
]) {
|
|
const cmd = validatorCommand({ version });
|
|
assert.equal(cmd.command, "npx", `expected npx for ${version.join(".")}`);
|
|
assert.deepEqual(cmd.args, npxArgs);
|
|
assert.match(cmd.via, /not the pinned/);
|
|
}
|
|
});
|
|
|
|
test("falls back to the pinned package when no CLI is present", () => {
|
|
const cmd = validatorCommand({ version: null });
|
|
assert.equal(cmd.command, "npx");
|
|
assert.deepEqual(cmd.args, npxArgs);
|
|
assert.match(cmd.via, /no local CLI/);
|
|
});
|
|
|
|
/** Collects what `runValidator` was asked to spawn. */
|
|
function harness({ probe, spawnResult }) {
|
|
const spawned = [];
|
|
const logs = [];
|
|
const errors = [];
|
|
const code = runValidator({
|
|
probe: () => probe,
|
|
spawn: (command, args, opts) => {
|
|
spawned.push({ command, args, opts });
|
|
return spawnResult;
|
|
},
|
|
log: (m) => logs.push(m),
|
|
error: (m) => errors.push(m),
|
|
});
|
|
return { code, spawned, logs, errors };
|
|
}
|
|
|
|
const okProbe = { status: 0, stdout: `${PINNED_CLI_VERSION} (Claude Code)\n` };
|
|
|
|
test("runs the local CLI and succeeds when the validator accepts", () => {
|
|
const r = harness({ probe: okProbe, spawnResult: { status: 0 } });
|
|
assert.equal(r.code, 0);
|
|
assert.equal(r.spawned[0].command, "claude");
|
|
assert.equal(r.errors.length, 0);
|
|
assert.match(r.logs.join(), /validating \.claude\/skills via local claude/);
|
|
});
|
|
|
|
test("treats an unusable probe as no CLI at all", () => {
|
|
// A `claude` that fails `--version` cannot be trusted to have the subcommand.
|
|
for (const probe of [
|
|
{ error: new Error("ENOENT") },
|
|
{ status: 1, stdout: "" },
|
|
{ status: 0, stdout: "not a version" },
|
|
]) {
|
|
const r = harness({ probe, spawnResult: { status: 0 } });
|
|
assert.equal(r.code, 0);
|
|
assert.equal(r.spawned[0].command, "npx");
|
|
}
|
|
});
|
|
|
|
test("fails when the validator rejects the skills", () => {
|
|
const r = harness({ probe: okProbe, spawnResult: { status: 1 } });
|
|
assert.equal(r.code, 1);
|
|
assert.match(r.errors.join(), /rejected the skills/);
|
|
});
|
|
|
|
test("fails when the validator cannot be spawned at all", () => {
|
|
const r = harness({
|
|
probe: okProbe,
|
|
spawnResult: { error: new Error("spawn npx ENOENT") },
|
|
});
|
|
assert.equal(r.code, 1);
|
|
assert.match(r.errors.join(), /could not run `claude`: spawn npx ENOENT/);
|
|
});
|