// Tests for the guaranteed authoritative-validator step (#2163, Copilot). // // Two things are covered, and neither is reachable from an ordinary run on a // machine that has a working, pinned CLI: // // 1. Which validator gets run. A local CLI is used only when it matches the // pin EXACTLY — accepting anything newer would mean a maintainer validates // against a different schema than CI, so the same `local:gate` could // disagree across machines, which is the failure a pin exists to prevent. // 2. The orchestration around it. A regression in the probe, the spawn-error // branch, or the propagation of a rejected validator would otherwise leave // `test:scripts` green while this gate stopped gating. import { test } from "node:test"; import assert from "node:assert/strict"; import { validatorCommand, runValidator } from "./verify-skills-cli.mjs"; import { PINNED_CLI_VERSION } from "./lib/skill-manifest.mjs"; const pinned = PINNED_CLI_VERSION.split(".").map(Number); const npxArgs = [ "-y", `@anthropic-ai/claude-code@${PINNED_CLI_VERSION}`, "plugin", "validate", ".claude/skills", ]; test("uses a local CLI only when it matches the pin exactly", () => { const cmd = validatorCommand({ version: pinned }); assert.equal(cmd.command, "claude"); assert.deepEqual(cmd.args, ["plugin", "validate", ".claude/skills"]); assert.match(cmd.via, /matches the pin/); }); test("falls back to the pinned package for any other local version", () => { // Newer as well as older: a newer CLI is a DIFFERENT schema from CI's, which // is exactly the cross-machine disagreement the pin exists to prevent. for (const version of [ [pinned[0], pinned[1], pinned[2] + 1], [pinned[0], pinned[1], pinned[2] - 1], [pinned[0] + 1, 0, 0], [2, 1, 232], ]) { const cmd = validatorCommand({ version }); assert.equal(cmd.command, "npx", `expected npx for ${version.join(".")}`); assert.deepEqual(cmd.args, npxArgs); assert.match(cmd.via, /not the pinned/); } }); test("falls back to the pinned package when no CLI is present", () => { const cmd = validatorCommand({ version: null }); assert.equal(cmd.command, "npx"); assert.deepEqual(cmd.args, npxArgs); assert.match(cmd.via, /no local CLI/); }); /** Collects what `runValidator` was asked to spawn. */ function harness({ probe, spawnResult }) { const spawned = []; const logs = []; const errors = []; const code = runValidator({ probe: () => probe, spawn: (command, args, opts) => { spawned.push({ command, args, opts }); return spawnResult; }, log: (m) => logs.push(m), error: (m) => errors.push(m), }); return { code, spawned, logs, errors }; } const okProbe = { status: 0, stdout: `${PINNED_CLI_VERSION} (Claude Code)\n` }; test("runs the local CLI and succeeds when the validator accepts", () => { const r = harness({ probe: okProbe, spawnResult: { status: 0 } }); assert.equal(r.code, 0); assert.equal(r.spawned[0].command, "claude"); assert.equal(r.errors.length, 0); assert.match(r.logs.join(), /validating \.claude\/skills via local claude/); }); test("treats an unusable probe as no CLI at all", () => { // A `claude` that fails `--version` cannot be trusted to have the subcommand. for (const probe of [ { error: new Error("ENOENT") }, { status: 1, stdout: "" }, { status: 0, stdout: "not a version" }, ]) { const r = harness({ probe, spawnResult: { status: 0 } }); assert.equal(r.code, 0); assert.equal(r.spawned[0].command, "npx"); } }); test("fails when the validator rejects the skills", () => { const r = harness({ probe: okProbe, spawnResult: { status: 1 } }); assert.equal(r.code, 1); assert.match(r.errors.join(), /rejected the skills/); }); test("fails when the validator cannot be spawned at all", () => { const r = harness({ probe: okProbe, spawnResult: { error: new Error("spawn npx ENOENT") }, }); assert.equal(r.code, 1); assert.match(r.errors.join(), /could not run `claude`: spawn npx ENOENT/); });