inspector/scripts/skill-eval.test.mjs

// Tests for the pure halves of the skill trigger eval (#2163).
//
// Everything else in `skill-eval.mjs` spawns a real `claude` CLI, so its stream
// parsing and its process-outcome handling are unreachable from an eval run's
// happy path — they would only ever be exercised by the very measurement they
// are supposed to make trustworthy. A nonzero exit read as "no skill invoked"
// (the bug these cases pin) passes every negative case and reads as a trigger
// miss on every positive one, so a rate-limited run reports a plausible number
// instead of failing.

import { test } from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
  collectCases,
  collectSkillInvocations,
  runRejection,
  invokedSkillNames,
  runPrompt,
  sampleHit,
} from "./skill-eval.mjs";
import { MIN_POSITIVE_CASES } from "./lib/skill-manifest.mjs";

const assistant = (...blocks) =>
  JSON.stringify({ type: "assistant", message: { content: blocks } });
const skillUse = (name) => ({
  type: "tool_use",
  name: "Skill",
  input: { skill: name },
});

test("collectSkillInvocations finds Skill tool_use payloads", () => {
  const { invoked } = collectSkillInvocations(
    assistant(skillUse("testing")) + "\n",
  );
  assert.equal(invoked.size, 1);
  assert.ok([...invoked][0].includes("testing"));
});

test("collectSkillInvocations ignores other tools and other event types", () => {
  const text =
    assistant({ type: "tool_use", name: "Read", input: { file_path: "a" } }) +
    "\n" +
    JSON.stringify({ type: "result", result: "Skill" }) +
    "\n";
  assert.equal(collectSkillInvocations(text).invoked.size, 0);
});

test("collectSkillInvocations survives malformed and blank lines", () => {
  const text = "not json\n\n" + assistant(skillUse("local-dev")) + "\n";
  const { invoked } = collectSkillInvocations(text);
  assert.equal(invoked.size, 1);
});

test("collectSkillInvocations holds back a trailing partial line", () => {
  const whole = assistant(skillUse("local-dev"));
  const first = collectSkillInvocations(whole.slice(0, 20));
  assert.equal(first.invoked.size, 0);
  assert.equal(first.rest, whole.slice(0, 20));
  // Feeding the remainder back with the held-over prefix completes the event.
  const second = collectSkillInvocations(first.rest + whole.slice(20) + "\n");
  assert.equal(second.invoked.size, 1);
});

test("collectSkillInvocations tolerates a tool_use with no input", () => {
  const text = assistant({ type: "tool_use", name: "Skill" }) + "\n";
  assert.deepEqual([...collectSkillInvocations(text).invoked], ["{}"]);
});

test("sampleHit scores positive and negative cases", () => {
  const fired = new Set(['{"skill":"testing"}']);
  const none = new Set();
  assert.equal(sampleHit("testing", fired), true);
  assert.equal(sampleHit("local-dev", fired), false);
  assert.equal(sampleHit(null, none), true);
  assert.equal(sampleHit(null, fired), false);
  assert.equal(sampleHit("testing", none), false);
});

test("invokedSkillNames matches structurally, not by substring", () => {
  // `{"skill":"not-testing"}` contains "testing" and must NOT count — a
  // substring match inflates the measured hit rate with invocations of a
  // different skill.
  assert.deepEqual(invokedSkillNames('{"skill":"not-testing"}'), [
    "not-testing",
  ]);
  assert.equal(
    sampleHit("testing", new Set(['{"skill":"not-testing"}'])),
    false,
  );
  assert.equal(sampleHit("testing", new Set(['{"skill":"testing"}'])), true);
  // The field name is not assumed, so any string value is a candidate.
  assert.equal(
    sampleHit("testing", new Set(['{"name":"testing","args":""}'])),
    true,
  );
});

test("invokedSkillNames tolerates payloads it cannot read", () => {
  for (const payload of ["not json", "[1,2]", "null", '"testing"']) {
    assert.deepEqual(invokedSkillNames(payload), []);
  }
});

/** Minimal stand-in for a ChildProcess: emits stdout chunks, then closes. */
function fakeSpawn({ chunks = [], code = 0, error = null }) {
  return () => {
    const child = new EventEmitter();
    child.stdout = new EventEmitter();
    // The prompt is written to stdin rather than passed in argv, so the
    // stand-in needs one or `runPrompt` throws before the child can close.
    child.stdin = { end: () => {} };
    queueMicrotask(() => {
      if (error) {
        child.emit("error", error);
        return;
      }
      for (const c of chunks) child.stdout.emit("data", Buffer.from(c));
      child.emit("close", code);
    });
    return child;
  };
}

const resultEvent = (subtype) =>
  JSON.stringify({ type: "result", subtype }) + "\n";

test("collectSkillInvocations reports the terminal result subtype", () => {
  assert.equal(
    collectSkillInvocations(resultEvent("success")).result,
    "success",
  );
  assert.equal(collectSkillInvocations("").result, null);
});

test("runRejection accepts a run that hit the turn limit", () => {
  // With `--max-turns 1`, a run in which a skill FIRES necessarily hits the
  // limit and the CLI exits 1. Rejecting on the exit code alone would throw away
  // exactly the observations the eval exists to count — verified against the
  // real CLI, which ends such a run `error_max_turns` with `num_turns: 2`.
  assert.equal(runRejection({ result: "error_max_turns", code: 1 }), null);
  assert.equal(runRejection({ result: "success", code: 0 }), null);
});

test("runRejection rejects a run that observed nothing", () => {
  // An auth failure, a rate limit, or a CLI that never started. Counting these
  // as "no skill invoked" passes every negative case and reads as a trigger
  // miss on every positive one.
  assert.match(runRejection({ result: null, code: 1 }) ?? "", /no terminal/);
  assert.match(runRejection({ result: null, code: 0 }) ?? "", /no terminal/);
  assert.match(
    runRejection({ result: "error_during_execution", code: 1 }) ?? "",
    /ended `error_during_execution`/,
  );
});

test("runPrompt collects invocations across chunk boundaries", async () => {
  const whole =
    JSON.stringify({
      type: "assistant",
      message: {
        content: [
          { type: "tool_use", name: "Skill", input: { skill: "testing" } },
        ],
      },
    }) + "\n";
  const invoked = await runPrompt("p", {
    spawnFn: fakeSpawn({
      chunks: [
        whole.slice(0, 30),
        whole.slice(30),
        resultEvent("error_max_turns"),
      ],
      code: 1,
    }),
  });
  assert.equal(sampleHit("testing", invoked), true);
});

test("runPrompt rejects a run that produced no terminal result", async () => {
  await assert.rejects(
    runPrompt("p", { spawnFn: fakeSpawn({ code: 1 }) }),
    /no terminal `result` event \(exit 1\) for prompt: p/,
  );
});

test("runPrompt rejects a run that ended in an unusable state", async () => {
  await assert.rejects(
    runPrompt("p", {
      spawnFn: fakeSpawn({
        chunks: [resultEvent("error_during_execution")],
        code: 1,
      }),
    }),
    /ended `error_during_execution`/,
  );
});

test("runPrompt propagates a spawn error", async () => {
  await assert.rejects(
    runPrompt("p", { spawnFn: fakeSpawn({ error: new Error("ENOENT") }) }),
    /ENOENT/,
  );
});

test("a negative case ignores skills that are not this repo's", () => {
  // A contributor's own `~/.claude/skills` entry, or a bundled skill, firing on
  // a negative prompt says nothing about these skills — failing on it would be
  // a false failure about someone else's environment.
  const ours = new Set(["testing", "local-dev"]);
  const foreign = new Set(['{"skill":"my-personal-notes"}']);
  const mine = new Set(['{"skill":"testing"}']);

  assert.equal(sampleHit(null, foreign, ours), true);
  assert.equal(sampleHit(null, mine, ours), false);
  assert.equal(sampleHit(null, new Set(), ours), true);
  // A positive case is unaffected: it names the skill it wants.
  assert.equal(sampleHit("testing", mine, ours), true);
  assert.equal(sampleHit("testing", foreign, ours), false);
  // With no repo set, any invocation still fails a negative case.
  assert.equal(sampleHit(null, foreign), false);
});

test("runPrompt sends the prompt on stdin, never in argv", () => {
  // On Windows the CLI is a `.cmd` shim and can only start through a shell,
  // where `cmd.exe` re-parses any metacharacter in an argument as syntax. Off
  // Windows it also keeps the prompt out of the process table.
  const prompt = "does it? (yes & no)";
  let seen;
  let written = null;
  runPrompt(prompt, {
    spawnFn: (command, args, options) => {
      seen = { command, args, options };
      const c = new EventEmitter();
      c.stdout = new EventEmitter();
      c.stdin = {
        end: (text) => {
          written = text;
        },
      };
      queueMicrotask(() => c.emit("close", 0));
      return c;
    },
    platform: "linux",
  }).catch(() => {});
  assert.equal(seen.command, "claude");
  assert.ok(!seen.args.includes(prompt), "prompt must not appear in argv");
  assert.deepEqual(seen.options.stdio, ["pipe", "pipe", "inherit"]);
  assert.equal(written, prompt);
});

test("runPrompt asks for a shell on Windows", () => {
  let seen;
  runPrompt("p", {
    spawnFn: (_c, _a, options) => {
      seen = options;
      const c = new EventEmitter();
      c.stdout = new EventEmitter();
      c.stdin = { end: () => {} };
      queueMicrotask(() => c.emit("close", 0));
      return c;
    },
    platform: "win32",
  }).catch(() => {});
  assert.equal(seen.shell, true);
});

/** A skills directory with two model-invoked skills and one name-only skill. */
function skillsFixture() {
  const root = mkdtempSync(path.join(tmpdir(), "skill-eval-"));
  const write = (name, frontmatter, evals) => {
    const dir = path.join(root, name);
    mkdirSync(dir, { recursive: true });
    writeFileSync(
      path.join(dir, "SKILL.md"),
      `---\nname: ${name}\ndescription: d\n${frontmatter}---\n\nBody\n`,
    );
    if (evals) {
      mkdirSync(path.join(dir, "evals"), { recursive: true });
      writeFileSync(
        path.join(dir, "evals", "evals.json"),
        JSON.stringify(evals),
      );
    }
  };
  // Enough positives to satisfy the same MIN_POSITIVE_CASES floor the real eval
  // files must clear; the prompts stay predictable so assertions can name them.
  const evalsFor = (letter, skill) => [
    ...Array.from({ length: MIN_POSITIVE_CASES }, (_, i) => ({
      prompt: `${letter}+${i}`,
      expect: skill,
    })),
    { prompt: `${letter}-`, expect: null },
  ];
  write("alpha", "disable-model-invocation: false\n", evalsFor("a", "alpha"));
  write("beta", "disable-model-invocation: false\n", evalsFor("b", "beta"));
  write("gamma", "disable-model-invocation: true\n");
  return root;
}

test("focused mode narrows the cases but not the repo's own skill set", () => {
  // `ours` decides what a NEGATIVE case counts as a false trigger. Narrowing it
  // with `only` would let `skills:eval -- alpha` score a `beta` invocation as
  // somebody else's skill and pass a negative case it should fail.
  const root = skillsFixture();

  const all = collectCases(undefined, root);
  assert.deepEqual([...all.ours].sort(), ["alpha", "beta"]);
  assert.equal(all.cases.length, (MIN_POSITIVE_CASES + 1) * 2);

  const focused = collectCases("alpha", root);
  assert.deepEqual([...focused.ours].sort(), ["alpha", "beta"]);
  assert.deepEqual(
    focused.cases.map((c) => c.prompt),
    ["a+0", "a+1", "a+2", "a+3", "a+4", "a-"],
  );
  // The consequence, stated as the assertion that matters:
  assert.equal(
    sampleHit(null, new Set(['{"skill":"beta"}']), focused.ours),
    false,
  );

  rmSync(root, { recursive: true, force: true });
});

test("focused mode accepts several skill names", () => {
  const root = skillsFixture();

  const focused = collectCases(["alpha", "beta"], root);
  assert.deepEqual(
    focused.cases.map((c) => c.prompt).sort(),
    [
      "a+0",
      "a+1",
      "a+2",
      "a+3",
      "a+4",
      "a-",
      "b+0",
      "b+1",
      "b+2",
      "b+3",
      "b+4",
      "b-",
    ].sort(),
  );
  // Narrowing to every skill is still not the same as no filter — `ours` is
  // unchanged either way, which is what keeps negative scoring honest.
  assert.deepEqual([...focused.ours].sort(), ["alpha", "beta"]);

  // A single name keeps working, so the one-argument form is unaffected.
  assert.deepEqual(
    collectCases("beta", root).cases.map((c) => c.prompt),
    ["b+0", "b+1", "b+2", "b+3", "b+4", "b-"],
  );

  rmSync(root, { recursive: true, force: true });
});

test("an unknown skill name is an error, not an empty run", () => {
  // A typo would otherwise enqueue zero cases and report a green 0/0, which
  // reads exactly like a clean pass of the skill that was meant.
  const root = skillsFixture();
  assert.throws(
    () => collectCases(["alpha", "aplha"], root),
    /no model-invoked skill named `aplha`/,
  );
  // A name-only skill is not model-invoked, so asking for it is the same error.
  assert.throws(() => collectCases(["gamma"], root), /no model-invoked skill/);
  rmSync(root, { recursive: true, force: true });
});

test("a name-only skill is not part of the repo's model-invoked set", () => {
  const root = skillsFixture();
  const { ours } = collectCases(undefined, root);
  assert.equal(ours.has("gamma"), false);
  // So its firing does not fail a negative case — it cannot fire on its own.
  assert.equal(sampleHit(null, new Set(['{"skill":"gamma"}']), ours), true);
  rmSync(root, { recursive: true, force: true });
});

test("collection fails loudly rather than silently shrinking the set", () => {
  const root = skillsFixture();
  // A model-invoked skill whose evals vanished.
  rmSync(path.join(root, "beta", "evals"), { recursive: true, force: true });
  assert.throws(
    () => collectCases(undefined, root),
    /beta is model-invoked but has no/,
  );
  rmSync(root, { recursive: true, force: true });
});