feat(skills): address Copilot review round 2 on #2204

Two findings, both correct, and the first one means round 1's containment
fix was only half a fix.

**`--allowedTools` is not the availability filter.** It grants permission;
it does not restrict which tools exist. So a tool a user's or a plugin's
settings already permit stayed reachable across all 14 turns, and the
bound still rested on the deny list being complete — the exact property
round 1 set out to stop depending on. `--tools` is the real restriction
(it selects from the built-in set), and both flags now carry the same
four names: `--tools` to bound, `--allowedTools` so those four never need
a permission prompt no headless run can answer. Verified against the real
CLI — the full flag set runs to a clean `end_turn`. The test asserts both
flags, and both probe snippets in `docs/skill-authoring.md` carry
`--tools` with a ⚠️ saying which one does the bounding.

**The threshold guard let `NaN` and negatives through.**
`CHAIN_THRESHOLD=abc` is `NaN`, which fails every comparison and prints an
`above NaN%` summary; a negative bar passes every chain unconditionally.
Both turn an advertised env knob into a measurement that quietly means
nothing. `CHAIN_THRESHOLD` must now be in `[0, 1)` — half-open because the
chain bar is strict — and `THRESHOLD` in `[0, 1]`, inclusive because a
first-move floor of 1.0 is meetable. The message names the offending
value. A new case spawns the script for each bad value rather than
asserting on the predicate, since these guards live in `main`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ABdTnqyBodw6mHVVkGehqe
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-04 20:03:11 -04:00
commit 3ec125725a

View file

@ -11,6 +11,8 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
@ -31,6 +33,9 @@ import { MIN_POSITIVE_CASES } from "./lib/skill-manifest.mjs";
/** The payloads a run records, in the order the skills fired. */
const fired = (...names) => names.map((n) => JSON.stringify({ skill: n }));
/** The eval script itself, for the cases that must exercise `main`'s guards. */
const SCRIPT_PATH = fileURLToPath(new URL("./skill-eval.mjs", import.meta.url));
const assistant = (...blocks) =>
JSON.stringify({ type: "assistant", message: { content: blocks } });
const skillUse = (name) => ({
@ -312,17 +317,47 @@ test("runPrompt bounds the run by what it needs, not only by what it forbids", (
return c;
},
}).catch(() => {});
assert.deepEqual(args[args.indexOf("--allowedTools") + 1].split(","), [
"Read",
"Glob",
"Grep",
"Skill",
]);
// `--tools` is the availability filter and is what actually bounds the run.
// `--allowedTools` only pre-approves: a tool a user's or a plugin's settings
// already permit would still be reachable across 14 turns without this.
for (const flag of ["--tools", "--allowedTools"]) {
assert.deepEqual(args[args.indexOf(flag) + 1].split(","), [
"Read",
"Glob",
"Grep",
"Skill",
]);
}
// No `--mcp-config` accompanies it, so this drops every configured server.
assert.ok(args.includes("--strict-mcp-config"));
assert.ok(!args.includes("--mcp-config"));
});
test("a malformed threshold is rejected rather than silently measured", () => {
// `Number("abc")` is NaN, which fails every comparison and would print an
// `above NaN%` summary; a negative bar passes every chain unconditionally.
// Either turns an advertised env knob into a measurement that means nothing.
const run = (env) =>
spawnSync(process.execPath, [SCRIPT_PATH], {
encoding: "utf8",
env: { ...process.env, ...env },
});
for (const bad of ["abc", "-0.5", "1", "1.5"]) {
const { status, stderr } = run({ CHAIN_THRESHOLD: bad });
assert.equal(status, 1, `CHAIN_THRESHOLD=${bad} must be rejected`);
assert.match(stderr, /CHAIN_THRESHOLD must be a number in \[0, 1\)/);
assert.ok(stderr.includes(bad), "the message names the offending value");
}
// The first-move bar is inclusive, so 1 is legitimate there and only the
// nonsensical values are refused.
for (const bad of ["abc", "-1", "1.5"]) {
const { status, stderr } = run({ THRESHOLD: bad });
assert.equal(status, 1, `THRESHOLD=${bad} must be rejected`);
assert.match(stderr, /THRESHOLD must be a number in \[0, 1\]/);
}
});
test("passesThreshold is a floor for a first move and strictly above for a chain", () => {
// "More often than not" is `> 0.5`. An inclusive compare passes 2/4 whenever
// RUNS is even, reporting a result the stated criterion does not license.