feat: Skills extension phase 3 — CLI, TUI, directory reads, frontmatter check
Closes #2248. Completes SEP-2640 support across all three clients, and closes the one obligation #2234 deliberately left open. - `checkSkillFrontmatterMatch` compares a served SKILL.md's own YAML frontmatter against the entry the listing advertised, field by field. No digest can cover this: a digest is taken over the bytes the server served, so it proves the file was not altered in transit and says nothing about whether the listing described it honestly. Needs a real YAML parser; `yaml` was already a root dependency, so this adds no package — but it does newly put it on core/'s import graph, hence the three bundler `external` lists. - `resources/directory/read`: result schemas defined against the normative text, `InspectorClient.readResourceDirectory`, a Directory section on the Skills screen, and `--method resources/directory/read` in the CLI. The client refuses the call locally when the server did not declare `directoryRead`, which is the SEP's MUST NOT. A child the directory lists but the entry does not is marked `not listed` rather than merged into the manifest — the SEP calls a directory result a live observation and forbids treating it as extending the manifest. - CLI: `skills/list`, `skills/get` and `resources/directory/read`, plus `--verify` — one NDJSON report per skill, a summary on stderr, exit 7 on a violation. Reads are sequential; a conforming manifest may declare 512 entries. - TUI: a Skills pane, shown only when the server declares the extension. Each row carries its conformance verdict as a glyph as well as a colour, since the pane is read over ssh and through script(1). Enter verifies. - `verifySkills` re-throws `AuthRecoveryRequiredError` rather than recording it per file: it says the session's authorization expired, so absorbing it would report N identical read failures and swallow the error the TUI and the web commands key off to reauthorize. Two open questions settled, both in code comments where they will be found: `skills/get` carries no caching attributes because SEP-2640 leaves the question open in as many words, so requiring them would fail a conforming server; and there is no `PagedSkillsState` because every consumer of this list is a whole-catalog verdict — computed over page one of three, "this server's skills conform" is not merely partial but wrong. The fixture grows two skills, both for checks that were otherwise undemonstrable: `lying-listing` (listing and file disagree, digest still verifies) and `stale-manifest` (serves a file its manifest does not declare). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
parent
219a08e1b5
commit
9a8488681e
1 changed files with 4750 additions and 191 deletions
150
clients/cli/__tests__/skills-verify-cli.test.ts
Normal file
150
clients/cli/__tests__/skills-verify-cli.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { runCli } from "../src/cli.js";
|
||||
import { consumeMethodOutcome } from "../src/handlers/consume-outcome.js";
|
||||
import { EXIT_CODES } from "../src/error-handler.js";
|
||||
|
||||
/**
|
||||
* `--verify`'s argument validation and its NDJSON consumption path (#2248).
|
||||
*
|
||||
* The validation sits with `--strict`'s, ahead of every short-circuit return in
|
||||
* `parseArgs`, for the same reason: the returns below it never reach
|
||||
* `runMethod`, so a check placed further down would let the flag be accepted
|
||||
* and then silently ignored.
|
||||
*/
|
||||
describe("--verify argument validation", () => {
|
||||
it("is rejected with a method other than skills/list or skills/get", async () => {
|
||||
await expect(
|
||||
runCli([
|
||||
"node",
|
||||
"cli",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--verify",
|
||||
"--server-url",
|
||||
"http://127.0.0.1:1/mcp",
|
||||
]),
|
||||
).rejects.toThrow(
|
||||
"--verify requires --method skills/list or --method skills/get.",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["servers/list", ["--method", "servers/list"]],
|
||||
["--list-stored-auth", ["--method", "servers/list", "--list-stored-auth"]],
|
||||
])(
|
||||
"is rejected on the %s short-circuit path, which never reaches the report",
|
||||
async (_label, extra) => {
|
||||
await expect(
|
||||
runCli(["node", "cli", "--cli", "--verify", ...extra]),
|
||||
).rejects.toThrow(
|
||||
"--verify requires --method skills/list or --method skills/get.",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("is accepted with skills/get", async () => {
|
||||
// Reaches the connect and fails there — which is the point: the flag
|
||||
// itself was not what was rejected.
|
||||
await expect(
|
||||
runCli([
|
||||
"node",
|
||||
"cli",
|
||||
"--cli",
|
||||
"--method",
|
||||
"skills/get",
|
||||
"--uri",
|
||||
"skill://demo/SKILL.md",
|
||||
"--verify",
|
||||
"--server-url",
|
||||
"http://127.0.0.1:1/mcp",
|
||||
]),
|
||||
).rejects.not.toThrow(/--verify requires/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("consumeMethodOutcome NDJSON summary and exit code (#2248)", () => {
|
||||
function captureStreams() {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const write = (sink: (s: string) => void) =>
|
||||
((chunk: unknown, ...rest: unknown[]) => {
|
||||
sink(typeof chunk === "string" ? chunk : String(chunk));
|
||||
const cb = rest.find((r) => typeof r === "function") as
|
||||
| (() => void)
|
||||
| undefined;
|
||||
cb?.();
|
||||
return true;
|
||||
}) as typeof process.stdout.write;
|
||||
const originalOut = process.stdout.write;
|
||||
const originalErr = process.stderr.write;
|
||||
process.stdout.write = write((s) => (stdout += s));
|
||||
process.stderr.write = write((s) => (stderr += s));
|
||||
return {
|
||||
get stdout() {
|
||||
return stdout;
|
||||
},
|
||||
get stderr() {
|
||||
return stderr;
|
||||
},
|
||||
restore() {
|
||||
process.stdout.write = originalOut;
|
||||
process.stderr.write = originalErr;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("writes the summary to stderr so it cannot contaminate the NDJSON", async () => {
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
await consumeMethodOutcome(
|
||||
{ kind: "ndjson", lines: [{ ok: true }], summary: "all good" },
|
||||
{},
|
||||
);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout.trim())).toEqual({ ok: true });
|
||||
expect(streams.stderr).toBe("all good\n");
|
||||
});
|
||||
|
||||
it("throws the exit code AFTER writing the report", async () => {
|
||||
// The report is the output a CI job reads; failing before writing it would
|
||||
// give the reader an exit code and nothing to act on.
|
||||
const streams = captureStreams();
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await consumeMethodOutcome(
|
||||
{
|
||||
kind: "ndjson",
|
||||
lines: [{ ok: false }],
|
||||
summary: "one failed",
|
||||
exitCode: EXIT_CODES.SKILL_NONCONFORMANT,
|
||||
},
|
||||
{},
|
||||
);
|
||||
} catch (err) {
|
||||
thrown = err;
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(streams.stdout.trim()).toBe('{"ok":false}');
|
||||
expect(thrown).toMatchObject({
|
||||
exitCode: EXIT_CODES.SKILL_NONCONFORMANT,
|
||||
envelope: { code: "skills_nonconformant" },
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves an --app-info NDJSON outcome unchanged", async () => {
|
||||
// No summary, no exit code — the field is additive and the older caller
|
||||
// must behave exactly as before.
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
await consumeMethodOutcome({ kind: "ndjson", lines: [{ a: 1 }] }, {});
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(streams.stderr).toBe("");
|
||||
expect(streams.stdout.trim()).toBe('{"a":1}');
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue