feat: Skills extension phase 3 — CLI, TUI, directory reads, frontmatter check
Closes #2248. Completes SEP-2640 support across all three clients, and closes the one obligation #2234 deliberately left open. - `checkSkillFrontmatterMatch` compares a served SKILL.md's own YAML frontmatter against the entry the listing advertised, field by field. No digest can cover this: a digest is taken over the bytes the server served, so it proves the file was not altered in transit and says nothing about whether the listing described it honestly. Needs a real YAML parser; `yaml` was already a root dependency, so this adds no package — but it does newly put it on core/'s import graph, hence the three bundler `external` lists. - `resources/directory/read`: result schemas defined against the normative text, `InspectorClient.readResourceDirectory`, a Directory section on the Skills screen, and `--method resources/directory/read` in the CLI. The client refuses the call locally when the server did not declare `directoryRead`, which is the SEP's MUST NOT. A child the directory lists but the entry does not is marked `not listed` rather than merged into the manifest — the SEP calls a directory result a live observation and forbids treating it as extending the manifest. - CLI: `skills/list`, `skills/get` and `resources/directory/read`, plus `--verify` — one NDJSON report per skill, a summary on stderr, exit 7 on a violation. Reads are sequential; a conforming manifest may declare 512 entries. - TUI: a Skills pane, shown only when the server declares the extension. Each row carries its conformance verdict as a glyph as well as a colour, since the pane is read over ssh and through script(1). Enter verifies. - `verifySkills` re-throws `AuthRecoveryRequiredError` rather than recording it per file: it says the session's authorization expired, so absorbing it would report N identical read failures and swallow the error the TUI and the web commands key off to reauthorize. Two open questions settled, both in code comments where they will be found: `skills/get` carries no caching attributes because SEP-2640 leaves the question open in as many words, so requiring them would fail a conforming server; and there is no `PagedSkillsState` because every consumer of this list is a whole-catalog verdict — computed over page one of three, "this server's skills conform" is not merely partial but wrong. The fixture grows two skills, both for checks that were otherwise undemonstrable: `lying-listing` (listing and file disagree, digest still verifies) and `stale-manifest` (serves a file its manifest does not declare). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall
parent
219a08e1b5
commit
9a8488681e
2 changed files with 4750 additions and 191 deletions
222
clients/cli/__tests__/run-method-skills.test.ts
Normal file
222
clients/cli/__tests__/run-method-skills.test.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
import { describe, it, expect, vi } from "vitest";
|
||||
import { runMethod } from "../src/handlers/run-method.js";
|
||||
import { summarizeSkillVerification } from "../src/handlers/skills-verify.js";
|
||||
import { EXIT_CODES } from "../src/error-handler.js";
|
||||
import type { InspectorClient } from "@inspector/core/mcp/index.js";
|
||||
import type { SkillEntry } from "@inspector/core/mcp/skillsSchemas.js";
|
||||
import { sha256Digest } from "@inspector/core/mcp/skills.js";
|
||||
import type { SkillVerifyReport } from "@inspector/core/mcp/skillsVerification.js";
|
||||
|
||||
/**
|
||||
* The three SEP-2640 methods the CLI gained in #2248, plus `--verify`.
|
||||
*
|
||||
* The store's cursor walk and the verification checks are covered where they
|
||||
* live (`managedSkillsState.test.ts`, `skillsVerification.test.ts`); what these
|
||||
* pin is the dispatcher's own decisions — which method reaches which client
|
||||
* call, what shape leaves as a result, and when the report sets a non-zero exit
|
||||
* code.
|
||||
*/
|
||||
const SKILL_MD = "---\nname: demo\ndescription: A demo\n---\n\n# Demo\n";
|
||||
|
||||
async function cleanEntry(): Promise<SkillEntry> {
|
||||
const bytes = new TextEncoder().encode(SKILL_MD);
|
||||
return {
|
||||
uri: "skill://demo/SKILL.md",
|
||||
frontmatter: { name: "demo", description: "A demo" },
|
||||
resources: [
|
||||
{
|
||||
uri: "skill://demo/SKILL.md",
|
||||
digest: await sha256Digest(bytes),
|
||||
size: bytes.byteLength,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function mockClient(overrides: Record<string, unknown> = {}): InspectorClient {
|
||||
return {
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
getStatus: vi.fn().mockReturnValue("connected"),
|
||||
getSkillsExtension: vi.fn().mockReturnValue({ directoryRead: true }),
|
||||
listSkills: vi.fn().mockResolvedValue({ skills: [] }),
|
||||
getSkill: vi.fn(),
|
||||
readResourceDirectory: vi.fn(),
|
||||
readResource: vi.fn().mockResolvedValue({
|
||||
result: { contents: [{ uri: "skill://demo/SKILL.md", text: SKILL_MD }] },
|
||||
}),
|
||||
...overrides,
|
||||
} as unknown as InspectorClient;
|
||||
}
|
||||
|
||||
describe("runMethod skills dispatch (#2248)", () => {
|
||||
it("returns the walked list for skills/list", async () => {
|
||||
const entry = await cleanEntry();
|
||||
const client = mockClient({
|
||||
listSkills: vi.fn().mockResolvedValue({ skills: [entry] }),
|
||||
});
|
||||
const outcome = await runMethod(client, { method: "skills/list" });
|
||||
expect(outcome).toEqual({
|
||||
kind: "result",
|
||||
result: { skills: [entry] },
|
||||
appInfo: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects skills/list with a usage exit code when the server declares no extension", async () => {
|
||||
// The store answers "no extension" with an empty list, which is right for
|
||||
// a UI that must render something and wrong for a CLI: "no skills" and
|
||||
// "does not serve skills" are answers a script has to tell apart.
|
||||
const client = mockClient({
|
||||
getSkillsExtension: vi.fn().mockReturnValue(undefined),
|
||||
});
|
||||
await expect(runMethod(client, { method: "skills/list" })).rejects.toThrow(
|
||||
/does not declare/i,
|
||||
);
|
||||
await expect(
|
||||
runMethod(client, { method: "skills/list" }),
|
||||
).rejects.toMatchObject({ exitCode: EXIT_CODES.USAGE });
|
||||
});
|
||||
|
||||
it("keeps the { skill } envelope on skills/get", async () => {
|
||||
// The client unwraps it for callers that want the entry; a CLI whose
|
||||
// contract is "print the result" must not quietly reshape the wire form.
|
||||
const entry = await cleanEntry();
|
||||
const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) });
|
||||
const outcome = await runMethod(client, {
|
||||
method: "skills/get",
|
||||
uri: entry.uri,
|
||||
});
|
||||
expect(outcome).toMatchObject({ result: { skill: entry } });
|
||||
});
|
||||
|
||||
it("requires --uri for skills/get", async () => {
|
||||
await expect(
|
||||
runMethod(mockClient(), { method: "skills/get" }),
|
||||
).rejects.toThrow(/URI is required/);
|
||||
});
|
||||
|
||||
it("requires --uri for resources/directory/read", async () => {
|
||||
await expect(
|
||||
runMethod(mockClient(), { method: "resources/directory/read" }),
|
||||
).rejects.toThrow(/URI is required/);
|
||||
});
|
||||
|
||||
it("returns one page of resources/directory/read and forwards the cursor", async () => {
|
||||
// One page, not a walk: the SEP says the listing is not recursive and the
|
||||
// client descends, so aggregating here would present a subtree as a
|
||||
// directory.
|
||||
const page = { resources: [], nextCursor: "2" };
|
||||
const readResourceDirectory = vi.fn().mockResolvedValue(page);
|
||||
const client = mockClient({ readResourceDirectory });
|
||||
const outcome = await runMethod(client, {
|
||||
method: "resources/directory/read",
|
||||
uri: "skill://demo",
|
||||
cursor: "1",
|
||||
});
|
||||
expect(readResourceDirectory).toHaveBeenCalledWith(
|
||||
"skill://demo",
|
||||
"1",
|
||||
undefined,
|
||||
);
|
||||
expect(outcome).toMatchObject({ result: page });
|
||||
});
|
||||
|
||||
it("--verify emits one NDJSON report per skill with no exit code when clean", async () => {
|
||||
const entry = await cleanEntry();
|
||||
const client = mockClient({
|
||||
listSkills: vi.fn().mockResolvedValue({ skills: [entry] }),
|
||||
});
|
||||
const outcome = await runMethod(client, {
|
||||
method: "skills/list",
|
||||
verify: true,
|
||||
});
|
||||
expect(outcome.kind).toBe("ndjson");
|
||||
if (outcome.kind !== "ndjson") throw new Error("unreachable");
|
||||
expect(outcome.lines).toHaveLength(1);
|
||||
expect((outcome.lines[0] as SkillVerifyReport).ok).toBe(true);
|
||||
expect(outcome.summary).toMatch(/no conformance errors/);
|
||||
expect(outcome.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("--verify sets the skills exit code when a skill fails", async () => {
|
||||
const entry = await cleanEntry();
|
||||
const client = mockClient({
|
||||
listSkills: vi.fn().mockResolvedValue({ skills: [entry] }),
|
||||
readResource: vi.fn().mockResolvedValue({
|
||||
result: {
|
||||
contents: [{ uri: entry.uri, text: "totally different bytes" }],
|
||||
},
|
||||
}),
|
||||
});
|
||||
const outcome = await runMethod(client, {
|
||||
method: "skills/list",
|
||||
verify: true,
|
||||
});
|
||||
if (outcome.kind !== "ndjson") throw new Error("unreachable");
|
||||
expect(outcome.exitCode).toBe(EXIT_CODES.SKILL_NONCONFORMANT);
|
||||
// Its own code, not SCHEMA_UNPORTABLE — an unportable tool schema and a
|
||||
// tampered skill digest are different CI failures.
|
||||
expect(EXIT_CODES.SKILL_NONCONFORMANT).not.toBe(
|
||||
EXIT_CODES.SCHEMA_UNPORTABLE,
|
||||
);
|
||||
});
|
||||
|
||||
it("--verify works on a single skills/get", async () => {
|
||||
const entry = await cleanEntry();
|
||||
const client = mockClient({ getSkill: vi.fn().mockResolvedValue(entry) });
|
||||
const outcome = await runMethod(client, {
|
||||
method: "skills/get",
|
||||
uri: entry.uri,
|
||||
verify: true,
|
||||
});
|
||||
expect(outcome.kind).toBe("ndjson");
|
||||
if (outcome.kind !== "ndjson") throw new Error("unreachable");
|
||||
expect(outcome.lines).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("summarizeSkillVerification (#2248)", () => {
|
||||
const report = (
|
||||
over: Partial<SkillVerifyReport> = {},
|
||||
): SkillVerifyReport => ({
|
||||
uri: "skill://demo/SKILL.md",
|
||||
name: "demo",
|
||||
conformance: [],
|
||||
frontmatter: [],
|
||||
files: [{ uri: "skill://demo/SKILL.md", status: "verified" }],
|
||||
ok: true,
|
||||
...over,
|
||||
});
|
||||
|
||||
it("reports a clean run with singular wording for one skill", () => {
|
||||
expect(summarizeSkillVerification([report()])).toBe(
|
||||
"Verified 1 skill and 1 file: no conformance errors.",
|
||||
);
|
||||
});
|
||||
|
||||
it("pluralizes for more than one", () => {
|
||||
expect(summarizeSkillVerification([report(), report()])).toBe(
|
||||
"Verified 2 skills and 2 files: no conformance errors.",
|
||||
);
|
||||
});
|
||||
|
||||
it("counts failures and digest mismatches separately", () => {
|
||||
// A skill can fail on a conformance error with no mismatched file at all,
|
||||
// so collapsing the two counts would misreport the cause.
|
||||
const failed = report({
|
||||
ok: false,
|
||||
files: [{ uri: "skill://demo/SKILL.md", status: "mismatch" }],
|
||||
});
|
||||
expect(summarizeSkillVerification([report(), failed])).toBe(
|
||||
"1 of 2 skills failed verification (1 digest/size mismatch across 2 files).",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports a failure with no mismatched file", () => {
|
||||
const failed = report({ ok: false, files: [] });
|
||||
expect(summarizeSkillVerification([failed])).toBe(
|
||||
"1 of 1 skill failed verification (0 digest/size mismatch across 0 files).",
|
||||
);
|
||||
});
|
||||
});
|
||||
150
clients/cli/__tests__/skills-verify-cli.test.ts
Normal file
150
clients/cli/__tests__/skills-verify-cli.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import { runCli } from "../src/cli.js";
|
||||
import { consumeMethodOutcome } from "../src/handlers/consume-outcome.js";
|
||||
import { EXIT_CODES } from "../src/error-handler.js";
|
||||
|
||||
/**
|
||||
* `--verify`'s argument validation and its NDJSON consumption path (#2248).
|
||||
*
|
||||
* The validation sits with `--strict`'s, ahead of every short-circuit return in
|
||||
* `parseArgs`, for the same reason: the returns below it never reach
|
||||
* `runMethod`, so a check placed further down would let the flag be accepted
|
||||
* and then silently ignored.
|
||||
*/
|
||||
describe("--verify argument validation", () => {
|
||||
it("is rejected with a method other than skills/list or skills/get", async () => {
|
||||
await expect(
|
||||
runCli([
|
||||
"node",
|
||||
"cli",
|
||||
"--cli",
|
||||
"--method",
|
||||
"tools/list",
|
||||
"--verify",
|
||||
"--server-url",
|
||||
"http://127.0.0.1:1/mcp",
|
||||
]),
|
||||
).rejects.toThrow(
|
||||
"--verify requires --method skills/list or --method skills/get.",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["servers/list", ["--method", "servers/list"]],
|
||||
["--list-stored-auth", ["--method", "servers/list", "--list-stored-auth"]],
|
||||
])(
|
||||
"is rejected on the %s short-circuit path, which never reaches the report",
|
||||
async (_label, extra) => {
|
||||
await expect(
|
||||
runCli(["node", "cli", "--cli", "--verify", ...extra]),
|
||||
).rejects.toThrow(
|
||||
"--verify requires --method skills/list or --method skills/get.",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("is accepted with skills/get", async () => {
|
||||
// Reaches the connect and fails there — which is the point: the flag
|
||||
// itself was not what was rejected.
|
||||
await expect(
|
||||
runCli([
|
||||
"node",
|
||||
"cli",
|
||||
"--cli",
|
||||
"--method",
|
||||
"skills/get",
|
||||
"--uri",
|
||||
"skill://demo/SKILL.md",
|
||||
"--verify",
|
||||
"--server-url",
|
||||
"http://127.0.0.1:1/mcp",
|
||||
]),
|
||||
).rejects.not.toThrow(/--verify requires/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("consumeMethodOutcome NDJSON summary and exit code (#2248)", () => {
|
||||
function captureStreams() {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const write = (sink: (s: string) => void) =>
|
||||
((chunk: unknown, ...rest: unknown[]) => {
|
||||
sink(typeof chunk === "string" ? chunk : String(chunk));
|
||||
const cb = rest.find((r) => typeof r === "function") as
|
||||
| (() => void)
|
||||
| undefined;
|
||||
cb?.();
|
||||
return true;
|
||||
}) as typeof process.stdout.write;
|
||||
const originalOut = process.stdout.write;
|
||||
const originalErr = process.stderr.write;
|
||||
process.stdout.write = write((s) => (stdout += s));
|
||||
process.stderr.write = write((s) => (stderr += s));
|
||||
return {
|
||||
get stdout() {
|
||||
return stdout;
|
||||
},
|
||||
get stderr() {
|
||||
return stderr;
|
||||
},
|
||||
restore() {
|
||||
process.stdout.write = originalOut;
|
||||
process.stderr.write = originalErr;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("writes the summary to stderr so it cannot contaminate the NDJSON", async () => {
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
await consumeMethodOutcome(
|
||||
{ kind: "ndjson", lines: [{ ok: true }], summary: "all good" },
|
||||
{},
|
||||
);
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(JSON.parse(streams.stdout.trim())).toEqual({ ok: true });
|
||||
expect(streams.stderr).toBe("all good\n");
|
||||
});
|
||||
|
||||
it("throws the exit code AFTER writing the report", async () => {
|
||||
// The report is the output a CI job reads; failing before writing it would
|
||||
// give the reader an exit code and nothing to act on.
|
||||
const streams = captureStreams();
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await consumeMethodOutcome(
|
||||
{
|
||||
kind: "ndjson",
|
||||
lines: [{ ok: false }],
|
||||
summary: "one failed",
|
||||
exitCode: EXIT_CODES.SKILL_NONCONFORMANT,
|
||||
},
|
||||
{},
|
||||
);
|
||||
} catch (err) {
|
||||
thrown = err;
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(streams.stdout.trim()).toBe('{"ok":false}');
|
||||
expect(thrown).toMatchObject({
|
||||
exitCode: EXIT_CODES.SKILL_NONCONFORMANT,
|
||||
envelope: { code: "skills_nonconformant" },
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves an --app-info NDJSON outcome unchanged", async () => {
|
||||
// No summary, no exit code — the field is additive and the older caller
|
||||
// must behave exactly as before.
|
||||
const streams = captureStreams();
|
||||
try {
|
||||
await consumeMethodOutcome({ kind: "ndjson", lines: [{ a: 1 }] }, {});
|
||||
} finally {
|
||||
streams.restore();
|
||||
}
|
||||
expect(streams.stderr).toBe("");
|
||||
expect(streams.stdout.trim()).toBe('{"a":1}');
|
||||
});
|
||||
});
|
||||
Loading…
Reference in a new issue