fix: address Copilot review round 15 on #2248

**Verification is a tri-state now, because two states could not be right.**
Round 12 was correct that `ok: true` for a manifest whose unread 513th file
may be tampered with is a false pass. Round 15 is correct that forcing `ok:
false` calls a server nonconformant for exceeding limits SEP-2640 states as
SHOULD NOT, with hosts free to support more — contradicting this module's own
rule that a warning never fails a report. Both hold, so:

  verified   -> exit 0   everything checked, everything passed
  failed     -> exit 7   a MUST was broken
  incomplete -> exit 8   nothing checked was wrong; the walk was cut short

`ok` keeps its narrow meaning, `allSkillsVerified` is stricter than
`every(r => r.ok)` since it selects the exit code, and `anySkillFailed`
separates 7 from 8. A job that tolerates oversized catalogs can allow 8 and
still fail on 7.

**`manifestKey` still used `JSON.stringify` on the entry** — the same crash I
fixed in the TUI in round 12, one file over.

**Directory children were navigable outside the skill root.** A server could
return `skill://other-skill/...` and clicking it left the tree, with "Up"
only comparing equality against the root. Containment is checked on the
normalized URI now, and an outside child renders as text rather than a link.
Worth recording from the tests: `..` cannot escape the authority, so
`skill://a/nested/../x.md` resolves back inside and must stay navigable.

**The TUI matched report rows by raw URI**, which missed a row recorded under
an equivalent spelling while `extraReportFiles` suppressed it as covered.
Normalized, like the membership test beside it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ND46Ci64DC3tzBy9aG81Nc
Signed-off-by: cliffhall <cliff@futurescale.com>
This commit is contained in:
cliffhall 2026-09-07 22:47:32 -04:00
commit 89438d1a4b
3 changed files with 434 additions and 141 deletions

View file

@ -35,6 +35,19 @@ export const EXIT_CODES = {
* collapsing them would make `if [ $? -eq 6 ]` ambiguous.
*/
SKILL_NONCONFORMANT: 7,
/**
* `--verify` could not check the whole catalog: the read bounds stopped the
* walk before it finished (#2248).
*
* Distinct from `SKILL_NONCONFORMANT` because the server has broken no
* **MUST** — SEP-2640 states the 512-entry and 16 MiB limits as SHOULD NOT,
* with hosts free to support more — so exiting 7 would call a conforming
* server nonconformant. It is still non-zero, because reporting success for
* a manifest whose unread entries were never fetched is a false pass. A CI
* job that wants to tolerate oversized catalogs can allow 8 and still fail
* on 7.
*/
SKILL_INCOMPLETE: 8,
} as const;
/** Machine-readable error envelope written as one JSON line on stderr. */

View file

@ -1,5 +1,5 @@
import { awaitableError, awaitableLog } from "../utils/awaitable-log.js";
import { CliExitCodeError } from "../error-handler.js";
import { CliExitCodeError, EXIT_CODES } from "../error-handler.js";
import { emitResult } from "./emit-result.js";
import type { MethodArgs, MethodOutcome } from "./method-types.js";
@ -30,7 +30,12 @@ export async function consumeMethodOutcome(
// last thing that happens.
if (outcome.exitCode) {
throw new CliExitCodeError(outcome.exitCode, outcome.summary ?? "", {
code: "skills_nonconformant",
// The envelope's `code` follows the exit code, so a caller reading one
// never has to reconcile it against the other.
code:
outcome.exitCode === EXIT_CODES.SKILL_INCOMPLETE
? "skills_incomplete"
: "skills_nonconformant",
});
}
return;

View file

@ -15,6 +15,7 @@ import { collectAppInfo } from "./collect-app-info.js";
import { summarizeSkillVerification } from "./skills-verify.js";
import {
allSkillsVerified,
anySkillFailed,
verifySkills,
} from "@inspector/core/mcp/skillsVerification.js";
import type {
@ -334,9 +335,17 @@ export async function runMethod(
kind: "ndjson",
lines: reports,
summary: summarizeSkillVerification(reports),
// Three outcomes, three exit codes: a broken MUST is 7, a walk the
// read bounds cut short is 8, and everything checked and passing is
// 0. Collapsing the middle case into either of the others reports
// something untrue about the server (Copilot).
...(allSkillsVerified(reports)
? {}
: { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }),
: {
exitCode: anySkillFailed(reports)
? EXIT_CODES.SKILL_NONCONFORMANT
: EXIT_CODES.SKILL_INCOMPLETE,
}),
};
}
result = { skills };
@ -371,9 +380,17 @@ export async function runMethod(
kind: "ndjson",
lines: reports,
summary: summarizeSkillVerification(reports),
// Three outcomes, three exit codes: a broken MUST is 7, a walk the
// read bounds cut short is 8, and everything checked and passing is
// 0. Collapsing the middle case into either of the others reports
// something untrue about the server (Copilot).
...(allSkillsVerified(reports)
? {}
: { exitCode: EXIT_CODES.SKILL_NONCONFORMANT }),
: {
exitCode: anySkillFailed(reports)
? EXIT_CODES.SKILL_NONCONFORMANT
: EXIT_CODES.SKILL_INCOMPLETE,
}),
};
}
result = envelope;