feat: sdk subcommand to run local conformance against any SDK ref (#277)

* feat: add sdk subcommand to run conformance against any SDK ref (#250)

* Revise sdk runner: explicit --mode, KNOWN_SDKS-only config, v1/v2 entries

Addresses review feedback on the sdk subcommand:

- Require --mode (client|server) and remove "both". Each invocation now
  tests exactly one side with its own exit code; the old default ran
  client then server but combined exit codes with ||=, which skipped the
  server side entirely whenever the client run failed.
- Resolve build/run config from KNOWN_SDKS + CLI flags only; drop the
  conformance.config.yaml loader (no SDK ships one yet). The Zod schema
  stays as the type for the built-in entries.
- Split the typescript entry by major version: typescript-sdk (v2/main,
  pnpm install + build:all, expected-failures.yaml) and typescript-sdk-v1
  (v1.x, npm ci + build, conformance-baseline.yml). An entry may set
  `repo` (real clone target for an alias) and `defaultRef` (branch used
  when no @ref is given). parseSdkSpec now leaves ref undefined when
  omitted so defaultRef can apply; a trailing @ is treated as no ref.
- Key the clone cache by ref (<repo>/<ref>) so different refs of the same
  repo no longer share one checkout.
- Bound the server readiness probe with a per-request AbortSignal timeout
  so a server that accepts the socket but never responds can't hang past
  the overall deadline.

* fix(sdk-runner): resolve -o to absolute path; add --expected-failures override; replaceAll for safeName
This commit is contained in:
Paul Carleton 2026-05-19 22:08:06 +01:00 • committed by GitHub
commit cfcecdba2c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -0,0 +1,61 @@
import type { SdkConfig } from './config';
/**
* Built-in conformance configs for official SDKs, keyed by repo name.
*
* These live here (not in the SDK repos) so adding an SDK to the matrix
* doesn't require a coordinated cross-repo PR. Any field can be overridden
* per-invocation via the CLI flags (--build-cmd / --client-cmd / etc.).
*/
export const KNOWN_SDKS: Record<string, SdkConfig> = {
// v2 — the monorepo on `main` (pnpm). Default ref is `main`.
'typescript-sdk': {
build: 'pnpm install && pnpm run build:all',
client: {
command: 'npx tsx test/conformance/src/everythingClient.ts'
},
server: {
command: 'npx tsx test/conformance/src/everythingServer.ts',
url: 'http://localhost:3000/mcp'
},
expectedFailures: 'test/conformance/expected-failures.yaml'
},
// v1.x — the published npm line. Same fixtures as v2; differs only in the
// build (npm, not pnpm) and the baseline filename. Clones the typescript-sdk
// repo, defaulting to the `v1.x` branch.
'typescript-sdk-v1': {
repo: 'typescript-sdk',
defaultRef: 'v1.x',
build: 'npm ci && npm run build',
client: {
command: 'npx tsx test/conformance/src/everythingClient.ts'
},
server: {
command: 'npx tsx test/conformance/src/everythingServer.ts',
url: 'http://localhost:3000/mcp'
},
expectedFailures: 'test/conformance/conformance-baseline.yml'
},
'go-sdk': {
build: 'go build -o ./.conformance-server ./examples/server/conformance',
// Upstream go-sdk has no client conformance fixture yet (see go-sdk#859).
server: {
command: './.conformance-server -http=:3000',
url: 'http://localhost:3000'
}
}
};
/**
* Look up a built-in config by SDK name. Accepts bare names (typescript-sdk),
* owner/repo (modelcontextprotocol/typescript-sdk), or a checkout path
* basename — only the final path segment is used as the key.
*/
export function lookupBuiltinConfig(name: string): SdkConfig | null {
const key = name.split('/').pop() ?? name;
return KNOWN_SDKS[key] ?? null;
}
export function knownSdkNames(): string[] {
return Object.keys(KNOWN_SDKS);
}