feat(sdk-runner): add rust-sdk + per-spec-version config overlays (#419)
* feat(sdk-runner): add rust-sdk to KNOWN_SDKS The mcp-conformance package's fixture bins follow the harness contract directly (URL argv + MCP_CONFORMANCE_SCENARIO env), and the server selects the SEP-2575 stateless lifecycle via STATELESS=1. Requires the repo's pinned Rust toolchain (rustup installs it on demand). * feat(sdk-runner): per-spec-version config overlays (specOverrides) Several KNOWN_SDKS entries needed a different invocation when targeting 2026-07-28 — go-sdk's -stateless flag, csharp-sdk's /stateless URL, rust-sdk's STATELESS=1 env, python-sdk's per-revision baseline — and each carried a comment telling the caller which flag to paste. Encode those variants in the entry instead: specOverrides maps a canonical spec version to a partial config that is merged over the base entry when a run targets that version (--spec-version, after 'draft' alias resolution, or the entry's own specVersion default). Explicit CLI flags still win. Keys are validated against the known spec versions at schema level so a 'draft' key or a typo'd date fails loudly instead of silently never matching. * style: prettier fixes in sdk-runner tests * refactor(sdk-runner): reuse isSpecVersion for specOverrides key validation Reuses the canonical spec-version list from #421 instead of keeping a separate copy; SPEC_VERSION_TIMELINE is now exported so the validation message can enumerate the valid keys.
This commit is contained in:
Paul Carleton
GitHub
parent
5cc567c399
commit
49103de6ed
6 changed files with 241 additions and 28 deletions
23
README.md
23
README.md
|
|
@ -299,14 +299,25 @@ npm start -- sdk typescript-sdk --mode client --spec-version draft
|
|||
|
||||
Build/run commands for each official SDK are looked up by name from [`src/sdk-runner/known-sdks.ts`](src/sdk-runner/known-sdks.ts) — no config file is required in the SDK repo. Resolution order is **CLI flag > built-in entry**, so any field can be overridden on the command line for refs that diverge from the built-in.
|
||||
|
||||
An SDK can have more than one entry when its layout differs across major versions — e.g. `typescript-sdk` (v2, the `main` monorepo) and `typescript-sdk-v1` (the published npm v1.x line). An entry may set `defaultRef` (the branch used when you don't pass `@<ref>`) and `repo` (the real clone target when the entry name is an alias). Overriding built-in fields for a one-off run:
|
||||
An SDK can have more than one entry when its layout differs across major versions — e.g. `typescript-sdk` (v2, the `main` monorepo) and `typescript-sdk-v1` (the published npm v1.x line). An entry may set `defaultRef` (the branch used when you don't pass `@<ref>`) and `repo` (the real clone target when the entry name is an alias).
|
||||
|
||||
When the right invocation depends on the spec version being targeted, the entry carries it in `specOverrides` instead of a comment to copy from. The matching entry is merged over the base config when you pass `--spec-version` (or when the entry's own `specVersion` default applies), so version-specific runs need no extra flags:
|
||||
|
||||
```bash
|
||||
# e.g. a 2026-07-28 run against go-sdk's server — the built-in server command
|
||||
# pins -stateless=false (what the dated-spec suites need), so override it to
|
||||
# get the SEP-2575 stateless lifecycle:
|
||||
npm start -- sdk go-sdk --mode server --suite all --spec-version 2026-07-28 \
|
||||
--server-cmd './.conformance-server -http=localhost:3000'
|
||||
# go-sdk's server pins -stateless=false for the dated-spec suites; its
|
||||
# specOverrides['2026-07-28'] entry swaps in the stateless invocation, so
|
||||
# this is the whole command:
|
||||
npm start -- sdk go-sdk --mode server --suite all --spec-version 2026-07-28
|
||||
|
||||
# same for csharp-sdk (stateless URL), rust-sdk (STATELESS=1 env), and
|
||||
# python-sdk (per-revision expected-failures baseline)
|
||||
```
|
||||
|
||||
Explicit CLI flags still beat everything, config included — overriding a field for a one-off run:
|
||||
|
||||
```bash
|
||||
npm start -- sdk go-sdk@my-fork-branch --mode server \
|
||||
--build-cmd 'go build -o ./.conformance-server ./experimental/server'
|
||||
```
|
||||
|
||||
To add a new SDK to the matrix, add an entry to `KNOWN_SDKS`.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,32 @@
|
|||
import { z } from 'zod';
|
||||
import {
|
||||
DRAFT_PROTOCOL_VERSION,
|
||||
SPEC_VERSION_TIMELINE,
|
||||
isSpecVersion
|
||||
} from '../types';
|
||||
|
||||
// Fields an entry may vary per targeted spec version. Identity fields
|
||||
// (repo / defaultRef / specVersion) can't vary — they pick what to clone,
|
||||
// not how to drive it. `server` is partial here so an override can change
|
||||
// just the command (go-sdk) or just the url (csharp-sdk).
|
||||
const SpecOverrideSchema = z.object({
|
||||
build: z.string().optional(),
|
||||
client: z
|
||||
.object({
|
||||
command: z.string()
|
||||
})
|
||||
.optional(),
|
||||
server: z
|
||||
.object({
|
||||
command: z.string().optional(),
|
||||
url: z.string().url().optional(),
|
||||
readyTimeoutMs: z.number().int().positive().optional()
|
||||
})
|
||||
.optional(),
|
||||
expectedFailures: z.string().optional()
|
||||
});
|
||||
|
||||
export type SpecOverride = z.infer<typeof SpecOverrideSchema>;
|
||||
|
||||
export const SdkConfigSchema = z.object({
|
||||
// Clone this repo instead of the KNOWN_SDKS key — lets an alias entry
|
||||
|
|
@ -23,7 +51,61 @@ export const SdkConfigSchema = z.object({
|
|||
// Spec version this SDK targets, used as the default --spec-version when
|
||||
// the flag isn't given (e.g. a v1 SDK pinned to the latest dated spec).
|
||||
// An explicit --spec-version on the sdk command always wins.
|
||||
specVersion: z.string().optional()
|
||||
specVersion: z.string().optional(),
|
||||
// Per-spec-version defaults, keyed by the canonical spec version a run
|
||||
// targets (--spec-version after alias resolution — so keys are the dated
|
||||
// strings, never 'draft' — or the entry's own specVersion). Matched entries
|
||||
// are merged over the base config field-by-field before CLI flags apply, so
|
||||
// `sdk go-sdk --mode server --spec-version 2026-07-28` picks up the right
|
||||
// server invocation with no manual overrides.
|
||||
// Precedence: CLI flag > specOverrides > base entry.
|
||||
specOverrides: z
|
||||
.record(z.string(), SpecOverrideSchema)
|
||||
.superRefine((overrides, ctx) => {
|
||||
// Keys must be canonical spec versions: a 'draft' key (or a typo'd
|
||||
// date) would silently never match, because the requested version is
|
||||
// resolved to its dated form before the lookup.
|
||||
for (const key of Object.keys(overrides)) {
|
||||
if (!isSpecVersion(key)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message:
|
||||
`specOverrides key '${key}' is not a spec version. ` +
|
||||
`Use one of: ${SPEC_VERSION_TIMELINE.join(', ')} ` +
|
||||
`('draft' resolves to ${DRAFT_PROTOCOL_VERSION} before lookup, so key the dated form).`
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
.optional()
|
||||
});
|
||||
|
||||
export type SdkConfig = z.infer<typeof SdkConfigSchema>;
|
||||
|
||||
/**
|
||||
* Resolve the config to use for a run targeting `specVersion`: the base
|
||||
* entry with the matching specOverrides entry (if any) merged on top.
|
||||
* Returns the input unchanged when nothing matches; never mutates it.
|
||||
*/
|
||||
export function resolveConfigForSpec(
|
||||
config: SdkConfig,
|
||||
specVersion: string | undefined
|
||||
): SdkConfig {
|
||||
const override = specVersion
|
||||
? config.specOverrides?.[specVersion]
|
||||
: undefined;
|
||||
if (!override) return config;
|
||||
const server =
|
||||
config.server || override.server
|
||||
? { ...config.server, ...override.server }
|
||||
: undefined;
|
||||
return {
|
||||
...config,
|
||||
build: override.build ?? config.build,
|
||||
client: override.client ?? config.client,
|
||||
// The base schema guarantees command+url when a base server exists; an
|
||||
// override without a base server must itself be complete to be usable.
|
||||
server: server as SdkConfig['server'],
|
||||
expectedFailures: override.expectedFailures ?? config.expectedFailures
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { spawn, ChildProcess } from 'child_process';
|
||||
import path from 'path';
|
||||
import { Command, Option } from 'commander';
|
||||
import { SdkConfig } from './config';
|
||||
import { SdkConfig, resolveConfigForSpec } from './config';
|
||||
import { parseSdkSpec, ensureCheckout } from './checkout';
|
||||
import { lookupBuiltinConfig, knownSdkNames } from './known-sdks';
|
||||
import { resolveSpecVersion } from '../scenarios';
|
||||
|
||||
type Mode = 'client' | 'server';
|
||||
|
||||
|
|
@ -190,7 +191,17 @@ export function createSdkCommand(): Command {
|
|||
spec?.name ?? path.basename(path.resolve(options.path!));
|
||||
|
||||
// Resolution: CLI flag > built-in entry (KNOWN_SDKS).
|
||||
const builtinConfig: SdkConfig = lookupBuiltinConfig(sdkName) ?? {};
|
||||
const baseConfig: SdkConfig = lookupBuiltinConfig(sdkName) ?? {};
|
||||
|
||||
// The targeted spec version (explicit flag wins over the per-SDK
|
||||
// default) selects that version's specOverrides entry, so version-
|
||||
// specific invocations live in config instead of copy-paste flags.
|
||||
// 'draft' resolves to its dated alias so overlay keys stay canonical.
|
||||
const requestedSpec = options.specVersion ?? baseConfig.specVersion;
|
||||
const specVersion: string | undefined = requestedSpec
|
||||
? resolveSpecVersion(requestedSpec)
|
||||
: undefined;
|
||||
const builtinConfig = resolveConfigForSpec(baseConfig, specVersion);
|
||||
|
||||
// The built-in entry may be an alias (e.g. typescript-sdk-v1): honor its
|
||||
// `repo` (real clone target) and `defaultRef` (branch when no @ref given).
|
||||
|
|
@ -223,10 +234,6 @@ export function createSdkCommand(): Command {
|
|||
const output = options.output
|
||||
? path.resolve(options.output)
|
||||
: undefined;
|
||||
// Explicit flag wins over the per-SDK default.
|
||||
const specVersion: string | undefined =
|
||||
options.specVersion ?? builtinConfig.specVersion;
|
||||
|
||||
if (buildCmd && !options.skipBuild) {
|
||||
console.error(`[sdk] Building: ${buildCmd}`);
|
||||
await execShell(buildCmd, dir);
|
||||
|
|
|
|||
|
|
@ -40,10 +40,9 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
},
|
||||
// Fixtures live under conformance/ (everything-client + everything-server,
|
||||
// mirroring scripts/{client,server}-conformance.sh). The server's -stateless
|
||||
// flag defaults to true; -stateless=false pins the stateful transport, which
|
||||
// the dated-spec (`active` suite) scenarios need — for a 2026-07-28 /
|
||||
// SEP-2575 run, override with --server-cmd './.conformance-server -http=localhost:3000'
|
||||
// to get the stateless lifecycle.
|
||||
// flag defaults to true; the dated-spec (`active` suite) scenarios need the
|
||||
// stateful transport, so the base command pins -stateless=false and the
|
||||
// 2026-07-28 override drops it for the SEP-2575 stateless lifecycle.
|
||||
'go-sdk': {
|
||||
build:
|
||||
'go build -o ./.conformance-server ./conformance/everything-server && go build -o ./.conformance-client ./conformance/everything-client',
|
||||
|
|
@ -54,14 +53,17 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
command: './.conformance-server -http=localhost:3000 -stateless=false',
|
||||
url: 'http://localhost:3000'
|
||||
},
|
||||
expectedFailures: 'conformance/baseline.yml'
|
||||
expectedFailures: 'conformance/baseline.yml',
|
||||
specOverrides: {
|
||||
'2026-07-28': {
|
||||
server: { command: './.conformance-server -http=localhost:3000' }
|
||||
}
|
||||
}
|
||||
},
|
||||
// main — targets the 2026-07-28 revision. Same uv workspace layout as v1.x
|
||||
// (client fixture in .github/actions/conformance/, mcp-everything-server
|
||||
// workspace package). Two baselines exist upstream: expected-failures.yml
|
||||
// covers runs at the latest dated spec (the default here), and
|
||||
// expected-failures.2026-07-28.yml covers --spec-version 2026-07-28 runs —
|
||||
// pass the latter via --expected-failures when targeting the new revision.
|
||||
// workspace package). Two baselines exist upstream, one per spec target;
|
||||
// the 2026-07-28 override picks the matching one.
|
||||
'python-sdk': {
|
||||
build: 'uv sync --frozen --all-extras --all-packages',
|
||||
client: {
|
||||
|
|
@ -71,7 +73,13 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
command: 'uv run --frozen mcp-everything-server --port 3000',
|
||||
url: 'http://localhost:3000/mcp'
|
||||
},
|
||||
expectedFailures: '.github/actions/conformance/expected-failures.yml'
|
||||
expectedFailures: '.github/actions/conformance/expected-failures.yml',
|
||||
specOverrides: {
|
||||
'2026-07-28': {
|
||||
expectedFailures:
|
||||
'.github/actions/conformance/expected-failures.2026-07-28.yml'
|
||||
}
|
||||
}
|
||||
},
|
||||
// v1.x — the stable, published line of the python-sdk, analogous to
|
||||
// typescript-sdk-v1. Clones the python-sdk repo, defaulting to the `v1.x`
|
||||
|
|
@ -96,6 +104,29 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
},
|
||||
expectedFailures: '.github/actions/conformance/expected-failures.yml'
|
||||
},
|
||||
// Fixtures live in conformance/ (the mcp-conformance package's
|
||||
// conformance-client + conformance-server bins; the package is excluded from
|
||||
// the workspace default-members, so build it explicitly). The server reads
|
||||
// PORT and STATELESS from the environment: the stateful (dated-spec)
|
||||
// lifecycle is the default, and the 2026-07-28 override sets STATELESS=1
|
||||
// for the SEP-2575 stateless lifecycle.
|
||||
'rust-sdk': {
|
||||
build: 'cargo build -p mcp-conformance',
|
||||
client: {
|
||||
command: './target/debug/conformance-client'
|
||||
},
|
||||
server: {
|
||||
command: 'PORT=3000 ./target/debug/conformance-server',
|
||||
url: 'http://localhost:3000/mcp'
|
||||
},
|
||||
specOverrides: {
|
||||
'2026-07-28': {
|
||||
server: {
|
||||
command: 'STATELESS=1 PORT=3000 ./target/debug/conformance-server'
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
// Fixtures live in tests/ModelContextProtocol.ConformanceClient and
|
||||
// tests/ModelContextProtocol.ConformanceServer (requires the .NET 10 SDK,
|
||||
// per global.json); build output goes to the repo-level artifacts/ tree
|
||||
|
|
@ -103,8 +134,8 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
// argument rather than reading MCP_CONFORMANCE_SCENARIO, so the command
|
||||
// bridges the env var into argv ($1 is the server URL the harness appends).
|
||||
// The server serves the stateful lifecycle at / and the SEP-2575 stateless
|
||||
// lifecycle at /stateless from the same port — for a 2026-07-28 run,
|
||||
// override with --server-url http://localhost:3000/stateless.
|
||||
// lifecycle at /stateless from the same port; the 2026-07-28 override
|
||||
// points runs at the stateless endpoint.
|
||||
'csharp-sdk': {
|
||||
build:
|
||||
'dotnet build tests/ModelContextProtocol.ConformanceClient -c Release -f net10.0 && dotnet build tests/ModelContextProtocol.ConformanceServer -c Release -f net10.0',
|
||||
|
|
@ -115,6 +146,11 @@ export const KNOWN_SDKS: Record<string, SdkConfig> = {
|
|||
command:
|
||||
'dotnet artifacts/bin/ModelContextProtocol.ConformanceServer/Release/net10.0/ModelContextProtocol.ConformanceServer.dll --urls http://localhost:3000',
|
||||
url: 'http://localhost:3000'
|
||||
},
|
||||
specOverrides: {
|
||||
'2026-07-28': {
|
||||
server: { url: 'http://localhost:3000/stateless' }
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { parseSdkSpec } from './checkout';
|
||||
import { SdkConfigSchema } from './config';
|
||||
import { lookupBuiltinConfig, KNOWN_SDKS } from './known-sdks';
|
||||
import { SdkConfigSchema, resolveConfigForSpec } from './config';
|
||||
|
||||
describe('parseSdkSpec', () => {
|
||||
it('leaves ref undefined when omitted (resolved later via defaultRef/main)', () => {
|
||||
|
|
@ -74,7 +74,7 @@ describe('lookupBuiltinConfig', () => {
|
|||
});
|
||||
|
||||
it('returns null for unknown SDKs', () => {
|
||||
expect(lookupBuiltinConfig('rust-sdk')).toBeNull();
|
||||
expect(lookupBuiltinConfig('swift-sdk')).toBeNull();
|
||||
});
|
||||
|
||||
it('exposes python-sdk-v1 with repo + defaultRef + specVersion and both commands', () => {
|
||||
|
|
@ -133,9 +133,86 @@ describe('lookupBuiltinConfig', () => {
|
|||
expect(cs?.server?.url).toBe('http://localhost:3000');
|
||||
});
|
||||
|
||||
it('exposes rust-sdk with the mcp-conformance fixture bins', () => {
|
||||
const rs = lookupBuiltinConfig('rust-sdk');
|
||||
expect(rs?.build).toBe('cargo build -p mcp-conformance');
|
||||
expect(rs?.client?.command).toBe('./target/debug/conformance-client');
|
||||
expect(rs?.server?.command).toContain('conformance-server');
|
||||
expect(rs?.server?.url).toBe('http://localhost:3000/mcp');
|
||||
});
|
||||
|
||||
it('every built-in entry validates against SdkConfigSchema', () => {
|
||||
for (const [name, cfg] of Object.entries(KNOWN_SDKS)) {
|
||||
expect(() => SdkConfigSchema.parse(cfg), name).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveConfigForSpec', () => {
|
||||
it("rejects a 'draft' or typo'd specOverrides key at schema level", () => {
|
||||
const bad = {
|
||||
server: { command: 'x', url: 'http://localhost:3000' },
|
||||
specOverrides: { draft: { server: { command: 'y' } } }
|
||||
};
|
||||
expect(() => SdkConfigSchema.parse(bad)).toThrow(/not a spec version/);
|
||||
const typo = { ...bad, specOverrides: { '2026-7-28': {} } };
|
||||
expect(() => SdkConfigSchema.parse(typo)).toThrow(/not a spec version/);
|
||||
});
|
||||
|
||||
it('returns the base config when no spec version is given', () => {
|
||||
const go = KNOWN_SDKS['go-sdk'];
|
||||
expect(resolveConfigForSpec(go, undefined)).toBe(go);
|
||||
});
|
||||
|
||||
it('returns the base config for a version with no override', () => {
|
||||
const go = KNOWN_SDKS['go-sdk'];
|
||||
expect(resolveConfigForSpec(go, '2025-11-25')).toBe(go);
|
||||
});
|
||||
|
||||
it('swaps the whole server command for go-sdk at 2026-07-28', () => {
|
||||
const resolved = resolveConfigForSpec(KNOWN_SDKS['go-sdk'], '2026-07-28');
|
||||
expect(resolved.server?.command).toBe(
|
||||
'./.conformance-server -http=localhost:3000'
|
||||
);
|
||||
// untouched fields carry through
|
||||
expect(resolved.server?.url).toBe('http://localhost:3000');
|
||||
expect(resolved.client?.command).toBe('./.conformance-client');
|
||||
expect(resolved.expectedFailures).toBe('conformance/baseline.yml');
|
||||
});
|
||||
|
||||
it('merges a partial server override (csharp url) over the base command', () => {
|
||||
const resolved = resolveConfigForSpec(
|
||||
KNOWN_SDKS['csharp-sdk'],
|
||||
'2026-07-28'
|
||||
);
|
||||
expect(resolved.server?.url).toBe('http://localhost:3000/stateless');
|
||||
expect(resolved.server?.command).toContain(
|
||||
'ModelContextProtocol.ConformanceServer.dll'
|
||||
);
|
||||
});
|
||||
|
||||
it('swaps the expected-failures baseline for python-sdk at 2026-07-28', () => {
|
||||
const resolved = resolveConfigForSpec(
|
||||
KNOWN_SDKS['python-sdk'],
|
||||
'2026-07-28'
|
||||
);
|
||||
expect(resolved.expectedFailures).toBe(
|
||||
'.github/actions/conformance/expected-failures.2026-07-28.yml'
|
||||
);
|
||||
expect(resolved.server?.command).toContain('mcp-everything-server');
|
||||
});
|
||||
|
||||
it('prefixes STATELESS=1 for rust-sdk at 2026-07-28', () => {
|
||||
const resolved = resolveConfigForSpec(KNOWN_SDKS['rust-sdk'], '2026-07-28');
|
||||
expect(resolved.server?.command).toBe(
|
||||
'STATELESS=1 PORT=3000 ./target/debug/conformance-server'
|
||||
);
|
||||
});
|
||||
|
||||
it('does not mutate the base entry', () => {
|
||||
const go = KNOWN_SDKS['go-sdk'];
|
||||
const before = JSON.stringify(go);
|
||||
resolveConfigForSpec(go, '2026-07-28');
|
||||
expect(JSON.stringify(go)).toBe(before);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ export const NEGOTIABLE_PROTOCOL_VERSIONS: readonly string[] = [
|
|||
export type SpecVersion = DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION;
|
||||
|
||||
/** Spec versions in timeline order, dated revisions followed by the draft. */
|
||||
const SPEC_VERSION_TIMELINE: readonly SpecVersion[] = [
|
||||
export const SPEC_VERSION_TIMELINE: readonly SpecVersion[] = [
|
||||
...DATED_SPEC_VERSIONS,
|
||||
DRAFT_PROTOCOL_VERSION
|
||||
];
|
||||
|
|
|
|||
Loading…
Reference in a new issue