Pin each conformance leg to a spec-revision wire (#3304)

.github/actions/conformance/expected-failures.2025-11-25.yml

@@ -0,0 +1,26 @@
+# Expected failures at the 2025-11-25 wire, read by the `--suite all
+# --spec-version 2025-11-25` server and client legs: every scenario the pinned
+# harness marks applicable at 2025-11-25, run over the stateful handshake.
+#
+# That selection is a superset of the frozen 2025-11-25 requirement set (`npx
+# $CONFORMANCE_PKG list --requirements 2025-11-25`), so an entry here for a
+# scenario in that set means `conformance tier-check` scores the SDK below 100%
+# for 2025-11-25 even though CI is green; link the tracking issue next to any
+# such entry.
+#
+# Kept separate from expected-failures.yml even though the bare `--suite all`
+# legs currently run these same scenarios at 2025-11-25 too: the harness's
+# default wire is due to move (modelcontextprotocol/conformance#426) and this
+# file must keep meaning 2025-11-25. Until then a failure here also needs an
+# entry there. The file is passed even while empty: run-client.sh keys its solo
+# re-verification off the harness's baseline output.
+#
+# Baseline established against the harness pinned via CONFORMANCE_PKG in
+# .github/workflows/conformance.yml. New conformance releases are adopted by
+# deliberately bumping that pin and reconciling all three expected-failures
+# files in the same change. Entries MUST be removed as they start passing (the
+# runner fails on stale entries).
+
+client: []
+
+server: []

.github/actions/conformance/expected-failures.2026-07-28.yml

@@ -1,19 +1,23 @@
-# Expected failures for the carried-forward 2026-07-28 legs
+# Expected failures at the 2026-07-28 wire, read by the `--suite all
-# (`--suite all --spec-version 2026-07-28` for both server and client).
+# --spec-version 2026-07-28` server and client legs: every scenario the pinned
+# harness marks applicable at 2026-07-28, run stateless with per-request _meta.
+#
+# That selection is a superset of the frozen 2026-07-28 requirement set (`npx
+# $CONFORMANCE_PKG list --requirements 2026-07-28`), so an entry here for a
+# scenario in that set means `conformance tier-check` scores the SDK below 100%
+# for 2026-07-28 even though CI is green; link the tracking issue next to any
+# such entry.
 #
 # This baseline is separate from expected-failures.yml because entries are
-# keyed by scenario name only: a scenario that passes at its default version
+# keyed by scenario name only: a scenario that passes at the harness default
-# in the 2025 legs but fails when forced to 2026-07-28 (or vice versa) cannot
+# wire in the bare `--suite all` legs but fails when forced to 2026-07-28 (or
-# be expressed in a shared file (the passing leg would flag the entry as
+# vice versa) cannot be expressed in a shared file. Where the bare legs already
-# stale). Like expected-failures.yml, this single file covers both
+# default a scenario to 2026-07-28, a failure here also needs an entry there.
-# directions: the client 2026 leg reads the `client:` section and the server
-# 2026 leg reads the `server:` section. Both burn down independently of the
-# 2025 legs.
 #
 # Baseline established against the harness pinned via CONFORMANCE_PKG in
 # .github/workflows/conformance.yml. New conformance releases are adopted by
-# deliberately bumping that pin and reconciling both this file and
+# deliberately bumping that pin and reconciling all three expected-failures
-# expected-failures.yml in the same change.
+# files in the same change.
 #
 # Entries are grouped by what unblocks them. As each gap closes the
 # corresponding scenarios start passing and MUST be removed from this list

.github/actions/conformance/expected-failures.yml

@@ -1,10 +1,16 @@
-# Conformance scenarios not yet passing against the Python SDK on main.
+# Conformance scenarios not yet passing against the Python SDK on main, read by
-# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
+# the bare `--suite all` server and client legs (every scenario shipped with the
+# pinned harness, each at the harness's default wire for it). CI exits 0 if only
+# these fail, exits 1 on unexpected failures or stale entries.
+#
+# The wire-pinned legs read expected-failures.2025-11-25.yml and
+# expected-failures.2026-07-28.yml instead; see the header of
+# .github/workflows/conformance.yml for the leg layout.
 #
 # Baseline established against the harness pinned via CONFORMANCE_PKG in
 # .github/workflows/conformance.yml. New conformance releases are adopted by
-# deliberately bumping that pin and reconciling both this file and
+# deliberately bumping that pin and reconciling all three expected-failures
-# expected-failures.2026-07-28.yml in the same change.
+# files in the same change.
 #
 # Entries are grouped by SEP. As each SEP lands in the SDK the corresponding
 # scenarios start passing and MUST be removed from this list (the runner fails
@@ -30,10 +36,9 @@ client:
 
 server:
   # SEP-2663 (io.modelcontextprotocol/tasks): the SDK does not implement the
-  # tasks extension yet. These extension-tagged scenarios are selected only by
+  # tasks extension yet. The bare `--suite all` leg is the only one that
-  # the bare `--suite all` leg — extension scenarios never match a
+  # reaches these extension-tagged scenarios (they never match a
-  # --spec-version filter and the active/draft suites exclude them — so these
+  # --spec-version filter), so these entries live here only.
-  # entries are inert for the other legs that read this file.
   #
   # `tasks-status-notifications` is intentionally NOT listed: the harness
   # skips it unconditionally (pending its rewrite against subscriptions/

.github/actions/conformance/run-client.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Run a client conformance suite, re-verifying unexpected failures solo.
-# Concurrent suite runs on a 2-vCPU runner can push scenarios with real-time
+# Concurrent suite runs on a hosted runner can push scenarios with real-time
 # waits past tolerance; solo, a real failure fails again while a contention
 # artifact passes. Failures that only reproduce under concurrency are excused.
 set -uo pipefail
@@ -101,4 +101,8 @@ if [ -n "$output_dir" ]; then
     printf '%s\n' "${scenarios[@]}" > "$output_dir/FLAKE_RESCUED"
 fi
 echo "All ${#scenarios[@]} unexpected failure(s) passed when re-run solo; the suite failures were parallel-run contention."
+# Surface the rescue on the run summary; the log line above and the
+# FLAKE_RESCUED marker in the uploaded results are otherwise easy to miss on a
+# green run.
+echo "::warning title=Conformance flake rescued::${scenarios[*]} failed under suite concurrency and passed solo${output_dir:+ (see ${output_dir}-solo and FLAKE_RESCUED in the uploaded results)}."
 exit 0

.github/workflows/conformance.yml

@@ -15,13 +15,28 @@ permissions:
 
 env:
   # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
-  # Bump deliberately and reconcile both
+  # Bump deliberately and reconcile the three
   # .github/actions/conformance/expected-failures*.yml files in the same change.
   CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"
 
+# Each job runs three `--suite all` legs, one per baseline file. The two
+# `--spec-version <rev>` legs run every scenario the pinned harness marks
+# applicable at that spec revision, at that revision's wire (stateful
+# handshake for 2025-11-25, stateless per-request _meta for 2026-07-28). Each
+# is a superset of the frozen requirement set that `conformance tier-check`
+# scores for that revision (`npx $CONFORMANCE_PKG list --requirements <rev>`
+# prints it), so an expected-failures.<rev>.yml entry for a scenario in that
+# set means tier-check reports the SDK below 100% for <rev>. The bare leg runs
+# every shipped scenario at whichever wire the harness defaults it to; it is
+# the only leg that reaches extension-tagged scenarios (tasks-*, auth/dpop*,
+# ...), which never match a --spec-version filter. For everything else it
+# overlaps one of the pinned legs, so a failure it also sees needs its own
+# expected-failures.yml entry besides the pinned file's.
+
 jobs:
   server-conformance:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
@@ -34,37 +49,21 @@ jobs:
         with:
           node-version: 24
       - run: uv sync --frozen --all-extras --package mcp-everything-server
-      - name: Run server conformance (active suite)
+      - name: Run server conformance (all suite, 2025-11-25 wire)
         run: >-
           ./.github/actions/conformance/run-server.sh
-          --suite active
+          --suite all
-          --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --spec-version 2025-11-25
-          --output-dir conformance-results/server-active
+          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
-      - name: Run server conformance (draft suite)
+          --output-dir conformance-results/server-2025-11-25
-        run: >-
+      - name: Run server conformance (all suite, 2026-07-28 wire)
-          ./.github/actions/conformance/run-server.sh
-          --suite draft
-          --expected-failures ./.github/actions/conformance/expected-failures.yml
-          --output-dir conformance-results/server-draft
-      - name: Run server conformance (2026-07-28 wire, all suite)
         run: >-
           ./.github/actions/conformance/run-server.sh
           --suite all
           --spec-version 2026-07-28
           --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
           --output-dir conformance-results/server-2026-07-28
-      - name: Run server conformance (all suite, extension scenarios)
+      - name: Run server conformance (all suite, default wire)
-        # A bare `--suite all` (no --spec-version) selects every scenario
-        # shipped with the pinned harness — including the extension-tagged
-        # tasks-* scenarios and pending-listed ones like server-sse-polling,
-        # which no other leg reaches (extension scenarios never match a
-        # --spec-version filter, and the pending list keeps them out of the
-        # active suite). Running the full set keeps unimplemented surfaces
-        # visible as baselined known failures in expected-failures.yml instead
-        # of silent exclusions, and stays robust to scenarios moving between
-        # harness suite lists across pin bumps. `--suite pending` would cover
-        # the same union slightly faster; the full set is preferred for the
-        # self-contained run and for parity with typescript-sdk's CI.
         run: >-
           ./.github/actions/conformance/run-server.sh
           --suite all
@@ -72,7 +71,9 @@ jobs:
           --output-dir conformance-results/server-all
       - name: Upload conformance results
         # The log has only summary counts; per-check data is in checks.json.
-        if: failure()
+        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
+        # and that is when the partial results matter most.
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: server-conformance-results
@@ -81,6 +82,7 @@ jobs:
 
   client-conformance:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
@@ -97,20 +99,21 @@ jobs:
       - run: uv sync --frozen --all-extras --package mcp --compile-bytecode
       - name: Pre-compile bytecode (editable sources)
         run: uv run --frozen python -m compileall -q src .github/actions/conformance
-      - name: Run client conformance (all suite)
+      # The harness runs a leg's scenarios via unbounded Promise.all; under
-        # The harness runs all scenarios via unbounded Promise.all; with 40
+      # that concurrency the slowest one (sse-retry, which has a real-time SSE
-        # scenarios on a 2-core runner the slowest one (sse-retry, which has a
+      # reconnect wait) needs more than the 30s default budget.
-        # real-time SSE reconnect wait) needs more than the 30s default budget.
+      # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
-        # `.venv/bin/python` (not `uv run`) avoids lockfile re-checks in ~40
+      # concurrent spawns; run-client.sh re-runs unexpected failures solo.
-        # concurrent spawns; run-client.sh re-runs unexpected failures solo.
+      - name: Run client conformance (all suite, 2025-11-25 wire)
         run: >-
           ./.github/actions/conformance/run-client.sh
           --command '.venv/bin/python .github/actions/conformance/client.py'
           --suite all
           --timeout 60000
-          --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --spec-version 2025-11-25
-          --output-dir conformance-results/client-all
+          --expected-failures ./.github/actions/conformance/expected-failures.2025-11-25.yml
-      - name: Run client conformance (2026-07-28 wire, all suite)
+          --output-dir conformance-results/client-2025-11-25
+      - name: Run client conformance (all suite, 2026-07-28 wire)
         run: >-
           ./.github/actions/conformance/run-client.sh
           --command '.venv/bin/python .github/actions/conformance/client.py'
@@ -119,10 +122,19 @@ jobs:
           --spec-version 2026-07-28
           --expected-failures ./.github/actions/conformance/expected-failures.2026-07-28.yml
           --output-dir conformance-results/client-2026-07-28
+      - name: Run client conformance (all suite, default wire)
+        run: >-
+          ./.github/actions/conformance/run-client.sh
+          --command '.venv/bin/python .github/actions/conformance/client.py'
+          --suite all
+          --timeout 60000
+          --expected-failures ./.github/actions/conformance/expected-failures.yml
+          --output-dir conformance-results/client-all
       - name: Upload conformance results
         # The log has only summary counts; per-check data is in checks.json.
-        # Also on FLAKE_RESCUED: rescued-flake evidence is otherwise discarded.
+        # always(), not !cancelled(): a timeout-minutes kill is a cancellation,
-        if: failure() || hashFiles('conformance-results/**/FLAKE_RESCUED') != ''
+        # and that is when the partial results matter most.
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: client-conformance-results