ci(conformance): bump harness to 0.2.0-alpha.5 preview (#2927)

.github/actions/conformance/expected-failures.2026-07-28.yml

@@ -10,10 +10,10 @@
 # 2026 leg reads the `server:` section. Both burn down independently of the
 # 2025 legs.
 #
-# Baseline established against @modelcontextprotocol/conformance pinned in
+# Baseline established against the harness pinned via CONFORMANCE_PKG in
-# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4).
+# .github/workflows/conformance.yml. New conformance releases are adopted by
-# New conformance releases are adopted by deliberately bumping that pin and
+# deliberately bumping that pin and reconciling both this file and
-# reconciling both this file and expected-failures.yml in the same change.
+# expected-failures.yml in the same change.
 #
 # Entries are grouped by what unblocks them. As each gap closes the
 # corresponding scenarios start passing and MUST be removed from this list

.github/actions/conformance/expected-failures.yml

@@ -1,11 +1,10 @@
 # Conformance scenarios not yet passing against the Python SDK on main.
 # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
 #
-# Baseline established against @modelcontextprotocol/conformance pinned in
+# Baseline established against the harness pinned via CONFORMANCE_PKG in
-# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4).
+# .github/workflows/conformance.yml. New conformance releases are adopted by
-# New conformance releases are adopted by deliberately bumping that pin and
+# deliberately bumping that pin and reconciling both this file and
-# reconciling both this file and expected-failures.2026-07-28.yml in the same
+# expected-failures.2026-07-28.yml in the same change.
-# change.
 #
 # Entries are grouped by SEP. As each SEP lands in the SDK the corresponding
 # scenarios start passing and MUST be removed from this list (the runner fails
@@ -41,9 +40,6 @@ client:
   - auth/iss-normalized
 
   # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 ---
-  # SEP-2350 (scope step-up): WARNING-only; the expected-failures evaluator
-  # counts WARNINGs as failures.
-  - auth/scope-step-up
   # SEP-990 (enterprise-managed authorization extension): no fixture handler /
   # client support for the token-exchange + JWT bearer flow.
   - auth/enterprise-managed-authorization

.github/actions/conformance/run-server.sh

@@ -47,5 +47,5 @@ done
 
 echo "Server ready at $SERVER_URL"
 
-npx --yes @modelcontextprotocol/conformance@"${CONFORMANCE_VERSION:?set CONFORMANCE_VERSION (pinned in .github/workflows/conformance.yml)}" \
+npx --yes "${CONFORMANCE_PKG:?set CONFORMANCE_PKG (pinned in .github/workflows/conformance.yml)}" \
     server --url "$SERVER_URL" "$@"

.github/workflows/conformance.yml

@@ -14,10 +14,16 @@ permissions:
   contents: read
 
 env:
-  # Pinned conformance harness version. Bump deliberately and reconcile
+  # Pinned conformance harness package spec (passed verbatim to `npx --yes`).
-  # both .github/actions/conformance/expected-failures*.yml files in the
+  # Use a published version, e.g. @modelcontextprotocol/conformance@0.2.0-alpha.5.
-  # same change.
+  # Bump deliberately and reconcile both
-  CONFORMANCE_VERSION: "0.2.0-alpha.4"
+  # .github/actions/conformance/expected-failures*.yml files in the same change.
+  #
+  # TODO: replace with @modelcontextprotocol/conformance@0.2.0-alpha.5 once
+  # https://github.com/modelcontextprotocol/conformance/pull/357 publishes, and
+  # drop CONFORMANCE_PKG_SHA256 plus the fetch-and-verify step below.
+  CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@65fcd39"
+  CONFORMANCE_PKG_SHA256: "9a381d7083f8be2fe7ae44efeca54530f18c61425805ddaf9cd88915efcc1574"
 
 jobs:
   server-conformance:
@@ -33,6 +39,19 @@ jobs:
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: 24
+      - name: Fetch and verify conformance harness
+        # Only when CONFORMANCE_PKG is a URL: download, check the recorded
+        # sha256, and re-point CONFORMANCE_PKG at the verified local tarball.
+        # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's
+        # own integrity check applies).
+        run: |
+          case "$CONFORMANCE_PKG" in
+            https://*)
+              curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz
+              echo "$CONFORMANCE_PKG_SHA256  /tmp/conformance.tgz" | sha256sum -c -
+              echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV"
+              ;;
+          esac
       - run: uv sync --frozen --all-extras --package mcp-everything-server
       - name: Run server conformance (active suite)
         run: >-
@@ -64,16 +83,25 @@ jobs:
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
           node-version: 24
+      - name: Fetch and verify conformance harness
+        run: |
+          case "$CONFORMANCE_PKG" in
+            https://*)
+              curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz
+              echo "$CONFORMANCE_PKG_SHA256  /tmp/conformance.tgz" | sha256sum -c -
+              echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV"
+              ;;
+          esac
       - run: uv sync --frozen --all-extras --package mcp
       - name: Run client conformance (all suite)
         run: >-
-          npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client
+          npx --yes "$CONFORMANCE_PKG" client
           --command 'uv run --frozen python .github/actions/conformance/client.py'
           --suite all
           --expected-failures ./.github/actions/conformance/expected-failures.yml
       - name: Run client conformance (2026-07-28 wire, all suite)
         run: >-
-          npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client
+          npx --yes "$CONFORMANCE_PKG" client
           --command 'uv run --frozen python .github/actions/conformance/client.py'
           --suite all
           --spec-version 2026-07-28