ci(conformance): pin harness to 0.2.0-alpha.3 with expected-failures baseline (#2877)

.github/actions/conformance/expected-failures.yml

@@ -0,0 +1,52 @@
+# Conformance scenarios not yet passing against the Python SDK on main.
+# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries.
+#
+# Baseline established against @modelcontextprotocol/conformance pinned in
+# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.3).
+# New conformance releases are adopted by deliberately bumping that pin and
+# reconciling this file in the same change.
+#
+# Entries are grouped by SEP. As each SEP lands in the SDK the corresponding
+# scenarios start passing and MUST be removed from this list (the runner fails
+# on stale entries), so the baseline burns down per milestone.
+
+client:
+  # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) ---
+  # SEP-2575 (request metadata / _meta envelope): client does not populate the
+  # _meta envelope or the MCP-Protocol-Version header semantics yet.
+  - request-metadata
+  # SEP-2322 (multi-round-trip requests): client does not echo requestState /
+  # handle IncompleteResult yet.
+  - sep-2322-client-request-state
+  # SEP-2243 (HTTP standardization): no fixture handler / client header support yet.
+  - http-custom-headers
+  - http-invalid-tool-headers
+  # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs.
+  - json-schema-ref-no-deref
+  # SEP-2468 (authorization response iss parameter): not implemented in the client.
+  - auth/iss-supported
+  - auth/iss-not-advertised
+  - auth/iss-supported-missing
+  - auth/iss-wrong-issuer
+  - auth/iss-unexpected
+  - auth/iss-normalized
+  - auth/metadata-issuer-mismatch
+  # SEP-2352 (authorization server migration): client does not re-register when
+  # PRM authorization_servers changes.
+  - auth/authorization-server-migration
+  # SEP-837 (application_type during DCR): the check only fires on draft-version
+  # runs; this draft scenario is the one place the client still hits it.
+  - auth/offline-access-not-supported
+
+  # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 ---
+  # SEP-2350 (scope step-up): WARNING-only; the expected-failures evaluator
+  # counts WARNINGs as failures.
+  - auth/scope-step-up
+  # SEP-990 (enterprise-managed authorization extension): no fixture handler /
+  # client support for the token-exchange + JWT bearer flow.
+  - auth/enterprise-managed-authorization
+
+# The `active` suite (30 scenarios / 42 assertions) is fully green against
+# mcp-everything-server. Draft-suite entries are added when the workflow
+# gains a `--suite draft` step.
+server: []

.github/actions/conformance/run-server.sh

@@ -7,15 +7,36 @@ SERVER_URL="http://localhost:${PORT}/mcp"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR/../../.."
 
-# Start everything-server
+# Refuse to start if something is already listening on the port. The readiness
+# check below cannot tell our server apart from a stale one, so a leftover
+# listener would mean silently running conformance against old code.
+if (: > "/dev/tcp/localhost/${PORT}") 2>/dev/null; then
+    echo "Error: port ${PORT} is already in use." >&2
+    echo "Stop the stale process first (lsof -ti:${PORT} -sTCP:LISTEN | xargs kill) or set PORT to a free port." >&2
+    exit 1
+fi
+
+echo "Starting mcp-everything-server on port ${PORT}..."
 uv run --frozen mcp-everything-server --port "$PORT" &
 SERVER_PID=$!
-trap "kill $SERVER_PID 2>/dev/null || true; wait $SERVER_PID 2>/dev/null || true" EXIT
 
-# Wait for server to be ready
+cleanup() {
+    echo "Stopping server (PID: ${SERVER_PID})..."
+    kill $SERVER_PID 2>/dev/null || true
+    wait $SERVER_PID 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Wait for server to be ready. --max-time keeps a hung listener from wedging
+# the loop, and a dead server process fails fast instead of retrying.
+echo "Waiting for server to be ready..."
 MAX_RETRIES=30
 RETRY_COUNT=0
-while ! curl -s "$SERVER_URL" > /dev/null 2>&1; do
+while ! curl -s --max-time 2 "$SERVER_URL" > /dev/null 2>&1; do
+    if ! kill -0 $SERVER_PID 2>/dev/null; then
+        echo "Server process exited unexpectedly" >&2
+        exit 1
+    fi
     RETRY_COUNT=$((RETRY_COUNT + 1))
     if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
         echo "Server failed to start after ${MAX_RETRIES} retries" >&2
@@ -26,5 +47,5 @@ done
 
 echo "Server ready at $SERVER_URL"
 
-# Run conformance tests
-npx @modelcontextprotocol/conformance@0.1.10 server --url "$SERVER_URL" "$@"
+npx --yes @modelcontextprotocol/conformance@"${CONFORMANCE_VERSION:?set CONFORMANCE_VERSION (pinned in .github/workflows/conformance.yml)}" \
+    server --url "$SERVER_URL" "$@"

.github/workflows/conformance.yml

@@ -13,10 +13,14 @@ concurrency:
 permissions:
   contents: read
 
+env:
+  # Pinned conformance harness version. Bump deliberately and reconcile
+  # .github/actions/conformance/expected-failures.yml in the same change.
+  CONFORMANCE_VERSION: "0.2.0-alpha.3"
+
 jobs:
   server-conformance:
     runs-on: ubuntu-latest
-    continue-on-error: true
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
@@ -29,11 +33,14 @@ jobs:
         with:
           node-version: 24
       - run: uv sync --frozen --all-extras --package mcp-everything-server
-      - run: ./.github/actions/conformance/run-server.sh
+      - name: Run server conformance (active suite)
+        run: >-
+          ./.github/actions/conformance/run-server.sh
+          --suite active
+          --expected-failures ./.github/actions/conformance/expected-failures.yml
 
   client-conformance:
     runs-on: ubuntu-latest
-    continue-on-error: true
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
@@ -46,4 +53,9 @@ jobs:
         with:
           node-version: 24
       - run: uv sync --frozen --all-extras --package mcp
-      - run: npx @modelcontextprotocol/conformance@0.1.13 client --command 'uv run --frozen python .github/actions/conformance/client.py' --suite all
+      - name: Run client conformance (all suite)
+        run: >-
+          npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client
+          --command 'uv run --frozen python .github/actions/conformance/client.py'
+          --suite all
+          --expected-failures ./.github/actions/conformance/expected-failures.yml