chore: run full conformance CI suite (#1010)

.github/workflows/conformance.yml

@@ -14,11 +14,15 @@ concurrency:
 env:
   # Pinned for reproducible runs; bump deliberately when the suite updates.
   CONFORMANCE_VERSION: "0.1.16"
+  # When updating DRAFT_CONFORMANCE_VERSION, diff
+  #   `conformance list --spec-version 2026-07-28`
+  # and update #977
   DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"
 
 jobs:
   server:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     permissions:
       contents: read
     steps:
@@ -81,7 +85,9 @@ jobs:
           echo "draft conformance server did not become ready" >&2
           exit 1
 
-      # Run discovery separately until #985 enables the full draft suite.
+      # useful for enforcing partial conformance for SEP-2575.
+      # when `server-stateless` is removed from `expected-failures-2026-07-28.yaml`,
+      # this step can be removed.
       - name: Run SEP-2575 discovery contract
         run: |
           endpoint=http://127.0.0.1:8002/mcp
@@ -91,6 +97,8 @@ jobs:
             -H "Mcp-Method: server/discover"
           )
 
+          # The stateless server streams responses as SSE, so unwrap the
+          # `data:` payload before parsing as JSON.
           discover_response="$(
             curl --fail-with-body --silent --show-error \
               "${common_headers[@]}" \
@@ -110,7 +118,8 @@ jobs:
                   }
                 }
               }' \
-              "$endpoint"
+              "$endpoint" \
+              | sed -n 's/^data: //p'
           )"
           jq -e '
             .result.resultType == "complete" and
@@ -152,35 +161,14 @@ jobs:
             (.error.data.supported | index("2026-07-28") != null)
           ' /tmp/unsupported-version.json
 
-      # Keep this explicit list until the full draft suite is enabled by #985.
+      - name: Run 2026-07-28 server suite
-      - name: Run supported draft server scenarios
         run: |
-          for scenario in \
+          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
-            sep-2164-resource-not-found \
+            --url http://127.0.0.1:8002/mcp \
-            caching \
+            --suite all \
-            http-header-validation \
+            --spec-version 2026-07-28 \
-            http-custom-header-server-validation \
+            --expected-failures conformance/expected-failures-2026-07-28.yaml \
-            input-required-result-basic-elicitation \
+            -o conformance-results
-            input-required-result-basic-sampling \
-            input-required-result-basic-list-roots \
-            input-required-result-request-state \
-            input-required-result-multiple-input-requests \
-            input-required-result-multi-round \
-            input-required-result-missing-input-response \
-            input-required-result-non-tool-request \
-            input-required-result-result-type \
-            input-required-result-unsupported-methods \
-            input-required-result-tampered-state \
-            input-required-result-capability-check \
-            input-required-result-ignore-extra-params \
-            input-required-result-validate-input \
-          ; do
-            npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
-              --url http://127.0.0.1:8002/mcp \
-              --scenario "$scenario" \
-              --spec-version draft \
-              -o conformance-results
-          done
 
       - name: Stop conformance servers
         if: always()
@@ -197,6 +185,7 @@ jobs:
 
   client:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     permissions:
       contents: read
     steps:
@@ -218,21 +207,14 @@ jobs:
             --spec-version 2025-11-25 \
             -o conformance-client-results/full
 
-      # SEP-2322 MRTR client scenario (spec 2026-07-28).
+      - name: Run 2026-07-28 client suite
-      - name: Run draft SEP-2322 client scenario
         run: |
           npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
             --command "$(pwd)/target/debug/conformance-client" \
-            --scenario sep-2322-client-request-state \
+            --suite all \
-            -o conformance-client-results/mrtr
+            --spec-version 2026-07-28 \
-
+            --expected-failures conformance/expected-failures-2026-07-28.yaml \
-      - name: Run draft SEP-2575 client scenario
+            -o conformance-client-results/draft
-        run: |
-          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
-            --command "$(pwd)/target/debug/conformance-client" \
-            --scenario request-metadata \
-            --spec-version draft \
-            -o conformance-client-results/sep-2575
 
       - name: Upload results
         if: always()

conformance/expected-failures-2026-07-28.yaml

@@ -0,0 +1,34 @@
+# Known failures for the pinned 2026-07-28 draft conformance suite
+# (@modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION).
+#
+# The full suites run in CI with `--expected-failures` pointing at this file:
+# - a scenario failing that is NOT listed here fails the build
+# - a scenario listed here that starts passing also fails the build (stale entry),
+#   so remove it from this list when the underlying issue is fixed.
+#
+# When bumping DRAFT_CONFORMANCE_VERSION, diff
+# `conformance list --spec-version 2026-07-28` and update #977.
+
+server:
+  # SEP-2575 stateless lifecycle gaps: discover capability declaration,
+  # missing-capability rejection, HTTP 404 for removed methods, and
+  # diagnostic tools (test_missing_capability, test_streaming_elicitation,
+  # test_logging_tool) not yet implemented.
+  # tracked in #1004
+  - server-stateless
+  # SEP-2106: composition/conditional/$anchor keywords are stripped from
+  # published tool input schemas.
+  # tracked in #1003
+  - json-schema-2020-12
+
+client:
+  # Client does not yet send MCP-Protocol-Version header pre-initialize as
+  # required by the 2026-07-28 stateless lifecycle.
+  # tracked in #1002
+  - tools_call
+  # Auth feature gaps in the 2026-07-28 auth scenarios.
+  # tracked in #1002
+  - auth/scope-step-up
+  # SEP-2352: SDK lacks issuer-stamped credential storage (#879), so the
+  # sep-2352-reregister-on-as-change check fails.
+  - auth/authorization-server-migration

conformance/src/bin/client.rs

@@ -1006,25 +1006,49 @@ async fn main() -> anyhow::Result<()> {
 
     tracing::info!("Running scenario '{}' against {}", scenario, server_url);
 
-    match scenario.as_str() {
+    // Safety net: some harness servers intentionally misbehave (e.g. reply
+    // with an id-less error instead of answering a request), which would
+    // leave the client waiting forever. Exit on our own before the harness's
+    // 30s client timeout so it never has to kill us (which has been observed
+    // to wedge the harness process in CI).
+    let timeout_secs: u64 = std::env::var("MCP_CONFORMANCE_TIMEOUT_SECS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(25);
+    tokio::time::timeout(
+        std::time::Duration::from_secs(timeout_secs),
+        run_scenario(&scenario, &server_url, &ctx),
+    )
+    .await
+    .map_err(|_| anyhow::anyhow!("Scenario '{scenario}' timed out after {timeout_secs}s"))??;
+
+    Ok(())
+}
+
+async fn run_scenario(
+    scenario: &str,
+    server_url: &str,
+    ctx: &ConformanceContext,
+) -> anyhow::Result<()> {
+    match scenario {
         // Non-auth scenarios
-        "initialize" => run_basic_client(&server_url).await?,
+        "initialize" => run_basic_client(server_url).await?,
         // SEP-2106: the scenario serves a tool whose schema carries a network
         // `$ref`; the check passes when the client lists tools without
         // dereferencing (fetching) that URL. A plain connect → list_tools →
         // close is sufficient; the scenario's mock server does not implement
         // the discover lifecycle, so `run_discover_client` hangs against it.
-        "json-schema-ref-no-deref" => run_basic_client(&server_url).await?,
+        "json-schema-ref-no-deref" => run_basic_client(server_url).await?,
-        "tools_call" => run_tools_call_client(&server_url, &ctx).await?,
+        "tools_call" => run_tools_call_client(server_url, ctx).await?,
         "elicitation-sep1034-client-defaults" => {
-            run_elicitation_defaults_client(&server_url).await?
+            run_elicitation_defaults_client(server_url).await?
         }
-        "sse-retry" => run_sse_retry_client(&server_url).await?,
+        "sse-retry" => run_sse_retry_client(server_url).await?,
         "request-metadata" | "sep-2322-client-request-state" => {
-            run_discover_client(&server_url).await?
+            run_discover_client(server_url).await?
         }
         "http-standard-headers" | "http-custom-headers" | "http-invalid-tool-headers" => {
-            run_discover_tools_call_client(&server_url, &ctx).await?
+            run_discover_tools_call_client(server_url, ctx).await?
         }
 
         // Auth scenarios - standard OAuth flow
@@ -1056,34 +1080,26 @@ async fn main() -> anyhow::Result<()> {
         | "auth/iss-wrong-issuer"
         | "auth/iss-unexpected"
         | "auth/iss-normalized"
-        | "auth/metadata-issuer-mismatch"
+        | "auth/metadata-issuer-mismatch" => run_auth_client(server_url, ctx).await?,
-        | "auth/metadata-issuer-mismatch"
-        // SEP-2352: PRM `authorization_servers` switches between calls; a
-        // compliant client re-registers at the new AS. Known partial failure:
-        // the SDK lacks issuer-stamped credential storage (#879), so the
-        // `sep-2352-reregister-on-as-change` check fails. Left on the standard
-        // flow rather than fixture-orchestrated re-registration so the
-        // conformance result reflects real SDK behavior.
-        | "auth/authorization-server-migration" => run_auth_client(&server_url, &ctx).await?,
 
         // Auth - scope step-up
-        "auth/scope-step-up" => run_auth_scope_step_up_client(&server_url, &ctx).await?,
+        "auth/scope-step-up" => run_auth_scope_step_up_client(server_url, ctx).await?,
 
         // Auth - scope retry limit
-        "auth/scope-retry-limit" => run_auth_scope_retry_limit_client(&server_url, &ctx).await?,
+        "auth/scope-retry-limit" => run_auth_scope_retry_limit_client(server_url, ctx).await?,
 
         // Auth - authorization server migration (SEP-2352)
         "auth/authorization-server-migration" => {
-            run_auth_server_migration_client(&server_url, &ctx).await?
+            run_auth_server_migration_client(server_url, ctx).await?
         }
 
         // Auth - pre-registration
-        "auth/pre-registration" => run_auth_preregistered_client(&server_url, &ctx).await?,
+        "auth/pre-registration" => run_auth_preregistered_client(server_url, ctx).await?,
 
         // Auth - resource mismatch (should fail to auth → pass)
         "auth/resource-mismatch" => {
             // Try to auth; it should fail because PRM resource doesn't match
-            match run_auth_client(&server_url, &ctx).await {
+            match run_auth_client(server_url, ctx).await {
                 Ok(_) => {
                     tracing::warn!("Auth succeeded despite resource mismatch!");
                 }
@@ -1094,12 +1110,12 @@ async fn main() -> anyhow::Result<()> {
         }
 
         // Auth - client credentials
-        "auth/client-credentials-basic" => run_client_credentials_basic(&server_url, &ctx).await?,
+        "auth/client-credentials-basic" => run_client_credentials_basic(server_url, ctx).await?,
-        "auth/client-credentials-jwt" => run_client_credentials_jwt(&server_url, &ctx).await?,
+        "auth/client-credentials-jwt" => run_client_credentials_jwt(server_url, ctx).await?,
 
         // Auth - cross-app access
         "auth/cross-app-access-complete-flow" => {
-            run_cross_app_access_client(&server_url, &ctx).await?
+            run_cross_app_access_client(server_url, ctx).await?
         }
 
         unknown => anyhow::bail!("Unsupported conformance scenario: {unknown}"),