ci: simplify conformance while running against both spec versions (#1060)

2026-07-27 18:32:07 -04:00 · 2026-07-27 18:32:07 -04:00 · 1f63e43479
commit 1f63e43479
parent 2047ae42ec
4 changed files with 22 additions and 62 deletions
--- a/.github/workflows/conformance.yml
+++ b/.github/workflows/conformance.yml
@ -12,12 +12,7 @@ concurrency:
  cancel-in-progress: true

 env:
-  # Pinned for reproducible runs; bump deliberately when the suite updates.
-  CONFORMANCE_VERSION: "0.1.16"
-  # When updating DRAFT_CONFORMANCE_VERSION, diff
-  #   `conformance list --spec-version 2026-07-28`
-  # and update #977
-  DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"
+  CONFORMANCE_VERSION: "0.2.0-alpha.9"

 jobs:
  server:
@ -33,15 +28,13 @@ jobs:

      - uses: Swatinem/rust-cache@v2

-      # Build the whole package (server + client bins): the conformance crate is
-      # excluded from the workspace default-members.
      - name: Build conformance binaries
        run: cargo build -p mcp-conformance

      - name: Test conformance server
        run: cargo test -p mcp-conformance --bin conformance-server

-      - name: Start 2025-11-25 server
+      - name: Start conformance server
        run: |
          PORT=8001 ./target/debug/conformance-server &
          echo $! > server.pid
@ -58,45 +51,19 @@ jobs:
        run: |
          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
            --url http://127.0.0.1:8001/mcp \
+            --suite all \
            --spec-version 2025-11-25 \
-            -o conformance-results
+            -o conformance-results/2025-11-25

-      # These pass today but are excluded from the default "active" suite;
-      # run them explicitly so regressions are still caught.
-      - name: Run 2025-11-25 pending scenarios
+      - name: Run 2026-07-28 server suite
        run: |
-          for scenario in json-schema-2020-12 server-sse-polling; do
-            npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
-              --url http://127.0.0.1:8001/mcp \
-              --scenario "$scenario" \
-              -o conformance-results
-          done
-
-      - name: Start draft server
-        run: |
-          STATELESS=1 PORT=8002 ./target/debug/conformance-server &
-          echo $! > draft-server.pid
-          for _ in $(seq 1 30); do
-            if curl -s -o /dev/null http://127.0.0.1:8002/mcp; then
-              exit 0
-            fi
-            sleep 1
-          done
-          echo "draft conformance server did not become ready" >&2
-          exit 1
-
-      - name: Run 2026-07-28 versioned-spec server suite
-        run: |
-          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
-            --url http://127.0.0.1:8002/mcp \
+          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
+            --url http://127.0.0.1:8001/mcp \
            --suite all \
            --spec-version 2026-07-28 \
-            -o conformance-results
+            -o conformance-results/2026-07-28

-      # Extension scenarios are excluded by the --spec-version filter and
-      # are informational for tiering. Run them explicitly so their gaps remain
-      # visible and newly passing scenarios make the strict baseline fail stale.
-      - name: Run Tasks extension server suite (informational)
+      - name: Run extension server scenarios (informational)
        run: |
          scenarios=(
            tasks-lifecycle
@ -110,20 +77,17 @@ jobs:
            tasks-required-task-error
            tasks-mrtr-composition
          )
-
          for scenario in "${scenarios[@]}"; do
-            npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
-              --url http://127.0.0.1:8002/mcp \
+            npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" server \
+              --url http://127.0.0.1:8001/mcp \
              --scenario "$scenario" \
              --expected-failures conformance/expected-failures-extensions.yaml \
              -o conformance-extension-results
          done

-      - name: Stop conformance servers
+      - name: Stop conformance server
        if: always()
-        run: |
-          kill "$(cat draft-server.pid)" 2>/dev/null || true
-          kill "$(cat server.pid)" 2>/dev/null || true
+        run: kill "$(cat server.pid)" 2>/dev/null || true

      - name: Upload results
        if: always()
@ -156,19 +120,19 @@ jobs:
            --command "$(pwd)/target/debug/conformance-client" \
            --suite all \
            --spec-version 2025-11-25 \
-            -o conformance-client-results/full
+            -o conformance-client-results/2025-11-25

-      - name: Run 2026-07-28 versioned-spec client suite
+      - name: Run 2026-07-28 client suite
        run: |
-          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
+          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --suite all \
            --spec-version 2026-07-28 \
-            -o conformance-client-results/draft
+            -o conformance-client-results/2026-07-28

      - name: Run extension client suite (informational)
        run: |
-          npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
+          npx -y "@modelcontextprotocol/conformance@${CONFORMANCE_VERSION}" client \
            --command "$(pwd)/target/debug/conformance-client" \
            --suite extensions \
            --expected-failures conformance/expected-failures-extensions.yaml \
--- a/conformance/expected-failures-extensions.yaml
+++ b/conformance/expected-failures-extensions.yaml
@ -1,5 +1,5 @@
 # Known failures for informational extension scenarios in
-# @modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION.
+# @modelcontextprotocol/conformance CONFORMANCE_VERSION.
 #
 # Extensions are not selected by a `--spec-version` run and do not count toward
 # SDK tiering. CI runs them separately so a green versioned-spec suite does not
@ -9,7 +9,7 @@
 # - an unlisted failure fails the build
 # - a listed scenario that starts passing fails the build as a stale entry
 #
-# When bumping DRAFT_CONFORMANCE_VERSION, review the available extension and
+# When bumping CONFORMANCE_VERSION, review the available extension and
 # pending scenarios and update this file deliberately.

 # The SEP-2663 Tasks Extension server scenarios (tracked in #868) all pass and
--- a/conformance/src/bin/client.rs
+++ b/conformance/src/bin/client.rs
@ -943,8 +943,7 @@ fn preferred_protocol_versions() -> Vec<ProtocolVersion> {
    preferred_versions
 }

-/// Runs draft stateless scenarios through the public discover lifecycle and
-/// Streamable HTTP transport.
+/// Runs scenarios through the discover lifecycle and Streamable HTTP transport.
 async fn run_discover_client(server_url: &str) -> anyhow::Result<()> {
    let preferred_versions = preferred_protocol_versions();
    let transport = StreamableHttpClientTransport::from_uri(server_url);
--- a/conformance/src/bin/server.rs
+++ b/conformance/src/bin/server.rs
@ -1740,10 +1740,7 @@ async fn main() -> anyhow::Result<()> {
    tracing::info!("Starting conformance server on {}", bind_addr);

    let server = ConformanceServer::new();
-    let stateless = std::env::var_os("STATELESS").is_some();
-    let config = StreamableHttpServerConfig::default()
-        .with_legacy_session_mode(!stateless)
-        .with_json_response(stateless);
+    let config = StreamableHttpServerConfig::default();
    let service = StreamableHttpService::new(
        move || Ok(server.clone()),
        LocalSessionManager::default().into(),