diff --git a/.github/workflows/conformance-weekly.yaml b/.github/workflows/conformance-weekly.yaml index f12f379..ecc262f 100644 --- a/.github/workflows/conformance-weekly.yaml +++ b/.github/workflows/conformance-weekly.yaml @@ -1,9 +1,9 @@ name: conformance-weekly -# Runs the MCP conformance suite weekly against the latest -# @modelcontextprotocol/conformance release. The on:pull_request pipeline -# pins to whatever version is available at PR time; this schedule catches -# upstream releases that add scenarios between PRs. +# Runs the MCP conformance suite weekly against the newest published +# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact +# version; this schedule catches upstream releases that add scenarios between +# PRs, so the pin never silently goes stale. # # It also scores each run and publishes the client/server pass-rate as # shields.io endpoint JSON to the orphan `badges` branch (consumed by the @@ -17,10 +17,26 @@ permissions: contents: write issues: write +# The PR pipeline pins an exact conformance version so a PR only goes red for +# reasons in the PR. That pin is only safe because this job tracks the moving +# target instead — on both revisions, since each rides a different dist-tag: +# the dated one is on `latest`, the draft one only on `alpha`. jobs: server: - name: conformance / server (latest) + name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spec-version: '2025-11-25' + dist-tag: latest + path: '/' + baseline: conformance-baseline.yml + - spec-version: '2026-07-28' + dist-tag: alpha + path: '/stateless' + baseline: conformance-baseline-2026-07-28.yml steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -35,9 +51,17 @@ jobs: sleep 5 - name: Run conformance tests working-directory: ./tests/Conformance - run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml --output-dir results + run: >- + npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server + --url http://localhost:8000${{ matrix.path }} + --suite all + --spec-version ${{ matrix.spec-version }} + --expected-failures ${{ matrix.baseline }} + --output-dir results + # The badge tracks the released revision, so only the dated entry feeds + # it; the draft entry runs purely to catch upstream drift. - name: Generate score badge - if: always() + if: always() && matrix.spec-version == '2025-11-25' run: php tests/Conformance/score.php server - name: Show docker logs on failure if: failure() @@ -46,20 +70,30 @@ jobs: if: failure() uses: actions/upload-artifact@v7 with: - name: conformance-server-results + name: conformance-server-results-${{ matrix.spec-version }} path: | tests/Conformance/logs tests/Conformance/results - name: Upload score badge - if: always() + if: always() && matrix.spec-version == '2025-11-25' uses: actions/upload-artifact@v7 with: name: server-badge path: tests/Conformance/server-conformance.json client: - name: conformance / client (latest) + name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spec-version: '2025-11-25' + dist-tag: latest + baseline: conformance-baseline.yml + - spec-version: '2026-07-28' + dist-tag: alpha + baseline: conformance-baseline-2026-07-28.yml steps: - uses: actions/checkout@v7 - uses: shivammathur/setup-php@v2 @@ -73,20 +107,26 @@ jobs: - run: mkdir -p tests/Conformance/logs - name: Run conformance tests working-directory: ./tests/Conformance - run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml --output-dir results + run: >- + npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client + --command "php ${{ github.workspace }}/tests/Conformance/client.php" + --suite all + --spec-version ${{ matrix.spec-version }} + --expected-failures ${{ matrix.baseline }} + --output-dir results - name: Generate score badge - if: always() + if: always() && matrix.spec-version == '2025-11-25' run: php tests/Conformance/score.php client - name: Upload conformance results if: failure() uses: actions/upload-artifact@v7 with: - name: conformance-client-results + name: conformance-client-results-${{ matrix.spec-version }} path: | tests/Conformance/logs tests/Conformance/results - name: Upload score badge - if: always() + if: always() && matrix.spec-version == '2025-11-25' uses: actions/upload-artifact@v7 with: name: client-badge diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f7396e6..402e22b 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -5,6 +5,12 @@ permissions: contents: read pull-requests: write +env: + # Pinned so a PR only goes red for reasons in the PR. The 2026-07-28 + # scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them. + # conformance-weekly.yaml is what tracks upstream releases. + CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11" + jobs: unit: runs-on: ubuntu-latest @@ -95,8 +101,24 @@ jobs: run: vendor/bin/phpunit --testsuite=inspector conformance-server: - name: conformance / server + name: conformance / server (${{ matrix.spec-version }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + # One entry per lifecycle, not per spec revision: `--spec-version` is + # cumulative across the dated revisions, so 2025-11-25 already covers + # 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it + # is the SEP-2575 stateless wire, served from its own endpoint with its + # own baseline, so it needs its own run. + matrix: + include: + - spec-version: '2025-11-25' + path: '/' + baseline: conformance-baseline.yml + - spec-version: '2026-07-28' + path: '/stateless' + baseline: conformance-baseline-2026-07-28.yml + steps: - name: Checkout uses: actions/checkout@v7 @@ -118,7 +140,12 @@ jobs: - name: Run conformance tests working-directory: ./tests/Conformance - run: npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --expected-failures conformance-baseline.yml + run: >- + npx --yes ${{ env.CONFORMANCE_PKG }} server + --url http://localhost:8000${{ matrix.path }} + --suite all + --spec-version ${{ matrix.spec-version }} + --expected-failures ${{ matrix.baseline }} - name: Show logs on failure if: failure() @@ -146,8 +173,17 @@ jobs: run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml down conformance-client: - name: conformance / client + name: conformance / client (${{ matrix.spec-version }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spec-version: '2025-11-25' + baseline: conformance-baseline.yml + - spec-version: '2026-07-28' + baseline: conformance-baseline-2026-07-28.yml + steps: - name: Checkout uses: actions/checkout@v7 @@ -171,7 +207,12 @@ jobs: - name: Run client conformance tests working-directory: ./tests/Conformance - run: npx @modelcontextprotocol/conformance client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml + run: >- + npx --yes ${{ env.CONFORMANCE_PKG }} client + --command "php ${{ github.workspace }}/tests/Conformance/client.php" + --suite all + --spec-version ${{ matrix.spec-version }} + --expected-failures ${{ matrix.baseline }} - name: Show logs on failure if: failure() diff --git a/Makefile b/Makefile index b019429..c756792 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client conformance-draft conformance-draft-server conformance-draft-client check-conformance-repo docs +.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client conformance-draft conformance-draft-server conformance-draft-client docs -# The published conformance release carries no 2026-07-28 scenarios, so the -# draft targets run a local checkout instead. -# Override with `make conformance-draft CONFORMANCE_REPO=/path/to/conformance`. -CONFORMANCE_REPO ?= $(CURDIR)/../conformance -CONFORMANCE_DRAFT = node $(CONFORMANCE_REPO)/dist/index.js +# The 2026-07-28 scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has +# none of them. Pinned to the same version CI runs (see +# .github/workflows/pipeline.yaml), so a local pass means a green pipeline. +CONFORMANCE_VERSION ?= 0.2.0-alpha.11 +CONFORMANCE = npx --yes @modelcontextprotocol/conformance@$(CONFORMANCE_VERSION) deps-stable: composer update --prefer-stable @@ -47,29 +47,20 @@ conformance-client: php tests/Conformance/score.php client # --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------ -# Local-checkout only until upstream publishes the draft scenarios. conformance-draft: conformance-draft-server conformance-draft-client -check-conformance-repo: - @test -f $(CONFORMANCE_REPO)/dist/index.js || { \ - echo "No conformance build at $(CONFORMANCE_REPO)/dist/index.js."; \ - echo "Clone modelcontextprotocol/conformance and run 'npm install && npm run build' there,"; \ - echo "or point CONFORMANCE_REPO at an existing checkout."; \ - exit 1; \ - } - -conformance-draft-server: check-conformance-repo +conformance-draft-server: docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d @echo "Waiting for server to start..." @sleep 5 rm -rf tests/Conformance/results-2026-07-28 - cd tests/Conformance && $(CONFORMANCE_DRAFT) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true + cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true docker compose -f tests/Conformance/Fixtures/docker-compose.yml down -conformance-draft-client: check-conformance-repo +conformance-draft-client: rm -rf tests/Conformance/results-2026-07-28 - cd tests/Conformance && $(CONFORMANCE_DRAFT) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true + cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true coverage: XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage diff --git a/tests/Conformance/conformance-baseline-2026-07-28.yml b/tests/Conformance/conformance-baseline-2026-07-28.yml index dfc892a..faaa602 100644 --- a/tests/Conformance/conformance-baseline-2026-07-28.yml +++ b/tests/Conformance/conformance-baseline-2026-07-28.yml @@ -25,4 +25,43 @@ server: - json-schema-2020-12:json-schema-2020-12-tool-found - resources-templates-read:wire-schema-valid -client: [] +# The client does not speak 2026-07-28 at all yet: it still opens with +# `initialize` regardless of revision, so it never reaches the stateless wire +# (no per-request `_meta`, no Mcp-Method/Mcp-Name headers, no client-side +# requestState) — and separately, the SDK ships no OAuth client, so every +# scenario driving an authorization flow has nothing to exercise. +client: + - request-metadata + - http-standard-headers + - http-custom-headers + - http-invalid-tool-headers:sep-2243-invalid-tool-tools-list-gate + - http-invalid-tool-headers:sep-2243-client-reject-invalid-tool + - sep-2322-client-request-state + - tools_call:tool-add-numbers + - json-schema-2020-12-preservation:json-schema-2020-12-client-tool-found + - json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed + + - auth/metadata-default + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/metadata-issuer-mismatch + - auth/basic-cimd + - auth/pre-registration + - auth/scope-from-www-authenticate + - auth/scope-from-scopes-supported + - auth/scope-omitted-when-undefined + - auth/scope-step-up + - auth/scope-retry-limit + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-post + - auth/token-endpoint-auth-none + - auth/offline-access-scope + - auth/offline-access-not-supported + - auth/authorization-server-migration:sep-2352-reregister-on-as-change + - auth/iss-supported + - auth/iss-not-advertised + - auth/iss-supported-missing + - auth/iss-wrong-issuer + - auth/iss-unexpected + - auth/iss-normalized diff --git a/tests/Conformance/conformance-baseline.yml b/tests/Conformance/conformance-baseline.yml index efda80a..cba53fd 100644 --- a/tests/Conformance/conformance-baseline.yml +++ b/tests/Conformance/conformance-baseline.yml @@ -1,8 +1,19 @@ -server: [] +# Both pre-existing fixture bugs, not lifecycle-specific: json-schema-2020-12 +# wants a `json_schema_2020_12_tool` fixture neither conformance server +# defines, and resources-templates-read emits a resource-template URI in a +# shape the schema rejects. Reproduce identically on the 2026-07-28 endpoint +# (see conformance-baseline-2026-07-28.yml). +server: + - json-schema-2020-12:json-schema-2020-12-tool-found + - resources-templates-read:wire-schema-valid client: - elicitation-sep1034-client-defaults - sse-retry + # The example client doesn't call add_numbers or echo the tool's inputSchema + # back, so these checks have nothing to exercise. + - tools_call:tool-add-numbers + - json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed - auth/metadata-default - auth/metadata-var1 - auth/metadata-var2