[CI] Matrix conformance jobs over both spec revisions (#458)
* [CI] Matrix the conformance jobs over both spec revisions Runs server/client conformance against both 2025-11-25 and 2026-07-28, each against its own endpoint and baseline. PR pipeline pins an exact conformance package version; conformance-weekly tracks the moving target on both dist-tags. * [CI] Run the draft conformance suite from the alpha package The conformance repo now ships 2026-07-28 scenarios on its alpha dist-tag, so the draft targets no longer need a local checkout. Pinned to the same version the pipeline runs. * [CI] Baseline the failures --suite all newly exposes The old workflow ran without --suite all/--spec-version, so several pre-existing gaps never surfaced in CI. Now that both revisions run the full suite: - server: json-schema-2020-12 and resources-templates-read are fixture bugs shared by both revisions; 2026-07-28 also needs resources/subscribe and resources/unsubscribe declared removed under the stateless wire, which the server doesn't yet do - client: the example client doesn't call add_numbers or echo back a tool's inputSchema; against 2026-07-28 it doesn't speak the lifecycle or OAuth at all yet, so every such scenario fails Verified locally: all four conformance jobs (server/client x 2025-11-25/2026-07-28) pass against these baselines.
This commit is contained in:
Christopher Hertel
GitHub
parent
dd3783a720
commit
7ffc739fe7
5 changed files with 161 additions and 39 deletions
68
.github/workflows/conformance-weekly.yaml
vendored
68
.github/workflows/conformance-weekly.yaml
vendored
|
|
@ -1,9 +1,9 @@
|
|||
name: conformance-weekly
|
||||
|
||||
# Runs the MCP conformance suite weekly against the latest
|
||||
# @modelcontextprotocol/conformance release. The on:pull_request pipeline
|
||||
# pins to whatever version is available at PR time; this schedule catches
|
||||
# upstream releases that add scenarios between PRs.
|
||||
# Runs the MCP conformance suite weekly against the newest published
|
||||
# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact
|
||||
# version; this schedule catches upstream releases that add scenarios between
|
||||
# PRs, so the pin never silently goes stale.
|
||||
#
|
||||
# It also scores each run and publishes the client/server pass-rate as
|
||||
# shields.io endpoint JSON to the orphan `badges` branch (consumed by the
|
||||
|
|
@ -17,10 +17,26 @@ permissions:
|
|||
contents: write
|
||||
issues: write
|
||||
|
||||
# The PR pipeline pins an exact conformance version so a PR only goes red for
|
||||
# reasons in the PR. That pin is only safe because this job tracks the moving
|
||||
# target instead — on both revisions, since each rides a different dist-tag:
|
||||
# the dated one is on `latest`, the draft one only on `alpha`.
|
||||
jobs:
|
||||
server:
|
||||
name: conformance / server (latest)
|
||||
name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
dist-tag: latest
|
||||
path: '/'
|
||||
baseline: conformance-baseline.yml
|
||||
- spec-version: '2026-07-28'
|
||||
dist-tag: alpha
|
||||
path: '/stateless'
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-node@v7
|
||||
|
|
@ -35,9 +51,17 @@ jobs:
|
|||
sleep 5
|
||||
- name: Run conformance tests
|
||||
working-directory: ./tests/Conformance
|
||||
run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml --output-dir results
|
||||
run: >-
|
||||
npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server
|
||||
--url http://localhost:8000${{ matrix.path }}
|
||||
--suite all
|
||||
--spec-version ${{ matrix.spec-version }}
|
||||
--expected-failures ${{ matrix.baseline }}
|
||||
--output-dir results
|
||||
# The badge tracks the released revision, so only the dated entry feeds
|
||||
# it; the draft entry runs purely to catch upstream drift.
|
||||
- name: Generate score badge
|
||||
if: always()
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
run: php tests/Conformance/score.php server
|
||||
- name: Show docker logs on failure
|
||||
if: failure()
|
||||
|
|
@ -46,20 +70,30 @@ jobs:
|
|||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: conformance-server-results
|
||||
name: conformance-server-results-${{ matrix.spec-version }}
|
||||
path: |
|
||||
tests/Conformance/logs
|
||||
tests/Conformance/results
|
||||
- name: Upload score badge
|
||||
if: always()
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: server-badge
|
||||
path: tests/Conformance/server-conformance.json
|
||||
|
||||
client:
|
||||
name: conformance / client (latest)
|
||||
name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
dist-tag: latest
|
||||
baseline: conformance-baseline.yml
|
||||
- spec-version: '2026-07-28'
|
||||
dist-tag: alpha
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: shivammathur/setup-php@v2
|
||||
|
|
@ -73,20 +107,26 @@ jobs:
|
|||
- run: mkdir -p tests/Conformance/logs
|
||||
- name: Run conformance tests
|
||||
working-directory: ./tests/Conformance
|
||||
run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml --output-dir results
|
||||
run: >-
|
||||
npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client
|
||||
--command "php ${{ github.workspace }}/tests/Conformance/client.php"
|
||||
--suite all
|
||||
--spec-version ${{ matrix.spec-version }}
|
||||
--expected-failures ${{ matrix.baseline }}
|
||||
--output-dir results
|
||||
- name: Generate score badge
|
||||
if: always()
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
run: php tests/Conformance/score.php client
|
||||
- name: Upload conformance results
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: conformance-client-results
|
||||
name: conformance-client-results-${{ matrix.spec-version }}
|
||||
path: |
|
||||
tests/Conformance/logs
|
||||
tests/Conformance/results
|
||||
- name: Upload score badge
|
||||
if: always()
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: client-badge
|
||||
|
|
|
|||
49
.github/workflows/pipeline.yaml
vendored
49
.github/workflows/pipeline.yaml
vendored
|
|
@ -5,6 +5,12 @@ permissions:
|
|||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
# Pinned so a PR only goes red for reasons in the PR. The 2026-07-28
|
||||
# scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them.
|
||||
# conformance-weekly.yaml is what tracks upstream releases.
|
||||
CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -95,8 +101,24 @@ jobs:
|
|||
run: vendor/bin/phpunit --testsuite=inspector
|
||||
|
||||
conformance-server:
|
||||
name: conformance / server
|
||||
name: conformance / server (${{ matrix.spec-version }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# One entry per lifecycle, not per spec revision: `--spec-version` is
|
||||
# cumulative across the dated revisions, so 2025-11-25 already covers
|
||||
# 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it
|
||||
# is the SEP-2575 stateless wire, served from its own endpoint with its
|
||||
# own baseline, so it needs its own run.
|
||||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
path: '/'
|
||||
baseline: conformance-baseline.yml
|
||||
- spec-version: '2026-07-28'
|
||||
path: '/stateless'
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
|
@ -118,7 +140,12 @@ jobs:
|
|||
|
||||
- name: Run conformance tests
|
||||
working-directory: ./tests/Conformance
|
||||
run: npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --expected-failures conformance-baseline.yml
|
||||
run: >-
|
||||
npx --yes ${{ env.CONFORMANCE_PKG }} server
|
||||
--url http://localhost:8000${{ matrix.path }}
|
||||
--suite all
|
||||
--spec-version ${{ matrix.spec-version }}
|
||||
--expected-failures ${{ matrix.baseline }}
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
|
|
@ -146,8 +173,17 @@ jobs:
|
|||
run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
|
||||
|
||||
conformance-client:
|
||||
name: conformance / client
|
||||
name: conformance / client (${{ matrix.spec-version }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
baseline: conformance-baseline.yml
|
||||
- spec-version: '2026-07-28'
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
|
|
@ -171,7 +207,12 @@ jobs:
|
|||
|
||||
- name: Run client conformance tests
|
||||
working-directory: ./tests/Conformance
|
||||
run: npx @modelcontextprotocol/conformance client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml
|
||||
run: >-
|
||||
npx --yes ${{ env.CONFORMANCE_PKG }} client
|
||||
--command "php ${{ github.workspace }}/tests/Conformance/client.php"
|
||||
--suite all
|
||||
--spec-version ${{ matrix.spec-version }}
|
||||
--expected-failures ${{ matrix.baseline }}
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
|
|
|
|||
29
Makefile
29
Makefile
|
|
@ -1,10 +1,10 @@
|
|||
.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client conformance-draft conformance-draft-server conformance-draft-client check-conformance-repo docs
|
||||
.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client conformance-draft conformance-draft-server conformance-draft-client docs
|
||||
|
||||
# The published conformance release carries no 2026-07-28 scenarios, so the
|
||||
# draft targets run a local checkout instead.
|
||||
# Override with `make conformance-draft CONFORMANCE_REPO=/path/to/conformance`.
|
||||
CONFORMANCE_REPO ?= $(CURDIR)/../conformance
|
||||
CONFORMANCE_DRAFT = node $(CONFORMANCE_REPO)/dist/index.js
|
||||
# The 2026-07-28 scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has
|
||||
# none of them. Pinned to the same version CI runs (see
|
||||
# .github/workflows/pipeline.yaml), so a local pass means a green pipeline.
|
||||
CONFORMANCE_VERSION ?= 0.2.0-alpha.11
|
||||
CONFORMANCE = npx --yes @modelcontextprotocol/conformance@$(CONFORMANCE_VERSION)
|
||||
|
||||
deps-stable:
|
||||
composer update --prefer-stable
|
||||
|
|
@ -47,29 +47,20 @@ conformance-client:
|
|||
php tests/Conformance/score.php client
|
||||
|
||||
# --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------
|
||||
# Local-checkout only until upstream publishes the draft scenarios.
|
||||
|
||||
conformance-draft: conformance-draft-server conformance-draft-client
|
||||
|
||||
check-conformance-repo:
|
||||
@test -f $(CONFORMANCE_REPO)/dist/index.js || { \
|
||||
echo "No conformance build at $(CONFORMANCE_REPO)/dist/index.js."; \
|
||||
echo "Clone modelcontextprotocol/conformance and run 'npm install && npm run build' there,"; \
|
||||
echo "or point CONFORMANCE_REPO at an existing checkout."; \
|
||||
exit 1; \
|
||||
}
|
||||
|
||||
conformance-draft-server: check-conformance-repo
|
||||
conformance-draft-server:
|
||||
docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d
|
||||
@echo "Waiting for server to start..."
|
||||
@sleep 5
|
||||
rm -rf tests/Conformance/results-2026-07-28
|
||||
cd tests/Conformance && $(CONFORMANCE_DRAFT) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
|
||||
|
||||
conformance-draft-client: check-conformance-repo
|
||||
conformance-draft-client:
|
||||
rm -rf tests/Conformance/results-2026-07-28
|
||||
cd tests/Conformance && $(CONFORMANCE_DRAFT) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
|
||||
coverage:
|
||||
XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage
|
||||
|
|
|
|||
|
|
@ -25,4 +25,43 @@ server:
|
|||
- json-schema-2020-12:json-schema-2020-12-tool-found
|
||||
- resources-templates-read:wire-schema-valid
|
||||
|
||||
client: []
|
||||
# The client does not speak 2026-07-28 at all yet: it still opens with
|
||||
# `initialize` regardless of revision, so it never reaches the stateless wire
|
||||
# (no per-request `_meta`, no Mcp-Method/Mcp-Name headers, no client-side
|
||||
# requestState) — and separately, the SDK ships no OAuth client, so every
|
||||
# scenario driving an authorization flow has nothing to exercise.
|
||||
client:
|
||||
- request-metadata
|
||||
- http-standard-headers
|
||||
- http-custom-headers
|
||||
- http-invalid-tool-headers:sep-2243-invalid-tool-tools-list-gate
|
||||
- http-invalid-tool-headers:sep-2243-client-reject-invalid-tool
|
||||
- sep-2322-client-request-state
|
||||
- tools_call:tool-add-numbers
|
||||
- json-schema-2020-12-preservation:json-schema-2020-12-client-tool-found
|
||||
- json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed
|
||||
|
||||
- auth/metadata-default
|
||||
- auth/metadata-var1
|
||||
- auth/metadata-var2
|
||||
- auth/metadata-var3
|
||||
- auth/metadata-issuer-mismatch
|
||||
- auth/basic-cimd
|
||||
- auth/pre-registration
|
||||
- auth/scope-from-www-authenticate
|
||||
- auth/scope-from-scopes-supported
|
||||
- auth/scope-omitted-when-undefined
|
||||
- auth/scope-step-up
|
||||
- auth/scope-retry-limit
|
||||
- auth/token-endpoint-auth-basic
|
||||
- auth/token-endpoint-auth-post
|
||||
- auth/token-endpoint-auth-none
|
||||
- auth/offline-access-scope
|
||||
- auth/offline-access-not-supported
|
||||
- auth/authorization-server-migration:sep-2352-reregister-on-as-change
|
||||
- auth/iss-supported
|
||||
- auth/iss-not-advertised
|
||||
- auth/iss-supported-missing
|
||||
- auth/iss-wrong-issuer
|
||||
- auth/iss-unexpected
|
||||
- auth/iss-normalized
|
||||
|
|
|
|||
|
|
@ -1,8 +1,19 @@
|
|||
server: []
|
||||
# Both pre-existing fixture bugs, not lifecycle-specific: json-schema-2020-12
|
||||
# wants a `json_schema_2020_12_tool` fixture neither conformance server
|
||||
# defines, and resources-templates-read emits a resource-template URI in a
|
||||
# shape the schema rejects. Reproduce identically on the 2026-07-28 endpoint
|
||||
# (see conformance-baseline-2026-07-28.yml).
|
||||
server:
|
||||
- json-schema-2020-12:json-schema-2020-12-tool-found
|
||||
- resources-templates-read:wire-schema-valid
|
||||
|
||||
client:
|
||||
- elicitation-sep1034-client-defaults
|
||||
- sse-retry
|
||||
# The example client doesn't call add_numbers or echo the tool's inputSchema
|
||||
# back, so these checks have nothing to exercise.
|
||||
- tools_call:tool-add-numbers
|
||||
- json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed
|
||||
- auth/metadata-default
|
||||
- auth/metadata-var1
|
||||
- auth/metadata-var2
|
||||
|
|
|
|||
Loading…
Reference in a new issue