modelcontextprotocol/php-sdk
Watch
1
0
Fork
You've already forked php-sdk
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[CI] Version both baseline files and split conformance badges per spec (#459)

Browse source 
- Rename conformance-baseline.yml to conformance-baseline-2025-11-25.yml
  so both baseline files carry their spec revision symmetrically, and
  bring conformance-server/conformance-client up to the same rigor
  (--suite all, pinned CONFORMANCE, baseline check) as the draft targets
- score.php now takes a required spec-version and writes
  <suite>-conformance-<spec-version>.json, so 2025-11-25 and 2026-07-28
  each get their own badge instead of only the dated one
- conformance-weekly.yaml generates and publishes badges for both
  matrix entries instead of gating on 2025-11-25 only, and drops the
  now-stale unversioned badge files from the badges branch
- README shows all four badges alongside the existing packagist/CI/PHP
  version/license row
- Trimmed workflow comments to the minimum needed
This commit is contained in:
Christopher Hertel 2026-08-18 23:42:36 +02:00 • committed by GitHub
commit 78a8926f17
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 66 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

55
.github/workflows/conformance-weekly.yaml vendored
View file

@ -1,13 +1,8 @@
name: conformance-weekly name: conformance-weekly
# Runs the MCP conformance suite weekly against the newest published # Tracks the newest published @modelcontextprotocol/conformance; the
# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact # on:pull_request pipeline pins an exact version instead. Also publishes
# version; this schedule catches upstream releases that add scenarios between # score badges to the orphan `badges` branch, consumed by the README.
# PRs, so the pin never silently goes stale.
#
# It also scores each run and publishes the client/server pass-rate as
# shields.io endpoint JSON to the orphan `badges` branch (consumed by the
# README); that branch is created on the first run.
on: on:
schedule: schedule:
- cron: '0 6 * * 1' # Mondays 06:00 UTC - cron: '0 6 * * 1' # Mondays 06:00 UTC
@ -17,10 +12,6 @@ permissions:
contents: write contents: write
issues: write issues: write
# The PR pipeline pins an exact conformance version so a PR only goes red for
# reasons in the PR. That pin is only safe because this job tracks the moving
# target instead — on both revisions, since each rides a different dist-tag:
# the dated one is on `latest`, the draft one only on `alpha`.
jobs: jobs:
server: server:
name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
@ -32,7 +23,7 @@ jobs:
- spec-version: '2025-11-25' - spec-version: '2025-11-25'
dist-tag: latest dist-tag: latest
path: '/' path: '/'
baseline: conformance-baseline.yml baseline: conformance-baseline-2025-11-25.yml
- spec-version: '2026-07-28' - spec-version: '2026-07-28'
dist-tag: alpha dist-tag: alpha
path: '/stateless' path: '/stateless'
@ -58,11 +49,9 @@ jobs:
--spec-version ${{ matrix.spec-version }} --spec-version ${{ matrix.spec-version }}
--expected-failures ${{ matrix.baseline }} --expected-failures ${{ matrix.baseline }}
--output-dir results --output-dir results
# The badge tracks the released revision, so only the dated entry feeds
# it; the draft entry runs purely to catch upstream drift.
- name: Generate score badge - name: Generate score badge
if: always() && matrix.spec-version == '2025-11-25' if: always()
run: php tests/Conformance/score.php server run: php tests/Conformance/score.php server ${{ matrix.spec-version }}
- name: Show docker logs on failure - name: Show docker logs on failure
if: failure() if: failure()
run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs
@ -75,11 +64,11 @@ jobs:
tests/Conformance/logs tests/Conformance/logs
tests/Conformance/results tests/Conformance/results
- name: Upload score badge - name: Upload score badge
if: always() && matrix.spec-version == '2025-11-25' if: always()
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v7
with: with:
name: server-badge name: badge-server-${{ matrix.spec-version }}
path: tests/Conformance/server-conformance.json path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json
client: client:
name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
@ -90,7 +79,7 @@ jobs:
include: include:
- spec-version: '2025-11-25' - spec-version: '2025-11-25'
dist-tag: latest dist-tag: latest
baseline: conformance-baseline.yml baseline: conformance-baseline-2025-11-25.yml
- spec-version: '2026-07-28' - spec-version: '2026-07-28'
dist-tag: alpha dist-tag: alpha
baseline: conformance-baseline-2026-07-28.yml baseline: conformance-baseline-2026-07-28.yml
@ -115,8 +104,8 @@ jobs:
--expected-failures ${{ matrix.baseline }} --expected-failures ${{ matrix.baseline }}
--output-dir results --output-dir results
- name: Generate score badge - name: Generate score badge
if: always() && matrix.spec-version == '2025-11-25' if: always()
run: php tests/Conformance/score.php client run: php tests/Conformance/score.php client ${{ matrix.spec-version }}
- name: Upload conformance results - name: Upload conformance results
if: failure() if: failure()
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v7
@ -126,11 +115,11 @@ jobs:
tests/Conformance/logs tests/Conformance/logs
tests/Conformance/results tests/Conformance/results
- name: Upload score badge - name: Upload score badge
if: always() && matrix.spec-version == '2025-11-25' if: always()
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v7
with: with:
name: client-badge name: badge-client-${{ matrix.spec-version }}
path: tests/Conformance/client-conformance.json path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json
notify: notify:
name: Open issue on failure name: Open issue on failure
@ -156,26 +145,21 @@ jobs:
- Run: $RUN_URL - Run: $RUN_URL
- Triggered: $(date -u +%FT%TZ) - Triggered: $(date -u +%FT%TZ)
Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline.yml\`." Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline-2025-11-25.yml\`."
fi fi
publish: publish:
name: Publish conformance badges name: Publish conformance badges
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [server, client] needs: [server, client]
# Publish even when the suite regressed (the badge should reflect reality);
# skip on forks, which cannot push the `badges` branch.
if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }} if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }}
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/download-artifact@v8 - uses: actions/download-artifact@v8
with: with:
name: server-badge pattern: badge-*
path: badges-in
- uses: actions/download-artifact@v8
with:
name: client-badge
path: badges-in path: badges-in
merge-multiple: true
- name: Publish to badges branch - name: Publish to badges branch
run: | run: |
git config user.name "github-actions[bot]" git config user.name "github-actions[bot]"
@ -190,7 +174,8 @@ jobs:
git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true
fi fi
cp badges-in/server-conformance.json badges-in/client-conformance.json badges-wt/ rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json
cp badges-in/*.json badges-wt/
cd badges-wt cd badges-wt
git add -A git add -A

13
.github/workflows/pipeline.yaml vendored
View file

@ -6,9 +6,7 @@ permissions:
pull-requests: write pull-requests: write
env: env:
# Pinned so a PR only goes red for reasons in the PR. The 2026-07-28 # Pinned exact version; conformance-weekly.yaml tracks upstream releases.
# scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them.
# conformance-weekly.yaml is what tracks upstream releases.
CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11" CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"
jobs: jobs:
@ -105,16 +103,11 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
fail-fast: false fail-fast: false
# One entry per lifecycle, not per spec revision: `--spec-version` is
# cumulative across the dated revisions, so 2025-11-25 already covers
# 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it
# is the SEP-2575 stateless wire, served from its own endpoint with its
# own baseline, so it needs its own run.
matrix: matrix:
include: include:
- spec-version: '2025-11-25' - spec-version: '2025-11-25'
path: '/' path: '/'
baseline: conformance-baseline.yml baseline: conformance-baseline-2025-11-25.yml
- spec-version: '2026-07-28' - spec-version: '2026-07-28'
path: '/stateless' path: '/stateless'
baseline: conformance-baseline-2026-07-28.yml baseline: conformance-baseline-2026-07-28.yml
@ -180,7 +173,7 @@ jobs:
matrix: matrix:
include: include:
- spec-version: '2025-11-25' - spec-version: '2025-11-25'
baseline: conformance-baseline.yml baseline: conformance-baseline-2025-11-25.yml
- spec-version: '2026-07-28' - spec-version: '2026-07-28'
baseline: conformance-baseline-2026-07-28.yml baseline: conformance-baseline-2026-07-28.yml

10
Makefile
View file

@ -37,14 +37,14 @@ conformance-server:
@echo "Waiting for server to start..." @echo "Waiting for server to start..."
@sleep 5 @sleep 5
rm -rf tests/Conformance/results rm -rf tests/Conformance/results
cd tests/Conformance && npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --spec-version 2025-11-25 --output-dir results || true cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/ --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true
php tests/Conformance/score.php server php tests/Conformance/score.php server 2025-11-25
docker compose -f tests/Conformance/Fixtures/docker-compose.yml down docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
conformance-client: conformance-client:
rm -rf tests/Conformance/results rm -rf tests/Conformance/results
cd tests/Conformance && npx @modelcontextprotocol/conformance client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline.yml --output-dir results || true cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true
php tests/Conformance/score.php client php tests/Conformance/score.php client 2025-11-25
# --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------ # --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------
@ -56,11 +56,13 @@ conformance-draft-server:
@sleep 5 @sleep 5
rm -rf tests/Conformance/results-2026-07-28 rm -rf tests/Conformance/results-2026-07-28
cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
php tests/Conformance/score.php server 2026-07-28 results-2026-07-28
docker compose -f tests/Conformance/Fixtures/docker-compose.yml down docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
conformance-draft-client: conformance-draft-client:
rm -rf tests/Conformance/results-2026-07-28 rm -rf tests/Conformance/results-2026-07-28
cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
php tests/Conformance/score.php client 2026-07-28 results-2026-07-28
coverage: coverage:
XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage

6
README.md
View file

@ -6,8 +6,10 @@
[![CI](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml/badge.svg)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml) [![CI](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml/badge.svg)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml)
[![PHP Version](https://img.shields.io/packagist/php-v/mcp/sdk.svg)](https://packagist.org/packages/mcp/sdk) [![PHP Version](https://img.shields.io/packagist/php-v/mcp/sdk.svg)](https://packagist.org/packages/mcp/sdk)
[![License](https://img.shields.io/packagist/l/mcp/sdk.svg)](LICENSE) [![License](https://img.shields.io/packagist/l/mcp/sdk.svg)](LICENSE)
[![Server Conformance](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) [![Server Conformance 2025-11-25](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance-2025-11-25.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
[![Client Conformance](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) [![Client Conformance 2025-11-25](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance-2025-11-25.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
[![Server Conformance 2026-07-28](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance-2026-07-28.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
[![Client Conformance 2026-07-28](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance-2026-07-28.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
</div> </div>

53
tests/Conformance/conformance-baseline-2025-11-25.yml Normal file
View file

@ -0,0 +1,53 @@
# Expected conformance failures for spec revision 2025-11-25.
#
# Separate from conformance-baseline-2026-07-28.yml because the two revisions
# exercise different lifecycles against different endpoints: this file covers
# the handshake era served at /, the other the modern (SEP-2575) era served
# at /stateless.
#
# Entries are per-check (scenario:check-id) rather than whole scenarios, so a
# scenario that is mostly conformant keeps reporting the parts that work and a
# new regression inside it still fails the run.
#
# Run with:
# make conformance-server
# make conformance-client
# Both pre-existing fixture bugs, not lifecycle-specific: json-schema-2020-12
# wants a `json_schema_2020_12_tool` fixture neither conformance server
# defines, and resources-templates-read emits a resource-template URI in a
# shape the schema rejects. Reproduce identically on the 2026-07-28 endpoint
# (see conformance-baseline-2026-07-28.yml).
server:
- json-schema-2020-12:json-schema-2020-12-tool-found
- resources-templates-read:wire-schema-valid
client:
- elicitation-sep1034-client-defaults
- sse-retry
# The example client doesn't call add_numbers or echo the tool's inputSchema
# back, so these checks have nothing to exercise.
- tools_call:tool-add-numbers
- json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed
- auth/metadata-default
- auth/metadata-var1
- auth/metadata-var2
- auth/metadata-var3
- auth/basic-cimd
- auth/scope-from-www-authenticate
- auth/scope-from-scopes-supported
- auth/scope-omitted-when-undefined
- auth/scope-step-up
- auth/scope-retry-limit
- auth/token-endpoint-auth-basic
- auth/token-endpoint-auth-post
- auth/token-endpoint-auth-none
- auth/pre-registration
- auth/2025-03-26-oauth-metadata-backcompat
- auth/2025-03-26-oauth-endpoint-fallback
- auth/offline-access-scope
- auth/offline-access-not-supported
- auth/client-credentials-jwt
- auth/client-credentials-basic
- auth/cross-app-access-complete-flow

7
tests/Conformance/conformance-baseline-2026-07-28.yml
View file

@ -1,8 +1,8 @@
# Expected conformance failures for spec revision 2026-07-28. # Expected conformance failures for spec revision 2026-07-28.
# #
# Separate from conformance-baseline.yml because the two revisions exercise # Separate from conformance-baseline-2025-11-25.yml because the two revisions
# different lifecycles against different endpoints: the dated baseline covers # exercise different lifecycles against different endpoints: the other file
# the handshake era served at /, this one covers the modern (SEP-2575) era # covers the handshake era served at /, this one the modern (SEP-2575) era
# served at /stateless. # served at /stateless.
# #
# Entries are per-check (scenario:check-id) rather than whole scenarios, so a # Entries are per-check (scenario:check-id) rather than whole scenarios, so a
@ -11,6 +11,7 @@
# #
# Run with: # Run with:
# make conformance-draft-server # make conformance-draft-server
# make conformance-draft-client
server: server:
# --- Pre-existing, not lifecycle-specific ---------------------------------- # --- Pre-existing, not lifecycle-specific ----------------------------------

24
tests/Conformance/score.php
View file

@ -13,13 +13,14 @@
* Turns a conformance run's results into a shields.io endpoint badge JSON, so * Turns a conformance run's results into a shields.io endpoint badge JSON, so
* the client/server conformance score can be rendered in the README. * the client/server conformance score can be rendered in the README.
* *
* php score.php <server|client> * php score.php <server|client> <spec-version> [results-dir]
* *
* The conformance CLI (run with `--output-dir results`) writes one * The conformance CLI writes one `checks.json` per scenario into
* `checks.json` per scenario into the `results/` directory next to this file. * `[results-dir]` (default `results`), relative to this file. A scenario
* A scenario counts as passing when none of its checks has a FAILURE status; * counts as passing when none of its checks has a FAILURE status; the badge
* the badge message is "<passed>/<total> (<pct>%)" and is written to * message is "<passed>/<total> (<pct>%)" and is written to
* `<suite>-conformance.json`. * `<suite>-conformance-<spec-version>.json`, so each revision gets its own
* badge.
*/ */
use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Command\Command;
@ -36,10 +37,13 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
->setName('conformance-score') ->setName('conformance-score')
->setDescription('Generates a shields.io endpoint badge from the conformance results') ->setDescription('Generates a shields.io endpoint badge from the conformance results')
->addArgument('suite', InputArgument::REQUIRED, 'Which conformance suite was run: "server" or "client"') ->addArgument('suite', InputArgument::REQUIRED, 'Which conformance suite was run: "server" or "client"')
->addArgument('spec-version', InputArgument::REQUIRED, 'The spec revision that was run, e.g. "2025-11-25"')
->addArgument('results-dir', InputArgument::OPTIONAL, 'Directory the conformance CLI wrote results into, relative to this file', 'results')
->setCode(static function (InputInterface $input, OutputInterface $output): int { ->setCode(static function (InputInterface $input, OutputInterface $output): int {
$io = new SymfonyStyle($input, $output); $io = new SymfonyStyle($input, $output);
$suite = $input->getArgument('suite'); $suite = $input->getArgument('suite');
$specVersion = $input->getArgument('spec-version');
if (!in_array($suite, ['server', 'client'], true)) { if (!in_array($suite, ['server', 'client'], true)) {
$io->error(sprintf('Suite must be "server" or "client", got "%s".', $suite)); $io->error(sprintf('Suite must be "server" or "client", got "%s".', $suite));
@ -47,10 +51,10 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
return Command::INVALID; return Command::INVALID;
} }
$resultsDir = __DIR__.'/results'; $resultsDir = __DIR__.'/'.$input->getArgument('results-dir');
if (!is_dir($resultsDir)) { if (!is_dir($resultsDir)) {
$io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir results` first.', $resultsDir)); $io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir` first.', $resultsDir));
return Command::FAILURE; return Command::FAILURE;
} }
@ -88,7 +92,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
$badge = [ $badge = [
'schemaVersion' => 1, 'schemaVersion' => 1,
'label' => $suite.' conformance', 'label' => sprintf('%s %s', $suite, $specVersion),
'message' => $total > 0 ? sprintf('%d/%d (%d%%)', $passed, $total, $pct) : 'no data', 'message' => $total > 0 ? sprintf('%d/%d (%d%%)', $passed, $total, $pct) : 'no data',
'color' => match (true) { 'color' => match (true) {
0 === $total => 'lightgrey', 0 === $total => 'lightgrey',
@ -99,7 +103,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
}, },
]; ];
$outputFile = __DIR__.'/'.$suite.'-conformance.json'; $outputFile = __DIR__.'/'.$suite.'-conformance-'.$specVersion.'.json';
if (false === file_put_contents($outputFile, json_encode($badge, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES)."\n")) { if (false === file_put_contents($outputFile, json_encode($badge, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES)."\n")) {
$io->error(sprintf('Could not write badge file "%s".', $outputFile)); $io->error(sprintf('Could not write badge file "%s".', $outputFile));