diff --git a/.github/workflows/conformance-weekly.yaml b/.github/workflows/conformance-weekly.yaml index ecc262f..3d9b614 100644 --- a/.github/workflows/conformance-weekly.yaml +++ b/.github/workflows/conformance-weekly.yaml @@ -1,13 +1,8 @@ name: conformance-weekly -# Runs the MCP conformance suite weekly against the newest published -# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact -# version; this schedule catches upstream releases that add scenarios between -# PRs, so the pin never silently goes stale. -# -# It also scores each run and publishes the client/server pass-rate as -# shields.io endpoint JSON to the orphan `badges` branch (consumed by the -# README); that branch is created on the first run. +# Tracks the newest published @modelcontextprotocol/conformance; the +# on:pull_request pipeline pins an exact version instead. Also publishes +# score badges to the orphan `badges` branch, consumed by the README. on: schedule: - cron: '0 6 * * 1' # Mondays 06:00 UTC @@ -17,10 +12,6 @@ permissions: contents: write issues: write -# The PR pipeline pins an exact conformance version so a PR only goes red for -# reasons in the PR. That pin is only safe because this job tracks the moving -# target instead — on both revisions, since each rides a different dist-tag: -# the dated one is on `latest`, the draft one only on `alpha`. jobs: server: name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) @@ -32,7 +23,7 @@ jobs: - spec-version: '2025-11-25' dist-tag: latest path: '/' - baseline: conformance-baseline.yml + baseline: conformance-baseline-2025-11-25.yml - spec-version: '2026-07-28' dist-tag: alpha path: '/stateless' @@ -58,11 +49,9 @@ jobs: --spec-version ${{ matrix.spec-version }} --expected-failures ${{ matrix.baseline }} --output-dir results - # The badge tracks the released revision, so only the dated entry feeds - # it; the draft entry runs purely to catch upstream drift. - name: Generate score badge - if: always() && matrix.spec-version == '2025-11-25' - run: php tests/Conformance/score.php server + if: always() + run: php tests/Conformance/score.php server ${{ matrix.spec-version }} - name: Show docker logs on failure if: failure() run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs @@ -75,11 +64,11 @@ jobs: tests/Conformance/logs tests/Conformance/results - name: Upload score badge - if: always() && matrix.spec-version == '2025-11-25' + if: always() uses: actions/upload-artifact@v7 with: - name: server-badge - path: tests/Conformance/server-conformance.json + name: badge-server-${{ matrix.spec-version }} + path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json client: name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }}) @@ -90,7 +79,7 @@ jobs: include: - spec-version: '2025-11-25' dist-tag: latest - baseline: conformance-baseline.yml + baseline: conformance-baseline-2025-11-25.yml - spec-version: '2026-07-28' dist-tag: alpha baseline: conformance-baseline-2026-07-28.yml @@ -115,8 +104,8 @@ jobs: --expected-failures ${{ matrix.baseline }} --output-dir results - name: Generate score badge - if: always() && matrix.spec-version == '2025-11-25' - run: php tests/Conformance/score.php client + if: always() + run: php tests/Conformance/score.php client ${{ matrix.spec-version }} - name: Upload conformance results if: failure() uses: actions/upload-artifact@v7 @@ -126,11 +115,11 @@ jobs: tests/Conformance/logs tests/Conformance/results - name: Upload score badge - if: always() && matrix.spec-version == '2025-11-25' + if: always() uses: actions/upload-artifact@v7 with: - name: client-badge - path: tests/Conformance/client-conformance.json + name: badge-client-${{ matrix.spec-version }} + path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json notify: name: Open issue on failure @@ -156,26 +145,21 @@ jobs: - Run: $RUN_URL - Triggered: $(date -u +%FT%TZ) - Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline.yml\`." + Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline-2025-11-25.yml\`." fi publish: name: Publish conformance badges runs-on: ubuntu-latest needs: [server, client] - # Publish even when the suite regressed (the badge should reflect reality); - # skip on forks, which cannot push the `badges` branch. if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }} steps: - uses: actions/checkout@v7 - uses: actions/download-artifact@v8 with: - name: server-badge - path: badges-in - - uses: actions/download-artifact@v8 - with: - name: client-badge + pattern: badge-* path: badges-in + merge-multiple: true - name: Publish to badges branch run: | git config user.name "github-actions[bot]" @@ -190,7 +174,8 @@ jobs: git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true fi - cp badges-in/server-conformance.json badges-in/client-conformance.json badges-wt/ + rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json + cp badges-in/*.json badges-wt/ cd badges-wt git add -A diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 402e22b..2a5475d 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -6,9 +6,7 @@ permissions: pull-requests: write env: - # Pinned so a PR only goes red for reasons in the PR. The 2026-07-28 - # scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them. - # conformance-weekly.yaml is what tracks upstream releases. + # Pinned exact version; conformance-weekly.yaml tracks upstream releases. CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11" jobs: @@ -105,16 +103,11 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - # One entry per lifecycle, not per spec revision: `--spec-version` is - # cumulative across the dated revisions, so 2025-11-25 already covers - # 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it - # is the SEP-2575 stateless wire, served from its own endpoint with its - # own baseline, so it needs its own run. matrix: include: - spec-version: '2025-11-25' path: '/' - baseline: conformance-baseline.yml + baseline: conformance-baseline-2025-11-25.yml - spec-version: '2026-07-28' path: '/stateless' baseline: conformance-baseline-2026-07-28.yml @@ -180,7 +173,7 @@ jobs: matrix: include: - spec-version: '2025-11-25' - baseline: conformance-baseline.yml + baseline: conformance-baseline-2025-11-25.yml - spec-version: '2026-07-28' baseline: conformance-baseline-2026-07-28.yml diff --git a/Makefile b/Makefile index c756792..7490515 100644 --- a/Makefile +++ b/Makefile @@ -37,14 +37,14 @@ conformance-server: @echo "Waiting for server to start..." @sleep 5 rm -rf tests/Conformance/results - cd tests/Conformance && npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --spec-version 2025-11-25 --output-dir results || true - php tests/Conformance/score.php server + cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/ --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true + php tests/Conformance/score.php server 2025-11-25 docker compose -f tests/Conformance/Fixtures/docker-compose.yml down conformance-client: rm -rf tests/Conformance/results - cd tests/Conformance && npx @modelcontextprotocol/conformance client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline.yml --output-dir results || true - php tests/Conformance/score.php client + cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true + php tests/Conformance/score.php client 2025-11-25 # --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------ @@ -56,11 +56,13 @@ conformance-draft-server: @sleep 5 rm -rf tests/Conformance/results-2026-07-28 cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true + php tests/Conformance/score.php server 2026-07-28 results-2026-07-28 docker compose -f tests/Conformance/Fixtures/docker-compose.yml down conformance-draft-client: rm -rf tests/Conformance/results-2026-07-28 cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true + php tests/Conformance/score.php client 2026-07-28 results-2026-07-28 coverage: XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage diff --git a/README.md b/README.md index 4e1a868..dfab676 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,10 @@ [![CI](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml/badge.svg)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml) [![PHP Version](https://img.shields.io/packagist/php-v/mcp/sdk.svg)](https://packagist.org/packages/mcp/sdk) [![License](https://img.shields.io/packagist/l/mcp/sdk.svg)](LICENSE) -[![Server Conformance](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) -[![Client Conformance](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) +[![Server Conformance 2025-11-25](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance-2025-11-25.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) +[![Client Conformance 2025-11-25](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance-2025-11-25.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) +[![Server Conformance 2026-07-28](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/server-conformance-2026-07-28.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) +[![Client Conformance 2026-07-28](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/modelcontextprotocol/php-sdk/badges/client-conformance-2026-07-28.json)](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml) diff --git a/tests/Conformance/conformance-baseline.yml b/tests/Conformance/conformance-baseline-2025-11-25.yml similarity index 70% rename from tests/Conformance/conformance-baseline.yml rename to tests/Conformance/conformance-baseline-2025-11-25.yml index cba53fd..2450aa6 100644 --- a/tests/Conformance/conformance-baseline.yml +++ b/tests/Conformance/conformance-baseline-2025-11-25.yml @@ -1,3 +1,18 @@ +# Expected conformance failures for spec revision 2025-11-25. +# +# Separate from conformance-baseline-2026-07-28.yml because the two revisions +# exercise different lifecycles against different endpoints: this file covers +# the handshake era served at /, the other the modern (SEP-2575) era served +# at /stateless. +# +# Entries are per-check (scenario:check-id) rather than whole scenarios, so a +# scenario that is mostly conformant keeps reporting the parts that work and a +# new regression inside it still fails the run. +# +# Run with: +# make conformance-server +# make conformance-client + # Both pre-existing fixture bugs, not lifecycle-specific: json-schema-2020-12 # wants a `json_schema_2020_12_tool` fixture neither conformance server # defines, and resources-templates-read emits a resource-template URI in a diff --git a/tests/Conformance/conformance-baseline-2026-07-28.yml b/tests/Conformance/conformance-baseline-2026-07-28.yml index faaa602..4b14d87 100644 --- a/tests/Conformance/conformance-baseline-2026-07-28.yml +++ b/tests/Conformance/conformance-baseline-2026-07-28.yml @@ -1,8 +1,8 @@ # Expected conformance failures for spec revision 2026-07-28. # -# Separate from conformance-baseline.yml because the two revisions exercise -# different lifecycles against different endpoints: the dated baseline covers -# the handshake era served at /, this one covers the modern (SEP-2575) era +# Separate from conformance-baseline-2025-11-25.yml because the two revisions +# exercise different lifecycles against different endpoints: the other file +# covers the handshake era served at /, this one the modern (SEP-2575) era # served at /stateless. # # Entries are per-check (scenario:check-id) rather than whole scenarios, so a @@ -11,6 +11,7 @@ # # Run with: # make conformance-draft-server +# make conformance-draft-client server: # --- Pre-existing, not lifecycle-specific ---------------------------------- diff --git a/tests/Conformance/score.php b/tests/Conformance/score.php index ddad90f..06c3409 100644 --- a/tests/Conformance/score.php +++ b/tests/Conformance/score.php @@ -13,13 +13,14 @@ * Turns a conformance run's results into a shields.io endpoint badge JSON, so * the client/server conformance score can be rendered in the README. * - * php score.php + * php score.php [results-dir] * - * The conformance CLI (run with `--output-dir results`) writes one - * `checks.json` per scenario into the `results/` directory next to this file. - * A scenario counts as passing when none of its checks has a FAILURE status; - * the badge message is "/ (%)" and is written to - * `-conformance.json`. + * The conformance CLI writes one `checks.json` per scenario into + * `[results-dir]` (default `results`), relative to this file. A scenario + * counts as passing when none of its checks has a FAILURE status; the badge + * message is "/ (%)" and is written to + * `-conformance-.json`, so each revision gets its own + * badge. */ use Symfony\Component\Console\Command\Command; @@ -36,10 +37,13 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php'; ->setName('conformance-score') ->setDescription('Generates a shields.io endpoint badge from the conformance results') ->addArgument('suite', InputArgument::REQUIRED, 'Which conformance suite was run: "server" or "client"') + ->addArgument('spec-version', InputArgument::REQUIRED, 'The spec revision that was run, e.g. "2025-11-25"') + ->addArgument('results-dir', InputArgument::OPTIONAL, 'Directory the conformance CLI wrote results into, relative to this file', 'results') ->setCode(static function (InputInterface $input, OutputInterface $output): int { $io = new SymfonyStyle($input, $output); $suite = $input->getArgument('suite'); + $specVersion = $input->getArgument('spec-version'); if (!in_array($suite, ['server', 'client'], true)) { $io->error(sprintf('Suite must be "server" or "client", got "%s".', $suite)); @@ -47,10 +51,10 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php'; return Command::INVALID; } - $resultsDir = __DIR__.'/results'; + $resultsDir = __DIR__.'/'.$input->getArgument('results-dir'); if (!is_dir($resultsDir)) { - $io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir results` first.', $resultsDir)); + $io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir` first.', $resultsDir)); return Command::FAILURE; } @@ -88,7 +92,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php'; $badge = [ 'schemaVersion' => 1, - 'label' => $suite.' conformance', + 'label' => sprintf('%s %s', $suite, $specVersion), 'message' => $total > 0 ? sprintf('%d/%d (%d%%)', $passed, $total, $pct) : 'no data', 'color' => match (true) { 0 === $total => 'lightgrey', @@ -99,7 +103,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php'; }, ]; - $outputFile = __DIR__.'/'.$suite.'-conformance.json'; + $outputFile = __DIR__.'/'.$suite.'-conformance-'.$specVersion.'.json'; if (false === file_put_contents($outputFile, json_encode($badge, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES)."\n")) { $io->error(sprintf('Could not write badge file "%s".', $outputFile));