[CI] Version both baseline files and split conformance badges per spec (#459)
- Rename conformance-baseline.yml to conformance-baseline-2025-11-25.yml so both baseline files carry their spec revision symmetrically, and bring conformance-server/conformance-client up to the same rigor (--suite all, pinned CONFORMANCE, baseline check) as the draft targets - score.php now takes a required spec-version and writes <suite>-conformance-<spec-version>.json, so 2025-11-25 and 2026-07-28 each get their own badge instead of only the dated one - conformance-weekly.yaml generates and publishes badges for both matrix entries instead of gating on 2025-11-25 only, and drops the now-stale unversioned badge files from the badges branch - README shows all four badges alongside the existing packagist/CI/PHP version/license row - Trimmed workflow comments to the minimum needed
This commit is contained in:
Christopher Hertel
GitHub
parent
cad686aba7
commit
78a8926f17
7 changed files with 66 additions and 64 deletions
55
.github/workflows/conformance-weekly.yaml
vendored
55
.github/workflows/conformance-weekly.yaml
vendored
|
|
@ -1,13 +1,8 @@
|
|||
name: conformance-weekly
|
||||
|
||||
# Runs the MCP conformance suite weekly against the newest published
|
||||
# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact
|
||||
# version; this schedule catches upstream releases that add scenarios between
|
||||
# PRs, so the pin never silently goes stale.
|
||||
#
|
||||
# It also scores each run and publishes the client/server pass-rate as
|
||||
# shields.io endpoint JSON to the orphan `badges` branch (consumed by the
|
||||
# README); that branch is created on the first run.
|
||||
# Tracks the newest published @modelcontextprotocol/conformance; the
|
||||
# on:pull_request pipeline pins an exact version instead. Also publishes
|
||||
# score badges to the orphan `badges` branch, consumed by the README.
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1' # Mondays 06:00 UTC
|
||||
|
|
@ -17,10 +12,6 @@ permissions:
|
|||
contents: write
|
||||
issues: write
|
||||
|
||||
# The PR pipeline pins an exact conformance version so a PR only goes red for
|
||||
# reasons in the PR. That pin is only safe because this job tracks the moving
|
||||
# target instead — on both revisions, since each rides a different dist-tag:
|
||||
# the dated one is on `latest`, the draft one only on `alpha`.
|
||||
jobs:
|
||||
server:
|
||||
name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
|
||||
|
|
@ -32,7 +23,7 @@ jobs:
|
|||
- spec-version: '2025-11-25'
|
||||
dist-tag: latest
|
||||
path: '/'
|
||||
baseline: conformance-baseline.yml
|
||||
baseline: conformance-baseline-2025-11-25.yml
|
||||
- spec-version: '2026-07-28'
|
||||
dist-tag: alpha
|
||||
path: '/stateless'
|
||||
|
|
@ -58,11 +49,9 @@ jobs:
|
|||
--spec-version ${{ matrix.spec-version }}
|
||||
--expected-failures ${{ matrix.baseline }}
|
||||
--output-dir results
|
||||
# The badge tracks the released revision, so only the dated entry feeds
|
||||
# it; the draft entry runs purely to catch upstream drift.
|
||||
- name: Generate score badge
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
run: php tests/Conformance/score.php server
|
||||
if: always()
|
||||
run: php tests/Conformance/score.php server ${{ matrix.spec-version }}
|
||||
- name: Show docker logs on failure
|
||||
if: failure()
|
||||
run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs
|
||||
|
|
@ -75,11 +64,11 @@ jobs:
|
|||
tests/Conformance/logs
|
||||
tests/Conformance/results
|
||||
- name: Upload score badge
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: server-badge
|
||||
path: tests/Conformance/server-conformance.json
|
||||
name: badge-server-${{ matrix.spec-version }}
|
||||
path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json
|
||||
|
||||
client:
|
||||
name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
|
||||
|
|
@ -90,7 +79,7 @@ jobs:
|
|||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
dist-tag: latest
|
||||
baseline: conformance-baseline.yml
|
||||
baseline: conformance-baseline-2025-11-25.yml
|
||||
- spec-version: '2026-07-28'
|
||||
dist-tag: alpha
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
|
|
@ -115,8 +104,8 @@ jobs:
|
|||
--expected-failures ${{ matrix.baseline }}
|
||||
--output-dir results
|
||||
- name: Generate score badge
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
run: php tests/Conformance/score.php client
|
||||
if: always()
|
||||
run: php tests/Conformance/score.php client ${{ matrix.spec-version }}
|
||||
- name: Upload conformance results
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
|
|
@ -126,11 +115,11 @@ jobs:
|
|||
tests/Conformance/logs
|
||||
tests/Conformance/results
|
||||
- name: Upload score badge
|
||||
if: always() && matrix.spec-version == '2025-11-25'
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: client-badge
|
||||
path: tests/Conformance/client-conformance.json
|
||||
name: badge-client-${{ matrix.spec-version }}
|
||||
path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json
|
||||
|
||||
notify:
|
||||
name: Open issue on failure
|
||||
|
|
@ -156,26 +145,21 @@ jobs:
|
|||
- Run: $RUN_URL
|
||||
- Triggered: $(date -u +%FT%TZ)
|
||||
|
||||
Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline.yml\`."
|
||||
Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to \`tests/Conformance/conformance-baseline-2025-11-25.yml\`."
|
||||
fi
|
||||
|
||||
publish:
|
||||
name: Publish conformance badges
|
||||
runs-on: ubuntu-latest
|
||||
needs: [server, client]
|
||||
# Publish even when the suite regressed (the badge should reflect reality);
|
||||
# skip on forks, which cannot push the `badges` branch.
|
||||
if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: server-badge
|
||||
path: badges-in
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: client-badge
|
||||
pattern: badge-*
|
||||
path: badges-in
|
||||
merge-multiple: true
|
||||
- name: Publish to badges branch
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
|
|
@ -190,7 +174,8 @@ jobs:
|
|||
git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
cp badges-in/server-conformance.json badges-in/client-conformance.json badges-wt/
|
||||
rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json
|
||||
cp badges-in/*.json badges-wt/
|
||||
|
||||
cd badges-wt
|
||||
git add -A
|
||||
|
|
|
|||
13
.github/workflows/pipeline.yaml
vendored
13
.github/workflows/pipeline.yaml
vendored
|
|
@ -6,9 +6,7 @@ permissions:
|
|||
pull-requests: write
|
||||
|
||||
env:
|
||||
# Pinned so a PR only goes red for reasons in the PR. The 2026-07-28
|
||||
# scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them.
|
||||
# conformance-weekly.yaml is what tracks upstream releases.
|
||||
# Pinned exact version; conformance-weekly.yaml tracks upstream releases.
|
||||
CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"
|
||||
|
||||
jobs:
|
||||
|
|
@ -105,16 +103,11 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# One entry per lifecycle, not per spec revision: `--spec-version` is
|
||||
# cumulative across the dated revisions, so 2025-11-25 already covers
|
||||
# 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it
|
||||
# is the SEP-2575 stateless wire, served from its own endpoint with its
|
||||
# own baseline, so it needs its own run.
|
||||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
path: '/'
|
||||
baseline: conformance-baseline.yml
|
||||
baseline: conformance-baseline-2025-11-25.yml
|
||||
- spec-version: '2026-07-28'
|
||||
path: '/stateless'
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
|
|
@ -180,7 +173,7 @@ jobs:
|
|||
matrix:
|
||||
include:
|
||||
- spec-version: '2025-11-25'
|
||||
baseline: conformance-baseline.yml
|
||||
baseline: conformance-baseline-2025-11-25.yml
|
||||
- spec-version: '2026-07-28'
|
||||
baseline: conformance-baseline-2026-07-28.yml
|
||||
|
||||
|
|
|
|||
10
Makefile
10
Makefile
|
|
@ -37,14 +37,14 @@ conformance-server:
|
|||
@echo "Waiting for server to start..."
|
||||
@sleep 5
|
||||
rm -rf tests/Conformance/results
|
||||
cd tests/Conformance && npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --spec-version 2025-11-25 --output-dir results || true
|
||||
php tests/Conformance/score.php server
|
||||
cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/ --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true
|
||||
php tests/Conformance/score.php server 2025-11-25
|
||||
docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
|
||||
|
||||
conformance-client:
|
||||
rm -rf tests/Conformance/results
|
||||
cd tests/Conformance && npx @modelcontextprotocol/conformance client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline.yml --output-dir results || true
|
||||
php tests/Conformance/score.php client
|
||||
cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2025-11-25 --expected-failures conformance-baseline-2025-11-25.yml --output-dir results || true
|
||||
php tests/Conformance/score.php client 2025-11-25
|
||||
|
||||
# --- 2026-07-28 (SEP-2575 stateless lifecycle) ------------------------------
|
||||
|
||||
|
|
@ -56,11 +56,13 @@ conformance-draft-server:
|
|||
@sleep 5
|
||||
rm -rf tests/Conformance/results-2026-07-28
|
||||
cd tests/Conformance && $(CONFORMANCE) server --url http://localhost:8000/stateless --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
php tests/Conformance/score.php server 2026-07-28 results-2026-07-28
|
||||
docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
|
||||
|
||||
conformance-draft-client:
|
||||
rm -rf tests/Conformance/results-2026-07-28
|
||||
cd tests/Conformance && $(CONFORMANCE) client --command "php $(CURDIR)/tests/Conformance/client.php" --suite all --spec-version 2026-07-28 --expected-failures conformance-baseline-2026-07-28.yml --output-dir results-2026-07-28 || true
|
||||
php tests/Conformance/score.php client 2026-07-28 results-2026-07-28
|
||||
|
||||
coverage:
|
||||
XDEBUG_MODE=coverage vendor/bin/phpunit --testsuite=unit --coverage-html=coverage
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@
|
|||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/pipeline.yaml)
|
||||
[](https://packagist.org/packages/mcp/sdk)
|
||||
[](LICENSE)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
[](https://github.com/modelcontextprotocol/php-sdk/actions/workflows/conformance-weekly.yaml)
|
||||
|
||||
</div>
|
||||
|
||||
|
|
|
|||
53
tests/Conformance/conformance-baseline-2025-11-25.yml
Normal file
53
tests/Conformance/conformance-baseline-2025-11-25.yml
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# Expected conformance failures for spec revision 2025-11-25.
|
||||
#
|
||||
# Separate from conformance-baseline-2026-07-28.yml because the two revisions
|
||||
# exercise different lifecycles against different endpoints: this file covers
|
||||
# the handshake era served at /, the other the modern (SEP-2575) era served
|
||||
# at /stateless.
|
||||
#
|
||||
# Entries are per-check (scenario:check-id) rather than whole scenarios, so a
|
||||
# scenario that is mostly conformant keeps reporting the parts that work and a
|
||||
# new regression inside it still fails the run.
|
||||
#
|
||||
# Run with:
|
||||
# make conformance-server
|
||||
# make conformance-client
|
||||
|
||||
# Both pre-existing fixture bugs, not lifecycle-specific: json-schema-2020-12
|
||||
# wants a `json_schema_2020_12_tool` fixture neither conformance server
|
||||
# defines, and resources-templates-read emits a resource-template URI in a
|
||||
# shape the schema rejects. Reproduce identically on the 2026-07-28 endpoint
|
||||
# (see conformance-baseline-2026-07-28.yml).
|
||||
server:
|
||||
- json-schema-2020-12:json-schema-2020-12-tool-found
|
||||
- resources-templates-read:wire-schema-valid
|
||||
|
||||
client:
|
||||
- elicitation-sep1034-client-defaults
|
||||
- sse-retry
|
||||
# The example client doesn't call add_numbers or echo the tool's inputSchema
|
||||
# back, so these checks have nothing to exercise.
|
||||
- tools_call:tool-add-numbers
|
||||
- json-schema-2020-12-preservation:json-schema-2020-12-client-echo-completed
|
||||
- auth/metadata-default
|
||||
- auth/metadata-var1
|
||||
- auth/metadata-var2
|
||||
- auth/metadata-var3
|
||||
- auth/basic-cimd
|
||||
- auth/scope-from-www-authenticate
|
||||
- auth/scope-from-scopes-supported
|
||||
- auth/scope-omitted-when-undefined
|
||||
- auth/scope-step-up
|
||||
- auth/scope-retry-limit
|
||||
- auth/token-endpoint-auth-basic
|
||||
- auth/token-endpoint-auth-post
|
||||
- auth/token-endpoint-auth-none
|
||||
- auth/pre-registration
|
||||
- auth/2025-03-26-oauth-metadata-backcompat
|
||||
- auth/2025-03-26-oauth-endpoint-fallback
|
||||
- auth/offline-access-scope
|
||||
- auth/offline-access-not-supported
|
||||
- auth/client-credentials-jwt
|
||||
- auth/client-credentials-basic
|
||||
- auth/cross-app-access-complete-flow
|
||||
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
# Expected conformance failures for spec revision 2026-07-28.
|
||||
#
|
||||
# Separate from conformance-baseline.yml because the two revisions exercise
|
||||
# different lifecycles against different endpoints: the dated baseline covers
|
||||
# the handshake era served at /, this one covers the modern (SEP-2575) era
|
||||
# Separate from conformance-baseline-2025-11-25.yml because the two revisions
|
||||
# exercise different lifecycles against different endpoints: the other file
|
||||
# covers the handshake era served at /, this one the modern (SEP-2575) era
|
||||
# served at /stateless.
|
||||
#
|
||||
# Entries are per-check (scenario:check-id) rather than whole scenarios, so a
|
||||
|
|
@ -11,6 +11,7 @@
|
|||
#
|
||||
# Run with:
|
||||
# make conformance-draft-server
|
||||
# make conformance-draft-client
|
||||
|
||||
server:
|
||||
# --- Pre-existing, not lifecycle-specific ----------------------------------
|
||||
|
|
|
|||
|
|
@ -13,13 +13,14 @@
|
|||
* Turns a conformance run's results into a shields.io endpoint badge JSON, so
|
||||
* the client/server conformance score can be rendered in the README.
|
||||
*
|
||||
* php score.php <server|client>
|
||||
* php score.php <server|client> <spec-version> [results-dir]
|
||||
*
|
||||
* The conformance CLI (run with `--output-dir results`) writes one
|
||||
* `checks.json` per scenario into the `results/` directory next to this file.
|
||||
* A scenario counts as passing when none of its checks has a FAILURE status;
|
||||
* the badge message is "<passed>/<total> (<pct>%)" and is written to
|
||||
* `<suite>-conformance.json`.
|
||||
* The conformance CLI writes one `checks.json` per scenario into
|
||||
* `[results-dir]` (default `results`), relative to this file. A scenario
|
||||
* counts as passing when none of its checks has a FAILURE status; the badge
|
||||
* message is "<passed>/<total> (<pct>%)" and is written to
|
||||
* `<suite>-conformance-<spec-version>.json`, so each revision gets its own
|
||||
* badge.
|
||||
*/
|
||||
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
|
|
@ -36,10 +37,13 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
|
|||
->setName('conformance-score')
|
||||
->setDescription('Generates a shields.io endpoint badge from the conformance results')
|
||||
->addArgument('suite', InputArgument::REQUIRED, 'Which conformance suite was run: "server" or "client"')
|
||||
->addArgument('spec-version', InputArgument::REQUIRED, 'The spec revision that was run, e.g. "2025-11-25"')
|
||||
->addArgument('results-dir', InputArgument::OPTIONAL, 'Directory the conformance CLI wrote results into, relative to this file', 'results')
|
||||
->setCode(static function (InputInterface $input, OutputInterface $output): int {
|
||||
$io = new SymfonyStyle($input, $output);
|
||||
|
||||
$suite = $input->getArgument('suite');
|
||||
$specVersion = $input->getArgument('spec-version');
|
||||
|
||||
if (!in_array($suite, ['server', 'client'], true)) {
|
||||
$io->error(sprintf('Suite must be "server" or "client", got "%s".', $suite));
|
||||
|
|
@ -47,10 +51,10 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
|
|||
return Command::INVALID;
|
||||
}
|
||||
|
||||
$resultsDir = __DIR__.'/results';
|
||||
$resultsDir = __DIR__.'/'.$input->getArgument('results-dir');
|
||||
|
||||
if (!is_dir($resultsDir)) {
|
||||
$io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir results` first.', $resultsDir));
|
||||
$io->error(sprintf('Results directory "%s" does not exist; run the conformance suite with `--output-dir` first.', $resultsDir));
|
||||
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
|
@ -88,7 +92,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
|
|||
|
||||
$badge = [
|
||||
'schemaVersion' => 1,
|
||||
'label' => $suite.' conformance',
|
||||
'label' => sprintf('%s %s', $suite, $specVersion),
|
||||
'message' => $total > 0 ? sprintf('%d/%d (%d%%)', $passed, $total, $pct) : 'no data',
|
||||
'color' => match (true) {
|
||||
0 === $total => 'lightgrey',
|
||||
|
|
@ -99,7 +103,7 @@ require_once dirname(__DIR__, 2).'/vendor/autoload.php';
|
|||
},
|
||||
];
|
||||
|
||||
$outputFile = __DIR__.'/'.$suite.'-conformance.json';
|
||||
$outputFile = __DIR__.'/'.$suite.'-conformance-'.$specVersion.'.json';
|
||||
|
||||
if (false === file_put_contents($outputFile, json_encode($badge, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES)."\n")) {
|
||||
$io->error(sprintf('Could not write badge file "%s".', $outputFile));
|
||||
|
|
|
|||
Loading…
Reference in a new issue