modelcontextprotocol/rust-sdk
Watch
1
0
Fork
You've already forked rust-sdk
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

chore: run full conformance CI suite (#1010)

Browse source
This commit is contained in:
Jack Amadeo 2026-07-20 17:24:30 -04:00 • committed by GitHub
commit dd11e5be99
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 99 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

66
.github/workflows/conformance.yml vendored
View file

@ -14,11 +14,15 @@ concurrency:
env:
# Pinned for reproducible runs; bump deliberately when the suite updates.
CONFORMANCE_VERSION: "0.1.16"
# When updating DRAFT_CONFORMANCE_VERSION, diff
# `conformance list --spec-version 2026-07-28`
# and update #977
DRAFT_CONFORMANCE_VERSION: "0.2.0-alpha.9"
jobs:
server:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
@ -81,7 +85,9 @@ jobs:
echo "draft conformance server did not become ready" >&2
exit 1
# Run discovery separately until #985 enables the full draft suite.
# useful for enforcing partial conformance for SEP-2575.
# when `server-stateless` is removed from `expected-failures-2026-07-28.yaml`,
# this step can be removed.
- name: Run SEP-2575 discovery contract
run: |
endpoint=http://127.0.0.1:8002/mcp
@ -91,6 +97,8 @@ jobs:
-H "Mcp-Method: server/discover"
)
# The stateless server streams responses as SSE, so unwrap the
# `data:` payload before parsing as JSON.
discover_response="$(
curl --fail-with-body --silent --show-error \
"${common_headers[@]}" \
@ -110,7 +118,8 @@ jobs:
}
}
}' \
"$endpoint"
"$endpoint" \
| sed -n 's/^data: //p'
)"
jq -e '
.result.resultType == "complete" and
@ -152,35 +161,14 @@ jobs:
(.error.data.supported | index("2026-07-28") != null)
' /tmp/unsupported-version.json
# Keep this explicit list until the full draft suite is enabled by #985.
- name: Run supported draft server scenarios
- name: Run 2026-07-28 server suite
run: |
for scenario in \
sep-2164-resource-not-found \
caching \
http-header-validation \
http-custom-header-server-validation \
input-required-result-basic-elicitation \
input-required-result-basic-sampling \
input-required-result-basic-list-roots \
input-required-result-request-state \
input-required-result-multiple-input-requests \
input-required-result-multi-round \
input-required-result-missing-input-response \
input-required-result-non-tool-request \
input-required-result-result-type \
input-required-result-unsupported-methods \
input-required-result-tampered-state \
input-required-result-capability-check \
input-required-result-ignore-extra-params \
input-required-result-validate-input \
; do
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8002/mcp \
--scenario "$scenario" \
--spec-version draft \
-o conformance-results
done
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" server \
--url http://127.0.0.1:8002/mcp \
--suite all \
--spec-version 2026-07-28 \
--expected-failures conformance/expected-failures-2026-07-28.yaml \
-o conformance-results
- name: Stop conformance servers
if: always()
@ -197,6 +185,7 @@ jobs:
client:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
@ -218,21 +207,14 @@ jobs:
--spec-version 2025-11-25 \
-o conformance-client-results/full
# SEP-2322 MRTR client scenario (spec 2026-07-28).
- name: Run draft SEP-2322 client scenario
- name: Run 2026-07-28 client suite
run: |
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
--command "$(pwd)/target/debug/conformance-client" \
--scenario sep-2322-client-request-state \
-o conformance-client-results/mrtr
- name: Run draft SEP-2575 client scenario
run: |
npx -y "@modelcontextprotocol/conformance@${DRAFT_CONFORMANCE_VERSION}" client \
--command "$(pwd)/target/debug/conformance-client" \
--scenario request-metadata \
--spec-version draft \
-o conformance-client-results/sep-2575
--suite all \
--spec-version 2026-07-28 \
--expected-failures conformance/expected-failures-2026-07-28.yaml \
-o conformance-client-results/draft
- name: Upload results
if: always()

34
conformance/expected-failures-2026-07-28.yaml Normal file
View file

@ -0,0 +1,34 @@
# Known failures for the pinned 2026-07-28 draft conformance suite
# (@modelcontextprotocol/conformance DRAFT_CONFORMANCE_VERSION).
#
# The full suites run in CI with `--expected-failures` pointing at this file:
# - a scenario failing that is NOT listed here fails the build
# - a scenario listed here that starts passing also fails the build (stale entry),
# so remove it from this list when the underlying issue is fixed.
#
# When bumping DRAFT_CONFORMANCE_VERSION, diff
# `conformance list --spec-version 2026-07-28` and update #977.
server:
# SEP-2575 stateless lifecycle gaps: discover capability declaration,
# missing-capability rejection, HTTP 404 for removed methods, and
# diagnostic tools (test_missing_capability, test_streaming_elicitation,
# test_logging_tool) not yet implemented.
# tracked in #1004
- server-stateless
# SEP-2106: composition/conditional/$anchor keywords are stripped from
# published tool input schemas.
# tracked in #1003
- json-schema-2020-12
client:
# Client does not yet send MCP-Protocol-Version header pre-initialize as
# required by the 2026-07-28 stateless lifecycle.
# tracked in #1002
- tools_call
# Auth feature gaps in the 2026-07-28 auth scenarios.
# tracked in #1002
- auth/scope-step-up
# SEP-2352: SDK lacks issuer-stamped credential storage (#879), so the
# sep-2352-reregister-on-as-change check fails.
- auth/authorization-server-migration

66
conformance/src/bin/client.rs
View file

@ -1006,25 +1006,49 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("Running scenario '{}' against {}", scenario, server_url);
match scenario.as_str() {
// Safety net: some harness servers intentionally misbehave (e.g. reply
// with an id-less error instead of answering a request), which would
// leave the client waiting forever. Exit on our own before the harness's
// 30s client timeout so it never has to kill us (which has been observed
// to wedge the harness process in CI).
let timeout_secs: u64 = std::env::var("MCP_CONFORMANCE_TIMEOUT_SECS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(25);
tokio::time::timeout(
std::time::Duration::from_secs(timeout_secs),
run_scenario(&scenario, &server_url, &ctx),
)
.await
.map_err(|_| anyhow::anyhow!("Scenario '{scenario}' timed out after {timeout_secs}s"))??;
Ok(())
}
async fn run_scenario(
scenario: &str,
server_url: &str,
ctx: &ConformanceContext,
) -> anyhow::Result<()> {
match scenario {
// Non-auth scenarios
"initialize" => run_basic_client(&server_url).await?,
"initialize" => run_basic_client(server_url).await?,
// SEP-2106: the scenario serves a tool whose schema carries a network
// `$ref`; the check passes when the client lists tools without
// dereferencing (fetching) that URL. A plain connect → list_tools →
// close is sufficient; the scenario's mock server does not implement
// the discover lifecycle, so `run_discover_client` hangs against it.
"json-schema-ref-no-deref" => run_basic_client(&server_url).await?,
"tools_call" => run_tools_call_client(&server_url, &ctx).await?,
"json-schema-ref-no-deref" => run_basic_client(server_url).await?,
"tools_call" => run_tools_call_client(server_url, ctx).await?,
"elicitation-sep1034-client-defaults" => {
run_elicitation_defaults_client(&server_url).await?
run_elicitation_defaults_client(server_url).await?
}
"sse-retry" => run_sse_retry_client(&server_url).await?,
"sse-retry" => run_sse_retry_client(server_url).await?,
"request-metadata" | "sep-2322-client-request-state" => {
run_discover_client(&server_url).await?
run_discover_client(server_url).await?
}
"http-standard-headers" | "http-custom-headers" | "http-invalid-tool-headers" => {
run_discover_tools_call_client(&server_url, &ctx).await?
run_discover_tools_call_client(server_url, ctx).await?
}
// Auth scenarios - standard OAuth flow
@ -1056,34 +1080,26 @@ async fn main() -> anyhow::Result<()> {
| "auth/iss-wrong-issuer"
| "auth/iss-unexpected"
| "auth/iss-normalized"
| "auth/metadata-issuer-mismatch"
| "auth/metadata-issuer-mismatch"
// SEP-2352: PRM `authorization_servers` switches between calls; a
// compliant client re-registers at the new AS. Known partial failure:
// the SDK lacks issuer-stamped credential storage (#879), so the
// `sep-2352-reregister-on-as-change` check fails. Left on the standard
// flow rather than fixture-orchestrated re-registration so the
// conformance result reflects real SDK behavior.
| "auth/authorization-server-migration" => run_auth_client(&server_url, &ctx).await?,
| "auth/metadata-issuer-mismatch" => run_auth_client(server_url, ctx).await?,
// Auth - scope step-up
"auth/scope-step-up" => run_auth_scope_step_up_client(&server_url, &ctx).await?,
"auth/scope-step-up" => run_auth_scope_step_up_client(server_url, ctx).await?,
// Auth - scope retry limit
"auth/scope-retry-limit" => run_auth_scope_retry_limit_client(&server_url, &ctx).await?,
"auth/scope-retry-limit" => run_auth_scope_retry_limit_client(server_url, ctx).await?,
// Auth - authorization server migration (SEP-2352)
"auth/authorization-server-migration" => {
run_auth_server_migration_client(&server_url, &ctx).await?
run_auth_server_migration_client(server_url, ctx).await?
}
// Auth - pre-registration
"auth/pre-registration" => run_auth_preregistered_client(&server_url, &ctx).await?,
"auth/pre-registration" => run_auth_preregistered_client(server_url, ctx).await?,
// Auth - resource mismatch (should fail to auth → pass)
"auth/resource-mismatch" => {
// Try to auth; it should fail because PRM resource doesn't match
match run_auth_client(&server_url, &ctx).await {
match run_auth_client(server_url, ctx).await {
Ok(_) => {
tracing::warn!("Auth succeeded despite resource mismatch!");
}
@ -1094,12 +1110,12 @@ async fn main() -> anyhow::Result<()> {
}
// Auth - client credentials
"auth/client-credentials-basic" => run_client_credentials_basic(&server_url, &ctx).await?,
"auth/client-credentials-jwt" => run_client_credentials_jwt(&server_url, &ctx).await?,
"auth/client-credentials-basic" => run_client_credentials_basic(server_url, ctx).await?,
"auth/client-credentials-jwt" => run_client_credentials_jwt(server_url, ctx).await?,
// Auth - cross-app access
"auth/cross-app-access-complete-flow" => {
run_cross_app_access_client(&server_url, &ctx).await?
run_cross_app_access_client(server_url, ctx).await?
}
unknown => anyhow::bail!("Unsupported conformance scenario: {unknown}"),