modelcontextprotocol/python-sdk
Watch
1
0
Fork
You've already forked python-sdk
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests/interaction: era-axis machinery for the requirements manifest (#2909)

Browse source
This commit is contained in:
Max 2026-06-19 12:59:39 +01:00 • committed by GitHub
commit 364b762ef6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1142 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

258
tests/interaction/test_coverage.py
View file

@ -11,10 +11,29 @@ import importlib
import re
from pathlib import Path
from types import ModuleType
from typing import cast
import pytest
from tests.interaction._requirements import REQUIREMENTS, Requirement, covered_by, requirement
from mcp.shared.version import KNOWN_PROTOCOL_VERSIONS
from mcp.types import LATEST_PROTOCOL_VERSION
from tests.interaction._requirements import (
CONNECTABLE_TRANSPORTS,
REQUIREMENTS,
SPEC_2026_BASE_URL,
SPEC_BASE_URL,
SPEC_VERSIONS,
ArmExclusion,
KnownFailure,
Requirement,
SpecVersion,
Transport,
cell_id,
compute_cells,
covered_by,
requirement,
)
from tests.interaction.conftest import _FACTORIES
_SUITE_ROOT = Path(__file__).parent
_REPO_ROOT = _SUITE_ROOT.parent.parent
@ -93,6 +112,87 @@ def test_deferral_reasons_cite_existing_paths() -> None:
assert not missing, f"Deferral reasons citing paths that do not exist: {missing}"
def test_spec_versions_are_known_and_include_latest() -> None:
"""Every active spec version is one the SDK knows about, and the SDK's latest is on the active axis."""
assert set(SPEC_VERSIONS) <= set(KNOWN_PROTOCOL_VERSIONS)
assert LATEST_PROTOCOL_VERSION in SPEC_VERSIONS
def test_spec_base_urls_are_pinned_to_their_revision() -> None:
"""SPEC_BASE_URL constants are pinned literals, so growing SPEC_VERSIONS cannot repoint existing source links."""
assert SPEC_BASE_URL == "https://modelcontextprotocol.io/specification/2025-11-25"
assert SPEC_2026_BASE_URL == "https://modelcontextprotocol.io/specification/2026-07-28"
def test_connectable_transports_match_connect_factories() -> None:
"""CONNECTABLE_TRANSPORTS and the conftest factory map name exactly the same transports."""
assert set(CONNECTABLE_TRANSPORTS) == set(_FACTORIES)
def test_supersession_links_are_symmetric_and_versioned() -> None:
"""``supersedes``/``superseded_by`` reference real entries, agree in both directions, and carry version bounds."""
broken = [
f"{req_id} -> {target}"
for req_id, req in REQUIREMENTS.items()
for target in req.supersedes
if target not in REQUIREMENTS or REQUIREMENTS[target].superseded_by != req_id or req.added_in is None
] + [
f"{req_id} <- {req.superseded_by}"
for req_id, req in REQUIREMENTS.items()
if req.superseded_by is not None
if req.superseded_by not in REQUIREMENTS
or req_id not in REQUIREMENTS[req.superseded_by].supersedes
or req.removed_in is None
]
assert not broken, f"Broken supersession links (forward '->' or back '<-'): {broken}"
def test_removed_entry_has_disposition() -> None:
"""Every retired requirement carries either a forward link or a prose note explaining the retirement."""
undisposed = [
req_id
for req_id, req in REQUIREMENTS.items()
if req.removed_in is not None and req.superseded_by is None and req.note is None
]
assert not undisposed, f"Requirements with removed_in but no superseded_by or note: {undisposed}"
def test_transport_restriction_has_note() -> None:
"""Every transport-restricted requirement carries a note explaining why it is transport-specific."""
missing = [req_id for req_id, req in REQUIREMENTS.items() if req.transports is not None and req.note is None]
assert not missing, f"Requirements with transports= but no note: {missing}"
def test_every_arm_exclusion_targets_a_reachable_cell() -> None:
"""Every arm exclusion names a connectable transport (or wildcards).
spec_version is type-checked against the SpecVersion Literal and may reference a version not yet
on the active SPEC_VERSIONS axis, so pre-staged exclusions for an upcoming revision are permitted.
"""
unreachable = [
f"{req_id}: {exclusion}"
for req_id, req in REQUIREMENTS.items()
for exclusion in req.arm_exclusions
if exclusion.transport is not None and exclusion.transport not in CONNECTABLE_TRANSPORTS
]
assert not unreachable, f"Arm exclusions targeting unreachable cells: {unreachable}"
def test_every_known_failure_targets_a_reachable_cell() -> None:
"""Every known failure names a connectable transport (or wildcards).
spec_version is type-checked against the SpecVersion Literal and may reference a version not yet
on the active SPEC_VERSIONS axis, so pre-staged exclusions for an upcoming revision are permitted.
"""
unreachable = [
f"{req_id}: {failure}"
for req_id, req in REQUIREMENTS.items()
for failure in req.known_failures
if failure.transport is not None and failure.transport not in CONNECTABLE_TRANSPORTS
]
assert not unreachable, f"Known failures targeting unreachable cells: {unreachable}"
def test_unknown_requirement_id_is_rejected() -> None:
"""Marking a test with an ID that is not in the manifest fails at decoration time."""
with pytest.raises(KeyError, match="Unknown requirement id 'tools:call:does-not-exist'"):
@ -103,3 +203,159 @@ def test_invalid_requirement_source_is_rejected() -> None:
"""A requirement whose source is not a spec URL, 'sdk', or an issue reference fails at construction."""
with pytest.raises(ValueError, match="source must be a specification URL"):
Requirement(source="https://example.com/not-the-spec", behavior="Never constructed.")
def test_arm_exclusion_with_unknown_spec_version_is_rejected() -> None:
"""An arm exclusion naming a spec version outside KNOWN_PROTOCOL_VERSIONS fails at construction."""
with pytest.raises(ValueError, match="is not in KNOWN_PROTOCOL_VERSIONS"):
ArmExclusion(reason="requires-session", spec_version=cast("SpecVersion", "2099-01-01"))
def test_known_failure_with_empty_note_is_rejected() -> None:
"""A known failure with a blank note fails at construction."""
with pytest.raises(ValueError, match="note must be non-empty"):
KnownFailure(note=" ")
def test_known_failure_with_unknown_spec_version_is_rejected() -> None:
"""A known failure naming a spec version outside KNOWN_PROTOCOL_VERSIONS fails at construction."""
with pytest.raises(ValueError, match="is not in KNOWN_PROTOCOL_VERSIONS"):
KnownFailure(note="x", spec_version=cast("SpecVersion", "2099-01-01"))
def test_known_failure_with_malformed_issue_is_rejected() -> None:
"""A known failure whose issue reference is neither '#<n>' nor a GitHub URL fails at construction."""
with pytest.raises(ValueError, match="must be '#<n>' or a GitHub URL"):
KnownFailure(note="x", issue="not-a-link")
def test_requirement_with_unknown_added_in_is_rejected() -> None:
"""A requirement whose added_in is outside KNOWN_PROTOCOL_VERSIONS fails at construction."""
with pytest.raises(ValueError, match="added_in .* is not in KNOWN_PROTOCOL_VERSIONS"):
Requirement(source="sdk", behavior="x", added_in=cast("SpecVersion", "2099-01-01"))
def test_requirement_with_unknown_removed_in_is_rejected() -> None:
"""A requirement whose removed_in is outside KNOWN_PROTOCOL_VERSIONS fails at construction."""
with pytest.raises(ValueError, match="removed_in .* is not in KNOWN_PROTOCOL_VERSIONS"):
Requirement(source="sdk", behavior="x", removed_in=cast("SpecVersion", "2099-01-01"))
def test_requirement_with_empty_version_range_is_rejected() -> None:
"""A requirement whose added_in is not strictly earlier than its removed_in fails at construction."""
with pytest.raises(ValueError, match="must be earlier than"):
Requirement(source="sdk", behavior="x", added_in="2025-11-25", removed_in="2025-11-25")
def _req(
*,
added_in: SpecVersion | None = None,
removed_in: SpecVersion | None = None,
transports: tuple[Transport, ...] | None = None,
arm_exclusions: tuple[ArmExclusion, ...] = (),
known_failures: tuple[KnownFailure, ...] = (),
) -> Requirement:
"""Build a synthetic Requirement for compute_cells() unit tests."""
return Requirement(
source="sdk",
behavior="x",
added_in=added_in,
removed_in=removed_in,
transports=transports,
arm_exclusions=arm_exclusions,
known_failures=known_failures,
)
def test_compute_cells_with_no_requirements_yields_full_grid() -> None:
"""With a single-version axis, an empty requirement list yields one cell per connectable transport."""
cells = compute_cells([], spec_versions=("2025-11-25",))
assert [c.id for c in cells] == ["in-memory", "sse", "streamable-http", "streamable-http-stateless"]
assert [c.values for c in cells] == [
(("in-memory", "2025-11-25"),),
(("sse", "2025-11-25"),),
(("streamable-http", "2025-11-25"),),
(("streamable-http-stateless", "2025-11-25"),),
]
def test_compute_cells_intersects_stacked_version_ranges() -> None:
"""Stacked requirements intersect their [added_in, removed_in) windows: a cell survives only if all admit it."""
cells = compute_cells(
[_req(removed_in="2026-07-28"), _req(added_in="2025-11-25")],
spec_versions=("2025-11-25", "2026-07-28"),
)
assert [c.id for c in cells] == [
"in-memory-2025-11-25",
"sse-2025-11-25",
"streamable-http-2025-11-25",
"streamable-http-stateless-2025-11-25",
]
def test_compute_cells_drops_era_locked_transport_outside_its_versions() -> None:
"""A transport listed in TRANSPORT_SPEC_VERSIONS only appears for the spec versions it serves."""
cells = compute_cells([], spec_versions=("2025-11-25", "2026-07-28"))
assert [c.id for c in cells] == [
"in-memory-2025-11-25",
"sse-2025-11-25",
"streamable-http-2025-11-25",
"streamable-http-stateless-2025-11-25",
"in-memory-2026-07-28",
"streamable-http-2026-07-28",
"streamable-http-stateless-2026-07-28",
]
def test_compute_cells_honours_arm_exclusion_from_any_stacked_requirement() -> None:
"""An arm exclusion on any stacked requirement drops the matching cell even when other requirements have none."""
cells = compute_cells(
[_req(), _req(arm_exclusions=(ArmExclusion(reason="requires-session", transport="sse"),))],
spec_versions=("2025-11-25",),
)
assert [c.id for c in cells] == ["in-memory", "streamable-http", "streamable-http-stateless"]
def test_compute_cells_wildcard_arm_exclusion_drops_every_cell() -> None:
"""An arm exclusion with both transport and spec_version unset matches every cell, leaving none."""
cells = compute_cells([_req(arm_exclusions=(ArmExclusion(reason="requires-session"),))])
assert cells == []
def test_compute_cells_marks_known_failure_as_strict_xfail() -> None:
"""A known failure attaches a strict xfail mark to exactly the matching cell and leaves others unmarked."""
cells = compute_cells(
[_req(known_failures=(KnownFailure(note="broken on sse", transport="sse"),))],
spec_versions=("2025-11-25",),
)
by_id = {c.id: c for c in cells}
assert set(by_id) == {"in-memory", "sse", "streamable-http", "streamable-http-stateless"}
assert by_id["sse"].marks[0].name == "xfail"
assert by_id["sse"].marks[0].kwargs == {"reason": "broken on sse", "strict": True}
assert by_id["in-memory"].marks == ()
assert by_id["streamable-http"].marks == ()
assert by_id["streamable-http-stateless"].marks == ()
def test_compute_cells_wildcard_known_failure_marks_every_cell() -> None:
"""A known failure with both transport and spec_version unset marks every emitted cell as strict xfail."""
cells = compute_cells([_req(known_failures=(KnownFailure(note="all broken"),))], spec_versions=("2025-11-25",))
assert len(cells) == 4
assert all(c.marks[0].name == "xfail" for c in cells)
assert all(c.marks[0].kwargs == {"reason": "all broken", "strict": True} for c in cells)
def test_compute_cells_ignores_transports_field() -> None:
"""Requirement.transports is descriptive metadata only and does not filter the cell grid."""
cells = compute_cells([_req(transports=("stdio",))], spec_versions=("2025-11-25",))
assert [c.id for c in cells] == list(CONNECTABLE_TRANSPORTS)
def test_cell_id_omits_version_when_single_spec_version() -> None:
"""With a single-version axis the cell id is just the transport name."""
assert cell_id("sse", "2025-11-25", spec_versions=("2025-11-25",)) == "sse"
def test_cell_id_appends_version_when_multiple_spec_versions() -> None:
"""With more than one active spec version the cell id gains a -<version> suffix."""
assert cell_id("sse", "2025-11-25", spec_versions=("2025-11-25", "2026-07-28")) == "sse-2025-11-25"