fix(validators): harden mcp-name matching (PyPI/NuGet anchoring, comment-form safe) + cargo follow-ups (#1331)

Registry-wide hardening of the `mcp-name:` ownership-token match, plus
the cargo follow-up fixes from the #1330 review — consolidated here per
maintainer request. Rebased onto current `main` (the merged #1330 cargo
commit drops out), so the diff is just the net-new work.

### Changes
- **PyPI/NuGet: boundary-anchored token match.** Replace
`strings.Contains(readme, "mcp-name: "+name)` with the shared
`containsMCPNameToken`, so a README declaring a longer name
(`…/widget-pro`) no longer satisfies an ownership claim for a shorter
prefix (`…/widget`). (Cargo already got this in #1330; NPM is unaffected
— it compares an exact metadata field.)
- **Matcher: treat the HTML comment close as a boundary.** `<!--
mcp-name: NAME-->` / `<!--mcp-name: NAME-->` (any spacing) validate
again — the documented hidden-comment form for PyPI/NuGet — while a
genuine longer name (`…/widget--pro`) still does not.
- **Cargo hardening (review of #1330):** pin scheme+port (not just host)
on the README fetch and redirects; a rate-limited/failed crate-version
existence probe now reports *transient* instead of "not found"; a 403
with the crate present no longer flatly asserts "no README".
- **Docs:** note the token must be followed by a boundary.

### ⚠️ Behavior change for PyPI/NuGet — read before merging
The match is **strictly stricter** (verified by fuzz, 2.3M execs: it can
only flip pass→fail, never fail→pass). After the comment-close fix, the
**only** forms that newly fail are unusual *inline* ones: the token
ending a sentence (`…/my-mcp.`) or glued to a trailing `/`. The
documented forms (own line, in `<!-- … -->`) are unaffected.

**Correcting an earlier claim:** this re-validates on **edits too**, not
just new versions — `edit.go → UpdateServer → ValidateUpdateRequest →
validateRegistryOwnership` runs the token check on any edit of a live
server (only delete-transitions skip it). So an existing PyPI/NuGet
server whose README uses one of the breaking inline forms would fail on
its next publish *or* edit. Given the v0.1 API-freeze posture, this
should land as a deliberate, noted change.

### Testing
`go build`, `vet`, `gofmt`, `golangci-lint` (prod files),
`check-schema`, `validate-examples` all clean. Hermetic + **live**
PyPI/NuGet/cargo suite passes. New: `TestCargoURLAllowed`, a
combined-fixture probe-429→transient case, comment-form matcher cases,
and `FuzzContainsMCPNameToken` (the strictly-stricter property, 2.3M
execs).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

internal/validators/registries/cargo.go

@@ -106,16 +106,37 @@ func cargoAllowedHosts(baseURL string) map[string]struct{} {
 	return hosts
 }
 
+// cargoURLAllowed reports whether u is a URL the validator may fetch for the
+// given base. The host must be in allowedHosts; additionally, for the real
+// crates.io base, the scheme must be https and the port must be the default —
+// so a metadata response or redirect cannot downgrade to http or steer the
+// validator at a non-standard port on an otherwise-trusted host. For test bases
+// (httptest servers) only the host is checked, so mocks keep working.
+func cargoURLAllowed(u *url.URL, baseURL string, allowedHosts map[string]struct{}) bool {
+	if _, ok := allowedHosts[u.Hostname()]; !ok {
+		return false
+	}
+	if baseURL == model.RegistryURLCrates {
+		if u.Scheme != "https" {
+			return false
+		}
+		if p := u.Port(); p != "" && p != "443" {
+			return false
+		}
+	}
+	return true
+}
+
 // newCargoHTTPClient builds the client used for all crates.io calls. The
-// CheckRedirect policy pins every redirect hop to allowedHosts, so even though
-// the initial URL is host-pinned, an upstream 3xx cannot redirect the validator
-// to an unexpected host.
-func newCargoHTTPClient(allowedHosts map[string]struct{}) *http.Client {
+// CheckRedirect policy pins every redirect hop via cargoURLAllowed, so even
+// though the initial URL is checked, an upstream 3xx cannot redirect the
+// validator to an unexpected host, scheme, or port.
+func newCargoHTTPClient(baseURL string, allowedHosts map[string]struct{}) *http.Client {
 	return &http.Client{
 		Timeout: 10 * time.Second,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			if _, ok := allowedHosts[req.URL.Hostname()]; !ok {
-				return fmt.Errorf("refusing redirect to unexpected host %q", req.URL.Hostname())
+			if !cargoURLAllowed(req.URL, baseURL, allowedHosts) {
+				return fmt.Errorf("refusing redirect to unexpected URL %q", req.URL.Redacted())
 			}
 			if len(via) >= 10 {
 				return errors.New("stopped after 10 redirects")
@@ -125,37 +146,53 @@ func newCargoHTTPClient(allowedHosts map[string]struct{}) *http.Client {
 	}
 }
 
-// cargoVersionExists checks whether a specific crate version exists on crates.io,
-// used to disambiguate a 403 from the README CDN. static.crates.io (S3) returns
-// 403 both for a genuinely-missing crate/version AND for a crate that exists but
-// has no rendered README, so a 403 alone cannot tell a publisher which it is.
-//
-// Returns (exists, determined): determined is false if the existence endpoint
-// itself was unreachable or returned an unexpected status, in which case the
-// caller should fall back to a generic message rather than assert existence.
-func cargoVersionExists(ctx context.Context, client *http.Client, baseURL, identifier, version string) (exists, determined bool) {
+// cargoVersionState is the outcome of probing the crate-version metadata
+// endpoint, used to disambiguate a 403 from the README CDN.
+type cargoVersionState int
+
+const (
+	// cargoVersionUnknown: the probe returned an unexpected status we can't classify.
+	cargoVersionUnknown cargoVersionState = iota
+	// cargoVersionExists: the crate version exists (200).
+	cargoVersionExists
+	// cargoVersionMissing: the crate version does not exist (404).
+	cargoVersionMissing
+	// cargoVersionTransient: the probe failed for a retryable reason (network
+	// error, 429, or 5xx) — existence is undetermined and the caller should not
+	// report "not found".
+	cargoVersionTransient
+)
+
+// probeCargoVersion checks whether a specific crate version exists on crates.io.
+// static.crates.io (S3) returns 403 both for a genuinely-missing crate/version
+// AND for a crate that exists but has no rendered README, so a 403 from the CDN
+// alone cannot tell a publisher which it is; this probe disambiguates, while
+// distinguishing a transient failure from a definitive missing/exists answer.
+func probeCargoVersion(ctx context.Context, client *http.Client, baseURL, identifier, version string) cargoVersionState {
 	versionURL := fmt.Sprintf("%s/api/v1/crates/%s/%s",
 		baseURL, url.PathEscape(identifier), url.PathEscape(version))
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, versionURL, nil)
 	if err != nil {
-		return false, false
+		return cargoVersionUnknown
 	}
 	req.Header.Set("User-Agent", cargoUserAgent)
 	req.Header.Set("Accept", "application/json")
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return false, false
+		return cargoVersionTransient
 	}
 	defer resp.Body.Close()
 
-	switch resp.StatusCode {
-	case http.StatusOK:
-		return true, true
-	case http.StatusNotFound:
-		return false, true
+	switch {
+	case resp.StatusCode == http.StatusOK:
+		return cargoVersionExists
+	case resp.StatusCode == http.StatusNotFound:
+		return cargoVersionMissing
+	case resp.StatusCode == http.StatusTooManyRequests, resp.StatusCode >= 500 && resp.StatusCode < 600:
+		return cargoVersionTransient
 	default:
-		return false, false
+		return cargoVersionUnknown
 	}
 }
 
@@ -196,15 +233,21 @@ func cargoReadmeStatusError(ctx context.Context, client *http.Client, pkg model.
 // endpoint so the publisher gets an actionable message rather than a blanket
 // "not found".
 func cargoReadme403Error(ctx context.Context, client *http.Client, pkg model.Package, serverName string) error {
-	exists, determined := cargoVersionExists(ctx, client, pkg.RegistryBaseURL, pkg.Identifier, pkg.Version)
-	switch {
-	case determined && exists:
-		return fmt.Errorf("cargo package '%s' version '%s' exists on crates.io but has no rendered README. Add a README containing 'mcp-name: %s' and publish a new version", pkg.Identifier, pkg.Version, serverName)
-	case determined && !exists:
+	switch probeCargoVersion(ctx, client, pkg.RegistryBaseURL, pkg.Identifier, pkg.Version) {
+	case cargoVersionExists:
+		// The crate/version exists but the README CDN returned 403. The likely
+		// cause is a missing README, but a 403 is not definitive proof (e.g. a
+		// transient CDN/WAF block), so don't flatly assert "no README".
+		return fmt.Errorf("cargo package '%s' version '%s' exists on crates.io, but its rendered README could not be retrieved (status: 403). If it has no README, add one containing 'mcp-name: %s' and publish a new version", pkg.Identifier, pkg.Version, serverName)
+	case cargoVersionMissing:
 		return fmt.Errorf("cargo package '%s' version '%s' not found on crates.io", pkg.Identifier, pkg.Version)
-	default:
-		return fmt.Errorf("cargo package '%s' version '%s' not found on crates.io (status: 403)", pkg.Identifier, pkg.Version)
+	case cargoVersionTransient:
+		return fmt.Errorf("crates.io could not confirm cargo package '%s' version '%s' (README status: 403, version check inconclusive) — likely transient, retry later", pkg.Identifier, pkg.Version)
+	case cargoVersionUnknown:
+		// Probe returned an unclassifiable status — fall through to the
+		// best-effort message below.
 	}
+	return fmt.Errorf("cargo package '%s' version '%s' not found on crates.io (status: 403)", pkg.Identifier, pkg.Version)
 }
 
 // validateCargoREADME performs the two-call README fetch and the mcp-name token
@@ -213,7 +256,7 @@ func cargoReadme403Error(ctx context.Context, client *http.Client, pkg model.Pac
 // bypassing the exact-baseURL guard that ValidateCargo enforces for callers.
 func validateCargoREADME(ctx context.Context, pkg model.Package, serverName string) error {
 	allowedHosts := cargoAllowedHosts(pkg.RegistryBaseURL)
-	client := newCargoHTTPClient(allowedHosts)
+	client := newCargoHTTPClient(pkg.RegistryBaseURL, allowedHosts)
 
 	// Step 1: fetch the README pointer from the documented API endpoint.
 	metaURL := fmt.Sprintf("%s/api/v1/crates/%s/%s/readme",
@@ -246,14 +289,15 @@ func validateCargoREADME(ctx context.Context, pkg model.Package, serverName stri
 		return fmt.Errorf("cargo package '%s' metadata response missing 'url' field", pkg.Identifier)
 	}
 
-	// Pin the README pointer to an allowed host before fetching it, so a metadata
-	// response cannot steer the validator at an internal or attacker-chosen host.
+	// Pin the README pointer to an allowed host/scheme/port before fetching it, so
+	// a metadata response cannot steer the validator at an internal or
+	// attacker-chosen URL.
 	readmeParsed, err := url.Parse(meta.URL)
 	if err != nil || readmeParsed.Hostname() == "" {
 		return fmt.Errorf("cargo package '%s': crates.io returned an unparseable README URL", pkg.Identifier)
 	}
-	if _, ok := allowedHosts[readmeParsed.Hostname()]; !ok {
-		return fmt.Errorf("cargo package '%s': crates.io returned a README URL on unexpected host %q — refusing to fetch", pkg.Identifier, readmeParsed.Hostname())
+	if !cargoURLAllowed(readmeParsed, pkg.RegistryBaseURL, allowedHosts) {
+		return fmt.Errorf("cargo package '%s': crates.io returned a README URL on an unexpected host/scheme %q — refusing to fetch", pkg.Identifier, readmeParsed.Redacted())
 	}
 
 	// Step 2: fetch the rendered README from the (now host-validated) URL.
@@ -287,5 +331,11 @@ func validateCargoREADME(ctx context.Context, pkg model.Package, serverName stri
 		return nil
 	}
 
+	// If the token IS present but glued to a trailing character, explain that
+	// rather than telling the publisher to add a token they can already see.
+	if trailing, glued := mcpNameTokenGluedTrailing(string(body), serverName); glued {
+		return fmt.Errorf("cargo package '%s' ownership validation failed: found 'mcp-name: %s' in the README, but it is immediately followed by %q rather than a boundary. The token must be followed by a space, newline, or an HTML tag — put it on its own line and publish a new version", pkg.Identifier, serverName, trailing)
+	}
+
 	return fmt.Errorf("cargo package '%s' ownership validation failed. The server name '%s' must appear as 'mcp-name: %s' in the package README", pkg.Identifier, serverName, serverName)
 }