From 80cb03ff7a6b515ed051681ac6e0c5abddd5d242 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 9 Dec 2025 10:19:38 -0500 Subject: [PATCH] Fix NuGet.org base URL default, improve API integration (#813) ## Motivation and Context This resolves https://github.com/modelcontextprotocol/registry/issues/738. There are two issues with the NuGet.org integration today: 1. Change the `registryBaseUrl` for NuGet to `https://api.nuget.org/v3/index.json`. `https://api.nuget.org` is not the entry point for our service and is not specific enough. `https://api.nuget.org/v3/index.json` is the value that users put into their tooling to connect to our public registry (it is installed by default in .NET client tooling). 2. Use the service index API to get the URL pattern for package READMEs. - This URL is not guaranteed to be stable. The service index URL (`registryBaseUrl`) is, however. Additionally, I polished up the error messages so various error modes are clearer. When packages are published to NuGet.org, they take 2-5 minutes to become available due to validations we run. Some possible states that are now handled and described more helpfully are: - Version exists, but the package has no README - The ID exists, but the version does not - The ID does not exist at all There are only 3 authors (me, another MSFT person, and a community member) explicitly setting the old value. The DB migration will fix the issue. A future publish will fail with the validation error but the error message is clear. `registry type and base URL do not match: 'https://api.nuget.org' is not valid for registry type 'nuget'. Expected: https://api.nuget.org/v3/index.json` ## How Has This Been Tested? - Unit tests, new and existing - Manual test locally - Set old data in the DB, `docker compose up`, confirm the migration updates the specific package ## Breaking Changes Yes, but minimal impact expected, unless there is a dependency on the specific `https://api.nuget.org` value downstream. I am not aware of any such dependency, and the new value is more proper and specific. ## Types of changes - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [x] Breaking change (fix or feature that would cause existing functionality to change) - [x] Documentation update ## Checklist - [x] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [x] My code follows the repository's style guidelines - [x] New and existing tests pass locally - [x] I have added appropriate error handling - [x] I have added or updated documentation as needed ## Additional context For more information about the NuGet API see: https://learn.microsoft.com/en-us/nuget/api/overview#service-index (I wrote most of the docs). --- complete.md | 11 +- .../modelcontextprotocol-io/package-types.mdx | 2 +- docs/reference/api/openapi.yaml | 2 +- .../server-json/generic-server-json.md | 4 +- .../official-registry-requirements.md | 2 +- docs/reference/server-json/server.schema.json | 2 +- .../012_fix_nuget_registry_base_url.sql | 58 ++++ internal/validators/registries/nuget.go | 254 ++++++++++++++++-- internal/validators/registries/nuget_test.go | 18 +- .../validators/registries/testutils_test.go | 8 - pkg/model/constants.go | 2 +- 11 files changed, 323 insertions(+), 40 deletions(-) create mode 100644 internal/database/migrations/012_fix_nuget_registry_base_url.sql diff --git a/complete.md b/complete.md index c42097d..1fe5d05 100644 --- a/complete.md +++ b/complete.md @@ -658,7 +658,7 @@ Only trusted public registries are supported. Private registries and alternative **Supported registries:** - **NPM**: `https://registry.npmjs.org` only - **PyPI**: `https://pypi.org` only -- **NuGet**: `https://api.nuget.org` only +- **NuGet**: `https://api.nuget.org/v3/index.json` - **Docker/OCI**: `https://docker.io` only - **MCPB**: `https://github.com` releases and `https://gitlab.com` releases only @@ -1549,7 +1549,8 @@ Include your server name in your package's README using this format: Add a README file to your NuGet package that includes the server name. This can be in a comment if you want to hide it from display elsewhere. ### How It Works -- Registry fetches README from `https://api.nuget.org/v3-flatcontainer/{id}/{version}/readme` +- Registry fetches and cached service index `https://api.nuget.org/v3/index.json` +- Registry uses `ReadmeUriTemplate/6.13.0` URL template in service index to fetch README - Passes if `mcp-name: server-name` is found in the README content ### Example server.json @@ -1566,7 +1567,7 @@ Add a README file to your NuGet package that includes the server name. This can } ``` -The official MCP registry currently only supports the official NuGet registry (`https://api.nuget.org`). +The official MCP registry currently only supports the official NuGet registry (`https://api.nuget.org/v3/index.json`). @@ -2341,7 +2342,7 @@ Suppose your MCP server application requires a `mcp start` CLI arguments to star "packages": [ { "registry_type": "nuget", - "registry_base_url": "https://api.nuget.org", + "registry_base_url": "https://api.nuget.org/v3/index.json", "identifier": "Knapcode.SampleMcpServer", "version": "0.4.0-beta", "transport": { @@ -2578,7 +2579,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6. "packages": [ { "registry_type": "nuget", - "registry_base_url": "https://api.nuget.org", + "registry_base_url": "https://api.nuget.org/v3/index.json", "identifier": "Knapcode.SampleMcpServer", "version": "0.5.0", "runtime_hint": "dnx", diff --git a/docs/modelcontextprotocol-io/package-types.mdx b/docs/modelcontextprotocol-io/package-types.mdx index 496ff53..57960a3 100644 --- a/docs/modelcontextprotocol-io/package-types.mdx +++ b/docs/modelcontextprotocol-io/package-types.mdx @@ -85,7 +85,7 @@ This MCP server executes SQL queries and manages database connections. ## NuGet Packages -For NuGet packages, the MCP Registry currently supports the official NuGet registry (`https://api.nuget.org`) only. +For NuGet packages, the MCP Registry currently supports the official NuGet registry (`https://api.nuget.org/v3/index.json`) only. NuGet packages use `"registryType": "nuget"` in `server.json`. For example: diff --git a/docs/reference/api/openapi.yaml b/docs/reference/api/openapi.yaml index 59d941a..c85dc42 100644 --- a/docs/reference/api/openapi.yaml +++ b/docs/reference/api/openapi.yaml @@ -414,7 +414,7 @@ components: - "https://registry.npmjs.org" - "https://pypi.org" - "https://docker.io" - - "https://api.nuget.org" + - "https://api.nuget.org/v3/index.json" - "https://github.com" - "https://gitlab.com" identifier: diff --git a/docs/reference/server-json/generic-server-json.md b/docs/reference/server-json/generic-server-json.md index 302a444..f037d2d 100644 --- a/docs/reference/server-json/generic-server-json.md +++ b/docs/reference/server-json/generic-server-json.md @@ -131,7 +131,7 @@ Suppose your MCP server application requires a `mcp start` CLI arguments to star "packages": [ { "registryType": "nuget", - "registryBaseUrl": "https://api.nuget.org", + "registryBaseUrl": "https://api.nuget.org/v3/index.json", "identifier": "Knapcode.SampleMcpServer", "version": "0.4.0-beta", "transport": { @@ -370,7 +370,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6. "packages": [ { "registryType": "nuget", - "registryBaseUrl": "https://api.nuget.org", + "registryBaseUrl": "https://api.nuget.org/v3/index.json", "identifier": "Knapcode.SampleMcpServer", "version": "0.5.0", "runtimeHint": "dnx", diff --git a/docs/reference/server-json/official-registry-requirements.md b/docs/reference/server-json/official-registry-requirements.md index ef3da63..494458f 100644 --- a/docs/reference/server-json/official-registry-requirements.md +++ b/docs/reference/server-json/official-registry-requirements.md @@ -32,7 +32,7 @@ Only trusted public registries are supported. Private registries and alternative **Supported registries:** - **NPM**: `https://registry.npmjs.org` only - **PyPI**: `https://pypi.org` only -- **NuGet**: `https://api.nuget.org` only +- **NuGet**: `https://api.nuget.org/v3/index.json` only - **Docker/OCI**: - Docker Hub (`docker.io`) - GitHub Container Registry (`ghcr.io`) diff --git a/docs/reference/server-json/server.schema.json b/docs/reference/server-json/server.schema.json index 2d55b58..221eb05 100644 --- a/docs/reference/server-json/server.schema.json +++ b/docs/reference/server-json/server.schema.json @@ -226,7 +226,7 @@ "https://registry.npmjs.org", "https://pypi.org", "https://docker.io", - "https://api.nuget.org", + "https://api.nuget.org/v3/index.json", "https://github.com", "https://gitlab.com" ], diff --git a/internal/database/migrations/012_fix_nuget_registry_base_url.sql b/internal/database/migrations/012_fix_nuget_registry_base_url.sql new file mode 100644 index 0000000..7527a1a --- /dev/null +++ b/internal/database/migrations/012_fix_nuget_registry_base_url.sql @@ -0,0 +1,58 @@ +-- Migration: Set NuGet package registryBaseUrl for specific server entries +-- +-- This migration updates the packages[].registryBaseUrl to +-- "https://api.nuget.org/v3/index.json" for NuGet packages on a small, +-- explicitly listed set of server entries. Only packages with +-- registryType == "nuget" are modified; other package types are left +-- unchanged. +-- +-- Entries to update (name, version): +-- com.joelverhagen.mcp/Knapcode.SampleMcpServer 0.7.0-beta, 0.10.0-beta.10 +-- io.github.joelverhagen/Knapcode.SampleMcpServer 0.7.0-beta +-- io.github.moonolgerd/game-mcp 1.0.0 +-- io.github.timheuer/sampledotnetmcpserver 0.1.56-beta-g9538a23d37, 0.1.57-beta + +BEGIN; + +WITH affected_entries AS ( + SELECT server_name, version FROM (VALUES + ('com.joelverhagen.mcp/Knapcode.SampleMcpServer', '0.7.0-beta'), + ('com.joelverhagen.mcp/Knapcode.SampleMcpServer', '0.10.0-beta.10'), + ('io.github.joelverhagen/Knapcode.SampleMcpServer', '0.7.0-beta'), + ('io.github.moonolgerd/game-mcp', '1.0.0'), + ('io.github.timheuer/sampledotnetmcpserver', '0.1.56-beta-g9538a23d37'), + ('io.github.timheuer/sampledotnetmcpserver', '0.1.57-beta') + ) AS t(server_name, version) +) +UPDATE servers s +SET value = jsonb_set( + s.value, + '{packages}', + ( + SELECT jsonb_agg( + CASE + WHEN p->>'registryType' = 'nuget' THEN + jsonb_set( + p, + '{registryBaseUrl}', + '"https://api.nuget.org/v3/index.json"'::jsonb, + true + ) + ELSE p + END + ) + FROM jsonb_array_elements(s.value#>'{packages}') AS p + ), + true +) +FROM affected_entries ae +WHERE s.server_name = ae.server_name + AND s.version = ae.version + AND s.value#>'{packages}' IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements(s.value#>'{packages}') AS p + WHERE p->>'registryType' = 'nuget' AND p->>'registryBaseUrl' IS NOT NULL + ); + +COMMIT; diff --git a/internal/validators/registries/nuget.go b/internal/validators/registries/nuget.go index 9e2f836..c3b95a0 100644 --- a/internal/validators/registries/nuget.go +++ b/internal/validators/registries/nuget.go @@ -2,11 +2,13 @@ package registries import ( "context" + "encoding/json" "errors" "fmt" "io" "net/http" "strings" + "sync" "time" "github.com/modelcontextprotocol/registry/pkg/model" @@ -17,9 +19,109 @@ var ( ErrMissingVersionForNuget = errors.New("package version is required for NuGet packages") ) +const userAgent = "MCP-Registry-Validator/1.0" + +type cachedServiceIndex struct { + index *serviceIndex + expiresAt time.Time +} + +var ( + serviceIndexCache = make(map[string]*cachedServiceIndex) + cacheMu sync.RWMutex + cacheDuration = 1 * time.Hour +) + +type serviceIndexResource struct { + ID string `json:"@id"` + Type string `json:"@type"` +} + +type serviceIndex struct { + Resources []serviceIndexResource `json:"resources"` +} + +type packageContentIndex struct { + Versions []string `json:"versions"` +} + +type ReadmeState int + +const ( + ValidReadme ReadmeState = iota + InvalidReadme + NoReadme +) + +type PackageExistenceState int + +const ( + PackageAndVersionExist PackageExistenceState = iota + PackageExistsVersionMissing + PackageIDNotFound +) + // ValidateNuGet validates that a NuGet package contains the correct MCP server name func ValidateNuGet(ctx context.Context, pkg model.Package, serverName string) error { - // Set default registry base URL if empty + err := validateAndNormalizeBaseURL(&pkg) + if err != nil { + return err + } + + if pkg.Version == "" { + return ErrMissingVersionForNuget + } + + client := &http.Client{Timeout: 10 * time.Second} + + // Fetch the service serviceIndex + serviceIndex, err := fetchAndCacheServiceIndex(ctx, client, pkg.RegistryBaseURL) + if err != nil { + return err + } + + lowerID := strings.ToLower(pkg.Identifier) + lowerVersion := strings.ToLower(pkg.Version) + + // remove any SemVer 2.0.0 build metadata suffix (e.g., +abc) + if i := strings.Index(lowerVersion, "+"); i >= 0 { + lowerVersion = lowerVersion[:i] + } + + status, err := validateReadme(ctx, serverName, lowerID, lowerVersion, client, serviceIndex) + if err != nil { + return err + } + + switch status { + case ValidReadme: + return nil + case InvalidReadme: + return fmt.Errorf("NuGet package '%s' ownership validation for version %s failed. The server name '%s' must appear as 'mcp-name: %s' in the package README. Add it to your README and publish a new package version", pkg.Identifier, pkg.Version, serverName, serverName) + case NoReadme: + // Continue to check if package exists + default: + return fmt.Errorf("unexpected readme state: %d", status) + } + + existenceState, err := validatePackageExists(ctx, lowerID, lowerVersion, client, serviceIndex) + if err != nil { + return err + } + + switch existenceState { + case PackageIDNotFound: + return fmt.Errorf("NuGet package '%s' does not exist in the registry. If you recently published the package for the first time, wait for validation to complete", pkg.Identifier) + case PackageExistsVersionMissing: + return fmt.Errorf("NuGet package '%s' exists but version %s does not exist in the registry. If you recently published the version, wait for validation to complete", pkg.Identifier, pkg.Version) + case PackageAndVersionExist: + return fmt.Errorf("NuGet package '%s' ownership validation for version %s failed because it does not have an embedded README. Add one to your package and publish a new version", pkg.Identifier, pkg.Version) + default: + return fmt.Errorf("unexpected package existence state: %d", existenceState) + } +} + +func validateAndNormalizeBaseURL(pkg *model.Package) error { if pkg.RegistryBaseURL == "" { pkg.RegistryBaseURL = model.RegistryURLNuGet } @@ -39,26 +141,87 @@ func ValidateNuGet(ctx context.Context, pkg model.Package, serverName string) er pkg.RegistryBaseURL, model.RegistryTypeNuGet, model.RegistryURLNuGet) } - client := &http.Client{Timeout: 10 * time.Second} + return nil +} - lowerID := strings.ToLower(pkg.Identifier) - lowerVersion := strings.ToLower(pkg.Version) - if lowerVersion == "" { - return ErrMissingVersionForNuget +func fetchAndCacheServiceIndex(ctx context.Context, client *http.Client, baseURL string) (*serviceIndex, error) { + cacheMu.RLock() + if cached, exists := serviceIndexCache[baseURL]; exists { + if time.Now().Before(cached.expiresAt) { + cacheMu.RUnlock() + return cached.index, nil + } + } + cacheMu.RUnlock() + + cacheMu.Lock() + defer cacheMu.Unlock() + + if cached, exists := serviceIndexCache[baseURL]; exists { + if time.Now().Before(cached.expiresAt) { + return cached.index, nil + } } - // Try to get README from the package - readmeURL := fmt.Sprintf("%s/v3-flatcontainer/%s/%s/readme", pkg.RegistryBaseURL, lowerID, lowerVersion) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, readmeURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL, nil) if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("failed to create NuGet service index request: %w", err) } - req.Header.Set("User-Agent", "MCP-Registry-Validator/1.0") + req.Header.Set("User-Agent", userAgent) resp, err := client.Do(req) if err != nil { - return fmt.Errorf("failed to fetch README from NuGet: %w", err) + return nil, fmt.Errorf("failed to fetch NuGet service index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("NuGet service index returned status %d", resp.StatusCode) + } + + var index serviceIndex + if err := json.NewDecoder(resp.Body).Decode(&index); err != nil { + return nil, fmt.Errorf("failed to parse NuGet service index: %w", err) + } + + serviceIndexCache[baseURL] = &cachedServiceIndex{ + index: &index, + expiresAt: time.Now().Add(cacheDuration), + } + + return &index, nil +} + +func getReadmeURLTemplate(index *serviceIndex) (string, error) { + for _, resource := range index.Resources { + if resource.Type == "ReadmeUriTemplate/6.13.0" { + return resource.ID, nil + } + } + + return "", fmt.Errorf("ReadmeUriTemplate/6.13.0 not found in service index") +} + +func validateReadme(ctx context.Context, serverName, lowerID, lowerVersion string, client *http.Client, index *serviceIndex) (ReadmeState, error) { + readmeURLTemplate, err := getReadmeURLTemplate(index) + if err != nil { + return NoReadme, fmt.Errorf("failed to get README URL template: %w", err) + } + + // Replace placeholders in the template + readmeURL := strings.ReplaceAll(readmeURLTemplate, "{lower_id}", lowerID) + readmeURL = strings.ReplaceAll(readmeURL, "{lower_version}", lowerVersion) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, readmeURL, nil) + if err != nil { + return NoReadme, fmt.Errorf("failed to create NuGet README request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + resp, err := client.Do(req) + if err != nil { + return NoReadme, fmt.Errorf("failed to fetch NuGet README: %w", err) } defer resp.Body.Close() @@ -66,7 +229,7 @@ func ValidateNuGet(ctx context.Context, pkg model.Package, serverName string) er // Check README content readmeBytes, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("failed to read README content: %w", err) + return NoReadme, fmt.Errorf("failed to read NuGet README content: %w", err) } readmeContent := string(readmeBytes) @@ -74,9 +237,70 @@ func ValidateNuGet(ctx context.Context, pkg model.Package, serverName string) er // Check for mcp-name: format (more specific) mcpNamePattern := "mcp-name: " + serverName if strings.Contains(readmeContent, mcpNamePattern) { - return nil // Found as mcp-name: format + return ValidReadme, nil // Found as mcp-name: format + } + + return InvalidReadme, nil + } + + if resp.StatusCode == http.StatusNotFound { + return NoReadme, nil + } + + return InvalidReadme, fmt.Errorf("NuGet README request returned status %d", resp.StatusCode) +} + +func getPackageContentBaseURL(index *serviceIndex) (string, error) { + for _, resource := range index.Resources { + if resource.Type == "PackageBaseAddress/3.0.0" { + return resource.ID, nil } } - return fmt.Errorf("NuGet package '%s' ownership validation failed. The server name '%s' must appear as 'mcp-name: %s' in the package README. Add it to your package README", pkg.Identifier, serverName, serverName) + return "", fmt.Errorf("PackageBaseAddress/3.0.0 not found in service index") +} + +func validatePackageExists(ctx context.Context, lowerID, lowerVersion string, client *http.Client, index *serviceIndex) (PackageExistenceState, error) { + packageBaseURL, err := getPackageContentBaseURL(index) + if err != nil { + return PackageIDNotFound, fmt.Errorf("failed to get Package Base URL: %w", err) + } + + // Fetch the package content index to check if package ID and version exist + indexURL := fmt.Sprintf("%s/%s/index.json", strings.TrimRight(packageBaseURL, "/"), lowerID) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) + if err != nil { + return PackageIDNotFound, fmt.Errorf("failed to create NuGet package index request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + resp, err := client.Do(req) + if err != nil { + return PackageIDNotFound, fmt.Errorf("failed to fetch NuGet package index: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return PackageIDNotFound, nil // Package ID does not exist + } + + if resp.StatusCode != http.StatusOK { + return PackageIDNotFound, fmt.Errorf("NuGet package index returned status %d", resp.StatusCode) + } + + var contentIndex packageContentIndex + if err := json.NewDecoder(resp.Body).Decode(&contentIndex); err != nil { + return PackageIDNotFound, fmt.Errorf("failed to parse NuGet package index: %w", err) + } + + // Check if the version exists in the versions list + for _, v := range contentIndex.Versions { + if strings.EqualFold(v, lowerVersion) { + return PackageAndVersionExist, nil + } + } + + return PackageExistsVersionMissing, nil } diff --git a/internal/validators/registries/nuget_test.go b/internal/validators/registries/nuget_test.go index 79955c5..32f14b1 100644 --- a/internal/validators/registries/nuget_test.go +++ b/internal/validators/registries/nuget_test.go @@ -46,11 +46,11 @@ func TestValidateNuGet_RealPackages(t *testing.T) { }, { name: "non-existent package should fail", - packageName: generateRandomNuGetPackageName(), + packageName: "will--never--exist", version: "1.0.0", serverName: "com.example/test", expectError: true, - errorMessage: "ownership validation failed", + errorMessage: "NuGet package 'will--never--exist' does not exist in the registry", }, { name: "real package without version should fail", @@ -66,7 +66,15 @@ func TestValidateNuGet_RealPackages(t *testing.T) { version: "999.999.999", // Version that doesn't exist serverName: "com.example/test", expectError: true, - errorMessage: "ownership validation failed", + errorMessage: "exists but version 999.999.999 does not exist in the registry", + }, + { + name: "real package with non-existent version should fail", + packageName: "Newtonsoft.Json", + version: "6.0.1", // README doesn't exist + serverName: "com.example/test", + expectError: true, + errorMessage: "because it does not have an embedded README", }, { name: "real package without server name in README should fail", @@ -74,7 +82,7 @@ func TestValidateNuGet_RealPackages(t *testing.T) { version: "13.0.3", // Popular version serverName: "com.example/test", expectError: true, - errorMessage: "ownership validation failed", + errorMessage: "The server name 'com.example/test' must appear as 'mcp-name: com.example/test' in the package README.", }, { name: "real package without server name in README should fail", @@ -82,7 +90,7 @@ func TestValidateNuGet_RealPackages(t *testing.T) { version: "1.0.0", serverName: "io.github.domdomegg/time-mcp-server", expectError: true, - errorMessage: "ownership validation failed", + errorMessage: "ownership validation for version", }, { name: "real package with server name in README should pass", diff --git a/internal/validators/registries/testutils_test.go b/internal/validators/registries/testutils_test.go index e5206b8..d652c7c 100644 --- a/internal/validators/registries/testutils_test.go +++ b/internal/validators/registries/testutils_test.go @@ -14,11 +14,3 @@ func generateRandomPackageName() string { } return fmt.Sprintf("nonexistent-pkg-%s", hex.EncodeToString(bytes)) } - -func generateRandomNuGetPackageName() string { - bytes := make([]byte, 16) - if _, err := rand.Read(bytes); err != nil { - return "NonExistent.Package.Fallback" - } - return fmt.Sprintf("NonExistent.Package.%s", hex.EncodeToString(bytes)[:16]) -} diff --git a/pkg/model/constants.go b/pkg/model/constants.go index 433319e..1473c92 100644 --- a/pkg/model/constants.go +++ b/pkg/model/constants.go @@ -13,7 +13,7 @@ const ( const ( RegistryURLNPM = "https://registry.npmjs.org" RegistryURLPyPI = "https://pypi.org" - RegistryURLNuGet = "https://api.nuget.org" + RegistryURLNuGet = "https://api.nuget.org/v3/index.json" RegistryURLGitHub = "https://github.com" RegistryURLGitLab = "https://gitlab.com" )