From 774a0df5923225cd6ba6ad07e935fb252af56878 Mon Sep 17 00:00:00 2001 From: Tadas Antanavicius Date: Tue, 15 Jul 2025 11:32:54 -0700 Subject: [PATCH] Add validation tools for server.json schemas and examples (#176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a first PR for #159 - adding some validation scripts. I want to get to a place where we are confident that making schema changes is in sync with the rest of our code. Step one is making sure the schemas are in sync with our examples. Step two will be making sure the schemas are in sync with our actual Registry implementation code. This is just for the JSON right now; if we're feeling good about this approach I'll do an analogous change for the OpenAPI schemas as well. I'll also follow with adding usage of these scripts to our CI pipeline so we have automated checks on them with every change. ## Summary - Added CLI validation tools for JSON schemas and examples in `docs/server-json/` - Fixed some minor validation errors found in `examples.md` - Ensures schema consistency and example validity in CI/CD pipelines ## Changes ### New Validation Tools 1. **`tools/validate-schemas`** - Validates that `schema.json` and `registry-schema.json` are valid JSON Schema documents 2. **`tools/validate-examples`** - Validates all JSON examples in `examples.md` against both schemas ### Key Features - ✅ Validates JSON schema syntax and structure - ✅ Validates all examples against both base and registry schemas - ✅ Tracks validation counts to prevent silent failures - ✅ Proper exit codes for CI integration - ✅ Clear error messages for debugging ### Fixes to examples.md - Added missing `repository.id` field in NuGet example - Changed `repository.source` from "gitlab" to "github" in Docker example ### Usage ```bash # From repository root ./tools/validate-schemas.sh ./tools/validate-examples.sh ``` ## Testing I ran these scripts locally with minor (breaking) changes, which they each correctly reported. As submitted, they are both passing. 🤖 Co-authored with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Claude Co-authored-by: Charles Lowell <10964656+chlowell@users.noreply.github.com> --- .gitignore | 2 + .golangci.yml | 5 + docs/server-json/README.md | 22 +++ docs/server-json/examples.md | 4 +- go.mod | 1 + go.sum | 2 + internal/api/handlers/v0/auth.go | 6 +- internal/api/handlers/v0/publish_test.go | 2 +- internal/database/database.go | 2 +- internal/database/import.go | 2 +- internal/database/memory.go | 2 +- internal/database/mongo.go | 2 +- tools/publisher/main.go | 2 +- tools/validate-examples.sh | 8 ++ tools/validate-examples/main.go | 168 +++++++++++++++++++++++ tools/validate-schemas.sh | 8 ++ tools/validate-schemas/main.go | 94 +++++++++++++ 17 files changed, 321 insertions(+), 11 deletions(-) create mode 100755 tools/validate-examples.sh create mode 100644 tools/validate-examples/main.go create mode 100755 tools/validate-schemas.sh create mode 100644 tools/validate-schemas/main.go diff --git a/.gitignore b/.gitignore index 8ea8087..6dda66f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ build/* **/bin cmd/registry/registry publisher +validate-examples +validate-schemas diff --git a/.golangci.yml b/.golangci.yml index 699cc57..8eb0a95 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -71,6 +71,11 @@ linters: - whitespace linters-settings: + revive: + rules: + - name: use-any + disabled: false + severity: error cyclop: max-complexity: 50 funlen: diff --git a/docs/server-json/README.md b/docs/server-json/README.md index b836133..527aed3 100644 --- a/docs/server-json/README.md +++ b/docs/server-json/README.md @@ -15,3 +15,25 @@ References: - [schema.json](./schema.json) - The official JSON schema specification for this representation - [examples.md](./examples.md) - Example manifestations of the JSON schema - [registry-schema.json](./registry-schema.json) - A more constrained version of `schema.json` that the official registry supports + +## Validation Tools + +Two validation tools are provided in the repository's `tools/` directory: +- **validate-schemas** - Validates that `schema.json` and `registry-schema.json` are valid JSON Schema documents +- **validate-examples** - Validates that all JSON examples in `examples.md` conform to both schemas + +### Usage + +From the repository root: + +#### Validate JSON Schemas + +```bash +./tools/validate-schemas.sh +``` + +#### Validate Examples + +```bash +./tools/validate-examples.sh +``` \ No newline at end of file diff --git a/docs/server-json/examples.md b/docs/server-json/examples.md index 3de955a..3c85e95 100644 --- a/docs/server-json/examples.md +++ b/docs/server-json/examples.md @@ -219,7 +219,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6. "repository": { "url": "https://github.com/joelverhagen/Knapcode.SampleMcpServer", "source": "github", - "id": "def456gh-ijkl-7890-mnop-qrstuvwxyz13" + "id": "example-nuget-id-0000-1111-222222222222" }, "version_detail": { "version": "0.3.0-beta" @@ -251,7 +251,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6. "description": "MCP server for database operations with support for multiple database types", "repository": { "url": "https://github.com/example/mcp-database", - "source": "gitlab", + "source": "github", "id": "ghi789jk-lmno-1234-pqrs-tuvwxyz56789" }, "version_detail": { diff --git a/go.mod b/go.mod index 29c67e5..5cbf514 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 require ( github.com/caarlos0/env/v11 v11.3.1 github.com/google/uuid v1.6.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/stretchr/testify v1.10.0 github.com/swaggo/files v1.0.1 github.com/swaggo/http-swagger v1.3.4 diff --git a/go.sum b/go.sum index 5c7a5a6..4b96ee6 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= diff --git a/internal/api/handlers/v0/auth.go b/internal/api/handlers/v0/auth.go index 38156d1..3abf011 100644 --- a/internal/api/handlers/v0/auth.go +++ b/internal/api/handlers/v0/auth.go @@ -64,7 +64,7 @@ func StartAuthHandler(authService auth.Service) http.HandlerFunc { // Return successful response w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(map[string]interface{}{ + if err := json.NewEncoder(w).Encode(map[string]any{ "flow_info": flowInfo, "status_token": statusToken, "expires_in": 300, // 5 minutes @@ -98,7 +98,7 @@ func CheckAuthStatusHandler(authService auth.Service) http.HandlerFunc { // Auth is still pending w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(map[string]interface{}{ + if err := json.NewEncoder(w).Encode(map[string]any{ "status": "pending", }); err != nil { http.Error(w, "Failed to encode response", http.StatusInternalServerError) @@ -115,7 +115,7 @@ func CheckAuthStatusHandler(authService auth.Service) http.HandlerFunc { // Authentication completed successfully w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(map[string]interface{}{ + if err := json.NewEncoder(w).Encode(map[string]any{ "status": "complete", "token": token, }); err != nil { diff --git a/internal/api/handlers/v0/publish_test.go b/internal/api/handlers/v0/publish_test.go index 641e730..cf94dd0 100644 --- a/internal/api/handlers/v0/publish_test.go +++ b/internal/api/handlers/v0/publish_test.go @@ -61,7 +61,7 @@ func TestPublishHandler(t *testing.T) { testCases := []struct { name string method string - requestBody interface{} + requestBody any authHeader string setupMocks func(*MockRegistryService, *MockAuthService) expectedStatus int diff --git a/internal/database/database.go b/internal/database/database.go index f2e8a0e..d145f0c 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -19,7 +19,7 @@ var ( // Database defines the interface for database operations on MCPRegistry entries type Database interface { // List retrieves all MCPRegistry entries with optional filtering - List(ctx context.Context, filter map[string]interface{}, cursor string, limit int) ([]*model.Server, string, error) + List(ctx context.Context, filter map[string]any, cursor string, limit int) ([]*model.Server, string, error) // GetByID retrieves a single ServerDetail by it's ID GetByID(ctx context.Context, id string) (*model.ServerDetail, error) // Publish adds a new ServerDetail to the database diff --git a/internal/database/import.go b/internal/database/import.go index 9df209c..e29965a 100644 --- a/internal/database/import.go +++ b/internal/database/import.go @@ -33,7 +33,7 @@ func ReadSeedFile(path string) ([]model.ServerDetail, error) { var servers []model.ServerDetail if err := json.Unmarshal(fileContent, &servers); err != nil { // Try parsing as a raw JSON array and then convert to our model - var rawData []map[string]interface{} + var rawData []map[string]any if jsonErr := json.Unmarshal(fileContent, &rawData); jsonErr != nil { return nil, fmt.Errorf("failed to parse JSON: %w (original error: %w)", jsonErr, err) } diff --git a/internal/database/memory.go b/internal/database/memory.go index fd892a6..6cd6cb0 100644 --- a/internal/database/memory.go +++ b/internal/database/memory.go @@ -90,7 +90,7 @@ func compareSemanticVersions(version1, version2 string) int { //gocognit:ignore func (db *MemoryDB) List( ctx context.Context, - filter map[string]interface{}, + filter map[string]any, cursor string, limit int, ) ([]*model.Server, string, error) { diff --git a/internal/database/mongo.go b/internal/database/mongo.go index 3dbf776..ac34805 100644 --- a/internal/database/mongo.go +++ b/internal/database/mongo.go @@ -75,7 +75,7 @@ func NewMongoDB(ctx context.Context, connectionURI, databaseName, collectionName // List retrieves MCPRegistry entries with optional filtering and pagination func (db *MongoDB) List( ctx context.Context, - filter map[string]interface{}, + filter map[string]any, cursor string, limit int, ) ([]*model.Server, string, error) { diff --git a/tools/publisher/main.go b/tools/publisher/main.go index 1ad2629..ec25b52 100644 --- a/tools/publisher/main.go +++ b/tools/publisher/main.go @@ -290,7 +290,7 @@ func createCommand() { // publishToRegistry sends the MCP server details to the registry with authentication func publishToRegistry(registryURL string, mcpData []byte, token string) error { // Parse the MCP JSON data - var mcpDetails map[string]interface{} + var mcpDetails map[string]any err := json.Unmarshal(mcpData, &mcpDetails) if err != nil { return fmt.Errorf("error parsing server.json file: %w", err) diff --git a/tools/validate-examples.sh b/tools/validate-examples.sh new file mode 100755 index 0000000..0e58fcb --- /dev/null +++ b/tools/validate-examples.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Validate examples in docs/server-json/examples.md +# For more information, see docs/server-json/README.md + +set -e + +cd "$(dirname "$0")/.." +exec go run tools/validate-examples/main.go "$@" diff --git a/tools/validate-examples/main.go b/tools/validate-examples/main.go new file mode 100644 index 0000000..85e9d5b --- /dev/null +++ b/tools/validate-examples/main.go @@ -0,0 +1,168 @@ +// validate-examples validates JSON examples in docs/server-json/examples.md +// against both schema.json and registry-schema.json. +// +// For more information, see docs/server-json/README.md +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v5" +) + +const ( + // expectedExampleCount is the number of JSON examples we expect to find in examples.md + // IMPORTANT: Only change this count if you have intentionally added or removed examples + // from the examples.md file. This check prevents accidental formatting changes from + // causing examples to be skipped during validation. + expectedExampleCount = 8 +) + +func main() { + log.SetFlags(0) // Remove timestamp from logs + + if err := runValidation(); err != nil { + log.Fatalf("Error: %v", err) + } +} + +func runValidation() error { + basePath := filepath.Join("docs", "server-json") + + examplesPath := filepath.Join(basePath, "examples.md") + schemaPath := filepath.Join(basePath, "schema.json") + registrySchemaPath := filepath.Join(basePath, "registry-schema.json") + + examples, err := extractExamples(examplesPath) + if err != nil { + return fmt.Errorf("failed to extract examples: %w", err) + } + + log.Printf("Found %d examples in examples.md\n", len(examples)) + + if len(examples) != expectedExampleCount { + return fmt.Errorf("expected %d examples but found %d - if this is intentional, update expectedExampleCount in %s", + expectedExampleCount, len(examples), "tools/validate-examples/main.go") + } + + log.Println() + + baseSchema, err := compileSchema(schemaPath) + if err != nil { + return fmt.Errorf("failed to compile schema.json: %w", err) + } + + registrySchema, err := compileSchema(registrySchemaPath) + if err != nil { + return fmt.Errorf("failed to compile registry-schema.json: %w", err) + } + + validatedCount := 0 + for i, example := range examples { + log.Printf("Example %d:", i+1) + + var data any + if err := json.Unmarshal([]byte(example.content), &data); err != nil { + log.Printf(" ❌ Invalid JSON: %v", err) + continue + } + + baseValid := false + registryValid := false + + if err := baseSchema.Validate(data); err != nil { + log.Printf(" Validating against schema.json: ❌") + log.Printf(" Error: %v", err) + } else { + log.Printf(" Validating against schema.json: ✅") + baseValid = true + } + + if err := registrySchema.Validate(data); err != nil { + log.Printf(" Validating against registry-schema.json: ❌") + log.Printf(" Error: %v", err) + } else { + log.Printf(" Validating against registry-schema.json: ✅") + registryValid = true + } + + // Only count as validated if both schemas passed + if baseValid && registryValid { + validatedCount++ + } + + log.Println() + } + + if validatedCount != expectedExampleCount { + return fmt.Errorf("validation failed: expected %d examples to pass both validations but only %d did", + expectedExampleCount, validatedCount) + } + + log.Printf("Successfully validated all %d examples!", validatedCount) + return nil +} + +type example struct { + content string + line int +} + +func extractExamples(path string) ([]example, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + content := string(data) + + // Regex to match JSON code blocks in markdown + // Captures everything between ```json and ``` + re := regexp.MustCompile("(?s)```json\n(.*?)\n```") + matches := re.FindAllStringSubmatchIndex(content, -1) + + var examples []example + for _, match := range matches { + if len(match) < 4 { + // should never happen + return nil, fmt.Errorf("invalid match - expected at least 4 indices but got %d", len(match)) + } + start, end := match[2], match[3] + // line numbers start at 1 + line := 1 + strings.Count(content[:start], "\n") + examples = append(examples, example{ + content: content[start:end], + line: line, + }) + } + + return examples, nil +} + +func compileSchema(path string) (*jsonschema.Schema, error) { + compiler := jsonschema.NewCompiler() + compiler.Draft = jsonschema.Draft7 + + // For registry-schema.json, we need to register the base schema it references + if strings.Contains(path, "registry-schema.json") { + basePath := filepath.Join(filepath.Dir(path), "schema.json") + baseData, err := os.ReadFile(basePath) + if err != nil { + return nil, fmt.Errorf("failed to read base schema: %w", err) + } + + // Add the base schema to the compiler with the expected URL + if err := compiler.AddResource("https://modelcontextprotocol.io/schemas/draft/2025-07-09/server.json", bytes.NewReader(baseData)); err != nil { + return nil, fmt.Errorf("failed to add base schema resource: %w", err) + } + } + + return compiler.Compile(path) +} diff --git a/tools/validate-schemas.sh b/tools/validate-schemas.sh new file mode 100755 index 0000000..2c39087 --- /dev/null +++ b/tools/validate-schemas.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Validate JSON schema files +# For more information, see docs/server-json/README.md + +set -e + +cd "$(dirname "$0")/.." +exec go run tools/validate-schemas/main.go "$@" diff --git a/tools/validate-schemas/main.go b/tools/validate-schemas/main.go new file mode 100644 index 0000000..90f21f8 --- /dev/null +++ b/tools/validate-schemas/main.go @@ -0,0 +1,94 @@ +// validate-schemas validates that schema.json and registry-schema.json +// are valid JSON Schema documents. +// +// For more information, see docs/server-json/README.md +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v5" +) + +func main() { + log.SetFlags(0) // Remove timestamp from logs + + if err := runValidation(); err != nil { + log.Fatalf("Error: %v", err) + } +} + +func runValidation() error { + basePath := filepath.Join("docs", "server-json") + + schemas := []struct { + name string + path string + }{ + {"schema.json", filepath.Join(basePath, "schema.json")}, + {"registry-schema.json", filepath.Join(basePath, "registry-schema.json")}, + } + + expectedSchemaCount := len(schemas) + validatedCount := 0 + + for _, schemaFile := range schemas { + log.Printf("Validating %s...", schemaFile.name) + + if err := validateSchema(schemaFile.path); err != nil { + log.Printf(" ❌ Invalid: %v", err) + } else { + log.Printf(" ✅ Valid JSON Schema") + validatedCount++ + } + } + + if validatedCount != expectedSchemaCount { + return fmt.Errorf("validation failed: expected to validate %d schemas but only %d passed", + expectedSchemaCount, validatedCount) + } + + log.Printf("\nSuccessfully validated all %d schemas!", validatedCount) + return nil +} + +func validateSchema(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + var schemaData any + if err := json.Unmarshal(data, &schemaData); err != nil { + return fmt.Errorf("invalid JSON: %w", err) + } + + compiler := jsonschema.NewCompiler() + compiler.Draft = jsonschema.Draft7 + + // For registry-schema.json, we need to register the base schema it references + if strings.Contains(path, "registry-schema.json") { + basePath := filepath.Join(filepath.Dir(path), "schema.json") + baseData, err := os.ReadFile(basePath) + if err != nil { + return fmt.Errorf("failed to read base schema: %w", err) + } + + // Add the base schema to the compiler with the expected URL + if err := compiler.AddResource("https://modelcontextprotocol.io/schemas/draft/2025-07-09/server.json", bytes.NewReader(baseData)); err != nil { + return fmt.Errorf("failed to add base schema resource: %w", err) + } + } + + if _, err := compiler.Compile(path); err != nil { + return fmt.Errorf("invalid schema: %w", err) + } + + return nil +}