Simplify seed data configuration by removing command line flags and renaming environment variable (#175)

This PR simplifies the seed data import configuration based on maintainer feedback by removing command line flags and using a clearer environment variable name. ## Key Changes ### Configuration Simplification - **Renamed**: `MCP_REGISTRY_SEED_FILE_PATH` → `MCP_REGISTRY_SEED_FROM` - **Removed**: `--seed-file-path` command line flag to eliminate configuration complexity ### Improved User Experience Users now configure seed data import using a single, clear environment variable: ```bash # Before (multiple ways to configure) MCP_REGISTRY_SEED_FILE_PATH=data/seed.json ./registry ./registry --seed-file-path=data/seed.json # After (simplified single method) MCP_REGISTRY_SEED_FROM=data/seed.json ./registry ``` ### Registry Composability The composability workflow is now simplified while maintaining all functionality: ```bash # Start first registry with seed data MCP_REGISTRY_SEED_FROM=data/seed.json ./registry # Start second registry importing from first via pagination MCP_REGISTRY_SEED_FROM=http://localhost:8080 ./registry ``` ## Technical Implementation - Maintains full backward compatibility for import functionality - All existing seed data sources continue to work (local files, HTTP URLs, registry URLs) - Automatic pagination through `/v0/servers` endpoint remains unchanged - Configurable delays between requests preserved ## Benefits - **Reduced complexity**: Single configuration method instead of multiple - **Clearer naming**: `SEED_FROM` better describes the source of seed data - **Easier maintenance**: Fewer configuration paths to test and document - **Consistent patterns**: Aligns with other environment-variable-only configuration All existing functionality is preserved while providing a cleaner, more maintainable configuration interface.  --- 💬 Share your feedback on Copilot coding agent for the chance to win a $200 gift card! Click [here](https://survey.alchemer.com/s3/8343779/Copilot-Coding-agent) to start the survey. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Avinash Sridhar <sridharavinash@users.noreply.github.com> Co-authored-by: Toby Padilla <toby@toby.sh> Co-authored-by: toby <83556+toby@users.noreply.github.com>
2025-08-08 13:42:16 +01:00 · 2025-08-08 13:42:16 +01:00 · 9c49c8b9b8
commit 9c49c8b9b8
parent f04c995f0a
4 changed files with 570 additions and 152 deletions
--- a/internal/database/import.go
+++ b/internal/database/import.go
@ -1,18 +1,27 @@
 package database

 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"

 	"github.com/modelcontextprotocol/registry/internal/model"
 )

-// ReadSeedFile reads and parses the seed.json file - exported for use by all database implementations
-func ReadSeedFile(path string) ([]model.ServerDetail, error) {
-	log.Printf("Reading seed file from %s", path)
+// ReadSeedFile reads seed data from various sources:
+// 1. Local file paths (*.json files)
+// 2. Direct HTTP URLs to seed.json files
+// 3. Registry root URLs (automatically appends /v0/servers and paginates)
+func ReadSeedFile(ctx context.Context, path string) ([]model.ServerDetail, error) {
+	log.Printf("Reading seed data from %s", path)

 	// Set default seed file path if not provided
 	if path == "" {
@ -23,13 +32,60 @@ func ReadSeedFile(path string) ([]model.ServerDetail, error) {
 		}
 	}

-	// Read the file content
+	// Check if path is an HTTP URL
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		// Determine if this is a direct seed file URL or a registry root URL
+		if strings.HasSuffix(path, ".json") || strings.Contains(path, "seed.json") {
+			// Direct seed file URL - read directly
+			fileContent, err := readFromHTTP(ctx, path)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read from HTTP URL: %w", err)
+			}
+			return parseSeedJSON(fileContent)
+		}
+		// Registry root URL - paginate through /v0/servers endpoint
+		return readFromRegistryWithContext(ctx, path)
+	}
+	// Read from local file
 	fileContent, err := os.ReadFile(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file: %w", err)
 	}
+	return parseSeedJSON(fileContent)
+}

-	// Parse the JSON content
+// readFromHTTP reads content from an HTTP URL with timeout
+func readFromHTTP(ctx context.Context, url string) ([]byte, error) {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch URL: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed with status %d: %s", resp.StatusCode, resp.Status)
+	}
+
+	// Read the response body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return body, nil
+}
+
+// parseSeedJSON parses JSON content into ServerDetail objects
+func parseSeedJSON(fileContent []byte) ([]model.ServerDetail, error) {
 	var servers []model.ServerDetail
 	if err := json.Unmarshal(fileContent, &servers); err != nil {
 		// Try parsing as a raw JSON array and then convert to our model
@ -39,6 +95,177 @@ func ReadSeedFile(path string) ([]model.ServerDetail, error) {
 		}
 	}

-	log.Printf("Found %d server entries in seed file", len(servers))
+	log.Printf("Found %d server entries in seed data", len(servers))
 	return servers, nil
 }
+
+// PaginatedResponse represents the paginated response from /v0/servers endpoint
+// PaginatedResponse represents the structure of a paginated response from /v0/servers endpoint
+type PaginatedResponse struct {
+	Data     []model.Server `json:"servers"`
+	Metadata Metadata       `json:"metadata,omitempty"`
+}
+
+// Metadata contains pagination metadata
+type Metadata struct {
+	NextCursor string `json:"next_cursor,omitempty"`
+	Count      int    `json:"count,omitempty"`
+	Total      int    `json:"total,omitempty"`
+}
+
+// readFromRegistryWithContext reads all servers from a registry by paginating through /v0/servers endpoint
+func readFromRegistryWithContext(ctx context.Context, registryURL string) ([]model.ServerDetail, error) {
+	log.Printf("Reading from registry: %s", registryURL)
+
+	// Ensure the URL doesn't have a trailing slash
+	registryURL = strings.TrimSuffix(registryURL, "/")
+
+	var allServers []model.ServerDetail
+	cursor := ""
+	pageCount := 0
+
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	for {
+		pageCount++
+
+		// Add delay between requests as requested (10 seconds by default)
+		// Can be overridden by SEED_IMPORT_DELAY environment variable for testing
+		if pageCount > 1 { // Don't delay before the first request
+			delay := 10 * time.Second
+			if delayStr := os.Getenv("SEED_IMPORT_DELAY"); delayStr != "" {
+				if parsedDelay, err := time.ParseDuration(delayStr); err == nil {
+					delay = parsedDelay
+				}
+			}
+			if delay > 0 {
+				log.Printf("Waiting %v before fetching page %d...", delay, pageCount)
+				time.Sleep(delay)
+			}
+		}
+
+		log.Printf("Fetching page %d from registry", pageCount)
+
+		// Build the URL for this page
+		serverURL := registryURL + "/v0/servers"
+		if cursor != "" {
+			// Add cursor parameter for pagination
+			parsed, err := url.Parse(serverURL)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse registry URL: %w", err)
+			}
+			query := parsed.Query()
+			query.Set("cursor", cursor)
+			query.Set("limit", "100") // Use maximum limit for efficiency
+			parsed.RawQuery = query.Encode()
+			serverURL = parsed.String()
+		} else {
+			// First page - use max limit
+			serverURL += "?limit=100"
+		}
+
+		// Fetch the page
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create request for %s: %w", serverURL, err)
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch servers from %s: %w", serverURL, err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			return nil, fmt.Errorf("HTTP request to %s failed with status %d: %s", serverURL, resp.StatusCode, resp.Status)
+		}
+
+		// Read and parse the response
+		body, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body from %s: %w", serverURL, err)
+		}
+
+		var pageResponse PaginatedResponse
+		if err := json.Unmarshal(body, &pageResponse); err != nil {
+			return nil, fmt.Errorf("failed to parse servers response from %s: %w", serverURL, err)
+		}
+
+		log.Printf("Retrieved %d servers from page %d", len(pageResponse.Data), pageCount)
+
+		// For each server in this page, get the detailed information
+		for _, server := range pageResponse.Data {
+			// Build URL for server detail
+			detailURL := registryURL + "/v0/servers/" + server.ID
+
+			detailReq, err := http.NewRequestWithContext(ctx, http.MethodGet, detailURL, nil)
+			if err != nil {
+				log.Printf("Warning: failed to create request for server %s: %v", server.ID, err)
+				// Fall back to basic server information
+				serverDetail := model.ServerDetail{
+					Server: server,
+				}
+				allServers = append(allServers, serverDetail)
+				continue
+			}
+
+			detailResp, err := client.Do(detailReq)
+			if err != nil {
+				log.Printf("Warning: failed to fetch details for server %s: %v", server.ID, err)
+				// Fall back to basic server information
+				serverDetail := model.ServerDetail{
+					Server: server,
+				}
+				allServers = append(allServers, serverDetail)
+				continue
+			}
+
+			if detailResp.StatusCode != http.StatusOK {
+				log.Printf("Warning: failed to fetch details for server %s (status %d)", server.ID, detailResp.StatusCode)
+				detailResp.Body.Close()
+				// Fall back to basic server information
+				serverDetail := model.ServerDetail{
+					Server: server,
+				}
+				allServers = append(allServers, serverDetail)
+				continue
+			}
+
+			detailBody, err := io.ReadAll(detailResp.Body)
+			detailResp.Body.Close()
+			if err != nil {
+				log.Printf("Warning: failed to read detail response for server %s: %v", server.ID, err)
+				// Fall back to basic server information
+				serverDetail := model.ServerDetail{
+					Server: server,
+				}
+				allServers = append(allServers, serverDetail)
+				continue
+			}
+
+			var serverDetail model.ServerDetail
+			if err := json.Unmarshal(detailBody, &serverDetail); err != nil {
+				log.Printf("Warning: failed to parse detail response for server %s: %v", server.ID, err)
+				// Fall back to basic server information
+				serverDetail = model.ServerDetail{
+					Server: server,
+				}
+			}
+
+			allServers = append(allServers, serverDetail)
+		}
+
+		// Check if there are more pages
+		if pageResponse.Metadata.NextCursor == "" {
+			log.Printf("Reached end of pagination after %d pages", pageCount)
+			break
+		}
+
+		cursor = pageResponse.Metadata.NextCursor
+	}
+
+	log.Printf("Successfully retrieved %d servers from registry %s", len(allServers), registryURL)
+	return allServers, nil
+}
--- a/internal/database/import_test.go
+++ b/internal/database/import_test.go
@ -0,0 +1,188 @@
+package database_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/modelcontextprotocol/registry/internal/database"
+	"github.com/modelcontextprotocol/registry/internal/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadSeedFile_LocalFile(t *testing.T) {
+	// Create a temporary seed file
+	tempFile := "/tmp/test_seed.json"
+	seedData := []model.ServerDetail{
+		{
+			Server: model.Server{
+				ID:          "test-id-1",
+				Name:        "test-server-1",
+				Description: "Test server 1",
+				Repository: model.Repository{
+					URL:    "https://github.com/test/repo1",
+					Source: "github",
+					ID:     "123",
+				},
+				VersionDetail: model.VersionDetail{
+					Version:     "1.0.0",
+					ReleaseDate: "2023-01-01T00:00:00Z",
+					IsLatest:    true,
+				},
+			},
+		},
+	}
+
+	// Write seed data to temp file
+	data, err := json.Marshal(seedData)
+	assert.NoError(t, err)
+
+	err = func() error {
+		f, err := os.Create(tempFile)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = f.Write(data)
+		return err
+	}()
+	assert.NoError(t, err)
+	defer os.Remove(tempFile)
+
+	// Test reading the file
+	result, err := database.ReadSeedFile(context.Background(), tempFile)
+	assert.NoError(t, err)
+	assert.Len(t, result, 1)
+	assert.Equal(t, "test-server-1", result[0].Name)
+}
+
+func TestReadSeedFile_DirectHTTPURL(t *testing.T) {
+	// Create a test HTTP server that serves seed JSON directly
+	seedData := []model.ServerDetail{
+		{
+			Server: model.Server{
+				ID:          "test-id-1",
+				Name:        "test-server-1",
+				Description: "Test server 1",
+			},
+		},
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(seedData); err != nil {
+			http.Error(w, "Failed to encode JSON", http.StatusInternalServerError)
+		}
+	}))
+	defer server.Close()
+
+	// Test reading from HTTP URL ending in .json
+	result, err := database.ReadSeedFile(context.Background(), server.URL+"/seed.json")
+	assert.NoError(t, err)
+	assert.Len(t, result, 1)
+	assert.Equal(t, "test-server-1", result[0].Name)
+}
+
+func TestReadSeedFile_RegistryURL(t *testing.T) {
+	// Create mock registry servers
+	server1 := model.Server{
+		ID:          "server-1",
+		Name:        "Test Server 1",
+		Description: "First test server",
+	}
+	server2 := model.Server{
+		ID:          "server-2",
+		Name:        "Test Server 2",
+		Description: "Second test server",
+	}
+
+	serverDetail1 := model.ServerDetail{
+		Server: server1,
+		Packages: []model.Package{
+			{
+				Name:    "test-package-1",
+				Version: "1.0.0",
+			},
+		},
+	}
+	serverDetail2 := model.ServerDetail{
+		Server: server2,
+		Packages: []model.Package{
+			{
+				Name:    "test-package-2",
+				Version: "2.0.0",
+			},
+		},
+	}
+
+	// Create a test HTTP server that simulates the registry API
+	mux := http.NewServeMux()
+
+	// Handle /v0/servers endpoint (paginated)
+	mux.HandleFunc("/v0/servers", func(w http.ResponseWriter, r *http.Request) {
+		cursor := r.URL.Query().Get("cursor")
+
+		var response database.PaginatedResponse
+		switch cursor {
+		case "":
+			// First page
+			response = database.PaginatedResponse{
+				Data: []model.Server{server1},
+				Metadata: database.Metadata{
+					NextCursor: "next-cursor-1",
+					Count:      1,
+				},
+			}
+		case "next-cursor-1":
+			// Second page
+			response = database.PaginatedResponse{
+				Data: []model.Server{server2},
+				Metadata: database.Metadata{
+					Count: 1,
+					// No NextCursor means end of pagination
+				},
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(response); err != nil {
+			http.Error(w, "Failed to encode JSON", http.StatusInternalServerError)
+		}
+	})
+
+	// Handle individual server detail endpoints
+	mux.HandleFunc("/v0/servers/server-1", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(serverDetail1); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	})
+
+	mux.HandleFunc("/v0/servers/server-2", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(serverDetail2); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	})
+
+	server := httptest.NewServer(mux)
+	defer server.Close()
+
+	// Test reading from registry root URL (this should trigger pagination)
+	result, err := database.ReadSeedFile(context.Background(), server.URL)
+	assert.NoError(t, err)
+	assert.Len(t, result, 2)
+
+	// Verify the servers were imported correctly
+	assert.Equal(t, "Test Server 1", result[0].Name)
+	assert.Equal(t, "Test Server 2", result[1].Name)
+
+	// Verify packages were included
+	assert.Len(t, result[0].Packages, 1)
+	assert.Equal(t, "test-package-1", result[0].Packages[0].Name)
+	assert.Len(t, result[1].Packages, 1)
+	assert.Equal(t, "test-package-2", result[1].Packages[0].Name)
+}
--- a/internal/database/memory.go
+++ b/internal/database/memory.go
@ -252,7 +252,7 @@ func (db *MemoryDB) ImportSeed(ctx context.Context, seedFilePath string) error {
 	}

 	// Read the seed file
-	seedData, err := ReadSeedFile(seedFilePath)
+	seedData, err := ReadSeedFile(ctx, seedFilePath)
 	if err != nil {
 		return fmt.Errorf("failed to read seed file: %w", err)
 	}
--- a/internal/database/mongo.go
+++ b/internal/database/mongo.go
@ -235,7 +235,7 @@ func (db *MongoDB) Publish(ctx context.Context, serverDetail *model.ServerDetail
 // ImportSeed imports initial data from a seed file into MongoDB
 func (db *MongoDB) ImportSeed(ctx context.Context, seedFilePath string) error {
 	// Read the seed file
-	servers, err := ReadSeedFile(seedFilePath)
+	servers, err := ReadSeedFile(ctx, seedFilePath)
 	if err != nil {
 		return fmt.Errorf("failed to read seed file: %w", err)
 	}