Database implementation cleanup (#335)
## Motivation and Context The database logic was getting a bit unwieldy and fragile - especially because business logic that should really have lived at the registry service level was being pushed into the persistence level. This PR tightens the interface, simplifies the code there, as well as rips out a bunch of mocks etc. we don't need. ## How Has This Been Tested? Locally everything works ## Breaking Changes The list endpoint was surprisingly always filtering for isLatest only, which seems like a bug? I have removed this filter.
This commit is contained in:
adam jones
GitHub
parent
07d353ba92
commit
9d9a02c57f
2 changed files with 669 additions and 1280 deletions
184
internal/importer/importer.go
Normal file
184
internal/importer/importer.go
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/modelcontextprotocol/registry/internal/database"
|
||||
"github.com/modelcontextprotocol/registry/internal/validators"
|
||||
apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0"
|
||||
)
|
||||
|
||||
// Service handles importing seed data into the registry
|
||||
type Service struct {
|
||||
db database.Database
|
||||
}
|
||||
|
||||
// NewService creates a new importer service
|
||||
func NewService(db database.Database) *Service {
|
||||
return &Service{db: db}
|
||||
}
|
||||
|
||||
// ImportFromPath imports seed data from various sources:
|
||||
// 1. Local file paths (*.json files) - expects extension wrapper format
|
||||
// 2. Direct HTTP URLs to seed.json files - expects extension wrapper format
|
||||
// 3. Registry root URLs (automatically appends /v0/servers and paginates)
|
||||
func (s *Service) ImportFromPath(ctx context.Context, path string) error {
|
||||
servers, err := readSeedFile(ctx, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read seed data: %w", err)
|
||||
}
|
||||
|
||||
// Import each server using CreateServer
|
||||
for _, server := range servers {
|
||||
_, err := s.db.CreateServer(ctx, server)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to import server %s: %w", server.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
// readSeedFile reads seed data from various sources
|
||||
func readSeedFile(ctx context.Context, path string) ([]*apiv0.ServerJSON, error) {
|
||||
var data []byte
|
||||
var err error
|
||||
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
// Handle HTTP URLs
|
||||
if strings.HasSuffix(path, "/v0/servers") || strings.Contains(path, "/v0/servers") {
|
||||
// This is a registry API endpoint - fetch paginated data
|
||||
return fetchFromRegistryAPI(ctx, path)
|
||||
}
|
||||
// This is a direct file URL
|
||||
data, err = fetchFromHTTP(ctx, path)
|
||||
} else {
|
||||
// Handle local file paths
|
||||
data, err = os.ReadFile(path)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read seed data from %s: %w", path, err)
|
||||
}
|
||||
|
||||
// Parse extension wrapper format (only supported format)
|
||||
var serverResponses []apiv0.ServerJSON
|
||||
if err := json.Unmarshal(data, &serverResponses); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse seed data as extension wrapper format: %w", err)
|
||||
}
|
||||
|
||||
if len(serverResponses) == 0 {
|
||||
return []*apiv0.ServerJSON{}, nil
|
||||
}
|
||||
|
||||
// Validate servers and collect warnings instead of failing the whole batch
|
||||
var validRecords []*apiv0.ServerJSON
|
||||
var invalidServers []string
|
||||
var validationFailures []string
|
||||
|
||||
for _, response := range serverResponses {
|
||||
if err := validators.ValidateServerJSON(&response); err != nil {
|
||||
// Log warning and track invalid server instead of failing
|
||||
invalidServers = append(invalidServers, response.Name)
|
||||
validationFailures = append(validationFailures, fmt.Sprintf("Server '%s': %v", response.Name, err))
|
||||
log.Printf("Warning: Skipping invalid server '%s': %v", response.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert valid ServerJSON to ServerRecord
|
||||
record := convertServerResponseToRecord(response)
|
||||
validRecords = append(validRecords, record)
|
||||
}
|
||||
|
||||
// Print summary of validation results
|
||||
if len(invalidServers) > 0 {
|
||||
log.Printf("Import summary: %d valid servers imported, %d invalid servers skipped", len(validRecords), len(invalidServers))
|
||||
log.Printf("Invalid servers: %v", invalidServers)
|
||||
for _, failure := range validationFailures {
|
||||
log.Printf(" - %s", failure)
|
||||
}
|
||||
} else {
|
||||
log.Printf("Import summary: All %d servers imported successfully", len(validRecords))
|
||||
}
|
||||
|
||||
return validRecords, nil
|
||||
}
|
||||
|
||||
func fetchFromHTTP(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HTTP request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch from HTTP: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP request failed with status: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func fetchFromRegistryAPI(ctx context.Context, baseURL string) ([]*apiv0.ServerJSON, error) {
|
||||
var allRecords []*apiv0.ServerJSON
|
||||
cursor := ""
|
||||
|
||||
for {
|
||||
url := baseURL
|
||||
if cursor != "" {
|
||||
if strings.Contains(url, "?") {
|
||||
url += "&cursor=" + cursor
|
||||
} else {
|
||||
url += "?cursor=" + cursor
|
||||
}
|
||||
}
|
||||
|
||||
data, err := fetchFromHTTP(ctx, url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch page from registry API: %w", err)
|
||||
}
|
||||
|
||||
var response struct {
|
||||
Servers []apiv0.ServerJSON `json:"servers"`
|
||||
Metadata *struct {
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse registry API response: %w", err)
|
||||
}
|
||||
|
||||
// Convert and add servers
|
||||
for _, serverResponse := range response.Servers {
|
||||
record := convertServerResponseToRecord(serverResponse)
|
||||
allRecords = append(allRecords, record)
|
||||
}
|
||||
|
||||
// Check if there's a next page
|
||||
if response.Metadata == nil || response.Metadata.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
cursor = response.Metadata.NextCursor
|
||||
}
|
||||
|
||||
return allRecords, nil
|
||||
}
|
||||
|
||||
func convertServerResponseToRecord(response apiv0.ServerJSON) *apiv0.ServerJSON {
|
||||
// The response is already in the correct flattened format
|
||||
// Just return a pointer to it
|
||||
return &response
|
||||
}
|
||||
201
internal/importer/importer_test.go
Normal file
201
internal/importer/importer_test.go
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
package importer_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/modelcontextprotocol/registry/internal/database"
|
||||
"github.com/modelcontextprotocol/registry/internal/importer"
|
||||
apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0"
|
||||
"github.com/modelcontextprotocol/registry/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestImportService_LocalFile(t *testing.T) {
|
||||
// Create a temporary seed file
|
||||
tempFile := "/tmp/test_import_seed.json"
|
||||
seedData := []apiv0.ServerJSON{
|
||||
{
|
||||
Name: "io.github.test/test-server-1",
|
||||
Description: "Test server 1",
|
||||
Repository: model.Repository{
|
||||
URL: "https://github.com/test/repo1",
|
||||
Source: "github",
|
||||
ID: "123",
|
||||
},
|
||||
VersionDetail: model.VersionDetail{
|
||||
Version: "1.0.0",
|
||||
},
|
||||
Meta: &apiv0.ServerMeta{
|
||||
IOModelContextProtocolRegistry: &apiv0.RegistryExtensions{
|
||||
ID: "test-id-1",
|
||||
PublishedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
IsLatest: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(seedData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = os.WriteFile(tempFile, jsonData, 0600)
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(tempFile)
|
||||
|
||||
memDB := database.NewMemoryDB()
|
||||
|
||||
// Create importer service and test import
|
||||
service := importer.NewService(memDB)
|
||||
err = service.ImportFromPath(context.Background(), tempFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the server was imported
|
||||
servers, _, err := memDB.List(context.Background(), nil, "", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, servers, 1)
|
||||
assert.Equal(t, "io.github.test/test-server-1", servers[0].Name)
|
||||
}
|
||||
|
||||
func TestImportService_HTTPFile(t *testing.T) {
|
||||
// Create a test HTTP server
|
||||
seedData := []apiv0.ServerJSON{
|
||||
{
|
||||
Name: "io.github.test/http-test-server",
|
||||
Description: "HTTP test server",
|
||||
Repository: model.Repository{
|
||||
URL: "https://github.com/test/repo1",
|
||||
Source: "github",
|
||||
ID: "123",
|
||||
},
|
||||
VersionDetail: model.VersionDetail{
|
||||
Version: "2.0.0",
|
||||
},
|
||||
Meta: &apiv0.ServerMeta{
|
||||
IOModelContextProtocolRegistry: &apiv0.RegistryExtensions{
|
||||
ID: "test-id-2",
|
||||
PublishedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
IsLatest: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(seedData)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
memDB := database.NewMemoryDB()
|
||||
|
||||
// Create importer service and test import
|
||||
service := importer.NewService(memDB)
|
||||
err := service.ImportFromPath(context.Background(), server.URL+"/seed.json")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the server was imported
|
||||
servers, _, err := memDB.List(context.Background(), nil, "", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, servers, 1)
|
||||
assert.Equal(t, "io.github.test/http-test-server", servers[0].Name)
|
||||
}
|
||||
|
||||
func TestImportService_RegistryAPI(t *testing.T) {
|
||||
// Create test data
|
||||
testServers := []apiv0.ServerJSON{
|
||||
{
|
||||
Name: "io.github.test/api-server-1",
|
||||
Description: "API server 1",
|
||||
Repository: model.Repository{
|
||||
URL: "https://github.com/test/repo1",
|
||||
Source: "github",
|
||||
ID: "123",
|
||||
},
|
||||
VersionDetail: model.VersionDetail{
|
||||
Version: "1.0.0",
|
||||
},
|
||||
Meta: &apiv0.ServerMeta{
|
||||
IOModelContextProtocolRegistry: &apiv0.RegistryExtensions{
|
||||
ID: "api-test-id-1",
|
||||
PublishedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
IsLatest: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "io.github.test/api-server-2",
|
||||
Description: "API server 2",
|
||||
Repository: model.Repository{
|
||||
URL: "https://github.com/test/repo2",
|
||||
Source: "github",
|
||||
ID: "456",
|
||||
},
|
||||
VersionDetail: model.VersionDetail{
|
||||
Version: "2.0.0",
|
||||
},
|
||||
Meta: &apiv0.ServerMeta{
|
||||
IOModelContextProtocolRegistry: &apiv0.RegistryExtensions{
|
||||
ID: "api-test-id-2",
|
||||
PublishedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
IsLatest: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Create test server that responds with paginated results
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
|
||||
var response struct {
|
||||
Servers []apiv0.ServerJSON `json:"servers"`
|
||||
Metadata *struct {
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
} `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
switch cursor {
|
||||
case "":
|
||||
// First page
|
||||
response.Servers = []apiv0.ServerJSON{testServers[0]}
|
||||
response.Metadata = &struct {
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
}{NextCursor: "page2"}
|
||||
case "page2":
|
||||
// Second page
|
||||
response.Servers = []apiv0.ServerJSON{testServers[1]}
|
||||
// No next cursor means end of results
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
memDB := database.NewMemoryDB()
|
||||
|
||||
// Create importer service and test import
|
||||
service := importer.NewService(memDB)
|
||||
err := service.ImportFromPath(context.Background(), server.URL+"/v0/servers")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify both servers were imported
|
||||
servers, _, err := memDB.List(context.Background(), nil, "", 10)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, servers, 2)
|
||||
|
||||
names := []string{servers[0].Name, servers[1].Name}
|
||||
assert.Contains(t, names, "io.github.test/api-server-1")
|
||||
assert.Contains(t, names, "io.github.test/api-server-2")
|
||||
}
|
||||
Loading…
Reference in a new issue