registry/internal/database/import.go

package database

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/modelcontextprotocol/registry/internal/model"
)

// ReadSeedFile reads seed data from various sources:
// 1. Local file paths (*.json files)
// 2. Direct HTTP URLs to seed.json files
// 3. Registry root URLs (automatically appends /v0/servers and paginates)
func ReadSeedFile(ctx context.Context, path string) ([]model.ServerDetail, error) {
	log.Printf("Reading seed data from %s", path)

	// Set default seed file path if not provided
	if path == "" {
		// Try to find the seed.json in the data directory
		path = filepath.Join("data", "seed.json")
		if _, err := os.Stat(path); os.IsNotExist(err) {
			return nil, fmt.Errorf("seed file not found at %s", path)
		}
	}

	// Check if path is an HTTP URL
	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
		// Determine if this is a direct seed file URL or a registry root URL
		if strings.HasSuffix(path, ".json") || strings.Contains(path, "seed.json") {
			// Direct seed file URL - read directly
			fileContent, err := readFromHTTP(ctx, path)
			if err != nil {
				return nil, fmt.Errorf("failed to read from HTTP URL: %w", err)
			}
			return parseSeedJSON(fileContent)
		}
		// Registry root URL - paginate through /v0/servers endpoint
		return readFromRegistryWithContext(ctx, path)
	}
	// Read from local file
	fileContent, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read file: %w", err)
	}
	return parseSeedJSON(fileContent)
}

// readFromHTTP reads content from an HTTP URL with timeout
func readFromHTTP(ctx context.Context, url string) ([]byte, error) {
	client := &http.Client{
		Timeout: 30 * time.Second,
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch URL: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP request failed with status %d: %s", resp.StatusCode, resp.Status)
	}

	// Read the response body
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body: %w", err)
	}

	return body, nil
}

// parseSeedJSON parses JSON content into ServerDetail objects
func parseSeedJSON(fileContent []byte) ([]model.ServerDetail, error) {
	var servers []model.ServerDetail
	if err := json.Unmarshal(fileContent, &servers); err != nil {
		// Try parsing as a raw JSON array and then convert to our model
		var rawData []map[string]any
		if jsonErr := json.Unmarshal(fileContent, &rawData); jsonErr != nil {
			return nil, fmt.Errorf("failed to parse JSON: %w (original error: %w)", jsonErr, err)
		}
	}

	log.Printf("Found %d server entries in seed data", len(servers))
	return servers, nil
}

// PaginatedResponse represents the paginated response from /v0/servers endpoint
// PaginatedResponse represents the structure of a paginated response from /v0/servers endpoint
type PaginatedResponse struct {
	Data     []model.Server `json:"servers"`
	Metadata Metadata       `json:"metadata,omitempty"`
}

// Metadata contains pagination metadata
type Metadata struct {
	NextCursor string `json:"next_cursor,omitempty"`
	Count      int    `json:"count,omitempty"`
	Total      int    `json:"total,omitempty"`
}

// readFromRegistryWithContext reads all servers from a registry by paginating through /v0/servers endpoint
func readFromRegistryWithContext(ctx context.Context, registryURL string) ([]model.ServerDetail, error) {
	log.Printf("Reading from registry: %s", registryURL)

	// Ensure the URL doesn't have a trailing slash
	registryURL = strings.TrimSuffix(registryURL, "/")

	var allServers []model.ServerDetail
	cursor := ""
	pageCount := 0

	client := &http.Client{
		Timeout: 30 * time.Second,
	}

	for {
		pageCount++

		// Add delay between requests as requested (10 seconds by default)
		// Can be overridden by SEED_IMPORT_DELAY environment variable for testing
		if pageCount > 1 { // Don't delay before the first request
			delay := 10 * time.Second
			if delayStr := os.Getenv("SEED_IMPORT_DELAY"); delayStr != "" {
				if parsedDelay, err := time.ParseDuration(delayStr); err == nil {
					delay = parsedDelay
				}
			}
			if delay > 0 {
				log.Printf("Waiting %v before fetching page %d...", delay, pageCount)
				time.Sleep(delay)
			}
		}

		log.Printf("Fetching page %d from registry", pageCount)

		// Build the URL for this page
		serverURL := registryURL + "/v0/servers"
		if cursor != "" {
			// Add cursor parameter for pagination
			parsed, err := url.Parse(serverURL)
			if err != nil {
				return nil, fmt.Errorf("failed to parse registry URL: %w", err)
			}
			query := parsed.Query()
			query.Set("cursor", cursor)
			query.Set("limit", "100") // Use maximum limit for efficiency
			parsed.RawQuery = query.Encode()
			serverURL = parsed.String()
		} else {
			// First page - use max limit
			serverURL += "?limit=100"
		}

		// Fetch the page
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL, nil)
		if err != nil {
			return nil, fmt.Errorf("failed to create request for %s: %w", serverURL, err)
		}
		resp, err := client.Do(req)
		if err != nil {
			return nil, fmt.Errorf("failed to fetch servers from %s: %w", serverURL, err)
		}

		if resp.StatusCode != http.StatusOK {
			resp.Body.Close()
			return nil, fmt.Errorf("HTTP request to %s failed with status %d: %s", serverURL, resp.StatusCode, resp.Status)
		}

		// Read and parse the response
		body, err := io.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			return nil, fmt.Errorf("failed to read response body from %s: %w", serverURL, err)
		}

		var pageResponse PaginatedResponse
		if err := json.Unmarshal(body, &pageResponse); err != nil {
			return nil, fmt.Errorf("failed to parse servers response from %s: %w", serverURL, err)
		}

		log.Printf("Retrieved %d servers from page %d", len(pageResponse.Data), pageCount)

		// For each server in this page, get the detailed information
		for _, server := range pageResponse.Data {
			// Build URL for server detail
			detailURL := registryURL + "/v0/servers/" + server.ID

			detailReq, err := http.NewRequestWithContext(ctx, http.MethodGet, detailURL, nil)
			if err != nil {
				log.Printf("Warning: failed to create request for server %s: %v", server.ID, err)
				// Fall back to basic server information
				serverDetail := model.ServerDetail{
					Server: server,
				}
				allServers = append(allServers, serverDetail)
				continue
			}

			detailResp, err := client.Do(detailReq)
			if err != nil {
				log.Printf("Warning: failed to fetch details for server %s: %v", server.ID, err)
				// Fall back to basic server information
				serverDetail := model.ServerDetail{
					Server: server,
				}
				allServers = append(allServers, serverDetail)
				continue
			}

			if detailResp.StatusCode != http.StatusOK {
				log.Printf("Warning: failed to fetch details for server %s (status %d)", server.ID, detailResp.StatusCode)
				detailResp.Body.Close()
				// Fall back to basic server information
				serverDetail := model.ServerDetail{
					Server: server,
				}
				allServers = append(allServers, serverDetail)
				continue
			}

			detailBody, err := io.ReadAll(detailResp.Body)
			detailResp.Body.Close()
			if err != nil {
				log.Printf("Warning: failed to read detail response for server %s: %v", server.ID, err)
				// Fall back to basic server information
				serverDetail := model.ServerDetail{
					Server: server,
				}
				allServers = append(allServers, serverDetail)
				continue
			}

			var serverDetail model.ServerDetail
			if err := json.Unmarshal(detailBody, &serverDetail); err != nil {
				log.Printf("Warning: failed to parse detail response for server %s: %v", server.ID, err)
				// Fall back to basic server information
				serverDetail = model.ServerDetail{
					Server: server,
				}
			}

			allServers = append(allServers, serverDetail)
		}

		// Check if there are more pages
		if pageResponse.Metadata.NextCursor == "" {
			log.Printf("Reached end of pagination after %d pages", pageCount)
			break
		}

		cursor = pageResponse.Metadata.NextCursor
	}

	log.Printf("Successfully retrieved %d servers from registry %s", len(allServers), registryURL)
	return allServers, nil
}