f52dc8525a## Summary - cache successful public list responses for `/v0/servers` and `/v0.1/servers` at ingress-nginx for 30 seconds - give only those exact cached paths a 600 RPM per-controller budget - preserve the existing 180 RPM limit for publish, auth, mutation, detail, and other routes - expose `X-Registry-Cache` to make cache MISS/HIT behavior directly observable ## Why Production ingress logs show that the current 180 RPM per-controller limit rejects legitimate bulk pagination before it reaches the application. The affected traffic is overwhelmingly registry-list reads, not publishing. A ten-minute production sample contained 17,836 successful list responses but only 3,188 unique URLs: 82.1% were repeated requests, representing roughly 1 GB of duplicate response bodies. Raising the list limit without caching would increase pressure on the application and its shared PostgreSQL pool. The short ingress cache absorbs repeated cursor-page reads before raising their scoped client budget. Publish latency is a separate concern. Reducing read pressure may help indirectly, but slow external package validation is intentionally out of scope for this PR. Relates to #826. ## Safety - caching is limited to exact public GET list paths and keyed by host plus full request URI - only 200 responses are cached - cache lifetime is 30 seconds; newly published entries can therefore take up to 30 seconds to appear on a cached page - cache locking coalesces simultaneous misses - stale cached responses may be served while refreshing or during transient upstream 5xx/timeouts - each ingress replica uses at most 1 GB of ephemeral on-disk cache ## Validation - `go test ./...` in `deploy/` - `go vet ./...` in `deploy/` - `git diff --check` - ingress-nginx chart 4.13.0 in a disposable kind cluster - generated NGINX configuration passed `nginx -t` - repeated list request verified `X-Registry-Cache: MISS` followed by `HIT` - non-list route verified uncached - exact list locations verified with the higher rate zone while the catch-all retained 180 RPM Nothing was applied to production during investigation or validation.
463 lines
17 KiB
Go
463 lines
17 KiB
Go
package k8s
|
||
|
||
import (
|
||
"os/exec"
|
||
"strings"
|
||
|
||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions"
|
||
v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1"
|
||
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
|
||
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||
networkingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
|
||
policyv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1"
|
||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
||
|
||
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
|
||
)
|
||
|
||
// getGitCommitHash returns the current git commit hash
|
||
func getGitCommitHash() string {
|
||
cmd := exec.Command("git", "rev-parse", "HEAD")
|
||
output, err := cmd.Output()
|
||
if err != nil {
|
||
// Fallback to a default value if git command fails
|
||
return "unknown"
|
||
}
|
||
return strings.TrimSpace(string(output))
|
||
}
|
||
|
||
// DeployMCPRegistry deploys the MCP Registry to the Kubernetes cluster
|
||
func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string, ingressNginx *helm.Chart, pgCluster *apiextensions.CustomResource) (*corev1.Service, error) {
|
||
conf := config.New(ctx, "mcp-registry")
|
||
githubClientId := conf.Require("githubClientId")
|
||
|
||
// Determine Docker image tag based on environment
|
||
imageTag := "main" // Default for staging
|
||
if environment == "prod" {
|
||
// Use explicitly configured image tag for production
|
||
// This prevents automatic promotion and requires manual control of production releases
|
||
imageTag = conf.Require("imageTag") // Explicit configuration required - no fallback
|
||
}
|
||
|
||
// Create Secret with sensitive configuration
|
||
secret, err := corev1.NewSecret(ctx, "mcp-registry-secrets", &corev1.SecretArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry-secrets"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
},
|
||
StringData: pulumi.StringMap{
|
||
"GITHUB_CLIENT_SECRET": conf.RequireSecret("githubClientSecret"),
|
||
"JWT_PRIVATE_KEY": conf.RequireSecret("jwtPrivateKey"),
|
||
},
|
||
Type: pulumi.String("Opaque"),
|
||
}, pulumi.Provider(cluster.Provider))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Create Deployment
|
||
_, err = v1.NewDeployment(ctx, "mcp-registry", &v1.DeploymentArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
},
|
||
Spec: &v1.DeploymentSpecArgs{
|
||
Replicas: pulumi.Int(2),
|
||
Strategy: &v1.DeploymentStrategyArgs{
|
||
Type: pulumi.String("RollingUpdate"),
|
||
RollingUpdate: &v1.RollingUpdateDeploymentArgs{
|
||
MaxUnavailable: pulumi.IntPtr(0), // Never reduce capacity during updates
|
||
MaxSurge: pulumi.IntPtr(1), // Create new pods first, then terminate old
|
||
},
|
||
},
|
||
Selector: &metav1.LabelSelectorArgs{
|
||
MatchLabels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
},
|
||
},
|
||
Template: &corev1.PodTemplateSpecArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
},
|
||
Annotations: pulumi.StringMap{
|
||
// Use git commit hash to trigger pod restarts when deploying new infra versions
|
||
"registry.modelcontextprotocol.io/deployCommit": pulumi.String(getGitCommitHash()),
|
||
},
|
||
},
|
||
Spec: &corev1.PodSpecArgs{
|
||
// Soft-spread pods across nodes so the PodDisruptionBudget below can't
|
||
// deadlock a node drain when both pods happen to co-locate.
|
||
// `ScheduleAnyway` keeps single-node clusters (e.g. constrained dev
|
||
// envs) schedulable while still preferring spread under normal load.
|
||
TopologySpreadConstraints: corev1.TopologySpreadConstraintArray{
|
||
&corev1.TopologySpreadConstraintArgs{
|
||
MaxSkew: pulumi.Int(1),
|
||
TopologyKey: pulumi.String("kubernetes.io/hostname"),
|
||
WhenUnsatisfiable: pulumi.String("ScheduleAnyway"),
|
||
LabelSelector: &metav1.LabelSelectorArgs{
|
||
MatchLabels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
},
|
||
},
|
||
},
|
||
},
|
||
Containers: corev1.ContainerArray{
|
||
&corev1.ContainerArgs{
|
||
Name: pulumi.String("mcp-registry"),
|
||
Image: pulumi.Sprintf("ghcr.io/modelcontextprotocol/registry:%s", imageTag),
|
||
ImagePullPolicy: pulumi.String("Always"),
|
||
Ports: corev1.ContainerPortArray{
|
||
&corev1.ContainerPortArgs{
|
||
ContainerPort: pulumi.Int(8080),
|
||
Name: pulumi.String("http"),
|
||
},
|
||
},
|
||
Env: corev1.EnvVarArray{
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_DATABASE_URL"),
|
||
ValueFrom: &corev1.EnvVarSourceArgs{
|
||
SecretKeyRef: &corev1.SecretKeySelectorArgs{
|
||
Name: pgCluster.Metadata.Name().ApplyT(func(name *string) string {
|
||
if name == nil {
|
||
return "registry-pg-app"
|
||
}
|
||
return *name + "-app"
|
||
}).(pulumi.StringOutput),
|
||
Key: pulumi.String("uri"),
|
||
},
|
||
},
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_GITHUB_CLIENT_ID"),
|
||
Value: pulumi.String(githubClientId),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_GITHUB_CLIENT_SECRET"),
|
||
ValueFrom: &corev1.EnvVarSourceArgs{
|
||
SecretKeyRef: &corev1.SecretKeySelectorArgs{
|
||
Name: secret.Metadata.Name(),
|
||
Key: pulumi.String("GITHUB_CLIENT_SECRET"),
|
||
},
|
||
},
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_JWT_PRIVATE_KEY"),
|
||
ValueFrom: &corev1.EnvVarSourceArgs{
|
||
SecretKeyRef: &corev1.SecretKeySelectorArgs{
|
||
Name: secret.Metadata.Name(),
|
||
Key: pulumi.String("JWT_PRIVATE_KEY"),
|
||
},
|
||
},
|
||
},
|
||
// Google Cloud Identity OIDC for admin access
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_ENABLED"),
|
||
Value: pulumi.String("true"),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_ISSUER"),
|
||
Value: pulumi.String("https://accounts.google.com"),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_CLIENT_ID"),
|
||
Value: pulumi.String("32555940559.apps.googleusercontent.com"),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_EXTRA_CLAIMS"),
|
||
Value: pulumi.String(`[{"hd":"modelcontextprotocol.io"}]`),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_EDIT_PERMISSIONS"),
|
||
Value: pulumi.String("*"),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_OIDC_PUBLISH_PERMISSIONS"),
|
||
Value: pulumi.String("*"),
|
||
},
|
||
&corev1.EnvVarArgs{
|
||
Name: pulumi.String("MCP_REGISTRY_GITHUB_OIDC_AUDIENCE"),
|
||
Value: pulumi.String(func() string {
|
||
if environment == "prod" {
|
||
return "https://registry.modelcontextprotocol.io"
|
||
}
|
||
return "https://" + environment + ".registry.modelcontextprotocol.io"
|
||
}()),
|
||
},
|
||
},
|
||
// StartupProbe protects the DB-retry budget in cmd/registry/main.go:
|
||
// 30 × 5s = 150s allowed before liveness/readiness take over, which
|
||
// covers the worst-case retry budget (8 attempts × 10s + ~40s sleep).
|
||
StartupProbe: &corev1.ProbeArgs{
|
||
HttpGet: &corev1.HTTPGetActionArgs{
|
||
Path: pulumi.String("/v0/health"),
|
||
Port: pulumi.Int(8080),
|
||
},
|
||
PeriodSeconds: pulumi.Int(5),
|
||
FailureThreshold: pulumi.Int(30),
|
||
TimeoutSeconds: pulumi.Int(3),
|
||
},
|
||
LivenessProbe: &corev1.ProbeArgs{
|
||
HttpGet: &corev1.HTTPGetActionArgs{
|
||
Path: pulumi.String("/v0/health"),
|
||
Port: pulumi.Int(8080),
|
||
},
|
||
InitialDelaySeconds: pulumi.Int(30),
|
||
TimeoutSeconds: pulumi.Int(5),
|
||
},
|
||
ReadinessProbe: &corev1.ProbeArgs{
|
||
HttpGet: &corev1.HTTPGetActionArgs{
|
||
Path: pulumi.String("/v0/health"),
|
||
Port: pulumi.Int(8080),
|
||
},
|
||
InitialDelaySeconds: pulumi.Int(5),
|
||
TimeoutSeconds: pulumi.Int(3),
|
||
},
|
||
Resources: &corev1.ResourceRequirementsArgs{
|
||
// Memory was 256Mi/512Mi when pgxpool MaxConns was 30. After
|
||
// #1221 doubled MaxConns to 60 per pod, peak per-pod memory
|
||
// shifted from ~330MB to ~450MB under scraper load and one
|
||
// pod was OOMKilled at 17:01 UTC on 2026-04-29. pgxpool keeps
|
||
// per-connection state (prepared statement cache, scratch
|
||
// buffers, row iteration state) so memory scales roughly
|
||
// linearly with the connection count. Doubled both request
|
||
// and limit to match.
|
||
Requests: pulumi.StringMap{
|
||
"memory": pulumi.String("512Mi"),
|
||
"cpu": pulumi.String("100m"),
|
||
},
|
||
Limits: pulumi.StringMap{
|
||
"memory": pulumi.String("1Gi"),
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
}, pulumi.Provider(cluster.Provider))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Keep at least one registry pod available during voluntary disruptions
|
||
// (node drains, evictions, etc.) so a synchronized eviction can't take both
|
||
// pods down at once.
|
||
_, err = policyv1.NewPodDisruptionBudget(ctx, "mcp-registry", &policyv1.PodDisruptionBudgetArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
},
|
||
Spec: &policyv1.PodDisruptionBudgetSpecArgs{
|
||
MinAvailable: pulumi.Int(1),
|
||
Selector: &metav1.LabelSelectorArgs{
|
||
MatchLabels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
},
|
||
},
|
||
},
|
||
}, pulumi.Provider(cluster.Provider))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Create Service
|
||
service, err := corev1.NewService(ctx, "mcp-registry", &corev1.ServiceArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
},
|
||
Spec: &corev1.ServiceSpecArgs{
|
||
Selector: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
},
|
||
Ports: corev1.ServicePortArray{
|
||
&corev1.ServicePortArgs{
|
||
Port: pulumi.Int(80),
|
||
TargetPort: pulumi.Int(8080),
|
||
Name: pulumi.String("http"),
|
||
},
|
||
},
|
||
Type: pulumi.String("ClusterIP"),
|
||
},
|
||
}, pulumi.Provider(cluster.Provider))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Create Ingress
|
||
hosts := []string{
|
||
environment + ".registry.modelcontextprotocol.io",
|
||
}
|
||
|
||
// Add root domain for prod environment
|
||
if environment == "prod" {
|
||
hosts = append(hosts, "registry.modelcontextprotocol.io")
|
||
}
|
||
|
||
ingress, err := networkingv1.NewIngress(ctx, "mcp-registry", &networkingv1.IngressArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
Annotations: pulumi.StringMap{
|
||
"cert-manager.io/cluster-issuer": pulumi.String("letsencrypt-prod"),
|
||
"kubernetes.io/ingress.class": pulumi.String("nginx"),
|
||
// Rate limiting to protect against abuse. The exact bulk-list paths get a
|
||
// separate, higher budget below; all other routes retain this limit.
|
||
// Allows 180 requests/minute (3 req/sec avg), with bursts up to 540 requests.
|
||
// Status code 429 is set globally via NGINX ConfigMap (per-Ingress annotation doesn't work).
|
||
"nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("180"),
|
||
"nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"),
|
||
},
|
||
},
|
||
Spec: &networkingv1.IngressSpecArgs{
|
||
Tls: networkingv1.IngressTLSArray{
|
||
&networkingv1.IngressTLSArgs{
|
||
Hosts: pulumi.ToStringArray(hosts),
|
||
SecretName: pulumi.Sprintf("mcp-registry-%s-tls", environment),
|
||
},
|
||
},
|
||
Rules: pulumi.ToStringArray(hosts).ToStringArrayOutput().ApplyT(func(hosts []string) networkingv1.IngressRuleArray {
|
||
rules := make(networkingv1.IngressRuleArray, 0, len(hosts))
|
||
for _, host := range hosts {
|
||
rules = append(rules, &networkingv1.IngressRuleArgs{
|
||
Host: pulumi.String(host),
|
||
Http: &networkingv1.HTTPIngressRuleValueArgs{
|
||
Paths: networkingv1.HTTPIngressPathArray{
|
||
&networkingv1.HTTPIngressPathArgs{
|
||
Path: pulumi.String("/"),
|
||
PathType: pulumi.String("Prefix"),
|
||
Backend: &networkingv1.IngressBackendArgs{
|
||
Service: &networkingv1.IngressServiceBackendArgs{
|
||
Name: service.Metadata.Name().Elem(),
|
||
Port: &networkingv1.ServiceBackendPortArgs{
|
||
Number: pulumi.Int(80),
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
})
|
||
}
|
||
return rules
|
||
}).(networkingv1.IngressRuleArrayOutput),
|
||
},
|
||
}, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(ingressNginx.Ready))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// Bulk registry consumers paginate the complete server list. Production
|
||
// traffic analysis on 2026-08-06 showed legitimate consumers sustaining
|
||
// roughly 10 requests/second per source IP while the catch-all Ingress only
|
||
// allowed 3 requests/second per controller replica. That caused NGINX to
|
||
// reject healthy pagination requests before they reached the application.
|
||
// The ingress controller caches these exact public paths for 30 seconds, so
|
||
// the higher client budget does not translate into the same increase in
|
||
// application and database load.
|
||
//
|
||
// ingress-nginx merges different paths for the same host across Ingress
|
||
// resources and applies each resource's annotations only to its own paths.
|
||
// Keep the canonical list endpoints on exact paths so publish, auth, edit,
|
||
// status, and individual-server reads remain protected by the lower limit.
|
||
_, err = networkingv1.NewIngress(ctx, "mcp-registry-bulk-read", &networkingv1.IngressArgs{
|
||
Metadata: &metav1.ObjectMetaArgs{
|
||
Name: pulumi.String("mcp-registry-bulk-read"),
|
||
Namespace: pulumi.String("default"),
|
||
Labels: pulumi.StringMap{
|
||
"app": pulumi.String("mcp-registry"),
|
||
"environment": pulumi.String(environment),
|
||
},
|
||
Annotations: pulumi.StringMap{
|
||
// ingress-nginx disables response buffering by default, but NGINX
|
||
// needs it to populate the on-disk proxy cache for these paths.
|
||
"nginx.ingress.kubernetes.io/proxy-buffering": pulumi.String("on"),
|
||
// Limits are maintained independently by each ingress-nginx replica.
|
||
// At two production replicas this permits up to 1,200 requests/minute
|
||
// per source IP, while a single replica still accommodates the observed
|
||
// 10 requests/second pagination rate.
|
||
"nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("600"),
|
||
"nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"),
|
||
},
|
||
},
|
||
Spec: &networkingv1.IngressSpecArgs{
|
||
IngressClassName: pulumi.String("nginx"),
|
||
Rules: pulumi.ToStringArray(hosts).ToStringArrayOutput().ApplyT(func(hosts []string) networkingv1.IngressRuleArray {
|
||
rules := make(networkingv1.IngressRuleArray, 0, len(hosts))
|
||
for _, host := range hosts {
|
||
rules = append(rules, &networkingv1.IngressRuleArgs{
|
||
Host: pulumi.String(host),
|
||
Http: &networkingv1.HTTPIngressRuleValueArgs{
|
||
Paths: networkingv1.HTTPIngressPathArray{
|
||
// Exact prevents the higher budget from applying to edit/status
|
||
// subpaths. The v0.1 dot is accepted because ingress.go disables
|
||
// strict path validation for this controller.
|
||
&networkingv1.HTTPIngressPathArgs{
|
||
Path: pulumi.String("/v0/servers"),
|
||
PathType: pulumi.String("Exact"),
|
||
Backend: &networkingv1.IngressBackendArgs{
|
||
Service: &networkingv1.IngressServiceBackendArgs{
|
||
Name: service.Metadata.Name().Elem(),
|
||
Port: &networkingv1.ServiceBackendPortArgs{
|
||
Number: pulumi.Int(80),
|
||
},
|
||
},
|
||
},
|
||
},
|
||
&networkingv1.HTTPIngressPathArgs{
|
||
Path: pulumi.String("/v0.1/servers"),
|
||
PathType: pulumi.String("Exact"),
|
||
Backend: &networkingv1.IngressBackendArgs{
|
||
Service: &networkingv1.IngressServiceBackendArgs{
|
||
Name: service.Metadata.Name().Elem(),
|
||
Port: &networkingv1.ServiceBackendPortArgs{
|
||
Number: pulumi.Int(80),
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
})
|
||
}
|
||
return rules
|
||
}).(networkingv1.IngressRuleArrayOutput),
|
||
},
|
||
}, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(ingressNginx.Ready))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
ctx.Export("ingressHosts", ingress.Spec.Rules().ApplyT(func(rules []networkingv1.IngressRule) []string {
|
||
hosts := make([]string, 0, len(rules))
|
||
for _, rule := range rules {
|
||
hosts = append(hosts, *rule.Host)
|
||
}
|
||
return hosts
|
||
}))
|
||
|
||
return service, nil
|
||
}
|