fix(deploy): cache bulk registry reads (#1509)
## Summary - cache successful public list responses for `/v0/servers` and `/v0.1/servers` at ingress-nginx for 30 seconds - give only those exact cached paths a 600 RPM per-controller budget - preserve the existing 180 RPM limit for publish, auth, mutation, detail, and other routes - expose `X-Registry-Cache` to make cache MISS/HIT behavior directly observable ## Why Production ingress logs show that the current 180 RPM per-controller limit rejects legitimate bulk pagination before it reaches the application. The affected traffic is overwhelmingly registry-list reads, not publishing. A ten-minute production sample contained 17,836 successful list responses but only 3,188 unique URLs: 82.1% were repeated requests, representing roughly 1 GB of duplicate response bodies. Raising the list limit without caching would increase pressure on the application and its shared PostgreSQL pool. The short ingress cache absorbs repeated cursor-page reads before raising their scoped client budget. Publish latency is a separate concern. Reducing read pressure may help indirectly, but slow external package validation is intentionally out of scope for this PR. Relates to #826. ## Safety - caching is limited to exact public GET list paths and keyed by host plus full request URI - only 200 responses are cached - cache lifetime is 30 seconds; newly published entries can therefore take up to 30 seconds to appear on a cached page - cache locking coalesces simultaneous misses - stale cached responses may be served while refreshing or during transient upstream 5xx/timeouts - each ingress replica uses at most 1 GB of ephemeral on-disk cache ## Validation - `go test ./...` in `deploy/` - `go vet ./...` in `deploy/` - `git diff --check` - ingress-nginx chart 4.13.0 in a disposable kind cluster - generated NGINX configuration passed `nginx -t` - repeated list request verified `X-Registry-Cache: MISS` followed by `HIT` - non-list route verified uncached - exact list locations verified with the higher rate zone while the catch-all retained 180 RPM Nothing was applied to production during investigation or validation.
This commit is contained in:
Radoslav Dimitrov
GitHub
parent
2b09bde534
commit
f52dc8525a
2 changed files with 112 additions and 5 deletions
|
|
@ -65,6 +65,31 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
|
|||
"annotations": pulumi.Map{},
|
||||
},
|
||||
"config": pulumi.Map{
|
||||
// Cache the public list endpoints at the ingress. Registry consumers
|
||||
// repeatedly walk the same cursor pages, so sending every walk to the
|
||||
// application needlessly consumes application CPU and database pool
|
||||
// connections. The cache is intentionally short-lived so newly
|
||||
// published servers become visible quickly.
|
||||
//
|
||||
// Use a variable cache zone so this global location snippet remains a
|
||||
// no-op for publish, auth, mutation, and individual-server routes.
|
||||
"http-snippet": pulumi.String(`
|
||||
proxy_cache_path /tmp/registry-read-cache levels=1:2 keys_zone=registry_read_cache:20m max_size=1g inactive=5m use_temp_path=off;
|
||||
map "$request_method:$uri" $registry_read_cache_zone {
|
||||
default off;
|
||||
~^GET:/v0(?:\.1)?/servers$ registry_read_cache;
|
||||
}
|
||||
`),
|
||||
"location-snippet": pulumi.String(`
|
||||
proxy_cache $registry_read_cache_zone;
|
||||
proxy_cache_key "$scheme$request_method$host$request_uri";
|
||||
proxy_cache_valid 200 30s;
|
||||
proxy_cache_lock on;
|
||||
proxy_cache_lock_timeout 5s;
|
||||
proxy_cache_background_update on;
|
||||
proxy_cache_use_stale updating error timeout http_500 http_502 http_503 http_504;
|
||||
add_header X-Registry-Cache $upstream_cache_status always;
|
||||
`),
|
||||
// Disable strict path validation, to work around a bug in ingress-nginx
|
||||
// https://cert-manager.io/docs/releases/release-notes/release-notes-1.18/#acme-http01-challenge-paths-now-use-pathtype-exact-in-ingress-routes
|
||||
// https://github.com/kubernetes/ingress-nginx/issues/11176
|
||||
|
|
|
|||
|
|
@ -324,11 +324,12 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
Annotations: pulumi.StringMap{
|
||||
"cert-manager.io/cluster-issuer": pulumi.String("letsencrypt-prod"),
|
||||
"kubernetes.io/ingress.class": pulumi.String("nginx"),
|
||||
// Rate limiting to protect against abuse
|
||||
// Allows 180 requests/minute (3 req/sec avg), with bursts up to 540 requests
|
||||
// Status code 429 is set globally via NGINX ConfigMap (per-Ingress annotation doesn't work)
|
||||
"nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("180"),
|
||||
"nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"),
|
||||
// Rate limiting to protect against abuse. The exact bulk-list paths get a
|
||||
// separate, higher budget below; all other routes retain this limit.
|
||||
// Allows 180 requests/minute (3 req/sec avg), with bursts up to 540 requests.
|
||||
// Status code 429 is set globally via NGINX ConfigMap (per-Ingress annotation doesn't work).
|
||||
"nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("180"),
|
||||
"nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"),
|
||||
},
|
||||
},
|
||||
Spec: &networkingv1.IngressSpecArgs{
|
||||
|
|
@ -369,6 +370,87 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// Bulk registry consumers paginate the complete server list. Production
|
||||
// traffic analysis on 2026-08-06 showed legitimate consumers sustaining
|
||||
// roughly 10 requests/second per source IP while the catch-all Ingress only
|
||||
// allowed 3 requests/second per controller replica. That caused NGINX to
|
||||
// reject healthy pagination requests before they reached the application.
|
||||
// The ingress controller caches these exact public paths for 30 seconds, so
|
||||
// the higher client budget does not translate into the same increase in
|
||||
// application and database load.
|
||||
//
|
||||
// ingress-nginx merges different paths for the same host across Ingress
|
||||
// resources and applies each resource's annotations only to its own paths.
|
||||
// Keep the canonical list endpoints on exact paths so publish, auth, edit,
|
||||
// status, and individual-server reads remain protected by the lower limit.
|
||||
_, err = networkingv1.NewIngress(ctx, "mcp-registry-bulk-read", &networkingv1.IngressArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Name: pulumi.String("mcp-registry-bulk-read"),
|
||||
Namespace: pulumi.String("default"),
|
||||
Labels: pulumi.StringMap{
|
||||
"app": pulumi.String("mcp-registry"),
|
||||
"environment": pulumi.String(environment),
|
||||
},
|
||||
Annotations: pulumi.StringMap{
|
||||
// ingress-nginx disables response buffering by default, but NGINX
|
||||
// needs it to populate the on-disk proxy cache for these paths.
|
||||
"nginx.ingress.kubernetes.io/proxy-buffering": pulumi.String("on"),
|
||||
// Limits are maintained independently by each ingress-nginx replica.
|
||||
// At two production replicas this permits up to 1,200 requests/minute
|
||||
// per source IP, while a single replica still accommodates the observed
|
||||
// 10 requests/second pagination rate.
|
||||
"nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("600"),
|
||||
"nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"),
|
||||
},
|
||||
},
|
||||
Spec: &networkingv1.IngressSpecArgs{
|
||||
IngressClassName: pulumi.String("nginx"),
|
||||
Rules: pulumi.ToStringArray(hosts).ToStringArrayOutput().ApplyT(func(hosts []string) networkingv1.IngressRuleArray {
|
||||
rules := make(networkingv1.IngressRuleArray, 0, len(hosts))
|
||||
for _, host := range hosts {
|
||||
rules = append(rules, &networkingv1.IngressRuleArgs{
|
||||
Host: pulumi.String(host),
|
||||
Http: &networkingv1.HTTPIngressRuleValueArgs{
|
||||
Paths: networkingv1.HTTPIngressPathArray{
|
||||
// Exact prevents the higher budget from applying to edit/status
|
||||
// subpaths. The v0.1 dot is accepted because ingress.go disables
|
||||
// strict path validation for this controller.
|
||||
&networkingv1.HTTPIngressPathArgs{
|
||||
Path: pulumi.String("/v0/servers"),
|
||||
PathType: pulumi.String("Exact"),
|
||||
Backend: &networkingv1.IngressBackendArgs{
|
||||
Service: &networkingv1.IngressServiceBackendArgs{
|
||||
Name: service.Metadata.Name().Elem(),
|
||||
Port: &networkingv1.ServiceBackendPortArgs{
|
||||
Number: pulumi.Int(80),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
&networkingv1.HTTPIngressPathArgs{
|
||||
Path: pulumi.String("/v0.1/servers"),
|
||||
PathType: pulumi.String("Exact"),
|
||||
Backend: &networkingv1.IngressBackendArgs{
|
||||
Service: &networkingv1.IngressServiceBackendArgs{
|
||||
Name: service.Metadata.Name().Elem(),
|
||||
Port: &networkingv1.ServiceBackendPortArgs{
|
||||
Number: pulumi.Int(80),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
return rules
|
||||
}).(networkingv1.IngressRuleArrayOutput),
|
||||
},
|
||||
}, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(ingressNginx.Ready))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx.Export("ingressHosts", ingress.Spec.Rules().ApplyT(func(rules []networkingv1.IngressRule) []string {
|
||||
hosts := make([]string, 0, len(rules))
|
||||
for _, rule := range rules {
|
||||
|
|
|
|||
Loading…
Reference in a new issue