fix: harden registry availability and add publish phase timings (#1211)
## Summary Daily ~17:00 UTC `Publish Endpoint Latency` and `Availability < 95%` alerts are caused by both registry pods exit-0'ing within seconds of each other and racing with the still-recovering `registry-pg-rw` endpoint on restart. Separately, the default compute SA had no project IAM roles, so `fluentbit-gke` and the GMP collector were silently dropping every log line and time series — which is why none of this was visible in Cloud Logging. All Kubernetes/IAM changes apply uniformly to staging and prod. ## Changes - **GCP IAM** (`deploy/pkg/providers/gcp/provider.go`) — grant the default compute SA `logging.logWriter`, `monitoring.metricWriter`, `monitoring.viewer`, and `stackdriver.resourceMetadata.writer`. Already applied live; this just encodes it in Pulumi. - **Pod resilience** (`deploy/pkg/k8s/registry.go`) — - `PodDisruptionBudget` (`minAvailable: 1`) prevents a synchronized voluntary disruption. - `TopologySpreadConstraints` (`ScheduleAnyway`, hostname) keeps the PDB from deadlocking node drains when both pods co-locate. - `StartupProbe` (30 × 5s = 150s) protects the DB-retry budget from the liveness probe. - Memory bump 128→256Mi request, 256→512Mi limit. - **Startup retry** (`cmd/registry/main.go`) — replace silent `return` on initial DB-ping failure with bounded retry-with-backoff (8 attempts, 1→8s capped, 10s per-attempt). A brief endpoint flap no longer cascades into a restart loop. - **Diagnostic log** (`internal/service/registry_service.go`) — emit `validate_ms` on each publish (success and failure). Validate fans out to npm/PyPI/OCI with 10s per-host timeouts and is the most likely contributor to publish latency. Total request duration is already covered by the existing `http.request.duration` histogram. ## Calibration The IAM, PDB, topology-spread, startup-probe, and DB-retry pieces are mitigations — they reduce blast radius regardless of root cause. The `validate_ms` log is the diagnostic for identifying the trigger when the alert fires next. Single-PG SPOF (`deploy/pkg/k8s/postgres.go:57`) remains a follow-up; today's evidence shows PG was a victim, not the cause. ## Test plan - [x] `go build ./...` and `go vet ./...` clean for both modules - [x] `make lint` clean (12 pre-existing issues under `deploy/` are not in `make lint`) - [x] `go test -race -count=1 ./internal/... ./cmd/...` green - [x] IAM applied live; `fluentbit-gke` and GMP no longer hitting `IAM_PERMISSION_DENIED` - [ ] After deploy, verify the next 17:00 UTC firing surfaces `validate_ms` in Cloud Logging 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Radoslav Dimitrov
GitHub
parent
7722031739
commit
b24d6c7657
1 changed files with 165 additions and 17 deletions
|
|
@ -10,6 +10,7 @@ import (
|
|||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
|
||||
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
networkingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
|
||||
policyv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
||||
|
||||
|
|
@ -95,6 +96,22 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
},
|
||||
},
|
||||
Spec: &corev1.PodSpecArgs{
|
||||
// Soft-spread pods across nodes so the PodDisruptionBudget below can't
|
||||
// deadlock a node drain when both pods happen to co-locate.
|
||||
// `ScheduleAnyway` keeps single-node clusters (e.g. constrained dev
|
||||
// envs) schedulable while still preferring spread under normal load.
|
||||
TopologySpreadConstraints: corev1.TopologySpreadConstraintArray{
|
||||
&corev1.TopologySpreadConstraintArgs{
|
||||
MaxSkew: pulumi.Int(1),
|
||||
TopologyKey: pulumi.String("kubernetes.io/hostname"),
|
||||
WhenUnsatisfiable: pulumi.String("ScheduleAnyway"),
|
||||
LabelSelector: &metav1.LabelSelectorArgs{
|
||||
MatchLabels: pulumi.StringMap{
|
||||
"app": pulumi.String("mcp-registry"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Containers: corev1.ContainerArray{
|
||||
&corev1.ContainerArgs{
|
||||
Name: pulumi.String("mcp-registry"),
|
||||
|
|
@ -169,6 +186,18 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
Value: pulumi.String("*"),
|
||||
},
|
||||
},
|
||||
// StartupProbe protects the DB-retry budget in cmd/registry/main.go:
|
||||
// 30 × 5s = 150s allowed before liveness/readiness take over, which
|
||||
// covers the worst-case retry budget (8 attempts × 10s + ~40s sleep).
|
||||
StartupProbe: &corev1.ProbeArgs{
|
||||
HttpGet: &corev1.HTTPGetActionArgs{
|
||||
Path: pulumi.String("/v0/health"),
|
||||
Port: pulumi.Int(8080),
|
||||
},
|
||||
PeriodSeconds: pulumi.Int(5),
|
||||
FailureThreshold: pulumi.Int(30),
|
||||
TimeoutSeconds: pulumi.Int(3),
|
||||
},
|
||||
LivenessProbe: &corev1.ProbeArgs{
|
||||
HttpGet: &corev1.HTTPGetActionArgs{
|
||||
Path: pulumi.String("/v0/health"),
|
||||
|
|
@ -187,11 +216,11 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
},
|
||||
Resources: &corev1.ResourceRequirementsArgs{
|
||||
Requests: pulumi.StringMap{
|
||||
"memory": pulumi.String("128Mi"),
|
||||
"memory": pulumi.String("256Mi"),
|
||||
"cpu": pulumi.String("100m"),
|
||||
},
|
||||
Limits: pulumi.StringMap{
|
||||
"memory": pulumi.String("256Mi"),
|
||||
"memory": pulumi.String("512Mi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -204,6 +233,31 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
|
|||
return nil, err
|
||||
}
|
||||
|
||||
// Keep at least one registry pod available during voluntary disruptions
|
||||
// (node drains, evictions, etc.) so a synchronized eviction can't take both
|
||||
// pods down at once.
|
||||
_, err = policyv1.NewPodDisruptionBudget(ctx, "mcp-registry", &policyv1.PodDisruptionBudgetArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Name: pulumi.String("mcp-registry"),
|
||||
Namespace: pulumi.String("default"),
|
||||
Labels: pulumi.StringMap{
|
||||
"app": pulumi.String("mcp-registry"),
|
||||
"environment": pulumi.String(environment),
|
||||
},
|
||||
},
|
||||
Spec: &policyv1.PodDisruptionBudgetSpecArgs{
|
||||
MinAvailable: pulumi.Int(1),
|
||||
Selector: &metav1.LabelSelectorArgs{
|
||||
MatchLabels: pulumi.StringMap{
|
||||
"app": pulumi.String("mcp-registry"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Provider(cluster.Provider))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create Service
|
||||
service, err := corev1.NewService(ctx, "mcp-registry", &corev1.ServiceArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
|
|
|
|||
Loading…
Reference in a new issue