fix: harden registry availability and add publish phase timings (#1211)

## Summary

Daily ~17:00 UTC `Publish Endpoint Latency` and `Availability < 95%`
alerts are caused by both registry pods exit-0'ing within seconds of
each other and racing with the still-recovering `registry-pg-rw`
endpoint on restart. Separately, the default compute SA had no project
IAM roles, so `fluentbit-gke` and the GMP collector were silently
dropping every log line and time series — which is why none of this was
visible in Cloud Logging.

All Kubernetes/IAM changes apply uniformly to staging and prod.

## Changes

- **GCP IAM** (`deploy/pkg/providers/gcp/provider.go`) — grant the
default compute SA `logging.logWriter`, `monitoring.metricWriter`,
`monitoring.viewer`, and `stackdriver.resourceMetadata.writer`. Already
applied live; this just encodes it in Pulumi.
- **Pod resilience** (`deploy/pkg/k8s/registry.go`) —
- `PodDisruptionBudget` (`minAvailable: 1`) prevents a synchronized
voluntary disruption.
- `TopologySpreadConstraints` (`ScheduleAnyway`, hostname) keeps the PDB
from deadlocking node drains when both pods co-locate.
- `StartupProbe` (30 × 5s = 150s) protects the DB-retry budget from the
liveness probe.
  - Memory bump 128→256Mi request, 256→512Mi limit.
- **Startup retry** (`cmd/registry/main.go`) — replace silent `return`
on initial DB-ping failure with bounded retry-with-backoff (8 attempts,
1→8s capped, 10s per-attempt). A brief endpoint flap no longer cascades
into a restart loop.
- **Diagnostic log** (`internal/service/registry_service.go`) — emit
`validate_ms` on each publish (success and failure). Validate fans out
to npm/PyPI/OCI with 10s per-host timeouts and is the most likely
contributor to publish latency. Total request duration is already
covered by the existing `http.request.duration` histogram.

## Calibration

The IAM, PDB, topology-spread, startup-probe, and DB-retry pieces are
mitigations — they reduce blast radius regardless of root cause. The
`validate_ms` log is the diagnostic for identifying the trigger when the
alert fires next. Single-PG SPOF (`deploy/pkg/k8s/postgres.go:57`)
remains a follow-up; today's evidence shows PG was a victim, not the
cause.

## Test plan

- [x] `go build ./...` and `go vet ./...` clean for both modules
- [x] `make lint` clean (12 pre-existing issues under `deploy/` are not
in `make lint`)
- [x] `go test -race -count=1 ./internal/... ./cmd/...` green
- [x] IAM applied live; `fluentbit-gke` and GMP no longer hitting
`IAM_PERMISSION_DENIED`
- [ ] After deploy, verify the next 17:00 UTC firing surfaces
`validate_ms` in Cloud Logging

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

deploy/pkg/providers/gcp/provider.go

@@ -3,9 +3,12 @@ package gcp
 import (
 	"encoding/base64"
 	"fmt"
+	"strings"
 
 	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
 	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
+	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/organizations"
+	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects"
 	"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
 	corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
@@ -58,6 +61,49 @@ func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error)
 	return nil, nil
 }
 
+// grantNodeServiceAccountRoles grants the default compute service account the standard
+// GKE node roles required for log shipping (fluentbit-gke) and metrics scraping
+// (managed Prometheus). New GCP projects no longer auto-grant Editor to the default
+// compute SA, so these have to be set explicitly or every pod log/metric is dropped.
+func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProvider *gcp.Provider) error {
+	invokeOpts := []pulumi.InvokeOption{}
+	resourceOpts := []pulumi.ResourceOption{}
+	if gcpProvider != nil {
+		invokeOpts = append(invokeOpts, pulumi.Provider(gcpProvider))
+		resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
+	}
+
+	project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{
+		ProjectId: pulumi.StringRef(projectID),
+	}, invokeOpts...)
+	if err != nil {
+		return fmt.Errorf("failed to look up project number for node SA bindings: %w", err)
+	}
+
+	nodeSA := fmt.Sprintf("serviceAccount:%s-compute@developer.gserviceaccount.com", project.Number)
+
+	roles := []string{
+		"roles/logging.logWriter",
+		"roles/monitoring.metricWriter",
+		"roles/monitoring.viewer",
+		"roles/stackdriver.resourceMetadata.writer",
+	}
+
+	for _, role := range roles {
+		name := fmt.Sprintf("node-sa-%s", strings.ReplaceAll(strings.TrimPrefix(role, "roles/"), ".", "-"))
+		_, err := projects.NewIAMMember(ctx, name, &projects.IAMMemberArgs{
+			Project: pulumi.String(projectID),
+			Role:    pulumi.String(role),
+			Member:  pulumi.String(nodeSA),
+		}, resourceOpts...)
+		if err != nil {
+			return fmt.Errorf("failed to grant %s to node SA: %w", role, err)
+		}
+	}
+
+	return nil
+}
+
 // CreateCluster creates a Google Kubernetes Engine cluster
 func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
 	// Get configuration
@@ -152,6 +198,13 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov
 		return nil, fmt.Errorf("failed to create node pool: %w", err)
 	}
 
+	// Grant the default compute service account the standard GKE node roles so the
+	// fluentbit-gke and managed Prometheus collectors can ship logs and metrics.
+	// Without these, every pod log line and time series fails with IAM_PERMISSION_DENIED.
+	if err := grantNodeServiceAccountRoles(ctx, projectID, gcpProvider); err != nil {
+		return nil, err
+	}
+
 	// Create Kubernetes provider using the cluster directly
 	k8sProvider, err := kubernetes.NewProvider(ctx, "k8s-provider", &kubernetes.ProviderArgs{
 		Kubeconfig: pulumi.All(cluster.Endpoint, cluster.MasterAuth).ApplyT(func(args []any) (string, error) {