b24d6c7657## Summary Daily ~17:00 UTC `Publish Endpoint Latency` and `Availability < 95%` alerts are caused by both registry pods exit-0'ing within seconds of each other and racing with the still-recovering `registry-pg-rw` endpoint on restart. Separately, the default compute SA had no project IAM roles, so `fluentbit-gke` and the GMP collector were silently dropping every log line and time series — which is why none of this was visible in Cloud Logging. All Kubernetes/IAM changes apply uniformly to staging and prod. ## Changes - **GCP IAM** (`deploy/pkg/providers/gcp/provider.go`) — grant the default compute SA `logging.logWriter`, `monitoring.metricWriter`, `monitoring.viewer`, and `stackdriver.resourceMetadata.writer`. Already applied live; this just encodes it in Pulumi. - **Pod resilience** (`deploy/pkg/k8s/registry.go`) — - `PodDisruptionBudget` (`minAvailable: 1`) prevents a synchronized voluntary disruption. - `TopologySpreadConstraints` (`ScheduleAnyway`, hostname) keeps the PDB from deadlocking node drains when both pods co-locate. - `StartupProbe` (30 × 5s = 150s) protects the DB-retry budget from the liveness probe. - Memory bump 128→256Mi request, 256→512Mi limit. - **Startup retry** (`cmd/registry/main.go`) — replace silent `return` on initial DB-ping failure with bounded retry-with-backoff (8 attempts, 1→8s capped, 10s per-attempt). A brief endpoint flap no longer cascades into a restart loop. - **Diagnostic log** (`internal/service/registry_service.go`) — emit `validate_ms` on each publish (success and failure). Validate fans out to npm/PyPI/OCI with 10s per-host timeouts and is the most likely contributor to publish latency. Total request duration is already covered by the existing `http.request.duration` histogram. ## Calibration The IAM, PDB, topology-spread, startup-probe, and DB-retry pieces are mitigations — they reduce blast radius regardless of root cause. The `validate_ms` log is the diagnostic for identifying the trigger when the alert fires next. Single-PG SPOF (`deploy/pkg/k8s/postgres.go:57`) remains a follow-up; today's evidence shows PG was a victim, not the cause. ## Test plan - [x] `go build ./...` and `go vet ./...` clean for both modules - [x] `make lint` clean (12 pre-existing issues under `deploy/` are not in `make lint`) - [x] `go test -race -count=1 ./internal/... ./cmd/...` green - [x] IAM applied live; `fluentbit-gke` and GMP no longer hitting `IAM_PERMISSION_DENIED` - [ ] After deploy, verify the next 17:00 UTC firing surfaces `validate_ms` in Cloud Logging 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
357 lines
12 KiB
Go
357 lines
12 KiB
Go
package gcp
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/organizations"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
|
|
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
|
|
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
|
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
|
|
|
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
|
|
)
|
|
|
|
// Provider implements the ClusterProvider interface for Google Kubernetes Engine
|
|
type Provider struct{}
|
|
|
|
// createGCPProvider creates a GCP provider with explicit credentials if configured
|
|
func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error) {
|
|
gcpConf := config.New(ctx, "gcp")
|
|
|
|
// Get project ID from config
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Get region from config or use default
|
|
region := gcpConf.Get("region")
|
|
if region == "" {
|
|
region = "us-central1"
|
|
}
|
|
|
|
// Get credentials from config (base64 encoded service account JSON)
|
|
credentials := gcpConf.Get("credentials")
|
|
if credentials != "" {
|
|
// Decode the base64 credentials
|
|
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
|
|
}
|
|
credentials = string(decodedCreds)
|
|
}
|
|
|
|
// Create a GCP provider with explicit credentials if provided
|
|
if credentials != "" {
|
|
return gcp.NewProvider(ctx, name, &gcp.ProviderArgs{
|
|
Project: pulumi.String(projectID),
|
|
Region: pulumi.String(region),
|
|
Credentials: pulumi.String(credentials),
|
|
})
|
|
}
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
// grantNodeServiceAccountRoles grants the default compute service account the standard
|
|
// GKE node roles required for log shipping (fluentbit-gke) and metrics scraping
|
|
// (managed Prometheus). New GCP projects no longer auto-grant Editor to the default
|
|
// compute SA, so these have to be set explicitly or every pod log/metric is dropped.
|
|
func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProvider *gcp.Provider) error {
|
|
invokeOpts := []pulumi.InvokeOption{}
|
|
resourceOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
invokeOpts = append(invokeOpts, pulumi.Provider(gcpProvider))
|
|
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{
|
|
ProjectId: pulumi.StringRef(projectID),
|
|
}, invokeOpts...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to look up project number for node SA bindings: %w", err)
|
|
}
|
|
|
|
nodeSA := fmt.Sprintf("serviceAccount:%s-compute@developer.gserviceaccount.com", project.Number)
|
|
|
|
roles := []string{
|
|
"roles/logging.logWriter",
|
|
"roles/monitoring.metricWriter",
|
|
"roles/monitoring.viewer",
|
|
"roles/stackdriver.resourceMetadata.writer",
|
|
}
|
|
|
|
for _, role := range roles {
|
|
name := fmt.Sprintf("node-sa-%s", strings.ReplaceAll(strings.TrimPrefix(role, "roles/"), ".", "-"))
|
|
_, err := projects.NewIAMMember(ctx, name, &projects.IAMMemberArgs{
|
|
Project: pulumi.String(projectID),
|
|
Role: pulumi.String(role),
|
|
Member: pulumi.String(nodeSA),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to grant %s to node SA: %w", role, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// CreateCluster creates a Google Kubernetes Engine cluster
|
|
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
|
|
// Get configuration
|
|
gcpConf := config.New(ctx, "gcp")
|
|
|
|
// Get project ID from config
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Get region from config or use default
|
|
region := gcpConf.Get("region")
|
|
if region == "" {
|
|
region = "us-central1"
|
|
}
|
|
|
|
// Create GCP provider with explicit credentials if configured
|
|
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create GKE cluster
|
|
clusterName := fmt.Sprintf("mcp-registry-%s", environment)
|
|
|
|
// Use a specific zone instead of region for zonal cluster
|
|
zone := fmt.Sprintf("%s-b", region)
|
|
|
|
// Configure the GKE cluster
|
|
clusterArgs := &container.ClusterArgs{
|
|
Name: pulumi.String(clusterName),
|
|
Location: pulumi.String(zone),
|
|
Project: pulumi.String(projectID),
|
|
Description: pulumi.String(fmt.Sprintf("MCP Registry %s GKE Cluster", environment)),
|
|
|
|
// Initial node count (will be managed by node pool)
|
|
InitialNodeCount: pulumi.Int(1),
|
|
|
|
// Remove default node pool after cluster creation
|
|
RemoveDefaultNodePool: pulumi.Bool(true),
|
|
|
|
AddonsConfig: &container.ClusterAddonsConfigArgs{
|
|
// Disable as we use ingress-nginx
|
|
HttpLoadBalancing: &container.ClusterAddonsConfigHttpLoadBalancingArgs{
|
|
Disabled: pulumi.Bool(true),
|
|
},
|
|
},
|
|
}
|
|
|
|
// Add provider if we have explicit credentials
|
|
clusterOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
clusterOpts = append(clusterOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
cluster, err := container.NewCluster(ctx, clusterName, clusterArgs, clusterOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create GKE cluster: %w", err)
|
|
}
|
|
|
|
// Create a managed node pool for the cluster
|
|
nodePoolName := fmt.Sprintf("%s-nodepool", clusterName)
|
|
nodePoolArgs := &container.NodePoolArgs{
|
|
Cluster: cluster.Name,
|
|
Location: pulumi.String(zone),
|
|
Project: pulumi.String(projectID),
|
|
|
|
// Node pool configuration
|
|
NodeCount: pulumi.Int(2),
|
|
NodeConfig: &container.NodePoolNodeConfigArgs{
|
|
MachineType: pulumi.String("e2-standard-2"),
|
|
DiskSizeGb: pulumi.Int(20),
|
|
DiskType: pulumi.String("pd-standard"),
|
|
},
|
|
|
|
// Node management configuration
|
|
Management: &container.NodePoolManagementArgs{
|
|
AutoRepair: pulumi.Bool(true),
|
|
AutoUpgrade: pulumi.Bool(true),
|
|
},
|
|
}
|
|
|
|
// Add provider if we have explicit credentials
|
|
nodePoolOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
nodePoolOpts = append(nodePoolOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
nodePool, err := container.NewNodePool(ctx, nodePoolName, nodePoolArgs, nodePoolOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create node pool: %w", err)
|
|
}
|
|
|
|
// Grant the default compute service account the standard GKE node roles so the
|
|
// fluentbit-gke and managed Prometheus collectors can ship logs and metrics.
|
|
// Without these, every pod log line and time series fails with IAM_PERMISSION_DENIED.
|
|
if err := grantNodeServiceAccountRoles(ctx, projectID, gcpProvider); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create Kubernetes provider using the cluster directly
|
|
k8sProvider, err := kubernetes.NewProvider(ctx, "k8s-provider", &kubernetes.ProviderArgs{
|
|
Kubeconfig: pulumi.All(cluster.Endpoint, cluster.MasterAuth).ApplyT(func(args []any) (string, error) {
|
|
endpoint := args[0].(string)
|
|
masterAuth := args[1].(container.ClusterMasterAuth)
|
|
context := fmt.Sprintf("%s_%s_%s", projectID, zone, clusterName)
|
|
|
|
// Extract CA certificate
|
|
caCert := ""
|
|
if masterAuth.ClusterCaCertificate != nil {
|
|
caCert = *masterAuth.ClusterCaCertificate
|
|
}
|
|
|
|
// Create kubeconfig using gke-gcloud-auth-plugin
|
|
kubeconfigYAML := fmt.Sprintf(`apiVersion: v1
|
|
clusters:
|
|
- cluster:
|
|
certificate-authority-data: %s
|
|
server: https://%s
|
|
name: %s
|
|
contexts:
|
|
- context:
|
|
cluster: %s
|
|
user: %s
|
|
name: %s
|
|
current-context: %s
|
|
kind: Config
|
|
preferences: {}
|
|
users:
|
|
- name: %s
|
|
user:
|
|
exec:
|
|
apiVersion: client.authentication.k8s.io/v1beta1
|
|
command: gke-gcloud-auth-plugin
|
|
installHint: Install gke-gcloud-auth-plugin for use with kubectl by following
|
|
https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke
|
|
provideClusterInfo: true
|
|
`, caCert, endpoint, context, context, context, context, context, context)
|
|
|
|
return kubeconfigYAML, nil
|
|
}).(pulumi.StringOutput),
|
|
}, pulumi.DependsOn([]pulumi.Resource{cluster, nodePool}))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create Kubernetes provider: %w", err)
|
|
}
|
|
|
|
return &providers.ProviderInfo{
|
|
Name: nodePool.Cluster,
|
|
Provider: k8sProvider,
|
|
}, nil
|
|
}
|
|
|
|
// CreateBackupStorage creates GCS bucket with HMAC credentials for S3-compatible access
|
|
func (p *Provider) CreateBackupStorage(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*providers.BackupStorageInfo, error) {
|
|
gcpConf := config.New(ctx, "gcp")
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Create GCP provider with explicit credentials if configured
|
|
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit-backup")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set resource options with provider if we have explicit credentials
|
|
resourceOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
// Create GCS bucket for backups
|
|
bucketName := fmt.Sprintf("mcp-registry-%s-backups", environment)
|
|
bucket, err := storage.NewBucket(ctx, "backup-bucket", &storage.BucketArgs{
|
|
Name: pulumi.String(bucketName),
|
|
Location: pulumi.String("US"),
|
|
StorageClass: pulumi.String("STANDARD"),
|
|
LifecycleRules: storage.BucketLifecycleRuleArray{
|
|
&storage.BucketLifecycleRuleArgs{
|
|
Action: &storage.BucketLifecycleRuleActionArgs{
|
|
Type: pulumi.String("Delete"),
|
|
},
|
|
Condition: &storage.BucketLifecycleRuleConditionArgs{
|
|
Age: pulumi.Int(60), // Keep backups for 60 days (K8up manages pruning at 28 days, GCS deletion is a safety net)
|
|
},
|
|
},
|
|
},
|
|
UniformBucketLevelAccess: pulumi.Bool(true),
|
|
Versioning: &storage.BucketVersioningArgs{
|
|
Enabled: pulumi.Bool(true),
|
|
},
|
|
Labels: pulumi.StringMap{
|
|
"environment": pulumi.String(environment),
|
|
"purpose": pulumi.String("k8up-backups"),
|
|
},
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create backup bucket: %w", err)
|
|
}
|
|
|
|
// Use the existing Pulumi service account instead of creating a new one
|
|
// The service account email is pulumi-svc@<project>.iam.gserviceaccount.com
|
|
serviceAccountEmail := fmt.Sprintf("pulumi-svc@%s.iam.gserviceaccount.com", projectID)
|
|
|
|
// Grant the service account access to the bucket
|
|
_, err = storage.NewBucketIAMMember(ctx, "backup-bucket-iam", &storage.BucketIAMMemberArgs{
|
|
Bucket: bucket.Name,
|
|
Role: pulumi.String("roles/storage.objectAdmin"),
|
|
Member: pulumi.Sprintf("serviceAccount:%s", pulumi.String(serviceAccountEmail)),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to grant bucket access: %w", err)
|
|
}
|
|
|
|
// Create HMAC key for S3-compatible access
|
|
hmacKey, err := storage.NewHmacKey(ctx, "backup-hmac-key", &storage.HmacKeyArgs{
|
|
ServiceAccountEmail: pulumi.String(serviceAccountEmail),
|
|
Project: pulumi.String(projectID),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create HMAC key: %w", err)
|
|
}
|
|
|
|
// Create Kubernetes secret with S3-compatible credentials
|
|
backupSecret, err := corev1.NewSecret(ctx, "k8up-backup-credentials", &corev1.SecretArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("k8up-backup-credentials"),
|
|
Namespace: pulumi.String("default"),
|
|
Labels: pulumi.StringMap{
|
|
"k8up.io/backup": pulumi.String("true"),
|
|
"environment": pulumi.String(environment),
|
|
},
|
|
},
|
|
Type: pulumi.String("Opaque"),
|
|
StringData: pulumi.StringMap{
|
|
"AWS_ACCESS_KEY_ID": hmacKey.AccessId,
|
|
"AWS_SECRET_ACCESS_KEY": hmacKey.Secret,
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create backup credentials secret: %w", err)
|
|
}
|
|
|
|
return &providers.BackupStorageInfo{
|
|
Endpoint: "https://storage.googleapis.com",
|
|
BucketName: bucketName,
|
|
Credentials: backupSecret,
|
|
}, nil
|
|
}
|