c6d3096d1b## Summary Follow-up to #1211 to encode the GCP API and IAM dependencies its node-SA bindings rely on. After #1211 merged, the staging deploy hit two undocumented prereqs in sequence — Cloud Resource Manager API not enabled (because `projects.NewIAMMember` calls SetIamPolicy through CRM), then the Pulumi service account missing `roles/resourcemanager.projectIamAdmin`. Both fixed manually to unblock; this PR makes them explicit so they don't bite again. ## Changes **`ensureRequiredAPIs` adopts five GCP APIs** as Pulumi-managed `projects.Service` resources, all with `DisableOnDestroy: false` and `DisableDependentServices: false` so a Pulumi destroy/refactor can never disable a shared API: | API | Why | |-----|-----| | `cloudresourcemanager` | `projects.NewIAMMember` → SetIamPolicy | | `compute` | `compute.GetDefaultServiceAccount` invoke + GKE | | `container` | GKE cluster | | `logging` | fluentbit-gke ships container logs | | `monitoring` | managed Prometheus collector ships metrics | CRM is created explicitly (not in the loop) so callers get a direct reference for `pulumi.DependsOn`. Storage is intentionally **not** Pulumi-managed — Pulumi state itself lives in a GCS bucket, chicken-and-egg. **`grantNodeServiceAccountRoles`** now `DependsOn` the CRM `projects.Service` for the four IAM bindings, and uses `compute.GetDefaultServiceAccount` to derive the SA email (Compute API was already required for GKE — avoids a CRM-dependent project-number lookup). **`deploy/README.md`** — adds `roles/resourcemanager.projectIamAdmin` to the Pulumi SA's required roles and clarifies which APIs must be enabled before the first `pulumi up` (storage, cloudresourcemanager, container) vs. which `ensureRequiredAPIs` adopts. ## Notes `projects.Service` is idempotent against already-enabled APIs — on existing projects (staging/prod), Pulumi adopts the live enablement into state without re-enabling. This was confirmed live: staging deploy went green after the manual unblocks above, validating the runtime model. ## Test plan - [x] `go build ./...` clean for `deploy/` - [x] `golangci-lint run` clean for `deploy/pkg/providers/gcp/...` (one pre-existing `nilnil` at line 61, unrelated) - [x] Manual unblock + green staging deploy verified the runtime behavior this PR encodes - [ ] On merge: staging deploy adopts the existing API enablements without churn, then proceeds normally - [ ] Prod deploy via #1212 (image promotion) inherits the same setup 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
433 lines
15 KiB
Go
433 lines
15 KiB
Go
package gcp
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/compute"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects"
|
|
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
|
|
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
|
|
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
|
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
|
|
|
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
|
|
)
|
|
|
|
// Provider implements the ClusterProvider interface for Google Kubernetes Engine
|
|
type Provider struct{}
|
|
|
|
// createGCPProvider creates a GCP provider with explicit credentials if configured
|
|
func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error) {
|
|
gcpConf := config.New(ctx, "gcp")
|
|
|
|
// Get project ID from config
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Get region from config or use default
|
|
region := gcpConf.Get("region")
|
|
if region == "" {
|
|
region = "us-central1"
|
|
}
|
|
|
|
// Get credentials from config (base64 encoded service account JSON)
|
|
credentials := gcpConf.Get("credentials")
|
|
if credentials != "" {
|
|
// Decode the base64 credentials
|
|
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
|
|
}
|
|
credentials = string(decodedCreds)
|
|
}
|
|
|
|
// Create a GCP provider with explicit credentials if provided
|
|
if credentials != "" {
|
|
return gcp.NewProvider(ctx, name, &gcp.ProviderArgs{
|
|
Project: pulumi.String(projectID),
|
|
Region: pulumi.String(region),
|
|
Credentials: pulumi.String(credentials),
|
|
})
|
|
}
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
// ensureRequiredAPIs adopts the GCP APIs the deploy depends on as Pulumi-managed
|
|
// services. Encoding them as Pulumi resources makes the dependency explicit and
|
|
// protects against drift (an org policy reset or accidental disable).
|
|
//
|
|
// Note: bootstrap APIs (storage, cloudresourcemanager, container — see
|
|
// deploy/README.md) must already be enabled before this runs, because the
|
|
// resources that need them are created earlier in the stack. For those, this
|
|
// function adopts them into Pulumi state after the fact. For the rest (logging,
|
|
// monitoring), the adoption happens before any consumer runs.
|
|
//
|
|
// DisableOnDestroy and DisableDependentServices are both false — a Pulumi destroy
|
|
// or refactor must never disable a shared API that other resources (and humans)
|
|
// rely on. Enable-only.
|
|
//
|
|
// Returns the Cloud Resource Manager API resource specifically, since the node-SA
|
|
// IAM bindings need to DependsOn it (SetIamPolicy is gated on CRM).
|
|
func ensureRequiredAPIs(ctx *pulumi.Context, projectID string, resourceOpts []pulumi.ResourceOption) (*projects.Service, error) {
|
|
// Service Usage API (which projects.NewService itself uses) is enabled by default
|
|
// on GCP projects, so we don't need to manage it here.
|
|
|
|
// CRM is created explicitly (not in the loop below) because callers need a
|
|
// direct reference to it for DependsOn — projects.NewIAMMember calls
|
|
// SetIamPolicy under the hood, which is gated on CRM.
|
|
crm, err := projects.NewService(ctx, "crm-api", &projects.ServiceArgs{
|
|
Project: pulumi.String(projectID),
|
|
Service: pulumi.String("cloudresourcemanager.googleapis.com"),
|
|
DisableOnDestroy: pulumi.Bool(false),
|
|
DisableDependentServices: pulumi.Bool(false),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to ensure cloudresourcemanager.googleapis.com is enabled: %w", err)
|
|
}
|
|
|
|
otherAPIs := []struct {
|
|
resourceName string
|
|
serviceName string
|
|
}{
|
|
// Required for compute.GetDefaultServiceAccount and the GKE cluster.
|
|
{"compute-api", "compute.googleapis.com"},
|
|
// Required for the GKE cluster.
|
|
{"container-api", "container.googleapis.com"},
|
|
// Required for fluentbit-gke to ship container logs.
|
|
{"logging-api", "logging.googleapis.com"},
|
|
// Required for the managed Prometheus collector to ship metrics.
|
|
{"monitoring-api", "monitoring.googleapis.com"},
|
|
}
|
|
|
|
for _, api := range otherAPIs {
|
|
_, err := projects.NewService(ctx, api.resourceName, &projects.ServiceArgs{
|
|
Project: pulumi.String(projectID),
|
|
Service: pulumi.String(api.serviceName),
|
|
DisableOnDestroy: pulumi.Bool(false),
|
|
DisableDependentServices: pulumi.Bool(false),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to ensure %s is enabled: %w", api.serviceName, err)
|
|
}
|
|
}
|
|
return crm, nil
|
|
}
|
|
|
|
// grantNodeServiceAccountRoles grants the default compute service account the standard
|
|
// GKE node roles required for log shipping (fluentbit-gke) and metrics scraping
|
|
// (managed Prometheus). New GCP projects no longer auto-grant Editor to the default
|
|
// compute SA, so these have to be set explicitly or every pod log/metric is dropped.
|
|
//
|
|
// The IAM bindings depend on Cloud Resource Manager API being enabled (managed by
|
|
// ensureRequiredAPIs above) — without it, projects.NewIAMMember fails because
|
|
// SetIamPolicy lives behind that API.
|
|
func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProvider *gcp.Provider) error {
|
|
invokeOpts := []pulumi.InvokeOption{}
|
|
resourceOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
invokeOpts = append(invokeOpts, pulumi.Provider(gcpProvider))
|
|
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
crmAPI, err := ensureRequiredAPIs(ctx, projectID, resourceOpts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Get the default compute SA email directly from the Compute API instead of
|
|
// constructing it from the project number — avoids needing CRM API just for
|
|
// the project lookup, and matches whatever GCP actually returns.
|
|
defaultSA, err := compute.GetDefaultServiceAccount(ctx, &compute.GetDefaultServiceAccountArgs{
|
|
Project: pulumi.StringRef(projectID),
|
|
}, invokeOpts...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to look up default compute service account: %w", err)
|
|
}
|
|
|
|
nodeSA := "serviceAccount:" + defaultSA.Email
|
|
|
|
roles := []string{
|
|
"roles/logging.logWriter",
|
|
"roles/monitoring.metricWriter",
|
|
"roles/monitoring.viewer",
|
|
"roles/stackdriver.resourceMetadata.writer",
|
|
}
|
|
|
|
bindingOpts := make([]pulumi.ResourceOption, 0, len(resourceOpts)+1)
|
|
bindingOpts = append(bindingOpts, resourceOpts...)
|
|
bindingOpts = append(bindingOpts, pulumi.DependsOn([]pulumi.Resource{crmAPI}))
|
|
for _, role := range roles {
|
|
name := fmt.Sprintf("node-sa-%s", strings.ReplaceAll(strings.TrimPrefix(role, "roles/"), ".", "-"))
|
|
_, err := projects.NewIAMMember(ctx, name, &projects.IAMMemberArgs{
|
|
Project: pulumi.String(projectID),
|
|
Role: pulumi.String(role),
|
|
Member: pulumi.String(nodeSA),
|
|
}, bindingOpts...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to grant %s to node SA: %w", role, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// CreateCluster creates a Google Kubernetes Engine cluster
|
|
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
|
|
// Get configuration
|
|
gcpConf := config.New(ctx, "gcp")
|
|
|
|
// Get project ID from config
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Get region from config or use default
|
|
region := gcpConf.Get("region")
|
|
if region == "" {
|
|
region = "us-central1"
|
|
}
|
|
|
|
// Create GCP provider with explicit credentials if configured
|
|
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create GKE cluster
|
|
clusterName := fmt.Sprintf("mcp-registry-%s", environment)
|
|
|
|
// Use a specific zone instead of region for zonal cluster
|
|
zone := fmt.Sprintf("%s-b", region)
|
|
|
|
// Configure the GKE cluster
|
|
clusterArgs := &container.ClusterArgs{
|
|
Name: pulumi.String(clusterName),
|
|
Location: pulumi.String(zone),
|
|
Project: pulumi.String(projectID),
|
|
Description: pulumi.String(fmt.Sprintf("MCP Registry %s GKE Cluster", environment)),
|
|
|
|
// Initial node count (will be managed by node pool)
|
|
InitialNodeCount: pulumi.Int(1),
|
|
|
|
// Remove default node pool after cluster creation
|
|
RemoveDefaultNodePool: pulumi.Bool(true),
|
|
|
|
AddonsConfig: &container.ClusterAddonsConfigArgs{
|
|
// Disable as we use ingress-nginx
|
|
HttpLoadBalancing: &container.ClusterAddonsConfigHttpLoadBalancingArgs{
|
|
Disabled: pulumi.Bool(true),
|
|
},
|
|
},
|
|
}
|
|
|
|
// Add provider if we have explicit credentials
|
|
clusterOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
clusterOpts = append(clusterOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
cluster, err := container.NewCluster(ctx, clusterName, clusterArgs, clusterOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create GKE cluster: %w", err)
|
|
}
|
|
|
|
// Create a managed node pool for the cluster
|
|
nodePoolName := fmt.Sprintf("%s-nodepool", clusterName)
|
|
nodePoolArgs := &container.NodePoolArgs{
|
|
Cluster: cluster.Name,
|
|
Location: pulumi.String(zone),
|
|
Project: pulumi.String(projectID),
|
|
|
|
// Node pool configuration
|
|
NodeCount: pulumi.Int(2),
|
|
NodeConfig: &container.NodePoolNodeConfigArgs{
|
|
MachineType: pulumi.String("e2-standard-2"),
|
|
DiskSizeGb: pulumi.Int(20),
|
|
DiskType: pulumi.String("pd-standard"),
|
|
},
|
|
|
|
// Node management configuration
|
|
Management: &container.NodePoolManagementArgs{
|
|
AutoRepair: pulumi.Bool(true),
|
|
AutoUpgrade: pulumi.Bool(true),
|
|
},
|
|
}
|
|
|
|
// Add provider if we have explicit credentials
|
|
nodePoolOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
nodePoolOpts = append(nodePoolOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
nodePool, err := container.NewNodePool(ctx, nodePoolName, nodePoolArgs, nodePoolOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create node pool: %w", err)
|
|
}
|
|
|
|
// Grant the default compute service account the standard GKE node roles so the
|
|
// fluentbit-gke and managed Prometheus collectors can ship logs and metrics.
|
|
// Without these, every pod log line and time series fails with IAM_PERMISSION_DENIED.
|
|
if err := grantNodeServiceAccountRoles(ctx, projectID, gcpProvider); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Create Kubernetes provider using the cluster directly
|
|
k8sProvider, err := kubernetes.NewProvider(ctx, "k8s-provider", &kubernetes.ProviderArgs{
|
|
Kubeconfig: pulumi.All(cluster.Endpoint, cluster.MasterAuth).ApplyT(func(args []any) (string, error) {
|
|
endpoint := args[0].(string)
|
|
masterAuth := args[1].(container.ClusterMasterAuth)
|
|
context := fmt.Sprintf("%s_%s_%s", projectID, zone, clusterName)
|
|
|
|
// Extract CA certificate
|
|
caCert := ""
|
|
if masterAuth.ClusterCaCertificate != nil {
|
|
caCert = *masterAuth.ClusterCaCertificate
|
|
}
|
|
|
|
// Create kubeconfig using gke-gcloud-auth-plugin
|
|
kubeconfigYAML := fmt.Sprintf(`apiVersion: v1
|
|
clusters:
|
|
- cluster:
|
|
certificate-authority-data: %s
|
|
server: https://%s
|
|
name: %s
|
|
contexts:
|
|
- context:
|
|
cluster: %s
|
|
user: %s
|
|
name: %s
|
|
current-context: %s
|
|
kind: Config
|
|
preferences: {}
|
|
users:
|
|
- name: %s
|
|
user:
|
|
exec:
|
|
apiVersion: client.authentication.k8s.io/v1beta1
|
|
command: gke-gcloud-auth-plugin
|
|
installHint: Install gke-gcloud-auth-plugin for use with kubectl by following
|
|
https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke
|
|
provideClusterInfo: true
|
|
`, caCert, endpoint, context, context, context, context, context, context)
|
|
|
|
return kubeconfigYAML, nil
|
|
}).(pulumi.StringOutput),
|
|
}, pulumi.DependsOn([]pulumi.Resource{cluster, nodePool}))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create Kubernetes provider: %w", err)
|
|
}
|
|
|
|
return &providers.ProviderInfo{
|
|
Name: nodePool.Cluster,
|
|
Provider: k8sProvider,
|
|
}, nil
|
|
}
|
|
|
|
// CreateBackupStorage creates GCS bucket with HMAC credentials for S3-compatible access
|
|
func (p *Provider) CreateBackupStorage(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*providers.BackupStorageInfo, error) {
|
|
gcpConf := config.New(ctx, "gcp")
|
|
projectID := gcpConf.Get("project")
|
|
if projectID == "" {
|
|
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
|
}
|
|
|
|
// Create GCP provider with explicit credentials if configured
|
|
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit-backup")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Set resource options with provider if we have explicit credentials
|
|
resourceOpts := []pulumi.ResourceOption{}
|
|
if gcpProvider != nil {
|
|
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
|
|
}
|
|
|
|
// Create GCS bucket for backups
|
|
bucketName := fmt.Sprintf("mcp-registry-%s-backups", environment)
|
|
bucket, err := storage.NewBucket(ctx, "backup-bucket", &storage.BucketArgs{
|
|
Name: pulumi.String(bucketName),
|
|
Location: pulumi.String("US"),
|
|
StorageClass: pulumi.String("STANDARD"),
|
|
LifecycleRules: storage.BucketLifecycleRuleArray{
|
|
&storage.BucketLifecycleRuleArgs{
|
|
Action: &storage.BucketLifecycleRuleActionArgs{
|
|
Type: pulumi.String("Delete"),
|
|
},
|
|
Condition: &storage.BucketLifecycleRuleConditionArgs{
|
|
Age: pulumi.Int(60), // Keep backups for 60 days (K8up manages pruning at 28 days, GCS deletion is a safety net)
|
|
},
|
|
},
|
|
},
|
|
UniformBucketLevelAccess: pulumi.Bool(true),
|
|
Versioning: &storage.BucketVersioningArgs{
|
|
Enabled: pulumi.Bool(true),
|
|
},
|
|
Labels: pulumi.StringMap{
|
|
"environment": pulumi.String(environment),
|
|
"purpose": pulumi.String("k8up-backups"),
|
|
},
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create backup bucket: %w", err)
|
|
}
|
|
|
|
// Use the existing Pulumi service account instead of creating a new one
|
|
// The service account email is pulumi-svc@<project>.iam.gserviceaccount.com
|
|
serviceAccountEmail := fmt.Sprintf("pulumi-svc@%s.iam.gserviceaccount.com", projectID)
|
|
|
|
// Grant the service account access to the bucket
|
|
_, err = storage.NewBucketIAMMember(ctx, "backup-bucket-iam", &storage.BucketIAMMemberArgs{
|
|
Bucket: bucket.Name,
|
|
Role: pulumi.String("roles/storage.objectAdmin"),
|
|
Member: pulumi.Sprintf("serviceAccount:%s", pulumi.String(serviceAccountEmail)),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to grant bucket access: %w", err)
|
|
}
|
|
|
|
// Create HMAC key for S3-compatible access
|
|
hmacKey, err := storage.NewHmacKey(ctx, "backup-hmac-key", &storage.HmacKeyArgs{
|
|
ServiceAccountEmail: pulumi.String(serviceAccountEmail),
|
|
Project: pulumi.String(projectID),
|
|
}, resourceOpts...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create HMAC key: %w", err)
|
|
}
|
|
|
|
// Create Kubernetes secret with S3-compatible credentials
|
|
backupSecret, err := corev1.NewSecret(ctx, "k8up-backup-credentials", &corev1.SecretArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("k8up-backup-credentials"),
|
|
Namespace: pulumi.String("default"),
|
|
Labels: pulumi.StringMap{
|
|
"k8up.io/backup": pulumi.String("true"),
|
|
"environment": pulumi.String(environment),
|
|
},
|
|
},
|
|
Type: pulumi.String("Opaque"),
|
|
StringData: pulumi.StringMap{
|
|
"AWS_ACCESS_KEY_ID": hmacKey.AccessId,
|
|
"AWS_SECRET_ACCESS_KEY": hmacKey.Secret,
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create backup credentials secret: %w", err)
|
|
}
|
|
|
|
return &providers.BackupStorageInfo{
|
|
Endpoint: "https://storage.googleapis.com",
|
|
BucketName: bucketName,
|
|
Credentials: backupSecret,
|
|
}, nil
|
|
}
|