From c6d3096d1b029278ed03891d08a20a0029b101ed Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 27 Apr 2026 12:32:22 -0400 Subject: [PATCH] fix(deploy): manage Cloud Resource Manager API enablement in Pulumi (#1213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Follow-up to #1211 to encode the GCP API and IAM dependencies its node-SA bindings rely on. After #1211 merged, the staging deploy hit two undocumented prereqs in sequence — Cloud Resource Manager API not enabled (because `projects.NewIAMMember` calls SetIamPolicy through CRM), then the Pulumi service account missing `roles/resourcemanager.projectIamAdmin`. Both fixed manually to unblock; this PR makes them explicit so they don't bite again. ## Changes **`ensureRequiredAPIs` adopts five GCP APIs** as Pulumi-managed `projects.Service` resources, all with `DisableOnDestroy: false` and `DisableDependentServices: false` so a Pulumi destroy/refactor can never disable a shared API: | API | Why | |-----|-----| | `cloudresourcemanager` | `projects.NewIAMMember` → SetIamPolicy | | `compute` | `compute.GetDefaultServiceAccount` invoke + GKE | | `container` | GKE cluster | | `logging` | fluentbit-gke ships container logs | | `monitoring` | managed Prometheus collector ships metrics | CRM is created explicitly (not in the loop) so callers get a direct reference for `pulumi.DependsOn`. Storage is intentionally **not** Pulumi-managed — Pulumi state itself lives in a GCS bucket, chicken-and-egg. **`grantNodeServiceAccountRoles`** now `DependsOn` the CRM `projects.Service` for the four IAM bindings, and uses `compute.GetDefaultServiceAccount` to derive the SA email (Compute API was already required for GKE — avoids a CRM-dependent project-number lookup). **`deploy/README.md`** — adds `roles/resourcemanager.projectIamAdmin` to the Pulumi SA's required roles and clarifies which APIs must be enabled before the first `pulumi up` (storage, cloudresourcemanager, container) vs. which `ensureRequiredAPIs` adopts. ## Notes `projects.Service` is idempotent against already-enabled APIs — on existing projects (staging/prod), Pulumi adopts the live enablement into state without re-enabling. This was confirmed live: staging deploy went green after the manual unblocks above, validating the runtime model. ## Test plan - [x] `go build ./...` clean for `deploy/` - [x] `golangci-lint run` clean for `deploy/pkg/providers/gcp/...` (one pre-existing `nilnil` at line 61, unrelated) - [x] Manual unblock + green staging deploy verified the runtime behavior this PR encodes - [ ] On merge: staging deploy adopts the existing API enablements without churn, then proceeds normally - [ ] Prod deploy via #1212 (image promotion) inherits the same setup 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- deploy/README.md | 17 +++++- deploy/pkg/providers/gcp/provider.go | 90 +++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 3125d61..392b244 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -65,8 +65,20 @@ Pre-requisites: 1. Create a project: `gcloud projects create mcp-registry-prod` 2. Set the project: `gcloud config set project mcp-registry-prod` -3. Enable required APIs: `gcloud services enable storage.googleapis.com && gcloud services enable container.googleapis.com` -4. Create a service account with necessary permissions, and get the key: +3. Enable the bootstrap APIs needed before Pulumi can run. The remaining APIs + (`cloudresourcemanager`, `compute`, `container`, `logging`, `monitoring`) are + adopted as Pulumi-managed `projects.Service` resources by `ensureRequiredAPIs` + in `pkg/providers/gcp/provider.go`, so they self-heal against drift but still + need to be reachable on the very first deploy: + ```bash + gcloud services enable storage.googleapis.com # for the Pulumi state bucket below + gcloud services enable cloudresourcemanager.googleapis.com # for the IAM bindings Pulumi creates + gcloud services enable container.googleapis.com # for the GKE cluster + ``` +4. Create the Pulumi service account and grant it the roles required to manage + the deploy. `projectIamAdmin` is needed because the deploy creates IAM bindings + on the project (for the default compute SA). The other four are general + resource management: ```bash gcloud iam service-accounts create pulumi-svc sleep 10 @@ -74,6 +86,7 @@ Pre-requisites: gcloud projects add-iam-policy-binding mcp-registry-prod --member="serviceAccount:pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com" --role="roles/compute.admin" gcloud projects add-iam-policy-binding mcp-registry-prod --member="serviceAccount:pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com" --role="roles/storage.admin" gcloud projects add-iam-policy-binding mcp-registry-prod --member="serviceAccount:pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com" --role="roles/storage.hmacKeyAdmin" + gcloud projects add-iam-policy-binding mcp-registry-prod --member="serviceAccount:pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com" --role="roles/resourcemanager.projectIamAdmin" gcloud iam service-accounts add-iam-policy-binding $(gcloud projects describe mcp-registry-prod --format="value(projectNumber)")-compute@developer.gserviceaccount.com --member="serviceAccount:pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com" --role="roles/iam.serviceAccountUser" gcloud iam service-accounts keys create sa-key.json --iam-account=pulumi-svc@mcp-registry-prod.iam.gserviceaccount.com ``` diff --git a/deploy/pkg/providers/gcp/provider.go b/deploy/pkg/providers/gcp/provider.go index 6a51e27..e4f1e76 100644 --- a/deploy/pkg/providers/gcp/provider.go +++ b/deploy/pkg/providers/gcp/provider.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp" + "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/compute" "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container" - "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/organizations" "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects" "github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" @@ -61,10 +61,75 @@ func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error) return nil, nil } +// ensureRequiredAPIs adopts the GCP APIs the deploy depends on as Pulumi-managed +// services. Encoding them as Pulumi resources makes the dependency explicit and +// protects against drift (an org policy reset or accidental disable). +// +// Note: bootstrap APIs (storage, cloudresourcemanager, container — see +// deploy/README.md) must already be enabled before this runs, because the +// resources that need them are created earlier in the stack. For those, this +// function adopts them into Pulumi state after the fact. For the rest (logging, +// monitoring), the adoption happens before any consumer runs. +// +// DisableOnDestroy and DisableDependentServices are both false — a Pulumi destroy +// or refactor must never disable a shared API that other resources (and humans) +// rely on. Enable-only. +// +// Returns the Cloud Resource Manager API resource specifically, since the node-SA +// IAM bindings need to DependsOn it (SetIamPolicy is gated on CRM). +func ensureRequiredAPIs(ctx *pulumi.Context, projectID string, resourceOpts []pulumi.ResourceOption) (*projects.Service, error) { + // Service Usage API (which projects.NewService itself uses) is enabled by default + // on GCP projects, so we don't need to manage it here. + + // CRM is created explicitly (not in the loop below) because callers need a + // direct reference to it for DependsOn — projects.NewIAMMember calls + // SetIamPolicy under the hood, which is gated on CRM. + crm, err := projects.NewService(ctx, "crm-api", &projects.ServiceArgs{ + Project: pulumi.String(projectID), + Service: pulumi.String("cloudresourcemanager.googleapis.com"), + DisableOnDestroy: pulumi.Bool(false), + DisableDependentServices: pulumi.Bool(false), + }, resourceOpts...) + if err != nil { + return nil, fmt.Errorf("failed to ensure cloudresourcemanager.googleapis.com is enabled: %w", err) + } + + otherAPIs := []struct { + resourceName string + serviceName string + }{ + // Required for compute.GetDefaultServiceAccount and the GKE cluster. + {"compute-api", "compute.googleapis.com"}, + // Required for the GKE cluster. + {"container-api", "container.googleapis.com"}, + // Required for fluentbit-gke to ship container logs. + {"logging-api", "logging.googleapis.com"}, + // Required for the managed Prometheus collector to ship metrics. + {"monitoring-api", "monitoring.googleapis.com"}, + } + + for _, api := range otherAPIs { + _, err := projects.NewService(ctx, api.resourceName, &projects.ServiceArgs{ + Project: pulumi.String(projectID), + Service: pulumi.String(api.serviceName), + DisableOnDestroy: pulumi.Bool(false), + DisableDependentServices: pulumi.Bool(false), + }, resourceOpts...) + if err != nil { + return nil, fmt.Errorf("failed to ensure %s is enabled: %w", api.serviceName, err) + } + } + return crm, nil +} + // grantNodeServiceAccountRoles grants the default compute service account the standard // GKE node roles required for log shipping (fluentbit-gke) and metrics scraping // (managed Prometheus). New GCP projects no longer auto-grant Editor to the default // compute SA, so these have to be set explicitly or every pod log/metric is dropped. +// +// The IAM bindings depend on Cloud Resource Manager API being enabled (managed by +// ensureRequiredAPIs above) — without it, projects.NewIAMMember fails because +// SetIamPolicy lives behind that API. func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProvider *gcp.Provider) error { invokeOpts := []pulumi.InvokeOption{} resourceOpts := []pulumi.ResourceOption{} @@ -73,14 +138,22 @@ func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProv resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider)) } - project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{ - ProjectId: pulumi.StringRef(projectID), - }, invokeOpts...) + crmAPI, err := ensureRequiredAPIs(ctx, projectID, resourceOpts) if err != nil { - return fmt.Errorf("failed to look up project number for node SA bindings: %w", err) + return err } - nodeSA := fmt.Sprintf("serviceAccount:%s-compute@developer.gserviceaccount.com", project.Number) + // Get the default compute SA email directly from the Compute API instead of + // constructing it from the project number — avoids needing CRM API just for + // the project lookup, and matches whatever GCP actually returns. + defaultSA, err := compute.GetDefaultServiceAccount(ctx, &compute.GetDefaultServiceAccountArgs{ + Project: pulumi.StringRef(projectID), + }, invokeOpts...) + if err != nil { + return fmt.Errorf("failed to look up default compute service account: %w", err) + } + + nodeSA := "serviceAccount:" + defaultSA.Email roles := []string{ "roles/logging.logWriter", @@ -89,13 +162,16 @@ func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProv "roles/stackdriver.resourceMetadata.writer", } + bindingOpts := make([]pulumi.ResourceOption, 0, len(resourceOpts)+1) + bindingOpts = append(bindingOpts, resourceOpts...) + bindingOpts = append(bindingOpts, pulumi.DependsOn([]pulumi.Resource{crmAPI})) for _, role := range roles { name := fmt.Sprintf("node-sa-%s", strings.ReplaceAll(strings.TrimPrefix(role, "roles/"), ".", "-")) _, err := projects.NewIAMMember(ctx, name, &projects.IAMMemberArgs{ Project: pulumi.String(projectID), Role: pulumi.String(role), Member: pulumi.String(nodeSA), - }, resourceOpts...) + }, bindingOpts...) if err != nil { return fmt.Errorf("failed to grant %s to node SA: %w", role, err) }