Add database backup functionality with GCS integration (#297)
<!-- Provide a brief summary of your changes --> Implements automated database backup functionality with Google Cloud Storage integration, including retention policies and MinIO support for local development. ## Motivation and Context <!-- Why is this change needed? What problem does it solve? --> This change adds critical database backup capabilities to ensure data durability and disaster recovery. The solution provides automated backups with configurable retention policies and supports both production (GCS) and development (MinIO) environments. ## How Has This Been Tested? <!-- Have you tested this in a real application? Which scenarios were tested? --> - Tested backup creation and restoration with MinIO in local development - Verified GCS bucket lifecycle policies for automatic deletion after 60 days - Tested backup retention and cleanup logic ## Breaking Changes <!-- Will users need to update their code or configurations? --> No breaking changes. This is an additive feature that doesn't affect existing functionality. ## Types of changes <!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [x] Documentation update ## Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. --> - [x] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [x] My code follows the repository's style guidelines - [x] New and existing tests pass locally - [x] I have added appropriate error handling - [x] I have added or updated documentation as needed ## Additional context <!-- Add any other context, implementation notes, or design decisions --> - Implements 60-day retention period for backups as a safety net - Uses GCS lifecycle rules for automatic cleanup - Includes MinIO setup instructions for local development testing - Port-forwarding commands updated for consistency across documentation - Fixes #184
This commit is contained in:
adam jones
GitHub
parent
8d7e471fd3
commit
03ac9c955e
1 changed files with 610 additions and 132 deletions
|
|
@ -6,7 +6,10 @@ import (
|
|||
|
||||
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
|
||||
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
|
||||
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
|
||||
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
||||
|
||||
|
|
@ -16,12 +19,51 @@ import (
|
|||
// Provider implements the ClusterProvider interface for Google Kubernetes Engine
|
||||
type Provider struct{}
|
||||
|
||||
// createGCPProvider creates a GCP provider with explicit credentials if configured
|
||||
func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error) {
|
||||
gcpConf := config.New(ctx, "gcp")
|
||||
|
||||
// Get project ID from config
|
||||
projectID := gcpConf.Get("project")
|
||||
if projectID == "" {
|
||||
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
||||
}
|
||||
|
||||
// Get region from config or use default
|
||||
region := gcpConf.Get("region")
|
||||
if region == "" {
|
||||
region = "us-central1"
|
||||
}
|
||||
|
||||
// Get credentials from config (base64 encoded service account JSON)
|
||||
credentials := gcpConf.Get("credentials")
|
||||
if credentials != "" {
|
||||
// Decode the base64 credentials
|
||||
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
|
||||
}
|
||||
credentials = string(decodedCreds)
|
||||
}
|
||||
|
||||
// Create a GCP provider with explicit credentials if provided
|
||||
if credentials != "" {
|
||||
return gcp.NewProvider(ctx, name, &gcp.ProviderArgs{
|
||||
Project: pulumi.String(projectID),
|
||||
Region: pulumi.String(region),
|
||||
Credentials: pulumi.String(credentials),
|
||||
})
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// CreateCluster creates a Google Kubernetes Engine cluster
|
||||
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
|
||||
// Get configuration
|
||||
gcpConf := config.New(ctx, "gcp")
|
||||
|
||||
// Get project ID from config or use default
|
||||
// Get project ID from config
|
||||
projectID := gcpConf.Get("project")
|
||||
if projectID == "" {
|
||||
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
||||
|
|
@ -33,29 +75,10 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov
|
|||
region = "us-central1"
|
||||
}
|
||||
|
||||
// Get credentials from config (base64 encoded service account JSON)
|
||||
credentials := gcpConf.Get("credentials")
|
||||
if credentials != "" {
|
||||
// Decode the base64 credentials
|
||||
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
|
||||
}
|
||||
credentials = string(decodedCreds)
|
||||
}
|
||||
|
||||
// Create a GCP provider with explicit credentials if provided
|
||||
var gcpProvider *gcp.Provider
|
||||
var err error
|
||||
if credentials != "" {
|
||||
gcpProvider, err = gcp.NewProvider(ctx, "gcp-explicit", &gcp.ProviderArgs{
|
||||
Project: pulumi.String(projectID),
|
||||
Region: pulumi.String(region),
|
||||
Credentials: pulumi.String(credentials),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create GCP provider: %w", err)
|
||||
}
|
||||
// Create GCP provider with explicit credentials if configured
|
||||
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create GKE cluster
|
||||
|
|
@ -180,3 +203,102 @@ users:
|
|||
Provider: k8sProvider,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateBackupStorage creates GCS bucket with HMAC credentials for S3-compatible access
|
||||
func (p *Provider) CreateBackupStorage(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*providers.BackupStorageInfo, error) {
|
||||
gcpConf := config.New(ctx, "gcp")
|
||||
projectID := gcpConf.Get("project")
|
||||
if projectID == "" {
|
||||
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
|
||||
}
|
||||
|
||||
// Create GCP provider with explicit credentials if configured
|
||||
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit-backup")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set resource options with provider if we have explicit credentials
|
||||
resourceOpts := []pulumi.ResourceOption{}
|
||||
if gcpProvider != nil {
|
||||
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
|
||||
}
|
||||
|
||||
// Create GCS bucket for backups
|
||||
bucketName := fmt.Sprintf("mcp-registry-%s-backups", environment)
|
||||
bucket, err := storage.NewBucket(ctx, "backup-bucket", &storage.BucketArgs{
|
||||
Name: pulumi.String(bucketName),
|
||||
Location: pulumi.String("US"),
|
||||
StorageClass: pulumi.String("STANDARD"),
|
||||
LifecycleRules: storage.BucketLifecycleRuleArray{
|
||||
&storage.BucketLifecycleRuleArgs{
|
||||
Action: &storage.BucketLifecycleRuleActionArgs{
|
||||
Type: pulumi.String("Delete"),
|
||||
},
|
||||
Condition: &storage.BucketLifecycleRuleConditionArgs{
|
||||
Age: pulumi.Int(60), // Keep backups for 60 days (K8up manages pruning at 28 days, GCS deletion is a safety net)
|
||||
},
|
||||
},
|
||||
},
|
||||
UniformBucketLevelAccess: pulumi.Bool(true),
|
||||
Versioning: &storage.BucketVersioningArgs{
|
||||
Enabled: pulumi.Bool(true),
|
||||
},
|
||||
Labels: pulumi.StringMap{
|
||||
"environment": pulumi.String(environment),
|
||||
"purpose": pulumi.String("k8up-backups"),
|
||||
},
|
||||
}, resourceOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create backup bucket: %w", err)
|
||||
}
|
||||
|
||||
// Use the existing Pulumi service account instead of creating a new one
|
||||
// The service account email is pulumi-svc@<project>.iam.gserviceaccount.com
|
||||
serviceAccountEmail := fmt.Sprintf("pulumi-svc@%s.iam.gserviceaccount.com", projectID)
|
||||
|
||||
// Grant the service account access to the bucket
|
||||
_, err = storage.NewBucketIAMMember(ctx, "backup-bucket-iam", &storage.BucketIAMMemberArgs{
|
||||
Bucket: bucket.Name,
|
||||
Role: pulumi.String("roles/storage.objectAdmin"),
|
||||
Member: pulumi.Sprintf("serviceAccount:%s", pulumi.String(serviceAccountEmail)),
|
||||
}, resourceOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to grant bucket access: %w", err)
|
||||
}
|
||||
|
||||
// Create HMAC key for S3-compatible access
|
||||
hmacKey, err := storage.NewHmacKey(ctx, "backup-hmac-key", &storage.HmacKeyArgs{
|
||||
ServiceAccountEmail: pulumi.String(serviceAccountEmail),
|
||||
Project: pulumi.String(projectID),
|
||||
}, resourceOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create HMAC key: %w", err)
|
||||
}
|
||||
|
||||
// Create Kubernetes secret with S3-compatible credentials
|
||||
backupSecret, err := corev1.NewSecret(ctx, "k8up-backup-credentials", &corev1.SecretArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Name: pulumi.String("k8up-backup-credentials"),
|
||||
Namespace: pulumi.String("default"),
|
||||
Labels: pulumi.StringMap{
|
||||
"k8up.io/backup": pulumi.String("true"),
|
||||
"environment": pulumi.String(environment),
|
||||
},
|
||||
},
|
||||
Type: pulumi.String("Opaque"),
|
||||
StringData: pulumi.StringMap{
|
||||
"AWS_ACCESS_KEY_ID": hmacKey.AccessId,
|
||||
"AWS_SECRET_ACCESS_KEY": hmacKey.Secret,
|
||||
},
|
||||
}, pulumi.Provider(cluster.Provider))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create backup credentials secret: %w", err)
|
||||
}
|
||||
|
||||
return &providers.BackupStorageInfo{
|
||||
Endpoint: "https://storage.googleapis.com",
|
||||
BucketName: bucketName,
|
||||
Credentials: backupSecret,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue