Add database backup functionality with GCS integration (#297)

<!-- Provide a brief summary of your changes -->
Implements automated database backup functionality with Google Cloud
Storage integration, including retention policies and MinIO support for
local development.

## Motivation and Context
<!-- Why is this change needed? What problem does it solve? -->
This change adds critical database backup capabilities to ensure data
durability and disaster recovery. The solution provides automated
backups with configurable retention policies and supports both
production (GCS) and development (MinIO) environments.

## How Has This Been Tested?
<!-- Have you tested this in a real application? Which scenarios were
tested? -->
- Tested backup creation and restoration with MinIO in local development
- Verified GCS bucket lifecycle policies for automatic deletion after 60
days
- Tested backup retention and cleanup logic

## Breaking Changes
<!-- Will users need to update their code or configurations? -->
No breaking changes. This is an additive feature that doesn't affect
existing functionality.

## Types of changes
<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [x] Documentation update

## Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply. -->
- [x] I have read the [MCP
Documentation](https://modelcontextprotocol.io)
- [x] My code follows the repository's style guidelines
- [x] New and existing tests pass locally
- [x] I have added appropriate error handling
- [x] I have added or updated documentation as needed

## Additional context
<!-- Add any other context, implementation notes, or design decisions
-->
- Implements 60-day retention period for backups as a safety net
- Uses GCS lifecycle rules for automatic cleanup
- Includes MinIO setup instructions for local development testing
- Port-forwarding commands updated for consistency across documentation
- Fixes #184
This commit is contained in:
adam jones 2025-08-25 17:27:40 +01:00 • committed by GitHub
commit 03ac9c955e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 610 additions and 132 deletions

View file

@ -1,99 +0,0 @@
package aks
import (
"encoding/base64"
"fmt"
"github.com/pulumi/pulumi-azure-native-sdk/containerservice"
"github.com/pulumi/pulumi-azure-native-sdk/resources"
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
)
// Provider implements the ClusterProvider interface for Azure Kubernetes Service
type Provider struct{}
// CreateCluster creates an Azure Kubernetes Service cluster
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
// Get resource group name from config or use default
conf := config.New(ctx, "mcp-registry")
resourceGroupName := conf.Get("resourceGroupName")
if resourceGroupName == "" {
resourceGroupName = "official-mcp-registry-prod"
}
// Get existing resource group
resourceGroup, err := resources.LookupResourceGroup(ctx, &resources.LookupResourceGroupArgs{
ResourceGroupName: resourceGroupName,
})
if err != nil {
return nil, fmt.Errorf("failed to find resource group '%s': %w", resourceGroupName, err)
}
// Create AKS cluster
clusterName := fmt.Sprintf("mcp-registry-%s", environment)
cluster, err := containerservice.NewManagedCluster(ctx, clusterName, &containerservice.ManagedClusterArgs{
ResourceGroupName: pulumi.String(resourceGroup.Name),
Location: pulumi.String(resourceGroup.Location),
DnsPrefix: pulumi.String(clusterName),
KubernetesVersion: pulumi.String("1.33.2"),
Identity: &containerservice.ManagedClusterIdentityArgs{
Type: containerservice.ResourceIdentityTypeSystemAssigned,
},
AgentPoolProfiles: containerservice.ManagedClusterAgentPoolProfileArray{
&containerservice.ManagedClusterAgentPoolProfileArgs{
Name: pulumi.String("nodepool1"),
Count: pulumi.Int(2),
VmSize: pulumi.String("Standard_B2s"),
OsType: pulumi.String("Linux"),
Mode: pulumi.String("System"),
},
},
NetworkProfile: &containerservice.ContainerServiceNetworkProfileArgs{
NetworkPlugin: pulumi.String("azure"),
ServiceCidr: pulumi.String("10.0.0.0/16"),
DnsServiceIP: pulumi.String("10.0.0.10"),
},
})
if err != nil {
return nil, err
}
// Get AKS credentials
creds := pulumi.All(cluster.Name, pulumi.String(resourceGroup.Name)).ApplyT(
func(args []any) (string, error) {
clusterName := args[0].(string)
rgName := args[1].(string)
credentials, err := containerservice.ListManagedClusterUserCredentials(ctx, &containerservice.ListManagedClusterUserCredentialsArgs{
ResourceGroupName: rgName,
ResourceName: clusterName,
})
if err != nil {
return "", err
}
// Decode base64 kubeconfig
kubeconfigData, err := base64.StdEncoding.DecodeString(credentials.Kubeconfigs[0].Value)
if err != nil {
return "", fmt.Errorf("failed to decode kubeconfig: %w", err)
}
return string(kubeconfigData), nil
},
).(pulumi.StringOutput)
// Create Kubernetes provider for AKS
k8sProvider, err := kubernetes.NewProvider(ctx, "k8s-provider", &kubernetes.ProviderArgs{
Kubeconfig: creds,
})
if err != nil {
return nil, err
}
return &providers.ProviderInfo{
Name: cluster.Name,
Provider: k8sProvider,
}, nil
}

View file

@ -6,7 +6,10 @@ import (
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
@ -16,12 +19,51 @@ import (
// Provider implements the ClusterProvider interface for Google Kubernetes Engine
type Provider struct{}
// createGCPProvider creates a GCP provider with explicit credentials if configured
func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error) {
gcpConf := config.New(ctx, "gcp")
// Get project ID from config
projectID := gcpConf.Get("project")
if projectID == "" {
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
}
// Get region from config or use default
region := gcpConf.Get("region")
if region == "" {
region = "us-central1"
}
// Get credentials from config (base64 encoded service account JSON)
credentials := gcpConf.Get("credentials")
if credentials != "" {
// Decode the base64 credentials
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
if err != nil {
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
}
credentials = string(decodedCreds)
}
// Create a GCP provider with explicit credentials if provided
if credentials != "" {
return gcp.NewProvider(ctx, name, &gcp.ProviderArgs{
Project: pulumi.String(projectID),
Region: pulumi.String(region),
Credentials: pulumi.String(credentials),
})
}
return nil, nil
}
// CreateCluster creates a Google Kubernetes Engine cluster
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
// Get configuration
gcpConf := config.New(ctx, "gcp")
// Get project ID from config or use default
// Get project ID from config
projectID := gcpConf.Get("project")
if projectID == "" {
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
@ -33,29 +75,10 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov
region = "us-central1"
}
// Get credentials from config (base64 encoded service account JSON)
credentials := gcpConf.Get("credentials")
if credentials != "" {
// Decode the base64 credentials
decodedCreds, err := base64.StdEncoding.DecodeString(credentials)
if err != nil {
return nil, fmt.Errorf("failed to decode GCP credentials: %w", err)
}
credentials = string(decodedCreds)
}
// Create a GCP provider with explicit credentials if provided
var gcpProvider *gcp.Provider
var err error
if credentials != "" {
gcpProvider, err = gcp.NewProvider(ctx, "gcp-explicit", &gcp.ProviderArgs{
Project: pulumi.String(projectID),
Region: pulumi.String(region),
Credentials: pulumi.String(credentials),
})
if err != nil {
return nil, fmt.Errorf("failed to create GCP provider: %w", err)
}
// Create GCP provider with explicit credentials if configured
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit")
if err != nil {
return nil, err
}
// Create GKE cluster
@ -180,3 +203,102 @@ users:
Provider: k8sProvider,
}, nil
}
// CreateBackupStorage creates GCS bucket with HMAC credentials for S3-compatible access
func (p *Provider) CreateBackupStorage(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*providers.BackupStorageInfo, error) {
gcpConf := config.New(ctx, "gcp")
projectID := gcpConf.Get("project")
if projectID == "" {
return nil, fmt.Errorf("GCP project ID not configured. Set gcp:project")
}
// Create GCP provider with explicit credentials if configured
gcpProvider, err := createGCPProvider(ctx, "gcp-explicit-backup")
if err != nil {
return nil, err
}
// Set resource options with provider if we have explicit credentials
resourceOpts := []pulumi.ResourceOption{}
if gcpProvider != nil {
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
}
// Create GCS bucket for backups
bucketName := fmt.Sprintf("mcp-registry-%s-backups", environment)
bucket, err := storage.NewBucket(ctx, "backup-bucket", &storage.BucketArgs{
Name: pulumi.String(bucketName),
Location: pulumi.String("US"),
StorageClass: pulumi.String("STANDARD"),
LifecycleRules: storage.BucketLifecycleRuleArray{
&storage.BucketLifecycleRuleArgs{
Action: &storage.BucketLifecycleRuleActionArgs{
Type: pulumi.String("Delete"),
},
Condition: &storage.BucketLifecycleRuleConditionArgs{
Age: pulumi.Int(60), // Keep backups for 60 days (K8up manages pruning at 28 days, GCS deletion is a safety net)
},
},
},
UniformBucketLevelAccess: pulumi.Bool(true),
Versioning: &storage.BucketVersioningArgs{
Enabled: pulumi.Bool(true),
},
Labels: pulumi.StringMap{
"environment": pulumi.String(environment),
"purpose": pulumi.String("k8up-backups"),
},
}, resourceOpts...)
if err != nil {
return nil, fmt.Errorf("failed to create backup bucket: %w", err)
}
// Use the existing Pulumi service account instead of creating a new one
// The service account email is pulumi-svc@<project>.iam.gserviceaccount.com
serviceAccountEmail := fmt.Sprintf("pulumi-svc@%s.iam.gserviceaccount.com", projectID)
// Grant the service account access to the bucket
_, err = storage.NewBucketIAMMember(ctx, "backup-bucket-iam", &storage.BucketIAMMemberArgs{
Bucket: bucket.Name,
Role: pulumi.String("roles/storage.objectAdmin"),
Member: pulumi.Sprintf("serviceAccount:%s", pulumi.String(serviceAccountEmail)),
}, resourceOpts...)
if err != nil {
return nil, fmt.Errorf("failed to grant bucket access: %w", err)
}
// Create HMAC key for S3-compatible access
hmacKey, err := storage.NewHmacKey(ctx, "backup-hmac-key", &storage.HmacKeyArgs{
ServiceAccountEmail: pulumi.String(serviceAccountEmail),
Project: pulumi.String(projectID),
}, resourceOpts...)
if err != nil {
return nil, fmt.Errorf("failed to create HMAC key: %w", err)
}
// Create Kubernetes secret with S3-compatible credentials
backupSecret, err := corev1.NewSecret(ctx, "k8up-backup-credentials", &corev1.SecretArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("k8up-backup-credentials"),
Namespace: pulumi.String("default"),
Labels: pulumi.StringMap{
"k8up.io/backup": pulumi.String("true"),
"environment": pulumi.String(environment),
},
},
Type: pulumi.String("Opaque"),
StringData: pulumi.StringMap{
"AWS_ACCESS_KEY_ID": hmacKey.AccessId,
"AWS_SECRET_ACCESS_KEY": hmacKey.Secret,
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create backup credentials secret: %w", err)
}
return &providers.BackupStorageInfo{
Endpoint: "https://storage.googleapis.com",
BucketName: bucketName,
Credentials: backupSecret,
}, nil
}

View file

@ -1,7 +1,15 @@
package local
import (
"fmt"
"os/exec"
"strings"
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1"
batchv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/batch/v1"
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
@ -10,6 +18,17 @@ import (
// Provider implements the ClusterProvider interface for local Kubernetes clusters
type Provider struct{}
// getClusterName attempts to get the current Kubernetes cluster name from kubectl config
func getClusterName() string {
cmd := exec.Command("kubectl", "config", "current-context")
output, err := cmd.Output()
if err != nil {
// Fallback to default if we can't get the context
return "local"
}
return strings.TrimSpace(string(output))
}
// CreateCluster configures access to a local Kubernetes cluster via kubeconfig
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
// Create Kubernetes provider for local cluster
@ -18,8 +37,237 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov
return nil, err
}
// Get the actual cluster name from kubectl config
clusterName := getClusterName()
return &providers.ProviderInfo{
Name: pulumi.String("local").ToStringOutput(),
Name: pulumi.String(clusterName).ToStringOutput(),
Provider: k8sProvider,
}, nil
}
// CreateBackupStorage creates MinIO for S3-compatible backup storage
func (p *Provider) CreateBackupStorage(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*providers.BackupStorageInfo, error) {
// Create MinIO namespace
minioNamespace, err := corev1.NewNamespace(ctx, "minio", &corev1.NamespaceArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio"),
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create MinIO namespace: %w", err)
}
// Create MinIO credentials secret
minioSecret, err := corev1.NewSecret(ctx, "minio-credentials", &corev1.SecretArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio-credentials"),
Namespace: minioNamespace.Metadata.Name(),
},
Type: pulumi.String("Opaque"),
StringData: pulumi.StringMap{
"accesskey": pulumi.String("minioadmin"),
"secretkey": pulumi.String("minioadmin"),
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create MinIO credentials: %w", err)
}
// Create MinIO PVC for data storage
_, err = corev1.NewPersistentVolumeClaim(ctx, "minio-pvc", &corev1.PersistentVolumeClaimArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio-pvc"),
Namespace: minioNamespace.Metadata.Name(),
},
Spec: &corev1.PersistentVolumeClaimSpecArgs{
AccessModes: pulumi.StringArray{
pulumi.String("ReadWriteOnce"),
},
Resources: &corev1.VolumeResourceRequirementsArgs{
Requests: pulumi.StringMap{
"storage": pulumi.String("10Gi"),
},
},
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create MinIO PVC: %w", err)
}
// Deploy MinIO
minioDeployment, err := appsv1.NewDeployment(ctx, "minio", &appsv1.DeploymentArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio"),
Namespace: minioNamespace.Metadata.Name(),
},
Spec: &appsv1.DeploymentSpecArgs{
Replicas: pulumi.Int(1),
Selector: &metav1.LabelSelectorArgs{
MatchLabels: pulumi.StringMap{
"app": pulumi.String("minio"),
},
},
Template: &corev1.PodTemplateSpecArgs{
Metadata: &metav1.ObjectMetaArgs{
Labels: pulumi.StringMap{
"app": pulumi.String("minio"),
},
},
Spec: &corev1.PodSpecArgs{
Containers: corev1.ContainerArray{
&corev1.ContainerArgs{
Name: pulumi.String("minio"),
Image: pulumi.String("minio/minio:RELEASE.2025-07-23T15-54-02Z"),
Args: pulumi.StringArray{
pulumi.String("server"),
pulumi.String("/data"),
pulumi.String("--console-address"),
pulumi.String(":9001"),
},
Env: corev1.EnvVarArray{
&corev1.EnvVarArgs{
Name: pulumi.String("MINIO_ROOT_USER"),
ValueFrom: &corev1.EnvVarSourceArgs{
SecretKeyRef: &corev1.SecretKeySelectorArgs{
Name: minioSecret.Metadata.Name(),
Key: pulumi.String("accesskey"),
},
},
},
&corev1.EnvVarArgs{
Name: pulumi.String("MINIO_ROOT_PASSWORD"),
ValueFrom: &corev1.EnvVarSourceArgs{
SecretKeyRef: &corev1.SecretKeySelectorArgs{
Name: minioSecret.Metadata.Name(),
Key: pulumi.String("secretkey"),
},
},
},
},
Ports: corev1.ContainerPortArray{
&corev1.ContainerPortArgs{
ContainerPort: pulumi.Int(9000),
Name: pulumi.String("api"),
},
&corev1.ContainerPortArgs{
ContainerPort: pulumi.Int(9001),
Name: pulumi.String("console"),
},
},
VolumeMounts: corev1.VolumeMountArray{
&corev1.VolumeMountArgs{
Name: pulumi.String("data"),
MountPath: pulumi.String("/data"),
},
},
},
},
Volumes: corev1.VolumeArray{
&corev1.VolumeArgs{
Name: pulumi.String("data"),
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSourceArgs{
ClaimName: pulumi.String("minio-pvc"),
},
},
},
},
},
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create MinIO deployment: %w", err)
}
// Create MinIO service
minioService, err := corev1.NewService(ctx, "minio-service", &corev1.ServiceArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio"),
Namespace: minioNamespace.Metadata.Name(),
},
Spec: &corev1.ServiceSpecArgs{
Selector: pulumi.StringMap{
"app": pulumi.String("minio"),
},
Ports: corev1.ServicePortArray{
&corev1.ServicePortArgs{
Name: pulumi.String("api"),
Port: pulumi.Int(9000),
TargetPort: pulumi.Int(9000),
},
&corev1.ServicePortArgs{
Name: pulumi.String("console"),
Port: pulumi.Int(9001),
TargetPort: pulumi.Int(9001),
},
},
},
}, pulumi.Provider(cluster.Provider))
if err != nil {
return nil, fmt.Errorf("failed to create MinIO service: %w", err)
}
// Create a Job to initialize the MinIO bucket
_, err = batchv1.NewJob(ctx, "minio-bucket-init", &batchv1.JobArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("minio-bucket-init"),
Namespace: minioNamespace.Metadata.Name(),
},
Spec: &batchv1.JobSpecArgs{
Template: &corev1.PodTemplateSpecArgs{
Spec: &corev1.PodSpecArgs{
RestartPolicy: pulumi.String("OnFailure"),
Containers: corev1.ContainerArray{
&corev1.ContainerArgs{
Name: pulumi.String("mc"),
Image: pulumi.String("minio/mc:RELEASE.2025-07-19T03-52-40Z"),
Command: pulumi.StringArray{
pulumi.String("/bin/sh"),
pulumi.String("-c"),
},
Args: pulumi.StringArray{
pulumi.String(`
until mc alias set myminio http://minio:9000 minioadmin minioadmin; do
echo 'Waiting for MinIO to be ready...'
sleep 5
done
mc mb -p myminio/k8up-backups
echo 'Bucket created successfully'
`),
},
},
},
},
},
},
}, pulumi.Provider(cluster.Provider), pulumi.DependsOn([]pulumi.Resource{minioService}))
if err != nil {
return nil, fmt.Errorf("failed to create bucket initialization job: %w", err)
}
// Create backup credentials secret for S3 access
backupSecret, err := corev1.NewSecret(ctx, "k8up-backup-credentials", &corev1.SecretArgs{
Metadata: &metav1.ObjectMetaArgs{
Name: pulumi.String("k8up-backup-credentials"),
Namespace: pulumi.String("default"),
Labels: pulumi.StringMap{
"k8up.io/backup": pulumi.String("true"),
"environment": pulumi.String(environment),
},
},
Type: pulumi.String("Opaque"),
StringData: pulumi.StringMap{
"AWS_ACCESS_KEY_ID": pulumi.String("minioadmin"),
"AWS_SECRET_ACCESS_KEY": pulumi.String("minioadmin"),
},
}, pulumi.Provider(cluster.Provider), pulumi.DependsOn([]pulumi.Resource{minioDeployment}))
if err != nil {
return nil, fmt.Errorf("failed to create backup credentials secret: %w", err)
}
return &providers.BackupStorageInfo{
Endpoint: "http://minio.minio:9000",
BucketName: "k8up-backups",
Credentials: backupSecret,
}, nil
}

View file

@ -2,6 +2,7 @@ package providers
import (
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
@ -11,8 +12,18 @@ type ProviderInfo struct {
Provider *kubernetes.Provider
}
// BackupStorageInfo represents S3-compatible backup storage configuration
type BackupStorageInfo struct {
Endpoint string // S3 endpoint (e.g., https://storage.googleapis.com or http://minio:9000)
BucketName string
Credentials *corev1.Secret // Should contain AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
}
// ClusterProvider defines the interface that all cluster providers must implement
type ClusterProvider interface {
// CreateCluster creates a Kubernetes cluster and returns provider info
CreateCluster(ctx *pulumi.Context, environment string) (*ProviderInfo, error)
// CreateBackupStorage creates backup storage infrastructure
CreateBackupStorage(ctx *pulumi.Context, cluster *ProviderInfo, environment string) (*BackupStorageInfo, error)
}