diff --git a/deploy/pkg/k8s/postgres.go b/deploy/pkg/k8s/postgres.go index 7a119fd..0b5e407 100644 --- a/deploy/pkg/k8s/postgres.go +++ b/deploy/pkg/k8s/postgres.go @@ -59,6 +59,21 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf "storage": map[string]any{ "size": "50Gi", }, + // Explicit resource budget. PG was previously unlimited which made + // node-level OOM behaviour unpredictable. With max_connections=200 + // the worst-case per-connection memory (~10–15MB each) plus + // shared_buffers and overhead lands around 3–4GiB; 4Gi limit gives + // headroom for query workspaces (work_mem) and OS-level effects. + "resources": map[string]any{ + "requests": map[string]any{ + "memory": "512Mi", + "cpu": "200m", + }, + "limits": map[string]any{ + "memory": "4Gi", + "cpu": "1500m", + }, + }, // Enable pg_stat_statements so we can attribute slow time to specific // queries (the 2026-04-27 incident took an EXPLAIN-the-one-query-I-saw // approach because we had no aggregate visibility). CNPG triggers a PG @@ -66,10 +81,16 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf // is brief downtime. CREATE EXTENSION still needs to run once as a // superuser on existing clusters; new clusters get it via the // postInitApplicationSQL hook below. + // + // max_connections raised from default 100 to 200 because the registry + // pgxpool was bumped from 30 to 60 per pod (120 total across 2 pods) + // + ~10 reserved for PG internals and ad-hoc admin = 130, with 70 + // headroom for future scale or burst. "postgresql": map[string]any{ "shared_preload_libraries": []any{"pg_stat_statements"}, "parameters": map[string]any{ "pg_stat_statements.track": "all", + "max_connections": "200", }, }, "bootstrap": map[string]any{ diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 105c570..1b0abdb 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -45,9 +45,13 @@ func NewPostgreSQL(ctx context.Context, connectionURI string) (*PostgreSQL, erro return nil, fmt.Errorf("failed to parse PostgreSQL config: %w", err) } - // Configure pool for stability-focused defaults - config.MaxConns = 30 // Handle good concurrent load - config.MinConns = 5 // Keep connections warm for fast response + // Configure pool for stability-focused defaults. + // MaxConns was 30 per pod (60 total across 2 replicas) which saturated under + // the 2026-04-28 scraper bursts (15 req/s on /v0/servers caused queue blowup + // even though individual queries were fast). 60 per pod gives 120 total, + // leaving 80 of PG max_connections=200 for autovacuum/admin/headroom. + config.MaxConns = 60 // Handle scraper-burst concurrent load + config.MinConns = 10 // Keep connections warm for fast response config.MaxConnIdleTime = 30 * time.Minute // Keep connections available for bursts config.MaxConnLifetime = 2 * time.Hour // Refresh connections regularly for stability