Preserve client IPs and bump NGINX replicas (#848)

<!-- Provide a brief summary of your changes -->

## Motivation and Context
<!-- Why is this change needed? What problem does it solve? -->
A few changes related to configuring the rate limiting:
* Set `externalTrafficPolicy: Local` to preserve real client IPs
(otherwise we only see the cluster IPs)
* Disable `use-forwarded-headers` (L4 LB doesn't set `X-Forwarded-For`,
prevents spoofing)
* Scale NGINX to 2 replicas in prod (costs nothing, gives us
zero-downtime deploys to prod)

## How Has This Been Tested?
<!-- Have you tested this in a real application? Which scenarios were
tested? -->

## Breaking Changes
<!-- Will users need to update their code or configurations? -->

## Types of changes
<!-- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update

## Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply. -->
- [ ] I have read the [MCP
Documentation](https://modelcontextprotocol.io)
- [ ] My code follows the repository's style guidelines
- [ ] New and existing tests pass locally
- [ ] I have added appropriate error handling
- [ ] I have added or updated documentation as needed

## Additional context
<!-- Add any other context, implementation notes, or design decisions
-->

Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com>

deploy/pkg/k8s/ingress.go

@@ -40,6 +40,14 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
 		return "LoadBalancer"
 	}).(pulumi.StringOutput)
 
+	// Configure replicas based on environment
+	// Staging: 1 replica (sufficient for testing, allows brief downtime during deploys)
+	// Production: 2 replicas (HA, zero-downtime deploys, node-level resilience)
+	replicaCount := 1
+	if environment == "prod" {
+		replicaCount = 2
+	}
+
 	// Install NGINX Ingress Controller
 	ingressNginx, err := helm.NewChart(ctx, "ingress-nginx", helm.ChartArgs{
 		Chart:   pulumi.String("ingress-nginx"),
@@ -50,9 +58,11 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
 		Namespace: ingressNginxNamespace.Metadata.Name().Elem(),
 		Values: pulumi.Map{
 			"controller": pulumi.Map{
+				"replicaCount": pulumi.Int(replicaCount),
 				"service": pulumi.Map{
-					"type":        serviceType,
-					"annotations": pulumi.Map{},
+					"type":                  serviceType,
+					"externalTrafficPolicy": pulumi.String("Local"),
+					"annotations":           pulumi.Map{},
 				},
 				"config": pulumi.Map{
 					// Disable strict path validation, to work around a bug in ingress-nginx
@@ -60,8 +70,10 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
 					// https://github.com/kubernetes/ingress-nginx/issues/11176
 					"strict-validate-path-type": pulumi.String("false"),
 
-					// Use forwarded headers for proper client IP handling
-					"use-forwarded-headers": pulumi.String("true"),
+					// Do NOT use forwarded headers with L4 load balancer
+					// GCP L4 Passthrough Network Load Balancer does not set X-Forwarded-For
+					// Real client IP comes from TCP connection source with externalTrafficPolicy: Local
+					"use-forwarded-headers": pulumi.String("false"),
 				},
 			},
 		},