fef7d3e2ac<!-- Provide a brief summary of your changes --> ## Motivation and Context <!-- Why is this change needed? What problem does it solve? --> A few changes related to configuring the rate limiting: * Set `externalTrafficPolicy: Local` to preserve real client IPs (otherwise we only see the cluster IPs) * Disable `use-forwarded-headers` (L4 LB doesn't set `X-Forwarded-For`, prevents spoofing) * Scale NGINX to 2 replicas in prod (costs nothing, gives us zero-downtime deploys to prod) ## How Has This Been Tested? <!-- Have you tested this in a real application? Which scenarios were tested? --> ## Breaking Changes <!-- Will users need to update their code or configurations? --> ## Types of changes <!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update ## Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. --> - [ ] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [ ] My code follows the repository's style guidelines - [ ] New and existing tests pass locally - [ ] I have added appropriate error handling - [ ] I have added or updated documentation as needed ## Additional context <!-- Add any other context, implementation notes, or design decisions --> Signed-off-by: Radoslav Dimitrov <radoslav@stacklok.com>
113 lines
4 KiB
Go
113 lines
4 KiB
Go
package k8s
|
|
|
|
import (
|
|
"strings"
|
|
|
|
v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
|
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
|
|
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
|
|
|
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
|
|
)
|
|
|
|
// SetupIngressController sets up the NGINX Ingress Controller
|
|
func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*helm.Chart, error) {
|
|
conf := config.New(ctx, "mcp-registry")
|
|
provider := conf.Get("provider")
|
|
if provider == "" {
|
|
provider = "local"
|
|
}
|
|
|
|
// Create namespace for ingress-nginx
|
|
ingressNginxNamespace, err := v1.NewNamespace(ctx, "ingress-nginx", &v1.NamespaceArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("ingress-nginx"),
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Usually we should expose the ingress to a LoadBalancer
|
|
// This works in GCP and most local setups e.g. minikube (with minikube tunnel)
|
|
// Kind unfortunately does not support LoadBalancer type, and hangs indefinitely. This is a workaround for that.
|
|
serviceType := cluster.Name.ApplyT(func(name string) string {
|
|
if name == "kind-kind" {
|
|
return "NodePort"
|
|
}
|
|
return "LoadBalancer"
|
|
}).(pulumi.StringOutput)
|
|
|
|
// Configure replicas based on environment
|
|
// Staging: 1 replica (sufficient for testing, allows brief downtime during deploys)
|
|
// Production: 2 replicas (HA, zero-downtime deploys, node-level resilience)
|
|
replicaCount := 1
|
|
if environment == "prod" {
|
|
replicaCount = 2
|
|
}
|
|
|
|
// Install NGINX Ingress Controller
|
|
ingressNginx, err := helm.NewChart(ctx, "ingress-nginx", helm.ChartArgs{
|
|
Chart: pulumi.String("ingress-nginx"),
|
|
Version: pulumi.String("4.13.0"),
|
|
FetchArgs: helm.FetchArgs{
|
|
Repo: pulumi.String("https://kubernetes.github.io/ingress-nginx"),
|
|
},
|
|
Namespace: ingressNginxNamespace.Metadata.Name().Elem(),
|
|
Values: pulumi.Map{
|
|
"controller": pulumi.Map{
|
|
"replicaCount": pulumi.Int(replicaCount),
|
|
"service": pulumi.Map{
|
|
"type": serviceType,
|
|
"externalTrafficPolicy": pulumi.String("Local"),
|
|
"annotations": pulumi.Map{},
|
|
},
|
|
"config": pulumi.Map{
|
|
// Disable strict path validation, to work around a bug in ingress-nginx
|
|
// https://cert-manager.io/docs/releases/release-notes/release-notes-1.18/#acme-http01-challenge-paths-now-use-pathtype-exact-in-ingress-routes
|
|
// https://github.com/kubernetes/ingress-nginx/issues/11176
|
|
"strict-validate-path-type": pulumi.String("false"),
|
|
|
|
// Do NOT use forwarded headers with L4 load balancer
|
|
// GCP L4 Passthrough Network Load Balancer does not set X-Forwarded-For
|
|
// Real client IP comes from TCP connection source with externalTrafficPolicy: Local
|
|
"use-forwarded-headers": pulumi.String("false"),
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Extract ingress IPs from the Helm chart's controller service
|
|
ingressIps := ingressNginx.Resources.ApplyT(func(resources interface{}) interface{} {
|
|
// Look for the ingress-nginx-controller service
|
|
resourceMap := resources.(map[string]pulumi.Resource)
|
|
for resourceName, resource := range resourceMap {
|
|
if strings.Contains(resourceName, "ingress-nginx-controller") &&
|
|
!strings.Contains(resourceName, "admission") &&
|
|
strings.Contains(resourceName, "Service") {
|
|
if svc, ok := resource.(*v1.Service); ok {
|
|
// Return the LoadBalancer ingress IPs
|
|
return svc.Status.LoadBalancer().Ingress().ApplyT(func(ingresses []v1.LoadBalancerIngress) []string {
|
|
var ips []string
|
|
for _, ingress := range ingresses {
|
|
if ip := ingress.Ip; ip != nil && *ip != "" {
|
|
ips = append(ips, *ip)
|
|
}
|
|
}
|
|
return ips
|
|
})
|
|
}
|
|
}
|
|
}
|
|
// Return empty array if no matching service found
|
|
return []string{}
|
|
})
|
|
ctx.Export("ingressIps", ingressIps)
|
|
|
|
return ingressNginx, nil
|
|
}
|