a9ff705d80This includes infra setup for metrics collection and visualisation ## Motivation and Context It is important to have basic observability setup, so that common issues can be identified quickly ## How Has This Been Tested? - Tested on local - Tested on one of the production setup ## Breaking Changes No ## Types of changes <!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> - [ ] Bug fix (non-breaking change which fixes an issue) - [X] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update ## Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. --> - [X] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [X] My code follows the repository's style guidelines - [X] New and existing tests pass locally - [X] I have added appropriate error handling - [ ] I have added or updated documentation as needed ## Additional context - This includes set up of following components: - Victoriametrics single node cluster for storing metrics data with persistent storage configuration for local and production env. - Vmagent with target discovery for mcp-registry pods with scrape interval of 30s. - Grafana for visualising metrics and setting alerts. - Datasource of victoriametrics is pre-configured as config map - Persistent volume for basic configuration - Sqlite for local and Postgres for production setup for storing dashboard, alerts etc details ## Metrics in action <img width="1422" height="666" alt="Screenshot 2025-08-31 at 4 35 28 AM" src="https://github.com/user-attachments/assets/e54ce846-653c-4fa3-aa7a-fe277f0810e6" /> --------- Co-authored-by: Adam Jones <adamj@anthropic.com> Co-authored-by: adam jones <adamj+git@anthropic.com>
329 lines
9.9 KiB
Go
329 lines
9.9 KiB
Go
package k8s
|
|
|
|
import (
|
|
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions"
|
|
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
|
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
|
|
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
|
networkingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
|
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
|
|
"gopkg.in/yaml.v2"
|
|
|
|
"github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers"
|
|
)
|
|
|
|
func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string, ingressNginx *helm.Chart) error {
|
|
// Create namespace
|
|
ns, err := corev1.NewNamespace(ctx, "monitoring", &corev1.NamespaceArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("monitoring"),
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Deploy VictoriaMetrics
|
|
_, err = helm.NewChart(ctx, "victoria-metrics", helm.ChartArgs{
|
|
Chart: pulumi.String("victoria-metrics-single"),
|
|
Version: pulumi.String("0.24.4"),
|
|
Namespace: ns.Metadata.Name().Elem(),
|
|
FetchArgs: helm.FetchArgs{
|
|
Repo: pulumi.String("https://victoriametrics.github.io/helm-charts/"),
|
|
},
|
|
Values: pulumi.Map{
|
|
"server": pulumi.Map{
|
|
"retentionPeriod": pulumi.String("14d"),
|
|
"resources": pulumi.Map{
|
|
"requests": pulumi.Map{
|
|
"memory": pulumi.String("2Gi"),
|
|
"cpu": pulumi.String("500m"),
|
|
},
|
|
"limits": pulumi.Map{
|
|
"memory": pulumi.String("4Gi"),
|
|
"cpu": pulumi.String("1"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Deploy VMAgent
|
|
_, err = helm.NewChart(ctx, "victoria-metrics-agent", helm.ChartArgs{
|
|
Chart: pulumi.String("victoria-metrics-agent"),
|
|
Version: pulumi.String("0.25.3"),
|
|
Namespace: ns.Metadata.Name().Elem(),
|
|
FetchArgs: helm.FetchArgs{
|
|
Repo: pulumi.String("https://victoriametrics.github.io/helm-charts/"),
|
|
},
|
|
Values: pulumi.Map{
|
|
"remoteWrite": pulumi.Array{
|
|
pulumi.Map{
|
|
"url": pulumi.String("http://victoria-metrics-victoria-metrics-single-server:8428/api/v1/write"),
|
|
},
|
|
},
|
|
"config": pulumi.Map{
|
|
"global": pulumi.Map{
|
|
"scrape_interval": pulumi.String("60s"),
|
|
},
|
|
"scrape_configs": pulumi.Array{
|
|
pulumi.Map{
|
|
"job_name": pulumi.String("mcp-registry"),
|
|
"kubernetes_sd_configs": pulumi.Array{
|
|
pulumi.Map{
|
|
"role": pulumi.String("pod"),
|
|
"namespaces": pulumi.Map{
|
|
"names": pulumi.Array{pulumi.String("default")},
|
|
},
|
|
},
|
|
},
|
|
"relabel_configs": pulumi.Array{
|
|
pulumi.Map{
|
|
"source_labels": pulumi.Array{pulumi.String("__meta_kubernetes_pod_label_app")},
|
|
"regex": pulumi.String("mcp-registry.*"),
|
|
"action": pulumi.String("keep"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"resources": pulumi.Map{
|
|
"requests": pulumi.Map{
|
|
"memory": pulumi.String("256Mi"),
|
|
"cpu": pulumi.String("100m"),
|
|
},
|
|
"limits": pulumi.Map{
|
|
"memory": pulumi.String("512Mi"),
|
|
"cpu": pulumi.String("200m"),
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Deploy Grafana
|
|
return deployGrafana(ctx, cluster, ns, environment, ingressNginx)
|
|
}
|
|
|
|
func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string, ingressNginx *helm.Chart) error {
|
|
conf := config.New(ctx, "mcp-registry")
|
|
grafanaSecret, err := corev1.NewSecret(ctx, "grafana-secrets", &corev1.SecretArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("grafana-secrets"),
|
|
Namespace: ns.Metadata.Name(),
|
|
},
|
|
StringData: pulumi.StringMap{
|
|
"GF_AUTH_GOOGLE_CLIENT_SECRET": conf.RequireSecret("googleOauthClientSecret"),
|
|
},
|
|
Type: pulumi.String("Opaque"),
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
grafanaPgCluster, err := apiextensions.NewCustomResource(ctx, "grafana-pg", &apiextensions.CustomResourceArgs{
|
|
ApiVersion: pulumi.String("postgresql.cnpg.io/v1"),
|
|
Kind: pulumi.String("Cluster"),
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("grafana-pg"),
|
|
Namespace: ns.Metadata.Name(),
|
|
Labels: pulumi.StringMap{
|
|
"app": pulumi.String("grafana-pg"),
|
|
"environment": pulumi.String(environment),
|
|
},
|
|
},
|
|
OtherFields: map[string]any{
|
|
"spec": map[string]any{
|
|
"instances": 1,
|
|
"storage": map[string]any{
|
|
"size": "10Gi",
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create VictoriaMetrics datasource
|
|
datasourcesConfig := map[string]interface{}{
|
|
"apiVersion": 1,
|
|
"datasources": []map[string]interface{}{
|
|
{
|
|
"name": "VictoriaMetrics",
|
|
"type": "prometheus",
|
|
"url": "http://victoria-metrics-victoria-metrics-single-server:8428",
|
|
"access": "proxy",
|
|
"isDefault": true,
|
|
},
|
|
},
|
|
}
|
|
|
|
datasourcesConfigYAML, _ := yaml.Marshal(datasourcesConfig)
|
|
grafanaDataSourcesConfigMap, err := corev1.NewConfigMap(ctx, "grafana-datasources", &corev1.ConfigMapArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("grafana-datasources"),
|
|
Namespace: ns.Metadata.Name(),
|
|
},
|
|
Data: pulumi.StringMap{
|
|
"datasources.yaml": pulumi.String(string(datasourcesConfigYAML)),
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Deploy Grafana
|
|
_, err = helm.NewChart(ctx, "grafana", helm.ChartArgs{
|
|
Chart: pulumi.String("grafana"),
|
|
Version: pulumi.String("9.4.4"),
|
|
FetchArgs: &helm.FetchArgs{
|
|
Repo: pulumi.String("https://grafana.github.io/helm-charts"),
|
|
},
|
|
Namespace: ns.Metadata.Name().Elem(),
|
|
Values: pulumi.Map{
|
|
"extraConfigmapMounts": pulumi.Array{
|
|
pulumi.Map{
|
|
"name": pulumi.String("grafana-datasources"),
|
|
"mountPath": pulumi.String("/etc/grafana/provisioning/datasources"),
|
|
"configMap": grafanaDataSourcesConfigMap.Metadata.Name(),
|
|
"readOnly": pulumi.Bool(true),
|
|
},
|
|
},
|
|
"grafana.ini": pulumi.Map{
|
|
"auth": pulumi.Map{
|
|
"disable_login_form": pulumi.Bool(true),
|
|
},
|
|
"auth.basic": pulumi.Map{
|
|
"enabled": pulumi.Bool(false),
|
|
},
|
|
"security": pulumi.Map{
|
|
"disable_initial_admin_creation": pulumi.Bool(true),
|
|
},
|
|
"users": pulumi.Map{
|
|
"auto_assign_org_role": pulumi.String("Admin"),
|
|
},
|
|
"auth.google": pulumi.Map{
|
|
"enabled": pulumi.Bool(true),
|
|
"client_id": pulumi.String("606636202366-tpjm7d5vpp4lp9helg5ld2vrcafnrgh7.apps.googleusercontent.com"),
|
|
"hosted_domain": pulumi.String("modelcontextprotocol.io"),
|
|
"allowed_domains": pulumi.String("modelcontextprotocol.io"),
|
|
"skip_org_role_sync": pulumi.Bool(true),
|
|
},
|
|
"database": pulumi.Map{
|
|
"type": pulumi.String("postgres"),
|
|
"host": pulumi.String("grafana-pg-rw:5432"),
|
|
},
|
|
},
|
|
"envValueFrom": pulumi.Map{
|
|
"GF_AUTH_GOOGLE_CLIENT_SECRET": pulumi.Map{
|
|
"secretKeyRef": pulumi.Map{
|
|
"name": grafanaSecret.Metadata.Name(),
|
|
"key": pulumi.String("GF_AUTH_GOOGLE_CLIENT_SECRET"),
|
|
},
|
|
},
|
|
"GF_DATABASE_USER": pulumi.Map{
|
|
"secretKeyRef": pulumi.Map{
|
|
"name": grafanaPgCluster.Metadata.Name().ApplyT(func(name *string) string {
|
|
if name == nil {
|
|
return "grafana-pg-app"
|
|
}
|
|
return *name + "-app"
|
|
}).(pulumi.StringOutput),
|
|
"key": pulumi.String("username"),
|
|
},
|
|
},
|
|
"GF_DATABASE_PASSWORD": pulumi.Map{
|
|
"secretKeyRef": pulumi.Map{
|
|
"name": grafanaPgCluster.Metadata.Name().ApplyT(func(name *string) string {
|
|
if name == nil {
|
|
return "grafana-pg-app"
|
|
}
|
|
return *name + "-app"
|
|
}).(pulumi.StringOutput),
|
|
"key": pulumi.String("password"),
|
|
},
|
|
},
|
|
"GF_DATABASE_NAME": pulumi.Map{
|
|
"secretKeyRef": pulumi.Map{
|
|
"name": grafanaPgCluster.Metadata.Name().ApplyT(func(name *string) string {
|
|
if name == nil {
|
|
return "grafana-pg-app"
|
|
}
|
|
return *name + "-app"
|
|
}).(pulumi.StringOutput),
|
|
"key": pulumi.String("dbname"),
|
|
},
|
|
},
|
|
},
|
|
"resources": pulumi.Map{
|
|
"requests": pulumi.Map{
|
|
"memory": pulumi.String("512Mi"),
|
|
"cpu": pulumi.String("200m"),
|
|
},
|
|
"limits": pulumi.Map{
|
|
"memory": pulumi.String("1Gi"),
|
|
"cpu": pulumi.String("500m"),
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create ingress for external access
|
|
grafanaHost := "grafana.registry." + environment + ".modelcontextprotocol.io"
|
|
|
|
_, err = networkingv1.NewIngress(ctx, "grafana-ingress", &networkingv1.IngressArgs{
|
|
Metadata: &metav1.ObjectMetaArgs{
|
|
Name: pulumi.String("grafana-ingress"),
|
|
Namespace: ns.Metadata.Name(),
|
|
Annotations: pulumi.StringMap{
|
|
"cert-manager.io/cluster-issuer": pulumi.String("letsencrypt-prod"),
|
|
"kubernetes.io/ingress.class": pulumi.String("nginx"),
|
|
},
|
|
},
|
|
Spec: &networkingv1.IngressSpecArgs{
|
|
Tls: networkingv1.IngressTLSArray{
|
|
&networkingv1.IngressTLSArgs{
|
|
Hosts: pulumi.StringArray{pulumi.String(grafanaHost)},
|
|
SecretName: pulumi.String("grafana-tls"),
|
|
},
|
|
},
|
|
Rules: networkingv1.IngressRuleArray{
|
|
&networkingv1.IngressRuleArgs{
|
|
Host: pulumi.String(grafanaHost),
|
|
Http: &networkingv1.HTTPIngressRuleValueArgs{
|
|
Paths: networkingv1.HTTPIngressPathArray{
|
|
&networkingv1.HTTPIngressPathArgs{
|
|
Path: pulumi.String("/"),
|
|
PathType: pulumi.String("Prefix"),
|
|
Backend: &networkingv1.IngressBackendArgs{
|
|
Service: &networkingv1.IngressServiceBackendArgs{
|
|
Name: pulumi.String("grafana"),
|
|
Port: &networkingv1.ServiceBackendPortArgs{
|
|
Number: pulumi.Int(80),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(ingressNginx.Ready))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx.Export("grafanaUrl", pulumi.Sprintf("https://%s", grafanaHost))
|
|
return nil
|
|
}
|