464 logs infra (#478)
## Motivation and Context Logs will help in fixing issues quickly, thereby reducing the downtime and MTTD. ## How Has This Been Tested? End to end local setup is tested ## Breaking Changes No ## Types of changes <!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update ## Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. --> - [ ] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [x] My code follows the repository's style guidelines - [x] New and existing tests pass locally - [ ] I have added appropriate error handling - [ ] I have added or updated documentation as needed ## Additional context - It handles container logs for mcp-registry pods. - It supports k8sattributes resources which helps in identifying from where the logs are coming. - Supports victorialogs and it's plugin in grafana. <img width="1129" height="383" alt="Screenshot 2025-09-15 at 1 43 51 AM" src="https://github.com/user-attachments/assets/a560c541-721d-415c-a0a8-8cbe008c3af6" /> <img width="1134" height="703" alt="Screenshot 2025-09-15 at 1 44 34 AM" src="https://github.com/user-attachments/assets/8b5b4132-9fc3-4b8e-8027-27c7787070f1" /> <img width="1134" height="727" alt="Screenshot 2025-09-15 at 1 46 10 AM" src="https://github.com/user-attachments/assets/bcdb4d9d-cea1-4d88-8dd5-fa4cdb51031f" /> --------- Co-authored-by: Radoslav Dimitrov <radoslav@stacklok.com>
This commit is contained in:
Preeti Dewani
GitHub
parent
2280e4da38
commit
6ed625f476
1 changed files with 284 additions and 1 deletions
|
|
@ -105,10 +105,281 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo,
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Deploy VictoriaLogs for log storage
|
||||||
|
err = deployVictoriaLogs(ctx, cluster, ns, environment)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deploy OpenTelemetry Collector DaemonSet
|
||||||
|
err = deployOtelCollectorDaemonSet(ctx, cluster, ns, environment)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Deploy Grafana
|
// Deploy Grafana
|
||||||
return deployGrafana(ctx, cluster, ns, environment, ingressNginx)
|
return deployGrafana(ctx, cluster, ns, environment, ingressNginx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deployVictoriaLogs deploys VictoriaLogs for log storage
|
||||||
|
func deployVictoriaLogs(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string) error {
|
||||||
|
// Deploy VictoriaLogs using Helm chart
|
||||||
|
_, err := helm.NewChart(ctx, "victoria-logs", helm.ChartArgs{
|
||||||
|
Chart: pulumi.String("victoria-logs-single"),
|
||||||
|
Version: pulumi.String("0.11.8"),
|
||||||
|
Namespace: ns.Metadata.Name().Elem(),
|
||||||
|
FetchArgs: helm.FetchArgs{
|
||||||
|
Repo: pulumi.String("https://victoriametrics.github.io/helm-charts/"),
|
||||||
|
},
|
||||||
|
Values: pulumi.Map{
|
||||||
|
"server": pulumi.Map{
|
||||||
|
"retentionPeriod": pulumi.String("15d"),
|
||||||
|
"resources": pulumi.Map{
|
||||||
|
"requests": pulumi.Map{
|
||||||
|
"memory": pulumi.String("256Mi"),
|
||||||
|
"cpu": pulumi.String("100m"),
|
||||||
|
},
|
||||||
|
"limits": pulumi.Map{
|
||||||
|
"memory": pulumi.String("2Gi"),
|
||||||
|
"cpu": pulumi.String("1000m"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"persistence": pulumi.Map{
|
||||||
|
"enabled": pulumi.Bool(true),
|
||||||
|
"size": pulumi.String("20Gi"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, pulumi.Provider(cluster.Provider))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deployOtelCollectorDaemonSet deploys OpenTelemetry Collector using Helm chart
|
||||||
|
func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string) error {
|
||||||
|
// Deploy OpenTelemetry Collector using Helm chart
|
||||||
|
_, err := helm.NewChart(ctx, "opentelemetry-collector", helm.ChartArgs{
|
||||||
|
Chart: pulumi.String("opentelemetry-collector"),
|
||||||
|
Version: pulumi.String("0.133.0"),
|
||||||
|
Namespace: ns.Metadata.Name().Elem(),
|
||||||
|
FetchArgs: helm.FetchArgs{
|
||||||
|
Repo: pulumi.String("https://open-telemetry.github.io/opentelemetry-helm-charts"),
|
||||||
|
},
|
||||||
|
Values: pulumi.Map{
|
||||||
|
"mode": pulumi.String("daemonset"),
|
||||||
|
"image": pulumi.Map{
|
||||||
|
"repository": pulumi.String("otel/opentelemetry-collector-contrib"),
|
||||||
|
"tag": pulumi.String("0.133.0"),
|
||||||
|
},
|
||||||
|
"clusterRole": pulumi.Map{
|
||||||
|
"create": pulumi.Bool(true),
|
||||||
|
"rules": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"apiGroups": pulumi.StringArray{pulumi.String("")},
|
||||||
|
"resources": pulumi.StringArray{
|
||||||
|
pulumi.String("pods"),
|
||||||
|
pulumi.String("pods/log"),
|
||||||
|
pulumi.String("nodes"),
|
||||||
|
pulumi.String("namespaces"),
|
||||||
|
},
|
||||||
|
"verbs": pulumi.StringArray{
|
||||||
|
pulumi.String("get"),
|
||||||
|
pulumi.String("list"),
|
||||||
|
pulumi.String("watch"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"config": pulumi.Map{
|
||||||
|
"receivers": pulumi.Map{
|
||||||
|
"filelog": pulumi.Map{
|
||||||
|
"include": pulumi.StringArray{pulumi.String("/var/log/pods/default_mcp-registry*/*/*.log")},
|
||||||
|
"exclude": pulumi.StringArray{pulumi.String("/var/log/pods/*/*-collector-*/*.log")},
|
||||||
|
"start_at": pulumi.String("end"),
|
||||||
|
"include_file_path": pulumi.Bool(true),
|
||||||
|
"include_file_name": pulumi.Bool(false),
|
||||||
|
"operators": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("regex_parser"),
|
||||||
|
"id": pulumi.String("extract_metadata_from_filepath"),
|
||||||
|
"regex": pulumi.String(`^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log`),
|
||||||
|
"parse_from": pulumi.String("attributes[\"log.file.path\"]"),
|
||||||
|
"cache": pulumi.Map{
|
||||||
|
"size": pulumi.Int(128),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("move"),
|
||||||
|
"from": pulumi.String("attributes.container_name"),
|
||||||
|
"to": pulumi.String("resource[\"k8s.container.name\"]"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("move"),
|
||||||
|
"from": pulumi.String("attributes.namespace"),
|
||||||
|
"to": pulumi.String("resource[\"k8s.namespace.name\"]"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("move"),
|
||||||
|
"from": pulumi.String("attributes.pod_name"),
|
||||||
|
"to": pulumi.String("resource[\"k8s.pod.name\"]"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("move"),
|
||||||
|
"from": pulumi.String("attributes.restart_count"),
|
||||||
|
"to": pulumi.String("resource[\"k8s.container.restart_count\"]"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"type": pulumi.String("move"),
|
||||||
|
"from": pulumi.String("attributes.uid"),
|
||||||
|
"to": pulumi.String("resource[\"k8s.pod.uid\"]"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"processors": pulumi.Map{
|
||||||
|
"batch": pulumi.Map{},
|
||||||
|
"k8sattributes": pulumi.Map{
|
||||||
|
"auth_type": pulumi.String("serviceAccount"),
|
||||||
|
"passthrough": pulumi.Bool(false),
|
||||||
|
"filter": pulumi.Map{
|
||||||
|
"node_from_env_var": pulumi.String("KUBERNETES_NODE_NAME"),
|
||||||
|
},
|
||||||
|
"extract": pulumi.Map{
|
||||||
|
"metadata": pulumi.StringArray{
|
||||||
|
pulumi.String("k8s.pod.name"),
|
||||||
|
pulumi.String("k8s.pod.uid"),
|
||||||
|
pulumi.String("k8s.deployment.name"),
|
||||||
|
pulumi.String("k8s.namespace.name"),
|
||||||
|
pulumi.String("k8s.node.name"),
|
||||||
|
pulumi.String("k8s.pod.start_time"),
|
||||||
|
pulumi.String("k8s.cluster.uid"),
|
||||||
|
},
|
||||||
|
"labels": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"tag_name": pulumi.String("app"),
|
||||||
|
"key": pulumi.String("app"),
|
||||||
|
"from": pulumi.String("pod"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"pod_association": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"sources": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"from": pulumi.String("resource_attribute"),
|
||||||
|
"name": pulumi.String("k8s.pod.name"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"from": pulumi.String("resource_attribute"),
|
||||||
|
"name": pulumi.String("k8s.namespace.name"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"exporters": pulumi.Map{
|
||||||
|
"otlphttp/victorialogs": pulumi.Map{
|
||||||
|
"logs_endpoint": pulumi.String("http://victoria-logs-victoria-logs-single-server:9428/insert/opentelemetry/v1/logs"),
|
||||||
|
"headers": pulumi.Map{
|
||||||
|
"VL-Msg-Field": pulumi.String("body"),
|
||||||
|
"VL-Time-Field": pulumi.String("timestamp"),
|
||||||
|
"VL-Stream-Fields": pulumi.String("k8s.namespace.name,k8s.pod.name,k8s.container.name,log.iostream"),
|
||||||
|
},
|
||||||
|
"timeout": pulumi.String("10s"),
|
||||||
|
"retry_on_failure": pulumi.Map{
|
||||||
|
"enabled": pulumi.Bool(true),
|
||||||
|
"initial_interval": pulumi.String("5s"),
|
||||||
|
"max_interval": pulumi.String("30s"),
|
||||||
|
"max_elapsed_time": pulumi.String("300s"),
|
||||||
|
},
|
||||||
|
"sending_queue": pulumi.Map{
|
||||||
|
"enabled": pulumi.Bool(true),
|
||||||
|
"num_consumers": pulumi.Int(10),
|
||||||
|
"queue_size": pulumi.Int(50),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"service": pulumi.Map{
|
||||||
|
"pipelines": pulumi.Map{
|
||||||
|
"logs": pulumi.Map{
|
||||||
|
"receivers": pulumi.StringArray{pulumi.String("filelog")},
|
||||||
|
"processors": pulumi.StringArray{pulumi.String("batch"), pulumi.String("k8sattributes")},
|
||||||
|
"exporters": pulumi.StringArray{pulumi.String("otlphttp/victorialogs")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"extraVolumes": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"name": pulumi.String("varlogpods"),
|
||||||
|
"hostPath": pulumi.Map{
|
||||||
|
"path": pulumi.String("/var/log/pods"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"name": pulumi.String("varlibdockercontainers"),
|
||||||
|
"hostPath": pulumi.Map{
|
||||||
|
"path": pulumi.String("/var/lib/docker/containers"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"extraVolumeMounts": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"name": pulumi.String("varlogpods"),
|
||||||
|
"mountPath": pulumi.String("/var/log/pods"),
|
||||||
|
"readOnly": pulumi.Bool(true),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"name": pulumi.String("varlibdockercontainers"),
|
||||||
|
"mountPath": pulumi.String("/var/lib/docker/containers"),
|
||||||
|
"readOnly": pulumi.Bool(true),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"extraEnvs": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"name": pulumi.String("KUBERNETES_NODE_NAME"),
|
||||||
|
"valueFrom": pulumi.Map{
|
||||||
|
"fieldRef": pulumi.Map{
|
||||||
|
"fieldPath": pulumi.String("spec.nodeName"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"resources": pulumi.Map{
|
||||||
|
"requests": pulumi.Map{
|
||||||
|
"memory": pulumi.String("200Mi"),
|
||||||
|
"cpu": pulumi.String("100m"),
|
||||||
|
},
|
||||||
|
"limits": pulumi.Map{
|
||||||
|
"memory": pulumi.String("400Mi"),
|
||||||
|
"cpu": pulumi.String("200m"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"tolerations": pulumi.Array{
|
||||||
|
pulumi.Map{
|
||||||
|
"key": pulumi.String("node-role.kubernetes.io/master"),
|
||||||
|
"operator": pulumi.String("Exists"),
|
||||||
|
"effect": pulumi.String("NoSchedule"),
|
||||||
|
},
|
||||||
|
pulumi.Map{
|
||||||
|
"key": pulumi.String("node-role.kubernetes.io/control-plane"),
|
||||||
|
"operator": pulumi.String("Exists"),
|
||||||
|
"effect": pulumi.String("NoSchedule"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, pulumi.Provider(cluster.Provider))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string, ingressNginx *helm.Chart) error {
|
func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string, ingressNginx *helm.Chart) error {
|
||||||
conf := config.New(ctx, "mcp-registry")
|
conf := config.New(ctx, "mcp-registry")
|
||||||
grafanaSecret, err := corev1.NewSecret(ctx, "grafana-secrets", &corev1.SecretArgs{
|
grafanaSecret, err := corev1.NewSecret(ctx, "grafana-secrets", &corev1.SecretArgs{
|
||||||
|
|
@ -149,7 +420,7 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create VictoriaMetrics datasource
|
// Create VictoriaMetrics and VictoriaLogs datasources
|
||||||
datasourcesConfig := map[string]interface{}{
|
datasourcesConfig := map[string]interface{}{
|
||||||
"apiVersion": 1,
|
"apiVersion": 1,
|
||||||
"datasources": []map[string]interface{}{
|
"datasources": []map[string]interface{}{
|
||||||
|
|
@ -160,6 +431,15 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
|
||||||
"access": "proxy",
|
"access": "proxy",
|
||||||
"isDefault": true,
|
"isDefault": true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "VictoriaLogs",
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"url": "http://victoria-logs-victoria-logs-single-server:9428",
|
||||||
|
"access": "proxy",
|
||||||
|
"jsonData": map[string]interface{}{
|
||||||
|
"maxLines": 1000,
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -187,6 +467,9 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor
|
||||||
},
|
},
|
||||||
Namespace: ns.Metadata.Name().Elem(),
|
Namespace: ns.Metadata.Name().Elem(),
|
||||||
Values: pulumi.Map{
|
Values: pulumi.Map{
|
||||||
|
"plugins": pulumi.Array{
|
||||||
|
pulumi.String("victoriametrics-logs-datasource"),
|
||||||
|
},
|
||||||
"extraConfigmapMounts": pulumi.Array{
|
"extraConfigmapMounts": pulumi.Array{
|
||||||
pulumi.Map{
|
pulumi.Map{
|
||||||
"name": pulumi.String("grafana-datasources"),
|
"name": pulumi.String("grafana-datasources"),
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue