Add OpenTelemetry metrics instrumentation (#290)
## Motivation and Context - It is important to have basic observability in place so that availability issues can be figured out easily ## How Has This Been Tested? - Registration of metrics is tested - Prometheus endpoint `/metrics` is included in test cases to test the metrics exposed - Test locally to understand any regression. ## Breaking Changes - No ## Types of changes <!-- What types of changes does your code introduce? Put an `x` in all the boxes that apply: --> - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update ## Checklist <!-- Go over all the following points, and put an `x` in all the boxes that apply. --> - [x] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [x] My code follows the repository's style guidelines - [x] New and existing tests pass locally - [x] I have added appropriate error handling - [ ] I have added or updated documentation as needed ## Additional context - Telemetry implementation is currently done by keeping metrics in mind as of now, other types of telemetry can be added on top of it later. - Instrumentation takes care of these problems already: - Cardinality control by limiting latency buckets and not including labels that grows with throughput. - Latency buckets are added as per SLA guidelines - Each metric prefixed with `mcp` to easily identify. - Pulling mechanism is used since the idea is to scrape metrics via an agent like vmagent, prometheus agent or otel-collector. - Metrics follow standard approach of defining counters, histogram and gauges as per opentelemetry guidelines.
This commit is contained in:
pree-dew
GitHub
parent
12ab632d32
commit
d237fc90f8
2 changed files with 538 additions and 21 deletions
147
internal/telemetry/metrics.go
Normal file
147
internal/telemetry/metrics.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/contrib/instrumentation/runtime"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/prometheus"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
|
||||
)
|
||||
|
||||
const (
|
||||
Namespace = "mcp_registry"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
// Requests tracks the number of HTTP requests
|
||||
Requests metric.Int64Counter
|
||||
|
||||
// RequestDuration tracks the duration of HTTP Requests
|
||||
RequestDuration metric.Float64Histogram
|
||||
|
||||
// ErrorCount tracks the number of errors
|
||||
ErrorCount metric.Int64Counter
|
||||
|
||||
// Up tracks the health of the service
|
||||
Up metric.Int64Gauge
|
||||
}
|
||||
|
||||
// ShutdownFunc is a delegate that shuts down the OpenTelemetry components.
|
||||
type ShutdownFunc func(ctx context.Context) error
|
||||
|
||||
func NewMetrics(meter metric.Meter) (*Metrics, error) {
|
||||
req, err := meter.Int64Counter(
|
||||
Namespace+".http.requests",
|
||||
metric.WithDescription("Total number of HTTP requests"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request counter: %w", err)
|
||||
}
|
||||
|
||||
reqDuration, err := meter.Float64Histogram(
|
||||
Namespace+".http.request.duration",
|
||||
metric.WithDescription("Duration of HTTP requests in seconds"),
|
||||
metric.WithExplicitBucketBoundaries(
|
||||
0.005, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 20.0, 50.0,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request duration histogram: %w", err)
|
||||
}
|
||||
|
||||
errCount, err := meter.Int64Counter(
|
||||
Namespace+".http.errors",
|
||||
metric.WithDescription("Total number of HTTP errors"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create error counter: %w", err)
|
||||
}
|
||||
|
||||
up, err := meter.Int64Gauge(
|
||||
Namespace+".service.up",
|
||||
metric.WithDescription("Service health status (1 for up, 0 for down)"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create service up gauge: %w", err)
|
||||
}
|
||||
|
||||
return &Metrics{
|
||||
Requests: req,
|
||||
RequestDuration: reqDuration,
|
||||
ErrorCount: errCount,
|
||||
Up: up,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewPrometheusMeterProvider(res *resource.Resource, exp *prometheus.Exporter) (*sdkmetric.MeterProvider, error) {
|
||||
if exp == nil {
|
||||
return nil, errors.New("exporter cannot be nil")
|
||||
}
|
||||
meterProvider := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithResource(res),
|
||||
sdkmetric.WithReader(exp),
|
||||
)
|
||||
|
||||
return meterProvider, nil
|
||||
}
|
||||
|
||||
func InitMetrics(version string) (ShutdownFunc, *Metrics, error) {
|
||||
// Initialized the returned shutdownFunc to no-op.
|
||||
shutdown := func(_ context.Context) error { return nil }
|
||||
|
||||
res, err := resource.New(context.Background(),
|
||||
resource.WithAttributes(
|
||||
semconv.ServiceName(Namespace),
|
||||
semconv.ServiceVersion(version),
|
||||
),
|
||||
resource.WithProcessRuntimeDescription(),
|
||||
)
|
||||
if err != nil {
|
||||
return shutdown, nil, fmt.Errorf("failed to create resource: %w", err)
|
||||
}
|
||||
|
||||
res, err = resource.Merge(resource.Default(), res)
|
||||
if err != nil {
|
||||
return shutdown, nil, fmt.Errorf("failed to merge resources: %w", err)
|
||||
}
|
||||
|
||||
exporter, err := prometheus.New()
|
||||
if err != nil {
|
||||
return shutdown, nil, fmt.Errorf("failed to create Prometheus exporter: %w", err)
|
||||
}
|
||||
|
||||
mp, err := NewPrometheusMeterProvider(res, exporter)
|
||||
if err != nil {
|
||||
return shutdown, nil, fmt.Errorf("failed to create Prometheus meter provider: %w", err)
|
||||
}
|
||||
otel.SetMeterProvider(mp)
|
||||
|
||||
// Update the returned shutdownFunc that calls metric provider
|
||||
// shutdown methods and make sure that a non-nil error is returned
|
||||
// if any returned an error.
|
||||
shutdown = func(ctx context.Context) error {
|
||||
var retErr error
|
||||
if err := mp.Shutdown(ctx); err != nil {
|
||||
retErr = err
|
||||
}
|
||||
return retErr
|
||||
}
|
||||
|
||||
meter := mp.Meter(Namespace, metric.WithSchemaURL(semconv.SchemaURL), metric.WithInstrumentationVersion(runtime.Version()))
|
||||
metrics, err := NewMetrics(meter)
|
||||
return shutdown, metrics, err
|
||||
}
|
||||
|
||||
// PrometheusHandler returns the HTTP handler for Prometheus metrics
|
||||
// This handler serves the metrics endpoint for Prometheus to scrape.
|
||||
func (m *Metrics) PrometheusHandler() http.Handler {
|
||||
return promhttp.Handler()
|
||||
}
|
||||
84
internal/telemetry/metrics_test.go
Normal file
84
internal/telemetry/metrics_test.go
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
package telemetry_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.opentelemetry.io/otel/exporters/prometheus"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/metric/noop"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
|
||||
"github.com/modelcontextprotocol/registry/internal/telemetry"
|
||||
)
|
||||
|
||||
func TestNewMetrics(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meter metric.Meter
|
||||
expectedMetric string
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Valid Meter",
|
||||
meter: noop.NewMeterProvider().Meter("test"),
|
||||
expectedMetric: "http_requests_total",
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
metrics, err := telemetry.NewMetrics(tt.meter)
|
||||
|
||||
// Ensure the metric is registered correctly
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, metrics)
|
||||
assert.NotNil(t, metrics.Requests)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewPrometheusMeterProvider(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
wantErr bool
|
||||
mockPrometheus func() (*prometheus.Exporter, error)
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Successful creation of meter provider",
|
||||
wantErr: false,
|
||||
mockPrometheus: func() (*prometheus.Exporter, error) {
|
||||
return &prometheus.Exporter{
|
||||
Reader: sdkmetric.NewManualReader(),
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Error on resource creation",
|
||||
wantErr: true,
|
||||
expectedError: errors.New("error creating prometheus resource"),
|
||||
mockPrometheus: func() (*prometheus.Exporter, error) {
|
||||
return nil, errors.New("error creating prometheus resource")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
res := resource.NewSchemaless() // Use an empty resource for testing.
|
||||
exp, _ := tt.mockPrometheus()
|
||||
mp, err := telemetry.NewPrometheusMeterProvider(res, exp)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.NotNil(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, mp)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue