Custom Metric Attributes

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/luraproject/lura/v2 v2.6.2
 	github.com/prometheus/client_golang v1.18.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
 	go.opentelemetry.io/otel v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0
@@ -24,6 +25,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect

go.sum

@@ -13,6 +13,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583j
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -93,6 +95,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
 go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0 h1:f2jriWfOdldanBwS9jNBdeOKAQN7b4ugAMaNu1/1k9g=

http/attributes.go

@@ -1,6 +1,8 @@
 package http
 
 import (
+	"context"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"net/http"
 
 	"go.opentelemetry.io/otel/attribute"
@@ -43,3 +45,33 @@ func TraceResponseAttrs(resp *http.Response) []attribute.KeyValue {
 		semconv.HTTPResponseBodySize(int(resp.ContentLength)),
 	}
 }
+
+type customLabelerType int
+
+const customLabelerKey customLabelerType = 0
+
+// Labeler For a server plugin wanting to add custom metric attributes
+func Labeler(ctx context.Context) (*otelhttp.Labeler, bool) {
+	l, ok := ctx.Value(customLabelerKey).(*otelhttp.Labeler)
+	if !ok {
+		l = &otelhttp.Labeler{}
+	}
+	return l, ok
+}
+
+// InjectLabeler Set up the labeler in the context
+func InjectLabeler(ctx context.Context) context.Context {
+	labeler, found := Labeler(ctx)
+
+	if !found {
+		ctx = context.WithValue(ctx, customLabelerKey, labeler)
+	}
+
+	return ctx
+}
+
+// CustomMetricAttributes Get custom attributes set up for this request
+func CustomMetricAttributes(r *http.Request) []attribute.KeyValue {
+	labeler, _ := Labeler(r.Context())
+	return labeler.Get()
+}

http/client/client_test.go

@@ -12,6 +12,7 @@ import (
 
 	// "go.opentelemetry.io/otel/codes"
 
+	otelhttp "github.com/krakend/krakend-otel/http"
 	"go.opentelemetry.io/otel/metric"
 	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/metric/metricdata"
@@ -223,3 +224,154 @@ func TestInstrumentedHTTPClient(t *testing.T) {
 		return
 	}
 }
+
+func TestCustomLabels(t *testing.T) {
+	server, otelInstance, c, done := setupTestEnvironment(t)
+
+	if done {
+		return
+	}
+
+	req, err := http.NewRequest("GET", server.URL, nil)
+	if err != nil {
+		t.Errorf("unexpected error creating request: %s", err.Error())
+		return
+	}
+
+	req = req.WithContext(otelhttp.InjectLabeler(req.Context()))
+	labeler, _ := otelhttp.Labeler(req.Context())
+
+	labeler.Add(attribute.String("testkey", "testvalue"))
+
+	resp, err := c.Do(req)
+	if err != nil {
+		t.Errorf("unexpected client error: %s", err.Error())
+		return
+	}
+
+	if resp == nil {
+		t.Errorf("nil response")
+		return
+	}
+
+	b, _ := io.ReadAll(resp.Body)
+	if len(b) == 0 {
+		t.Errorf("no bytes read")
+		return
+	}
+
+	mdata := metricdata.ResourceMetrics{}
+	err = otelInstance.metricReader.Collect(context.Background(), &mdata)
+	if err != nil {
+		t.Errorf("cannot collect the recorded metrics")
+		return
+	}
+
+	if len(mdata.ScopeMetrics) != 1 {
+		t.Errorf("wrong amount of metrics, want: 1, got: %d", len(mdata.ScopeMetrics))
+		for idx, sm := range mdata.ScopeMetrics {
+			t.Errorf("%d -> %#v", idx, sm)
+		}
+		return
+	}
+
+	// --> check that we have all the metrics we want to report
+	sm := mdata.ScopeMetrics[0]
+	wantedMetrics := map[string]bool{
+		// "requests-failed-count": false // <- we do not have requests that failed
+		// "requests-canceled-count": false // <- we do not have requests cancelled
+		// "requests-timedout-count": false // <- we do not have requests timed out
+		"http.client.request.started.count":   false,
+		"http.client.request.size":            false,
+		"http.client.duration":                false,
+		"http.client.response.size":           false,
+		"http.client.response.read.size":      false,
+		"http.client.response.read.size-hist": false,
+		"http.client.response.read.time":      false,
+		"http.client.response.read.time-hist": false,
+		// "reader-errors":    false,
+	}
+	numWantedMetrics := len(wantedMetrics)
+	gotMetrics := make(map[string]metricdata.Metrics, numWantedMetrics)
+	for _, m := range sm.Metrics {
+		gotMetrics[m.Name] = m
+	}
+	for k := range wantedMetrics {
+		if _, ok := gotMetrics[k]; !ok {
+			t.Errorf("missing metric %s", k)
+			return
+		}
+	}
+	// check that we do not have not expected metrics:
+	for k := range gotMetrics {
+		if _, ok := wantedMetrics[k]; !ok {
+			t.Errorf("got unexpected metric %s", k)
+			return
+		}
+	}
+
+	verifyCustomAttributesOnMetrics(t, gotMetrics)
+}
+
+func setupTestEnvironment(t *testing.T) (*httptest.Server, *testOTEL, *http.Client, bool) {
+	svc := &fakeService{}
+	server := httptest.NewServer(svc)
+
+	innerClient := &http.Client{}
+	otelInstance := newTestOTEL()
+	transportOptions := &TransportOptions{
+		OTELInstance: otelInstance,
+		TracesOpts: TransportTracesOptions{
+			RoundTrip:   true,
+			ReadPayload: true,
+			FixedAttributes: []attribute.KeyValue{
+				attribute.String("test-trace-attr", "a_trace"),
+			},
+		},
+		MetricsOpts: TransportMetricsOptions{
+			RoundTrip:   true,
+			ReadPayload: true,
+			FixedAttributes: []attribute.KeyValue{
+				attribute.String("test-metric-attr", "a_metric"),
+			},
+		},
+	}
+
+	c := InstrumentedHTTPClient(innerClient, transportOptions, "test-http-client")
+	if c == nil {
+		t.Error("unable to create client")
+		return nil, nil, nil, true
+	}
+	return server, otelInstance, c, false
+}
+
+func verifyCustomAttributesOnMetrics(t *testing.T, gotMetrics map[string]metricdata.Metrics) {
+	readSize := gotMetrics["http.client.request.started.count"]
+	readSizeSum, ok := readSize.Data.(metricdata.Sum[int64])
+	if !ok {
+		t.Errorf("cannot access read size aggregation: %#v", readSize.Data)
+		return
+	}
+	if len(readSizeSum.DataPoints) != 1 {
+		t.Errorf("read sum data points, want: 1, got: %d", len(readSizeSum.DataPoints))
+		return
+	}
+	dp := readSizeSum.DataPoints[0]
+
+	if dp.Attributes.Len() != 5 {
+		t.Errorf("missing attributes, want 5, got: %d\n%#v", dp.Attributes.Len(), dp.Attributes)
+		return
+	}
+
+	var found = false
+	for i := 0; i < dp.Attributes.Len(); i++ {
+		value, foundItem := dp.Attributes.Get(i)
+		if foundItem == true && string(value.Key) == "testkey" && value.Value.AsString() == "testvalue" {
+			found = true
+		}
+	}
+
+	if !found {
+		t.Errorf("Expected metric to have label testkey with value testvalue but the metric had no such label")
+	}
+}

http/client/transport.go

@@ -12,6 +12,7 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/trace"
 
+	otelhttp "github.com/krakend/krakend-otel/http"
 	otelio "github.com/krakend/krakend-otel/io"
 	"github.com/krakend/krakend-otel/state"
 )
@@ -133,7 +134,7 @@ func newTransport(base http.RoundTripper, metricsOpts TransportMetricsOptions,
 		otelState:     otelState,
 		tracesOpts:    tracesOpts,
 		metricsOpts:   metricsOpts,
-		metrics:       newTransportMetrics(&metricsOpts, meter, clientName),
+		metrics:       newTransportMetrics(&metricsOpts, meter),
 		traces:        newTransportTraces(&tracesOpts, tracer, clientName),
 		readerWrapper: readWrapperBuilder(&metricsOpts, &tracesOpts, meter, tracer),
 	}
@@ -154,7 +155,8 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
 	rtt.resp, rtt.err = t.base.RoundTrip(rtt.req)
 	rtt.latencyInSecs = float64(time.Since(requestSentAt)) / float64(time.Second)
 
-	t.metrics.report(&rtt, t.metricsOpts.FixedAttributes)
+	attributes := append(otelhttp.CustomMetricAttributes(req), t.metricsOpts.FixedAttributes...)
+	t.metrics.report(&rtt, attributes)
 
 	if rtt.resp != nil && rtt.resp.Body != nil {
 		rtt.resp.Body = t.readerWrapper(rtt.resp.Body, rtt.req.Context())

http/client/transport_metrics.go

@@ -9,6 +9,7 @@ import (
 	"go.opentelemetry.io/otel/semconv/v1.21.0"
 
 	kotelconfig "github.com/krakend/krakend-otel/config"
+	otelhttp "github.com/krakend/krakend-otel/http"
 )
 
 // TransportMetricsOptions contains the options to enable / disable
@@ -56,7 +57,7 @@ type transportMetrics struct {
 	clientName string
 }
 
-func newTransportMetrics(metricsOpts *TransportMetricsOptions, meter metric.Meter, clientName string) *transportMetrics {
+func newTransportMetrics(metricsOpts *TransportMetricsOptions, meter metric.Meter) *transportMetrics {
 	if meter == nil {
 		return nil
 	}
@@ -86,14 +87,17 @@ func (m *transportMetrics) report(rtt *roundTripTracking, attrs []attribute.KeyV
 		return
 	}
 
-	attrM := make([]attribute.KeyValue, len(attrs), len(attrs)+4)
+	customAttributes := otelhttp.CustomMetricAttributes(rtt.req)
+	attrM := make([]attribute.KeyValue, len(attrs), len(attrs)+4+len(customAttributes))
+
 	copy(attrM, attrs)
 	if len(m.clientName) > 0 {
 		attrM = append(attrM, attribute.Key("clientname").String(m.clientName))
 	}
 	attrM = append(attrM, semconv.HTTPRequestMethodKey.String(rtt.req.Method))
 	attrM = append(attrM, semconv.ServerAddress(rtt.req.RemoteAddr))
 
+	attrM = append(attrM, customAttributes...)
 	statusCode := 0
 	if rtt.err == nil {
 		// if we fail on the client side, we do not have a status code, but we

http/server/metrics.go

@@ -1,11 +1,11 @@
 package server
 
 import (
-	"net/http"
-
+	otelhttp "github.com/krakend/krakend-otel/http"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/semconv/v1.21.0"
+	"net/http"
 
 	kotelconfig "github.com/krakend/krakend-otel/config"
 )
@@ -36,11 +36,16 @@ func (m *metricsHTTP) report(t *tracking, r *http.Request) {
 	if m == nil || m.latency == nil {
 		return
 	}
+
 	dynAttrsOpts := metric.WithAttributes(
-		semconv.HTTPRoute(t.EndpointPattern()),
-		semconv.HTTPRequestMethodKey.String(r.Method),
-		semconv.HTTPResponseStatusCode(t.responseStatus),
+		append(
+			otelhttp.CustomMetricAttributes(r),
+			semconv.HTTPRoute(t.EndpointPattern()),
+			semconv.HTTPRequestMethodKey.String(r.Method),
+			semconv.HTTPResponseStatusCode(t.responseStatus),
+		)...,
 	)
+
 	m.latency.Record(t.ctx, t.latencyInSecs, m.fixedAttrsOpts, dynAttrsOpts)
 	m.size.Record(t.ctx, int64(t.responseSize), m.fixedAttrsOpts, dynAttrsOpts)
 }

http/server/server.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	otelhttp "github.com/krakend/krakend-otel/http"
 	"net"
 	"net/http"
 
@@ -37,6 +38,7 @@ func (h *trackingHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		}
 	}
 	t.ctx = context.WithValue(t.ctx, krakenDContextTrackingStrKey, t)
+	t.ctx = otelhttp.InjectLabeler(t.ctx)
 	r = r.WithContext(t.ctx)
 
 	if h.metrics != nil || h.traces != nil {