Add exemplar support to the prometheus exporter (#5111)

Author: dashpole

CHANGELOG.md

@@ -25,6 +25,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Add `otel.scope.name` and `otel.scope.version` tags to spans exported by `go.opentelemetry.io/otel/exporters/zipkin`. (#5108)
 - Add support for `AddLink` to `go.opentelemetry.io/otel/bridge/opencensus`. (#5116)
 - Add `String` method to `Value` and `KeyValue` in `go.opentelemetry.io/otel/log`. (#5117)
+- Add Exemplar support to `go.opentelemetry.io/otel/exporters/prometheus`. (#5111)
 
 ### Changed
 

exporters/prometheus/exporter.go

@@ -5,6 +5,7 @@ package prometheus // import "go.opentelemetry.io/otel/exporters/prometheus"
 
 import (
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"slices"
@@ -32,6 +33,9 @@ const (
 
 	scopeInfoMetricName  = "otel_scope_info"
 	scopeInfoDescription = "Instrumentation Scope metadata"
+
+	traceIDExemplarKey = "trace_id"
+	spanIDExemplarKey  = "span_id"
 )
 
 var (
@@ -238,7 +242,6 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) {
 }
 
 func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogram metricdata.Histogram[N], m metricdata.Metrics, ks, vs [2]string, name string, resourceKV keyVals) {
-	// TODO(https://github.com/open-telemetry/opentelemetry-go/issues/3163): support exemplars
 	for _, dp := range histogram.DataPoints {
 		keys, values := getAttrs(dp.Attributes, ks, vs, resourceKV)
 
@@ -255,6 +258,7 @@ func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogra
 			otel.Handle(err)
 			continue
 		}
+		m = addExemplars(m, dp.Exemplars)
 		ch <- m
 	}
 }
@@ -274,6 +278,7 @@ func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata
 			otel.Handle(err)
 			continue
 		}
+		m = addExemplars(m, dp.Exemplars)
 		ch <- m
 	}
 }
@@ -549,3 +554,37 @@ func (c *collector) validateMetrics(name, description string, metricType *dto.Me
 
 	return false, ""
 }
+
+func addExemplars[N int64 | float64](m prometheus.Metric, exemplars []metricdata.Exemplar[N]) prometheus.Metric {
+	if len(exemplars) == 0 {
+		return m
+	}
+	promExemplars := make([]prometheus.Exemplar, len(exemplars))
+	for i, exemplar := range exemplars {
+		labels := attributesToLabels(exemplar.FilteredAttributes)
+		// Overwrite any existing trace ID or span ID attributes
+		labels[traceIDExemplarKey] = hex.EncodeToString(exemplar.TraceID[:])
+		labels[spanIDExemplarKey] = hex.EncodeToString(exemplar.SpanID[:])
+		promExemplars[i] = prometheus.Exemplar{
+			Value:     float64(exemplar.Value),
+			Timestamp: exemplar.Time,
+			Labels:    labels,
+		}
+	}
+	metricWithExemplar, err := prometheus.NewMetricWithExemplars(m, promExemplars...)
+	if err != nil {
+		// If there are errors creating the metric with exemplars, just warn
+		// and return the metric without exemplars.
+		otel.Handle(err)
+		return m
+	}
+	return metricWithExemplar
+}
+
+func attributesToLabels(attrs []attribute.KeyValue) prometheus.Labels {
+	labels := make(map[string]string)
+	for _, attr := range attrs {
+		labels[string(attr.Key)] = attr.Value.Emit()
+	}
+	return labels
+}

exporters/prometheus/exporter_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -22,6 +23,7 @@ import (
 	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
 	semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
+	"go.opentelemetry.io/otel/trace"
 )
 
 func TestPrometheusExporter(t *testing.T) {
@@ -898,3 +900,118 @@ func TestShutdownExporter(t *testing.T) {
 	// ensure we aren't unnecessarily logging errors from the shutdown MeterProvider
 	require.NoError(t, handledError)
 }
+
+func TestExemplars(t *testing.T) {
+	attrsOpt := otelmetric.WithAttributes(
+		attribute.Key("A").String("B"),
+		attribute.Key("C").String("D"),
+		attribute.Key("E").Bool(true),
+		attribute.Key("F").Int(42),
+	)
+	for _, tc := range []struct {
+		name                  string
+		recordMetrics         func(ctx context.Context, meter otelmetric.Meter)
+		expectedExemplarValue float64
+	}{
+		{
+			name: "counter",
+			recordMetrics: func(ctx context.Context, meter otelmetric.Meter) {
+				counter, err := meter.Float64Counter("foo")
+				require.NoError(t, err)
+				counter.Add(ctx, 9, attrsOpt)
+			},
+			expectedExemplarValue: 9,
+		},
+		{
+			name: "histogram",
+			recordMetrics: func(ctx context.Context, meter otelmetric.Meter) {
+				hist, err := meter.Int64Histogram("foo")
+				require.NoError(t, err)
+				hist.Record(ctx, 9, attrsOpt)
+			},
+			expectedExemplarValue: 9,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("OTEL_GO_X_EXEMPLAR", "true")
+			// initialize registry exporter
+			ctx := context.Background()
+			registry := prometheus.NewRegistry()
+			exporter, err := New(WithRegisterer(registry), WithoutTargetInfo(), WithoutScopeInfo())
+			require.NoError(t, err)
+
+			// initialize resource
+			res, err := resource.New(ctx,
+				resource.WithAttributes(semconv.ServiceName("prometheus_test")),
+				resource.WithAttributes(semconv.TelemetrySDKVersion("latest")),
+			)
+			require.NoError(t, err)
+			res, err = resource.Merge(resource.Default(), res)
+			require.NoError(t, err)
+
+			// initialize provider and meter
+			provider := metric.NewMeterProvider(
+				metric.WithReader(exporter),
+				metric.WithResource(res),
+				metric.WithView(metric.NewView(
+					metric.Instrument{Name: "*"},
+					metric.Stream{
+						// filter out all attributes so they are added as filtered
+						// attributes to the exemplar
+						AttributeFilter: attribute.NewAllowKeysFilter(),
+					},
+				)),
+			)
+			meter := provider.Meter("meter", otelmetric.WithInstrumentationVersion("v0.1.0"))
+
+			// Add a sampled span context so that measurements get exemplars added
+			sc := trace.NewSpanContext(trace.SpanContextConfig{
+				SpanID:     trace.SpanID{0o1},
+				TraceID:    trace.TraceID{0o1},
+				TraceFlags: trace.FlagsSampled,
+			})
+			ctx = trace.ContextWithSpanContext(ctx, sc)
+			// Record a single observation with the exemplar
+			tc.recordMetrics(ctx, meter)
+
+			// Verify that the exemplar is present in the proto version of the
+			// prometheus metrics.
+			got, done, err := prometheus.ToTransactionalGatherer(registry).Gather()
+			defer done()
+			require.NoError(t, err)
+
+			require.Len(t, got, 1)
+			family := got[0]
+			require.Len(t, family.GetMetric(), 1)
+			metric := family.GetMetric()[0]
+			var exemplar *dto.Exemplar
+			switch family.GetType() {
+			case dto.MetricType_COUNTER:
+				exemplar = metric.GetCounter().GetExemplar()
+			case dto.MetricType_HISTOGRAM:
+				for _, b := range metric.GetHistogram().GetBucket() {
+					if b.GetExemplar() != nil {
+						exemplar = b.GetExemplar()
+						continue
+					}
+				}
+			}
+			require.NotNil(t, exemplar)
+			require.Equal(t, exemplar.GetValue(), tc.expectedExemplarValue)
+			expectedLabels := map[string]string{
+				traceIDExemplarKey: "01000000000000000000000000000000",
+				spanIDExemplarKey:  "0100000000000000",
+				"A":                "B",
+				"C":                "D",
+				"E":                "true",
+				"F":                "42",
+			}
+			require.Equal(t, len(expectedLabels), len(exemplar.GetLabel()))
+			for _, label := range exemplar.GetLabel() {
+				val, ok := expectedLabels[label.GetName()]
+				require.True(t, ok)
+				require.Equal(t, label.GetValue(), val)
+			}
+		})
+	}
+}

exporters/prometheus/go.mod

@@ -10,6 +10,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.24.0
 	go.opentelemetry.io/otel/sdk v1.24.0
 	go.opentelemetry.io/otel/sdk/metric v1.24.0
+	go.opentelemetry.io/otel/trace v1.24.0
 	google.golang.org/protobuf v1.33.0
 )
 
@@ -23,7 +24,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )