Add span size as a new metrics to metrics generator (#1662)

* add span size metrics to spanmetrics in metrics generator * add changelog
grafana · Aug 19, 2022 · 08aef83 · 08aef83
1 parent b78c20f
commit 08aef83
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## main / unreleased
 
+* [ENHANCEMENT] metrics-generator: expose span size as a metric [#1662](https://github.com/grafana/tempo/pull/1662) (@ie-pham)
+
 ## v1.5.0 / 2022-08-17
 
 * [CHANGE] metrics-generator: Changed added metric label `instance` to `__metrics_gen_instance` to reduce collisions with custom dimensions. [#1439](https://github.com/grafana/tempo/pull/1439) (@joe-elliott)

diff --git a/integration/e2e/metrics_generator_test.go b/integration/e2e/metrics_generator_test.go
@@ -147,6 +147,7 @@ func TestMetricsGenerator(t *testing.T) {
 	// Span metrics
 	lbls = []string{"service", "lb", "span_name", "lb-get", "span_kind", "SPAN_KIND_CLIENT", "status_code", "STATUS_CODE_UNSET"}
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
+	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
 	assert.Equal(t, 0.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
@@ -155,6 +156,7 @@ func TestMetricsGenerator(t *testing.T) {
 
 	lbls = []string{"service", "app", "span_name", "app-handle", "span_kind", "SPAN_KIND_SERVER", "status_code", "STATUS_CODE_UNSET"}
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_calls_total", lbls))
+	assert.NotEqual(t, 0, sumValues(metricFamilies, "traces_spanmetrics_size_total", lbls))
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "1")))
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "2")))
 	assert.Equal(t, 1.0, sumValues(metricFamilies, "traces_spanmetrics_latency_bucket", append(lbls, "le", "+Inf")))
@@ -164,9 +166,9 @@ func TestMetricsGenerator(t *testing.T) {
 	// Verify metrics
 	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(2), "tempo_metrics_generator_spans_received_total"))
 
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(23), "tempo_metrics_generator_registry_active_series"))
+	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_active_series"))
 	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(1000), "tempo_metrics_generator_registry_max_active_series"))
-	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(23), "tempo_metrics_generator_registry_series_added_total"))
+	assert.NoError(t, tempoMetricsGenerator.WaitSumMetrics(e2e.Equals(25), "tempo_metrics_generator_registry_series_added_total"))
 }
 
 func newPrometheus() *e2e.HTTPService {

diff --git a/modules/generator/processor/spanmetrics/spanmetrics.go b/modules/generator/processor/spanmetrics/spanmetrics.go
@@ -19,13 +19,15 @@ import (
 const (
 	metricCallsTotal      = "traces_spanmetrics_calls_total"
 	metricDurationSeconds = "traces_spanmetrics_latency"
+	metricSizeTotal       = "traces_spanmetrics_size_total"
 )
 
 type Processor struct {
 	Cfg Config
 
 	spanMetricsCallsTotal      registry.Counter
 	spanMetricsDurationSeconds registry.Histogram
+	spanMetricsSizeTotal       registry.Counter
 
 	// for testing
 	now func() time.Time
@@ -41,6 +43,7 @@ func New(cfg Config, registry registry.Registry) gen.Processor {
 		Cfg:                        cfg,
 		spanMetricsCallsTotal:      registry.NewCounter(metricCallsTotal, labels),
 		spanMetricsDurationSeconds: registry.NewHistogram(metricDurationSeconds, labels, cfg.HistogramBuckets),
+		spanMetricsSizeTotal:       registry.NewCounter(metricSizeTotal, labels),
 		now:                        time.Now,
 	}
 }
@@ -86,5 +89,6 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, rs *v1.Resource, spa
 	registryLabelValues := registry.NewLabelValues(labelValues)
 
 	p.spanMetricsCallsTotal.Inc(registryLabelValues, 1)
+	p.spanMetricsSizeTotal.Inc(registryLabelValues, float64(span.Size()))
 	p.spanMetricsDurationSeconds.ObserveWithExemplar(registryLabelValues, latencySeconds, tempo_util.TraceIDToHexString(span.TraceId))
 }