diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bdc7397fd6..2c9c2fd266f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [BUGFIX] Metrics generators: Correctly drop from the ring before stopping ingestion to reduce drops during a rollout. [#4101](https://github.com/grafana/tempo/pull/4101) (@joe-elliott) * [BUGFIX] Bring back application-json content-type header. [#4121](https://github.com/grafana/tempo/pull/4121) (@javiermolinar) * [BUGFIX] Correctly handle 400 Bad Request and 404 Not Found in gRPC streaming [#4144](https://github.com/grafana/tempo/pull/4144) (@mapno) +* [BUGFIX] Pushes a 0 to classic histogram's counter when the series is new to allow Prometheus to start from a non-null value. [#4140](https://github.com/grafana/tempo/pull/4140) (@mapno) * [CHANGE] TraceByID: don't allow concurrent_shards greater than query_shards. [#4074](https://github.com/grafana/tempo/pull/4074) (@electron0zero) * **BREAKING CHANGE** tempo-query is no longer a jaeger instance with grpcPlugin. Its now a standalone server. Serving a grpc api for jaeger on `0.0.0.0:7777` by default. [#3840](https://github.com/grafana/tempo/issues/3840) (@frzifus) * [CHANGE] **BREAKING CHANGE** The dynamic injection of X-Scope-OrgID header for metrics generator remote-writes is changed. If the header is aleady set in per-tenant overrides or global tempo configuration, then it is honored and not overwritten. [#4021](https://github.com/grafana/tempo/pull/4021) (@mdisibio) diff --git a/modules/generator/registry/histogram.go b/modules/generator/registry/histogram.go index 2c4a4c6e071..ed1e64c4f14 100644 --- a/modules/generator/registry/histogram.go +++ b/modules/generator/registry/histogram.go @@ -44,6 +44,19 @@ type histogramSeries struct { exemplars []*atomic.String exemplarValues []*atomic.Float64 lastUpdated *atomic.Int64 + // firstSeries is used to track if this series is new to the counter. This + // is used to ensure that new counters being with 0, and then are incremented + // to the desired value. This avoids Prometheus throwing away the first + // value in the series, due to the transition from null -> x. + firstSeries *atomic.Bool +} + +func (hs *histogramSeries) isNew() bool { + return hs.firstSeries.Load() +} + +func (hs *histogramSeries) registerSeenSeries() { + hs.firstSeries.Store(false) } var ( @@ -114,6 +127,7 @@ func (h *histogram) newSeries(labelValueCombo *LabelValueCombo, value float64, t buckets: nil, exemplars: nil, lastUpdated: atomic.NewInt64(0), + firstSeries: atomic.NewBool(true), } for i := 0; i < len(h.buckets); i++ { newSeries.buckets = append(newSeries.buckets, atomic.NewFloat64(0)) @@ -151,6 +165,8 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte h.seriesMtx.Lock() defer h.seriesMtx.Unlock() + t := timeMs + activeSeries = len(h.series) * int(h.activeSeriesPerHistogramSerie()) labelsCount := 0 @@ -171,16 +187,27 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte lb.Set(name, s.labels.values[i]) } + // If we are about to call Append for the first time on a series, + // we need to first insert a 0 value to allow Prometheus to start from a non-null value. + if s.isNew() { + lb.Set(labels.MetricName, h.nameCount) + _, err = appender.Append(0, lb.Labels(), t-1, 0) // t-1 to ensure that the next value is not at the same time + if err != nil { + return + } + s.registerSeenSeries() + } + // sum lb.Set(labels.MetricName, h.nameSum) - _, err = appender.Append(0, lb.Labels(), timeMs, s.sum.Load()) + _, err = appender.Append(0, lb.Labels(), t, s.sum.Load()) if err != nil { return } // count lb.Set(labels.MetricName, h.nameCount) - _, err = appender.Append(0, lb.Labels(), timeMs, s.count.Load()) + _, err = appender.Append(0, lb.Labels(), t, s.count.Load()) if err != nil { return } @@ -190,7 +217,7 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte for i, bucketLabel := range h.bucketLabels { lb.Set(labels.BucketLabel, bucketLabel) - ref, err := appender.Append(0, lb.Labels(), timeMs, s.buckets[i].Load()) + ref, err := appender.Append(0, lb.Labels(), t, s.buckets[i].Load()) if err != nil { return activeSeries, err } @@ -203,7 +230,7 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte Value: ex, }}, Value: s.exemplarValues[i].Load(), - Ts: timeMs, + Ts: t, }) if err != nil { return activeSeries, err diff --git a/modules/generator/registry/histogram_test.go b/modules/generator/registry/histogram_test.go index 77da6b3e733..731ee539234 100644 --- a/modules/generator/registry/histogram_test.go +++ b/modules/generator/registry/histogram_test.go @@ -27,12 +27,15 @@ func Test_histogram(t *testing.T) { assert.Equal(t, 2, seriesAdded) collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // Zero entry for value-1 series newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), // Zero entry for value-2 series newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -59,6 +62,7 @@ func Test_histogram(t *testing.T) { assert.Equal(t, 3, seriesAdded) collectionTimeMs = time.Now().UnixMilli() + collectionTimeWithOffsetMs = collectionTimeMs - 1 expectedSamples = []sample{ newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), @@ -70,6 +74,7 @@ func Test_histogram(t *testing.T) { newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-3"}, collectionTimeWithOffsetMs, 0), // Zero entry for value-3 series newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-3"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-3"}, collectionTimeMs, 3), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, collectionTimeMs, 0), @@ -150,12 +155,15 @@ func Test_histogram_cantAdd(t *testing.T) { h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0) collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -204,12 +212,15 @@ func Test_histogram_removeStaleSeries(t *testing.T) { assert.Equal(t, 0, removedSeries) collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -246,12 +257,15 @@ func Test_histogram_externalLabels(t *testing.T) { h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0) collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1", "external_label": "external_value"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1", "external_label": "external_value"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1", "external_label": "external_value"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1", "external_label": "external_value"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2", "external_label": "external_value"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf", "external_label": "external_value"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2", "external_label": "external_value"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2", "external_label": "external_value"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2", "external_label": "external_value"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1", "external_label": "external_value"}, collectionTimeMs, 0), @@ -337,7 +351,9 @@ func Test_histogram_concurrencyCorrectness(t *testing.T) { wg.Wait() collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, float64(totalCount.Load())), newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 2*float64(totalCount.Load())), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 0), @@ -353,9 +369,11 @@ func Test_histogram_span_multiplier(t *testing.T) { h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 2.0, "", 5) collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 expectedSamples := []sample{ - newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 11.5), + newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 6.5), + newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 11.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 6.5), newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 6.5), diff --git a/modules/generator/registry/native_histogram.go b/modules/generator/registry/native_histogram.go index b7a431cb692..b19d6e56632 100644 --- a/modules/generator/registry/native_histogram.go +++ b/modules/generator/registry/native_histogram.go @@ -11,6 +11,7 @@ import ( promhistogram "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" + "go.uber.org/atomic" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -46,6 +47,20 @@ type nativeHistogramSeries struct { promHistogram prometheus.Histogram lastUpdated int64 histogram *dto.Histogram + + // firstSeries is used to track if this series is new to the counter. + // This is used in classic histograms to ensure that new counters begin with 0. + // This avoids Prometheus throwing away the first value in the series, + // due to the transition from null -> x. + firstSeries *atomic.Bool +} + +func (hs *nativeHistogramSeries) isNew() bool { + return hs.firstSeries.Load() +} + +func (hs *nativeHistogramSeries) registerSeenSeries() { + hs.firstSeries.Store(false) } var ( @@ -112,6 +127,7 @@ func (h *nativeHistogram) newSeries(labelValueCombo *LabelValueCombo, value floa NativeHistogramMinResetDuration: 15 * time.Minute, }), lastUpdated: 0, + firstSeries: atomic.NewBool(true), } h.updateSeries(newSeries, value, traceID, multiplier) @@ -267,6 +283,15 @@ func (h *nativeHistogram) nativeHistograms(appender storage.Appender, lb *labels } func (h *nativeHistogram) classicHistograms(appender storage.Appender, lb *labels.Builder, timeMs int64, s *nativeHistogramSeries) (activeSeries int, err error) { + if s.isNew() { + lb.Set(labels.MetricName, h.metricName+"_count") + _, err = appender.Append(0, lb.Labels(), timeMs-1, 0) + if err != nil { + return activeSeries, err + } + s.registerSeenSeries() + } + // sum lb.Set(labels.MetricName, h.metricName+"_sum") _, err = appender.Append(0, lb.Labels(), timeMs, s.histogram.GetSampleSum()) diff --git a/modules/generator/registry/native_histogram_test.go b/modules/generator/registry/native_histogram_test.go index 8e7d801be80..a740a174753 100644 --- a/modules/generator/registry/native_histogram_test.go +++ b/modules/generator/registry/native_histogram_test.go @@ -48,6 +48,7 @@ func Test_Histograms(t *testing.T) { } collectionTimeMs := time.Now().UnixMilli() + collectionTimeWithOffsetMs := collectionTimeMs - 1 cases := []struct { name string @@ -70,6 +71,7 @@ func Test_Histograms(t *testing.T) { }, }, expectedSamples: []sample{ + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), @@ -106,11 +108,13 @@ func Test_Histograms(t *testing.T) { }, }, expectedSamples: []sample{ + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -152,11 +156,13 @@ func Test_Histograms(t *testing.T) { }, }, expectedSamples: []sample{ + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -202,6 +208,7 @@ func Test_Histograms(t *testing.T) { newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-3"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-3"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-3"}, collectionTimeMs, 3), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-3", "le": "1"}, collectionTimeMs, 0), @@ -243,11 +250,13 @@ func Test_Histograms(t *testing.T) { }, }, expectedSamples: []sample{ + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeMs, 20), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-1"}, collectionTimeMs, 20*1.5), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 0), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 20), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 20), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeMs, 13), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-2"}, collectionTimeMs, 13*3), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -290,11 +299,13 @@ func Test_Histograms(t *testing.T) { }, }, expectedSamples: []sample{ + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), @@ -340,6 +351,7 @@ func Test_Histograms(t *testing.T) { newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2), + newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-3"}, collectionTimeWithOffsetMs, 0), // zero count at the beginning newSample(map[string]string{"__name__": "test_histogram_count", "label": "value-3"}, collectionTimeMs, 1), newSample(map[string]string{"__name__": "test_histogram_sum", "label": "value-3"}, collectionTimeMs, 3), newSample(map[string]string{"__name__": "test_histogram_bucket", "label": "value-3", "le": "1"}, collectionTimeMs, 0), @@ -427,7 +439,7 @@ func Test_Histograms(t *testing.T) { h.ObserveWithExemplar(obs.labelValueCombo, obs.value, obs.traceID, obs.multiplier) } - collectMetricsAndAssertSeries(t, h, collectionTimeMs, len(c.expectedSamples), appender) + collectMetricsAndAssertSeries(t, h, collectionTimeMs, expectedSeriesLen(c.expectedSamples), appender) if len(c.expectedSamples) > 0 { assertAppenderSamples(t, appender, c.expectedSamples) } @@ -441,11 +453,7 @@ func Test_Histograms(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Run("classic", func(t *testing.T) { - var seriesAdded int - onAdd := func(count uint32) bool { - seriesAdded += int(count) - return true - } + onAdd := func(uint32) bool { return true } h := newHistogram("test_histogram", tc.buckets, onAdd, nil, "trace_id") testHistogram(t, h, tc.collections) }) @@ -454,12 +462,7 @@ func Test_Histograms(t *testing.T) { t.SkipNow() } - var seriesAdded int - onAdd := func(count uint32) bool { - seriesAdded += int(count) - return true - } - + onAdd := func(uint32) bool { return true } h := newNativeHistogram("test_histogram", tc.buckets, onAdd, nil, "trace_id", HistogramModeBoth) testHistogram(t, h, tc.collections) }) @@ -533,3 +536,11 @@ func assertAppenderExemplars(t *testing.T, appender *capturingAppender, expected } }) } + +func expectedSeriesLen(samples []sample) int { + series := make(map[string]struct{}) + for _, s := range samples { + series[s.l.String()] = struct{}{} + } + return len(series) +} diff --git a/modules/generator/registry/registry_test.go b/modules/generator/registry/registry_test.go index 714555ac92e..f97f9ff3aa1 100644 --- a/modules/generator/registry/registry_test.go +++ b/modules/generator/registry/registry_test.go @@ -85,11 +85,12 @@ func TestManagedRegistry_histogram(t *testing.T) { histogram.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0) expectedSamples := []sample{ - newSample(map[string]string{"__name__": "histogram_count", "label": "value-1", "__metrics_gen_instance": mustGetHostname()}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_sum", "label": "value-1", "__metrics_gen_instance": mustGetHostname()}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "1"}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "2"}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "+Inf"}, 0, 1.0), + newSample(map[string]string{"__name__": "histogram_count", "label": "value-1", "__metrics_gen_instance": mustGetHostname()}, 0, 0), + newSample(map[string]string{"__name__": "histogram_count", "label": "value-1", "__metrics_gen_instance": mustGetHostname()}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_sum", "label": "value-1", "__metrics_gen_instance": mustGetHostname()}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "1"}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "2"}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_bucket", "label": "value-1", "__metrics_gen_instance": mustGetHostname(), "le": "+Inf"}, 1, 1.0), } collectRegistryMetricsAndAssert(t, registry, appender, expectedSamples) } @@ -236,11 +237,12 @@ func TestManagedRegistry_maxLabelNameLength(t *testing.T) { expectedSamples := []sample{ newSample(map[string]string{"__name__": "counter", "very_len": "very_", "__metrics_gen_instance": mustGetHostname()}, 0, 0.0), - newSample(map[string]string{"__name__": "counter", "very_len": "very_", "__metrics_gen_instance": mustGetHostname()}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_count", "another_": "anoth", "__metrics_gen_instance": mustGetHostname()}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_sum", "another_": "anoth", "__metrics_gen_instance": mustGetHostname()}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_bucket", "another_": "anoth", "__metrics_gen_instance": mustGetHostname(), "le": "1"}, 0, 1.0), - newSample(map[string]string{"__name__": "histogram_bucket", "another_": "anoth", "__metrics_gen_instance": mustGetHostname(), "le": "+Inf"}, 0, 1.0), + newSample(map[string]string{"__name__": "counter", "very_len": "very_", "__metrics_gen_instance": mustGetHostname()}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_count", "another_": "anoth", "__metrics_gen_instance": mustGetHostname()}, 0, 0), + newSample(map[string]string{"__name__": "histogram_count", "another_": "anoth", "__metrics_gen_instance": mustGetHostname()}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_sum", "another_": "anoth", "__metrics_gen_instance": mustGetHostname()}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_bucket", "another_": "anoth", "__metrics_gen_instance": mustGetHostname(), "le": "1"}, 1, 1.0), + newSample(map[string]string{"__name__": "histogram_bucket", "another_": "anoth", "__metrics_gen_instance": mustGetHostname(), "le": "+Inf"}, 1, 1.0), } collectRegistryMetricsAndAssert(t, registry, appender, expectedSamples) }