metric integration

Signed-off-by: Owen Diehl <[email protected]>

Author: owen-d

pkg/storage/bloom/v1/bloom_tokenizer.go

@@ -137,10 +137,7 @@ func (bt *BloomTokenizer) Populate(
 			// If a bloom is full, the chunk wasn't completely added
 			// so we'll submit this bloom, start a new one, and continue indexing
 			if full {
-				ch <- &BloomCreation{
-					Bloom:            bloom,
-					SourceBytesAdded: bytesAdded,
-				}
+				bt.sendBloom(ch, bloom, bytesAdded)
 
 				// start a new bloom + reset bytesAdded counter
 				bytesAdded = 0
@@ -158,11 +155,26 @@ func (bt *BloomTokenizer) Populate(
 	}
 
 	// Send the last bloom
+	bt.sendBloom(ch, bloom, bytesAdded)
+	close(ch)
+}
+
+func (bt *BloomTokenizer) sendBloom(
+	ch chan<- *BloomCreation,
+	bloom *Bloom,
+	bytesAdded int,
+) {
+	fillRatio := bloom.ScalableBloomFilter.FillRatio()
+	bt.metrics.hammingWeightRatio.Observe(fillRatio)
+	bt.metrics.estimatedCount.Observe(
+		float64(estimatedCount(bloom.ScalableBloomFilter.Capacity(), fillRatio)),
+	)
+	bt.metrics.bloomSize.Observe(float64(bloom.ScalableBloomFilter.Capacity() / eightBits))
+	bt.metrics.bloomsTotal.Inc()
 	ch <- &BloomCreation{
 		Bloom:            bloom,
 		SourceBytesAdded: bytesAdded,
 	}
-	close(ch)
 }
 
 // addChunkToBloom adds the tokens from the given chunk to the given bloom.

pkg/storage/bloom/v1/metrics.go

@@ -9,11 +9,10 @@ import (
 
 type Metrics struct {
 	// writes
-	bloomsTotal         *prometheus.CounterVec // number of blooms created
-	sbfCreationTime     *prometheus.CounterVec // time spent creating sbfs
-	bloomSize           prometheus.Histogram   // size of the bloom filter in bytes
-	hammingWeightRatio  prometheus.Histogram   // ratio of the hamming weight of the bloom filter to the number of bits in the bloom filter
-	estimatedCount      prometheus.Histogram   // estimated number of elements in the bloom filter
+	bloomsTotal         prometheus.Counter   // number of blooms created
+	bloomSize           prometheus.Histogram // size of the bloom filter in bytes
+	hammingWeightRatio  prometheus.Histogram // ratio of the hamming weight of the bloom filter to the number of bits in the bloom filter
+	estimatedCount      prometheus.Histogram // estimated number of elements in the bloom filter
 	chunksIndexed       *prometheus.CounterVec
 	chunksPerSeries     prometheus.Histogram
 	blockSeriesIterated prometheus.Counter
@@ -51,9 +50,6 @@ const (
 	skipReasonErr      = "err"
 	skipReasonOOB      = "out_of_bounds"
 
-	bloomCreationTypeIndexed = "indexed"
-	bloomCreationTypeSkipped = "skipped"
-
 	recorderRequested = "requested"
 	recorderFound     = "found"
 	recorderSkipped   = "skipped"
@@ -63,16 +59,11 @@ const (
 
 func NewMetrics(r prometheus.Registerer) *Metrics {
 	return &Metrics{
-		bloomsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+		bloomsTotal: promauto.With(r).NewCounter(prometheus.CounterOpts{
 			Namespace: constants.Loki,
 			Name:      "blooms_created_total",
 			Help:      "Number of blooms created",
-		}, []string{"type"}),
-		sbfCreationTime: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
-			Namespace: constants.Loki,
-			Name:      "bloom_creation_time_total",
-			Help:      "Time spent creating scalable bloom filters",
-		}, []string{"type"}),
+		}),
 		bloomSize: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
 			Namespace: constants.Loki,
 			Name:      "bloom_size",