metric integration

Signed-off-by: Owen Diehl <[email protected]>

Author: owen-d

pkg/storage/bloom/v1/bloom_tokenizer.go

@@ -137,10 +137,7 @@ func (bt *BloomTokenizer) Populate(
 			// If a bloom is full, the chunk wasn't completely added
 			// so we'll submit this bloom, start a new one, and continue indexing
 			if full {
-				ch <- &BloomCreation{
-					Bloom:            bloom,
-					SourceBytesAdded: bytesAdded,
-				}
+				bt.sendBloom(ch, bloom, bytesAdded)
 
 				// start a new bloom + reset bytesAdded counter
 				bytesAdded = 0
@@ -158,11 +155,25 @@ func (bt *BloomTokenizer) Populate(
 	}
 
 	// Send the last bloom
+	bt.sendBloom(ch, bloom, bytesAdded)
+	close(ch)
+}
+
+func (bt *BloomTokenizer) sendBloom(
+	ch chan<- *BloomCreation,
+	bloom *Bloom,
+	bytesAdded int,
+) {
+	fillRatio := bloom.ScalableBloomFilter.FillRatio()
+	bt.metrics.hammingWeightRatio.Observe(fillRatio)
+	bt.metrics.estimatedCount.Observe(
+		float64(estimatedCount(bloom.ScalableBloomFilter.Capacity(), fillRatio)),
+	)
+	bt.metrics.bloomSize.Observe(float64(bloom.ScalableBloomFilter.Capacity() / eightBits))
 	ch <- &BloomCreation{
 		Bloom:            bloom,
 		SourceBytesAdded: bytesAdded,
 	}
-	close(ch)
 }
 
 // addChunkToBloom adds the tokens from the given chunk to the given bloom.

pkg/storage/bloom/v1/metrics.go

@@ -10,7 +10,6 @@ import (
 type Metrics struct {
 	// writes
 	bloomsTotal         *prometheus.CounterVec // number of blooms created
-	sbfCreationTime     *prometheus.CounterVec // time spent creating sbfs
 	bloomSize           prometheus.Histogram   // size of the bloom filter in bytes
 	hammingWeightRatio  prometheus.Histogram   // ratio of the hamming weight of the bloom filter to the number of bits in the bloom filter
 	estimatedCount      prometheus.Histogram   // estimated number of elements in the bloom filter
@@ -68,11 +67,6 @@ func NewMetrics(r prometheus.Registerer) *Metrics {
 			Name:      "blooms_created_total",
 			Help:      "Number of blooms created",
 		}, []string{"type"}),
-		sbfCreationTime: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
-			Namespace: constants.Loki,
-			Name:      "bloom_creation_time_total",
-			Help:      "Time spent creating scalable bloom filters",
-		}, []string{"type"}),
 		bloomSize: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
 			Namespace: constants.Loki,
 			Name:      "bloom_size",