Add a Benchmark for collecting douplicates

pkg/collector/distinct_value_collector.go

@@ -112,6 +112,7 @@ func (d *DistinctValue[T]) Diff() []T {
 	d.mtx.Lock()
 	defer d.mtx.Unlock()
 
+	// TODO: return error if Diff is called on diffEnabled=false collector
 	if !d.diffEnabled {
 		return nil
 	}

pkg/collector/distinct_value_collector_test.go

@@ -32,7 +32,7 @@ func stringsSlicesEqual(t *testing.T, a, b []string) {
 	require.Equal(t, a, b)
 }
 
-func BenchmarkCollect(b *testing.B) {
+func BenchmarkCollectUniques(b *testing.B) {
 	// simulate 100 ingesters, each returning 10_000 tag values
 	numIngesters := 100
 	numTagValuesPerIngester := 10_000
@@ -72,3 +72,45 @@ func BenchmarkCollect(b *testing.B) {
 		})
 	}
 }
+
+func BenchmarkCollectDuplicates(b *testing.B) {
+	// simulate 100 ingesters, each returning 10_000 tag values each with same value to simulate duplicates
+	numIngesters := 100
+	numTagValuesPerIngester := 10_000
+	ingesterTagValues := make([][]tempopb.TagValue, numIngesters)
+	for i := 0; i < numIngesters; i++ {
+		tagValues := make([]tempopb.TagValue, numTagValuesPerIngester)
+		tv := tempopb.TagValue{
+			Type:  "string",
+			Value: fmt.Sprintf("value_%d", i),
+		}
+		for j := 0; j < numTagValuesPerIngester; j++ {
+			tagValues[j] = tv
+		}
+		ingesterTagValues[i] = tagValues
+	}
+
+	limits := []int{
+		0,          // no limit
+		100_000,    // 100KB
+		1_000_000,  // 1MB
+		10_000_000, // 10MB
+	}
+
+	b.ResetTimer() // to exclude the setup time for generating tag values
+	for _, lim := range limits {
+		b.Run("limit:"+strconv.Itoa(lim), func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				// NewDistinctValue is collecting tag values without diff support
+				distinctValues := NewDistinctValue(lim, func(v tempopb.TagValue) int { return len(v.Type) + len(v.Value) })
+				for _, tagValues := range ingesterTagValues {
+					for _, v := range tagValues {
+						if distinctValues.Collect(v) {
+							break // stop early if limit is reached
+						}
+					}
+				}
+			}
+		})
+	}
+}