Mimir: Add attributed prefix to labels of cost attribution metrics

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Grafana Mimir
 
 * [FEATURE] Ingester/Distributor: Add support for exporting cost attribution metrics (`cortex_ingester_attributed_active_series`, `cortex_distributor_received_attributed_samples_total`, and `cortex_discarded_attributed_samples_total`) with labels specified by customers to a custom Prometheus registry. This feature enables more flexible billing data tracking. #10269
+* [CHANGE] Ingester/Distributor: Prefix cost attribution metric labels with `attributed_` to prevent collisions with system cluster and namespace labels. #10509
 * [CHANGE] Querier: pass context to queryable `IsApplicable` hook. #10451
 * [CHANGE] Distributor: OTLP and push handler replace all non-UTF8 characters with the unicode replacement character `\uFFFD` in error messages before propagating them. #10236
 * [CHANGE] Querier: pass query matchers to queryable `IsApplicable` hook. #10256

cmd/mimir/config-descriptor.json

@@ -4414,7 +4414,7 @@
           "kind": "field",
           "name": "cost_attribution_labels",
           "required": false,
-          "desc": "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{team='frontend', service='api'}.",
+          "desc": "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{attributed_team='frontend', attributed_service='api'}.",
           "fieldValue": null,
           "fieldDefaultValue": "",
           "fieldFlag": "validation.cost-attribution-labels",

cmd/mimir/help-all.txt.tmpl

@@ -3332,7 +3332,7 @@ Usage of ./cmd/mimir/mimir:
   -validation.cost-attribution-cooldown duration
     	[experimental] Defines how long cost attribution stays in overflow before attempting a reset, with received/discarded samples extending the cooldown if overflow persists, while active series reset and restart tracking after the cooldown.
   -validation.cost-attribution-labels comma-separated-list-of-strings
-    	[experimental] Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{team='frontend', service='api'}.
+    	[experimental] Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{attributed_team='frontend', attributed_service='api'}.
   -validation.create-grace-period duration
     	Controls how far into the future incoming samples and exemplars are accepted compared to the wall clock. Any sample or exemplar will be rejected if its timestamp is greater than '(now + creation_grace_period)'. This configuration is enforced in the distributor and ingester. (default 10m)
   -validation.enforce-metadata-metric-name

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -3623,8 +3623,8 @@ The `limits` block configures default and per-tenant limits imposed by component
 # (experimental) Defines labels for cost attribution. Applies to metrics like
 # cortex_distributor_received_attributed_samples_total. To disable, set to an
 # empty string. For example, 'team,service' produces metrics such as
-# cortex_distributor_received_attributed_samples_total{team='frontend',
-# service='api'}.
+# cortex_distributor_received_attributed_samples_total{attributed_team='frontend',
+# attributed_service='api'}.
 # CLI flag: -validation.cost-attribution-labels
 [cost_attribution_labels: <string> | default = ""]
 

pkg/costattribution/active_tracker.go

@@ -34,6 +34,14 @@ type ActiveSeriesTracker struct {
 	overflowCounter atomic.Int64
 }
 
+func addLabelsPrefix(labels []string) []string {
+	out := make([]string, 0, len(labels))
+	for _, l := range labels {
+		out = append(out, strings.Join([]string{usagePrefix, l}, "_"))
+	}
+	return out
+}
+
 func newActiveSeriesTracker(userID string, trackedLabels []string, limit int, cooldownDuration time.Duration, logger log.Logger) *ActiveSeriesTracker {
 	// Create a map for overflow labels to export when overflow happens
 	overflowLabels := make([]string, len(trackedLabels)+2)
@@ -54,11 +62,10 @@ func newActiveSeriesTracker(userID string, trackedLabels []string, limit int, co
 		cooldownDuration: cooldownDuration,
 	}
 
-	variableLabels := slices.Clone(trackedLabels)
-	variableLabels = append(variableLabels, tenantLabel, "reason")
-
+	labelsWithPrefix := addLabelsPrefix(trackedLabels)
+	labelsWithPrefix = append(labelsWithPrefix, tenantLabel)
 	ast.activeSeriesPerUserAttribution = prometheus.NewDesc("cortex_ingester_attributed_active_series",
-		"The total number of active series per user and attribution.", variableLabels[:len(variableLabels)-1],
+		"The total number of active series per user and attribution.", labelsWithPrefix,
 		prometheus.Labels{trackerLabel: defaultTrackerName})
 
 	return ast

pkg/costattribution/active_tracker_test.go

@@ -70,7 +70,7 @@ func TestActiveTracker_Concurrency(t *testing.T) {
 	expectedMetrics := `
     # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution.
     # TYPE cortex_ingester_attributed_active_series gauge
-	cortex_ingester_attributed_active_series{team="__overflow__",tenant="user1",tracker="cost-attribution"} 100
+	cortex_ingester_attributed_active_series{attributed_team="__overflow__",tenant="user1",tracker="cost-attribution"} 100
 `
 	assert.NoError(t, testutil.GatherAndCompare(m.reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series"))
 

pkg/costattribution/manager.go

@@ -21,6 +21,7 @@ const (
 	defaultTrackerName = "cost-attribution"
 	missingValue       = "__missing__"
 	overflowValue      = "__overflow__"
+	usagePrefix        = "attributed"
 )
 
 type Manager struct {

pkg/costattribution/manager_test.go

@@ -60,14 +60,14 @@ func TestManager_CreateDeleteTracker(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
 		# TYPE cortex_discarded_attributed_samples_total counter
-		cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="bar",tenant="user1",tracker="cost-attribution"} 1
-		cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_team="bar",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1
 		# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 		# TYPE cortex_distributor_received_attributed_samples_total counter
-		cortex_distributor_received_attributed_samples_total{department="foo",service="dodo",tenant="user3",tracker="cost-attribution"} 1
+		cortex_distributor_received_attributed_samples_total{attributed_department="foo",attributed_service="dodo",tenant="user3",tracker="cost-attribution"} 1
 		# HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution.
         # TYPE cortex_ingester_attributed_active_series gauge
-        cortex_ingester_attributed_active_series{team="bar",tenant="user1",tracker="cost-attribution"} 1
+        cortex_ingester_attributed_active_series{attributed_team="bar",tenant="user1",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_distributor_received_attributed_samples_total", "cortex_ingester_attributed_active_series"))
 	})
@@ -78,10 +78,10 @@ func TestManager_CreateDeleteTracker(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
 		# TYPE cortex_discarded_attributed_samples_total counter
-		cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1
 		# HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution.
         # TYPE cortex_ingester_attributed_active_series gauge
-        cortex_ingester_attributed_active_series{team="bar",tenant="user1",tracker="cost-attribution"} 1
+        cortex_ingester_attributed_active_series{attributed_team="bar",tenant="user1",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_ingester_attributed_active_series"))
 	})
@@ -96,7 +96,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 		# TYPE cortex_distributor_received_attributed_samples_total counter
-		cortex_distributor_received_attributed_samples_total{department="foo",service="dodo",tenant="user3",tracker="cost-attribution"} 1
+		cortex_distributor_received_attributed_samples_total{attributed_department="foo",attributed_service="dodo",tenant="user3",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_distributor_received_attributed_samples_total", "cortex_ingester_attributed_active_series"))
 	})
@@ -114,7 +114,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
 		# TYPE cortex_discarded_attributed_samples_total counter
-		cortex_discarded_attributed_samples_total{feature="__missing__",reason="invalid-metrics-name",team="foo",tenant="user3",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_feature="__missing__",attributed_team="foo",reason="invalid-metrics-name",tenant="user3",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total"))
 	})
@@ -126,7 +126,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 		# TYPE cortex_distributor_received_attributed_samples_total counter
-		cortex_distributor_received_attributed_samples_total{feature="__overflow__",team="__overflow__",tenant="user3",tracker="cost-attribution"} 2
+		cortex_distributor_received_attributed_samples_total{attributed_feature="__overflow__",attributed_team="__overflow__",tenant="user3",tracker="cost-attribution"} 2
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total"))
 	})
@@ -146,8 +146,8 @@ func TestManager_PurgeInactiveAttributionsUntil(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
 		# TYPE cortex_discarded_attributed_samples_total counter
-		cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1
-		cortex_discarded_attributed_samples_total{department="foo",reason="out-of-window",service="bar",tenant="user3",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_department="foo",attributed_service="bar",reason="out-of-window",tenant="user3",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total"))
 	})
@@ -165,7 +165,7 @@ func TestManager_PurgeInactiveAttributionsUntil(t *testing.T) {
 		expectedMetrics := `
 		# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
 		# TYPE cortex_discarded_attributed_samples_total counter
-		cortex_discarded_attributed_samples_total{department="foo",reason="out-of-window",service="bar",tenant="user3",tracker="cost-attribution"} 1
+		cortex_discarded_attributed_samples_total{attributed_department="foo",attributed_service="bar",reason="out-of-window",tenant="user3",tracker="cost-attribution"} 1
 		`
 		assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total"))
 	})

pkg/costattribution/sample_tracker.go

@@ -66,16 +66,17 @@ func newSampleTracker(userID string, trackedLabels []string, limit int, cooldown
 		overflowCounter:  observation{},
 	}
 
-	variableLabels := slices.Clone(trackedLabels)
-	variableLabels = append(variableLabels, tenantLabel, "reason")
+	labelsWithPrefix := addLabelsPrefix(trackedLabels)
+	labelsWithPrefix = append(labelsWithPrefix, tenantLabel, "reason")
+
 	tracker.discardedSampleAttribution = prometheus.NewDesc("cortex_discarded_attributed_samples_total",
 		"The total number of samples that were discarded per attribution.",
-		variableLabels,
+		labelsWithPrefix,
 		prometheus.Labels{trackerLabel: defaultTrackerName})
 
 	tracker.receivedSamplesAttribution = prometheus.NewDesc("cortex_distributor_received_attributed_samples_total",
 		"The total number of samples that were received per attribution.",
-		variableLabels[:len(variableLabels)-1],
+		labelsWithPrefix[:len(labelsWithPrefix)-1],
 		prometheus.Labels{trackerLabel: defaultTrackerName})
 	return tracker
 }

pkg/costattribution/sample_tracker_test.go

@@ -30,7 +30,7 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) {
 		expectedMetrics := `
 	# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 	# TYPE cortex_distributor_received_attributed_samples_total counter
-	cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 3
+	cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 3
 	`
 		assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total"))
 	})
@@ -43,8 +43,8 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) {
 		expectedMetrics := `
 	# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 	# TYPE cortex_distributor_received_attributed_samples_total counter
-	cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 6
-	cortex_distributor_received_attributed_samples_total{platform="bar",tenant="user4",tracker="cost-attribution"} 5
+	cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 6
+	cortex_distributor_received_attributed_samples_total{attributed_platform="bar",tenant="user4",tracker="cost-attribution"} 5
 	`
 		assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total"))
 	})
@@ -58,8 +58,8 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) {
 		expectedMetrics := `
 	# HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
 	# TYPE cortex_distributor_received_attributed_samples_total counter
-	cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 14
-	cortex_distributor_received_attributed_samples_total{platform="bar",tenant="user4",tracker="cost-attribution"} 5
+	cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 14
+	cortex_distributor_received_attributed_samples_total{attributed_platform="bar",tenant="user4",tracker="cost-attribution"} 5
 	`
 		assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total"))
 	})
@@ -148,11 +148,10 @@ func TestSampleTracker_Concurrency(t *testing.T) {
 	expectedMetrics := `
 	# HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution.
     # TYPE cortex_discarded_attributed_samples_total counter
-    cortex_discarded_attributed_samples_total{reason="__overflow__",team="__overflow__",tenant="user1",tracker="cost-attribution"} 95
+    cortex_discarded_attributed_samples_total{attributed_team="__overflow__",reason="__overflow__",tenant="user1",tracker="cost-attribution"} 95
     # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution.
     # TYPE cortex_distributor_received_attributed_samples_total counter
-	cortex_distributor_received_attributed_samples_total{team="__overflow__",tenant="user1",tracker="cost-attribution"} 95
-
+	cortex_distributor_received_attributed_samples_total{attributed_team="__overflow__",tenant="user1",tracker="cost-attribution"} 95
 `
 	assert.NoError(t, testutil.GatherAndCompare(m.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total", "cortex_discarded_attributed_samples_total"))
 }