diff --git a/CHANGELOG.md b/CHANGELOG.md index b1c6fab317..f5e39d0da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Grafana Mimir -* [FEATURE] Ingester/Distributor: Add support for exporting cost attribution metrics (`cortex_ingester_attributed_active_series`, `cortex_distributor_received_attributed_samples_total`, and `cortex_discarded_attributed_samples_total`) with labels specified by customers to a custom Prometheus registry. This feature enables more flexible billing data tracking. #10269 +* [FEATURE] Ingester/Distributor: Add support for exporting cost attribution metrics (`cortex_ingester_attributed_active_series`, `cortex_distributor_received_attributed_samples_total`, and `cortex_discarded_attributed_samples_total`) with labels specified by customers to a custom Prometheus registry. This feature enables more flexible billing data tracking. #10269 #10509 * [CHANGE] Querier: pass context to queryable `IsApplicable` hook. #10451 * [CHANGE] Distributor: OTLP and push handler replace all non-UTF8 characters with the unicode replacement character `\uFFFD` in error messages before propagating them. #10236 * [CHANGE] Querier: pass query matchers to queryable `IsApplicable` hook. #10256 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index fbfbe2af75..5e48004224 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -4414,7 +4414,7 @@ "kind": "field", "name": "cost_attribution_labels", "required": false, - "desc": "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{team='frontend', service='api'}.", + "desc": "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{attributed_team='frontend', attributed_service='api'}.", "fieldValue": null, "fieldDefaultValue": "", "fieldFlag": "validation.cost-attribution-labels", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index b5c57b3780..26ff910534 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -3332,7 +3332,7 @@ Usage of ./cmd/mimir/mimir: -validation.cost-attribution-cooldown duration [experimental] Defines how long cost attribution stays in overflow before attempting a reset, with received/discarded samples extending the cooldown if overflow persists, while active series reset and restart tracking after the cooldown. -validation.cost-attribution-labels comma-separated-list-of-strings - [experimental] Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{team='frontend', service='api'}. + [experimental] Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{attributed_team='frontend', attributed_service='api'}. -validation.create-grace-period duration Controls how far into the future incoming samples and exemplars are accepted compared to the wall clock. Any sample or exemplar will be rejected if its timestamp is greater than '(now + creation_grace_period)'. This configuration is enforced in the distributor and ingester. (default 10m) -validation.enforce-metadata-metric-name diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index fd481fc24a..8ed7dec563 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -3623,8 +3623,8 @@ The `limits` block configures default and per-tenant limits imposed by component # (experimental) Defines labels for cost attribution. Applies to metrics like # cortex_distributor_received_attributed_samples_total. To disable, set to an # empty string. For example, 'team,service' produces metrics such as -# cortex_distributor_received_attributed_samples_total{team='frontend', -# service='api'}. +# cortex_distributor_received_attributed_samples_total{attributed_team='frontend', +# attributed_service='api'}. # CLI flag: -validation.cost-attribution-labels [cost_attribution_labels: | default = ""] diff --git a/pkg/costattribution/active_tracker.go b/pkg/costattribution/active_tracker.go index 0ecd3e3c53..a5dc14750f 100644 --- a/pkg/costattribution/active_tracker.go +++ b/pkg/costattribution/active_tracker.go @@ -34,6 +34,14 @@ type ActiveSeriesTracker struct { overflowCounter atomic.Int64 } +func addLabelsPrefix(labels []string) []string { + out := make([]string, 0, len(labels)) + for _, l := range labels { + out = append(out, strings.Join([]string{usagePrefix, l}, "")) + } + return out +} + func newActiveSeriesTracker(userID string, trackedLabels []string, limit int, cooldownDuration time.Duration, logger log.Logger) *ActiveSeriesTracker { // Create a map for overflow labels to export when overflow happens overflowLabels := make([]string, len(trackedLabels)+2) @@ -54,11 +62,10 @@ func newActiveSeriesTracker(userID string, trackedLabels []string, limit int, co cooldownDuration: cooldownDuration, } - variableLabels := slices.Clone(trackedLabels) - variableLabels = append(variableLabels, tenantLabel, "reason") - + labelsWithPrefix := addLabelsPrefix(trackedLabels) + labelsWithPrefix = append(labelsWithPrefix, tenantLabel) ast.activeSeriesPerUserAttribution = prometheus.NewDesc("cortex_ingester_attributed_active_series", - "The total number of active series per user and attribution.", variableLabels[:len(variableLabels)-1], + "The total number of active series per user and attribution.", labelsWithPrefix, prometheus.Labels{trackerLabel: defaultTrackerName}) return ast diff --git a/pkg/costattribution/active_tracker_test.go b/pkg/costattribution/active_tracker_test.go index 68793c390f..2585947281 100644 --- a/pkg/costattribution/active_tracker_test.go +++ b/pkg/costattribution/active_tracker_test.go @@ -70,7 +70,7 @@ func TestActiveTracker_Concurrency(t *testing.T) { expectedMetrics := ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{team="__overflow__",tenant="user1",tracker="cost-attribution"} 100 + cortex_ingester_attributed_active_series{attributed_team="__overflow__",tenant="user1",tracker="cost-attribution"} 100 ` assert.NoError(t, testutil.GatherAndCompare(m.reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) diff --git a/pkg/costattribution/manager.go b/pkg/costattribution/manager.go index 5f7884f2b6..53bcd60728 100644 --- a/pkg/costattribution/manager.go +++ b/pkg/costattribution/manager.go @@ -21,6 +21,7 @@ const ( defaultTrackerName = "cost-attribution" missingValue = "__missing__" overflowValue = "__overflow__" + usagePrefix = "attributed_" ) type Manager struct { diff --git a/pkg/costattribution/manager_test.go b/pkg/costattribution/manager_test.go index 550e1d67a7..2f8db77c48 100644 --- a/pkg/costattribution/manager_test.go +++ b/pkg/costattribution/manager_test.go @@ -60,14 +60,14 @@ func TestManager_CreateDeleteTracker(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="bar",tenant="user1",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="bar",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1 # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{department="foo",service="dodo",tenant="user3",tracker="cost-attribution"} 1 + cortex_distributor_received_attributed_samples_total{attributed_department="foo",attributed_service="dodo",tenant="user3",tracker="cost-attribution"} 1 # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{team="bar",tenant="user1",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_team="bar",tenant="user1",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_distributor_received_attributed_samples_total", "cortex_ingester_attributed_active_series")) }) @@ -78,10 +78,10 @@ func TestManager_CreateDeleteTracker(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1 # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{team="bar",tenant="user1",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_team="bar",tenant="user1",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_ingester_attributed_active_series")) }) @@ -96,7 +96,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) { expectedMetrics := ` # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{department="foo",service="dodo",tenant="user3",tracker="cost-attribution"} 1 + cortex_distributor_received_attributed_samples_total{attributed_department="foo",attributed_service="dodo",tenant="user3",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total", "cortex_distributor_received_attributed_samples_total", "cortex_ingester_attributed_active_series")) }) @@ -114,7 +114,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{feature="__missing__",reason="invalid-metrics-name",team="foo",tenant="user3",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_feature="__missing__",attributed_team="foo",reason="invalid-metrics-name",tenant="user3",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total")) }) @@ -126,7 +126,7 @@ func TestManager_CreateDeleteTracker(t *testing.T) { expectedMetrics := ` # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{feature="__overflow__",team="__overflow__",tenant="user3",tracker="cost-attribution"} 2 + cortex_distributor_received_attributed_samples_total{attributed_feature="__overflow__",attributed_team="__overflow__",tenant="user3",tracker="cost-attribution"} 2 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total")) }) @@ -146,8 +146,8 @@ func TestManager_PurgeInactiveAttributionsUntil(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="invalid-metrics-name",team="foo",tenant="user1",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{department="foo",reason="out-of-window",service="bar",tenant="user3",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="foo",reason="invalid-metrics-name",tenant="user1",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_department="foo",attributed_service="bar",reason="out-of-window",tenant="user3",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total")) }) @@ -165,7 +165,7 @@ func TestManager_PurgeInactiveAttributionsUntil(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{department="foo",reason="out-of-window",service="bar",tenant="user3",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_department="foo",attributed_service="bar",reason="out-of-window",tenant="user3",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(manager.reg, strings.NewReader(expectedMetrics), "cortex_discarded_attributed_samples_total")) }) diff --git a/pkg/costattribution/sample_tracker.go b/pkg/costattribution/sample_tracker.go index bab830f8c3..54fd897dac 100644 --- a/pkg/costattribution/sample_tracker.go +++ b/pkg/costattribution/sample_tracker.go @@ -66,16 +66,17 @@ func newSampleTracker(userID string, trackedLabels []string, limit int, cooldown overflowCounter: observation{}, } - variableLabels := slices.Clone(trackedLabels) - variableLabels = append(variableLabels, tenantLabel, "reason") + labelsWithPrefix := addLabelsPrefix(trackedLabels) + labelsWithPrefix = append(labelsWithPrefix, tenantLabel, "reason") + tracker.discardedSampleAttribution = prometheus.NewDesc("cortex_discarded_attributed_samples_total", "The total number of samples that were discarded per attribution.", - variableLabels, + labelsWithPrefix, prometheus.Labels{trackerLabel: defaultTrackerName}) tracker.receivedSamplesAttribution = prometheus.NewDesc("cortex_distributor_received_attributed_samples_total", "The total number of samples that were received per attribution.", - variableLabels[:len(variableLabels)-1], + labelsWithPrefix[:len(labelsWithPrefix)-1], prometheus.Labels{trackerLabel: defaultTrackerName}) return tracker } diff --git a/pkg/costattribution/sample_tracker_test.go b/pkg/costattribution/sample_tracker_test.go index b68d4eff67..d485840f11 100644 --- a/pkg/costattribution/sample_tracker_test.go +++ b/pkg/costattribution/sample_tracker_test.go @@ -30,7 +30,7 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) { expectedMetrics := ` # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 3 + cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 3 ` assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total")) }) @@ -43,8 +43,8 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) { expectedMetrics := ` # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 6 - cortex_distributor_received_attributed_samples_total{platform="bar",tenant="user4",tracker="cost-attribution"} 5 + cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 6 + cortex_distributor_received_attributed_samples_total{attributed_platform="bar",tenant="user4",tracker="cost-attribution"} 5 ` assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total")) }) @@ -58,8 +58,8 @@ func TestSampleTracker_IncrementReceviedSamples(t *testing.T) { expectedMetrics := ` # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{platform="foo",tenant="user4",tracker="cost-attribution"} 14 - cortex_distributor_received_attributed_samples_total{platform="bar",tenant="user4",tracker="cost-attribution"} 5 + cortex_distributor_received_attributed_samples_total{attributed_platform="foo",tenant="user4",tracker="cost-attribution"} 14 + cortex_distributor_received_attributed_samples_total{attributed_platform="bar",tenant="user4",tracker="cost-attribution"} 5 ` assert.NoError(t, testutil.GatherAndCompare(tManager.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total")) }) @@ -148,11 +148,10 @@ func TestSampleTracker_Concurrency(t *testing.T) { expectedMetrics := ` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="__overflow__",team="__overflow__",tenant="user1",tracker="cost-attribution"} 95 + cortex_discarded_attributed_samples_total{attributed_team="__overflow__",reason="__overflow__",tenant="user1",tracker="cost-attribution"} 95 # HELP cortex_distributor_received_attributed_samples_total The total number of samples that were received per attribution. # TYPE cortex_distributor_received_attributed_samples_total counter - cortex_distributor_received_attributed_samples_total{team="__overflow__",tenant="user1",tracker="cost-attribution"} 95 - + cortex_distributor_received_attributed_samples_total{attributed_team="__overflow__",tenant="user1",tracker="cost-attribution"} 95 ` assert.NoError(t, testutil.GatherAndCompare(m.reg, strings.NewReader(expectedMetrics), "cortex_distributor_received_attributed_samples_total", "cortex_discarded_attributed_samples_total")) } diff --git a/pkg/distributor/validate_test.go b/pkg/distributor/validate_test.go index 594afc53cd..4148a5686e 100644 --- a/pkg/distributor/validate_test.go +++ b/pkg/distributor/validate_test.go @@ -266,14 +266,14 @@ func TestValidateLabels(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(careg, strings.NewReader(` # HELP cortex_discarded_attributed_samples_total The total number of samples that were discarded per attribution. # TYPE cortex_discarded_attributed_samples_total counter - cortex_discarded_attributed_samples_total{reason="label_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_name_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_value_invalid",team="plof",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="label_value_too_long",team="biz",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="max_label_names_per_info_series",team="a",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="max_label_names_per_series",team="plof",tenant="testUser",tracker="cost-attribution"} 1 - cortex_discarded_attributed_samples_total{reason="metric_name_invalid",team="a",tenant="testUser",tracker="cost-attribution"} 2 - cortex_discarded_attributed_samples_total{reason="missing_metric_name",team="a",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="a",reason="label_invalid",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="biz",reason="label_name_too_long",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="plof",reason="label_value_invalid",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="biz",reason="label_value_too_long",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="a",reason="max_label_names_per_info_series",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="plof",reason="max_label_names_per_series",tenant="testUser",tracker="cost-attribution"} 1 + cortex_discarded_attributed_samples_total{attributed_team="a",reason="metric_name_invalid",tenant="testUser",tracker="cost-attribution"} 2 + cortex_discarded_attributed_samples_total{attributed_team="a",reason="missing_metric_name",tenant="testUser",tracker="cost-attribution"} 1 `), "cortex_discarded_attributed_samples_total")) s.deleteUserMetrics(userID) diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index a565c2019f..3cce22c3c7 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -277,7 +277,7 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics := ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) @@ -287,8 +287,8 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics = ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="2",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="2",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) @@ -298,9 +298,9 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics = ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="2",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="3",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="2",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="3",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) @@ -311,9 +311,9 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics = ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="2",tenant="user5",tracker="cost-attribution"} 2 - cortex_ingester_attributed_active_series{a="3",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="2",tenant="user5",tracker="cost-attribution"} 2 + cortex_ingester_attributed_active_series{attributed_a="3",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) @@ -323,10 +323,10 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics = ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="2",tenant="user5",tracker="cost-attribution"} 2 - cortex_ingester_attributed_active_series{a="3",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="4",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="2",tenant="user5",tracker="cost-attribution"} 2 + cortex_ingester_attributed_active_series{attributed_a="3",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="4",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) @@ -336,11 +336,11 @@ func testCostAttributionUpdateSeries(t *testing.T, c *ActiveSeries, reg *prometh expectedMetrics = ` # HELP cortex_ingester_attributed_active_series The total number of active series per user and attribution. # TYPE cortex_ingester_attributed_active_series gauge - cortex_ingester_attributed_active_series{a="1",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="2",tenant="user5",tracker="cost-attribution"} 2 - cortex_ingester_attributed_active_series{a="3",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="4",tenant="user5",tracker="cost-attribution"} 1 - cortex_ingester_attributed_active_series{a="5",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="1",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="2",tenant="user5",tracker="cost-attribution"} 2 + cortex_ingester_attributed_active_series{attributed_a="3",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="4",tenant="user5",tracker="cost-attribution"} 1 + cortex_ingester_attributed_active_series{attributed_a="5",tenant="user5",tracker="cost-attribution"} 1 ` assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expectedMetrics), "cortex_ingester_attributed_active_series")) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 4e771bfb99..b3b8895061 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -317,7 +317,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.StringVar(&l.SeparateMetricsGroupLabel, "validation.separate-metrics-group-label", "", "Label used to define the group label for metrics separation. For each write request, the group is obtained from the first non-empty group label from the first timeseries in the incoming list of timeseries. Specific distributor and ingester metrics will be further separated adding a 'group' label with group label's value. Currently applies to the following metrics: cortex_discarded_samples_total") - f.Var(&l.CostAttributionLabels, costAttributionLabelsFlag, "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{team='frontend', service='api'}.") + f.Var(&l.CostAttributionLabels, costAttributionLabelsFlag, "Defines labels for cost attribution. Applies to metrics like cortex_distributor_received_attributed_samples_total. To disable, set to an empty string. For example, 'team,service' produces metrics such as cortex_distributor_received_attributed_samples_total{attributed_team='frontend', attributed_service='api'}.") f.IntVar(&l.MaxCostAttributionLabelsPerUser, maxCostAttributionLabelsPerUserFlag, 2, "Maximum number of cost attribution labels allowed per user, the value is capped at 4.") f.IntVar(&l.MaxCostAttributionCardinalityPerUser, "validation.max-cost-attribution-cardinality-per-user", 10000, "Maximum cardinality of cost attribution labels allowed per user.") f.Var(&l.CostAttributionCooldown, "validation.cost-attribution-cooldown", "Defines how long cost attribution stays in overflow before attempting a reset, with received/discarded samples extending the cooldown if overflow persists, while active series reset and restart tracking after the cooldown.")