From 17b4dfe53206cbc0d0e6eed0a0cde2d9d38b9005 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Mon, 13 Sep 2021 10:59:29 +0200 Subject: [PATCH 1/2] Split limits config into a variable we can reuse Signed-off-by: Goutham Veeramachaneni --- cortex/config.libsonnet | 23 +++++++++++++++++++++++ cortex/distributor.libsonnet | 3 +-- cortex/ingester.libsonnet | 7 +------ cortex/ruler.libsonnet | 2 -- cortex/tsdb.libsonnet | 2 +- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 3c4cf545..2c27e9dc 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -312,6 +312,29 @@ // These are the defaults. limits: $._config.overrides.extra_small_user, + // These are all the flags for the default limits. + distributorLimitsConfig: { + 'distributor.ingestion-rate-limit': $._config.limits.ingestion_rate, + 'distributor.ingestion-burst-size': $._config.limits.ingestion_burst_size, + }, + ingesterLimitsConfig: { + 'ingester.max-series-per-user': $._config.limits.max_series_per_user, + 'ingester.max-series-per-metric': $._config.limits.max_series_per_metric, + 'ingester.max-global-series-per-user': $._config.limits.max_global_series_per_user, + 'ingester.max-global-series-per-metric': $._config.limits.max_global_series_per_metric, + 'ingester.max-series-per-query': $._config.limits.max_series_per_query, + 'ingester.max-samples-per-query': $._config.limits.max_samples_per_query, + }, + rulerLimitsConfig: { + 'ruler.max-rules-per-rule-group': $._config.limits.ruler_max_rules_per_rule_group, + 'ruler.max-rule-groups-per-tenant': $._config.limits.ruler_max_rule_groups_per_tenant, + }, + compactorLimitsConfig: { + 'compactor.blocks-retention-period': $._config.limits.compactor_blocks_retention_period, + }, + + limitsConfig: self.distributorLimitsConfig + self.ingesterLimitsConfig + self.rulerLimitsConfig + self.compactorLimitsConfig, + overrides_configmap: 'overrides', overrides: { diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet index 02c8767b..c9df411c 100644 --- a/cortex/distributor.libsonnet +++ b/cortex/distributor.libsonnet @@ -6,6 +6,7 @@ $._config.grpcConfig + $._config.ringConfig + $._config.distributorConfig + + $._config.distributorLimitsConfig + { target: 'distributor', @@ -30,8 +31,6 @@ 'server.grpc.keepalive.max-connection-idle': '1m', 'distributor.ingestion-rate-limit-strategy': 'global', - 'distributor.ingestion-rate-limit': $._config.limits.ingestion_rate, - 'distributor.ingestion-burst-size': $._config.limits.ingestion_burst_size, // The ingestion rate global limit requires the distributors to form a ring. 'distributor.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index e0753d84..3078db36 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -6,6 +6,7 @@ $._config.storageConfig + $._config.blocksStorageConfig + $._config.distributorConfig + // This adds the distributor ring flags to the ingester. + $._config.ingesterLimitsConfig + { target: 'ingester', @@ -24,12 +25,6 @@ // Limits config. 'ingester.max-chunk-idle': $._config.max_chunk_idle, - 'ingester.max-series-per-user': $._config.limits.max_series_per_user, - 'ingester.max-series-per-metric': $._config.limits.max_series_per_metric, - 'ingester.max-global-series-per-user': $._config.limits.max_global_series_per_user, - 'ingester.max-global-series-per-metric': $._config.limits.max_global_series_per_metric, - 'ingester.max-series-per-query': $._config.limits.max_series_per_query, - 'ingester.max-samples-per-query': $._config.limits.max_samples_per_query, 'runtime-config.file': '/etc/cortex/overrides.yaml', 'server.grpc-max-concurrent-streams': 10000, 'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024, diff --git a/cortex/ruler.libsonnet b/cortex/ruler.libsonnet index dfb5727e..c503defe 100644 --- a/cortex/ruler.libsonnet +++ b/cortex/ruler.libsonnet @@ -22,8 +22,6 @@ 'ruler.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, // Limits - 'ruler.max-rules-per-rule-group': $._config.limits.ruler_max_rules_per_rule_group, - 'ruler.max-rule-groups-per-tenant': $._config.limits.ruler_max_rule_groups_per_tenant, 'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024, 'server.grpc-max-recv-msg-size-bytes': 10 * 1024 * 1024, diff --git a/cortex/tsdb.libsonnet b/cortex/tsdb.libsonnet index 2a91f8b4..1ce3c028 100644 --- a/cortex/tsdb.libsonnet +++ b/cortex/tsdb.libsonnet @@ -144,6 +144,7 @@ $._config.grpcConfig + $._config.storageConfig + $._config.blocksStorageConfig + + $._config.compactorLimitsConfig + { target: 'compactor', @@ -162,7 +163,6 @@ // Limits config. 'runtime-config.file': '/etc/cortex/overrides.yaml', - 'compactor.blocks-retention-period': $._config.limits.compactor_blocks_retention_period, }, compactor_ports:: $.util.defaultPorts, From 6ed56f7e1e0946d0ee288fb1fd389bd735c66815 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Mon, 13 Sep 2021 15:17:00 +0200 Subject: [PATCH 2/2] Review feedback Signed-off-by: Goutham Veeramachaneni --- cortex/config.libsonnet | 1 + cortex/distributor.libsonnet | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 2c27e9dc..6369513c 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -314,6 +314,7 @@ // These are all the flags for the default limits. distributorLimitsConfig: { + 'distributor.ingestion-rate-limit-strategy': 'global', 'distributor.ingestion-rate-limit': $._config.limits.ingestion_rate, 'distributor.ingestion-burst-size': $._config.limits.ingestion_burst_size, }, diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet index c9df411c..436c7fc8 100644 --- a/cortex/distributor.libsonnet +++ b/cortex/distributor.libsonnet @@ -30,8 +30,6 @@ 'server.grpc.keepalive.max-connection-age-grace': '5m', 'server.grpc.keepalive.max-connection-idle': '1m', - 'distributor.ingestion-rate-limit-strategy': 'global', - // The ingestion rate global limit requires the distributors to form a ring. 'distributor.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, 'distributor.ring.prefix': '',