diff --git a/CHANGELOG.md b/CHANGELOG.md index 317993ca..76dd06a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287 * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294 +* [ENHANCEMENT] Added ingester instance limits to config. #296 +* [ENHANCEMENT] Added alerts for ingester reaching the instance limits (if limits are configured). #296: + * `CortexIngesterReachingSeriesLimit` + * `CortexIngesterReachingTenantsLimit` * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288 ## 1.8.0 / 2021-03-25 diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index cad56a1b..c5211ca6 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -255,6 +255,87 @@ }, ], }, + { + name: 'cortex_ingester_instance_alerts', + rules: [ + { + alert: 'CortexIngesterReachingSeriesLimit', + expr: ||| + ( + (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_series"} > 0) + ) > 0.7 + |||, + 'for': '5m', + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachingSeriesLimit', + expr: ||| + ( + (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_series"} > 0) + ) > 0.8 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachingTenantsLimit', + expr: ||| + ( + (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_tenants"} > 0) + ) > 0.7 + |||, + 'for': '5m', + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachingTenantsLimit', + expr: ||| + ( + (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_tenants"} > 0) + ) > 0.8 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit. + |||, + }, + }, + ], + }, { name: 'cortex_wal_alerts', rules: [ diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 14fc54f9..c4173bd0 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -436,6 +436,15 @@ // Enables streaming of chunks from ingesters using blocks. ingester_stream_chunks_when_using_blocks: true, + + // Ingester limits are put directly into runtime config, if not null. Available limits: + // ingester_instance_limits: { + // max_inflight_push_requests: 0, // Max inflight push requests per ingester. 0 = no limit. + // max_ingestion_rate: 0, // Max ingestion rate (samples/second) per ingester. 0 = no limit. + // max_series: 0, // Max number of series per ingester. 0 = no limit. + // max_tenants: 0, // Max number of tenants per ingester. 0 = no limit. + // }, + ingester_instance_limits: null, }, local configMap = $.core.v1.configMap, @@ -447,6 +456,7 @@ { overrides: $._config.overrides } + (if std.length($._config.multi_kv_config) > 0 then { multi_kv_config: $._config.multi_kv_config } else {}) + (if $._config.ingester_stream_chunks_when_using_blocks then { ingester_stream_chunks_when_using_blocks: true } else {}) + + (if $._config.ingester_instance_limits != null then { ingester_limits: $._config.ingester_instance_limits } else {}), ), }),