From 40dbe6aa0aa5142654333c51444d300ba7efd986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 22 Apr 2021 09:59:35 +0200 Subject: [PATCH 1/3] Introduce ingester instance limits to configuration, and add alerts. --- cortex-mixin/alerts/alerts.libsonnet | 81 ++++++++++++++++++++++++++++ cortex/config.libsonnet | 10 ++++ 2 files changed, 91 insertions(+) diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index cad56a1b..91da0a68 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -255,6 +255,87 @@ }, ], }, + { + name: 'cortex_ingester_instance_alerts', + rules: [ + { + alert: 'CortexIngesterReachedSeriesLimitWarning', + expr: ||| + ( + (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_series"} > 0) + ) > 0.7 + |||, + 'for': '5m', + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachedSeriesLimitCritical', + expr: ||| + ( + (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_series"} > 0) + ) > 0.8 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachedTenantsLimitWarning', + expr: ||| + ( + (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_tenants"} > 0) + ) > 0.7 + |||, + 'for': '5m', + labels: { + severity: 'warning', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit. + |||, + }, + }, + { + alert: 'CortexIngesterReachedTenantsLimitCritical', + expr: ||| + ( + (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"}) + and ignoring (limit) + (cortex_ingester_instance_limits{limit="max_tenants"} > 0) + ) > 0.8 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + message: ||| + Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit. + |||, + }, + }, + ], + }, { name: 'cortex_wal_alerts', rules: [ diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 14fc54f9..c4173bd0 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -436,6 +436,15 @@ // Enables streaming of chunks from ingesters using blocks. ingester_stream_chunks_when_using_blocks: true, + + // Ingester limits are put directly into runtime config, if not null. Available limits: + // ingester_instance_limits: { + // max_inflight_push_requests: 0, // Max inflight push requests per ingester. 0 = no limit. + // max_ingestion_rate: 0, // Max ingestion rate (samples/second) per ingester. 0 = no limit. + // max_series: 0, // Max number of series per ingester. 0 = no limit. + // max_tenants: 0, // Max number of tenants per ingester. 0 = no limit. + // }, + ingester_instance_limits: null, }, local configMap = $.core.v1.configMap, @@ -447,6 +456,7 @@ { overrides: $._config.overrides } + (if std.length($._config.multi_kv_config) > 0 then { multi_kv_config: $._config.multi_kv_config } else {}) + (if $._config.ingester_stream_chunks_when_using_blocks then { ingester_stream_chunks_when_using_blocks: true } else {}) + + (if $._config.ingester_instance_limits != null then { ingester_limits: $._config.ingester_instance_limits } else {}), ), }), From 4fdcb8199a37a4ea6615aa826b63fa566416a697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 22 Apr 2021 10:02:26 +0200 Subject: [PATCH 2/3] CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 317993ca..34a00f09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287 * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294 +* [ENHANCEMENT] Added ingester instance limits to config. #296 +* [ENHANCEMENT] Added alerts for ingester reaching the instance limits (if limits are configured). #296: + * `CortexIngesterReachedSeriesLimitWarning` + * `CortexIngesterReachedSeriesLimitCritical` + * `CortexIngesterReachedTenantsLimitWarning` + * `CortexIngesterReachedTenantsLimitCritical` * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288 ## 1.8.0 / 2021-03-25 From c437a55cd597350d43f51a29505dc41b0c88411f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Thu, 22 Apr 2021 10:12:45 +0200 Subject: [PATCH 3/3] Address (internal) review feedback. --- CHANGELOG.md | 6 ++---- cortex-mixin/alerts/alerts.libsonnet | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34a00f09..76dd06a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,8 @@ * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294 * [ENHANCEMENT] Added ingester instance limits to config. #296 * [ENHANCEMENT] Added alerts for ingester reaching the instance limits (if limits are configured). #296: - * `CortexIngesterReachedSeriesLimitWarning` - * `CortexIngesterReachedSeriesLimitCritical` - * `CortexIngesterReachedTenantsLimitWarning` - * `CortexIngesterReachedTenantsLimitCritical` + * `CortexIngesterReachingSeriesLimit` + * `CortexIngesterReachingTenantsLimit` * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288 ## 1.8.0 / 2021-03-25 diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index 91da0a68..c5211ca6 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -259,7 +259,7 @@ name: 'cortex_ingester_instance_alerts', rules: [ { - alert: 'CortexIngesterReachedSeriesLimitWarning', + alert: 'CortexIngesterReachingSeriesLimit', expr: ||| ( (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) @@ -278,7 +278,7 @@ }, }, { - alert: 'CortexIngesterReachedSeriesLimitCritical', + alert: 'CortexIngesterReachingSeriesLimit', expr: ||| ( (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"}) @@ -297,7 +297,7 @@ }, }, { - alert: 'CortexIngesterReachedTenantsLimitWarning', + alert: 'CortexIngesterReachingTenantsLimit', expr: ||| ( (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"}) @@ -316,7 +316,7 @@ }, }, { - alert: 'CortexIngesterReachedTenantsLimitCritical', + alert: 'CortexIngesterReachingTenantsLimit', expr: ||| ( (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"})