Introduce ingester instance limits to configuration, and add alerts. (#…

…296) * Introduce ingester instance limits to configuration, and add alerts. * CHANGELOG.md * Address (internal) review feedback.
grafana · Apr 22, 2021 · ce896a7 · ce896a7
1 parent 410f9fe
commit ce896a7
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@
 * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287
 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287
 * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294
+* [ENHANCEMENT] Added ingester instance limits to config. #296
+* [ENHANCEMENT] Added alerts for ingester reaching the instance limits (if limits are configured). #296:
+  * `CortexIngesterReachingSeriesLimit`
+  * `CortexIngesterReachingTenantsLimit`
 * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288
 
 ## 1.8.0 / 2021-03-25

diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet
@@ -255,6 +255,87 @@
         },
       ],
     },
+    {
+      name: 'cortex_ingester_instance_alerts',
+      rules: [
+        {
+          alert: 'CortexIngesterReachingSeriesLimit',
+          expr: |||
+            (
+                (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"})
+                and ignoring (limit)
+                (cortex_ingester_instance_limits{limit="max_series"} > 0)
+            ) > 0.7
+          |||,
+          'for': '5m',
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexIngesterReachingSeriesLimit',
+          expr: |||
+            (
+                (cortex_ingester_memory_series / ignoring(limit) cortex_ingester_instance_limits{limit="max_series"})
+                and ignoring (limit)
+                (cortex_ingester_instance_limits{limit="max_series"} > 0)
+            ) > 0.8
+          |||,
+          'for': '5m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexIngesterReachingTenantsLimit',
+          expr: |||
+            (
+                (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"})
+                and ignoring (limit)
+                (cortex_ingester_instance_limits{limit="max_tenants"} > 0)
+            ) > 0.7
+          |||,
+          'for': '5m',
+          labels: {
+            severity: 'warning',
+          },
+          annotations: {
+            message: |||
+              Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit.
+            |||,
+          },
+        },
+        {
+          alert: 'CortexIngesterReachingTenantsLimit',
+          expr: |||
+            (
+                (cortex_ingester_memory_users / ignoring(limit) cortex_ingester_instance_limits{limit="max_tenants"})
+                and ignoring (limit)
+                (cortex_ingester_instance_limits{limit="max_tenants"} > 0)
+            ) > 0.8
+          |||,
+          'for': '5m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Ingester {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its tenant limit.
+            |||,
+          },
+        },
+      ],
+    },
     {
       name: 'cortex_wal_alerts',
       rules: [

diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet
@@ -436,6 +436,15 @@
 
     // Enables streaming of chunks from ingesters using blocks.
     ingester_stream_chunks_when_using_blocks: true,
+
+    // Ingester limits are put directly into runtime config, if not null. Available limits:
+    //    ingester_instance_limits: {
+    //      max_inflight_push_requests: 0,  // Max inflight push requests per ingester. 0 = no limit.
+    //      max_ingestion_rate: 0,  // Max ingestion rate (samples/second) per ingester. 0 = no limit.
+    //      max_series: 0,  // Max number of series per ingester. 0 = no limit.
+    //      max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
+    //    },
+    ingester_instance_limits: null,
   },
 
   local configMap = $.core.v1.configMap,
@@ -447,6 +456,7 @@
         { overrides: $._config.overrides }
         + (if std.length($._config.multi_kv_config) > 0 then { multi_kv_config: $._config.multi_kv_config } else {})
         + (if $._config.ingester_stream_chunks_when_using_blocks then { ingester_stream_chunks_when_using_blocks: true } else {})
+        + (if $._config.ingester_instance_limits != null then { ingester_limits: $._config.ingester_instance_limits } else {}),
       ),
     }),