Merge pull request #294 from grafana/fix-compactor-alert

Fix CortexCompactorHasNotSuccessfullyRunCompaction to avoid false positives
grafana · Apr 21, 2021 · 410f9fe · 410f9fe
2 parents 1fb3caa + 90f451e
commit 410f9fe
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 * [FEATURE] Added "Cortex / Rollout progress" dashboard. #289 #290
 * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287
 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287
+* [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294
 * [BUGFIX] Fixed `CortexCompactorRunFailed` false positives. #288
 
 ## 1.8.0 / 2021-03-25

diff --git a/cortex-mixin/alerts/compactor.libsonnet b/cortex-mixin/alerts/compactor.libsonnet
@@ -18,17 +18,33 @@
           },
         },
         {
-          // Alert if the compactor has not successfully run compaction in the last 6h.
+          // Alert if the compactor has not successfully run compaction in the last 24h.
           alert: 'CortexCompactorHasNotSuccessfullyRunCompaction',
           'for': '1h',
           expr: |||
-            time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 6
+            (time() - cortex_compactor_last_successful_run_timestamp_seconds > 60 * 60 * 24)
+            and
+            (cortex_compactor_last_successful_run_timestamp_seconds > 0)
+          |||,
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 24 hours.',
+          },
+        },
+        {
+          // Alert if the compactor has not successfully run compaction in the last 24h since startup.
+          alert: 'CortexCompactorHasNotSuccessfullyRunCompaction',
+          'for': '24h',
+          expr: |||
+            cortex_compactor_last_successful_run_timestamp_seconds == 0
           |||,
           labels: {
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 6 hours.',
+            message: 'Cortex Compactor {{ $labels.namespace }}/{{ $labels.instance }} has not run compaction in the last 24 hours.',
           },
         },
         {