Merge pull request #412 from aallawala/aja_bugfix_alerts

Span the annotation.message in alerts as YAML multiline strings.
grafana · Nov 2, 2021 · 591e1f8 · 591e1f8
2 parents 92f3b64 + d6bdd88
commit 591e1f8
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -81,6 +81,7 @@
 * [BUGFIX] Upstream recording rule `node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate` renamed. #379
 * [BUGFIX] Treat `compactor_blocks_retention_period` type as string rather than int.#395
 * [BUGFIX] Fixed writes/reads/alertmanager resources dashboards to use `$._config.job_names.gateway`. #403
+* [BUGFIX] Span the annotation.message in alerts as YAML multiline strings. #412
 
 ## 1.9.0 / 2021-05-18
 

diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet
@@ -18,7 +18,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).' % $._config,
+            message: |||
+              Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).
+            ||| % $._config,
           },
         },
         {
@@ -199,7 +201,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.
+            |||,
           },
         },
         {
@@ -212,7 +216,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} transfer failed.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} transfer failed.
+            |||,
           },
         },
         {
@@ -266,7 +272,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.
+            |||,
           },
         },
       ],
@@ -707,7 +715,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.' % $._config,
+            message: |||
+              Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.
+            ||| % $._config,
           },
         },
       ],

diff --git a/cortex-mixin/alerts/blocks.libsonnet b/cortex-mixin/alerts/blocks.libsonnet
@@ -26,7 +26,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
+            ||| % $._config,
           },
         },
         {
@@ -43,7 +45,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
+            ||| % $._config,
           },
         },
         {
@@ -61,7 +65,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: "Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet." % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet.
+            ||| % $._config,
           },
         },
         {
@@ -77,7 +83,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.
+            ||| % $._config,
           },
         },
         {
@@ -89,7 +97,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.
+            ||| % $._config,
           },
         },
         {
@@ -101,7 +111,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.
+            ||| % $._config,
           },
         },
         {
@@ -113,7 +125,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.
+            ||| % $._config,
           },
         },
         {
@@ -125,7 +139,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -137,7 +153,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -150,7 +168,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -166,7 +186,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -190,7 +212,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.' % $._config,
+            message: |||
+              Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.
+            ||| % $._config,
           },
         },
         {
@@ -206,7 +230,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -219,7 +245,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -233,7 +261,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.' % $._config,
+            message: |||
+              Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.
+            ||| % $._config,
           },
         },
       ],

diff --git a/cortex-mixin/alerts/compactor.libsonnet b/cortex-mixin/alerts/compactor.libsonnet
@@ -14,7 +14,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.
+            ||| % $._config,
           },
         },
         {
@@ -30,7 +32,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -44,7 +48,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -57,7 +63,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.
+            ||| % $._config,
           },
         },
         {
@@ -73,7 +81,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -87,7 +97,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
+            ||| % $._config,
           },
         },
       ],