From 33dd2e98fe2a94a875ec191a6dff5b8ecaec72f6 Mon Sep 17 00:00:00 2001
From: "Abdurrahman J. Allawala" <aallawala1@gmail.com>
Date: Mon, 18 Oct 2021 15:11:30 -0700
Subject: [PATCH 1/5] multiline annotation for CortexIngesterUnhealthy alert

---
 cortex-mixin/alerts/alerts.libsonnet | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet
index 59022dd..245aa6e 100644
--- a/cortex-mixin/alerts/alerts.libsonnet
+++ b/cortex-mixin/alerts/alerts.libsonnet
@@ -18,7 +18,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).' % $._config,
+            message: |||
+              Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).
+            ||| % $._config,
           },
         },
         {

From 929623fd40deb7626dcdb57509551779ba192c8b Mon Sep 17 00:00:00 2001
From: "Abdurrahman J. Allawala" <aallawala1@gmail.com>
Date: Mon, 18 Oct 2021 15:24:24 -0700
Subject: [PATCH 2/5] add changelog entry

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 698d38c..694d783 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -77,6 +77,7 @@
 * [BUGFIX] Upstream recording rule `node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate` renamed. #379
 * [BUGFIX] Treat `compactor_blocks_retention_period` type as string rather than int.#395
 * [BUGFIX] Fixed writes/reads/alertmanager resources dashboards to use `$._config.job_names.gateway`. #403
+* [BUGFIX] Span the annotation.message field for `CortexIngesterUnhealthy` alert in one line. #412
 
 ## 1.9.0 / 2021-05-18
 

From 181f725ade4be895b8707b7bddc4c84d7317642b Mon Sep 17 00:00:00 2001
From: "Abdurrahman J. Allawala" <aallawala1@gmail.com>
Date: Mon, 18 Oct 2021 15:44:08 -0700
Subject: [PATCH 3/5] use multiline wrapper for the rest of annotation.message
 templates

---
 cortex-mixin/alerts/alerts.libsonnet | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet
index 245aa6e..3ab6751 100644
--- a/cortex-mixin/alerts/alerts.libsonnet
+++ b/cortex-mixin/alerts/alerts.libsonnet
@@ -201,7 +201,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.
+            |||,
           },
         },
         {
@@ -214,7 +216,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} transfer failed.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} transfer failed.
+            |||,
           },
         },
         {
@@ -268,7 +272,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: '{{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.',
+            message: |||
+              {{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.
+            |||,
           },
         },
       ],
@@ -690,7 +696,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.' % $._config,
+            message: |||
+              Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.
+            ||| % $._config,
           },
         },
       ],

From 8ad55b92a070009a544f6448d8956509f16a1b7f Mon Sep 17 00:00:00 2001
From: "Abdurrahman J. Allawala" <aallawala1@gmail.com>
Date: Mon, 18 Oct 2021 15:45:26 -0700
Subject: [PATCH 4/5] update changelog

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 694d783..ee1df28 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -77,7 +77,7 @@
 * [BUGFIX] Upstream recording rule `node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate` renamed. #379
 * [BUGFIX] Treat `compactor_blocks_retention_period` type as string rather than int.#395
 * [BUGFIX] Fixed writes/reads/alertmanager resources dashboards to use `$._config.job_names.gateway`. #403
-* [BUGFIX] Span the annotation.message field for `CortexIngesterUnhealthy` alert in one line. #412
+* [BUGFIX] Span the annotation.message in alerts as YAML multiline strings. #412
 
 ## 1.9.0 / 2021-05-18
 

From d6bdd88245d9aedfe4f5393eecdff6f420d4d5f3 Mon Sep 17 00:00:00 2001
From: "Abdurrahman J. Allawala" <aallawala1@gmail.com>
Date: Mon, 18 Oct 2021 16:08:15 -0700
Subject: [PATCH 5/5] add in compactor|blocks.libsonnet as well

---
 cortex-mixin/alerts/blocks.libsonnet    | 60 ++++++++++++++++++-------
 cortex-mixin/alerts/compactor.libsonnet | 24 +++++++---
 2 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/cortex-mixin/alerts/blocks.libsonnet b/cortex-mixin/alerts/blocks.libsonnet
index a60ac2d..428418b 100644
--- a/cortex-mixin/alerts/blocks.libsonnet
+++ b/cortex-mixin/alerts/blocks.libsonnet
@@ -26,7 +26,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
+            ||| % $._config,
           },
         },
         {
@@ -43,7 +45,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
+            ||| % $._config,
           },
         },
         {
@@ -61,7 +65,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: "Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet." % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet.
+            ||| % $._config,
           },
         },
         {
@@ -77,7 +83,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.
+            ||| % $._config,
           },
         },
         {
@@ -89,7 +97,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.
+            ||| % $._config,
           },
         },
         {
@@ -101,7 +111,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.
+            ||| % $._config,
           },
         },
         {
@@ -113,7 +125,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.
+            ||| % $._config,
           },
         },
         {
@@ -125,7 +139,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -137,7 +153,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -150,7 +168,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.' % $._config,
+            message: |||
+              Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.
+            ||| % $._config,
           },
         },
         {
@@ -166,7 +186,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -190,7 +212,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.' % $._config,
+            message: |||
+              Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.
+            ||| % $._config,
           },
         },
         {
@@ -206,7 +230,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -219,7 +245,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.' % $._config,
+            message: |||
+              Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.
+            ||| % $._config,
           },
         },
         {
@@ -233,7 +261,9 @@
             severity: 'warning',
           },
           annotations: {
-            message: 'Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.' % $._config,
+            message: |||
+              Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.
+            ||| % $._config,
           },
         },
       ],
diff --git a/cortex-mixin/alerts/compactor.libsonnet b/cortex-mixin/alerts/compactor.libsonnet
index 5538545..73e50a4 100644
--- a/cortex-mixin/alerts/compactor.libsonnet
+++ b/cortex-mixin/alerts/compactor.libsonnet
@@ -14,7 +14,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.
+            ||| % $._config,
           },
         },
         {
@@ -30,7 +32,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -44,7 +48,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -57,7 +63,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.
+            ||| % $._config,
           },
         },
         {
@@ -73,7 +81,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
+            ||| % $._config,
           },
         },
         {
@@ -87,7 +97,9 @@
             severity: 'critical',
           },
           annotations: {
-            message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
+            message: |||
+              Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
+            ||| % $._config,
           },
         },
       ],