Test and fix custom labels support in mixin

.github/workflows/test-build-deploy.yml

@@ -62,6 +62,8 @@ jobs:
           ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir
       - name: Check Mixin
         run: make BUILD_IN_CONTAINER=false check-mixin
+      - name: Check Mixin Tests
+        run: make BUILD_IN_CONTAINER=false check-mixin-tests
       - name: Check Jsonnet Manifests
         run: make BUILD_IN_CONTAINER=false check-jsonnet-manifests
       - name: Check Jsonnet Getting Started

CHANGELOG.md

@@ -51,6 +51,7 @@
   - `MimirContinuousTestFailed`
 * [ENHANCEMENT] Added `per_cluster_label` support to allow to change the label name used to differentiate between Kubernetes clusters. #1651
 * [BUGFIX] Dashboards: Fix "Failed evaluation rate" panel on Tenants dashboard. #1629
+* [BUGFIX] Honor the configured `per_instance_label` in all dashboards and alerts. #1697
 
 ### Jsonnet
 

Makefile

@@ -207,7 +207,7 @@ GOVOLUMES=	-v $(shell pwd)/.cache:/go/cache:delegated,z \
 # Mount local ssh credentials to be able to clone private repos when doing `mod-check`
 SSHVOLUME=  -v ~/.ssh/:/root/.ssh:delegated,z
 
-exes $(EXES) protos $(PROTO_GOS) lint test test-with-race cover shell mod-check check-protos doc format dist build-mixin format-mixin: fetch-build-image
+exes $(EXES) protos $(PROTO_GOS) lint test test-with-race cover shell mod-check check-protos doc format dist build-mixin format-mixin check-mixin-tests license check-license: fetch-build-image
 	@mkdir -p $(shell pwd)/.pkg
 	@mkdir -p $(shell pwd)/.cache
 	@echo
@@ -331,7 +331,7 @@ doc: clean-doc $(DOC_TEMPLATES:.template=.md) $(DOC_EMBED:.md=.md.embedmd)
 
 # Add license header to files.
 license:
-	go run ./tools/add-license ./cmd ./integration ./pkg ./tools ./development ./mimir-build-image
+	go run ./tools/add-license ./cmd ./integration ./pkg ./tools ./development ./mimir-build-image ./operations
 
 check-license: license
 	@git diff --exit-code || (echo "Please add the license header running 'make BUILD_IN_CONTAINER=false license'" && false)
@@ -375,6 +375,9 @@ build-mixin: check-mixin-jb
 	@cd $(MIXIN_OUT_PATH)/.. && zip -q -r mimir-mixin.zip $$(basename "$(MIXIN_OUT_PATH)")
 	@echo "The mixin has been compiled to $(MIXIN_OUT_PATH) and archived to $$(realpath --relative-to=$$(pwd) $(MIXIN_OUT_PATH)/../mimir-mixin.zip)"
 
+check-mixin-tests:
+	@./operations/mimir-mixin-tests/run.sh || (echo "Mixin tests are failing. Please fix the reported issues. You can run mixin tests with 'make check-mixin-tests'" && false)
+
 format-mixin:
 	@find $(MIXIN_PATH) -type f -name '*.libsonnet' | xargs jsonnetfmt -i
 

operations/mimir-mixin-tests/.gitignore

@@ -0,0 +1,2 @@
+# Any test output directory.
+test-*/

operations/mimir-mixin-tests/run.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+MIXIN_DIR="${SCRIPT_DIR}/../mimir-mixin"
+MIXIN_TEST_FILEPATH="${MIXIN_DIR}/mixin-test.libsonnet"
+
+# Cleanup temporary copy of test file when this script exits.
+trap "rm -f ${MIXIN_TEST_FILEPATH}" EXIT
+
+TESTS=$(ls -1 "${SCRIPT_DIR}"/test-*.libsonnet)
+
+for FILEPATH in $TESTS; do
+  # Extract the filename (without extension).
+  TEST_NAME=$(basename -s '.libsonnet' "$FILEPATH")
+  TEST_DIR="${SCRIPT_DIR}/${TEST_NAME}"
+
+  # Begin with a clean output dir.
+  rm -rf "${TEST_DIR}" && mkdir "${TEST_DIR}"
+
+  # Temporarily copy the test file to the mixin directory. This file is deleted once
+  # this script exits.
+  cp "${SCRIPT_DIR}/${TEST_NAME}.libsonnet" "${MIXIN_TEST_FILEPATH}"
+
+  mixtool generate all \
+    --output-alerts "${TEST_DIR}/alerts.yaml" \
+    --output-rules "${TEST_DIR}/rules.yaml" \
+    --directory "${TEST_DIR}/dashboards" \
+    "${MIXIN_TEST_FILEPATH}"
+
+  # Run assertions.
+  "${SCRIPT_DIR}"/"${TEST_NAME}-asserts.sh"
+done

operations/mimir-mixin-tests/test-custom-labels-asserts.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+DASHBOARDS_DIR="${SCRIPT_DIR}"/test-custom-labels/dashboards
+DASHBOARDS_FILES=$(ls -1 "${DASHBOARDS_DIR}"/*.json)
+ALERTS_FILE="${SCRIPT_DIR}"/test-custom-labels/alerts.yaml
+RULES_FILE="${SCRIPT_DIR}"/test-custom-labels/rules.yaml
+FAILED=0
+
+assert_failed() {
+  MSG=$1
+
+  echo ""
+  echo -e "$MSG"
+  echo ""
+  FAILED=1
+}
+
+# In this test we customize some labels. We expect to not find any reference to default labels.
+FORBIDDEN_LABELS="cluster instance pod"
+
+for LABEL in ${FORBIDDEN_LABELS}; do
+  QUERY_REGEX="[^\$a-z_]${LABEL}[^a-z_]"
+  ALERT_VARIABLE_REGEX="\\\$labels.${LABEL}"
+  RULE_PREFIX_REGEX="(^|[^a-z])${LABEL}(\$|[^a-z])"
+
+  # Check dashboards.
+  for FILEPATH in ${DASHBOARDS_FILES}; do
+    echo "Checking ${FILEPATH}"
+
+    MATCHES=$(jq '.. | select(.expr?).expr' "${FILEPATH}" | grep -E "${QUERY_REGEX}" || true)
+    if [ -n "$MATCHES" ]; then
+      assert_failed "The dashboard at ${FILEPATH} contains unexpected references to '${LABEL}' label in some queries:\n$MATCHES"
+    fi
+  done
+
+  # Check alerts.
+  echo "Checking ${ALERTS_FILE}"
+  MATCHES=$(yq eval '.. | select(.expr?).expr | sub("\n", " ")' "${ALERTS_FILE}" | grep -E "${QUERY_REGEX}" || true)
+  if [ -n "$MATCHES" ]; then
+    assert_failed "The alerts at ${ALERTS_FILE} contains unexpected references to '${LABEL}' label in some queries:\n$MATCHES"
+  fi
+
+  MATCHES=$(yq eval '.. | select(.message?).message | sub("\n", " ")' "${ALERTS_FILE}" | grep -E "${ALERT_VARIABLE_REGEX}" || true)
+  if [ -n "$MATCHES" ]; then
+    assert_failed "The alerts at ${ALERTS_FILE} contains unexpected references to '${LABEL}' label in some messages:\n$MATCHES"
+  fi
+
+  # Check rules.
+  echo "Checking ${RULES_FILE}"
+  MATCHES=$(yq eval '.. | select(.expr?).expr | sub("\n", " ")' "${RULES_FILE}" | grep -E "${QUERY_REGEX}" || true)
+  if [ -n "$MATCHES" ]; then
+    assert_failed "The rules at ${RULES_FILE} contains unexpected references to '${LABEL}' label in some queries:\n$MATCHES"
+  fi
+
+  MATCHES=$(yq eval '.. | select(.record?).record | sub("\n", " ")' "${RULES_FILE}" | grep -Eo '^[^:]+' | grep -E "${RULE_PREFIX_REGEX}" || true)
+  if [ -n "$MATCHES" ]; then
+    assert_failed "The rules at ${RULES_FILE} contains unexpected references to '${LABEL}' label in some rule name prefix:\n$MATCHES"
+  fi
+done
+
+if [ $FAILED -ne 0 ]; then
+  exit 1
+fi

operations/mimir-mixin-tests/test-custom-labels.libsonnet

@@ -0,0 +1,7 @@
+(import 'mixin-compiled.libsonnet') + {
+  _config+:: {
+    per_cluster_label: 'mycluster',
+    per_instance_label: 'myinstance',
+    per_node_label: 'mynode',
+  },
+}

operations/mimir-mixin-tools/screenshots/Dockerfile

@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
 FROM node:17-bullseye
 
 # Install Chrome dependencies.

operations/mimir-mixin-tools/screenshots/run.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
 set -e
 
 SCRIPT_DIR=$(cd `dirname $0` && pwd)

operations/mimir-mixin-tools/serve/run.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
 set -e
 
 SCRIPT_DIR=$(cd `dirname $0` && pwd)

operations/mimir-mixin/.gitignore

@@ -1 +1,2 @@
 /vendor/
+mixin-test.libsonnet

operations/mimir-mixin/alerts.libsonnet

@@ -1,10 +1,10 @@
 {
   prometheusAlerts+::
+    { _config:: $._config + $._group_config } +
     (import 'alerts/alerts.libsonnet') +
     (import 'alerts/alertmanager.libsonnet') +
     (import 'alerts/blocks.libsonnet') +
     (import 'alerts/compactor.libsonnet') +
     (import 'alerts/autoscaling.libsonnet') +
-    (import 'alerts/continuous-test.libsonnet') +
-    { _config:: $._config + $._group_config },
+    (import 'alerts/continuous-test.libsonnet'),
 }

operations/mimir-mixin/alerts/alertmanager.libsonnet

@@ -14,7 +14,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to read tenant configurations from storage.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s is failing to read tenant configurations from storage.
             ||| % $._config,
           },
         },
@@ -29,7 +29,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} is unable to check tenants ownership via the ring.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s is unable to check tenants ownership via the ring.
             ||| % $._config,
           },
         },
@@ -44,7 +44,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to merge partial state changes received from a replica.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s is failing to merge partial state changes received from a replica.
             ||| % $._config,
           },
         },
@@ -59,7 +59,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} is failing to replicating partial state to its replicas.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s is failing to replicating partial state to its replicas.
             ||| % $._config,
           },
         },
@@ -74,7 +74,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} is unable to persist full state snaphots to remote storage.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s is unable to persist full state snaphots to remote storage.
             ||| % $._config,
           },
         },
@@ -88,7 +88,7 @@
           },
           annotations: {
             message: |||
-              %(product)s Alertmanager {{ $labels.job }}/{{ $labels.instance }} was unable to obtain some initial state when starting up.
+              %(product)s Alertmanager {{ $labels.job }}/%(alert_instance_variable)s was unable to obtain some initial state when starting up.
             ||| % $._config,
           },
         },
@@ -105,7 +105,7 @@
           },
           annotations: {
             message: |||
-              Alertmanager {{ $labels.pod }} in %(alert_aggregation_variables)s is using too much memory.
+              Alertmanager %(alert_instance_variable)s in %(alert_aggregation_variables)s is using too much memory.
             ||| % $._config,
           },
         },
@@ -122,7 +122,7 @@
           },
           annotations: {
             message: |||
-              Alertmanager {{ $labels.pod }} in %(alert_aggregation_variables)s is using too much memory.
+              Alertmanager %(alert_instance_variable)s in %(alert_aggregation_variables)s is using too much memory.
             ||| % $._config,
           },
         },

operations/mimir-mixin/alerts/alerts-utils.libsonnet

@@ -1,4 +1,10 @@
 {
+  _config+:: {
+    alert_cluster_variable: '{{ $labels.%s }}' % $._config.per_cluster_label,
+    alert_instance_variable: '{{ $labels.%s }}' % $._config.per_instance_label,
+    alert_node_variable: '{{ $labels.%s }}' % $._config.per_cluster_label,
+  },
+
   // The alert name is prefixed with the product name (eg. AlertName -> MimirAlertName).
   alertName(name)::
     $._config.product + name,