fix: cluster_id label override in mixins

production/loki-mixin-compiled-ssd/alerts.yaml

@@ -1,41 +1,41 @@
 groups:
-- name: loki_alerts
-  rules:
-  - alert: LokiRequestErrors
-    annotations:
-      message: |
-        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
-    expr: |
-      100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route)
-        /
-      sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route)
-        > 10
-    for: 15m
-    labels:
-      severity: critical
-  - alert: LokiRequestPanics
-    annotations:
-      message: |
-        {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics.
-    expr: |
-      sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
-    labels:
-      severity: critical
-  - alert: LokiRequestLatency
-    annotations:
-      message: |
-        {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
-    expr: |
-      cluster_namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > 1
-    for: 15m
-    labels:
-      severity: critical
-  - alert: LokiTooManyCompactorsRunning
-    annotations:
-      message: |
-        {{ $labels.cluster }} {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time.
-    expr: |
-      sum(loki_boltdb_shipper_compactor_running) by (namespace, cluster) > 1
-    for: 5m
-    labels:
-      severity: warning
+    - name: loki_alerts
+      rules:
+        - alert: LokiRequestErrors
+          annotations:
+            message: |
+                {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
+          expr: |
+            100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[2m])) by (namespace, job, route)
+              /
+            sum(rate(loki_request_duration_seconds_count[2m])) by (namespace, job, route)
+              > 10
+          for: 15m
+          labels:
+            severity: critical
+        - alert: LokiRequestPanics
+          annotations:
+            message: |
+                {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics.
+          expr: |
+            sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
+          labels:
+            severity: critical
+        - alert: LokiRequestLatency
+          annotations:
+            message: |
+                {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
+          expr: |
+            cluster_namespace_job_route:loki_request_duration_seconds:99quantile{route!~"(?i).*tail.*|/schedulerpb.SchedulerForQuerier/QuerierLoop"} > 1
+          for: 15m
+          labels:
+            severity: critical
+        - alert: LokiTooManyCompactorsRunning
+          annotations:
+            message: |
+                {{ $labels.cluster }} {{ $labels.namespace }} has had {{ printf "%.0f" $value }} compactors running for more than 5m. Only one compactor should run at a time.
+          expr: |
+            sum(loki_boltdb_shipper_compactor_running) by (namespace, cluster) > 1
+          for: 5m
+          labels:
+            severity: warning

production/loki-mixin-compiled-ssd/dashboards/loki-logs.json

@@ -63,7 +63,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})",
+                  "expr": "sum(go_goroutines{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})",
                   "refId": "A"
                }
             ],
@@ -149,7 +149,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)",
+                  "expr": "sum(go_gc_duration_seconds{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)",
                   "legendFormat": "{{quantile}}",
                   "refId": "A"
                }
@@ -236,7 +236,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))",
+                  "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))",
                   "refId": "A"
                }
             ],
@@ -322,7 +322,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})",
+                  "expr": "sum(container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})",
                   "refId": "A"
                }
             ],
@@ -408,7 +408,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
+                  "expr": "sum(rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
                   "refId": "A"
                }
             ],
@@ -494,7 +494,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
+                  "expr": "sum(rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))",
                   "refId": "A"
                }
             ],
@@ -580,7 +580,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0",
+                  "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0",
                   "legendFormat": "{{reason}}",
                   "refId": "A"
                }
@@ -667,7 +667,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)",
+                  "expr": "sum(rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)",
                   "legendFormat": "{{level}}",
                   "refId": "A"
                }
@@ -771,7 +771,7 @@
             "steppedLine": false,
             "targets": [
                {
-                  "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)",
+                  "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)",
                   "intervalFactor": 3,
                   "legendFormat": "{{level}}",
                   "refId": "A"
@@ -836,7 +836,7 @@
             },
             "targets": [
                {
-                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"",
+                  "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"",
                   "refId": "A"
                }
             ],
@@ -935,7 +935,7 @@
                "multi": false,
                "name": "deployment",
                "options": [ ],
-               "query": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)",
+               "query": "label_values(kube_deployment_created{cluster=~\"$cluster\", namespace=\"$namespace\"}, deployment)",
                "refresh": 0,
                "regex": "",
                "sort": 1,
@@ -955,7 +955,7 @@
                "multi": false,
                "name": "pod",
                "options": [ ],
-               "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)",
+               "query": "label_values(kube_pod_container_info{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)",
                "refresh": 0,
                "regex": "",
                "sort": 1,
@@ -975,7 +975,7 @@
                "multi": false,
                "name": "container",
                "options": [ ],
-               "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)",
+               "query": "label_values(kube_pod_container_info{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)",
                "refresh": 0,
                "regex": "",
                "sort": 1,

production/loki-mixin-compiled-ssd/dashboards/loki-mixin-recording-rules.json

@@ -674,7 +674,7 @@
                "multi": false,
                "name": "tenant",
                "options": [ ],
-               "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))",
+               "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=~\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))",
                "refresh": 0,
                "regex": "/\"([^\"]+)\"/",
                "sort": 1,