Merge r190 into out of order

CHANGELOG.md

@@ -34,6 +34,8 @@
 * [BUGFIX] API: gzip is now enforced when identity encoding is explicitly rejected. #1864
 * [BUGFIX] Fix panic at startup when Mimir is running in monolithic mode and query sharding is enabled. #2036
 * [BUGFIX] Ruler: report failed evaluation metric for any 5xx status code returned by the query-frontend when remote operational mode is enabled. #2053
+* [BUGFIX] Ingester: fix slow rollout when using `-ingester.ring.unregister-on-shutdown=false` with long `-ingester.ring.heartbeat-period`. #2085
+* [BUGFIX] Ruler: add timeout for remote rule evaluation queries to prevent rule group evaluations getting stuck indefinitely. The duration is configurable with  (`-ruler.query-frontend.timeout` (default `2m`). #2090
 
 ### Mixin
 

cmd/mimir/config-descriptor.json

@@ -7404,6 +7404,16 @@
               "fieldFlag": "ruler.query-frontend.address",
               "fieldType": "string"
             },
+            {
+              "kind": "field",
+              "name": "timeout",
+              "required": false,
+              "desc": "The timeout for a rule query being evaluated by the query-frontend.",
+              "fieldValue": null,
+              "fieldDefaultValue": 120000000000,
+              "fieldFlag": "ruler.query-frontend.timeout",
+              "fieldType": "duration"
+            },
             {
               "kind": "block",
               "name": "grpc_client_config",

cmd/mimir/help-all.txt.tmpl

@@ -1450,6 +1450,8 @@ Usage of ./cmd/mimir/mimir:
     	Path to the key file for the client certificate. Also requires the client certificate to be configured.
   -ruler.query-frontend.grpc-client-config.tls-server-name string
     	Override the expected name on the server certificate.
+  -ruler.query-frontend.timeout duration
+    	The timeout for a rule query being evaluated by the query-frontend. (default 2m0s)
   -ruler.query-stats-enabled
     	Report the wall time for ruler queries to complete as a per-tenant metric and as an info level log message.
   -ruler.resend-delay duration

cmd/mimir/help.txt.tmpl

@@ -450,6 +450,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of rules per rule group per-tenant. 0 to disable. (default 20)
   -ruler.query-frontend.address string
     	GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) to enable client side load balancing.
+  -ruler.query-frontend.timeout duration
+    	The timeout for a rule query being evaluated by the query-frontend. (default 2m0s)
   -ruler.ring.consul.hostname string
     	Hostname and port of Consul. (default "localhost:8500")
   -ruler.ring.etcd.endpoints value

docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md

@@ -1400,6 +1400,10 @@ query_frontend:
   # CLI flag: -ruler.query-frontend.address
   [address: <string> | default = ""]
 
+  # The timeout for a rule query being evaluated by the query-frontend.
+  # CLI flag: -ruler.query-frontend.timeout
+  [timeout: <duration> | default = 2m]
+
   grpc_client_config:
     # (advanced) gRPC client max receive message size (bytes).
     # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.0
-	github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b
+	github.com/grafana/dskit v0.0.0-20220613090928-ebb5c6de233d
 	github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/json-iterator/go v1.1.12

go.sum

@@ -738,8 +738,8 @@ github.com/gosimple/slug v1.1.1 h1:fRu/digW+NMwBIP+RmviTK97Ho/bEj/C9swrCspN3D4=
 github.com/gosimple/slug v1.1.1/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0=
 github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM=
 github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE=
-github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b h1:9h79WowXGj6wErMzoi35pBECxnh7ucKJlQTs4Gs0yOI=
-github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b/go.mod h1:9It/K30QPyj/FuTqBb/SYnaS4/BJCP5YL4SRfXB7dG0=
+github.com/grafana/dskit v0.0.0-20220613090928-ebb5c6de233d h1:cHmYkgZr5vdkxt7EbzFIwdm9fS/0E3wyTMa1g2vezLg=
+github.com/grafana/dskit v0.0.0-20220613090928-ebb5c6de233d/go.mod h1:9It/K30QPyj/FuTqBb/SYnaS4/BJCP5YL4SRfXB7dG0=
 github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe h1:mxrRWDjKtob43xF9nEhJthdtCzX35/800Sk7nE//YHQ=
 github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo=
 github.com/grafana/memberlist v0.3.1-0.20220425183535-6b97a09b7167 h1:PgEQkGHR4YimSCEGT5IoswN9gJKZDVskf+he6UClCLw=

operations/helm/charts/mimir-distributed/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-version: 2.2.0-weekly.189
+version: 2.2.0-weekly.190
 appVersion: 2.1.0
 description: "Grafana Mimir"
 engine: gotpl

operations/helm/charts/mimir-distributed/README.md

@@ -4,7 +4,7 @@ Helm chart for deploying [Grafana Mimir](https://grafana.com/docs/mimir/v2.1.x/)
 
 # mimir-distributed
 
-![Version: 2.2.0-weekly.189](https://img.shields.io/badge/Version-2.2.0--weekly.189-informational?style=flat-square) ![AppVersion: 2.1.0](https://img.shields.io/badge/AppVersion-2.1.0-informational?style=flat-square)
+![Version: 2.2.0-weekly.190](https://img.shields.io/badge/Version-2.2.0--weekly.190-informational?style=flat-square) ![AppVersion: 2.1.0](https://img.shields.io/badge/AppVersion-2.1.0-informational?style=flat-square)
 
 Grafana Mimir
 

operations/helm/charts/mimir-distributed/values.yaml

@@ -16,7 +16,7 @@ fullnameOverride: null
 # Since the image is unique for all microservices, so are image settings.
 image:
   repository: grafana/mimir
-  tag: 2.1.0
+  tag: r190-fe20bbd
   pullPolicy: IfNotPresent
   # Optionally specify an array of imagePullSecrets.
   # Secrets must be manually created in the namespace.
@@ -1429,7 +1429,7 @@ enterprise:
   # Container image settings for enterprise, note that pullPolicy and pullSecrets are set in top level .image
   image:
     repository: grafana/enterprise-metrics
-    tag: v2.1.0
+    tag: r190-9abd21a8
 
 # In order to use Grafana Enterprise Metrics features, you will need to provide the contents of your Grafana Enterprise Metrics
 # license, either by providing the contents of the license.jwt, or the name Kubernetes Secret that contains your license.jwt.

operations/helm/tests/build.sh

@@ -19,5 +19,5 @@ for FILEPATH in $TESTS; do
   TEST_NAME=$(basename -s '.yaml' "$FILEPATH")
 
   echo "Templating $TEST_NAME"
-  helm template "${TEST_NAME}" ${CHART_PATH} -f "${FILEPATH}" --output-dir "operations/helm/tests/${TEST_NAME}-generated"
+  helm template "${TEST_NAME}" ${CHART_PATH} -f "${FILEPATH}" --output-dir "operations/helm/tests/${TEST_NAME}-generated" --namespace citestns
 done

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml

@@ -16,4 +16,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: test-enterprise-values-minio-update-prometheus-secret
-    namespace: "default"
+    namespace: "citestns"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml

@@ -4,7 +4,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: "test-enterprise-values-minio"
-  namespace: "default"
+  namespace: "citestns"
   labels:
     app: minio
     chart: minio-8.0.10

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-dep.yaml

@@ -6,7 +6,7 @@ metadata:
   annotations:
     {}
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: admin-api
@@ -29,23 +29,23 @@ spec:
   template:
     metadata:
       labels:
-        helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+        helm.sh/chart: mimir-distributed-2.2.0-weekly.190
         app.kubernetes.io/name: mimir
         app.kubernetes.io/instance: test-enterprise-values
         app.kubernetes.io/version: "2.1.0"
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/component: admin-api
         app.kubernetes.io/part-of: memberlist
       annotations:
-        checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99
+        checksum/config: deee5d2bcf924492c2988a219d9cf89cde36027ee6b101cee76fde256a0b9cef
     spec:
       serviceAccountName: test-enterprise-values-mimir
       securityContext:
         {}
       initContainers:
       containers:
         - name: admin-api
-          image: "grafana/enterprise-metrics:v2.1.0"
+          image: "grafana/enterprise-metrics:r190-9abd21a8"
           imagePullPolicy: IfNotPresent
           args:
             - "-target=admin-api"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-admin-api
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: admin-api

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml

@@ -5,7 +5,7 @@ kind: StatefulSet
 metadata:
   name: test-enterprise-values-mimir-alertmanager
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: alertmanager
@@ -27,15 +27,15 @@ spec:
   template:
     metadata:
       labels:
-        helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+        helm.sh/chart: mimir-distributed-2.2.0-weekly.190
         app.kubernetes.io/name: mimir
         app.kubernetes.io/instance: test-enterprise-values
         app.kubernetes.io/version: "2.1.0"
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/component: alertmanager
         app.kubernetes.io/part-of: memberlist
       annotations:
-        checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99
+        checksum/config: deee5d2bcf924492c2988a219d9cf89cde36027ee6b101cee76fde256a0b9cef
     spec:
       serviceAccountName: test-enterprise-values-mimir
       securityContext:
@@ -65,7 +65,7 @@ spec:
           emptyDir: {}
       containers:
         - name: alertmanager
-          image: "grafana/enterprise-metrics:v2.1.0"
+          image: "grafana/enterprise-metrics:r190-9abd21a8"
           imagePullPolicy: IfNotPresent
           args:
             - "-target=alertmanager"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-alertmanager-headless
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: alertmanager

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-alertmanager
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: alertmanager

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml

@@ -5,7 +5,7 @@ kind: StatefulSet
 metadata:
   name: test-enterprise-values-mimir-compactor
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: compactor
@@ -27,15 +27,15 @@ spec:
   template:
     metadata:
       labels:
-        helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+        helm.sh/chart: mimir-distributed-2.2.0-weekly.190
         app.kubernetes.io/name: mimir
         app.kubernetes.io/instance: test-enterprise-values
         app.kubernetes.io/version: "2.1.0"
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/component: compactor
         app.kubernetes.io/part-of: memberlist
       annotations:
-        checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99
+        checksum/config: deee5d2bcf924492c2988a219d9cf89cde36027ee6b101cee76fde256a0b9cef
     spec:
       serviceAccountName: test-enterprise-values-mimir
       securityContext:
@@ -73,7 +73,7 @@ spec:
           emptyDir: {}
       containers:
         - name: compactor
-          image: "grafana/enterprise-metrics:v2.1.0"
+          image: "grafana/enterprise-metrics:r190-9abd21a8"
           imagePullPolicy: IfNotPresent
           args:
             - "-target=compactor"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-compactor
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: compactor

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml

@@ -5,7 +5,7 @@ kind: Deployment
 metadata:
   name: test-enterprise-values-mimir-distributor
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: distributor
@@ -29,15 +29,15 @@ spec:
   template:
     metadata:
       labels:
-        helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+        helm.sh/chart: mimir-distributed-2.2.0-weekly.190
         app.kubernetes.io/name: mimir
         app.kubernetes.io/instance: test-enterprise-values
         app.kubernetes.io/version: "2.1.0"
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/component: distributor
         app.kubernetes.io/part-of: memberlist
       annotations:
-        checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99
+        checksum/config: deee5d2bcf924492c2988a219d9cf89cde36027ee6b101cee76fde256a0b9cef
     spec:
       serviceAccountName: test-enterprise-values-mimir
       securityContext:
@@ -46,7 +46,7 @@ spec:
         []
       containers:
         - name: distributor
-          image: "grafana/enterprise-metrics:v2.1.0"
+          image: "grafana/enterprise-metrics:r190-9abd21a8"
           imagePullPolicy: IfNotPresent
           args:
             - "-target=distributor"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-distributor-headless
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: distributor

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-distributor
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: distributor

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-dep.yaml

@@ -6,7 +6,7 @@ metadata:
   annotations:
     {}
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: gateway
@@ -28,14 +28,14 @@ spec:
   template:
     metadata:
       labels:
-        helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+        helm.sh/chart: mimir-distributed-2.2.0-weekly.190
         app.kubernetes.io/name: mimir
         app.kubernetes.io/instance: test-enterprise-values
         app.kubernetes.io/version: "2.1.0"
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/component: gateway
       annotations:
-        checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99
+        checksum/config: deee5d2bcf924492c2988a219d9cf89cde36027ee6b101cee76fde256a0b9cef
     spec:
       serviceAccountName: test-enterprise-values-mimir
       securityContext:
@@ -44,7 +44,7 @@ spec:
         []
       containers:
         - name: gateway
-          image: "grafana/enterprise-metrics:v2.1.0"
+          image: "grafana/enterprise-metrics:r190-9abd21a8"
           imagePullPolicy: IfNotPresent
           args:
             - "-target=gateway"

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-gateway
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: gateway

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml

@@ -5,7 +5,7 @@ kind: Service
 metadata:
   name: test-enterprise-values-mimir-gossip-ring
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: gossip-ring

operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml

@@ -5,7 +5,7 @@ kind: PodDisruptionBudget
 metadata:
   name: test-enterprise-values-mimir-ingester
   labels:
-    helm.sh/chart: mimir-distributed-2.2.0-weekly.189
+    helm.sh/chart: mimir-distributed-2.2.0-weekly.190
     app.kubernetes.io/name: mimir
     app.kubernetes.io/instance: test-enterprise-values
     app.kubernetes.io/component: ingester