From 5fa86178a6cd5532da4d55cea42bedd73e949eb2 Mon Sep 17 00:00:00 2001 From: Douglas Camata <159076+douglascamata@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:44:43 +0200 Subject: [PATCH 1/3] Make the Store liveness probe timeout configurable Under heavy load, the 1s default timeout for the Store liveness probe can be often triggered, which leads to Store restarts. The default value is 1s, to keep the current behavior, and in affected environments one can increase it. Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> --- .../thanos-store-shard0-statefulSet.yaml | 1 + .../thanos-store-shard1-statefulSet.yaml | 1 + .../thanos-store-shard2-statefulSet.yaml | 1 + .../all/manifests/thanos-store-statefulSet.yaml | 1 + .../kube-thanos-store-default-params.libsonnet | 3 +++ jsonnet/kube-thanos/kube-thanos-store.libsonnet | 15 ++++++++++----- manifests/thanos-store-statefulSet.yaml | 1 + 7 files changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/all/manifests/thanos-store-shard0-statefulSet.yaml b/examples/all/manifests/thanos-store-shard0-statefulSet.yaml index 07259f29..ed5080d7 100644 --- a/examples/all/manifests/thanos-store-shard0-statefulSet.yaml +++ b/examples/all/manifests/thanos-store-shard0-statefulSet.yaml @@ -124,6 +124,7 @@ spec: port: 10902 scheme: HTTP periodSeconds: 30 + timeoutSeconds: 1 name: thanos-store ports: - containerPort: 10901 diff --git a/examples/all/manifests/thanos-store-shard1-statefulSet.yaml b/examples/all/manifests/thanos-store-shard1-statefulSet.yaml index 657a3c56..1c561eef 100644 --- a/examples/all/manifests/thanos-store-shard1-statefulSet.yaml +++ b/examples/all/manifests/thanos-store-shard1-statefulSet.yaml @@ -124,6 +124,7 @@ spec: port: 10902 scheme: HTTP periodSeconds: 30 + timeoutSeconds: 1 name: thanos-store ports: - containerPort: 10901 diff --git a/examples/all/manifests/thanos-store-shard2-statefulSet.yaml b/examples/all/manifests/thanos-store-shard2-statefulSet.yaml index 0a76de05..7d9a1353 100644 --- a/examples/all/manifests/thanos-store-shard2-statefulSet.yaml +++ b/examples/all/manifests/thanos-store-shard2-statefulSet.yaml @@ -124,6 +124,7 @@ spec: port: 10902 scheme: HTTP periodSeconds: 30 + timeoutSeconds: 1 name: thanos-store ports: - containerPort: 10901 diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml index 4e162fb1..7ea32dc7 100644 --- a/examples/all/manifests/thanos-store-statefulSet.yaml +++ b/examples/all/manifests/thanos-store-statefulSet.yaml @@ -112,6 +112,7 @@ spec: port: 10902 scheme: HTTP periodSeconds: 30 + timeoutSeconds: 1 name: thanos-store ports: - containerPort: 10901 diff --git a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet index a989c0cc..e288416d 100644 --- a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet @@ -22,6 +22,9 @@ grpc: 10901, http: 10902, }, + livenessProbe: { + timeoutSeconds: 1, + }, tracing: {}, minTime: '', maxTime: '', diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet index faad4187..d2264963 100644 --- a/jsonnet/kube-thanos/kube-thanos-store.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-store.libsonnet @@ -135,11 +135,16 @@ function(params) { { name: 'tls-secret', mountPath: ts.config.objectStorageConfig.tlsSecretMountPath }, ] else [] ), - livenessProbe: { failureThreshold: 8, periodSeconds: 30, httpGet: { - scheme: 'HTTP', - port: ts.config.ports.http, - path: '/-/healthy', - } }, + livenessProbe: { + failureThreshold: 8, + periodSeconds: 30, + timeoutSeconds: ts.config.livenessProbe.timeoutSeconds, + httpGet: { + scheme: 'HTTP', + port: ts.config.ports.http, + path: '/-/healthy', + }, + }, readinessProbe: { failureThreshold: 20, periodSeconds: 5, httpGet: { scheme: 'HTTP', port: ts.config.ports.http, diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml index 7da4ae25..34d0eb97 100644 --- a/manifests/thanos-store-statefulSet.yaml +++ b/manifests/thanos-store-statefulSet.yaml @@ -71,6 +71,7 @@ spec: port: 10902 scheme: HTTP periodSeconds: 30 + timeoutSeconds: 1 name: thanos-store ports: - containerPort: 10901 From 7361d9315558065cba7e7e25b0b2654aff86e719 Mon Sep 17 00:00:00 2001 From: Douglas Camata <159076+douglascamata@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:48:12 +0200 Subject: [PATCH 2/3] Update CHANGELOG Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56fcf74c..b877aaac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#263](https://github.com/thanos-io/kube-thanos/pull/263) Add support for stateless Rulers. - [#271](https://github.com/thanos-io/kube-thanos/pull/271) Add annotation support for ServiceAccount. +- [#286](https://github.com/thanos-io/kube-thanos/pull/286) Store: make the liveness probe timeout configurable. ### Fixed From 449e423dc618f2911038a1b0111ab5a6a7c7174e Mon Sep 17 00:00:00 2001 From: Douglas Camata <159076+douglascamata@users.noreply.github.com> Date: Tue, 11 Oct 2022 12:04:46 +0200 Subject: [PATCH 3/3] Make Store liveness probe more configurable Adds configuration for failure threshold and period seconds. Signed-off-by: Douglas Camata <159076+douglascamata@users.noreply.github.com> --- .../kube-thanos/kube-thanos-store-default-params.libsonnet | 2 ++ jsonnet/kube-thanos/kube-thanos-store.libsonnet | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet index e288416d..0a48f057 100644 --- a/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-store-default-params.libsonnet @@ -24,6 +24,8 @@ }, livenessProbe: { timeoutSeconds: 1, + failureThreshold: 8, + periodSeconds: 30, }, tracing: {}, minTime: '', diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet index d2264963..bf5e5b9a 100644 --- a/jsonnet/kube-thanos/kube-thanos-store.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-store.libsonnet @@ -136,8 +136,8 @@ function(params) { ] else [] ), livenessProbe: { - failureThreshold: 8, - periodSeconds: 30, + failureThreshold: ts.config.livenessProbe.failureThreshold, + periodSeconds: ts.config.livenessProbe.periodSeconds, timeoutSeconds: ts.config.livenessProbe.timeoutSeconds, httpGet: { scheme: 'HTTP',