From c08f7f64b8aa5ac8317da060f41610649e3c9e9d Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 13 Aug 2021 08:54:16 +0200 Subject: [PATCH] Improve config settings based on recent learnings Signed-off-by: Marco Pracucci --- operations/mimir/ingester.libsonnet | 2 +- operations/mimir/querier.libsonnet | 4 ++++ operations/mimir/ruler.libsonnet | 4 ++++ operations/mimir/tsdb.libsonnet | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet index 34b4d9872e2..e0753d843ee 100644 --- a/operations/mimir/ingester.libsonnet +++ b/operations/mimir/ingester.libsonnet @@ -31,7 +31,7 @@ 'ingester.max-series-per-query': $._config.limits.max_series_per_query, 'ingester.max-samples-per-query': $._config.limits.max_samples_per_query, 'runtime-config.file': '/etc/cortex/overrides.yaml', - 'server.grpc-max-concurrent-streams': 100000, + 'server.grpc-max-concurrent-streams': 10000, 'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024, 'server.grpc-max-recv-msg-size-bytes': 10 * 1024 * 1024, } + ( diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet index 5dc3c834aa6..574358d8b0a 100644 --- a/operations/mimir/querier.libsonnet +++ b/operations/mimir/querier.libsonnet @@ -26,6 +26,10 @@ 'querier.second-store-engine': $._config.querier_second_storage_engine, + // We request high memory but the Go heap is typically very low (< 100MB) and this causes + // the GC to trigger continuously. Setting a ballast of 256MB reduces GC. + 'mem-ballast-size-bytes': 1 << 28, // 256M + 'log.level': 'debug', }, diff --git a/operations/mimir/ruler.libsonnet b/operations/mimir/ruler.libsonnet index 73029a20360..dfb5727e9e7 100644 --- a/operations/mimir/ruler.libsonnet +++ b/operations/mimir/ruler.libsonnet @@ -29,6 +29,10 @@ // Storage 'querier.second-store-engine': $._config.querier_second_storage_engine, + + // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown" + // is set to false. + 'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown, }, ruler_container:: diff --git a/operations/mimir/tsdb.libsonnet b/operations/mimir/tsdb.libsonnet index 5c69bf004fc..1c77abd99e3 100644 --- a/operations/mimir/tsdb.libsonnet +++ b/operations/mimir/tsdb.libsonnet @@ -115,7 +115,7 @@ statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + // When the ingester needs to flush blocks to the storage, it may take quite a lot of time. // For this reason, we grant an high termination period (80 minutes). - statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(1200) + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') + $.util.podPriority('high') +