Merge pull request grafana/cortex-jsonnet#369 from grafana/config-cha…

…nges Improve config settings based on recent learnings
grafana · Aug 13, 2021 · 2fa5bcb · 2fa5bcb
2 parents f330b19 + c08f7f6
commit 2fa5bcb
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 2 deletions.
diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet
@@ -31,7 +31,7 @@
       'ingester.max-series-per-query': $._config.limits.max_series_per_query,
       'ingester.max-samples-per-query': $._config.limits.max_samples_per_query,
       'runtime-config.file': '/etc/cortex/overrides.yaml',
-      'server.grpc-max-concurrent-streams': 100000,
+      'server.grpc-max-concurrent-streams': 10000,
       'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024,
       'server.grpc-max-recv-msg-size-bytes': 10 * 1024 * 1024,
     } + (

diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet
@@ -26,6 +26,10 @@
 
       'querier.second-store-engine': $._config.querier_second_storage_engine,
 
+      // We request high memory but the Go heap is typically very low (< 100MB) and this causes
+      // the GC to trigger continuously. Setting a ballast of 256MB reduces GC.
+      'mem-ballast-size-bytes': 1 << 28,  // 256M
+
       'log.level': 'debug',
     },
 

diff --git a/operations/mimir/ruler.libsonnet b/operations/mimir/ruler.libsonnet
@@ -29,6 +29,10 @@
 
       // Storage
       'querier.second-store-engine': $._config.querier_second_storage_engine,
+
+      // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown"
+      // is set to false.
+      'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown,
     },
 
   ruler_container::

diff --git a/operations/mimir/tsdb.libsonnet b/operations/mimir/tsdb.libsonnet
@@ -115,7 +115,7 @@
     statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) +
     // When the ingester needs to flush blocks to the storage, it may take quite a lot of time.
     // For this reason, we grant an high termination period (80 minutes).
-    statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) +
+    statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(1200) +
     statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') +
     $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
     $.util.podPriority('high') +