Merge pull request grafana/cortex-jsonnet#420 from grafana/backport-i…

…mprovements Upstream some config improvements
grafana · Nov 22, 2021 · a1c68bd · a1c68bd
2 parents 98a694e + 2bbb7de
commit a1c68bd
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 8 deletions.
diff --git a/operations/mimir/distributor.libsonnet b/operations/mimir/distributor.libsonnet
@@ -57,7 +57,9 @@
   distributor_deployment:
     deployment.new('distributor', 3, [$.distributor_container], $.distributor_deployment_labels) +
     (if $._config.cortex_distributor_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
-    $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex'),
+    $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   local service = $.core.v1.service,
 

diff --git a/operations/mimir/memcached.libsonnet b/operations/mimir/memcached.libsonnet
@@ -34,6 +34,7 @@ memcached {
     $.memcached {
       name: 'memcached-index-queries',
       max_item_size: '%dm' % [$._config.memcached_index_queries_max_item_size_mb],
+      connection_limit: 16384,
     }
   else {},
 
@@ -54,7 +55,7 @@ memcached {
       // Save memory by more tightly provisioning memcached chunks.
       memory_limit_mb: 6 * 1024,
       overprovision_factor: 1.05,
-      connection_limit: 4096,
+      connection_limit: 16384,
 
       local container = $.core.v1.container,
     }
@@ -65,7 +66,7 @@ memcached {
     $.memcached {
       name: 'memcached-metadata',
       max_item_size: '%dm' % [$._config.memcached_metadata_max_item_size_mb],
-      connection_limit: 4096,
+      connection_limit: 16384,
 
       // Metadata cache doesn't need much memory.
       memory_limit_mb: 512,

diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet
@@ -58,6 +58,8 @@
     deployment.new(name, $._config.querier.replicas, [container], $.querier_deployment_labels) +
     (if $._config.cortex_querier_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
     $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) +
     $.storage_config_mixin,
 
   querier_deployment:

diff --git a/operations/mimir/query-frontend.libsonnet b/operations/mimir/query-frontend.libsonnet
@@ -17,7 +17,7 @@
       'querier.split-queries-by-interval': '24h',
 
       // Cache query results.
-      'querier.align-querier-with-step': true,
+      'querier.align-querier-with-step': false,
       'querier.cache-results': true,
       'frontend.memcached.hostname': 'memcached-frontend.%s.svc.cluster.local' % $._config.namespace,
       'frontend.memcached.service': 'memcached-client',
@@ -52,7 +52,9 @@
   newQueryFrontendDeployment(name, container)::
     deployment.new(name, $._config.queryFrontend.replicas, [container]) +
     $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') +
-    (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity),
+    (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(1) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   query_frontend_deployment: self.newQueryFrontendDeployment('query-frontend', $.query_frontend_container),
 

diff --git a/operations/mimir/query-scheduler.libsonnet b/operations/mimir/query-scheduler.libsonnet
@@ -25,7 +25,10 @@
   newQuerySchedulerDeployment(name, container)::
     deployment.new(name, 2, [container]) +
     $.util.configVolumeMount('overrides', '/etc/cortex') +
-    $.util.antiAffinity,
+    $.util.antiAffinity +
+    // Do not run more query-schedulers than expected.
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else
     self.newQuerySchedulerDeployment('query-scheduler', $.query_scheduler_container),

diff --git a/operations/mimir/tsdb.libsonnet b/operations/mimir/tsdb.libsonnet
@@ -172,8 +172,9 @@
     container.withPorts($.compactor_ports) +
     container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
-    $.util.resourcesRequests('1', '6Gi') +
-    $.util.resourcesLimits($._config.cortex_compactor_max_concurrency, '6Gi') +
+    // Do not limit compactor CPU and request enough cores to honor configured max concurrency.
+    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') +
+    $.util.resourcesLimits(null, '6Gi') +
     $.util.readinessProbe +
     $.jaeger_mixin,