diff --git a/CHANGELOG.md b/CHANGELOG.md index 749ed43..9599d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,10 @@ * [CHANGE] Split `cortex_api` recording rule group into three groups. This is a workaround for large clusters where this group can become slow to evaluate. #401 * [CHANGE] Increased `CortexIngesterReachingSeriesLimit` warning threshold from 70% to 80% and critical threshold from 85% to 90%. #404 * [CHANGE] Rename ruler_s3_bucket_name and ruler_gcs_bucket_name to ruler_storage_bucket_name: #415 +* [CHANGE] Fine-tuned rolling update policy for distributor, querier, query-frontend, query-scheduler. #420 +* [CHANGE] Increased memcached metadata/chunks/index-queries max connections from 4k to 16k. #420 +* [CHANGE] Disabled step alignment in query-frontend to be compliant with PromQL. #420 +* [CHANGE] Do not limit compactor CPU and request a number of cores equal to the configured concurrency. #420 * [ENHANCEMENT] Add overrides config to compactor. This allows setting retention configs per user. #386 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317 * [ENHANCEMENT] Cortex-mixin: Include `cortex-gw-internal` naming variation in default `gateway` job names. #328 diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet index ae1c9ff..ea22523 100644 --- a/cortex/distributor.libsonnet +++ b/cortex/distributor.libsonnet @@ -57,7 +57,9 @@ distributor_deployment: deployment.new('distributor', 3, [$.distributor_container], $.distributor_deployment_labels) + (if $._config.cortex_distributor_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) + - $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex'), + $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') + + deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1), local service = $.core.v1.service, diff --git a/cortex/memcached.libsonnet b/cortex/memcached.libsonnet index e303f51..011328c 100644 --- a/cortex/memcached.libsonnet +++ b/cortex/memcached.libsonnet @@ -34,6 +34,7 @@ memcached { $.memcached { name: 'memcached-index-queries', max_item_size: '%dm' % [$._config.memcached_index_queries_max_item_size_mb], + connection_limit: 16384, } else {}, @@ -54,7 +55,7 @@ memcached { // Save memory by more tightly provisioning memcached chunks. memory_limit_mb: 6 * 1024, overprovision_factor: 1.05, - connection_limit: 4096, + connection_limit: 16384, local container = $.core.v1.container, } @@ -65,7 +66,7 @@ memcached { $.memcached { name: 'memcached-metadata', max_item_size: '%dm' % [$._config.memcached_metadata_max_item_size_mb], - connection_limit: 4096, + connection_limit: 16384, // Metadata cache doesn't need much memory. memory_limit_mb: 512, diff --git a/cortex/querier.libsonnet b/cortex/querier.libsonnet index 0256a91..eb807ee 100644 --- a/cortex/querier.libsonnet +++ b/cortex/querier.libsonnet @@ -58,6 +58,8 @@ deployment.new(name, $._config.querier.replicas, [container], $.querier_deployment_labels) + (if $._config.cortex_querier_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) + $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') + + deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + $.storage_config_mixin, querier_deployment: diff --git a/cortex/query-frontend.libsonnet b/cortex/query-frontend.libsonnet index 7cc5a8c..80f36d0 100644 --- a/cortex/query-frontend.libsonnet +++ b/cortex/query-frontend.libsonnet @@ -17,7 +17,7 @@ 'querier.split-queries-by-interval': '24h', // Cache query results. - 'querier.align-querier-with-step': true, + 'querier.align-querier-with-step': false, 'querier.cache-results': true, 'frontend.memcached.hostname': 'memcached-frontend.%s.svc.cluster.local' % $._config.namespace, 'frontend.memcached.service': 'memcached-client', @@ -52,7 +52,9 @@ newQueryFrontendDeployment(name, container):: deployment.new(name, $._config.queryFrontend.replicas, [container]) + $.util.configVolumeMount($._config.overrides_configmap, '/etc/cortex') + - (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity), + (if $._config.cortex_query_frontend_allow_multiple_replicas_on_same_node then {} else $.util.antiAffinity) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(1) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1), query_frontend_deployment: self.newQueryFrontendDeployment('query-frontend', $.query_frontend_container), diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet index 130325e..604d258 100644 --- a/cortex/query-scheduler.libsonnet +++ b/cortex/query-scheduler.libsonnet @@ -25,7 +25,10 @@ newQuerySchedulerDeployment(name, container):: deployment.new(name, 2, [container]) + $.util.configVolumeMount('overrides', '/etc/cortex') + - $.util.antiAffinity, + $.util.antiAffinity + + // Do not run more query-schedulers than expected. + deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1), query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else self.newQuerySchedulerDeployment('query-scheduler', $.query_scheduler_container), diff --git a/cortex/tsdb.libsonnet b/cortex/tsdb.libsonnet index 491adb0..aa47082 100644 --- a/cortex/tsdb.libsonnet +++ b/cortex/tsdb.libsonnet @@ -172,8 +172,9 @@ container.withPorts($.compactor_ports) + container.withArgsMixin($.util.mapToFlags($.compactor_args)) + container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) + - $.util.resourcesRequests('1', '6Gi') + - $.util.resourcesLimits($._config.cortex_compactor_max_concurrency, '6Gi') + + // Do not limit compactor CPU and request enough cores to honor configured max concurrency. + $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') + + $.util.resourcesLimits(null, '6Gi') + $.util.readinessProbe + $.jaeger_mixin,