From a760772b6c926a5d91e71d742de4f30700e46afa Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 26 Aug 2020 11:59:49 +0530 Subject: [PATCH 1/4] Add jsonnet for ingester StatefulSet with WAL Signed-off-by: Ganesh Vernekar --- cortex/config.libsonnet | 4 +++ cortex/ingester.libsonnet | 73 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 7e323886..9d2773e7 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -63,6 +63,10 @@ store_gateway_replication_factor: 3, + // By default ingesters will be run as StatefulSet with WAL. + // If this is set to true, ingesters will use staless deployments without WAL. + ingester_deployment_without_wal: false, + // Blocks storage engine doesn't require the table manager. // When running blocks with chunks as secondary storage engine for querier only, we need table-manager to apply // retention policies. diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index 2a2b2619..8f8379b7 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -1,6 +1,4 @@ { - local container = $.core.v1.container, - ingester_args:: $._config.ringConfig + $._config.storeConfig + @@ -40,9 +38,28 @@ else {} ), + ingester_statefulset_args:: { + 'ingester.wal-enabled': true, + 'ingester.checkpoint-enabled': true, + 'ingester.recover-from-wal': true, + 'ingester.wal-dir': $._config.wal_dir, + 'ingester.checkpoint-duration': '15m', + '-log.level': 'info', + 'ingester.tokens-file-path': $._config.wal_dir + '/tokens', + }, + + _config+:: { + wal_dir: '/wal_data', + ingester+: { + statefulset_replicas: 3, + statefulset_disk: '150Gi', + }, + }, + ingester_ports:: $.util.defaultPorts, local name = 'ingester', + local container = $.core.v1.container, ingester_container:: container.new(name, $._images.ingester) + @@ -53,11 +70,53 @@ $.util.readinessProbe + $.jaeger_mixin, - local deployment = $.apps.v1.deployment, + local volumeMount = $.core.v1.volumeMount, + + ingester_statefulset_container:: + $.ingester_container + + container.withArgsMixin($.util.mapToFlags($.ingester_statefulset_args)) + + container.withVolumeMountsMixin([ + volumeMount.new('ingester-pvc', $._config.wal_dir), + ]), ingester_deployment_labels:: {}, + local pvc = $.core.v1.persistentVolumeClaim, + local volume = $.core.v1.volume, + local statefulSet = $.apps.v1.statefulSet, + + local ingester_pvc = + pvc.new('ingester-pvc') + + pvc.mixin.spec.resources.withRequests({ storage: $._config.ingester.statefulset_disk }) + + pvc.mixin.spec.withAccessModes(['ReadWriteOnce']) + + pvc.mixin.spec.withStorageClassName('fast'), + + statefulset_storage_config_mixin:: + statefulSet.mixin.spec.template.metadata.withAnnotationsMixin({ schemaID: $._config.schemaID },) + + $.util.configVolumeMount('schema-' + $._config.schemaID, '/etc/cortex/schema'), + + ingester_statefulset: + if $._config.ingester_deployment_without_wal == false then + statefulSet.new('ingester', $._config.ingester.statefulset_replicas, [$.ingester_statefulset_container], ingester_pvc) + + statefulSet.mixin.spec.withServiceName('ingester') + + statefulSet.mixin.spec.template.spec.withVolumes([volume.fromPersistentVolumeClaim('ingester-pvc', 'ingester-pvc')]) + + statefulSet.mixin.metadata.withNamespace($._config.namespace) + + statefulSet.mixin.metadata.withLabels({ name: 'ingester' }) + + statefulSet.mixin.spec.template.metadata.withLabels({ name: 'ingester' } + $.ingester_deployment_labels) + + statefulSet.mixin.spec.selector.withMatchLabels({ name: 'ingester' }) + + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + $.statefulset_storage_config_mixin + + $.util.configVolumeMount('overrides', '/etc/cortex') + + $.util.podPriority('high') + + $.util.antiAffinityStatefulSet + else null, + + local deployment = $.apps.v1.deployment, + ingester_deployment: + if $._config.ingester_deployment_without_wal then deployment.new(name, 3, [$.ingester_container], $.ingester_deployment_labels) + $.util.antiAffinity + $.util.configVolumeMount('overrides', '/etc/cortex') + @@ -67,12 +126,16 @@ deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + $.storage_config_mixin + - $.util.podPriority('high'), + $.util.podPriority('high') + else null, ingester_service_ignored_labels:: [], ingester_service: - $.util.serviceFor($.ingester_deployment, $.ingester_service_ignored_labels), + if $._config.ingester_deployment_without_wal then + $.util.serviceFor($.ingester_deployment, $.ingester_service_ignored_labels) + else + $.util.serviceFor($.ingester_statefulset, $.ingester_service_ignored_labels), local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, From 79c8dad9c640929f348217681378b03e381f5a10 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 26 Aug 2020 12:07:12 +0530 Subject: [PATCH 2/4] Add CHANGELOG entry Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d45664f..1d4e06da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## master / unreleased +* [CHANGE] Ingesters now default to running as `StatefulSet` with WAL enabled. It is controlled by the config `$._config.ingester_deployment_without_wal` which is `false` by default. Setting the config to `true` will yeild the old behaviour (stateless `Deployment` without WAL enabled). #72 + ## 1.3.0 / 2020-08-21 This version is compatible with Cortex `1.3.0`. From d37136bed96a6afd5dd7b87d373e0734fc50e2ff Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 26 Aug 2020 12:12:15 +0530 Subject: [PATCH 3/4] Fix lint Signed-off-by: Ganesh Vernekar --- cortex/ingester.libsonnet | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index 8f8379b7..6c9c29da 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -96,46 +96,46 @@ $.util.configVolumeMount('schema-' + $._config.schemaID, '/etc/cortex/schema'), ingester_statefulset: - if $._config.ingester_deployment_without_wal == false then - statefulSet.new('ingester', $._config.ingester.statefulset_replicas, [$.ingester_statefulset_container], ingester_pvc) + - statefulSet.mixin.spec.withServiceName('ingester') + - statefulSet.mixin.spec.template.spec.withVolumes([volume.fromPersistentVolumeClaim('ingester-pvc', 'ingester-pvc')]) + - statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: 'ingester' }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: 'ingester' } + $.ingester_deployment_labels) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: 'ingester' }) + - statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + - statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + - statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + - $.statefulset_storage_config_mixin + - $.util.configVolumeMount('overrides', '/etc/cortex') + - $.util.podPriority('high') + - $.util.antiAffinityStatefulSet - else null, + if $._config.ingester_deployment_without_wal == false then + statefulSet.new('ingester', $._config.ingester.statefulset_replicas, [$.ingester_statefulset_container], ingester_pvc) + + statefulSet.mixin.spec.withServiceName('ingester') + + statefulSet.mixin.spec.template.spec.withVolumes([volume.fromPersistentVolumeClaim('ingester-pvc', 'ingester-pvc')]) + + statefulSet.mixin.metadata.withNamespace($._config.namespace) + + statefulSet.mixin.metadata.withLabels({ name: 'ingester' }) + + statefulSet.mixin.spec.template.metadata.withLabels({ name: 'ingester' } + $.ingester_deployment_labels) + + statefulSet.mixin.spec.selector.withMatchLabels({ name: 'ingester' }) + + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + $.statefulset_storage_config_mixin + + $.util.configVolumeMount('overrides', '/etc/cortex') + + $.util.podPriority('high') + + $.util.antiAffinityStatefulSet + else null, local deployment = $.apps.v1.deployment, ingester_deployment: - if $._config.ingester_deployment_without_wal then - deployment.new(name, 3, [$.ingester_container], $.ingester_deployment_labels) + - $.util.antiAffinity + - $.util.configVolumeMount('overrides', '/etc/cortex') + - deployment.mixin.metadata.withLabels({ name: name }) + - deployment.mixin.spec.withMinReadySeconds(60) + - deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) + - deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + - deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + - $.storage_config_mixin + - $.util.podPriority('high') - else null, + if $._config.ingester_deployment_without_wal then + deployment.new(name, 3, [$.ingester_container], $.ingester_deployment_labels) + + $.util.antiAffinity + + $.util.configVolumeMount('overrides', '/etc/cortex') + + deployment.mixin.metadata.withLabels({ name: name }) + + deployment.mixin.spec.withMinReadySeconds(60) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(0) + + deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1) + + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(4800) + + $.storage_config_mixin + + $.util.podPriority('high') + else null, ingester_service_ignored_labels:: [], ingester_service: - if $._config.ingester_deployment_without_wal then - $.util.serviceFor($.ingester_deployment, $.ingester_service_ignored_labels) - else - $.util.serviceFor($.ingester_statefulset, $.ingester_service_ignored_labels), + if $._config.ingester_deployment_without_wal then + $.util.serviceFor($.ingester_deployment, $.ingester_service_ignored_labels) + else + $.util.serviceFor($.ingester_statefulset, $.ingester_service_ignored_labels), local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, From cac287f519bc4d3050b53ba11f54011c02ec7f3f Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 26 Aug 2020 18:05:09 +0530 Subject: [PATCH 4/4] Fix review comments Signed-off-by: Ganesh Vernekar --- cortex/config.libsonnet | 7 +++++++ cortex/ingester.libsonnet | 14 +++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index 9d2773e7..5851d067 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -67,6 +67,13 @@ // If this is set to true, ingesters will use staless deployments without WAL. ingester_deployment_without_wal: false, + ingester: { + // These config options are only for the chunks storage. + wal_dir: '/wal_data', + statefulset_replicas: 3, + statefulset_disk: '150Gi', + }, + // Blocks storage engine doesn't require the table manager. // When running blocks with chunks as secondary storage engine for querier only, we need table-manager to apply // retention policies. diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index 6c9c29da..69b3c2e1 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -42,18 +42,10 @@ 'ingester.wal-enabled': true, 'ingester.checkpoint-enabled': true, 'ingester.recover-from-wal': true, - 'ingester.wal-dir': $._config.wal_dir, + 'ingester.wal-dir': $._config.ingester.wal_dir, 'ingester.checkpoint-duration': '15m', '-log.level': 'info', - 'ingester.tokens-file-path': $._config.wal_dir + '/tokens', - }, - - _config+:: { - wal_dir: '/wal_data', - ingester+: { - statefulset_replicas: 3, - statefulset_disk: '150Gi', - }, + 'ingester.tokens-file-path': $._config.ingester.wal_dir + '/tokens', }, ingester_ports:: $.util.defaultPorts, @@ -76,7 +68,7 @@ $.ingester_container + container.withArgsMixin($.util.mapToFlags($.ingester_statefulset_args)) + container.withVolumeMountsMixin([ - volumeMount.new('ingester-pvc', $._config.wal_dir), + volumeMount.new('ingester-pvc', $._config.ingester.wal_dir), ]), ingester_deployment_labels:: {},