Chunks blocks migration (grafana/cortex-jsonnet#148)

* Allow configuring querier with second store engine. * Introduced newIngesterStatefulSet and newIngesterPdb functions. * Rename parameters to be more clear.
grafana · Jul 27, 2020 · 0c1a3c7 · 0c1a3c7
1 parent 66e79ea
commit 0c1a3c7
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 20 deletions.
diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet
@@ -58,6 +58,8 @@
     // Use the Cortex chunks storage engine by default, while giving the ability
     // to switch to tsdb storage.
     storage_engine: 'chunks',
+    // Secondary storage engine is only used for querying.
+    querier_second_storage_engine: null,
     storage_tsdb_bucket_name: error 'must specify GCS bucket name to store TSDB blocks',
 
     store_gateway_replication_factor: 3,
@@ -113,7 +115,7 @@
 
     storeConfig: self.storeMemcachedChunksConfig,
 
-    storeMemcachedChunksConfig: if $._config.memcached_chunks_enabled && $._config.storage_engine == 'chunks' then
+    storeMemcachedChunksConfig: if $._config.memcached_chunks_enabled && ($._config.storage_engine == 'chunks' || $._config.querier_second_storage_engine == 'chunks') then
       {
         'store.chunks-cache.memcached.hostname': 'memcached.%s.svc.cluster.local' % $._config.namespace,
         'store.chunks-cache.memcached.service': 'memcached-client',
@@ -131,8 +133,8 @@
     // TSDB blocks storage configuration, used only when 'tsdb' storage
     // engine is explicitly enabled.
     storageTSDBConfig: (
-      if $._config.storage_engine != 'tsdb' then {} else {
-        'store.engine': 'tsdb',
+      if $._config.storage_engine == 'tsdb' || $._config.querier_second_storage_engine == 'tsdb' then {
+        'store.engine': $._config.storage_engine,  // May still be chunks
         'experimental.tsdb.dir': '/data/tsdb',
         'experimental.tsdb.bucket-store.sync-dir': '/data/tsdb',
         'experimental.tsdb.bucket-store.ignore-deletion-marks-delay': '1h',
@@ -147,7 +149,7 @@
         'experimental.store-gateway.sharding-ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace,
         'experimental.store-gateway.sharding-ring.prefix': '',
         'experimental.store-gateway.replication-factor': $._config.store_gateway_replication_factor,
-      }
+      } else {}
     ),
 
     // Shared between the Ruler and Querier

diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet
@@ -75,10 +75,12 @@
 
   local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget,
 
-  ingester_pdb:
+  newIngesterPdb(pdbName, ingesterName)::
     podDisruptionBudget.new() +
-    podDisruptionBudget.mixin.metadata.withName('ingester-pdb') +
-    podDisruptionBudget.mixin.metadata.withLabels({ name: 'ingester-pdb' }) +
-    podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: name }) +
+    podDisruptionBudget.mixin.metadata.withName(pdbName) +
+    podDisruptionBudget.mixin.metadata.withLabels({ name: pdbName }) +
+    podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: ingesterName }) +
     podDisruptionBudget.mixin.spec.withMaxUnavailable(1),
+
+  ingester_pdb: self.newIngesterPdb('ingester-pdb', name),
 }
diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet
@@ -10,7 +10,7 @@
     {
       target: 'querier',
 
-      // Increase HTTP server response write timeout, as we were seeing some
+      // Increase HTTP server response write timeout, as we were seeing some
       // queries that return a lot of data timeing out.
       'server.http-write-timeout': '1m',
 
@@ -22,6 +22,8 @@
       'querier.frontend-address': 'query-frontend-discovery.%(namespace)s.svc.cluster.local:9095' % $._config,
       'querier.frontend-client.grpc-max-send-msg-size': 100 << 20,
 
+      'querier.second-store-engine': $._config.querier_second_storage_engine,
+
       'log.level': 'debug',
     },
 

diff --git a/operations/mimir/tsdb.libsonnet b/operations/mimir/tsdb.libsonnet
@@ -84,18 +84,17 @@
     'ingester.tokens-file-path': '/data/tokens',
   },
 
-  ingester_container+::
-    container.withVolumeMountsMixin([
-      volumeMount.new('ingester-data', '/data'),
-    ]),
-
-  ingester_statefulset:
-    statefulSet.new('ingester', 3, [$.ingester_container], ingester_data_pvc) +
-    statefulSet.mixin.spec.withServiceName('ingester') +
+  newIngesterStatefulSet(name, container)::
+    statefulSet.new(name, 3, [
+      container + $.core.v1.container.withVolumeMountsMixin([
+        volumeMount.new('ingester-data', '/data'),
+      ]),
+    ], ingester_data_pvc) +
+    statefulSet.mixin.spec.withServiceName(name) +
     statefulSet.mixin.metadata.withNamespace($._config.namespace) +
-    statefulSet.mixin.metadata.withLabels({ name: 'ingester' }) +
-    statefulSet.mixin.spec.template.metadata.withLabels({ name: 'ingester' } + $.ingester_deployment_labels) +
-    statefulSet.mixin.spec.selector.withMatchLabels({ name: 'ingester' }) +
+    statefulSet.mixin.metadata.withLabels({ name: name }) +
+    statefulSet.mixin.spec.template.metadata.withLabels({ name: name } + $.ingester_deployment_labels) +
+    statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) +
     statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) +
     statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(600) +
     statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') +
@@ -108,6 +107,8 @@
     // ready).
     statefulSet.mixin.spec.withPodManagementPolicy('Parallel'),
 
+  ingester_statefulset: self.newIngesterStatefulSet('ingester', $.ingester_container),
+
   ingester_service:
     $.util.serviceFor($.ingester_statefulset, $.ingester_service_ignored_labels),