Add query-scheduler.libsonnet

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## master / unreleased
 
 * [FEATURE] Added "Cortex / Rollout progress" dashboard. #289 #290
+* [FEATURE] Added support for using query-scheduler component, which moves the queue out of query-frontend, and allows scaling up of query-frontend component. #295
 * [ENHANCEMENT] Added `newCompactorStatefulSet()` function to create a custom statefulset for the compactor. #287
 * [ENHANCEMENT] Added option to configure compactor job name used in dashboards and alerts. #287
 * [ENHANCEMENT] Added `CortexCompactorHasNotSuccessfullyRunCompaction` alert. #292 #294

cortex/config.libsonnet

@@ -434,6 +434,9 @@
 
     alertmanager_enabled: false,
 
+    // Enables query-scheduler component, and reconfigures querier and query-frontend to use it.
+    query_scheduler_enabled: false,
+
     // Enables streaming of chunks from ingesters using blocks.
     ingester_stream_chunks_when_using_blocks: true,
   },

cortex/cortex.libsonnet

@@ -13,6 +13,7 @@
 (import 'table-manager.libsonnet') +
 (import 'ruler.libsonnet') +
 (import 'alertmanager.libsonnet') +
+(import 'query-scheduler.libsonnet') +
 
 // Supporting services
 (import 'etcd.libsonnet') +

cortex/images.libsonnet

@@ -17,6 +17,7 @@
     flusher: self.cortex,
     ruler: self.cortex,
     store_gateway: self.cortex,
+    query_scheduler: self.cortex,
 
     cortex_tools: 'grafana/cortex-tools:v0.4.0',
     query_tee: 'quay.io/cortexproject/query-tee:v1.8.0',

cortex/query-scheduler.libsonnet

@@ -0,0 +1,52 @@
+// Query-scheduler is optional service. When query-scheduler.libsonnet is added to Cortex, querier and frontend
+// are reconfigured to use query-scheduler service.
+{
+  local container = $.core.v1.container,
+  local deployment = $.apps.v1.deployment,
+  local service = $.core.v1.service,
+
+  query_scheduler_args+::
+    $._config.grpcConfig
+    {
+      target: 'query-scheduler',
+      'log.level': 'debug',
+      'query-scheduler.max-outstanding-requests-per-tenant': 100,
+    },
+
+  query_scheduler_container::
+    container.new('query-scheduler', $._images.query_scheduler) +
+    container.withPorts($.util.defaultPorts) +
+    container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) +
+    $.jaeger_mixin +
+    $.util.readinessProbe +
+    $.util.resourcesRequests('2', '1Gi') +
+    $.util.resourcesLimits(null, '2Gi'),
+
+
+  query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else
+    deployment.new('query-scheduler', 2, [$.query_scheduler_container]) +
+    $.util.configVolumeMount('overrides', '/etc/cortex') +
+    $.util.antiAffinity,
+
+  query_scheduler_service: if !$._config.query_scheduler_enabled then {} else
+    $.util.serviceFor($.query_scheduler_deployment),
+
+  // Headless to make sure resolution gets IP address of target pods, and not service IP.
+  query_scheduler_discovery_service: if !$._config.query_scheduler_enabled then {} else
+    $.util.serviceFor($.query_scheduler_deployment) +
+    service.mixin.spec.withPublishNotReadyAddresses(true) +
+    service.mixin.spec.withClusterIp('None') +
+    service.mixin.metadata.withName('query-scheduler-discovery'),
+
+  // Reconfigure querier and query-frontend to use scheduler.
+  querier_args+:: if !$._config.query_scheduler_enabled then {} else {
+    'querier.worker-match-max-concurrent': 'true',
+    'querier.worker-parallelism': null,  // Disabled since we set worker-match-max-concurrent.
+    'querier.frontend-address': null,
+    'querier.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config,
+  },
+
+  query_frontend_args+:: if !$._config.query_scheduler_enabled then {} else {
+    'frontend.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config,
+  },
+}