From e90e0de42193f6ffbb38e5f806b9534060a1833a Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Tue, 11 May 2021 17:44:15 +0100 Subject: [PATCH 1/6] feat: Allow configuration of ring members in gossip alerts Signed-off-by: Jack Baldry --- cortex-mixin/alerts/alerts.libsonnet | 4 ++-- cortex-mixin/config.libsonnet | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index 42c1e5f7..81ef396a 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -593,8 +593,8 @@ expr: ||| memberlist_client_cluster_members_count != on (%s) group_left - sum by (%s) (up{job=~".+/(admin-api|compactor|store-gateway|distributor|ingester.*|querier.*|cortex|ruler)"}) - ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels], + sum by (%s) (up{job=~".+/%s"}) + ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels, $._config.job_names.ring_members], 'for': '5m', labels: { severity: 'warning', diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index 94de5dd7..e0e76a92 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,6 +33,7 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', + ring_members: '(distributor|ingester|querier|cortex|ruler)', store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. From 36cf15b5db36e503e3200bf35ff9062ab42b975d Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Tue, 11 May 2021 17:45:08 +0100 Subject: [PATCH 2/6] fix: Add store-gateway and compactor ring_members Also re-order names for readability. Signed-off-by: Jack Baldry --- cortex-mixin/config.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index e0e76a92..803bbb79 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,7 +33,7 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', - ring_members: '(distributor|ingester|querier|cortex|ruler)', + ring_members: '(compactor|distributor|ingester|querier|ruler|store-gateway|cortex)', store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. From a95639d83b5c0e7a21e40834959ee8fd82ccdb9b Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Mon, 24 May 2021 10:54:46 +0100 Subject: [PATCH 3/6] fix: Match all ingester workloads and avoid matching the cortex-gateway Signed-off-by: Jack Baldry --- cortex-mixin/config.libsonnet | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index 803bbb79..a9eccf75 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,7 +33,9 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', - ring_members: '(compactor|distributor|ingester|querier|ruler|store-gateway|cortex)', + // ingester-.* accommodates multiple ingester StatefulSets or Deployments. + // cortex$ prevents matching the cortex-gateway. + ring_members: '(compactor|distributor|ingester-.*|querier|ruler|store-gateway|cortex$)', store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. From 4ddad08fbc2b0660064dc02173320aac5de1d823 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Thu, 3 Jun 2021 11:51:26 +0100 Subject: [PATCH 4/6] feat: Optionally allow use of array or string to configure ring members Signed-off-by: Jack Baldry --- cortex-mixin/config.libsonnet | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index a9eccf75..386745c2 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,9 +33,13 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', + // If non-null, this list is joined together to form the regexp for the ring_members matcher value. + // to completely, override the matcher value, you can make this null and override the `ring_members` + // field instead of this. // ingester-.* accommodates multiple ingester StatefulSets or Deployments. // cortex$ prevents matching the cortex-gateway. - ring_members: '(compactor|distributor|ingester-.*|querier|ruler|store-gateway|cortex$)', + ring_members_list: ['compactor', 'distributor', 'ingester-.*', 'querier', 'ruler', 'store-gateway', 'cortex$'], + ring_members: if self.ring_members_list != null then '(%s)' % std.join('|', self.ring_members_list) else '', store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. From 9c152e4723d5c245b70c1971c4429c32eff2921e Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Tue, 20 Jul 2021 10:40:39 +0100 Subject: [PATCH 5/6] address review feedback Signed-off-by: Jack Baldry --- cortex-mixin/alerts/alerts.libsonnet | 16 +++++++++++----- cortex-mixin/config.libsonnet | 8 +------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index 81ef396a..3c407b48 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -1,4 +1,9 @@ { + // simpleRegexpOpt produces a simple regexp that matches all strings in the input array. + local simpleRegexpOpt(strings) = + assert std.isArray(strings) : 'simpleRegexpOpt requires that `strings` is an array of strings`'; + '(' + std.join('|', strings) + ')', + groups+: [ { name: 'cortex_alerts', @@ -590,11 +595,12 @@ rules: [ { alert: 'CortexGossipMembersMismatch', - expr: ||| - memberlist_client_cluster_members_count - != on (%s) group_left - sum by (%s) (up{job=~".+/%s"}) - ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels, $._config.job_names.ring_members], + expr: + ||| + memberlist_client_cluster_members_count + != on (%s) group_left + sum by (%s) (up{job=~".+/%s"}) + ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels, simpleRegexpOpt($._config.job_names.ring_members)], 'for': '5m', labels: { severity: 'warning', diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index 386745c2..3a5ac33c 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,13 +33,7 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', - // If non-null, this list is joined together to form the regexp for the ring_members matcher value. - // to completely, override the matcher value, you can make this null and override the `ring_members` - // field instead of this. - // ingester-.* accommodates multiple ingester StatefulSets or Deployments. - // cortex$ prevents matching the cortex-gateway. - ring_members_list: ['compactor', 'distributor', 'ingester-.*', 'querier', 'ruler', 'store-gateway', 'cortex$'], - ring_members: if self.ring_members_list != null then '(%s)' % std.join('|', self.ring_members_list) else '', + ring_members: ['compactor', 'distributor', 'ingester-.*', 'querier', 'ruler', 'store-gateway', 'cortex'], store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments. From c3d6055f337b11fa50735a67475ca5ccaf998d80 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Fri, 27 Aug 2021 09:35:21 +0100 Subject: [PATCH 6/6] fix: Correct ingester and querier regexps Signed-off-by: Jack Baldry --- cortex-mixin/config.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet index 3a5ac33c..9b8d81a2 100644 --- a/cortex-mixin/config.libsonnet +++ b/cortex-mixin/config.libsonnet @@ -33,7 +33,7 @@ query_frontend: '(query-frontend.*|cortex$)', // Match also custom query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. table_manager: '(table-manager|cortex$)', - ring_members: ['compactor', 'distributor', 'ingester-.*', 'querier', 'ruler', 'store-gateway', 'cortex'], + ring_members: ['compactor', 'distributor', 'ingester.*', 'querier.*', 'ruler', 'store-gateway', 'cortex'], store_gateway: '(store-gateway|cortex$)', gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*', // Match also custom compactor deployments.