From 5e5077048dd6f3996fff0318d0f378e31fc5d20d Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 14 Dec 2020 10:24:46 +0100 Subject: [PATCH 1/2] Added avg and max blocks / tenant Signed-off-by: Marco Pracucci --- cortex-mixin/dashboards/compactor.libsonnet | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cortex-mixin/dashboards/compactor.libsonnet b/cortex-mixin/dashboards/compactor.libsonnet index 5d656c3f..36bff1d3 100644 --- a/cortex-mixin/dashboards/compactor.libsonnet +++ b/cortex-mixin/dashboards/compactor.libsonnet @@ -5,7 +5,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; ($.dashboard('Cortex / Compactor') + { uid: '9c408e1d55681ecb8a22c9fab46875cc' }) .addClusterSelectorTemplates() .addRow( - $.row('Compactions') + $.row('Summary') .addPanel( $.textPanel('', ||| - **Per-instance runs**: number of times a compactor instance triggers a compaction across all tenants its shard manage. @@ -52,6 +52,23 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher('compactor')) ) ) + .addRow( + $.row('') + .addPanel( + $.textPanel('', ||| + - **Average blocks / tenant**: the average number of blocks per tenant. + - **Tenants with largest number of blocks**: the 10 tenants with the largest number of blocks. + |||), + ) + .addPanel( + $.panel('Average blocks / tenant') + + $.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), 'avg'), + ) + .addPanel( + $.panel('Tenants with largest number of blocks') + + $.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), '{{user}}'), + ) + ) .addRow( $.row('Garbage Collector') .addPanel( From ca1b50bb2e91412953bc931e2cd77ace2c0f034b Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 14 Dec 2020 10:26:32 +0100 Subject: [PATCH 2/2] Updated CHANGELOG Signed-off-by: Marco Pracucci --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26735c9a..1c974bca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,12 @@ * [ENHANCEMENT] Introduce a resources dashboard for the Alertmanager. #219 * [ENHANCEMENT] Improves query visibility in the Ruler Dashboard for both chunks and blocks storage. #226 * [ENHANCEMENT] Add query-scheduler to dashboards. Add alert for queries stuck in scheduler. #228 -* [ENHANCEMENT] Improved blocks storage observability: #224 +* [ENHANCEMENT] Improved blocks storage observability: #224 #230 - Cortex / Writes: added current number of tenants in the cluster - Cortex / Writes Resources: added ingester disk read/writes/utilisation - Cortex / Reads Resources: added store-gateway disk read/writes/utilisation - Cortex / Queries: added "Lazy loaded index-headers" and "Index-header lazy load duration" - - Cortex / Compactor: added "Tenants compaction progress" + - Cortex / Compactor: added "Tenants compaction progress", "Average blocks / tenant" and "Tenants with largest number of blocks" - Alerts: added "CortexMemoryMapAreasTooHigh" * [BUGFIX] Fixed workingset memory panel while rolling out a StatefulSet. #229