Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add a version of the mixin dashboards for meta monitoring #12700

Merged
merged 10 commits into from
Apr 26, 2024
Merged

feat: Add a version of the mixin dashboards for meta monitoring #12700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
56a0acd
Add mixin for meta monitoring
MichelHollands Apr 19, 2024
73a9337
Run fmt-jsonnet
MichelHollands Apr 19, 2024
a7cfa41
Merge branch 'main' into add_combined_mixin_state
MichelHollands Apr 24, 2024
8748f30
Fix path for alerts for meta monitoring
MichelHollands Apr 24, 2024
91fcc3d
Run mixin generation with latest build image
MichelHollands Apr 24, 2024
14c2d7c
Merge branch 'main' into add_combined_mixin_state
MichelHollands Apr 24, 2024
2ef846e
Merge branch 'main' into add_combined_mixin_state
MichelHollands Apr 25, 2024
4f0834d
Do not build meta monitoring version here
MichelHollands Apr 26, 2024
af9435b
Merge branch 'add_combined_mixin_state' of github.com:grafana/loki in…
MichelHollands Apr 26, 2024
0c20db9
Merge branch 'main' of github.com:grafana/loki into add_combined_mixi…
MichelHollands Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@
"span": 6,
"targets": [
{
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval])) by (user)",
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-backend.*\"}[$__rate_interval])) by (user)",
"format": "time_series",
"legendFormat": "{{user}}",
"legendLink": null
Expand All @@ -606,7 +606,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-backend.*\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"refId": "A"
}
],
Expand All @@ -619,7 +619,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-backend.*\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"refId": "A"
}
],
Expand Down
4 changes: 2 additions & 2 deletions production/loki-mixin-compiled-ssd/dashboards/loki-retention.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@
"span": 4,
"targets": [
{
"expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})",
"expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-backend\"})",
"format": "time_series",
"legendFormat": "{{pod}}",
"legendLink": null
Expand Down Expand Up @@ -1367,7 +1367,7 @@
"span": 12,
"targets": [
{
"expr": "{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}",
"expr": "{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-backend\"}",
"refId": "A"
}
],
Expand Down
5 changes: 5 additions & 0 deletions production/loki-mixin/config.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,10 @@
// The prefix used to match the write and read pods on SSD mode.
pod_prefix_matcher: '(loki|enterprise-logs)',
},

// Meta-monitoring related configuration
meta_monitoring: {
enabled: false,
},
},
}
6 changes: 5 additions & 1 deletion production/loki-mixin/dashboards/loki-chunks.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ local utils = import 'mixin-utils/utils.libsonnet';
local dashboards = self,
'loki-chunks.json': {
local cfg = self,
labelsSelector:: $._config.per_cluster_label + '="$cluster", job=~"$namespace/%s"' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*'),
labelsSelector:: $._config.per_cluster_label + '="$cluster", job=~"$namespace/%s"' % (
if $._config.meta_monitoring.enabled
then '(ingester.*|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*'
),
} +
$.dashboard('Loki / Chunks', uid='chunks')
.addCluster()
Expand Down
4 changes: 3 additions & 1 deletion production/loki-mixin/dashboards/loki-deletion.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ local g = import 'grafana-builder/grafana.libsonnet';
local utils = import 'mixin-utils/utils.libsonnet';

(import 'dashboard-utils.libsonnet') {
local compactor_matcher = if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
local compactor_matcher = if $._config.meta_monitoring.enabled
then 'pod=~"(compactor|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-backend.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
grafanaDashboards+::
{
'loki-deletion.json':
Expand Down
28 changes: 21 additions & 7 deletions production/loki-mixin/dashboards/loki-operational.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,31 @@ local utils = import 'mixin-utils/utils.libsonnet';

jobMatchers:: {
cortexgateway: [utils.selector.re('job', '($namespace)/cortex-gw(-internal)?')],
distributor: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'distributor'))],
ingester: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*'))],
querier: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'querier'))],
queryFrontend: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))],
distributor: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(distributor|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'distributor'))],
ingester: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(ingester|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*'))],
querier: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(querier|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'querier'))],
queryFrontend: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(query-frontend|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))],
},

podMatchers:: {
cortexgateway: [utils.selector.re('pod', 'cortex-gw')],
distributor: [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else 'distributor.*'))],
ingester: [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else 'ingester.*'))],
querier: [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else 'querier.*'))],
distributor: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(distributor|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else 'distributor.*'))],
ingester: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(ingester|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else 'ingester.*'))],
querier: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(querier|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else 'querier.*'))],
},
}
+ lokiOperational + {
Expand Down
16 changes: 12 additions & 4 deletions production/loki-mixin/dashboards/loki-reads-resources.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,19 @@ local grafana = import 'grafonnet/grafana.libsonnet';
local utils = import 'mixin-utils/utils.libsonnet';

(import 'dashboard-utils.libsonnet') {
local index_gateway_pod_matcher = if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="index-gateway"',
local index_gateway_job_matcher = if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'index-gateway',
local index_gateway_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|index-gateway", pod=~"(index-gateway.*|%s-read.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="index-gateway"',
local index_gateway_job_matcher = if $._config.meta_monitoring.enabled
then '(index-gateway.*|%s-read.*|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'index-gateway',

local ingester_pod_matcher = if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"',
local ingester_job_matcher = if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.+',
local ingester_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|ingester", pod=~"(ingester.*|%s-write.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"',
local ingester_job_matcher = if $._config.meta_monitoring.enabled
then '(ingester.+|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.+',

grafanaDashboards+::
{
Expand Down
28 changes: 21 additions & 7 deletions production/loki-mixin/dashboards/loki-reads.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,27 @@ local utils = import 'mixin-utils/utils.libsonnet';

matchers:: {
cortexgateway: [utils.selector.re('job', '($namespace)/cortex-gw(-internal)?')],
queryFrontend: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))],
querier: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'querier'))],
ingester: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester'))],
ingesterZoneAware: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester-zone.*'))],
querierOrIndexGateway: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else '(querier|index-gateway)'))],
indexGateway: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-backend' % $._config.ssd.pod_prefix_matcher else 'index-gateway'))],
bloomGateway: [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-backend' % $._config.ssd.pod_prefix_matcher else 'bloom-gateway'))],
queryFrontend: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(query-frontend|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))],
querier: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(querier|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'querier'))],
ingester: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(ingester|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester'))],
ingesterZoneAware: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(ingester-zone-.*|%s-write|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester-zone.*'))],
querierOrIndexGateway: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(querier|index-gateway|%s-read|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else '(querier|index-gateway)'))],
indexGateway: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(index-gateway|%s-backend|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-backend' % $._config.ssd.pod_prefix_matcher else 'index-gateway'))],
bloomGateway: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(bloom-gateway|%s-backend|loki-single-binary' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-backend' % $._config.ssd.pod_prefix_matcher else 'bloom-gateway'))],
},

local selector(matcherId) =
Expand Down
8 changes: 6 additions & 2 deletions production/loki-mixin/dashboards/loki-retention.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
local utils = import 'mixin-utils/utils.libsonnet';

(import 'dashboard-utils.libsonnet') {
local compactor_pod_matcher = if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
local compactor_job_matcher = if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'compactor',
local compactor_pod_matcher = if $._config.meta_monitoring.enabled
then 'pod=~(compactor.*|%s-backend.*|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
local compactor_job_matcher = if $._config.meta_monitoring.enabled
then '(compactor|%s-backend.*|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then '%s-backend' % $._config.ssd.pod_prefix_matcher else 'compactor',
grafanaDashboards+::
{
'loki-retention.json':
Expand Down
8 changes: 6 additions & 2 deletions production/loki-mixin/dashboards/loki-writes-resources.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ local grafana = import 'grafonnet/grafana.libsonnet';
local utils = import 'mixin-utils/utils.libsonnet';

(import 'dashboard-utils.libsonnet') {
local ingester_pod_matcher = if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"',
local ingester_job_matcher = if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*',
local ingester_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|ingester", pod=~"(ingester.*|%s-write.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"',
local ingester_job_matcher = if $._config.meta_monitoring.enabled
then '(ingester.*|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*',

grafanaDashboards+::
{
Expand Down
Loading
Loading