Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mixin: fix errors on autoscaling metrics after series churn #9412

Merged
merged 5 commits into from
Sep 26, 2024
Merged

mixin: fix errors on autoscaling metrics after series churn #9412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef99b2a
WIP
dimitarvdimitrov Sep 25, 2024
5e41f39
mixin: fix errors on autoscaling metrics after series churn
dimitarvdimitrov Sep 25, 2024
609aa91
Add CHANGELOG.md entry
dimitarvdimitrov Sep 25, 2024
02920a5
Also fix alert
dimitarvdimitrov Sep 25, 2024
8248721
Regenerate helm tests
dimitarvdimitrov Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@
* [BUGFIX] Alerts: do not fire `MimirRingMembersMismatch` during the migration to experimental ingest storage. #8727
* [BUGFIX] Dashboards: avoid over-counting of ingesters metrics when migrating to experimental ingest storage. #9170
* [BUGFIX] Dashboards: fix `job_prefix` not utilized in `jobSelector`. #9155
* [BUGFIX] Dashboards: Fix autoscaling metrics joins when series churn. #9412
* [BUGFIX] Alerts: Fix autoscaling metrics joins in `MimirAutoscalerNotActive` when series churn. #9412

### Jsonnet

Expand Down
16 changes: 8 additions & 8 deletions ...oring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20357,7 +20357,7 @@ data:
"span": 4,
"targets": [
{
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, metric, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"format": "time_series",
"legendFormat": "{{scaler}} failures",
"legendLink": null
Expand Down Expand Up @@ -26151,7 +26151,7 @@ data:
"span": 6,
"targets": [
{
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, metric, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"} * 0),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"format": "time_series",
"legendFormat": "{{scaler}} failures",
"legendLink": null
Expand Down Expand Up @@ -26212,7 +26212,7 @@ data:
"span": 4,
"targets": [
{
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"expr": "sum by (scaler) (\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric, scaler) keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric_name) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"format": "time_series",
"legendFormat": "{{ scaler }}",
"legendLink": null
Expand Down Expand Up @@ -26261,7 +26261,7 @@ data:
"span": 4,
"targets": [
{
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"expr": "sum by (scaler) (\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric, scaler) keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric_name) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"format": "time_series",
"legendFormat": "{{ scaler }}",
"legendLink": null
Expand Down Expand Up @@ -26310,7 +26310,7 @@ data:
"span": 4,
"targets": [
{
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*queries.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"expr": "sum by (scaler) (\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric, scaler) keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*queries.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric_name) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"format": "time_series",
"legendFormat": "{{ scaler }}",
"legendLink": null
Expand Down Expand Up @@ -40428,7 +40428,7 @@ data:
"span": 3,
"targets": [
{
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"expr": "sum by (scaler) (\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric, scaler) keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric_name) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"format": "time_series",
"legendFormat": "{{ scaler }}",
"legendLink": null
Expand Down Expand Up @@ -40477,7 +40477,7 @@ data:
"span": 3,
"targets": [
{
"expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"expr": "sum by (scaler) (\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric, scaler) keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, scaledObject, metric_name) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
"format": "time_series",
"legendFormat": "{{ scaler }}",
"legendLink": null
Expand Down Expand Up @@ -40526,7 +40526,7 @@ data:
"span": 3,
"targets": [
{
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster, namespace, metric, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"} * 0),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
"format": "time_series",
"legendFormat": "{{scaler}} failures",
"legendLink": null
Expand Down
13 changes: 10 additions & 3 deletions ...amonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -990,16 +990,23 @@ spec:
# Match only Mimir namespaces.
* on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
# Add "metric" label.
+ on(cluster, namespace, horizontalpodautoscaler) group_right label_replace(kube_horizontalpodautoscaler_spec_target_metric*0, "metric", "$1", "metric_name", "(.+)")
+ on(cluster, namespace, horizontalpodautoscaler) group_right
# Using `max by ()` so that series churn doesn't break the promQL join
max by (cluster, namespace, horizontalpodautoscaler) (
label_replace(kube_horizontalpodautoscaler_spec_target_metric*0, "metric", "$1", "metric_name", "(.+)")
)
> 0),
"scaledObject", "$1", "horizontalpodautoscaler", "keda-hpa-(.*)"
)
)
# Alert only if the scaling metric exists and is > 0. If the KEDA ScaledObject is configured to scale down 0,
# then HPA ScalingActive may be false when expected to run 0 replicas. In this case, the scaling metric exported
# by KEDA could not exist at all or being exposed with a value of 0.
and on (cluster, namespace, metric, scaledObject)
(label_replace(keda_scaler_metrics_value, "namespace", "$0", "exported_namespace", ".+") > 0)
and on (cluster, namespace, metric, scaledObject) (
max by (cluster, namespace, metric, scaledObject) (
label_replace(keda_scaler_metrics_value, "namespace", "$0", "exported_namespace", ".+") > 0
)
)
for: 1h
labels:
severity: critical
Expand Down
13 changes: 10 additions & 3 deletions operations/mimir-mixin-compiled-baremetal/alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -964,16 +964,23 @@ groups:
# Match only Mimir namespaces.
* on(cluster, namespace) group_left max by(cluster, namespace) (cortex_build_info)
# Add "metric" label.
+ on(cluster, namespace, horizontalpodautoscaler) group_right label_replace(kube_horizontalpodautoscaler_spec_target_metric*0, "metric", "$1", "metric_name", "(.+)")
+ on(cluster, namespace, horizontalpodautoscaler) group_right
# Using `max by ()` so that series churn doesn't break the promQL join
max by (cluster, namespace, horizontalpodautoscaler) (
label_replace(kube_horizontalpodautoscaler_spec_target_metric*0, "metric", "$1", "metric_name", "(.+)")
)
> 0),
"scaledObject", "$1", "horizontalpodautoscaler", "keda-hpa-(.*)"
)
)
# Alert only if the scaling metric exists and is > 0. If the KEDA ScaledObject is configured to scale down 0,
# then HPA ScalingActive may be false when expected to run 0 replicas. In this case, the scaling metric exported
# by KEDA could not exist at all or being exposed with a value of 0.
and on (cluster, namespace, metric, scaledObject)
(label_replace(keda_scaler_metrics_value, "namespace", "$0", "exported_namespace", ".+") > 0)
and on (cluster, namespace, metric, scaledObject) (
max by (cluster, namespace, metric, scaledObject) (
label_replace(keda_scaler_metrics_value, "namespace", "$0", "exported_namespace", ".+") > 0
)
)
for: 1h
labels:
severity: critical
Expand Down
2 changes: 1 addition & 1 deletion operations/mimir-mixin-compiled-baremetal/dashboards/mimir-reads.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading