From 43f209462974360682bbec4e79dc048355849a78 Mon Sep 17 00:00:00 2001 From: Damien Grisonnet Date: Tue, 27 Aug 2024 21:48:38 +0200 Subject: [PATCH 1/2] jsonnet: add component SLI metrics Signed-off-by: Damien Grisonnet --- .../components/k8s-control-plane.libsonnet | 151 ++++++++++++------ .../components/prometheus.libsonnet | 2 +- 2 files changed, 101 insertions(+), 52 deletions(-) diff --git a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet index a771e95dbe..8692f38df1 100644 --- a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet +++ b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet @@ -71,13 +71,23 @@ function(params) { }, spec: { jobLabel: 'app.kubernetes.io/name', - endpoints: [{ - port: 'https-metrics', - interval: '30s', - scheme: 'https', - bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', - tlsConfig: { insecureSkipVerify: true }, - }], + endpoints: [ + { + port: 'https-metrics', + interval: '30s', + scheme: 'https', + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { insecureSkipVerify: true }, + }, + { + port: 'https-metrics', + interval: '5s', + scheme: 'https', + path: '/metrics/slis', + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { insecureSkipVerify: true }, + }, + ], selector: { matchLabels: { 'app.kubernetes.io/name': 'kube-scheduler' }, }, @@ -174,6 +184,20 @@ function(params) { targetLabel: 'metrics_path', }], }, + { + port: 'https-metrics', + scheme: 'https', + path: '/metrics/slis', + interval: '5s', + honorLabels: true, + tlsConfig: { insecureSkipVerify: true }, + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + relabelings: [{ + action: 'replace', + sourceLabels: ['__metrics_path__'], + targetLabel: 'metrics_path', + }], + }, ], selector: { matchLabels: { 'app.kubernetes.io/name': 'kubelet' }, @@ -193,22 +217,34 @@ function(params) { }, spec: { jobLabel: 'app.kubernetes.io/name', - endpoints: [{ - port: 'https-metrics', - interval: '30s', - scheme: 'https', - bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', - tlsConfig: { - insecureSkipVerify: true, + endpoints: [ + { + port: 'https-metrics', + interval: '30s', + scheme: 'https', + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { + insecureSkipVerify: true, + }, + metricRelabelings: relabelings + [ + { + sourceLabels: ['__name__'], + regex: 'etcd_(debugging|disk|request|server).*', + action: 'drop', + }, + ], }, - metricRelabelings: relabelings + [ - { - sourceLabels: ['__name__'], - regex: 'etcd_(debugging|disk|request|server).*', - action: 'drop', + { + port: 'https-metrics', + interval: '5s', + scheme: 'https', + path: '/metrics/slis', + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { + insecureSkipVerify: true, }, - ], - }], + }, + ], selector: { matchLabels: { 'app.kubernetes.io/name': 'kube-controller-manager' }, }, @@ -236,38 +272,51 @@ function(params) { namespaceSelector: { matchNames: ['default'], }, - endpoints: [{ - port: 'https', - interval: '30s', - scheme: 'https', - tlsConfig: { - caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - serverName: 'kubernetes', - }, - bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', - metricRelabelings: relabelings + [ - { - sourceLabels: ['__name__'], - regex: 'etcd_(debugging|disk|server).*', - action: 'drop', - }, - { - sourceLabels: ['__name__'], - regex: 'apiserver_admission_controller_admission_latencies_seconds_.*', - action: 'drop', - }, - { - sourceLabels: ['__name__'], - regex: 'apiserver_admission_step_admission_latencies_seconds_.*', - action: 'drop', + endpoints: [ + { + port: 'https', + interval: '30s', + scheme: 'https', + tlsConfig: { + caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + serverName: 'kubernetes', }, - { - sourceLabels: ['__name__', 'le'], - regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)', - action: 'drop', + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + metricRelabelings: relabelings + [ + { + sourceLabels: ['__name__'], + regex: 'etcd_(debugging|disk|server).*', + action: 'drop', + }, + { + sourceLabels: ['__name__'], + regex: 'apiserver_admission_controller_admission_latencies_seconds_.*', + action: 'drop', + }, + { + sourceLabels: ['__name__'], + regex: 'apiserver_admission_step_admission_latencies_seconds_.*', + action: 'drop', + }, + { + sourceLabels: ['__name__', 'le'], + regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)', + action: 'drop', + }, + ], + }, + { + port: 'https', + interval: '5s', + scheme: 'https', + path: '/metrics/slis', + tlsConfig: { + caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + serverName: 'kubernetes', }, - ], - }], + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + }, + ], }, }, diff --git a/jsonnet/kube-prometheus/components/prometheus.libsonnet b/jsonnet/kube-prometheus/components/prometheus.libsonnet index 72d5019cd5..8aaad02aec 100644 --- a/jsonnet/kube-prometheus/components/prometheus.libsonnet +++ b/jsonnet/kube-prometheus/components/prometheus.libsonnet @@ -223,7 +223,7 @@ function(params) { verbs: ['get'], }, { - nonResourceURLs: ['/metrics'], + nonResourceURLs: ['/metrics', '/metrics/slis'], verbs: ['get'], }, ], From 89f1ee2c213fef8133a01dfd48b434b98b4cb3ea Mon Sep 17 00:00:00 2001 From: Damien Grisonnet Date: Tue, 27 Aug 2024 21:48:53 +0200 Subject: [PATCH 2/2] manifests: regenerate Signed-off-by: Damien Grisonnet --- ...ernetesControlPlane-serviceMonitorApiserver.yaml | 8 ++++++++ ...olPlane-serviceMonitorKubeControllerManager.yaml | 7 +++++++ ...tesControlPlane-serviceMonitorKubeScheduler.yaml | 7 +++++++ ...ubernetesControlPlane-serviceMonitorKubelet.yaml | 13 +++++++++++++ manifests/prometheus-clusterRole.yaml | 1 + 5 files changed, 36 insertions(+) diff --git a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml index bfc1f31545..75fe828d2c 100644 --- a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml +++ b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml @@ -65,6 +65,14 @@ spec: tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt serverName: kubernetes + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 5s + path: /metrics/slis + port: https + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: kubernetes jobLabel: component namespaceSelector: matchNames: diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml index 1a71e8e458..e8955d208c 100644 --- a/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml +++ b/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml @@ -51,6 +51,13 @@ spec: scheme: https tlsConfig: insecureSkipVerify: true + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 5s + path: /metrics/slis + port: https-metrics + scheme: https + tlsConfig: + insecureSkipVerify: true jobLabel: app.kubernetes.io/name namespaceSelector: matchNames: diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml index 7fd84fc373..19a6626f8d 100644 --- a/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml +++ b/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml @@ -14,6 +14,13 @@ spec: scheme: https tlsConfig: insecureSkipVerify: true + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 5s + path: /metrics/slis + port: https-metrics + scheme: https + tlsConfig: + insecureSkipVerify: true jobLabel: app.kubernetes.io/name namespaceSelector: matchNames: diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml index 96bbdbab72..2321391741 100644 --- a/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml +++ b/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml @@ -96,6 +96,19 @@ spec: scheme: https tlsConfig: insecureSkipVerify: true + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + interval: 5s + path: /metrics/slis + port: https-metrics + relabelings: + - action: replace + sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + tlsConfig: + insecureSkipVerify: true jobLabel: app.kubernetes.io/name namespaceSelector: matchNames: diff --git a/manifests/prometheus-clusterRole.yaml b/manifests/prometheus-clusterRole.yaml index 7abc593300..ea971084e7 100644 --- a/manifests/prometheus-clusterRole.yaml +++ b/manifests/prometheus-clusterRole.yaml @@ -17,5 +17,6 @@ rules: - get - nonResourceURLs: - /metrics + - /metrics/slis verbs: - get