Merge pull request #2496 from dgrisonnet/sli-metrics

Add Kubernetes components SLI metrics
prometheus-operator · Aug 28, 2024 · d04c1e1 · d04c1e1
2 parents 5d03655 + 89f1ee2
commit d04c1e1
Show file tree

Hide file tree

Showing 7 changed files with 137 additions and 52 deletions.
diff --git a/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet b/jsonnet/kube-prometheus/components/k8s-control-plane.libsonnet
@@ -71,13 +71,23 @@ function(params) {
     },
     spec: {
       jobLabel: 'app.kubernetes.io/name',
-      endpoints: [{
-        port: 'https-metrics',
-        interval: '30s',
-        scheme: 'https',
-        bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
-        tlsConfig: { insecureSkipVerify: true },
-      }],
+      endpoints: [
+        {
+          port: 'https-metrics',
+          interval: '30s',
+          scheme: 'https',
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          tlsConfig: { insecureSkipVerify: true },
+        },
+        {
+          port: 'https-metrics',
+          interval: '5s',
+          scheme: 'https',
+          path: '/metrics/slis',
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          tlsConfig: { insecureSkipVerify: true },
+        },
+      ],
       selector: {
         matchLabels: { 'app.kubernetes.io/name': 'kube-scheduler' },
       },
@@ -174,6 +184,20 @@ function(params) {
             targetLabel: 'metrics_path',
           }],
         },
+        {
+          port: 'https-metrics',
+          scheme: 'https',
+          path: '/metrics/slis',
+          interval: '5s',
+          honorLabels: true,
+          tlsConfig: { insecureSkipVerify: true },
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          relabelings: [{
+            action: 'replace',
+            sourceLabels: ['__metrics_path__'],
+            targetLabel: 'metrics_path',
+          }],
+        },
       ],
       selector: {
         matchLabels: { 'app.kubernetes.io/name': 'kubelet' },
@@ -193,22 +217,34 @@ function(params) {
     },
     spec: {
       jobLabel: 'app.kubernetes.io/name',
-      endpoints: [{
-        port: 'https-metrics',
-        interval: '30s',
-        scheme: 'https',
-        bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
-        tlsConfig: {
-          insecureSkipVerify: true,
+      endpoints: [
+        {
+          port: 'https-metrics',
+          interval: '30s',
+          scheme: 'https',
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          tlsConfig: {
+            insecureSkipVerify: true,
+          },
+          metricRelabelings: relabelings + [
+            {
+              sourceLabels: ['__name__'],
+              regex: 'etcd_(debugging|disk|request|server).*',
+              action: 'drop',
+            },
+          ],
         },
-        metricRelabelings: relabelings + [
-          {
-            sourceLabels: ['__name__'],
-            regex: 'etcd_(debugging|disk|request|server).*',
-            action: 'drop',
+        {
+          port: 'https-metrics',
+          interval: '5s',
+          scheme: 'https',
+          path: '/metrics/slis',
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          tlsConfig: {
+            insecureSkipVerify: true,
           },
-        ],
-      }],
+        },
+      ],
       selector: {
         matchLabels: { 'app.kubernetes.io/name': 'kube-controller-manager' },
       },
@@ -236,38 +272,51 @@ function(params) {
       namespaceSelector: {
         matchNames: ['default'],
       },
-      endpoints: [{
-        port: 'https',
-        interval: '30s',
-        scheme: 'https',
-        tlsConfig: {
-          caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
-          serverName: 'kubernetes',
-        },
-        bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
-        metricRelabelings: relabelings + [
-          {
-            sourceLabels: ['__name__'],
-            regex: 'etcd_(debugging|disk|server).*',
-            action: 'drop',
-          },
-          {
-            sourceLabels: ['__name__'],
-            regex: 'apiserver_admission_controller_admission_latencies_seconds_.*',
-            action: 'drop',
-          },
-          {
-            sourceLabels: ['__name__'],
-            regex: 'apiserver_admission_step_admission_latencies_seconds_.*',
-            action: 'drop',
+      endpoints: [
+        {
+          port: 'https',
+          interval: '30s',
+          scheme: 'https',
+          tlsConfig: {
+            caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
+            serverName: 'kubernetes',
           },
-          {
-            sourceLabels: ['__name__', 'le'],
-            regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
-            action: 'drop',
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+          metricRelabelings: relabelings + [
+            {
+              sourceLabels: ['__name__'],
+              regex: 'etcd_(debugging|disk|server).*',
+              action: 'drop',
+            },
+            {
+              sourceLabels: ['__name__'],
+              regex: 'apiserver_admission_controller_admission_latencies_seconds_.*',
+              action: 'drop',
+            },
+            {
+              sourceLabels: ['__name__'],
+              regex: 'apiserver_admission_step_admission_latencies_seconds_.*',
+              action: 'drop',
+            },
+            {
+              sourceLabels: ['__name__', 'le'],
+              regex: 'apiserver_request_duration_seconds_bucket;(0.15|0.25|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2.5|3|3.5|4.5|6|7|8|9|15|25|30|50)',
+              action: 'drop',
+            },
+          ],
+        },
+        {
+          port: 'https',
+          interval: '5s',
+          scheme: 'https',
+          path: '/metrics/slis',
+          tlsConfig: {
+            caFile: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
+            serverName: 'kubernetes',
           },
-        ],
-      }],
+          bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+        },
+      ],
     },
   },
 

diff --git a/jsonnet/kube-prometheus/components/prometheus.libsonnet b/jsonnet/kube-prometheus/components/prometheus.libsonnet
@@ -223,7 +223,7 @@ function(params) {
         verbs: ['get'],
       },
       {
-        nonResourceURLs: ['/metrics'],
+        nonResourceURLs: ['/metrics', '/metrics/slis'],
         verbs: ['get'],
       },
     ],

diff --git a/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml b/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
@@ -65,6 +65,14 @@ spec:
     tlsConfig:
       caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
       serverName: kubernetes
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 5s
+    path: /metrics/slis
+    port: https
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      serverName: kubernetes
   jobLabel: component
   namespaceSelector:
     matchNames:

diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml
@@ -51,6 +51,13 @@ spec:
     scheme: https
     tlsConfig:
       insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 5s
+    path: /metrics/slis
+    port: https-metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
   jobLabel: app.kubernetes.io/name
   namespaceSelector:
     matchNames:

diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml
@@ -14,6 +14,13 @@ spec:
     scheme: https
     tlsConfig:
       insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 5s
+    path: /metrics/slis
+    port: https-metrics
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
   jobLabel: app.kubernetes.io/name
   namespaceSelector:
     matchNames:

diff --git a/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml b/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml
@@ -96,6 +96,19 @@ spec:
     scheme: https
     tlsConfig:
       insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 5s
+    path: /metrics/slis
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
   jobLabel: app.kubernetes.io/name
   namespaceSelector:
     matchNames:

diff --git a/manifests/prometheus-clusterRole.yaml b/manifests/prometheus-clusterRole.yaml
@@ -17,5 +17,6 @@ rules:
   - get
 - nonResourceURLs:
   - /metrics
+  - /metrics/slis
   verbs:
   - get