From 9de782e719d6a616bcccd8e73ea4d8ba71bffa89 Mon Sep 17 00:00:00 2001 From: Floppy Disk Date: Thu, 16 Jan 2025 13:23:56 +0300 Subject: [PATCH 1/5] fix --- packages/system/kubevirt-operator/alerts/PrometheusRule.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/system/kubevirt-operator/alerts/PrometheusRule.yaml b/packages/system/kubevirt-operator/alerts/PrometheusRule.yaml index bf8604d0a..f20762dfb 100644 --- a/packages/system/kubevirt-operator/alerts/PrometheusRule.yaml +++ b/packages/system/kubevirt-operator/alerts/PrometheusRule.yaml @@ -2,7 +2,6 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: vm-not-running-alert - namespace: monitoring spec: groups: - name: kubevirt-alerts From 46556574c4a8f16cbe06c113d48634c3859b85de Mon Sep 17 00:00:00 2001 From: Floppy Disk Date: Wed, 5 Feb 2025 13:20:36 +0300 Subject: [PATCH 2/5] add-managed-etcd-monitoring --- .../core/platform/bundles/distro-full.yaml | 4 + .../core/platform/bundles/distro-hosted.yaml | 4 + packages/core/platform/bundles/paas-full.yaml | 4 + .../core/platform/bundles/paas-hosted.yaml | 4 + .../templates/etcd-proxy-scrape.yaml | 133 ++++++++++++++++++ .../templates/etcd-scrape.yaml | 69 ++++----- packages/system/monitoring-agents/values.yaml | 4 + 7 files changed, 188 insertions(+), 34 deletions(-) create mode 100644 packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml diff --git a/packages/core/platform/bundles/distro-full.yaml b/packages/core/platform/bundles/distro-full.yaml index 19ad78ec3..3c99734f8 100644 --- a/packages/core/platform/bundles/distro-full.yaml +++ b/packages/core/platform/bundles/distro-full.yaml @@ -75,6 +75,10 @@ releases: privileged: true optional: true dependsOn: [cilium,victoria-metrics-operator] + values: + scrapeRules: + etcd: + enabled: true - name: metallb releaseName: metallb diff --git a/packages/core/platform/bundles/distro-hosted.yaml b/packages/core/platform/bundles/distro-hosted.yaml index 45f58a8c2..650efccde 100644 --- a/packages/core/platform/bundles/distro-hosted.yaml +++ b/packages/core/platform/bundles/distro-hosted.yaml @@ -58,6 +58,10 @@ releases: privileged: true optional: true dependsOn: [victoria-metrics-operator] + values: + scrapeRules: + etcd: + enabled: true - name: etcd-operator releaseName: etcd-operator diff --git a/packages/core/platform/bundles/paas-full.yaml b/packages/core/platform/bundles/paas-full.yaml index 328d0bbbf..b45992ecd 100644 --- a/packages/core/platform/bundles/paas-full.yaml +++ b/packages/core/platform/bundles/paas-full.yaml @@ -97,6 +97,10 @@ releases: namespace: cozy-monitoring privileged: true dependsOn: [cilium,kubeovn,victoria-metrics-operator] + values: + scrapeRules: + etcd: + enabled: true - name: kubevirt-operator releaseName: kubevirt-operator diff --git a/packages/core/platform/bundles/paas-hosted.yaml b/packages/core/platform/bundles/paas-hosted.yaml index 3b1085cda..82edc2ab7 100644 --- a/packages/core/platform/bundles/paas-hosted.yaml +++ b/packages/core/platform/bundles/paas-hosted.yaml @@ -70,6 +70,10 @@ releases: namespace: cozy-monitoring privileged: true dependsOn: [victoria-metrics-operator] + values: + scrapeRules: + etcd: + enabled: true - name: etcd-operator releaseName: etcd-operator diff --git a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml new file mode 100644 index 000000000..183a60b5b --- /dev/null +++ b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml @@ -0,0 +1,133 @@ +{{- if not .Values.scrapeRules.etcd.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-rbac-proxy + namespace: cozy-monitoring + labels: + app: kube-rbac-proxy +spec: + selector: + matchLabels: + app: kube-rbac-proxy + template: + metadata: + labels: + app: kube-rbac-proxy + spec: + serviceAccountName: kube-rbac-proxy + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + containers: + - name: kube-rbac-proxy + image: quay.io/brancz/kube-rbac-proxy:v0.11.0 + args: + - "--secure-listen-address=0.0.0.0:9443" + - "--upstream=http://127.0.0.1:2381/" + ports: + - containerPort: 9443 + name: etcd-metrics + securityContext: + runAsUser: 1000 + runAsNonRoot: true + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-rbac-proxy + namespace: cozy-monitoring + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-rbac-proxy-auth +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-rbac-proxy-auth-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-rbac-proxy-auth +subjects: + - kind: ServiceAccount + name: kube-rbac-proxy + namespace: cozy-monitoring + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vm-scrape + namespace: cozy-monitoring + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: etcd-metrics-reader +rules: +- nonResourceURLs: ["/metrics"] + verbs: ["get"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: etcd-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: etcd-metrics-reader +subjects: +- kind: ServiceAccount + name: vm-scrape + namespace: cozy-monitoring + +--- + +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: vm-token + annotations: + kubernetes.io/service-account.name: vm-scrape + +--- + +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: etcd-managment-scrape +spec: + podMetricsEndpoints: + - port: etcd-metrics + scheme: https + tlsConfig: + insecureSkipVerify: true + bearerTokenSecret: + name: vm-token + key: token + selector: + matchLabels: + app: kube-rbac-proxy +{{- end }} diff --git a/packages/system/monitoring-agents/templates/etcd-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-scrape.yaml index 829e0040b..d776e9da2 100644 --- a/packages/system/monitoring-agents/templates/etcd-scrape.yaml +++ b/packages/system/monitoring-agents/templates/etcd-scrape.yaml @@ -1,34 +1,35 @@ -#--- -#apiVersion: operator.victoriametrics.com/v1beta1 -#kind: VMNodeScrape -#metadata: -# name: kube-etcd -# namespace: cozy-monitoring -#spec: -# selector: -# node-role.kubernetes.io/control-plane: "" -# bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token -# honorLabels: true -# metricRelabelConfigs: -# - action: labeldrop -# regex: (uid) -# - action: labeldrop -# regex: (id|name) -# - action: drop -# regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) -# source_labels: -# - __name__ -# port: "2379" -# relabelConfigs: -# - action: labelmap -# regex: __meta_kubernetes_node_label_(.+) -# - sourceLabels: -# - __metrics_path__ -# targetLabel: metrics_path -# - replacement: etcd -# targetLabel: job -# scheme: https -# scrapeTimeout: 5s -# tlsConfig: -# caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -# insecureSkipVerify: true +{{- if .Values.scrapeRules.etcd.enabled }} +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMNodeScrape +metadata: + name: kube-etcd + namespace: cozy-monitoring +spec: + selector: + node-role.kubernetes.io/control-plane: "" + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + metricRelabelConfigs: + - action: labeldrop + regex: (uid) + - action: labeldrop + regex: (id|name) + - action: drop + regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) + source_labels: + - __name__ + port: "2379" + relabelConfigs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + - replacement: etcd + targetLabel: job + scheme: http + scrapeTimeout: 5s + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true +{{- end }} diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index 250db7d82..d7cb50afd 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -359,3 +359,7 @@ fluent-bit: Name modify Match * Add cluster root-cluster + +scrapeRules: + etcd: + enabled: false From dfce1b094b02f80f8b1448b119c8f925f3a56ee0 Mon Sep 17 00:00:00 2001 From: klinch0 <68821526+klinch0@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:21:12 +0300 Subject: [PATCH 3/5] Update packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml Co-authored-by: Andrei Kvapil --- .../system/monitoring-agents/templates/etcd-proxy-scrape.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml index 183a60b5b..bf0db42af 100644 --- a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml +++ b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml @@ -23,7 +23,7 @@ spec: - name: kube-rbac-proxy image: quay.io/brancz/kube-rbac-proxy:v0.11.0 args: - - "--secure-listen-address=0.0.0.0:9443" + - "--secure-listen-address=$(NODE_IP):9443" - "--upstream=http://127.0.0.1:2381/" ports: - containerPort: 9443 From c5b89cbda7780ca2bf29821a056ea9fa1d463eee Mon Sep 17 00:00:00 2001 From: klinch0 <68821526+klinch0@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:21:19 +0300 Subject: [PATCH 4/5] Update packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml Co-authored-by: Andrei Kvapil --- .../monitoring-agents/templates/etcd-proxy-scrape.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml index bf0db42af..73a97206d 100644 --- a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml +++ b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml @@ -25,6 +25,11 @@ spec: args: - "--secure-listen-address=$(NODE_IP):9443" - "--upstream=http://127.0.0.1:2381/" + env: + - name: NODE_IP + valueFrom: + fieldRef: + status.hostIP ports: - containerPort: 9443 name: etcd-metrics From 86b686124ad20b06aae2b6fb6ed85a5f251a80c4 Mon Sep 17 00:00:00 2001 From: Floppy Disk Date: Thu, 6 Feb 2025 14:41:59 +0300 Subject: [PATCH 5/5] del nodescrape --- .../templates/etcd-proxy-scrape.yaml | 4 +-- .../templates/etcd-scrape.yaml | 35 ------------------- 2 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 packages/system/monitoring-agents/templates/etcd-scrape.yaml diff --git a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml index 73a97206d..275a6f479 100644 --- a/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml +++ b/packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.scrapeRules.etcd.enabled }} +{{- if .Values.scrapeRules.etcd.enabled }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -29,7 +29,7 @@ spec: - name: NODE_IP valueFrom: fieldRef: - status.hostIP + fieldPath: status.hostIP ports: - containerPort: 9443 name: etcd-metrics diff --git a/packages/system/monitoring-agents/templates/etcd-scrape.yaml b/packages/system/monitoring-agents/templates/etcd-scrape.yaml deleted file mode 100644 index d776e9da2..000000000 --- a/packages/system/monitoring-agents/templates/etcd-scrape.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if .Values.scrapeRules.etcd.enabled }} -apiVersion: operator.victoriametrics.com/v1beta1 -kind: VMNodeScrape -metadata: - name: kube-etcd - namespace: cozy-monitoring -spec: - selector: - node-role.kubernetes.io/control-plane: "" - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - honorLabels: true - metricRelabelConfigs: - - action: labeldrop - regex: (uid) - - action: labeldrop - regex: (id|name) - - action: drop - regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count) - source_labels: - - __name__ - port: "2379" - relabelConfigs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - sourceLabels: - - __metrics_path__ - targetLabel: metrics_path - - replacement: etcd - targetLabel: job - scheme: http - scrapeTimeout: 5s - tlsConfig: - caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecureSkipVerify: true -{{- end }}