From d7193ed932105fca931bbbff86d3ded009c6b9e4 Mon Sep 17 00:00:00 2001 From: Philip Schmid Date: Thu, 27 Feb 2025 18:52:35 +0100 Subject: [PATCH] helm: Added support for operator failover Added support to run multiple instances of the Tetragon Operator. This also includes slight adjustments to the rollingUpdate strategy and adding a podAntiAffinity. This prevents situations where upgrades are stuck because of impossible to reach maxUnavailable values and running both replicas on the same node. Signed-off-by: Philip Schmid --- docs/content/en/docs/reference/helm-chart.md | 6 +++-- install/kubernetes/tetragon/README.md | 6 +++-- .../templates/operator_configmap.yaml | 3 +++ .../templates/operator_deployment.yaml | 20 +++++++++++++---- .../tetragon/templates/operator_role.yaml | 22 +++++++++++++++++++ .../templates/operator_rolebinding.yaml | 17 ++++++++++++++ install/kubernetes/tetragon/values.yaml | 16 ++++++++++++-- 7 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 install/kubernetes/tetragon/templates/operator_role.yaml create mode 100644 install/kubernetes/tetragon/templates/operator_rolebinding.yaml diff --git a/docs/content/en/docs/reference/helm-chart.md b/docs/content/en/docs/reference/helm-chart.md index f9f470654d5..4230347944e 100644 --- a/docs/content/en/docs/reference/helm-chart.md +++ b/docs/content/en/docs/reference/helm-chart.md @@ -137,7 +137,8 @@ To use [the values available](#values), with `helm install` or `helm upgrade`, u | tetragon.redactionFilters | string | `""` | Filters to redact secrets from the args fields in Tetragon events. To perform redactions, redaction filters define RE2 regular expressions in the `redact` field. Any capture groups in these RE2 regular expressions are redacted and replaced with "*****". For more control, you can select which binary or binaries should have their arguments redacted with the `binary_regex` field. NOTE: This feature uses RE2 as its regular expression library. Make sure that you follow RE2 regular expression guidelines as you may observe unexpected results otherwise. More information on RE2 syntax can be found [here](https://github.com/google/re2/wiki/Syntax). NOTE: When writing regular expressions in JSON, it is important to escape backslash characters. For instance `\Wpasswd\W?` would be written as `{"redact": "\\Wpasswd\\W?"}`. As a concrete example, the following will redact all passwords passed to processes with the "--password" argument: {"redact": ["--password(?:\\s+|=)(\\S*)"]} Now, an event which contains the string "--password=foo" would have that string replaced with "--password=*****". Suppose we also see some passwords passed via the -p shorthand for a specific binary, foo. We can also redact these as follows: {"binary_regex": ["(?:^|/)foo$"], "redact": ["-p(?:\\s+|=)(\\S*)"]} With both of the above redaction filters in place, we are now redacting all password arguments. | | tetragon.resources | object | `{}` | | | tetragon.securityContext.privileged | bool | `true` | | -| tetragonOperator.affinity | object | `{}` | | +| tetragonOperator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchLabels."app.kubernetes.io/name" | string | `"tetragon-operator"` | | +| tetragonOperator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey | string | `"kubernetes.io/hostname"` | | | tetragonOperator.annotations | object | `{}` | Annotations for the Tetragon Operator Deployment. | | tetragonOperator.enabled | bool | `true` | Enables the Tetragon Operator. | | tetragonOperator.extraLabels | object | `{}` | Extra labels to be added on the Tetragon Operator Deployment. | @@ -158,10 +159,11 @@ To use [the values available](#values), with `helm install` or `helm upgrade`, u | tetragonOperator.prometheus.serviceMonitor.extraLabels | object | `{}` | Extra labels to be added on the Tetragon Operator ServiceMonitor. | | tetragonOperator.prometheus.serviceMonitor.labelsOverride | object | `{}` | The set of labels to place on the 'ServiceMonitor' resource. | | tetragonOperator.prometheus.serviceMonitor.scrapeInterval | string | `"10s"` | Interval at which metrics should be scraped. If not specified, Prometheus' global scrape interval is used. | +| tetragonOperator.replicas | int | `1` | Number of replicas to run for the tetragon-operator deployment | | tetragonOperator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | resources for the Tetragon Operator Deployment Pod container. | | tetragonOperator.securityContext | object | `{}` | securityContext for the Tetragon Operator Deployment Pods. | | tetragonOperator.serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | tetragon-operator service account. | -| tetragonOperator.strategy | object | `{}` | resources for the Tetragon Operator Deployment update strategy | +| tetragonOperator.strategy | object | `{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"}` | resources for the Tetragon Operator Deployment update strategy | | tetragonOperator.tolerations | list | `[]` | | | tetragonOperator.tracingPolicy.enabled | bool | `true` | Enables the TracingPolicy and TracingPolicyNamespaced CRD creation. | | tolerations[0].operator | string | `"Exists"` | | diff --git a/install/kubernetes/tetragon/README.md b/install/kubernetes/tetragon/README.md index 02197d8c12f..0674a0b6471 100644 --- a/install/kubernetes/tetragon/README.md +++ b/install/kubernetes/tetragon/README.md @@ -119,7 +119,8 @@ Helm chart for Tetragon | tetragon.redactionFilters | string | `""` | Filters to redact secrets from the args fields in Tetragon events. To perform redactions, redaction filters define RE2 regular expressions in the `redact` field. Any capture groups in these RE2 regular expressions are redacted and replaced with "*****". For more control, you can select which binary or binaries should have their arguments redacted with the `binary_regex` field. NOTE: This feature uses RE2 as its regular expression library. Make sure that you follow RE2 regular expression guidelines as you may observe unexpected results otherwise. More information on RE2 syntax can be found [here](https://github.com/google/re2/wiki/Syntax). NOTE: When writing regular expressions in JSON, it is important to escape backslash characters. For instance `\Wpasswd\W?` would be written as `{"redact": "\\Wpasswd\\W?"}`. As a concrete example, the following will redact all passwords passed to processes with the "--password" argument: {"redact": ["--password(?:\\s+|=)(\\S*)"]} Now, an event which contains the string "--password=foo" would have that string replaced with "--password=*****". Suppose we also see some passwords passed via the -p shorthand for a specific binary, foo. We can also redact these as follows: {"binary_regex": ["(?:^|/)foo$"], "redact": ["-p(?:\\s+|=)(\\S*)"]} With both of the above redaction filters in place, we are now redacting all password arguments. | | tetragon.resources | object | `{}` | | | tetragon.securityContext.privileged | bool | `true` | | -| tetragonOperator.affinity | object | `{}` | | +| tetragonOperator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchLabels."app.kubernetes.io/name" | string | `"tetragon-operator"` | | +| tetragonOperator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey | string | `"kubernetes.io/hostname"` | | | tetragonOperator.annotations | object | `{}` | Annotations for the Tetragon Operator Deployment. | | tetragonOperator.enabled | bool | `true` | Enables the Tetragon Operator. | | tetragonOperator.extraLabels | object | `{}` | Extra labels to be added on the Tetragon Operator Deployment. | @@ -140,10 +141,11 @@ Helm chart for Tetragon | tetragonOperator.prometheus.serviceMonitor.extraLabels | object | `{}` | Extra labels to be added on the Tetragon Operator ServiceMonitor. | | tetragonOperator.prometheus.serviceMonitor.labelsOverride | object | `{}` | The set of labels to place on the 'ServiceMonitor' resource. | | tetragonOperator.prometheus.serviceMonitor.scrapeInterval | string | `"10s"` | Interval at which metrics should be scraped. If not specified, Prometheus' global scrape interval is used. | +| tetragonOperator.replicas | int | `1` | Number of replicas to run for the tetragon-operator deployment | | tetragonOperator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | resources for the Tetragon Operator Deployment Pod container. | | tetragonOperator.securityContext | object | `{}` | securityContext for the Tetragon Operator Deployment Pods. | | tetragonOperator.serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | tetragon-operator service account. | -| tetragonOperator.strategy | object | `{}` | resources for the Tetragon Operator Deployment update strategy | +| tetragonOperator.strategy | object | `{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"50%"},"type":"RollingUpdate"}` | resources for the Tetragon Operator Deployment update strategy | | tetragonOperator.tolerations | list | `[]` | | | tetragonOperator.tracingPolicy.enabled | bool | `true` | Enables the TracingPolicy and TracingPolicyNamespaced CRD creation. | | tolerations[0].operator | string | `"Exists"` | | diff --git a/install/kubernetes/tetragon/templates/operator_configmap.yaml b/install/kubernetes/tetragon/templates/operator_configmap.yaml index 2b31687a46d..04a1666d39d 100644 --- a/install/kubernetes/tetragon/templates/operator_configmap.yaml +++ b/install/kubernetes/tetragon/templates/operator_configmap.yaml @@ -15,4 +15,7 @@ data: skip-pod-info-crd: {{ not .Values.tetragonOperator.podInfo.enabled | quote }} skip-tracing-policy-crd: {{ not .Values.tetragonOperator.tracingPolicy.enabled | quote }} force-update-crds: {{ .Values.tetragonOperator.forceUpdateCRDs | quote }} + {{- if gt (int .Values.tetragonOperator.replicas) 1 }} + leader-elect: "true" + {{- end }} {{- end }} diff --git a/install/kubernetes/tetragon/templates/operator_deployment.yaml b/install/kubernetes/tetragon/templates/operator_deployment.yaml index f2862a6e9ec..8d5e19d1399 100644 --- a/install/kubernetes/tetragon/templates/operator_deployment.yaml +++ b/install/kubernetes/tetragon/templates/operator_deployment.yaml @@ -17,7 +17,7 @@ spec: selector: matchLabels: {{- include "tetragon-operator.selectorLabels" . | nindent 6 }} - replicas: 1 + replicas: {{ .Values.tetragonOperator.replicas }} template: metadata: {{- with .Values.tetragonOperator.podAnnotations }} @@ -107,8 +107,20 @@ spec: {{- with .Values.tetragonOperator.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} -{{- with .Values.tetragonOperator.strategy }} + # Ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # Otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + {{- if and (eq (.Values.tetragonOperator.replicas | toString) "1") (eq .Values.tetragonOperator.strategy.type "RollingUpdate") }} strategy: - {{- toYaml . | nindent 4 }} -{{- end }} + rollingUpdate: + maxSurge: {{ .Values.tetragonOperator.strategy.rollingUpdate.maxSurge }} + maxUnavailable: 100% + type: RollingUpdate + {{- else }} + {{- with .Values.tetragonOperator.strategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- end }} {{- end }} diff --git a/install/kubernetes/tetragon/templates/operator_role.yaml b/install/kubernetes/tetragon/templates/operator_role.yaml new file mode 100644 index 00000000000..0fa33731f5e --- /dev/null +++ b/install/kubernetes/tetragon/templates/operator_role.yaml @@ -0,0 +1,22 @@ +{{- if and (and .Values.tetragonOperator.enabled .Values.serviceAccount.create) (gt (int .Values.tetragonOperator.replicas) 1) }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-operator + namespace: {{ .Release.Namespace }} + labels: + {{- include "tetragon-operator.labels" . | nindent 4 }} +rules: +# For tetragon-operator running with multiple replicas +# +# Tetragon operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +{{- end }} diff --git a/install/kubernetes/tetragon/templates/operator_rolebinding.yaml b/install/kubernetes/tetragon/templates/operator_rolebinding.yaml new file mode 100644 index 00000000000..071e785dc2b --- /dev/null +++ b/install/kubernetes/tetragon/templates/operator_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and (and .Values.tetragonOperator.enabled .Values.serviceAccount.create) (gt (int .Values.tetragonOperator.replicas) 1) }} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-operator-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "tetragon-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-operator +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ include "tetragon-operator.serviceAccount" . }} +{{- end }} diff --git a/install/kubernetes/tetragon/values.yaml b/install/kubernetes/tetragon/values.yaml index e02e21a3af0..20e5a3f51a6 100644 --- a/install/kubernetes/tetragon/values.yaml +++ b/install/kubernetes/tetragon/values.yaml @@ -251,6 +251,8 @@ tetragon: tetragonOperator: # -- Enables the Tetragon Operator. enabled: true + # -- Number of replicas to run for the tetragon-operator deployment + replicas: 1 # -- Annotations for the Tetragon Operator Deployment. annotations: {} # -- Annotations for the Tetragon Operator Deployment Pods. @@ -283,11 +285,21 @@ tetragonOperator: cpu: 10m memory: 64Mi # -- resources for the Tetragon Operator Deployment update strategy - strategy: {} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% # -- Steer the Tetragon Operator Deployment Pod placement via nodeSelector, tolerations and affinity rules. nodeSelector: {} tolerations: [] - affinity: {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: tetragon-operator # -- tetragon-operator image. image: override: ~