helm: Added support for operator failover

Added support to run multiple instances of the Tetragon Operator. This also includes slight adjustments to the rollingUpdate strategy and adding a podAntiAffinity. This prevents situations where upgrades are stuck because of impossible to reach maxUnavailable values and running both replicas on the same node. Signed-off-by: Philip Schmid <[email protected]>
cilium · Feb 27, 2025 · d7193ed · d7193ed
1 parent 9f60b23
commit d7193ed
Show file tree

Hide file tree

Showing 7 changed files with 80 additions and 10 deletions.
diff --git a/docs/content/en/docs/reference/helm-chart.md b/docs/content/en/docs/reference/helm-chart.md
diff --git a/install/kubernetes/tetragon/README.md b/install/kubernetes/tetragon/README.md
diff --git a/install/kubernetes/tetragon/templates/operator_configmap.yaml b/install/kubernetes/tetragon/templates/operator_configmap.yaml
@@ -15,4 +15,7 @@ data:
   skip-pod-info-crd: {{ not .Values.tetragonOperator.podInfo.enabled | quote }}
   skip-tracing-policy-crd: {{ not .Values.tetragonOperator.tracingPolicy.enabled | quote }}
   force-update-crds: {{ .Values.tetragonOperator.forceUpdateCRDs | quote }}
+  {{- if gt (int .Values.tetragonOperator.replicas) 1 }}
+  leader-elect: "true"
+  {{- end }}
 {{- end }}
diff --git a/install/kubernetes/tetragon/templates/operator_deployment.yaml b/install/kubernetes/tetragon/templates/operator_deployment.yaml
@@ -17,7 +17,7 @@ spec:
   selector:
     matchLabels:
       {{- include "tetragon-operator.selectorLabels" . | nindent 6 }}
-  replicas: 1
+  replicas: {{ .Values.tetragonOperator.replicas }}
   template:
     metadata:
       {{- with .Values.tetragonOperator.podAnnotations }}
@@ -107,8 +107,20 @@ spec:
       {{- with .Values.tetragonOperator.extraVolumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
-{{- with .Values.tetragonOperator.strategy }}
+  # Ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case
+  # of one replica and no user configured Recreate strategy.
+  # Otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the
+  # podAntiAffinity which prevents deployments of multiple operator replicas on the same node.
+  {{- if and (eq (.Values.tetragonOperator.replicas | toString) "1") (eq .Values.tetragonOperator.strategy.type "RollingUpdate") }}
   strategy:
-    {{- toYaml . | nindent 4 }}
-{{- end }}
+    rollingUpdate:
+      maxSurge: {{ .Values.tetragonOperator.strategy.rollingUpdate.maxSurge }}
+      maxUnavailable: 100%
+    type: RollingUpdate
+  {{- else }}
+  {{- with .Values.tetragonOperator.strategy }}
+  strategy:
+    {{- toYaml . | trim | nindent 4 }}
+  {{- end }}
+  {{- end }}
 {{- end }}
diff --git a/install/kubernetes/tetragon/templates/operator_role.yaml b/install/kubernetes/tetragon/templates/operator_role.yaml
@@ -0,0 +1,22 @@
+{{- if and (and .Values.tetragonOperator.enabled .Values.serviceAccount.create) (gt (int .Values.tetragonOperator.replicas) 1) }}
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-operator
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "tetragon-operator.labels" . | nindent 4 }}
+rules:
+# For tetragon-operator running with multiple replicas
+#
+# Tetragon operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+{{- end }}
diff --git a/install/kubernetes/tetragon/templates/operator_rolebinding.yaml b/install/kubernetes/tetragon/templates/operator_rolebinding.yaml
@@ -0,0 +1,17 @@
+{{- if and (and .Values.tetragonOperator.enabled .Values.serviceAccount.create) (gt (int .Values.tetragonOperator.replicas) 1) }}
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-operator-rolebinding
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "tetragon-operator.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-operator
+subjects:
+  - kind: ServiceAccount
+    namespace: {{ .Release.Namespace }}
+    name: {{ include "tetragon-operator.serviceAccount" . }}
+{{- end }}
diff --git a/install/kubernetes/tetragon/values.yaml b/install/kubernetes/tetragon/values.yaml
@@ -251,6 +251,8 @@ tetragon:
 tetragonOperator:
   # -- Enables the Tetragon Operator.
   enabled: true
+  # -- Number of replicas to run for the tetragon-operator deployment
+  replicas: 1
   # -- Annotations for the Tetragon Operator Deployment.
   annotations: {}
   # -- Annotations for the Tetragon Operator Deployment Pods.
@@ -283,11 +285,21 @@ tetragonOperator:
       cpu: 10m
       memory: 64Mi
   # -- resources for the Tetragon Operator Deployment update strategy
-  strategy: {}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 50%
   # -- Steer the Tetragon Operator Deployment Pod placement via nodeSelector, tolerations and affinity rules.
   nodeSelector: {}
   tolerations: []
-  affinity: {}
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: tetragon-operator
   # -- tetragon-operator image.
   image:
     override: ~