Adding back support for the old leader-elector

Adds the leader-elector container support that was removed in PR #568. The new vault-k8s uses an internal mechanism for leader determination, so this is just for backwards compatibility, and can be removed in the near future.
hashicorp · Sep 3, 2021 · 447e668 · 447e668
1 parent 87a0a56
commit 447e668
Show file tree

Hide file tree

Showing 7 changed files with 173 additions and 5 deletions.
diff --git a/templates/injector-deployment.yaml b/templates/injector-deployment.yaml
@@ -137,6 +137,35 @@ spec:
             periodSeconds: 2
             successThreshold: 1
             timeoutSeconds: 5
+        {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+        - name: leader-elector
+          image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }}
+          args:
+            - --election={{ template "vault.fullname" . }}-agent-injector-leader
+            - --election-namespace={{ .Release.Namespace }}
+            - --http=0.0.0.0:4040
+            - --ttl={{ .Values.injector.leaderElector.ttl }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 5
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 5
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+        {{- end }}
 {{- if .Values.injector.certs.secretName }}
           volumeMounts:
             - name: webhook-certs

diff --git a/templates/injector-leader-endpoint.yaml b/templates/injector-leader-endpoint.yaml
@@ -0,0 +1,12 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+# This is created here so it can be cleaned up easily, since if
+# the endpoint is left around the leader won't expire for about a minute.
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
diff --git a/templates/injector-role.yaml b/templates/injector-role.yaml
@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 rules:
   - apiGroups: [""]
-    resources: ["secrets", "configmaps"]
+    resources: ["secrets", "configmaps", "endpoints"]
     verbs:
       - "create"
       - "get"
@@ -22,4 +22,4 @@ rules:
       - "get"
       - "patch"
       - "delete"
-{{- end }}
+{{- end }}
diff --git a/test/acceptance/injector-leader-elector.bats b/test/acceptance/injector-leader-elector.bats
@@ -22,15 +22,21 @@ load _helpers
   tries=0
   until [ $tries -ge 60 ]
   do
+    ## The new internal leader mechanism uses a ConfigMap
     owner=$(kubectl get configmaps vault-k8s-leader -o json | jq -r .metadata.ownerReferences\[0\].name)
     leader=$(kubectl get pods $owner -o json | jq -r .metadata.name)
     [ -n "${leader}" ] && [ "${leader}" != "null" ] && break
-    let "tries=tries+1"
+
+    ## Also check the old leader-elector container
+    old_leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+    [ -n "${old_leader}" ] && break
+
+    ((++tries))
     sleep .5
   done
 
   # Check the leader name is valid - i.e. one of the 3 pods
-  [[ " ${pods[@]} " =~ " ${leader} " ]]
+  [[ " ${pods[@]} " =~ " ${leader} " || " ${pods[@]} " =~ " ${old_leader} " ]]
 
 }
 

diff --git a/test/unit/injector-leader-elector.bats b/test/unit/injector-leader-elector.bats
@@ -165,4 +165,104 @@ load _helpers
       . || echo "---") | tee /dev/stderr |
       yq 'length > 0' | tee /dev/stderr)
   [ "${actual}" = "true" ]
-}
+}
+
+#--------------------------------------------------------------------
+# Old leader-elector container support
+# Note: deprecated and will be removed soon
+
+@test "injector/deployment: leader elector - sidecar is created only when enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "1" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "1" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "2" ]
+}
+
+@test "injector/deployment: leader elector image name is configurable" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.image.repository=SomeOtherImage" \
+      --set "injector.leaderElector.image.tag=SomeOtherTag" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].image' | tee /dev/stderr)
+  [ "${actual}" = "SomeOtherImage:SomeOtherTag" ]
+}
+
+@test "injector/deployment: leader elector TTL is configurable" {
+  cd `chart_dir`
+  # Default value 60s
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
+  [ "${actual}" = "--ttl=60s" ]
+
+  # Configured to 30s
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.ttl=30s" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
+  [ "${actual}" = "--ttl=30s" ]
+}
+
+@test "injector/leader-endpoint: created/skipped as appropriate" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "global.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
diff --git a/values.schema.json b/values.schema.json
@@ -281,6 +281,20 @@
                     "properties": {
                         "enabled": {
                             "type": "boolean"
+                        },
+                        "image": {
+                            "type": "object",
+                            "properties": {
+                                "repository": {
+                                    "type": "string"
+                                },
+                                "tag": {
+                                    "type": "string"
+                                }
+                            }
+                        },
+                        "ttl": {
+                            "type": "string"
                         }
                     }
                 },

diff --git a/values.yaml b/values.yaml
@@ -37,6 +37,13 @@ injector:
   # so that only one injector attempts to create TLS certificates.
   leaderElector:
     enabled: true
+    # Note: The deployment of a separate leader-elector container will soon be
+    # removed from this chart since vault-k8s now uses an internal mechanism to
+    # determine leadership.
+    image:
+      repository: "gcr.io/google_containers/leader-elector"
+      tag: "0.4"
+    ttl: 60s
 
   # If true, will enable a node exporter metrics endpoint at /metrics.
   metrics: