From 447e668230bb84adc91213b9b15b09f47473e25d Mon Sep 17 00:00:00 2001 From: Theron Voran Date: Fri, 3 Sep 2021 14:24:58 -0700 Subject: [PATCH] Adding back support for the old leader-elector Adds the leader-elector container support that was removed in PR #568. The new vault-k8s uses an internal mechanism for leader determination, so this is just for backwards compatibility, and can be removed in the near future. --- templates/injector-deployment.yaml | 29 ++++++ templates/injector-leader-endpoint.yaml | 12 +++ templates/injector-role.yaml | 4 +- test/acceptance/injector-leader-elector.bats | 10 +- test/unit/injector-leader-elector.bats | 102 ++++++++++++++++++- values.schema.json | 14 +++ values.yaml | 7 ++ 7 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 templates/injector-leader-endpoint.yaml diff --git a/templates/injector-deployment.yaml b/templates/injector-deployment.yaml index e753c9c95..0ef336d9b 100644 --- a/templates/injector-deployment.yaml +++ b/templates/injector-deployment.yaml @@ -137,6 +137,35 @@ spec: periodSeconds: 2 successThreshold: 1 timeoutSeconds: 5 + {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} + - name: leader-elector + image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }} + args: + - --election={{ template "vault.fullname" . }}-agent-injector-leader + - --election-namespace={{ .Release.Namespace }} + - --http=0.0.0.0:4040 + - --ttl={{ .Values.injector.leaderElector.ttl }} + livenessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + {{- end }} {{- if .Values.injector.certs.secretName }} volumeMounts: - name: webhook-certs diff --git a/templates/injector-leader-endpoint.yaml b/templates/injector-leader-endpoint.yaml new file mode 100644 index 000000000..0055c035a --- /dev/null +++ b/templates/injector-leader-endpoint.yaml @@ -0,0 +1,12 @@ +{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} +# This is created here so it can be cleaned up easily, since if +# the endpoint is left around the leader won't expire for about a minute. +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "vault.fullname" . }}-agent-injector-leader + labels: + app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} diff --git a/templates/injector-role.yaml b/templates/injector-role.yaml index 88fe53f62..446efaf59 100644 --- a/templates/injector-role.yaml +++ b/templates/injector-role.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} rules: - apiGroups: [""] - resources: ["secrets", "configmaps"] + resources: ["secrets", "configmaps", "endpoints"] verbs: - "create" - "get" @@ -22,4 +22,4 @@ rules: - "get" - "patch" - "delete" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/test/acceptance/injector-leader-elector.bats b/test/acceptance/injector-leader-elector.bats index 4c7154cb2..a399124ab 100644 --- a/test/acceptance/injector-leader-elector.bats +++ b/test/acceptance/injector-leader-elector.bats @@ -22,15 +22,21 @@ load _helpers tries=0 until [ $tries -ge 60 ] do + ## The new internal leader mechanism uses a ConfigMap owner=$(kubectl get configmaps vault-k8s-leader -o json | jq -r .metadata.ownerReferences\[0\].name) leader=$(kubectl get pods $owner -o json | jq -r .metadata.name) [ -n "${leader}" ] && [ "${leader}" != "null" ] && break - let "tries=tries+1" + + ## Also check the old leader-elector container + old_leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)" + [ -n "${old_leader}" ] && break + + ((++tries)) sleep .5 done # Check the leader name is valid - i.e. one of the 3 pods - [[ " ${pods[@]} " =~ " ${leader} " ]] + [[ " ${pods[@]} " =~ " ${leader} " || " ${pods[@]} " =~ " ${old_leader} " ]] } diff --git a/test/unit/injector-leader-elector.bats b/test/unit/injector-leader-elector.bats index a1b27a455..82aec119e 100644 --- a/test/unit/injector-leader-elector.bats +++ b/test/unit/injector-leader-elector.bats @@ -165,4 +165,104 @@ load _helpers . || echo "---") | tee /dev/stderr | yq 'length > 0' | tee /dev/stderr) [ "${actual}" = "true" ] -} \ No newline at end of file +} + +#-------------------------------------------------------------------- +# Old leader-elector container support +# Note: deprecated and will be removed soon + +@test "injector/deployment: leader elector - sidecar is created only when enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "1" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "1" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "2" ] +} + +@test "injector/deployment: leader elector image name is configurable" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.image.repository=SomeOtherImage" \ + --set "injector.leaderElector.image.tag=SomeOtherTag" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].image' | tee /dev/stderr) + [ "${actual}" = "SomeOtherImage:SomeOtherTag" ] +} + +@test "injector/deployment: leader elector TTL is configurable" { + cd `chart_dir` + # Default value 60s + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr) + [ "${actual}" = "--ttl=60s" ] + + # Configured to 30s + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.ttl=30s" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr) + [ "${actual}" = "--ttl=30s" ] +} + +@test "injector/leader-endpoint: created/skipped as appropriate" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "global.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "injector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} diff --git a/values.schema.json b/values.schema.json index c6d7d1039..c159c0102 100644 --- a/values.schema.json +++ b/values.schema.json @@ -281,6 +281,20 @@ "properties": { "enabled": { "type": "boolean" + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "ttl": { + "type": "string" } } }, diff --git a/values.yaml b/values.yaml index 4e3f70c9d..9e5263798 100644 --- a/values.yaml +++ b/values.yaml @@ -37,6 +37,13 @@ injector: # so that only one injector attempts to create TLS certificates. leaderElector: enabled: true + # Note: The deployment of a separate leader-elector container will soon be + # removed from this chart since vault-k8s now uses an internal mechanism to + # determine leadership. + image: + repository: "gcr.io/google_containers/leader-elector" + tag: "0.4" + ttl: 60s # If true, will enable a node exporter metrics endpoint at /metrics. metrics: