Add support for running acceptance tests in a kind cluster

* make the injector-leader-elector a bit more reliable when run locally

Author: benashz

Makefile

@@ -7,6 +7,15 @@ ACCEPTANCE_TESTS?=acceptance
 # filter bats unit tests to run.
 UNIT_TESTS_FILTER?='.*'
 
+# set to 'true' to run acceptance tests locally in a kind cluster
+LOCAL_ACCEPTANCE_TESTS?=false
+
+# kind cluster name
+KIND_CLUSTER_NAME?=vault-helm
+
+# kind k8s version
+KIND_K8S_VERSION?=v1.20.2
+
 # Generate json schema for chart values. See test/README.md for more details.
 values-schema:
 	helm schema-gen values.yaml > values.schema.json
@@ -24,14 +33,18 @@ test: test-image test-bats
 # run acceptance tests on GKE
 # set google project/credential vars above
 test-acceptance:
+ifeq ($(LOCAL_ACCEPTANCE_TESTS),true)
+	make setup-kind acceptance
+else
 	@docker run -it -v ${PWD}:/helm-test \
 	-e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \
 	-e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \
 	-e KUBECONFIG=/helm-test/.kube/config \
 	-w /helm-test \
 	$(TEST_IMAGE) \
 	make acceptance
-	
+endif
+
 # destroy GKE cluster using terraform
 test-destroy:
 	@docker run -it -v ${PWD}:/helm-test \
@@ -54,7 +67,9 @@ test-provision:
 # this target is for running the acceptance tests
 # it is run in the docker container above when the test-acceptance target is invoked
 acceptance:
+ifneq ($(LOCAL_ACCEPTANCE_TESTS),true)
 	gcloud auth activate-service-account --key-file=${GOOGLE_CREDENTIALS}
+endif
 	bats test/${ACCEPTANCE_TESTS}
 
 # this target is for provisioning the GKE cluster
@@ -69,4 +84,17 @@ provision-cluster:
 destroy-cluster:
 	terraform destroy -auto-approve
 
-.PHONY: values-schema test-image test-unit test-bats test test-acceptance test-destroy test-provision acceptance provision-cluster destroy-cluster
+# create a kind cluster for running the acceptance tests locally
+setup-kind:
+	kind get clusters | grep -q "^${KIND_CLUSTER_NAME}$$" || \
+	kind create cluster \
+	--image kindest/node:${KIND_K8S_VERSION} \
+	--name ${KIND_CLUSTER_NAME}  \
+	--config $(CURDIR)/test/kind/config.yaml
+	kubectl config use-context kind-${KIND_CLUSTER_NAME}
+
+# delete the kind cluster
+delete-kind:
+	kind delete cluster --name ${KIND_CLUSTER_NAME} || :
+
+.PHONY: values-schema test-image test-unit test-bats test test-acceptance test-destroy test-provision acceptance provision-cluster destroy-cluster

test/README.md

@@ -2,15 +2,25 @@
 
 ## Running Vault Helm Acceptance tests
 
-The Makefile at the top level of this repo contains a few target that should help with running acceptance tests in your own GKE instance.
+The Makefile at the top level of this repo contains a few target that should help with running acceptance tests in your own GKE instance or in kind cluster.
 
-* Set the GOOGLE_CREDENTIALS and CLOUDSDK_CORE_PROJECT variables at the top of the file. GOOGLE_CREDENTIALS should contain the local path to your Google Cloud Platform account credentials in JSON format. CLOUDSDK_CORE_PROJECT should be set to the ID of your GCP project.
+### Running in a GKE cluster
+* Set the `GOOGLE_CREDENTIALS` and `CLOUDSDK_CORE_PROJECT` variables at the top of the file. `GOOGLE_CREDENTIALS` should contain the local path to your Google Cloud Platform account credentials in JSON format. `CLOUDSDK_CORE_PROJECT` should be set to the ID of your GCP project.
 * Run `make test-image` to create the docker image (with dependencies installed) that will be re-used in the below steps.
 * Run `make test-provision` to provision the GKE cluster using terraform.
 * Run `make test-acceptance` to run the acceptance tests in this already provisioned cluster.
 * You can choose to only run certain tests by setting the ACCEPTANCE_TESTS variable and re-running the above target.
 * Run `make test-destroy` when you have finished testing and want to tear-down and remove the cluster.
 
+### Running in a kind cluster
+* Run `make test-acceptance LOCAL_ACCEPTANCE_TESTS=true`
+* You can choose to only run certain tests by setting the `ACCEPTANCE_TESTS` variable and re-running the above target.
+* Run `make delete-kind` when you have finished testing and want to tear-down and remove the cluster.
+* You can set an alternate kind cluster name by specifying the `KIND_CLUSTER_NAME` variable for any of the above targets.
+* You can set an alternate K8S version by specifying the `KIND_K8S_VERSION` variable for any of the above targets.
+
+See [kind-quick-start](https://kind.sigs.k8s.io/docs/user/quick-start/) if you don't have kind installed on your system.
+
 ## Running chart verification tests
 
 If [chart-verifier](https://github.com/redhat-certification/chart-verifier) is built and available in your PATH, run:

test/acceptance/injector-leader-elector.bats

@@ -4,19 +4,30 @@ load _helpers
 
 @test "injector: testing leader elector" {
   cd `chart_dir`
-  
+
   kubectl delete namespace acceptance --ignore-not-found=true
   kubectl create namespace acceptance
   kubectl config set-context --current --namespace=acceptance
 
   helm install "$(name_prefix)" \
+    --wait \
+    --timeout=5m \
     --set="injector.replicas=3" .
   kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=vault-agent-injector --timeout=5m
 
   pods=($(kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json | jq -r '.items[] | .metadata.name'))
   [ "${#pods[@]}" == 3 ]
 
-  leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+  leader=''
+  tries=0
+  until [ $tries -ge 60 ]
+  do
+    leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+    [ -n "${leader}" ] && break
+    ((tries++))
+    sleep .5
+  done
+
   # Check the leader name is valid - i.e. one of the 3 pods
   [[ " ${pods[@]} " =~ " ${leader} " ]]
 

test/kind/config.yaml

@@ -0,0 +1,7 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker