testing: Add kuttl test suite to operator (PROJQUAY-2622)

- Add kuttl testing framework to Github Actions
- Update vendor directory

.github/workflows/ci.yaml

@@ -1,5 +1,5 @@
 name: CI
-on: [push, pull_request]
+on: [pull_request]
 jobs:
   build:
     name: Build
@@ -32,3 +32,9 @@ jobs:
           export PATH=$PATH:/usr/local/kubebuilder/bin
       - name: Tests
         run: go test -v ./...
+      - name: E2E Tests
+        env: 
+          KUBECONFIG: ${{ github.workspace }}/kubeconfig
+        run: |
+          printf "${{ secrets.KUBE_CONFIG }}" | base64 --decode > "${KUBECONFIG}"
+          make test-e2e;

.gitignore

@@ -1,3 +1,4 @@
 bin
 tmp
 cover.out
+kubeconfig

Makefile

@@ -17,6 +17,12 @@ all: manager
 test: generate fmt vet manifests
 	go test ./... -coverprofile cover.out
 
+test-e2e:
+	mkdir ./bin
+	curl -L https://github.com/kudobuilder/kuttl/releases/download/v0.11.1/kubectl-kuttl_0.11.1_linux_x86_64 -o ./bin/kuttl;
+	chmod +x ./bin/kuttl;
+	./bin/kuttl test;
+
 # Build manager binary
 manager: generate fmt vet
 	go build -o bin/manager main.go

e2e/min_deployment/00-assert.yaml

@@ -0,0 +1,81 @@
+apiVersion: quay.redhat.com/v1
+kind: QuayRegistry
+metadata:
+  finalizers:
+  - quay-operator/finalizer
+  name: skynet
+spec:
+  components:
+  - kind: clair
+    managed: false
+  - kind: postgres
+    managed: true
+  - kind: redis
+    managed: true
+  - kind: horizontalpodautoscaler
+    managed: true
+  - kind: objectstorage
+    managed: true
+  - kind: route
+    managed: true
+  - kind: mirror
+    managed: true
+  - kind: monitoring
+    managed: true
+  - kind: tls
+    managed: true
+status:
+  conditions:
+  - message: Horizontal pod autoscaler found
+    reason: ComponentReady
+    status: "True"
+    type: ComponentHPAReady
+  - message: Route admitted
+    reason: ComponentReady
+    status: "True"
+    type: ComponentRouteReady
+  - message: ServiceMonitor and PrometheusRules created
+    reason: ComponentReady
+    status: "True"
+    type: ComponentMonitoringReady
+  - message: Deployment skynet-quay-database healthy
+    reason: ComponentReady
+    status: "True"
+    type: ComponentPostgresReady
+  - message: Object bucket claim bound
+    reason: ComponentReady
+    status: "True"
+    type: ComponentObjectStorageReady
+  - message: Clair not managed by the operator
+    reason: ComponentNotManaged
+    status: "True"
+    type: ComponentClairReady
+  - message: Using cluster wildcard certs
+    reason: ComponentReady
+    status: "True"
+    type: ComponentTLSReady
+  - message: Deployment skynet-quay-redis healthy
+    reason: ComponentReady
+    status: "True"
+    type: ComponentRedisReady
+  - message: Base component healthy
+    reason: ComponentReady
+    status: "True"
+    type: ComponentBaseReady
+  - message: Deployment skynet-quay-mirror healthy
+    reason: ComponentReady
+    status: "True"
+    type: ComponentMirrorReady
+  - message: All components reporting as healthy
+    reason: HealthChecksPassing
+    status: "True"
+    type: Available
+  - message: All registry components created
+    reason: ComponentsCreationSuccess
+    status: "True"
+    type: ComponentsCreated
+  - message: All objects created/updated successfully
+    reason: ComponentsCreationSuccess
+    status: "False"
+    type: RolloutBlocked
+  currentVersion: dev

e2e/min_deployment/00-create-quay-registry.yaml

@@ -0,0 +1,8 @@
+apiVersion: quay.redhat.com/v1
+kind: QuayRegistry
+metadata:
+  name: skynet
+spec:
+  components:
+  - kind: clair
+    managed: false

e2e/min_deployment/01-delete-registry.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: quay.redhat.com/v1
+  kind: QuayRegistry
+  metadata:
+    name: skynet

e2e/min_deployment/01-errors.yaml

@@ -0,0 +1,4 @@
+apiVersion: quay.redhat.com/v1
+kind: QuayRegistry
+metadata:
+  name: skynet

hack/deploy.sh

@@ -78,26 +78,25 @@ done
 
 info 'installing the operator'
 
-yq e -i '.spec.targetNamespaces[0] = env(NAMESPACE)' "${OG_PATH}"
-oc apply -n "${NAMESPACE}" -f "${OG_PATH}"
+oc apply -f "${OG_PATH}"
 
 yq e -i '
 	.spec.channel = "test" |
 	.spec.startingCSV = env(OPERATOR_PKG_NAME) |
 	.spec.name = env(OPERATOR_PKG_NAME) |
 	.spec.source = env(OPERATOR_PKG_NAME)
 ' "${SUBSCRIPTION_PATH}"
-oc apply -n "${NAMESPACE}" -f "${SUBSCRIPTION_PATH}"
+oc apply -f "${SUBSCRIPTION_PATH}"
 
 info 'waiting for CSV...'
 
 for n in {1..60}; do
-	phase="$(oc get csv "${OPERATOR_PKG_NAME}" -n "${NAMESPACE}" -o jsonpath='{.status.phase}' 2> /dev/null)"
+	phase="$(oc get csv "${OPERATOR_PKG_NAME}" -o jsonpath='{.status.phase}' 2> /dev/null)"
 	if [ ! "$phase" = 'Succeeded' ]; then
 		if [ "${n}" = "60" ]; then
 			error 'timed out waiting'
 			info 'csv contents:'
-			oc get csv "${OPERATOR_PKG_NAME}" -n "${NAMESPACE}" -o yaml
+			oc get csv "${OPERATOR_PKG_NAME}" -o yaml
 			exit 1
 		fi
 		sleep 10
@@ -107,113 +106,7 @@ for n in {1..60}; do
 	break
 done
 
-info 'deploy Quay'
-oc apply -n "${NAMESPACE}" -f "${QUAY_SAMPLE_PATH}"
-
-# wait for quay deployment to come up. the next checks will fail if the
-# resources don't exist.
-
-info 'waiting for quay deployment...'
-
-for n in {1..60}; do
-	deploy="$(oc get deploy skynet-quay-app -n "${NAMESPACE}" -o jsonpath='{.metadata.name}' 2> /dev/null)"
-	if [ ! "$deploy" = 'skynet-quay-app' ]; then
-		if [ "${n}" = "60" ]; then
-			error 'timed out waiting'
-			exit 1
-		fi
-		sleep 10
-		continue
-	fi
-	info 'quay deployment created'
-	break
-done
-
-# the reconcile loop recreates resources a few times before stabilising,
-# and the wait command isn't very smart (similar problems exist when waiting
-# for rollouts), so we just sleep a bit before waiting.
-# see https://github.com/kubernetes/kubectl/issues/1120 for details.
-info 'sleeping two minutes before waiting for pods...'
-sleep 120
-
-# the order below matters!
-# the mirror is the last to come up, so after the dependencies are ready
-# we wait for the mirror to be ready, then we check the quay app as it's
-# more likely to have stabilized after the mirror is up and running.
-info 'waiting for postgres pod...'
-oc -n "${NAMESPACE}" wait pods -l=quay-component=postgres --for=condition=Ready --timeout="${WAIT_TIMEOUT}"
-
-info 'waiting for redis pod...'
-oc -n "${NAMESPACE}" wait pods -l=quay-component=redis --for=condition=Ready --timeout="${WAIT_TIMEOUT}"
-
-info 'waiting for mirror pod...'
-oc -n "${NAMESPACE}" wait pods -l=quay-component=quay-mirror --for=condition=Ready --timeout="${WAIT_TIMEOUT}"
-
-info 'waiting for config-editor pod...'
-oc -n "${NAMESPACE}" wait pods -l=quay-component=quay-config-editor --for=condition=Ready --timeout="${WAIT_TIMEOUT}"
-
-info 'waiting for quay pod...'
-oc -n "${NAMESPACE}" wait pods -l=quay-component=quay-app --for=condition=Ready --timeout="${WAIT_TIMEOUT}"
-
-
-# manually check quay's health to ensure it can successfuly connect to its components.
-endpoint="$(oc -n "${NAMESPACE}" get quayregistry skynet -o jsonpath='{.status.registryEndpoint}')"
-result="$(curl -s -k "${endpoint}"/health/instance)"
-
-if [ "$(echo "${result}" | jq '.status_code')" != "200" ]; then
-	error 'quay health check did not return 200'
-	info "${result}"
-	info 'quayregistry CR yaml:'
-	oc -n "${NAMESPACE}" get quayregistry skynet -o yaml
-	info 'quay-app pod logs:'
-	oc -n "${NAMESPACE}" logs -l=quay-component=quay-app
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.auth')" != "true" ]; then
-	error 'quay auth health check was false'
-	info "${result}"
-	oc -n "${NAMESPACE}" logs -l=quay-component=quay-app
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.database')" != "true" ]; then
-	error 'quay database health check was false'
-	info "${result}"
-	oc -n "${NAMESPACE}" logs -l=quay-component=postgres
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.disk_space')" != "true" ]; then
-	error 'quay disk_space health check was false'
-	info "${result}"
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.registry_gunicorn')" != "true" ]; then
-	error 'quay registry_gunicorn health check was false'
-	info "${result}"
-	oc -n "${NAMESPACE}" logs -l=quay-component=quay-app
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.service_key')" != "true" ]; then
-	error 'quay service_key health check was false'
-	info "${result}"
-	oc -n "${NAMESPACE}" logs -l=quay-component=quay-app
-	exit 1
-fi
-
-if [ "$(echo "${result}" | jq '.data.services.web_gunicorn')" != "true" ]; then
-	error 'quay web_gunicorn health check was false'
-	info "${result}"
-	oc -n "${NAMESPACE}" logs -l=quay-component=quay-app
-	exit 1
-fi
-
-info 'all healthchecks passed'
-info 'successfully installed Quay!'
-info 'use hack/teardown.sh to remove all created resources'
+make test-e2e;
 
 # shellcheck disable=SC2046
 if [ -x $(command -v git >/dev/null 2>&1) ]; then

hack/teardown.sh

@@ -14,18 +14,11 @@ function info {
 export OPERATOR_PKG_NAME=${OPERATOR_PKG_NAME:-'quay-operator-test'}
 export OG_PATH=${OG_PATH:-'./bundle/quay-operator.operatorgroup.yaml'}
 export SUBSCRIPTION_PATH=${SUBSCRIPTION_PATH:-'./bundle/quay-operator.subscription.yaml'}
-export QUAY_SAMPLE_PATH=${QUAY_SAMPLE_PATH:-'./config/samples/managed.quayregistry.yaml'}
-export NAMESPACE=${NAMESPACE:-'quay-operator-e2e-nightly'}
-
-info 'tearing down created resources...'
-
-info 'uninstall Quay'
-oc delete -n "${NAMESPACE}" -f "${QUAY_SAMPLE_PATH}"
 
 info 'uninstalling operator'
-oc delete -n "${NAMESPACE}" operatorgroup "$(yq e '.metadata.name' "${OG_PATH}")"
-oc delete -n "${NAMESPACE}" subscription "$(yq e '.metadata.name' "${SUBSCRIPTION_PATH}")"
-oc delete -n "${NAMESPACE}" csv "${OPERATOR_PKG_NAME}"
+oc delete operatorgroup "$(yq e '.metadata.name' "${OG_PATH}")"
+oc delete subscription "$(yq e '.metadata.name' "${SUBSCRIPTION_PATH}")"
+oc delete csv "${OPERATOR_PKG_NAME}"
 
 info 'deleting catalog source'
 oc delete catsrc "${OPERATOR_PKG_NAME}" -n openshift-marketplace

kuttl-test.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+  - ./e2e
+timeout: 500
+crdDir: ./bundle/upstream/manifests
+suppress: 
+ - "events"
+commands:
+  - command: /bin/bash -c 'QUAY_VERSION=dev go run main.go' 
+    background: true
+