Golang E2E in Openshift API-CI (#1384)

* *: reworking for openshift-ci

* test/e2e/_incluster-test-code/memcached_test.go: don't use parallel tests

Travis still doesn't like running these tests in parallel, so
keep them sequential until we fully migrate to api-ci

* scaffold,pkg/{metrics,leader,k8sutil}: add force local mode

* cmd/.../{test,up}/local: set env var for commands

Makefile

@@ -124,6 +124,9 @@ test/e2e: test/e2e/go test/e2e/ansible test/e2e/ansible-molecule test/e2e/helm
 test/e2e/go:
 	./hack/tests/e2e-go.sh $(ARGS)
 
+test/e2e/go2:
+	./ci/tests/e2e-go.sh $(ARGS)
+
 test/e2e/ansible: image/build/ansible
 	./hack/tests/e2e-ansible.sh
 

ci/dockerfiles/builder.Dockerfile

@@ -1,8 +1,7 @@
 FROM openshift/origin-release:golang-1.12
 
 WORKDIR /go/src/github.com/operator-framework/operator-sdk
-# Set gopath before build and include build destination in PATH
-ENV GOPATH=/go PATH=/go/src/github.com/operator-framework/operator-sdk/build:$PATH
+ENV GOPATH=/go PATH=/go/src/github.com/operator-framework/operator-sdk/build:$PATH GOPROXY=https://proxy.golang.org/ GO111MODULE=on
 
 COPY . .
 

ci/dockerfiles/go-e2e.Dockerfile

@@ -0,0 +1,19 @@
+FROM osdk-builder as builder
+
+RUN ci/tests/e2e-go-scaffold.sh
+
+FROM registry.access.redhat.com/ubi7/ubi-minimal:latest
+
+ENV OPERATOR=/usr/local/bin/memcached-operator \
+    USER_UID=1001 \
+    USER_NAME=memcached-operator
+
+# install operator binary
+COPY --from=builder /memcached-operator ${OPERATOR}
+COPY test/test-framework/build/bin /usr/local/bin
+
+RUN  /usr/local/bin/user_setup
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+
+USER ${USER_UID}

ci/tests/e2e-go-scaffold.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ex
+
+# Scaffold project (image name and local image args don't matter at this stage as those only affect the operator manifest)
+source ./hack/tests/scaffolding/e2e-go-scaffold.sh
+
+pushd $BASEPROJECTDIR/memcached-operator
+go build -gcflags "all=-trimpath=${GOPATH}" -asmflags "all=-trimpath=${GOPATH}" -o /memcached-operator $BASEPROJECTDIR/memcached-operator/cmd/manager
+popd

ci/tests/e2e-go.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -ex
+
+make install
+# this configures the image correctly for memcached-operator
+component="memcached-operator"
+eval IMAGE=$IMAGE_FORMAT
+set -- "--image-name=$IMAGE --local-image=false"
+source ./hack/tests/scaffolding/e2e-go-scaffold.sh
+
+pushd $BASEPROJECTDIR/memcached-operator
+operator-sdk test local ./test/e2e
+popd
+
+go test ./test/e2e/... -root=. -globalMan=testdata/empty.yaml

cmd/operator-sdk/test/local.go

@@ -26,6 +26,7 @@ import (
 	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 	"github.com/operator-framework/operator-sdk/internal/util/yamlutil"
+	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/operator-framework/operator-sdk/pkg/test"
 
 	"github.com/ghodss/yaml"
@@ -207,7 +208,9 @@ func testLocalGoFunc(cmd *cobra.Command, args []string) error {
 	if tlConfig.namespace != "" || tlConfig.noSetup {
 		testArgs = append(testArgs, "-"+test.SingleNamespaceFlag, "-parallel=1")
 	}
+	env := append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace))
 	if tlConfig.upLocal {
+		env = append(env, fmt.Sprintf("%s=%s", k8sutil.ForceRunModeEnv, k8sutil.LocalRunMode))
 		testArgs = append(testArgs, "-"+test.LocalOperatorFlag)
 		if tlConfig.localOperatorFlags != "" {
 			testArgs = append(testArgs, "-"+test.LocalOperatorArgs, tlConfig.localOperatorFlags)
@@ -216,7 +219,7 @@ func testLocalGoFunc(cmd *cobra.Command, args []string) error {
 	opts := projutil.GoTestOptions{
 		GoCmdOptions: projutil.GoCmdOptions{
 			PackagePath: args[0] + "/...",
-			Env:         append(os.Environ(), fmt.Sprintf("%v=%v", test.TestNamespaceEnv, tlConfig.namespace)),
+			Env:         env,
 			Dir:         projutil.MustGetwd(),
 			GoMod:       projutil.IsDepManagerGoMod(),
 		},

cmd/operator-sdk/up/local.go

@@ -139,6 +139,7 @@ func upLocal() error {
 		os.Exit(0)
 	}()
 	dc.Env = os.Environ()
+	dc.Env = append(dc.Env, fmt.Sprintf("%s=%s", k8sutil.ForceRunModeEnv, k8sutil.LocalRunMode))
 	// only set env var if user explicitly specified a kubeconfig path
 	if kubeConfig != "" {
 		dc.Env = append(dc.Env, fmt.Sprintf("%v=%v", k8sutil.KubeConfigEnvVar, kubeConfig))

hack/tests/scaffolding/e2e-go-scaffold.sh

@@ -5,8 +5,6 @@ set -ex
 source hack/lib/test_lib.sh
 
 ROOTDIR="$(pwd)"
-# TODO: remove once PR 1566 is merged
-trap_add 'rm -f $ROOTDIR/go.mod' EXIT
 BASEPROJECTDIR="$(mktemp -d)"
 IMAGE_NAME="quay.io/example/memcached-operator:v0.0.1"
 

pkg/k8sutil/k8sutil.go

@@ -29,6 +29,17 @@ import (
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 )
 
+// ForceRunModeEnv indicates if the operator should be forced to run in either local
+// or cluster mode (currently only used for local mode)
+var ForceRunModeEnv = "OSDK_FORCE_RUN_MODE"
+
+type RunModeType string
+
+const (
+	LocalRunMode   RunModeType = "local"
+	ClusterRunMode RunModeType = "cluster"
+)
+
 var log = logf.Log.WithName("k8sutil")
 
 // GetWatchNamespace returns the namespace the operator should be watching for changes
@@ -40,12 +51,19 @@ func GetWatchNamespace() (string, error) {
 	return ns, nil
 }
 
-// errNoNS indicates that a namespace could not be found for the current
+// ErrNoNamespace indicates that a namespace could not be found for the current
 // environment
 var ErrNoNamespace = fmt.Errorf("namespace not found for current environment")
 
+// ErrRunLocal indicates that the operator is set to run in local mode (this error
+// is returned by functions that only work on operators running in cluster mode)
+var ErrRunLocal = fmt.Errorf("operator run mode forced to local")
+
 // GetOperatorNamespace returns the namespace the operator should be running in.
 func GetOperatorNamespace() (string, error) {
+	if isRunModeLocal() {
+		return "", ErrRunLocal
+	}
 	nsBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -93,6 +111,9 @@ func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind strin
 // is currently running.
 // It expects the environment variable POD_NAME to be set by the downwards API.
 func GetPod(ctx context.Context, client crclient.Client, ns string) (*corev1.Pod, error) {
+	if isRunModeLocal() {
+		return nil, ErrRunLocal
+	}
 	podName := os.Getenv(PodNameEnvVar)
 	if podName == "" {
 		return nil, fmt.Errorf("required env %s not set, please configure downward API", PodNameEnvVar)
@@ -156,3 +177,7 @@ func isKubeMetaKind(kind string) bool {
 
 	return false
 }
+
+func isRunModeLocal() bool {
+	return os.Getenv(ForceRunModeEnv) == string(LocalRunMode)
+}

pkg/leader/leader.go

@@ -47,7 +47,7 @@ func Become(ctx context.Context, lockName string) error {
 
 	ns, err := k8sutil.GetOperatorNamespace()
 	if err != nil {
-		if err == k8sutil.ErrNoNamespace {
+		if err == k8sutil.ErrNoNamespace || err == k8sutil.ErrRunLocal {
 			log.Info("Skipping leader election; not running in a cluster.")
 			return nil
 		}

pkg/metrics/metrics.go

@@ -53,7 +53,7 @@ func CreateMetricsService(ctx context.Context, cfg *rest.Config, servicePorts []
 	}
 	s, err := initOperatorService(ctx, client, servicePorts)
 	if err != nil {
-		if err == k8sutil.ErrNoNamespace {
+		if err == k8sutil.ErrNoNamespace || err == k8sutil.ErrRunLocal {
 			log.Info("Skipping metrics Service creation; not running in a cluster.")
 			return nil, nil
 		}

test/test-framework/build/bin/entrypoint

@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+# This is documented here:
+# https://docs.openshift.com/container-platform/3.11/creating_images/guidelines.html#openshift-specific-guidelines
+
+if ! whoami &>/dev/null; then
+  if [ -w /etc/passwd ]; then
+    echo "${USER_NAME:-memcached-operator}:x:$(id -u):$(id -g):${USER_NAME:-memcached-operator} user:${HOME}:/sbin/nologin" >> /etc/passwd
+  fi
+fi
+
+exec ${OPERATOR} $@

test/test-framework/build/bin/user_setup

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -x
+
+# ensure $HOME exists and is accessible by group 0 (we don't know what the runtime UID will be)
+mkdir -p ${HOME}
+chown ${USER_UID}:0 ${HOME}
+chmod ug+rwx ${HOME}
+
+# runtime user will need to be able to self-insert in /etc/passwd
+chmod g+rw /etc/passwd
+
+# no need for this script to remain in the image after running
+rm $0