Cherry pick E2E test changes and fix failing tagging controller test for 1.20

hack/e2e/run.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+function test_run_id() {
+    echo "$(date '+%Y%m%d%H%M%S')"
+}
+
+test_run_id="$(test_run_id)"
+repo_root="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/../.." &> /dev/null && pwd )"
+output="${repo_root}/_output"
+test_output_root="${output}/test"
+test_run="${test_output_root}/${test_run_id}"
+
+# Set in Makefile
+BUILD_VERSION="${BUILD_VERSION:-}"
+BUILD_IMAGE="${BUILD_IMAGE:-}"
+
+if [[ -z "${BUILD_VERSION}" || -z "${BUILD_IMAGE}" ]]; then
+    echo "$0: Execute with 'make test-e2e'"
+    exit 1
+fi
+
+# Configurable
+KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config}"
+SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-}"
+
+# If UP==yes, provision a cluster as part of testing
+# Otherwise, rely on KUBECONFIG to determine test cluster.
+UP="${UP:-yes}"
+# if DOWN==yes, delete cluster after test
+DOWN="${DOWN:-yes}"
+
+KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.20.15}"
+CLUSTER_NAME="test-cluster-${test_run_id}.k8s.local"
+KOPS_STATE_STORE="${KOPS_STATE_STORE:-}"
+REGION="${AWS_REGION:-us-west-2}"
+ZONES="${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}"
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/amazon/cloud-controller-manager}
+IMAGE_TAG=${IMAGE_TAG:-${BUILD_VERSION}-${test_run_id}}
+
+# Test args
+GINKGO_FOCUS=${GINKGO_FOCUS:-"\[cloud-provider-aws-e2e\]"}
+GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
+GINKGO_NODES=${GINKGO_NODES:-4}
+
+EXPANDED_TEST_EXTRA_FLAGS="${EXPANDED_TEST_EXTRA_FLAGS:-}"
+
+mkdir -p "${test_run}"
+
+if [[ -z "${KOPS_STATE_STORE}" ]]; then
+    echo "KOPS_STATE_STORE must be set"
+    exit 1
+fi
+
+if [[ ! -f "${repo_root}/e2e.test" ]]; then
+    echo "Missing e2e.test binary"
+    exit 1
+fi
+
+if [[ -z "${SSH_PUBLIC_KEY_PATH}" ]]; then
+    ssh_key_path=${test_run}/sshkey
+    ssh-keygen -b 2048 -t rsa -f ${ssh_key_path} -q -N ""
+    SSH_PUBLIC_KEY_PATH=${ssh_key_path}.pub
+fi
+
+yes_or_no="^(yes|no)$"
+
+if [[ ! "${UP}" =~ $yes_or_no ]]; then
+    echo "Invalid UP: ${UP} (valid: [yes|no])"
+    exit 1
+fi
+
+if [[ ! "${DOWN}" =~ $yes_or_no ]]; then
+    echo "Invalid DOWN: ${DOWN} (valid: [yes|no])"
+    exit 1
+fi
+
+if [[ -z "${INSTALL_PATH}" ]]; then
+    echo "INSTALL_PATH must be set"
+    exit 1
+fi
+
+export PATH="${INSTALL_PATH}:${PATH}"
+
+echo "Starting test run ---"
+echo " + Region:              ${REGION} (${ZONES})"
+echo " + Cluster name:        ${CLUSTER_NAME}"
+echo " + Kubernetes version:  ${KUBERNETES_VERSION}"
+echo " + Focus:               ${GINKGO_FOCUS}"
+echo " + Skip:                ${GINKGO_SKIP}"
+echo " + Kops state store:    ${KOPS_STATE_STORE}"
+echo " + SSH public key path: ${SSH_PUBLIC_KEY_PATH}"
+echo " + Test run ID:         ${test_run_id}"
+echo " + Kubetest run dir:    ${test_run}"
+echo " + Image:               ${IMAGE_NAME}:${IMAGE_TAG}"
+echo " + Create cluster:      ${UP}"
+echo " + Delete cluster:      ${DOWN}"
+
+export KOPS_STATE_STORE
+# kubetest2 sets RunDir as filepath.Join(artifacts.BaseDir(), o.RunID())
+export ARTIFACTS="${test_output_root}"
+export KUBETEST2_RUN_DIR="${test_run}"
+
+echo "Installing e2e.test to ${test_run}"
+cp "${repo_root}/e2e.test" "${test_run}"
+
+echo "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
+aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
+docker tag "${BUILD_IMAGE}" "${IMAGE_NAME}:${IMAGE_TAG}"
+docker push "${IMAGE_NAME}:${IMAGE_TAG}"
+
+if [[ "${UP}" = "yes" ]]; then
+    kubetest2 kops \
+      -v 2 \
+      --up \
+      --run-id="${test_run_id}" \
+      --cloud-provider=aws \
+      --cluster-name="${CLUSTER_NAME}" \
+      --create-args="--zones=${ZONES} --node-size=m5.large --master-size=m5.large --override=cluster.spec.kubeAPIServer.cloudProvider=external --override=cluster.spec.kubeControllerManager.cloudProvider=external --override=cluster.spec.kubelet.cloudProvider=external --override=cluster.spec.cloudControllerManager.cloudProvider=aws --override=cluster.spec.cloudControllerManager.image=${IMAGE_NAME}:${IMAGE_TAG} --override=spec.cloudConfig.awsEBSCSIDriver.enabled=true" \
+      --admin-access="0.0.0.0/0" \
+      --kubernetes-version="${KUBERNETES_VERSION}" \
+      --ssh-public-key="${SSH_PUBLIC_KEY_PATH}" \
+      --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
+
+      # Use the kops tester once we have a way of consuming an arbitrary e2e.test binary.
+      #--test=kops \
+      #-- \
+      #--use-built-binaries=true \
+      #--focus-regex="${GINKGO_FOCUS}" \
+      #--parallel 25
+fi
+
+pushd ./tests/e2e
+ginkgo . -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "" -- -kubeconfig="${KUBECONFIG}" -report-dir="${test_run}" -gce-zone="${ZONES%,*}" "${EXPANDED_TEST_EXTRA_FLAGS}"
+popd
+
+if [[ "${DOWN}" = "yes" ]]; then
+    ${test_run}/kops delete cluster --name "${CLUSTER_NAME}" --yes
+fi

hack/install-e2e-tools.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GINKGO_VERSION="${GINKGO_VERSION:-v1.14.0}"
+KOPS_ROOT="${KOPS_ROOT:-}"
+export GO111MODULE=on
+
+if [[ -n ${INSTALL_PATH} ]]; then
+    export GOBIN="${INSTALL_PATH}"
+fi
+
+cd $(mktemp -d) > /dev/null
+
+echo " + Installing kubetest2"
+go install "sigs.k8s.io/kubetest2@latest"
+
+echo " + Installing ginkgo"
+go install "github.com/onsi/ginkgo/ginkgo@${GINKGO_VERSION}"
+
+if [[ -z "${KOPS_ROOT}" ]]; then
+    git clone https://github.com/kubernetes/kops.git
+    KOPS_ROOT="$(pwd)/kops"
+fi
+
+cd "${KOPS_ROOT}/tests/e2e" > /dev/null
+
+echo " + Installing kubetest2-tester-kops"
+go install ./kubetest2-tester-kops
+
+echo " + Installing kubetest2-kops"
+go install ./kubetest2-kops

pkg/controllers/tagging/tagging_controller_test.go

@@ -34,7 +34,6 @@ import (
 const TestClusterID = "clusterid.test"
 
 func Test_NodesJoiningAndLeaving(t *testing.T) {
-	klog.InitFlags(nil)
 	flag.CommandLine.Parse([]string{"--logtostderr=false"})
 	testcases := []struct {
 		name             string

tests/e2e/go.mod

@@ -0,0 +1,47 @@
+module k8s.io/cloud-provider-aws/tests/e2e
+
+go 1.16
+
+require (
+	github.com/google/uuid v1.1.4 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/onsi/ginkgo v1.14.1
+	github.com/onsi/gomega v1.10.3
+	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
+	k8s.io/api v0.23.0
+	k8s.io/apimachinery v0.23.0
+	k8s.io/client-go v9.0.0+incompatible
+	k8s.io/kubectl v0.17.2 // indirect
+	k8s.io/kubernetes v1.23.0
+)
+
+replace (
+	k8s.io/api => k8s.io/api v0.23.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.23.0
+	k8s.io/apiserver => k8s.io/apiserver v0.23.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.0
+	k8s.io/client-go => k8s.io/client-go v0.23.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.0
+	k8s.io/code-generator => k8s.io/code-generator v0.23.0
+	k8s.io/component-base => k8s.io/component-base v0.23.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.23.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.23.0
+	k8s.io/cri-api => k8s.io/cri-api v0.17.4-beta.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.0
+	k8s.io/kubectl => k8s.io/kubectl v0.23.0
+	k8s.io/kubelet => k8s.io/kubelet v0.23.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.0
+	k8s.io/metrics => k8s.io/metrics v0.23.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.23.0
+	k8s.io/node-api => k8s.io/node-api v0.23.0
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.0
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.23.0
+	k8s.io/sample-controller => k8s.io/sample-controller v0.23.0
+)