Skip to content

Commit

Permalink
WIP E2E
Browse files Browse the repository at this point in the history
  • Loading branch information
nckturner committed Feb 4, 2022
1 parent 454ed78 commit b0353e9
Show file tree
Hide file tree
Showing 13 changed files with 2,576 additions and 7 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ _output/
/kops-example
docs/book/_book/
site/

.vscode/
e2e.test
20 changes: 20 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

.EXPORT_ALL_VARIABLES:

SHELL := /bin/bash
SOURCES := $(shell find . -name '*.go')
GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
Expand Down Expand Up @@ -66,6 +67,11 @@ docker-build:
--platform linux/amd64,linux/arm64 \
--tag $(IMAGE) .

e2e.test:
pushd tests/e2e > /dev/null && \
go test -c && popd
mv tests/e2e/e2e.test e2e.test

.PHONY: check
check: verify-fmt verify-lint vet

Expand Down Expand Up @@ -105,3 +111,17 @@ publish-docs:
.PHONY: kops-example
kops-example:
./hack/kops-example.sh

.PHONY: test-e2e
test-e2e: e2e.test
AWS_REGION=us-west-2 \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[cloud-provider-aws-e2e\]" \
./hack/e2e/run.sh

# Use `make install-e2e-tools KOPS_ROOT=<local-kops-installation>`
# to skip the kops download, test local changes to the kubetest2-kops
# deployer, etc.
.PHONY: install-e2e-tools
install-e2e-tools:
./hack/install-e2e-tools.sh
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
gopkg.in/gcfg.v1 v1.2.0
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
k8s.io/client-go v9.0.0+incompatible
k8s.io/cloud-provider v0.23.0
k8s.io/code-generator v0.23.0
k8s.io/component-base v0.23.0
Expand Down Expand Up @@ -44,11 +44,11 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/google/uuid v1.1.4 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/imdario/mergo v0.3.5 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand Down Expand Up @@ -102,7 +102,7 @@ require (
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/warnings.v0 v0.1.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/apiserver v0.23.0 // indirect
Expand All @@ -112,5 +112,6 @@ require (
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/kubetest2 v0.0.0-20220127004650-cc0c36d0693e // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
)
986 changes: 984 additions & 2 deletions go.sum

Large diffs are not rendered by default.

97 changes: 97 additions & 0 deletions hack/e2e/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/bin/bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

function test_run_id() {
echo "$(date '+%Y%m%d%H%M%S')"
}

test_run_id="$(test_run_id)"
repo_root="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/../.." &> /dev/null && pwd )"
output="${repo_root}/_output"
test_output_root="${output}/test"
test_run="${test_output_root}/${test_run_id}"

# Configurable
#KUBECONFIG="${KUBECONFIG:-}"
SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-}"

KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.23.2}"
GINKGO_VERSION="v1.14.0"
CLUSTER_NAME="test-cluster-${test_run_id}.k8s.local"
KOPS_STATE_STORE="${KOPS_STATE_STORE:-}"
REGION="${AWS_REGION:-us-west-2}"
ZONES="${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}"

# Test args
GINKGO_FOCUS=${GINKGO_FOCUS:-"\[cloud-provider-aws-e2e\]"}
GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
GINKGO_NODES=${GINKGO_NODES:-4}

if [[ -z "${KOPS_STATE_STORE}" ]]; then
echo "KOPS_STATE_STORE must be set"
exit 1
fi

if [[ ! -f "${repo_root}/e2e.test" ]]; then
echo "Missing e2e.test binary"
exit 1
fi

echo "Starting test run ---"
echo " + Region: ${REGION} (${ZONES})"
echo " + Cluster name: ${CLUSTER_NAME}"
echo " + Kubernetes version: ${KUBERNETES_VERSION}"
echo " + Focus: ${GINKGO_FOCUS}"
echo " + Skip: ${GINKGO_SKIP}"
echo " + Kops state store: ${KOPS_STATE_STORE}"
echo " + SSH key path: ${SSH_PUBLIC_KEY_PATH}"
echo " + Test run ID: ${test_run_id}"
echo " + Kubetest run dir: ${test_run}"

mkdir -p "${test_run}"

export KOPS_STATE_STORE
export ARTIFACTS="${test_output_root}"
export KUBETEST2_RUN_DIR="${test_output_root}"

echo "Installing e2e.test to ${test_run}"
cp "${repo_root}/e2e.test" "${test_run}"

echo "Installing ginkgo to ${test_run}"
GINKGO_BIN=${test_run}/ginkgo
if [[ ! -f ${GINKGO_BIN} ]]; then
GOBIN=${test_run} go install "github.com/onsi/ginkgo/ginkgo@${GINKGO_VERSION}"
fi

kubetest2 kops \
-v 2 \
--up \
--run-id=${test_run_id} \
--cloud-provider=aws \
--cluster-name=${CLUSTER_NAME} \
--create-args="--zones=${ZONES} --node-size=m5.large --master-size=m5.large" \
--admin-access="0.0.0.0/0" \
--kubernetes-version=${KUBERNETES_VERSION} \
--ssh-public-key="${SSH_PUBLIC_KEY_PATH}" \
--kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
--test=kops \
-- \
--use-built-binaries=true \
--focus-regex="${GINKGO_FOCUS}" \
--parallel 25

36 changes: 36 additions & 0 deletions hack/install-e2e-tools.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env bash

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

KOPS_ROOT="${KOPS_ROOT:-}"
export GO111MODULE=on

if [[ -z "${KOPS_ROOT}" ]]; then
cd $(mktemp -d) > /dev/null
git clone https://github.com/kubernetes/kops.git
KOPS_ROOT="$(cwd)/kops"
fi

cd "$HOME/Projects/kubetest2/" > /dev/null
#go install sigs.k8s.io/kubetest2/...@latest
go install ./kubetest2-tester-ginkgo

cd "${KOPS_ROOT}/tests/e2e" > /dev/null
go install ./kubetest2-tester-kops
go install ./kubetest2-kops
47 changes: 47 additions & 0 deletions tests/e2e/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
module k8s.io/cloud-provider-aws/tests/e2e

go 1.16

require (
github.com/google/uuid v1.1.4 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/onsi/ginkgo v1.14.1
github.com/onsi/gomega v1.10.3
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
k8s.io/client-go v9.0.0+incompatible
k8s.io/kubectl v0.17.2 // indirect
k8s.io/kubernetes v1.23.0
)

replace (
k8s.io/api => k8s.io/api v0.23.0
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.0
k8s.io/apimachinery => k8s.io/apimachinery v0.23.0
k8s.io/apiserver => k8s.io/apiserver v0.23.0
k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.0
k8s.io/client-go => k8s.io/client-go v0.23.0
k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.0
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.0
k8s.io/code-generator => k8s.io/code-generator v0.23.0
k8s.io/component-base => k8s.io/component-base v0.23.0
k8s.io/component-helpers => k8s.io/component-helpers v0.23.0
k8s.io/controller-manager => k8s.io/controller-manager v0.23.0
k8s.io/cri-api => k8s.io/cri-api v0.17.4-beta.0
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.0
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.0
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.0
k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.0
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.0
k8s.io/kubectl => k8s.io/kubectl v0.23.0
k8s.io/kubelet => k8s.io/kubelet v0.23.0
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.0
k8s.io/metrics => k8s.io/metrics v0.23.0
k8s.io/mount-utils => k8s.io/mount-utils v0.23.0
k8s.io/node-api => k8s.io/node-api v0.23.0
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.0
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.0
k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.23.0
k8s.io/sample-controller => k8s.io/sample-controller v0.23.0
)
Loading

0 comments on commit b0353e9

Please sign in to comment.