From 87a6e534b3467159e28001a768230ae6bbeb8bcb Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Wed, 11 Dec 2019 19:43:47 -0800 Subject: [PATCH] Add integration test framework. (#10) * Move logic from cmd/manager to a package for easier testing. Signed-off-by: Anthony Yeh * Add integration test framework. Signed-off-by: Anthony Yeh * Add integration-test workflow. Signed-off-by: Anthony Yeh * integration/framework: Wait for CRDs to be ready. Signed-off-by: Anthony Yeh * Switch to controller-runtime client. Signed-off-by: Anthony Yeh --- .github/workflows/integration-test.yaml | 32 ++++ .github/workflows/unit-test.yaml | 2 +- .gitignore | 2 + Makefile | 12 +- cmd/manager/main.go | 115 +------------ go.mod | 3 +- go.sum | 14 +- .../controllermanager/controller_manager.go | 54 +++++++ pkg/operator/controllermanager/flags.go | 50 ++++++ pkg/operator/controllermanager/scheme.go | 45 ++++++ test/integration/framework/apiserver.go | 112 +++++++++++++ test/integration/framework/etcd.go | 114 +++++++++++++ test/integration/framework/fixture.go | 145 +++++++++++++++++ test/integration/framework/main.go | 151 ++++++++++++++++++ .../vitesscluster/vitesscluster_test.go | 21 +++ tools/get-kube-binaries.sh | 53 ++++++ 16 files changed, 801 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/integration-test.yaml create mode 100644 pkg/operator/controllermanager/controller_manager.go create mode 100644 pkg/operator/controllermanager/flags.go create mode 100644 pkg/operator/controllermanager/scheme.go create mode 100644 test/integration/framework/apiserver.go create mode 100644 test/integration/framework/etcd.go create mode 100644 test/integration/framework/fixture.go create mode 100644 test/integration/framework/main.go create mode 100644 test/integration/vitesscluster/vitesscluster_test.go create mode 100755 tools/get-kube-binaries.sh diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 00000000..ee52532e --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,32 @@ +name: integration-test +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + build: + name: Integration Test + runs-on: ubuntu-latest + steps: + + - name: Set up Go + uses: actions/setup-go@v1 + with: + go-version: 1.12 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Go module cache + id: go-mod-cache + uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: go-mod-cache + + - name: Integration Test + run: make integration-test diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 9ce73c1c..a5173c06 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -32,4 +32,4 @@ jobs: run: go install -v ./... - name: Unit Test - run: go test ./... + run: make unit-test diff --git a/.gitignore b/.gitignore index 7c504700..165521f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Temporary Build Files build/_output build/_test +/tools/_bin/ + # Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode ### Emacs ### # -*- mode: gitignore; -*- diff --git a/Makefile b/Makefile index cfc75960..8477db0b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build release-build generate push-only push +.PHONY: build release-build unit-test integration-test generate push-only push IMAGE_REGISTRY:=docker.io IMAGE_TAG:=latest @@ -22,6 +22,16 @@ build: release-build: docker build -f build/Dockerfile.release -t $(IMAGE_NAME):$(IMAGE_TAG) . +unit-test: + pkgs="$$(go list ./... | grep -v '/test/integration/')" && \ + go test -i $${pkgs} && \ + go test $${pkgs} + +integration-test: + tools/get-kube-binaries.sh + go test -i ./test/integration/... + PATH="$(PWD)/tools/_bin:$(PATH)" go test -v -timeout 5m ./test/integration/... -args --logtostderr -v=6 + generate: operator-sdk-$(OPERATOR_SDK_VERSION) generate k8s operator-sdk-$(OPERATOR_SDK_VERSION) generate openapi diff --git a/cmd/manager/main.go b/cmd/manager/main.go index c29e8846..e48e9f62 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -24,18 +24,9 @@ import ( goruntime "runtime" "time" - "k8s.io/klog" - - "github.com/spf13/pflag" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/apimachinery/pkg/runtime" - kubernetesscheme "k8s.io/client-go/kubernetes/scheme" - appsv1defaults "k8s.io/kubernetes/pkg/apis/apps/v1" - batchv1defaults "k8s.io/kubernetes/pkg/apis/batch/v1" - corev1defaults "k8s.io/kubernetes/pkg/apis/core/v1" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" @@ -43,12 +34,9 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/operator-framework/operator-sdk/pkg/leader" - "github.com/operator-framework/operator-sdk/pkg/log/zap" sdkVersion "github.com/operator-framework/operator-sdk/version" - planetscaleapis "planetscale.dev/vitess-operator/pkg/apis" - "planetscale.dev/vitess-operator/pkg/controller" - vbssubcontroller "planetscale.dev/vitess-operator/pkg/controller/vitessbackupstorage/subcontroller" + "planetscale.dev/vitess-operator/pkg/operator/controllermanager" "planetscale.dev/vitess-operator/pkg/operator/fork" ) @@ -62,30 +50,7 @@ var ( metricsHost = "0.0.0.0" metricsPort int32 = 8383 ) -var log = logf.Log.WithName("cmd") - -// schemeAddFuncs are all the things we register into our compiled-in API type system (Scheme). -var schemeAddFuncs = []func(*runtime.Scheme) error{ - // This is the types only (no defaulters) for all built-in APIs supported by client-go. - kubernetesscheme.AddToScheme, - - /* - These are the defaulters for some built-in APIs that we use. - - This code doesn't come with the Kubernetes client library, - so we have to vendor it from the main server repository. - - Doing this lets us achieve `kubectl apply`-like semantics while - still using statically-typed Go structs instead of JSON/YAML - to define our objects. - */ - corev1defaults.AddToScheme, - appsv1defaults.AddToScheme, - batchv1defaults.AddToScheme, - - // This is our own CRDs. - planetscaleapis.AddToScheme, -} +var log = logf.Log.WithName("manager") func printVersion() { log.Info(fmt.Sprintf("Go Version: %s", goruntime.Version())) @@ -98,41 +63,7 @@ func main() { // do something other than run the main operator code. forkPath := fork.Path() - // Add the zap logger flag set to the CLI. The flag set must - // be added before calling pflag.Parse(). - pflag.CommandLine.AddFlagSet(zap.FlagSet()) - - // Add flags registered by imported packages (e.g. glog and - // controller-runtime) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - pflag.Parse() - - // Initialize flags for klog, which is necessary to configure logging from - // the low-level k8s client libraries. We don't use glog ourselves, but we - // have dependencies that use it, so we have to follow the instructions for - // making klog coexist with glog: - // https://github.com/kubernetes/klog/blob/master/examples/coexist_glog/coexist_glog.go - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - // Sync the glog and klog flags. - pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - f2.Value.Set(value) - } - }) - - // Use a zap logr.Logger implementation. If none of the zap - // flags are configured (or if the zap flag set is not being - // used), this defaults to a production zap logger. - // - // The logger instantiated here can be changed to any logger - // implementing the logr.Logger interface. This logger will - // be propagated through the whole operator, generating - // uniform and structured logs. - logf.SetLogger(zap.Logger()) + controllermanager.InitFlags() printVersion() @@ -161,18 +92,7 @@ func main() { } } - // Set up scheme for all resources we depend on. - scheme := runtime.NewScheme() - for _, add := range schemeAddFuncs { - if err := add(scheme); err != nil { - log.Error(err, "") - os.Exit(1) - } - } - - // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{ - Scheme: scheme, + mgr, err := controllermanager.New(forkPath, cfg, manager.Options{ Namespace: namespace, SyncPeriod: cacheInvalidateInterval, MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), @@ -182,32 +102,9 @@ func main() { os.Exit(1) } - log.Info("Registering Components.") - - // We use the fork path primarily to decide which controllers to run in this - // manager process. Not all controllers run in the root process, for example. - switch forkPath { - case "": - // Run all root controllers, defined as anything that registers itself - // in the top-level 'pkg/controller' at package init time. - if err := controller.AddToManager(mgr); err != nil { - log.Error(err, "") - os.Exit(1) - } - case vbssubcontroller.ForkPath: - // Run only the vitessbackupstorage subcontroller. - if err := vbssubcontroller.Add(mgr); err != nil { - log.Error(err, "") - os.Exit(1) - } - default: - log.Error(fmt.Errorf("undefined fork path: %v", forkPath), "") - os.Exit(1) - } - - log.Info("Starting the Cmd.") + log.Info("Starting the manager.") - // Start the Cmd + // Start the manager if err := mgr.Start(signals.SetupSignalHandler()); err != nil { log.Error(err, "Manager exited non-zero") os.Exit(1) diff --git a/go.mod b/go.mod index fb9694e6..38372a5d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.12 require ( contrib.go.opencensus.io/exporter/ocagent v0.4.12 // indirect github.com/Azure/go-autorest v11.7.1+incompatible // indirect - github.com/ahmetb/gen-crd-api-reference-docs v0.1.5-0.20190629210212-52e137b8d003 // indirect + github.com/ahmetb/gen-crd-api-reference-docs v0.1.5-0.20190629210212-52e137b8d003 github.com/aws/aws-sdk-go v1.20.4 // indirect github.com/coreos/etcd v3.3.12+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -27,6 +27,7 @@ require ( github.com/stretchr/testify v1.4.0 gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20190612125737-db0771252981 + k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236 k8s.io/apimachinery v0.0.0-20190612125636-6a5db36e93ad k8s.io/apiserver v0.0.0-20190228174905-79427f02047f // indirect k8s.io/client-go v11.0.0+incompatible diff --git a/go.sum b/go.sum index 335c0073..8536db19 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,7 @@ github.com/evanphx/json-patch v3.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.1.0+incompatible h1:K1MDoo4AZ4wU0GIU/fPmtZg7VpzLjCxu+UwBD1FvwOc= github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= @@ -239,12 +240,10 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20180418170936-39de4380c2e0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v0.0.0-20161128002007-199c40a060d1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -444,8 +443,6 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= -github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= @@ -456,8 +453,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190403104016-ea9eea638872/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 h1:JO6VBMEDSBX/LT4GKwSdvuFigZNwVD4lkPyUE4BDCKE= -github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= @@ -604,8 +599,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 h1:qPnAdmjNA41t3QBTx2mFGf/SD1IoslhYu7AmdsVzCcs= golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -649,8 +642,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0 h1:7z820YPX9pxWR59qM7BE5+fglp4D/mKqAwCvGt11b+8= -golang.org/x/sys v0.0.0-20190830142957-1e83adbbebd0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190926180325-855e68c8590b h1:/8GN4qrAmRZQXgjWZHj9z/UJI5vNqQhPtgcw02z2f+8= golang.org/x/sys v0.0.0-20190926180325-855e68c8590b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -681,6 +672,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190830154057-c17b040389b9 h1:5/jaG/gKlo3xxvUn85ReNyTlN7BvlPPsxC6sHZKjGEE= golang.org/x/tools v0.0.0-20190830154057-c17b040389b9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -804,7 +796,5 @@ sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= -vitess.io/vitess v0.0.0-20190910173716-ab425723d591 h1:K6fl8U71PECvq0qNjJpeMelXBEzraWuOBEz5TPEcLOA= -vitess.io/vitess v0.0.0-20190910173716-ab425723d591/go.mod h1:a2W63geOKZ66uYs+/4fCC84TE6ADmQcAqOKMiaNUb4w= vitess.io/vitess v0.0.0-20191106000153-8de7237f7b75 h1:NsQR+7ALrP7+0AEY7pm3Z0rb7F0tFYRVL2SdOG23M+s= vitess.io/vitess v0.0.0-20191106000153-8de7237f7b75/go.mod h1:Sk+YyPzRiOS3a6ToCSbno8wUQuxgqsiJat+lyXm5smw= diff --git a/pkg/operator/controllermanager/controller_manager.go b/pkg/operator/controllermanager/controller_manager.go new file mode 100644 index 00000000..fce49526 --- /dev/null +++ b/pkg/operator/controllermanager/controller_manager.go @@ -0,0 +1,54 @@ +package controllermanager + +import ( + "fmt" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" + + "sigs.k8s.io/controller-runtime/pkg/manager" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + "planetscale.dev/vitess-operator/pkg/controller" + vbssubcontroller "planetscale.dev/vitess-operator/pkg/controller/vitessbackupstorage/subcontroller" +) + +var log = logf.Log.WithName("controller-manager") + +func New(forkPath string, cfg *rest.Config, opts manager.Options) (manager.Manager, error) { + // Set up scheme for all resources we depend on. + var err error + opts.Scheme, err = NewScheme() + if err != nil { + return nil, err + } + + // Create a new manager to provide shared dependencies and start components + mgr, err := manager.New(cfg, opts) + if err != nil { + return nil, err + } + + log.Info("Registering Components.") + + // We use the fork path primarily to decide which controllers to run in this + // manager process. Not all controllers run in the root process, for example. + switch forkPath { + case "": + // Run all root controllers, defined as anything that registers itself + // in the top-level 'pkg/controller' at package init time. + if err := controller.AddToManager(mgr); err != nil { + return nil, err + } + case vbssubcontroller.ForkPath: + // Run only the vitessbackupstorage subcontroller. + if err := vbssubcontroller.Add(mgr); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("undefined fork path: %v", forkPath) + } + + return mgr, nil +} diff --git a/pkg/operator/controllermanager/flags.go b/pkg/operator/controllermanager/flags.go new file mode 100644 index 00000000..89632aa3 --- /dev/null +++ b/pkg/operator/controllermanager/flags.go @@ -0,0 +1,50 @@ +package controllermanager + +import ( + "flag" + + "github.com/spf13/pflag" + + "k8s.io/klog" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + "github.com/operator-framework/operator-sdk/pkg/log/zap" +) + +func InitFlags() { + // Add the zap logger flag set to the CLI. The flag set must + // be added before calling pflag.Parse(). + pflag.CommandLine.AddFlagSet(zap.FlagSet()) + + // Add flags registered by imported packages (e.g. glog and + // controller-runtime) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + pflag.Parse() + + // Initialize flags for klog, which is necessary to configure logging from + // the low-level k8s client libraries. We don't use glog ourselves, but we + // have dependencies that use it, so we have to follow the instructions for + // making klog coexist with glog: + // https://github.com/kubernetes/klog/blob/master/examples/coexist_glog/coexist_glog.go + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + // Sync the glog and klog flags. + pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + f2.Value.Set(value) + } + }) + + // Use a zap logr.Logger implementation. If none of the zap + // flags are configured (or if the zap flag set is not being + // used), this defaults to a production zap logger. + // + // The logger instantiated here can be changed to any logger + // implementing the logr.Logger interface. This logger will + // be propagated through the whole operator, generating + // uniform and structured logs. + logf.SetLogger(zap.Logger()) +} diff --git a/pkg/operator/controllermanager/scheme.go b/pkg/operator/controllermanager/scheme.go new file mode 100644 index 00000000..4ca5a1c9 --- /dev/null +++ b/pkg/operator/controllermanager/scheme.go @@ -0,0 +1,45 @@ +package controllermanager + +import ( + "k8s.io/apimachinery/pkg/runtime" + kubernetesscheme "k8s.io/client-go/kubernetes/scheme" + appsv1defaults "k8s.io/kubernetes/pkg/apis/apps/v1" + batchv1defaults "k8s.io/kubernetes/pkg/apis/batch/v1" + corev1defaults "k8s.io/kubernetes/pkg/apis/core/v1" + + planetscaleapis "planetscale.dev/vitess-operator/pkg/apis" +) + +// schemeAddFuncs are all the things we register into our compiled-in API type system (Scheme). +var schemeAddFuncs = []func(*runtime.Scheme) error{ + // This is the types only (no defaulters) for all built-in APIs supported by client-go. + kubernetesscheme.AddToScheme, + + /* + These are the defaulters for some built-in APIs that we use. + + This code doesn't come with the Kubernetes client library, + so we have to vendor it from the main server repository. + + Doing this lets us achieve `kubectl apply`-like semantics while + still using statically-typed Go structs instead of JSON/YAML + to define our objects. + */ + corev1defaults.AddToScheme, + appsv1defaults.AddToScheme, + batchv1defaults.AddToScheme, + + // This is our own CRDs. + planetscaleapis.AddToScheme, +} + +// NewScheme returns a Scheme with all the types that the operator needs. +func NewScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + for _, add := range schemeAddFuncs { + if err := add(scheme); err != nil { + return nil, err + } + } + return scheme, nil +} diff --git a/test/integration/framework/apiserver.go b/test/integration/framework/apiserver.go new file mode 100644 index 00000000..112ac1da --- /dev/null +++ b/test/integration/framework/apiserver.go @@ -0,0 +1,112 @@ +/* +Copyright 2019 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is forked from github.com/GoogleCloudPlatform/metacontroller. + +package framework + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strconv" + + "k8s.io/client-go/rest" + "k8s.io/klog" +) + +var apiserverURL = "" + +const installApiserver = ` +Cannot find kube-apiserver, cannot run integration tests + +Please download kube-apiserver and ensure it is somewhere in the PATH. +See tools/get-kube-binaries.sh + +` + +// getApiserverPath returns a path to a kube-apiserver executable. +func getApiserverPath() (string, error) { + return exec.LookPath("kube-apiserver") +} + +// startApiserver executes a kube-apiserver instance. +// The returned function will signal the process and wait for it to exit. +func startApiserver() (func(), error) { + apiserverPath, err := getApiserverPath() + if err != nil { + fmt.Fprintf(os.Stderr, installApiserver) + return nil, fmt.Errorf("could not find kube-apiserver in PATH: %v", err) + } + apiserverPort, err := getAvailablePort() + if err != nil { + return nil, fmt.Errorf("could not get a port: %v", err) + } + apiserverURL = fmt.Sprintf("http://127.0.0.1:%d", apiserverPort) + klog.Infof("starting kube-apiserver on %s", apiserverURL) + + apiserverDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_apiserver_data") + if err != nil { + return nil, fmt.Errorf("unable to make temp kube-apiserver data dir: %v", err) + } + klog.Infof("storing kube-apiserver data in: %v", apiserverDataDir) + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext( + ctx, + apiserverPath, + "--cert-dir", apiserverDataDir, + "--insecure-port", strconv.Itoa(apiserverPort), + // We don't use the secure port, but we need to pick something that + // doesn't conflict with other test apiservers. + "--secure-port", strconv.Itoa(apiserverPort+1), + "--etcd-servers", etcdURL, + ) + + // Uncomment these to see kube-apiserver output in test logs. + // For operator tests, we generally don't expect problems at this level. + //cmd.Stdout = os.Stdout + //cmd.Stderr = os.Stderr + + stop := func() { + cancel() + err := cmd.Wait() + klog.Infof("kube-apiserver exit status: %v", err) + err = os.RemoveAll(apiserverDataDir) + if err != nil { + klog.Warningf("error during kube-apiserver cleanup: %v", err) + } + } + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to run kube-apiserver: %v", err) + } + return stop, nil +} + +// ApiserverURL returns the URL of the kube-apiserver instance started by TestMain. +func ApiserverURL() string { + return apiserverURL +} + +// ApiserverConfig returns a rest.Config to connect to the test instance. +func ApiserverConfig() *rest.Config { + return &rest.Config{ + Host: ApiserverURL(), + } +} diff --git a/test/integration/framework/etcd.go b/test/integration/framework/etcd.go new file mode 100644 index 00000000..69eb7411 --- /dev/null +++ b/test/integration/framework/etcd.go @@ -0,0 +1,114 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is forked from k8s.io/kubernetes/test/integration/framework/. + +package framework + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + + "k8s.io/klog" +) + +var etcdURL = "" + +const installEtcd = ` +Cannot find etcd, cannot run integration tests + +Please download kube-apiserver and ensure it is somewhere in the PATH. +See tools/get-kube-binaries.sh + +` + +// getEtcdPath returns a path to an etcd executable. +func getEtcdPath() (string, error) { + bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd") + p, err := exec.LookPath(bazelPath) + if err == nil { + return p, nil + } + return exec.LookPath("etcd") +} + +// getAvailablePort returns a TCP port that is available for binding. +func getAvailablePort() (int, error) { + l, err := net.Listen("tcp", ":0") + if err != nil { + return 0, fmt.Errorf("could not bind to a port: %v", err) + } + // It is possible but unlikely that someone else will bind this port before we + // get a chance to use it. + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +// startEtcd executes an etcd instance. The returned function will signal the +// etcd process and wait for it to exit. +func startEtcd() (func(), error) { + etcdPath, err := getEtcdPath() + if err != nil { + fmt.Fprintf(os.Stderr, installEtcd) + return nil, fmt.Errorf("could not find etcd in PATH: %v", err) + } + etcdPort, err := getAvailablePort() + if err != nil { + return nil, fmt.Errorf("could not get a port: %v", err) + } + etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort) + klog.Infof("starting etcd on %s", etcdURL) + + etcdDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_etcd_data") + if err != nil { + return nil, fmt.Errorf("unable to make temp etcd data dir: %v", err) + } + klog.Infof("storing etcd data in: %v", etcdDataDir) + + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext( + ctx, + etcdPath, + "--data-dir", etcdDataDir, + "--listen-client-urls", etcdURL, + "--advertise-client-urls", etcdURL, + "--listen-peer-urls", "http://127.0.0.1:0", + ) + + // Uncomment these to see etcd output in test logs. + //cmd.Stdout = os.Stdout + //cmd.Stderr = os.Stderr + + stop := func() { + cancel() + err := cmd.Wait() + klog.Infof("etcd exit status: %v", err) + err = os.RemoveAll(etcdDataDir) + if err != nil { + klog.Warningf("error during etcd cleanup: %v", err) + } + } + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to run etcd: %v", err) + } + return stop, nil +} diff --git a/test/integration/framework/fixture.go b/test/integration/framework/fixture.go new file mode 100644 index 00000000..f0ca0770 --- /dev/null +++ b/test/integration/framework/fixture.go @@ -0,0 +1,145 @@ +/* +Copyright 2019 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is forked from github.com/GoogleCloudPlatform/metacontroller. + +package framework + +import ( + "context" + "fmt" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "planetscale.dev/vitess-operator/pkg/operator/controllermanager" +) + +const ( + defaultWaitTimeout = 30 * time.Second + defaultWaitInterval = 250 * time.Millisecond +) + +// Fixture is a collection of scaffolding for each integration test method. +type Fixture struct { + t *testing.T + + teardownFuncs []func(ctx context.Context) error + + client client.Client +} + +func NewFixture(t *testing.T) *Fixture { + config := ApiserverConfig() + + scheme, err := controllermanager.NewScheme() + if err != nil { + t.Fatalf("can't create Scheme: %v", err) + } + + mapper, err := apiutil.NewDiscoveryRESTMapper(config) + if err != nil { + t.Fatalf("can't create Mapper: %v", err) + } + + kubeClient, err := client.New(config, client.Options{ + Scheme: scheme, + Mapper: mapper, + }) + if err != nil { + t.Fatalf("can't create Client: %v", err) + } + + return &Fixture{ + t: t, + client: kubeClient, + } +} + +// Client returns the Kubernetes client. +func (f *Fixture) Client() client.Client { + return f.client +} + +// CreateNamespace creates a namespace that will be deleted after this test +// finishes. +func (f *Fixture) CreateNamespace(ctx context.Context, namespace string) *corev1.Namespace { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + if err := f.client.Create(ctx, ns); err != nil { + f.t.Fatal(err) + } + f.deferTeardown(func(ctx context.Context) error { + // Make a fresh object with just the name, so the delete is unconditional. + return f.client.Delete(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + }, client.PropagationPolicy(metav1.DeletePropagationForeground)) + }) + return ns +} + +// TearDown cleans up resources created through this instance of the test fixture. +func (f *Fixture) TearDown(ctx context.Context) { + for i := len(f.teardownFuncs) - 1; i >= 0; i-- { + teardown := f.teardownFuncs[i] + if err := teardown(ctx); err != nil { + f.t.Logf("Error during teardown: %v", err) + // Mark the test as failed, but continue trying to tear down. + f.t.Fail() + } + } +} + +// Wait polls the condition until it's true, with a default interval and timeout. +// This is meant for use in integration tests, so frequent polling is fine. +// +// The condition function returns a bool indicating whether it is satisfied, +// as well as an error which should be non-nil if and only if the function was +// unable to determine whether or not the condition is satisfied (for example +// if the check involves calling a remote server and the request failed). +// +// If the condition function returns a non-nil error, Wait will log the error +// and continue retrying until the timeout. +func (f *Fixture) Wait(condition func() (bool, error)) error { + start := time.Now() + for { + ok, err := condition() + if err == nil && ok { + return nil + } + if err != nil { + // Log error, but keep trying until timeout. + f.t.Logf("error while waiting for condition: %v", err) + } + if time.Since(start) > defaultWaitTimeout { + return fmt.Errorf("timed out waiting for condition (%v)", err) + } + time.Sleep(defaultWaitInterval) + } +} + +func (f *Fixture) deferTeardown(teardown func(ctx context.Context) error) { + f.teardownFuncs = append(f.teardownFuncs, teardown) +} diff --git a/test/integration/framework/main.go b/test/integration/framework/main.go new file mode 100644 index 00000000..a6976d29 --- /dev/null +++ b/test/integration/framework/main.go @@ -0,0 +1,151 @@ +/* +Copyright 2019 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is forked from github.com/GoogleCloudPlatform/metacontroller. + +package framework + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path" + "time" + + "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "planetscale.dev/vitess-operator/pkg/operator/controllermanager" +) + +const installKubectl = ` +Cannot find kubectl, cannot run integration tests + +Please download kubectl and ensure it is somewhere in the PATH. +See tools/get-kube-binaries.sh + +` + +// deployDir is the path from the integration test binary working dir to the +// directory containing manifests to install vitess-operator. +const deployDir = "../../../deploy" + +// getKubectlPath returns a path to a kube-apiserver executable. +func getKubectlPath() (string, error) { + return exec.LookPath("kubectl") +} + +// TestMain starts etcd, kube-apiserver, and vitess-operator before running tests. +func TestMain(tests func() int) { + if err := testMain(tests); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func testMain(tests func() int) error { + controllermanager.InitFlags() + + if _, err := getKubectlPath(); err != nil { + return errors.New(installKubectl) + } + + stopEtcd, err := startEtcd() + if err != nil { + return fmt.Errorf("cannot run integration tests: unable to start etcd: %v", err) + } + defer stopEtcd() + + stopApiserver, err := startApiserver() + if err != nil { + return fmt.Errorf("cannot run integration tests: unable to start kube-apiserver: %v", err) + } + defer stopApiserver() + + klog.Info("Waiting for kube-apiserver to be ready...") + start := time.Now() + for { + out, kubectlErr := execKubectl("version") + if kubectlErr == nil { + break + } + if time.Since(start) > 60*time.Second { + return fmt.Errorf("timed out waiting for kube-apiserver to be ready: %v\n%s", kubectlErr, out) + } + time.Sleep(time.Second) + } + + // Install vites-operator base files, but not the Deployment itself. + files := []string{ + "service_account.yaml", + "role.yaml", + "role_binding.yaml", + "priority.yaml", + "crds/", + } + for _, file := range files { + filePath := path.Join(deployDir, file) + klog.Infof("Installing %v...", filePath) + if out, err := execKubectl("apply", "-f", filePath); err != nil { + return fmt.Errorf("cannot install %v: %v\n%s", filePath, err, out) + } + } + + klog.Info("Waiting for CRDs to be ready...") + start = time.Now() + for { + out, kubectlErr := execKubectl("get", "vt,vtc,vtk,vts,vtbs,vtb,etcdls") + if kubectlErr == nil { + break + } + if time.Since(start) > 30*time.Second { + return fmt.Errorf("timed out waiting for CRDs to be ready: %v\n%s", kubectlErr, out) + } + time.Sleep(time.Second) + } + + // Start vitess-operator in this test process. + mgr, err := controllermanager.New("", ApiserverConfig(), manager.Options{ + Namespace: "default", + }) + if err != nil { + return fmt.Errorf("cannot create controller-manager: %v", err) + } + stop := make(chan struct{}) + defer close(stop) + go func() { + if err := mgr.Start(stop); err != nil { + klog.Errorf("cannot start controller-manager: %v", err) + } + }() + + // Now actually run the tests. + if exitCode := tests(); exitCode != 0 { + return fmt.Errorf("one or more tests failed with exit code: %v", exitCode) + } + return nil +} + +func execKubectl(args ...string) ([]byte, error) { + execPath, err := exec.LookPath("kubectl") + if err != nil { + return nil, fmt.Errorf("cannot exec kubectl: %v", err) + } + cmdline := append([]string{"--server", ApiserverURL()}, args...) + cmd := exec.Command(execPath, cmdline...) + return cmd.CombinedOutput() +} diff --git a/test/integration/vitesscluster/vitesscluster_test.go b/test/integration/vitesscluster/vitesscluster_test.go new file mode 100644 index 00000000..62925e6a --- /dev/null +++ b/test/integration/vitesscluster/vitesscluster_test.go @@ -0,0 +1,21 @@ +package vitesscluster + +import ( + "context" + "testing" + + "planetscale.dev/vitess-operator/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.TestMain(m.Run) +} + +func TestVitesCluster(t *testing.T) { + ctx := context.Background() + + f := framework.NewFixture(t) + defer f.TearDown(ctx) + + // TODO: Test something. This is just a skeleton. +} diff --git a/tools/get-kube-binaries.sh b/tools/get-kube-binaries.sh new file mode 100755 index 00000000..60ebe5c7 --- /dev/null +++ b/tools/get-kube-binaries.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +# This script downloads etcd and Kubernetes binaries that are +# used as part of the integration test environment, +# and places them in tools/_bin/. +# +# The integration test framework expects these binaries to be found in the PATH. + +# This is the kube-apiserver version to test against. +KUBE_VERSION="${KUBE_VERSION:-v1.13.4}" +KUBERNETES_RELEASE_URL="${KUBERNETES_RELEASE_URL:-https://dl.k8s.io}" + +# This should be the etcd version downloaded by kubernetes/hack/lib/etcd.sh +# as of the above Kubernetes version. +ETCD_VERSION="${ETCD_VERSION:-v3.2.24}" +ETCD_RELEASE_URL="${ETCD_RELEASE_URL:-https://github.com/coreos/etcd/releases/download}" + +DIR="${BASH_SOURCE%/*}" +mkdir -p "${DIR}/_bin" +cd "${DIR}/_bin" + +# Download kube-apiserver if needed. +if [ ! -f "kube-apiserver-${KUBE_VERSION}" ]; then + echo "Downloading kube-apiserver ${KUBE_VERSION}..." + curl -L "${KUBERNETES_RELEASE_URL}/${KUBE_VERSION}/bin/linux/amd64/kube-apiserver" > "kube-apiserver-${KUBE_VERSION}" + chmod +x "kube-apiserver-${KUBE_VERSION}" +fi +echo "Using kube-apiserver ${KUBE_VERSION}." +ln -sf "kube-apiserver-${KUBE_VERSION}" kube-apiserver + +# Download kubectl if needed. +if [ ! -f "kubectl-${KUBE_VERSION}" ]; then + echo "Downloading kubectl ${KUBE_VERSION}..." + curl -L "${KUBERNETES_RELEASE_URL}/${KUBE_VERSION}/bin/linux/amd64/kubectl" > "kubectl-${KUBE_VERSION}" + chmod +x "kubectl-${KUBE_VERSION}" +fi +echo "Using kubectl ${KUBE_VERSION}." +ln -sf "kubectl-${KUBE_VERSION}" kubectl + +# Download etcd if needed. +if [ ! -f "etcd-${ETCD_VERSION}" ]; then + echo "Downloading etcd ${ETCD_VERSION}..." + basename="etcd-${ETCD_VERSION}-linux-amd64" + tarfile="${basename}.tar.gz" + url="${ETCD_RELEASE_URL}/${ETCD_VERSION}/${tarfile}" + curl -L "${url}" | tar -zx + mv "${basename}/etcd" "etcd-${ETCD_VERSION}" + rm -rf "${basename}" +fi +echo "Using etcd ${ETCD_VERSION}." +ln -sf "etcd-${ETCD_VERSION}" etcd