Merge pull request #515 from saikat-royc/issue-511

Add option to deploy GKE managed PD CSI driver for integration tests
kubernetes-sigs · Jun 12, 2020 · 7e03d6d · 7e03d6d
2 parents 13325ce + 77f4be8
commit 7e03d6d
Show file tree

Hide file tree

Showing 7 changed files with 667 additions and 68 deletions.
diff --git a/test/k8s-integration/cluster.go b/test/k8s-integration/cluster.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -9,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
 	"k8s.io/klog"
 )
 
@@ -131,7 +133,7 @@ func setImageTypeEnvs(imageType string) error {
 	return nil
 }
 
-func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) error {
+func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string, useManagedDriver bool) error {
 	locationArg, locationVal, err := gkeLocationArgs(gceZone, gceRegion)
 	if err != nil {
 		return err
@@ -150,9 +152,23 @@ func clusterUpGKE(gceZone, gceRegion string, numNodes int, imageType string) err
 			return err
 		}
 	}
-	cmd := exec.Command("gcloud", "container", "clusters", "create", gkeTestClusterName,
-		locationArg, locationVal, "--cluster-version", *gkeClusterVer, "--num-nodes", strconv.Itoa(numNodes),
-		"--quiet", "--machine-type", "n1-standard-2", "--image-type", imageType)
+
+	var cmd *exec.Cmd
+	cmdParams := []string{"container", "clusters", "create", gkeTestClusterName,
+		locationArg, locationVal, "--num-nodes", strconv.Itoa(numNodes)}
+	if isVariableSet(gkeClusterVer) {
+		cmdParams = append(cmdParams, "--cluster-version", *gkeClusterVer)
+	} else {
+		cmdParams = append(cmdParams, "--release-channel", *gkeReleaseChannel)
+	}
+
+	if useManagedDriver {
+		// PD CSI Driver add on is enabled only in gcloud beta.
+		cmdParams = append([]string{"beta"}, cmdParams...)
+		cmdParams = append(cmdParams, "--addons", "GcePersistentDiskCsiDriver")
+	}
+
+	cmd = exec.Command("gcloud", cmdParams...)
 	err = runCommand("Staring E2E Cluster on GKE", cmd)
 	if err != nil {
 		return fmt.Errorf("failed to bring up kubernetes e2e cluster on gke: %v", err)
@@ -290,3 +306,30 @@ func getNormalizedVersion(kubeVersion, gkeVersion string) (string, error) {
 	return strings.Join(toks[:2], "."), nil
 
 }
+
+func getKubeClusterVersion() (string, error) {
+	out, err := exec.Command("kubectl", "version", "-o=json").CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to obtain cluster version, error: %v", err)
+	}
+	type version struct {
+		ClientVersion *apimachineryversion.Info `json:"clientVersion,omitempty" yaml:"clientVersion,omitempty"`
+		ServerVersion *apimachineryversion.Info `json:"serverVersion,omitempty" yaml:"serverVersion,omitempty"`
+	}
+
+	var v version
+	err = json.Unmarshal(out, &v)
+	if err != nil {
+		return "", fmt.Errorf("Failed to parse kubectl version output, error: %v", err)
+	}
+
+	return v.ServerVersion.GitVersion, nil
+}
+
+func mustGetKubeClusterVersion() string {
+	ver, err := getKubeClusterVersion()
+	if err != nil {
+		klog.Fatalf("Error: %v", err)
+	}
+	return ver
+}
diff --git a/test/k8s-integration/main.go b/test/k8s-integration/main.go
@@ -22,27 +22,28 @@ import (
 	"path/filepath"
 	"syscall"
 
-	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
-
 	"k8s.io/apimachinery/pkg/util/uuid"
+	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/klog"
+	testutils "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/test/e2e/utils"
 )
 
 var (
 	// Kubernetes cluster flags
-	teardownCluster  = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
-	teardownDriver   = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
-	bringupCluster   = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
-	gceZone          = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
-	gceRegion        = flag.String("gce-region", "", "region that gke regional cluster should be created in")
-	kubeVersion      = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
-	testVersion      = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
-	kubeFeatureGates = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
-	localK8sDir      = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
-	deploymentStrat  = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
-	gkeClusterVer    = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
-	numNodes         = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
-	imageType        = flag.String("image-type", "cos", "the image type to use for the cluster")
+	teardownCluster   = flag.Bool("teardown-cluster", true, "teardown the cluster after the e2e test")
+	teardownDriver    = flag.Bool("teardown-driver", true, "teardown the driver after the e2e test")
+	bringupCluster    = flag.Bool("bringup-cluster", true, "build kubernetes and bringup a cluster")
+	gceZone           = flag.String("gce-zone", "", "zone that the gce k8s cluster is created/found in")
+	gceRegion         = flag.String("gce-region", "", "region that gke regional cluster should be created in")
+	kubeVersion       = flag.String("kube-version", "", "version of Kubernetes to download and use for the cluster")
+	testVersion       = flag.String("test-version", "", "version of Kubernetes to download and use for tests")
+	kubeFeatureGates  = flag.String("kube-feature-gates", "", "feature gates to set on new kubernetes cluster")
+	localK8sDir       = flag.String("local-k8s-dir", "", "local prebuilt kubernetes/kubernetes directory to use for cluster and test binaries")
+	deploymentStrat   = flag.String("deployment-strategy", "gce", "choose between deploying on gce or gke")
+	gkeClusterVer     = flag.String("gke-cluster-version", "", "version of Kubernetes master and node for gke")
+	numNodes          = flag.Int("num-nodes", -1, "the number of nodes in the test cluster")
+	imageType         = flag.String("image-type", "cos", "the image type to use for the cluster")
+	gkeReleaseChannel = flag.String("gke-release-channel", "", "GKE release channel to be used for cluster deploy. One of 'rapid', 'stable' or 'regular'")
 
 	// Test infrastructure flags
 	boskosResourceType = flag.String("boskos-resource-type", "gce-project", "name of the boskos resource type to reserve")
@@ -51,10 +52,11 @@ var (
 	inProw             = flag.Bool("run-in-prow", false, "is the test running in PROW")
 
 	// Driver flags
-	stagingImage      = flag.String("staging-image", "", "name of image to stage to")
-	saFile            = flag.String("service-account-file", "", "path of service account file")
-	deployOverlayName = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
-	doDriverBuild     = flag.Bool("do-driver-build", true, "building the driver from source")
+	stagingImage        = flag.String("staging-image", "", "name of image to stage to")
+	saFile              = flag.String("service-account-file", "", "path of service account file")
+	deployOverlayName   = flag.String("deploy-overlay-name", "", "which kustomize overlay to deploy the driver with")
+	doDriverBuild       = flag.Bool("do-driver-build", true, "building the driver from source")
+	useGKEManagedDriver = flag.Bool("use-gke-managed-driver", false, "use GKE managed PD CSI driver for the tests")
 
 	// Test flags
 	migrationTest = flag.Bool("migration-test", false, "sets the flag on the e2e binary signalling migration")
@@ -75,12 +77,23 @@ func init() {
 func main() {
 	flag.Parse()
 
-	if !*inProw {
+	if !*inProw && !*useGKEManagedDriver {
 		ensureVariable(stagingImage, true, "staging-image is a required flag, please specify the name of image to stage to")
 	}
 
+	if *useGKEManagedDriver {
+		ensureVariableVal(deploymentStrat, "gke", "deployment strategy must be GKE for using managed driver")
+		ensureFlag(doDriverBuild, false, "'do-driver-build' must be false when using GKE managed driver")
+		ensureFlag(teardownDriver, false, "'teardown-driver' must be false when using GKE managed driver")
+		ensureVariable(stagingImage, false, "'staging-image' must not be set when using GKE managed driver")
+		ensureVariable(deployOverlayName, false, "'deploy-overlay-name' must not be set when using GKE managed driver")
+	}
+
 	ensureVariable(saFile, true, "service-account-file is a required flag")
-	ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
+	if !*useGKEManagedDriver {
+		ensureVariable(deployOverlayName, true, "deploy-overlay-name is a required flag")
+	}
+
 	ensureVariable(testFocus, true, "test-focus is a required flag")
 	ensureVariable(imageType, true, "image type is a required flag. Available options include 'cos' and 'ubuntu'")
 
@@ -103,7 +116,8 @@ func main() {
 	if *deploymentStrat == "gke" {
 		ensureFlag(migrationTest, false, "Cannot set deployment strategy to 'gke' for migration tests.")
 		ensureVariable(kubeVersion, false, "Cannot set kube-version when using deployment strategy 'gke'. Use gke-cluster-version.")
-		ensureVariable(gkeClusterVer, true, "Must set gke-cluster-version when using deployment strategy 'gke'.")
+		ensureExactlyOneVariableSet([]*string{gkeClusterVer, gkeReleaseChannel},
+			"For GKE cluster deployment, exactly one of 'gke-cluster-version' or 'gke-release-channel' must be set")
 		ensureVariable(kubeFeatureGates, false, "Cannot set feature gates when using deployment strategy 'gke'.")
 		if len(*localK8sDir) == 0 {
 			ensureVariable(testVersion, true, "Must set either test-version or local k8s dir when using deployment strategy 'gke'.")
@@ -243,7 +257,7 @@ func handle() error {
 		case "gce":
 			err = clusterUpGCE(k8sDir, *gceZone, *numNodes, *imageType)
 		case "gke":
-			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType)
+			err = clusterUpGKE(*gceZone, *gceRegion, *numNodes, *imageType, *useGKEManagedDriver)
 		default:
 			err = fmt.Errorf("deployment-strategy must be set to 'gce' or 'gke', but is: %s", *deploymentStrat)
 		}
@@ -272,21 +286,24 @@ func handle() error {
 		}()
 	}
 
-	// Install the driver and defer its teardown
-	err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
-	if *teardownDriver {
-		defer func() {
-			// TODO (#140): collect driver logs
-			if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
-				klog.Errorf("failed to delete driver: %v", teardownErr)
-			}
-		}()
-	}
-	if err != nil {
-		return fmt.Errorf("failed to install CSI Driver: %v", err)
+	if !*useGKEManagedDriver {
+		// Install the driver and defer its teardown
+		err := installDriver(goPath, pkgDir, *stagingImage, stagingVersion, *deployOverlayName, *doDriverBuild)
+		if *teardownDriver {
+			defer func() {
+				// TODO (#140): collect driver logs
+				if teardownErr := deleteDriver(goPath, pkgDir, *deployOverlayName); teardownErr != nil {
+					klog.Errorf("failed to delete driver: %v", teardownErr)
+				}
+			}()
+		}
+		if err != nil {
+			return fmt.Errorf("failed to install CSI Driver: %v", err)
+		}
 	}
 
 	var cloudProviderArgs []string
+	var err error
 	switch *deploymentStrat {
 	case "gke":
 		cloudProviderArgs, err = getGKEKubeTestArgs(*gceZone, *gceRegion, *imageType)
@@ -295,12 +312,20 @@ func handle() error {
 		}
 	}
 
-	normalizedVersion, err := getNormalizedVersion(*kubeVersion, *gkeClusterVer)
-	if err != nil {
-		return fmt.Errorf("failed to get cluster minor version: %v", err)
+	// Kubernetes version of GKE deployments are expected to be of the pattern x.y.z-gke.k,
+	// hence we use the main.Version utils to parse and compare GKE managed cluster versions.
+	// For clusters deployed on GCE, use the apimachinery version utils (which supports non-gke based semantic versioning).
+	clusterVersion := mustGetKubeClusterVersion()
+	var testSkip string
+	switch *deploymentStrat {
+	case "gce":
+		testSkip = generateGCETestSkip(clusterVersion)
+	case "gke":
+		testSkip = generateGKETestSkip(clusterVersion, *useGKEManagedDriver)
+	default:
+		return fmt.Errorf("Unknown deployment strategy %s", *deploymentStrat)
 	}
 
-	testSkip := generateTestSkip(normalizedVersion)
 	// Run the tests using the testDir kubernetes
 	if len(*storageClassFile) != 0 {
 		err = runCSITests(pkgDir, testDir, *testFocus, testSkip, *storageClassFile, *snapshotClassFile, cloudProviderArgs, *deploymentStrat)
@@ -317,30 +342,40 @@ func handle() error {
 	return nil
 }
 
-func generateTestSkip(normalizedVersion string) string {
+func generateGCETestSkip(clusterVersion string) string {
 	skipString := "\\[Disruptive\\]|\\[Serial\\]"
-	switch normalizedVersion {
-	// Fall-through versioning since all test cases we want to skip in 1.15
-	// should also be skipped in 1.14
-	case "1.13":
-		fallthrough
-	case "1.14":
-		fallthrough
-	case "1.15":
-		fallthrough
-	case "1.16":
-		// "volumeMode should not mount / map unused volumes in a pod" tests a
-		// bug-fix introduced in 1.17
-		// (https://github.com/kubernetes/kubernetes/pull/81163)
+	v := apimachineryversion.MustParseSemantic(clusterVersion)
+
+	// "volumeMode should not mount / map unused volumes in a pod" tests a
+	// (https://github.com/kubernetes/kubernetes/pull/81163)
+	// bug-fix introduced in 1.16
+	if v.LessThan(apimachineryversion.MustParseSemantic("1.16.0")) {
 		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
-		// Skip Snapshot tests pre 1.17
-		skipString = skipString + "|snapshot"
-		fallthrough
-	case "1.17":
-	case "latest":
-	case "master":
-	default:
 	}
+
+	if v.LessThan(apimachineryversion.MustParseSemantic("1.17.0")) {
+		skipString = skipString + "|VolumeSnapshotDataSource"
+	}
+	return skipString
+}
+
+func generateGKETestSkip(clusterVersion string, use_gke_managed_driver bool) string {
+	skipString := "\\[Disruptive\\]|\\[Serial\\]"
+	curVer := mustParseVersion(clusterVersion)
+
+	// "volumeMode should not mount / map unused volumes in a pod" tests a
+	// (https://github.com/kubernetes/kubernetes/pull/81163)
+	// bug-fix introduced in 1.16
+	if curVer.lessThan(mustParseVersion("1.16.0")) {
+		skipString = skipString + "|volumeMode\\sshould\\snot\\smount\\s/\\smap\\sunused\\svolumes\\sin\\sa\\spod"
+	}
+
+	// For GKE deployed PD CSI snapshot is enabled in 1.17.6-gke.4(and higher), 1.18.3-gke.0(and higher).
+	if (use_gke_managed_driver && curVer.lessThan(mustParseVersion("1.17.6-gke.4"))) ||
+		(!use_gke_managed_driver && (*curVer).lessThan(mustParseVersion("1.17.0"))) {
+		skipString = skipString + "|VolumeSnapshotDataSource"
+	}
+
 	return skipString
 }
 

diff --git a/test/k8s-integration/utils.go b/test/k8s-integration/utils.go
@@ -58,6 +58,19 @@ func ensureFlag(v *bool, setTo bool, msgOnError string) {
 	}
 }
 
+func ensureExactlyOneVariableSet(vars []*string, msgOnError string) {
+	var count int
+	for _, v := range vars {
+		if len(*v) != 0 {
+			count++
+		}
+	}
+
+	if count != 1 {
+		klog.Fatal(msgOnError)
+	}
+}
+
 func shredFile(filePath string) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		klog.V(4).Infof("File %v was not found, skipping shredding", filePath)
@@ -79,3 +92,13 @@ func shredFile(filePath string) {
 		klog.V(4).Infof("Failed to remove service account file %s: %v", filePath, err)
 	}
 }
+
+func ensureVariableVal(v *string, val string, msgOnError string) {
+	if *v != val {
+		klog.Fatal(msgOnError)
+	}
+}
+
+func isVariableSet(v *string) bool {
+	return len(*v) != 0
+}