Dump logs for kubetest2 GCE deployer for cloud-provider-gcp

Like the initial Up() and Down() commits, this only supports the
k/cloud-provider/gcp repo. Handles checking and creation of logs
directory and using kubectl and log-dump.sh to get logs. Opted
to use the repo/cluster/kubectl.sh script instead of just calling
the local kubectl for consistency. Had to add one environment var
$USER for the reasons explained in the code comment.

kubetest2-gce/deployer/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "build.go",
         "deployer.go",
         "down.go",
+        "dumplogs.go",
         "up.go",
     ],
     importpath = "k8s.io/test-infra/kubetest2/kubetest2-gce/deployer",

kubetest2-gce/deployer/deployer.go

@@ -39,6 +39,7 @@ type deployer struct {
 
 	kubeconfigPath string
 	kubectl        string
+	logsDir        string
 	RepoRoot       string `desc:"The path to the root of the local kubernetes/cloud-provider-gcp repo. Necessary to call certain scripts. Defaults to the current directory. If operating in legacy mode, this should be set to the local kubernetes/kubernetes repo."`
 	GCPProject     string `desc:"GCP Project to create VMs in. Must be set."`
 }
@@ -48,6 +49,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 	d := &deployer{
 		commonOptions:  opts,
 		kubeconfigPath: filepath.Join(opts.ArtifactsDir(), "kubetest2-kubeconfig"),
+		logsDir:        filepath.Join(opts.ArtifactsDir(), "cluster-logs"),
 	}
 
 	flagSet, err := gpflag.Parse(d)
@@ -96,8 +98,6 @@ func (d *deployer) IsUp() (up bool, err error) {
 	return len(lines) > 0, nil
 }
 
-func (d *deployer) DumpClusterLogs() error { return nil }
-
 func (d *deployer) Kubeconfig() (string, error) {
 	_, err := os.Stat(d.kubeconfigPath)
 	if err == nil {

kubetest2-gce/deployer/dumplogs.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"k8s.io/klog"
+	"k8s.io/test-infra/kubetest2/pkg/exec"
+)
+
+func (d *deployer) DumpClusterLogs() error {
+	klog.Info("GCE deployer starting DumpClusterLogs()")
+
+	if err := d.verifyFlags(); err != nil {
+		return fmt.Errorf("dump cluster logs could not verify flags: %s", err)
+	}
+
+	if err := d.makeLogsDir(); err != nil {
+		return fmt.Errorf("couldn't make logs dir: %s", err)
+	}
+
+	env := d.buildEnv()
+	outfile, err := os.Create(filepath.Join(d.logsDir, "cluster-info.log"))
+	if err != nil {
+		return fmt.Errorf("failed to create cluster-info log file: %s", err)
+	}
+	defer outfile.Close()
+
+	command := []string{
+		d.kubectl,
+		"cluster-info",
+		"dump",
+	}
+	klog.Infof("About to run: %s", command)
+
+	cmd := exec.Command(command[0], command[1:]...)
+	cmd.SetEnv(env...)
+	exec.InheritOutput(cmd)
+	cmd.SetStderr(os.Stderr)
+	cmd.SetStdout(outfile)
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("couldn't use kubectl to dump cluster info: %s", err)
+	}
+
+	env = append(env, "KUBE_GCE_INSTANCE_PREFIX=kubernetes")
+
+	command = []string{
+		filepath.Join(d.RepoRoot, "cluster", "log-dump", "log-dump.sh"),
+		d.logsDir,
+	}
+	klog.Infof("About to run: %s", command)
+
+	cmd = exec.Command(command[0], command[1:]...)
+	cmd.SetEnv(env...)
+	exec.InheritOutput(cmd)
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("failed to use log-dump.sh for cluster logs: %s", err)
+	}
+
+	return nil
+}
+
+func (d *deployer) makeLogsDir() error {
+	_, err := os.Stat(d.logsDir)
+
+	if err == nil {
+		// TODO: removeall instead?
+		return fmt.Errorf("cluster logs directory %s already exists, please clean up manually before continuing", d.logsDir)
+	} else if os.IsNotExist(err) {
+		err := os.Mkdir(d.logsDir, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("failed to create %s: %s", d.logsDir, err)
+		}
+		return nil
+	}
+
+	return fmt.Errorf("unexpected exception when making cluster logs directory: %s", err)
+}

kubetest2-gce/deployer/up.go

@@ -36,6 +36,12 @@ func (d *deployer) Up() error {
 		return fmt.Errorf("up couldn't enable compute API: %s", err)
 	}
 
+	defer func() {
+		if err := d.DumpClusterLogs(); err != nil {
+			klog.Warningf("Dumping cluster logs at the end of Up() failed: %s", err)
+		}
+	}()
+
 	env := d.buildEnv()
 	script := filepath.Join(d.RepoRoot, "cluster", "kube-up.sh")
 	klog.Infof("About to run script at: %s", script)
@@ -70,8 +76,14 @@ func (d *deployer) buildEnv() []string {
 	var env []string
 
 	// path is necessary for scripts to find gsutil, gcloud, etc
+	// can be removed if env is inherited from the os
 	env = append(env, fmt.Sprintf("PATH=%s", os.Getenv("PATH")))
 
+	// used by config-test.sh to set $NETWORK in the default case
+	// if unset, bash's set -u gets angry and kills the log dump script
+	// can be removed if env is inherited from the os
+	env = append(env, fmt.Sprintf("USER=%s", os.Getenv("USER")))
+
 	// kube-up.sh, kube-down.sh etc. use PROJECT as a parameter for all gcloud commands
 	env = append(env, fmt.Sprintf("PROJECT=%s", d.GCPProject))