From fb21a08299d15675caf10f2cf9966faf6933ce94 Mon Sep 17 00:00:00 2001 From: Michael Dresser Date: Wed, 17 Jun 2020 20:18:29 +0000 Subject: [PATCH] Dump logs for kubetest2 GCE deployer for cloud-provider-gcp Like the initial Up() and Down() commits, this only supports the k/cloud-provider/gcp repo. Handles checking and creation of logs directory and using kubectl and log-dump.sh to get logs. Opted to use the repo/cluster/kubectl.sh script instead of just calling the local kubectl for consistency. Had to add one environment var $USER for the reasons explained in the code comment. --- kubetest2-gce/deployer/BUILD.bazel | 1 + kubetest2-gce/deployer/deployer.go | 4 +- kubetest2-gce/deployer/dumplogs.go | 97 ++++++++++++++++++++++++++++++ kubetest2-gce/deployer/up.go | 12 ++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 kubetest2-gce/deployer/dumplogs.go diff --git a/kubetest2-gce/deployer/BUILD.bazel b/kubetest2-gce/deployer/BUILD.bazel index c71dc9b6..5b0c0840 100644 --- a/kubetest2-gce/deployer/BUILD.bazel +++ b/kubetest2-gce/deployer/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "build.go", "deployer.go", "down.go", + "dumplogs.go", "up.go", ], importpath = "k8s.io/test-infra/kubetest2/kubetest2-gce/deployer", diff --git a/kubetest2-gce/deployer/deployer.go b/kubetest2-gce/deployer/deployer.go index 9cf1b18d..6b173707 100644 --- a/kubetest2-gce/deployer/deployer.go +++ b/kubetest2-gce/deployer/deployer.go @@ -39,6 +39,7 @@ type deployer struct { kubeconfigPath string kubectl string + logsDir string RepoRoot string `desc:"The path to the root of the local kubernetes/cloud-provider-gcp repo. Necessary to call certain scripts. Defaults to the current directory. If operating in legacy mode, this should be set to the local kubernetes/kubernetes repo."` GCPProject string `desc:"GCP Project to create VMs in. Must be set."` } @@ -48,6 +49,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { d := &deployer{ commonOptions: opts, kubeconfigPath: filepath.Join(opts.ArtifactsDir(), "kubetest2-kubeconfig"), + logsDir: filepath.Join(opts.ArtifactsDir(), "cluster-logs"), } flagSet, err := gpflag.Parse(d) @@ -96,8 +98,6 @@ func (d *deployer) IsUp() (up bool, err error) { return len(lines) > 0, nil } -func (d *deployer) DumpClusterLogs() error { return nil } - func (d *deployer) Kubeconfig() (string, error) { _, err := os.Stat(d.kubeconfigPath) if err == nil { diff --git a/kubetest2-gce/deployer/dumplogs.go b/kubetest2-gce/deployer/dumplogs.go new file mode 100644 index 00000000..7e4f9b9e --- /dev/null +++ b/kubetest2-gce/deployer/dumplogs.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/klog" + "k8s.io/test-infra/kubetest2/pkg/exec" +) + +func (d *deployer) DumpClusterLogs() error { + klog.Info("GCE deployer starting DumpClusterLogs()") + + if err := d.verifyFlags(); err != nil { + return fmt.Errorf("dump cluster logs could not verify flags: %s", err) + } + + if err := d.makeLogsDir(); err != nil { + return fmt.Errorf("couldn't make logs dir: %s", err) + } + + env := d.buildEnv() + outfile, err := os.Create(filepath.Join(d.logsDir, "cluster-info.log")) + if err != nil { + return fmt.Errorf("failed to create cluster-info log file: %s", err) + } + defer outfile.Close() + + command := []string{ + d.kubectl, + "cluster-info", + "dump", + } + klog.Infof("About to run: %s", command) + + cmd := exec.Command(command[0], command[1:]...) + cmd.SetEnv(env...) + exec.InheritOutput(cmd) + cmd.SetStderr(os.Stderr) + cmd.SetStdout(outfile) + err = cmd.Run() + if err != nil { + return fmt.Errorf("couldn't use kubectl to dump cluster info: %s", err) + } + + env = append(env, "KUBE_GCE_INSTANCE_PREFIX=kubernetes") + + command = []string{ + filepath.Join(d.RepoRoot, "cluster", "log-dump", "log-dump.sh"), + d.logsDir, + } + klog.Infof("About to run: %s", command) + + cmd = exec.Command(command[0], command[1:]...) + cmd.SetEnv(env...) + exec.InheritOutput(cmd) + err = cmd.Run() + if err != nil { + return fmt.Errorf("failed to use log-dump.sh for cluster logs: %s", err) + } + + return nil +} + +func (d *deployer) makeLogsDir() error { + _, err := os.Stat(d.logsDir) + + if err == nil { + // TODO: removeall instead? + return fmt.Errorf("cluster logs directory %s already exists, please clean up manually before continuing", d.logsDir) + } else if os.IsNotExist(err) { + err := os.Mkdir(d.logsDir, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create %s: %s", d.logsDir, err) + } + return nil + } + + return fmt.Errorf("unexpected exception when making cluster logs directory: %s", err) +} diff --git a/kubetest2-gce/deployer/up.go b/kubetest2-gce/deployer/up.go index 73a203b7..aef949ca 100644 --- a/kubetest2-gce/deployer/up.go +++ b/kubetest2-gce/deployer/up.go @@ -36,6 +36,12 @@ func (d *deployer) Up() error { return fmt.Errorf("up couldn't enable compute API: %s", err) } + defer func() { + if err := d.DumpClusterLogs(); err != nil { + klog.Warningf("Dumping cluster logs at the end of Up() failed: %s", err) + } + }() + env := d.buildEnv() script := filepath.Join(d.RepoRoot, "cluster", "kube-up.sh") klog.Infof("About to run script at: %s", script) @@ -70,8 +76,14 @@ func (d *deployer) buildEnv() []string { var env []string // path is necessary for scripts to find gsutil, gcloud, etc + // can be removed if env is inherited from the os env = append(env, fmt.Sprintf("PATH=%s", os.Getenv("PATH"))) + // used by config-test.sh to set $NETWORK in the default case + // if unset, bash's set -u gets angry and kills the log dump script + // can be removed if env is inherited from the os + env = append(env, fmt.Sprintf("USER=%s", os.Getenv("USER"))) + // kube-up.sh, kube-down.sh etc. use PROJECT as a parameter for all gcloud commands env = append(env, fmt.Sprintf("PROJECT=%s", d.GCPProject))