From 6152161994477c0d5c6d61ccf1f843777f491912 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 23 Nov 2018 19:26:02 +0100 Subject: [PATCH 1/4] Compress docker artifacts with gzip --- dev-tools/mage/dockerbuilder.go | 36 +++++++++++++++++++++++++++-- dev-tools/packaging/package_test.go | 10 ++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 24baefd1eafb..b3b5204cf8a0 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -18,8 +18,11 @@ package mage import ( + "compress/gzip" "fmt" + "io" "os" + "os/exec" "path/filepath" "strings" @@ -141,13 +144,42 @@ func (b *dockerBuilder) dockerSave(tag string) error { // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { - outputTar, err := b.Expand(defaultBinaryName + ".docker.tar") + outputTar, err := b.Expand(defaultBinaryName + ".docker.gz") if err != nil { return err } outputFile = filepath.Join(distributionsDir, outputTar) } - if err := sh.Run("docker", "save", "-o", outputFile, tag); err != nil { + cmd := exec.Command("docker", "save", tag) + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err = cmd.Start(); err != nil { + return err + } + + err = func() error { + f, err := os.Create(outputFile) + if err != nil { + return err + } + defer f.Close() + + w := gzip.NewWriter(f) + defer w.Close() + + _, err = io.Copy(w, stdout) + if err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + + if err = cmd.Wait(); err != nil { return err } return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file") diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index d447e22ade73..27f2a8a1ebdc 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -94,7 +94,7 @@ func TestZip(t *testing.T) { } func TestDocker(t *testing.T) { - dockers := getFiles(t, regexp.MustCompile(`\.docker.tar$`)) + dockers := getFiles(t, regexp.MustCompile(`\.docker.gz$`)) for _, docker := range dockers { checkDocker(t, docker) } @@ -514,7 +514,13 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { var info *dockerInfo layers := make(map[string]*packageFile) - tarReader := tar.NewReader(file) + gzipReader, err := gzip.NewReader(file) + if err != nil { + return nil, nil, err + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() if err != nil { From 0808336765cacab4329d5aafe8dcbb5c2baf9f47 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 24 Nov 2018 13:16:50 +0100 Subject: [PATCH 2/4] Return stderr as part of the error on docker save failure --- dev-tools/mage/dockerbuilder.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index b3b5204cf8a0..36583afb152f 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -18,6 +18,8 @@ package mage import ( + "bufio" + "bytes" "compress/gzip" "fmt" "io" @@ -150,7 +152,9 @@ func (b *dockerBuilder) dockerSave(tag string) error { } outputFile = filepath.Join(distributionsDir, outputTar) } + var stderr bytes.Buffer cmd := exec.Command("docker", "save", tag) + cmd.Stderr = bufio.NewWriter(&stderr) stdout, err := cmd.StdoutPipe() if err != nil { return err @@ -180,6 +184,9 @@ func (b *dockerBuilder) dockerSave(tag string) error { } if err = cmd.Wait(); err != nil { + if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" { + err = errors.Wrap(errors.New(errmsg), err.Error()) + } return err } return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file") From 2b1cafdabfbfb99dcf32e4255317ad89551c46e5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 24 Nov 2018 13:35:40 +0100 Subject: [PATCH 3/4] Use tar.gz as extension for docker artifacts --- dev-tools/mage/dockerbuilder.go | 2 +- dev-tools/packaging/package_test.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 36583afb152f..90e72ba2cc34 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -146,7 +146,7 @@ func (b *dockerBuilder) dockerSave(tag string) error { // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { - outputTar, err := b.Expand(defaultBinaryName + ".docker.gz") + outputTar, err := b.Expand(defaultBinaryName + ".docker.tar.gz") if err != nil { return err } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index 27f2a8a1ebdc..cea1e9e9b69b 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -80,7 +80,8 @@ func TestDeb(t *testing.T) { } func TestTar(t *testing.T) { - tars := getFiles(t, regexp.MustCompile(`\.tar\.gz$`)) + // Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz + tars := getFiles(t, regexp.MustCompile(`-\w+\.tar\.gz$`)) for _, tar := range tars { checkTar(t, tar) } @@ -94,7 +95,7 @@ func TestZip(t *testing.T) { } func TestDocker(t *testing.T) { - dockers := getFiles(t, regexp.MustCompile(`\.docker.gz$`)) + dockers := getFiles(t, regexp.MustCompile(`\.docker\.tar\.gz$`)) for _, docker := range dockers { checkDocker(t, docker) } From 4185211d1f5bfd100f1cf0c28182b50546c04cdf Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 24 Nov 2018 14:21:40 +0100 Subject: [PATCH 4/4] Remove not needed bufio writer --- dev-tools/mage/dockerbuilder.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 90e72ba2cc34..44791010217e 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -18,7 +18,6 @@ package mage import ( - "bufio" "bytes" "compress/gzip" "fmt" @@ -154,7 +153,7 @@ func (b *dockerBuilder) dockerSave(tag string) error { } var stderr bytes.Buffer cmd := exec.Command("docker", "save", tag) - cmd.Stderr = bufio.NewWriter(&stderr) + cmd.Stderr = &stderr stdout, err := cmd.StdoutPipe() if err != nil { return err