Specifying alpine image version in 'Integration-linux' test

playing around WIP: this works WIP: TODO handle single test command input WIP: general pattern finished, need to refactor and optimize and error handle WIP: handling error. TODO: push rec downstream Finished bulk test, would be nice to parse out by file
commander-cli · Apr 19, 2020 · 0a98ef4 · 0a98ef4
1 parent 36ed967
commit 0a98ef4
Show file tree

Hide file tree

Showing 12 changed files with 189 additions and 27 deletions.
diff --git a/examples/commander.yaml b/examples/commander.yaml
@@ -19,4 +19,4 @@ tests:
   it should fail:
     command: invalid
     stderr: "/bin/sh: 1: invalid: not found"
-    exit-code: 127
+    exit-code: 127
diff --git a/examples/environment.yaml b/examples/environment.yaml
@@ -12,4 +12,4 @@ tests:
     config:
       env:
         KEY: local
-    exit-code: 0
+    exit-code: 0
diff --git a/examples/minimal_test.yaml b/examples/minimal_test.yaml
@@ -1,3 +1,3 @@
 tests:
   echo hello:
-    exit-code: 0
+    exit-code: 0
diff --git a/go.mod298498081.tmp b/go.mod298498081.tmp
@@ -0,0 +1,27 @@
+module github.com/SimonBaeumer/commander
+
+require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/SimonBaeumer/cmd v1.1.0
+	github.com/antchfx/xmlquery v1.1.0
+	github.com/antchfx/xpath v1.1.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tidwall/gjson v1.3.2
+	github.com/urfave/cli v1.20.0
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v2 v2.2.4
+)
+
+go 1.13
diff --git a/go.mod427131847.tmp b/go.mod427131847.tmp
@@ -0,0 +1,27 @@
+module github.com/SimonBaeumer/commander
+
+require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/SimonBaeumer/cmd v1.1.0
+	github.com/antchfx/xmlquery v1.1.0
+	github.com/antchfx/xpath v1.1.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tidwall/gjson v1.3.2
+	github.com/urfave/cli v1.20.0
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v2 v2.2.4
+)
+
+go 1.13
diff --git a/go.mod939984059.tmp b/go.mod939984059.tmp
@@ -0,0 +1,27 @@
+module github.com/SimonBaeumer/commander
+
+require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/SimonBaeumer/cmd v1.1.0
+	github.com/antchfx/xmlquery v1.1.0
+	github.com/antchfx/xpath v1.1.0 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.4.0
+	github.com/tidwall/gjson v1.3.2
+	github.com/urfave/cli v1.20.0
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v2 v2.2.4
+)
+
+go 1.13
diff --git a/integration/linux/docker.yaml b/integration/linux/docker.yaml
@@ -20,4 +20,4 @@ tests:
         BUG_REPORT_URL="https://bugs.alpinelinux.org/"
 
   "id -u":
-    stdout: "1001"
+    stdout: "1001"
diff --git a/pkg/app/test_command.go b/pkg/app/test_command.go
@@ -2,53 +2,110 @@ package app
 
 import (
 	"fmt"
-	"github.com/SimonBaeumer/commander/pkg/output"
-	"github.com/SimonBaeumer/commander/pkg/runtime"
-	"github.com/SimonBaeumer/commander/pkg/suite"
 	"io/ioutil"
 	"log"
 	"os"
+	"sync"
+
+	"github.com/SimonBaeumer/commander/pkg/output"
+	"github.com/SimonBaeumer/commander/pkg/runtime"
+	"github.com/SimonBaeumer/commander/pkg/suite"
 )
 
 // TestCommand executes the test argument
 // file is the path to the configuration file
 // title ist the title of test which should be executed, if empty it will execute all tests
 // ctx holds the command flags
-func TestCommand(file string, title string, ctx AddCommandContext) error {
+func TestCommand(input string, title string, ctx AddCommandContext) error {
 	if ctx.Verbose == true {
 		log.SetOutput(os.Stdout)
 	}
 
-	if file == "" {
-		file = CommanderFile
+	out := output.NewCliOutput(!ctx.NoColor)
+
+	if input == "" {
+		input = CommanderFile
+	}
+	inputType, err := os.Stat(input)
+	if err != nil {
+		return fmt.Errorf("Error " + err.Error())
+	}
+
+	var results <-chan runtime.TestResult
+	if inputType.IsDir() {
+		fmt.Println("Starting test against directory: " + input + "...")
+		fmt.Println("")
+
+		results, err = testDir(input, title)
+
+	} else {
+		fmt.Println("Starting test file " + input + "...")
+		fmt.Println("")
+		results, err = testFile(input, title)
 	}
 
-	fmt.Println("Starting test file " + file + "...")
-	fmt.Println("")
-	content, err := ioutil.ReadFile(file)
 	if err != nil {
 		return fmt.Errorf("Error " + err.Error())
 	}
 
+	if !out.Start(results) {
+		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")
+	}
+
+	return nil
+}
+
+func testDir(directory string, title string) (<-chan runtime.TestResult, error) {
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, fmt.Errorf("Error " + err.Error())
+	}
+	results := make(chan runtime.TestResult)
+
+	var wg sync.WaitGroup
+	for _, f := range files {
+		wg.Add(1)
+		go func(f os.FileInfo) {
+			defer wg.Done()
+			fileResults, err := testFile(directory+"/"+f.Name(), title)
+			if err != nil {
+				results <- runtime.TestResult{FileError: err}
+				return
+			}
+			for bb := range fileResults {
+				results <- bb //fan in results
+			}
+		}(f)
+	}
+
+	go func(ch chan runtime.TestResult) {
+		wg.Wait()
+		close(results)
+	}(results)
+
+	return results, nil
+}
+
+func testFile(input string, title string) (<-chan runtime.TestResult, error) {
+	content, err := ioutil.ReadFile(input)
+	if err != nil {
+		return nil, fmt.Errorf("Error " + err.Error())
+	}
+
 	var s suite.Suite
 	s = suite.ParseYAML(content)
 	tests := s.GetTests()
 	// Filter tests if test title was given
 	if title != "" {
 		test, err := s.GetTestByTitle(title)
 		if err != nil {
-			return err
+			return nil, fmt.Errorf("Error " + err.Error())
 		}
 		tests = []runtime.TestCase{test}
 	}
 
 	r := runtime.NewRuntime(s.Nodes...)
-
 	results := r.Start(tests)
-	out := output.NewCliOutput(!ctx.NoColor)
-	if !out.Start(results) {
-		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")
-	}
 
-	return nil
+	return results, nil
 }
diff --git a/pkg/app/test_command_test.go b/pkg/app/test_command_test.go
@@ -2,13 +2,14 @@ package app
 
 import (
 	"bytes"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"log"
 	"os"
 	"runtime"
 	"sync"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_TestCommand_Verbose(t *testing.T) {
@@ -31,7 +32,7 @@ func Test_TestCommand(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		assert.Contains(t, err.Error(), "Error open commander.yaml:")
 	} else {
-		assert.Equal(t, "Error open commander.yaml: no such file or directory", err.Error())
+		assert.Equal(t, "Error stat commander.yaml: no such file or directory", err.Error())
 	}
 }
 
@@ -41,7 +42,17 @@ func Test_TestCommand_ShouldUseCustomFile(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		assert.Contains(t, err.Error(), "Error open my-test.yaml: ")
 	} else {
-		assert.Equal(t, "Error open my-test.yaml: no such file or directory", err.Error())
+		assert.Equal(t, "Error stat my-test.yaml: no such file or directory", err.Error())
+	}
+}
+
+func Test_TestCommand_Dir(t *testing.T) {
+	err := TestCommand("../../examples", "echo hello", AddCommandContext{})
+
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "Error open my-test.yaml: ")
+	} else {
+		assert.Equal(t, "Error stat my-test.yaml: no such file or directory", err.Error())
 	}
 }
 

diff --git a/pkg/output/cli.go b/pkg/output/cli.go
@@ -35,11 +35,16 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 		au = aurora.NewAurora(false)
 	}
 
+	fileErrors := make([]error, 0)
 	failed := 0
 	testResults := []runtime.TestResult{}
 	start := time.Now()
-
 	for r := range results {
+		if r.FileError != nil {
+			fileErrors = append(fileErrors, r.FileError)
+			continue
+		}
+
 		testResults = append(testResults, r)
 		if r.ValidationResult.Success {
 			str := fmt.Sprintf("✓ [%s] %s", r.Node, r.TestCase.Title)
@@ -63,6 +68,11 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 		w.printFailures(testResults)
 	}
 
+	w.fprintf("")
+	for _, e := range fileErrors {
+		fmt.Println(e)
+	}
+
 	w.fprintf("")
 	w.fprintf(fmt.Sprintf("Duration: %.3fs", duration.Seconds()))
 	summary := fmt.Sprintf("Count: %d, Failed: %d", len(testResults), failed)

diff --git a/pkg/output/cli_test.go b/pkg/output/cli_test.go
@@ -3,11 +3,12 @@ package output
 import (
 	"bytes"
 	"fmt"
-	"github.com/SimonBaeumer/commander/pkg/runtime"
-	"github.com/stretchr/testify/assert"
 	"strings"
 	"sync"
 	"testing"
+
+	"github.com/SimonBaeumer/commander/pkg/runtime"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_NewCliOutput(t *testing.T) {

diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go
@@ -2,12 +2,13 @@ package runtime
 
 import (
 	"fmt"
-	"github.com/SimonBaeumer/cmd"
 	"log"
 	"os"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/SimonBaeumer/cmd"
 )
 
 // Constants for defining the various tested properties
@@ -132,6 +133,7 @@ type TestResult struct {
 	Tries            int
 	Node             string
 	Error            error
+	FileError        error
 }
 
 // Start starts the given test suite and executes all tests