Add exit-code validation

Makefile

@@ -1,5 +1,11 @@
 build:
+	$(info INFO: Starting build $@)
 	go build cmd/commander/commander.go
 
 test:
-	go test ./...
+	$(info INFO: Starting build $@)
+	go test ./...
+
+test-integration: build
+	$(info INFO: Starting build $@)
+	./commander test

README.md

@@ -13,6 +13,8 @@ $ ./commander ./example/commander.yaml
 ```
 
 ## Todo:
+ - go api
+ - logging / verbose output
  - command execution
    - environment variables
    - arguments?

cmd/commander/commander.go

@@ -3,10 +3,52 @@ package main
 import (
     "github.com/SimonBaeumer/commander/pkg"
     "github.com/SimonBaeumer/commander/pkg/runtime"
+    "github.com/urfave/cli"
+    "io/ioutil"
+    "log"
     "os"
 )
 
+const (
+    AppName = "Commander"
+    CommanderFile = "commander.yaml"
+)
+
 func main() {
-    suite := commander.ParseYAMLFile(os.Args[1])
-    runtime.Start(suite)
+    log.SetOutput(ioutil.Discard)
+
+    log.Println("Starting commander")
+
+    app := cli.NewApp()
+    app.Name = AppName
+
+    app.Flags = []cli.Flag{
+        cli.BoolFlag{
+            Name: "verbose",
+            Usage: "More output for debugging",
+            EnvVar: "COMMANDER_VERBOSE",
+        },
+    }
+
+    app.Commands = []cli.Command{
+        {
+            Name: "test",
+            Usage: "Execute the test suite",
+            ArgsUsage: "[file]",
+            Action: func(c *cli.Context) {
+                log.Println("Starting test suite")
+                file := c.Args().First()
+                if file == "" {
+                    file = CommanderFile
+                }
+
+                suite := commander.ParseYAMLFile(file)
+                runtime.Start(suite)
+            },
+        },
+    }
+
+    if err := app.Run(os.Args); err != nil {
+        log.Fatal(err)
+    }
 }

commander.yaml

@@ -0,0 +1,7 @@
+tests:
+  it should fail with invalid argument:
+    command: ./commander asfdf
+    exit-code: 3
+  it should display help:
+    command: ./commander
+    exit-code: 0

examples/commander.yaml

@@ -2,12 +2,4 @@ tests:
   "it should print hello world":
     command: "echo hello world"
     stdout: hello world
-    exit-code: 0
-  "it should print something":
-    command: "echo soething"
-    stdout: something
-    exit-code: 0
-  "more printing":
-    command: "echo soething"
-    stdout: something
-    exit-code: 1
+    exit-code: 0

go.mod

@@ -2,5 +2,6 @@ module github.com/SimonBaeumer/commander
 
 require (
 	github.com/stretchr/testify v1.3.0
+	github.com/urfave/cli v1.20.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -5,6 +5,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/_fixtures/commander.yaml

@@ -2,4 +2,4 @@ tests:
   "it should print hello world":
     command: "echo hello world"
     stdout: hello world
-    exit-code: 0
+    exit-code: 0

pkg/output/human_output.go

@@ -0,0 +1,59 @@
+package output
+
+import (
+    "fmt"
+    "log"
+)
+
+type HumanOutput struct {
+    buffer []string
+}
+
+func (o HumanOutput) BuildHeader() {
+    s := "Starting test suite...\n"
+    o.add(s)
+}
+
+func (o HumanOutput) BuildTestResult(test TestCase) string {
+    var s string
+
+    if test.Result.Success {
+        s = fmt.Sprintf("✓ %s", test.Title)
+    } else {
+        s = fmt.Sprintf("✗ %s\n", test.Title)
+        for _, p := range test.Result.FailureProperties {
+            log.Printf("Printing property result '%s'", p)
+            if p == "Stdout" {
+                s +=  fmt.Sprintf("Got '%s', expected '%s'\n", test.Result.Stdout, test.Stdout)
+            }
+            if p == "Stderr" {
+                s += fmt.Sprintf("Got %s, expected %s\n", test.Result.Stderr, test.Stderr)
+            }
+            if p == "ExitCode" {
+                s += fmt.Sprintf("Got %d, expected %d\n", test.Result.ExitCode, test.ExitCode)
+            }
+        }
+    }
+
+    o.add(s)
+    return s
+}
+
+func (o HumanOutput) BuildSuiteResult() {
+    s := "Duration: 3.24sec"
+    o.add(s)
+}
+
+func (o HumanOutput) add(out string) {
+    o.buffer = append(o.buffer, out)
+}
+
+func (o HumanOutput) GetBuffer() []string {
+    return o.buffer
+}
+
+func (o HumanOutput) Print() {
+    for _, s := range o.GetBuffer() {
+        fmt.Println(s)
+    }
+}

pkg/output/output.go

@@ -0,0 +1,14 @@
+package output
+
+import (
+    "github.com/SimonBaeumer/commander/pkg"
+)
+
+type TestCase commander.TestCase
+
+type Output interface {
+    BuildHeader()
+    BuildTestResult(test TestCase)
+    BuildSuiteResult()
+    GetBuffer() []string
+}

pkg/runtime/runtime.go

@@ -4,9 +4,12 @@ import (
     "bytes"
     "fmt"
     "github.com/SimonBaeumer/commander/pkg"
+    "github.com/SimonBaeumer/commander/pkg/output"
     "log"
+    "os"
     "os/exec"
     "strings"
+    "syscall"
 )
 
 type Command struct {
@@ -33,17 +36,14 @@ func Start(suite commander.Suite) {
 }
 
 func printResults(c chan commander.TestCase, suite commander.Suite) {
+    o := &output.HumanOutput{}
     counter := 0
     for r := range c {
-        // Validate result
-        if !r.Result.Success {
-            fmt.Println("✗ ", r.Title)
-        } else {
-            fmt.Println("✓ ", r.Title)
-        }
+        s := o.BuildTestResult(output.TestCase(r))
+        fmt.Println(s)
 
         counter++
-        if (counter >= len(suite.GetTestCases())) {
+        if counter >= len(suite.GetTestCases()) {
             close(c)
         }
     }
@@ -67,7 +67,7 @@ func runTest(test commander.TestCase, results chan<- commander.TestCase) {
 
     result := Validate(test)
     test.Result.Success = result.Success
-    test.Result.FailureProperty = result.Property
+    test.Result.FailureProperties = result.Properties
 
     // Send to result channel
     results <- test
@@ -89,6 +89,8 @@ func compile(command string) *Command {
 // Execute executes a command on the system
 func (c *Command) Execute() error {
     cmd := exec.Command(c.cmd, c.args)
+    env := os.Environ()
+    cmd.Env = env
 
     var (
         outBuff bytes.Buffer
@@ -97,15 +99,25 @@ func (c *Command) Execute() error {
     cmd.Stdout = &outBuff
     cmd.Stderr = &errBuff
 
-    err := cmd.Run()
+    err := cmd.Start()
+    log.Println("Started command " + c.cmd)
     if err != nil {
-        return err
+        log.Println("Started command " + c.cmd + " err: " + err.Error())
+    }
+
+    if err := cmd.Wait(); err != nil {
+        if exiterr, ok := err.(*exec.ExitError); ok {
+            fmt.Println(exiterr)
+            if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+                c.exitCode = status.ExitStatus()
+                //log.Printf("Exit Status: %d", status.ExitStatus())
+            }
+        }
     } else {
         c.exitCode = 0
     }
-
-    c.stderr = errBuff.String()
-    c.stdout = outBuff.String()
+    c.stderr = strings.Trim(errBuff.String(), "\n")
+    c.stdout = strings.Trim(outBuff.String(), "\n")
 
     return nil
 }

pkg/runtime/validator.go

@@ -3,45 +3,31 @@ package runtime
 import "github.com/SimonBaeumer/commander/pkg"
 
 type ValidationResult struct {
-    Success  bool
-    Property string
+    Success    bool
+    Properties []string
 }
 
 func Validate(test commander.TestCase) ValidationResult {
-    r := ValidationResult{}
-
-    result := validateOutput(test.Stdout, test.Result.Stdout)
-    if !result {
-        r.Success = result
-        test.Result.FailureProperty = commander.Stdout
+    r := &ValidationResult{
+        Success: true,
+        Properties: []string{},
     }
 
-    result = validateOutput(test.Stderr, test.Result.Stderr)
-    if !result {
-        r.Success = result
-        r.Property = commander.Stderr
+    if test.Stdout != "" && (test.Stdout != test.Result.Stdout) {
+        r.Properties = append(r.Properties, commander.Stdout)
     }
 
-    result = validateExitCode(test.ExitCode, test.Result.ExitCode)
-    if !result {
-        r.Success = result
-        r.Property = commander.ExitCode
+    if test.Stderr != "" && (test.Stderr != test.Result.Stderr) {
+        r.Properties = append(r.Properties, commander.Stderr)
     }
 
-    r.Success = true
-    return r
-}
-
-func validateOutput(expected string, actual string) bool {
-    if expected != actual {
-        return false
+    if test.ExitCode != test.Result.ExitCode {
+        r.Properties = append(r.Properties, commander.ExitCode)
     }
-    return true
-}
 
-func validateExitCode(expected int, actual int) bool {
-    if expected != actual {
-        return false
+    if len(r.Properties) > 0 {
+        r.Success = false
     }
-    return true
-}
+
+    return *r
+}