diff --git a/Makefile b/Makefile index 4d8c61e3..d4f4862b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,11 @@ build: + $(info INFO: Starting build $@) go build cmd/commander/commander.go test: - go test ./... \ No newline at end of file + $(info INFO: Starting build $@) + go test ./... + +test-integration: build + $(info INFO: Starting build $@) + ./commander test \ No newline at end of file diff --git a/README.md b/README.md index 0a1adc05..507d816b 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ $ ./commander ./example/commander.yaml ``` ## Todo: + - go api + - logging / verbose output - command execution - environment variables - arguments? diff --git a/cmd/commander/commander.go b/cmd/commander/commander.go index 4de925a4..40f66f6b 100644 --- a/cmd/commander/commander.go +++ b/cmd/commander/commander.go @@ -3,10 +3,52 @@ package main import ( "github.com/SimonBaeumer/commander/pkg" "github.com/SimonBaeumer/commander/pkg/runtime" + "github.com/urfave/cli" + "io/ioutil" + "log" "os" ) +const ( + AppName = "Commander" + CommanderFile = "commander.yaml" +) + func main() { - suite := commander.ParseYAMLFile(os.Args[1]) - runtime.Start(suite) + log.SetOutput(ioutil.Discard) + + log.Println("Starting commander") + + app := cli.NewApp() + app.Name = AppName + + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "verbose", + Usage: "More output for debugging", + EnvVar: "COMMANDER_VERBOSE", + }, + } + + app.Commands = []cli.Command{ + { + Name: "test", + Usage: "Execute the test suite", + ArgsUsage: "[file]", + Action: func(c *cli.Context) { + log.Println("Starting test suite") + file := c.Args().First() + if file == "" { + file = CommanderFile + } + + suite := commander.ParseYAMLFile(file) + runtime.Start(suite) + }, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } } diff --git a/commander.yaml b/commander.yaml new file mode 100644 index 00000000..c7f85e25 --- /dev/null +++ b/commander.yaml @@ -0,0 +1,7 @@ +tests: + it should fail with invalid argument: + command: ./commander asfdf + exit-code: 3 + it should display help: + command: ./commander + exit-code: 0 \ No newline at end of file diff --git a/examples/commander.yaml b/examples/commander.yaml index b9cd8bd4..a18fd881 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -2,12 +2,4 @@ tests: "it should print hello world": command: "echo hello world" stdout: hello world - exit-code: 0 - "it should print something": - command: "echo soething" - stdout: something - exit-code: 0 - "more printing": - command: "echo soething" - stdout: something - exit-code: 1 \ No newline at end of file + exit-code: 0 \ No newline at end of file diff --git a/examples/simple_test.sh b/examples/simple_test.sh deleted file mode 100644 index 9323456c..00000000 --- a/examples/simple_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -@test "heyho" { - export ENV_TEST="heyho" - - run echo "hello world" - - assert $output "test" - assert $line[0] "test" - assert $output ./example.out - assert $status SUCCESS - assert $status 127 -} \ No newline at end of file diff --git a/go.mod b/go.mod index 20fa36e7..3323d718 100644 --- a/go.mod +++ b/go.mod @@ -2,5 +2,6 @@ module github.com/SimonBaeumer/commander require ( github.com/stretchr/testify v1.3.0 + github.com/urfave/cli v1.20.0 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 1f4f4c59..c24e6068 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/_fixtures/commander.yaml b/pkg/_fixtures/commander.yaml index a18fd881..882182b0 100644 --- a/pkg/_fixtures/commander.yaml +++ b/pkg/_fixtures/commander.yaml @@ -2,4 +2,4 @@ tests: "it should print hello world": command: "echo hello world" stdout: hello world - exit-code: 0 \ No newline at end of file + exit-code: 0 diff --git a/pkg/output/human_output.go b/pkg/output/human_output.go new file mode 100644 index 00000000..dcd9ee7a --- /dev/null +++ b/pkg/output/human_output.go @@ -0,0 +1,59 @@ +package output + +import ( + "fmt" + "log" +) + +type HumanOutput struct { + buffer []string +} + +func (o HumanOutput) BuildHeader() { + s := "Starting test suite...\n" + o.add(s) +} + +func (o HumanOutput) BuildTestResult(test TestCase) string { + var s string + + if test.Result.Success { + s = fmt.Sprintf("✓ %s", test.Title) + } else { + s = fmt.Sprintf("✗ %s\n", test.Title) + for _, p := range test.Result.FailureProperties { + log.Printf("Printing property result '%s'", p) + if p == "Stdout" { + s += fmt.Sprintf("Got '%s', expected '%s'\n", test.Result.Stdout, test.Stdout) + } + if p == "Stderr" { + s += fmt.Sprintf("Got %s, expected %s\n", test.Result.Stderr, test.Stderr) + } + if p == "ExitCode" { + s += fmt.Sprintf("Got %d, expected %d\n", test.Result.ExitCode, test.ExitCode) + } + } + } + + o.add(s) + return s +} + +func (o HumanOutput) BuildSuiteResult() { + s := "Duration: 3.24sec" + o.add(s) +} + +func (o HumanOutput) add(out string) { + o.buffer = append(o.buffer, out) +} + +func (o HumanOutput) GetBuffer() []string { + return o.buffer +} + +func (o HumanOutput) Print() { + for _, s := range o.GetBuffer() { + fmt.Println(s) + } +} diff --git a/pkg/output/output.go b/pkg/output/output.go new file mode 100644 index 00000000..f1e70098 --- /dev/null +++ b/pkg/output/output.go @@ -0,0 +1,14 @@ +package output + +import ( + "github.com/SimonBaeumer/commander/pkg" +) + +type TestCase commander.TestCase + +type Output interface { + BuildHeader() + BuildTestResult(test TestCase) + BuildSuiteResult() + GetBuffer() []string +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 02d9292e..36ba5bd7 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -4,9 +4,12 @@ import ( "bytes" "fmt" "github.com/SimonBaeumer/commander/pkg" + "github.com/SimonBaeumer/commander/pkg/output" "log" + "os" "os/exec" "strings" + "syscall" ) type Command struct { @@ -33,17 +36,14 @@ func Start(suite commander.Suite) { } func printResults(c chan commander.TestCase, suite commander.Suite) { + o := &output.HumanOutput{} counter := 0 for r := range c { - // Validate result - if !r.Result.Success { - fmt.Println("✗ ", r.Title) - } else { - fmt.Println("✓ ", r.Title) - } + s := o.BuildTestResult(output.TestCase(r)) + fmt.Println(s) counter++ - if (counter >= len(suite.GetTestCases())) { + if counter >= len(suite.GetTestCases()) { close(c) } } @@ -67,7 +67,7 @@ func runTest(test commander.TestCase, results chan<- commander.TestCase) { result := Validate(test) test.Result.Success = result.Success - test.Result.FailureProperty = result.Property + test.Result.FailureProperties = result.Properties // Send to result channel results <- test @@ -89,6 +89,8 @@ func compile(command string) *Command { // Execute executes a command on the system func (c *Command) Execute() error { cmd := exec.Command(c.cmd, c.args) + env := os.Environ() + cmd.Env = env var ( outBuff bytes.Buffer @@ -97,15 +99,25 @@ func (c *Command) Execute() error { cmd.Stdout = &outBuff cmd.Stderr = &errBuff - err := cmd.Run() + err := cmd.Start() + log.Println("Started command " + c.cmd) if err != nil { - return err + log.Println("Started command " + c.cmd + " err: " + err.Error()) + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + fmt.Println(exiterr) + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + c.exitCode = status.ExitStatus() + //log.Printf("Exit Status: %d", status.ExitStatus()) + } + } } else { c.exitCode = 0 } - - c.stderr = errBuff.String() - c.stdout = outBuff.String() + c.stderr = strings.Trim(errBuff.String(), "\n") + c.stdout = strings.Trim(outBuff.String(), "\n") return nil } diff --git a/pkg/runtime/validator.go b/pkg/runtime/validator.go index 004b9977..6f33cc45 100644 --- a/pkg/runtime/validator.go +++ b/pkg/runtime/validator.go @@ -3,45 +3,31 @@ package runtime import "github.com/SimonBaeumer/commander/pkg" type ValidationResult struct { - Success bool - Property string + Success bool + Properties []string } func Validate(test commander.TestCase) ValidationResult { - r := ValidationResult{} - - result := validateOutput(test.Stdout, test.Result.Stdout) - if !result { - r.Success = result - test.Result.FailureProperty = commander.Stdout + r := &ValidationResult{ + Success: true, + Properties: []string{}, } - result = validateOutput(test.Stderr, test.Result.Stderr) - if !result { - r.Success = result - r.Property = commander.Stderr + if test.Stdout != "" && (test.Stdout != test.Result.Stdout) { + r.Properties = append(r.Properties, commander.Stdout) } - result = validateExitCode(test.ExitCode, test.Result.ExitCode) - if !result { - r.Success = result - r.Property = commander.ExitCode + if test.Stderr != "" && (test.Stderr != test.Result.Stderr) { + r.Properties = append(r.Properties, commander.Stderr) } - r.Success = true - return r -} - -func validateOutput(expected string, actual string) bool { - if expected != actual { - return false + if test.ExitCode != test.Result.ExitCode { + r.Properties = append(r.Properties, commander.ExitCode) } - return true -} -func validateExitCode(expected int, actual int) bool { - if expected != actual { - return false + if len(r.Properties) > 0 { + r.Success = false } - return true -} \ No newline at end of file + + return *r +} diff --git a/pkg/runtime/validator_test.go b/pkg/runtime/validator_test.go new file mode 100644 index 00000000..70335244 --- /dev/null +++ b/pkg/runtime/validator_test.go @@ -0,0 +1,119 @@ +package runtime + +import ( + "github.com/SimonBaeumer/commander/pkg" + "github.com/stretchr/testify/assert" + "testing" +) + +const ( + SuccessCode = 0 + ErrorCode = 1 +) + +func Test_ValidateExitCodeFail(t *testing.T) { + test := commander.TestCase{} + test.ExitCode = SuccessCode + test.Result = commander.TestResult{ + ExitCode: ErrorCode, + } + + got := Validate(test) + + assert.False(t, got.Success) +} + +func Test_ValidateExitCodeSuccess(t *testing.T) { + test := commander.TestCase{} + test.ExitCode = SuccessCode + test.Result = commander.TestResult{ + ExitCode: SuccessCode, + } + + got := Validate(test) + + assert.True(t, got.Success) +} + +func Test_ValidateStdoutFail(t *testing.T) { + test := commander.TestCase{} + test.Stdout = "foo" + test.Result = commander.TestResult{ + Stdout: "bar", + } + + got := Validate(test) + + assert.False(t, got.Success) +} + +func Test_ValidateStdoutSuccess(t *testing.T) { + test := commander.TestCase{} + test.Stdout = "foo" + test.Result = commander.TestResult{ + Stdout: "foo", + } + + got := Validate(test) + + assert.True(t, got.Success) +} + +func Test_ValidateStderrFail(t *testing.T) { + test := commander.TestCase{Stdout: "foo"} + test.Result = commander.TestResult{ + Stdout: "bar", + } + + got := Validate(test) + + assert.False(t, got.Success) +} + +func Test_ValidateStderrSuccess(t *testing.T) { + test := commander.TestCase{Stdout: "foo"} + test.Result = commander.TestResult{ + Stdout: "foo", + } + + got := Validate(test) + + assert.True(t, got.Success) +} + +func Test_Validate(t *testing.T) { + test := commander.TestCase{ + Stdout: "foo", + Stderr: "bar", + ExitCode: 0, + } + test.Result = commander.TestResult{ + Stdout: "foo", + Stderr: "bar", + ExitCode: 0, + } + + got := Validate(test) + + assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) +} + +func Test_ValidateFail(t *testing.T) { + test := commander.TestCase{ + Stdout: "fail", + Stderr: "fail", + ExitCode: 1, + } + test.Result = commander.TestResult{ + Stdout: "foo", + Stderr: "bar", + ExitCode: 0, + } + + got := Validate(test) + + + assert.False(t, got.Success) + assert.Equal(t, []string{"Stdout", "Stderr", "ExitCode"}, got.Properties) +} \ No newline at end of file diff --git a/pkg/suite.go b/pkg/suite.go index 9586d932..702f07ad 100644 --- a/pkg/suite.go +++ b/pkg/suite.go @@ -20,10 +20,10 @@ type TestCase struct { type TestResult struct { Success bool //Skipped bool - Stdout string - Stderr string - ExitCode int - FailureProperty string + Stdout string + Stderr string + ExitCode int + FailureProperties []string } // Suite