diff --git a/README.md b/README.md index f9cba6e5..5ed0081f 100644 --- a/README.md +++ b/README.md @@ -16,22 +16,28 @@ $ ./commander ./example/commander.yaml ``` ## Todo: - - go api + - suite fails -> error exit code - logging / verbose output + - print errors in colors + - execute a single test + + - go api - command execution - environment variables - arguments? - timeout - - exit code + - exit code *done* - stdout - - Validate against string + - Validate against string *done* - Validate against file - Validate against line - Validate with wildcards / regex + - Validate against template - stderr - - Validate against string + - Validate against string *done* - Validate against file - Validate with wildcards + - Validate against template - testing interactive applications? - Support different os - Windows diff --git a/commander.yaml b/commander.yaml index 8b7f6464..fd07d32a 100644 --- a/commander.yaml +++ b/commander.yaml @@ -2,11 +2,20 @@ tests: it should fail with invalid argument: command: ./commander asfdf exit-code: 3 + it should display help: command: ./commander exit-code: 0 + it should execute tests: - command: ./commander test ./examples/commander.yaml + command: ./commander test ./integration/commander_test.yaml + stdout: ✓ it should exit with error code + exit-code: 0 + + it should assert that commander will fail: + command: ./commander test ./integration/failing_suite.yaml stdout: >- - ✓ it should fail + ✗ it will fail + + Got 1, expected 0 exit-code: 0 \ No newline at end of file diff --git a/examples/commander.yaml b/examples/commander.yaml index 7ca4ef0c..f8a4a482 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -1,5 +1,8 @@ tests: - it should fail: - command: /bin/bash invalid - stderr: "/bin/bash: invalid: No such file or directory" - exit-code: 127 \ No newline at end of file + it should print hello world: + command: echo hello world + stdout: hello world + exit-code: 0 + it should exit with error code: + command: exit 1 + exit-code: 1 \ No newline at end of file diff --git a/integration/commander_test.yaml b/integration/commander_test.yaml new file mode 100644 index 00000000..08ef30fe --- /dev/null +++ b/integration/commander_test.yaml @@ -0,0 +1,19 @@ +tests: + it should fail: + command: /bin/bash invalid + stderr: "/bin/bash: invalid: No such file or directory" + exit-code: 127 + + it should exit with error code: + command: exit 1 + exit-code: 1 + + it should assert stdout: + command: echo hello + stdout: hello + exit-code: 0 + + it should assert stderr: + command: '>&2 echo "error"' + stderr: error + exit-code: 0 \ No newline at end of file diff --git a/integration/failing_suite.yaml b/integration/failing_suite.yaml new file mode 100644 index 00000000..a662cda3 --- /dev/null +++ b/integration/failing_suite.yaml @@ -0,0 +1,4 @@ +tests: + it will fail: + command: exit 1 + exit-code: 0 \ No newline at end of file diff --git a/pkg/runtime/validator.go b/pkg/runtime/validator.go index 49202568..3a13ce40 100644 --- a/pkg/runtime/validator.go +++ b/pkg/runtime/validator.go @@ -1,6 +1,9 @@ package runtime -import "github.com/SimonBaeumer/commander/pkg" +import ( + "github.com/SimonBaeumer/commander/pkg" + "strings" +) type ValidationResult struct { Success bool @@ -13,11 +16,11 @@ func Validate(test commander.TestCase) ValidationResult { Properties: []string{}, } - if test.Stdout != "" && (test.Stdout != test.Result.Stdout) { + if test.Stdout != "" && !strings.Contains(test.Result.Stdout, test.Stdout) { r.Properties = append(r.Properties, commander.Stdout) } - if test.Stderr != "" && (test.Stderr != test.Result.Stderr) { + if test.Stderr != "" && !strings.Contains(test.Result.Stderr, test.Stderr) { r.Properties = append(r.Properties, commander.Stderr) } diff --git a/pkg/runtime/validator_test.go b/pkg/runtime/validator_test.go index c50e45f9..1e4c4828 100644 --- a/pkg/runtime/validator_test.go +++ b/pkg/runtime/validator_test.go @@ -21,6 +21,7 @@ func Test_ValidateExitCodeFail(t *testing.T) { got := Validate(test) assert.False(t, got.Success) + assert.Len(t, got.Properties, 1) } func Test_ValidateExitCodeSuccess(t *testing.T) { @@ -33,6 +34,7 @@ func Test_ValidateExitCodeSuccess(t *testing.T) { got := Validate(test) assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) } func Test_ValidateStdoutFail(t *testing.T) { @@ -45,6 +47,7 @@ func Test_ValidateStdoutFail(t *testing.T) { got := Validate(test) assert.False(t, got.Success) + assert.Len(t, got.Properties, 1) } func Test_ValidateStdoutSuccess(t *testing.T) { @@ -57,6 +60,7 @@ func Test_ValidateStdoutSuccess(t *testing.T) { got := Validate(test) assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) } func Test_ValidateStderrFail(t *testing.T) { @@ -68,6 +72,7 @@ func Test_ValidateStderrFail(t *testing.T) { got := Validate(test) assert.False(t, got.Success) + assert.Len(t, got.Properties, 1) } func Test_ValidateStderrSuccess(t *testing.T) { @@ -79,18 +84,19 @@ func Test_ValidateStderrSuccess(t *testing.T) { got := Validate(test) assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) } func Test_Validate(t *testing.T) { test := commander.TestCase{ Stdout: "foo", Stderr: "bar", - ExitCode: 0, + ExitCode: SuccessCode, } test.Result = commander.TestResult{ Stdout: "foo", Stderr: "bar", - ExitCode: 0, + ExitCode: SuccessCode, } got := Validate(test) @@ -103,12 +109,12 @@ func Test_ValidateFail(t *testing.T) { test := commander.TestCase{ Stdout: "fail", Stderr: "fail", - ExitCode: 1, + ExitCode: ErrorCode, } test.Result = commander.TestResult{ Stdout: "foo", Stderr: "bar", - ExitCode: 0, + ExitCode: SuccessCode, } got := Validate(test) @@ -116,3 +122,50 @@ func Test_ValidateFail(t *testing.T) { assert.False(t, got.Success) assert.Equal(t, []string{"Stdout", "Stderr", "ExitCode"}, got.Properties) } + +func Test_ValidateContains(t *testing.T) { + test := commander.TestCase{ + Stdout: ` +✓ it should assert stderr +`, + Stderr: ` +this +`, + ExitCode: 0, + } + test.Result = commander.TestResult{ + Stdout: ` +✓ it should exit with error code +✓ it should assert stderr +✓ it should assert stdout +✓ it should fail' +`, + Stderr: ` +this +is +my +stderr +and +more`, + ExitCode: SuccessCode, + } + + got := Validate(test) + + assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) +} + +func Test_ValidateWithEmptyStdoutAndStderr(t *testing.T) { + test := commander.TestCase{ExitCode: SuccessCode} + test.Result = commander.TestResult{ + Stdout: "out", + Stderr: "err", + ExitCode: SuccessCode, + } + + got := Validate(test) + + assert.True(t, got.Success) + assert.Len(t, got.Properties, 0) +} \ No newline at end of file