Add integration tests and asser that stdout / stderr contain string

README.md

@@ -16,22 +16,28 @@ $ ./commander ./example/commander.yaml
 ```
 
 ## Todo:
- - go api
+ - suite fails -> error exit code
  - logging / verbose output
+ - print errors in colors
+ - execute a single test
+
+ - go api
  - command execution
    - environment variables
    - arguments?
    - timeout
- - exit code
+ - exit code *done*
  - stdout
-    - Validate against string
+    - Validate against string *done*
     - Validate against file
     - Validate against line
     - Validate with wildcards / regex
+    - Validate against template
  - stderr
-    - Validate against string
+    - Validate against string *done*
     - Validate against file
     - Validate with wildcards
+    - Validate against template
  - testing interactive applications?
  - Support different os
    - Windows

commander.yaml

@@ -2,11 +2,20 @@ tests:
   it should fail with invalid argument:
     command: ./commander asfdf
     exit-code: 3
+
   it should display help:
     command: ./commander
     exit-code: 0
+
   it should execute tests:
-    command: ./commander test ./examples/commander.yaml
+    command: ./commander test ./integration/commander_test.yaml
+    stdout: ✓ it should exit with error code
+    exit-code: 0
+
+  it should assert that commander will fail:
+    command: ./commander test ./integration/failing_suite.yaml
     stdout: >-
-      ✓ it should fail
+      ✗ it will fail
+
+      Got 1, expected 0
     exit-code: 0

examples/commander.yaml

@@ -1,5 +1,8 @@
 tests:
-  it should fail:
-    command: /bin/bash invalid
-    stderr: "/bin/bash: invalid: No such file or directory"
-    exit-code: 127
+  it should print hello world:
+    command: echo hello world
+    stdout: hello world
+    exit-code: 0
+  it should exit with error code:
+    command: exit 1
+    exit-code: 1

integration/commander_test.yaml

@@ -0,0 +1,19 @@
+tests:
+  it should fail:
+    command: /bin/bash invalid
+    stderr: "/bin/bash: invalid: No such file or directory"
+    exit-code: 127
+
+  it should exit with error code:
+      command: exit 1
+      exit-code: 1
+
+  it should assert stdout:
+      command: echo hello
+      stdout: hello
+      exit-code: 0
+
+  it should assert stderr:
+      command: '>&2 echo "error"'
+      stderr: error
+      exit-code: 0

integration/failing_suite.yaml

@@ -0,0 +1,4 @@
+tests:
+  it will fail:
+    command: exit 1
+    exit-code: 0

pkg/runtime/validator.go

@@ -1,6 +1,9 @@
 package runtime
 
-import "github.com/SimonBaeumer/commander/pkg"
+import (
+	"github.com/SimonBaeumer/commander/pkg"
+	"strings"
+)
 
 type ValidationResult struct {
 	Success    bool
@@ -13,11 +16,11 @@ func Validate(test commander.TestCase) ValidationResult {
 		Properties: []string{},
 	}
 
-	if test.Stdout != "" && (test.Stdout != test.Result.Stdout) {
+	if test.Stdout != "" && !strings.Contains(test.Result.Stdout, test.Stdout) {
 		r.Properties = append(r.Properties, commander.Stdout)
 	}
 
-	if test.Stderr != "" && (test.Stderr != test.Result.Stderr) {
+	if test.Stderr != "" && !strings.Contains(test.Result.Stderr, test.Stderr) {
 		r.Properties = append(r.Properties, commander.Stderr)
 	}
 

pkg/runtime/validator_test.go

@@ -21,6 +21,7 @@ func Test_ValidateExitCodeFail(t *testing.T) {
 	got := Validate(test)
 
 	assert.False(t, got.Success)
+    assert.Len(t, got.Properties, 1)
 }
 
 func Test_ValidateExitCodeSuccess(t *testing.T) {
@@ -33,6 +34,7 @@ func Test_ValidateExitCodeSuccess(t *testing.T) {
 	got := Validate(test)
 
 	assert.True(t, got.Success)
+    assert.Len(t, got.Properties, 0)
 }
 
 func Test_ValidateStdoutFail(t *testing.T) {
@@ -45,6 +47,7 @@ func Test_ValidateStdoutFail(t *testing.T) {
 	got := Validate(test)
 
 	assert.False(t, got.Success)
+    assert.Len(t, got.Properties, 1)
 }
 
 func Test_ValidateStdoutSuccess(t *testing.T) {
@@ -57,6 +60,7 @@ func Test_ValidateStdoutSuccess(t *testing.T) {
 	got := Validate(test)
 
 	assert.True(t, got.Success)
+    assert.Len(t, got.Properties, 0)
 }
 
 func Test_ValidateStderrFail(t *testing.T) {
@@ -68,6 +72,7 @@ func Test_ValidateStderrFail(t *testing.T) {
 	got := Validate(test)
 
 	assert.False(t, got.Success)
+    assert.Len(t, got.Properties, 1)
 }
 
 func Test_ValidateStderrSuccess(t *testing.T) {
@@ -79,18 +84,19 @@ func Test_ValidateStderrSuccess(t *testing.T) {
 	got := Validate(test)
 
 	assert.True(t, got.Success)
+    assert.Len(t, got.Properties, 0)
 }
 
 func Test_Validate(t *testing.T) {
 	test := commander.TestCase{
 		Stdout:   "foo",
 		Stderr:   "bar",
-		ExitCode: 0,
+		ExitCode: SuccessCode,
 	}
 	test.Result = commander.TestResult{
 		Stdout:   "foo",
 		Stderr:   "bar",
-		ExitCode: 0,
+		ExitCode: SuccessCode,
 	}
 
 	got := Validate(test)
@@ -103,16 +109,63 @@ func Test_ValidateFail(t *testing.T) {
 	test := commander.TestCase{
 		Stdout:   "fail",
 		Stderr:   "fail",
-		ExitCode: 1,
+		ExitCode: ErrorCode,
 	}
 	test.Result = commander.TestResult{
 		Stdout:   "foo",
 		Stderr:   "bar",
-		ExitCode: 0,
+		ExitCode: SuccessCode,
 	}
 
 	got := Validate(test)
 
 	assert.False(t, got.Success)
 	assert.Equal(t, []string{"Stdout", "Stderr", "ExitCode"}, got.Properties)
 }
+
+func Test_ValidateContains(t *testing.T) {
+	test := commander.TestCase{
+		Stdout:   `
+✓ it should assert stderr
+`,
+		Stderr:   `
+this
+`,
+		ExitCode: 0,
+	}
+	test.Result = commander.TestResult{
+		Stdout:   `
+✓ it should exit with error code
+✓ it should assert stderr
+✓ it should assert stdout
+✓ it should fail'
+`,
+		Stderr:   `
+this
+is
+my
+stderr
+and
+more`,
+		ExitCode: SuccessCode,
+	}
+
+	got := Validate(test)
+
+	assert.True(t, got.Success)
+	assert.Len(t, got.Properties, 0)
+}
+
+func Test_ValidateWithEmptyStdoutAndStderr(t *testing.T) {
+    test := commander.TestCase{ExitCode: SuccessCode}
+    test.Result = commander.TestResult{
+        Stdout: "out",
+        Stderr: "err",
+        ExitCode: SuccessCode,
+    }
+
+    got := Validate(test)
+
+    assert.True(t, got.Success)
+    assert.Len(t, got.Properties, 0)
+}