Ordered cli tests

api/server/server_utils.go

@@ -79,7 +79,7 @@ func StartTestServer() (Config, *grpc.ClientConn) {
 	go Start(config)
 
 	// Wait for swarm to be ready
-	log.Println("Waiting swarm to be ready")
+	log.Println("Waiting for swarm to be ready")
 	if err := initDependencies(config); err != nil {
 		log.Panicln("Dependencies are not ready", err)
 	}

cmd/amp/cli/cli_test.go

@@ -6,7 +6,9 @@ import (
 	"io/ioutil"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -20,11 +22,18 @@ type TestSpec struct {
 
 type CommandSpec struct {
 	Cmd         string   `yaml:"cmd"`
+	Step        int      `yaml:"step"`
 	Args        []string `yaml:"args"`
 	Options     []string `yaml:"options"`
 	Expectation string   `yaml:"expectation"`
 }
 
+type CommandSpecByStep []CommandSpec
+
+func (a CommandSpecByStep) Len() int           { return len(a) }
+func (a CommandSpecByStep) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a CommandSpecByStep) Less(i, j int) bool { return a[i].Step < a[j].Step }
+
 var (
 	testDir = "./test_samples"
 )
@@ -52,6 +61,10 @@ func loadFiles(t *testing.T) []*TestSpec {
 	}
 	for _, f := range files {
 		name := f.Name()
+		if filepath.Ext(name) != ".yml" {
+			t.Log("Ignoring file:", name)
+			continue
+		}
 		t.Log("Loading file:", name)
 		valid := false
 		if !strings.HasPrefix(name, "00-") {
@@ -72,12 +85,17 @@ func loadFiles(t *testing.T) []*TestSpec {
 }
 
 func parseCmd(t *testing.T, test *TestSpec) {
+	var sortedCmdSpecs []CommandSpec
 	commandMap, err := generateCmdSpec(test.contents)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-	for _, cmdSpec := range commandMap {
+	for _, k := range commandMap {
+		sortedCmdSpecs = append(sortedCmdSpecs, k)
+	}
+	sort.Sort(CommandSpecByStep(sortedCmdSpecs))
+	for _, cmdSpec := range sortedCmdSpecs {
 		cmdString := generateCmdString(cmdSpec)
 		t.Log(cmdString, "Command passed.")
 		for i := 0; i < 10; i++ {

cmd/amp/cli/test_samples/00-invalid-sample-01.yml

@@ -1,5 +1,6 @@
 invalid-sample:
   cmd: amp service creat
+  step: 1
   args:
     - appcelerator/pinger
   options:

cmd/amp/cli/test_samples/00-invalid-sample-02.yml

@@ -1,5 +1,6 @@
 invalid-sample:
   cmd: curl
+  step: 1
   args:
     - localhost:90/ping
   options:

cmd/amp/cli/test_samples/01-create-service.yml

@@ -1,5 +1,6 @@
 create-service:
   cmd: amp service create
+  step: 1
   args:
     - appcelerator/pinger
   options:

cmd/amp/cli/test_samples/02-list-service.yml

@@ -1,5 +1,6 @@
 list-service:
   cmd: docker service ls
+  step: 1
   args:
   options:
   expectation: ((ID)(\s)+(NAME)(\s)+(REPLICAS)(\s)+(IMAGE)(\s)+(COMMAND)((.)|(\s))+(pinger)(.)+(appcelerator/pinger)(\s)+(.)*)

cmd/amp/cli/test_samples/03-curl-service.yml

@@ -1,5 +1,6 @@
 curl-service:
   cmd: curl
+  step: 1
   args:
     - localhost:90/ping
   options:

cmd/amp/cli/test_samples/04-remove-service.yml

@@ -1,11 +1,13 @@
 remove-service:
   cmd: amp service rm
+  step: 1
   args:
     - pinger
   options:
   expectation: (pinger)
 list-service:
   cmd: docker service ls
+  step: 2
   args:
   options:
   expectation: ((ID)(\s)+(NAME)(\s)+(REPLICAS)(\s)+(IMAGE)(\s)+(COMMAND)((.)|(\s))+)

cmd/amp/cli/test_samples/05-create-stack.yml

@@ -1,5 +1,6 @@
 create-stack:
   cmd: amp stack up
+  step: 1
   args:
   options:
      - -f ../../../api/rpc/stack/test_samples/sample-04.yml

cmd/amp/cli/test_samples/06-list-stack.yml

@@ -1,5 +1,6 @@
 list-stack:
   cmd: amp stack ls
+  step: 1
   args:
   options:
   expectation: ((NAME)(\s)+(ID)(\s)+(STATE)((.)|(\s))+(stack1)(\s)+([a-z0-9]{64})(\s)+(Running)((.)|(\s))(.)*)

cmd/amp/cli/test_samples/07-stop-stack.yml

@@ -1,11 +1,13 @@
 stop-stack:
   cmd: amp stack stop
+  step: 1
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
 list-stack:
   cmd: amp stack ls
+  step: 2
   args:
   options:
   expectation: "No stack is available"

cmd/amp/cli/test_samples/08-restart-stack.yml

@@ -1,11 +1,13 @@
 restart-stack:
   cmd: amp stack restart
+  step: 1
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
 list-stack:
   cmd: amp stack ls
+  step: 2
   args:
   options:
   expectation: ((NAME)(\s)+(ID)(\s)+(STATE)((.)|(\s))+(stack1)(\s)+([a-z0-9]{64})(\s)+(Running)((.)|(\s))(.)*)

cmd/amp/cli/test_samples/09-remove-stack.yml

@@ -1,17 +1,20 @@
 stop-stack:
   cmd: amp stack stop
+  step: 1
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
 remove-stack:
   cmd: amp stack rm
+  step: 2
   args:
     - stack1
   options:
   expectation: ([a-z0-9]{64})
 list-stack:
   cmd: amp stack ls
+  step: 3
   args:
   options:
   expectation: "No stack is available"

cmd/amp/cli/test_samples/10-cpu-stats.yml

@@ -1,5 +1,6 @@
 cpu-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/11-mem-stats.yml

@@ -1,5 +1,6 @@
 mem-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/12-io-stats.yml

@@ -1,5 +1,6 @@
 io-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/13-task-stats.yml

@@ -1,5 +1,6 @@
 task-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/14-container-stats.yml

@@ -1,5 +1,6 @@
 container-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/15-net-stats.yml

@@ -1,5 +1,6 @@
 net-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/16-node-stats.yml

@@ -1,5 +1,6 @@
 node-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/17-service-stats.yml

@@ -1,5 +1,6 @@
 service-stats:
   cmd: amp stats
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/18-stack-logs.yml

@@ -1,5 +1,6 @@
 stack-logs:
   cmd: amp logs
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/19-numbered-logs.yml

@@ -1,5 +1,6 @@
 numbered-logs:
   cmd: amp logs
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/20-metadata-logs.yml

@@ -1,5 +1,6 @@
 metadata-logs:
   cmd: amp logs
+  step: 1
   args:
     -
   options:

cmd/amp/cli/test_samples/21-all-logs.yml

@@ -1,5 +1,6 @@
 all-logs:
   cmd: amp logs
+  step: 1
   args:
   options:
   expectation: