Make gateway error structure more flexible and single test runner (#277)

.gencode_hash.txt

@@ -8,7 +8,7 @@ a82821e72af6d0ee35800e6262eb9bb05256309b98aed2dad1a368fd2d6882bb  gencode/docs/i
 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930  gencode/docs/schema_doc.css
 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9  gencode/docs/schema_doc.min.js
 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a  gencode/docs/schema_extras.js
-5ed3aec445c397d50eef8c6d2baa01c8d9b59efaa83640d1d4033c13dca9b044  gencode/docs/state.html
+7c956996257531d257f72b3c9c190fe41a37d6e06bdb8ac43a06bb3ead94e501  gencode/docs/state.html
 68f8919d59556c7c781958baaac0b8cc629b6c4ce86e6ffd0171d23536747ec6  gencode/java/udmi/schema/Ancillary.java
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
@@ -31,8 +31,8 @@ ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454  gencode/java/u
 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d  gencode/java/udmi/schema/FamilyDiscoveryState.java
 60a8115ae1acae7c199b63180823198d38ec50d57b48dd85aca1ccc865058f85  gencode/java/udmi/schema/GatewayConfig.java
 4e9a913d5cf47a5901a63ec005115c58884e06c2cd6ba6bb786ffbb7c7fdaf74  gencode/java/udmi/schema/GatewayMetadata.java
-c1f0e51c41fb044e1d1bb5f975e5e686be3fb36eaf716da6ba7d9980897d45a0  gencode/java/udmi/schema/GatewayState.java
-a10ca7383711269fbd2bb5ed8de9438f71673c47dd68de7bbd32bf8ba79913d6  gencode/java/udmi/schema/Level.java
+e0e7739046e834c0f0ca6a70b38b4579618899be3162887a0fa7ab60bbff22a5  gencode/java/udmi/schema/GatewayState.java
+a5e5adfc187709e8646a11c92e804acfb67743f9d72149008aaca954df3177f6  gencode/java/udmi/schema/Level.java
 f53c891932643871f93368cfe797cac6fd4ed0f64e71c893f275cd7184cb8019  gencode/java/udmi/schema/LocalnetConfig.java
 fc2bf2d94a9cbfb600a381c7b3dad66850b2646a388183848e913de1b102a40f  gencode/java/udmi/schema/LocalnetMetadata.java
 2df4ae32d0bbecc21f7c3f6a416a195baa766a6210cfa8abca4a7bb45b9c7961  gencode/java/udmi/schema/Location.java
@@ -95,7 +95,7 @@ c8c8ecae303d9c96fb7a97106d722b32db8c3728a44a46db028cf0376d9dc79f  gencode/python
 a5a914cb5d74c29671a4d29dfa6c700b3fec27d695d607e28814ad31307e82da  gencode/python/udmi/schema/state_blobset_blob.py
 26443a1f6d0be3469ff93aa7fdb4e6682e0439a3b29a8e237998dcebec5f6901  gencode/python/udmi/schema/state_discovery.py
 187400078dfc89912062ca1ad92f61e32d28126ae56119d83e6767d58cda1117  gencode/python/udmi/schema/state_discovery_family.py
-fc40816c3483ede5ad4e4a74a8b494a3582085cdd1f49da4155e40f6ae7dae32  gencode/python/udmi/schema/state_gateway.py
+05e82aa15c64842e206ae8ce3d5810d115bb890d009ea5d657822fad0e0d2165  gencode/python/udmi/schema/state_gateway.py
 8bac9baabe83e2bf682b6376fee1eba2564a34c867f47f0c73eda44d6b3d8bb1  gencode/python/udmi/schema/state_pointset.py
 837ecc89c477abe3a1faf837733ca05475774891b55353d84ca231d90a1fbf31  gencode/python/udmi/schema/state_pointset_point.py
 c7e5245ac7fb8a17691e8bd5b71ff47b6b586f33b01e5c0150706598ee58aed5  gencode/python/udmi/schema/state_system.py

.github/workflows/testing.yml

@@ -66,8 +66,8 @@ jobs:
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
           bin/test_validator $GCP_TARGET_PROJECT
-          cat out/validation_report.json && echo
-          cat out/devices/AHU-1/state.json && echo
+          cat udmi_site_model/out/validation_report.json && echo
+          cat udmi_site_model/out/devices/AHU-1/state.json && echo
           diff -u /tmp/validator.out etc/validator.out && echo No validator diff detected.
       - name: sequence tests
         env:
@@ -85,9 +85,9 @@ jobs:
           validator/bin/build check
       - name: device ouput logs
         if: ${{ always() }}
-        run: more `find out/devices/ -type f`
+        run: more `find udmi_site_model/out/devices/ -type f`
       - name: pubber logs
         if: ${{ always() }}
         run: |
           cat pubber.out || true
-          more `find out/groups/ -type f`
+          more out/*.json

bin/loop_sequences

@@ -1,15 +1,24 @@
-#!/bin/bash
+#!/bin/bash -e
 
 export VALIDATOR_CONFIG=/tmp/validator_config.json
 
 ROOT_DIR=$(realpath $(dirname $0)/..)
 cd $ROOT_DIR
 
-if [[ $# != 0 ]]; then
-    echo Usage: $0 [-n]
+if [[ $1 == '-h' ]]; then
+    echo Usage: $0 [TEST_NAMES...]
     false
 fi
 
+test_names="$@"
+if [[ -z $test_names ]]; then
+    test_names="."
+    test_prefix=
+else
+    test_prefix=\#
+    echo Executing specific tests $test_names
+fi
+
 echo Parsing $VALIDATOR_CONFIG:
 cat $VALIDATOR_CONFIG
 project_id=`jq -r .project_id $VALIDATOR_CONFIG`
@@ -21,6 +30,11 @@ echo Site model $site_model
 echo Target device $device_id
 echo Device serial $serial_no
 
+if [[ ! -d $site_model ]]; then
+    echo Site model $site_model not found.
+    false
+fi
+
 awc=$(echo $project_id $site_model $device_id $serial_no | wc -w)
 if [[ $awc != 4 ]]; then
     echo Missing configuration paramaters.
@@ -32,29 +46,32 @@ EXPECTED_SEQUENCES=2
 validator/bin/build
 
 JARFILE=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
-JAVA_CMD="java -cp $JARFILE org.junit.runner.JUnitCore"
+JAVA_CMD="java -cp $JARFILE com.google.daq.mqtt.validator.SequenceTestRunner"
 
-rm -rf out/devices/$device_id
+rm -rf $site_model/out/devices/$device_id
 
-exit_code=0
-for test_class in DiscoveryValidator ConfigValidator WritebackValidator; do
+exit_code=2
+for test_name in $test_names; do
+  if [[ $test_name == . ]]; then
+    test_name=
+  fi
+  for test_class in ConfigValidator WritebackValidator; do
     rm -f out/*.json
-    CLASS=com.google.daq.mqtt.validator.validations.$test_class
+    target=$test_class$test_prefix$test_name
+    CLASS=com.google.daq.mqtt.validator.validations.$target
     echo $JAVA_CMD $CLASS
     timeout 5m $JAVA_CMD $CLASS
     result=$?
 
-    if [[ $result != 0 ]]; then
-        echo Sequence test $test_class exited with error code $result
+    if [[ $result == 2 ]]; then
+        echo Sequence test $target found no matching tests.
+    elif [[ $result != 0 ]]; then
+        echo Sequence test $target exited with error code $result.
         exit_code=$result
+    elif [[ $exit_code == 2 ]]; then
+        exit_code=0;
     fi
-
-    mkdir -p out/groups/$test_class/
-    for file in out/*.json; do
-        # Format output so it looks nice...
-        jq < $file > out/groups/$test_class/${file#out/}
-        rm $file
-    done
+  done
 done
 
 echo Done with test sequence execution, exit $exit_code.

bin/sequencer

@@ -6,9 +6,20 @@
 
 ROOT_DIR=$(realpath $(dirname $0)/..)
 TEST_LOG=/tmp/test_log.txt
+log_level=INFO
 
-if [ $# -lt 3 -o $# -gt 4 ]; then
-    echo Usage: $0 SITE_PATH PROJECT_ID DEVICE_ID [SERIAL_NO]
+if [[ $1 == '-v' ]]; then
+    log_level=DEBUG
+    shift
+fi
+
+if [[ $1 == '-v' || $1 == '-vv' ]]; then
+    log_level=TRACE
+    shift
+fi
+
+if [[ $# < 3 ]]; then
+    echo Usage: $0 SITE_PATH PROJECT_ID DEVICE_ID [SERIAL_NO] [TEST_NAMES...]
     false
 fi
 
@@ -18,6 +29,9 @@ device_id=$3
 shift 3
 
 serial_no=${1:-//}
+shift || true
+
+test_names="$@"
 
 cd $ROOT_DIR
 
@@ -28,19 +42,24 @@ cat <<EOF > /tmp/validator_config.json
   "site_model": "$site_path",
   "device_id": "$device_id",
   "serial_no": "$serial_no",
+  "log_level": "$log_level",
   "key_file": "$site_path/validator/rsa_private.pkcs8"
 }
 EOF
 
 cat /tmp/validator_config.json
+echo Running tools version `(cd $ROOT_DIR; git describe)`
 
-bin/loop_sequences 2>&1 | tee $TEST_LOG
+bin/loop_sequences $test_names 2>&1 | tee $TEST_LOG
 
 echo
 # Sort by test name (6th field)
 fgrep 'RESULT ' $TEST_LOG | sort -k 6 | tee /tmp/sequencer.out
 
-echo Running tools version `(cd $ROOT_DIR; git describe)`
+if [[ `wc -l < /tmp/sequencer.out` == 0 ]]; then
+    echo No test results found.
+    exit 1
+fi
 
 bin/check_version
 

bin/test_schema

@@ -33,7 +33,7 @@ while getopts "d:s:fnp" opt; do
             parallel=y
             ;;
         \?)
-            echo "Usage: $0 [-p] [-f] [-n] [-d TEST_DATA_DIR] [-s SEQUENCE]"
+            echo "Usage: $0 [-p] [-f] [-n] [-d TEST_DATA_DIR] [-s TEST_SUBSET]"
             exit -1
             ;;
     esac

bin/test_validator

@@ -1,5 +1,8 @@
 #!/bin/bash -e
 
+# Force consistent sort order
+export LC_ALL=C
+
 ROOT_DIR=$(dirname $0)/..
 cd $ROOT_DIR
 
@@ -24,11 +27,10 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
-rm -rf out/devices
+rm -rf $site_path/out/devices
 
 # Prepare auth key used by reflector
-cp udmi_site_model/devices/AHU-1/rsa_private.pkcs8 \
-   udmi_site_model/gcp_reflect_key.pkcs8
+cp $site_path/devices/AHU-1/rsa_private.pkcs8 $site_path/gcp_reflect_key.pkcs8
 
 bin/validator $site_path $project_id &
 vpid=$!
@@ -63,6 +65,6 @@ pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
 echo Killing pids $vpid $pids
 kill $vpid $pids
 
-outfiles=`find out/devices -name \*.out` || true
+outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 echo Found .out files $outfiles, copying to /tmp/validator.out
 more $outfiles > /tmp/validator.out || true