From 7c8a2aa16555bedc25cfcd386d452faa9b3d39eb Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 7 Mar 2023 19:17:53 -0800 Subject: [PATCH 1/4] Sorting sequence.out --- bin/test_sequencer | 12 ++++++------ etc/sequencer.out | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/test_sequencer b/bin/test_sequencer index 8df422a4da..988619a257 100755 --- a/bin/test_sequencer +++ b/bin/test_sequencer @@ -27,7 +27,7 @@ device_id=AHU-1 device_out_base=$site_path/out/devices/$device_id test_out_base=$device_out_base/tests rm -rf $test_out_base -mkdir -p $test_out_base +mkdir -p $test_out_base out/ serial_no=sequencer-$RANDOM echo Using pubber with serial $serial_no @@ -68,8 +68,8 @@ fi bin/sequencer -a $SEQUENCER_OPTS $site_path $project_id $device_id $serial_no $targets 2>&1 | tee sequencer.out echo Completed execution of sequencer test run. -sed -i -e 's/.*sequence RESULT/RESULT/' /tmp/sequencer.out -echo Copied output to /tmp/sequencer.out for later examination. +sed -e 's/.*sequence RESULT/RESULT/' /tmp/sequencer.out | sort > out/sequencer.out +echo Copied output to out/sequencer.out for later examination. pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'` if [[ -n $pids ]]; then @@ -84,11 +84,11 @@ generated=docs/specs/sequences/generated.md cp $generated /tmp/ # Save for test/comparison later bin/gencode_seq -echo Comparing diff /tmp/sequencer.out etc/sequencer.out -diff -u /tmp/sequencer.out etc/sequencer.out +echo Comparing diff out/sequencer.out etc/sequencer.out +diff -u out/sequencer.out etc/sequencer.out echo Comparing diff /tmp/generated.md docs/specs/sequences/generated.md -diff -u /tmp/generated.md docs/specs/sequences/generated.md || ( +diff -u out/generated.md docs/specs/sequences/generated.md || ( echo Run bin/gencode_seq to regenerate output, if necessary. false ) diff --git a/etc/sequencer.out b/etc/sequencer.out index 493179b7c0..963faf36a3 100644 --- a/etc/sequencer.out +++ b/etc/sequencer.out @@ -6,15 +6,16 @@ RESULT pass endpoint endpoint_connection_retry ALPHA 5 Sequence complete RESULT pass endpoint endpoint_connection_success_alternate ALPHA 5 Sequence complete RESULT pass endpoint endpoint_connection_success_reconnect ALPHA 5 Sequence complete RESULT pass enumeration empty_enumeration ALPHA 5 Sequence complete -RESULT pass enumeration multi_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.families family_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.features feature_enumeration ALPHA 5 Sequence complete +RESULT pass enumeration multi_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.pointset pointset_enumeration ALPHA 5 Sequence complete RESULT pass pointset pointset_publish_interval ALPHA 5 Sequence complete RESULT pass pointset pointset_sample_rate ALPHA 5 Sequence complete RESULT pass system broken_config ALPHA 5 Sequence complete RESULT pass system device_config_acked ALPHA 5 Sequence complete RESULT pass system extra_config BETA 5 Sequence complete +RESULT pass system.mode system_mode_restart ALPHA 5 Sequence complete RESULT pass system system_last_update STABLE 5 Sequence complete RESULT pass system system_min_loglevel ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete @@ -22,7 +23,6 @@ RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete -RESULT pass system.mode system_mode_restart ALPHA 5 Sequence complete RESULT pass writeback writeback_failure ALPHA 5 Sequence complete RESULT pass writeback writeback_invalid ALPHA 5 Sequence complete RESULT pass writeback writeback_success ALPHA 5 Sequence complete From 431b58c78fb39293cd2eb9044fcaed7a8d463a2e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 7 Mar 2023 21:02:41 -0800 Subject: [PATCH 2/4] Fixing some test weirdness --- bin/fail | 10 ++++++++++ bin/run_tests | 4 ++-- bin/test_sequencer | 1 + bin/test_sites | 3 +-- tests/downgrade.site/registration_summary.json | 2 +- 5 files changed, 15 insertions(+), 5 deletions(-) create mode 100755 bin/fail diff --git a/bin/fail b/bin/fail new file mode 100755 index 0000000000..6f97a8a185 --- /dev/null +++ b/bin/fail @@ -0,0 +1,10 @@ +#!/bin/bash -e +# +# Convenience script to simply output a message and then fail. Useful for testing in the form of: +# +# do_test_stuff || bin/fail the testing script failed +# + +echo fail because $@ + +false diff --git a/bin/run_tests b/bin/run_tests index fffc8d6b2d..ddc889f5f0 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -8,10 +8,10 @@ $0 [step] where [step] is one of: install_dependencies code_tests - util_tests schema_tests trace_tests registrar_tests + util_tests all_tests EOF false @@ -46,10 +46,10 @@ case "$1" in all_tests) bin/run_tests install_dependencies bin/run_tests code_tests - bin/run_tests util_tests bin/run_tests schema_tests bin/run_tests trace_tests bin/run_tests registrar_tests + bin/run_tests util_tests ;; *) usage diff --git a/bin/test_sequencer b/bin/test_sequencer index 988619a257..fea7934ffc 100755 --- a/bin/test_sequencer +++ b/bin/test_sequencer @@ -85,6 +85,7 @@ cp $generated /tmp/ # Save for test/comparison later bin/gencode_seq echo Comparing diff out/sequencer.out etc/sequencer.out +md5sum out/sequencer.out etc/sequencer.out diff -u out/sequencer.out etc/sequencer.out echo Comparing diff /tmp/generated.md docs/specs/sequences/generated.md diff --git a/bin/test_sites b/bin/test_sites index 928f8e9759..b0f140eb62 100755 --- a/bin/test_sites +++ b/bin/test_sites @@ -58,8 +58,7 @@ echo Testing simple mock sequencer... bin/sequencer tests/downgrade.site/ -- DWN-2 54321 system_mode_restart RESULTLOG=tests/downgrade.site/out/devices/DWN-2/RESULT.log -echo Checking expected output from $RESULTLOG -fgrep "RESULT fail system.mode system_mode_restart STABLE 5 null" $RESULTLOG +fgrep "RESULT fail system.mode system_mode_restart ALPHA 5 null" $RESULTLOG || bin/fail unexpected output from $RESULTLOG echo echo Done with site tests diff --git a/tests/downgrade.site/registration_summary.json b/tests/downgrade.site/registration_summary.json index c52734bb6c..c936f97d2a 100644 --- a/tests/downgrade.site/registration_summary.json +++ b/tests/downgrade.site/registration_summary.json @@ -11,6 +11,6 @@ "DWN-2" : "devices/DWN-2" }, "Version" : { - "main" : "1.4.0-147-g792ad526" + "main" : "1.4.0-151-g7c8a2aa1" } } \ No newline at end of file From 601c6435891ad86e0d53cba9adfe0f6bfd215402 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 7 Mar 2023 21:04:39 -0800 Subject: [PATCH 3/4] Few more testing fixes --- .github/workflows/testing.yml | 3 +++ tests/downgrade.site/registration_summary.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 580d1c458c..a87b1fedf5 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -38,6 +38,9 @@ jobs: - name: bin/run_tests registrar_tests if: ${{ always() }} run: bin/run_tests registrar_tests + - name: bin/run_tests util_tests + if: ${{ always() }} + run: bin/run_tests util_tests spelling: name: Docs Spelling Checks diff --git a/tests/downgrade.site/registration_summary.json b/tests/downgrade.site/registration_summary.json index c936f97d2a..8427f2b698 100644 --- a/tests/downgrade.site/registration_summary.json +++ b/tests/downgrade.site/registration_summary.json @@ -11,6 +11,6 @@ "DWN-2" : "devices/DWN-2" }, "Version" : { - "main" : "1.4.0-151-g7c8a2aa1" + "main" : "1.4.0-152-g431b58c7" } } \ No newline at end of file From 8caf87afad3bf1d2b91bae9f527eb02a6b41aece Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 7 Mar 2023 21:39:09 -0800 Subject: [PATCH 4/4] Fixing sort order for sequencer.out --- bin/test_sequencer | 3 +++ etc/sequencer.out | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/test_sequencer b/bin/test_sequencer index fea7934ffc..3e64091771 100755 --- a/bin/test_sequencer +++ b/bin/test_sequencer @@ -3,6 +3,9 @@ ROOT_DIR=$(dirname $0)/.. cd $ROOT_DIR +# Fix sort order difference on different platforms. +export LC_ALL=C + if [[ $# -lt 1 ]]; then echo Usage: $0 PROJECT_ID [tests...] false diff --git a/etc/sequencer.out b/etc/sequencer.out index 963faf36a3..493179b7c0 100644 --- a/etc/sequencer.out +++ b/etc/sequencer.out @@ -6,16 +6,15 @@ RESULT pass endpoint endpoint_connection_retry ALPHA 5 Sequence complete RESULT pass endpoint endpoint_connection_success_alternate ALPHA 5 Sequence complete RESULT pass endpoint endpoint_connection_success_reconnect ALPHA 5 Sequence complete RESULT pass enumeration empty_enumeration ALPHA 5 Sequence complete +RESULT pass enumeration multi_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.families family_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.features feature_enumeration ALPHA 5 Sequence complete -RESULT pass enumeration multi_enumeration ALPHA 5 Sequence complete RESULT pass enumeration.pointset pointset_enumeration ALPHA 5 Sequence complete RESULT pass pointset pointset_publish_interval ALPHA 5 Sequence complete RESULT pass pointset pointset_sample_rate ALPHA 5 Sequence complete RESULT pass system broken_config ALPHA 5 Sequence complete RESULT pass system device_config_acked ALPHA 5 Sequence complete RESULT pass system extra_config BETA 5 Sequence complete -RESULT pass system.mode system_mode_restart ALPHA 5 Sequence complete RESULT pass system system_last_update STABLE 5 Sequence complete RESULT pass system system_min_loglevel ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete @@ -23,6 +22,7 @@ RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete RESULT pass system valid_serial_no ALPHA 5 Sequence complete +RESULT pass system.mode system_mode_restart ALPHA 5 Sequence complete RESULT pass writeback writeback_failure ALPHA 5 Sequence complete RESULT pass writeback writeback_invalid ALPHA 5 Sequence complete RESULT pass writeback writeback_success ALPHA 5 Sequence complete