diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 83e3171623..ffcd786549 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -8,7 +8,7 @@ a82821e72af6d0ee35800e6262eb9bb05256309b98aed2dad1a368fd2d6882bb gencode/docs/i 741b880216be3743f6747800a042f2dbd89f3b0344c6b0a965f4bc010f03a930 gencode/docs/schema_doc.css 878ea88206c974f40643c3cc430875f9c4e8c5e3fd6bcd6358bd3eb6d48699a9 gencode/docs/schema_doc.min.js 7ed934930aee763e0beebc349725ba3909115e8d346bb762f28bcbe745bb163a gencode/docs/schema_extras.js -5ed3aec445c397d50eef8c6d2baa01c8d9b59efaa83640d1d4033c13dca9b044 gencode/docs/state.html +7c956996257531d257f72b3c9c190fe41a37d6e06bdb8ac43a06bb3ead94e501 gencode/docs/state.html 68f8919d59556c7c781958baaac0b8cc629b6c4ce86e6ffd0171d23536747ec6 gencode/java/udmi/schema/Ancillary.java d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b gencode/java/udmi/schema/Asset.java 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d gencode/java/udmi/schema/Aux.java @@ -31,8 +31,8 @@ ae4a645f199c8e24b3303463d428ca17af7603ae9ae9238397a6a82e752ab454 gencode/java/u 0afc15acd72874e5a0c47f546abc0c4569f5bc37838fdcac77bc7bd55cc53a6d gencode/java/udmi/schema/FamilyDiscoveryState.java 60a8115ae1acae7c199b63180823198d38ec50d57b48dd85aca1ccc865058f85 gencode/java/udmi/schema/GatewayConfig.java 4e9a913d5cf47a5901a63ec005115c58884e06c2cd6ba6bb786ffbb7c7fdaf74 gencode/java/udmi/schema/GatewayMetadata.java -c1f0e51c41fb044e1d1bb5f975e5e686be3fb36eaf716da6ba7d9980897d45a0 gencode/java/udmi/schema/GatewayState.java -a10ca7383711269fbd2bb5ed8de9438f71673c47dd68de7bbd32bf8ba79913d6 gencode/java/udmi/schema/Level.java +e0e7739046e834c0f0ca6a70b38b4579618899be3162887a0fa7ab60bbff22a5 gencode/java/udmi/schema/GatewayState.java +a5e5adfc187709e8646a11c92e804acfb67743f9d72149008aaca954df3177f6 gencode/java/udmi/schema/Level.java f53c891932643871f93368cfe797cac6fd4ed0f64e71c893f275cd7184cb8019 gencode/java/udmi/schema/LocalnetConfig.java fc2bf2d94a9cbfb600a381c7b3dad66850b2646a388183848e913de1b102a40f gencode/java/udmi/schema/LocalnetMetadata.java 2df4ae32d0bbecc21f7c3f6a416a195baa766a6210cfa8abca4a7bb45b9c7961 gencode/java/udmi/schema/Location.java @@ -95,7 +95,7 @@ c8c8ecae303d9c96fb7a97106d722b32db8c3728a44a46db028cf0376d9dc79f gencode/python a5a914cb5d74c29671a4d29dfa6c700b3fec27d695d607e28814ad31307e82da gencode/python/udmi/schema/state_blobset_blob.py 26443a1f6d0be3469ff93aa7fdb4e6682e0439a3b29a8e237998dcebec5f6901 gencode/python/udmi/schema/state_discovery.py 187400078dfc89912062ca1ad92f61e32d28126ae56119d83e6767d58cda1117 gencode/python/udmi/schema/state_discovery_family.py -fc40816c3483ede5ad4e4a74a8b494a3582085cdd1f49da4155e40f6ae7dae32 gencode/python/udmi/schema/state_gateway.py +05e82aa15c64842e206ae8ce3d5810d115bb890d009ea5d657822fad0e0d2165 gencode/python/udmi/schema/state_gateway.py 8bac9baabe83e2bf682b6376fee1eba2564a34c867f47f0c73eda44d6b3d8bb1 gencode/python/udmi/schema/state_pointset.py 837ecc89c477abe3a1faf837733ca05475774891b55353d84ca231d90a1fbf31 gencode/python/udmi/schema/state_pointset_point.py c7e5245ac7fb8a17691e8bd5b71ff47b6b586f33b01e5c0150706598ee58aed5 gencode/python/udmi/schema/state_system.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index aedba50641..9a8943128f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -66,8 +66,8 @@ jobs: if: "${{ env.GCP_TARGET_PROJECT != '' }}" run: | bin/test_validator $GCP_TARGET_PROJECT - cat out/validation_report.json && echo - cat out/devices/AHU-1/state.json && echo + cat udmi_site_model/out/validation_report.json && echo + cat udmi_site_model/out/devices/AHU-1/state.json && echo diff -u /tmp/validator.out etc/validator.out && echo No validator diff detected. - name: sequence tests env: @@ -85,9 +85,9 @@ jobs: validator/bin/build check - name: device ouput logs if: ${{ always() }} - run: more `find out/devices/ -type f` + run: more `find udmi_site_model/out/devices/ -type f` - name: pubber logs if: ${{ always() }} run: | cat pubber.out || true - more `find out/groups/ -type f` + more out/*.json diff --git a/bin/loop_sequences b/bin/loop_sequences index 4ed152eb15..6ab3c394cc 100755 --- a/bin/loop_sequences +++ b/bin/loop_sequences @@ -1,15 +1,24 @@ -#!/bin/bash +#!/bin/bash -e export VALIDATOR_CONFIG=/tmp/validator_config.json ROOT_DIR=$(realpath $(dirname $0)/..) cd $ROOT_DIR -if [[ $# != 0 ]]; then - echo Usage: $0 [-n] +if [[ $1 == '-h' ]]; then + echo Usage: $0 [TEST_NAMES...] false fi +test_names="$@" +if [[ -z $test_names ]]; then + test_names="." + test_prefix= +else + test_prefix=\# + echo Executing specific tests $test_names +fi + echo Parsing $VALIDATOR_CONFIG: cat $VALIDATOR_CONFIG project_id=`jq -r .project_id $VALIDATOR_CONFIG` @@ -21,6 +30,11 @@ echo Site model $site_model echo Target device $device_id echo Device serial $serial_no +if [[ ! -d $site_model ]]; then + echo Site model $site_model not found. + false +fi + awc=$(echo $project_id $site_model $device_id $serial_no | wc -w) if [[ $awc != 4 ]]; then echo Missing configuration paramaters. @@ -32,29 +46,32 @@ EXPECTED_SEQUENCES=2 validator/bin/build JARFILE=validator/build/libs/validator-1.0-SNAPSHOT-all.jar -JAVA_CMD="java -cp $JARFILE org.junit.runner.JUnitCore" +JAVA_CMD="java -cp $JARFILE com.google.daq.mqtt.validator.SequenceTestRunner" -rm -rf out/devices/$device_id +rm -rf $site_model/out/devices/$device_id -exit_code=0 -for test_class in DiscoveryValidator ConfigValidator WritebackValidator; do +exit_code=2 +for test_name in $test_names; do + if [[ $test_name == . ]]; then + test_name= + fi + for test_class in ConfigValidator WritebackValidator; do rm -f out/*.json - CLASS=com.google.daq.mqtt.validator.validations.$test_class + target=$test_class$test_prefix$test_name + CLASS=com.google.daq.mqtt.validator.validations.$target echo $JAVA_CMD $CLASS timeout 5m $JAVA_CMD $CLASS result=$? - if [[ $result != 0 ]]; then - echo Sequence test $test_class exited with error code $result + if [[ $result == 2 ]]; then + echo Sequence test $target found no matching tests. + elif [[ $result != 0 ]]; then + echo Sequence test $target exited with error code $result. exit_code=$result + elif [[ $exit_code == 2 ]]; then + exit_code=0; fi - - mkdir -p out/groups/$test_class/ - for file in out/*.json; do - # Format output so it looks nice... - jq < $file > out/groups/$test_class/${file#out/} - rm $file - done + done done echo Done with test sequence execution, exit $exit_code. diff --git a/bin/sequencer b/bin/sequencer index 5a19c7f7e8..6fdba33814 100755 --- a/bin/sequencer +++ b/bin/sequencer @@ -6,9 +6,20 @@ ROOT_DIR=$(realpath $(dirname $0)/..) TEST_LOG=/tmp/test_log.txt +log_level=INFO -if [ $# -lt 3 -o $# -gt 4 ]; then - echo Usage: $0 SITE_PATH PROJECT_ID DEVICE_ID [SERIAL_NO] +if [[ $1 == '-v' ]]; then + log_level=DEBUG + shift +fi + +if [[ $1 == '-v' || $1 == '-vv' ]]; then + log_level=TRACE + shift +fi + +if [[ $# < 3 ]]; then + echo Usage: $0 SITE_PATH PROJECT_ID DEVICE_ID [SERIAL_NO] [TEST_NAMES...] false fi @@ -18,6 +29,9 @@ device_id=$3 shift 3 serial_no=${1:-//} +shift || true + +test_names="$@" cd $ROOT_DIR @@ -28,19 +42,24 @@ cat < /tmp/validator_config.json "site_model": "$site_path", "device_id": "$device_id", "serial_no": "$serial_no", + "log_level": "$log_level", "key_file": "$site_path/validator/rsa_private.pkcs8" } EOF cat /tmp/validator_config.json +echo Running tools version `(cd $ROOT_DIR; git describe)` -bin/loop_sequences 2>&1 | tee $TEST_LOG +bin/loop_sequences $test_names 2>&1 | tee $TEST_LOG echo # Sort by test name (6th field) fgrep 'RESULT ' $TEST_LOG | sort -k 6 | tee /tmp/sequencer.out -echo Running tools version `(cd $ROOT_DIR; git describe)` +if [[ `wc -l < /tmp/sequencer.out` == 0 ]]; then + echo No test results found. + exit 1 +fi bin/check_version diff --git a/bin/test_schema b/bin/test_schema index f56d4aa79b..9090f7204d 100755 --- a/bin/test_schema +++ b/bin/test_schema @@ -33,7 +33,7 @@ while getopts "d:s:fnp" opt; do parallel=y ;; \?) - echo "Usage: $0 [-p] [-f] [-n] [-d TEST_DATA_DIR] [-s SEQUENCE]" + echo "Usage: $0 [-p] [-f] [-n] [-d TEST_DATA_DIR] [-s TEST_SUBSET]" exit -1 ;; esac diff --git a/bin/test_validator b/bin/test_validator index 37b0ad12eb..e96062a0ac 100755 --- a/bin/test_validator +++ b/bin/test_validator @@ -1,5 +1,8 @@ #!/bin/bash -e +# Force consistent sort order +export LC_ALL=C + ROOT_DIR=$(dirname $0)/.. cd $ROOT_DIR @@ -24,11 +27,10 @@ if [[ -n $pids ]]; then kill $pids fi -rm -rf out/devices +rm -rf $site_path/out/devices # Prepare auth key used by reflector -cp udmi_site_model/devices/AHU-1/rsa_private.pkcs8 \ - udmi_site_model/gcp_reflect_key.pkcs8 +cp $site_path/devices/AHU-1/rsa_private.pkcs8 $site_path/gcp_reflect_key.pkcs8 bin/validator $site_path $project_id & vpid=$! @@ -63,6 +65,6 @@ pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'` echo Killing pids $vpid $pids kill $vpid $pids -outfiles=`find out/devices -name \*.out` || true +outfiles=`find $site_path/out/devices -name \*.out | sort` || true echo Found .out files $outfiles, copying to /tmp/validator.out more $outfiles > /tmp/validator.out || true diff --git a/dashboard/functions/index.js b/dashboard/functions/index.js index 4289321e8c..98fde8088f 100644 --- a/dashboard/functions/index.js +++ b/dashboard/functions/index.js @@ -2,27 +2,34 @@ * Simple function to ingest test results event from DAQ. */ +// Hacky stuff to work with "maybe have firestore enabled" +const PROJECT_ID = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT; +const useFirestore = !!process.env.FIREBASE_CONFIG; +if (!process.env.GCLOUD_PROJECT) { + console.log("Setting GCLOUD_PROJECT to " + PROJECT_ID); + process.env.GCLOUD_PROJECT = PROJECT_ID; +} + const functions = require('firebase-functions'); const admin = require('firebase-admin'); const { PubSub } = require(`@google-cloud/pubsub`); const iot = require('@google-cloud/iot'); const pubsub = new PubSub(); const REFLECT_REGISTRY = 'UDMS-REFLECT'; -const UDMI_VERSION = '1'; +const UDMI_VERSION = '1.3.14'; const EVENT_TYPE = 'event'; const CONFIG_TYPE = 'config'; const STATE_TYPE = 'state'; -const PROJECT_ID = process.env.GCP_PROJECT || process.env.GCLOUD_PROJECT; const ALL_REGIONS = ['us-central1', 'europe-west1', 'asia-east1']; let registry_regions = null; -if (process.env.FIREBASE_CONFIG) { +if (useFirestore) { admin.initializeApp(functions.config().firebase); } else { console.log('No FIREBASE_CONFIG defined'); } -const firestore = process.env.FIREBASE_CONFIG ? admin.firestore() : null; +const firestore = useFirestore ? admin.firestore() : null; const iotClient = new iot.v1.DeviceManagerClient({ // optional auth parameters. @@ -48,9 +55,9 @@ function recordMessage(attributes, message) { message.version = message.version || UDMI_VERSION; } - console.log('record', registryId, deviceId, subType, subFolder, message); + console.log('Message record', registryId, deviceId, subType, subFolder); - if (firestore) { + if (useFirestore) { const reg_doc = firestore.collection('registries').doc(registryId); promises.push(reg_doc.set({ 'updated': timestamp @@ -74,18 +81,18 @@ function recordMessage(attributes, message) { } function sendCommand(registryId, deviceId, subFolder, message) { + const messageStr = JSON.stringify(message); return registry_promise.then(() => { - return sendCommandSafe(registryId, deviceId, subFolder, message); + return sendCommandSafe(registryId, deviceId, subFolder, messageStr); }); } -function sendCommandSafe(registryId, deviceId, subFolder, message) { +function sendCommandSafe(registryId, deviceId, subFolder, messageStr) { const cloudRegion = registry_regions[registryId]; const formattedName = iotClient.devicePath(PROJECT_ID, cloudRegion, registryId, deviceId); - const messageStr = JSON.stringify(message); console.log('command', formattedName, subFolder); const binaryData = Buffer.from(messageStr); @@ -97,7 +104,7 @@ function sendCommandSafe(registryId, deviceId, subFolder, message) { return iotClient.sendCommandToDevice(request) .catch((e) => { - console.error('error sending command:', e.details); + console.error('Command error', e.details); }); } @@ -214,7 +221,6 @@ function process_states_update(attributes, msgObject) { const registryId = attributes.deviceRegistryId; const commandFolder = `devices/${deviceId}/update/states`; - promises.push(sendCommand(REFLECT_REGISTRY, registryId, commandFolder, msgObject)); attributes.subType = STATE_TYPE; @@ -223,6 +229,7 @@ function process_states_update(attributes, msgObject) { if (typeof subMsg === 'object') { attributes.subFolder = block; subMsg.timestamp = msgObject.timestamp; + subMsg.version = msgObject.version; promises.push(publishPubsubMessage('udmi_target', attributes, subMsg)); const new_promises = recordMessage(attributes, subMsg); promises.push(...new_promises); @@ -253,7 +260,9 @@ exports.udmi_config = functions.pubsub.topic('udmi_config').onPublish((event) => const promises = recordMessage(attributes, msgObject); promises.push(publishPubsubMessage('udmi_target', attributes, msgObject)); - if (!firestore) { + if (useFirestore) { + console.info('Deferring to firestore trigger for IoT Core modification.'); + } else { promises.push(modify_device_config(registryId, deviceId, subFolder, msgObject)); } @@ -261,14 +270,31 @@ exports.udmi_config = functions.pubsub.topic('udmi_config').onPublish((event) => }); async function modify_device_config(registryId, deviceId, subFolder, subContents) { - console.log('Config modify subFolder', subFolder); const [oldConfig, version] = await get_device_config(registryId, deviceId); - const message = JSON.parse(oldConfig); + let newConfig = {}; + try { + const resetConfig = subFolder === "system" && subContents.extra_field === "reset_config"; + if (!resetConfig && oldConfig) { + newConfig = JSON.parse(oldConfig); + } else { + console.log("Config reset explicit=" + resetConfig); + resetConfig && delete subContents.extra_field; + } + } catch (e) { + console.warn('Previous config parse error, ignoring update'); + return; + } + + newConfig.version = UDMI_VERSION; + newConfig.timestamp = currentTimestamp(); + console.log('Config modify version', version, subFolder); if (subContents) { - message[subFolder] = subContents; + delete subContents.version; + delete subContents.timestamp; + newConfig[subFolder] = subContents; } else { - delete message[subFolder]; + delete newConfig[subFolder]; } const attributes = { projectId: PROJECT_ID, @@ -276,7 +302,7 @@ async function modify_device_config(registryId, deviceId, subFolder, subContents deviceId: deviceId, deviceRegistryId: registryId }; - return update_device_config(message, attributes, version) + return update_device_config(newConfig, attributes, version) .catch(e => { console.log('Config update rejected, retry', subFolder); return modify_device_config(registryId, deviceId, subFolder, subContents); @@ -320,9 +346,8 @@ function update_device_config(message, attributes, preVersion) { const extraField = message.system && message.system.extra_field; const normalJson = extraField !== 'break_json'; - if (!normalJson) { - console.log('breaking json for test'); - } + console.log('Config extra field is ' + extraField + ' ' + normalJson); + const msgString = normalJson ? JSON.stringify(message) : '{ broken because extra_field == ' + message.system.extra_field; const binaryData = Buffer.from(msgString); @@ -403,7 +428,7 @@ function publishPubsubMessage(topicName, attributes, data) { const dataBuffer = Buffer.from(dataStr); var attr_copy = Object.assign({}, attributes); - console.log('publish', topicName, attributes, dataStr); + console.log('Message publish', topicName, JSON.stringify(attributes)); return pubsub .topic(topicName) diff --git a/docs/tools/sequencer.md b/docs/tools/sequencer.md index 35239c593f..fc91c68205 100644 --- a/docs/tools/sequencer.md +++ b/docs/tools/sequencer.md @@ -1,8 +1,8 @@ [**UDMI**](../../) / [**Docs**](../) / [**Tools**](./) / [Sequencer](#) -# Sequence Validator Setup +# Sequencer Setup -The UDMI sequence validator tool monitors a sequence of messages from a device's stream and +The UDMI _sequencer_ tool monitors a sequence of messages from a device's stream and validates that the composition of sequential messsages is compliant with the UDMI Schema 1. Ensure you have [deployed the necessary cloud functions](../cloud/gcp/dashboard.md) to your GCP project @@ -17,19 +17,24 @@ validates that the composition of sequential messsages is compliant with the UDM * device_id: Use the `` as defined in Site Model for the devices to be tested. * auth_key: Use the public key you just created from `validator/rsa_public.pem` -## Running Validator +## Running Sequencer -To run the sequence validator and pubber device, run the command from the top-level -of the _site_model_ directory: -``` -~/udmi/bin/sequence ${GCP_PROJECT_NAME} ${TARGET_DEVICE} -``` - -To test against a real device, add the appropriate device serial number at the end: +To run the _sequencer_, run the command from the top-level of the _site_model_ directory, with +an optional device serial number: ``` ~/udmi/bin/sequence ${GCP_PROJECT_NAME} ${TARGET_DEVICE} ${SERIAL_NUMBER} ``` +## Troubleshooting + +To increase the logging verbosity add a `-v` at the beginning of the command options, or add +another `-v` or just `-vv` to add even more details. + +Adding a specific test names (e.g. `broken_config`) to the end will trigger just those explicitly +specified tests. + +## Example Output + An example output (using the pubber device), looks something like: ``` diff --git a/etc/Level.java b/etc/Level.java index 7a1449ea1a..561f133cfa 100644 --- a/etc/Level.java +++ b/etc/Level.java @@ -7,6 +7,7 @@ // proper source and don't be fooled! This is subset of the StackDriver LogSeverity levels. public enum Level { + TRACE(50), DEBUG(100), INFO(200), NOTICE(300), diff --git a/etc/sequencer.out b/etc/sequencer.out index 6f806f8352..8b7c990216 100644 --- a/etc/sequencer.out +++ b/etc/sequencer.out @@ -1,6 +1,6 @@ RESULT pass broken_config Sequence complete RESULT pass extra_config Sequence complete +RESULT pass provided_serial_no Sequence complete +RESULT pass provided_serial_no Sequence complete RESULT pass system_last_update Sequence complete -RESULT pass valid_serial_no Sequence complete -RESULT pass valid_serial_no Sequence complete RESULT skip writeback_states Missing 'invalid' target specification diff --git a/etc/validator.out b/etc/validator.out index 0c8cebf6a4..8730f69c19 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -1,24 +1,24 @@ :::::::::::::: -out/devices/AHU-1/state_pointset.out -:::::::::::::: -While converting to json node: 1 schema violations found - 1 schema violations found - object instance has properties which are not allowed by the schema: ["version"] -:::::::::::::: -out/devices/AHU-1/state_system.out +udmi_site_model/out/devices/AHU-1/state.out :::::::::::::: +While converting to json node: 2 schema violations found + 2 schema violations found + instance value ("states") not found in enum (possible values: ["update","discovery","system","gateway","localnet","metadata","pointset","blobset"]) + instance value ("update") not found in enum (possible values: ["event","command","state","config"]) While converting to json node: 2 schema violations found 2 schema violations found object has missing required properties (["last_config"]) + object has missing required properties (["version"]) +:::::::::::::: +udmi_site_model/out/devices/AHU-1/state_pointset.out +:::::::::::::: +While converting to json node: 1 schema violations found + 1 schema violations found object instance has properties which are not allowed by the schema: ["version"] :::::::::::::: -out/devices/AHU-1/state.out +udmi_site_model/out/devices/AHU-1/state_system.out :::::::::::::: -While converting to json node: 2 schema violations found - 2 schema violations found - instance value ("states") not found in enum (possible values: ["update","discovery","system","gateway","localnet","metadata","pointset","blobset"]) - instance value ("update") not found in enum (possible values: ["event","command","state","config"]) While converting to json node: 2 schema violations found 2 schema violations found object has missing required properties (["last_config"]) - object has missing required properties (["version"]) + object instance has properties which are not allowed by the schema: ["version"] diff --git a/gencode/docs/state.html b/gencode/docs/state.html index 8fced841c7..9b33607ab7 100644 --- a/gencode/docs/state.html +++ b/gencode/docs/state.html @@ -875,18 +875,18 @@

-
+
-
+

- +

-
+
root @@ -903,17 +903,316 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - error_ids

Type: array of string
+ devices
Type: object
-

Each item of this array must be:

+ +
+
+
+

+ +

+
+ +
+

Each additional property must conform to the following schema

+ +
root + + + + gateway + + + + devices + + + + additionalProperties
Type: object
+ No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + gateway + + + + devices + + + + additionalProperties + + + + status
Type: object
+ + + No Additional Properties + + + + + + +
+
+
+

+ +

+
+ +
+
+ +
root + + + + gateway + + + + devices + + + + additionalProperties + + + + status + + + + message
Type: string
+

A human-readable one-line description of the entry

+ + + + + + +
+
Example:
+
"Point is not writable"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + gateway + + + + devices + + + + additionalProperties + + + + status + + + + detail
Type: string
+

An optional extensive entry which can include more detail, e.g. a complete program stack-trace

+ + + + + + + +
+
+
+
+
-
+
+

+ +

+
+ +
+
+ +
root + + + + gateway + + + + devices + + + + additionalProperties + + + + status + + + + category
Type: string
+

A device-specific representation of the category an entry pertains to

+Must match regular expression: ^[a-z][._a-zA-Z]*[a-zA-Z]$ + + + + +
+
Example:
+
"pointset.points.config"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
root @@ -929,23 +1228,118 @@

Each item of this array must be:

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - error_ids + devices - error_ids items
Type: string
-Must match regular expression: ^[A-Z]{3}-[1-9][0-9]{0,2}$ + additionalProperties + + + + status + + + + timestamp
Type: string
+

Timestamp the condition was triggered, or most recently updated

+ +
+
Example:
+
"2018-08-26T21:39:28.364Z"
+
+
+
+
+
+
+
+
+
+

+ +

+
+ +
+
+ +
root + + + + gateway + + + + devices + + + + additionalProperties + + + + status + + + + level
Type: integer
+

The status level should conform to the numerical Stackdriver LogEntry levels. The DEFAULT value of 0 is not allowed (lowest value is 100, maximum 800).

+ + + +

Value must be greater or equal to 100 and lesser or equal to 800 and a multiple of 1

+ +
+
Example:
+
600
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/gencode/java/udmi/schema/GatewayState.java b/gencode/java/udmi/schema/GatewayState.java index 51d355f1f8..3ccab4340b 100644 --- a/gencode/java/udmi/schema/GatewayState.java +++ b/gencode/java/udmi/schema/GatewayState.java @@ -1,8 +1,6 @@ package udmi.schema; -import java.util.ArrayList; -import java.util.List; import javax.annotation.processing.Generated; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -17,18 +15,18 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ - "error_ids" + "devices" }) @Generated("jsonschema2pojo") public class GatewayState { - @JsonProperty("error_ids") - public List error_ids = new ArrayList(); + @JsonProperty("devices") + public Object devices; @Override public int hashCode() { int result = 1; - result = ((result* 31)+((this.error_ids == null)? 0 :this.error_ids.hashCode())); + result = ((result* 31)+((this.devices == null)? 0 :this.devices.hashCode())); return result; } @@ -41,7 +39,7 @@ public boolean equals(Object other) { return false; } GatewayState rhs = ((GatewayState) other); - return ((this.error_ids == rhs.error_ids)||((this.error_ids!= null)&&this.error_ids.equals(rhs.error_ids))); + return ((this.devices == rhs.devices)||((this.devices!= null)&&this.devices.equals(rhs.devices))); } } diff --git a/gencode/java/udmi/schema/Level.java b/gencode/java/udmi/schema/Level.java index 7a1449ea1a..561f133cfa 100644 --- a/gencode/java/udmi/schema/Level.java +++ b/gencode/java/udmi/schema/Level.java @@ -7,6 +7,7 @@ // proper source and don't be fooled! This is subset of the StackDriver LogSeverity levels. public enum Level { + TRACE(50), DEBUG(100), INFO(200), NOTICE(300), diff --git a/gencode/python/udmi/schema/state_gateway.py b/gencode/python/udmi/schema/state_gateway.py index a546b344f2..981aeabc74 100644 --- a/gencode/python/udmi/schema/state_gateway.py +++ b/gencode/python/udmi/schema/state_gateway.py @@ -5,14 +5,14 @@ class GatewayState: """Generated schema class""" def __init__(self): - self.error_ids = None + self.devices = None @staticmethod def from_dict(source): if not source: return None result = GatewayState() - result.error_ids = source.get('error_ids') + result.devices = source.get('devices') return result @staticmethod @@ -33,6 +33,6 @@ def expand_dict(input): def to_dict(self): result = {} - if self.error_ids: - result['error_ids'] = self.error_ids # 1 + if self.devices: + result['devices'] = self.devices # 5 return result diff --git a/pubber/pubber.iml b/pubber/pubber.iml index ed4eafbc1a..2c39d50b51 100644 --- a/pubber/pubber.iml +++ b/pubber/pubber.iml @@ -9,6 +9,7 @@ + diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index f188c3c88b..70116f10d4 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -10,7 +10,10 @@ import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PubSubClient.Bundle; import daq.pubber.SwarmMessage.Attributes; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.OutputStream; +import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -48,19 +51,18 @@ */ public class Pubber { + public static final String UDMI_VERSION = "1.3.14"; private static final Logger LOG = LoggerFactory.getLogger(Pubber.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() .setDateFormat(new ISO8601DateFormat()) .setSerializationInclusion(JsonInclude.Include.NON_NULL); - private static final String HOSTNAME = System.getenv("HOSTNAME"); - private static final String POINTSET_TOPIC = "events/pointset"; private static final String SYSTEM_TOPIC = "events/system"; private static final String STATE_TOPIC = "state"; private static final String CONFIG_TOPIC = "config"; private static final String ERROR_TOPIC = "errors"; - private static final int MIN_REPORT_MS = 200; private static final int DEFAULT_REPORT_SEC = 10; private static final int CONFIG_WAIT_TIME_MS = 10000; @@ -79,12 +81,12 @@ public class Pubber { ); private static final int MESSAGE_REPORT_INTERVAL = 100; private static final Map> LOG_MAP = ImmutableMap.of( + Level.TRACE, LOG::trace, Level.DEBUG, LOG::debug, Level.INFO, LOG::info, Level.WARNING, LOG::warn, Level.ERROR, LOG::error ); - public static final String UDMI_VERSION = "1.3.14"; private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final Configuration configuration; private final AtomicInteger messageDelayMs = new AtomicInteger(DEFAULT_REPORT_SEC * 1000); @@ -541,12 +543,22 @@ private Entry entryFromException(String category, Throwable e) { Entry entry = new Entry(); entry.category = category; entry.timestamp = new Date(); - entry.message = success ? "success" : e.getMessage(); - entry.detail = success ? null : e.toString(); + entry.message = success ? "success" + : e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + entry.detail = success ? null : exceptionDetail(e); entry.level = success ? Level.INFO.value() : Level.ERROR.value(); return entry; } + private String exceptionDetail(Throwable e) { + StringBuilder buffer = new StringBuilder(); + while (e != null) { + buffer.append(e).append(';'); + e = e.getCause(); + } + return buffer.toString(); + } + private void gatewayHandler(Config config) { info(String.format("%s gateway config %s", getTimestamp(), isoConvert(config.timestamp))); } @@ -574,10 +586,19 @@ private void configHandler(Config config) { publisherConfigLog("apply", null); } catch (Exception e) { publisherConfigLog("apply", e); + trace(stackTraceString(e)); } publishStateMessage(); } + private String stackTraceString(Throwable e) { + OutputStream outputStream = new ByteArrayOutputStream(); + try (PrintStream ps = new PrintStream(outputStream)) { + e.printStackTrace(ps); + } + return outputStream.toString(); + } + private String getTimestamp() { return isoConvert(new Date()); } @@ -593,7 +614,11 @@ private Date isoConvert(String timestamp) { private String isoConvert(Date timestamp) { try { + if (timestamp == null) { + return "null"; + } String dateString = OBJECT_MAPPER.writeValueAsString(timestamp); + // Strip off the leading and trailing quotes from the JSON string-as-string representation. return dateString.substring(1, dateString.length() - 1); } catch (Exception e) { throw new RuntimeException("Creating timestamp", e); @@ -602,8 +627,10 @@ private String isoConvert(Date timestamp) { private void updatePointsetConfig(PointsetConfig pointsetConfig) { PointsetConfig useConfig = pointsetConfig != null ? pointsetConfig : new PointsetConfig(); + Map points = + useConfig.points != null ? useConfig.points : new HashMap<>(); allPoints.forEach(point -> - updatePointConfig(point, useConfig.points.get(point.getName()))); + updatePointConfig(point, points.get(point.getName()))); deviceState.pointset.state_etag = useConfig.state_etag; } @@ -669,7 +696,8 @@ private void publishStateMessage() { } deviceState.timestamp = new Date(); String deviceId = configuration.deviceId; - info(String.format("update state %s", isoConvert(deviceState.timestamp))); + info(String.format("update state %s last_config %s", isoConvert(deviceState.timestamp), + isoConvert(deviceState.system.last_config))); stateDirty = false; publishMessage(deviceId, STATE_TOPIC, deviceState); lastStateTimeMs = System.currentTimeMillis(); @@ -721,6 +749,10 @@ private void info(String message) { cloudLog(message, Level.INFO); } + private void trace(String message) { + cloudLog(message, Level.TRACE); + } + private void warn(String message) { cloudLog(message, Level.WARNING); } @@ -730,9 +762,9 @@ private void error(String message) { } private void error(String message, Throwable e) { - LOG.error(message, e); String longMessage = message + ": " + e.getMessage(); cloudLog(longMessage, Level.ERROR); + trace(stackTraceString(e)); } static class ExtraPointsetEvent extends PointsetEvent { diff --git a/schema/state_gateway.json b/schema/state_gateway.json index e83a2b8a02..779f9d1611 100644 --- a/schema/state_gateway.json +++ b/schema/state_gateway.json @@ -5,11 +5,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { - "error_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[A-Z]{3}-[1-9][0-9]{0,2}$" + "devices": { + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "status": { + "$ref": "file:common.json#/definitions/entry" + } + } } } } diff --git a/tests/state.tests/gateway.json b/tests/state.tests/gateway.json index 0f208a7046..44d6ae2291 100644 --- a/tests/state.tests/gateway.json +++ b/tests/state.tests/gateway.json @@ -31,6 +31,23 @@ } }, "gateway": { - "error_ids": [ "991", "SMS-91" ] + "devices": { + "991": { + "status": { + "message": "Error attaching device", + "category": "device.state.com", + "timestamp": "2018-08-26T21:39:30.364Z", + "level": 600 + } + }, + "SMS-91": { + "status": { + "message": "Error attaching device", + "category": "device.state.com", + "timestamp": "2018-08-26T21:39:30.364Z", + "level": 600 + } + } + } } } diff --git a/tests/state.tests/gateway.out b/tests/state.tests/gateway.out index 7f3c915b16..e69de29bb2 100644 --- a/tests/state.tests/gateway.out +++ b/tests/state.tests/gateway.out @@ -1,2 +0,0 @@ -1 schema violations found - ECMA 262 regex "^[A-Z]{3}-[1-9][0-9]{0,2}$" does not match input string "991" diff --git a/validator/.idea/runConfigurations/BaselineValidator_broken_config.xml b/validator/.idea/runConfigurations/BaselineValidator_broken_config.xml index e1cc0e02f7..bc48a429d2 100644 --- a/validator/.idea/runConfigurations/BaselineValidator_broken_config.xml +++ b/validator/.idea/runConfigurations/BaselineValidator_broken_config.xml @@ -1,6 +1,6 @@ - +