Update validator to handle issues found in QA

bin/run_tests

@@ -37,6 +37,7 @@ case "$1" in
     ;;
   trace_tests)
     bin/test_trace simple
+    bin/test_trace upgrade
     ;;
   registrar_tests)
     bin/test_registrar

bin/test_trace

@@ -8,8 +8,17 @@ fi
 trace_name=$1
 shift
 
-trace_out=validator/traces/${trace_name}.out
-trace_in=validator/traces/${trace_name}.in
+function redact() {
+    sed -E -i $file \
+        -e "s/[0-9-]{10}T[0-9:]{8}Z/1999-10-20T01:02:03Z/" \
+        -e "s/Validator.java:[0-9]+/redacted/" \
+        -e 's/\\t.*\.java:[0-9]+\)/\\tredacted\\n/g' \
+        -e 's/\\n.*\.java:[0-9]+\)/\\tredacted\\n/g'
+}
+
+
+trace_in=tests/${trace_name}.trace
+trace_out=$trace_in/expected/
 site_model=sites/udmi_site_model
 site_out=$site_model/out
 
@@ -19,15 +28,9 @@ export UDMI_TOOLS=test_trace
 rm -rf $site_out
 validator/bin/validate -- schema trace $trace_in $site_model
 
-echo
-
-for file in `find $site_out -type f`; do
-    sed -E -i $file \
-        -e "s/[0-9-]{10}T[0-9:]{8}Z/1999-10-20T01:02:03Z/" \
-        -e "s/Validator.java:[0-9]+/redacted/" \
-        -e 's/\\t.*\.java:[0-9]+\)/\\tredacted\\n/g' \
-        -e 's/\\n.*\.java:[0-9]+\)/\\tredacted\\n/g'
-done
+echo Redacting output files...
+for file in `find $site_out -type f`; do redact; done
+for file in `find $trace_out -type f`; do redact; done
 
 echo Checking diff -r $site_out $trace_out
 diff -r $site_out $trace_out

common/src/main/java/com/google/udmi/util/Common.java

@@ -19,13 +19,14 @@
 public abstract class Common {
 
   public static final String STATE_QUERY_TOPIC = "update/query";
-  public static final String TIMESTAMP_PROPERTY_KEY = "timestamp";
-  public static final String VERSION_PROPERTY_KEY = "version";
+  public static final String EXCEPTION_KEY = "exception";
+  public static final String MESSAGE_KEY = "message";
+  public static final String TIMESTAMP_KEY = "timestamp";
+  public static final String VERSION_KEY = "version";
   public static final String SUBTYPE_PROPERTY_KEY = "subType";
   public static final String SUBFOLDER_PROPERTY_KEY = "subFolder";
   public static final String NO_SITE = "--";
   public static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
-  public static final String EXCEPTION_KEY = "exception";
   private static final String UDMI_VERSION_KEY = "UDMI_VERSION";
   public static final char DETAIL_SEPARATOR_CHAR = ';';
   public static final String DETAIL_SEPARATOR = DETAIL_SEPARATOR_CHAR + " ";

validator/traces/simple.in/SNS-4/002_event_discovery.json → tests/simple.trace/devices/SNS-4/002_event_discovery.json

@@ -1,4 +1,4 @@
-{{{{{{
+{{{{{{{{
   "points" : {
     "split_threshold" : {
       "present_value" : true

tests/upgrade.trace/devices/AHU-1/001_state_update.json

@@ -0,0 +1,14 @@
+{
+  "timestamp": "2023-03-02T19:27:35.819Z",
+  "version": 1,
+  "system": {
+    "last_config": "2021-05-14T15:53:26Z",
+    "firmware": {
+      "version": [
+        "D_IP_BW6_V35_6_S14"
+      ]
+    },
+    "operational": true,
+    "make_model": "Delmatic IP_BW6"
+  }
+}

tests/upgrade.trace/expected/devices/AHU-1/state.attr

@@ -0,0 +1,9 @@
+{
+  "deviceNumId" : "00000062256946",
+  "subFolder" : "update",
+  "deviceRegistryId" : "ZZ-TRI-FECTA",
+  "subType" : "state",
+  "msgSource" : "001_state_update.json",
+  "deviceId" : "AHU-1",
+  "projectId" : "playback-project"
+}

tests/upgrade.trace/expected/devices/AHU-1/state.json

@@ -0,0 +1,17 @@
+{
+  "system" : {
+    "last_config" : "1999-10-20T01:02:03Z",
+    "hardware" : {
+      "model" : "Delmatic IP_BW6",
+      "make" : "unknown"
+    },
+    "software" : {
+      "firmware" : [ "D_IP_BW6_V35_6_S14" ]
+    },
+    "operation" : {
+      "operational" : true
+    }
+  },
+  "timestamp" : "2023-03-02T19:27:35.819Z",
+  "version" : "1.4.0"
+}

tests/upgrade.trace/expected/devices/AHU-1/state.out

@@ -0,0 +1,20 @@
+{
+  "timestamp" : "1999-10-20T01:02:03Z",
+  "version" : "1.4.0",
+  "sub_folder" : "update",
+  "sub_type" : "state",
+  "status" : {
+    "message" : "While converting to json node: 2 schema violations found",
+    "detail" : "state_update: While converting to json node: 2 schema violations found @Validator.validateMessage(redacted); 2 schema violations found; /system: object has missing required properties ([\"serial_no\"]); /system/software/firmware: instance type (array) does not match any allowed primitive type (allowed: [\"string\"])",
+    "category" : "validation.device.schema",
+    "timestamp" : "1999-10-20T01:02:03Z",
+    "level" : 500
+  },
+  "errors" : [ {
+    "message" : "While converting to json node: 2 schema violations found",
+    "detail" : "state_update: While converting to json node: 2 schema violations found @Validator.validateMessage(redacted); 2 schema violations found; /system: object has missing required properties ([\"serial_no\"]); /system/software/firmware: instance type (array) does not match any allowed primitive type (allowed: [\"string\"])",
+    "category" : "validation.device.schema",
+    "timestamp" : "1999-10-20T01:02:03Z",
+    "level" : 500
+  } ]
+}

tests/upgrade.trace/expected/validation_report.json

@@ -0,0 +1,24 @@
+{
+  "timestamp" : "1999-10-20T01:02:03Z",
+  "version" : "1.4.0",
+  "tools" : "test_trace",
+  "start_time" : "1999-10-20T01:02:03Z",
+  "summary" : {
+    "correct_devices" : [ ],
+    "extra_devices" : [ ],
+    "missing_devices" : [ "AHU-22", "GAT-123", "SNS-4" ],
+    "error_devices" : [ "AHU-1" ]
+  },
+  "devices" : {
+    "AHU-1" : {
+      "last_seen" : "1999-10-20T01:02:03Z",
+      "status" : {
+        "message" : "While converting to json node: 2 schema violations found",
+        "detail" : "state_update: While converting to json node: 2 schema violations found @Validator.validateMessage(redacted); 2 schema violations found; /system: object has missing required properties ([\"serial_no\"]); /system/software/firmware: instance type (array) does not match any allowed primitive type (allowed: [\"string\"])",
+        "category" : "validation.device.schema",
+        "timestamp" : "1999-10-20T01:02:03Z",
+        "level" : 500
+      }
+    }
+  }
+}

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -8,7 +8,7 @@
 import static com.google.daq.mqtt.registrar.Registrar.METADATA_JSON;
 import static com.google.daq.mqtt.registrar.Registrar.NORMALIZED_JSON;
 import static com.google.daq.mqtt.util.MessageUpgrader.METADATA_SCHEMA;
-import static com.google.udmi.util.Common.VERSION_PROPERTY_KEY;
+import static com.google.udmi.util.Common.VERSION_KEY;
 import static com.google.udmi.util.JsonUtil.asMap;
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -297,7 +297,7 @@ private Metadata readMetadataWithValidation(boolean validate) {
       Metadata loadedMetadata = SiteModel.loadDeviceMetadata(siteDir.getPath(), deviceId,
           LocalDevice.class);
       instance = JsonUtil.convertTo(JsonNode.class, loadedMetadata);
-      baseVersion = instance.get(VERSION_PROPERTY_KEY);
+      baseVersion = instance.get(VERSION_KEY);
       new MessageUpgrader(METADATA_SCHEMA, instance).upgrade(false);
     } catch (Exception exception) {
       exceptionMap.put(EXCEPTION_LOADING, exception);

validator/src/main/java/com/google/daq/mqtt/registrar/MessageDowngrader.java

@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.registrar;
 
-import static com.google.udmi.util.Common.VERSION_PROPERTY_KEY;
+import static com.google.udmi.util.Common.VERSION_KEY;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -58,7 +58,7 @@ public void downgrade(JsonNode versionNode) {
       }
     }
 
-    message.set(VERSION_PROPERTY_KEY, versionNode);
+    message.set(VERSION_KEY, versionNode);
   }
 
   private String convertVersion(JsonNode versionNode) {

validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java

@@ -6,7 +6,7 @@
 import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize;
 import static com.google.udmi.util.CleanDateFormat.dateEquals;
 import static com.google.udmi.util.Common.EXCEPTION_KEY;
-import static com.google.udmi.util.Common.TIMESTAMP_PROPERTY_KEY;
+import static com.google.udmi.util.Common.TIMESTAMP_KEY;
 import static com.google.udmi.util.JsonUtil.getTimestamp;
 import static com.google.udmi.util.JsonUtil.safeSleep;
 import static com.google.udmi.util.JsonUtil.stringify;
@@ -1036,7 +1036,7 @@ private synchronized void handleReflectorMessage(String subTypeRaw,
       }
       if (message.containsKey(EXCEPTION_KEY)) {
         debug("Ignoring reflector exception:\n" + message.get(EXCEPTION_KEY).toString());
-        configExceptionTimestamp = (String) message.get(TIMESTAMP_PROPERTY_KEY);
+        configExceptionTimestamp = (String) message.get(TIMESTAMP_KEY);
         return;
       }
       configExceptionTimestamp = null;

validator/src/main/java/com/google/daq/mqtt/util/FileDataSink.java

@@ -50,6 +50,7 @@ public String publish(String deviceId, String topic, String data) {
     if (outFile == null) {
       return null;
     }
+    System.err.println("Updating " + outFile.getAbsolutePath());
     try (PrintWriter out = new PrintWriter(Files.newOutputStream(outFile.toPath()))) {
       out.println(data);
     } catch (Exception e) {

validator/src/main/java/com/google/daq/mqtt/util/MessageUpgrader.java

@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.util;
 
-import static com.google.udmi.util.Common.VERSION_PROPERTY_KEY;
+import static com.google.udmi.util.Common.VERSION_KEY;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
@@ -31,7 +31,7 @@ public MessageUpgrader(String schemaName, JsonNode message) {
     this.message = message;
     this.schemaName = schemaName;
 
-    JsonNode version = message.get(VERSION_PROPERTY_KEY);
+    JsonNode version = message.get(VERSION_KEY);
     String verStr =
         version != null ? version.isNumber() ? Integer.toString(version.asInt()) : version.asText()
             : "1";
@@ -90,8 +90,8 @@ public boolean upgrade(boolean forceUpgrade) {
       patch = 0;
     }
 
-    if (upgraded && message.has(VERSION_PROPERTY_KEY)) {
-      ((ObjectNode) message).put(VERSION_PROPERTY_KEY,
+    if (upgraded && message.has(VERSION_KEY)) {
+      ((ObjectNode) message).put(VERSION_KEY,
           String.format(TARGET_FORMAT, major, minor, patch));
     }
 
@@ -162,10 +162,10 @@ private void upgradeStatuses(ObjectNode system) {
   private void upgradeFirmware(ObjectNode system) {
     JsonNode firmware = system.remove("firmware");
     if (firmware != null) {
-      JsonNode version = ((ObjectNode) firmware).remove(VERSION_PROPERTY_KEY);
+      JsonNode version = ((ObjectNode) firmware).remove(VERSION_KEY);
       if (version != null && !system.has("software")) {
         ObjectNode softwareNode = new ObjectNode(NODE_FACTORY);
-        softwareNode.put("firmware", version.asText());
+        softwareNode.set("firmware", version);
         system.set("software", softwareNode);
       }
     }

validator/src/main/java/com/google/daq/mqtt/validator/MessageReadingClient.java

@@ -1,5 +1,7 @@
 package com.google.daq.mqtt.validator;
 
+import static com.google.udmi.util.Common.EXCEPTION_KEY;
+import static com.google.udmi.util.Common.MESSAGE_KEY;
 import static com.google.udmi.util.Common.SUBFOLDER_PROPERTY_KEY;
 import static com.google.udmi.util.Common.SUBTYPE_PROPERTY_KEY;
 
@@ -63,11 +65,12 @@ public MessageReadingClient(String registryId, String dirStr) {
     if (!messageDir.exists() || !messageDir.isDirectory()) {
       throw new RuntimeException("Message directory not found " + messageDir.getAbsolutePath());
     }
-    Arrays.stream(Objects.requireNonNull(messageDir.list())).forEach(this::prepDevice);
+    File devicesDir = new File(messageDir, "devices");
+    Arrays.stream(Objects.requireNonNull(devicesDir.list())).forEach(this::prepDevice);
   }
 
   private void prepDevice(String deviceId) {
-    File deviceDir = new File(messageDir, deviceId);
+    File deviceDir = new File(messageDir, "devices/" + deviceId);
     List<String> messages = Arrays.stream(Objects.requireNonNull(deviceDir.list()))
         .filter(filename -> filename.endsWith(TRACE_FILE_SUFFIX))
         .sorted()
@@ -97,14 +100,14 @@ private void prepNextMessage(String deviceId) {
   }
 
   private Map<String, Object> getMessageObject(String deviceId, String msgName) {
+    File deviceDir = new File(messageDir, "devices/" + deviceId);
+    File msgFile = new File(deviceDir, msgName);
     try {
-      File deviceDir = new File(messageDir, deviceId);
-      File msgFile = new File(deviceDir, msgName);
       @SuppressWarnings("unchecked")
       Map<String, Object> treeMap = OBJECT_MAPPER.readValue(msgFile, TreeMap.class);
       return treeMap;
     } catch (Exception e) {
-      return new ErrorContainer(e, msgName, lastValidTimestamp);
+      return new ErrorContainer(e, "Reading from " + msgFile.getAbsolutePath(), lastValidTimestamp);
     }
   }
 

validator/src/main/java/com/google/daq/mqtt/validator/Validator.java

@@ -4,13 +4,15 @@
 import static com.google.daq.mqtt.util.ConfigUtil.UDMI_TOOLS;
 import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
 import static com.google.daq.mqtt.util.ConfigUtil.readExecutionConfiguration;
+import static com.google.udmi.util.Common.EXCEPTION_KEY;
 import static com.google.udmi.util.Common.GCP_REFLECT_KEY_PKCS8;
+import static com.google.udmi.util.Common.MESSAGE_KEY;
 import static com.google.udmi.util.Common.NO_SITE;
 import static com.google.udmi.util.Common.STATE_QUERY_TOPIC;
 import static com.google.udmi.util.Common.SUBFOLDER_PROPERTY_KEY;
 import static com.google.udmi.util.Common.SUBTYPE_PROPERTY_KEY;
-import static com.google.udmi.util.Common.TIMESTAMP_PROPERTY_KEY;
-import static com.google.udmi.util.Common.VERSION_PROPERTY_KEY;
+import static com.google.udmi.util.Common.TIMESTAMP_KEY;
+import static com.google.udmi.util.Common.VERSION_KEY;
 import static com.google.udmi.util.Common.removeNextArg;
 import static com.google.udmi.util.JsonUtil.JSON_SUFFIX;
 import static com.google.udmi.util.JsonUtil.OBJECT_MAPPER;
@@ -92,9 +94,6 @@
  */
 public class Validator {
 
-  public static final String EXCEPTION_KEY = "exception";
-  public static final String TIMESTAMP_KEY = "timestamp";
-  public static final String MESSAGE_KEY = "message";
   private static final String ERROR_FORMAT_INDENT = "  ";
   private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas";
   private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s";
@@ -103,7 +102,6 @@ public class Validator {
   private static final String MESSAGE_FILE_FORMAT = "%s.json";
   private static final String SCHEMA_SKIP_FORMAT = "Unknown schema subFolder '%s' for %s";
   private static final String ENVELOPE_SCHEMA_ID = "envelope";
-  private static final String METADATA_JSON = "metadata.json";
   private static final String DEVICES_SUBDIR = "devices";
   private static final String DEVICE_REGISTRY_ID_KEY = "deviceRegistryId";
   private static final String UNKNOWN_FOLDER_DEFAULT = "unknown";
@@ -462,8 +460,8 @@ private String sanitizeMessage(String schemaName, Map<String, Object> message) {
     String nonce = (String) message.remove(SequenceBase.CONFIG_NONCE_KEY);
     message.values().forEach(this::sanitizeBlock);
     if (schemaName.startsWith(CONFIG_PREFIX) || schemaName.startsWith(STATE_PREFIX)) {
-      message.remove(VERSION_PROPERTY_KEY);
-      message.remove(TIMESTAMP_PROPERTY_KEY);
+      message.remove(VERSION_KEY);
+      message.remove(TIMESTAMP_KEY);
     }
     return nonce;
   }
@@ -496,8 +494,8 @@ private ReportingDevice validateUpdate(
         base64Devices.add(deviceId);
       }
 
-      if (message.containsKey(Common.EXCEPTION_KEY)) {
-        device.addError((Exception) message.get(Common.EXCEPTION_KEY), attributes,
+      if (message.containsKey(EXCEPTION_KEY)) {
+        device.addError((Exception) message.get(EXCEPTION_KEY), attributes,
             Category.VALIDATION_DEVICE_RECEIVE);
         return device;
       }
@@ -506,7 +504,7 @@ private ReportingDevice validateUpdate(
       upgradeMessage(schemaName, message);
       prepareDeviceOutDir(message, attributes, deviceId, schemaName);
 
-      String timeString = (String) message.get(TIMESTAMP_PROPERTY_KEY);
+      String timeString = (String) message.get(TIMESTAMP_KEY);
       String subTypeRaw = Optional.ofNullable(attributes.get(SUBTYPE_PROPERTY_KEY))
           .orElse(UNKNOWN_TYPE_DEFAULT);
       if (timeString != null && LAST_SEEN_SUBTYPES.contains(SubType.fromValue(subTypeRaw))) {

validator/src/test/java/com/google/daq/mqtt/validator/BasicTest.java

@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.validator;
 
-import static com.google.udmi.util.Common.TIMESTAMP_PROPERTY_KEY;
+import static com.google.udmi.util.Common.TIMESTAMP_KEY;
 import static com.google.udmi.util.JsonUtil.getTimestamp;
 import static com.google.udmi.util.JsonUtil.safeSleep;
 import static org.junit.Assert.assertEquals;
@@ -141,7 +141,7 @@ public void lastSeenUpdate() {
     ValidationState report = getValidationReport();
     DeviceValidationEvent deviceValidationEvent = report.devices.get(TestCommon.DEVICE_ID);
     Date lastSeen = deviceValidationEvent.last_seen;
-    Instant parse = Instant.parse((String) eventBundle.message.get(TIMESTAMP_PROPERTY_KEY));
+    Instant parse = Instant.parse((String) eventBundle.message.get(TIMESTAMP_KEY));
     assertEquals("device last seen", getTimestamp(Date.from(parse)), getTimestamp(lastSeen));
   }