From 74a72bd993695c673486f8102e831a08d4035e7e Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Tue, 2 Aug 2022 17:06:23 -0700
Subject: [PATCH 01/49] IoT COnfig redirect

---
 pubber/.idea/runConfigurations/Pubber.xml   |  2 +-
 pubber/src/main/java/daq/pubber/Pubber.java | 57 +++++++++++++++++++--
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/pubber/.idea/runConfigurations/Pubber.xml b/pubber/.idea/runConfigurations/Pubber.xml
index e60f811adc..71f95c4348 100644
--- a/pubber/.idea/runConfigurations/Pubber.xml
+++ b/pubber/.idea/runConfigurations/Pubber.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="Pubber" type="Application" factoryName="Application" nameIsGenerated="true">
     <option name="MAIN_CLASS_NAME" value="daq.pubber.Pubber" />
     <module name="pubber" />
-    <option name="PROGRAM_PARAMETERS" value="bos-udmi-proto sites/udmi_site_model AHU-22 54335" />
+    <option name="PROGRAM_PARAMETERS" value="bos-peringknife-dev sites/udmi_site_model AHU-1 54335" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
     <extension name="coverage">
       <pattern>
diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java
index 575e91e5c0..aeec9da092 100644
--- a/pubber/src/main/java/daq/pubber/Pubber.java
+++ b/pubber/src/main/java/daq/pubber/Pubber.java
@@ -43,6 +43,7 @@
 import org.apache.http.ConnectionClosedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import udmi.schema.BlobBlobsetConfig;
 import udmi.schema.CloudModel.Auth_type;
 import udmi.schema.Config;
 import udmi.schema.DiscoveryConfig;
@@ -73,6 +74,8 @@ public class Pubber {
 
   public static final int SCAN_DURATION_SEC = 10;
   public static final String DISCOVERY_ID = "RANDOM_ID";
+  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
+  public static final String BLOB_COMMIT_PHASE = "commit";
   private static final String UDMI_VERSION = "1.3.14";
   private static final Logger LOG = LoggerFactory.getLogger(Pubber.class);
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
@@ -136,6 +139,8 @@ public class Pubber {
   private Consumer<String> onDone;
   private boolean publishingLog;
   private Map<String, Metadata> allMetadata;
+  private String appliedEndpoint;
+  private EndpointConfiguration extractedEndpoint;
 
   /**
    * Start an instance from a configuration file.
@@ -486,7 +491,7 @@ private void sendMessages() {
       updatePoints();
       sendDeviceMessage();
       flushDirtyState();
-      // Some things can't be done from a on-message callback, so do them here instead.
+      // Some things can't be done from an on-message callback, so do them here instead.
       maybeRedirectEndpoint();
     } catch (Exception e) {
       error("Fatal error during execution", e);
@@ -576,6 +581,7 @@ private void initializeMqtt() {
     }
     Preconditions.checkState(mqttPublisher == null, "mqttPublisher already defined");
     ensureKeyBytes();
+    appliedEndpoint = toJson(configuration.endpoint);
     mqttPublisher = new MqttPublisher(configuration, this::publisherException);
     if (configuration.gatewayId != null) {
       mqttPublisher.registerHandler(configuration.gatewayId, CONFIG_TOPIC,
@@ -587,6 +593,14 @@ private void initializeMqtt() {
         this::configHandler, Config.class);
   }
 
+  private String toJson(Object configuration) {
+    try {
+      return OBJECT_MAPPER.writeValueAsString(configuration);
+    } catch (Exception e) {
+      throw new RuntimeException("While converting object to string", e);
+    }
+  }
+
   private void ensureKeyBytes() {
     if (configuration.keyBytes != null) {
       return;
@@ -594,6 +608,7 @@ private void ensureKeyBytes() {
     Preconditions.checkNotNull(configuration.keyFile, "configuration keyFile not defined");
     info("Loading device key bytes from " + configuration.keyFile);
     configuration.keyBytes = getFileBytes(configuration.keyFile);
+    configuration.keyFile = null;
   }
 
   private String getDeviceKeyPrefix() {
@@ -705,6 +720,7 @@ private void processConfigUpdate(Config config) {
       actualInterval = updateSystemConfig(config.pointset);
       updatePointsetConfig(config.pointset);
       updateDiscoveryConfig(config.discovery);
+      extractedEndpoint = extractEndpointBlobConfig();
     } else {
       info(getTimestamp() + " defaulting empty config");
       actualInterval = DEFAULT_REPORT_SEC * 1000;
@@ -712,15 +728,32 @@ private void processConfigUpdate(Config config) {
     maybeRestartExecutor(actualInterval);
   }
 
+  private EndpointConfiguration extractEndpointBlobConfig() {
+    try {
+      String iot_config = extractConfigBlob(IOT_ENDPOINT_BLOB_NAME);
+      if (iot_config == null) {
+        return null;
+      }
+      return OBJECT_MAPPER.readValue(iot_config, EndpointConfiguration.class);
+    } catch (Exception e) {
+      throw new RuntimeException("While extracting endpoint blob config", e);
+    }
+  }
+
   private void maybeRedirectEndpoint() {
     String redirectRegistry = configuration.options.redirectRegistry;
-    if (redirectRegistry == null || redirectRegistry.equals(configuration.endpoint.registryId)
-        || configLatch.getCount() > 0) {
+    if (extractedEndpoint != null && !toJson(extractedEndpoint).equals(appliedEndpoint)) {
+      info("New config blob endpoint detected");
+      configuration.endpoint = extractedEndpoint;
+    } else if (redirectRegistry != null && configLatch.getCount() <= 0 &&
+        !redirectRegistry.equals(configuration.endpoint.registryId)) {
+      info("Mismatched redirectRegistry detected");
+      configuration.endpoint.registryId = redirectRegistry;
+    } else {
       return;
     }
     try {
       disconnectMqtt();
-      configuration.endpoint.registryId = redirectRegistry;
       initializeMqtt();
       startConnection(onDone);
     } catch (Exception e) {
@@ -728,6 +761,22 @@ private void maybeRedirectEndpoint() {
     }
   }
 
+  private String extractConfigBlob(String blobName) {
+    try {
+      if (deviceConfig == null || deviceConfig.blobset == null || deviceConfig.blobset.blobs == null) {
+        return null;
+      }
+      BlobBlobsetConfig blobBlobsetConfig = deviceConfig.blobset.blobs.get(blobName);
+      if (blobBlobsetConfig != null && BLOB_COMMIT_PHASE.equals(blobBlobsetConfig.phase)
+          && blobBlobsetConfig.base64 != null) {
+        return new String(Base64.getDecoder().decode(blobBlobsetConfig.base64));
+      }
+      return null;
+    } catch (Exception e) {
+      throw new RuntimeException("While extracting config blob " + blobName, e);
+    }
+  }
+
   private void updateDiscoveryConfig(DiscoveryConfig discovery) {
     DiscoveryConfig discoveryConfig = discovery == null ? new DiscoveryConfig() : discovery;
     if (deviceState.discovery == null) {

From 9c978e5f379d73ad2860581f6f63530b6f0ddf28 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 04:41:24 -0700
Subject: [PATCH 02/49] Reset device config

---
 bin/test_redirect | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/bin/test_redirect b/bin/test_redirect
index 026e76d45c..a2fdee7d88 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -21,6 +21,10 @@ fi
 site_path=sites/udmi_site_model
 device_id=AHU-22 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
 
+site_config=$site_path/cloud_iot_config.json
+cloud_region=$(jq -r .cloud_region $site_config)
+registry_id=$(jq -r .registry_id $site_config)
+
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
@@ -33,6 +37,16 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
+device_config=/tmp/${device_id}_config.json
+cp sites/udmi_site_model/devices/${device_id}/out/generated_config.json $device_config
+now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
+echo Setting config timestamp $now_date
+jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
+
+echo Resetting device $device_id config...
+gcloud iot devices configs update --config-file=$device_config \
+       --device=$device_id --region=$cloud_region --registry=$registry_id --project=$project_id
+
 echo Writing pubber output to $PUBBER_OUT
 echo bin/pubber $site_path $project_id $device_id $serial_no
 

From 358e73c7504bfbe0d62f1ecdd7efd9b9bd3852d8 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 05:02:36 -0700
Subject: [PATCH 03/49] Style checks

---
 pubber/src/main/java/daq/pubber/Pubber.java | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java
index aeec9da092..5f6f1d18e9 100644
--- a/pubber/src/main/java/daq/pubber/Pubber.java
+++ b/pubber/src/main/java/daq/pubber/Pubber.java
@@ -730,11 +730,11 @@ private void processConfigUpdate(Config config) {
 
   private EndpointConfiguration extractEndpointBlobConfig() {
     try {
-      String iot_config = extractConfigBlob(IOT_ENDPOINT_BLOB_NAME);
-      if (iot_config == null) {
+      String iotConfig = extractConfigBlob(IOT_ENDPOINT_BLOB_NAME);
+      if (iotConfig == null) {
         return null;
       }
-      return OBJECT_MAPPER.readValue(iot_config, EndpointConfiguration.class);
+      return OBJECT_MAPPER.readValue(iotConfig, EndpointConfiguration.class);
     } catch (Exception e) {
       throw new RuntimeException("While extracting endpoint blob config", e);
     }
@@ -745,8 +745,8 @@ private void maybeRedirectEndpoint() {
     if (extractedEndpoint != null && !toJson(extractedEndpoint).equals(appliedEndpoint)) {
       info("New config blob endpoint detected");
       configuration.endpoint = extractedEndpoint;
-    } else if (redirectRegistry != null && configLatch.getCount() <= 0 &&
-        !redirectRegistry.equals(configuration.endpoint.registryId)) {
+    } else if (redirectRegistry != null && configLatch.getCount() <= 0
+        && !redirectRegistry.equals(configuration.endpoint.registryId)) {
       info("Mismatched redirectRegistry detected");
       configuration.endpoint.registryId = redirectRegistry;
     } else {
@@ -763,7 +763,8 @@ private void maybeRedirectEndpoint() {
 
   private String extractConfigBlob(String blobName) {
     try {
-      if (deviceConfig == null || deviceConfig.blobset == null || deviceConfig.blobset.blobs == null) {
+      if (deviceConfig == null || deviceConfig.blobset == null
+          || deviceConfig.blobset.blobs == null) {
         return null;
       }
       BlobBlobsetConfig blobBlobsetConfig = deviceConfig.blobset.blobs.get(blobName);

From 129633abf39e847bbd2af3ceb1022395ad4a0039 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 05:04:07 -0700
Subject: [PATCH 04/49] Adding apt-get utils

---
 bin/setup_base | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/setup_base b/bin/setup_base
index a9d150f00b..ce8da695fe 100755
--- a/bin/setup_base
+++ b/bin/setup_base
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sudo apt-get install -y hxtools
+sudo apt-get install -y hxtools moreutils
 
 python3 -m venv venv
 

From 9f05ac5e9b5b40c50e8b54aed5de3c7c1cf7f372 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 06:00:44 -0700
Subject: [PATCH 05/49] Adding needs clause

---
 .github/workflows/testing.yml                 |  1 +
 .../.idea/runConfigurations/Reflector.xml     | 17 ++++
 .../google/daq/mqtt/registrar/Registrar.java  |  4 +-
 .../java/com/google/daq/mqtt/util/Common.java | 34 ++++++++
 .../google/daq/mqtt/validator/Reflector.java  | 82 +++++++++++++++++++
 .../daq/mqtt/validator/SequenceValidator.java |  3 +-
 .../google/daq/mqtt/validator/Validator.java  | 38 ++-------
 7 files changed, 146 insertions(+), 33 deletions(-)
 create mode 100644 validator/.idea/runConfigurations/Reflector.xml
 create mode 100644 validator/src/main/java/com/google/daq/mqtt/util/Common.java
 create mode 100644 validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 97d82955a3..f0f0add296 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -78,6 +78,7 @@ jobs:
     name: Sequence tests
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    needs: redirect  # Access to UDMI-REFLECTOR is mutually exclusive
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v1
diff --git a/validator/.idea/runConfigurations/Reflector.xml b/validator/.idea/runConfigurations/Reflector.xml
new file mode 100644
index 0000000000..3df26cd507
--- /dev/null
+++ b/validator/.idea/runConfigurations/Reflector.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Reflector" type="Application" factoryName="Application" nameIsGenerated="true">
+    <option name="MAIN_CLASS_NAME" value="com.google.daq.mqtt.validator.Reflector" />
+    <module name="validator" />
+    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="com.google.daq.mqtt.validator.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>
\ No newline at end of file
diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
index 5932fbce9a..a8f4e88ff3 100644
--- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
+++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.registrar;
 
-import static com.google.daq.mqtt.validator.Validator.NO_SITE;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -9,7 +9,6 @@
 import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
-import com.github.fge.jsonschema.core.report.ProcessingReport;
 import com.github.fge.jsonschema.main.JsonSchema;
 import com.github.fge.jsonschema.main.JsonSchemaFactory;
 import com.google.api.services.cloudiot.v1.model.Device;
@@ -30,7 +29,6 @@
 import java.io.FilenameFilter;
 import java.io.InputStream;
 import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
 import java.math.BigInteger;
 import java.net.URI;
 import java.time.Duration;
diff --git a/validator/src/main/java/com/google/daq/mqtt/util/Common.java b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
new file mode 100644
index 0000000000..4e53e81a01
--- /dev/null
+++ b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
@@ -0,0 +1,34 @@
+package com.google.daq.mqtt.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
+import java.util.List;
+import java.util.MissingFormatArgumentException;
+
+public abstract class Common {
+
+  public static final String STATE_QUERY_TOPIC = "query/state";
+  public static final String TIMESTAMP_ATTRIBUTE = "timestamp";
+  public static final String NO_SITE = "--";
+  public static final ObjectMapper OBJECT_MAPPER =
+      new ObjectMapper()
+          .enable(Feature.ALLOW_COMMENTS)
+          .enable(SerializationFeature.INDENT_OUTPUT)
+          .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+          .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
+          .setDateFormat(new ISO8601DateFormat())
+          .setSerializationInclusion(Include.NON_NULL);
+  public static final String JSON_SUFFIX = ".json";
+  public static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
+
+  public static String removeNextArg(List<String> argList) {
+    if (argList.isEmpty()) {
+      throw new MissingFormatArgumentException("Missing argument");
+    }
+    return argList.remove(0);
+  }
+}
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
new file mode 100644
index 0000000000..18a87af700
--- /dev/null
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -0,0 +1,82 @@
+package com.google.daq.mqtt.validator;
+
+import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
+import static com.google.daq.mqtt.util.Common.removeNextArg;
+
+import com.google.bos.iot.core.proxy.IotReflectorClient;
+import com.google.daq.mqtt.util.CloudIotConfig;
+import com.google.daq.mqtt.util.CloudIotManager;
+import com.google.daq.mqtt.util.ConfigUtil;
+import com.google.daq.mqtt.util.FileDataSink;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class Reflector {
+
+  private String projectId;
+  private String siteDir;
+  private CloudIotConfig cloudIotConfig;
+  private File baseDir;
+  private IotReflectorClient client;
+
+  public Reflector(List<String> argsList) {
+    parseArgs(argsList);
+  }
+
+  public static void main(String[] args) {
+    Reflector reflector = new Reflector(Arrays.asList(args));
+    reflector.initialize();
+    reflector.reflect();
+  }
+
+  private void reflect() {
+    client.publish();
+  }
+
+  private void initialize() {
+    String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath();
+    System.err.println("Loading reflector key file from " + keyFile);
+    client = new IotReflectorClient(projectId, cloudIotConfig, keyFile);
+  }
+
+  private List<String> parseArgs(List<String> argsList) {
+    List<String> listCopy = new ArrayList<>(argsList);
+    while (!listCopy.isEmpty()) {
+      String option = removeNextArg(listCopy);
+      try {
+        switch (option) {
+          case "-p":
+            projectId = removeNextArg(listCopy);
+            break;
+          case "-s":
+            setSiteDir(removeNextArg(listCopy));
+            break;
+        }
+      } catch (Exception e) {
+        throw new RuntimeException("While processing option " + option, e);
+      }
+    }
+    return listCopy;
+  }
+
+  /**
+   * Set the site directory to use for this run.
+   *
+   * @param siteDir site model directory
+   */
+  public void setSiteDir(String siteDir) {
+    if (NO_SITE.equals(siteDir)) {
+      siteDir = null;
+      baseDir = new File(".");
+    } else {
+      this.siteDir = siteDir;
+      baseDir = new File(siteDir);
+      File cloudConfig = new File(siteDir, "cloud_iot_config.json");
+      cloudIotConfig = CloudIotManager.validate(ConfigUtil.readCloudIotConfig(cloudConfig),
+          projectId);
+    }
+  }
+}
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
index 685e3be544..6a57d2681a 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
@@ -13,6 +13,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.daq.mqtt.util.CloudIotConfig;
 import com.google.daq.mqtt.util.ConfigUtil;
+import com.google.daq.mqtt.util.Common;
 import com.google.daq.mqtt.util.ValidatorConfig;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -454,7 +455,7 @@ private String writeLogEntry(Entry logEntry, String filename) {
   }
 
   protected void queryState() {
-    client.publish(deviceId, Validator.STATE_QUERY_TOPIC, EMPTY_MESSAGE);
+    client.publish(deviceId, Common.STATE_QUERY_TOPIC, EMPTY_MESSAGE);
   }
 
   /**
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
index b2220e2ee8..a2ef5db888 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
@@ -1,15 +1,16 @@
 package com.google.daq.mqtt.validator;
 
+import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8;
 import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
+import static com.google.daq.mqtt.util.Common.JSON_SUFFIX;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
+import static com.google.daq.mqtt.util.Common.OBJECT_MAPPER;
+import static com.google.daq.mqtt.util.Common.STATE_QUERY_TOPIC;
+import static com.google.daq.mqtt.util.Common.TIMESTAMP_ATTRIBUTE;
+import static com.google.daq.mqtt.util.Common.removeNextArg;
 
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.core.JsonParser.Feature;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.github.fge.jsonschema.core.exceptions.ProcessingException;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
@@ -25,13 +26,13 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.daq.mqtt.util.CloudIotConfig;
 import com.google.daq.mqtt.util.CloudIotManager;
+import com.google.daq.mqtt.util.Common;
 import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.ExceptionMap;
 import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
 import com.google.daq.mqtt.util.FileDataSink;
 import com.google.daq.mqtt.util.PubSubClient;
 import com.google.daq.mqtt.util.ValidationException;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -39,7 +40,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.PrintStream;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
@@ -78,19 +78,7 @@
  */
 public class Validator {
 
-  public static final String STATE_QUERY_TOPIC = "query/state";
-  public static final String TIMESTAMP_ATTRIBUTE = "timestamp";
-  public static final String NO_SITE = "--";
-  private static final ObjectMapper OBJECT_MAPPER =
-      new ObjectMapper()
-          .enable(Feature.ALLOW_COMMENTS)
-          .enable(SerializationFeature.INDENT_OUTPUT)
-          .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
-          .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
-          .setDateFormat(new ISO8601DateFormat())
-          .setSerializationInclusion(Include.NON_NULL);
   private static final String ERROR_FORMAT_INDENT = "  ";
-  private static final String JSON_SUFFIX = ".json";
   private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas";
   private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s";
   private static final String DEVICE_FILE_FORMAT = "devices/%s";
@@ -104,7 +92,6 @@ public class Validator {
   private static final String UNKNOWN_FOLDER_DEFAULT = "unknown";
   private static final String EVENT_POINTSET = "event_pointset";
   private static final String STATE_POINTSET = "state_pointset";
-  private static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
   private static final String EMPTY_MESSAGE = "{}";
   private static final String CONFIG_PREFIX = "config_";
   private static final String STATE_PREFIX = "state_";
@@ -181,7 +168,7 @@ public static void main(String[] args) {
   }
 
   private void parseArgs(List<String> argList) {
-    while (argList.size() > 0) {
+    while (!argList.isEmpty()) {
       String option = removeNextArg(argList);
       try {
         switch (option) {
@@ -232,13 +219,6 @@ private void validateMessageTrace(String messageDir) {
     reportAfterEveryMessage = true;
   }
 
-  private String removeNextArg(List<String> argList) {
-    if (argList.isEmpty()) {
-      throw new MissingFormatArgumentException("Missing argument");
-    }
-    return argList.remove(0);
-  }
-
   /**
    * Set the site directory to use for this validation run.
    *

From 713a0c3c3c91c2470b3b8b6294fa2f2a433027b5 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 06:06:11 -0700
Subject: [PATCH 06/49] Fix compiile

---
 .../src/main/java/com/google/daq/mqtt/validator/Reflector.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
index 18a87af700..103c8acbcc 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -33,7 +33,6 @@ public static void main(String[] args) {
   }
 
   private void reflect() {
-    client.publish();
   }
 
   private void initialize() {

From c1b2b0e13a30e247dc28e9b13c7f134abbaef0e1 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 06:07:48 -0700
Subject: [PATCH 07/49] Disable reconfig

---
 bin/test_redirect | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index a2fdee7d88..56bd812a4d 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -43,9 +43,9 @@ now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
 echo Setting config timestamp $now_date
 jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
 
-echo Resetting device $device_id config...
-gcloud iot devices configs update --config-file=$device_config \
-       --device=$device_id --region=$cloud_region --registry=$registry_id --project=$project_id
+#echo Resetting device $device_id config...
+#gcloud iot devices configs update --config-file=$device_config \
+#       --device=$device_id --region=$cloud_region --registry=$registry_id --project=$project_id
 
 echo Writing pubber output to $PUBBER_OUT
 echo bin/pubber $site_path $project_id $device_id $serial_no

From dd8e1c8475dfa285db4501d7e864e4a8dae4b1af Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 06:20:08 -0700
Subject: [PATCH 08/49] Linty fixes

---
 .../java/com/google/daq/mqtt/util/Common.java    |  9 +++++++++
 .../com/google/daq/mqtt/validator/Reflector.java | 16 +++++++++++++++-
 .../daq/mqtt/validator/SequenceValidator.java    |  2 +-
 .../com/google/daq/mqtt/validator/Validator.java |  3 +--
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/validator/src/main/java/com/google/daq/mqtt/util/Common.java b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
index 4e53e81a01..2894868efd 100644
--- a/validator/src/main/java/com/google/daq/mqtt/util/Common.java
+++ b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
@@ -9,6 +9,9 @@
 import java.util.List;
 import java.util.MissingFormatArgumentException;
 
+/**
+ * Collection of common constants and minor utilities.
+ */
 public abstract class Common {
 
   public static final String STATE_QUERY_TOPIC = "query/state";
@@ -25,6 +28,12 @@ public abstract class Common {
   public static final String JSON_SUFFIX = ".json";
   public static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
 
+  /**
+   * Remove the next item from the list in an exception-safe way.
+   *
+   * @param argList list of arguments
+   * @return removed argument
+   */
   public static String removeNextArg(List<String> argList) {
     if (argList.isEmpty()) {
       throw new MissingFormatArgumentException("Missing argument");
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
index 103c8acbcc..61ca82f268 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -8,12 +8,14 @@
 import com.google.daq.mqtt.util.CloudIotConfig;
 import com.google.daq.mqtt.util.CloudIotManager;
 import com.google.daq.mqtt.util.ConfigUtil;
-import com.google.daq.mqtt.util.FileDataSink;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+/**
+ * General utility for working with UDMI Reflector messages.
+ */
 public class Reflector {
 
   private String projectId;
@@ -22,10 +24,20 @@ public class Reflector {
   private File baseDir;
   private IotReflectorClient client;
 
+  /**
+   * Create an instance of the Reflector class.
+   *
+   * @param argsList command-line arguments
+   */
   public Reflector(List<String> argsList) {
     parseArgs(argsList);
   }
 
+  /**
+   * Let's go.
+   *
+   * @param args command-line arguments
+   */
   public static void main(String[] args) {
     Reflector reflector = new Reflector(Arrays.asList(args));
     reflector.initialize();
@@ -53,6 +65,8 @@ private List<String> parseArgs(List<String> argsList) {
           case "-s":
             setSiteDir(removeNextArg(listCopy));
             break;
+          default:
+            throw new IllegalArgumentException("Unknown command line option " + option);
         }
       } catch (Exception e) {
         throw new RuntimeException("While processing option " + option, e);
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
index 6a57d2681a..6dd894c96d 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
@@ -12,8 +12,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.daq.mqtt.util.CloudIotConfig;
-import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.Common;
+import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.ValidatorConfig;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
index a2ef5db888..d85e5eaaa8 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
@@ -1,13 +1,13 @@
 package com.google.daq.mqtt.validator;
 
 import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8;
-import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
 import static com.google.daq.mqtt.util.Common.JSON_SUFFIX;
 import static com.google.daq.mqtt.util.Common.NO_SITE;
 import static com.google.daq.mqtt.util.Common.OBJECT_MAPPER;
 import static com.google.daq.mqtt.util.Common.STATE_QUERY_TOPIC;
 import static com.google.daq.mqtt.util.Common.TIMESTAMP_ATTRIBUTE;
 import static com.google.daq.mqtt.util.Common.removeNextArg;
+import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -26,7 +26,6 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.daq.mqtt.util.CloudIotConfig;
 import com.google.daq.mqtt.util.CloudIotManager;
-import com.google.daq.mqtt.util.Common;
 import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.ExceptionMap;
 import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;

From c4f400a91c016bdd92d6c602946c465a4b6c3730 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 14:44:23 -0700
Subject: [PATCH 09/49] Reflector tweaking

---
 .../.idea/runConfigurations/Reflector.xml     |  2 +-
 .../iot/core/proxy/IotReflectorClient.java    |  2 +-
 .../google/daq/mqtt/validator/Reflector.java  | 38 +++++++++++++++++--
 .../daq/mqtt/validator/SequenceValidator.java |  1 +
 4 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/validator/.idea/runConfigurations/Reflector.xml b/validator/.idea/runConfigurations/Reflector.xml
index 3df26cd507..b4f9770b09 100644
--- a/validator/.idea/runConfigurations/Reflector.xml
+++ b/validator/.idea/runConfigurations/Reflector.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="Reflector" type="Application" factoryName="Application" nameIsGenerated="true">
     <option name="MAIN_CLASS_NAME" value="com.google.daq.mqtt.validator.Reflector" />
     <module name="validator" />
-    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model" />
+    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model -d AHU-1 update/config:config.json" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
     <extension name="coverage">
       <pattern>
diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
index 2316ce69c1..c8e2b2b506 100644
--- a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
+++ b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
@@ -59,7 +59,7 @@ public IotReflectorClient(String projectId, CloudIotConfig iotConfig, String key
       mqttPublisher = new MqttPublisher(projectId, cloudRegion, UDMS_REFLECT,
           siteName, keyBytes, IOT_KEY_ALGORITHM, this::messageHandler, this::errorHandler);
     } catch (Exception e) {
-      throw new RuntimeException("While connecting subscription " + subscriptionId, e);
+      throw new RuntimeException("While connecting MQTT endpoint " + subscriptionId, e);
     }
 
     active = true;
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
index 61ca82f268..6b716a865c 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -9,6 +9,7 @@
 import com.google.daq.mqtt.util.CloudIotManager;
 import com.google.daq.mqtt.util.ConfigUtil;
 import java.io.File;
+import java.io.FileInputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -18,11 +19,13 @@
  */
 public class Reflector {
 
+  private final List<String> reflectCommands;
   private String projectId;
   private String siteDir;
   private CloudIotConfig cloudIotConfig;
   private File baseDir;
   private IotReflectorClient client;
+  private String deviceId;
 
   /**
    * Create an instance of the Reflector class.
@@ -30,7 +33,7 @@ public class Reflector {
    * @param argsList command-line arguments
    */
   public Reflector(List<String> argsList) {
-    parseArgs(argsList);
+    reflectCommands = parseArgs(argsList);
   }
 
   /**
@@ -45,6 +48,33 @@ public static void main(String[] args) {
   }
 
   private void reflect() {
+    System.err.printf("Reflecting %d directives...%n", reflectCommands.size());
+    while (!reflectCommands.isEmpty()) {
+      reflect(reflectCommands.remove(0));
+    }
+  }
+
+  private void reflect(String directive) {
+    System.err.printf("Reflecting %s%n", directive);
+    String[] parts = directive.split(":", 2);
+    if (parts.length != 2) {
+      throw new RuntimeException("Expected topic:file format reflect directive " + directive);
+    }
+    File file = new File(parts[1]);
+    reflect(parts[0], file);
+  }
+
+  private void reflect(String topic, File dataFile) {
+    try (FileInputStream fis = new FileInputStream(dataFile)) {
+      String data = new String(fis.readAllBytes());
+      reflect(topic, data);
+    } catch (Exception e) {
+      throw new RuntimeException("While processing input file " + dataFile.getAbsolutePath());
+    }
+  }
+
+  private void reflect(String topic, String data) {
+    client.publish(deviceId, topic, data);
   }
 
   private void initialize() {
@@ -65,14 +95,16 @@ private List<String> parseArgs(List<String> argsList) {
           case "-s":
             setSiteDir(removeNextArg(listCopy));
             break;
+          case "-d":
+            deviceId = removeNextArg(listCopy);
           default:
-            throw new IllegalArgumentException("Unknown command line option " + option);
+            return listCopy; // default case is the remaining list of reflection directives
         }
       } catch (Exception e) {
         throw new RuntimeException("While processing option " + option, e);
       }
     }
-    return listCopy;
+    throw new IllegalArgumentException("No reflect directives specified!");
   }
 
   /**
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
index 6dd894c96d..317c27268b 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
@@ -149,6 +149,7 @@ public abstract class SequenceValidator {
     resultSummary.delete();
     System.err.println("Writing results to " + resultSummary.getAbsolutePath());
 
+    System.err.printf("Loading reflector key file from %s%n", new File(key_file).getAbsolutePath());
     System.err.printf("Validating against device %s serial %s%n", deviceId, serialNo);
     client = new IotReflectorClient(projectId, cloudIotConfig, key_file);
     setReflectorState();

From 9e63742cfa295b091f0ee1054b78e626cee833ba Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 16:56:08 -0700
Subject: [PATCH 10/49] Working reflector

---
 validator/.idea/runConfigurations/Reflector.xml   |  2 +-
 .../com/google/daq/mqtt/validator/Reflector.java  | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/validator/.idea/runConfigurations/Reflector.xml b/validator/.idea/runConfigurations/Reflector.xml
index b4f9770b09..d25ad94d5a 100644
--- a/validator/.idea/runConfigurations/Reflector.xml
+++ b/validator/.idea/runConfigurations/Reflector.xml
@@ -2,7 +2,7 @@
   <configuration default="false" name="Reflector" type="Application" factoryName="Application" nameIsGenerated="true">
     <option name="MAIN_CLASS_NAME" value="com.google.daq.mqtt.validator.Reflector" />
     <module name="validator" />
-    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model -d AHU-1 update/config:config.json" />
+    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model -d AHU-1 update/config:sites/udmi_site_model/devices/AHU-1/out/generated_config.json" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
     <extension name="coverage">
       <pattern>
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
index 6b716a865c..ef88026b9d 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -45,6 +45,16 @@ public static void main(String[] args) {
     Reflector reflector = new Reflector(Arrays.asList(args));
     reflector.initialize();
     reflector.reflect();
+    reflector.shutdown();
+  }
+
+  private void shutdown() {
+    try {
+      Thread.sleep(5000);
+    } catch (Exception e) {
+      throw new RuntimeException("While sleeping", e);
+    }
+    client.close();
   }
 
   private void reflect() {
@@ -52,6 +62,7 @@ private void reflect() {
     while (!reflectCommands.isEmpty()) {
       reflect(reflectCommands.remove(0));
     }
+    System.err.println("Done with all reflective directives!");
   }
 
   private void reflect(String directive) {
@@ -69,7 +80,7 @@ private void reflect(String topic, File dataFile) {
       String data = new String(fis.readAllBytes());
       reflect(topic, data);
     } catch (Exception e) {
-      throw new RuntimeException("While processing input file " + dataFile.getAbsolutePath());
+      throw new RuntimeException("While processing input file " + dataFile.getAbsolutePath(), e);
     }
   }
 
@@ -97,7 +108,9 @@ private List<String> parseArgs(List<String> argsList) {
             break;
           case "-d":
             deviceId = removeNextArg(listCopy);
+            break;
           default:
+            listCopy.add(option);
             return listCopy; // default case is the remaining list of reflection directives
         }
       } catch (Exception e) {

From 37cd4f868172c952cc344e2882cbafa99869a0d5 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 21:11:57 -0700
Subject: [PATCH 11/49] Adding reflector to test

---
 bin/test_redirect       |  5 ++---
 validator/bin/reflector | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)
 create mode 100755 validator/bin/reflector

diff --git a/bin/test_redirect b/bin/test_redirect
index 56bd812a4d..825226fa5d 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -43,9 +43,8 @@ now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
 echo Setting config timestamp $now_date
 jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
 
-#echo Resetting device $device_id config...
-#gcloud iot devices configs update --config-file=$device_config \
-#       --device=$device_id --region=$cloud_region --registry=$registry_id --project=$project_id
+echo Resetting device $device_id config...
+validator/bin/reflector $site_path $project_id $device_id update/config:$device_config
 
 echo Writing pubber output to $PUBBER_OUT
 echo bin/pubber $site_path $project_id $device_id $serial_no
diff --git a/validator/bin/reflector b/validator/bin/reflector
new file mode 100755
index 0000000000..7e68092f2d
--- /dev/null
+++ b/validator/bin/reflector
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/../..)
+
+if [[ $# < 4 ]]; then
+    echo Usage: $0 site_dir project_id device_id directive [directives...]
+    echo "  Directive is something like update/config:sites/udmi_site_model/devices/AHU-1/out/generated_config.json"
+    false
+fi
+
+site_dir=$(realpath $1)
+project_id=$2
+device_id=$3
+shift 3
+cd $ROOT
+
+validator/bin/build
+
+jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
+mainclass=com.google.daq.mqtt.validator.Reflector
+
+cmd="java -cp $jarfile $mainclass -p $project_id -s $site_dir -d $device_id $*"
+$cmd

From 4fd8d3828e73a2a3081c24a037f5f8fdeec14f79 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 21:17:34 -0700
Subject: [PATCH 12/49] Adding reset_config

---
 bin/reset_config   | 23 +++++++++++++++++++++++
 bin/test_redirect  |  9 +--------
 bin/test_validator |  2 ++
 3 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100755 bin/reset_config

diff --git a/bin/reset_config b/bin/reset_config
new file mode 100755
index 0000000000..bbdc6ef09d
--- /dev/null
+++ b/bin/reset_config
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+
+if [[ $# != 3 ]]; then
+    echo Usage: $0 site_dir project_id device_id
+    false
+fi
+
+site_dir=$(realpath $1)
+project_id=$2
+device_id=$3
+shift 3
+cd $ROOT
+
+device_config=/tmp/${device_id}_config.json
+cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
+now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
+echo Setting config timestamp $now_date
+jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
+
+echo Resetting device $device_id config...
+validator/bin/reflector $site_dir $project_id $device_id update/config:$device_config
diff --git a/bin/test_redirect b/bin/test_redirect
index 825226fa5d..42114106d6 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -37,14 +37,7 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
-device_config=/tmp/${device_id}_config.json
-cp sites/udmi_site_model/devices/${device_id}/out/generated_config.json $device_config
-now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
-echo Setting config timestamp $now_date
-jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
-
-echo Resetting device $device_id config...
-validator/bin/reflector $site_path $project_id $device_id update/config:$device_config
+bin/reset_config $site_path $project_id $device_id
 
 echo Writing pubber output to $PUBBER_OUT
 echo bin/pubber $site_path $project_id $device_id $serial_no
diff --git a/bin/test_validator b/bin/test_validator
index 15b4f76a72..bb540a9052 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -38,6 +38,8 @@ vpid=$!
 sleep 10
 echo Started validator pid $vpid
 
+bin/reset_config $site_path $project_id $device_id
+
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT

From 9765b3d8a12b81378686092e6e28b82bea85bf01 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 21:22:16 -0700
Subject: [PATCH 13/49] Add reflector

---
 bin/reset_config                              |  23 +++
 bin/setup_base                                |   2 +-
 bin/test_redirect                             |   2 +
 bin/test_validator                            |   2 +
 .../.idea/runConfigurations/Reflector.xml     |  17 +++
 validator/bin/reflector                       |  23 +++
 .../iot/core/proxy/IotReflectorClient.java    |   2 +-
 .../google/daq/mqtt/registrar/Registrar.java  |   4 +-
 .../java/com/google/daq/mqtt/util/Common.java |  43 ++++++
 .../google/daq/mqtt/validator/Reflector.java  | 140 ++++++++++++++++++
 .../daq/mqtt/validator/SequenceValidator.java |   4 +-
 .../google/daq/mqtt/validator/Validator.java  |  37 +----
 12 files changed, 264 insertions(+), 35 deletions(-)
 create mode 100755 bin/reset_config
 create mode 100644 validator/.idea/runConfigurations/Reflector.xml
 create mode 100755 validator/bin/reflector
 create mode 100644 validator/src/main/java/com/google/daq/mqtt/util/Common.java
 create mode 100644 validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java

diff --git a/bin/reset_config b/bin/reset_config
new file mode 100755
index 0000000000..bbdc6ef09d
--- /dev/null
+++ b/bin/reset_config
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/..)
+
+if [[ $# != 3 ]]; then
+    echo Usage: $0 site_dir project_id device_id
+    false
+fi
+
+site_dir=$(realpath $1)
+project_id=$2
+device_id=$3
+shift 3
+cd $ROOT
+
+device_config=/tmp/${device_id}_config.json
+cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
+now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
+echo Setting config timestamp $now_date
+jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
+
+echo Resetting device $device_id config...
+validator/bin/reflector $site_dir $project_id $device_id update/config:$device_config
diff --git a/bin/setup_base b/bin/setup_base
index a9d150f00b..ce8da695fe 100755
--- a/bin/setup_base
+++ b/bin/setup_base
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sudo apt-get install -y hxtools
+sudo apt-get install -y hxtools moreutils
 
 python3 -m venv venv
 
diff --git a/bin/test_redirect b/bin/test_redirect
index 026e76d45c..59b5b079bb 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -33,6 +33,8 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
+bin/reset_config $site_path $project_id $device_id
+
 echo Writing pubber output to $PUBBER_OUT
 echo bin/pubber $site_path $project_id $device_id $serial_no
 
diff --git a/bin/test_validator b/bin/test_validator
index 15b4f76a72..bb540a9052 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -38,6 +38,8 @@ vpid=$!
 sleep 10
 echo Started validator pid $vpid
 
+bin/reset_config $site_path $project_id $device_id
+
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT
diff --git a/validator/.idea/runConfigurations/Reflector.xml b/validator/.idea/runConfigurations/Reflector.xml
new file mode 100644
index 0000000000..d25ad94d5a
--- /dev/null
+++ b/validator/.idea/runConfigurations/Reflector.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Reflector" type="Application" factoryName="Application" nameIsGenerated="true">
+    <option name="MAIN_CLASS_NAME" value="com.google.daq.mqtt.validator.Reflector" />
+    <module name="validator" />
+    <option name="PROGRAM_PARAMETERS" value="-p bos-peringknife-dev -s sites/udmi_site_model -d AHU-1 update/config:sites/udmi_site_model/devices/AHU-1/out/generated_config.json" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/.." />
+    <extension name="coverage">
+      <pattern>
+        <option name="PATTERN" value="com.google.daq.mqtt.validator.*" />
+        <option name="ENABLED" value="true" />
+      </pattern>
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>
\ No newline at end of file
diff --git a/validator/bin/reflector b/validator/bin/reflector
new file mode 100755
index 0000000000..7e68092f2d
--- /dev/null
+++ b/validator/bin/reflector
@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+ROOT=$(realpath $(dirname $0)/../..)
+
+if [[ $# < 4 ]]; then
+    echo Usage: $0 site_dir project_id device_id directive [directives...]
+    echo "  Directive is something like update/config:sites/udmi_site_model/devices/AHU-1/out/generated_config.json"
+    false
+fi
+
+site_dir=$(realpath $1)
+project_id=$2
+device_id=$3
+shift 3
+cd $ROOT
+
+validator/bin/build
+
+jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
+mainclass=com.google.daq.mqtt.validator.Reflector
+
+cmd="java -cp $jarfile $mainclass -p $project_id -s $site_dir -d $device_id $*"
+$cmd
diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
index 2316ce69c1..c8e2b2b506 100644
--- a/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
+++ b/validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java
@@ -59,7 +59,7 @@ public IotReflectorClient(String projectId, CloudIotConfig iotConfig, String key
       mqttPublisher = new MqttPublisher(projectId, cloudRegion, UDMS_REFLECT,
           siteName, keyBytes, IOT_KEY_ALGORITHM, this::messageHandler, this::errorHandler);
     } catch (Exception e) {
-      throw new RuntimeException("While connecting subscription " + subscriptionId, e);
+      throw new RuntimeException("While connecting MQTT endpoint " + subscriptionId, e);
     }
 
     active = true;
diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
index 5932fbce9a..a8f4e88ff3 100644
--- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
+++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java
@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.registrar;
 
-import static com.google.daq.mqtt.validator.Validator.NO_SITE;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -9,7 +9,6 @@
 import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
-import com.github.fge.jsonschema.core.report.ProcessingReport;
 import com.github.fge.jsonschema.main.JsonSchema;
 import com.github.fge.jsonschema.main.JsonSchemaFactory;
 import com.google.api.services.cloudiot.v1.model.Device;
@@ -30,7 +29,6 @@
 import java.io.FilenameFilter;
 import java.io.InputStream;
 import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
 import java.math.BigInteger;
 import java.net.URI;
 import java.time.Duration;
diff --git a/validator/src/main/java/com/google/daq/mqtt/util/Common.java b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
new file mode 100644
index 0000000000..2894868efd
--- /dev/null
+++ b/validator/src/main/java/com/google/daq/mqtt/util/Common.java
@@ -0,0 +1,43 @@
+package com.google.daq.mqtt.util;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
+import java.util.List;
+import java.util.MissingFormatArgumentException;
+
+/**
+ * Collection of common constants and minor utilities.
+ */
+public abstract class Common {
+
+  public static final String STATE_QUERY_TOPIC = "query/state";
+  public static final String TIMESTAMP_ATTRIBUTE = "timestamp";
+  public static final String NO_SITE = "--";
+  public static final ObjectMapper OBJECT_MAPPER =
+      new ObjectMapper()
+          .enable(Feature.ALLOW_COMMENTS)
+          .enable(SerializationFeature.INDENT_OUTPUT)
+          .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
+          .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
+          .setDateFormat(new ISO8601DateFormat())
+          .setSerializationInclusion(Include.NON_NULL);
+  public static final String JSON_SUFFIX = ".json";
+  public static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
+
+  /**
+   * Remove the next item from the list in an exception-safe way.
+   *
+   * @param argList list of arguments
+   * @return removed argument
+   */
+  public static String removeNextArg(List<String> argList) {
+    if (argList.isEmpty()) {
+      throw new MissingFormatArgumentException("Missing argument");
+    }
+    return argList.remove(0);
+  }
+}
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
new file mode 100644
index 0000000000..ef88026b9d
--- /dev/null
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Reflector.java
@@ -0,0 +1,140 @@
+package com.google.daq.mqtt.validator;
+
+import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
+import static com.google.daq.mqtt.util.Common.removeNextArg;
+
+import com.google.bos.iot.core.proxy.IotReflectorClient;
+import com.google.daq.mqtt.util.CloudIotConfig;
+import com.google.daq.mqtt.util.CloudIotManager;
+import com.google.daq.mqtt.util.ConfigUtil;
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * General utility for working with UDMI Reflector messages.
+ */
+public class Reflector {
+
+  private final List<String> reflectCommands;
+  private String projectId;
+  private String siteDir;
+  private CloudIotConfig cloudIotConfig;
+  private File baseDir;
+  private IotReflectorClient client;
+  private String deviceId;
+
+  /**
+   * Create an instance of the Reflector class.
+   *
+   * @param argsList command-line arguments
+   */
+  public Reflector(List<String> argsList) {
+    reflectCommands = parseArgs(argsList);
+  }
+
+  /**
+   * Let's go.
+   *
+   * @param args command-line arguments
+   */
+  public static void main(String[] args) {
+    Reflector reflector = new Reflector(Arrays.asList(args));
+    reflector.initialize();
+    reflector.reflect();
+    reflector.shutdown();
+  }
+
+  private void shutdown() {
+    try {
+      Thread.sleep(5000);
+    } catch (Exception e) {
+      throw new RuntimeException("While sleeping", e);
+    }
+    client.close();
+  }
+
+  private void reflect() {
+    System.err.printf("Reflecting %d directives...%n", reflectCommands.size());
+    while (!reflectCommands.isEmpty()) {
+      reflect(reflectCommands.remove(0));
+    }
+    System.err.println("Done with all reflective directives!");
+  }
+
+  private void reflect(String directive) {
+    System.err.printf("Reflecting %s%n", directive);
+    String[] parts = directive.split(":", 2);
+    if (parts.length != 2) {
+      throw new RuntimeException("Expected topic:file format reflect directive " + directive);
+    }
+    File file = new File(parts[1]);
+    reflect(parts[0], file);
+  }
+
+  private void reflect(String topic, File dataFile) {
+    try (FileInputStream fis = new FileInputStream(dataFile)) {
+      String data = new String(fis.readAllBytes());
+      reflect(topic, data);
+    } catch (Exception e) {
+      throw new RuntimeException("While processing input file " + dataFile.getAbsolutePath(), e);
+    }
+  }
+
+  private void reflect(String topic, String data) {
+    client.publish(deviceId, topic, data);
+  }
+
+  private void initialize() {
+    String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath();
+    System.err.println("Loading reflector key file from " + keyFile);
+    client = new IotReflectorClient(projectId, cloudIotConfig, keyFile);
+  }
+
+  private List<String> parseArgs(List<String> argsList) {
+    List<String> listCopy = new ArrayList<>(argsList);
+    while (!listCopy.isEmpty()) {
+      String option = removeNextArg(listCopy);
+      try {
+        switch (option) {
+          case "-p":
+            projectId = removeNextArg(listCopy);
+            break;
+          case "-s":
+            setSiteDir(removeNextArg(listCopy));
+            break;
+          case "-d":
+            deviceId = removeNextArg(listCopy);
+            break;
+          default:
+            listCopy.add(option);
+            return listCopy; // default case is the remaining list of reflection directives
+        }
+      } catch (Exception e) {
+        throw new RuntimeException("While processing option " + option, e);
+      }
+    }
+    throw new IllegalArgumentException("No reflect directives specified!");
+  }
+
+  /**
+   * Set the site directory to use for this run.
+   *
+   * @param siteDir site model directory
+   */
+  public void setSiteDir(String siteDir) {
+    if (NO_SITE.equals(siteDir)) {
+      siteDir = null;
+      baseDir = new File(".");
+    } else {
+      this.siteDir = siteDir;
+      baseDir = new File(siteDir);
+      File cloudConfig = new File(siteDir, "cloud_iot_config.json");
+      cloudIotConfig = CloudIotManager.validate(ConfigUtil.readCloudIotConfig(cloudConfig),
+          projectId);
+    }
+  }
+}
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
index 685e3be544..317c27268b 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/SequenceValidator.java
@@ -12,6 +12,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.daq.mqtt.util.CloudIotConfig;
+import com.google.daq.mqtt.util.Common;
 import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.ValidatorConfig;
 import java.io.ByteArrayOutputStream;
@@ -148,6 +149,7 @@ public abstract class SequenceValidator {
     resultSummary.delete();
     System.err.println("Writing results to " + resultSummary.getAbsolutePath());
 
+    System.err.printf("Loading reflector key file from %s%n", new File(key_file).getAbsolutePath());
     System.err.printf("Validating against device %s serial %s%n", deviceId, serialNo);
     client = new IotReflectorClient(projectId, cloudIotConfig, key_file);
     setReflectorState();
@@ -454,7 +456,7 @@ private String writeLogEntry(Entry logEntry, String filename) {
   }
 
   protected void queryState() {
-    client.publish(deviceId, Validator.STATE_QUERY_TOPIC, EMPTY_MESSAGE);
+    client.publish(deviceId, Common.STATE_QUERY_TOPIC, EMPTY_MESSAGE);
   }
 
   /**
diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
index b2220e2ee8..d85e5eaaa8 100644
--- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
+++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java
@@ -1,15 +1,16 @@
 package com.google.daq.mqtt.validator;
 
+import static com.google.daq.mqtt.util.Common.GCP_REFLECT_KEY_PKCS8;
+import static com.google.daq.mqtt.util.Common.JSON_SUFFIX;
+import static com.google.daq.mqtt.util.Common.NO_SITE;
+import static com.google.daq.mqtt.util.Common.OBJECT_MAPPER;
+import static com.google.daq.mqtt.util.Common.STATE_QUERY_TOPIC;
+import static com.google.daq.mqtt.util.Common.TIMESTAMP_ATTRIBUTE;
+import static com.google.daq.mqtt.util.Common.removeNextArg;
 import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
 
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
-import com.fasterxml.jackson.core.JsonParser.Feature;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.github.fge.jsonschema.core.exceptions.ProcessingException;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
@@ -31,7 +32,6 @@
 import com.google.daq.mqtt.util.FileDataSink;
 import com.google.daq.mqtt.util.PubSubClient;
 import com.google.daq.mqtt.util.ValidationException;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -39,7 +39,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.PrintStream;
 import java.net.URI;
 import java.net.URL;
 import java.util.ArrayList;
@@ -78,19 +77,7 @@
  */
 public class Validator {
 
-  public static final String STATE_QUERY_TOPIC = "query/state";
-  public static final String TIMESTAMP_ATTRIBUTE = "timestamp";
-  public static final String NO_SITE = "--";
-  private static final ObjectMapper OBJECT_MAPPER =
-      new ObjectMapper()
-          .enable(Feature.ALLOW_COMMENTS)
-          .enable(SerializationFeature.INDENT_OUTPUT)
-          .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
-          .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
-          .setDateFormat(new ISO8601DateFormat())
-          .setSerializationInclusion(Include.NON_NULL);
   private static final String ERROR_FORMAT_INDENT = "  ";
-  private static final String JSON_SUFFIX = ".json";
   private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas";
   private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s";
   private static final String DEVICE_FILE_FORMAT = "devices/%s";
@@ -104,7 +91,6 @@ public class Validator {
   private static final String UNKNOWN_FOLDER_DEFAULT = "unknown";
   private static final String EVENT_POINTSET = "event_pointset";
   private static final String STATE_POINTSET = "state_pointset";
-  private static final String GCP_REFLECT_KEY_PKCS8 = "validator/rsa_private.pkcs8";
   private static final String EMPTY_MESSAGE = "{}";
   private static final String CONFIG_PREFIX = "config_";
   private static final String STATE_PREFIX = "state_";
@@ -181,7 +167,7 @@ public static void main(String[] args) {
   }
 
   private void parseArgs(List<String> argList) {
-    while (argList.size() > 0) {
+    while (!argList.isEmpty()) {
       String option = removeNextArg(argList);
       try {
         switch (option) {
@@ -232,13 +218,6 @@ private void validateMessageTrace(String messageDir) {
     reportAfterEveryMessage = true;
   }
 
-  private String removeNextArg(List<String> argList) {
-    if (argList.isEmpty()) {
-      throw new MissingFormatArgumentException("Missing argument");
-    }
-    return argList.remove(0);
-  }
-
   /**
    * Set the site directory to use for this validation run.
    *

From 15c897ae33861036bb78e302021d391c79e75f00 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 21:44:11 -0700
Subject: [PATCH 14/49] Add REFLECTOR mutual exclusion

---
 .github/workflows/testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 97d82955a3..f0f0add296 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -78,6 +78,7 @@ jobs:
     name: Sequence tests
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    needs: redirect  # Access to UDMI-REFLECTOR is mutually exclusive
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-java@v1

From 39f44f50245a53014ffb0352179062daa9d8fc55 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 22:01:41 -0700
Subject: [PATCH 15/49] Basics for extended test

---
 .github/workflows/testing.yml | 13 +++++++++----
 bin/reset_config              | 14 +++++++++++---
 bin/test_redirect             | 19 ++++++++++---------
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index f0f0add296..a21ee520c5 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -65,10 +65,15 @@ jobs:
         run: |
           # Simple checks that a redirect happened and failed
           set +x
-          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out
-          fgrep 'system.config.parse success' pubber.out
-          fgrep registries/missing/devices/GAT-123 pubber.out
-          fgrep 'Not authorized to connect' pubber.out
+          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.1
+          fgrep 'system.config.parse success' pubber.out.1
+          fgrep registries/missing/devices/GAT-123 pubber.out.1
+          fgrep 'Not authorized to connect' pubber.out.1
+          # Now check the redirect-by-config setup
+          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.2
+          fgrep 'system.config.parse success' pubber.out.2
+          fgrep registries/reconfigure/devices/GAT-123 pubber.out.2
+          fgrep 'Not authorized to connect' pubber.out.2
       - name: pubber logs
         if: ${{ always() }}
         run: |
diff --git a/bin/reset_config b/bin/reset_config
index bbdc6ef09d..64197b8b6f 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -2,8 +2,9 @@
 
 ROOT=$(realpath $(dirname $0)/..)
 
-if [[ $# != 3 ]]; then
-    echo Usage: $0 site_dir project_id device_id
+if [[ $# != 3 && $# != 4 ]]; then
+    echo Usage: $0 site_dir project_id device_id [config_filename]
+    echo "  config_filename defaults to generated_config.json"
     false
 fi
 
@@ -11,10 +12,17 @@ site_dir=$(realpath $1)
 project_id=$2
 device_id=$3
 shift 3
+
+filename=generated_config.json
+if [[ -n $1 ]]; then
+    filename=$1
+    shift
+fi
+
 cd $ROOT
 
 device_config=/tmp/${device_id}_config.json
-cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
+cp $site_dir/devices/${device_id}/out/$filename $device_config
 now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
 echo Setting config timestamp $now_date
 jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
diff --git a/bin/test_redirect b/bin/test_redirect
index 42114106d6..8220d159c7 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -21,10 +21,6 @@ fi
 site_path=sites/udmi_site_model
 device_id=AHU-22 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
 
-site_config=$site_path/cloud_iot_config.json
-cloud_region=$(jq -r .cloud_region $site_config)
-registry_id=$(jq -r .registry_id $site_config)
-
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
@@ -39,10 +35,15 @@ fi
 
 bin/reset_config $site_path $project_id $device_id
 
-echo Writing pubber output to $PUBBER_OUT
-echo bin/pubber $site_path $project_id $device_id $serial_no
-
+echo Writing pubber output to $PUBBER_OUT.1
 result=0
-timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT 2>&1 || result=$?
-
+timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
 echo Pubber exit code $result
+
+echo Writing pubber output to $PUBBER_OUT.2
+bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
+
+# Wait for initial connection, then reset config to redirect
+sleep 20
+bin/reset_config $site_path $project_id $device_id redirect_config.json
+sleep 20

From 7447fcb562cc0c8be064dccd56afce9f0446d581 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Wed, 3 Aug 2022 22:02:58 -0700
Subject: [PATCH 16/49] Disable redirect

---
 bin/test_redirect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index 8220d159c7..c6dfb73b39 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -45,5 +45,5 @@ bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect
 sleep 20
-bin/reset_config $site_path $project_id $device_id redirect_config.json
+bin/reset_config $site_path $project_id $device_id #redirect_config.json
 sleep 20

From 215b6318836171688c8255f1d06b8f67c5839784 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 07:49:44 -0700
Subject: [PATCH 17/49] Review tweaks

---
 bin/reset_config        | 2 +-
 validator/bin/reflector | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/reset_config b/bin/reset_config
index bbdc6ef09d..028e4a38b1 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -15,7 +15,7 @@ cd $ROOT
 
 device_config=/tmp/${device_id}_config.json
 cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
-now_date=$(date -Ins -u | sed -E 's/.{6}\+.*/Z/' | tr , .)
+now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp $now_date
 jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
 
diff --git a/validator/bin/reflector b/validator/bin/reflector
index 7e68092f2d..eda35acf84 100755
--- a/validator/bin/reflector
+++ b/validator/bin/reflector
@@ -20,4 +20,5 @@ jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 mainclass=com.google.daq.mqtt.validator.Reflector
 
 cmd="java -cp $jarfile $mainclass -p $project_id -s $site_dir -d $device_id $*"
+echo $cmd
 $cmd

From 35514b1b7537df4c580c3b828a9341e8896c703c Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 07:52:08 -0700
Subject: [PATCH 18/49] Add spongs check

---
 bin/reset_config | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/reset_config b/bin/reset_config
index 028e4a38b1..1cd419837c 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -17,6 +17,7 @@ device_config=/tmp/${device_id}_config.json
 cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp $now_date
+sponge < /dev/null || echo sponge command not found, might need workaround for osx or other
 jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
 
 echo Resetting device $device_id config...

From c235c8e87a36fa88190086c78f4e5217553c267e Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 07:53:44 -0700
Subject: [PATCH 19/49] Adding {} for all vars

---
 bin/reset_config | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/reset_config b/bin/reset_config
index 1cd419837c..6d965dfe78 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -11,14 +11,14 @@ site_dir=$(realpath $1)
 project_id=$2
 device_id=$3
 shift 3
-cd $ROOT
+cd ${ROOT}
 
 device_config=/tmp/${device_id}_config.json
-cp $site_dir/devices/${device_id}/out/generated_config.json $device_config
+cp ${site_dir}/devices/${device_id}/out/generated_config.json ${device_config}
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
-echo Setting config timestamp $now_date
+echo Setting config timestamp ${now_date}
 sponge < /dev/null || echo sponge command not found, might need workaround for osx or other
-jq .timestamp=\"$now_date\" < $device_config | sponge $device_config
+jq .timestamp=\"${now_date}\" < ${device_config} | sponge ${device_config}
 
-echo Resetting device $device_id config...
-validator/bin/reflector $site_dir $project_id $device_id update/config:$device_config
+echo Resetting device ${device_id} config...
+validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${device_config}

From f5eed7ee1f284fc45429e5e97e4580e1b5e49e40 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 08:57:56 -0700
Subject: [PATCH 20/49] Debugging

---
 bin/test_validator | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/test_validator b/bin/test_validator
index bb540a9052..091949a224 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -1,9 +1,9 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 # Force consistent sort order
 export LC_ALL=C
 
-ROOT_DIR=$(dirname $0)/..
+ROOT_DIR=$(realpath $(dirname $0)/..)
 cd $ROOT_DIR
 
 if [[ $# != 1 ]]; then

From 15733d13fbb61546a399a0975833012da1dee959 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 09:45:57 -0700
Subject: [PATCH 21/49] Debugging

---
 bin/validator | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/validator b/bin/validator
index 65055f3563..dad2419fa9 100755
--- a/bin/validator
+++ b/bin/validator
@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 ROOT_DIR=$(realpath $(dirname $0)/..)
 
@@ -23,6 +23,9 @@ fi
 echo Building validator...
 $ROOT_DIR/validator/bin/build > /dev/null
 
+echo Files built
+(cd $ROOT_DIR; find validator/build -type f) || true
+
 echo Running tools version `(cd $ROOT_DIR; git describe)`
 
 if [[ -n $subscription ]]; then

From f66dab47c5640c1ac179cd360e726e7ea32af662 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 11:20:29 -0700
Subject: [PATCH 22/49] Diagnostics

---
 bin/reset_config | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/reset_config b/bin/reset_config
index 6d965dfe78..28e509b3a4 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 ROOT=$(realpath $(dirname $0)/..)
 
@@ -13,6 +13,8 @@ device_id=$3
 shift 3
 cd ${ROOT}
 
+pwd
+
 device_config=/tmp/${device_id}_config.json
 cp ${site_dir}/devices/${device_id}/out/generated_config.json ${device_config}
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')

From d095b88374fa5eaeb69749b66510da6107344e4f Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 11:39:18 -0700
Subject: [PATCH 23/49] Diagnostics

---
 validator/bin/reflector | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/validator/bin/reflector b/validator/bin/reflector
index eda35acf84..c5794ae619 100755
--- a/validator/bin/reflector
+++ b/validator/bin/reflector
@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 
 ROOT=$(realpath $(dirname $0)/../..)
 
@@ -14,8 +14,12 @@ device_id=$3
 shift 3
 cd $ROOT
 
+pwd
+
 validator/bin/build
 
+find validator/build -type f || true
+
 jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 mainclass=com.google.daq.mqtt.validator.Reflector
 

From c6f5b32aa733c0beabff1524f563ead156b1ce8e Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 12:08:43 -0700
Subject: [PATCH 24/49] Dont' die

---
 bin/test_validator | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_validator b/bin/test_validator
index 091949a224..512cb53629 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -67,7 +67,7 @@ sleep 30
 
 pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
 echo Killing pids $vpid $pids
-kill $vpid $pids
+kill $vpid $pids || true
 
 outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 if [[ -z $outfiles ]]; then

From aa4cef81ed063babccc3462995fefca50da0bb07 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 12:41:04 -0700
Subject: [PATCH 25/49] Remove some debugging

---
 bin/test_validator  | 2 +-
 bin/validator       | 2 +-
 validator/bin/build | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/test_validator b/bin/test_validator
index 512cb53629..62d890e104 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 # Force consistent sort order
 export LC_ALL=C
diff --git a/bin/validator b/bin/validator
index dad2419fa9..8f55149e06 100755
--- a/bin/validator
+++ b/bin/validator
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 ROOT_DIR=$(realpath $(dirname $0)/..)
 
diff --git a/validator/bin/build b/validator/bin/build
index 3576b0d868..4cbae7b554 100755
--- a/validator/bin/build
+++ b/validator/bin/build
@@ -14,7 +14,7 @@ cd $ROOT
 echo Building validataor in $PWD
 
 rm -rf build
-./gradlew shadow $check
+./gradlew --info shadow $check
 
 ls -l build/libs/validator-1.0-SNAPSHOT-all.jar
 

From bea11a59351674f951e2384ef5cc8f9259e000fa Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:03:09 -0700
Subject: [PATCH 26/49] Adding shared constants file

---
 .gencode_hash.txt                        |  1 +
 bin/gencode_java                         |  4 ++--
 etc/Predefined.java                      | 10 ++++++++++
 gencode/java/udmi/schema/Predefined.java | 10 ++++++++++
 4 files changed, 23 insertions(+), 2 deletions(-)
 create mode 100644 etc/Predefined.java
 create mode 100644 gencode/java/udmi/schema/Predefined.java

diff --git a/.gencode_hash.txt b/.gencode_hash.txt
index 279f5c6b00..a624e28aa4 100644
--- a/.gencode_hash.txt
+++ b/.gencode_hash.txt
@@ -53,6 +53,7 @@ bb1c7e0df78870b7f7bd78fdfb1ed115cb121613ffcd98ec6b95e41c01624d41  gencode/java/u
 fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa  gencode/java/udmi/schema/PointsetState.java
 61afd6c9c0364b8982c90df7664147829bad8858408b2141299ee8bbc35f1612  gencode/java/udmi/schema/PointsetSummary.java
 ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/udmi/schema/Position.java
+4823fe7ebf7e0c6c99b4759169cd75f5bb93f4e96903d72d82cada0dcdf4b5e7  gencode/java/udmi/schema/Predefined.java
 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70  gencode/java/udmi/schema/Properties.java
 7bc53cb9a61df8f804a75aa4860403d19281f7c665cbddd5364a2aa746a1f651  gencode/java/udmi/schema/ReflectorConfig.java
 abe99dd74122c186403baa6982300a9d5968f8bbb7a67b1689104111b98f32fb  gencode/java/udmi/schema/ReflectorState.java
diff --git a/bin/gencode_java b/bin/gencode_java
index 1205ef6879..a5a8b9b958 100755
--- a/bin/gencode_java
+++ b/bin/gencode_java
@@ -31,8 +31,8 @@ echo Generating code in $OUTDIR
 $JARBIN $JOPTS --source . --target $OUTDIR
 
 # There is no way to specify enum constants in generated code, so just hack it in manually.
-echo Copying Level.java
-cp -n $ROOT_DIR/etc/Level.java $OUTDIR/udmi/schema/Level.java
+echo Copying shared constants $ROOT_DIR/etc/*.java
+cp -n $ROOT_DIR/etc/*.java $OUTDIR/udmi/schema/
 
 echo Cleaning up __ classes...
 find $OUTDIR -name \*__\*.java | xargs rm
diff --git a/etc/Predefined.java b/etc/Predefined.java
new file mode 100644
index 0000000000..6e9f537edb
--- /dev/null
+++ b/etc/Predefined.java
@@ -0,0 +1,10 @@
+package udmi.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// This class is manually curated and then copied into the gencode directory. Look for the
+// proper source and don't be fooled! These are generally useful constants.
+public static class Predefined {
+  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
+}
diff --git a/gencode/java/udmi/schema/Predefined.java b/gencode/java/udmi/schema/Predefined.java
new file mode 100644
index 0000000000..6e9f537edb
--- /dev/null
+++ b/gencode/java/udmi/schema/Predefined.java
@@ -0,0 +1,10 @@
+package udmi.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+
+// This class is manually curated and then copied into the gencode directory. Look for the
+// proper source and don't be fooled! These are generally useful constants.
+public static class Predefined {
+  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
+}

From 9c8900e19b6bd99f76ed4dd430fba1e5f1408ddd Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:10:54 -0700
Subject: [PATCH 27/49] Cleanup

---
 bin/reset_config        | 2 +-
 bin/test_validator      | 2 +-
 bin/validator           | 3 ---
 validator/bin/build     | 2 +-
 validator/bin/reflector | 4 +---
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/bin/reset_config b/bin/reset_config
index 28e509b3a4..4917dd7508 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 ROOT=$(realpath $(dirname $0)/..)
 
diff --git a/bin/test_validator b/bin/test_validator
index 62d890e104..8ec6777502 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -67,7 +67,7 @@ sleep 30
 
 pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
 echo Killing pids $vpid $pids
-kill $vpid $pids || true
+kill $vpid $pids
 
 outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 if [[ -z $outfiles ]]; then
diff --git a/bin/validator b/bin/validator
index 8f55149e06..65055f3563 100755
--- a/bin/validator
+++ b/bin/validator
@@ -23,9 +23,6 @@ fi
 echo Building validator...
 $ROOT_DIR/validator/bin/build > /dev/null
 
-echo Files built
-(cd $ROOT_DIR; find validator/build -type f) || true
-
 echo Running tools version `(cd $ROOT_DIR; git describe)`
 
 if [[ -n $subscription ]]; then
diff --git a/validator/bin/build b/validator/bin/build
index 4cbae7b554..3576b0d868 100755
--- a/validator/bin/build
+++ b/validator/bin/build
@@ -14,7 +14,7 @@ cd $ROOT
 echo Building validataor in $PWD
 
 rm -rf build
-./gradlew --info shadow $check
+./gradlew shadow $check
 
 ls -l build/libs/validator-1.0-SNAPSHOT-all.jar
 
diff --git a/validator/bin/reflector b/validator/bin/reflector
index c5794ae619..3db9c7ade9 100755
--- a/validator/bin/reflector
+++ b/validator/bin/reflector
@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 ROOT=$(realpath $(dirname $0)/../..)
 
@@ -18,8 +18,6 @@ pwd
 
 validator/bin/build
 
-find validator/build -type f || true
-
 jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 mainclass=com.google.daq.mqtt.validator.Reflector
 

From 03b9ffbb08cd567718588e3f42030bc47c6679e5 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:24:09 -0700
Subject: [PATCH 28/49] Adding constants

---
 .gencode_hash.txt                           |  2 +-
 etc/Blob.java                               |  8 ++++++++
 etc/Predefined.java                         | 10 ----------
 gencode/java/udmi/schema/Blob.java          |  8 ++++++++
 gencode/java/udmi/schema/Predefined.java    | 10 ----------
 pubber/src/main/java/daq/pubber/Pubber.java |  8 ++++----
 6 files changed, 21 insertions(+), 25 deletions(-)
 create mode 100644 etc/Blob.java
 delete mode 100644 etc/Predefined.java
 create mode 100644 gencode/java/udmi/schema/Blob.java
 delete mode 100644 gencode/java/udmi/schema/Predefined.java

diff --git a/.gencode_hash.txt b/.gencode_hash.txt
index a624e28aa4..f1b12a1933 100644
--- a/.gencode_hash.txt
+++ b/.gencode_hash.txt
@@ -12,6 +12,7 @@ f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/i
 43019239edfdad71647c40d254b9b3aec68f3deb0c808ba636530c1cf9985fe0  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
+c2205c3e95800cae639d0b043979e156d0b7142dc72719e64aaca3a397515033  gencode/java/udmi/schema/Blob.java
 d5adb804697243f97cdd8589750401654f3fab075a9aeac4f2851e46695ef11d  gencode/java/udmi/schema/BlobBlobsetConfig.java
 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0  gencode/java/udmi/schema/BlobBlobsetState.java
 d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124  gencode/java/udmi/schema/BlobsetConfig.java
@@ -53,7 +54,6 @@ bb1c7e0df78870b7f7bd78fdfb1ed115cb121613ffcd98ec6b95e41c01624d41  gencode/java/u
 fc3a9415c04d8a06954dbdbfdff5d68ab113cce3948532c19df555778ffb04fa  gencode/java/udmi/schema/PointsetState.java
 61afd6c9c0364b8982c90df7664147829bad8858408b2141299ee8bbc35f1612  gencode/java/udmi/schema/PointsetSummary.java
 ca2e7566106818ca7e5190c8041eb86f0c9b3251b0bda8c3ea7ce11a0c891a0a  gencode/java/udmi/schema/Position.java
-4823fe7ebf7e0c6c99b4759169cd75f5bb93f4e96903d72d82cada0dcdf4b5e7  gencode/java/udmi/schema/Predefined.java
 3df66bb1a37a9e0b2b6cf392f8c64d404a73c83e5e13c02bb4844f09b9a04b70  gencode/java/udmi/schema/Properties.java
 7bc53cb9a61df8f804a75aa4860403d19281f7c665cbddd5364a2aa746a1f651  gencode/java/udmi/schema/ReflectorConfig.java
 abe99dd74122c186403baa6982300a9d5968f8bbb7a67b1689104111b98f32fb  gencode/java/udmi/schema/ReflectorState.java
diff --git a/etc/Blob.java b/etc/Blob.java
new file mode 100644
index 0000000000..97d3dc926d
--- /dev/null
+++ b/etc/Blob.java
@@ -0,0 +1,8 @@
+package udmi.schema;
+
+// This class is manually curated and then copied into the gencode directory. Look for the
+// proper source and don't be fooled! These are general constants for blob management.
+public abstract class Blob {
+    public static final String COMMIT_PHASE = "commit";
+    public static final String IOT_CONFIG_BLOB = "_iot_config";
+}
diff --git a/etc/Predefined.java b/etc/Predefined.java
deleted file mode 100644
index 6e9f537edb..0000000000
--- a/etc/Predefined.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package udmi.schema;
-
-import java.util.HashMap;
-import java.util.Map;
-
-// This class is manually curated and then copied into the gencode directory. Look for the
-// proper source and don't be fooled! These are generally useful constants.
-public static class Predefined {
-  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
-}
diff --git a/gencode/java/udmi/schema/Blob.java b/gencode/java/udmi/schema/Blob.java
new file mode 100644
index 0000000000..97d3dc926d
--- /dev/null
+++ b/gencode/java/udmi/schema/Blob.java
@@ -0,0 +1,8 @@
+package udmi.schema;
+
+// This class is manually curated and then copied into the gencode directory. Look for the
+// proper source and don't be fooled! These are general constants for blob management.
+public abstract class Blob {
+    public static final String COMMIT_PHASE = "commit";
+    public static final String IOT_CONFIG_BLOB = "_iot_config";
+}
diff --git a/gencode/java/udmi/schema/Predefined.java b/gencode/java/udmi/schema/Predefined.java
deleted file mode 100644
index 6e9f537edb..0000000000
--- a/gencode/java/udmi/schema/Predefined.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package udmi.schema;
-
-import java.util.HashMap;
-import java.util.Map;
-
-// This class is manually curated and then copied into the gencode directory. Look for the
-// proper source and don't be fooled! These are generally useful constants.
-public static class Predefined {
-  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
-}
diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java
index 5f6f1d18e9..bfd712463e 100644
--- a/pubber/src/main/java/daq/pubber/Pubber.java
+++ b/pubber/src/main/java/daq/pubber/Pubber.java
@@ -1,6 +1,8 @@
 package daq.pubber;
 
 import static java.util.stream.Collectors.toMap;
+import static udmi.schema.Blob.COMMIT_PHASE;
+import static udmi.schema.Blob.IOT_CONFIG_BLOB;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -74,8 +76,6 @@ public class Pubber {
 
   public static final int SCAN_DURATION_SEC = 10;
   public static final String DISCOVERY_ID = "RANDOM_ID";
-  public static final String IOT_ENDPOINT_BLOB_NAME = "_iot_config";
-  public static final String BLOB_COMMIT_PHASE = "commit";
   private static final String UDMI_VERSION = "1.3.14";
   private static final Logger LOG = LoggerFactory.getLogger(Pubber.class);
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
@@ -730,7 +730,7 @@ private void processConfigUpdate(Config config) {
 
   private EndpointConfiguration extractEndpointBlobConfig() {
     try {
-      String iotConfig = extractConfigBlob(IOT_ENDPOINT_BLOB_NAME);
+      String iotConfig = extractConfigBlob(IOT_CONFIG_BLOB);
       if (iotConfig == null) {
         return null;
       }
@@ -768,7 +768,7 @@ private String extractConfigBlob(String blobName) {
         return null;
       }
       BlobBlobsetConfig blobBlobsetConfig = deviceConfig.blobset.blobs.get(blobName);
-      if (blobBlobsetConfig != null && BLOB_COMMIT_PHASE.equals(blobBlobsetConfig.phase)
+      if (blobBlobsetConfig != null && COMMIT_PHASE.equals(blobBlobsetConfig.phase)
           && blobBlobsetConfig.base64 != null) {
         return new String(Base64.getDecoder().decode(blobBlobsetConfig.base64));
       }

From 3b9db9c7a735def9385526836fba825665259443 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:35:24 -0700
Subject: [PATCH 29/49] COMMIT to FINAL

---
 .gencode_hash.txt                           | 2 +-
 etc/Blob.java                               | 2 +-
 gencode/java/udmi/schema/Blob.java          | 2 +-
 pubber/src/main/java/daq/pubber/Pubber.java | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gencode_hash.txt b/.gencode_hash.txt
index f1b12a1933..98bf659276 100644
--- a/.gencode_hash.txt
+++ b/.gencode_hash.txt
@@ -12,7 +12,7 @@ f48026471ae3cd7867bce416dc21c2fb728f48d8476a8d6e95f6acaf1d8b6cf3  gencode/docs/i
 43019239edfdad71647c40d254b9b3aec68f3deb0c808ba636530c1cf9985fe0  gencode/docs/state.html
 d39d7fe37a41c74a40080af7b0a429d201ab1fdff7444428c4b98eb7b38c332b  gencode/java/udmi/schema/Asset.java
 0825a5cec83003bb0a6488c4ed7010a04ae0d3848ef36fe01bb4e6718ba7b96d  gencode/java/udmi/schema/Aux.java
-c2205c3e95800cae639d0b043979e156d0b7142dc72719e64aaca3a397515033  gencode/java/udmi/schema/Blob.java
+df118b8e5abebac38579027b5a17192163f0cc97e55a8e44b847b1a965263636  gencode/java/udmi/schema/Blob.java
 d5adb804697243f97cdd8589750401654f3fab075a9aeac4f2851e46695ef11d  gencode/java/udmi/schema/BlobBlobsetConfig.java
 2c03651cb2ecda072b1418222eebb5560185669f8ffdd03021ad5ad8ff7ba3b0  gencode/java/udmi/schema/BlobBlobsetState.java
 d2c5b5aae8db27b68104fc83a1f38de0a3f1b5d683f2b13599adf24e96c7d124  gencode/java/udmi/schema/BlobsetConfig.java
diff --git a/etc/Blob.java b/etc/Blob.java
index 97d3dc926d..933264fdac 100644
--- a/etc/Blob.java
+++ b/etc/Blob.java
@@ -3,6 +3,6 @@
 // This class is manually curated and then copied into the gencode directory. Look for the
 // proper source and don't be fooled! These are general constants for blob management.
 public abstract class Blob {
-    public static final String COMMIT_PHASE = "commit";
+    public static final String FINAL_PHASE = "final";
     public static final String IOT_CONFIG_BLOB = "_iot_config";
 }
diff --git a/gencode/java/udmi/schema/Blob.java b/gencode/java/udmi/schema/Blob.java
index 97d3dc926d..933264fdac 100644
--- a/gencode/java/udmi/schema/Blob.java
+++ b/gencode/java/udmi/schema/Blob.java
@@ -3,6 +3,6 @@
 // This class is manually curated and then copied into the gencode directory. Look for the
 // proper source and don't be fooled! These are general constants for blob management.
 public abstract class Blob {
-    public static final String COMMIT_PHASE = "commit";
+    public static final String FINAL_PHASE = "final";
     public static final String IOT_CONFIG_BLOB = "_iot_config";
 }
diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java
index bfd712463e..1bffbf76d6 100644
--- a/pubber/src/main/java/daq/pubber/Pubber.java
+++ b/pubber/src/main/java/daq/pubber/Pubber.java
@@ -1,7 +1,7 @@
 package daq.pubber;
 
 import static java.util.stream.Collectors.toMap;
-import static udmi.schema.Blob.COMMIT_PHASE;
+import static udmi.schema.Blob.FINAL_PHASE;
 import static udmi.schema.Blob.IOT_CONFIG_BLOB;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -768,7 +768,7 @@ private String extractConfigBlob(String blobName) {
         return null;
       }
       BlobBlobsetConfig blobBlobsetConfig = deviceConfig.blobset.blobs.get(blobName);
-      if (blobBlobsetConfig != null && COMMIT_PHASE.equals(blobBlobsetConfig.phase)
+      if (blobBlobsetConfig != null && FINAL_PHASE.equals(blobBlobsetConfig.phase)
           && blobBlobsetConfig.base64 != null) {
         return new String(Base64.getDecoder().decode(blobBlobsetConfig.base64));
       }

From d49bf30673945ddd746c01bacb7381e4861a2854 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:51:25 -0700
Subject: [PATCH 30/49] Adding mergey

---
 bin/test_redirect  | 33 +++++++++++++++++++++++++++++++--
 bin/test_validator |  4 ++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index 37f43331fb..928cda0446 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -24,6 +24,10 @@ device_id=AHU-22 # Static device for testing. Needs to be different than other t
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
+site_config=$site_path/cloud_iot_config.json
+cloud_region=$(jq -r .cloud_region $site_config)
+registry_id=$(jq -r .registry_id $site_config)
+
 PUBBER_OUT=pubber.out
 
 # Ideally use lock files, not grep-and-kill...
@@ -39,7 +43,7 @@ echo bin/pubber $site_path $project_id $device_id $serial_no
 
 echo Writing pubber output to $PUBBER_OUT.1
 result=0
-timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
+#timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
 echo Pubber exit code $result
 
 echo Writing pubber output to $PUBBER_OUT.2
@@ -47,5 +51,30 @@ bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect
 sleep 20
-bin/reset_config $site_path $project_id $device_id #redirect_config.json
+
+cat <<EOF > endpoint.json
+{
+  "projectId": "$project_id",
+  "registryId": "$registry_id",
+  "cloudRegion": "$cloud_region"
+}
+EOF
+
+base64=$(base64 endpoint.json)
+
+cat <<EOF > endpoint.json
+{
+  "blobs": {
+    "_iot_config": {
+      "phase": "final",
+      "base64": "$base64"
+    }
+  }
+}
+EOF
+
+out_base=$site_path/devices/$device_id/out
+jq -s '.[0] * .[1]' $out_base/generated_config.json endpoint.json > $out_base/redirect_config.json
+
+bin/reset_config $site_path $project_id $device_id redirect_config.json
 sleep 20
diff --git a/bin/test_validator b/bin/test_validator
index 62d890e104..1410391faa 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -33,13 +33,13 @@ rm -rf $site_path/out/devices
 mkdir -p $site_path/validator
 cp $site_path/devices/AHU-1/rsa_private.pkcs8 $site_path/validator/rsa_private.pkcs8
 
+bin/reset_config $site_path $project_id $device_id
+
 bin/validator $site_path $project_id &
 vpid=$!
 sleep 10
 echo Started validator pid $vpid
 
-bin/reset_config $site_path $project_id $device_id
-
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT

From 35073cb8f17630bca2d308108b350ca554852de6 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 14:51:46 -0700
Subject: [PATCH 31/49] Change order

---
 bin/test_validator | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/test_validator b/bin/test_validator
index 8ec6777502..184913875a 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -33,13 +33,13 @@ rm -rf $site_path/out/devices
 mkdir -p $site_path/validator
 cp $site_path/devices/AHU-1/rsa_private.pkcs8 $site_path/validator/rsa_private.pkcs8
 
+bin/reset_config $site_path $project_id $device_id
+
 bin/validator $site_path $project_id &
 vpid=$!
 sleep 10
 echo Started validator pid $vpid
 
-bin/reset_config $site_path $project_id $device_id
-
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT

From ea57cb880ce673808f84fac8de609a960737f7d0 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 15:56:26 -0700
Subject: [PATCH 32/49] Fixing config redirect

---
 bin/reset_config  | 21 +++++++++++++--------
 bin/test_redirect | 15 ++++++++-------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/bin/reset_config b/bin/reset_config
index 4917dd7508..03203473c7 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -2,8 +2,8 @@
 
 ROOT=$(realpath $(dirname $0)/..)
 
-if [[ $# != 3 ]]; then
-    echo Usage: $0 site_dir project_id device_id
+if [[ $# != 3 && $# != 4 ]]; then
+    echo Usage: $0 site_dir project_id device_id [config_file]
     false
 fi
 
@@ -11,16 +11,21 @@ site_dir=$(realpath $1)
 project_id=$2
 device_id=$3
 shift 3
-cd ${ROOT}
 
-pwd
+config_file=generated_config.json
+if [[ -n $1 ]]; then
+    config_file=$1
+    shift
+fi
+
+cd ${ROOT}
 
-device_config=/tmp/${device_id}_config.json
-cp ${site_dir}/devices/${device_id}/out/generated_config.json ${device_config}
+dst_config=/tmp/${device_id}_config.json
+src_config=${site_dir}/devices/${device_id}/out/$config_file
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp ${now_date}
 sponge < /dev/null || echo sponge command not found, might need workaround for osx or other
-jq .timestamp=\"${now_date}\" < ${device_config} | sponge ${device_config}
+jq .timestamp=\"${now_date}\" < ${src_config} > ${dst_config}
 
 echo Resetting device ${device_id} config...
-validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${device_config}
+validator/bin/reflector ${site_dir} ${project_id} ${device_id} update/config:${dst_config}
diff --git a/bin/test_redirect b/bin/test_redirect
index 928cda0446..cd9c6a85df 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -37,7 +37,7 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
-bin/reset_config $site_path $project_id $device_id
+#bin/reset_config $site_path $project_id $device_id
 
 echo bin/pubber $site_path $project_id $device_id $serial_no
 
@@ -47,10 +47,10 @@ result=0
 echo Pubber exit code $result
 
 echo Writing pubber output to $PUBBER_OUT.2
-bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
+#bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect
-sleep 20
+#sleep 20
 
 cat <<EOF > endpoint.json
 {
@@ -60,9 +60,9 @@ cat <<EOF > endpoint.json
 }
 EOF
 
-base64=$(base64 endpoint.json)
+base64=$(base64 -w 0 endpoint.json)
 
-cat <<EOF > endpoint.json
+cat <<EOF > blobs.json
 {
   "blobs": {
     "_iot_config": {
@@ -74,7 +74,8 @@ cat <<EOF > endpoint.json
 EOF
 
 out_base=$site_path/devices/$device_id/out
-jq -s '.[0] * .[1]' $out_base/generated_config.json endpoint.json > $out_base/redirect_config.json
+jq -s '.[0] * .[1]' $out_base/generated_config.json blobs.json > $out_base/redirect_config.json
 
+echo bin/reset_config $site_path $project_id $device_id redirect_config.json
 bin/reset_config $site_path $project_id $device_id redirect_config.json
-sleep 20
+#sleep 20

From 8a828a56472f031174c1a7682eada55bfba9b3d7 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 16:18:13 -0700
Subject: [PATCH 33/49] Updating test script

---
 bin/test_redirect | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index cd9c6a85df..cf7ebf141b 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -37,20 +37,20 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
-#bin/reset_config $site_path $project_id $device_id
+bin/reset_config $site_path $project_id $device_id
 
 echo bin/pubber $site_path $project_id $device_id $serial_no
 
 echo Writing pubber output to $PUBBER_OUT.1
 result=0
-#timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
+timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
 echo Pubber exit code $result
 
 echo Writing pubber output to $PUBBER_OUT.2
-#bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
+bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect
-#sleep 20
+sleep 20
 
 cat <<EOF > endpoint.json
 {
@@ -78,4 +78,6 @@ jq -s '.[0] * .[1]' $out_base/generated_config.json blobs.json > $out_base/redir
 
 echo bin/reset_config $site_path $project_id $device_id redirect_config.json
 bin/reset_config $site_path $project_id $device_id redirect_config.json
-#sleep 20
+
+# Let pubber react to the new configuration
+sleep 20

From 8e668844d418fe9f74405cc76fa634f2285d813a Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 16:28:58 -0700
Subject: [PATCH 34/49] Cleanup debug

---
 .github/workflows/testing.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index a21ee520c5..d5f19437a7 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -63,13 +63,12 @@ jobs:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
-          # Simple checks that a redirect happened and failed
-          set +x
+          echo Simple checks that a redirect happened and failed
           fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.1
           fgrep 'system.config.parse success' pubber.out.1
           fgrep registries/missing/devices/GAT-123 pubber.out.1
           fgrep 'Not authorized to connect' pubber.out.1
-          # Now check the redirect-by-config setup
+          echo Check the redirect-by-config setup
           fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.2
           fgrep 'system.config.parse success' pubber.out.2
           fgrep registries/reconfigure/devices/GAT-123 pubber.out.2
@@ -77,7 +76,10 @@ jobs:
       - name: pubber logs
         if: ${{ always() }}
         run: |
-          cat pubber.out || true
+          echo ::::::: pubber.out.1
+          cat pubber.out.1 || true
+          echo ::::::: pubber.out.2
+          cat pubber.out.2 || true
     
   udmi:
     name: Sequence tests

From 318f1c11b20e8bea1b20a8d2b67bb44cf5b280a0 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 16:57:34 -0700
Subject: [PATCH 35/49] Cleanup testing scripts

---
 bin/pubber         |  2 +-
 bin/test_redirect  | 15 +++++++--------
 bin/test_validator | 12 ++++--------
 3 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/bin/pubber b/bin/pubber
index 55cc8173ba..35d7b03c82 100755
--- a/bin/pubber
+++ b/bin/pubber
@@ -19,7 +19,7 @@ if [ ! -f $site_path/cloud_iot_config.json ]; then
 fi
 
 echo Building pubber...
-$ROOT_DIR/pubber/bin/build > /dev/null
+$ROOT_DIR/pubber/bin/build
 
 echo Running tools version `(cd $ROOT_DIR; git describe)`
 
diff --git a/bin/test_redirect b/bin/test_redirect
index cf7ebf141b..541ecd5e01 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -30,12 +30,8 @@ registry_id=$(jq -r .registry_id $site_config)
 
 PUBBER_OUT=pubber.out
 
-# Ideally use lock files, not grep-and-kill...
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-if [[ -n $pids ]]; then
-    echo Killing pubber pids $pids
-    kill $pids
-fi
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
 bin/reset_config $site_path $project_id $device_id
 
@@ -43,8 +39,7 @@ echo bin/pubber $site_path $project_id $device_id $serial_no
 
 echo Writing pubber output to $PUBBER_OUT.1
 result=0
-timeout 5m bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || result=$?
-echo Pubber exit code $result
+timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1
 
 echo Writing pubber output to $PUBBER_OUT.2
 bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
@@ -81,3 +76,7 @@ bin/reset_config $site_path $project_id $device_id redirect_config.json
 
 # Let pubber react to the new configuration
 sleep 20
+
+# Ideally use lock files, not grep-and-kill...
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
diff --git a/bin/test_validator b/bin/test_validator
index 184913875a..5656b76a4a 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -21,11 +21,8 @@ echo Using pubber with serial $serial_no
 
 PUBBER_OUT=pubber.out
 
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-if [[ -n $pids ]]; then
-    echo Killing pubber pids $pids
-    kill $pids
-fi
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
 rm -rf $site_path/out/devices
 
@@ -65,9 +62,8 @@ fi
 echo Waiting for system to run for a bit...
 sleep 30
 
-pids=`ps ax | fgrep pubber | fgrep java | awk '{print $1}'`
-echo Killing pids $vpid $pids
-kill $vpid $pids
+echo Killing running pubber instances...
+ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
 outfiles=`find $site_path/out/devices -name \*.out | sort` || true
 if [[ -z $outfiles ]]; then

From 1758786876c36800f3ddf0c0fa8a14af5a9dd031 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:09:15 -0700
Subject: [PATCH 36/49] Cleanup timeout error

---
 bin/test_redirect | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index dc7ebd0c08..ebd578f1b4 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -35,14 +35,10 @@ ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 
 bin/reset_config $site_path $project_id $device_id
 
-echo Writing pubber output to $PUBBER_OUT
-echo bin/pubber $site_path $project_id $device_id $serial_no
+echo Running pubber with redirectRegistry on timeout
+timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
 
-echo Writing pubber output to $PUBBER_OUT.1
-result=0
-timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1
-
-echo Writing pubber output to $PUBBER_OUT.2
+echo Running pubber in background...
 bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect

From 57ed11b504cff8cb26c7ae8c1035193b87914592 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:14:07 -0700
Subject: [PATCH 37/49] Add missing dep

---
 bin/setup_base    | 2 +-
 bin/test_redirect | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/setup_base b/bin/setup_base
index ce8da695fe..fa9721123f 100755
--- a/bin/setup_base
+++ b/bin/setup_base
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-sudo apt-get install -y hxtools moreutils
+sudo apt-get install -y hxtools moreutils expect
 
 python3 -m venv venv
 
diff --git a/bin/test_redirect b/bin/test_redirect
index ebd578f1b4..de6f45069f 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -77,3 +77,5 @@ sleep 20
 # Ideally use lock files, not grep-and-kill...
 echo Killing running pubber instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
+
+echo Done with redirect test.

From dac16d7481dfa6d8b186c23b33082e68622ff9e2 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:15:49 -0700
Subject: [PATCH 38/49] Cleanup output log reporting

---
 .github/workflows/testing.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d5f19437a7..9010b912a8 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -58,28 +58,32 @@ jobs:
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
           bin/test_redirect $GCP_TARGET_PROJECT
-      - name: pubber checks
+      - name: pubber cmd checks
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
+          echo ::::::: pubber.out.1
+          cat pubber.out.1
+          echo :::::::
           echo Simple checks that a redirect happened and failed
           fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.1
           fgrep 'system.config.parse success' pubber.out.1
           fgrep registries/missing/devices/GAT-123 pubber.out.1
           fgrep 'Not authorized to connect' pubber.out.1
+      - name: pubber config checks
+        env:
+          GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
+        if: "${{ env.GCP_TARGET_PROJECT != '' }}"
+        run: |
+          echo ::::::: pubber.out.2
+          cat pubber.out.2
+          echo :::::::
           echo Check the redirect-by-config setup
           fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.2
           fgrep 'system.config.parse success' pubber.out.2
           fgrep registries/reconfigure/devices/GAT-123 pubber.out.2
           fgrep 'Not authorized to connect' pubber.out.2
-      - name: pubber logs
-        if: ${{ always() }}
-        run: |
-          echo ::::::: pubber.out.1
-          cat pubber.out.1 || true
-          echo ::::::: pubber.out.2
-          cat pubber.out.2 || true
     
   udmi:
     name: Sequence tests

From 75d071a39cb217e89a2a4e02b82c0b8eae57e3da Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:19:53 -0700
Subject: [PATCH 39/49] Slight name relabel

---
 .github/workflows/testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 9010b912a8..c4f09291c8 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           bin/setup_base
           bin/clone_model
-      - name: pubber run
+      - name: pubber runs
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"

From 0ccaf40344c5e58ff769fe2e755ccb8a60253ee1 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:26:00 -0700
Subject: [PATCH 40/49] Testing

---
 bin/test_redirect | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index de6f45069f..b469c55299 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -36,7 +36,7 @@ ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 bin/reset_config $site_path $project_id $device_id
 
 echo Running pubber with redirectRegistry on timeout
-timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
+#timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
 
 echo Running pubber in background...
 bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
@@ -66,12 +66,16 @@ cat <<EOF > blobs.json
 EOF
 
 out_base=$site_path/devices/$device_id/out
+# Merge JSON files together into new redirect config
 jq -s '.[0] * .[1]' $out_base/generated_config.json blobs.json > $out_base/redirect_config.json
 
+echo New redirection config:
+cat $out_base/redirect_config.json
+
 echo bin/reset_config $site_path $project_id $device_id redirect_config.json
 bin/reset_config $site_path $project_id $device_id redirect_config.json
 
-# Let pubber react to the new configuration
+echo Let pubber react to the new configuration...
 sleep 20
 
 # Ideally use lock files, not grep-and-kill...

From 99b2fc8dc43013a8506e5101a2b382724570cb9f Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:26:40 -0700
Subject: [PATCH 41/49] Restore timeout

---
 bin/test_redirect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index b469c55299..48b56939da 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -36,7 +36,7 @@ ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
 bin/reset_config $site_path $project_id $device_id
 
 echo Running pubber with redirectRegistry on timeout
-#timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
+timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
 
 echo Running pubber in background...
 bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &

From c9852e89bdc5a4fab8ad50fb2db3e91a47fedd13 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:36:11 -0700
Subject: [PATCH 42/49] Use redirect registry

---
 .github/workflows/testing.yml | 8 ++++----
 bin/test_redirect             | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index c4f09291c8..813d57a9f2 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -67,9 +67,9 @@ jobs:
           cat pubber.out.1
           echo :::::::
           echo Simple checks that a redirect happened and failed
-          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.1
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1
           fgrep 'system.config.parse success' pubber.out.1
-          fgrep registries/missing/devices/GAT-123 pubber.out.1
+          fgrep registries/missing/devices pubber.out.1
           fgrep 'Not authorized to connect' pubber.out.1
       - name: pubber config checks
         env:
@@ -80,9 +80,9 @@ jobs:
           cat pubber.out.2
           echo :::::::
           echo Check the redirect-by-config setup
-          fgrep registries/ZZ-TRI-FECTA/devices/GAT-123 pubber.out.2
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2
           fgrep 'system.config.parse success' pubber.out.2
-          fgrep registries/reconfigure/devices/GAT-123 pubber.out.2
+          fgrep registries/reconfigure/devices pubber.out.2
           fgrep 'Not authorized to connect' pubber.out.2
     
   udmi:
diff --git a/bin/test_redirect b/bin/test_redirect
index 48b56939da..793ac96ca8 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -19,7 +19,7 @@ else
 fi
 
 site_path=sites/udmi_site_model
-device_id=AHU-22 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
+device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
 
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
@@ -47,7 +47,7 @@ sleep 20
 cat <<EOF > endpoint.json
 {
   "projectId": "$project_id",
-  "registryId": "$registry_id",
+  "registryId": "redirect",
   "cloudRegion": "$cloud_region"
 }
 EOF
@@ -70,7 +70,7 @@ out_base=$site_path/devices/$device_id/out
 jq -s '.[0] * .[1]' $out_base/generated_config.json blobs.json > $out_base/redirect_config.json
 
 echo New redirection config:
-cat $out_base/redirect_config.json
+cat /tmp/${device_id}_config.json
 
 echo bin/reset_config $site_path $project_id $device_id redirect_config.json
 bin/reset_config $site_path $project_id $device_id redirect_config.json

From 626313e14df687ffd2478a4d44ac1356befb3675 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:53:19 -0700
Subject: [PATCH 43/49] Fix reflect update function

---
 bin/test_redirect            | 10 +++++----
 dashboard/functions/index.js | 41 +++++++++++++++++++++---------------
 2 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index 793ac96ca8..bbd9265c1c 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -56,10 +56,12 @@ base64=$(base64 -w 0 endpoint.json)
 
 cat <<EOF > blobs.json
 {
-  "blobs": {
-    "_iot_config": {
-      "phase": "final",
-      "base64": "$base64"
+  "blobset": {
+    "blobs": {
+      "_iot_config": {
+        "phase": "final",
+        "base64": "$base64"
+      }
     }
   }
 }
diff --git a/dashboard/functions/index.js b/dashboard/functions/index.js
index 463bed3360..f961000eee 100644
--- a/dashboard/functions/index.js
+++ b/dashboard/functions/index.js
@@ -327,28 +327,35 @@ function parse_old_config(oldConfig, resetConfig) {
 
 async function modify_device_config(registryId, deviceId, subFolder, subContents, startTime) {
   const [oldConfig, version] = await get_device_config(registryId, deviceId);
+  var newConfig;
 
-  const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config';
-  const newConfig = parse_old_config(oldConfig, resetConfig);
-  if (newConfig === null) {
-    return;
-  }
-
-  newConfig.version = UDMI_VERSION;
-  newConfig.timestamp = currentTimestamp();
-
-  console.log('Config modify version', subFolder, version, startTime);
-  if (subContents) {
-    delete subContents.version;
-    delete subContents.timestamp;
-    newConfig[subFolder] = subContents;
+  if (subFolder == 'update') {
+    console.log('Config replace version', version, startTime);
+    newConfig = subContents;
   } else {
-    if (!newConfig[subFolder]) {
-      console.log('Config target already null', subFolder, version, startTime);
+    const resetConfig = subFolder == 'system' && subContents && subContents.extra_field == 'reset_config';
+    newConfig = parse_old_config(oldConfig, resetConfig);
+    if (newConfig === null) {
       return;
     }
-    delete newConfig[subFolder];
+
+    newConfig.version = UDMI_VERSION;
+    newConfig.timestamp = currentTimestamp();
+
+    console.log('Config modify version', subFolder, version, startTime);
+    if (subContents) {
+      delete subContents.version;
+      delete subContents.timestamp;
+      newConfig[subFolder] = subContents;
+    } else {
+      if (!newConfig[subFolder]) {
+        console.log('Config target already null', subFolder, version, startTime);
+        return;
+      }
+      delete newConfig[subFolder];
+    }
   }
+
   const attributes = {
     projectId: PROJECT_ID,
     cloudRegion: registry_regions[registryId],

From e7f9f72bbce44b474eafd4a7f90a5398b22e7f86 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 17:54:49 -0700
Subject: [PATCH 44/49] Tweak output

---
 bin/test_redirect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index bbd9265c1c..56a65fe2ab 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -38,7 +38,7 @@ bin/reset_config $site_path $project_id $device_id
 echo Running pubber with redirectRegistry on timeout
 timeout 3m unbuffer bin/pubber $site_path $project_id $device_id $serial_no redirectRegistry=missing > $PUBBER_OUT.1 2>&1 || true
 
-echo Running pubber in background...
+echo Done with timeout, running pubber in background...
 bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 
 # Wait for initial connection, then reset config to redirect

From 4985822606ec0f9e1c8900ee60232560ed8fe38e Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 20:25:51 -0700
Subject: [PATCH 45/49] Fix target registry

---
 bin/test_redirect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/test_redirect b/bin/test_redirect
index 56a65fe2ab..b467d2723d 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -47,7 +47,7 @@ sleep 20
 cat <<EOF > endpoint.json
 {
   "projectId": "$project_id",
-  "registryId": "redirect",
+  "registryId": "reconfigure",
   "cloudRegion": "$cloud_region"
 }
 EOF

From 619d9cb060f62f9c45b8d2731a2dd1ee339a5fd7 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 20:39:31 -0700
Subject: [PATCH 46/49] Review cleanup

---
 .github/workflows/testing.yml | 24 ++++++++++++------------
 bin/reset_config              |  1 -
 bin/test_redirect             | 11 ++++++-----
 bin/test_validator            |  5 +++--
 4 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 813d57a9f2..d45e5681bd 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -64,26 +64,26 @@ jobs:
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
           echo ::::::: pubber.out.1
-          cat pubber.out.1
+          cat out/pubber.out.1
           echo :::::::
           echo Simple checks that a redirect happened and failed
-          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1
-          fgrep 'system.config.parse success' pubber.out.1
-          fgrep registries/missing/devices pubber.out.1
-          fgrep 'Not authorized to connect' pubber.out.1
+          fgrep registries/ZZ-TRI-FECTA/devices out/pubber.out.1
+          fgrep 'system.config.parse success' out/pubber.out.1
+          fgrep registries/missing/devices out/pubber.out.1
+          fgrep 'Not authorized to connect' out/pubber.out.1
       - name: pubber config checks
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
-          echo ::::::: pubber.out.2
-          cat pubber.out.2
+          echo ::::::: out/pubber.out.2
+          cat out/pubber.out.2
           echo :::::::
           echo Check the redirect-by-config setup
-          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2
-          fgrep 'system.config.parse success' pubber.out.2
-          fgrep registries/reconfigure/devices pubber.out.2
-          fgrep 'Not authorized to connect' pubber.out.2
+          fgrep registries/ZZ-TRI-FECTA/devices out/pubber.out.2
+          fgrep 'system.config.parse success' out/pubber.out.2
+          fgrep registries/reconfigure/devices out/pubber.out.2
+          fgrep 'Not authorized to connect' out/pubber.out.2
     
   udmi:
     name: Sequence tests
@@ -127,5 +127,5 @@ jobs:
       - name: pubber logs
         if: ${{ always() }}
         run: |
-          cat pubber.out || true
+          cat out/pubber.out || true
           more out/*.json
diff --git a/bin/reset_config b/bin/reset_config
index 03203473c7..b084c514f9 100755
--- a/bin/reset_config
+++ b/bin/reset_config
@@ -24,7 +24,6 @@ dst_config=/tmp/${device_id}_config.json
 src_config=${site_dir}/devices/${device_id}/out/$config_file
 now_date=$(python3 -c 'import datetime; print(datetime.datetime.utcnow().isoformat() + "Z")')
 echo Setting config timestamp ${now_date}
-sponge < /dev/null || echo sponge command not found, might need workaround for osx or other
 jq .timestamp=\"${now_date}\" < ${src_config} > ${dst_config}
 
 echo Resetting device ${device_id} config...
diff --git a/bin/test_redirect b/bin/test_redirect
index b467d2723d..0e6afb4596 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -20,6 +20,7 @@ fi
 
 site_path=sites/udmi_site_model
 device_id=AHU-1 # Static device for testing. Needs to be different than other tests scripts so as not to conflict during CI.
+mkdir -p out
 
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
@@ -28,7 +29,7 @@ site_config=$site_path/cloud_iot_config.json
 cloud_region=$(jq -r .cloud_region $site_config)
 registry_id=$(jq -r .registry_id $site_config)
 
-PUBBER_OUT=pubber.out
+PUBBER_OUT=out/pubber.out
 
 echo Killing running pubber instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
@@ -44,7 +45,7 @@ bin/pubber $site_path $project_id $device_id $serial_no > $PUBBER_OUT.2 2>&1 &
 # Wait for initial connection, then reset config to redirect
 sleep 20
 
-cat <<EOF > endpoint.json
+cat <<EOF > out/endpoint.json
 {
   "projectId": "$project_id",
   "registryId": "reconfigure",
@@ -52,9 +53,9 @@ cat <<EOF > endpoint.json
 }
 EOF
 
-base64=$(base64 -w 0 endpoint.json)
+base64=$(base64 -w 0 out/endpoint.json)
 
-cat <<EOF > blobs.json
+cat <<EOF > out/blobs.json
 {
   "blobset": {
     "blobs": {
@@ -69,7 +70,7 @@ EOF
 
 out_base=$site_path/devices/$device_id/out
 # Merge JSON files together into new redirect config
-jq -s '.[0] * .[1]' $out_base/generated_config.json blobs.json > $out_base/redirect_config.json
+jq -s '.[0] * .[1]' $out_base/generated_config.json out/blobs.json > $out_base/redirect_config.json
 
 echo New redirection config:
 cat /tmp/${device_id}_config.json
diff --git a/bin/test_validator b/bin/test_validator
index 7346fc9c7c..90e8fa9f06 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -15,12 +15,13 @@ shift
 
 site_path=sites/udmi_site_model
 device_id=AHU-1
+mkdir -p out
 
 serial_no=validator-$RANDOM
 echo Using pubber with serial $serial_no
 
-PUBBER_OUT=pubber.out
-VALIDATOR_OUT=validator.out
+PUBBER_OUT=out/pubber.out
+VALIDATOR_OUT=out/validator.out
 WAITING=10
 
 echo Killing running pubber/validator instances...

From 9d29a88ef5d7ed30b255f39428815aec3ab08b5f Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 20:44:26 -0700
Subject: [PATCH 47/49] Reverting random changes

---
 .../src/device/dao/mongodb/MongoDeviceDAO.ts  |  2 +-
 .../src/device/dao/static/StaticDeviceDAO.ts  |  1 +
 udmif/event-handler/src/DeviceDao.ts          |  4 ++--
 .../src/DeviceDocumentFactory.ts              | 18 +++++++++++++----
 udmif/event-handler/src/DocumentTypeUtil.ts   |  2 ++
 udmif/event-handler/src/UdmiMessageHandler.ts |  2 +-
 .../src/__tests__/DeviceDao.spec.ts           |  5 +++--
 .../src/__tests__/DeviceDaoFactory.spec.ts    |  1 +
 .../src/__tests__/DeviceDocument.spec.ts      |  2 +-
 .../__tests__/DeviceDocumentFactory.spec.ts   | 20 ++++++++++++++++---
 .../src/__tests__/DeviceTypeUtils.spec.ts     |  2 ++
 .../src/__tests__/Points.spec.ts              |  9 ++-------
 udmif/web/src/app/app.module.ts               |  2 +-
 udmif/web/src/app/auth/auth.module.ts         |  2 +-
 udmif/web/src/app/auth/auth.service.ts        |  4 ++--
 udmif/web/src/app/graphql/graphql.module.ts   |  2 +-
 16 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts b/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts
index cdf5f0bde7..88604238d6 100644
--- a/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts
+++ b/udmif/api/src/device/dao/mongodb/MongoDeviceDAO.ts
@@ -18,7 +18,7 @@ import { getAggregate } from './MongoAggregateBuilder';
 
 // this class exists to return sorted, and filtered data from MongoDB
 export class MongoDeviceDAO implements DeviceDAO {
-  constructor(private db: Db) {}
+  constructor(private db: Db) { }
 
   async getDevices(searchOptions: SearchOptions): Promise<Device[]> {
     return this.db
diff --git a/udmif/api/src/device/dao/static/StaticDeviceDAO.ts b/udmif/api/src/device/dao/static/StaticDeviceDAO.ts
index e899bc69a0..fa86bd5455 100644
--- a/udmif/api/src/device/dao/static/StaticDeviceDAO.ts
+++ b/udmif/api/src/device/dao/static/StaticDeviceDAO.ts
@@ -50,6 +50,7 @@ export class StaticDeviceDAO implements DeviceDAO {
   }
 
   public async getFilteredDeviceCount(searchOptions: SearchOptions): Promise<number> {
+
     let filteredDevices = this.devices;
     if (searchOptions.filter) {
       const filters: Filter[] = fromString(searchOptions.filter);
diff --git a/udmif/event-handler/src/DeviceDao.ts b/udmif/event-handler/src/DeviceDao.ts
index 43dec976ad..92e8c76f87 100644
--- a/udmif/event-handler/src/DeviceDao.ts
+++ b/udmif/event-handler/src/DeviceDao.ts
@@ -10,12 +10,12 @@ export interface DeviceDao {
 }
 
 export class DefaultDeviceDao implements DeviceDao {
-  constructor(private collection: Collection<DeviceDocument>) {}
+  constructor(private collection: Collection<DeviceDocument>) { }
 
   /**
    * Updates a device document if it is found using the device key, else it will insert a new device document
    * @param {DeviceKey} deviceKey
-   * @param {DeviceDocument} deviceDocument
+   * @param {DeviceDocument} deviceDocument 
    */
   async upsert(deviceKey: DeviceKey, deviceDocument: DeviceDocument): Promise<void> {
     // we're using upsert which will allow document updates if it already exists and a document cretion if it does not
diff --git a/udmif/event-handler/src/DeviceDocumentFactory.ts b/udmif/event-handler/src/DeviceDocumentFactory.ts
index 4b85f88b8f..3eb4901505 100644
--- a/udmif/event-handler/src/DeviceDocumentFactory.ts
+++ b/udmif/event-handler/src/DeviceDocumentFactory.ts
@@ -1,11 +1,16 @@
 import { DeviceDocumentBuilder, DeviceDocument } from './DeviceDocument';
-import { isPointset, isSystem } from './DocumentTypeUtil';
+import {
+  isPointset,
+  isSystem,
+} from './DocumentTypeUtil';
 import { UdmiMessage } from './UdmiMessage';
 import { PointBuilder, Point } from './Point';
 
 export function createDeviceDocument(udmiMessage: UdmiMessage, existingPoints: Point[]): DeviceDocument {
   const builder: DeviceDocumentBuilder = new DeviceDocumentBuilder();
-  builder.id(udmiMessage.attributes.deviceNumId).name(udmiMessage.attributes.deviceId);
+  builder
+    .id(udmiMessage.attributes.deviceNumId)
+    .name(udmiMessage.attributes.deviceId);
 
   if (isSystem(udmiMessage)) {
     return buildDeviceDocumentFromSystem(udmiMessage, builder);
@@ -14,7 +19,10 @@ export function createDeviceDocument(udmiMessage: UdmiMessage, existingPoints: P
   }
 }
 
-function buildDeviceDocumentFromSystem(udmiMessage: UdmiMessage, builder: DeviceDocumentBuilder): DeviceDocument {
+function buildDeviceDocumentFromSystem(
+  udmiMessage: UdmiMessage,
+  builder: DeviceDocumentBuilder
+): DeviceDocument {
   return builder
     .lastPayload(udmiMessage.data.timestamp)
     .operational(udmiMessage.data.operational)
@@ -35,7 +43,7 @@ function buildDeviceDocumentFromPointset(
   const points: Point[] = [];
 
   for (var pointCode in udmiMessage.data.points) {
-    const existingPoint = existingPoints.find((point) => point.name === pointCode);
+    const existingPoint = existingPoints.find(point => point.name === pointCode);
     const point: Point = buildPoint(udmiMessage, existingPoint, pointCode);
     points.push(point);
   }
@@ -44,6 +52,7 @@ function buildDeviceDocumentFromPointset(
 }
 
 function buildPoint(udmiMessage: UdmiMessage, existingPoint: Point, pointCode: string): Point {
+
   const pointValue = udmiMessage.data.points[pointCode];
 
   // we get the value from either the message or the existing point
@@ -61,3 +70,4 @@ function buildPoint(udmiMessage: UdmiMessage, existingPoint: Point, pointCode: s
     .metaCode(pointCode)
     .build();
 }
+
diff --git a/udmif/event-handler/src/DocumentTypeUtil.ts b/udmif/event-handler/src/DocumentTypeUtil.ts
index 4e04e84c4e..34e4f7438f 100644
--- a/udmif/event-handler/src/DocumentTypeUtil.ts
+++ b/udmif/event-handler/src/DocumentTypeUtil.ts
@@ -15,3 +15,5 @@ export function isSystem(message): boolean {
 export function isSubFolder(message, folderName: string): boolean {
   return message.attributes.subFolder === folderName;
 }
+
+
diff --git a/udmif/event-handler/src/UdmiMessageHandler.ts b/udmif/event-handler/src/UdmiMessageHandler.ts
index d07ee707f7..db06311d95 100644
--- a/udmif/event-handler/src/UdmiMessageHandler.ts
+++ b/udmif/event-handler/src/UdmiMessageHandler.ts
@@ -5,7 +5,7 @@ import { UdmiMessage } from './UdmiMessage';
 import { DeviceKey } from './DeviceKey';
 
 export default class UdmiMessageHandler {
-  constructor(private deviceDao: DeviceDao) {}
+  constructor(private deviceDao: DeviceDao) { }
 
   async handleUdmiEvent(udmiMessage: UdmiMessage) {
     try {
diff --git a/udmif/event-handler/src/__tests__/DeviceDao.spec.ts b/udmif/event-handler/src/__tests__/DeviceDao.spec.ts
index ecf12c4d40..f7640b970b 100644
--- a/udmif/event-handler/src/__tests__/DeviceDao.spec.ts
+++ b/udmif/event-handler/src/__tests__/DeviceDao.spec.ts
@@ -23,7 +23,7 @@ afterAll(async () => {
 beforeEach(async () => {
   // clean the collection before each test
   await deviceCollection.deleteMany({});
-});
+})
 
 describe('DeviceDao.upsert', () => {
   test('upsert calls the updateOne method on the provided collection', () => {
@@ -44,6 +44,7 @@ describe('DeviceDao.upsert', () => {
 });
 
 describe('DeviceDao.get', () => {
+
   test('get method is called and returns the matching document', async () => {
     // arrange
     const findOneSpy = jest.spyOn(deviceCollection, 'findOne');
@@ -63,4 +64,4 @@ describe('DeviceDao.get', () => {
     expect(retrievedDeviceDocument).toEqual(insertedDeviceDocument);
     expect(findOneSpy).toHaveBeenCalledWith(deviceKey);
   });
-});
+});
\ No newline at end of file
diff --git a/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts b/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts
index 4f7955b52b..68ce119e10 100644
--- a/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts
+++ b/udmif/event-handler/src/__tests__/DeviceDaoFactory.spec.ts
@@ -23,6 +23,7 @@ describe('DeviceDaoFactory.getDeviceDAO()', () => {
 });
 
 describe('DeviceDaoFactory.getUri()', () => {
+
   // take a backup of the environment prior to running the tests
   const ENV_BACKUP = { ...process.env };
 
diff --git a/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts b/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts
index a6217a9ce2..2d4f6a0fc7 100644
--- a/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts
+++ b/udmif/event-handler/src/__tests__/DeviceDocument.spec.ts
@@ -60,7 +60,7 @@ describe('DeviceDocument.DeviceDocumentBuilder', () => {
       site,
       serialNumber,
       points,
-      tags,
+      tags
     });
   });
 });
diff --git a/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts b/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts
index 0837e55e9f..3cbd8cd3a2 100644
--- a/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts
+++ b/udmif/event-handler/src/__tests__/DeviceDocumentFactory.spec.ts
@@ -1,6 +1,12 @@
 import { createDeviceDocument } from '../DeviceDocumentFactory';
 import { DeviceDocument } from '../DeviceDocument';
-import { CONFIG, MODEL, POINTSET_SUB_FOLDER, STATE, SYSTEM_SUB_FOLDER } from '../DocumentTypeUtil';
+import {
+  CONFIG,
+  MODEL,
+  POINTSET_SUB_FOLDER,
+  STATE,
+  SYSTEM_SUB_FOLDER,
+} from '../DocumentTypeUtil';
 import { UdmiMessage } from '../UdmiMessage';
 import { Point } from '../Point';
 
@@ -10,6 +16,7 @@ const BASIC_SYSTEM_ATTRIBUTES = { deviceId: name, deviceNumId: id, subFolder: SY
 const BASIC_POINTSET_ATTRIBUTES = { deviceId: name, deviceNumId: id, subFolder: POINTSET_SUB_FOLDER };
 
 describe('DeviceDocumentFactory.createDeviceDocument.default', () => {
+
   const points: Point[] = [];
   const tags: string[] = [];
 
@@ -77,9 +84,10 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => {
 
   beforeEach(() => {
     existingPoints = [];
-  });
+  })
 
   test('creates a device document with pointset', () => {
+
     // arrange
     const inputMessage: UdmiMessage = {
       attributes: { ...BASIC_POINTSET_ATTRIBUTES },
@@ -106,6 +114,7 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => {
   });
 
   test('creates a device document with pointset model', () => {
+
     const inputMessage: UdmiMessage = {
       attributes: { ...BASIC_POINTSET_ATTRIBUTES, subType: MODEL },
       data: {
@@ -187,8 +196,11 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => {
   });
 
   test('merges a device document with pointset', () => {
+
     // existing point has units and a value
-    existingPoints.push({ name: faps, id: faps, value: '70', units: 'Bars', meta: { code: faps, units: 'Bars' } });
+    existingPoints.push(
+      { name: faps, id: faps, value: '70', units: 'Bars', meta: { code: faps, units: 'Bars' } }
+    )
 
     // arrange
     const inputMessage: UdmiMessage = {
@@ -214,4 +226,6 @@ describe('DeviceDocumentFactory.createDeviceDocument.pointset', () => {
     // act and assert
     expect(createDeviceDocument(inputMessage, existingPoints)).toEqual(expectedDeviceDocument);
   });
+
 });
+
diff --git a/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts b/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts
index e63d9ce1ab..3a48532cba 100644
--- a/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts
+++ b/udmif/event-handler/src/__tests__/DeviceTypeUtils.spec.ts
@@ -24,6 +24,7 @@ describe('DeviceTypeUtils.isSystemState', () => {
     const inputMessage: UdmiMessage = { attributes: { ...BASIC_SYSTEM_ATTRIBUTES, subType: MODEL }, data: {} };
     expect(isSystem(inputMessage)).toEqual(true);
   });
+
 });
 
 describe('DeviceDocumentFactory.isPointset...', () => {
@@ -47,3 +48,4 @@ describe('DeviceDocumentFactory.isPointset...', () => {
     expect(isPointset(inputMessage)).toEqual(true);
   });
 });
+
diff --git a/udmif/event-handler/src/__tests__/Points.spec.ts b/udmif/event-handler/src/__tests__/Points.spec.ts
index e6251779c9..3cc843d900 100644
--- a/udmif/event-handler/src/__tests__/Points.spec.ts
+++ b/udmif/event-handler/src/__tests__/Points.spec.ts
@@ -33,18 +33,13 @@ describe('Points.PointBuilder', () => {
     expect(builder.build()).toEqual({ id, name, state, units, value });
   });
   test('Builder allows optional meta attributes', () => {
-    builder
-      .id(id)
-      .name('filter_alarm_pressure_status')
-      .units('Degrees-Celsius')
-      .metaCode('filter_alarm_pressure_status')
-      .metaUnit('Degrees-Celsius');
+    builder.id(id).name('filter_alarm_pressure_status').units('Degrees-Celsius').metaCode('filter_alarm_pressure_status').metaUnit('Degrees-Celsius');
     expect(builder.build()).toEqual({
       id,
       name: 'filter_alarm_pressure_status',
       units: 'Degrees-Celsius',
       meta: { code: 'filter_alarm_pressure_status', units: 'Degrees-Celsius' },
-      state: '',
+      state: ''
     });
   });
 });
diff --git a/udmif/web/src/app/app.module.ts b/udmif/web/src/app/app.module.ts
index 36880444b3..0f3294cb45 100644
--- a/udmif/web/src/app/app.module.ts
+++ b/udmif/web/src/app/app.module.ts
@@ -21,4 +21,4 @@ import { EnvModule } from './env/env.module';
   ],
   bootstrap: [AppComponent],
 })
-export class AppModule {}
+export class AppModule { }
diff --git a/udmif/web/src/app/auth/auth.module.ts b/udmif/web/src/app/auth/auth.module.ts
index d5164aef03..acac42819c 100644
--- a/udmif/web/src/app/auth/auth.module.ts
+++ b/udmif/web/src/app/auth/auth.module.ts
@@ -26,4 +26,4 @@ export function SocialAuthFactory(env: EnvService): SocialAuthServiceConfig {
     },
   ],
 })
-export class AuthModule {}
+export class AuthModule { }
diff --git a/udmif/web/src/app/auth/auth.service.ts b/udmif/web/src/app/auth/auth.service.ts
index 1375b71176..7128b059bb 100644
--- a/udmif/web/src/app/auth/auth.service.ts
+++ b/udmif/web/src/app/auth/auth.service.ts
@@ -24,7 +24,7 @@ export class AuthService {
     try {
       await this.socialAuth.signIn(GoogleLoginProvider.PROVIDER_ID);
       this.router.navigateByUrl('/devices');
-    } catch {}
+    } catch { }
   }
 
   async logout(): Promise<void> {
@@ -35,7 +35,7 @@ export class AuthService {
       // and the AuthService needs to makes use of the Apollo service,
       // we need to use the injector.get pattern to avoid circular dep issue.
       this.injector.get<Apollo>(Apollo).client.clearStore();
-    } catch {}
+    } catch { }
   }
 
   async refreshToken(): Promise<void> {
diff --git a/udmif/web/src/app/graphql/graphql.module.ts b/udmif/web/src/app/graphql/graphql.module.ts
index 9beb6349d1..86e2d9c2f7 100644
--- a/udmif/web/src/app/graphql/graphql.module.ts
+++ b/udmif/web/src/app/graphql/graphql.module.ts
@@ -68,4 +68,4 @@ export function ApolloFactory(httpLink: HttpLink, env: EnvService, auth: AuthSer
     },
   ],
 })
-export class GraphQLModule {}
+export class GraphQLModule { }

From 8d62d7f007b39ba9747c2f9b9a81f822b080dda4 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 21:22:33 -0700
Subject: [PATCH 48/49] Revert out/ changes

---
 .github/workflows/testing.yml | 24 ++++++++++++------------
 bin/test_redirect             |  2 +-
 bin/test_validator            |  4 ++--
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d45e5681bd..813d57a9f2 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -64,26 +64,26 @@ jobs:
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
           echo ::::::: pubber.out.1
-          cat out/pubber.out.1
+          cat pubber.out.1
           echo :::::::
           echo Simple checks that a redirect happened and failed
-          fgrep registries/ZZ-TRI-FECTA/devices out/pubber.out.1
-          fgrep 'system.config.parse success' out/pubber.out.1
-          fgrep registries/missing/devices out/pubber.out.1
-          fgrep 'Not authorized to connect' out/pubber.out.1
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.1
+          fgrep 'system.config.parse success' pubber.out.1
+          fgrep registries/missing/devices pubber.out.1
+          fgrep 'Not authorized to connect' pubber.out.1
       - name: pubber config checks
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
         if: "${{ env.GCP_TARGET_PROJECT != '' }}"
         run: |
-          echo ::::::: out/pubber.out.2
-          cat out/pubber.out.2
+          echo ::::::: pubber.out.2
+          cat pubber.out.2
           echo :::::::
           echo Check the redirect-by-config setup
-          fgrep registries/ZZ-TRI-FECTA/devices out/pubber.out.2
-          fgrep 'system.config.parse success' out/pubber.out.2
-          fgrep registries/reconfigure/devices out/pubber.out.2
-          fgrep 'Not authorized to connect' out/pubber.out.2
+          fgrep registries/ZZ-TRI-FECTA/devices pubber.out.2
+          fgrep 'system.config.parse success' pubber.out.2
+          fgrep registries/reconfigure/devices pubber.out.2
+          fgrep 'Not authorized to connect' pubber.out.2
     
   udmi:
     name: Sequence tests
@@ -127,5 +127,5 @@ jobs:
       - name: pubber logs
         if: ${{ always() }}
         run: |
-          cat out/pubber.out || true
+          cat pubber.out || true
           more out/*.json
diff --git a/bin/test_redirect b/bin/test_redirect
index 0e6afb4596..e6e91e50ae 100755
--- a/bin/test_redirect
+++ b/bin/test_redirect
@@ -29,7 +29,7 @@ site_config=$site_path/cloud_iot_config.json
 cloud_region=$(jq -r .cloud_region $site_config)
 registry_id=$(jq -r .registry_id $site_config)
 
-PUBBER_OUT=out/pubber.out
+PUBBER_OUT=pubber.out
 
 echo Killing running pubber instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs kill || true
diff --git a/bin/test_validator b/bin/test_validator
index 90e8fa9f06..738cce7ae2 100755
--- a/bin/test_validator
+++ b/bin/test_validator
@@ -20,8 +20,8 @@ mkdir -p out
 serial_no=validator-$RANDOM
 echo Using pubber with serial $serial_no
 
-PUBBER_OUT=out/pubber.out
-VALIDATOR_OUT=out/validator.out
+PUBBER_OUT=pubber.out
+VALIDATOR_OUT=validator.out
 WAITING=10
 
 echo Killing running pubber/validator instances...

From 9bd06b2e587ff9e3b1950100e0bef07be661f210 Mon Sep 17 00:00:00 2001
From: Trevor Pering <peringknife@google.com>
Date: Thu, 4 Aug 2022 22:18:15 -0700
Subject: [PATCH 49/49] Reset config before launching pubber

---
 bin/test_sequencer | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/test_sequencer b/bin/test_sequencer
index cd6df9eddf..b355765201 100755
--- a/bin/test_sequencer
+++ b/bin/test_sequencer
@@ -32,6 +32,8 @@ if [[ -n $pids ]]; then
     kill $pids
 fi
 
+bin/reset_config $site_path $project_id $device_id
+
 pubber/bin/build
 
 echo Writing pubber output to $PUBBER_OUT