Cleanup multiple tool invocation errors (#756)

bin/registrar

@@ -3,6 +3,10 @@
 UDMI_ROOT=$(dirname $0)/..
 source $UDMI_ROOT/etc/shell_common.sh
 
+OUT_DIR=$UDMI_ROOT/out
+mkdir -p $OUT_DIR
+echo $(date --utc +%Y%m%d-%H%M%S) $0 $@ >> $OUT_DIR/cmd_history.log
+
 config_file=/tmp/registrar_config.json
 
 [[ $# == 0 ]] && usage 'SITE_{PATH/FILE}' [PROJECT_ID] [OPTIONS] [devices...]
@@ -65,7 +69,7 @@ else
 fi
 
 if [[ -f $site_path ]]; then
-    rm -f $config_file
+    [[ $site_path -ef $config_file ]] || rm -f $config_file
     config_file=$site_path
 else
     echo Writing config to $config_file
@@ -85,6 +89,6 @@ echo $config_file:
 cat $config_file
 echo
 
-validator/bin/registrar $config_file $* 2>&1
+validator/bin/registrar $config_file $* 2>&1 | tee $OUT_DIR/registrar.log
 
 bin/check_version

bin/sequencer

@@ -8,6 +8,9 @@ UDMI_ROOT=$(realpath $(dirname $0)/..)
 source $UDMI_ROOT/etc/shell_common.sh
 
 OUT_DIR=$UDMI_ROOT/out
+mkdir -p $OUT_DIR
+echo $(date --utc +%Y%m%d-%H%M%S) $0 $@ >> $OUT_DIR/cmd_history.log
+
 SEQUENCER_LOG=$OUT_DIR/sequencer.log
 SEQUENCER_OUT=$OUT_DIR/sequencer.out
 SCHEMA_OUT=$OUT_DIR/schema.out
@@ -75,7 +78,6 @@ test_names="$@"
 udmi_version=`git describe --always --dirty`
 
 cd $UDMI_ROOT
-mkdir -p $OUT_DIR
 
 if ! bin/which_venv_pip3; then
     echo ERROR venv pip3 not installed, please run bin/setup_base and try again

bin/support_process

@@ -4,7 +4,7 @@ archive=$1
 shift || true
 
 if [[ -z $archive ]]; then
-    archive=$(ls -t ~/Downloads/udmi-support_*.zip | head -n 1)
+    archive=$(ls -t ~/Downloads/udmi-support_*.zip ~/Downloads/udmi-support_*.tgz | head -n 1)
     echo Auto-detected archive $archive
 fi
 
@@ -21,6 +21,12 @@ rm -f *_udmi-support_*.tgz
 rm -rf out $sites_out
 mkdir -p out $sites_out
 
+if [[ ${archive}\$ =~ '.tgz$' ]]; then
+    tar -xzvf $archive
+    echo Extracted singular archive $archive
+    exit 0
+fi
+
 if [[ -f $archive ]]; then
     rm -rf $archive_dir
     echo Extracting archive to $archive_dir

bin/validator

@@ -3,6 +3,10 @@
 UDMI_ROOT=$(realpath $(dirname $0)/..)
 source $UDMI_ROOT/etc/shell_common.sh
 
+OUT_DIR=$UDMI_ROOT/out
+mkdir -p $OUT_DIR
+echo $(date --utc +%Y%m%d-%H%M%S) $0 $@ >> $OUT_DIR/cmd_history.log
+
 [[ $# == 0 ]] && usage 'SITE_{PATH/FILE}' [PROJECT_ID] [SUBSCRIPTION_ID]
 
 site_path=$(realpath $1)
@@ -72,6 +76,6 @@ cat $site_file
 
 echo Running tools version $udmi_version
 
-$UDMI_ROOT/validator/bin/validate $site_file $*
+$UDMI_ROOT/validator/bin/validate $site_file $* 2>&1 | tee $OUT_DIR/validator.log
 
 bin/check_version

common/src/main/java/com/google/udmi/util/Common.java

@@ -5,6 +5,7 @@
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
 import com.google.common.reflect.ClassPath;
 import com.google.common.reflect.ClassPath.ClassInfo;
 import java.util.ArrayList;
@@ -161,6 +162,6 @@ private static String getExceptionMessage(Throwable exception, Class<?> containe
   }
 
   public static String getNamespacePrefix(String udmiNamespace) {
-    return ofNullable(udmiNamespace).map(x -> x + PREFIX_SEPARATOR).orElse("");
+    return Strings.isNullOrEmpty(udmiNamespace) ? "" : udmiNamespace + PREFIX_SEPARATOR;
   }
 }

validator/src/main/java/com/google/bos/iot/core/proxy/IotReflectorClient.java

@@ -14,7 +14,6 @@
 import static com.google.udmi.util.JsonUtil.getDate;
 import static com.google.udmi.util.JsonUtil.getTimestamp;
 import static com.google.udmi.util.JsonUtil.stringify;
-import static com.google.udmi.util.JsonUtil.stringifyTerse;
 import static com.google.udmi.util.JsonUtil.toMap;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
@@ -46,6 +45,7 @@
 import udmi.schema.Envelope.SubFolder;
 import udmi.schema.Envelope.SubType;
 import udmi.schema.ExecutionConfiguration;
+import udmi.schema.IotAccess.IotProvider;
 import udmi.schema.SetupUdmiConfig;
 import udmi.schema.SetupUdmiState;
 import udmi.schema.UdmiConfig;
@@ -57,8 +57,8 @@
 public class IotReflectorClient implements MessagePublisher {
 
   public static final String UDMI_FOLDER = "udmi";
-  static final String REFLECTOR_KEY_ALGORITHM = "RS256";
   public static final String UDMI_REFLECT = "UDMI-REFLECT";
+  static final String REFLECTOR_KEY_ALGORITHM = "RS256";
   private static final String MOCK_DEVICE_NUM_ID = "123456789101112";
   private static final String UDMI_TOPIC = "events/" + UDMI_FOLDER;
   private static final long CONFIG_TIMEOUT_SEC = 10;
@@ -100,7 +100,8 @@ public IotReflectorClient(ExecutionConfiguration iotConfig, int requiredVersion)
     String cloudRegion = Optional.ofNullable(iotConfig.reflect_region)
         .orElse(iotConfig.cloud_region);
     String prefix = getNamespacePrefix(iotConfig.udmi_namespace);
-    String iotProvider = iotConfig.iot_provider.value();
+    String iotProvider = ifNotNullGet(iotConfig.iot_provider, IotProvider::value,
+        IotProvider.IMPLICIT.value());
     subscriptionId = format("%s/%s/%s/%s%s/%s",
         projectId, iotProvider, cloudRegion, prefix, UDMI_REFLECT, registryId);
 

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -3,9 +3,10 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.daq.mqtt.registrar.Registrar.DEVICE_ERRORS_MAP;
-import static com.google.daq.mqtt.registrar.Registrar.ENVELOPE_JSON;
+import static com.google.daq.mqtt.registrar.Registrar.ENVELOPE_SCHEMA_JSON;
 import static com.google.daq.mqtt.registrar.Registrar.GENERATED_CONFIG_JSON;
 import static com.google.daq.mqtt.registrar.Registrar.METADATA_JSON;
+import static com.google.daq.mqtt.registrar.Registrar.METADATA_SCHEMA_JSON;
 import static com.google.daq.mqtt.registrar.Registrar.NORMALIZED_JSON;
 import static com.google.udmi.util.Common.VERSION_KEY;
 import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_STRICT;
@@ -288,7 +289,7 @@ private Metadata readMetadataWithValidation(boolean validate) {
     JsonNode mergedMetadata = getMergedMetadata(instance);
 
     try {
-      ProcessingReport report = schemas.get(METADATA_JSON).validate(mergedMetadata);
+      ProcessingReport report = schemas.get(METADATA_SCHEMA_JSON).validate(mergedMetadata);
       if (validate) {
         parseMetadataValidateProcessingReport(report);
       }
@@ -620,11 +621,11 @@ public void validateEnvelope(String registryId, String siteName) {
       envelope.projectId = fakeProjectId();
       envelope.deviceNumId = makeNumId(envelope);
       String envelopeJson = OBJECT_MAPPER_STRICT.writeValueAsString(envelope);
-      ProcessingReport processingReport = schemas.get(ENVELOPE_JSON)
+      ProcessingReport processingReport = schemas.get(ENVELOPE_SCHEMA_JSON)
           .validate(OBJECT_MAPPER.readTree(envelopeJson));
       if (!processingReport.isSuccess()) {
         processingReport.forEach(action -> {
-          throw new RuntimeException("against schema", action.asException());
+          throw new RuntimeException("Against envelope schema", action.asException());
         });
       }
     } catch (Exception e) {

validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java

@@ -18,6 +18,8 @@
 import static java.util.Optional.ofNullable;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
 import com.github.fge.jsonschema.main.JsonSchema;
@@ -81,7 +83,8 @@ public class Registrar {
   public static final Joiner JOIN_CSV = Joiner.on(", ");
   public static final File BASE_DIR = new File(".");
   static final String METADATA_JSON = "metadata.json";
-  static final String ENVELOPE_JSON = "envelope.json";
+  static final String ENVELOPE_SCHEMA_JSON = "envelope.json";
+  static final String METADATA_SCHEMA_JSON = "metadata.json";
   static final String NORMALIZED_JSON = "metadata_norm.json";
   static final String DEVICE_ERRORS_MAP = "errors.map";
   static final String GENERATED_CONFIG_JSON = "generated_config.json";
@@ -990,14 +993,20 @@ protected void setToolRoot(String toolRoot) {
   private void loadSchema(String key) {
     File schemaFile = new File(schemaBase, key);
     try (InputStream schemaStream = Files.newInputStream(schemaFile.toPath())) {
+      JsonNode schemaTree = OBJECT_MAPPER.readTree(schemaStream);
+      if (schemaTree instanceof ObjectNode schemaObject) {
+        Set<String> toRemove = new HashSet<>();
+        schemaObject.fields().forEachRemaining(entry -> {
+          ifTrueThen(entry.getKey().startsWith("$"), () -> toRemove.add(entry.getKey()));
+        });
+        toRemove.forEach(schemaObject::remove);
+      }
       JsonSchema schema =
           JsonSchemaFactory.newBuilder()
               .setLoadingConfiguration(
                   LoadingConfiguration.newBuilder()
                       .addScheme("file", new RelativeDownloader())
-                      .freeze())
-              .freeze()
-              .getJsonSchema(OBJECT_MAPPER.readTree(schemaStream));
+                      .freeze()).freeze().getJsonSchema(schemaTree);
       schemas.put(key, schema);
     } catch (Exception e) {
       throw new RuntimeException("While loading schema " + schemaFile.getAbsolutePath(), e);
@@ -1016,7 +1025,7 @@ private void loadSiteMetadata() {
       // At this time, do not validate the Metadata schema because, by its nature of being
       // a partial overlay on each device Metadata, this Metadata will likely be incomplete
       // and fail validation.
-      schemas.get(METADATA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
+      schemas.get(METADATA_SCHEMA_JSON).validate(OBJECT_MAPPER.readTree(targetStream));
     } catch (FileNotFoundException e) {
       return;
     } catch (Exception e) {

validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java

@@ -1,11 +1,12 @@
 package com.google.daq.mqtt.util;
 
 import static com.google.udmi.util.Common.getNamespacePrefix;
-import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
+import static java.util.Optional.ofNullable;
+import static udmi.schema.IotAccess.IotProvider.IMPLICIT;
+import static udmi.schema.IotAccess.IotProvider.PUBSUB;
 
 import com.google.bos.iot.core.proxy.MqttPublisher;
 import com.google.daq.mqtt.validator.Validator.MessageBundle;
-import com.google.udmi.util.Common;
 import com.google.udmi.util.PubSubReflector;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
@@ -32,7 +33,11 @@ static String getRegistryId(ExecutionConfiguration config) {
    */
   static MessagePublisher from(ExecutionConfiguration iotConfig,
       BiConsumer<String, String> messageHandler, Consumer<Throwable> errorHandler) {
-    if (IotAccess.IotProvider.PUBSUB == iotConfig.iot_provider) {
+    IotAccess.IotProvider iotProvider = ofNullable(iotConfig.iot_provider).orElse(IMPLICIT);
+    if (iotConfig.reflector_endpoint != null && iotProvider != IMPLICIT) {
+      throw new RuntimeException("Explicit endpoint conflicts with iot_provider " + iotProvider);
+    }
+    if (PUBSUB == iotConfig.iot_provider) {
       return PubSubReflector.from(iotConfig, messageHandler, errorHandler);
     }
     return MqttPublisher.from(iotConfig, messageHandler, errorHandler);