Move MessageValidator to common (#1091)

common/src/main/java/com/google/udmi/util/Common.java

@@ -9,7 +9,7 @@
 import com.google.common.base.Strings;
 import com.google.common.reflect.ClassPath;
 import com.google.common.reflect.ClassPath.ClassInfo;
-import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
+import com.google.udmi.util.ExceptionMap.ExceptionCategory;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.MissingFormatArgumentException;

common/src/main/java/com/google/daq/mqtt/util/ContextWrapper.java → common/src/main/java/com/google/udmi/util/ContextWrapper.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import java.util.List;
 import java.util.ArrayList;

common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java → common/src/main/java/com/google/udmi/util/ErrorMap.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import java.util.HashMap;
 import java.util.Map;

common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java → common/src/main/java/com/google/udmi/util/ExceptionList.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import static java.lang.String.format;
 

common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java → common/src/main/java/com/google/udmi/util/ExceptionMap.java

@@ -1,8 +1,8 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import static com.google.udmi.util.GeneralUtils.multiTrim;
 
-import com.google.daq.mqtt.util.ErrorMap.ErrorMapException;
+import com.google.udmi.util.ErrorMap.ErrorMapException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -19,7 +19,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
-import com.google.daq.mqtt.util.ValidationException;
 import com.google.udmi.util.ProperPrinter.OutputFormat;
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;

validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java → common/src/main/java/com/google/udmi/util/MessageValidator.java

@@ -1,17 +1,21 @@
-package com.google.bos.iot.core.proxy;
+package com.google.udmi.util;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static java.lang.String.format;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.fge.jsonschema.core.exceptions.ProcessingException;
 import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration;
 import com.github.fge.jsonschema.core.load.download.URIDownloader;
+import com.github.fge.jsonschema.core.report.LogLevel;
+import com.github.fge.jsonschema.core.report.ProcessingMessage;
 import com.github.fge.jsonschema.core.report.ProcessingReport;
 import com.github.fge.jsonschema.main.JsonSchema;
 import com.github.fge.jsonschema.main.JsonSchemaFactory;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
-import com.google.daq.mqtt.util.ValidationException;
-import com.google.daq.mqtt.validator.Validator;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -20,13 +24,20 @@
 import java.net.URL;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.StreamSupport;
 
 /**
  * Validation wrapper for processing individual messages.
  */
 public class MessageValidator {
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+  @SuppressWarnings("checkstyle:linelength")
+  private static final List<String> IGNORE_LIST = ImmutableList.of(
+      "instance type \\(string\\) does not match any allowed primitive type \\(allowed: \\[.*\"number\"\\]\\)");
+  private static final List<Pattern> IGNORE_PATTERNS = IGNORE_LIST.stream().map(Pattern::compile)
+      .toList();
 
   private final Map<String, JsonSchema> schemaMap = Maps.newConcurrentMap();
   private final File schemaRoot;
@@ -44,6 +55,39 @@ public MessageValidator(String schemaRootPath) {
     }
   }
 
+  /**
+   * From an external processing report.
+   *
+   * @param report Report to convert
+   * @return Converted exception.
+   */
+  public static ValidationException fromProcessingReport(ProcessingReport report) {
+    checkArgument(!report.isSuccess(), "Report must not be successful");
+    ImmutableList<ValidationException> causingExceptions =
+        StreamSupport.stream(report.spliterator(), false)
+            .filter(MessageValidator::errorOrWorse)
+            .filter(MessageValidator::notOnIgnoreList)
+            .map(MessageValidator::convertMessage).collect(toImmutableList());
+    return causingExceptions.isEmpty() ? null : new ValidationException(
+        format("%d schema violations found", causingExceptions.size()), causingExceptions);
+  }
+
+  private static boolean notOnIgnoreList(ProcessingMessage processingMessage) {
+    return IGNORE_PATTERNS.stream()
+        .noneMatch(p -> p.matcher(processingMessage.getMessage()).matches());
+  }
+
+  private static boolean errorOrWorse(ProcessingMessage processingMessage) {
+    return processingMessage.getLogLevel().compareTo(LogLevel.ERROR) >= 0;
+  }
+
+  private static ValidationException convertMessage(ProcessingMessage processingMessage) {
+    String pointer = processingMessage.asJson().get("instance").get("pointer").asText();
+    String prefix =
+        com.google.api.client.util.Strings.isNullOrEmpty(pointer) ? "" : (pointer + ": ");
+    return new ValidationException(prefix + processingMessage.getMessage());
+  }
+
   /**
    * Validate the indicated message.
    *
@@ -58,7 +102,7 @@ public List<String> validateMessage(String subFolder, String data) {
       if (report.isSuccess()) {
         return ImmutableList.of();
       }
-      throw Validator.fromProcessingReport(report);
+      throw fromProcessingReport(report);
     } catch (ValidationException e) {
       return e.getAllMessages();
     } catch (IOException | ProcessingException ex) {

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -35,8 +35,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.util.ISO8601DateFormat;
 import com.google.common.collect.ImmutableList;
-import com.google.daq.mqtt.util.ExceptionMap;
-import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
+import com.google.udmi.util.ExceptionMap.ExceptionCategory;
 import java.io.File;
 import java.util.Arrays;
 import java.util.Collection;

common/src/main/java/com/google/daq/mqtt/util/ValidationError.java → common/src/main/java/com/google/udmi/util/ValidationError.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 public class ValidationError extends RuntimeException {
 

common/src/main/java/com/google/daq/mqtt/util/ValidationException.java → common/src/main/java/com/google/udmi/util/ValidationException.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 import com.google.common.collect.ImmutableList;
 

common/src/main/java/com/google/daq/mqtt/util/ValidationWarning.java → common/src/main/java/com/google/udmi/util/ValidationWarning.java

@@ -1,4 +1,4 @@
-package com.google.daq.mqtt.util;
+package com.google.udmi.util;
 
 public class ValidationWarning extends ValidationError {
 

pubber/build.gradle

@@ -100,6 +100,7 @@ publishing {
 
 dependencies {
     implementation 'io.jsonwebtoken:jjwt:0.9.1'
+    implementation group: 'com.github.java-json-tools', name: 'json-schema-validator', version: '2.2.14'
     implementation 'javax.xml.bind:jaxb-api:2.3.1'
     implementation 'com.sun.xml.bind:jaxb-impl:3.0.1'
     implementation 'commons-io:commons-io:2.5'

udmis/build.gradle

@@ -84,6 +84,7 @@ repositories {
 dependencies {
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.4'
     implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.0'
+    implementation group: 'com.github.java-json-tools', name: 'json-schema-validator', version: '2.2.14'
     implementation 'commons-io:commons-io:2.5'
 
     implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: '0.9.10'

validator/src/main/java/com/google/bos/iot/core/proxy/IotCoreProxy.java

@@ -10,6 +10,7 @@
 import com.google.cloud.ServiceOptions;
 import com.google.daq.mqtt.util.PubSubClient;
 import com.google.daq.mqtt.util.PubSubPusher;
+import com.google.udmi.util.MessageValidator;
 import java.io.File;
 import java.util.Date;
 import java.util.HashMap;

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -7,9 +7,9 @@
 import static com.google.daq.mqtt.registrar.Registrar.METADATA_SCHEMA_JSON;
 import static com.google.daq.mqtt.util.ConfigManager.GENERATED_CONFIG_JSON;
 import static com.google.daq.mqtt.util.ConfigManager.configFrom;
-import static com.google.daq.mqtt.util.ContextWrapper.runInContext;
 import static com.google.udmi.util.Common.DEVICE_ID_ALLOWABLE;
 import static com.google.udmi.util.Common.POINT_NAME_ALLOWABLE;
+import static com.google.udmi.util.ContextWrapper.runInContext;
 import static com.google.udmi.util.GeneralUtils.CSV_JOINER;
 import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_STRICT;
 import static com.google.udmi.util.GeneralUtils.catchToNull;
@@ -44,19 +44,19 @@
 import com.google.daq.mqtt.util.CloudIotManager;
 import com.google.daq.mqtt.util.ConfigManager;
 import com.google.daq.mqtt.util.DeviceExceptionManager;
-import com.google.daq.mqtt.util.ErrorMap;
-import com.google.daq.mqtt.util.ErrorMap.ErrorMapException;
-import com.google.daq.mqtt.util.ExceptionMap;
-import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
-import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
-import com.google.daq.mqtt.util.ValidationError;
-import com.google.daq.mqtt.util.ValidationException;
-import com.google.daq.mqtt.util.ValidationWarning;
-import com.google.daq.mqtt.validator.Validator;
+import com.google.udmi.util.ErrorMap;
+import com.google.udmi.util.ErrorMap.ErrorMapException;
+import com.google.udmi.util.ExceptionMap;
+import com.google.udmi.util.ExceptionMap.ErrorTree;
+import com.google.udmi.util.ExceptionMap.ExceptionCategory;
 import com.google.udmi.util.JsonUtil;
 import com.google.udmi.util.MessageDowngrader;
+import com.google.udmi.util.MessageValidator;
 import com.google.udmi.util.SiteModel;
 import com.google.udmi.util.SiteModel.MetadataException;
+import com.google.udmi.util.ValidationError;
+import com.google.udmi.util.ValidationException;
+import com.google.udmi.util.ValidationWarning;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileWriter;
@@ -238,7 +238,7 @@ public static void parseMetadataValidateProcessingReport(ProcessingReport report
 
     for (ProcessingMessage msg : report) {
       if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) {
-        throw Validator.fromProcessingReport(report);
+        throw MessageValidator.fromProcessingReport(report);
       }
     }
   }

validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java

@@ -59,16 +59,16 @@
 import com.google.daq.mqtt.util.CloudDeviceSettings;
 import com.google.daq.mqtt.util.CloudIotManager;
 import com.google.daq.mqtt.util.DeviceGatewayBoundException;
-import com.google.daq.mqtt.util.ExceptionMap;
-import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
-import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
 import com.google.daq.mqtt.util.PubSubPusher;
-import com.google.daq.mqtt.util.ValidationError;
 import com.google.udmi.util.CommandLineOption;
 import com.google.udmi.util.CommandLineProcessor;
 import com.google.udmi.util.Common;
+import com.google.udmi.util.ExceptionMap;
+import com.google.udmi.util.ExceptionMap.ErrorTree;
+import com.google.udmi.util.ExceptionMap.ExceptionCategory;
 import com.google.udmi.util.JsonUtil;
 import com.google.udmi.util.SiteModel;
+import com.google.udmi.util.ValidationError;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;

validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java

@@ -17,7 +17,8 @@
 import static udmi.schema.IotAccess.IotProvider.PUBSUB;
 
 import com.google.common.collect.ImmutableList;
-import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory;
+import com.google.udmi.util.ExceptionMap;
+import com.google.udmi.util.ExceptionMap.ExceptionCategory;
 import com.google.udmi.util.GeneralUtils;
 import com.google.udmi.util.IotProvider;
 import com.google.udmi.util.MetadataMapKeys;

validator/src/main/java/com/google/daq/mqtt/util/ConfigManager.java

@@ -1,10 +1,10 @@
 package com.google.daq.mqtt.util;
 
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.daq.mqtt.util.ContextWrapper.getCurrentContext;
-import static com.google.daq.mqtt.util.ContextWrapper.runInContext;
-import static com.google.daq.mqtt.util.ContextWrapper.wrapExceptionWithContext;
 import static com.google.daq.mqtt.util.providers.FamilyProvider.NAMED_FAMILIES;
+import static com.google.udmi.util.ContextWrapper.getCurrentContext;
+import static com.google.udmi.util.ContextWrapper.runInContext;
+import static com.google.udmi.util.ContextWrapper.wrapExceptionWithContext;
 import static com.google.udmi.util.GeneralUtils.catchToNull;
 import static com.google.udmi.util.GeneralUtils.deepCopy;
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;

validator/src/main/java/com/google/daq/mqtt/util/FirestoreDataSink.java

@@ -5,8 +5,8 @@
 import com.google.cloud.firestore.Firestore;
 import com.google.cloud.firestore.FirestoreOptions;
 import com.google.common.base.Preconditions;
-import com.google.daq.mqtt.util.ExceptionMap.ErrorTree;
 import com.google.daq.mqtt.validator.ReportingDevice;
+import com.google.udmi.util.ExceptionMap.ErrorTree;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;

validator/src/main/java/com/google/daq/mqtt/validator/PointsetValidator.java

@@ -6,9 +6,9 @@
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.daq.mqtt.util.ValidationException;
 import com.google.daq.mqtt.validator.ReportingDevice.MetadataDiff;
 import com.google.udmi.util.JsonUtil;
+import com.google.udmi.util.ValidationException;
 import java.util.Date;
 import java.util.Map;
 import java.util.Set;

validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java

@@ -8,9 +8,9 @@
 import static java.util.Optional.ofNullable;
 import static org.junit.Assert.assertTrue;
 
-import com.google.daq.mqtt.util.ExceptionList;
-import com.google.daq.mqtt.util.ValidationException;
 import com.google.udmi.util.Common;
+import com.google.udmi.util.ExceptionList;
+import com.google.udmi.util.ValidationException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Date;

validator/src/main/java/com/google/daq/mqtt/validator/SchemaTester.java

@@ -1,6 +1,6 @@
 package com.google.daq.mqtt.validator;
 
-import com.google.daq.mqtt.util.ExceptionMap;
+import com.google.udmi.util.ExceptionMap;
 import java.util.Arrays;
 
 /**