From 518c2a1d09cead3eacee42eaaf79991bc277e9a5 Mon Sep 17 00:00:00 2001 From: Trevor Date: Fri, 14 Feb 2025 06:27:26 -0800 Subject: [PATCH] Move MessageValidator to common (#1091) --- .../java/com/google/udmi/util/Common.java | 2 +- .../mqtt => udmi}/util/ContextWrapper.java | 2 +- .../{daq/mqtt => udmi}/util/ErrorMap.java | 2 +- .../mqtt => udmi}/util/ExceptionList.java | 2 +- .../{daq/mqtt => udmi}/util/ExceptionMap.java | 4 +- .../com/google/udmi/util/GeneralUtils.java | 1 - .../google/udmi/util}/MessageValidator.java | 52 +++++++++++++++-- .../java/com/google/udmi/util/SiteModel.java | 3 +- .../mqtt => udmi}/util/ValidationError.java | 2 +- .../util/ValidationException.java | 2 +- .../mqtt => udmi}/util/ValidationWarning.java | 2 +- pubber/build.gradle | 1 + udmis/build.gradle | 1 + .../bos/iot/core/proxy/IotCoreProxy.java | 1 + .../daq/mqtt/registrar/LocalDevice.java | 22 +++---- .../google/daq/mqtt/registrar/Registrar.java | 8 +-- .../google/daq/mqtt/util/CloudIotManager.java | 3 +- .../google/daq/mqtt/util/ConfigManager.java | 6 +- .../daq/mqtt/util/FirestoreDataSink.java | 2 +- .../daq/mqtt/validator/PointsetValidator.java | 2 +- .../daq/mqtt/validator/ReportingDevice.java | 4 +- .../daq/mqtt/validator/SchemaTester.java | 2 +- .../google/daq/mqtt/validator/Validator.java | 57 +++---------------- .../daq/mqtt/registrar/RegistrarTest.java | 2 +- 24 files changed, 94 insertions(+), 91 deletions(-) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ContextWrapper.java (97%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ErrorMap.java (95%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ExceptionList.java (89%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ExceptionMap.java (98%) rename {validator/src/main/java/com/google/bos/iot/core/proxy => common/src/main/java/com/google/udmi/util}/MessageValidator.java (60%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ValidationError.java (78%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ValidationException.java (97%) rename common/src/main/java/com/google/{daq/mqtt => udmi}/util/ValidationWarning.java (79%) diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java index b23da97150..c433c1f4e2 100644 --- a/common/src/main/java/com/google/udmi/util/Common.java +++ b/common/src/main/java/com/google/udmi/util/Common.java @@ -9,7 +9,7 @@ import com.google.common.base.Strings; import com.google.common.reflect.ClassPath; import com.google.common.reflect.ClassPath.ClassInfo; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import java.util.ArrayList; import java.util.List; import java.util.MissingFormatArgumentException; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ContextWrapper.java b/common/src/main/java/com/google/udmi/util/ContextWrapper.java similarity index 97% rename from common/src/main/java/com/google/daq/mqtt/util/ContextWrapper.java rename to common/src/main/java/com/google/udmi/util/ContextWrapper.java index 11b14527bd..67b092aad7 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ContextWrapper.java +++ b/common/src/main/java/com/google/udmi/util/ContextWrapper.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import java.util.List; import java.util.ArrayList; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java b/common/src/main/java/com/google/udmi/util/ErrorMap.java similarity index 95% rename from common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java rename to common/src/main/java/com/google/udmi/util/ErrorMap.java index cee835895a..52cbbd56fd 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ErrorMap.java +++ b/common/src/main/java/com/google/udmi/util/ErrorMap.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import java.util.HashMap; import java.util.Map; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java b/common/src/main/java/com/google/udmi/util/ExceptionList.java similarity index 89% rename from common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java rename to common/src/main/java/com/google/udmi/util/ExceptionList.java index c37fccdfa6..41e25e1240 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ExceptionList.java +++ b/common/src/main/java/com/google/udmi/util/ExceptionList.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import static java.lang.String.format; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java b/common/src/main/java/com/google/udmi/util/ExceptionMap.java similarity index 98% rename from common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java rename to common/src/main/java/com/google/udmi/util/ExceptionMap.java index 6ded34e087..f046cea46c 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ExceptionMap.java +++ b/common/src/main/java/com/google/udmi/util/ExceptionMap.java @@ -1,8 +1,8 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import static com.google.udmi.util.GeneralUtils.multiTrim; -import com.google.daq.mqtt.util.ErrorMap.ErrorMapException; +import com.google.udmi.util.ErrorMap.ErrorMapException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index 8ead465d55..e239c81550 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.common.hash.Hashing; -import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.ProperPrinter.OutputFormat; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java b/common/src/main/java/com/google/udmi/util/MessageValidator.java similarity index 60% rename from validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java rename to common/src/main/java/com/google/udmi/util/MessageValidator.java index 6c7125ca39..7fefb659c5 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/MessageValidator.java +++ b/common/src/main/java/com/google/udmi/util/MessageValidator.java @@ -1,17 +1,21 @@ -package com.google.bos.iot.core.proxy; +package com.google.udmi.util; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.fge.jsonschema.core.exceptions.ProcessingException; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; +import com.github.fge.jsonschema.core.report.LogLevel; +import com.github.fge.jsonschema.core.report.ProcessingMessage; import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import com.google.daq.mqtt.util.ValidationException; -import com.google.daq.mqtt.validator.Validator; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -20,6 +24,8 @@ import java.net.URL; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.StreamSupport; /** * Validation wrapper for processing individual messages. @@ -27,6 +33,11 @@ public class MessageValidator { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @SuppressWarnings("checkstyle:linelength") + private static final List IGNORE_LIST = ImmutableList.of( + "instance type \\(string\\) does not match any allowed primitive type \\(allowed: \\[.*\"number\"\\]\\)"); + private static final List IGNORE_PATTERNS = IGNORE_LIST.stream().map(Pattern::compile) + .toList(); private final Map schemaMap = Maps.newConcurrentMap(); private final File schemaRoot; @@ -44,6 +55,39 @@ public MessageValidator(String schemaRootPath) { } } + /** + * From an external processing report. + * + * @param report Report to convert + * @return Converted exception. + */ + public static ValidationException fromProcessingReport(ProcessingReport report) { + checkArgument(!report.isSuccess(), "Report must not be successful"); + ImmutableList causingExceptions = + StreamSupport.stream(report.spliterator(), false) + .filter(MessageValidator::errorOrWorse) + .filter(MessageValidator::notOnIgnoreList) + .map(MessageValidator::convertMessage).collect(toImmutableList()); + return causingExceptions.isEmpty() ? null : new ValidationException( + format("%d schema violations found", causingExceptions.size()), causingExceptions); + } + + private static boolean notOnIgnoreList(ProcessingMessage processingMessage) { + return IGNORE_PATTERNS.stream() + .noneMatch(p -> p.matcher(processingMessage.getMessage()).matches()); + } + + private static boolean errorOrWorse(ProcessingMessage processingMessage) { + return processingMessage.getLogLevel().compareTo(LogLevel.ERROR) >= 0; + } + + private static ValidationException convertMessage(ProcessingMessage processingMessage) { + String pointer = processingMessage.asJson().get("instance").get("pointer").asText(); + String prefix = + com.google.api.client.util.Strings.isNullOrEmpty(pointer) ? "" : (pointer + ": "); + return new ValidationException(prefix + processingMessage.getMessage()); + } + /** * Validate the indicated message. * @@ -58,7 +102,7 @@ public List validateMessage(String subFolder, String data) { if (report.isSuccess()) { return ImmutableList.of(); } - throw Validator.fromProcessingReport(report); + throw fromProcessingReport(report); } catch (ValidationException e) { return e.getAllMessages(); } catch (IOException | ProcessingException ex) { diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index b393c024d4..cea04f1ef9 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -35,8 +35,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; import com.google.common.collect.ImmutableList; -import com.google.daq.mqtt.util.ExceptionMap; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import java.io.File; import java.util.Arrays; import java.util.Collection; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ValidationError.java b/common/src/main/java/com/google/udmi/util/ValidationError.java similarity index 78% rename from common/src/main/java/com/google/daq/mqtt/util/ValidationError.java rename to common/src/main/java/com/google/udmi/util/ValidationError.java index 3d043c33bd..1de62294c6 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ValidationError.java +++ b/common/src/main/java/com/google/udmi/util/ValidationError.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; public class ValidationError extends RuntimeException { diff --git a/common/src/main/java/com/google/daq/mqtt/util/ValidationException.java b/common/src/main/java/com/google/udmi/util/ValidationException.java similarity index 97% rename from common/src/main/java/com/google/daq/mqtt/util/ValidationException.java rename to common/src/main/java/com/google/udmi/util/ValidationException.java index e300dd02b3..4da5bf7fbd 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ValidationException.java +++ b/common/src/main/java/com/google/udmi/util/ValidationException.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; import com.google.common.collect.ImmutableList; diff --git a/common/src/main/java/com/google/daq/mqtt/util/ValidationWarning.java b/common/src/main/java/com/google/udmi/util/ValidationWarning.java similarity index 79% rename from common/src/main/java/com/google/daq/mqtt/util/ValidationWarning.java rename to common/src/main/java/com/google/udmi/util/ValidationWarning.java index 4ee4bf1100..4121f6e663 100644 --- a/common/src/main/java/com/google/daq/mqtt/util/ValidationWarning.java +++ b/common/src/main/java/com/google/udmi/util/ValidationWarning.java @@ -1,4 +1,4 @@ -package com.google.daq.mqtt.util; +package com.google.udmi.util; public class ValidationWarning extends ValidationError { diff --git a/pubber/build.gradle b/pubber/build.gradle index ed91fd68c9..cca6207c37 100644 --- a/pubber/build.gradle +++ b/pubber/build.gradle @@ -100,6 +100,7 @@ publishing { dependencies { implementation 'io.jsonwebtoken:jjwt:0.9.1' + implementation group: 'com.github.java-json-tools', name: 'json-schema-validator', version: '2.2.14' implementation 'javax.xml.bind:jaxb-api:2.3.1' implementation 'com.sun.xml.bind:jaxb-impl:3.0.1' implementation 'commons-io:commons-io:2.5' diff --git a/udmis/build.gradle b/udmis/build.gradle index d11a1aaadd..711466cd15 100644 --- a/udmis/build.gradle +++ b/udmis/build.gradle @@ -84,6 +84,7 @@ repositories { dependencies { implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.4' implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.0' + implementation group: 'com.github.java-json-tools', name: 'json-schema-validator', version: '2.2.14' implementation 'commons-io:commons-io:2.5' implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: '0.9.10' diff --git a/validator/src/main/java/com/google/bos/iot/core/proxy/IotCoreProxy.java b/validator/src/main/java/com/google/bos/iot/core/proxy/IotCoreProxy.java index 198e37ced2..d5faf1a8b6 100644 --- a/validator/src/main/java/com/google/bos/iot/core/proxy/IotCoreProxy.java +++ b/validator/src/main/java/com/google/bos/iot/core/proxy/IotCoreProxy.java @@ -10,6 +10,7 @@ import com.google.cloud.ServiceOptions; import com.google.daq.mqtt.util.PubSubClient; import com.google.daq.mqtt.util.PubSubPusher; +import com.google.udmi.util.MessageValidator; import java.io.File; import java.util.Date; import java.util.HashMap; diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java index ac0f6bc7ea..8e97c7bafb 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java @@ -7,9 +7,9 @@ import static com.google.daq.mqtt.registrar.Registrar.METADATA_SCHEMA_JSON; import static com.google.daq.mqtt.util.ConfigManager.GENERATED_CONFIG_JSON; import static com.google.daq.mqtt.util.ConfigManager.configFrom; -import static com.google.daq.mqtt.util.ContextWrapper.runInContext; import static com.google.udmi.util.Common.DEVICE_ID_ALLOWABLE; import static com.google.udmi.util.Common.POINT_NAME_ALLOWABLE; +import static com.google.udmi.util.ContextWrapper.runInContext; import static com.google.udmi.util.GeneralUtils.CSV_JOINER; import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_STRICT; import static com.google.udmi.util.GeneralUtils.catchToNull; @@ -44,19 +44,19 @@ import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.ConfigManager; import com.google.daq.mqtt.util.DeviceExceptionManager; -import com.google.daq.mqtt.util.ErrorMap; -import com.google.daq.mqtt.util.ErrorMap.ErrorMapException; -import com.google.daq.mqtt.util.ExceptionMap; -import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; -import com.google.daq.mqtt.util.ValidationError; -import com.google.daq.mqtt.util.ValidationException; -import com.google.daq.mqtt.util.ValidationWarning; -import com.google.daq.mqtt.validator.Validator; +import com.google.udmi.util.ErrorMap; +import com.google.udmi.util.ErrorMap.ErrorMapException; +import com.google.udmi.util.ExceptionMap; +import com.google.udmi.util.ExceptionMap.ErrorTree; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.MessageDowngrader; +import com.google.udmi.util.MessageValidator; import com.google.udmi.util.SiteModel; import com.google.udmi.util.SiteModel.MetadataException; +import com.google.udmi.util.ValidationError; +import com.google.udmi.util.ValidationException; +import com.google.udmi.util.ValidationWarning; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; @@ -238,7 +238,7 @@ public static void parseMetadataValidateProcessingReport(ProcessingReport report for (ProcessingMessage msg : report) { if (msg.getLogLevel().compareTo(LogLevel.ERROR) >= 0) { - throw Validator.fromProcessingReport(report); + throw MessageValidator.fromProcessingReport(report); } } } diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 5682f4e99c..217381d63e 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -59,16 +59,16 @@ import com.google.daq.mqtt.util.CloudDeviceSettings; import com.google.daq.mqtt.util.CloudIotManager; import com.google.daq.mqtt.util.DeviceGatewayBoundException; -import com.google.daq.mqtt.util.ExceptionMap; -import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; import com.google.daq.mqtt.util.PubSubPusher; -import com.google.daq.mqtt.util.ValidationError; import com.google.udmi.util.CommandLineOption; import com.google.udmi.util.CommandLineProcessor; import com.google.udmi.util.Common; +import com.google.udmi.util.ExceptionMap; +import com.google.udmi.util.ExceptionMap.ErrorTree; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.SiteModel; +import com.google.udmi.util.ValidationError; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; diff --git a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java index 292fd91475..8ce5817ada 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java @@ -17,7 +17,8 @@ import static udmi.schema.IotAccess.IotProvider.PUBSUB; import com.google.common.collect.ImmutableList; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; +import com.google.udmi.util.ExceptionMap; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.IotProvider; import com.google.udmi.util.MetadataMapKeys; diff --git a/validator/src/main/java/com/google/daq/mqtt/util/ConfigManager.java b/validator/src/main/java/com/google/daq/mqtt/util/ConfigManager.java index a670ddb989..730504df88 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/ConfigManager.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/ConfigManager.java @@ -1,10 +1,10 @@ package com.google.daq.mqtt.util; import static com.google.common.base.Preconditions.checkState; -import static com.google.daq.mqtt.util.ContextWrapper.getCurrentContext; -import static com.google.daq.mqtt.util.ContextWrapper.runInContext; -import static com.google.daq.mqtt.util.ContextWrapper.wrapExceptionWithContext; import static com.google.daq.mqtt.util.providers.FamilyProvider.NAMED_FAMILIES; +import static com.google.udmi.util.ContextWrapper.getCurrentContext; +import static com.google.udmi.util.ContextWrapper.runInContext; +import static com.google.udmi.util.ContextWrapper.wrapExceptionWithContext; import static com.google.udmi.util.GeneralUtils.catchToNull; import static com.google.udmi.util.GeneralUtils.deepCopy; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; diff --git a/validator/src/main/java/com/google/daq/mqtt/util/FirestoreDataSink.java b/validator/src/main/java/com/google/daq/mqtt/util/FirestoreDataSink.java index 8af6afe4cc..6fe193c414 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/FirestoreDataSink.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/FirestoreDataSink.java @@ -5,8 +5,8 @@ import com.google.cloud.firestore.Firestore; import com.google.cloud.firestore.FirestoreOptions; import com.google.common.base.Preconditions; -import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; import com.google.daq.mqtt.validator.ReportingDevice; +import com.google.udmi.util.ExceptionMap.ErrorTree; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/PointsetValidator.java b/validator/src/main/java/com/google/daq/mqtt/validator/PointsetValidator.java index 2f797ebdce..61703a413f 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/PointsetValidator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/PointsetValidator.java @@ -6,9 +6,9 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.daq.mqtt.util.ValidationException; import com.google.daq.mqtt.validator.ReportingDevice.MetadataDiff; import com.google.udmi.util.JsonUtil; +import com.google.udmi.util.ValidationException; import java.util.Date; import java.util.Map; import java.util.Set; diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java b/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java index 230e61fa50..476d7fd8b5 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/ReportingDevice.java @@ -8,9 +8,9 @@ import static java.util.Optional.ofNullable; import static org.junit.Assert.assertTrue; -import com.google.daq.mqtt.util.ExceptionList; -import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.Common; +import com.google.udmi.util.ExceptionList; +import com.google.udmi.util.ValidationException; import java.time.Instant; import java.util.ArrayList; import java.util.Date; diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/SchemaTester.java b/validator/src/main/java/com/google/daq/mqtt/validator/SchemaTester.java index 3ada106540..ea1b6bc3f8 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/SchemaTester.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/SchemaTester.java @@ -1,6 +1,6 @@ package com.google.daq.mqtt.validator; -import com.google.daq.mqtt.util.ExceptionMap; +import com.google.udmi.util.ExceptionMap; import java.util.Arrays; /** diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 04098e189c..07cdbe1604 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -2,7 +2,6 @@ import static com.google.api.client.util.Preconditions.checkState; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.daq.mqtt.registrar.Registrar.BASE_DIR; import static com.google.daq.mqtt.sequencer.SequenceBase.EMPTY_MESSAGE; import static com.google.daq.mqtt.util.ConfigUtil.UDMI_ROOT; @@ -45,8 +44,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.github.fge.jsonschema.core.load.configuration.LoadingConfiguration; import com.github.fge.jsonschema.core.load.download.URIDownloader; -import com.github.fge.jsonschema.core.report.LogLevel; -import com.github.fge.jsonschema.core.report.ProcessingMessage; import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.github.fge.jsonschema.main.JsonSchemaFactory; @@ -59,22 +56,22 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.daq.mqtt.util.CloudIotManager; -import com.google.daq.mqtt.util.ErrorMap; -import com.google.daq.mqtt.util.ErrorMap.ErrorMapException; -import com.google.daq.mqtt.util.ExceptionMap; -import com.google.daq.mqtt.util.ExceptionMap.ErrorTree; import com.google.daq.mqtt.util.FileDataSink; import com.google.daq.mqtt.util.ImpulseRunningAverage; import com.google.daq.mqtt.util.MessagePublisher; import com.google.daq.mqtt.util.MessagePublisher.QuerySpeed; import com.google.daq.mqtt.util.PubSubClient; -import com.google.daq.mqtt.util.ValidationException; import com.google.udmi.util.CommandLineOption; import com.google.udmi.util.CommandLineProcessor; import com.google.udmi.util.Common; +import com.google.udmi.util.ErrorMap; +import com.google.udmi.util.ErrorMap.ErrorMapException; +import com.google.udmi.util.ExceptionMap; +import com.google.udmi.util.ExceptionMap.ErrorTree; import com.google.udmi.util.GeneralUtils; import com.google.udmi.util.JsonUtil; import com.google.udmi.util.MessageUpgrader; +import com.google.udmi.util.MessageValidator; import com.google.udmi.util.SiteModel; import java.io.File; import java.io.FilenameFilter; @@ -106,9 +103,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.apache.commons.io.FileUtils; import org.slf4j.impl.SimpleLogger; import udmi.schema.Category; @@ -159,12 +154,6 @@ public class Validator { SubType.EVENTS, SubType.STATE); - @SuppressWarnings("checkstyle:linelength") - private static final List IGNORE_LIST = ImmutableList.of( - "instance type \\(string\\) does not match any allowed primitive type \\(allowed: \\[.*\"number\"\\]\\)"); - private static final List IGNORE_PATTERNS = IGNORE_LIST.stream().map(Pattern::compile) - .toList(); - private static final long DEFAULT_INTERVAL_SEC = 60; private static final long REPORTS_PER_SEC = 10; private static final String VALIDATION_SITE_REPORT_DEVICE_ID = null; @@ -300,39 +289,6 @@ private static void sanitizeMessageException(Map message) { } } - /** - * From an external processing report. - * - * @param report Report to convert - * @return Converted exception. - */ - public static ValidationException fromProcessingReport(ProcessingReport report) { - checkArgument(!report.isSuccess(), "Report must not be successful"); - ImmutableList causingExceptions = - StreamSupport.stream(report.spliterator(), false) - .filter(Validator::errorOrWorse) - .filter(Validator::notOnIgnoreList) - .map(Validator::convertMessage).collect(toImmutableList()); - return causingExceptions.isEmpty() ? null : new ValidationException( - format("%d schema violations found", causingExceptions.size()), causingExceptions); - } - - private static boolean notOnIgnoreList(ProcessingMessage processingMessage) { - return IGNORE_PATTERNS.stream() - .noneMatch(p -> p.matcher(processingMessage.getMessage()).matches()); - } - - private static boolean errorOrWorse(ProcessingMessage processingMessage) { - return processingMessage.getLogLevel().compareTo(LogLevel.ERROR) >= 0; - } - - private static ValidationException convertMessage(ProcessingMessage processingMessage) { - String pointer = processingMessage.asJson().get("instance").get("pointer").asText(); - String prefix = - com.google.api.client.util.Strings.isNullOrEmpty(pointer) ? "" : (pointer + ": "); - return new ValidationException(prefix + processingMessage.getMessage()); - } - private List parseArgs(List argList) { if (!argList.isEmpty() && !argList.get(0).startsWith("-")) { processProfile(new File(argList.remove(0))); @@ -686,7 +642,8 @@ private void validateMessage(Object msgObject, Map attributes) { private void validateMessage(JsonSchema schema, Object message) throws Exception { ProcessingReport report = schema.validate(OBJECT_MAPPER.valueToTree(message), true); - ifTrueThen(!report.isSuccess(), () -> ifNotNullThrow(fromProcessingReport(report))); + ifTrueThen(!report.isSuccess(), + () -> ifNotNullThrow(MessageValidator.fromProcessingReport(report))); } private Instant getInstant(Object msgObject, Map attributes) { diff --git a/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java index 9e575e80c4..50bfb4bd89 100644 --- a/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ b/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -23,10 +23,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import com.google.daq.mqtt.util.ExceptionMap.ExceptionCategory; import com.google.daq.mqtt.util.IotMockProvider; import com.google.daq.mqtt.util.IotMockProvider.ActionType; import com.google.daq.mqtt.util.IotMockProvider.MockAction; +import com.google.udmi.util.ExceptionMap.ExceptionCategory; import java.util.ArrayList; import java.util.HashSet; import java.util.List;