From 53cde7d29e0e8bd18a8e20f2ff6965b1447dda01 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 18 Jul 2022 09:57:10 -0700 Subject: [PATCH 1/5] Initial conversion --- validator/bin/validate | 19 +++- .../google/daq/mqtt/validator/Validator.java | 89 +++++++++++-------- 2 files changed, 69 insertions(+), 39 deletions(-) diff --git a/validator/bin/validate b/validator/bin/validate index 794dc9ea9b..a293bb87b1 100755 --- a/validator/bin/validate +++ b/validator/bin/validate @@ -15,6 +15,11 @@ shift 5 ROOT=$(dirname $0)/../.. cd $ROOT +if [[ "$schema" != schema ]]; then + echo Currently only default schema supported. + false +fi + jarfile=validator/build/libs/validator-1.0-SNAPSHOT-all.jar if [ ! -f $jarfile ]; then @@ -30,10 +35,20 @@ rm -rf $sitepath/out/ echo Executing validator $schema $target... -echo java -jar $jarfile $project $schema $target $subscription $sitepath +if [[ $target == reflect ]]; then + srcargs="-r" +elif [[ $target == pubsub ]]; then + srcargs="-t $subscription" +else + echo Unsupported target $target + false +fi + +args="-p $project -s $sitepath $srcargs" +echo java -jar $jarfile $args error=0 -java -jar $jarfile $project $schema $target $subscription $sitepath || error=$? +java -jar $jarfile $args || error=$? echo Validation complete, exit $error exit $error diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 19d48bc22d..e3c157f0f4 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -40,11 +40,13 @@ import java.io.PrintStream; import java.net.URI; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.MissingFormatArgumentException; import java.util.Objects; import java.util.Set; import java.util.TreeMap; @@ -105,12 +107,12 @@ public class Validator { EVENT_POINTSET, PointsetEvent.class, STATE_POINTSET, PointsetState.class ); - private final String projectId; private final Map expectedDevices = new TreeMap<>(); private final Set extraDevices = new TreeSet<>(); private final Set processedDevices = new TreeSet<>(); private final Set base64Devices = new TreeSet<>(); private final Set ignoredRegistries = new HashSet(); + private String projectId; private File outBaseDir; private File metadataReportFile; private DataSink dataSink; @@ -123,46 +125,15 @@ public class Validator { private MessagePublisher client; private Map schemaMap; - /** - * Create validator for the given project id. - * - * @param projectId Target cloud project id - */ - public Validator(String projectId) { - this.projectId = projectId; - } - /** * Let's go. * * @param args Arguments for program execution */ public static void main(String[] args) { - if (args.length != 5) { - throw new IllegalArgumentException("Args: [project] [schema] [target] [instance] [site]"); - } try { - Validator validator = new Validator(args[0]); - validator.setSchemaSpec(args[1]); - String targetSpec = args[2]; - String instName = args[3]; - String siteDir = args[4]; - validator.setSiteDir(siteDir); - switch (targetSpec) { - case PUBSUB_MARKER: - validator.initializeCloudIoT(); - validator.initializeFirestoreDataSink(); - validator.validatePubSub(instName); - break; - case FILES_MARKER: - validator.validateFilesOutput(instName); - break; - case REFLECT_MARKER: - validator.validateReflector(instName); - break; - default: - throw new RuntimeException("Unknown target spec " + targetSpec); - } + List arrayList = Arrays.stream(args).collect(Collectors.toList()); + Validator validator = new Validator(arrayList); validator.messageLoop(); } catch (ExceptionMap processingException) { System.exit(2); @@ -174,6 +145,50 @@ public static void main(String[] args) { System.exit(0); } + /** + * Create validator with the given args. + * + * @param argList Argument list + */ + public Validator(List argList) { + setSchemaSpec("schema"); + while (argList.size() > 0) { + String option = removeNextArg(argList); + try { + switch (option) { + case "-p": + projectId = removeNextArg(argList); + break; + case "-s": + setSiteDir(removeNextArg(argList)); + break; + case "-t": + initializeCloudIoT(); + initializeFirestoreDataSink(); + validatePubSub(removeNextArg(argList)); + break; + case "-f": + validateFilesOutput(removeNextArg(argList)); + break; + case "-r": + validateReflector(); + break; + default: + throw new RuntimeException("Unknown cmdline option " + option); + } + } catch (MissingFormatArgumentException e) { + throw new RuntimeException("For command line option " + option, e); + } + } + } + + private String removeNextArg(List argList) { + if (argList.isEmpty()) { + throw new MissingFormatArgumentException("Missing argument"); + } + return argList.remove(0); + } + /** * Set the site directory to use for this validation run. * @@ -292,8 +307,8 @@ private void validatePubSub(String instName) { } } - private void validateReflector(String instName) { - deviceId = NO_SITE.equals(instName) ? null : instName; + private void validateReflector() { + deviceId = null; String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); System.err.println("Loading reflector key file from " + keyFile); client = new IotCoreClient(projectId, cloudIotConfig, keyFile); @@ -301,7 +316,7 @@ private void validateReflector(String instName) { private void messageLoop() { if (client == null) { - return; + throw new RuntimeException("No message loop client defined"); } System.err.println( "Entering message loop on " + client.getSubscriptionId() + " with device " + deviceId); From ab7ea9b16b305595c868889a3e715a82315969de Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 18 Jul 2022 10:02:45 -0700 Subject: [PATCH 2/5] Cleanup --- .../google/daq/mqtt/validator/Validator.java | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index e3c157f0f4..07202a342b 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -40,7 +40,6 @@ import java.io.PrintStream; import java.net.URI; import java.net.URL; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashSet; @@ -78,9 +77,6 @@ public class Validator { private static final String JSON_SUFFIX = ".json"; private static final String SCHEMA_VALIDATION_FORMAT = "Validating %d schemas"; private static final String TARGET_VALIDATION_FORMAT = "Validating %d files against %s"; - private static final String PUBSUB_MARKER = "pubsub"; - private static final String FILES_MARKER = "files"; - private static final String REFLECT_MARKER = "reflect"; private static final String DEVICE_FILE_FORMAT = "devices/%s"; private static final String ATTRIBUTE_FILE_FORMAT = "%s.attr"; private static final String MESSAGE_FILE_FORMAT = "%s.json"; @@ -125,26 +121,6 @@ public class Validator { private MessagePublisher client; private Map schemaMap; - /** - * Let's go. - * - * @param args Arguments for program execution - */ - public static void main(String[] args) { - try { - List arrayList = Arrays.stream(args).collect(Collectors.toList()); - Validator validator = new Validator(arrayList); - validator.messageLoop(); - } catch (ExceptionMap processingException) { - System.exit(2); - } catch (Exception e) { - e.printStackTrace(); - System.err.flush(); - System.exit(-1); - } - System.exit(0); - } - /** * Create validator with the given args. * @@ -182,6 +158,26 @@ public Validator(List argList) { } } + /** + * Let's go. + * + * @param args Arguments for program execution + */ + public static void main(String[] args) { + try { + List arrayList = Arrays.stream(args).collect(Collectors.toList()); + Validator validator = new Validator(arrayList); + validator.messageLoop(); + } catch (ExceptionMap processingException) { + System.exit(2); + } catch (Exception e) { + e.printStackTrace(); + System.err.flush(); + System.exit(-1); + } + System.exit(0); + } + private String removeNextArg(List argList) { if (argList.isEmpty()) { throw new MissingFormatArgumentException("Missing argument"); @@ -435,7 +431,6 @@ private boolean validateUpdate( updated = true; } - try { validateMessage(schemaMap.get(ENVELOPE_SCHEMA_ID), attributes); } catch (Exception e) { From 244d9c72cec72bf040a67c00c1d4e533d6ebff61 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 18 Jul 2022 10:39:14 -0700 Subject: [PATCH 3/5] Fix file validation --- bin/test_schema | 2 +- .../java/com/google/daq/mqtt/validator/Validator.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/test_schema b/bin/test_schema index f5c1661a39..bdc5f6352c 100755 --- a/bin/test_schema +++ b/bin/test_schema @@ -89,7 +89,7 @@ for subset in $subsets; do reltest=$reldir:$(realpath --relative-to $testdir $testpath) fi ( - (cd $schemadir; java -jar -Dnashorn.args=--no-deprecation-warning $jarfile -- $schemaname files $reltest --) || true + (cd $schemadir; java -jar -Dnashorn.args=--no-deprecation-warning $jarfile -a $schemaname -f $reltest) || true if [ $force == y ]; then diff $output $expected || echo Updating $expected && cp $output $expected else diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index 07202a342b..d7d5ac8359 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -127,7 +127,6 @@ public class Validator { * @param argList Argument list */ public Validator(List argList) { - setSchemaSpec("schema"); while (argList.size() > 0) { String option = removeNextArg(argList); try { @@ -138,6 +137,9 @@ public Validator(List argList) { case "-s": setSiteDir(removeNextArg(argList)); break; + case "-a": + setSchemaSpec(removeNextArg(argList)); + break; case "-t": initializeCloudIoT(); initializeFirestoreDataSink(); @@ -156,6 +158,9 @@ public Validator(List argList) { throw new RuntimeException("For command line option " + option, e); } } + if (schemaMap == null) { + setSchemaSpec("schema"); + } } /** @@ -312,7 +317,7 @@ private void validateReflector() { private void messageLoop() { if (client == null) { - throw new RuntimeException("No message loop client defined"); + return; } System.err.println( "Entering message loop on " + client.getSubscriptionId() + " with device " + deviceId); From 8f429050c40b461aa7252ace80ffc382a3b04a0a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 18 Jul 2022 10:56:50 -0700 Subject: [PATCH 4/5] Fix unit tests --- .../com/google/daq/mqtt/validator/Validator.java | 7 ++++--- .../google/daq/mqtt/validator/ValidatorTest.java | 13 +++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index d7d5ac8359..ea6af90b62 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -40,6 +40,7 @@ import java.io.PrintStream; import java.net.URI; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashSet; @@ -126,7 +127,8 @@ public class Validator { * * @param argList Argument list */ - public Validator(List argList) { + public Validator(List immutableList) { + List argList = new ArrayList<>(immutableList); while (argList.size() > 0) { String option = removeNextArg(argList); try { @@ -170,8 +172,7 @@ public Validator(List argList) { */ public static void main(String[] args) { try { - List arrayList = Arrays.stream(args).collect(Collectors.toList()); - Validator validator = new Validator(arrayList); + Validator validator = new Validator(Arrays.asList(args)); validator.messageLoop(); } catch (ExceptionMap processingException) { System.exit(2); diff --git a/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java b/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java index 9e8c69ffd6..af98d927bc 100644 --- a/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java +++ b/validator/src/test/java/com/google/daq/mqtt/validator/ValidatorTest.java @@ -7,11 +7,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.util.ISO8601DateFormat; +import com.google.common.collect.ImmutableList; import com.google.daq.mqtt.validator.Validator.MessageBundle; import com.google.daq.mqtt.validator.Validator.MetadataReport; import java.io.File; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -45,12 +47,11 @@ public class ValidatorTest { private static final File REPORT_FILE = new File(SITE_DIR + "/out/validation_report.json"); private static final String DEVICE_NUM_ID = "97216312321"; private static final String REGISTRY_ID = "ZZ-TRI-FECTA"; - private final Validator validator = new Validator(PROJECT_ID); - - { - validator.setSchemaSpec(SCHEMA_SPEC); - validator.setSiteDir(SITE_DIR); - } + private static final List testArgs = ImmutableList.of( + "-p", PROJECT_ID, + "-a", SCHEMA_SPEC, + "-s", SITE_DIR); + private final Validator validator = new Validator(testArgs); @Test public void emptySystemBlock() { From 2605d7c0d7c2a9b8308ccef2685959af5ff6cd7e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 18 Jul 2022 11:23:17 -0700 Subject: [PATCH 5/5] Restore device query init capability --- .../google/daq/mqtt/validator/Validator.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java index ea6af90b62..f500b0cd34 100644 --- a/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java +++ b/validator/src/main/java/com/google/daq/mqtt/validator/Validator.java @@ -118,7 +118,7 @@ public class Validator { private CloudIotConfig cloudIotConfig; private CloudIotManager cloudIotManager; private String siteDir; - private String deviceId; + private List deviceIds; private MessagePublisher client; private Map schemaMap; @@ -127,8 +127,16 @@ public class Validator { * * @param argList Argument list */ - public Validator(List immutableList) { - List argList = new ArrayList<>(immutableList); + public Validator(List argList) { + List listCopy = new ArrayList<>(argList); + parseArgs(listCopy); + if (schemaMap == null) { + setSchemaSpec("schema"); + } + deviceIds = listCopy; + } + + private void parseArgs(List argList) { while (argList.size() > 0) { String option = removeNextArg(argList); try { @@ -153,6 +161,9 @@ public Validator(List immutableList) { case "-r": validateReflector(); break; + case "--": + // All remaining arguments remain in the return list. + return; default: throw new RuntimeException("Unknown cmdline option " + option); } @@ -160,9 +171,6 @@ public Validator(List immutableList) { throw new RuntimeException("For command line option " + option, e); } } - if (schemaMap == null) { - setSchemaSpec("schema"); - } } /** @@ -310,7 +318,6 @@ private void validatePubSub(String instName) { } private void validateReflector() { - deviceId = null; String keyFile = new File(siteDir, GCP_REFLECT_KEY_PKCS8).getAbsolutePath(); System.err.println("Loading reflector key file from " + keyFile); client = new IotCoreClient(projectId, cloudIotConfig, keyFile); @@ -320,16 +327,11 @@ private void messageLoop() { if (client == null) { return; } - System.err.println( - "Entering message loop on " + client.getSubscriptionId() + " with device " + deviceId); + sendInitializationQuery(); + System.err.println("Entering message loop on " + client.getSubscriptionId()); BiConsumer, Map> validator = messageValidator(); - boolean initialized = false; while (client.isActive()) { try { - if (!initialized) { - initialized = true; - sendInitializationQuery(); - } client.processMessage(validator); } catch (Exception e) { e.printStackTrace(); @@ -339,7 +341,7 @@ private void messageLoop() { } private void sendInitializationQuery() { - if (deviceId != null) { + for (String deviceId : deviceIds) { System.err.println("Sending initialization query messages for device " + deviceId); client.publish(deviceId, STATE_QUERY_TOPIC, EMPTY_MESSAGE); }