Merge pull request #566 from Hurence/webanalyticBug

Webanalytic bug, simplification of engine code
Hurence · Apr 29, 2021 · f621f64 · f621f64
2 parents a53abbe + d58275d
commit f621f64
Show file tree

Hide file tree

Showing 322 changed files with 75,685 additions and 6,977 deletions.
diff --git a/logisland-assembly/README.md b/logisland-assembly/README.md
diff --git a/logisland-assembly/compare-assembly.sh b/logisland-assembly/compare-assembly.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+TMP="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+declare -r current_dir="$TMP"
+
+assembly1=$1
+assembly2=$2
+
+
+#
+# dif between content of directories
+#
+cd "${assembly1}"
+content_assembly1=$(find logisland-*-full-bin/ > "${current_dir}/assembly.txt")
+
+cd "${current_dir}"
+cd "${assembly2}"
+content_assembly2=$(find logisland-*-full-bin/ > "${current_dir}/assembly2.txt")
+
+cd "${current_dir}"
+echo "diff between files in assembly"
+diff assembly.txt assembly2.txt
+
+#
+# dif between individual jars
+#
+
+
diff --git a/logisland-assembly/pom.xml b/logisland-assembly/pom.xml
@@ -42,7 +42,7 @@
                     <attach>false</attach>
                     <appendAssemblyId>true</appendAssemblyId>
                 </configuration>
-                <version>3.1.1</version>
+                <version>3.3.0</version>
                 <executions>
                     <execution>
                         <id>make shared resource</id>

diff --git a/logisland-assembly/src/assembly/full-assembly.xml b/logisland-assembly/src/assembly/full-assembly.xml
@@ -77,6 +77,7 @@
                 <outputDirectory>lib/plugins</outputDirectory>
                 <includeDependencies>false</includeDependencies>
                 <unpack>false</unpack>
+                <attachmentClassifier>repackaged</attachmentClassifier>
             </binaries>
         </moduleSet>
     </moduleSets>

diff --git a/logisland-assembly/src/assembly/light-assembly.xml b/logisland-assembly/src/assembly/light-assembly.xml
@@ -41,6 +41,7 @@
                 <outputDirectory>lib/plugins</outputDirectory>
                 <includeDependencies>false</includeDependencies>
                 <unpack>false</unpack>
+                <attachmentClassifier>repackaged</attachmentClassifier>
             </binaries>
         </moduleSet>
     </moduleSets>

diff --git a/logisland-assembly/src/assembly/shared-dependencies.xml b/logisland-assembly/src/assembly/shared-dependencies.xml
@@ -55,7 +55,6 @@
                 <include>com.hurence.logisland:logisland-plugin-support</include>
                 <include>com.hurence.logisland:logisland-scripting-base</include>
                 <include>com.hurence.logisland:logisland-scripting-mvel</include>
-                <include>com.hurence.logisland:logisland-timeseries</include>
             </includes>
             <binaries>
                 <directoryMode>0770</directoryMode>

diff --git a/logisland-components/logisland-connectors/logisland-connector-opc/pom.xml b/logisland-components/logisland-connectors/logisland-connector-opc/pom.xml
@@ -51,6 +51,9 @@
                 <artifactId>spring-boot-maven-plugin</artifactId>
                 <executions>
                     <execution>
+                        <configuration>
+                            <classifier>repackaged</classifier>
+                        </configuration>
                         <phase>package</phase>
                     </execution>
                 </executions>

diff --git a/logisland-components/logisland-connectors/logisland-connector-spooldir/pom.xml b/logisland-components/logisland-connectors/logisland-connector-spooldir/pom.xml
@@ -107,6 +107,9 @@
                 <artifactId>spring-boot-maven-plugin</artifactId>
                 <executions>
                     <execution>
+                        <configuration>
+                            <classifier>repackaged</classifier>
+                        </configuration>
                         <phase>package</phase>
                     </execution>
                 </executions>

diff --git a/logisland-components/logisland-connectors/pom.xml b/logisland-components/logisland-connectors/pom.xml
@@ -35,10 +35,13 @@
                     <version>2.0.0.RELEASE</version>
                     <executions>
                         <execution>
+                            <configuration>
+                                <classifier>repackaged</classifier>
+                            </configuration>
+                            <phase>none</phase>
                             <goals>
                                 <goal>repackage</goal>
                             </goals>
-                            <phase>none</phase>
                         </execution>
                     </executions>
                     <configuration>

diff --git a/logisland-components/logisland-processors/logisland-processor-common/pom.xml b/logisland-components/logisland-processors/logisland-processor-common/pom.xml
@@ -143,20 +143,10 @@
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                    </execution>
-                </executions>
             </plugin>
             <plugin>
                 <groupId>com.hurence.logisland</groupId>
                 <artifactId>logisland-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                    </execution>
-                </executions>
             </plugin>
         </plugins>
     </build>

diff --git a/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/AddFields.java b/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/AddFields.java
@@ -117,7 +117,6 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String
         return new PropertyDescriptor.Builder()
                 .name(propertyDescriptorName)
                 .expressionLanguageSupported(true)
-                .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
                 .required(false)
                 .dynamic(true)
                 .build();

diff --git a/...-processor-common/src/main/java/com/hurence/logisland/processor/ComputeAggsProcessor.java b/...-processor-common/src/main/java/com/hurence/logisland/processor/ComputeAggsProcessor.java
@@ -41,7 +41,7 @@
 import com.hurence.logisland.record.FieldType;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 import org.apache.commons.lang3.StringUtils;
 
@@ -71,7 +71,7 @@ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
     }
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
         final Collection<ValidationResult> validations = new ArrayList<>();
         Set<Map.Entry<PropertyDescriptor, String>> props = context.getProperties().entrySet();
         if (props.isEmpty()) {
@@ -85,7 +85,7 @@ protected Collection<ValidationResult> customValidate(ValidationContext context)
         final Set<PropertyDescriptor> dynamicFieldPropertiesTmp = new HashSet<>();
         final Map<String, PropertyDescriptor> dynamicAggTypePropertiesTmp = new HashMap<>();
         final Map<String, PropertyDescriptor> dynamicTypeResultPropertiesTmp = new HashMap<>();
-        for (final Map.Entry<PropertyDescriptor, String> entry : props) {//TODO
+        for (final Map.Entry<PropertyDescriptor, String> entry : props) {
             if (!entry.getKey().isDynamic()) {
                 continue;
             }

diff --git a/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/DebugStream.java b/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/DebugStream.java
@@ -109,17 +109,10 @@ public Collection<Record> process(final ProcessContext context, final Collection
         getLogger().info("processing {} records", new Object[]{collection.size()});
 
         if (collection.size() != 0) {
-            collection.forEach(event -> {
-                //TODO seems tp not work when record contains other records !
-                ByteArrayOutputStream baos = new ByteArrayOutputStream();
-                try {
-                    serializer.serialize(baos, event);
-                    baos.close();
-                } catch (IOException e) {
-                    getLogger().error("error while trying to deserialize record !", e);
-                }
-
-                getLogger().info(new String(baos.toByteArray()));
+            //Do not use serialization ! It is pointless as at this point the object is already deserialized into a Record !
+            //Moreover trying to serialize the record may fail for a lot of reason (if record contains some objects without appropriate bean pattern.
+            collection.forEach(record -> {
+                getLogger().info(record.toString(1));
             });
         }
 

diff --git a/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/EncodeBase64.java b/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/EncodeBase64.java
@@ -66,21 +66,16 @@ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
     @Override
     public void init(ProcessContext context) throws InitializationException {
         super.init(context);
+        context.getPropertyValue(SOURCE_FIELDS).asStringOpt().ifPresent(s ->
+                sourceFieldNames.addAll(Arrays.asList((s.split(",")))));
+        context.getPropertyValue(DESTINATION_FIELDS).asStringOpt().ifPresent(s ->
+                destinationFieldNames.addAll(Arrays.asList((s.split(",")))));
+        if (sourceFieldNames.size() != destinationFieldNames.size()) {
+            throw new InitializationException(String.format("Processor properties %s and %s must contains the same number of elements. " +
+                            "Actual are: %d and %d", SOURCE_FIELDS.getName(), DESTINATION_FIELDS.getName(),
+                    sourceFieldNames.size(), destinationFieldNames.size()));
 
-        if(!isInitialized){
-            context.getPropertyValue(SOURCE_FIELDS).asStringOpt().ifPresent(s ->
-                    sourceFieldNames.addAll(Arrays.asList((s.split(",")))));
-            context.getPropertyValue(DESTINATION_FIELDS).asStringOpt().ifPresent(s ->
-                    destinationFieldNames.addAll(Arrays.asList((s.split(",")))));
-            if (sourceFieldNames.size() != destinationFieldNames.size()) {
-                throw new InitializationException(String.format("Processor properties %s and %s must contains the same number of elements. " +
-                                "Actual are: %d and %d", SOURCE_FIELDS.getName(), DESTINATION_FIELDS.getName(),
-                        sourceFieldNames.size(), destinationFieldNames.size()));
-
-            }
-            isInitialized = true;
         }
-
     }
 
     @Override

diff --git a/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/EncryptField.java b/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/EncryptField.java
@@ -26,7 +26,7 @@
 import com.hurence.logisland.record.FieldType;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 
 
@@ -245,15 +245,15 @@ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
     protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
         return new PropertyDescriptor.Builder()
                 .name(propertyDescriptorName)
-                .expressionLanguageSupported(false)  // TODO understand what expressionLanguage is !!!
+                .expressionLanguageSupported(false)
                 .required(false)
                 .dynamic(true)
                 .build();
     }
 
 
     @Override
-    protected Collection<ValidationResult> customValidate(final ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(final Configuration context) {
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
         try {
             String keyFile;

diff --git a/...land-processor-common/src/main/java/com/hurence/logisland/processor/EvaluateJsonPath.java b/...land-processor-common/src/main/java/com/hurence/logisland/processor/EvaluateJsonPath.java
@@ -21,7 +21,7 @@
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.FieldType;
 import com.hurence.logisland.record.Record;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.InvalidJsonException;
@@ -92,7 +92,7 @@ public class EvaluateJsonPath extends AbstractJsonPathProcessor {
     private transient final ConcurrentMap<String, JsonPath> cachedJsonPathMap = new ConcurrentHashMap<>();
 
     @Override
-    protected Collection<ValidationResult> customValidate(final ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(final Configuration context) {
         final List<ValidationResult> results = new ArrayList<>(super.customValidate(context));
 
         int jsonPathCount = 0;

diff --git a/...gisland-processor-common/src/main/java/com/hurence/logisland/processor/FilterRecords.java b/...gisland-processor-common/src/main/java/com/hurence/logisland/processor/FilterRecords.java
@@ -23,7 +23,7 @@
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -87,7 +87,7 @@ public class FilterRecords extends AbstractProcessor {
     private final Set<PropertyDescriptor> dynamicMethodProperties = new HashSet<>();
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
         if (context.getPropertyValue(FIELD_NAME).isSet() && !context.getPropertyValue(FIELD_VALUE).isSet()) {
             validationResults.add(
@@ -213,7 +213,7 @@ private void initDynamicProperties(ProcessContext context) {
     }
 
 
-    private boolean noDynamicProps(ValidationContext context) {
+    private boolean noDynamicProps(Configuration context) {
         for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
             if (!entry.getKey().isDynamic()) {
                 continue;

diff --git a/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/ModifyId.java b/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/ModifyId.java
@@ -23,7 +23,7 @@
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 import org.apache.commons.codec.binary.Hex;
 import org.slf4j.Logger;
@@ -138,7 +138,7 @@ public class ModifyId extends AbstractProcessor {
 
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
         if (context.getPropertyValue(STRATEGY).isSet()) {
             if (context.getPropertyValue(STRATEGY).getRawValue().equals(JAVA_FORMAT_STRING_WITH_FIELDS_STRATEGY.getValue())) {

diff --git a/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/RemoveFields.java b/...ogisland-processor-common/src/main/java/com/hurence/logisland/processor/RemoveFields.java
@@ -25,7 +25,7 @@
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -142,7 +142,7 @@ public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
     }
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
 
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
 

diff --git a/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/SendMail.java b/...rs/logisland-processor-common/src/main/java/com/hurence/logisland/processor/SendMail.java
@@ -23,7 +23,7 @@
 import com.hurence.logisland.record.Field;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.ValidationResult;
 
 import org.apache.commons.mail.EmailException;
@@ -628,7 +628,7 @@ private String getStringField(Record record, String fieldName)
     }
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
 
         logger.debug("customValidate");

diff --git a/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitRecord.java b/...logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitRecord.java
@@ -5,13 +5,9 @@
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.Field;
-import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.record.StandardRecord;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
-import com.hurence.logisland.validator.ValidationResult;
-
 
 
 import java.util.*;

diff --git a/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitText.java b/...s/logisland-processor-common/src/main/java/com/hurence/logisland/processor/SplitText.java
@@ -20,8 +20,8 @@
 import com.hurence.logisland.component.PropertyDescriptor;
 import com.hurence.logisland.record.*;
 import com.hurence.logisland.util.time.DateUtil;
+import com.hurence.logisland.validator.Configuration;
 import com.hurence.logisland.validator.StandardValidators;
-import com.hurence.logisland.validator.ValidationContext;
 import com.hurence.logisland.validator.ValidationResult;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -148,7 +148,7 @@ protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String
     }
 
     @Override
-    protected Collection<ValidationResult> customValidate(ValidationContext context) {
+    protected Collection<ValidationResult> customValidate(Configuration context) {
         final List<ValidationResult> validationResults = new ArrayList<>(super.customValidate(context));
 
         // key regex and fields must be set together

diff --git a/...ocessor-common/src/main/java/com/hurence/logisland/processor/datastore/EnrichRecords.java b/...ocessor-common/src/main/java/com/hurence/logisland/processor/datastore/EnrichRecords.java
@@ -20,10 +20,10 @@
 import com.hurence.logisland.processor.ProcessContext;
 import com.hurence.logisland.record.FieldDictionary;
 import com.hurence.logisland.record.Record;
-import com.hurence.logisland.service.datastore.InvalidMultiGetQueryRecordException;
-import com.hurence.logisland.service.datastore.MultiGetQueryRecord;
-import com.hurence.logisland.service.datastore.MultiGetQueryRecordBuilder;
-import com.hurence.logisland.service.datastore.MultiGetResponseRecord;
+import com.hurence.logisland.service.datastore.model.exception.InvalidMultiGetQueryRecordException;
+import com.hurence.logisland.service.datastore.model.MultiGetQueryRecord;
+import com.hurence.logisland.service.datastore.model.MultiGetQueryRecordBuilder;
+import com.hurence.logisland.service.datastore.model.MultiGetResponseRecord;
 import com.hurence.logisland.validator.StandardValidators;
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 import org.apache.commons.lang3.tuple.Triple;

diff --git a/...nd-processor-common/src/main/java/com/hurence/logisland/processor/datastore/MultiGet.java b/...nd-processor-common/src/main/java/com/hurence/logisland/processor/datastore/MultiGet.java
@@ -23,9 +23,9 @@
 import com.hurence.logisland.record.FieldType;
 import com.hurence.logisland.record.Record;
 import com.hurence.logisland.record.StandardRecord;
-import com.hurence.logisland.service.datastore.InvalidMultiGetQueryRecordException;
-import com.hurence.logisland.service.datastore.MultiGetQueryRecord;
-import com.hurence.logisland.service.datastore.MultiGetResponseRecord;
+import com.hurence.logisland.service.datastore.model.exception.InvalidMultiGetQueryRecordException;
+import com.hurence.logisland.service.datastore.model.MultiGetQueryRecord;
+import com.hurence.logisland.service.datastore.model.MultiGetResponseRecord;
 import com.hurence.logisland.validator.StandardValidators;
 
 import java.util.*;