Merge branch 'aws-merge-to-develop' of https://github.com/AbsaOSS/enc…

…eladus into aws-merge-to-develop

.gitignore

@@ -61,6 +61,7 @@ myTestCheckpoints
 **/main/resources/application.yml
 **/main/resources/spline.properties
 **/main/resources/kafka.properties
+**/menas/*/main/resources/*.properties
 **/enceladus_env.sh
 *.config
 *.keytab

dao/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

data-model/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

examples/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

menas/Dockerfile

@@ -15,12 +15,17 @@ LABEL \
     license="Apache License, version 2.0" \
     name="Menas"
 
-ADD src/main/resources/start_menas.sh start_menas.sh
+ARG WAR_FILE
+ARG PROPERTY_FILE
+
+ENV SPRING_CONFIG_NAME=${PROPERTY_FILE:-"application"}
+
+ADD ./src/main/resources/docker/start_menas.sh start_menas.sh
 RUN chmod +x start_menas.sh && \
     rm -rf webapps/*
 
-ADD ./target/*.war webapps/ROOT.war
-ADD ./src/main/resources/server.xml /tmp/server.xml
+ADD ./target/${WAR_FILE} webapps/ROOT.war
+ADD src/main/resources/docker/server.xml /tmp/server.xml
 
 EXPOSE 8080
 EXPOSE 8443

menas/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>
@@ -299,6 +299,19 @@
     </dependencies>
     <build>
         <plugins>
+            <plugin>
+                <groupId>com.spotify</groupId>
+                <artifactId>dockerfile-maven-plugin</artifactId>
+                <version>${dockerfile.maven.version}</version>
+                <configuration>
+                    <repository>${dockerfile.repository}</repository>
+                    <tag>${dockerfile.tag}</tag>
+                    <buildArgs>
+                        <WAR_FILE>${project.build.finalName}.war</WAR_FILE>
+                        <PROPERTY_FILE>${dockerfile.property.file}</PROPERTY_FILE>
+                    </buildArgs>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.scalastyle</groupId>
                 <artifactId>scalastyle-maven-plugin</artifactId>
@@ -365,6 +378,9 @@
                                 <copy file="${project.basedir}/src/main/resources/application.properties.template"
                                       tofile="${project.basedir}/src/main/resources/application.properties"
                                       overwrite="false"/>
+                                <copy file="${project.basedir}/src/main/resources/docker.properties.template"
+                                      tofile="${project.basedir}/src/main/resources/docker.properties"
+                                      overwrite="false"/>
                             </target>
                         </configuration>
                         <goals>

menas/src/main/resources/docker.properties.template

@@ -0,0 +1,62 @@
+#
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# A template property file for docker image. Most of the properties are expected to come from the environment variables.
+
+# Set the file size limit (default 1Mb). If you want to specify that files be
+# unlimited set the property to -1.
+spring.servlet.multipart.max-file-size=16MB
+
+# Set the timeout a single connection request can wait for a controller to return a value (in milliseconds)
+spring.mvc.async.request-timeout=60000
+
+# Set the total request size for a multipart/form-data (default 10Mb)
+spring.servlet.multipart.max-request-size=16MB
+
+# Number of seconds to cache the static resources for
+spring.resources.cache.cachecontrol.max-age=2678000
+spring.resources.chain.strategy.fixed.enabled=true
+spring.resources.chain.strategy.fixed.paths=/components/**,/css/**,/generic/**,/service/**
+[email protected]@
+
+# Disable second security filter chain passes for ASYNC requests
+spring.security.filter.dispatcher-types=REQUEST,ERROR
+
+# Timeout (in seconds) for MongoDB migration queries
+migrations.mongo.query.timeout.seconds=300
+
+menas.spark.master=local[1]
+
+#system-wide time zone
+timezone=UTC
+
+#---------- Actuator
+management.endpoints.web.base-path=/admin
+management.endpoints.jmx.exposure.exclude=*
+management.endpoints.web.exposure.include=health,threaddump,heapdump,loggers
+
+management.endpoints.enabled-by-default=false
+
+management.endpoint.health.enabled=true
+management.endpoint.health.show-details=always
+management.endpoint.threaddump.enabled=true
+management.endpoint.heapdump.enabled=true
+management.endpoint.loggers.enabled=true
+
+management.health.jms.enabled=false
+
+#---------- Monitoring
+# Limit on the number of documents to be fetch in a single mongodb query and shown in UI
+menas.monitoring.fetch.limit=500

menas/src/main/resources/docker/start_menas.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#
+# Copyright 2018 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [[ -n ${PRIVATE_KEY} && -n ${CERTIFICATE} && -n ${CA_CHAIN} ]]; then
+    echo "Certificate, chain and private key present, running secured version"
+    echo "${PRIVATE_KEY}" >> conf/private.pem
+    echo "${CERTIFICATE}" >> conf/certificate.pem
+    echo "${CA_CHAIN}" >> conf/cachain.pem
+    rm conf/server.xml
+    cp /tmp/server.xml conf/server.xml
+fi
+catalina.sh run

menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala

@@ -294,7 +294,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository
                                                   cr: C): RuleValidationsAndFields = {
     val withOutputValidated = validateOutputColumn(fields, cr.outputColumn)
     val validationInputFields = validateInputColumn(fields, cr.inputColumn)
-    withOutputValidated.update(validationInputFields)
+    validationInputFields.update(withOutputValidated)
   }
 
   def validateMappingTable(fields: Future[Set[String]],

migrations-cli/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

migrations/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

plugins-api/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

plugins-builtin/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

pom.xml

@@ -18,7 +18,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>za.co.absa.enceladus</groupId>
     <artifactId>parent</artifactId>
-    <version>2.18.0-SNAPSHOT</version>
+    <version>2.19.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>Enceladus</name>
@@ -134,6 +134,7 @@
         <maven.ant.plugin.version>1.8</maven.ant.plugin.version>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
+        <dockerfile.maven.version>1.4.13</dockerfile.maven.version>
         <maven.gpg.plugin.version>1.6</maven.gpg.plugin.version>
         <maven.jar.plugin.version>3.2.0</maven.jar.plugin.version>
         <maven.rat.plugin.version>0.12</maven.rat.plugin.version>
@@ -196,6 +197,10 @@
         <!--other properties-->
         <quiet.scalastyle.check>true</quiet.scalastyle.check>
         <scalastyle.configLocation>${project.basedir}/scalastyle-config.xml</scalastyle.configLocation>
+        <dockerfile.repository>docker.io/absaoss/menas</dockerfile.repository>
+        <dockerfile.tag>${project.version}</dockerfile.tag>
+        <!-- Name of property file that will be used by the Docker image. -->
+        <dockerfile.property.file>docker</dockerfile.property.file>
         <skip.integration.tests>true</skip.integration.tests>
         <!-- logging-->
         <log.level.base>WARN</log.level.base>

spark-jobs/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>

utils/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>za.co.absa.enceladus</groupId>
         <artifactId>parent</artifactId>
-        <version>2.18.0-SNAPSHOT</version>
+        <version>2.19.0-SNAPSHOT</version>
     </parent>
 
     <properties>