From 4d7b0769221cad5fc4f2f0967364ee642bdc5e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Zejnilovi=C4=87?= Date: Wed, 20 Jan 2021 13:10:01 +0200 Subject: [PATCH 1/6] #1638 Fix validation of conformance rules on import entity --- .../za/co/absa/enceladus/menas/services/DatasetService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala index 9c56bd0d4..6af2d87e6 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala @@ -208,7 +208,7 @@ class DatasetService @Autowired() (datasetMongoRepository: DatasetMongoRepositor cr: C): RuleValidationsAndFields = { val withOutputValidated = validateOutputColumn(fields, cr.outputColumn) val validationInputFields = validateInputColumn(fields, cr.inputColumn) - withOutputValidated.update(validationInputFields) + validationInputFields.update(withOutputValidated) } def validateMappingTable(fields: Future[Set[String]], From f200c41e244ccf9497012fe681392c000a18eb5e Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Thu, 21 Jan 2021 15:18:48 +0100 Subject: [PATCH 2/6] v2.17.1 --- dao/pom.xml | 2 +- data-model/pom.xml | 2 +- examples/pom.xml | 2 +- menas/pom.xml | 2 +- migrations-cli/pom.xml | 2 +- migrations/pom.xml | 2 +- plugins-api/pom.xml | 2 +- plugins-builtin/pom.xml | 2 +- pom.xml | 2 +- spark-jobs/pom.xml | 2 +- utils/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dao/pom.xml b/dao/pom.xml index 1766367fb..fbb6ebeed 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/data-model/pom.xml b/data-model/pom.xml index c48105841..aeac68b84 100644 --- a/data-model/pom.xml +++ b/data-model/pom.xml @@ -24,7 +24,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/examples/pom.xml b/examples/pom.xml index 722f0d32a..4a5cd109a 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/menas/pom.xml b/menas/pom.xml index d8ba1f54d..5263762f1 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/migrations-cli/pom.xml b/migrations-cli/pom.xml index 732f4541a..47c6d207c 100644 --- a/migrations-cli/pom.xml +++ b/migrations-cli/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/migrations/pom.xml b/migrations/pom.xml index f9c8db362..fbec1942b 100644 --- a/migrations/pom.xml +++ b/migrations/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/plugins-api/pom.xml b/plugins-api/pom.xml index 61a97a675..6e19b49bf 100644 --- a/plugins-api/pom.xml +++ b/plugins-api/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/plugins-builtin/pom.xml b/plugins-builtin/pom.xml index c81dddf1e..5f3f76f9d 100644 --- a/plugins-builtin/pom.xml +++ b/plugins-builtin/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/pom.xml b/pom.xml index bf6185a4a..a2cf7a26a 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 za.co.absa.enceladus parent - 2.17.0 + 2.17.1 pom Enceladus diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 5eba4726e..66dcf29f6 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -20,7 +20,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 diff --git a/utils/pom.xml b/utils/pom.xml index d241e5649..d9944a50d 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.17.0 + 2.17.1 From 3050cbc1429b2ca283a78fb5395aafb385c4482e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Zejnilovi=C4=87?= Date: Mon, 25 Jan 2021 14:21:56 +0100 Subject: [PATCH 3/6] 1434 Add new way of serving properties to Docker --- .gitignore | 1 + menas/Dockerfile | 11 +++- menas/pom.xml | 16 +++++ .../main/resources/docker.properties.template | 62 +++++++++++++++++++ .../resources/{ => docker}/envoy/Dockerfile | 0 .../resources/{ => docker}/envoy/envoy.yaml | 0 .../{ => docker}/envoy/start_envoy.sh | 0 .../main/resources/{ => docker}/server.xml | 0 .../src/main/resources/docker/start_menas.sh | 26 ++++++++ menas/src/main/resources/start_menas.sh | 10 --- pom.xml | 5 ++ 11 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 menas/src/main/resources/docker.properties.template rename menas/src/main/resources/{ => docker}/envoy/Dockerfile (100%) rename menas/src/main/resources/{ => docker}/envoy/envoy.yaml (100%) rename menas/src/main/resources/{ => docker}/envoy/start_envoy.sh (100%) rename menas/src/main/resources/{ => docker}/server.xml (100%) create mode 100644 menas/src/main/resources/docker/start_menas.sh delete mode 100644 menas/src/main/resources/start_menas.sh diff --git a/.gitignore b/.gitignore index 676b6298d..dd1fa6d78 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ myTestCheckpoints **/main/resources/application.yml **/main/resources/spline.properties **/main/resources/kafka.properties +**/menas/*/main/resources/*.properties **/enceladus_env.sh *.config *.keytab diff --git a/menas/Dockerfile b/menas/Dockerfile index 8346d7cda..e9e7d1839 100644 --- a/menas/Dockerfile +++ b/menas/Dockerfile @@ -15,12 +15,17 @@ LABEL \ license="Apache License, version 2.0" \ name="Menas" -ADD src/main/resources/start_menas.sh start_menas.sh +ARG WAR_FILE +ARG PROPERTY_FILE + +ENV SPRING_CONFIG_NAME=${PROPERTY_FILE:-"application"} + +ADD ./src/main/resources/docker/start_menas.sh start_menas.sh RUN chmod +x start_menas.sh && \ rm -rf webapps/* -ADD ./target/*.war webapps/ROOT.war -ADD ./src/main/resources/server.xml /tmp/server.xml +ADD ./target/${WAR_FILE} webapps/ROOT.war +ADD src/main/resources/docker/server.xml /tmp/server.xml EXPOSE 8080 EXPOSE 8443 diff --git a/menas/pom.xml b/menas/pom.xml index 68aee93b7..78bd456d6 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -299,6 +299,19 @@ + + com.spotify + dockerfile-maven-plugin + ${dockerfile.maven.version} + + ${dockerfile.repository} + ${dockerfile.tag} + + ${project.build.finalName}.war + ${dockerfile.property.file} + + + org.scalastyle scalastyle-maven-plugin @@ -365,6 +378,9 @@ + diff --git a/menas/src/main/resources/docker.properties.template b/menas/src/main/resources/docker.properties.template new file mode 100644 index 000000000..4079d5acf --- /dev/null +++ b/menas/src/main/resources/docker.properties.template @@ -0,0 +1,62 @@ +# +# Copyright 2018 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A template property file for docker image. Most of the properties are expected to come from the environment variables. + +# Set the file size limit (default 1Mb). If you want to specify that files be +# unlimited set the property to -1. +spring.servlet.multipart.max-file-size=16MB + +# Set the timeout a single connection request can wait for a controller to return a value (in milliseconds) +spring.mvc.async.request-timeout=60000 + +# Set the total request size for a multipart/form-data (default 10Mb) +spring.servlet.multipart.max-request-size=16MB + +# Number of seconds to cache the static resources for +spring.resources.cache.cachecontrol.max-age=2678000 +spring.resources.chain.strategy.fixed.enabled=true +spring.resources.chain.strategy.fixed.paths=/components/**,/css/**,/generic/**,/service/** +spring.resources.chain.strategy.fixed.version=@project.version@ + +# Disable second security filter chain passes for ASYNC requests +spring.security.filter.dispatcher-types=REQUEST,ERROR + +# Timeout (in seconds) for MongoDB migration queries +migrations.mongo.query.timeout.seconds=300 + +menas.spark.master=local[1] + +#system-wide time zone +timezone=UTC + +#---------- Actuator +management.endpoints.web.base-path=/admin +management.endpoints.jmx.exposure.exclude=* +management.endpoints.web.exposure.include=health,threaddump,heapdump,loggers + +management.endpoints.enabled-by-default=false + +management.endpoint.health.enabled=true +management.endpoint.health.show-details=always +management.endpoint.threaddump.enabled=true +management.endpoint.heapdump.enabled=true +management.endpoint.loggers.enabled=true + +management.health.jms.enabled=false + +#---------- Monitoring +# Limit on the number of documents to be fetch in a single mongodb query and shown in UI +menas.monitoring.fetch.limit=500 diff --git a/menas/src/main/resources/envoy/Dockerfile b/menas/src/main/resources/docker/envoy/Dockerfile similarity index 100% rename from menas/src/main/resources/envoy/Dockerfile rename to menas/src/main/resources/docker/envoy/Dockerfile diff --git a/menas/src/main/resources/envoy/envoy.yaml b/menas/src/main/resources/docker/envoy/envoy.yaml similarity index 100% rename from menas/src/main/resources/envoy/envoy.yaml rename to menas/src/main/resources/docker/envoy/envoy.yaml diff --git a/menas/src/main/resources/envoy/start_envoy.sh b/menas/src/main/resources/docker/envoy/start_envoy.sh similarity index 100% rename from menas/src/main/resources/envoy/start_envoy.sh rename to menas/src/main/resources/docker/envoy/start_envoy.sh diff --git a/menas/src/main/resources/server.xml b/menas/src/main/resources/docker/server.xml similarity index 100% rename from menas/src/main/resources/server.xml rename to menas/src/main/resources/docker/server.xml diff --git a/menas/src/main/resources/docker/start_menas.sh b/menas/src/main/resources/docker/start_menas.sh new file mode 100644 index 000000000..e2f7521cd --- /dev/null +++ b/menas/src/main/resources/docker/start_menas.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# +# Copyright 2018 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [[ -n ${PRIVATE_KEY} && -n ${CERTIFICATE} && -n ${CA_CHAIN} ]]; then + echo "Certificate, chain and private key present, running secured version" + echo "${PRIVATE_KEY}" >> conf/private.pem + echo "${CERTIFICATE}" >> conf/certificate.pem + echo "${CA_CHAIN}" >> conf/cachain.pem + rm conf/server.xml + cp /tmp/server.xml conf/server.xml +fi +catalina.sh run diff --git a/menas/src/main/resources/start_menas.sh b/menas/src/main/resources/start_menas.sh deleted file mode 100644 index ea9eaf275..000000000 --- a/menas/src/main/resources/start_menas.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ -n ${PRIVATE_KEY} && -n ${CERTIFICATE} && -n ${CA_CHAIN} ]]; then - echo "Certificate, chain and private key present, running secured version" - echo "${PRIVATE_KEY}" >> conf/private.pem - echo "${CERTIFICATE}" >> conf/certificate.pem - echo "${CA_CHAIN}" >> conf/cachain.pem - rm conf/server.xml - cp /tmp/server.xml conf/server.xml -fi -catalina.sh run diff --git a/pom.xml b/pom.xml index 516bc5e0e..1116ab83f 100644 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 1.8 1.8 1.8 + 1.4.13 1.6 3.2.0 0.12 @@ -196,6 +197,10 @@ true ${project.basedir}/scalastyle-config.xml + docker.io/absaoss/menas + ${project.version} + + docker true WARN From ddc93852d9fbb225b3b86997c8668cc6ac9914ac Mon Sep 17 00:00:00 2001 From: David Benedeki <14905969+benedeki@users.noreply.github.com> Date: Mon, 25 Jan 2021 14:37:40 +0100 Subject: [PATCH 4/6] #1638 Fix validation of conformance rules on import entity (#1647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Saša Zejnilović --- .../za/co/absa/enceladus/menas/services/DatasetService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala index 0e327b815..e381fe806 100644 --- a/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala +++ b/menas/src/main/scala/za/co/absa/enceladus/menas/services/DatasetService.scala @@ -294,7 +294,7 @@ class DatasetService @Autowired()(datasetMongoRepository: DatasetMongoRepository cr: C): RuleValidationsAndFields = { val withOutputValidated = validateOutputColumn(fields, cr.outputColumn) val validationInputFields = validateInputColumn(fields, cr.inputColumn) - withOutputValidated.update(validationInputFields) + validationInputFields.update(withOutputValidated) } def validateMappingTable(fields: Future[Set[String]], From a3c0d793b21a8725460c9552af0c9553df9eb2ab Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 27 Jan 2021 09:28:44 +0100 Subject: [PATCH 5/6] Update versions for release v2.18.0 --- dao/pom.xml | 2 +- data-model/pom.xml | 2 +- examples/pom.xml | 2 +- menas/pom.xml | 2 +- migrations-cli/pom.xml | 2 +- migrations/pom.xml | 2 +- plugins-api/pom.xml | 2 +- plugins-builtin/pom.xml | 2 +- pom.xml | 2 +- spark-jobs/pom.xml | 2 +- utils/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dao/pom.xml b/dao/pom.xml index 1e4e4bcc3..369dc195b 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/data-model/pom.xml b/data-model/pom.xml index fd9986491..4504a5a90 100644 --- a/data-model/pom.xml +++ b/data-model/pom.xml @@ -24,7 +24,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/examples/pom.xml b/examples/pom.xml index 5ba36c847..aca6a62ac 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/menas/pom.xml b/menas/pom.xml index 3d73ceacd..ae34e701e 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/migrations-cli/pom.xml b/migrations-cli/pom.xml index 126393819..0227e7355 100644 --- a/migrations-cli/pom.xml +++ b/migrations-cli/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/migrations/pom.xml b/migrations/pom.xml index b19e5ec71..e7b55b235 100644 --- a/migrations/pom.xml +++ b/migrations/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/plugins-api/pom.xml b/plugins-api/pom.xml index b33d8c5eb..ebf805438 100644 --- a/plugins-api/pom.xml +++ b/plugins-api/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/plugins-builtin/pom.xml b/plugins-builtin/pom.xml index 49df03953..fceb11eff 100644 --- a/plugins-builtin/pom.xml +++ b/plugins-builtin/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/pom.xml b/pom.xml index 31029b246..384b7b2b0 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 pom Enceladus diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 6c937263d..573383bf5 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 diff --git a/utils/pom.xml b/utils/pom.xml index c847b3f6f..ef4061b0e 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0-SNAPSHOT + 2.18.0 From 23384e92a83f6645733a7a6390babd7aa98dbfff Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 27 Jan 2021 17:18:12 +0100 Subject: [PATCH 6/6] Update for next development version 2.19.0-SNAPSHOT --- dao/pom.xml | 2 +- data-model/pom.xml | 2 +- examples/pom.xml | 2 +- menas/pom.xml | 2 +- migrations-cli/pom.xml | 2 +- migrations/pom.xml | 2 +- plugins-api/pom.xml | 2 +- plugins-builtin/pom.xml | 2 +- pom.xml | 2 +- spark-jobs/pom.xml | 2 +- utils/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dao/pom.xml b/dao/pom.xml index 369dc195b..0768ce085 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/data-model/pom.xml b/data-model/pom.xml index 4504a5a90..8aa2c5f81 100644 --- a/data-model/pom.xml +++ b/data-model/pom.xml @@ -24,7 +24,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index aca6a62ac..048f1d06e 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/menas/pom.xml b/menas/pom.xml index ae34e701e..7c98e665a 100644 --- a/menas/pom.xml +++ b/menas/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/migrations-cli/pom.xml b/migrations-cli/pom.xml index 0227e7355..47f3006b9 100644 --- a/migrations-cli/pom.xml +++ b/migrations-cli/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/migrations/pom.xml b/migrations/pom.xml index e7b55b235..975155f39 100644 --- a/migrations/pom.xml +++ b/migrations/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/plugins-api/pom.xml b/plugins-api/pom.xml index ebf805438..17c36d000 100644 --- a/plugins-api/pom.xml +++ b/plugins-api/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/plugins-builtin/pom.xml b/plugins-builtin/pom.xml index fceb11eff..342c16806 100644 --- a/plugins-builtin/pom.xml +++ b/plugins-builtin/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 384b7b2b0..f2d518c29 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT pom Enceladus diff --git a/spark-jobs/pom.xml b/spark-jobs/pom.xml index 573383bf5..120673ac9 100644 --- a/spark-jobs/pom.xml +++ b/spark-jobs/pom.xml @@ -21,7 +21,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT diff --git a/utils/pom.xml b/utils/pom.xml index ef4061b0e..5d267ebf2 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -22,7 +22,7 @@ za.co.absa.enceladus parent - 2.18.0 + 2.19.0-SNAPSHOT