Aws-POC merge to develop

README.md

@@ -67,11 +67,11 @@ The coverage reports are written in each module's `target` directory and aggrega
 #### Menas requirements:
 - [**Tomcat 8.5/9.0** installation](https://tomcat.apache.org/download-90.cgi)
 - [**MongoDB 4.0** installation](https://docs.mongodb.com/manual/administration/install-community/)
-- [**Spline service deployment**](https://absaoss.github.io/spline/#get-spline)
+- [**Spline UI deployment**](https://absaoss.github.io/spline/) - place the [spline.war](https://search.maven.org/remotecontent?filepath=za/co/absa/spline/spline-web/0.3.9/spline-web-0.3.9.war)
+ in your Tomcat webapps directory (rename after downloading to _spline.war_); NB! don't forget to set up the `spline.mongodb.url` configuration for the _war_
 - **HADOOP_CONF_DIR** environment variable, pointing to the location of your hadoop configuration (pointing to a hadoop installation)
 
-The _Spline service_ can be omitted; in such case the **Standardization** and **Conformance** `spline.producer.url` setting
-as well as **Menas** `menas.lineage.readApiUrl` and `menas.oozie.lineageWriteApiUrl` settings should be all set to empty string. 
+The _Spline UI_ can be omitted; in such case the **Menas** `spline.urlTemplate` setting should be set to empty string. 
 
 #### Deploying Menas
 Simply copy the **menas.war** file produced when building the project into Tomcat's webapps directory. 
@@ -106,7 +106,7 @@ password=changeme
 --deploy-mode <client/cluster> \
 --driver-cores <num> \
 --driver-memory <num>G \
---conf "spark.driver.extraJavaOptions=-Dmenas.rest.uri=<menas_api_uri:port> -Dstandardized.hdfs.path=<path_for_standardized_output>-{0}-{1}-{2}-{3} -Dspline.producer.url=<url_for_spline_consumer> -Dhdp.version=<hadoop_version>" \
+--conf "spark.driver.extraJavaOptions=-Dmenas.rest.uri=<menas_api_uri:port> -Dstandardized.hdfs.path=<path_for_standardized_output>-{0}-{1}-{2}-{3} -Dspline.mongodb.url=<mongo_url_for_spline> -Dspline.mongodb.name=<spline_database_name> -Dhdp.version=<hadoop_version>" \
 --class za.co.absa.enceladus.standardization.StandardizationJob \
 <spark-jobs_<build_version>.jar> \
 --menas-auth-keytab <path_to_keytab_file> \
@@ -130,7 +130,7 @@ password=changeme
 --driver-cores <num> \
 --driver-memory <num>G \
 --conf 'spark.ui.port=29000' \
---conf "spark.driver.extraJavaOptions=-Dmenas.rest.uri=<menas_api_uri:port> -Dstandardized.hdfs.path=<path_of_standardized_input>-{0}-{1}-{2}-{3} -Dconformance.mappingtable.pattern=reportDate={0}-{1}-{2} -Dspline.producer.url=<url_for_spline_consumer> -Dhdp.version=<hadoop_version>" \
+--conf "spark.driver.extraJavaOptions=-Dmenas.rest.uri=<menas_api_uri:port> -Dstandardized.hdfs.path=<path_of_standardized_input>-{0}-{1}-{2}-{3} -Dconformance.mappingtable.pattern=reportDate={0}-{1}-{2} -Dspline.mongodb.url=<mongo_url_for_spline> -Dspline.mongodb.name=<spline_database_name>" -Dhdp.version=<hadoop_version> \
 --packages za.co.absa:enceladus-parent:<version>,za.co.absa:enceladus-conformance:<version> \
 --class za.co.absa.enceladus.conformance.DynamicConformanceJob \
 <spark-jobs_<build_version>.jar> \

examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample1.scala

@@ -23,10 +23,9 @@ import za.co.absa.enceladus.dao.auth.MenasKerberosCredentials
 import za.co.absa.enceladus.dao.rest.RestDaoFactory
 import za.co.absa.enceladus.examples.interpreter.rules.custom.UppercaseCustomConformanceRule
 import za.co.absa.enceladus.model.Dataset
-import za.co.absa.enceladus.utils.testUtils.HadoopFsTestBase
 import za.co.absa.enceladus.utils.time.TimeZoneNormalizer
 
-object CustomRuleSample1 extends HadoopFsTestBase {
+object CustomRuleSample1 extends CustomRuleSampleFs {
 
   case class ExampleRow(id: Int, makeUpper: String, leave: String)
   case class OutputRow(id: Int, makeUpper: String, leave: String, doneUpper: String)

examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample2.scala

@@ -24,10 +24,9 @@ import za.co.absa.enceladus.dao.auth.MenasKerberosCredentials
 import za.co.absa.enceladus.dao.rest.{MenasConnectionStringParser, RestDaoFactory}
 import za.co.absa.enceladus.examples.interpreter.rules.custom.LPadCustomConformanceRule
 import za.co.absa.enceladus.model.Dataset
-import za.co.absa.enceladus.utils.testUtils.HadoopFsTestBase
 import za.co.absa.enceladus.utils.time.TimeZoneNormalizer
 
-object CustomRuleSample2 extends HadoopFsTestBase {
+object CustomRuleSample2 extends CustomRuleSampleFs {
 
   case class ExampleRow(id: Int, addPad: String, leave: String)
   case class OutputRow(id: Int, addPad: String, leave: String, donePad: String)

examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample3.scala

@@ -24,10 +24,9 @@ import za.co.absa.enceladus.dao.auth.MenasKerberosCredentials
 import za.co.absa.enceladus.dao.rest.{MenasConnectionStringParser, RestDaoFactory}
 import za.co.absa.enceladus.examples.interpreter.rules.custom.{LPadCustomConformanceRule, UppercaseCustomConformanceRule}
 import za.co.absa.enceladus.model.Dataset
-import za.co.absa.enceladus.utils.testUtils.HadoopFsTestBase
 import za.co.absa.enceladus.utils.time.TimeZoneNormalizer
 
-object CustomRuleSample3 extends HadoopFsTestBase {
+object CustomRuleSample3 extends CustomRuleSampleFs {
   implicit val spark: SparkSession = SparkSession.builder
     .master("local[*]")
     .appName("CustomRuleSample3")

examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSample4.scala

@@ -26,10 +26,9 @@ import za.co.absa.enceladus.dao.auth.MenasKerberosCredentials
 import za.co.absa.enceladus.dao.rest.{MenasConnectionStringParser, RestDaoFactory}
 import za.co.absa.enceladus.examples.interpreter.rules.custom.{LPadCustomConformanceRule, UppercaseCustomConformanceRule}
 import za.co.absa.enceladus.model.Dataset
-import za.co.absa.enceladus.utils.testUtils.HadoopFsTestBase
 import za.co.absa.enceladus.utils.time.TimeZoneNormalizer
 
-object CustomRuleSample4 extends HadoopFsTestBase {
+object CustomRuleSample4 extends CustomRuleSampleFs {
   TimeZoneNormalizer.normalizeJVMTimeZone() //normalize JVM time zone as soon as possible
 
   /**

examples/src/main/scala/za/co/absa/enceladus/examples/CustomRuleSampleFs.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.examples
+
+import org.apache.hadoop.fs.FileSystem
+import org.apache.spark.sql.SparkSession
+import za.co.absa.enceladus.utils.fs.HadoopFsUtils
+
+trait CustomRuleSampleFs {
+  def spark: SparkSession
+
+  implicit val fs: FileSystem = FileSystem.get(spark.sparkContext.hadoopConfiguration)
+  implicit val fsUtils: HadoopFsUtils = HadoopFsUtils.getOrCreate(fs)
+}

menas/src/main/scala/za/co/absa/enceladus/menas/WebSecurityConfig.scala

@@ -72,9 +72,6 @@ class WebSecurityConfig @Autowired()(beanFactory: BeanFactory,
         .and()
           .addFilterBefore(kerberosFilter, classOf[UsernamePasswordAuthenticationFilter])
           .addFilterAfter(jwtAuthFilter, classOf[SpnegoAuthenticationProcessingFilter])
-        .headers()
-          .frameOptions()
-          .sameOrigin()
     }
 
     @Bean

menas/ui/css/style.css

@@ -44,25 +44,25 @@ html, body {
 }
 
 .monitoringTitle {
-  textAlign: Center;
+  text-align: Center;
   width: 100%;
 }
 
-.monitoringRecordspopover {
+.monitoringRecordsPopover {
     top: 5rem !important;
     left: calc(50% - 18rem) !important;
 }
 
 .auditTrailNestedListItem {
   max-width: 100%;
   background-color: inherit !important;
-  border-bottom: 0px !important;
+  border-bottom: 0 !important;
 }
 
 .auditTrailNestedList > ul > .sapMListNoData {
   background-color: inherit !important;
-  border-bottom: 0px !important;
-  padding-left: 0px !important;
+  border-bottom: 0 !important;
+  padding-left: 0 !important;
 }
 
 .lineageIframe {
@@ -72,8 +72,8 @@ html, body {
 
 .auditTrailNestedList > ul > .sapMListNoData {
   background-color: inherit !important;
-  border-bottom: 0px !important;
-  padding-left: 0px !important;
+  border-bottom: 0 !important;
+  padding-left: 0 !important;
 }
 
 .lineageIframe {

pom.xml

@@ -178,7 +178,7 @@
         <scala.version>2.11.12</scala.version>
         <scalatest.maven.version>2.0.0</scalatest.maven.version>
         <scalatest.version>3.2.2</scalatest.version>
-        <scopt.version>4.0.0-RC2</scopt.version>
+        <scopt.version>4.0.0</scopt.version>
         <spark.compat.version>2.4</spark.compat.version>
         <spark.hats.version>0.2.1</spark.hats.version>
         <spark.hofs.version>0.4.0</spark.hofs.version>

scripts/bash/enceladus_env.template.sh

@@ -16,9 +16,10 @@
 # Environment configuration
 STD_HDFS_PATH="/bigdata/std/std-{0}-{1}-{2}-{3}"
 
-# Configuration for Spline
+# MongoDB connection configuration for Spline
 # Important! Special characters should be escaped using triple backslashes (\\\)
-SPLINE_PRODUCER_URL="http://localhost:8080/spline/producer"
+SPLINE_MONGODB_URL="mongodb://localhost:27017"
+SPLINE_MONGODB_NAME="spline"
 
 export SPARK_HOME="/opt/spark-2.4.4"
 SPARK_SUBMIT="$SPARK_HOME/bin/spark-submit"

scripts/bash/run_enceladus.sh

@@ -416,7 +416,7 @@ if [ "$DRA_ENABLED" = true ] ; then
 fi
 
 JVM_CONF="spark.driver.extraJavaOptions=-Dstandardized.hdfs.path=$STD_HDFS_PATH \
--Dspline.producer.url=$SPLINE_PRODUCER_URL -Dhdp.version=$HDP_VERSION \
+-Dspline.mongodb.url=$SPLINE_MONGODB_URL -Dspline.mongodb.name=$SPLINE_MONGODB_NAME -Dhdp.version=$HDP_VERSION \
 $MT_PATTERN"
 
 if [ "$HELP_CALL" == "1" ]; then

spark-jobs/src/main/resources/spline.properties.template

@@ -4,7 +4,6 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ -13,12 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Spline mode - the way how Spline is integrated. For details see Spline documentation
-# possible values (default is BEST_EFFORT):
-#  DISABLED - no Spline integration (no lineage will be recorded)
-#  REQUIRED - Spline service has to be running on the spline.producer.url address; if not, job exits without execution
-#  BEST_EFFORT - job tries to connect to the provided Spline service (spline.producer.url address); but if that fails, job will still execute
-spline.mode=BEST_EFFORT
+
+#
+# Spline properties template.
+# Uncomment the following lines to override corresponding Hadoop environment configuration properties.
+#
+# Set of properties for setting up persistence to MongoDB.
 #
+spline.persistence.factory=za.co.absa.spline.persistence.api.composition.ParallelCompositeFactory
+spline.persistence.composition.factories=za.co.absa.spline.persistence.mongo.MongoPersistenceFactory,za.co.absa.spline.persistence.hdfs.HdfsPersistenceFactory
+
+spline.mongodb.url=mongodb://localhost:27017
+spline.mongodb.name=spline
+
 #
-spline.producer.url=http://localhost:8080/spline/producer
+# A property for setting up persistence to Apache Atlas. Additional properties defining connectivity to Atlas are required to be part of this configuration file. (see Atlas configuration file)
+# spline.persistence.factory=za.co.absa.spline.persistence.atlas.AtlasPersistenceFactory