Aws-POC merge to develop

data-model/src/main/scala/za/co/absa/enceladus/model/Dataset.scala

@@ -64,6 +64,7 @@ case class Dataset(name: String,
   def setHDFSPath(newPath: String): Dataset = this.copy(hdfsPath = newPath)
   def setHDFSPublishPath(newPublishPath: String): Dataset = this.copy(hdfsPublishPath = newPublishPath)
   def setConformance(newConformance: List[ConformanceRule]): Dataset = this.copy(conformance = newConformance)
+  def setSchedule(newSchedule: Option[OozieSchedule]): Dataset = this.copy(schedule = newSchedule)
   override def setParent(newParent: Option[MenasReference]): Dataset = this.copy(parent = newParent)
 
   def propertiesAsMap: Map[String, String] = properties.getOrElse(Map.empty)
@@ -88,11 +89,10 @@ case class Dataset(name: String,
 
   private def substituteMappingConformanceRuleCharacter(dataset: Dataset, from: Char, to: Char): Dataset = {
     val conformanceRules = dataset.conformance.map {
-      case m: MappingConformanceRule => {
+      case m: MappingConformanceRule =>
         m.copy(attributeMappings = m.attributeMappings.map(key => {
           (key._1.replace(from, to), key._2)
         }))
-      }
       case c: ConformanceRule => c
     }
 

data-model/src/main/scala/za/co/absa/enceladus/model/menas/HDFSFolder.scala

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2018 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.enceladus.model.menas
+
+case class HDFSFolder
+(
+  path: String,
+  name: String,
+  children: Option[Seq[HDFSFolder]]
+)

data-model/src/test/scala/za/co/absa/enceladus/model/dataFrameFilter/DataFrameFilterSuite.scala

@@ -1,10 +1,10 @@
 package za.co.absa.enceladus.model.dataFrameFilter
 
-import org.scalatest.FunSuite
 import org.apache.spark.sql.functions.{col, lit, not => columnNot}
 import org.apache.spark.sql.types._
+import org.scalatest.funsuite.AnyFunSuite
 
-class DataFrameFilterSuite extends FunSuite {
+class DataFrameFilterSuite extends AnyFunSuite {
   test("Filter for equal value") {
     val filterExpr = EqualsFilter("column1", "v").filter.expr
     val expected = (col("column1") === lit("v")).expr

menas/pom.xml

@@ -236,11 +236,6 @@
             <artifactId>webjars-locator-core</artifactId>
             <version>${webjars.locator.version}</version>
         </dependency>
-        <dependency>
-            <groupId>za.co.absa.spline</groupId>
-            <artifactId>client-ui</artifactId>
-            <version>${spline.ui.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.oozie</groupId>
             <artifactId>oozie-client</artifactId>
@@ -270,6 +265,17 @@
             <artifactId>guava</artifactId>
             <version>${guava.version}</version>
         </dependency>
+        <dependency>
+            <groupId>za.co.absa</groupId>
+            <artifactId>atum_${scala.compat.version}</artifactId>
+            <version>${atum.version}</version>
+            <exclusions>
+                <exclusion>  <!-- declare the exclusion here -->
+                    <groupId>software.amazon.awssdk</groupId>
+                    <artifactId>s3</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <!-- Test -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

menas/src/main/resources/application.properties.template

@@ -73,9 +73,9 @@ menas.environment=Dev
 
 menas.spark.master=local[1]
 
-# The api gateway for Lineage (Spline) display. If empty Lineage button is not shown.
+# The template for Lineage (Spline) display. If empty Lineage button is not shown.
 # Avoiding the http(s) protocol will cause that the same protocol will be used as for Menas itself, avoiding issues
-menas.lineage.readApiUrl=//localhost:8080/spline-gw/consumer
+spline.urlTemplate=//localhost:8080/spline/dataset/lineage/_search?path=%s&application_id=%s
 
 #system-wide time zone
 timezone=UTC
@@ -99,6 +99,51 @@ management.health.jms.enabled=false
 # Limit on the number of documents to be fetch in a single mongodb query and shown in UI
 menas.monitoring.fetch.limit=500
 
+#--------Oozie
+#menas.oozie.oozieUrl=http://localhost:11000/oozie/
+
+#Path where oozie coordinators and workflows are stored on HDFS
+#menas.oozie.schedule.hdfs.path=/tmp
+#menas.oozie.timeZone=Africa/Ceuta
+
+#This is passed into oozie.action.sharelib.for.spark
+#This ensures that schedules are run with correct version of spark without updating oozie sharelibs globally
+menas.oozie.sharelibForSpark=spark-menas
+
+#This parameter disables sharelibForSpark. It's the absolute path to a location containing all relevant spark sharelibs
+#menas.oozie.libpath=/some/path/libpath
+
+#Enceladus jars are going to be loaded into this location
+menas.oozie.enceladusJarLocation=hdfs:///tmp/enceladus-jars
+
+#Maven repo from which to resolve enceladus jars.. Can be local repo, maven central, nexus
+menas.oozie.mavenRepoLocation=https://repo.maven.apache.org/maven2/
+
+#standardization and conformance path in the repo
+menas.oozie.mavenSparkJobsJarLocation=/za/co/absa/enceladus/spark-jobs/@project.version@/[email protected]@.jar
+
+#Menas URL for submitted std and conf jobs
+menas.oozie.menasApiURL=http://menasHostname:8080/menas/api
+
+#Mongo address for spline for the submitted jobs
+menas.oozie.splineMongoURL=mongodb://localhost:27017
+
+#In some cases extra driver options need to be surrounded by double quotes
+#This seems to depend of the version of oozie
+#menas.oozie.sparkConf.surroundingQuoteChar="
+
+#Configure the following three parameters to use oozie impersonation
+#menas.oozie.proxyUser=oozieProxyUser
+#menas.oozie.proxyGroup=oozieProxyGroup
+#menas.oozie.proxyUserKeytab=/path/to/keytab
+
+#The following property allows customization of the error message presented to the user if they're missing the impersonation group
+#menas.oozie.customImpersonationExceptionMessage="Please add the system user into the '${menas.oozie.proxyGroup}' group to use this feature."
+
+#Provide optional spark config options to be passed onto standardization and conformance jobs
+#menas.oozie.extraSparkConfigs={"spark.eventLog.dir" : "hdfs:///spark2-history/", \
+#"spark.history.fs.logDirectory": "hdfs:///spark2-history" }
+
 #----------- Schema Registry
 # URL of schema registry [optional]:
 # when unset, the ability to load schema by subject name from the schema repository will not be present

menas/src/main/scala/za/co/absa/enceladus/menas/MvcConfig.scala

@@ -15,24 +15,13 @@
 
 package za.co.absa.enceladus.menas
 
-import org.springframework.context.annotation.{Bean, Configuration}
-import org.springframework.web.servlet.config.annotation.{ResourceHandlerRegistry, ViewControllerRegistry, WebMvcConfigurer}
-import org.webjars.WebJarAssetLocator
+import org.springframework.context.annotation.Configuration
+import org.springframework.web.servlet.config.annotation.ViewControllerRegistry
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer
 
 @Configuration
 class MvcConfig extends WebMvcConfigurer {
-  @Bean
-  def webJarAssetLocator: WebJarAssetLocator = new WebJarAssetLocator()
-
-  override def addResourceHandlers(registry: ResourceHandlerRegistry) {
-    registry
-      .addResourceHandler(LineageConfig.resourceHandler)
-      .addResourceLocations(LineageConfig.resourceLocation)
-      .resourceChain(true)
-      .addResolver(new WebJarsResourceFuzzyResolver(webJarAssetLocator))
-  }
-
-  override def addViewControllers(registry: ViewControllerRegistry) {
+  def addViewControllers(registry: ViewControllerRegistry) {
     registry.addViewController("/login").setViewName("login")
   }
 }

menas/src/main/scala/za/co/absa/enceladus/menas/controllers/ConfigurationController.scala

@@ -17,7 +17,6 @@ package za.co.absa.enceladus.menas.controllers
 
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.web.bind.annotation.{GetMapping, RequestMapping, RestController}
-import za.co.absa.enceladus.menas.LineageConfig
 
 @RestController
 @RequestMapping(Array("/api/configuration"))
@@ -29,9 +28,4 @@ class ConfigurationController extends BaseController {
   def getEnvironment(): String = {
     menasEnvironment
   }
-
-  @GetMapping(path = Array("/lineageExecutionIdApiTemplate"))
-  def getLineageExecutionIdApiTemplate(): String = {
-    LineageConfig.executionIdApiTemplate.getOrElse("")
-  }
 }

menas/src/main/scala/za/co/absa/enceladus/menas/controllers/RestExceptionHandler.scala

@@ -27,16 +27,20 @@ import org.springframework.web.method.annotation.MethodArgumentTypeMismatchExcep
 import za.co.absa.enceladus.menas.exceptions._
 import za.co.absa.enceladus.menas.models.RestError
 import za.co.absa.enceladus.menas.models.rest.RestResponse
-import za.co.absa.enceladus.model.properties.propertyType.PropertyTypeValidationException
-import za.co.absa.enceladus.model.{UsedIn, Validation}
-import za.co.absa.enceladus.menas.models.Validation
 import za.co.absa.enceladus.menas.models.rest.errors.{RemoteSchemaRetrievalError, RequestTimeoutExpiredError, SchemaFormatError, SchemaParsingError}
 import za.co.absa.enceladus.menas.models.rest.exceptions.{RemoteSchemaRetrievalException, SchemaFormatException, SchemaParsingException}
-import za.co.absa.enceladus.model.UsedIn
+import za.co.absa.enceladus.model.properties.propertyType.PropertyTypeValidationException
+import za.co.absa.enceladus.model.{UsedIn, Validation}
 
 @ControllerAdvice(annotations = Array(classOf[RestController]))
 class RestExceptionHandler {
 
+  @Value("${menas.oozie.customImpersonationExceptionMessage:}")
+  val oozieImpersonationExceptionMessage: String = ""
+
+  @Value("${menas.oozie.proxyGroup:}")
+  val oozieProxyGroup: String = ""
+
   private val logger = LoggerFactory.getLogger(this.getClass)
 
   @ExceptionHandler(value = Array(classOf[AsyncRequestTimeoutException]))
@@ -105,4 +109,24 @@ class RestExceptionHandler {
     ResponseEntity.notFound().build[Any]()
   }
 
+  @ExceptionHandler(Array(classOf[OozieActionException]))
+  def handleOozieActionException(ex: OozieActionException): ResponseEntity[RestError] = {
+    val err = RestError(ex.getMessage)
+    logger.error(s"Exception: $err", ex)
+    new ResponseEntity(err, HttpStatus.INTERNAL_SERVER_ERROR)
+  }
+
+  @ExceptionHandler(Array(classOf[OozieClientException]))
+  def handleOozieClientException(ex: OozieClientException): ResponseEntity[RestError] = {
+    val err = if (ex.getMessage.toLowerCase.contains("unauthorized proxyuser")) {
+      val message = if (oozieImpersonationExceptionMessage.nonEmpty) oozieImpersonationExceptionMessage else
+        s"Please add the system user into ${oozieProxyGroup} group to use this feature."
+      RestError(message)
+    } else {
+      RestError(ex.getMessage)
+    }
+
+    logger.error(s"Exception: $err", ex)
+    new ResponseEntity(err, HttpStatus.INTERNAL_SERVER_ERROR)
+  }
 }