Extend Lighter to support query testing (#8)

* Dependencies upgrade * Keep spark session alive for query testing Co-authored-by: Denisas <[email protected]>
exacaster · Jan 19, 2022 · c524a3f · c524a3f
1 parent 587ea65
commit c524a3f
Show file tree

Hide file tree

Showing 16 changed files with 355 additions and 161 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM openjdk:11-jre-slim-stretch as server
 
-ARG SPARK_VERSION=3.0.3
+ARG SPARK_VERSION=3.2.0
 
 WORKDIR /home/app/
 COPY server/ ./server/
@@ -10,7 +10,7 @@ RUN ./gradlew build -PSPARK_VERSION=${SPARK_VERSION}
 
 FROM node:lts-alpine3.14 as frontend
 
-ARG SPARK_VERSION=3.0.3
+ARG SPARK_VERSION=3.2.0
 
 ENV REACT_APP_API_BASE_URL='/lighter'
 
@@ -23,7 +23,7 @@ RUN yarn install && yarn build
 
 FROM openjdk:11-jre-slim-stretch
 
-ARG SPARK_VERSION=3.0.3
+ARG SPARK_VERSION=3.2.0
 
 ENV FRONTEND_PATH=/home/app/frontend/
 ENV SPARK_HOME=/home/app/spark/
@@ -33,9 +33,9 @@ RUN ln -s /etc/hadoop/conf.cloudera.yarn /etc/alternatives/hadoop-conf \
   && ln -s /etc/hive/conf.cloudera.hive /etc/alternatives/hive-conf
 
 WORKDIR /home/app/
-COPY --from=server /home/app/server/build/docker/layers/libs /home/app/libs
-COPY --from=server /home/app/server/build/docker/layers/resources /home/app/resources
-COPY --from=server /home/app/server/build/docker/layers/application.jar /home/app/application.jar
+COPY --from=server /home/app/server/build/docker/main/layers/libs /home/app/libs
+COPY --from=server /home/app/server/build/docker/main/layers/resources /home/app/resources
+COPY --from=server /home/app/server/build/docker/main/layers/application.jar /home/app/application.jar
 
 COPY --from=frontend /home/app/frontend/build/ ./frontend/
 COPY --from=frontend /home/app/spark-${SPARK_VERSION}-bin-hadoop3.2/ ./spark/

diff --git a/server/build.gradle b/server/build.gradle
@@ -1,7 +1,8 @@
 plugins {
     id "java"
     id "groovy"
-    id "io.micronaut.application" version "2.0.6"
+    id "io.micronaut.application" version "3.1.1"
+    id "com.github.ben-manes.versions" version "0.41.0"
 }
 
 group "org.exacaster.lighter"
@@ -16,12 +17,12 @@ java {
 }
 
 micronaut {
-    version = "3.0.3"
+    version = "3.1.4"
     runtime "netty"
 }
 mainClassName = "com.exacaster.lighter.Application"
 
-def java.lang.String sparkVersion = project.hasProperty("SPARK_VERSION") ? project.getProperty("SPARK_VERSION") : '3.0.3'
+String sparkVersion = project.hasProperty("SPARK_VERSION") ? project.getProperty("SPARK_VERSION") : '3.2.0'
 
 dependencies {
     annotationProcessor "io.micronaut:micronaut-validation"
@@ -37,11 +38,11 @@ dependencies {
     implementation "io.micronaut:micronaut-http-client"
 
     implementation "org.apache.spark:spark-launcher_2.12:${sparkVersion}"
-    implementation 'io.fabric8:kubernetes-client:5.4.0'
-    implementation 'net.sf.py4j:py4j:0.10.9.2'
+    implementation 'io.fabric8:kubernetes-client:5.11.2'
+    implementation 'net.sf.py4j:py4j:0.10.9.3'
 
-    implementation "net.javacrumbs.shedlock:shedlock-micronaut:4.25.0"
-    implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-micronaut:4.25.0"
+    implementation "net.javacrumbs.shedlock:shedlock-micronaut:4.31.0"
+    implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-micronaut:4.31.0"
 
     runtimeOnly "io.micronaut.micrometer:micronaut-micrometer-core"
     runtimeOnly "io.micronaut.micrometer:micronaut-micrometer-registry-prometheus"
@@ -50,11 +51,11 @@ dependencies {
     runtimeOnly "com.h2database:h2"
     runtimeOnly "org.postgresql:postgresql"
 
-    testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:5.3.1"
+    testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:5.8.2"
     testImplementation "io.micronaut:micronaut-inject-groovy"
     testImplementation "io.micronaut.test:micronaut-test-spock"
     testImplementation "io.micronaut:micronaut-http-client"
-    testImplementation "io.fabric8:kubernetes-server-mock:5.7.3"
+    testImplementation "io.fabric8:kubernetes-server-mock:5.11.2"
 }
 
 tasks.withType(JavaCompile) {

diff --git a/server/gradle/wrapper/gradle-wrapper.jar b/server/gradle/wrapper/gradle-wrapper.jar
diff --git a/server/gradle/wrapper/gradle-wrapper.properties b/server/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists