From 1ca31de58cf2634fa4ce2fbe164cf7894ce2fc56 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 17:05:50 +0800 Subject: [PATCH 1/8] Add scala version suffix on scala modules --- build/dist | 10 +++++----- dev/kyuubi-codecov/pom.xml | 16 ++++++++-------- dev/kyuubi-extension-spark-3-1/pom.xml | 2 +- dev/kyuubi-tpcds/pom.xml | 2 +- externals/kyuubi-spark-monitor/pom.xml | 4 ++-- externals/kyuubi-spark-sql-engine/pom.xml | 12 ++++++------ kubernetes/integration-tests/pom.xml | 18 +++++++++--------- kyuubi-assembly/pom.xml | 12 ++++++------ kyuubi-common/pom.xml | 2 +- kyuubi-ctl/pom.xml | 10 +++++----- kyuubi-ha/pom.xml | 8 ++++---- kyuubi-hive-jdbc/pom.xml | 9 ++++++--- kyuubi-metrics/pom.xml | 7 +++---- kyuubi-server/pom.xml | 14 +++++++------- kyuubi-zookeeper/pom.xml | 6 +++--- tools/spark-block-cleaner/pom.xml | 6 +++--- 16 files changed, 70 insertions(+), 68 deletions(-) diff --git a/build/dist b/build/dist index 97f7ff6691a..6df56e60fbd 100755 --- a/build/dist +++ b/build/dist @@ -191,22 +191,22 @@ echo "Build flags: $@" >> "$DISTDIR/RELEASE" cp -r "$KYUUBI_HOME/kyuubi-assembly/target/scala-$SCALA_VERSION/jars/" "$DISTDIR/jars" # Copy spark engines -cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engine-$VERSION.jar" "$DISTDIR/externals/engines/spark" +cp "$KYUUBI_HOME/externals/kyuubi-spark-sql-engine/target/kyuubi-spark-sql-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/spark" # Copy kyuubi tools -if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" ]]; then +if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes" mkdir -p "$DISTDIR/tools/spark-block-cleaner/jars" cp -r "$KYUUBI_HOME/tools/spark-block-cleaner/kubernetes/" "$DISTDIR/tools/spark-block-cleaner/" - cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner-$VERSION.jar" "$DISTDIR/tools/spark-block-cleaner/jars/" + cp "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/tools/spark-block-cleaner/jars/" fi # Copy Kyuubi extension SPARK_EXTENSION_VERSIONS=('3-1') for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do - if [[ -f $"$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION-$VERSION.jar" ]]; then + if [[ -f $"$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION_${SCALA_VERSION}-${VERSION}.jar" ]]; then mkdir -p "$DISTDIR/extension" - cp "$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION-$VERSION.jar" "$DISTDIR/extension" + cp "$KYUUBI_HOME/dev/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/extension" fi done diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index fb5a8e35e9e..4fd145709fe 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -27,50 +27,50 @@ 4.0.0 - kyuubi-codecov + kyuubi-codecov_2.12 pom Kyuubi Project Dev Code Coverage org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-ctl + kyuubi-ctl_2.12 ${project.version} org.apache.kyuubi - kyuubi-zookeeper + kyuubi-zookeeper_2.12 ${project.version} org.apache.kyuubi - kyuubi-ha + kyuubi-ha_2.12 ${project.version} org.apache.kyuubi - kyuubi-metrics + kyuubi-metrics_2.12 ${project.version} org.apache.kyuubi - kyuubi-server + kyuubi-server_2.12 ${project.version} org.apache.kyuubi - kyuubi-spark-sql-engine + kyuubi-spark-sql-engine_2.12 ${project.version} diff --git a/dev/kyuubi-extension-spark-3-1/pom.xml b/dev/kyuubi-extension-spark-3-1/pom.xml index 59f37e47c31..6f3eb7a9fd0 100644 --- a/dev/kyuubi-extension-spark-3-1/pom.xml +++ b/dev/kyuubi-extension-spark-3-1/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - kyuubi-extension-spark-3-1 + kyuubi-extension-spark-3-1_2.12 jar Kyuubi Project Dev Spark Extensions (for Spark 3.1) diff --git a/dev/kyuubi-tpcds/pom.xml b/dev/kyuubi-tpcds/pom.xml index 77f41f4fe90..08c11c83129 100644 --- a/dev/kyuubi-tpcds/pom.xml +++ b/dev/kyuubi-tpcds/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - kyuubi-tpcds + kyuubi-tpcds_2.12 jar Kyuubi Project Dev TPCDS Generator diff --git a/externals/kyuubi-spark-monitor/pom.xml b/externals/kyuubi-spark-monitor/pom.xml index d1af4dd6dc5..e0ea286e7df 100644 --- a/externals/kyuubi-spark-monitor/pom.xml +++ b/externals/kyuubi-spark-monitor/pom.xml @@ -27,14 +27,14 @@ 4.0.0 - kyuubi-spark-monitor + kyuubi-spark-monitor_2.12 jar Kyuubi Project Spark Monitor org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml index 7063cb51e06..931031c19f1 100644 --- a/externals/kyuubi-spark-sql-engine/pom.xml +++ b/externals/kyuubi-spark-sql-engine/pom.xml @@ -27,26 +27,26 @@ 4.0.0 - kyuubi-spark-sql-engine + kyuubi-spark-sql-engine_2.12 Kyuubi Project Engine Spark SQL jar org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-ha + kyuubi-ha_2.12 ${project.version} org.apache.kyuubi - kyuubi-spark-monitor + kyuubi-spark-monitor_2.12 ${project.version} @@ -58,7 +58,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test @@ -165,7 +165,7 @@ org.apache.kyuubi - kyuubi-zookeeper + kyuubi-zookeeper_2.12 ${project.version} test diff --git a/kubernetes/integration-tests/pom.xml b/kubernetes/integration-tests/pom.xml index 97962912ca9..d568e93f4cc 100644 --- a/kubernetes/integration-tests/pom.xml +++ b/kubernetes/integration-tests/pom.xml @@ -26,32 +26,32 @@ ../../pom.xml + kubernetes-integration-tests_2.12 Kyuubi Kubernetes Integration Tests 4.0.0 - kubernetes-integration-tests - - - io.fabric8 - kubernetes-client - - org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test + + io.fabric8 + kubernetes-client + test + + org.apache.kyuubi diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index c72e5f97e1e..a6d66ebebb0 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -28,38 +28,38 @@ ../pom.xml - kyuubi-assembly + kyuubi-assembly_2.12 pom Kyuubi Project Assembly org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-ha + kyuubi-ha_2.12 ${project.version} org.apache.kyuubi - kyuubi-metrics + kyuubi-metrics_2.12 ${project.version} org.apache.kyuubi - kyuubi-server + kyuubi-server_2.12 ${project.version} org.apache.kyuubi - kyuubi-ctl + kyuubi-ctl_2.12 ${project.version} diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index 48aed51afc6..739a030742f 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -27,7 +27,7 @@ 4.0.0 - kyuubi-common + kyuubi-common_2.12 jar Kyuubi Project Common diff --git a/kyuubi-ctl/pom.xml b/kyuubi-ctl/pom.xml index 5ad02c5288d..42596aa4827 100644 --- a/kyuubi-ctl/pom.xml +++ b/kyuubi-ctl/pom.xml @@ -27,19 +27,19 @@ 4.0.0 - kyuubi-ctl + kyuubi-ctl_2.12 jar Kyuubi Project Control org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-ha + kyuubi-ha_2.12 ${project.version} @@ -85,7 +85,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test @@ -105,7 +105,7 @@ org.apache.kyuubi - kyuubi-zookeeper + kyuubi-zookeeper_2.12 ${project.version} test diff --git a/kyuubi-ha/pom.xml b/kyuubi-ha/pom.xml index 5ba8a1861d2..0f94fbf5748 100644 --- a/kyuubi-ha/pom.xml +++ b/kyuubi-ha/pom.xml @@ -27,14 +27,14 @@ 4.0.0 - kyuubi-ha + kyuubi-ha_2.12 jar Kyuubi Project High Availability org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} @@ -55,7 +55,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test @@ -75,7 +75,7 @@ org.apache.kyuubi - kyuubi-zookeeper + kyuubi-zookeeper_2.12 ${project.version} test diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml index f4eba55c138..872c7b570e4 100644 --- a/kyuubi-hive-jdbc/pom.xml +++ b/kyuubi-hive-jdbc/pom.xml @@ -26,6 +26,9 @@ 4.0.0 + kyuubi-hive-jdbc Kyuubi Project Hive JDBC Client jar @@ -64,20 +67,20 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar org.apache.kyuubi - kyuubi-spark-sql-engine + kyuubi-spark-sql-engine_2.12 ${project.version} test org.apache.kyuubi - kyuubi-spark-sql-engine + kyuubi-spark-sql-engine_2.12 ${project.version} test-jar diff --git a/kyuubi-metrics/pom.xml b/kyuubi-metrics/pom.xml index 843469f3c7a..ac0500e2f96 100644 --- a/kyuubi-metrics/pom.xml +++ b/kyuubi-metrics/pom.xml @@ -26,15 +26,14 @@ 4.0.0 - kyuubi-metrics - + kyuubi-metrics_2.12 jar Kyuubi Project Metrics org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} @@ -90,7 +89,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index 2e1e4214c42..883def4e04c 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -26,32 +26,32 @@ 4.0.0 - kyuubi-server + kyuubi-server_2.12 Kyuubi Project Server jar org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-ha + kyuubi-ha_2.12 ${project.version} org.apache.kyuubi - kyuubi-zookeeper + kyuubi-zookeeper_2.12 ${project.version} org.apache.kyuubi - kyuubi-metrics + kyuubi-metrics_2.12 ${project.version} @@ -68,7 +68,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test @@ -84,7 +84,7 @@ org.apache.kyuubi - kyuubi-spark-sql-engine + kyuubi-spark-sql-engine_2.12 ${project.version} test diff --git a/kyuubi-zookeeper/pom.xml b/kyuubi-zookeeper/pom.xml index 841911d6901..24d0fb91dc0 100644 --- a/kyuubi-zookeeper/pom.xml +++ b/kyuubi-zookeeper/pom.xml @@ -26,14 +26,14 @@ 4.0.0 - kyuubi-zookeeper + kyuubi-zookeeper_2.12 Kyuubi Project Embedded Zookeeper jar org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} @@ -51,7 +51,7 @@ org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index 54d4ba33765..a3627bc84e4 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -26,20 +26,20 @@ 4.0.0 - spark-block-cleaner + spark-block-cleaner_2.12 Kyuubi Project Spark Block Cleaner jar org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} org.apache.kyuubi - kyuubi-common + kyuubi-common_2.12 ${project.version} test-jar test From cac96531af653d481cdd7cb701d4e08e00ba33dc Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 17:07:11 +0800 Subject: [PATCH 2/8] Enable kyuubi-extension-spark-3-1 in daily publish snapshot --- .github/workflows/publish-snapshot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index c48bb37f87c..3855281a8d8 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -36,4 +36,4 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -DskipTests -Pspark-provided -s ./build/release/asf-settings.xml + ./build/mvn clean deploy -DskipTests -Pkyuubi-extension-spark-3-1,spark-provided -s ./build/release/asf-settings.xml From 2cf4793e34c0bc5dbc938b7168e8f280677e5711 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 19:19:01 +0800 Subject: [PATCH 3/8] fix --- .github/workflows/master.yml | 2 +- .github/workflows/nightly.yml | 2 +- build/dependency.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 772c742cbab..0f7d7568791 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -137,7 +137,7 @@ jobs: ${{ runner.os }}-maven-io- - name: Run TPC-DS Tests run: | - ./build/mvn clean install -Pspark-3.1 -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper,:kyuubi-spark-monitor + ./build/mvn clean install -Pspark-3.1 -DskipTests -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12 ./build/mvn test -Pspark-3.1 -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds -Dmaven.plugin.scalatest.exclude.tags='' minikube-it: name: Minikube Integration Test diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 577b52af61d..af6f2f4ccc4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: profiles: - - '-Pspark-master -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper,:kyuubi-spark-monitor' + - '-Pspark-master -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12' env: SPARK_LOCAL_IP: localhost steps: diff --git a/build/dependency.sh b/build/dependency.sh index 9fce0522f9e..345bab2a60e 100755 --- a/build/dependency.sh +++ b/build/dependency.sh @@ -32,7 +32,7 @@ DEP="${PWD}"/dev/dependencyList function build_classpath() { - $MVN dependency:build-classpath -pl :kyuubi-assembly |\ + $MVN dependency:build-classpath -pl :kyuubi-assembly_2.12 |\ grep -v "INFO" | \ tr ":" "\n" | \ awk -F '/' '{ From a15477536ee62be270cba8e97714025022084c8f Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 19:30:48 +0800 Subject: [PATCH 4/8] Fix engine jar name --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 8d0d7a9f306..3adc8f0fa79 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -65,7 +65,7 @@ class SparkProcessBuilder( override def mainResource: Option[String] = { // 1. get the main resource jar for user specified config first - val jarName = s"$module-$KYUUBI_VERSION.jar" + val jarName = s"$module-_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" conf.get(ENGINE_SPARK_MAIN_RESOURCE).filter { userSpecified => // skip check exist if not local file. val uri = new URI(userSpecified) From 5178847f75f8a560b2032e8c71c3ea721ada4fb6 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 19:57:05 +0800 Subject: [PATCH 5/8] Fixup --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 3adc8f0fa79..af5b49458fe 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -65,7 +65,7 @@ class SparkProcessBuilder( override def mainResource: Option[String] = { // 1. get the main resource jar for user specified config first - val jarName = s"$module-_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + val jarName = s"${module}_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" conf.get(ENGINE_SPARK_MAIN_RESOURCE).filter { userSpecified => // skip check exist if not local file. val uri = new URI(userSpecified) From 6090c798c5c11e2cff129d574d4b7818cd2576dd Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 20:14:08 +0800 Subject: [PATCH 6/8] Fix shade --- externals/kyuubi-spark-sql-engine/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml index 931031c19f1..70390489856 100644 --- a/externals/kyuubi-spark-sql-engine/pom.xml +++ b/externals/kyuubi-spark-sql-engine/pom.xml @@ -182,9 +182,9 @@ false - org.apache.kyuubi:kyuubi-common - org.apache.kyuubi:kyuubi-ha - org.apache.kyuubi:kyuubi-spark-monitor + org.apache.kyuubi:kyuubi-common_2.12 + org.apache.kyuubi:kyuubi-ha_2.12 + org.apache.kyuubi:kyuubi-spark-monitor_2.12 org.apache.curator:curator-client org.apache.curator:curator-framework org.apache.curator:curator-recipes From 16cb263a5fadb3f7674b30a165db2f5f1675c155 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 20:31:55 +0800 Subject: [PATCH 7/8] remove hardcode scala version in pom --- dev/kyuubi-codecov/pom.xml | 14 +++++++------- externals/kyuubi-spark-monitor/pom.xml | 2 +- externals/kyuubi-spark-sql-engine/pom.xml | 16 ++++++++-------- kubernetes/integration-tests/pom.xml | 4 ++-- kyuubi-assembly/pom.xml | 10 +++++----- kyuubi-ctl/pom.xml | 8 ++++---- kyuubi-ha/pom.xml | 6 +++--- kyuubi-hive-jdbc/pom.xml | 6 +++--- kyuubi-metrics/pom.xml | 4 ++-- kyuubi-server/pom.xml | 12 ++++++------ kyuubi-zookeeper/pom.xml | 4 ++-- tools/spark-block-cleaner/pom.xml | 4 ++-- 12 files changed, 45 insertions(+), 45 deletions(-) diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 4fd145709fe..a747bb2a415 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -34,43 +34,43 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ctl_2.12 + kyuubi-ctl_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-zookeeper_2.12 + kyuubi-zookeeper_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-metrics_2.12 + kyuubi-metrics_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-server_2.12 + kyuubi-server_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-spark-sql-engine_2.12 + kyuubi-spark-sql-engine_${scala.binary.version} ${project.version} diff --git a/externals/kyuubi-spark-monitor/pom.xml b/externals/kyuubi-spark-monitor/pom.xml index e0ea286e7df..ed58baa2e4e 100644 --- a/externals/kyuubi-spark-monitor/pom.xml +++ b/externals/kyuubi-spark-monitor/pom.xml @@ -34,7 +34,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml index 70390489856..d93ba30eaae 100644 --- a/externals/kyuubi-spark-sql-engine/pom.xml +++ b/externals/kyuubi-spark-sql-engine/pom.xml @@ -34,19 +34,19 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-spark-monitor_2.12 + kyuubi-spark-monitor_${scala.binary.version} ${project.version} @@ -58,7 +58,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test @@ -165,7 +165,7 @@ org.apache.kyuubi - kyuubi-zookeeper_2.12 + kyuubi-zookeeper_${scala.binary.version} ${project.version} test @@ -182,9 +182,9 @@ false - org.apache.kyuubi:kyuubi-common_2.12 - org.apache.kyuubi:kyuubi-ha_2.12 - org.apache.kyuubi:kyuubi-spark-monitor_2.12 + org.apache.kyuubi:kyuubi-common_${scala.binary.version} + org.apache.kyuubi:kyuubi-ha_${scala.binary.version} + org.apache.kyuubi:kyuubi-spark-monitor_${scala.binary.version} org.apache.curator:curator-client org.apache.curator:curator-framework org.apache.curator:curator-recipes diff --git a/kubernetes/integration-tests/pom.xml b/kubernetes/integration-tests/pom.xml index d568e93f4cc..ef8264c1a75 100644 --- a/kubernetes/integration-tests/pom.xml +++ b/kubernetes/integration-tests/pom.xml @@ -33,14 +33,14 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index a6d66ebebb0..90cde7d961b 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -35,31 +35,31 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-metrics_2.12 + kyuubi-metrics_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-server_2.12 + kyuubi-server_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ctl_2.12 + kyuubi-ctl_${scala.binary.version} ${project.version} diff --git a/kyuubi-ctl/pom.xml b/kyuubi-ctl/pom.xml index 42596aa4827..a0495bb1644 100644 --- a/kyuubi-ctl/pom.xml +++ b/kyuubi-ctl/pom.xml @@ -34,12 +34,12 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} ${project.version} @@ -85,7 +85,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test @@ -105,7 +105,7 @@ org.apache.kyuubi - kyuubi-zookeeper_2.12 + kyuubi-zookeeper_${scala.binary.version} ${project.version} test diff --git a/kyuubi-ha/pom.xml b/kyuubi-ha/pom.xml index 0f94fbf5748..6c34751337d 100644 --- a/kyuubi-ha/pom.xml +++ b/kyuubi-ha/pom.xml @@ -34,7 +34,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} @@ -55,7 +55,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test @@ -75,7 +75,7 @@ org.apache.kyuubi - kyuubi-zookeeper_2.12 + kyuubi-zookeeper_${scala.binary.version} ${project.version} test diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml index 872c7b570e4..d0061812b0b 100644 --- a/kyuubi-hive-jdbc/pom.xml +++ b/kyuubi-hive-jdbc/pom.xml @@ -67,20 +67,20 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar org.apache.kyuubi - kyuubi-spark-sql-engine_2.12 + kyuubi-spark-sql-engine_${scala.binary.version} ${project.version} test org.apache.kyuubi - kyuubi-spark-sql-engine_2.12 + kyuubi-spark-sql-engine_${scala.binary.version} ${project.version} test-jar diff --git a/kyuubi-metrics/pom.xml b/kyuubi-metrics/pom.xml index ac0500e2f96..894ea3824f3 100644 --- a/kyuubi-metrics/pom.xml +++ b/kyuubi-metrics/pom.xml @@ -33,7 +33,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} @@ -89,7 +89,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index 883def4e04c..6565cd4296e 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -33,25 +33,25 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-zookeeper_2.12 + kyuubi-zookeeper_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-metrics_2.12 + kyuubi-metrics_${scala.binary.version} ${project.version} @@ -68,7 +68,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test @@ -84,7 +84,7 @@ org.apache.kyuubi - kyuubi-spark-sql-engine_2.12 + kyuubi-spark-sql-engine_${scala.binary.version} ${project.version} test diff --git a/kyuubi-zookeeper/pom.xml b/kyuubi-zookeeper/pom.xml index 24d0fb91dc0..e0751c819f4 100644 --- a/kyuubi-zookeeper/pom.xml +++ b/kyuubi-zookeeper/pom.xml @@ -33,7 +33,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} @@ -51,7 +51,7 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index a3627bc84e4..04811ce51b1 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -33,13 +33,13 @@ org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} org.apache.kyuubi - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} ${project.version} test-jar test From 571eb9c31862429dc79763a2ffbd5df5aacc4523 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 3 Aug 2021 20:42:55 +0800 Subject: [PATCH 8/8] comments --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index af5b49458fe..47351d4e1a2 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -65,6 +65,7 @@ class SparkProcessBuilder( override def mainResource: Option[String] = { // 1. get the main resource jar for user specified config first + // TODO use SPARK_SCALA_VERSION instead of SCALA_COMPILE_VERSION val jarName = s"${module}_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" conf.get(ENGINE_SPARK_MAIN_RESOURCE).filter { userSpecified => // skip check exist if not local file.