From ced8d987e93bdf1d1bf23fc5f39227240686d2c0 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 19 Oct 2021 14:58:47 +0800 Subject: [PATCH 01/15] Reduce unnecessary maven profile --- dev/kyuubi-extension-spark-3-1/pom.xml | 9 ------- dev/kyuubi-extension-spark-3-2/pom.xml | 10 ------- dev/kyuubi-extension-spark-common/pom.xml | 30 --------------------- pom.xml | 32 +++++++---------------- tools/spark-block-cleaner/pom.xml | 5 ---- 5 files changed, 9 insertions(+), 77 deletions(-) diff --git a/dev/kyuubi-extension-spark-3-1/pom.xml b/dev/kyuubi-extension-spark-3-1/pom.xml index 1a7d517bf91..f43e933333d 100644 --- a/dev/kyuubi-extension-spark-3-1/pom.xml +++ b/dev/kyuubi-extension-spark-3-1/pom.xml @@ -155,13 +155,4 @@ - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - diff --git a/dev/kyuubi-extension-spark-3-2/pom.xml b/dev/kyuubi-extension-spark-3-2/pom.xml index 92d80648118..3b09706c04b 100644 --- a/dev/kyuubi-extension-spark-3-2/pom.xml +++ b/dev/kyuubi-extension-spark-3-2/pom.xml @@ -160,14 +160,4 @@ - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - - diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml index aa981e51d21..f0d9a18ebe9 100644 --- a/dev/kyuubi-extension-spark-common/pom.xml +++ b/dev/kyuubi-extension-spark-common/pom.xml @@ -134,16 +134,6 @@ - - net.alchim31.maven - scala-maven-plugin - - - - org.scalatest - scalatest-maven-plugin - - org.apache.maven.plugins maven-jar-plugin @@ -157,26 +147,6 @@ - - - org.scalastyle - scalastyle-maven-plugin - - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - diff --git a/pom.xml b/pom.xml index 88d8f1ce892..2c8c08ecbc2 100644 --- a/pom.xml +++ b/pom.xml @@ -1775,6 +1775,10 @@ 1.0.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-1 + @@ -1783,6 +1787,10 @@ 3.2.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-2 + @@ -1826,33 +1834,11 @@ - - kyuubi-extension-spark-3-1 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-1 - - - - - kyuubi-extension-spark-3-2 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-2 - - - - - spark-block-cleaner - - tools/spark-block-cleaner - - - kubernetes kubernetes/integration-tests + tools/spark-block-cleaner diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index e186693f4f2..e09d6aa9132 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -55,10 +55,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - - From 8d400798d1e21e7a1aeadfa8aec525f4b3b14f13 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 19 Oct 2021 16:28:24 +0800 Subject: [PATCH 02/15] Reduce unnecessary maven profile --- .github/workflows/master.yml | 4 ++-- .github/workflows/nightly.yml | 2 +- .github/workflows/publish-snapshot.yml | 2 +- build/release/create-package.sh | 2 +- docs/community/release.md | 2 +- docs/sql/rules.md | 11 +++++++---- pom.xml | 8 +++++++- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 983c4ab37bf..1ca73d4f97e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -43,12 +43,12 @@ jobs: - java: 8 spark: '3.1' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-1' + profiles: '-Pspark-3.1' codecov: 'true' - java: 8 spark: '3.2' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-2' + profiles: '-Pspark-3.2' - java: 11 spark: '3.1' spark-hadoop: '3.2' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index af6f2f4ccc4..2d1fd6abbbf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: profiles: - - '-Pspark-master -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12' + - '-Pspark-master -pl externals/kyuubi-spark-sql-engine,kyuubi-common,kyuubi-ha,kyuubi-zookeeper' env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 3855281a8d8..cdafcb4640f 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -36,4 +36,4 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -DskipTests -Pkyuubi-extension-spark-3-1,spark-provided -s ./build/release/asf-settings.xml + ./build/mvn clean deploy -DskipTests -Pspark-3.1,spark-provided -s ./build/release/asf-settings.xml diff --git a/build/release/create-package.sh b/build/release/create-package.sh index 9b0bb9f4566..f9cf52dafa5 100755 --- a/build/release/create-package.sh +++ b/build/release/create-package.sh @@ -75,7 +75,7 @@ package_binary() { echo "Creating binary release tarball ${BIN_TGZ_FILE}" - ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pkyuubi-extension-spark-3-1 + ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pspark-3.1 cp "${BIN_TGZ_FILE}" "${RELEASE_DIR}" diff --git a/docs/community/release.md b/docs/community/release.md index d2d34bde7c1..1a9afe7d151 100644 --- a/docs/community/release.md +++ b/docs/community/release.md @@ -159,7 +159,7 @@ export RELEASE_RC_NO= ```shell build/mvn versions:set -DgenerateBackupPoms=false \ -DnewVersion="${RELEASE_VERSION}" \ - -Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds + -Pspark-3.1,spark-block-cleaner git commit -am "[RELEASE] Bump ${RELEASE_VERSION}" ``` diff --git a/docs/sql/rules.md b/docs/sql/rules.md index a1d810bb0bf..d76fa4d5864 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -53,10 +53,13 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha ## How to use Kyuubi SQL extension -1. you need to choose Apache Spark branch-3.1 or higher version with Kyuubi binary tgz. -2. if you want to compile Kyuubi by yourself, the maven opt should add `-Pkyuubi-extension-spark-3-1` -3. move the jar(`kyuubi-extension-spark-*.jar`) which is in `$KYUUBI_HOME/extension` into `$SPARK_HOME/jars` -4. add a config into `spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` +1. Since v1.3.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-1-*.jar` for Apache Spark 3.1.x +2. Since v1.4.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-2-*.jar` for Apache Spark 3.2.x +3. You need to choose the supported Spark version with cooperated Kyuubi Spark SQL Extension Jar +5. The Kyuubi binary release tarball only contains one Kyuubi Spark SQL Extension jar which is Kyuubi the current default build Spark version, you can find all supported versions jars in Maven Central, or build by yourself +4. If you want to compile Kyuubi Spark SQL Extension by yourself, the supported Spark version profile will activate the cooperated Kyuubi Spark SQL Extension, i.e. you can get `kyuubi-extension-spark-3-1` when `-Pspark-3.1` is activated +6. Put the jar(`kyuubi-extension-spark-*.jar`) into `$SPARK_HOME/jars` +7. Add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` Now, you can enjoy the Kyuubi SQL Extension, and also Kyuubi provides some configs to make these feature easy to use. diff --git a/pom.xml b/pom.xml index 2c8c08ecbc2..d8ffe7f9198 100644 --- a/pom.xml +++ b/pom.xml @@ -1834,11 +1834,17 @@ + + spark-block-cleaner + + tools/spark-block-cleaner + + + kubernetes kubernetes/integration-tests - tools/spark-block-cleaner From 49a41e86bf6cd8f3f076b700484f7397537854c8 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 19 Oct 2021 16:31:36 +0800 Subject: [PATCH 03/15] nit --- .github/workflows/master.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 1ca73d4f97e..07623eb0558 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -43,12 +43,10 @@ jobs: - java: 8 spark: '3.1' spark-hadoop: '3.2' - profiles: '-Pspark-3.1' codecov: 'true' - java: 8 spark: '3.2' spark-hadoop: '3.2' - profiles: '-Pspark-3.2' - java: 11 spark: '3.1' spark-hadoop: '3.2' From 3055c2a83883545c3eb317ae7afd7e64b4b4c6e1 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 19 Oct 2021 16:38:15 +0800 Subject: [PATCH 04/15] nit --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2d1fd6abbbf..b5a9583bc01 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: profiles: - - '-Pspark-master -pl externals/kyuubi-spark-sql-engine,kyuubi-common,kyuubi-ha,kyuubi-zookeeper' + - '-Pspark-master -pl externals/kyuubi-spark-sql-engine -am' env: SPARK_LOCAL_IP: localhost steps: From 23b81dcd009f6f2eeba4d108d488911f246352d1 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 14:29:49 +0800 Subject: [PATCH 05/15] Address comments --- build/release/release.sh | 6 ++++++ dev/kyuubi-extension-spark-common/pom.xml | 8 ++++++++ docs/sql/rules.md | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/build/release/release.sh b/build/release/release.sh index 0cd69192970..99689bd2a37 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -96,6 +96,12 @@ upload_svn_staging() { upload_nexus_staging() { ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-3.1 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-1 -am + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-3.2 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-2 -am } finalize_svn() { diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml index f0d9a18ebe9..ed86ed679bf 100644 --- a/dev/kyuubi-extension-spark-common/pom.xml +++ b/dev/kyuubi-extension-spark-common/pom.xml @@ -134,6 +134,14 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + true + + + org.apache.maven.plugins maven-jar-plugin diff --git a/docs/sql/rules.md b/docs/sql/rules.md index d76fa4d5864..5ff09f61900 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -56,7 +56,7 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha 1. Since v1.3.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-1-*.jar` for Apache Spark 3.1.x 2. Since v1.4.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-2-*.jar` for Apache Spark 3.2.x 3. You need to choose the supported Spark version with cooperated Kyuubi Spark SQL Extension Jar -5. The Kyuubi binary release tarball only contains one Kyuubi Spark SQL Extension jar which is Kyuubi the current default build Spark version, you can find all supported versions jars in Maven Central, or build by yourself +5. The Kyuubi binary release tarball only contains one Kyuubi Spark SQL Extension jar which is Kyuubi the current default build Spark version, you can find all supported versions jars in [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark), or build by yourself 4. If you want to compile Kyuubi Spark SQL Extension by yourself, the supported Spark version profile will activate the cooperated Kyuubi Spark SQL Extension, i.e. you can get `kyuubi-extension-spark-3-1` when `-Pspark-3.1` is activated 6. Put the jar(`kyuubi-extension-spark-*.jar`) into `$SPARK_HOME/jars` 7. Add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` From ccff63c1804c46628b4ad881db3e1f33a642e373 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 14:40:13 +0800 Subject: [PATCH 06/15] update GA --- .github/workflows/publish-snapshot.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index cdafcb4640f..c871bf95b17 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -36,4 +36,6 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -DskipTests -Pspark-3.1,spark-provided -s ./build/release/asf-settings.xml + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-3.1 -s ./build/release/asf-settings.xml -pl dev/kyuubi-extension-spark-3-1 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-3.2 -s ./build/release/asf-settings.xml -pl dev/kyuubi-extension-spark-3-2 -am From 087d3fde281a7f12b359cc19c5efee7c0c7fa7a0 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 15:11:03 +0800 Subject: [PATCH 07/15] Add spark 3.2 binary test --- .github/workflows/master.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 07623eb0558..83cf824994c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -40,6 +40,10 @@ jobs: spark: '3.1' spark-hadoop: '2.7' profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' + - java: 8 + spark: '3.1' + spark-hadoop: '3.2' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.1' spark-hadoop: '3.2' From 6615d7cc273de15e877145fc5cae7b4e1a60ea7a Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 15:49:26 +0800 Subject: [PATCH 08/15] Exclude HudiTest in cross test --- .github/workflows/master.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 83cf824994c..86d13955a81 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -39,11 +39,11 @@ jobs: - java: 8 spark: '3.1' spark-hadoop: '2.7' - profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.1' spark-hadoop: '3.2' - profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.1' spark-hadoop: '3.2' From 98347c8c2e8aa3dcf64c97153ef1dce321427725 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 16:54:15 +0800 Subject: [PATCH 09/15] Update doc --- docs/sql/rules.md | 21 ++++++++++++--------- docs/sql/z-order-benchmark.md | 2 -- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/sql/rules.md b/docs/sql/rules.md index 5ff09f61900..c88475798a1 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) @@ -53,13 +51,18 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha ## How to use Kyuubi SQL extension -1. Since v1.3.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-1-*.jar` for Apache Spark 3.1.x -2. Since v1.4.0-incubating, Kyuubi provides `kyuubi-extension-spark-3-2-*.jar` for Apache Spark 3.2.x -3. You need to choose the supported Spark version with cooperated Kyuubi Spark SQL Extension Jar -5. The Kyuubi binary release tarball only contains one Kyuubi Spark SQL Extension jar which is Kyuubi the current default build Spark version, you can find all supported versions jars in [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark), or build by yourself -4. If you want to compile Kyuubi Spark SQL Extension by yourself, the supported Spark version profile will activate the cooperated Kyuubi Spark SQL Extension, i.e. you can get `kyuubi-extension-spark-3-1` when `-Pspark-3.1` is activated -6. Put the jar(`kyuubi-extension-spark-*.jar`) into `$SPARK_HOME/jars` -7. Add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` +| Kyuubi Spark SQL extension | Supported Spark version(s) | Available since | EOL | Bundled in Binary release tarball | Maven profile +| -------------------------- | -------------------------- | ---------------- | ---------------- | --------------------------------- | ------------- +| kyuubi-extension-spark-3-1 | 3.1.x | 1.3.0-incubating | N/A | 1.3.0-incubating | spark-3.1 +| kyuubi-extension-spark-3-2 | 3.2.x | 1.4.0-incubating | N/A | 1.4.0-incubating | spark-3.2 + +1. Check the matrix that if you are using the supported Spark version, and find the cooperated Kyuubi Spark SQL Extension Jar +2. Get the Kyuubi Spark SQL Extension Jar + 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` + 2. All supported versions of Kyuubi Spark SQL Extension will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) + 3. If you like, you can compile Kyuubi Spark SQL Extension by yourself, please activate the cooperated Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated +3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` +4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` Now, you can enjoy the Kyuubi SQL Extension, and also Kyuubi provides some configs to make these feature easy to use. diff --git a/docs/sql/z-order-benchmark.md b/docs/sql/z-order-benchmark.md index d60cb7c3999..5beb18058f0 100644 --- a/docs/sql/z-order-benchmark.md +++ b/docs/sql/z-order-benchmark.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) From e85116cdd150aa69c3ab2066c5abea98f4c4ac05 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 16:55:24 +0800 Subject: [PATCH 10/15] nit --- docs/sql/rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql/rules.md b/docs/sql/rules.md index c88475798a1..0bde265d1c9 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -56,8 +56,8 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha | kyuubi-extension-spark-3-1 | 3.1.x | 1.3.0-incubating | N/A | 1.3.0-incubating | spark-3.1 | kyuubi-extension-spark-3-2 | 3.2.x | 1.4.0-incubating | N/A | 1.4.0-incubating | spark-3.2 -1. Check the matrix that if you are using the supported Spark version, and find the cooperated Kyuubi Spark SQL Extension Jar -2. Get the Kyuubi Spark SQL Extension Jar +1. Check the matrix that if you are using the supported Spark version, and find the cooperated Kyuubi Spark SQL Extension jar +2. Get the Kyuubi Spark SQL Extension jar 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` 2. All supported versions of Kyuubi Spark SQL Extension will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) 3. If you like, you can compile Kyuubi Spark SQL Extension by yourself, please activate the cooperated Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated From bd96097e8f283ddefe396175d2e6a1b5c9d90629 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 16:55:53 +0800 Subject: [PATCH 11/15] nit --- docs/sql/rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql/rules.md b/docs/sql/rules.md index 0bde265d1c9..0efeeb53544 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -59,8 +59,8 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha 1. Check the matrix that if you are using the supported Spark version, and find the cooperated Kyuubi Spark SQL Extension jar 2. Get the Kyuubi Spark SQL Extension jar 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` - 2. All supported versions of Kyuubi Spark SQL Extension will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) - 3. If you like, you can compile Kyuubi Spark SQL Extension by yourself, please activate the cooperated Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated + 2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) + 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the cooperated Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated 3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` 4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` From 445666eec7d7d151fb23efb51d545ca971aa01ab Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 16:58:52 +0800 Subject: [PATCH 12/15] typo --- docs/sql/rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql/rules.md b/docs/sql/rules.md index 0efeeb53544..1609c53b3cb 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -56,11 +56,11 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha | kyuubi-extension-spark-3-1 | 3.1.x | 1.3.0-incubating | N/A | 1.3.0-incubating | spark-3.1 | kyuubi-extension-spark-3-2 | 3.2.x | 1.4.0-incubating | N/A | 1.4.0-incubating | spark-3.2 -1. Check the matrix that if you are using the supported Spark version, and find the cooperated Kyuubi Spark SQL Extension jar +1. Check the matrix that if you are using the supported Spark version, and find the corresponding Kyuubi Spark SQL Extension jar 2. Get the Kyuubi Spark SQL Extension jar 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` 2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) - 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the cooperated Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated + 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated 3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` 4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` From 00a32b25f2b527078077419ae1b1e35304d973be Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 20 Oct 2021 17:00:54 +0800 Subject: [PATCH 13/15] nit --- docs/sql/rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql/rules.md b/docs/sql/rules.md index 1609c53b3cb..0723ed4f256 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -60,7 +60,7 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha 2. Get the Kyuubi Spark SQL Extension jar 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` 2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) - 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when `-Pspark-3.1` is activated + 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when compile with `-Pspark-3.1` 3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` 4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` From 874f30c197bacc48162a8226ada8ba2018fed1e7 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 22 Oct 2021 11:32:11 +0800 Subject: [PATCH 14/15] Address comments --- .github/workflows/publish-snapshot.yml | 4 ++-- dev/kyuubi-codecov/pom.xml | 8 ++++++++ dev/kyuubi-extension-spark-common/pom.xml | 8 ++++++++ externals/kyuubi-download/pom.xml | 8 ++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index c871bf95b17..e1e4ff2dcde 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -37,5 +37,5 @@ jobs: run: | rm -rf ~/.m2/repository/org/apache/kyuubi ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-3.1 -s ./build/release/asf-settings.xml -pl dev/kyuubi-extension-spark-3-1 -am - ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-3.2 -s ./build/release/asf-settings.xml -pl dev/kyuubi-extension-spark-3-2 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 18ca2390ad3..3a3f0c0a9aa 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -111,6 +111,14 @@ true + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml index ed86ed679bf..b3fb584acdd 100644 --- a/dev/kyuubi-extension-spark-common/pom.xml +++ b/dev/kyuubi-extension-spark-common/pom.xml @@ -155,6 +155,14 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/externals/kyuubi-download/pom.xml b/externals/kyuubi-download/pom.xml index 7954567446c..a5613c531bd 100644 --- a/externals/kyuubi-download/pom.xml +++ b/externals/kyuubi-download/pom.xml @@ -57,6 +57,14 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + From 1d871b6cf515fa7930e48fd225122216fde56509 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 22 Oct 2021 12:50:38 +0800 Subject: [PATCH 15/15] nit --- build/release/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/release/release.sh b/build/release/release.sh index 99689bd2a37..00d7fa87e23 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -96,10 +96,10 @@ upload_svn_staging() { upload_nexus_staging() { ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-3.1 \ + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl dev/kyuubi-extension-spark-3-1 -am - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-3.2 \ + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl dev/kyuubi-extension-spark-3-2 -am }