diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 983c4ab37bf..86d13955a81 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -39,16 +39,18 @@ jobs: - java: 8 spark: '3.1' spark-hadoop: '2.7' - profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' + - java: 8 + spark: '3.1' + spark-hadoop: '3.2' + profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.1' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-1' codecov: 'true' - java: 8 spark: '3.2' spark-hadoop: '3.2' - profiles: '-Pkyuubi-extension-spark-3-2' - java: 11 spark: '3.1' spark-hadoop: '3.2' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index af6f2f4ccc4..b5a9583bc01 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: profiles: - - '-Pspark-master -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12' + - '-Pspark-master -pl externals/kyuubi-spark-sql-engine -am' env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 3855281a8d8..e1e4ff2dcde 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -36,4 +36,6 @@ jobs: ASF_PASSWORD: ${{ secrets.NEXUS_PW }} run: | rm -rf ~/.m2/repository/org/apache/kyuubi - ./build/mvn clean deploy -DskipTests -Pkyuubi-extension-spark-3-1,spark-provided -s ./build/release/asf-settings.xml + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am + ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am diff --git a/build/release/create-package.sh b/build/release/create-package.sh index 9b0bb9f4566..f9cf52dafa5 100755 --- a/build/release/create-package.sh +++ b/build/release/create-package.sh @@ -75,7 +75,7 @@ package_binary() { echo "Creating binary release tarball ${BIN_TGZ_FILE}" - ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pkyuubi-extension-spark-3-1 + ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pspark-3.1 cp "${BIN_TGZ_FILE}" "${RELEASE_DIR}" diff --git a/build/release/release.sh b/build/release/release.sh index 0cd69192970..00d7fa87e23 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -96,6 +96,12 @@ upload_svn_staging() { upload_nexus_staging() { ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-1 -am + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl dev/kyuubi-extension-spark-3-2 -am } finalize_svn() { diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 18ca2390ad3..3a3f0c0a9aa 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -111,6 +111,14 @@ true + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/dev/kyuubi-extension-spark-3-1/pom.xml b/dev/kyuubi-extension-spark-3-1/pom.xml index 1a7d517bf91..f43e933333d 100644 --- a/dev/kyuubi-extension-spark-3-1/pom.xml +++ b/dev/kyuubi-extension-spark-3-1/pom.xml @@ -155,13 +155,4 @@ - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - diff --git a/dev/kyuubi-extension-spark-3-2/pom.xml b/dev/kyuubi-extension-spark-3-2/pom.xml index 92d80648118..3b09706c04b 100644 --- a/dev/kyuubi-extension-spark-3-2/pom.xml +++ b/dev/kyuubi-extension-spark-3-2/pom.xml @@ -160,14 +160,4 @@ - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - - diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml index aa981e51d21..b3fb584acdd 100644 --- a/dev/kyuubi-extension-spark-common/pom.xml +++ b/dev/kyuubi-extension-spark-common/pom.xml @@ -135,13 +135,11 @@ - net.alchim31.maven - scala-maven-plugin - - - - org.scalatest - scalatest-maven-plugin + org.apache.maven.plugins + maven-dependency-plugin + + true + @@ -159,24 +157,12 @@ - org.scalastyle - scalastyle-maven-plugin + org.apache.maven.plugins + maven-deploy-plugin + + true + - - - - kyuubi-extension-spark-3-1 - - 3.1.2 - - - - kyuubi-extension-spark-3-2 - - 3.2.0 - - - diff --git a/docs/community/release.md b/docs/community/release.md index d2d34bde7c1..1a9afe7d151 100644 --- a/docs/community/release.md +++ b/docs/community/release.md @@ -159,7 +159,7 @@ export RELEASE_RC_NO= ```shell build/mvn versions:set -DgenerateBackupPoms=false \ -DnewVersion="${RELEASE_VERSION}" \ - -Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds + -Pspark-3.1,spark-block-cleaner git commit -am "[RELEASE] Bump ${RELEASE_VERSION}" ``` diff --git a/docs/sql/rules.md b/docs/sql/rules.md index a1d810bb0bf..0723ed4f256 100644 --- a/docs/sql/rules.md +++ b/docs/sql/rules.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) @@ -53,10 +51,18 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha ## How to use Kyuubi SQL extension -1. you need to choose Apache Spark branch-3.1 or higher version with Kyuubi binary tgz. -2. if you want to compile Kyuubi by yourself, the maven opt should add `-Pkyuubi-extension-spark-3-1` -3. move the jar(`kyuubi-extension-spark-*.jar`) which is in `$KYUUBI_HOME/extension` into `$SPARK_HOME/jars` -4. add a config into `spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` +| Kyuubi Spark SQL extension | Supported Spark version(s) | Available since | EOL | Bundled in Binary release tarball | Maven profile +| -------------------------- | -------------------------- | ---------------- | ---------------- | --------------------------------- | ------------- +| kyuubi-extension-spark-3-1 | 3.1.x | 1.3.0-incubating | N/A | 1.3.0-incubating | spark-3.1 +| kyuubi-extension-spark-3-2 | 3.2.x | 1.4.0-incubating | N/A | 1.4.0-incubating | spark-3.2 + +1. Check the matrix that if you are using the supported Spark version, and find the corresponding Kyuubi Spark SQL Extension jar +2. Get the Kyuubi Spark SQL Extension jar + 1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension` + 2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark) + 3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when compile with `-Pspark-3.1` +3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars` +4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` Now, you can enjoy the Kyuubi SQL Extension, and also Kyuubi provides some configs to make these feature easy to use. diff --git a/docs/sql/z-order-benchmark.md b/docs/sql/z-order-benchmark.md index d60cb7c3999..5beb18058f0 100644 --- a/docs/sql/z-order-benchmark.md +++ b/docs/sql/z-order-benchmark.md @@ -15,8 +15,6 @@ - limitations under the License. --> - -
![](../imgs/kyuubi_logo.png) diff --git a/externals/kyuubi-download/pom.xml b/externals/kyuubi-download/pom.xml index 7954567446c..a5613c531bd 100644 --- a/externals/kyuubi-download/pom.xml +++ b/externals/kyuubi-download/pom.xml @@ -57,6 +57,14 @@ + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + diff --git a/pom.xml b/pom.xml index 88d8f1ce892..d8ffe7f9198 100644 --- a/pom.xml +++ b/pom.xml @@ -1775,6 +1775,10 @@ 1.0.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-1 + @@ -1783,6 +1787,10 @@ 3.2.0 org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest + + dev/kyuubi-extension-spark-common + dev/kyuubi-extension-spark-3-2 + @@ -1826,22 +1834,6 @@ - - kyuubi-extension-spark-3-1 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-1 - - - - - kyuubi-extension-spark-3-2 - - dev/kyuubi-extension-spark-common - dev/kyuubi-extension-spark-3-2 - - - spark-block-cleaner diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml index e186693f4f2..e09d6aa9132 100644 --- a/tools/spark-block-cleaner/pom.xml +++ b/tools/spark-block-cleaner/pom.xml @@ -55,10 +55,5 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes - - - ${project.basedir}/src/main/resources - -