[KYUUBI #1257] [BUILD] Reduce unnecessary maven profile

### _Why are the changes needed?_  The changes based on #1226 (comment) In this PR, I'm going to remove profiles `kyuubi-extension-spark-3-1` and `kyuubi-extension-spark-3-2`, and keep `spark-3.0`, `spark-3.1`, `spark-3.2`. After changes, when `spark-3.1` is active, set `spark.version` to 3.1.2 and enable `kyuubi-extension-spark-common` module and `kyuubi-extension-spark-3-1` module, `spark-3.2` does the same thing ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1257 from pan3793/pom. Closes #1257 1d871b6 [Cheng Pan] nit 874f30c [Cheng Pan] Address comments 00a32b2 [Cheng Pan] nit 445666e [Cheng Pan] typo bd96097 [Cheng Pan] nit e85116c [Cheng Pan] nit 98347c8 [Cheng Pan] Update doc 6615d7c [Cheng Pan] Exclude HudiTest in cross test 087d3fd [Cheng Pan] Add spark 3.2 binary test ccff63c [Cheng Pan] update GA 23b81dc [Cheng Pan] Address comments 3055c2a [Cheng Pan] nit 49a41e8 [Cheng Pan] nit 8d40079 [Cheng Pan] Reduce unnecessary maven profile ced8d98 [Cheng Pan] Reduce unnecessary maven profile Lead-authored-by: Cheng Pan <[email protected]> Co-authored-by: Cheng Pan <[email protected]> Signed-off-by: ulysses-you <[email protected]>
apache · Oct 22, 2021 · 589b23f · 589b23f
1 parent f5b792a
commit 589b23f
Show file tree

Hide file tree

Showing 15 changed files with 63 additions and 79 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -39,16 +39,18 @@ jobs:
           - java: 8
             spark: '3.1'
             spark-hadoop: '2.7'
-            profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
+            profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
+          - java: 8
+            spark: '3.1'
+            spark-hadoop: '3.2'
+            profiles: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.0 -Dspark.archive.name=spark-3.2.0-bin-hadoop3.2.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
           - java: 8
             spark: '3.1'
             spark-hadoop: '3.2'
-            profiles: '-Pkyuubi-extension-spark-3-1'
             codecov: 'true'
           - java: 8
             spark: '3.2'
             spark-hadoop: '3.2'
-            profiles: '-Pkyuubi-extension-spark-3-2'
           - java: 11
             spark: '3.1'
             spark-hadoop: '3.2'

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         profiles:
-          - '-Pspark-master -pl :kyuubi-spark-sql-engine_2.12,:kyuubi-common_2.12,:kyuubi-ha_2.12,:kyuubi-zookeeper_2.12,:kyuubi-spark-monitor_2.12'
+          - '-Pspark-master -pl externals/kyuubi-spark-sql-engine -am'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
@@ -36,4 +36,6 @@ jobs:
         ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
       run: |
         rm -rf ~/.m2/repository/org/apache/kyuubi
-        ./build/mvn clean deploy -DskipTests -Pkyuubi-extension-spark-3-1,spark-provided -s ./build/release/asf-settings.xml
+        ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided
+        ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.1 -pl dev/kyuubi-extension-spark-3-1 -am
+        ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests -Pspark-provided,spark-3.2 -pl dev/kyuubi-extension-spark-3-2 -am
diff --git a/build/release/create-package.sh b/build/release/create-package.sh
@@ -75,7 +75,7 @@ package_binary() {
 
   echo "Creating binary release tarball ${BIN_TGZ_FILE}"
 
-  ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pkyuubi-extension-spark-3-1
+  ${KYUUBI_DIR}/build/dist --tgz --spark-provided -Pspark-3.1
 
   cp "${BIN_TGZ_FILE}" "${RELEASE_DIR}"
 

diff --git a/build/release/release.sh b/build/release/release.sh
@@ -96,6 +96,12 @@ upload_svn_staging() {
 upload_nexus_staging() {
   ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided \
     -s "${KYUUBI_DIR}/build/release/asf-settings.xml"
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.1 \
+    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
+    -pl dev/kyuubi-extension-spark-3-1 -am
+  ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,spark-provided,spark-3.2 \
+    -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \
+    -pl dev/kyuubi-extension-spark-3-2 -am
 }
 
 finalize_svn() {

diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml
@@ -111,6 +111,14 @@
                     <skip>true</skip>
                 </configuration>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/dev/kyuubi-extension-spark-3-1/pom.xml b/dev/kyuubi-extension-spark-3-1/pom.xml
@@ -155,13 +155,4 @@
             </plugin>
         </plugins>
     </build>
-
-    <profiles>
-        <profile>
-            <id>kyuubi-extension-spark-3-1</id>
-            <properties>
-                <spark.version>3.1.2</spark.version>
-            </properties>
-        </profile>
-    </profiles>
 </project>
diff --git a/dev/kyuubi-extension-spark-3-2/pom.xml b/dev/kyuubi-extension-spark-3-2/pom.xml
@@ -161,14 +161,4 @@
             </plugin>
         </plugins>
     </build>
-
-    <profiles>
-        <profile>
-            <id>kyuubi-extension-spark-3-2</id>
-            <properties>
-                <spark.version>3.2.0</spark.version>
-            </properties>
-        </profile>
-    </profiles>
-
 </project>
diff --git a/dev/kyuubi-extension-spark-common/pom.xml b/dev/kyuubi-extension-spark-common/pom.xml
@@ -135,13 +135,11 @@
             </plugin>
 
             <plugin>
-                <groupId>net.alchim31.maven</groupId>
-                <artifactId>scala-maven-plugin</artifactId>
-            </plugin>
-
-            <plugin>
-                <groupId>org.scalatest</groupId>
-                <artifactId>scalatest-maven-plugin</artifactId>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
             </plugin>
 
             <plugin>
@@ -159,24 +157,12 @@
             </plugin>
 
             <plugin>
-                <groupId>org.scalastyle</groupId>
-                <artifactId>scalastyle-maven-plugin</artifactId>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
             </plugin>
         </plugins>
     </build>
-
-    <profiles>
-        <profile>
-            <id>kyuubi-extension-spark-3-1</id>
-            <properties>
-                <spark.version>3.1.2</spark.version>
-            </properties>
-        </profile>
-        <profile>
-            <id>kyuubi-extension-spark-3-2</id>
-            <properties>
-                <spark.version>3.2.0</spark.version>
-            </properties>
-        </profile>
-    </profiles>
 </project>
diff --git a/docs/community/release.md b/docs/community/release.md
@@ -159,7 +159,7 @@ export RELEASE_RC_NO=<RC number, e.g. 0>
 ```shell
 build/mvn versions:set -DgenerateBackupPoms=false \
   -DnewVersion="${RELEASE_VERSION}" \
-  -Pkubernetes,kyuubi-extension-spark-3-1,spark-block-cleaner,tpcds
+  -Pspark-3.1,spark-block-cleaner
 
 git commit -am "[RELEASE] Bump ${RELEASE_VERSION}"
 ```

diff --git a/docs/sql/rules.md b/docs/sql/rules.md
@@ -15,8 +15,6 @@
  - limitations under the License.
  -->
 
-<!-- DO NOT MODIFY THIS FILE DIRECTLY, IT IS AUTO GENERATED BY [org.apache.kyuubi.engine.spark.udf.KyuubiUDFRegistrySuite] -->
-
 <div align=center>
 
 ![](../imgs/kyuubi_logo.png)
@@ -53,10 +51,18 @@ And don't worry, Kyuubi will support the new Apache Spark version in future. Tha
 
 ## How to use Kyuubi SQL extension
 
-1. you need to choose Apache Spark branch-3.1 or higher version with Kyuubi binary tgz.
-2. if you want to compile Kyuubi by yourself, the maven opt should add `-Pkyuubi-extension-spark-3-1`
-3. move the jar(`kyuubi-extension-spark-*.jar`) which is in `$KYUUBI_HOME/extension` into `$SPARK_HOME/jars`
-4. add a config into `spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension`
+| Kyuubi Spark SQL extension | Supported Spark version(s) | Available since  | EOL              | Bundled in Binary release tarball | Maven profile
+| -------------------------- | -------------------------- | ---------------- | ---------------- | --------------------------------- | -------------
+| kyuubi-extension-spark-3-1 | 3.1.x                      | 1.3.0-incubating | N/A              | 1.3.0-incubating                  | spark-3.1
+| kyuubi-extension-spark-3-2 | 3.2.x                      | 1.4.0-incubating | N/A              | 1.4.0-incubating                  | spark-3.2
+
+1. Check the matrix that if you are using the supported Spark version, and find the corresponding Kyuubi Spark SQL Extension jar
+2. Get the Kyuubi Spark SQL Extension jar
+   1. Each Kyuubi binary release tarball only contains one default version of Kyuubi Spark SQL Extension jar, if you are looking for such version, you can find it under `$KYUUBI_HOME/extension`
+   2. All supported versions of Kyuubi Spark SQL Extension jar will be deployed to [Maven Central](https://search.maven.org/search?q=kyuubi-extension-spark)
+   3. If you like, you can compile Kyuubi Spark SQL Extension jar by yourself, please activate the corresponding Maven's profile on you compile command, i.e. you can get Kyuubi Spark SQL Extension jar for Spark 3.1 under `dev/kyuubi-extension-spark-3-1/target` when compile with `-Pspark-3.1`
+3. Put the Kyuubi Spark SQL extension jar `kyuubi-extension-spark-*.jar` into `$SPARK_HOME/jars`
+4. Enable `KyuubiSparkSQLExtension`, i.e. add a config into `$SPARK_HOME/conf/spark-defaults.conf`, `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension`
 
 Now, you can enjoy the Kyuubi SQL Extension, and also Kyuubi provides some configs to make these feature easy to use.
 

diff --git a/docs/sql/z-order-benchmark.md b/docs/sql/z-order-benchmark.md
@@ -15,8 +15,6 @@
  - limitations under the License.
  -->
 
-<!-- DO NOT MODIFY THIS FILE DIRECTLY, IT IS AUTO GENERATED BY [org.apache.kyuubi.engine.spark.udf.KyuubiUDFRegistrySuite] -->
-
 <div align=center>
 
 ![](../imgs/kyuubi_logo.png)

diff --git a/externals/kyuubi-download/pom.xml b/externals/kyuubi-download/pom.xml
@@ -57,6 +57,14 @@
                     </execution>
                 </executions>
             </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/pom.xml b/pom.xml
@@ -1775,6 +1775,10 @@
                 <delta.version>1.0.0</delta.version>
                 <maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
             </properties>
+            <modules>
+                <module>dev/kyuubi-extension-spark-common</module>
+                <module>dev/kyuubi-extension-spark-3-1</module>
+            </modules>
         </profile>
 
         <profile>
@@ -1783,6 +1787,10 @@
                 <spark.version>3.2.0</spark.version>
                 <maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
             </properties>
+            <modules>
+                <module>dev/kyuubi-extension-spark-common</module>
+                <module>dev/kyuubi-extension-spark-3-2</module>
+            </modules>
         </profile>
 
         <profile>
@@ -1826,22 +1834,6 @@
             </properties>
         </profile>
 
-        <profile>
-            <id>kyuubi-extension-spark-3-1</id>
-            <modules>
-                <module>dev/kyuubi-extension-spark-common</module>
-                <module>dev/kyuubi-extension-spark-3-1</module>
-            </modules>
-        </profile>
-
-        <profile>
-            <id>kyuubi-extension-spark-3-2</id>
-            <modules>
-                <module>dev/kyuubi-extension-spark-common</module>
-                <module>dev/kyuubi-extension-spark-3-2</module>
-            </modules>
-        </profile>
-
         <profile>
             <id>spark-block-cleaner</id>
             <modules>

diff --git a/tools/spark-block-cleaner/pom.xml b/tools/spark-block-cleaner/pom.xml
@@ -55,10 +55,5 @@
     <build>
         <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
         <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-        <resources>
-            <resource>
-                <directory>${project.basedir}/src/main/resources</directory>
-            </resource>
-        </resources>
     </build>
 </project>