Improve CI with cancel & concurrency & paths filter

.github/workflows/dep.yml

@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Dependency
+
+on:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths:
+      # dependency check happens only pom changes
+      - '**/pom.xml'
+
+concurrency:
+  group: dep-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dep:
+    name: Dependency check
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup java
+        uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: 8
+          cache: 'maven'
+          check-latest: false
+      - name: build
+        env:
+          MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
+        run: >-
+          build/mvn clean install
+          -Pflink-provided,spark-provided
+          -Dmaven.javadoc.skip=true
+          -Drat.skip=true
+          -Dscalastyle.skip=true
+          -Dspotless.check.skip
+          -DskipTests
+          -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+      - name: Check dependency list
+        run: build/dependency.sh

.github/workflows/license.yml

@@ -25,6 +25,10 @@ on:
       - master
       - branch-*
 
+concurrency:
+  group: lincense-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   rat:
     name: License
@@ -36,6 +40,8 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
+          cache: 'maven'
+          check-latest: false
       - run:  build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds -Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2
       - name: Upload rat report
         if: failure()

.github/workflows/master.yml

@@ -27,6 +27,10 @@ on:
       - master
       - branch-*
 
+concurrency:
+  group: test-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   build:
@@ -95,7 +99,6 @@ jobs:
             **/kyuubi-flink-sql-engine.log*
             **/kyuubi-spark-sql-engine.log*
             **/kyuubi-trino-sql-engine.log*
-            **/target/scalastyle-output.xml
 
   tpcds:
     name: TPC-DS Tests
@@ -115,7 +118,7 @@ jobs:
           check-latest: false
       - name: Run TPC-DS Tests
         run: >-
-          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
+          ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn
           -pl kyuubi-server -am
           -Dmaven.plugin.scalatest.exclude.tags=''
           -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds

.github/workflows/nightly.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-name: Kyuubi Nightly
+name: Kyuubi Spark Nightly
 
 on:
   schedule:
@@ -41,30 +41,8 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-      - uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository/com
-          key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-com-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository/org
-          key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-org-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository/net
-          key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-net-
-      - uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository/io
-          key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-io-
+          cache: 'maven'
+          check-latest: false
       - name: Build with Maven
         run: ./build/mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
       - name: Upload test logs
@@ -74,6 +52,4 @@ jobs:
           name: unit-tests-log
           path: |
             **/target/unit-tests.log
-            **/kyuubi-flink-sql-engine.log*
             **/kyuubi-spark-sql-engine.log*
-            **/kyuubi-trino-sql-engine.log*

.github/workflows/style.yml

@@ -15,19 +15,26 @@
 # limitations under the License.
 #
 
-name: Style check
-
-# This GitHub workflow checks style & dependency issues.
+name: Style
 
 on:
   pull_request:
     branches:
       - master
       - branch-*
 
+concurrency:
+  group: linter-${{ github.ref }}
+  cancel-in-progress: true
+
+# no install runs before style check because it is expensive, we can skip build if style fails
+# Well, sometimes when we introduce a new module, it may 'fail' the style check for missing
+# dependency we just create for other module to inherit.
+# We can ignore this failure and merge the PR, if we see the style check pass on the new module
+# or via a local quick verification.
 jobs:
   linter:
-    name: Style and Dependency check
+    name: Style check
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -40,18 +47,12 @@ jobs:
         with:
           distribution: zulu
           java-version: 8
-      - name: Install
-        run: >-
-          build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
-              -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \
-              -Pflink-provided,spark-provided \
-              -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
-      - name: Scalastyle with Maven
-        run: build/mvn scalastyle:check ${{ matrix.profiles }}
+          cache: 'maven'
+          check-latest: false
+      - name: Scalastyle with maven
+        run: build/mvn scalastyle:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'
       - name: Upload scalastyle report
         if: failure()
         run: for log in `find * -name "scalastyle-output.xml"`;  do echo "=========$log========="; grep "error" $log; done
-      - name: JavaStyle with Maven
-        run: build/mvn spotless:check ${{ matrix.profiles }}
-      - name: Check dependency list
-        run: build/dependency.sh
+      - name: JavaStyle with maven
+        run: build/mvn spotless:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'

dev/kyuubi-codecov/pom.xml

@@ -110,6 +110,12 @@
             <artifactId>kyuubi-hive-beeline</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-hive-sql-engine_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
     <build>