diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml
new file mode 100644
index 00000000000..ee3134a989f
--- /dev/null
+++ b/.github/workflows/dep.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Dependency
+
+on:
+ pull_request:
+ branches:
+ - master
+ - branch-*
+ paths:
+ # dependency check happens only pom changes
+ - '**/pom.xml'
+
+concurrency:
+ group: dep-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ dep:
+ name: Dependency check
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v2
+ - name: setup java
+ uses: actions/setup-java@v2
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: 'maven'
+ check-latest: false
+ - name: build
+ env:
+ MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
+ run: >-
+ build/mvn clean install
+ -Pflink-provided,spark-provided
+ -Dmaven.javadoc.skip=true
+ -Drat.skip=true
+ -Dscalastyle.skip=true
+ -Dspotless.check.skip
+ -DskipTests
+ -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+ - name: Check dependency list
+ run: build/dependency.sh
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index fa70b317ea2..f2cf380c2ae 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -25,6 +25,10 @@ on:
- master
- branch-*
+concurrency:
+ group: lincense-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
rat:
name: License
@@ -36,6 +40,8 @@ jobs:
with:
distribution: zulu
java-version: 8
+ cache: 'maven'
+ check-latest: false
- run: build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds -Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2
- name: Upload rat report
if: failure()
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 38cae0b78e8..af2feee8d51 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -27,6 +27,10 @@ on:
- master
- branch-*
+concurrency:
+ group: test-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
build:
@@ -95,7 +99,6 @@ jobs:
**/kyuubi-flink-sql-engine.log*
**/kyuubi-spark-sql-engine.log*
**/kyuubi-trino-sql-engine.log*
- **/target/scalastyle-output.xml
tpcds:
name: TPC-DS Tests
@@ -115,7 +118,7 @@ jobs:
check-latest: false
- name: Run TPC-DS Tests
run: >-
- ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V
+ ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn
-pl kyuubi-server -am
-Dmaven.plugin.scalatest.exclude.tags=''
-Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index f52805a7e4a..bd2c23fbf1b 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: Kyuubi Nightly
+name: Kyuubi Spark Nightly
on:
schedule:
@@ -41,30 +41,8 @@ jobs:
with:
distribution: zulu
java-version: 8
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/com
- key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-com-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/org
- key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-org-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/net
- key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-net-
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository/io
- key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-io-
+ cache: 'maven'
+ check-latest: false
- name: Build with Maven
run: ./build/mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
- name: Upload test logs
@@ -74,6 +52,4 @@ jobs:
name: unit-tests-log
path: |
**/target/unit-tests.log
- **/kyuubi-flink-sql-engine.log*
**/kyuubi-spark-sql-engine.log*
- **/kyuubi-trino-sql-engine.log*
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 8d995c8c14e..b32ad10c1d8 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -15,9 +15,7 @@
# limitations under the License.
#
-name: Style check
-
-# This GitHub workflow checks style & dependency issues.
+name: Style
on:
pull_request:
@@ -25,9 +23,18 @@ on:
- master
- branch-*
+concurrency:
+ group: linter-${{ github.ref }}
+ cancel-in-progress: true
+
+# no install runs before style check because it is expensive, we can skip build if style fails
+# Well, sometimes when we introduce a new module, it may 'fail' the style check for missing
+# dependency we just create for other module to inherit.
+# We can ignore this failure and merge the PR, if we see the style check pass on the new module
+# or via a local quick verification.
jobs:
linter:
- name: Style and Dependency check
+ name: Style check
runs-on: ubuntu-20.04
strategy:
matrix:
@@ -40,18 +47,12 @@ jobs:
with:
distribution: zulu
java-version: 8
- - name: Install
- run: >-
- build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \
- -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \
- -Pflink-provided,spark-provided \
- -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
- - name: Scalastyle with Maven
- run: build/mvn scalastyle:check ${{ matrix.profiles }}
+ cache: 'maven'
+ check-latest: false
+ - name: Scalastyle with maven
+ run: build/mvn scalastyle:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'
- name: Upload scalastyle report
if: failure()
run: for log in `find * -name "scalastyle-output.xml"`; do echo "=========$log========="; grep "error" $log; done
- - name: JavaStyle with Maven
- run: build/mvn spotless:check ${{ matrix.profiles }}
- - name: Check dependency list
- run: build/dependency.sh
+ - name: JavaStyle with maven
+ run: build/mvn spotless:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12'
diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml
index f844944b059..3672ab4a5d6 100644
--- a/dev/kyuubi-codecov/pom.xml
+++ b/dev/kyuubi-codecov/pom.xml
@@ -110,6 +110,12 @@
kyuubi-hive-beeline
${project.version}
+
+
+ org.apache.kyuubi
+ kyuubi-hive-sql-engine_${scala.binary.version}
+ ${project.version}
+