From 3eb1fa9e48b4c533f05b3719486df198c6e29751 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 19 Mar 2022 21:06:08 +0800 Subject: [PATCH] [KYUUBI #2175] Improve CI with cancel & concurrency & paths filter ### _Why are the changes needed?_ Better ASF GHA resource usage, see more https://cwiki.apache.org/confluence/display/BUILDS/GitHub+Actions+status - cancel-in-progress enabled. w/ this, a previous build can be canceled if new commit(s) have arrived - style check w/o install. currently, the style check takes 8~10min to finish because an install step before it. now, it shall return within a minute. note that, this might 'fail' itself when we introduce a new module and others depend on it, but it's ok to verify by the log whether a PR ok to be merged. - dependency check runs only when pom.xml updates. this flow now runs only pom changes as it is the only way that may cause dependency changes. ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [x] Add screenshots for manual tests if appropriate **Canceled**: https://github.com/apache/incubator-kyuubi/runs/5604809742?check_suite_focus=true image - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #2175 from yaooqinn/gha. Closes #2175 33777bc5 [Kent Yao] Update .github/workflows/style.yml ec41ddd0 [Kent Yao] Update .github/workflows/style.yml 79db981f [Kent Yao] fix 4e1fb387 [Kent Yao] fix 733df9cb [Kent Yao] Improve CI with cancel & concurrency 494c3a84 [Kent Yao] Revert "Improve CI with cancel & concurrency" 57641ac4 [Kent Yao] Improve CI with cancel & concurrency 6dafe970 [Kent Yao] Improve CI with cancel & concurrency 40160ab9 [Kent Yao] Improve CI with cancel & concurrency 377285c2 [Kent Yao] Improve CI with cancel & concurrency dbe8999e [Kent Yao] Improve CI with cancel & concurrency 78a97260 [Kent Yao] Improve CI with cancel & concurrency 7e984ebd [Kent Yao] Improve CI with cancel & concurrency d2538210 [Kent Yao] Improve CI with cancel & concurrency 51a7c48c [Kent Yao] Improve CI with cancel & concurrency c5e3177a [Kent Yao] Improve CI with cancel & concurrency Authored-by: Kent Yao Signed-off-by: Kent Yao --- .github/workflows/dep.yml | 59 +++++++++++++++++++++++++++++++++++ .github/workflows/license.yml | 6 ++++ .github/workflows/master.yml | 7 +++-- .github/workflows/nightly.yml | 30 ++---------------- .github/workflows/style.yml | 33 ++++++++++---------- dev/kyuubi-codecov/pom.xml | 6 ++++ 6 files changed, 96 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/dep.yml diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml new file mode 100644 index 00000000000..ee3134a989f --- /dev/null +++ b/.github/workflows/dep.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Dependency + +on: + pull_request: + branches: + - master + - branch-* + paths: + # dependency check happens only pom changes + - '**/pom.xml' + +concurrency: + group: dep-${{ github.ref }} + cancel-in-progress: true + +jobs: + dep: + name: Dependency check + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: setup java + uses: actions/setup-java@v2 + with: + distribution: zulu + java-version: 8 + cache: 'maven' + check-latest: false + - name: build + env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error + run: >- + build/mvn clean install + -Pflink-provided,spark-provided + -Dmaven.javadoc.skip=true + -Drat.skip=true + -Dscalastyle.skip=true + -Dspotless.check.skip + -DskipTests + -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am + - name: Check dependency list + run: build/dependency.sh diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index fa70b317ea2..f2cf380c2ae 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -25,6 +25,10 @@ on: - master - branch-* +concurrency: + group: lincense-${{ github.ref }} + cancel-in-progress: true + jobs: rat: name: License @@ -36,6 +40,8 @@ jobs: with: distribution: zulu java-version: 8 + cache: 'maven' + check-latest: false - run: build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds -Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2 - name: Upload rat report if: failure() diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 38cae0b78e8..af2feee8d51 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -27,6 +27,10 @@ on: - master - branch-* +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + jobs: build: @@ -95,7 +99,6 @@ jobs: **/kyuubi-flink-sql-engine.log* **/kyuubi-spark-sql-engine.log* **/kyuubi-trino-sql-engine.log* - **/target/scalastyle-output.xml tpcds: name: TPC-DS Tests @@ -115,7 +118,7 @@ jobs: check-latest: false - name: Run TPC-DS Tests run: >- - ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V + ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -pl kyuubi-server -am -Dmaven.plugin.scalatest.exclude.tags='' -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f52805a7e4a..bd2c23fbf1b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,7 +15,7 @@ # limitations under the License. # -name: Kyuubi Nightly +name: Kyuubi Spark Nightly on: schedule: @@ -41,30 +41,8 @@ jobs: with: distribution: zulu java-version: 8 - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/com - key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-com- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/org - key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-org- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/net - key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-net- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/io - key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-io- + cache: 'maven' + check-latest: false - name: Build with Maven run: ./build/mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V - name: Upload test logs @@ -74,6 +52,4 @@ jobs: name: unit-tests-log path: | **/target/unit-tests.log - **/kyuubi-flink-sql-engine.log* **/kyuubi-spark-sql-engine.log* - **/kyuubi-trino-sql-engine.log* diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 8d995c8c14e..b32ad10c1d8 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -15,9 +15,7 @@ # limitations under the License. # -name: Style check - -# This GitHub workflow checks style & dependency issues. +name: Style on: pull_request: @@ -25,9 +23,18 @@ on: - master - branch-* +concurrency: + group: linter-${{ github.ref }} + cancel-in-progress: true + +# no install runs before style check because it is expensive, we can skip build if style fails +# Well, sometimes when we introduce a new module, it may 'fail' the style check for missing +# dependency we just create for other module to inherit. +# We can ignore this failure and merge the PR, if we see the style check pass on the new module +# or via a local quick verification. jobs: linter: - name: Style and Dependency check + name: Style check runs-on: ubuntu-20.04 strategy: matrix: @@ -40,18 +47,12 @@ jobs: with: distribution: zulu java-version: 8 - - name: Install - run: >- - build/mvn clean install -V -Pflink-provided,spark-provided -Dorg.slf4j.simpleLogger.defaultLogLevel=warn \ - -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests \ - -Pflink-provided,spark-provided \ - -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - - name: Scalastyle with Maven - run: build/mvn scalastyle:check ${{ matrix.profiles }} + cache: 'maven' + check-latest: false + - name: Scalastyle with maven + run: build/mvn scalastyle:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12' - name: Upload scalastyle report if: failure() run: for log in `find * -name "scalastyle-output.xml"`; do echo "=========$log========="; grep "error" $log; done - - name: JavaStyle with Maven - run: build/mvn spotless:check ${{ matrix.profiles }} - - name: Check dependency list - run: build/dependency.sh + - name: JavaStyle with maven + run: build/mvn spotless:check ${{ matrix.profiles }} -pl '!:kyuubi-codecov_2.12,!:kyuubi-assembly_2.12' diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index f844944b059..3672ab4a5d6 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -110,6 +110,12 @@ kyuubi-hive-beeline ${project.version} + + + org.apache.kyuubi + kyuubi-hive-sql-engine_${scala.binary.version} + ${project.version} +