From 5836db6004de001c15341be0a7a36bd7cc5209aa Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 22 Feb 2022 13:39:54 +0800 Subject: [PATCH] [KYUUBI #1947] Revamp Github Action settings for CI ### _Why are the changes needed?_ 1. Add missing asf header 2. Disable fail-fast to let jobs run individually 3. Group defaults maven builds with java & spark version matrix - so this PR brings fully tests with JDK 11 back 4. Group others(now cross-version check only) in `include` 5. replace `actions/cachev2` with `actions/setup-javav2` - `cache: 'maven'` ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Need to pass all CI jobs Closes #1947 from yaooqinn/ga. Closes #1947 7d5d2472 [Kent Yao] address comment d8d347b0 [Kent Yao] address comment 8c3cba99 [Kent Yao] address comment 7400cdc6 [Kent Yao] checkout 20eeb7bb [Kent Yao] test b1c70e42 [Kent Yao] test 4ca4f233 [Kent Yao] test b588cd32 [Kent Yao] test da4f6aea [Kent Yao] test 057790d5 [Kent Yao] test 01b07b25 [Kent Yao] test 2906ec66 [Kent Yao] test 4d97bf19 [Kent Yao] test cf16f169 [Kent Yao] test 8c12b8ef [Kent Yao] test 3b84e450 [Kent Yao] test 6e21d97a [Kent Yao] test dc2d9004 [Kent Yao] test 1cff392d [Kent Yao] test a4b47476 [Kent Yao] test c4718022 [Kent Yao] test d264d33d [Kent Yao] test ee293c13 [Kent Yao] test 903cb0bb [Kent Yao] test f74fa1a5 [Kent Yao] test Authored-by: Kent Yao Signed-off-by: Kent Yao --- .github/workflows/docker-image.yml | 19 +++- .github/workflows/greetings.yml | 17 ++++ .github/workflows/license.yml | 7 +- .github/workflows/master.yml | 131 ++++++++++--------------- .github/workflows/nightly.yml | 19 +++- .github/workflows/publish-snapshot.yml | 19 +++- .github/workflows/style.yml | 7 +- 7 files changed, 131 insertions(+), 88 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index ffa8fca252b..683a886ddcf 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Publish Docker image on: @@ -16,7 +33,7 @@ jobs: cancel-in-progress: true steps: - name: Checkout - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2 - name: Login to Docker Hub uses: docker/login-action@v1 with: diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 9372b6f05eb..7f4300253a3 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Greetings # on: [pull_request, issues] diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 23022829062..fa70b317ea2 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -1,5 +1,3 @@ -name: License check - # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -17,8 +15,9 @@ name: License check # limitations under the License. # -# This GitHub workflow checks Apache License v2.0 Header is added if needed +name: License check +# This GitHub workflow checks Apache License v2.0 Header is added if needed on: pull_request: @@ -31,7 +30,7 @@ jobs: name: License runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2 - name: Setup JDK 8 uses: actions/setup-java@v2 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 277df1da45b..62e890fda26 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1,4 +1,21 @@ -name: Kyuubi +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: CI on: push: @@ -13,39 +30,33 @@ on: jobs: build: - name: Build + name: Test runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: + java: + - 8 + - 11 + spark: + - '3.0' + - '3.1' + - '3.2' + spark-archive: [""] + exclude-tags: [""] include: - - java: 8 - spark: '3.0' - profiles: -Pspark-hadoop-2.7 - - java: 8 - spark: '3.1' - - java: 8 - spark: '3.2' - profiles: >- - -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 - -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz - -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest - java: 8 spark: '3.2' - profiles: >- - -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.2 - -Dspark.archive.name=spark-3.1.2-bin-hadoop3.2.tgz - -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.0.3 -Dspark.archive.name=spark-3.0.3-bin-hadoop2.7.tgz' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' - java: 8 spark: '3.2' - codecov: 'true' - - java: 11 - spark: '3.2' - profiles: -DskipTests -Pflink-provided,spark-provided - + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.2 -Dspark.archive.name=spark-3.1.2-bin-hadoop3.2.tgz' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK ${{ matrix.java }} @@ -53,36 +64,24 @@ jobs: with: distribution: zulu java-version: ${{ matrix.java }} - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/com - key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-com- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/org - key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-org- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/net - key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-net- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/io - key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-io- - - name: Build with Maven + cache: 'maven' + check-latest: false + - name: Build and test with maven w/o linters run: >- - ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V - -Pspark-${{ matrix.spark }} ${{ matrix.profiles }} + ./build/mvn clean install + -Dmaven.javadoc.skip=true + -Drat.skip=true + -Dscalastyle.skip=true + -Dspotless.check.skip + -Dorg.slf4j.simpleLogger.defaultLogLevel=warn + -Pspark-${{ matrix.spark }} + ${{ matrix.spark-archive }} + ${{ matrix.exclude-tags }} - name: Code coverage - if: ${{ matrix.codecov == 'true' }} + if: | + matrix.java == 8 && + matrix.spark == '3.2' && + matrix.spark-archive == '' uses: codecov/codecov-action@v2 with: verbose: true @@ -103,7 +102,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK 8 @@ -111,30 +110,8 @@ jobs: with: distribution: zulu java-version: 8 - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/com - key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-com- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/org - key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-org- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/net - key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-net- - - uses: actions/cache@v2 - with: - path: ~/.m2/repository/io - key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven-io- + cache: 'maven' + check-latest: false - name: Run TPC-DS Tests run: >- ./build/mvn clean install -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -V @@ -148,7 +125,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2 # https://github.com/docker/build-push-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fd7be3fdb89..8d181f34820 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Kyuubi Nightly on: @@ -16,7 +33,7 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Setup JDK 8 diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index fbf2fffb377..b0811e952de 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Publish Snapshot on: @@ -16,7 +33,7 @@ jobs: - branch-1.3 steps: - name: Checkout repository - uses: actions/checkout@v2.3.5 + uses: actions/checkout@v2 with: ref: ${{ matrix.branch }} - name: Cache Maven local repository diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index afb84b3a792..8d995c8c14e 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -1,5 +1,3 @@ -name: Style check - # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -17,8 +15,9 @@ name: Style check # limitations under the License. # -# This GitHub workflow checks style & dependency issues. +name: Style check +# This GitHub workflow checks style & dependency issues. on: pull_request: @@ -35,7 +34,7 @@ jobs: profiles: - '-Ptpcds -Pspark-block-cleaner -Pkubernetes-deployment-it -Pspark-3.1 -Pspark-3.2' steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2 - name: Setup JDK 8 uses: actions/setup-java@v2 with: