From 3687b2b732be22dc59f6ba08f90bce9e8b4ee45b Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Mon, 20 Mar 2023 15:31:28 -0700 Subject: [PATCH] Trying to setup node before Compatibility Regression Testing (#10440) --- .../workflows/pinot_compatibility_tests.yml | 12 ++++++- .github/workflows/pinot_tests.yml | 30 ++++++++++++------ .../scripts/.pinot_compatibility_verifier.sh | 15 +++++++++ compatibility-verifier/checkoutAndBuild.sh | 31 ++++++++++++++----- 4 files changed, 70 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pinot_compatibility_tests.yml b/.github/workflows/pinot_compatibility_tests.yml index 2b7425569b58..f02304dc858a 100644 --- a/.github/workflows/pinot_compatibility_tests.yml +++ b/.github/workflows/pinot_compatibility_tests.yml @@ -43,6 +43,16 @@ jobs: java-version: 11 distribution: 'adopt' cache: 'maven' + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: v10.16.1 + cache: 'npm' + cache-dependency-path: pinot-controller/src/main/resources/package-lock.json + - name: Install npm + run: | + npm install -g npm@6.10.0 + npm --version - name: Pinot Compatibility Regression Testing if : ${{github.event_name == 'workflow_dispatch'}} env: @@ -52,7 +62,7 @@ jobs: TEST_SUITE: ${{ matrix.test_suite }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml index 1c6dd883c804..97d917b1689a 100644 --- a/.github/workflows/pinot_tests.yml +++ b/.github/workflows/pinot_tests.yml @@ -60,7 +60,7 @@ jobs: env: MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED @@ -98,7 +98,7 @@ jobs: RUN_TEST_SET: ${{ matrix.testset }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED @@ -144,7 +144,7 @@ jobs: RUN_TEST_SET: ${{ matrix.testset }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED @@ -166,16 +166,28 @@ jobs: strategy: matrix: test_suite: [ "compatibility-verifier/sample-test-suite" ] - old_commit: [ "release-0.9.0", "master" ] + old_commit: [ + "release-0.9.3", "release-0.10.0", "release-0.11.0", "release-0.12.1", "master" + ] name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }} steps: - uses: actions/checkout@v2 - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: - java-version: 8 + java-version: 11 distribution: 'adopt' cache: 'maven' + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: v10.16.1 + cache: 'npm' + cache-dependency-path: pinot-controller/src/main/resources/package-lock.json + - name: Install npm + run: | + npm install -g npm@6.10.0 + npm --version # Step that does that actual cache save and restore - uses: actions/cache@v3 env: @@ -192,7 +204,7 @@ jobs: TEST_SUITE: ${{ matrix.test_suite }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED @@ -246,7 +258,7 @@ jobs: env: MAVEN_OPTS: > -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 - -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -XX:+IgnoreUnrecognizedVMOptions --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED @@ -263,6 +275,6 @@ jobs: cache: 'maven' - name: Build presto pinot driver with JDK 8 env: - MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false run: | mvn clean install -DskipTests -Ppresto-driver -am -B -pl ':presto-pinot-driver' -Djdk.version=8 -T 16 || exit 1 diff --git a/.github/workflows/scripts/.pinot_compatibility_verifier.sh b/.github/workflows/scripts/.pinot_compatibility_verifier.sh index 0897eacdd809..5a27b60b75dd 100755 --- a/.github/workflows/scripts/.pinot_compatibility_verifier.sh +++ b/.github/workflows/scripts/.pinot_compatibility_verifier.sh @@ -41,6 +41,21 @@ echo " https://packages.confluent.io/maven/">> ${SETTINGS_FILE} echo " false">> ${SETTINGS_FILE} echo " ">> ${SETTINGS_FILE} echo " ">> ${SETTINGS_FILE} + +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " central">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " 120000">> ${SETTINGS_FILE} +echo " 120000">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} +echo " ">> ${SETTINGS_FILE} + echo "">> ${SETTINGS_FILE} # PINOT_MAVEN_OPTS is used to provide additional maven options to the checkoutAndBuild.sh command diff --git a/compatibility-verifier/checkoutAndBuild.sh b/compatibility-verifier/checkoutAndBuild.sh index 702e01b1f4aa..df11b5469c04 100755 --- a/compatibility-verifier/checkoutAndBuild.sh +++ b/compatibility-verifier/checkoutAndBuild.sh @@ -73,7 +73,8 @@ function build() { local buildTests=$2 local buildId=$3 local repoOption="" - local versionOption="-Djdk.version=8" + local versionOption="-Djdk.version=11" + local maxRetry=5 mkdir -p ${MVN_CACHE_DIR} @@ -85,14 +86,28 @@ function build() { mvn versions:commit -q -B 1>${outFile} 2>&1 repoOption="-Dmaven.repo.local=${mvnCache}/${buildId}" fi - - mvn install package -DskipTests -Pbin-dist ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1 - if [ $? -ne 0 ]; then exit 1; fi - mvn -pl pinot-tools package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1 - if [ $? -ne 0 ]; then exit 1; fi + for i in $(seq 1 $maxRetry); do + mvn clean install -DskipTests -Pbin-dist ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1 + if [ $? -eq 0 ]; then break; fi + if [ $i -eq $maxRetry ]; then exit 1; fi + echo "Build failed, retrying after 30 seconds" + sleep 30 + done + for i in $(seq 1 $maxRetry); do + mvn -pl pinot-tools package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1 + if [ $? -eq 0 ]; then break; fi + if [ $i -eq $maxRetry ]; then exit 1; fi + echo "Build failed, retrying after 30 seconds" + sleep 30 + done if [ $buildTests -eq 1 ]; then - mvn -pl pinot-integration-tests package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1 - if [ $? -ne 0 ]; then exit 1; fi + for i in $(seq 1 $maxRetry); do + mvn -pl pinot-integration-tests package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1 + if [ $? -eq 0 ]; then break; fi + if [ $i -eq $maxRetry ]; then exit 1; fi + echo "Build failed, retrying after 30 seconds" + sleep 30 + done fi }