Trying to setup node before Compatibility Regression Testing (#10440)

apache · Mar 20, 2023 · 3687b2b · 3687b2b
1 parent cb1069f
commit 3687b2b
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 18 deletions.
diff --git a/.github/workflows/pinot_compatibility_tests.yml b/.github/workflows/pinot_compatibility_tests.yml
@@ -43,6 +43,16 @@ jobs:
           java-version: 11
           distribution: 'adopt'
           cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: v10.16.1
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g [email protected]
+          npm --version
       - name: Pinot Compatibility Regression Testing
         if : ${{github.event_name == 'workflow_dispatch'}}
         env:
@@ -52,7 +62,7 @@ jobs:
           TEST_SUITE: ${{ matrix.test_suite }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED

diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml
@@ -60,7 +60,7 @@ jobs:
         env:
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -98,7 +98,7 @@ jobs:
           RUN_TEST_SET: ${{ matrix.testset }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -144,7 +144,7 @@ jobs:
           RUN_TEST_SET: ${{ matrix.testset }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -166,16 +166,28 @@ jobs:
     strategy:
       matrix:
         test_suite: [ "compatibility-verifier/sample-test-suite" ]
-        old_commit: [ "release-0.9.0", "master" ]
+        old_commit: [
+          "release-0.9.3", "release-0.10.0", "release-0.11.0", "release-0.12.1", "master"
+        ]
     name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 8
+      - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
-          java-version: 8
+          java-version: 11
           distribution: 'adopt'
           cache: 'maven'
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: v10.16.1
+          cache: 'npm'
+          cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
+      - name: Install npm
+        run: |
+          npm install -g [email protected]
+          npm --version
       # Step that does that actual cache save and restore
       - uses: actions/cache@v3
         env:
@@ -192,7 +204,7 @@ jobs:
           TEST_SUITE: ${{ matrix.test_suite }}
           MAVEN_OPTS: >
             -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -246,7 +258,7 @@ jobs:
         env:
           MAVEN_OPTS: >
             -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
-            -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+            -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
             -XX:+IgnoreUnrecognizedVMOptions
             --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
@@ -263,6 +275,6 @@ jobs:
           cache: 'maven'
       - name: Build presto pinot driver with JDK 8
         env:
-          MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+          MAVEN_OPTS: -Xmx2G -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
         run: |
           mvn clean install -DskipTests -Ppresto-driver -am -B -pl ':presto-pinot-driver' -Djdk.version=8 -T 16 || exit 1
diff --git a/.github/workflows/scripts/.pinot_compatibility_verifier.sh b/.github/workflows/scripts/.pinot_compatibility_verifier.sh
@@ -41,6 +41,21 @@ echo "      <url>https://packages.confluent.io/maven/</url>">> ${SETTINGS_FILE}
 echo "      <blocked>false</blocked>">> ${SETTINGS_FILE}
 echo "    </mirror>">> ${SETTINGS_FILE}
 echo "  </mirrors>">> ${SETTINGS_FILE}
+
+echo "  <servers>">> ${SETTINGS_FILE}
+echo "    <server>">> ${SETTINGS_FILE}
+echo "      <id>central</id>">> ${SETTINGS_FILE}
+echo "      <configuration>">> ${SETTINGS_FILE}
+echo "        <httpConfiguration>">> ${SETTINGS_FILE}
+echo "          <all>">> ${SETTINGS_FILE}
+echo "            <connectionTimeout>120000</connectionTimeout>">> ${SETTINGS_FILE}
+echo "            <readTimeout>120000</readTimeout>">> ${SETTINGS_FILE}
+echo "          </all>">> ${SETTINGS_FILE}
+echo "        </httpConfiguration>">> ${SETTINGS_FILE}
+echo "      </configuration>">> ${SETTINGS_FILE}
+echo "    </server>">> ${SETTINGS_FILE}
+echo "  </servers>">> ${SETTINGS_FILE}
+
 echo "</settings>">> ${SETTINGS_FILE}
 
 # PINOT_MAVEN_OPTS is used to provide additional maven options to the checkoutAndBuild.sh command

diff --git a/compatibility-verifier/checkoutAndBuild.sh b/compatibility-verifier/checkoutAndBuild.sh
@@ -73,7 +73,8 @@ function build() {
   local buildTests=$2
   local buildId=$3
   local repoOption=""
-  local versionOption="-Djdk.version=8"
+  local versionOption="-Djdk.version=11"
+  local maxRetry=5
 
   mkdir -p ${MVN_CACHE_DIR}
 
@@ -85,14 +86,28 @@ function build() {
     mvn versions:commit -q -B 1>${outFile} 2>&1
     repoOption="-Dmaven.repo.local=${mvnCache}/${buildId}"
   fi
-
-  mvn install package -DskipTests -Pbin-dist ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
-  if [ $? -ne 0 ]; then exit 1; fi
-  mvn -pl pinot-tools package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
-  if [ $? -ne 0 ]; then exit 1; fi
+  for i in $(seq 1 $maxRetry); do
+    mvn clean install -DskipTests -Pbin-dist ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>${outFile} 2>&1
+    if [ $? -eq 0 ]; then break; fi
+    if [ $i -eq $maxRetry ]; then exit 1; fi
+    echo "Build failed, retrying after 30 seconds"
+    sleep 30
+  done
+  for i in $(seq 1 $maxRetry); do
+    mvn -pl pinot-tools package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
+    if [ $? -eq 0 ]; then break; fi
+    if [ $i -eq $maxRetry ]; then exit 1; fi
+    echo "Build failed, retrying after 30 seconds"
+    sleep 30
+  done
   if [ $buildTests -eq 1 ]; then
-    mvn -pl pinot-integration-tests package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
-    if [ $? -ne 0 ]; then exit 1; fi
+    for i in $(seq 1 $maxRetry); do
+      mvn -pl pinot-integration-tests package -DskipTests ${versionOption} ${repoOption} ${PINOT_MAVEN_OPTS} 1>>${outFile} 2>&1
+      if [ $? -eq 0 ]; then break; fi
+      if [ $i -eq $maxRetry ]; then exit 1; fi
+      echo "Build failed, retrying after 30 seconds"
+      sleep 30
+    done
   fi
 }