ci: install and test only necessary modules (#10811)

* ci: install and test only necessary modules * ci: debugging mvn install * ci: remove quotes from INTEGRATION_TEST_ARGS * ci: remove debugging lines * ci: use mvn to determine submodules * ci: ensure IFS is maintained * ci: debug statements * ci: debug statements * ci: prevent persisted change to IFS * ci: remove debug statements * ci: perform graalvm tests in parallel
googleapis · May 9, 2024 · 139b277 · 139b277
1 parent 46fa9ba
commit 139b277
Show file tree

Hide file tree

Showing 9 changed files with 145 additions and 67 deletions.
diff --git a/.cloud/helpers/destroy.sh b/.cloud/helpers/destroy.sh
@@ -22,6 +22,7 @@ pushd "$helperDir/.." >/dev/null
 source ./helpers/common.sh
 allModules=$(listAllModules)
 activeModules=$(getActiveTerraformModules)
+OLD_IFS="$IFS"
 IFS=','
 for module in $allModules; do
   friendlyName=$(getFriendlyOutputName "$module")
@@ -34,6 +35,7 @@ for module in $allModules; do
     source "../$module/.cloud/predestroy.sh"
   fi
 done
+IFS="$OLD_IFS"
 
 terraform destroy -auto-approve
 popd >/dev/null
diff --git a/.cloud/helpers/generate-config.sh b/.cloud/helpers/generate-config.sh
@@ -48,13 +48,15 @@ function appendAllModules() {
   else
     modules=$(listAllModules)
   fi
+  OLD_IFS="$IFS"
   IFS=','
   for module in $modules; do
     # Only include modules with a .cloud subdirectory in the generated config.
     if [ -d "../$module/.cloud" ]; then
       appendModule "${module%/}" # Remove possible trailing '/'
     fi
   done
+  IFS="$OLD_IFS"
 }
 
 # Ensure current directory is <root>/.cloud

diff --git a/.cloud/helpers/plan.sh b/.cloud/helpers/plan.sh
@@ -44,6 +44,7 @@ else
   modules=$(listAllModules)
 fi
 echo "Planning around modules $modules"
+OLD_IFS="$IFS"
 IFS=','
 for module in $modules; do
   # Only include modules with a .cloud subdirectory in the generated config.
@@ -52,6 +53,7 @@ for module in $modules; do
     source "../$module/.cloud/preplan.sh" generated.auto.tfvars
   fi
 done
+IFS="$OLD_IFS"
 
 terraform fmt -list=false generated.auto.tfvars
 terraform plan -out generated.tfplan

diff --git a/.cloud/helpers/populate-env.sh b/.cloud/helpers/populate-env.sh
@@ -30,6 +30,7 @@ pushd "$helperDir/.." >/dev/null
 source ./helpers/common.sh
 allModules=$(listAllModules)
 activeModules=$(getActiveTerraformModules)
+OLD_IFS="$IFS"
 IFS=','
 for module in $allModules; do
   friendlyName=$(getFriendlyOutputName "$module")
@@ -39,5 +40,6 @@ for module in $allModules; do
 
   modifyEnvironment "$module"
 done
+IFS="$OLD_IFS"
 
 popd >/dev/null
diff --git a/.cloud/setup.sh b/.cloud/setup.sh
@@ -29,5 +29,5 @@ popd >/dev/null
 # Ensure all SNAPSHOTs are available in the local mvn repository
 pushd "$scriptDir/.." >/dev/null
 source ./.kokoro/common.sh
-install_modules
+install_modules "$1"
 popd >/dev/null
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -38,16 +38,17 @@ RETURN_CODE=0
 case ${JOB_TYPE} in
   test)
     retry_with_backoff 3 10 \
-      mvn -B -ntp \
-      -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-      -Dclirr.skip=true \
-      -Denforcer.skip=true \
-      -Dcheckstyle.skip=true \
-      -Dflatten.skip=true \
-      -Danimal.sniffer.skip=true \
-      -Dmaven.wagon.http.retryHandler.count=5 \
-      -T 1C \
-      test
+      mvn test \
+        -B -ntp \
+        -Dorg.slf4j.simpleLogger.showDateTime=true \
+        -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+        -Dclirr.skip=true \
+        -Denforcer.skip=true \
+        -Dcheckstyle.skip=true \
+        -Dflatten.skip=true \
+        -Danimal.sniffer.skip=true \
+        -Dmaven.wagon.http.retryHandler.count=5 \
+        -T 1C
     RETURN_CODE=$?
     echo "Finished running unit tests"
     ;;
@@ -59,7 +60,7 @@ case ${JOB_TYPE} in
         echo "${modified_module_list[*]}"
       )
       setup_cloud "$module_list"
-      install_modules
+      install_modules "$module_list"
       run_integration_tests "$module_list"
     else
       echo "No Integration Tests to run"
@@ -69,31 +70,20 @@ case ${JOB_TYPE} in
     generate_graalvm_presubmit_modules_list
     printf "Running GraalVM presubmit checks for:\n%s\n" "${module_list}"
     setup_cloud "$module_list"
-    install_modules
+    install_modules "$module_list"
     run_graalvm_tests "$module_list"
     ;;
   graalvm)
     generate_graalvm_modules_list
     if [ ! -z "${module_list}" ]; then
       printf "Running GraalVM checks for:\n%s\n" "${module_list}"
       setup_cloud "$module_list"
-      install_modules
+      install_modules "$module_list"
       run_graalvm_tests "$module_list"
     else
       echo "Not running GraalVM checks -- No changes in relevant modules"
     fi
     ;;
-  graalvm17)
-    generate_graalvm_modules_list
-    if [ ! -z "${module_list}" ]; then
-      printf "Running GraalVM 17 checks for:\n%s\n" "${module_list}"
-      setup_cloud "$module_list"
-      install_modules
-      run_graalvm_tests "$module_list"
-    else
-      echo "Not running GraalVM 17 checks -- No changes in relevant modules"
-    fi
-    ;;
   *) ;;
 
 esac

diff --git a/.kokoro/common.sh b/.kokoro/common.sh
@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: remove java-core once we figure out how setup_cloud understands Maven's
-# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
-excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')
+excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent')
 
 function retry_with_backoff {
   attempts_left=$1
@@ -55,6 +53,55 @@ function retry_with_backoff {
   return $exit_code
 }
 
+# Given a folder containing a maven multi-module, assign the variable 'submodules' to a
+# comma-delimited list of <folder>/<submodule>.
+function parse_submodules() {
+  pushd "$1" >/dev/null
+
+  # New-line-delimited string containing the current folder's pom.xml <module> names.
+  mvn_submodules=$(mvn help:evaluate -Dexpression=project.modules \
+        | grep '<.*>.*</.*>' \
+        | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')
+
+  submodules_array=()
+  for submodule in $mvn_submodules; do
+    # Each entry = <folder>/<submodule>
+    submodules_array+=("$1/${submodule}");
+  done
+
+  # Convert from array to comma-delimited string
+  submodules=$(
+    IFS=,
+    echo "${submodules_array[*]}"
+  )
+  export submodules
+
+  popd >/dev/null
+}
+
+# Given a list of folders containing maven multi-modules, assign the variable 'all_submodules' to a
+# comma-delimited list of <folder>/<submodule>.
+#
+# See also parse_submodules()
+function parse_all_submodules() {
+  # Parse the comma-delimited input into an array.
+  IFS=',' read -ra input_modules <<< "$1"
+
+  all_submodules_array=()
+  for module in "${input_modules[@]}"; do
+    # For each module, parse its submodules and store the result in an array.
+    parse_submodules "$module"
+    all_submodules_array+=("$submodules")
+  done
+
+  # Convert from array to comma-delimited string
+  all_submodules=$(
+    IFS=,
+    echo "${all_submodules_array[*]}"
+  )
+  export all_submodules
+}
+
 ## Helper functions
 function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
 function msg() { println "$*" >&2; }
@@ -126,57 +173,56 @@ function generate_modified_modules_list() {
 }
 
 function run_integration_tests() {
-  printf "Running Integration Tests for:\n%s\n" "$1"
-  # --also-make-dependents to run other modules that use the affected module
-  mvn -B ${INTEGRATION_TEST_ARGS} \
-    -pl "$1" \
-    --also-make-dependents \
-    -ntp \
-    -Penable-integration-tests \
+  printf "Running integration tests for modules:\n%s\n" "$1"
+  parse_all_submodules "$1"
+  printf "Running integration tests for submodules:\n%s\n" "$all_submodules"
+
+  mvn verify -Penable-integration-tests --projects "$all_submodules" \
+    ${INTEGRATION_TEST_ARGS} \
+    -B -ntp -fae \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true \
+    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
     -Danimal.sniffer.skip=true \
     -Djacoco.skip=true \
     -DskipUnitTests=true \
     -Dmaven.wagon.http.retryHandler.count=5 \
-    -fae \
-    -T 1C \
-    verify
+    -T 1C
 
   RETURN_CODE=$?
-  printf "Finished Integration Tests for:\n%s\n" "$1"
+  printf "Finished integration tests for modules:\n%s\n" "$1"
 }
 
 function run_graalvm_tests() {
-  printf "Running GraalVM ITs on:\n%s\n" "$1"
+  printf "Running GraalVM ITs for modules:\n%s\n" "$1"
+  parse_all_submodules "$1"
+  printf "Running GraalVM ITs for submodules:\n%s\n" "$all_submodules"
 
-  mvn -B ${INTEGRATION_TEST_ARGS} \
-    -pl "$1" \
-    --also-make-dependents \
-    -ntp \
+  mvn test -Pnative --projects "$all_submodules" \
+    ${INTEGRATION_TEST_ARGS} \
+    -B -ntp -fae \
     -DtrimStackTrace=false \
     -Dclirr.skip=true \
     -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+    -Dorg.slf4j.simpleLogger.showDateTime=true \
+    -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
     -Dcheckstyle.skip=true \
     -Dflatten.skip=true \
     -Danimal.sniffer.skip=true \
-    -Pnative \
-    -fae \
-    test
+    -T 1C
 
   RETURN_CODE=$?
-  printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
+  printf "Finished GraalVM ITs for modules:\n%s\n" "$1"
 }
 
 function generate_graalvm_presubmit_modules_list() {
   modules_assigned_list=()
   generate_modified_modules_list
-  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 10 ]]; then
+  if [[ ${#modified_module_list[@]} -gt 0 && ${#modified_module_list[@]} -lt 5 ]]; then
     # If only a few modules have been modified, focus presubmit testing only on them.
     module_list=$(
       IFS=,
@@ -229,18 +275,52 @@ function generate_graalvm_modules_list() {
 }
 
 function install_modules() {
-  retry_with_backoff 3 10 \
-    mvn -B \
-    -ntp \
-    -DtrimStackTrace=false \
-    -Dclirr.skip=true \
-    -Denforcer.skip=true \
-    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
-    -Dcheckstyle.skip=true \
-    -Dflatten.skip=true \
-    -Danimal.sniffer.skip=true \
-    -DskipTests=true \
-    -Djacoco.skip=true \
-    -T 1C \
-    install
+  if [ -z "$1" ]; then
+    mvn install \
+      -B -ntp \
+      -DtrimStackTrace=false \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -Dorg.slf4j.simpleLogger.showDateTime=true \
+      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+      -Dcheckstyle.skip=true \
+      -Dflatten.skip=true \
+      -Danimal.sniffer.skip=true \
+      -DskipTests=true \
+      -Djacoco.skip=true \
+      -T 1C
+  else
+    printf "Installing modules:\n%s\n" "$1"
+    parse_all_submodules "$1"
+    printf "Installing submodules:\n%s\n" "$all_submodules"
+
+    # When working with a maven multi-module project containing other multi-module projects,
+    # to build a module with its dependencies and without building its dependents:
+    # Perform the install command on a grandchild module with the --also-make flag.
+    #
+    # Examples:
+    #
+    #   mvn install --projects java-asset --also-make
+    #      ! Does not work. Maven reactor will not build java-asset's child modules, such as the
+    #        gapic, proto, and grpc modules.
+    #
+    #   mvn install --projects java-kms --also-make-dependents
+    #      ! Does not work. Maven reactor will include java-kmsinventory in its build.
+    #
+    #   mvn install --projects java-kms/google-cloud-kms --also-make
+    #      Correctly builds dependencies without building dependents.
+    mvn install --projects "$all_submodules" --also-make \
+      -B -ntp \
+      -DtrimStackTrace=false \
+      -Dclirr.skip=true \
+      -Denforcer.skip=true \
+      -Dorg.slf4j.simpleLogger.showDateTime=true \
+      -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
+      -Dcheckstyle.skip=true \
+      -Dflatten.skip=true \
+      -Danimal.sniffer.skip=true \
+      -DskipTests=true \
+      -Djacoco.skip=true \
+      -T 1C
+  fi
 }
diff --git a/.kokoro/nightly/graalvm-native-17.cfg b/.kokoro/nightly/graalvm-native-17.cfg
@@ -8,7 +8,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm17"
+  value: "graalvm"
 }
 
 # TODO: remove this after we've migrated all tests and scripts

diff --git a/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg b/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg
@@ -29,7 +29,7 @@ env_vars: {
 
 env_vars: {
   key: "JOB_TYPE"
-  value: "graalvm17"
+  value: "graalvm"
 }
 
 # TODO: remove this after we've migrated all tests and scripts