Fix calling of get-changed-trilinos-packages.sh and other improvements (

trilinos#3133, trilinos#3265) Now calling this script will select the correct set of file changes and correct set of enables as part the goals of trilinos#3133. I have tested this manally on several different PRs and verified that the correct set of enables are being selected. I did a few other things while editing the script: * The script can now be run on a different scratch Trilinos repo. This allows it to be manually tested without messing up the status of your main Trilinos git repo (see details in PR trilinos#3258) * The Trilinos git repo where the PR branch is merged is now automatically updated to the correct state, no matter what its prior state is before it is run. The only requirement is that the 'origin' remote is set correctly. None of the other local branches have to be exist or set correctly at all. See the new comments at the top. * Make all formating if statements the same with the same indentation. * Removed code that was not used or did nothing. Without this change, the auto PR tester will test more package that is it supposed to.
tjfulle · Dec 6, 2018 · 5cc45a6 · 5cc45a6
1 parent 62771dc
commit 5cc45a6
Showing 1 changed file with 129 additions and 65 deletions.
diff --git a/cmake/std/PullRequestLinuxDriver.sh b/cmake/std/PullRequestLinuxDriver.sh
@@ -1,6 +1,25 @@
 #!/usr/bin/env bash
 # set -x  # echo commands
 
+#
+# This script drives a PR testing build.  It assume that Trilinos is already
+# cloned under $PWD/Trilinos and that the 'origin' remote points to
+# $TRILINOS_TARGET_REPO (but that is not checked here).
+#
+# As long as the ${PWD}/Trilinos git repo has the correct 'origin', this
+# script will automatically set it up to do the merge correctly, no matter
+# what its state before this script is called (i.e. from a past PR
+# attempt). Unless the Trilinos/.git directory becomes corrupted, there should
+# *NEVER* be any need to delete and reclone this Trilinos git repo.
+#
+# This script can be run in a mode where the driver scripts are run from one
+# Trilinos git repo and operate on another Trilinos git repo that gets
+# manipulated in order to merge the "source" topic branch into the "target"
+# branch.  This makes it easy to test changes to the PR scripts.  But if this
+# script is run from ${PWD}/Trilinos, then these repos are one and the same
+# and we get the correct behavior for PR testing.
+#
+
 export https_proxy=https://wwwproxy.sandia.gov:80
 export http_proxy=http://wwwproxy.sandia.gov:80
 
@@ -25,13 +44,36 @@ declare -i ierror=0
 #Have to keep loading git
 module load sems-git/2.10.1
 
-ls
-cd Trilinos
+
+#--------------------------------------------
+# Get Trilinos scripts and PR merge repo dirs
+#--------------------------------------------
+
+# The Trilinos dir that the PR merge will be done in
+PR_TRILINOS_DIR=$PWD/Trilinos
+echo "PR_TRILINOS_DIR = $PR_TRILINOS_DIR"
+
+# Get the Trilinos scripts driver dir where this script is run from.  First,t
+# ry to grab from the symlink (only works on Linux)
+_ABS_FILE_PATH=`readlink -f $0` || \
+  echo "Could not follow symlink to set TRILINOS_DRIVER_SRC_DIR!"
+if [ "$_ABS_FILE_PATH" != "" ] ; then
+  _SCRIPT_DIR=`dirname $_ABS_FILE_PATH`
+  TRILINOS_DRIVER_SRC_DIR=$_SCRIPT_DIR/../..
+else
+  # If that did not work, then we are not on Linux so give up and assume the
+  # standard location
+  TRILINOS_DRIVER_SRC_DIR=${PR_TRILINOS_DIR}
+fi
+echo "TRILINOS_DRIVER_SRC_DIR = $TRILINOS_DRIVER_SRC_DIR"
+
 
 #------------------------------
 # Doing merge of pull request
 #------------------------------
 
+ls
+cd Trilinos
 
 # Check for existence of source_remote and remove if it exists
 git_remote_text=`git remote -v | grep "source_remote"`
@@ -40,33 +82,62 @@ if [[ "$git_remote_text" != "" ]]; then
   git remote rm source_remote
 fi
 
-#Add the necessary remote
+# Add the necessary remote
 git remote add source_remote ${TRILINOS_SOURCE_REPO:?}
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "There was a problem adding the remote for the source repo. The error code was: $ierror"
-#just in case somehow a previously defined source_remote caused this failure
-#would be better to check prior to the add. Don't want to issue command that will be known to fail typically.
-#git remote rm source_remote
-exit $ierror
+  echo "There was a problem adding the remote for the source repo. The error code was: $ierror"
+  #just in case somehow a previously defined source_remote caused this failure
+  #would be better to check prior to the add. Don't want to issue command that
+  #will be known to fail typically.  git remote rm source_remote
+  exit $ierror
 fi
 
+git remote -v
+
 git fetch source_remote
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "Source remote fetch failed. The error code was: $ierror"
-#git remote rm source_remote
-exit $ierror
+  echo "Source remote fetch failed. The error code was: $ierror"
+  exit $ierror
+fi
+
+git fetch origin
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "Origin target remote fetch failed. The error code was: $ierror"
+  exit $ierror
 fi
 
-git remote -v
+git status
+
+# Clean out any non-committed changes so that we can checkout the correct
+# branch
+git reset --hard HEAD
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "There was an error clearing out any local uncommitted changes. The error code was: $ierror"
+  exit $ierror
+fi
+
+git status
+
+# Get on the right local branch
+git checkout -B ${TRILINOS_TARGET_BRANCH:?} origin/${TRILINOS_TARGET_BRANCH:?}
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "There was an error checking out and updating to the target remote branch. The error code was: $ierror"
+  exit $ierror
+fi
 
-git merge source_remote/${TRILINOS_SOURCE_BRANCH:?}
+git status
+
+# Merge the souce branch into the local target branch
+git merge --no-edit source_remote/${TRILINOS_SOURCE_BRANCH:?}
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "There was an issue merging changes from ${TRILINOS_SOURCE_REPO:?} ${TRILINOS_SOURCE_BRANCH:?} onto ${TRILINOS_TARGET_REPO:?} ${TRILINOS_TARGET_BRANCH:?}. The error code was: $ierror"
-#git remote rm source_remote
-exit $ierror
+  echo "There was an issue merging changes from ${TRILINOS_SOURCE_REPO:?} ${TRILINOS_SOURCE_BRANCH:?} onto ${TRILINOS_TARGET_REPO:?} ${TRILINOS_TARGET_BRANCH:?}. The error code was: $ierror"
+  exit $ierror
 fi
 
 #Need to compare expected SOURCE SHA to actual SHA! This will prevent a security hole.
@@ -77,7 +148,6 @@ echo "The most recent SHA for repo: ${TRILINOS_SOURCE_REPO:?} on branch: ${TRILI
 if [[ ABC != ${TRILINOS_SOURCE_SHA:?} ]]; then
   if [[ $source_sha != ${TRILINOS_SOURCE_SHA:?} ]]; then
     echo "The SHA ($source_sha) for the last commit on branch ${TRILINOS_SOURCE_BRANCH:?} in repo ${TRILINOS_SOURCE_REPO:?} is different than the expected SHA, which is: ${TRILINOS_SOURCE_SHA:?}. The error code was: $ierror"
-    #git remote rm source_remote
   exit -1
   fi
 fi
@@ -90,43 +160,42 @@ fi
 # Doing setup for build
 #------------------------------
 
-
-git status
-git diff origin/${TRILINOS_TARGET_BRANCH:?} --numstat > ../gitchanges.txt
-ierror=$?
-if [[ $ierror != 0 ]]; then
-echo "There was an issue getting the list of changed files. The error code was: $ierror"
-
-exit $ierror
-fi
-
 cd ../
 
 # Set up the full environment for the build
-if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestGCC4.8.4TestingEnv.sh
+if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.8.4TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the gcc environment. The error code was: $ierror"
     exit $ierror
   fi
-elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh
+elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the gcc environment. The error code was: $ierror"
     exit $ierror
   fi
-elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh
+elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the intel environment. The error code was: $ierror"
     exit $ierror
   fi
+# --- BEGIN SCAFFOLDING ---
+# nurgleflurgle... name match!  can we change this to a regex or something to
+# allow separate testing jobs!?
+#elif [ "Trilinos_pullrequest_gcc_4.9.3_wcmclen_test" == "${JOB_BASE_NAME:?}" ]
+#then
+#  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh
+#  ierror=$?
+#  if [[ $ierror != 0 ]]; then
+#    echo "There was an issue loading the gcc environment. The error code was: $ierror"
+#    exit $ierror
+#  fi
+# --- END SCAFFOLDING ---  
 else
   ierror=42
   echo "There was an issue loading the proper environment. The error code was: $ierror"
@@ -147,32 +216,40 @@ echo "CDash Track = ${CDASH_TRACK:?}"
 #-------------------------------------
 # Doing configure/build/test/submit
 #-------------------------------------
+
 rm packageEnables.cmake
-changed_packages_app=Trilinos/commonTools/framework/get-changed-trilinos-packages.sh
-${changed_packages_app} ${TRILINOS_SOURCE_SHA} ${TRILINOS_TARGET_SHA} packageEnables.cmake
+env \
+  TRILINOS_DIR=${PR_TRILINOS_DIR} \
+  TRILINOS_SCRIPTS_DIR=${TRILINOS_DRIVER_SRC_DIR} \
+${TRILINOS_DRIVER_SRC_DIR}/commonTools/framework/get-changed-trilinos-packages.sh \
+  origin/${TRILINOS_TARGET_BRANCH:?} HEAD packageEnables.cmake
 
 ierror=$?
 if [[ $ierror != 0 ]]; then
-    echo "There was an issue generating packageEnables.cmake.  The error code was: $ierror"
-    exit $ierror
+  echo "There was an issue generating packageEnables.cmake.  The error code was: $ierror"
+  exit $ierror
 fi
 
-
 build_name="PR-$PULLREQUESTNUM-test-$JOB_BASE_NAME-$BUILD_NUMBER"
 
 #This should be runnable from anywhere, but all the tests so far have been from the 
 #same dir the simple_testing.cmake file was in.
 cd TFW_testing_single_configure_prototype
 
-if [ "icc" == ${CC:?} ]
-then
-    CONFIG_SCRIPT=PullRequestLinuxIntelTestingSettings.cmake
+if [ "icc" == ${CC:?} ] ; then
+  CONFIG_SCRIPT=PullRequestLinuxIntelTestingSettings.cmake
 else
-    if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]; then
-        CONFIG_SCRIPT=PullRequestLinuxGCC4.8.4TestingSettings.cmake
-    elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]; then
-        CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake
-    fi
+  if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]; then
+    CONFIG_SCRIPT=PullRequestLinuxGCC4.8.4TestingSettings.cmake
+  elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]; then
+    CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake
+  # --- BEGIN SCAFFOLDING ---
+  # nurgleflurgle... name match!  can we change this to a regex or something to
+  # allow separate testing jobs!?
+  #elif [ "Trilinos_pullrequest_gcc_4.9.3_wcmclen_test" == "${JOB_BASE_NAME:?}" ]; then
+  #    CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake
+  # --- END SCAFFOLDING ---
+  fi
 fi
 
 ctest -S simple_testing.cmake \
@@ -183,28 +260,15 @@ ctest -S simple_testing.cmake \
   -Ddashboard_track="${CDASH_TRACK:?}" \
   -DPARALLEL_LEVEL=22 \
   -Dbuild_dir="${WORKSPACE:?}/pull_request_test" \
-  -Dconfigure_script=../Trilinos/cmake/std/${CONFIG_SCRIPT:?} \
+  -Dconfigure_script=${TRILINOS_DRIVER_SRC_DIR}/cmake/std/${CONFIG_SCRIPT:?} \
   -Dpackage_enables=../packageEnables.cmake \
   -Dsubprojects_file=../TFW_single_configure_support_scripts/package_subproject_list.cmake
 
 ierror=$?
-
 if [[ $ierror != 0 ]]; then
-echo "Single configure/build/test failed. The error code was: $ierror"
-exit $ierror
-fi
-
-
-ierror=$?
-if [[ $ierror != 0 ]]; then
-echo "There was an error removing the source remote. The error code was: $ierror"
-exit $ierror
+  echo "Single configure/build/test failed. The error code was: $ierror"
+  exit $ierror
 fi
 
-#NEED TO MAKE SURE THE REPOS ARE CLEAN FOR NEW PULL REQUESTS!
-
 #pushd Trilinos/cmake/ctest/drivers/parameterized
 #ctest -S ctest_linux_nightly_generic.cmake
-
-
-