Merge Pull Request #3258 from bartlettroscoe/Trilinos/3133-fix-logic

Automatically Merged using Trilinos Pull Request AutoTester
PR Title: Fix change logic in get-changed-trilinos-packages.sh and PullRequestLinuxDriver.sh (#3133)
PR Author: bartlettroscoe

cmake/std/PullRequestLinuxDriver.sh

@@ -1,6 +1,25 @@
 #!/usr/bin/env bash
 # set -x  # echo commands
 
+#
+# This script drives a PR testing build.  It assume that Trilinos is already
+# cloned under $PWD/Trilinos and that the 'origin' remote points to
+# $TRILINOS_TARGET_REPO (but that is not checked here).
+#
+# As long as the ${PWD}/Trilinos git repo has the correct 'origin', this
+# script will automatically set it up to do the merge correctly, no matter
+# what its state before this script is called (i.e. from a past PR
+# attempt). Unless the Trilinos/.git directory becomes corrupted, there should
+# *NEVER* be any need to delete and reclone this Trilinos git repo.
+#
+# This script can be run in a mode where the driver scripts are run from one
+# Trilinos git repo and operate on another Trilinos git repo that gets
+# manipulated in order to merge the "source" topic branch into the "target"
+# branch.  This makes it easy to test changes to the PR scripts.  But if this
+# script is run from ${PWD}/Trilinos, then these repos are one and the same
+# and we get the correct behavior for PR testing.
+#
+
 export https_proxy=https://wwwproxy.sandia.gov:80
 export http_proxy=http://wwwproxy.sandia.gov:80
 
@@ -25,13 +44,36 @@ declare -i ierror=0
 #Have to keep loading git
 module load sems-git/2.10.1
 
-ls
-cd Trilinos
+
+#--------------------------------------------
+# Get Trilinos scripts and PR merge repo dirs
+#--------------------------------------------
+
+# The Trilinos dir that the PR merge will be done in
+PR_TRILINOS_DIR=$PWD/Trilinos
+echo "PR_TRILINOS_DIR = $PR_TRILINOS_DIR"
+
+# Get the Trilinos scripts driver dir where this script is run from.  First,t
+# ry to grab from the symlink (only works on Linux)
+_ABS_FILE_PATH=`readlink -f $0` || \
+  echo "Could not follow symlink to set TRILINOS_DRIVER_SRC_DIR!"
+if [ "$_ABS_FILE_PATH" != "" ] ; then
+  _SCRIPT_DIR=`dirname $_ABS_FILE_PATH`
+  TRILINOS_DRIVER_SRC_DIR=$_SCRIPT_DIR/../..
+else
+  # If that did not work, then we are not on Linux so give up and assume the
+  # standard location
+  TRILINOS_DRIVER_SRC_DIR=${PR_TRILINOS_DIR}
+fi
+echo "TRILINOS_DRIVER_SRC_DIR = $TRILINOS_DRIVER_SRC_DIR"
+
 
 #------------------------------
 # Doing merge of pull request
 #------------------------------
 
+ls
+cd Trilinos
 
 # Check for existence of source_remote and remove if it exists
 git_remote_text=`git remote -v | grep "source_remote"`
@@ -40,33 +82,62 @@ if [[ "$git_remote_text" != "" ]]; then
   git remote rm source_remote
 fi
 
-#Add the necessary remote
+# Add the necessary remote
 git remote add source_remote ${TRILINOS_SOURCE_REPO:?}
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "There was a problem adding the remote for the source repo. The error code was: $ierror"
-#just in case somehow a previously defined source_remote caused this failure
-#would be better to check prior to the add. Don't want to issue command that will be known to fail typically.
-#git remote rm source_remote
-exit $ierror
+  echo "There was a problem adding the remote for the source repo. The error code was: $ierror"
+  #just in case somehow a previously defined source_remote caused this failure
+  #would be better to check prior to the add. Don't want to issue command that
+  #will be known to fail typically.  git remote rm source_remote
+  exit $ierror
 fi
 
+git remote -v
+
 git fetch source_remote
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "Source remote fetch failed. The error code was: $ierror"
-#git remote rm source_remote
-exit $ierror
+  echo "Source remote fetch failed. The error code was: $ierror"
+  exit $ierror
 fi
 
-git remote -v
+git fetch origin
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "Origin target remote fetch failed. The error code was: $ierror"
+  exit $ierror
+fi
+
+git status
+
+# Clean out any non-committed changes so that we can checkout the correct
+# branch
+git reset --hard HEAD
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "There was an error clearing out any local uncommitted changes. The error code was: $ierror"
+  exit $ierror
+fi
+
+git status
 
-git merge source_remote/${TRILINOS_SOURCE_BRANCH:?}
+# Get on the right local branch
+git checkout -B ${TRILINOS_TARGET_BRANCH:?} origin/${TRILINOS_TARGET_BRANCH:?}
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "There was an issue merging changes from ${TRILINOS_SOURCE_REPO:?} ${TRILINOS_SOURCE_BRANCH:?} onto ${TRILINOS_TARGET_REPO:?} ${TRILINOS_TARGET_BRANCH:?}. The error code was: $ierror"
-#git remote rm source_remote
-exit $ierror
+  echo "There was an error checking out and updating to the target remote branch. The error code was: $ierror"
+  exit $ierror
+fi
+
+git status
+
+# Merge the souce branch into the local target branch
+git merge --no-edit source_remote/${TRILINOS_SOURCE_BRANCH:?}
+ierror=$?
+if [[ $ierror != 0 ]]; then
+  echo "There was an issue merging changes from ${TRILINOS_SOURCE_REPO:?} ${TRILINOS_SOURCE_BRANCH:?} onto ${TRILINOS_TARGET_REPO:?} ${TRILINOS_TARGET_BRANCH:?}. The error code was: $ierror"
+  exit $ierror
 fi
 
 #Need to compare expected SOURCE SHA to actual SHA! This will prevent a security hole.
@@ -77,7 +148,6 @@ echo "The most recent SHA for repo: ${TRILINOS_SOURCE_REPO:?} on branch: ${TRILI
 if [[ ABC != ${TRILINOS_SOURCE_SHA:?} ]]; then
   if [[ $source_sha != ${TRILINOS_SOURCE_SHA:?} ]]; then
     echo "The SHA ($source_sha) for the last commit on branch ${TRILINOS_SOURCE_BRANCH:?} in repo ${TRILINOS_SOURCE_REPO:?} is different than the expected SHA, which is: ${TRILINOS_SOURCE_SHA:?}. The error code was: $ierror"
-    #git remote rm source_remote
   exit -1
   fi
 fi
@@ -90,38 +160,25 @@ fi
 # Doing setup for build
 #------------------------------
 
-
-git status
-git diff origin/${TRILINOS_TARGET_BRANCH:?} --numstat > ../gitchanges.txt
-ierror=$?
-if [[ $ierror != 0 ]]; then
-echo "There was an issue getting the list of changed files. The error code was: $ierror"
-
-exit $ierror
-fi
-
 cd ../
 
 # Set up the full environment for the build
-if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestGCC4.8.4TestingEnv.sh
+if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.8.4TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the gcc environment. The error code was: $ierror"
     exit $ierror
   fi
-elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh
+elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the gcc environment. The error code was: $ierror"
     exit $ierror
   fi
-elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ]
-then
-  source Trilinos/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh
+elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then
+  source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh
   ierror=$?
   if [[ $ierror != 0 ]]; then
     echo "There was an issue loading the intel environment. The error code was: $ierror"
@@ -147,32 +204,44 @@ echo "CDash Track = ${CDASH_TRACK:?}"
 #-------------------------------------
 # Doing configure/build/test/submit
 #-------------------------------------
+
 rm packageEnables.cmake
-changed_packages_app=Trilinos/commonTools/framework/get-changed-trilinos-packages.sh
-${changed_packages_app} ${TRILINOS_SOURCE_SHA} ${TRILINOS_TARGET_SHA} packageEnables.cmake
+env \
+  TRILINOS_DIR=${PR_TRILINOS_DIR} \
+  TRILINOS_SCRIPTS_DIR=${TRILINOS_DRIVER_SRC_DIR} \
+${TRILINOS_DRIVER_SRC_DIR}/commonTools/framework/get-changed-trilinos-packages.sh \
+  origin/${TRILINOS_TARGET_BRANCH:?} HEAD packageEnables.cmake
+# NOTE: Above we use the git diff origin/<target-branch>..HEAD to give us the
+# correct list of changed files.  This works because this is done after
+# merging the target branch and the soruce branch.  With that, git diff will
+# show all of the changes in the merged copy from what is in <target-branch>.
+# This with give the correct set of changed files even if older versions of
+# <target-branch> were merged multiple times into <source-branch>.  It turns
+# out that the only way to have git show the correct set of diffs in that case
+# is to actually do the merge and then do the diff.
 
 ierror=$?
 if [[ $ierror != 0 ]]; then
-    echo "There was an issue generating packageEnables.cmake.  The error code was: $ierror"
-    exit $ierror
+  echo "There was an issue generating packageEnables.cmake.  The error code was: $ierror"
+  exit $ierror
 fi
 
+echo "Enabled packages:"
+cmake -P packageEnables.cmake
 
 build_name="PR-$PULLREQUESTNUM-test-$JOB_BASE_NAME-$BUILD_NUMBER"
 
 #This should be runnable from anywhere, but all the tests so far have been from the 
 #same dir the simple_testing.cmake file was in.
 cd TFW_testing_single_configure_prototype
 
-if [ "icc" == ${CC:?} ]
-then
-    CONFIG_SCRIPT=PullRequestLinuxIntelTestingSettings.cmake
+if [ "icc" == ${CC:?} ] ; then
+  CONFIG_SCRIPT=PullRequestLinuxIntelTestingSettings.cmake
 else
-    if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]; then
-        CONFIG_SCRIPT=PullRequestLinuxGCC4.8.4TestingSettings.cmake
-    elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]; then
-        CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake
-    fi
+  if [ "Trilinos_pullrequest_gcc_4.8.4" == "${JOB_BASE_NAME:?}" ]; then
+    CONFIG_SCRIPT=PullRequestLinuxGCC4.8.4TestingSettings.cmake
+  elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ]; then
+    CONFIG_SCRIPT=PullRequestLinuxGCC4.9.3TestingSettings.cmake
 fi
 
 ctest -S simple_testing.cmake \
@@ -183,28 +252,15 @@ ctest -S simple_testing.cmake \
   -Ddashboard_track="${CDASH_TRACK:?}" \
   -DPARALLEL_LEVEL=22 \
   -Dbuild_dir="${WORKSPACE:?}/pull_request_test" \
-  -Dconfigure_script=../Trilinos/cmake/std/${CONFIG_SCRIPT:?} \
+  -Dconfigure_script=${TRILINOS_DRIVER_SRC_DIR}/cmake/std/${CONFIG_SCRIPT:?} \
   -Dpackage_enables=../packageEnables.cmake \
   -Dsubprojects_file=../TFW_single_configure_support_scripts/package_subproject_list.cmake
 
-ierror=$?
-
-if [[ $ierror != 0 ]]; then
-echo "Single configure/build/test failed. The error code was: $ierror"
-exit $ierror
-fi
-
-
 ierror=$?
 if [[ $ierror != 0 ]]; then
-echo "There was an error removing the source remote. The error code was: $ierror"
-exit $ierror
+  echo "Single configure/build/test failed. The error code was: $ierror"
+  exit $ierror
 fi
 
-#NEED TO MAKE SURE THE REPOS ARE CLEAN FOR NEW PULL REQUESTS!
-
 #pushd Trilinos/cmake/ctest/drivers/parameterized
 #ctest -S ctest_linux_nightly_generic.cmake
-
-
-

cmake/tribits/ci_support/TribitsPackageFilePathUtils.py

@@ -113,6 +113,15 @@ def getProjectCiFileChangeLogic(projectDir):
 
   if not projectDir:
     return DefaultProjectCiFileChangeLogic()
+  else:
+    projectCiFileChangeLogicFileBaseDir = os.path.join(projectDir, "cmake")
+    projectCiFileChangeLogicFile = os.path.join(
+      projectCiFileChangeLogicFileBaseDir, "ProjectCiFileChangeLogic.py")
+    if not os.path.isfile(projectCiFileChangeLogicFile):
+      return DefaultProjectCiFileChangeLogic()
+
+  # Else, if we get here, then the file
+  # <projectDir>/cmake/ProjectCiFileChangeLogic.py exists so lets read it in!
 
   tribitsDirPath = os.path.abspath(
     os.path.join(
@@ -124,7 +133,7 @@ def getProjectCiFileChangeLogic(projectDir):
   old_sys_path = sys.path
 
   try:
-    sys.path = [tribitsDirPath+"/examples/TribitsExampleProject/cmake"] + sys.path
+    sys.path = [projectCiFileChangeLogicFileBaseDir] + sys.path
     import ProjectCiFileChangeLogic
     return ProjectCiFileChangeLogic.ProjectCiFileChangeLogic()
   finally:

cmake/tribits/ci_support/filter-packages-list.py

@@ -70,7 +70,7 @@
 
   filter-packages-list.py --keep-test-test-categories=PT,ST,EX [other args]
 
-(or don't both running the script).
+(or don't bother running the script).
 """
 
 from optparse import OptionParser

cmake/tribits/doc/developers_guide/TribitsDevelopersGuide.rst

@@ -4512,7 +4512,7 @@ components.  However, detailed descriptions of these processes are deferred to
 the later sections `Pre-push Testing using checkin-test.py`_ and `TriBITS
 CTest/CDash Driver`_.
 
-The standard TriBITS-suupported project testing processes are:
+The standard TriBITS-supported project testing processes are:
 
 * `Pre-Push CI Testing`_
 * `Post-Push CI Testing`_
@@ -5058,7 +5058,7 @@ TriBITS packages, one must perform the following actions**:
 1) Change the TriBITS CMake files as described above that will result in the
    desired email addresses in the ``CDashSubprojectDependencies.xml``
    file. One can debug this by generating the file
-   `<Project>PackageDependencies.xml`_ as using the cmake -P script
+   `<Project>PackageDependencies.xml`_ by using the cmake -P script
    `TribitsDumpDepsXmlScript.cmake`_.
 
 2) Log onto the CDash server using an administrator account and then remove