Merge Pull Request #6840 from KyleFromKitware/Trilinos/ctest-resource…

…-groups

Automatically Merged using Trilinos Pull Request AutoTester
PR Title: Use RESOURCE_GROUPS property of CTest
PR Author: KyleFromKitware

cmake/std/atdm/README.md

@@ -1034,7 +1034,7 @@ command to run if using a CUDA build.  For example, to configure, build and
 run the tests for the default `cuda-debug` build for say `MueLu` (after
 cloning Trilinos on the `develop` branch) one would do:
 
-```
+```bash
 $ cd <some_build_dir>/
 
 $ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh cuda-debug
@@ -1047,13 +1047,20 @@ $ cmake \
 
 $ make NP=20
 
-$ bsub -x -Is -n 20 ctest -j4
+$ bsub -x -Is -n 20 ctest -j2
 ```
 
 **NOTE:** While the above example shows loading the environment, configuring
 and building on the login node, one can also do these on the compute nodes as
 well.  In fact, that is what the CTest -S drivers do in automated testing on
-'waterman'.
+'waterman'.  To get an interactive compute node, do:
+
+```
+$ bsub -x -Is -n 20 bash
+```
+
+Then one can configure, build, and run tests interactively on that compute
+node.
 
 Note that one can also run the same build and tests using the <a
 href="#checkin-test-atdmsh">checkin-test-atdm.sh</a> script as:

cmake/std/atdm/checkin-test-atdm.sh

@@ -19,14 +19,18 @@ if [ "$ATDM_TRILINOS_DIR" == "" ] ; then
     export ATDM_TRILINOS_DIR=$STD_ATDM_DIR/../../..
   fi
 fi
-
 echo "ATDM_TRILINOS_DIR = '$ATDM_TRILINOS_DIR'"
 
 if [ "$ATDM_TRILINOS_DIR" == "" ] ; then
   echo "ERROR: Cannot determine TRILINOS_DIR (you must be on a non-Linux system or you must have copied the script instead of symlinking it as per instructions)."
   exit 1
 fi
 
+if [ "$ATDM_TRIBITS_DIR" == "" ] ; then
+  export ATDM_TRIBITS_DIR=$ATDM_TRILINOS_DIR/cmake/tribits
+fi
+echo "ATDM_TRIBITS_DIR = '${ATDM_TRIBITS_DIR}'"
+
 #
 # Load a default env for the system
 #
@@ -241,7 +245,8 @@ echo
 echo "  ==> See output file checkin-test.final.out" 
 echo
 
-$ATDM_TRILINOS_DIR/cmake/tribits/ci_support/checkin-test.py \
+${ATDM_TRIBITS_DIR}/ci_support/checkin-test.py \
+  --src-dir=$ATDM_TRILINOS_DIR \
   --default-builds= --st-extra-builds=$ATDM_BUILD_NAME_KEYS_COMMA_LIST \
   --allow-no-pull "$ATDM_CHT_ENABLE_PACKAGES_ARG" \
   $ATDM_CHT_SEND_EMAIL_TO_ARG \

cmake/std/atdm/ride/environment.sh

@@ -225,7 +225,7 @@ export ATDM_CONFIG_NETCDF_LIBS="-L${NETCDF_ROOT}/lib;-L${HDF5_ROOT}/lib;${NETCDF
 
 # Use manually installed cmake and ninja to try to avoid module loading
 # problems (see TRIL-208)
-export PATH=/ascldap/users/rabartl/install/white-ride/cmake-3.11.2/bin:/ascldap/users/rabartl/install/white-ride/ninja-1.8.2/bin:$PATH
+export PATH=/home/atdm-devops-admin/tools/ride/cmake-3.17.2/bin:/home/rabartl/install/white-ride/ninja-1.8.2/bin:$PATH
 
 # Set MPI wrappers
 export MPICC=`which mpicc`

cmake/std/atdm/ride/tweaks/Tweaks.cmake

@@ -1,3 +1,11 @@
+#
+# Set up to limit running on GPUs
+#
+
+ATDM_SET_CACHE(Trilinos_AUTOGENERATE_TEST_RESOURCE_FILE ON CACHE BOOL)
+ATDM_SET_CACHE(Trilinos_CUDA_NUM_GPUS 2 CACHE STRING)
+ATDM_SET_CACHE(Trilinos_CUDA_SLOTS_PER_GPU 2 CACHE STRING)
+
 #
 # Disables across multiple builds on 'ride'
 #

cmake/std/atdm/utils/checkin-test-atdm-single.sh

@@ -33,7 +33,8 @@ fi
 
 set -x
 
-$ATDM_TRILINOS_DIR/cmake/tribits/ci_support/checkin-test.py \
+${ATDM_TRIBITS_DIR}/ci_support/checkin-test.py \
+  --src-dir=$ATDM_TRILINOS_DIR \
   --make-options="${make_options}" \
   --ctest-options="-j $ATDM_CONFIG_CTEST_PARALLEL_LEVEL" \
   --st-extra-builds=$ATDM_BUILD_NAME_KEYS "$@" \

cmake/std/atdm/waterman/environment.sh

@@ -146,15 +146,19 @@ elif [[ "$ATDM_CONFIG_COMPILER" == "CUDA"* ]] ; then
   export CUDA_LAUNCH_BLOCKING=1
   export CUDA_MANAGED_FORCE_DEVICE_ALLOC=1
   export KOKKOS_NUM_DEVICES=2
-  export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=4
-  # Avoids timeouts due to not running on separate GPUs (see #2446)
+  export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=2
+  # Avoids timeouts due to not running on separate GPUs (e.g. see #2446)
 
 fi
 
-# CMake and ninja
-module swap cmake/3.6.2 cmake/3.12.3
+# Ninja
 module load ninja/1.7.2
 
+# CMake
+#module swap cmake/3.6.2 cmake/3.12.3
+module unload cmake/3.6.2
+export PATH=/home/atdm-devops-admin/tools/waterman/cmake-3.17.2/bin:$PATH
+
 # HWLOC
 
 export ATDM_CONFIG_USE_HWLOC=OFF

cmake/std/atdm/waterman/tweaks/Tweaks.cmake

@@ -1,3 +1,11 @@
+#
+# Set up to limit running on GPUs
+#
+
+ATDM_SET_CACHE(Trilinos_AUTOGENERATE_TEST_RESOURCE_FILE ON CACHE BOOL)
+ATDM_SET_CACHE(Trilinos_CUDA_NUM_GPUS 2 CACHE STRING)
+ATDM_SET_CACHE(Trilinos_CUDA_SLOTS_PER_GPU 2 CACHE STRING)
+
 #
 # Disables across multiple builds on 'waterman'
 #

cmake/tribits/ci_support/checkin-test.py

@@ -741,7 +741,9 @@ def runProjectTestsWithCommandLineArgs(commandLineArgs, configuration = {}):
 
   clp.add_option(
     "--project-name", dest="projectName", action="store",
-    help="Set the project's name. This is used to locate various files.",
+    help="Set the project's name. This is used to locate various files."+\
+      "  If not set, then it reads the project name from the PROJECT_NAME"+\
+      " varaible set in the file SRCDIR/ProjectName.cmake.",
     default=None)
 
   clp.add_option(

cmake/tribits/core/package_arch/TribitsAddAdvancedTest.cmake

@@ -1399,6 +1399,9 @@ FUNCTION(TRIBITS_ADD_ADVANCED_TEST TEST_NAME_IN)
         PROCESSORS "${MAX_NUM_PROCESSORS_USED}")
     ENDIF()
 
+    TRIBITS_PRIVATE_ADD_TEST_ADD_ENVIRONMENT_AND_RESOURCE(${TEST_NAME}
+      ${MAX_NUM_PROCESSORS_USED})
+
     IF (SET_DISABLED_AND_MSG)
       TRIBITS_SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES DISABLED ON)
     ENDIF()

cmake/tribits/core/package_arch/TribitsAddTestHelpers.cmake

@@ -51,6 +51,21 @@ INCLUDE(AdvancedSet)
 INCLUDE(MessageWrapper)
 INCLUDE(TribitsGetCategoriesString)
 
+
+#
+# Do initialization for test helpers
+#
+# This must be run just before the packages define their tests and this macro
+# must be run in the base-level project scope.
+#
+MACRO(TRIBITS_ADD_TEST_HELPERS_INIT)
+  IF (TPL_ENABLE_CUDA)
+    SET(TRIBITS_TEST_EXTRA_ENVIRONMENT CTEST_KOKKOS_DEVICE_TYPE=gpus)
+    SET(TRIBITS_RESOURCES_PER_PROCESS gpus:1)
+  ENDIF()
+ENDMACRO()
+
+
 #
 # Wrapper function for SET_TESTS_PROPERTIES() to be used in unit testing.
 #
@@ -874,6 +889,8 @@ FUNCTION(TRIBITS_PRIVATE_ADD_TEST_POST_PROCESS_ADDED_TEST  TEST_NAME_IN
     TRIBITS_SET_TESTS_PROPERTIES(${TEST_NAME_IN} PROPERTIES DISABLED ON)
   ENDIF()
 
+  TRIBITS_PRIVATE_ADD_TEST_ADD_ENVIRONMENT_AND_RESOURCE(${TEST_NAME_IN} ${NUM_PROCS_USED_IN})
+
   TRIBITS_PRIVATE_ADD_TEST_ADD_LABEL_AND_KEYWORDS(${TEST_NAME_IN})
 
   TRIBITS_PRIVATE_ADD_TEST_PRINT_ADDED(${TEST_NAME_IN}
@@ -883,3 +900,24 @@ FUNCTION(TRIBITS_PRIVATE_ADD_TEST_POST_PROCESS_ADDED_TEST  TEST_NAME_IN
 ENDFUNCTION()
 
 
+#
+# Add environment and resource properties to a test
+#
+
+FUNCTION(TRIBITS_PRIVATE_ADD_TEST_ADD_ENVIRONMENT_AND_RESOURCE  TEST_NAME_IN
+  NUM_PROCS_USED_IN
+  )
+  IF(TRIBITS_TEST_EXTRA_ENVIRONMENT)
+    TRIBITS_SET_TEST_PROPERTY(${TEST_NAME_IN} APPEND PROPERTY ENVIRONMENT
+      "${TRIBITS_TEST_EXTRA_ENVIRONMENT}")
+  ENDIF()
+
+  IF(TRIBITS_RESOURCES_PER_PROCESS)
+    SET(NUM_PROCESSES ${NUM_PROCS_USED_IN})
+    IF(NOT NUM_PROCESSES OR NUM_PROCESSES LESS 1)
+      SET(NUM_PROCESSES 1)
+    ENDIF()
+    TRIBITS_SET_TESTS_PROPERTIES(${TEST_NAME_IN} PROPERTIES RESOURCE_GROUPS
+      "${NUM_PROCESSES},${TRIBITS_RESOURCES_PER_PROCESS}")
+  ENDIF()
+ENDFUNCTION()

cmake/tribits/core/package_arch/TribitsGenerateResourceSpecFile.cmake

@@ -0,0 +1,94 @@
+# @HEADER
+# ************************************************************************
+#
+#            TriBITS: Tribal Build, Integrate, and Test System
+#                    Copyright 2013 Sandia Corporation
+#
+# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+# the U.S. Government retains certain rights in this software.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the Corporation nor the names of the
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# ************************************************************************
+# @HEADER
+
+
+#
+# Top-level project logic to generate resources spec file
+#
+FUNCTION(TRIBITS_GENERATE_CTEST_RESOURCE_SPEC_FILE_PROJECT_LOGIC)
+  IF (${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE)
+    IF (CTEST_RESOURCE_SPEC_FILE STREQUAL CTEST_RESOURCE_SPEC_FILE_DEFAULT)
+      TRIBITS_GENERATE_CTEST_RESOURCE_SPEC_FILE()
+    ELSE()
+      MESSAGE("NOTE: The test resource file CTEST_RESOURCE_SPEC_FILE='${CTEST_RESOURCE_SPEC_FILE}'"
+        " will not be auto-generated even through"
+        " ${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE=${${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE}"
+        " because its location does not match the default"
+        " location '${CTEST_RESOURCE_SPEC_FILE_DEFAULT}'."
+        "  If you want to auto-generate this file, please clear CTEST_RESOURCE_SPEC_FILE and"
+        " reconfigure or create that file on your own and clear"
+        " ${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE."
+        )
+    ENDIF()
+  ENDIF()
+ENDFUNCTION()
+
+
+#
+# Generate resource spec file
+#
+FUNCTION(TRIBITS_GENERATE_CTEST_RESOURCE_SPEC_FILE)
+  SET(GPUS_JSON)
+  MATH(EXPR LAST_GPU "${${PROJECT_NAME}_CUDA_NUM_GPUS} - 1")
+  SET(FIRST 1)
+  FOREACH(GPU RANGE 0 ${LAST_GPU})
+    IF(NOT FIRST)
+      STRING(APPEND GPUS_JSON ",\n")
+    ENDIF()
+    SET(FIRST 0)
+    STRING(APPEND GPUS_JSON "        {
+          \"id\": \"${GPU}\",
+          \"slots\": ${${PROJECT_NAME}_CUDA_SLOTS_PER_GPU}
+        }")
+  ENDFOREACH()
+  FILE(WRITE "${CMAKE_BINARY_DIR}/ctest_resources.json" "{
+  \"version\": {
+    \"major\": 1,
+    \"minor\": 0
+  },
+  \"local\": [
+    {
+      \"gpus\": [
+${GPUS_JSON}
+      ]
+    }
+  ]
+}
+")
+ENDFUNCTION()

cmake/tribits/core/package_arch/TribitsGlobalMacros.cmake

@@ -487,6 +487,38 @@ MACRO(TRIBITS_DEFINE_GLOBAL_OPTIONS_AND_DEFINE_EXTRA_REPOS)
     "Enable explicit template instantiation in all packages that support it"
     )
 
+  ADVANCED_SET(${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE OFF
+    CACHE BOOL
+    "Auto-generate a resource spec file for use with CTest."
+    )
+
+  ADVANCED_SET(${PROJECT_NAME}_CUDA_NUM_GPUS 1
+    CACHE STRING
+    "Number of GPUS to make available in the auto-generated resource spec file."
+    )
+
+  ADVANCED_SET(${PROJECT_NAME}_CUDA_SLOTS_PER_GPU 1
+    CACHE STRING
+    "Number of slots per GPU in the auto-generated resource spec file."
+    )
+
+  SET(CTEST_RESOURCE_SPEC_FILE_DOC_EXTRA "")
+  IF (${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE)
+    SET(CTEST_RESOURCE_SPEC_FILE_DEFAULT  ${CMAKE_BINARY_DIR}/ctest_resources.json)
+    IF ("${CTEST_RESOURCE_SPEC_FILE}" STREQUAL "")
+      SET(CTEST_RESOURCE_SPEC_FILE_DOC_EXTRA
+         "  This file is autogenerated by default since ${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE=${${PROJECT_NAME}_AUTOGENERATE_TEST_RESOURCE_FILE}!" )
+    ENDIF()
+  ELSE()
+    SET(CTEST_RESOURCE_SPEC_FILE_DEFAULT "")
+  ENDIF()
+
+  ADVANCED_SET(CTEST_RESOURCE_SPEC_FILE
+    "${CTEST_RESOURCE_SPEC_FILE_DEFAULT}"
+    CACHE FILEPATH
+    "Resource spec file for CTest.${CTEST_RESOURCE_SPEC_FILE_DOC_EXTRA}"
+    )
+
   IF (USE_XSDK_DEFAULTS)
     # Need to set BUILD_SHARED_LIBS default here based on USE_XSDK_DEFAULTS
     # and not in TRIBITS_SETUP_ENV() in case there is logic in TriBITS or
@@ -2264,6 +2296,8 @@ MACRO(TRIBITS_INCLUDE_CTEST_SUPPORT)
   TRIBITS_CONFIGURE_CTEST_CUSTOM(${${PROJECT_NAME}_SOURCE_DIR}
     ${${PROJECT_NAME}_BINARY_DIR})
 
+  TRIBITS_ADD_TEST_HELPERS_INIT()
+
 ENDMACRO()
 # NOTE: The above logic with DART_TESTING_TIMEOUT is a huge hack.  For some
 # reason, on the first configure CMake will not put the local value of the

cmake/tribits/core/package_arch/TribitsProjectImpl.cmake

@@ -66,6 +66,7 @@ INCLUDE(TribitsIncludeDirectories)
 INCLUDE(TribitsFindPythonInterp)
 INCLUDE(TribitsGlobalMacros)
 INCLUDE(TribitsConfigureCTestCustom)
+INCLUDE(TribitsGenerateResourceSpecFile)
 
 INCLUDE(AdvancedSet)
 INCLUDE(AdvancedOption)
@@ -321,7 +322,13 @@ MACRO(TRIBITS_PROJECT_IMPL)
   ENDIF()
 
   #
-  # N) Show final timing and end
+  # N) Generate resource spec file if applicable
+  #
+
+  TRIBITS_GENERATE_CTEST_RESOURCE_SPEC_FILE_PROJECT_LOGIC()
+
+  #
+  # O) Show final timing and end
   #
 
   MESSAGE("")