diff --git a/cmake/ctest/drivers/atdm/ats2/local-driver.sh b/cmake/ctest/drivers/atdm/ats2/local-driver.sh index 8f12bd099d07..eaeab9c33132 100755 --- a/cmake/ctest/drivers/atdm/ats2/local-driver.sh +++ b/cmake/ctest/drivers/atdm/ats2/local-driver.sh @@ -16,6 +16,9 @@ source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME set -x +# Allow default setting for TPETRA_ASSUME_CUDA_AWARE_MPI=0 in trilinos_jsrun +unset TPETRA_ASSUME_CUDA_AWARE_MPI + atdm_run_script_on_compute_node \ $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver.sh \ $PWD/ctest-s-driver.out \ diff --git a/cmake/std/atdm/README.md b/cmake/std/atdm/README.md index 61c1808590e3..e3cdc553edb1 100644 --- a/cmake/std/atdm/README.md +++ b/cmake/std/atdm/README.md @@ -1037,12 +1037,12 @@ $ bsub -x -Is -n 20 \ ### ATS-2 -Once logged on a suppported ATS-2 system like 'vortex' (SRN), one can either -build and configure on the login node or the compute node. Make sure to setup -SSH keys as described in `/opt/VORTEX_INTRO` before trying to build on a -compute node. For example to configure, build and run the tests for the -default `cuda-debug` build for `Kokkos` (after cloning Trilinos on the -`develop` branch), do: +Once logged on a supported ATS-2 system like 'vortex' (SRN), one can either +build and configure on a login node or a compute node. Make sure to setup SSH +keys as described in `/opt/VORTEX_INTRO` before trying to build on a compute +node. For example, to configure, build and run the tests for the default +`cuda-debug` build for `Kokkos` (after cloning Trilinos on the `develop` +branch), do: ```bash $ cd / @@ -1058,7 +1058,8 @@ $ cmake -GNinja \ $ make NP=20 ``` -You may run the above commands from an interactive bsub session as well: +You may run the above commands from an interactive bsub session as well using: + ```bash $ bsub -J -W 4:00 -Is bash ``` @@ -1067,10 +1068,36 @@ CTest runs everything using the `jsrun` command. You must run jsrun from a compute node which can be acquired using the above bsub command. Once you're on a compute node, you can run ctest. For example: + ```bash $ ctest -j4 ``` +The MPI test exectuables are run by a wrapper script `trilinos_jsrun` which +calls the `jsrun` command which modifies the input arguments to accommodate +the MPI test suite in Trilinos (see the implementation of the script +`trilinos_jsrun` for details). By default, the script `trilinos_jsrun` will +set `export TPETRA_ASSUME_CUDA_AWARE_MPI=0` if `TPETRA_ASSUME_CUDA_AWARE_MPI` +is unset in the environment. Therefore, by default, the tests are run without +CUDA-aware MPI on this system. + +To explicitly **disable CUDA-aware MPI** when running the test suite, set the +environment variable: + +```bash +$ export TPETRA_ASSUME_CUDA_AWARE_MPI=0 +$ ctest -j4 +``` + +and to explicitly **enable CUDA-aware MPI** when running the test suite set: + +```bash +$ export TPETRA_ASSUME_CUDA_AWARE_MPI=1 +$ ctest -j4 +``` + +before running `ctest`. + **NOTES:** - Do NOT do `module purge` before loading the environment. Simply start off with a clean default environment on vortex. diff --git a/cmake/std/atdm/ats2/environment.sh b/cmake/std/atdm/ats2/environment.sh index bfd8b093d1d1..539277be4546 100644 --- a/cmake/std/atdm/ats2/environment.sh +++ b/cmake/std/atdm/ats2/environment.sh @@ -186,8 +186,6 @@ if [[ "$ATDM_CONFIG_COMPILER" == "CUDA-10.1.243_"* ]]; then export KOKKOS_NUM_DEVICES=4 # CTEST Settings - # TPETRA_ASSUME_CUDA_AWARE_MPI is used by cmake/std/atdm/ats2/trilinos_jsrun - export TPETRA_ASSUME_CUDA_AWARE_MPI=0 # Trilinos_CTEST_RUN_CUDA_AWARE_MPI is used by cmake/ctest/driver/atdm/ats2/local-driver.sh export Trilinos_CTEST_RUN_CUDA_AWARE_MPI=1 diff --git a/cmake/std/atdm/ats2/trilinos_jsrun b/cmake/std/atdm/ats2/trilinos_jsrun index 7b952e7cbfbb..c6289bc4c22a 100755 --- a/cmake/std/atdm/ats2/trilinos_jsrun +++ b/cmake/std/atdm/ats2/trilinos_jsrun @@ -72,7 +72,7 @@ function evaluate_jsrun_command { # Check if TPETRA_ASSUME_CUDA_AWARE_MPI is set and default to 0 if unset. if [[ "$TPETRA_ASSUME_CUDA_AWARE_MPI" != "0" ]] && [[ "$TPETRA_ASSUME_CUDA_AWARE_MPI" != "1" ]]; then echo "WARNING, you have not set TPETRA_ASSUME_CUDA_AWARE_MPI=0 or 1, defaulting to TPETRA_ASSUME_CUDA_AWARE_MPI=0" - export TPETRA_ASSUME_CUDA_AWARE=0 + export TPETRA_ASSUME_CUDA_AWARE_MPI=0 fi # Parse input arguments and modify them for jsrun diff --git a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh index 4312ded823f8..2fefbf12f49a 100644 --- a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh +++ b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh @@ -9,7 +9,6 @@ unset OMP_PLACES unset OMPI_CC unset OMPI_CXX unset OMPI_FC -unset TPETRA_ASSUME_CUDA_AWARE_MPI unset Trilinos_CTEST_RUN_CUDA_AWARE_MPI unset ATDM_CONFIG_ENABLE_SPARC_SETTINGS unset ATDM_CONFIG_USE_NINJA