Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add infrastructure for CUDA PR build #4217

Merged
merged 21 commits into from
Jan 23, 2019
Merged

Add infrastructure for CUDA PR build #4217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3a32119
Add infrastructure for testing cuda build
ZUUL42 Dec 13, 2018
7993516
Removing sems module init for cuda & fix workspace ref
ZUUL42 Jan 10, 2019
c476288
Fix MPI and sems init calls for cuda
ZUUL42 Jan 10, 2019
04bfef8
MPI_NAME -> MPI_VENDOR
ZUUL42 Jan 10, 2019
6daed62
Setting KOKKOS_ARCH
jwillenbring Jan 11, 2019
aa161c9
Fixing CUDA configuration
jwillenbring Jan 11, 2019
a5b2204
Added BLAS and LAPACK options.
jwillenbring Jan 11, 2019
cb2edca
Explicitly disabling Scotch and ParMetis.
jwillenbring Jan 11, 2019
df416a1
Adding Netcdf library configuration.
jwillenbring Jan 11, 2019
60b4424
Disabled SuperLU and boostlib
jwillenbring Jan 11, 2019
969e5a5
Disabled SEACAS for now.
jwillenbring Jan 15, 2019
915ed4a
Disabled Moertel.
jwillenbring Jan 16, 2019
14c493a
Disabling shared libs and Komplex
jwillenbring Jan 17, 2019
b5aebc2
Disabled debug to match ATDM build
jwillenbring Jan 17, 2019
1fc7808
Set Tpetra_INST_SERIAL=ON to match atdm build.
jwillenbring Jan 17, 2019
c504c36
Turning off Debug symbols.
jwillenbring Jan 17, 2019
559a7e4
Turning off secondary tested code.
jwillenbring Jan 17, 2019
5fefffa
Disable a failing test.
jwillenbring Jan 18, 2019
1fbde49
Disabled remaining failed tests.
jwillenbring Jan 18, 2019
26ce0f0
Added check for cuda build for if node is ride
ZUUL42 Jan 21, 2019
d8cad58
Added check for cuda build for if node is ride
ZUUL42 Jan 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions cmake/std/PullRequestLinuxCuda9.2TestingSettings.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
# This file contains the options needed to both run the pull request testing
# for Trilinos for the CUDA 9.2 pull request testing builds, and to reproduce
# the errors reported by those builds. Prior to using this this file, the
# appropriate set of modules must be loaded and path must be augmented.
# (See the sems/PullRequestCuda9.2TestingEnv.sh files.)

# Usage: cmake -C PullRequestLinuxCUDA9.2TestingSettings.cmake

# Misc options typically added by CI testing mode in TriBITS

# Use the below option only when submitting to the dashboard
set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing")

# Options necessary for CUDA build
set (TPL_ENABLE_MPI ON CACHE BOOL "Set by default for CUDA PR testing")
set (TPL_ENABLE_CUDA ON CACHE BOOL "Set by default for CUDA PR testing")
set (Kokkos_ENABLE_Cuda ON CACHE BOOL "Set by default for CUDA PR testing")
set (Kokkos_ENABLE_Cuda_UVM ON CACHE BOOL "Set by default for CUDA PR testing")
set (KOKKOS_ARCH Power8 CACHE STRING "Set by default for CUDA PR testing")

# TPL settings specific to CUDA build
set (TPL_BLAS_LIBRARIES "-L${BLAS_ROOT}/lib -lblas -lgfortran -lgomp -lm" CACHE STRING "Set by default for CUDA PR testing")
set (TPL_LAPACK_LIBRARIES "-L${LAPACK_ROOT}/lib -llapack -lgfortran -lgomp" CACHE STRING "Set by default for CUDA PR testing")
set (BUILD_SHARED_LIBS OFF CACHE BOOL "Set by default for CUDA PR testing")
set (Tpetra_INST_SERIAL ON CACHE BOOL "Set by default for CUDA PR testing")
set (Trilinos_ENABLE_SECONDARY_TESTED_CODE OFF CACHE BOOL "Set by default for CUDA PR testing")
set (TPL_ENABLE_Scotch OFF CACHE BOOL "Set by default for CUDA PR testing")
# Parmetis is available on ride and could be enabled for the CUDA PR build
set (TPL_ENABLE_ParMETIS OFF CACHE BOOL "Set by default for CUDA PR testing")
set (TPL_Netcdf_LIBRARIES "-L${BOOST_ROOT}/lib;-L${NETCDF_ROOT}/lib;-L${NETCDF_ROOT}/lib;-L${PNETCDF_ROOT}/lib;-L${HDF5_ROOT}/lib;${BOOST_ROOT}/lib/libboost_program_options.a;${BOOST_ROOT}/lib/libboost_system.a;${NETCDF_ROOT}/lib/libnetcdf.a;${PNETCDF_ROOT}/lib/libpnetcdf.a;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl" CACHE STRING "Set by default for CUDA PR testing")
# SuperLU is available on ride and could be enabled for the CUDA PR build
set (TPL_ENABLE_SuperLU OFF CACHE BOOL "Set by default for CUDA PR testing")
set (TPL_ENABLE_BoostLib OFF CACHE BOOL "Set by default for CUDA PR testing")
set (Trilinos_ENABLE_Moertel OFF CACHE BOOL "Disable for CUDA PR testing")
set (Trilinos_ENABLE_Komplex OFF CACHE BOOL "Disable for CUDA PR testing")

# Temporary options to clean up build
set (Trilinos_ENABLE_SEACAS OFF CACHE BOOL "Temporary disable for CUDA PR testing")
set (Trilinos_ENABLE_DEBUG OFF CACHE BOOL "Temporary disable for CUDA PR testing")
set (Trilinos_ENABLE_DEBUG_SYMBOLS OFF CACHE BOOL "Temporary disable for CUDA PR testing")
set (STK_ENABLE_TESTS OFF CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_interfacepartitionofunity_DIM2_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_interfacesets_DIM2_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_localpartitionofunitybasis_EPETRA_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_GDSWP_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_GDSWP_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_GDSWP_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_GDSWP_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_RGDSWP_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_RGDSWP_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_RGDSWP_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_RGDSWP_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB2_GDSW_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB2_GDSW_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB2_GDSW_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB2_GDSW_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB3_GDSW_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB3_GDSW_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB3_GDSW_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLBP_NB3_GDSW_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_GDSW_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_GDSW_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_GDSW_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_GDSW_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_IPOUHarmonic_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_IPOUHarmonic_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_IPOUHarmonic_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_IPOUHarmonic_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_RGDSW_DIM2_DPN1_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_RGDSW_DIM2_DPN1_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_RGDSW_DIM2_DPN2_ORD0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_laplace_TLP_RGDSW_DIM2_DPN2_ORD1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_stokes_hdf5_TLBP_GDSW_O0_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ShyLU_DDFROSch_test_thyra_xpetra_stokes_hdf5_TLBP_GDSW_O1_EPETRA_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_0ld_adv-diff-react_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_0ld_adv-diff-react_example_02_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_0ld_poisson_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_0ld_stefan-boltzmann_example_03_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_navier-stokes_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_navier-stokes_example_02_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_nonlinear-elliptic_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_nonlinear-elliptic_example_02_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_obstacle_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_example_PDE-OPT_topo-opt_poisson_example_01_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (ROL_test_elementwise_TpetraMultiVector_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_initialize_where_tpetra_initializes_kokkos_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_initialize_where_tpetra_initializes_mpi_and_user_initializes_kokkos_MPI_2_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_initialize_where_user_initializes_kokkos_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_initialize_where_user_initializes_mpi_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_ScopeGuard_where_tpetra_initializes_kokkos_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_ScopeGuard_where_tpetra_initializes_mpi_and_user_initializes_kokkos_MPI_2_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_ScopeGuard_where_user_initializes_kokkos_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TpetraCore_Core_ScopeGuard_where_user_initializes_mpi_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TrilinosCouplings_Example_Maxwell_MueLu_MPI_1_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")
set (TrilinosCouplings_Example_Maxwell_MueLu_MPI_4_DISABLE ON CACHE BOOL "Temporary disable for CUDA PR testing")

include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake")

23 changes: 23 additions & 0 deletions cmake/std/PullRequestLinuxCudaDriver.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash -l

if [ "${BSUB_CTEST_TIME_LIMIT}" == "" ] ; then
Copy link
Contributor

@william76 william76 Jan 21, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is BSUB_CTEST_TIME_LIMIT a system defined environment variable or is this a new Jenkins parameter that's being introduced?
If this doesn't exist, can it break things?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bartlettroscoe @ZUUL42 @william76

Is BSUB_CTEST_TIME_LIMIT a system defined environment variable or is this a new Jenkins parameter that's being introduced?
If this doesn't exist, can it break things?

I am not certain, but I think it is just an environment variable that can be defined by a user.
@bartlettroscoe may be able to confirm or correct this.
If it is not defined, it is set to a default value, so it shouldn't break things. The variable is specific to this build as far as current PR tests go (because ride uses bsub).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was just an env var that allowed specialization in the individual driver scripts. This came from the file:

  • Trilinos/cmake/ctest/drivers/atdm/ride/local-driver.sh

which has:

#!/bin/bash -l

if [ "${BSUB_CTEST_TIME_LIMIT}" == "" ] ; then
  export BSUB_CTEST_TIME_LIMIT=12:00
fi

...

source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME

set -x

bsub -x -Is -q $ATDM_CONFIG_QUEUE -n 16 -J $JOB_NAME -W $BSUB_CTEST_TIME_LIMIT \
  $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver.sh

Any questions about this?

export BSUB_CTEST_TIME_LIMIT=12:00
fi

if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then
export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE
fi

# comment out sh and add what we need individually.
#source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME

set -x

#TODO: review appropriate job size
bsub -x -Is -q rhel7F -n 16 -J $JOB_NAME -W $BSUB_CTEST_TIME_LIMIT \
$WORKSPACE/Trilinos/cmake/std/PullRequestLinuxDriver.sh

# NOTE: Above, this bsub command should grab a single rhel7F (Firestone,
# Dual-Socket POWER8, 8 cores per socket, K80 GPUs) node. The option '-x'
# makes sure that only this job runs on that node. The options '-n 16' and
# '-q rhel7G' should make bsub allocate a single one of these nodes.
19 changes: 14 additions & 5 deletions cmake/std/PullRequestLinuxDriver-Merge.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,23 @@ echo -e ""
: ${BUILD_NUMBER:?}
: ${WORKSPACE:?}

source /projects/sems/modulefiles/utils/sems-modules-init.sh

declare -i ierror=0
#Have to keep loading git
module load sems-git/2.10.1

cuda_regex=".*(_cuda_).*"
ride_regex=".*(ride).*"
if [[ ${JOB_BASE_NAME:?} =~ ${cuda_regex} ]]; then
if [[ ${NODE_NAME:?} =~ ${ride_regex} ]]; then
echo -e "Job is CUDA"
module load git/2.10.1
else
echo -e "ERROR: Unable to find matching environment for CUDA job not on Ride."
exit -1
fi
else
source /projects/sems/modulefiles/utils/sems-modules-init.sh
module load sems-git/2.10.1
fi

#--------------------------------------------
# Get Trilinos scripts and PR merge repo dirs
Expand Down Expand Up @@ -220,5 +231,3 @@ fi
# Return to previous directory
popd &> /dev/null
echo -e "Set CWD = `pwd`"


53 changes: 38 additions & 15 deletions cmake/std/PullRequestLinuxDriver-Test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

#
# Functions
#
#

# Test the branch constraints for a Pull Request:
# - Only pull requests from trilinos/Trilnos::develop are allowed
Expand Down Expand Up @@ -146,17 +146,27 @@ echo -e ""
: ${TRILINOS_TARGET_BRANCH:?}
: ${TRILINOS_SOURCE_SHA:?}
: ${PULLREQUESTNUM:?}
: ${NODE_NAME:?}
: ${JOB_BASE_NAME:?}
: ${BUILD_NUMBER:?}
: ${WORKSPACE:?}

source /projects/sems/modulefiles/utils/sems-modules-init.sh

declare -i ierror=0
#Have to keep loading git
module load sems-git/2.10.1


cuda_regex=".*(_cuda_).*"
ride_regex=".*(ride).*"
if [[ ${JOB_BASE_NAME:?} =~ ${cuda_regex} ]]; then
if [[ ${NODE_NAME:?} =~ ${ride_regex} ]]; then
echo -e "Job is CUDA"
module load git/2.10.1
else
echo -e "ERROR: Unable to find matching environment for CUDA job not on Ride."
exit -1
fi
else
source /projects/sems/modulefiles/utils/sems-modules-init.sh
module load sems-git/2.10.1
fi

#--------------------------------------------
# Apply Guards
Expand Down Expand Up @@ -222,7 +232,7 @@ elif [ "Trilinos_pullrequest_gcc_4.9.3" == "${JOB_BASE_NAME:?}" ] ; then
fi
elif [ "Trilinos_pullrequest_gcc_4.9.3_SERIAL" == "${JOB_BASE_NAME:?}" ] ; then
# TODO: Update this to use a 4.9.3 SERIAL testing environment script.
source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnvSERIAL.sh
source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestGCC4.9.3TestingEnvSERIAL.sh
ierror=$?
if [[ $ierror != 0 ]]; then
echo -e "There was an issue loading the gcc environment. The error code was: $ierror"
Expand All @@ -242,6 +252,13 @@ elif [ "Trilinos_pullrequest_gcc_7.3.0" == "${JOB_BASE_NAME:?}" ] ; then
echo -e "There was an issue loading the gcc environment. The error code was: $ierror"
exit $ierror
fi
elif [ "Trilinos_pullrequest_cuda_9.2" == "${JOB_BASE_NAME:?}" ] ; then
source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestCuda9.2TestingEnv.sh
ierror=$?
if [[ $ierror != 0 ]]; then
echo -e "There was an issue loading the cuda environment. The error code was: $ierror"
exit $ierror
fi
elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then
source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestIntel17.0.1TestingEnv.sh
ierror=$?
Expand All @@ -264,11 +281,18 @@ module list
# This crashes for the serial case since MPI variables are not set
# - See Issue #3625
# - wcm: bugfix #3673
regex=".*(_SERIAL)$"
if [[ ! ${JOB_BASE_NAME:?} =~ ${regex} ]]; then
echo -e "MPI type = sems-${SEMS_MPI_NAME:?}/${SEMS_MPI_VERSION:?}"
else
serial_regex=".*(_SERIAL)$"
if [[ ${JOB_BASE_NAME:?} =~ ${serial_regex} ]]; then
echo -e "Job is SERIAL"
elif [[ ${JOB_BASE_NAME:?} =~ ${cuda_regex} ]]; then
if [[ ${NODE_NAME:?} =~ ${ride_regex} ]]; then
echo -e "Job is CUDA"
echo -e "MPI type = sems-${MPI_VENDOR:?}/${MPI_VERSION:?}"
else
echo -e "MPI Vendor and Versions may not be set correctly for CUDA job not on Ride."
fi
else
echo -e "MPI type = sems-${SEMS_MPI_NAME:?}/${SEMS_MPI_VERSION:?}"
fi

CDASH_TRACK="Pull Request"
Expand Down Expand Up @@ -311,7 +335,7 @@ cmake -P packageEnables.cmake

build_name="PR-$PULLREQUESTNUM-test-$JOB_BASE_NAME-$BUILD_NUMBER"

#This should be runnable from anywhere, but all the tests so far have been from the
#This should be runnable from anywhere, but all the tests so far have been from the
#same dir the simple_testing.cmake file was in.
cd TFW_testing_single_configure_prototype &> /dev/null
echo -e "Set CWD = `pwd`"
Expand All @@ -330,6 +354,8 @@ else
CONFIG_SCRIPT=PullRequestLinuxGCC7.2.0TestingSettings.cmake
elif [ "Trilinos_pullrequest_gcc_7.3.0" == "${JOB_BASE_NAME:?}" ]; then
CONFIG_SCRIPT=PullRequestLinuxGCC7.3.0TestingSettings.cmake
elif [ "Trilinos_pullrequest_cuda_9.2" == "${JOB_BASE_NAME:?}" ]; then
CONFIG_SCRIPT=PullRequestLinuxCuda9.2TestingSettings.cmake
fi
fi

Expand All @@ -354,6 +380,3 @@ fi
# Reset to known directory location.
# - ${WORKSPACE} is set by Jenkins and points to the root workspace directory.
cd ${WORKSPACE:?}



20 changes: 20 additions & 0 deletions cmake/std/sems/PullRequestCuda9.2TestingEnv.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# This script can be used to load the appropriate environment for the
# PR build on ride using CUDA.

# usage: $ source PullRequestCUDA9.2TestingEnv.sh

#No SEMS NFS mount on ride
#source /projects/sems/modulefiles/utils/sems-modules-init.sh
module load git/2.10.1
module load devpack/20180521/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88
module swap openblas/0.2.20/gcc/7.2.0 netlib/3.8.0/gcc/7.2.0
#export OMPI_CXX=`which g++`
export OMPI_CXX=$WORKSPACE/Trilinos/packages/kokkos/bin/nvcc_wrapper
export OMPI_CC=`which gcc`
export OMPI_FC=`which gfortran`
export NVCC_WRAPPER_DEFAULT_COMPILER=`which g++`
export CUDA_LAUNCH_BLOCKING=1

# Use manually installed cmake and ninja to try to avoid module loading
# problems (see TRIL-208)
export PATH=/ascldap/users/rabartl/install/white-ride/cmake-3.11.2/bin:/ascldap/users/rabartl/install/white-ride/ninja-1.8.2/bin:$PATH