From 7ca2ddc854e408d64ea75dab61edd6df126e234f Mon Sep 17 00:00:00 2001 From: Jeffrey Edward Forster Date: Tue, 15 Oct 2019 13:12:11 -0600 Subject: [PATCH 01/34] Added files needed for the cts1 env --- ...nos-atdm-cts1-intel-debug-openmp-panzer.sh | 6 ++++ .../Trilinos-atdm-cts1-intel-debug-openmp.sh | 6 ++++ ...linos-atdm-cts1-intel-opt-openmp-panzer.sh | 6 ++++ .../Trilinos-atdm-cts1-intel-opt-openmp.sh | 6 ++++ cmake/ctest/drivers/atdm/cts1/local-driver.sh | 21 ++++++++++++++ cmake/std/atdm/cts1/all_supported_builds.sh | 7 +++++ cmake/std/atdm/cts1/environment.sh | 29 +++++++++++++++++++ .../cts1/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake | 2 ++ .../tweaks/INTEL_RELEASE_OPENMP_BDW.cmake | 2 ++ 9 files changed, 85 insertions(+) create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh create mode 100755 cmake/ctest/drivers/atdm/cts1/local-driver.sh create mode 100644 cmake/std/atdm/cts1/all_supported_builds.sh create mode 100755 cmake/std/atdm/cts1/environment.sh create mode 100644 cmake/std/atdm/cts1/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake create mode 100644 cmake/std/atdm/cts1/tweaks/INTEL_RELEASE_OPENMP_BDW.cmake diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh new file mode 100755 index 000000000000..960ca6bcd266 --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh @@ -0,0 +1,6 @@ +#!/bin/bash +export SALLOC_CTEST_TIME_LIMIT_MINUTES=0:45:00 +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=ATDM +fi +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh new file mode 100755 index 000000000000..627fd02f47ae --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=ATDM +fi +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh new file mode 100755 index 000000000000..b7f92d619b02 --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=ATDM +fi +export SALLOC_CTEST_TIME_LIMIT_MINUTES=0:30:00 +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh new file mode 100755 index 000000000000..627fd02f47ae --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=ATDM +fi +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/local-driver.sh b/cmake/ctest/drivers/atdm/cts1/local-driver.sh new file mode 100755 index 000000000000..0d8a13048b98 --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/local-driver.sh @@ -0,0 +1,21 @@ +#!/bin/bash -l + +if [ "${SALLOC_CTEST_TIME_LIMIT_MINUTES}" == "" ] ; then + SALLOC_CTEST_TIME_LIMIT_MINUTES=1:30:00 + # This is just running tests, not the entire build! +fi + +if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then + export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE +fi + +set -x + +source $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-config-build.sh + +set -x + +atdm_run_script_on_compute_node \ + $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-test.sh \ + $PWD/ctest-s-driver-test.out \ + ${SALLOC_CTEST_TIME_LIMIT_MINUTES} diff --git a/cmake/std/atdm/cts1/all_supported_builds.sh b/cmake/std/atdm/cts1/all_supported_builds.sh new file mode 100644 index 000000000000..af6d0a76d4d5 --- /dev/null +++ b/cmake/std/atdm/cts1/all_supported_builds.sh @@ -0,0 +1,7 @@ + +export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-cts1- + +export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=( + intel-debug-openmp + intel-opt-openmp + ) diff --git a/cmake/std/atdm/cts1/environment.sh b/cmake/std/atdm/cts1/environment.sh new file mode 100755 index 000000000000..948550be2dba --- /dev/null +++ b/cmake/std/atdm/cts1/environment.sh @@ -0,0 +1,29 @@ +################################################################################ +# +# Set up env on cts1 for ATMD builds of Trilinos +# +# This source script gets the settings from the ATDM_CONFIG_BUILD_NAME var. +# +################################################################################ + +# Chama and cts1 jobs all use the same environmnet changes to the +# sourced script below will impact jobs on both of those +# machines. please be mindful of this when making changes + +if [ "$ATDM_CONFIG_KOKKOS_ARCH" == "DEFAULT" ] ; then + export ATDM_CONFIG_KOKKOS_ARCH=BDW +fi + +if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then + echo "***" + echo "*** ERROR: KOKKOS_ARCH=$ATDM_CONFIG_KOKKOS_ARCH is not a valid option on this system." + echo "*** '$ATDM_CONFIG_KOKKOS_ARCH' appears in $ATDM_CONFIG_BUILD_NAME which then sets the KOKKOS_ARCH" + echo "*** on cts1 'BDW' is the only valid KOKKOS_ARCH. If no KOKKOS_ARCH is specified then" + echo "*** 'BDW' will be used by default" + echo "***" + return +fi + +export ATDM_CONFIG_ENABLE_SPARC_SETTINGS=ON + +source $ATDM_SCRIPT_DIR/common/toss3/environment.sh diff --git a/cmake/std/atdm/cts1/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake b/cmake/std/atdm/cts1/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake new file mode 100644 index 000000000000..ee3ffaba0139 --- /dev/null +++ b/cmake/std/atdm/cts1/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake @@ -0,0 +1,2 @@ +# Disable test that is failing or timing out in this build (see #2751) +ATDM_SET_ENABLE(PanzerAdaptersSTK_MixedPoissonExample-ConvTest-Hex-Order-3_DISABLE ON) diff --git a/cmake/std/atdm/cts1/tweaks/INTEL_RELEASE_OPENMP_BDW.cmake b/cmake/std/atdm/cts1/tweaks/INTEL_RELEASE_OPENMP_BDW.cmake new file mode 100644 index 000000000000..ee3ffaba0139 --- /dev/null +++ b/cmake/std/atdm/cts1/tweaks/INTEL_RELEASE_OPENMP_BDW.cmake @@ -0,0 +1,2 @@ +# Disable test that is failing or timing out in this build (see #2751) +ATDM_SET_ENABLE(PanzerAdaptersSTK_MixedPoissonExample-ConvTest-Hex-Order-3_DISABLE ON) From 88d1aeee896691dee2d243216282caf8fe53a378 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Mon, 28 Oct 2019 13:10:29 -0600 Subject: [PATCH 02/34] added enviroment_new.sh to toss3 for cts1 builds --- .../std/atdm/common/toss3/environment_new.sh | 193 ++++++++++++++++++ cmake/std/atdm/cts1/environment.sh | 4 +- 2 files changed, 194 insertions(+), 3 deletions(-) create mode 100755 cmake/std/atdm/common/toss3/environment_new.sh diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh new file mode 100755 index 000000000000..15efa8dac0e5 --- /dev/null +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -0,0 +1,193 @@ +################################################################################ +# +# Set up env on toss3 (chama and serrano) for ATMD builds of Trilinos +# +# This source script gets the settings from the ATDM_CONFIG_BUILD_NAME var. +# +################################################################################ + +if [ "$ATDM_CONFIG_COMPILER" == "DEFAULT" ] ; then + export ATDM_CONFIG_COMPILER=INTEL +fi + +echo "Using toss3 compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_CONFIG_BUILD_TYPE code with Kokkos node type $ATDM_CONFIG_NODE_TYPE" + +export ATDM_CONFIG_ENABLE_SPARC_SETTINGS=ON +export ATDM_CONFIG_USE_NINJA=ON +export ATDM_CONFIG_BUILD_COUNT=8 +# export ATDM_CONFIG_CMAKE_JOB_POOL_LINK=2 +# NOTE: Above, currently setting CMAKE_JOB_POOL_LINK results in a build +# failures with Ninja. See https://gitlab.kitware.com/snl/project-1/issues/60 + +# We do this twice since sems modules are wacked and we get errors to the screen on a purge +# The second purge will catch any real errors with purging ... +module purge &> /dev/null +module purge +. /projects/sems/modulefiles/utils/sems-modules-init.sh +module load sems-env +module load atdm-env +module load atdm-ninja_fortran/1.7.2 + +if [ "$ATDM_CONFIG_NODE_TYPE" == "OPENMP" ] ; then + export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=8 + export OMP_NUM_THREADS=2 +else + export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=16 +fi + +if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then + module load intel/18.0.2.199 + module load openmpi-intel/2.0 + + module load sparc-cmake/3.12.3 + + export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-18.0.2 + export CGNS_ROOT=/projects/sparc/tpls/cts1-bdw/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export HDF5_ROOT=/projects/sparc/tpls/cts1-bdw/hdf5-1.10.5/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export METIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export NETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export PARMETIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + + export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/include + export ATDM_CONFIG_SUPERLUDIST_LIBS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/lib64 + export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" + + export OMPI_CXX=`which icpc` + export OMPI_CC=`which icc` + export OMPI_FC=`which ifort` + export ATDM_CONFIG_LAPACK_LIBS="-mkl" + export ATDM_CONFIG_BLAS_LIBS="-mkl" +else + echo + echo "***" + echo "*** ERROR: COMPILER=$ATDM_CONFIG_COMPILER is not supported on this system!" + echo "***" + return +fi + +export ATDM_CONFIG_USE_HWLOC=OFF +export ATDM_CONFIG_HDF5_LIBS="-L${HDF5_ROOT}/lib;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl" +export ATDM_CONFIG_NETCDF_LIBS="-L${BOOST_ROOT}/lib;-L${NETCDF_ROOT}/lib;-L${PNETCDF_ROOT}/lib;-L${HDF5_ROOT}/lib;${BOOST_ROOT}/lib/libboost_program_options.a;${BOOST_ROOT}/lib/libboost_system.a;${NETCDF_ROOT}/lib/libnetcdf.a;${NETCDF_ROOT}/lib/libpnetcdf.a;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl;-lcurl" + +# not sure what below does. It was in the original environment script +#unset ATTB_ENV + +# Set MPI wrappers +export MPICC=`which mpicc` +export MPICXX=`which mpicxx` +export MPIF90=`which mpif90` + +export ATDM_CONFIG_MPI_EXEC=srun +export ATDM_CONFIG_MPI_PRE_FLAGS="--mpi=pmi2;--ntasks-per-node;36" +export ATDM_CONFIG_MPI_EXEC_NUMPROCS_FLAG=--ntasks + +# Set the default compilers +export CC=mpicc +export CXX=mpicxx +export FC=mpif77 +export F90=mpif90 + +export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE + + +# +# Run a script on the compute node send STDOUT and STDERR output to a file +# while also echo output to the console. The primary purpose is to run the +# tests on the compute node. +# +# Usage: +# +# atdm_run_script_on_comput_node \ +# [] [] +# +# If and/or are not given, then defaults are provided that +# work for the Jenkins driver process. +# +# In this case, sbatch is used to run the script but it also sends ouptut to +# STDOUT in real-time while it is running in addition to writing to the +# . The job name for the sbatch script is taken from the env var +# 'ATDM_CONFIG_BUILD_NAME'. This works for local builds since ATDM_CONFIG_BUILD_NAME. +# +# Note that you can pass in the script to run with arguments such as with +# " " and it will work. But note that this has to +# be bash script that 'sbatch' can copy and run form a temp location and it +# still has to work. So the script has to use absolute directory paths, not +# relative paths or asume sym links, etc. +# +function atdm_run_script_on_compute_node { + + set +x + + script_to_run=$1 + output_file=$2 + timeout_input=$3 + account_input=$4 + + echo + echo "***" + echo "*** atdm_run_script_on_compute_node '${script_to_run}' '${output_file}' '${timeout_input}' '${account_input}'" + echo "***" + echo + + if [ "${timeout_input}" == "" ] ; then + timeout=1:30:00 + else + timeout=${timeout_input} + fi + + if [ "${account_input}" == "" ] ; then + account=fy150090 + else + account=${account_input} + fi + + if [ -e $output_file ] ; then + echo "Remove existing file $output_file" + rm $output_file + fi + echo "Create empty file $output_file" + touch $output_file + + echo + echo "Running '$script_to_run' using sbatch in the background ..." + set -x + sbatch --output=$output_file --wait -N1 --time=${timeout} \ + -J $ATDM_CONFIG_BUILD_NAME --account=${account} ${script_to_run} & + SBATCH_PID=$! + set +x + + echo + echo "Tailing output file $output_file in the background ..." + set -x + tail -f $output_file & + TAIL_BID=$! + set +x + + echo + echo "Waiting for SBATCH_PID=$SBATCH_PID ..." + wait $SBATCH_PID + + echo + echo "Kill TAIL_BID=$TAIL_BID" + kill -s 9 $TAIL_BID + + echo + echo "Finished running ${script_to_run}!" + echo + +} + +export -f atdm_run_script_on_compute_node + +# NOTE: The above function is implemented in this way using 'sbatch' so that +# we can avoid using 'salloc' which is belived to cause ORTE errors. But we +# still want to see live ouput from the script so that we can report it on +# Jenkins. Therefore, the above approach is to use 'sbatch' and write its +# output to a known file-name. Then, we use `tail -f` to print that file as +# it gets filled in from the 'sbatch' command. The 'sbatch' command is run +# with --wait but is backgrouned to allow this to happen. Then we wait for +# the 'sbatch' command to complete and then we kill the 'tail -f' command. +# That might seem overly complex but that gets the job done. + + diff --git a/cmake/std/atdm/cts1/environment.sh b/cmake/std/atdm/cts1/environment.sh index 948550be2dba..19fa5bd5d863 100755 --- a/cmake/std/atdm/cts1/environment.sh +++ b/cmake/std/atdm/cts1/environment.sh @@ -24,6 +24,4 @@ if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then return fi -export ATDM_CONFIG_ENABLE_SPARC_SETTINGS=ON - -source $ATDM_SCRIPT_DIR/common/toss3/environment.sh +source $ATDM_SCRIPT_DIR/common/toss3/environment_new.sh From 3a43e8548e366714c4546e8c4aa93028e1f2ccb3 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Mon, 28 Oct 2019 13:26:22 -0600 Subject: [PATCH 03/34] updated superlu variable to point to the library file instead of just the directory --- cmake/std/atdm/common/toss3/environment_new.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 15efa8dac0e5..f62d9c436ea6 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -50,7 +50,7 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/include - export ATDM_CONFIG_SUPERLUDIST_LIBS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/lib64 + export ATDM_CONFIG_SUPERLUDIST_LIBS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/lib64/libsuperlu_dist.a export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" export OMPI_CXX=`which icpc` From 976adfcc258c2edf53d8042af95d17ade0111359 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Mon, 28 Oct 2019 16:13:15 -0600 Subject: [PATCH 04/34] Added mkl module --- cmake/std/atdm/common/toss3/environment_new.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index f62d9c436ea6..10408b7cb36e 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -38,6 +38,7 @@ fi if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then module load intel/18.0.2.199 module load openmpi-intel/2.0 + module load mkl/18.0.5.274 module load sparc-cmake/3.12.3 From d1ee9b88c1bab0c6695f5814bc59779a5f4b267b Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Thu, 7 Nov 2019 07:13:04 -0700 Subject: [PATCH 05/34] Updated the system name to cts1 for serrano, eclipse and ghost --- cmake/std/atdm/utils/get_known_system_info.sh | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index d681c35fa251..9fc9c0ea97b6 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -61,6 +61,52 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=( knownSystemNameInBuildName=`get_knownSystemNameInBuildName` #echo "knownSystemNameInBuildName = '${knownSystemNameInBuildName}'" +ATDM_IS_CEE_RHEL6_MACHINE= + +# Specifically named systems +if [[ $ATDM_CONFIG_REAL_HOSTNAME == "hansen"* ]] ; then + ATDM_HOSTNAME=hansen + ATDM_SYSTEM_NAME=shiller +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "shiller"* ]] ; then + ATDM_HOSTNAME=shiller + ATDM_SYSTEM_NAME=shiller +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "white"* ]] ; then + ATDM_HOSTNAME=white + ATDM_SYSTEM_NAME=ride +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "ride"* ]] ; then + ATDM_HOSTNAME=ride + ATDM_SYSTEM_NAME=ride +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "mutrino"* ]] ; then + ATDM_HOSTNAME=mutrino + ATDM_SYSTEM_NAME=mutrino +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "waterman"* ]] ; then + ATDM_HOSTNAME=waterman + ATDM_SYSTEM_NAME=waterman + +# cts1 systems +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "serrano"* ]] \ + || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ ser[0-9]+ ]] ; then + ATDM_HOSTNAME=serrano + ATDM_SYSTEM_NAME=cts1 +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "eclipse"* ]] \ + || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ ec[0-9]+ ]] ; then + ATDM_HOSTNAME=eclipse + ATDM_SYSTEM_NAME=cts1 +elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "ghost"* ]] \ + || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ gho[0-9]+ ]] ; then + ATDM_HOSTNAME=ghost + ATDM_SYSTEM_NAME=cts1 + +# tlcc2 systems +elif [[ $SNLSYSTEM == "tlcc2"* ]] ; then + ATDM_SYSTEM_NAME=tlcc2 + if [[ $SNLCLUSTER == "" ]] ; then + ATDM_HOSTNAME=$ATDM_CONFIG_REAL_HOSTNAME + else + ATDM_HOSTNAME=$SNLCLUSTER + fi + + # System name and hostname matching systemNameTypeMatchedList=() # In order of match preference unset systemNameTypeMatchedListHostNames @@ -108,7 +154,7 @@ elif [[ $realHostname == "ghost"* ]] || [[ $realHostname =~ gho[0-9]+ ]] ; then elif [[ $realHostname == "attaway"* ]] || [[ $realHostname =~ swa[0-9]+ ]] ; then hostnameMatch=attaway hostnameMatchSystemName=serrano - + # End specifically named systems fi From e974215f3860cd94bdeb54251b84ae73bc163ef7 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Thu, 7 Nov 2019 08:32:11 -0700 Subject: [PATCH 06/34] removed leftover merge line --- cmake/std/atdm/utils/get_known_system_info.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index 9fc9c0ea97b6..621d762a26fa 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -243,6 +243,12 @@ if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${knownSystemNameInBuildName}" != "" assert_selected_system_matches_known_system_type_mathces || return fi +# D.2) Second, go with matches based on hostname +if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${hostnameMatch}" != "" ]] ; then + ATDM_SYSTEM_NAME=${hostnameMatchSystemName} + ATDM_HOSTNAME=${hostnameMatch} +fi + # D.2) Last, go with the hostname match or matching system type if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${systemNameTypeMatchedList}" != "" ]] ; then ATDM_SYSTEM_NAME=${systemNameTypeMatchedList[0]} # First matching system type is preferred! From b912e7b9620476fa1fe967e5ddcc6311a76bd4a2 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Fri, 15 Nov 2019 14:26:08 -0700 Subject: [PATCH 07/34] ATDM: get_known_system_info.sh - Add missing fi. --- cmake/std/atdm/utils/get_known_system_info.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index 621d762a26fa..ea926b8af187 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -105,6 +105,7 @@ elif [[ $SNLSYSTEM == "tlcc2"* ]] ; then else ATDM_HOSTNAME=$SNLCLUSTER fi +fi # System name and hostname matching From 0836c5ab9b1160588df8e622e41b03f9d952a224 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Tue, 19 Nov 2019 06:46:21 -0700 Subject: [PATCH 08/34] updated atdm builds for cts1 machines Conflicts: cmake/std/atdm/utils/get_known_system_info.sh --- .../std/atdm/common/toss3/environment_new.sh | 4 ++ cmake/std/atdm/utils/get_known_system_info.sh | 56 ++----------------- 2 files changed, 9 insertions(+), 51 deletions(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 10408b7cb36e..5485c92c4665 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -44,11 +44,15 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-18.0.2 export CGNS_ROOT=/projects/sparc/tpls/cts1-bdw/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export EUCLID_ROOT=/projects/sparc/tpls/cts1-bdw/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export HDF5_ROOT=/projects/sparc/tpls/cts1-bdw/hdf5-1.10.5/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export LIBHIO_ROOT=/projects/sparc/tpls/cts1-bdw/libhio-1.4.1.2/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export METIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export NETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export PARMETIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export SGM_ROOT=/projects/sparc/tpls/cts1-bdw/sgm-19.30/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export SUPERLUDIST_ROOT=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/include export ATDM_CONFIG_SUPERLUDIST_LIBS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/lib64/libsuperlu_dist.a diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index ea926b8af187..45e7d3529beb 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -46,7 +46,7 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=( ride mutrino # Will be repalced by 'ats1' waterman - serrano # Will be replaced by 'cts1' + cts1 tlcc2 sems-rhel7 sems-rhel6 @@ -63,51 +63,6 @@ knownSystemNameInBuildName=`get_knownSystemNameInBuildName` ATDM_IS_CEE_RHEL6_MACHINE= -# Specifically named systems -if [[ $ATDM_CONFIG_REAL_HOSTNAME == "hansen"* ]] ; then - ATDM_HOSTNAME=hansen - ATDM_SYSTEM_NAME=shiller -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "shiller"* ]] ; then - ATDM_HOSTNAME=shiller - ATDM_SYSTEM_NAME=shiller -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "white"* ]] ; then - ATDM_HOSTNAME=white - ATDM_SYSTEM_NAME=ride -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "ride"* ]] ; then - ATDM_HOSTNAME=ride - ATDM_SYSTEM_NAME=ride -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "mutrino"* ]] ; then - ATDM_HOSTNAME=mutrino - ATDM_SYSTEM_NAME=mutrino -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "waterman"* ]] ; then - ATDM_HOSTNAME=waterman - ATDM_SYSTEM_NAME=waterman - -# cts1 systems -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "serrano"* ]] \ - || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ ser[0-9]+ ]] ; then - ATDM_HOSTNAME=serrano - ATDM_SYSTEM_NAME=cts1 -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "eclipse"* ]] \ - || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ ec[0-9]+ ]] ; then - ATDM_HOSTNAME=eclipse - ATDM_SYSTEM_NAME=cts1 -elif [[ $ATDM_CONFIG_REAL_HOSTNAME == "ghost"* ]] \ - || [[ $ATDM_CONFIG_REAL_HOSTNAME =~ gho[0-9]+ ]] ; then - ATDM_HOSTNAME=ghost - ATDM_SYSTEM_NAME=cts1 - -# tlcc2 systems -elif [[ $SNLSYSTEM == "tlcc2"* ]] ; then - ATDM_SYSTEM_NAME=tlcc2 - if [[ $SNLCLUSTER == "" ]] ; then - ATDM_HOSTNAME=$ATDM_CONFIG_REAL_HOSTNAME - else - ATDM_HOSTNAME=$SNLCLUSTER - fi -fi - - # System name and hostname matching systemNameTypeMatchedList=() # In order of match preference unset systemNameTypeMatchedListHostNames @@ -145,17 +100,16 @@ elif [[ $realHostname == "waterman"* ]] ; then # Specifically named cts1 systems elif [[ $realHostname == "serrano"* ]] || [[ $realHostname =~ ser[0-9]+ ]] ; then hostnameMatch=serrano - hostnameMatchSystemName=serrano + hostnameMatchSystemName=cts1 elif [[ $realHostname == "eclipse"* ]] || [[ $realHostname =~ ec[0-9]+ ]] ; then hostnameMatch=eclipse - hostnameMatchSystemName=serrano + hostnameMatchSystemName=cts1 elif [[ $realHostname == "ghost"* ]] || [[ $realHostname =~ gho[0-9]+ ]] ; then hostnameMatch=ghost - hostnameMatchSystemName=serrano + hostnameMatchSystemName=cts1 elif [[ $realHostname == "attaway"* ]] || [[ $realHostname =~ swa[0-9]+ ]] ; then hostnameMatch=attaway - hostnameMatchSystemName=serrano - + hostnameMatchSystemName=cts1 # End specifically named systems fi From 67cd92652474e982b5a919a370cb00785af7b101 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Tue, 19 Nov 2019 10:38:12 -0700 Subject: [PATCH 09/34] ATDM: Updated get_known_system_info.sh script (ATDV-212) - Added cts1 and "this" hostname to known system array. --- cmake/std/atdm/utils/get_known_system_info.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index 45e7d3529beb..9f9bcc7f1ff8 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -137,6 +137,13 @@ if [[ $SNLSYSTEM == "tlcc2"* ]] ; then systemNameTypeMatchedListHostNames[tlcc2]=$SNLCLUSTER fi +# CTS1 systems +if [[ $SNLSYSTEM == "cts1" ]] ; then + systemNameTypeMatchedList+=(cts1) + systemNameTypeMatchedListHostNames[cts1]=$SNLCLUSTER +fi + + # SEMS RHEL6 and RHEL7 systems if [[ "${SEMS_PLATFORM}" == "rhel6-x86_64" ]] ; then systemNameTypeMatchedList+=(sems-rhel6) From 52ea42b9283d9b9d7bc00619873b1e09083978d1 Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Tue, 19 Nov 2019 11:36:22 -0700 Subject: [PATCH 10/34] fixed some issues from the last merge --- cmake/std/atdm/common/toss3/environment_new.sh | 7 +++++-- cmake/std/atdm/utils/get_known_system_info.sh | 10 +--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 5485c92c4665..ab909b0aa9ef 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -54,10 +54,13 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then export SGM_ROOT=/projects/sparc/tpls/cts1-bdw/sgm-19.30/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export SUPERLUDIST_ROOT=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/include - export ATDM_CONFIG_SUPERLUDIST_LIBS=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3/lib64/libsuperlu_dist.a + export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include + export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" + export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH} + export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} + export OMPI_CXX=`which icpc` export OMPI_CC=`which icc` export OMPI_FC=`which ifort` diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index 9f9bcc7f1ff8..dc7c2297e98a 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -61,8 +61,6 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=( knownSystemNameInBuildName=`get_knownSystemNameInBuildName` #echo "knownSystemNameInBuildName = '${knownSystemNameInBuildName}'" -ATDM_IS_CEE_RHEL6_MACHINE= - # System name and hostname matching systemNameTypeMatchedList=() # In order of match preference unset systemNameTypeMatchedListHostNames @@ -205,12 +203,6 @@ if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${knownSystemNameInBuildName}" != "" assert_selected_system_matches_known_system_type_mathces || return fi -# D.2) Second, go with matches based on hostname -if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${hostnameMatch}" != "" ]] ; then - ATDM_SYSTEM_NAME=${hostnameMatchSystemName} - ATDM_HOSTNAME=${hostnameMatch} -fi - # D.2) Last, go with the hostname match or matching system type if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${systemNameTypeMatchedList}" != "" ]] ; then ATDM_SYSTEM_NAME=${systemNameTypeMatchedList[0]} # First matching system type is preferred! @@ -230,4 +222,4 @@ if [[ $ATDM_SYSTEM_NAME != "" ]] ; then export ATDM_CONFIG_CDASH_HOSTNAME=$ATDM_HOSTNAME export ATDM_CONFIG_SYSTEM_NAME=$ATDM_SYSTEM_NAME export ATDM_CONFIG_SYSTEM_DIR=${ATDM_CONFIG_SCRIPT_DIR}/${ATDM_CONFIG_SYSTEM_NAME} -fi +fi \ No newline at end of file From ba07917a884aa2a99270000e95e594f050e0ca64 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Tue, 19 Nov 2019 12:40:32 -0700 Subject: [PATCH 11/34] ATDM: Updated environment_new.sh and cts/custom_builds.sh (ATDV-212) - Added cts1 support for intel-19 and ompi-4. --- .../std/atdm/common/toss3/environment_new.sh | 59 ++++++++++++++----- cmake/std/atdm/cts1/custom_builds.sh | 36 +++++++++++ 2 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 cmake/std/atdm/cts1/custom_builds.sh diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index ab909b0aa9ef..a23e879df7d9 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -35,11 +35,10 @@ else export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=16 fi -if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then +if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then module load intel/18.0.2.199 module load openmpi-intel/2.0 module load mkl/18.0.5.274 - module load sparc-cmake/3.12.3 export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-18.0.2 @@ -60,12 +59,37 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL" ]; then export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} +elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then + module load intel/19.0.5.281 + module load openmpi-intel/4.0 + module load mkl/18.0.5.274 + module load sparc-cmake/3.12.3 + + export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux + export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-19.0.5 + export COMPILER_ROOT=/projects/global/toss3/compilers/intel/intel_2017/compilers_and_libraries_2019.5.281/linux + export HDF5_ROOT=/projects/sparc/tpls/cts1-bdw/hdf5-1.10.5/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export SPARC_HDF5=hdf5-1.10.5 + export CGNS_ROOT=/projects/sparc/tpls/cts1-bdw/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export NETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export LIBHIO_ROOT=/projects/sparc/tpls/cts1-bdw/libhio-1.4.1.2/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export METIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export PARMETIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export SUPERLUDIST_ROOT=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export TRILINOS_BASE=/projects/sparc/tpls/cts1-bdw/Trilinos/2019-10-01/00000001/cts1-bdw_intel-19.0.5_openmp_openmpi-4.0.1 + export SGM_ROOT=/projects/sparc/tpls/cts1-bdw/sgm-19.30/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export EUCLID_ROOT=/projects/sparc/tpls/cts1-bdw/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + + + export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include + export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a + export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" + + export PATH=/usr/tce/packages/gcc/gcc-4.9.3/bin:${PATH} + export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-4.9.3/lib64:${LD_LIBRARY_PATH} - export OMPI_CXX=`which icpc` - export OMPI_CC=`which icc` - export OMPI_FC=`which ifort` - export ATDM_CONFIG_LAPACK_LIBS="-mkl" - export ATDM_CONFIG_BLAS_LIBS="-mkl" + export F77=mpif77 else echo echo "***" @@ -74,6 +98,13 @@ else return fi + +export OMPI_CXX=`which icpc` +export OMPI_CC=`which icc` +export OMPI_FC=`which ifort` +export ATDM_CONFIG_LAPACK_LIBS="-mkl" +export ATDM_CONFIG_BLAS_LIBS="-mkl" + export ATDM_CONFIG_USE_HWLOC=OFF export ATDM_CONFIG_HDF5_LIBS="-L${HDF5_ROOT}/lib;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl" export ATDM_CONFIG_NETCDF_LIBS="-L${BOOST_ROOT}/lib;-L${NETCDF_ROOT}/lib;-L${PNETCDF_ROOT}/lib;-L${HDF5_ROOT}/lib;${BOOST_ROOT}/lib/libboost_program_options.a;${BOOST_ROOT}/lib/libboost_system.a;${NETCDF_ROOT}/lib/libnetcdf.a;${NETCDF_ROOT}/lib/libpnetcdf.a;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl;-lcurl" @@ -149,14 +180,14 @@ function atdm_run_script_on_compute_node { else account=${account_input} fi - + if [ -e $output_file ] ; then echo "Remove existing file $output_file" rm $output_file fi echo "Create empty file $output_file" touch $output_file - + echo echo "Running '$script_to_run' using sbatch in the background ..." set -x @@ -164,22 +195,22 @@ function atdm_run_script_on_compute_node { -J $ATDM_CONFIG_BUILD_NAME --account=${account} ${script_to_run} & SBATCH_PID=$! set +x - + echo echo "Tailing output file $output_file in the background ..." set -x tail -f $output_file & TAIL_BID=$! set +x - + echo echo "Waiting for SBATCH_PID=$SBATCH_PID ..." wait $SBATCH_PID - + echo echo "Kill TAIL_BID=$TAIL_BID" kill -s 9 $TAIL_BID - + echo echo "Finished running ${script_to_run}!" echo @@ -197,5 +228,3 @@ export -f atdm_run_script_on_compute_node # with --wait but is backgrouned to allow this to happen. Then we wait for # the 'sbatch' command to complete and then we kill the 'tail -f' command. # That might seem overly complex but that gets the job done. - - diff --git a/cmake/std/atdm/cts1/custom_builds.sh b/cmake/std/atdm/cts1/custom_builds.sh new file mode 100644 index 000000000000..ec87bc81e9a0 --- /dev/null +++ b/cmake/std/atdm/cts1/custom_builds.sh @@ -0,0 +1,36 @@ +# +# Custom builds for cee-rhel6 env +# +# NOTE: This file gets sourced in atdm/utils/set_build_options.sh before the +# default grep logic is applied. +# + +# Custom compiler selection logic + +# TODO: Check with EMPIRE developers about default compiler/mpi ver +if [[ $ATDM_CONFIG_BUILD_NAME == *"intel-19.0.5-openmpi-4.0.1"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-19.0.5_openmpi-4.0.1"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-19.0.5"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-19"* ]] \ + ; then + export ATDM_CONFIG_COMPILER=INTEL-19.0.5_OPENMPI-4.0.1 +elif [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2-openmpi-2.0.3"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2_openmpi-2.0.3"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"intel"* ]] \ + || [[ $ATDM_CONFIG_BUILD_NAME == *"default" ]] \ + ; then + export ATDM_CONFIG_COMPILER=INTEL-18.0.2_OPENMPI-2.0.3 +else + echo + echo "***" + echo "*** ERROR: A supported compiler was not selected for 'cts1' env" + echo "***" + echo "*** Supported compilers include:" + echo "***" + echo "**** intel-18.0.2_openmpi-2.0.3 (default)" + echo "**** intel-19.0.5_openmpi-4.0.1" + echo "***" + return + +fi From 46e61c6ad02b4b4a70401808bb2bbc1c17eccddf Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Tue, 19 Nov 2019 13:56:33 -0700 Subject: [PATCH 12/34] ATDM: Added Trilinos-atdm-cts1-intel-19-opt-openmp.sh (ATDV-212) --- .../cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh new file mode 100755 index 000000000000..627fd02f47ae --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=ATDM +fi +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh From 40c8d81f8081aff75d79eec56fe191d2bd2ed9d0 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Wed, 20 Nov 2019 10:40:54 -0700 Subject: [PATCH 13/34] ATMD: Implemented PR feedback (ATDV-212). --- .../drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh | 6 ------ .../drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh | 6 ------ cmake/std/atdm/common/toss3/environment_new.sh | 2 -- 3 files changed, 14 deletions(-) delete mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh delete mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh deleted file mode 100755 index 960ca6bcd266..000000000000 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp-panzer.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -export SALLOC_CTEST_TIME_LIMIT_MINUTES=0:45:00 -if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=ATDM -fi -$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh deleted file mode 100755 index b7f92d619b02..000000000000 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp-panzer.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=ATDM -fi -export SALLOC_CTEST_TIME_LIMIT_MINUTES=0:30:00 -$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index a23e879df7d9..f5ba99d6a65d 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -57,7 +57,6 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" - export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then module load intel/19.0.5.281 @@ -86,7 +85,6 @@ elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" - export PATH=/usr/tce/packages/gcc/gcc-4.9.3/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-4.9.3/lib64:${LD_LIBRARY_PATH} export F77=mpif77 From 492e2c6e35bd7c784fdbbe74a2c5ef0496e23d1f Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Wed, 20 Nov 2019 11:49:09 -0700 Subject: [PATCH 14/34] ATMD: Cleanup (ATDV-212). --- .../std/atdm/common/toss3/environment_new.sh | 72 +++++++++---------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index f5ba99d6a65d..405fd3b30d63 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -35,56 +35,38 @@ else export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=16 fi +# Common paths and modules for both intel-1{8,9} +sparc_tpl_base=/projects/sparc/tpls/cts1-bdw +module load mkl/18.0.5.274 +module load sparc-cmake/3.12.3 + if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then module load intel/18.0.2.199 module load openmpi-intel/2.0 - module load mkl/18.0.5.274 - module load sparc-cmake/3.12.3 - - export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-18.0.2 - export CGNS_ROOT=/projects/sparc/tpls/cts1-bdw/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export EUCLID_ROOT=/projects/sparc/tpls/cts1-bdw/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export HDF5_ROOT=/projects/sparc/tpls/cts1-bdw/hdf5-1.10.5/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export LIBHIO_ROOT=/projects/sparc/tpls/cts1-bdw/libhio-1.4.1.2/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export METIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export NETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export PARMETIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export SGM_ROOT=/projects/sparc/tpls/cts1-bdw/sgm-19.30/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export SUPERLUDIST_ROOT=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - - export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include - export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a - export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" + + sparc_tpl_ext=cts1-bdw_intel-18.0.2 + sparc_tpl_mpi_ext=cts1-bdw_intel-18.0.2_openmpi-2.0.3 + + export SGM_ROOT=${sparc_tpl_base}/sgm-19.30/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3 export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then module load intel/19.0.5.281 module load openmpi-intel/4.0 - module load mkl/18.0.5.274 - module load sparc-cmake/3.12.3 - export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux - export BOOST_ROOT=/projects/sparc/tpls/cts1-bdw/boost-1.65.1/00000000/cts1-bdw_intel-19.0.5 + sparc_tpl_ext=cts1-bdw_intel-19.0.5 + sparc_tpl_mpi_ext=cts1-bdw_intel-19.0.5_openmpi-4.0.1 + + export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux= export COMPILER_ROOT=/projects/global/toss3/compilers/intel/intel_2017/compilers_and_libraries_2019.5.281/linux - export HDF5_ROOT=/projects/sparc/tpls/cts1-bdw/hdf5-1.10.5/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 export SPARC_HDF5=hdf5-1.10.5 - export CGNS_ROOT=/projects/sparc/tpls/cts1-bdw/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export PNETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export NETCDF_ROOT=/projects/sparc/tpls/cts1-bdw/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export LIBHIO_ROOT=/projects/sparc/tpls/cts1-bdw/libhio-1.4.1.2/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export METIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export PARMETIS_ROOT=/projects/sparc/tpls/cts1-bdw/parmetis-4.0.3/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export SUPERLUDIST_ROOT=/projects/sparc/tpls/cts1-bdw/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export TRILINOS_BASE=/projects/sparc/tpls/cts1-bdw/Trilinos/2019-10-01/00000001/cts1-bdw_intel-19.0.5_openmp_openmpi-4.0.1 - export SGM_ROOT=/projects/sparc/tpls/cts1-bdw/sgm-19.30/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export EUCLID_ROOT=/projects/sparc/tpls/cts1-bdw/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - - - export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include - export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a - export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" + export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + export TRILINOS_BASE=${sparc_tpl_base}/Trilinos/2019-10-01/00000001/cts1-bdw_intel-19.0.5_openmp_openmpi-4.0.1 + export SGM_ROOT=${sparc_tpl_base}/sgm-19.30/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 + + export PATH=/usr/tce/packages/gcc/gcc-4.9.3/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-4.9.3/lib64:${LD_LIBRARY_PATH} export F77=mpif77 @@ -96,16 +78,28 @@ else return fi - export OMPI_CXX=`which icpc` export OMPI_CC=`which icc` export OMPI_FC=`which ifort` export ATDM_CONFIG_LAPACK_LIBS="-mkl" export ATDM_CONFIG_BLAS_LIBS="-mkl" +export BOOST_ROOT=${sparc_tpl_base}/boost-1.65.1/00000000/${sparc_tpl_ext} +export HDF5_ROOT=${sparc_tpl_base}/hdf5-1.10.5/00000000/${sparc_tpl_mpi_ext} +export CGNS_ROOT=${sparc_tpl_base}/cgns-c09a5cd/27e5681f1b74c679b5dcb337ac71036d16c47977/${sparc_tpl_mpi_ext} +export PNETCDF_ROOT=${sparc_tpl_base}/pnetcdf-1.10.0/6144dc67b2041e4093063a04e89fc1e33398bd09/${sparc_tpl_mpi_ext} +export NETCDF_ROOT=${sparc_tpl_base}/netcdf-4.7.0/58bc48d95be2cc9272a18488fea52e1be1f0b42a/${sparc_tpl_mpi_ext} +export PARMETIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_ext} +export METIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_ext} +export LIBHIO_ROOT=${sparc_tpl_base}/libhio-1.4.1.2/00000000/${sparc_tpl_mpi_ext} +export EUCLID_ROOT=${sparc_tpl_base}/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/${sparc_tpl_mpi_ext} + export ATDM_CONFIG_USE_HWLOC=OFF export ATDM_CONFIG_HDF5_LIBS="-L${HDF5_ROOT}/lib;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl" export ATDM_CONFIG_NETCDF_LIBS="-L${BOOST_ROOT}/lib;-L${NETCDF_ROOT}/lib;-L${PNETCDF_ROOT}/lib;-L${HDF5_ROOT}/lib;${BOOST_ROOT}/lib/libboost_program_options.a;${BOOST_ROOT}/lib/libboost_system.a;${NETCDF_ROOT}/lib/libnetcdf.a;${NETCDF_ROOT}/lib/libpnetcdf.a;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl;-lcurl" +export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include +export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a +export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a" # not sure what below does. It was in the original environment script #unset ATTB_ENV From 84828fea843bc1cfc168095325a2c76fbbebe8fc Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Wed, 20 Nov 2019 15:09:45 -0700 Subject: [PATCH 15/34] ATDM: Cleanup (ATDV-212). --- ...sh => Trilinos-atdm-cts1-intel-18-debug-openmp.sh} | 2 +- ...p.sh => Trilinos-atdm-cts1-intel-18-opt-openmp.sh} | 2 +- .../drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh | 2 +- cmake/std/atdm/common/toss3/environment_new.sh | 11 +++-------- 4 files changed, 6 insertions(+), 11 deletions(-) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-debug-openmp.sh => Trilinos-atdm-cts1-intel-18-debug-openmp.sh} (82%) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-opt-openmp.sh => Trilinos-atdm-cts1-intel-18-opt-openmp.sh} (82%) diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-debug-openmp.sh similarity index 82% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-debug-openmp.sh index 627fd02f47ae..92de5ae9c246 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-debug-openmp.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-debug-openmp.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=ATDM + export Trilinos_TRACK=Experimental fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-opt-openmp.sh similarity index 82% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-opt-openmp.sh index 627fd02f47ae..92de5ae9c246 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-opt-openmp.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-opt-openmp.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=ATDM + export Trilinos_TRACK=Experimental fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh index 627fd02f47ae..92de5ae9c246 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=ATDM + export Trilinos_TRACK=Experimental fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 405fd3b30d63..a3e614f46d6a 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -47,9 +47,6 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then sparc_tpl_ext=cts1-bdw_intel-18.0.2 sparc_tpl_mpi_ext=cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export SGM_ROOT=${sparc_tpl_base}/sgm-19.30/00000000/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-18.0.2_openmpi-2.0.3 - export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then module load intel/19.0.5.281 @@ -58,14 +55,10 @@ elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then sparc_tpl_ext=cts1-bdw_intel-19.0.5 sparc_tpl_mpi_ext=cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux= + export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux export COMPILER_ROOT=/projects/global/toss3/compilers/intel/intel_2017/compilers_and_libraries_2019.5.281/linux export SPARC_HDF5=hdf5-1.10.5 - export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export TRILINOS_BASE=${sparc_tpl_base}/Trilinos/2019-10-01/00000001/cts1-bdw_intel-19.0.5_openmp_openmpi-4.0.1 - export SGM_ROOT=${sparc_tpl_base}/sgm-19.30/00000000/cts1-bdw_intel-19.0.5_openmpi-4.0.1 - export PATH=/usr/tce/packages/gcc/gcc-4.9.3/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-4.9.3/lib64:${LD_LIBRARY_PATH} @@ -93,6 +86,8 @@ export PARMETIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_e export METIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_ext} export LIBHIO_ROOT=${sparc_tpl_base}/libhio-1.4.1.2/00000000/${sparc_tpl_mpi_ext} export EUCLID_ROOT=${sparc_tpl_base}/euclid-19.30/95b8242ad729449fdb86591b9643c11463b61fa2/${sparc_tpl_mpi_ext} +export SGM_ROOT=${sparc_tpl_base}/sgm-19.30/00000000/${sparc_tpl_mpi_ext} +export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/${sparc_tpl_mpi_ext} export ATDM_CONFIG_USE_HWLOC=OFF export ATDM_CONFIG_HDF5_LIBS="-L${HDF5_ROOT}/lib;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl" From b434d1b4144cfd445608beba12fb14cc92c9f66c Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Thu, 21 Nov 2019 10:45:03 -0700 Subject: [PATCH 16/34] ATDM: Add required path to GCC binaries (ATDV-212) - The intel-18 build cannot find cxx symbols without these binaries in PATH. --- cmake/std/atdm/common/toss3/environment_new.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index a3e614f46d6a..eeb7b35df483 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -47,6 +47,7 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then sparc_tpl_ext=cts1-bdw_intel-18.0.2 sparc_tpl_mpi_ext=cts1-bdw_intel-18.0.2_openmpi-2.0.3 + export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then module load intel/19.0.5.281 From db6378e85c64477d43bdf37d6e1085d0b97c0fcd Mon Sep 17 00:00:00 2001 From: "Forster, Jeffrey Edward" Date: Thu, 21 Nov 2019 13:46:10 -0700 Subject: [PATCH 17/34] updated cts1 driver script names to be usable with SPARC builds --- ...m-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh} | 0 ...m-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh} | 0 ...dm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh} | 0 ...tdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh | 6 ++++++ cmake/std/atdm/cts1/all_supported_builds.sh | 6 ++++-- 5 files changed, 10 insertions(+), 2 deletions(-) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-18-debug-openmp.sh => Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh} (100%) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-18-opt-openmp.sh => Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh} (100%) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-19-opt-openmp.sh => Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh} (100%) create mode 100755 cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-debug-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh similarity index 100% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-debug-openmp.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh similarity index 100% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18-opt-openmp.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh similarity index 100% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19-opt-openmp.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh new file mode 100755 index 000000000000..92de5ae9c246 --- /dev/null +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh @@ -0,0 +1,6 @@ +#!/bin/bash +#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +if [ "${Trilinos_TRACK}" == "" ] ; then + export Trilinos_TRACK=Experimental +fi +$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/std/atdm/cts1/all_supported_builds.sh b/cmake/std/atdm/cts1/all_supported_builds.sh index af6d0a76d4d5..51e311ae345a 100644 --- a/cmake/std/atdm/cts1/all_supported_builds.sh +++ b/cmake/std/atdm/cts1/all_supported_builds.sh @@ -2,6 +2,8 @@ export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-cts1- export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=( - intel-debug-openmp - intel-opt-openmp + cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg + cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt + cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg + cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt ) From 6e318167651a3bcdb203e8333e5e5f66d9b171cc Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Mon, 25 Nov 2019 12:45:10 -0700 Subject: [PATCH 18/34] ATDM: Direct cts environment.sh to install builds. (ATDV-212) --- cmake/std/atdm/cts1/environment.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/std/atdm/cts1/environment.sh b/cmake/std/atdm/cts1/environment.sh index 19fa5bd5d863..dc035536e4c2 100755 --- a/cmake/std/atdm/cts1/environment.sh +++ b/cmake/std/atdm/cts1/environment.sh @@ -11,7 +11,7 @@ # machines. please be mindful of this when making changes if [ "$ATDM_CONFIG_KOKKOS_ARCH" == "DEFAULT" ] ; then - export ATDM_CONFIG_KOKKOS_ARCH=BDW + export ATDM_CONFIG_KOKKOS_ARCH=BDW fi if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then @@ -25,3 +25,5 @@ if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then fi source $ATDM_SCRIPT_DIR/common/toss3/environment_new.sh + +export ATDM_CONFIG_TRIL_CMAKE_INSTALL_PREFIX_DATE_BASE_DEFAULT=/projects/atdm_devops/trilinos_installs/ From 6cffeb5564c7c3e6d7d3f49fe2fbeb0fc65381e4 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Mon, 25 Nov 2019 15:30:45 -0700 Subject: [PATCH 19/34] ATDM: change cmake module (ATDV-212). --- cmake/std/atdm/common/toss3/environment_new.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index eeb7b35df483..588a52b48426 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -38,7 +38,7 @@ fi # Common paths and modules for both intel-1{8,9} sparc_tpl_base=/projects/sparc/tpls/cts1-bdw module load mkl/18.0.5.274 -module load sparc-cmake/3.12.3 +module load cmake/3.12.2 if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then module load intel/18.0.2.199 From 4244d9782a4d6a73303302e767ad3215c7c43ee2 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Mon, 2 Dec 2019 08:58:40 -0700 Subject: [PATCH 20/34] ATDM: Updated SPARC Enables (ATDV-212) --- cmake/std/atdm/apps/sparc/SPARCTrilinosPackagesList.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/atdm/apps/sparc/SPARCTrilinosPackagesList.cmake b/cmake/std/atdm/apps/sparc/SPARCTrilinosPackagesList.cmake index e9b24f0e5865..8f989abbe541 100644 --- a/cmake/std/atdm/apps/sparc/SPARCTrilinosPackagesList.cmake +++ b/cmake/std/atdm/apps/sparc/SPARCTrilinosPackagesList.cmake @@ -17,7 +17,6 @@ SET(SPARC_Trilinos_Packages Amesos2 Ifpack2 MueLu - Zoltan2 STKTransfer STKSearch STKUtil @@ -26,4 +25,5 @@ SET(SPARC_Trilinos_Packages SEACAS Gtest ROL + TriKota ) From 00164883002589c0f857582833739e830e3a6153 Mon Sep 17 00:00:00 2001 From: Evan Harvey Date: Tue, 3 Dec 2019 15:01:51 -0700 Subject: [PATCH 21/34] ATDM: Load sparc-git. --- cmake/std/atdm/common/toss3/environment_new.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 588a52b48426..aebfb1d8d463 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -27,6 +27,7 @@ module purge module load sems-env module load atdm-env module load atdm-ninja_fortran/1.7.2 +module load sparc-git/2.19.1 if [ "$ATDM_CONFIG_NODE_TYPE" == "OPENMP" ] ; then export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=8 From 9e450bc6fdfda520a265842bb83acca840b095a5 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 19 Dec 2019 09:06:11 -0700 Subject: [PATCH 22/34] Factoring atdm_run_script_on_comput_node into reusable file and other improvements (ATDV-212, #6467) This gets us closer to a single atdm_run_script_on_comput_node function that can be used on all platforms that use SLURM. Some other things: * Defined new ATDM_CONFIG vars ATDM_CONFIG_SPARC_TPL_BASE, ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT and ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT to allow these new files to reused. * Removed bad logic for "DEFAULT" compiler since a custom_builds.sh script is now being used. * Added the system-name to the echo command for the build being configured (so it says 'cts1' instead of just 'toss3'). --- .../define_run_on_slurm_compute_node_func.sh | 110 ++++++++++++++++++ .../std/atdm/common/toss3/environment_new.sh | 110 +----------------- cmake/std/atdm/cts1/environment.sh | 6 + .../unset_atdm_config_vars_environment.sh | 4 + 4 files changed, 125 insertions(+), 105 deletions(-) create mode 100644 cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh diff --git a/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh b/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh new file mode 100644 index 000000000000..17dbe11f4149 --- /dev/null +++ b/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh @@ -0,0 +1,110 @@ +# +# Run a script on the compute node send STDOUT and STDERR output to a file +# while also echo output to the console. The primary purpose is to run the +# tests on the compute node. +# +# Usage: +# +# atdm_run_script_on_comput_node \ +# [] [] +# +# If is not given, then it is set to the env var +# ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT if != "". Otherwise, a default is set +# internally. +# +# If is not given, then it is set to the env var +# ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT if != "". Otherwise, a default is set +# internally. +# +# In this case, sbatch is used to run the script but it also sends ouptut to +# STDOUT in real-time while it is running in addition to writing to the +# . The SLURM job name for the sbatch script is taken from the +# env var 'ATDM_CONFIG_BUILD_NAME'. This works for local builds as well since +# ATDM_CONFIG_BUILD_NAME is always set by the atdm/load-env.sh script. +# +# Note that you can pass in the script to run with arguments such as with +# " " and it will work. But note that this script +# has to be bash script that 'sbatch' can copy and run from a temp location +# and it still has to work. Therefore, this script has to use absolute +# directory paths, not relative paths or asume sym links, etc (using env vars +# to know what to run and where). +# +function atdm_run_script_on_compute_node { + + set +x + + script_to_run=$1 + output_file=$2 + timeout_input=$3 + account_input=$4 + + echo + echo "***" + echo "*** atdm_run_script_on_compute_node '${script_to_run}' '${output_file}' '${timeout_input}' '${account_input}'" + echo "***" + echo + + if [[ "${timeout_input}" != "" ]] ; then + timeout=${timeout_input} + elif [[ "${ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT}" != "" ]] ; then + timeout=${ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT} + else + timeout=1:30:00 + fi + + if [ "${account_input}" != "" ] ; then + account=${account_input} + elif [[ "${ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT}" != "" ]] ; then + timeout=${ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT} + else + account=fy150090 + fi + + if [ -e $output_file ] ; then + echo "Remove existing file $output_file" + rm $output_file + fi + echo "Create empty file $output_file" + touch $output_file + + echo + echo "Running '$script_to_run' using sbatch in the background ..." + set -x + sbatch --output=$output_file --wait -N1 --time=${timeout} \ + -J $ATDM_CONFIG_BUILD_NAME --account=${account} ${script_to_run} & + SBATCH_PID=$! + set +x + + echo + echo "Tailing output file $output_file in the background ..." + set -x + tail -f $output_file & + TAIL_BID=$! + set +x + + echo + echo "Waiting for SBATCH_PID=$SBATCH_PID ..." + wait $SBATCH_PID + + echo + echo "Kill TAIL_BID=$TAIL_BID" + kill -s 9 $TAIL_BID + + echo + echo "Finished running ${script_to_run}!" + echo + +} + +export -f atdm_run_script_on_compute_node + +# NOTE: The above function is implemented in this way using 'sbatch' so that +# we can avoid using 'salloc' which is belived to cause ORTE errors. But we +# still want to see live ouput from the script so that we can report it on +# Jenkins and see it running in real-time on Jenins. Therefore, the above +# approach is to use 'sbatch' and write its output to a known file-name. +# Then, we use `tail -f` to print that file as it gets filled in from the +# 'sbatch' command. The 'sbatch' command is run with --wait but is +# backgrouned to allow this to happen. Then we wait for the 'sbatch' command +# to complete and then we kill the 'tail -f' command. That might seem overly +# complex but that gets the job done. diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index aebfb1d8d463..291f41b877e2 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -6,11 +6,7 @@ # ################################################################################ -if [ "$ATDM_CONFIG_COMPILER" == "DEFAULT" ] ; then - export ATDM_CONFIG_COMPILER=INTEL -fi - -echo "Using toss3 compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_CONFIG_BUILD_TYPE code with Kokkos node type $ATDM_CONFIG_NODE_TYPE" +echo "Using $ATDM_CONFIG_SYSTEM_NAME toss3 compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_CONFIG_BUILD_TYPE code with Kokkos node type $ATDM_CONFIG_NODE_TYPE" export ATDM_CONFIG_ENABLE_SPARC_SETTINGS=ON export ATDM_CONFIG_USE_NINJA=ON @@ -37,7 +33,7 @@ else fi # Common paths and modules for both intel-1{8,9} -sparc_tpl_base=/projects/sparc/tpls/cts1-bdw +sparc_tpl_base=${ATDM_CONFIG_SPARC_TPL_BASE} module load mkl/18.0.5.274 module load cmake/3.12.2 @@ -116,104 +112,8 @@ export CXX=mpicxx export FC=mpif77 export F90=mpif90 -export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE +# Define function atdm_run_script_on_compute_node +source $ATDM_SCRIPT_DIR/common/define_run_on_slurm_compute_node_func.sh +export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE -# -# Run a script on the compute node send STDOUT and STDERR output to a file -# while also echo output to the console. The primary purpose is to run the -# tests on the compute node. -# -# Usage: -# -# atdm_run_script_on_comput_node \ -# [] [] -# -# If and/or are not given, then defaults are provided that -# work for the Jenkins driver process. -# -# In this case, sbatch is used to run the script but it also sends ouptut to -# STDOUT in real-time while it is running in addition to writing to the -# . The job name for the sbatch script is taken from the env var -# 'ATDM_CONFIG_BUILD_NAME'. This works for local builds since ATDM_CONFIG_BUILD_NAME. -# -# Note that you can pass in the script to run with arguments such as with -# " " and it will work. But note that this has to -# be bash script that 'sbatch' can copy and run form a temp location and it -# still has to work. So the script has to use absolute directory paths, not -# relative paths or asume sym links, etc. -# -function atdm_run_script_on_compute_node { - - set +x - - script_to_run=$1 - output_file=$2 - timeout_input=$3 - account_input=$4 - - echo - echo "***" - echo "*** atdm_run_script_on_compute_node '${script_to_run}' '${output_file}' '${timeout_input}' '${account_input}'" - echo "***" - echo - - if [ "${timeout_input}" == "" ] ; then - timeout=1:30:00 - else - timeout=${timeout_input} - fi - - if [ "${account_input}" == "" ] ; then - account=fy150090 - else - account=${account_input} - fi - - if [ -e $output_file ] ; then - echo "Remove existing file $output_file" - rm $output_file - fi - echo "Create empty file $output_file" - touch $output_file - - echo - echo "Running '$script_to_run' using sbatch in the background ..." - set -x - sbatch --output=$output_file --wait -N1 --time=${timeout} \ - -J $ATDM_CONFIG_BUILD_NAME --account=${account} ${script_to_run} & - SBATCH_PID=$! - set +x - - echo - echo "Tailing output file $output_file in the background ..." - set -x - tail -f $output_file & - TAIL_BID=$! - set +x - - echo - echo "Waiting for SBATCH_PID=$SBATCH_PID ..." - wait $SBATCH_PID - - echo - echo "Kill TAIL_BID=$TAIL_BID" - kill -s 9 $TAIL_BID - - echo - echo "Finished running ${script_to_run}!" - echo - -} - -export -f atdm_run_script_on_compute_node - -# NOTE: The above function is implemented in this way using 'sbatch' so that -# we can avoid using 'salloc' which is belived to cause ORTE errors. But we -# still want to see live ouput from the script so that we can report it on -# Jenkins. Therefore, the above approach is to use 'sbatch' and write its -# output to a known file-name. Then, we use `tail -f` to print that file as -# it gets filled in from the 'sbatch' command. The 'sbatch' command is run -# with --wait but is backgrouned to allow this to happen. Then we wait for -# the 'sbatch' command to complete and then we kill the 'tail -f' command. -# That might seem overly complex but that gets the job done. diff --git a/cmake/std/atdm/cts1/environment.sh b/cmake/std/atdm/cts1/environment.sh index dc035536e4c2..d16f0f341048 100755 --- a/cmake/std/atdm/cts1/environment.sh +++ b/cmake/std/atdm/cts1/environment.sh @@ -24,6 +24,12 @@ if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then return fi +export ATDM_CONFIG_SPARC_TPL_BASE=/projects/sparc/tpls/cts1-bdw + +export ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT=4:00:00 + +export ATDM_CONFIG_SLURM_DEFAULT_ACCOUNT=fy150090 + source $ATDM_SCRIPT_DIR/common/toss3/environment_new.sh export ATDM_CONFIG_TRIL_CMAKE_INSTALL_PREFIX_DATE_BASE_DEFAULT=/projects/atdm_devops/trilinos_installs/ diff --git a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh index 2ca1b803488f..d94a2066581b 100644 --- a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh +++ b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh @@ -27,6 +27,7 @@ unset ATDM_CONFIG_PARALLEL_COMPILE_JOBS_LIMIT unset ATDM_CONFIG_PARALLEL_LINK_JOBS_LIMIT unset ATDM_CONFIG_CTEST_PARALLEL_LEVEL unset ATDM_CONFIG_NVCC_WRAPPER +unset ATDM_CONFIG_SPARC_TPL_BASE unset ATDM_CONFIG_BINUTILS_LIBS unset ATDM_CONFIG_BLAS_LIBS unset ATDM_CONFIG_LAPACK_LIBS @@ -46,4 +47,7 @@ unset ATDM_CONFIG_MPI_POST_FLAGS unset ATDM_CONFIG_WORKSPACE_BASE_DEFAULT unset ATDM_CONFIG_INSTALL_PBP_RUNNER_DEFAULT unset ATDM_CONFIG_TRIL_CMAKE_INSTALL_PREFIX_DATE_BASE_DEFAULT +unset ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT +unset ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT +unset atdm_run_script_on_compute_node unset ATDM_CONFIG_COMPLETED_ENV_SETUP From edf7f0e455361459e9260c3df5cd84b25e442d6c Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 19 Dec 2019 09:27:11 -0700 Subject: [PATCH 23/34] Load mkl/18.0.5.274 for both intel versions (ATDV-212) Apparently this is needed. Not sure why MKL is not found when just loading module intel/19.0.5.281. --- cmake/std/atdm/common/toss3/environment_new.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 291f41b877e2..1fdf645a1fd8 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -34,11 +34,11 @@ fi # Common paths and modules for both intel-1{8,9} sparc_tpl_base=${ATDM_CONFIG_SPARC_TPL_BASE} -module load mkl/18.0.5.274 module load cmake/3.12.2 if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then module load intel/18.0.2.199 + module load mkl/18.0.5.274 # Needed to address defect in MKL (#5316, #3992, #3914) module load openmpi-intel/2.0 sparc_tpl_ext=cts1-bdw_intel-18.0.2 @@ -48,6 +48,7 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then module load intel/19.0.5.281 + module load mkl/18.0.5.274 # Needed to find libmkl_intel_lp64.so (ATDV-212) module load openmpi-intel/4.0 sparc_tpl_ext=cts1-bdw_intel-19.0.5 From bdb5e74ef9472c18340f9335771e38a181bdd55a Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 19 Dec 2019 09:40:28 -0700 Subject: [PATCH 24/34] Fixed names of builds to match SPARC names for these (ATDV-212) --- ...dm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh} | 0 ...dm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh} | 0 cmake/std/atdm/cts1/all_supported_builds.sh | 6 +++--- 3 files changed, 3 insertions(+), 3 deletions(-) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh => Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh} (100%) rename cmake/ctest/drivers/atdm/cts1/drivers/{Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh => Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh} (100%) diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh similarity index 100% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh similarity index 100% rename from cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt.sh rename to cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh diff --git a/cmake/std/atdm/cts1/all_supported_builds.sh b/cmake/std/atdm/cts1/all_supported_builds.sh index 51e311ae345a..b7d69fd8887b 100644 --- a/cmake/std/atdm/cts1/all_supported_builds.sh +++ b/cmake/std/atdm/cts1/all_supported_builds.sh @@ -1,9 +1,9 @@ -export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-cts1- +export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm- export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=( - cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_dbg - cts1-intel-18.0.2_openmpi-2.0.3_openmpi_static_opt + cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg + cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt ) From 0e176ab0b71bae5d9106e1a5ca53fb2d45c5803b Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 19 Dec 2019 14:15:32 -0700 Subject: [PATCH 25/34] Make 'serrano' hostname still match 'serrano' system by default and fix error check (ATDV-212) There was a logic error in the check that did not allow one to select the 'cts1' env on an existing machine that matched the 'serrano' system by hostname. --- cmake/std/atdm/utils/get_known_system_info.sh | 16 ++++---- cmake/std/atdm/utils/get_system_info_utils.sh | 41 ++----------------- 2 files changed, 10 insertions(+), 47 deletions(-) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index dc7c2297e98a..b75cc3f5dd46 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -95,19 +95,19 @@ elif [[ $realHostname == "waterman"* ]] ; then hostnameMatch=waterman hostnameMatchSystemName=waterman -# Specifically named cts1 systems +# Specifically named cts1 systems (currently maps to 'serrano' env) elif [[ $realHostname == "serrano"* ]] || [[ $realHostname =~ ser[0-9]+ ]] ; then hostnameMatch=serrano - hostnameMatchSystemName=cts1 + hostnameMatchSystemName=serrano elif [[ $realHostname == "eclipse"* ]] || [[ $realHostname =~ ec[0-9]+ ]] ; then hostnameMatch=eclipse - hostnameMatchSystemName=cts1 + hostnameMatchSystemName=serrano elif [[ $realHostname == "ghost"* ]] || [[ $realHostname =~ gho[0-9]+ ]] ; then hostnameMatch=ghost - hostnameMatchSystemName=cts1 + hostnameMatchSystemName=serrano elif [[ $realHostname == "attaway"* ]] || [[ $realHostname =~ swa[0-9]+ ]] ; then hostnameMatch=attaway - hostnameMatchSystemName=cts1 + hostnameMatchSystemName=serrano # End specifically named systems fi @@ -141,7 +141,6 @@ if [[ $SNLSYSTEM == "cts1" ]] ; then systemNameTypeMatchedListHostNames[cts1]=$SNLCLUSTER fi - # SEMS RHEL6 and RHEL7 systems if [[ "${SEMS_PLATFORM}" == "rhel6-x86_64" ]] ; then systemNameTypeMatchedList+=(sems-rhel6) @@ -199,8 +198,7 @@ ATDM_SYSTEM_NAME= if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${knownSystemNameInBuildName}" != "" ]] ; then ATDM_SYSTEM_NAME=${knownSystemNameInBuildName} ATDM_HOSTNAME=${systemNameTypeMatchedListHostNames[${ATDM_SYSTEM_NAME}]} - assert_selected_system_matches_known_host_in_build_name || return - assert_selected_system_matches_known_system_type_mathces || return + assert_selected_system_matches_known_system_type_matches || return fi # D.2) Last, go with the hostname match or matching system type @@ -222,4 +220,4 @@ if [[ $ATDM_SYSTEM_NAME != "" ]] ; then export ATDM_CONFIG_CDASH_HOSTNAME=$ATDM_HOSTNAME export ATDM_CONFIG_SYSTEM_NAME=$ATDM_SYSTEM_NAME export ATDM_CONFIG_SYSTEM_DIR=${ATDM_CONFIG_SCRIPT_DIR}/${ATDM_CONFIG_SYSTEM_NAME} -fi \ No newline at end of file +fi diff --git a/cmake/std/atdm/utils/get_system_info_utils.sh b/cmake/std/atdm/utils/get_system_info_utils.sh index 261ec033bc0f..72f15c58c6d7 100644 --- a/cmake/std/atdm/utils/get_system_info_utils.sh +++ b/cmake/std/atdm/utils/get_system_info_utils.sh @@ -26,47 +26,12 @@ function list_contains() { } -function assert_selected_system_matches_known_host_in_build_name() { - - if [[ "${hostnameMatch}" == "" ]] ; then - return 0 - fi - - if [[ "${ATDM_SYSTEM_NAME}" != "${hostnameMatchSystemName}" ]] ; then - echo - echo "***" - echo "*** Error, the system name '${knownSystemNameInBuildName}' given in the build name:" - echo "***" - echo "*** ${ATDM_CONFIG_BUILD_NAME}" - echo "***" - echo "*** does not match the system type based on the hostname:" - echo "***" - echo "*** $realHostname" - echo "***" - echo "*** where this machine matches the known host system:" - echo "***" - echo "*** $hostnameMatch" - echo "***" - echo "*** with the associated system name:" - echo "***" - echo "*** $hostnameMatchSystemName" - echo "***" - echo "*** To address this, either remove '${knownSystemNameInBuildName}' from the build name" - echo "*** or change it to '${hostnameMatchSystemName}'." - echo "***" - echo - return 1 - fi - return 0 -} - - -function assert_selected_system_matches_known_system_type_mathces() { +function assert_selected_system_matches_known_system_type_matches() { if [[ "${systemNameTypeMatchedList}" == "" ]] ; then return 0 fi - + local selectedSystemNameMatchesSystemTypeMatches=false list_contains systemNameTypeMatchedList ${ATDM_SYSTEM_NAME} \ && selectedSystemNameMatchesSystemTypeMatches=true @@ -82,7 +47,7 @@ function assert_selected_system_matches_known_system_type_mathces() { echo "***" echo "*** $realHostname" echo "***" - echo "**** and does not match a supported system type on this machine which includes:" + echo "**** and does not match one of the supported system types on this machine which includes:" echo "***" echo "*** (${systemNameTypeMatchedList[@]})" echo "***" From 3dfbe0385777829912a96e24f6de2b05f5b3069b Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 19 Dec 2019 14:50:02 -0700 Subject: [PATCH 26/34] Add todos to make reusable (ATDV-212, #6467) We can do the refactorings to make this reusable once we actually reuse it. --- cmake/std/atdm/common/toss3/environment_new.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/std/atdm/common/toss3/environment_new.sh b/cmake/std/atdm/common/toss3/environment_new.sh index 1fdf645a1fd8..e1a9cf96ef75 100755 --- a/cmake/std/atdm/common/toss3/environment_new.sh +++ b/cmake/std/atdm/common/toss3/environment_new.sh @@ -43,6 +43,8 @@ if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-2.0.3" ]; then sparc_tpl_ext=cts1-bdw_intel-18.0.2 sparc_tpl_mpi_ext=cts1-bdw_intel-18.0.2_openmpi-2.0.3 + # rabartl: ToDo: Above, we need to find a way to extract 'cts1-bdw' out of + # this file for this to be general! export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH} export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH} @@ -53,6 +55,8 @@ elif [ "$ATDM_CONFIG_COMPILER" == "INTEL-19.0.5_OPENMPI-4.0.1" ]; then sparc_tpl_ext=cts1-bdw_intel-19.0.5 sparc_tpl_mpi_ext=cts1-bdw_intel-19.0.5_openmpi-4.0.1 + # rabartl: ToDo: Above, we need to find a way to extract 'cts1-bdw' out of + # this file for this to be general! export CBLAS_ROOT=/projects/global/toss3/compilers/intel/intel_2019/compilers_and_libraries_2019.5.281/linux export COMPILER_ROOT=/projects/global/toss3/compilers/intel/intel_2017/compilers_and_libraries_2019.5.281/linux From d2ed4c784287dfbaaccb2e5681623997ccc9f15b Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Sat, 21 Dec 2019 11:39:11 -0700 Subject: [PATCH 27/34] Change CDash group for new 'cts1' builds to 'Specialized' (ATDV-212) There are a very small number of test failures in these builds so they ae ready to go to 'Specialized' and be cleaned up. --- ...os-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh | 2 +- ...os-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh | 2 +- ...os-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh | 2 +- ...os-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh index 92de5ae9c246..297ca71f1b3a 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=Experimental + export Trilinos_TRACK=Specialized fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh index 92de5ae9c246..297ca71f1b3a 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=Experimental + export Trilinos_TRACK=Specialized fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh index 92de5ae9c246..297ca71f1b3a 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=Experimental + export Trilinos_TRACK=Specialized fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh index 92de5ae9c246..297ca71f1b3a 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh @@ -1,6 +1,6 @@ #!/bin/bash #export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then - export Trilinos_TRACK=Experimental + export Trilinos_TRACK=Specialized fi $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh From f58e77197d42a5abf42222453838050656aecf30 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 2 Jan 2020 07:58:14 -0700 Subject: [PATCH 28/34] Fix copy and paste doc typo (ATDV-212) --- cmake/std/atdm/cts1/custom_builds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/atdm/cts1/custom_builds.sh b/cmake/std/atdm/cts1/custom_builds.sh index ec87bc81e9a0..ef7da19ddfff 100644 --- a/cmake/std/atdm/cts1/custom_builds.sh +++ b/cmake/std/atdm/cts1/custom_builds.sh @@ -1,5 +1,5 @@ # -# Custom builds for cee-rhel6 env +# Custom builds for 'cts1' env # # NOTE: This file gets sourced in atdm/utils/set_build_options.sh before the # default grep logic is applied. From e422484f6e1b593d72287b709c789bbb59dfdbb1 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 2 Jan 2020 07:59:51 -0700 Subject: [PATCH 29/34] Add back 'serrano' as default system over 'cts1' on CTS-1 systems (ATDV-212) To avoid breaking backward comparbility, make default to 'serrano' on CTS-1 systems if no system-name is given in the build name. --- cmake/std/atdm/utils/get_known_system_info.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index b75cc3f5dd46..1181b82ecde2 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -46,6 +46,7 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=( ride mutrino # Will be repalced by 'ats1' waterman + serrano # Default to 'seranno' for backward compatibility for now cts1 tlcc2 sems-rhel7 From 89d90d574b7991a749bdb4a22247b4b6606c2cda Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 2 Jan 2020 08:32:02 -0700 Subject: [PATCH 30/34] More accurately document how system envs are selected (ATDV-212) Hopefully this is less confusing now. This script really needs strong automated testing :-( --- cmake/std/atdm/README.md | 8 +++++--- cmake/std/atdm/utils/get_known_system_info.sh | 17 +++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cmake/std/atdm/README.md b/cmake/std/atdm/README.md index fd822d3e707b..69938e04bb60 100644 --- a/cmake/std/atdm/README.md +++ b/cmake/std/atdm/README.md @@ -103,12 +103,14 @@ href="#sems-rhel6-environment">sems-rhel6 and cee-rhel6 environments are supported. On these CEE LAN RHEL6 machines, when `cee-rhel6` is included in ``, then the `cee-rhel6` env will be selected. But if `sems-rhel6` is included in -the build name or no system name is given, then the `sems-rhel6` env will be -selected by default on such machines. Likewise for CEE LAN RHEL7 machines -with the sems-rhel7 and sems-rhel7 and cee-rhel6 environments. And if `spack-rhel` is included in ``, then the spack-rhel will attempted to be loaded. +(In that case, one must ensure that the ATDM Spack modules have been defined.) diff --git a/cmake/std/atdm/utils/get_known_system_info.sh b/cmake/std/atdm/utils/get_known_system_info.sh index 1181b82ecde2..86bf8d1cfe3f 100644 --- a/cmake/std/atdm/utils/get_known_system_info.sh +++ b/cmake/std/atdm/utils/get_known_system_info.sh @@ -36,9 +36,14 @@ realHostname=`hostname` # # List out all of the known system envs # -# These are listed in order of presidence where if the current machine matches -# more than one of these, the first match will be selected if the system name -# is not given in the build name. +# The order these are listed in this array matters only if multiple known +# system name keywords are listed in the build name string. For example, if +# both 'ride' and 'cts1' are listed in the build name, then 'ride' will be the +# one recognized and 'cts1' will be ignored (because 'ride' is listed above +# 'cts1'). +# +# However, it is important that "all" of the known systems be listed in this +# array in order for each system name to recognized in the build name string. # ATDM_KNOWN_SYSTEM_NAMES_LIST=( @@ -46,7 +51,7 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=( ride mutrino # Will be repalced by 'ats1' waterman - serrano # Default to 'seranno' for backward compatibility for now + serrano cts1 tlcc2 sems-rhel7 @@ -195,14 +200,14 @@ fi ATDM_HOSTNAME= ATDM_SYSTEM_NAME= -# D.1) First, go with the system name in the build name. +# D.1) First, go with the system name in the build name if one was recognised if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${knownSystemNameInBuildName}" != "" ]] ; then ATDM_SYSTEM_NAME=${knownSystemNameInBuildName} ATDM_HOSTNAME=${systemNameTypeMatchedListHostNames[${ATDM_SYSTEM_NAME}]} assert_selected_system_matches_known_system_type_matches || return fi -# D.2) Last, go with the hostname match or matching system type +# D.2) Last, go with the first matching system name on this machine if [[ "${ATDM_SYSTEM_NAME}" == "" ]] && [[ "${systemNameTypeMatchedList}" != "" ]] ; then ATDM_SYSTEM_NAME=${systemNameTypeMatchedList[0]} # First matching system type is preferred! ATDM_HOSTNAME=${systemNameTypeMatchedListHostNames[${ATDM_SYSTEM_NAME}]} From b5a350af0f67d0d8bc6686ae493a5bd12f24fb41 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 2 Jan 2020 08:48:34 -0700 Subject: [PATCH 31/34] Fix misspelling of DEFAULT (ATDV-212) --- .../common/define_run_on_slurm_compute_node_func.sh | 12 ++++++------ cmake/std/atdm/cts1/environment.sh | 2 +- .../atdm/utils/unset_atdm_config_vars_environment.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh b/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh index 17dbe11f4149..edeeaf692d53 100644 --- a/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh +++ b/cmake/std/atdm/common/define_run_on_slurm_compute_node_func.sh @@ -9,11 +9,11 @@ # [] [] # # If is not given, then it is set to the env var -# ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT if != "". Otherwise, a default is set +# ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT if != "". Otherwise, a default is set # internally. # # If is not given, then it is set to the env var -# ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT if != "". Otherwise, a default is set +# ATDM_CONFIG_SBATCH_DEFAULT_ACCOUNT if != "". Otherwise, a default is set # internally. # # In this case, sbatch is used to run the script but it also sends ouptut to @@ -46,16 +46,16 @@ function atdm_run_script_on_compute_node { if [[ "${timeout_input}" != "" ]] ; then timeout=${timeout_input} - elif [[ "${ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT}" != "" ]] ; then - timeout=${ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT} + elif [[ "${ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT}" != "" ]] ; then + timeout=${ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT} else timeout=1:30:00 fi if [ "${account_input}" != "" ] ; then account=${account_input} - elif [[ "${ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT}" != "" ]] ; then - timeout=${ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT} + elif [[ "${ATDM_CONFIG_SBATCH_DEFAULT_ACCOUNT}" != "" ]] ; then + timeout=${ATDM_CONFIG_SBATCH_DEFAULT_ACCOUNT} else account=fy150090 fi diff --git a/cmake/std/atdm/cts1/environment.sh b/cmake/std/atdm/cts1/environment.sh index d16f0f341048..048c68663226 100755 --- a/cmake/std/atdm/cts1/environment.sh +++ b/cmake/std/atdm/cts1/environment.sh @@ -26,7 +26,7 @@ fi export ATDM_CONFIG_SPARC_TPL_BASE=/projects/sparc/tpls/cts1-bdw -export ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT=4:00:00 +export ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT=4:00:00 export ATDM_CONFIG_SLURM_DEFAULT_ACCOUNT=fy150090 diff --git a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh index d94a2066581b..dfae22bf1892 100644 --- a/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh +++ b/cmake/std/atdm/utils/unset_atdm_config_vars_environment.sh @@ -47,7 +47,7 @@ unset ATDM_CONFIG_MPI_POST_FLAGS unset ATDM_CONFIG_WORKSPACE_BASE_DEFAULT unset ATDM_CONFIG_INSTALL_PBP_RUNNER_DEFAULT unset ATDM_CONFIG_TRIL_CMAKE_INSTALL_PREFIX_DATE_BASE_DEFAULT -unset ATDM_CONFIG_SBATCH_DEFUALT_TIMEOUT -unset ATDM_CONFIG_SBATCH_DEFUALT_ACCOUNT +unset ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT +unset ATDM_CONFIG_SBATCH_DEFAULT_ACCOUNT unset atdm_run_script_on_compute_node unset ATDM_CONFIG_COMPLETED_ENV_SETUP From 1986eefc18075b40964bc35cd99ef8e818362d17 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 2 Jan 2020 09:55:57 -0700 Subject: [PATCH 32/34] Change name of SALLOC_CTEST_TIME_LIMIT_MINUTES to SLURM_CTEST_TIMEOUT and set to 1:20:00 (ATDV-212) The prior name was technically wrong (since we are using sbatch not salloc) and the timeout is in hh:mm:ss, not minutes! Also, I changed the default timeout from 1:30:00 to 1:20:00. I did this for two reasons. First, all of the builds ran had the ctest suite finish in less than 50 min (even the 'dbg' builds). Also, I want this to be different than the built-in timeout of 1:30:00 so I don't get confused again. --- ...tdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh | 2 +- ...tdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh | 2 +- ...tdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh | 2 +- ...tdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh | 2 +- cmake/ctest/drivers/atdm/cts1/local-driver.sh | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh index 297ca71f1b3a..d433540983d4 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_dbg.sh @@ -1,5 +1,5 @@ #!/bin/bash -#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +#export SLURM_CTEST_TIMEOUT=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then export Trilinos_TRACK=Specialized fi diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh index 297ca71f1b3a..d433540983d4 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-18.0.2_openmpi-2.0.3_openmp_static_opt.sh @@ -1,5 +1,5 @@ #!/bin/bash -#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +#export SLURM_CTEST_TIMEOUT=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then export Trilinos_TRACK=Specialized fi diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh index 297ca71f1b3a..d433540983d4 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_dbg.sh @@ -1,5 +1,5 @@ #!/bin/bash -#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +#export SLURM_CTEST_TIMEOUT=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then export Trilinos_TRACK=Specialized fi diff --git a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh index 297ca71f1b3a..d433540983d4 100755 --- a/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh +++ b/cmake/ctest/drivers/atdm/cts1/drivers/Trilinos-atdm-cts1-intel-19.0.5_openmpi-4.0.1_openmp_static_opt.sh @@ -1,5 +1,5 @@ #!/bin/bash -#export SALLOC_CTEST_TIME_LIMIT_MINUTES=1:00:00 +#export SLURM_CTEST_TIMEOUT=1:00:00 if [ "${Trilinos_TRACK}" == "" ] ; then export Trilinos_TRACK=Specialized fi diff --git a/cmake/ctest/drivers/atdm/cts1/local-driver.sh b/cmake/ctest/drivers/atdm/cts1/local-driver.sh index 0d8a13048b98..2a877912eecc 100755 --- a/cmake/ctest/drivers/atdm/cts1/local-driver.sh +++ b/cmake/ctest/drivers/atdm/cts1/local-driver.sh @@ -1,7 +1,7 @@ #!/bin/bash -l -if [ "${SALLOC_CTEST_TIME_LIMIT_MINUTES}" == "" ] ; then - SALLOC_CTEST_TIME_LIMIT_MINUTES=1:30:00 +if [ "${SLURM_CTEST_TIMEOUT}" == "" ] ; then + SLURM_CTEST_TIMEOUT=1:20:00 # This is just running tests, not the entire build! fi @@ -18,4 +18,4 @@ set -x atdm_run_script_on_compute_node \ $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-test.sh \ $PWD/ctest-s-driver-test.out \ - ${SALLOC_CTEST_TIME_LIMIT_MINUTES} + ${SLURM_CTEST_TIMEOUT} From d34778d6f4a5ab4c971d50696da63411e916966c Mon Sep 17 00:00:00 2001 From: Mauro Perego Date: Thu, 2 Jan 2020 12:15:42 -0700 Subject: [PATCH 33/34] Piro: fix redundant calls to observeSolution --- packages/piro/src/Piro_NOXSolver_Def.hpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/piro/src/Piro_NOXSolver_Def.hpp b/packages/piro/src/Piro_NOXSolver_Def.hpp index 631de018cb67..d3eef0c4632b 100644 --- a/packages/piro/src/Piro_NOXSolver_Def.hpp +++ b/packages/piro/src/Piro_NOXSolver_Def.hpp @@ -141,15 +141,19 @@ void Piro::NOXSolver::evalModelImpl( int iteration = optimizationParams.template get("Optimizer Iteration Number"); int write_interval = analysisParams.get("Write Interval",1); Teuchos::RCP writeParams = Teuchos::rcp(new Teuchos::ParameterList()); - if (write_interval > 0) { - observeFinalSolution = (iteration >= 0 && solving_sensitivities && iteration != current_iteration) ? (iteration%write_interval == 0) : false; - if(observeFinalSolution) - current_iteration = iteration; + if(analysisParams.isSublist("ROL") && analysisParams.sublist("ROL").get("Use Old Reduced Space Interface", false)) { + if (write_interval > 0) { + observeFinalSolution = (iteration >= 0 && solving_sensitivities && iteration != current_iteration) ? (iteration%write_interval == 0) : false; + if(observeFinalSolution) + current_iteration = iteration; + } + else if (write_interval == 0) + observeFinalSolution = false; + else if (write_interval == -1) + observeFinalSolution = true; } - else if (write_interval == 0) + else observeFinalSolution = false; - else if (write_interval == -1) - observeFinalSolution = true; } // Call observer method for each parameter if optimization variables have changed @@ -233,9 +237,6 @@ void Piro::NOXSolver::evalModelImpl( const RCP > finalSolution = solver->get_current_x(); modelInArgs.set_x(finalSolution); - if (Teuchos::nonnull(this->observer) && (solve_status.solveStatus == ::Thyra::SOLVE_STATUS_CONVERGED || !writeOnlyConvergedSol) && observeFinalSolution) { - this->observer->observeSolution(*finalSolution); - } std::string sensitivity_method = appParams->get("Sensitivity Method", "Forward"); From 88d1fff9ec0a144d2958b8d603532f05a6c83224 Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Thu, 2 Jan 2020 16:06:13 -0700 Subject: [PATCH 34/34] MueLu: implementing logic to build composite nullspace for Elasticity in SetupRegionHierarchy The logic will not work for a vector laplacian but will do just fine for scalar laplacian and elasticity 2D/3D --- .../src/SetupRegionHierarchy_def.hpp | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/packages/muelu/research/mmayr/composite_to_regions/src/SetupRegionHierarchy_def.hpp b/packages/muelu/research/mmayr/composite_to_regions/src/SetupRegionHierarchy_def.hpp index d86861f16e00..d653aac065dc 100644 --- a/packages/muelu/research/mmayr/composite_to_regions/src/SetupRegionHierarchy_def.hpp +++ b/packages/muelu/research/mmayr/composite_to_regions/src/SetupRegionHierarchy_def.hpp @@ -281,6 +281,8 @@ void MakeCoarseCompositeOperator(const int maxRegPerProc, TEUCHOS_ASSERT(check == 0); } + coarseCompOp->SetFixedBlockSize(dofsPerNode); + RCP compCoordMap; std::vector > > regCoordImporter(maxRegPerProc); if(dofsPerNode == 1) { @@ -388,6 +390,7 @@ MakeCompositeAMGHierarchy(RCP::coordinateType, LocalOrdinal, GlobalOrdinal, Node> > coordinates) { #include "MueLu_UseShortNames.hpp" + using coordinates_type = typename Teuchos::ScalarTraits::coordinateType; const Scalar one = Teuchos::ScalarTraits::one(); @@ -405,8 +408,73 @@ MakeCompositeAMGHierarchy(RCP nullspace = MultiVectorFactory::Build(compOp->getRowMap(), 1); - nullspace->putScalar(one); + RCP nullspace; + if(compOp->GetFixedBlockSize() == 1) { // Scalar problem, constant nullspace + nullspace = MultiVectorFactory::Build(compOp->getRowMap(), 1); + nullspace->putScalar(one); + } else if(compOp->GetFixedBlockSize() == 2) { // 2D Elasticity + nullspace = MultiVectorFactory::Build(compOp->getRowMap(), 3); + Array > nullspaceData(3); + Array > coordinateData(2); + + // Calculate center + const coordinates_type cx = coordinates->getVector(0)->meanValue(); + const coordinates_type cy = coordinates->getVector(1)->meanValue(); + + coordinateData[0] = coordinates->getData(0); + coordinateData[1] = coordinates->getData(1); + + for(int vecIdx = 0; vecIdx < 3; ++vecIdx) { + nullspaceData[vecIdx] = nullspace->getDataNonConst(vecIdx); + } + + for(size_t nodeIdx = 0; nodeIdx < coordinates->getLocalLength(); ++nodeIdx) { + // translations + nullspaceData[0][2*nodeIdx + 0] = one; + nullspaceData[1][2*nodeIdx + 1] = one; + + // rotation about z axis + nullspaceData[2][2*nodeIdx + 0] = -(coordinateData[1][nodeIdx] - cy); + nullspaceData[2][2*nodeIdx + 1] = (coordinateData[0][nodeIdx] - cx); + } + + } else if(compOp->GetFixedBlockSize() == 3) { // 3D Elasticity + nullspace = MultiVectorFactory::Build(compOp->getRowMap(), 6); + Array > nullspaceData(6); + Array > coordinateData(3); + + // Calculate center + const coordinates_type cx = coordinates->getVector(0)->meanValue(); + const coordinates_type cy = coordinates->getVector(1)->meanValue(); + const coordinates_type cz = coordinates->getVector(2)->meanValue(); + + coordinateData[0] = coordinates->getData(0); + coordinateData[1] = coordinates->getData(1); + coordinateData[2] = coordinates->getData(2); + + for(int vecIdx = 0; vecIdx < 6; ++vecIdx) { + nullspaceData[vecIdx] = nullspace->getDataNonConst(vecIdx); + } + + for(size_t nodeIdx = 0; nodeIdx < coordinates->getLocalLength(); ++nodeIdx) { + // translations + nullspaceData[0][3*nodeIdx + 0] = one; + nullspaceData[1][3*nodeIdx + 1] = one; + nullspaceData[2][3*nodeIdx + 2] = one; + + // rotation about z axis + nullspaceData[3][3*nodeIdx + 0] = -(coordinateData[1][nodeIdx] - cy); + nullspaceData[3][3*nodeIdx + 1] = (coordinateData[0][nodeIdx] - cx); + + // rotation about x axis + nullspaceData[4][3*nodeIdx + 1] = -(coordinateData[2][nodeIdx] - cz); + nullspaceData[4][3*nodeIdx + 2] = (coordinateData[1][nodeIdx] - cy); + + // rotation about y axis + nullspaceData[5][3*nodeIdx + 0] = (coordinateData[2][nodeIdx] - cz); + nullspaceData[5][3*nodeIdx + 2] = -(coordinateData[0][nodeIdx] - cx); + } + } // Insert into parameter list userParamList.set("Nullspace", nullspace); @@ -664,13 +732,13 @@ void createRegionHierarchy(const int maxRegPerProc, if (coarseSolverType == "direct") { RCP coarseDirectSolver = MakeCompositeDirectSolver(coarseCompOp); - coarseSolverData->set>("direct solver object", coarseDirectSolver); + coarseSolverData->set >("direct solver object", coarseDirectSolver); } else if (coarseSolverType == "amg") { std::string amgXmlFileName = coarseSolverData->get("amg xml file"); RCP coarseAMGHierarchy = MakeCompositeAMGHierarchy(coarseCompOp, amgXmlFileName, compCoarseCoordinates); - coarseSolverData->set>("amg hierarchy object", coarseAMGHierarchy); + coarseSolverData->set >("amg hierarchy object", coarseAMGHierarchy); } else {