Merge 'trilinos/Trilinos:develop' (aa11e22) into 'tcad-charon/Trilino…

…s:develop' (2bfd2c7).

* trilinos-develop: (186 commits)
  zoltan2:  upgrading testing for issues fixed in trilinos#6375
  tpetra: disable kokkos warnings in initialize tests
  Tacho - disable matrix market reader/writer test to improve PR test stability.
  kokkos: cmake fixes for clang +/- cuda
  kokkos/cmake/kokkos_arch.cmake: Fix for clang + NO cuda
  Fix some scopes in nlnml_nonlinearlevel.cpp
  Zoltan2: fix reversal of Cuthill McKee ordering
  Add a fix for a stk cmake file
  Promote atdm ats2 gnu+dbg and cuda+gnu+dbg to 'Specialized' (CDOFA-72)
  Intrepid2: remove unnecessary finalize calls in unit tests
  Disable STEQR() LAPACK test on ats2 deug builds (trilinos#2410, trilinos#6166)
  Disable some timing out ROL tests (trilinos#6124)
  Disable timing out Tempus tests on ats2 (trilinos#6009)
  Intrepid2: reenabling JacobiLegendrePolynomial_Tests and Hierarchical_Basis_Tests.
  fixed some broken teuchos unit tests and removed missed deprecated methods
  Promoting ats2+gnu+opt build which is 100% clean (CDOFA-27)
  removed deprecated overload of << in SerialDenseMatrix, SerialBandDenseMatrix, SerialSymDenseMatrix, and SerialDenseVector
  removed deprecated Teuchos::Comm helpers reduceAll and scan that take pointers to return arguments
  removed deprecated MPITraits class
  removed deprecated ArrayArg class
  ...

cmake/ProjectCompilerPostConfig.cmake

@@ -1,115 +1,15 @@
-IF (${Trilinos_ENABLE_Kokkos})
+IF (${PROJECT_NAME}_ENABLE_Kokkos)
 
-  PRINT_VAR(KOKKOS_ARCH)
-
-  # This is where to generate the gen_kokkos.cmake and KokkosCore_config.h 
-  # that we will use in the configuration
-  set(Kokkos_GEN_DIR ${CMAKE_BINARY_DIR})
-
-  # Enable debug checking in Kokkos by default if
-  # ${PROJECT_NAME}_ENABLE_DEBUG=ON
-  set(KOKKOS_ENABLE_DEBUG ${${PROJECT_NAME}_ENABLE_DEBUG}
-    CACHE BOOL
-    "Enable debug checking in Kokkos.")
-  set(Kokkos_ENABLE_Debug_Bounds_Check ${KOKKOS_ENABLE_DEBUG}
-    CACHE BOOL
-    "Enable bounds checking in Kokkos array classes.")
-  set(Kokkos_ENABLE_Profiling_DEFAULT ON)
-  if (DEFINED TPL_ENABLE_DLlib)
-    if (NOT TPL_ENABLE_DLlib)
-      message(STATUS  "Setting Kokkos_ENABLE_Profiling_DEFAULT=OFF because TPL_ENABLE_DLlib=${TPL_ENABLE_DLlib}")
-      set(Kokkos_ENABLE_Profiling_DEFAULT OFF)
-    endif()
-  endif()
-  set(Kokkos_ENABLE_Profiling ${Kokkos_ENABLE_Profiling_DEFAULT}
-    CACHE BOOL
-    "Enable Kokkos profiling hooks.")
-
-  # Basic initialization (Used in KOKKOS_SETTINGS)
-  set(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR})
-  set(KOKKOS_PATH ${KOKKOS_SRC_PATH})
-
-  #------------ COMPILER AND FEATURE CHECKS ------------------------------------
-  include(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake)
-  set_kokkos_cxx_compiler()
-  set_kokkos_cxx_standard()
-
-  #------------ GET OPTIONS ----------------------------------------------------
-  set(KOKKOS_CMAKE_VERBOSE True)
-  set(KOKKOS_HAS_TRILINOS True)
-  include(${KOKKOS_SRC_PATH}/cmake/kokkos_options.cmake)
-
-  #------------ COMPUTE KOKKOS_SETTINGS ----------------------------------------
-  include(${KOKKOS_SRC_PATH}/cmake/kokkos_settings.cmake)
-
-  #------------ GENERATE HEADER AND SOURCE FILES -------------------------------
-  execute_process(
-    COMMAND ${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} generate_build_settings
-    WORKING_DIRECTORY "${Kokkos_GEN_DIR}"
-    OUTPUT_FILE ${Kokkos_GEN_DIR}/core_src_make.out
-    RESULT_VARIABLE GEN_SETTINGS_RESULT
-  )
-  if (GEN_SETTINGS_RESULT)
-    message(FATAL_ERROR "Kokkos settings generation failed:\n"
-        "${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} generate_build_settings")
-  endif()
-  include(${Kokkos_GEN_DIR}/kokkos_generated_settings.cmake)
-  set(libdir lib)
-  if (${PROJECT_NAME}_INSTALL_LIB_DIR)
-    set(libdir ${${PROJECT_NAME}_INSTALL_LIB_DIR})
-  endif()
-  if (INSTALL_LIB_DIR)
-    set(libdir ${INSTALL_LIB_DIR})
-  endif()
-  install(FILES ${Kokkos_GEN_DIR}/kokkos_generated_settings.cmake DESTINATION ${libdir}/cmake/Kokkos)
-
-  IF (NOT KOKKOS_ARCH STREQUAL "None")
-
-    # Convert KOKKOS_CXX_FLAGS, which is a CMake list, into a string for CXX_FLAGS
-    set(KOKKOS_CXX_FLAGS_str "")
-    # When compiling CUDA with Clang, the flags "-x cuda" and "--cuda-gpu-arch=sm_??"
-    # cannot be passed to the link line, so we sneak these into the lesser-used
-    # add_compile_options() function, which only affects the compile line and not the link line
-    foreach(opt ${KOKKOS_CXX_FLAGS})
-      if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-        if (opt MATCHES "--cuda-gpu-arch")
-          # Furthermore, add_compile_options normally affects all languages, so
-          # we need a generator expression to prevent CUDA flags being passed to C or Fortran
-          add_compile_options($<$<COMPILE_LANGUAGE:CXX>:${opt}>)
-        else()
-          set(KOKKOS_CXX_FLAGS_str "${KOKKOS_CXX_FLAGS_str} ${opt}")
-        endif()
-      else()
-        set(KOKKOS_CXX_FLAGS_str "${KOKKOS_CXX_FLAGS_str} ${opt}")
-      endif()
-    endforeach()
-    if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-      # Since "-x cuda" shows up as two arguments, its easier to filter out here:
-      if (KOKKOS_CXX_FLAGS_str MATCHES "-x cuda")
-        string(REPLACE "-x cuda" "" KOKKOS_CXX_FLAGS_str "${KOKKOS_CXX_FLAGS_str}")
-        add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-x>)
-        add_compile_options($<$<COMPILE_LANGUAGE:CXX>:cuda>)
-      endif()
-    endif()
-
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${KOKKOS_CXX_FLAGS_str}")
-
-    # TODO -- need to remove the -lkokkos.  Check on LDFlags
-    #set(KOKKOS_LINK_DEPENDS libkokkos.a CACHE STRING "")
-    #set(KOKKOS_LIBS -lkokkos -ldl -lpthread CACHE STRING "")
-    #set(KOKKOS_LDFLAGS -L/scr_gabrielle/kruger/builds-ptsolve/trilinos/par2 --gcc-toolchain=/usr CACHE STRING "")
+  MESSAGE("-- " "Skip adding flags for C++11 because Kokkos flags does that ...")
+  # Set this to empty to trick Tribits into passing the C++11 flag check
+  SET(${PROJECT_NAME}_CXX11_FLAGS)
 
-    MESSAGE("-- " "Skip adding flags for C++11 because Kokkos flags does that ...")
-    SET(${PROJECT_NAME}_CXX11_FLAGS " ")
-
-    MESSAGE("-- " "Skip adding flags for OpenMP because Kokkos flags does that ...")
-    SET(OpenMP_CXX_FLAGS_OVERRIDE " ")
-
-  ENDIF()
-
-  # Above, It is important not to distrube the default configuraiton of
-  # Trilinos if KOKKOS_ARCH is not set.  But the implementation of the new
-  # Kokkos TriBITS CMake files requires kokkos_generated_settings.cmake be
-  # included.
+  MESSAGE("-- " "Skip adding flags for OpenMP because Kokkos flags does that ...")
+  SET(OpenMP_CXX_FLAGS_OVERRIDE " ")
 
+  # There is a top-level CMAKE_CXX_FLAGS. Warnings and other flags get added
+  # in the sub-scope of each individual package
+  # Grab those variables here so we know what was the original top-level flags
+  # and what are the CMAKE_CXX_FLAGS added afterwards for an individual package
+  SET(TRILINOS_TOPLEVEL_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 ENDIF()

cmake/RepositoryDependenciesSetup.cmake

@@ -8,7 +8,7 @@ if (TPL_ENABLE_CUDA AND NOT Kokkos_ENABLE_Cuda_Relocatable_Device_Code)
     # ${PROJECT_NAME}_ENABLE_ShyLU_NodeTacho so that each reconfigure will
     # show this same note.
   elseif (${PROJECT_NAME}_ENABLE_ShyLU_NodeTacho)
-    message(FATAL_ERROR "ERROR: ${PROJECT_NAME}_ENABLE_ShyLU_NodeTacho=ON but TPL_ENABLE_CUDA='${TPL_ENABLE_CUDA}' AND Kokkos_ENABLE_Cuda_Relocatable_Device_Code='${Kokkos_ENABLE_Cuda_Relocatable_Device_Code}' which is not allowed!")
+    message(FATAL_ERROR "ERROR: ${PROJECT_NAME}_ENABLE_ShyLU_NodeTacho=ON but TPL_ENABLE_CUDA='${TPL_ENABLE_CUDA}' AND Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE='${Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE}' which is not allowed!")
   endif()
 endif() 
 

cmake/ctest/drivers/atdm/ats2/drivers/Trilinos-atdm-ats2-cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24_static_dbg.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Specialized
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats2/local-driver.sh

cmake/ctest/drivers/atdm/ats2/drivers/Trilinos-atdm-ats2-cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24_static_opt.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats2/local-driver.sh

cmake/ctest/drivers/atdm/ats2/drivers/Trilinos-atdm-ats2-gnu-7.3.1-spmpi-2019.06.24_serial_static_dbg.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Specialized
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats2/local-driver.sh

cmake/ctest/drivers/atdm/ats2/drivers/Trilinos-atdm-ats2-gnu-7.3.1-spmpi-2019.06.24_serial_static_opt.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=ATDM
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats2/local-driver.sh

cmake/ctest/drivers/atdm/ats2/local-driver.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+if [ "${LSF_CTEST_TIMEOUT}" == "" ] ; then
+  LSF_CTEST_TIMEOUT=4:00
+  # This is just running tests, not the entire build!
+fi
+
+if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then
+  export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE
+fi
+
+set -x
+
+source $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-config-build.sh
+
+set -x
+
+atdm_run_script_on_compute_node \
+  $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-test.sh \
+  $PWD/ctest-s-driver-test.out \
+  ${LSF_CTEST_TIMEOUT}

cmake/ctest/drivers/atdm/utils/setup_env.sh

@@ -9,8 +9,10 @@ echo
 echo "cmake in path:"
 which cmake
 echo
-echo "ninja in path:"
-which ninja
+if [[ "$ATDM_CONFIG_USE_NINJA" == "ON" ]]; then
+  echo "ninja in path:"
+  which ninja
+fi
 echo
 echo "ATDM config env vars:"
 set | grep ATDM_CONFIG_

cmake/std/PullRequestLinuxCommonTestingSettings.cmake

@@ -60,6 +60,9 @@ set (Piro_EpetraSolver_MPI_4_DISABLE ON CACHE BOOL "Set by default for PR testin
 # Disable tests that timeout in PR testing until it can be fixed (#4614)
 set (PanzerAdaptersSTK_PoissonInterfaceExample_2d_diffsideids_MPI_1_DISABLE ON CACHE BOOL "Set by default for PR testing")
 
+# Disable long-failing Anazazi test until it can be fixed (#3585)
+set (Anasazi_Epetra_OrthoManagerGenTester_0_MPI_4_DISABLE ON CACHE BOOL "Set by default for PR testing")
+
 # Options from SEMSDevEnv.cmake
 
 SET(CMAKE_C_COMPILER "$ENV{MPICC}" CACHE FILEPATH "Set by default for PR testing")

cmake/std/atdm/ATDMDevEnvSettings.cmake

@@ -57,6 +57,8 @@ ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(CUDA_RDC OFF)
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(FPIC OFF)
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(COMPLEX OFF)
 
+ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(Kokkos_ENABLE_Serial ON)
+
 IF (ATDM_USE_PTHREADS)
   MESSAGE(FATAL_ERROR "Error, the Kokkos Pthreads backend is no longer supported!"
     "  Please unset env var ATDM_CONFIG_USE_PTHREADS!")
@@ -281,6 +283,8 @@ ATDM_SET_CACHE(Kokkos_ENABLE_Pthread "${ATDM_USE_PTHREADS}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_UVM "${ATDM_USE_CUDA}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_Relocatable_Device_Code "$ENV{ATDM_CONFIG_CUDA_RDC}"
   CACHE BOOL)
+ATDM_SET_CACHE(Kokkos_ENABLE_Serial "${ATDM_Kokkos_ENABLE_Serial}" CACHE BOOL)
+ATDM_SET_CACHE(Kokkos_ENABLE_Cuda "${ATDM_USE_CUDA}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_CXX11_DISPATCH_LAMBDA ON CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Cuda_Lambda "${ATDM_USE_CUDA}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_Debug_Bounds_Check "${Trilinos_ENABLE_DEBUG}" CACHE BOOL)
@@ -307,6 +311,21 @@ IF (ATDM_ENABLE_SPARC_SETTINGS)
   ATDM_SET_CACHE(Anasazi_ENABLE_RBGen ON CACHE BOOL)
 ENDIF()
 
+# Disable/hide deprecated code
+ATDM_SET_CACHE(KOKKOS_ENABLE_DEPRECATED_CODE OFF CACHE BOOL)
+ATDM_SET_CACHE(Tpetra_ENABLE_DEPRECATED_CODE OFF CACHE BOOL)
+ATDM_SET_CACHE(Belos_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Epetra_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Ifpack2_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Ifpack2_ENABLE_DEPRECATED_CODE OFF CACHE BOOL)
+ATDM_SET_CACHE(MueLu_ENABLE_DEPRECATED_CODE OFF CACHE BOOL)
+ATDM_SET_CACHE(Panzer_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Phalanx_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(STK_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Teuchos_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Thyra_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+ATDM_SET_CACHE(Trios_HIDE_DEPRECATED_CODE ON CACHE BOOL)
+
 #
 # F) TPL locations and enables
 #

cmake/std/atdm/ats2/all_supported_builds.sh

@@ -0,0 +1,11 @@
+
+export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-
+
+#  ats2-xl-2019.08.20-spmpi-2019.06.24_openmp_static_opt
+#  ats2-xl-2019.08.20-spmpi-2019.06.24_openmp_static_dbg
+export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=(
+  ats2-gnu-7.3.1-spmpi-2019.06.24_serial_static_opt
+  ats2-gnu-7.3.1-spmpi-2019.06.24_serial_static_dbg
+  ats2-cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24_static_opt
+  ats2-cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24_static_dbg
+  )

cmake/std/atdm/ats2/custom_builds.sh

@@ -0,0 +1,64 @@
+#
+# Custom builds for 'ats2' env
+#
+# NOTE: This file gets sourced in atdm/utils/set_build_options.sh before the
+# default grep logic is applied.
+#
+
+# Custom compiler selection logic
+
+if   [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-xl-2019.08.20-spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_xl-2019.08.20_spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-xl-2019.08.20"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_xl-2019.08.20"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-xl-2019"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_xl-2019"* ]] \
+  ; then
+  export ATDM_CONFIG_COMPILER=CUDA-10.1.243_XL-2019.08.20_SPMPI-2019.06.24
+
+elif [[ $ATDM_CONFIG_BUILD_NAME == *"xl-2019.08.20-spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"xl-2019.08.20_spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"xl-2019.08.20"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"xl-2019"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"xl"* ]] \
+  ; then
+  export ATDM_CONFIG_COMPILER=XL-2019.08.20_SPMPI-2019.06.24
+
+elif [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_gnu-7.3.1_spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-gnu-7.3.1"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_gnu-7.3.1"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243-gnu-7"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243_gnu-7"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10.1.243"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda-10"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"cuda"* ]] \
+  ; then
+  export ATDM_CONFIG_COMPILER=CUDA-10.1.243_GNU-7.3.1_SPMPI-2019.06.24
+  # NOTE: Default 'cuda' must be last cuda listed!
+
+elif [[ $ATDM_CONFIG_BUILD_NAME == *"gnu-7.3.1-spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"gnu-7.3.1_spmpi-2019.06.24"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"gnu-7.3.1"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"gnu-7"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"gnu"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"default" ]] \
+  ; then
+  export ATDM_CONFIG_COMPILER=GNU-7.3.1_SPMPI-2019.06.24
+  # NOTE: Defaut 'gnu' must be last 'gnu' listed!
+
+else
+  echo
+  echo "***"
+  echo "*** ERROR: A supported compiler was not selected for 'ats2' env - $ATDM_CONFIG_BUILD_NAME"
+  echo "***"
+  echo "*** Supported compilers include:"
+  echo "***"
+  echo "****  gnu-7.3.1_spmpi-2019.06.24                  (default, default gnu)"
+  echo "****  cuda-10.1.243_gnu-7.3.1_spmpi-2019.06.24    (default cuda)"
+  echo "****  xl-2019.08.20_spmpi-2019.06.24              (disabled)"
+  echo "****  cuda-10.1.243-gnu-7.3.1-spmpi-2019.06.24    (disabled)"
+  echo "***"
+  return
+
+fi