From 892a1a6a1bfda96f887d184e2d471787d3734bf2 Mon Sep 17 00:00:00 2001 From: "K. Devine" Date: Tue, 15 Sep 2020 20:54:23 -0600 Subject: [PATCH 1/3] Tpetra: removed CUDA-specific code that doesn't work with CUDA 11. #7923 CUDA-specific code fits better in Kokkos, but @crtrott says this particular capability is too compiler specific even for Kokkos. --- .../core/src/Tpetra_Details_checkPointer.cpp | 171 ------------------ .../core/src/Tpetra_Details_checkPointer.hpp | 116 ------------ .../core/src/Tpetra_Details_checkView.hpp | 47 +---- packages/tpetra/core/src/Tpetra_Map_def.hpp | 18 -- .../tpetra/core/test/Utils/CMakeLists.txt | 9 - 5 files changed, 7 insertions(+), 354 deletions(-) delete mode 100644 packages/tpetra/core/src/Tpetra_Details_checkPointer.cpp delete mode 100644 packages/tpetra/core/src/Tpetra_Details_checkPointer.hpp diff --git a/packages/tpetra/core/src/Tpetra_Details_checkPointer.cpp b/packages/tpetra/core/src/Tpetra_Details_checkPointer.cpp deleted file mode 100644 index 7317b6eeb9be..000000000000 --- a/packages/tpetra/core/src/Tpetra_Details_checkPointer.cpp +++ /dev/null @@ -1,171 +0,0 @@ -// @HEADER -// *********************************************************************** -// -// Tpetra: Templated Linear Algebra Services Package -// Copyright (2008) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// ************************************************************************ -// @HEADER - -#include "Tpetra_Details_checkPointer.hpp" -#include "Kokkos_Core.hpp" -#ifdef HAVE_TPETRACORE_CUDA -# include "cuda_runtime_api.h" -#endif // HAVE_TPETRACORE_CUDA - -namespace Tpetra { -namespace Details { -namespace Impl { - -enum class EMemoryType { - HOST, - CUDA_HOST_PINNED, - CUDA, - CUDA_UVM, - ERROR -}; - -EMemoryType getCudaMemoryType (const void* ptr) -{ -#ifdef HAVE_TPETRACORE_CUDA - cudaPointerAttributes attr; - const cudaError_t err = cudaPointerGetAttributes (&attr, ptr); - (void) cudaGetLastError (); // reset error state for future CUDA ops - - if (err == cudaErrorInvalidValue) { - // CUDA 11.0 supports passing in an unregistered host pointer. In - // that case, attr.type will be cudaMemoryTypeUnregistered. CUDA - // 9.2 doesn't yet have the 'type' field in the - // cudaPointerAttributes struct, and this function just returns - // this error code if given an unregistered host pointer. - return EMemoryType::HOST; - } - else if (err != cudaSuccess) { - return EMemoryType::ERROR; - } - -# if 1 - // cudaPointerAttributes::type doesn't exist yet in CUDA 9.2. We - // must use memoryType, which does not distinguish between types of - // device memory. - if (attr.memoryType == cudaMemoryTypeDevice) { - if (attr.isManaged) { - return EMemoryType::CUDA_UVM; - } - else { // if (attr.hostPointer == nullptr) { - return EMemoryType::CUDA; - } - // else { - // return EMemoryType::ERROR; - // } - } - else if (attr.memoryType == cudaMemoryTypeHost) { - if (attr.devicePointer == nullptr) { - return EMemoryType::HOST; // not device accessible - } - else { // device-accessible host memory is pinned - return EMemoryType::CUDA_HOST_PINNED; - } - } - else { - return EMemoryType::ERROR; - } - -# else - - const enum cudaMemoryType theType = attr.type; - if (theType == cudaMemoryTypeManaged) { - return EMemoryType::CUDA_UVM; - } - else if (theType == cudaMemoryTypeDevice) { - return EMemoryType::CUDA; - } - else if (theType == cudaMemoryTypeHost) { - return EMemoryType::CUDA_HOST_PINNED; - } - else { - return EMemoryType::HOST; - } - -# endif // 1 -#else // NOT HAVE_TPETRACORE_CUDA - return EMemoryType::HOST; -#endif // HAVE_TPETRACORE_CUDA -} - -#ifdef HAVE_TPETRACORE_CUDA -bool -CheckPointerAccessibility:: -accessible (const void* ptr, - const Kokkos::Cuda& /* space */) -{ - const EMemoryType type = getCudaMemoryType (ptr); - return type != EMemoryType::HOST && type != EMemoryType::ERROR; -} -#endif // HAVE_TPETRACORE_CUDA - -bool isHostAccessible (const void* ptr) -{ -#ifdef HAVE_TPETRACORE_CUDA - return getCudaMemoryType (ptr) != EMemoryType::CUDA; -#else - return true; -#endif // HAVE_TPETRACORE_CUDA -} - -} // namespace Impl - -std::string memorySpaceName (const void* ptr) -{ - using Impl::EMemoryType; - - const EMemoryType type = Impl::getCudaMemoryType (ptr); - if (type == EMemoryType::CUDA_UVM) { - return "CudaUVMSpace"; - } - else if (type == EMemoryType::CUDA) { - return "CudaSpace"; - } - else if (type == EMemoryType::CUDA_HOST_PINNED) { - return "CudaHostPinnedSpace"; - } - else if (type == EMemoryType::HOST) { - return "HostSpace"; - } - else { // EMemoryType::ERROR - return "ERROR"; - } -} - -} // namespace Details -} // namespace Tpetra diff --git a/packages/tpetra/core/src/Tpetra_Details_checkPointer.hpp b/packages/tpetra/core/src/Tpetra_Details_checkPointer.hpp deleted file mode 100644 index 08f053e84687..000000000000 --- a/packages/tpetra/core/src/Tpetra_Details_checkPointer.hpp +++ /dev/null @@ -1,116 +0,0 @@ -// @HEADER -// *********************************************************************** -// -// Tpetra: Templated Linear Algebra Services Package -// Copyright (2008) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// ************************************************************************ -// @HEADER - -#ifndef TPETRA_DETAILS_CHECKPOINTER_HPP -#define TPETRA_DETAILS_CHECKPOINTER_HPP - -/// \file Tpetra_Details_checkPointer.hpp -/// \brief Declaration of functions for checking whether a given -/// pointer is accessible from a given Kokkos execution space. -/// -/// \warning This header file and its contents are implementation -/// details of Tpetra. - -#include "TpetraCore_config.h" -#include - -#ifdef HAVE_TPETRACORE_CUDA -#ifndef DOXYGEN_SHOULD_SKIP_THIS -namespace Kokkos { - class Cuda; // forward declaration -} // namespace Kokkos -#endif // DOXYGEN_SHOULD_SKIP_THIS -#endif // HAVE_TPETRACORE_CUDA - -namespace Tpetra { -namespace Details { -namespace Impl { - -bool isHostAccessible (const void* ptr); - -template -struct CheckPointerAccessibility { - static bool - accessible (const void* ptr, - const ExecutionSpace& /* space */) - { - return isHostAccessible (ptr); - } -}; - -#ifdef HAVE_TPETRACORE_CUDA -template<> -struct CheckPointerAccessibility { - static bool - accessible (const void* ptr, - const Kokkos::Cuda& space); -}; -#endif // HAVE_TPETRACORE_CUDA - -} // namespace Impl - -/// \brief Is the given nonnull ptr accessible from the given -/// execution space? -/// -/// This function doesn't promise exact results for anything other -/// than CudaSpace, CudaUVMSpace, or CudaHostPinnedSpace. The point -/// of this function is for Tpetra classes to debug user error in -/// which users create a Kokkos::View with a raw pointer in the wrong -/// memory space (e.g., a host pointer, when they should have used a -/// UVM pointer). -template -bool -pointerAccessibleFromExecutionSpace (const void* ptr, - const ExecutionSpace& space) -{ - using impl_type = Impl::CheckPointerAccessibility; - return impl_type::accessible (ptr, space); -} - -/// \brief Return the Kokkos memory space name (without "Kokkos::") -/// corresponding to the given nonnull pointer. -/// -/// This function doesn't promise exact results for anything other -/// than CudaSpace, CudaUVMSpace, or CudaHostPinnedSpace. -std::string memorySpaceName (const void* ptr); - -} // namespace Details -} // namespace Tpetra - -#endif // TPETRA_DETAILS_CHECKPOINTER_HPP diff --git a/packages/tpetra/core/src/Tpetra_Details_checkView.hpp b/packages/tpetra/core/src/Tpetra_Details_checkView.hpp index 9be7674c1671..1f56b19d1c2f 100644 --- a/packages/tpetra/core/src/Tpetra_Details_checkView.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_checkView.hpp @@ -40,14 +40,13 @@ #ifndef TPETRA_DETAILS_CHECKVIEW_HPP #define TPETRA_DETAILS_CHECKVIEW_HPP -/// \file Tpetra_Details_checkPointer.hpp +/// \file Tpetra_Details_checkView.hpp /// \brief Declaration of functions for checking whether a given /// pointer is accessible from a given Kokkos execution space. /// /// \warning This header file and its contents are implementation /// details of Tpetra. -#include "Tpetra_Details_checkPointer.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Kokkos_DualView.hpp" #include "Teuchos_TypeNameTraits.hpp" @@ -58,11 +57,6 @@ namespace Tpetra { namespace Details { -template -bool -pointerAccessibleFromExecutionSpace (const void* ptr, - const ExecutionSpace& space); - std::string memorySpaceName (const void* ptr); /// \brief Is the given View valid? @@ -76,13 +70,6 @@ std::string memorySpaceName (const void* ptr); /// host pointer with execution_space Kokkos::Cuda). /// /// -/// This function doesn't promise exact results for anything other -/// than CudaSpace, CudaUVMSpace, or CudaHostPinnedSpace. The point -/// of this function is for Tpetra classes to debug user error in -/// which users create a Kokkos::View with a raw pointer in the wrong -/// memory space (e.g., a host pointer, when they should have used a -/// UVM pointer). -/// /// \param lclErrStrm [out] If the View is invalid, and this pointer /// is nonnull, then write a human-readable explanation of what's /// wrong with the View to the stream. @@ -121,21 +108,8 @@ checkLocalViewValidity } return false; } - else { // nonnull pointer, nonzero size - const bool canAcc = pointerAccessibleFromExecutionSpace (ptr, ES ()); - if (! canAcc && lclErrStrm != nullptr) { - const std::string viewName = TypeNameTraits::name (); - const std::string esName = TypeNameTraits::name (); - *lclErrStrm << "Proc " << myMpiProcessRank << ": Kokkos::View " - "of type " << viewName << " and nonzero size " << view.size () - << " has a pointer " << ptr << " which is nonnull, but not " - "accessible from the View's claimed execution space " - << esName << ". As far as I can tell, the View's pointer " - "(view.data()) lives in memory space " << memorySpaceName (ptr) - << "." << endl; - } - return canAcc; - } + else + return true; } } @@ -218,18 +192,11 @@ checkGlobalDualViewValidity if (gblSuccess != 1 && gblErrStrm != nullptr) { *gblErrStrm << "On at least one (MPI) process, the " - "Kokkos::DualView has either device or host pointer that " - "is not accessible from the corresponding execution space. " - "This can happen if you, the user, created the DualView " - "with raw pointer(s) that is/are not in the correct memory " - "space. For example, you may have a Kokkos::DualView whose " - "device memory_space is Kokkos::CudaUVMSpace, but you gave " - "the device View's constructor a raw host pointer. It may " - "also happen if either the device or host pointer in the " - "DualView is null, but the DualView has a nonzero number of " + "Kokkos::DualView has " + "either the device or host pointer in the " + "DualView equal to null, but the DualView has a nonzero number of " "rows. For more detailed information, please rerun with the " - "TPETRA_VERBOSE environment variable set to 1. (You do not " - "need to recompile.)"; + "TPETRA_VERBOSE environment variable set to 1. "; if (verbose) { *gblErrStrm << " Here are error messages from all " "processes:" << endl; diff --git a/packages/tpetra/core/src/Tpetra_Map_def.hpp b/packages/tpetra/core/src/Tpetra_Map_def.hpp index 37bd5156c790..b6848568bb65 100644 --- a/packages/tpetra/core/src/Tpetra_Map_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Map_def.hpp @@ -47,7 +47,6 @@ #include "Tpetra_Directory.hpp" // must include for implicit instantiation to work #include "Tpetra_Details_Behavior.hpp" -#include "Tpetra_Details_checkPointer.hpp" #include "Tpetra_Details_FixedHashTable.hpp" #include "Tpetra_Details_gathervPrint.hpp" #include "Tpetra_Details_printOnce.hpp" @@ -78,7 +77,6 @@ namespace { // (anonymous) const bool debug = Behavior::debug("Map"); if (debug) { - using Tpetra::Details::pointerAccessibleFromExecutionSpace; using Teuchos::outArg; using Teuchos::REDUCE_MIN; using Teuchos::reduceAll; @@ -96,22 +94,6 @@ namespace { // (anonymous) "but indexListSize=" << indexListSize << " != 0." << endl; } } - else { - if (indexListSize != 0 && indexList != nullptr && - ! pointerAccessibleFromExecutionSpace (indexList, execSpace)) { - lclSuccess = 0; - if (verbose) { - using ::Tpetra::Details::memorySpaceName; - const std::string memSpaceName = memorySpaceName (indexList); - const std::string execSpaceName = - Teuchos::TypeNameTraits::name (); - lclErrStrm << "Proc " << myRank << ": Input array is not " - "accessible from the required execution space " << - execSpaceName << ". As far as I can tell, array lives " - "in memory space " << memSpaceName << "." << endl; - } - } - } int gblSuccess = 0; // output argument reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess)); if (gblSuccess != 1) { diff --git a/packages/tpetra/core/test/Utils/CMakeLists.txt b/packages/tpetra/core/test/Utils/CMakeLists.txt index a9280f194091..fbb42d9014e5 100644 --- a/packages/tpetra/core/test/Utils/CMakeLists.txt +++ b/packages/tpetra/core/test/Utils/CMakeLists.txt @@ -127,15 +127,6 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( STANDARD_PASS_OUTPUT ) -TRIBITS_ADD_EXECUTABLE_AND_TEST( - CheckMemoryType - SOURCES - CheckMemoryType - COMM serial mpi - NUM_MPI_PROCS 1 - STANDARD_PASS_OUTPUT - ) - TRIBITS_ADD_EXECUTABLE_AND_TEST( verbosePrintArray SOURCES From 4f708fda40062025cdae66ec71c6b46019bc1dde Mon Sep 17 00:00:00 2001 From: "K. Devine" Date: Tue, 15 Sep 2020 21:08:07 -0600 Subject: [PATCH 2/3] tpetra: removing test that is no longer needed --- .../core/test/Utils/CheckMemoryType.cpp | 256 ------------------ 1 file changed, 256 deletions(-) delete mode 100644 packages/tpetra/core/test/Utils/CheckMemoryType.cpp diff --git a/packages/tpetra/core/test/Utils/CheckMemoryType.cpp b/packages/tpetra/core/test/Utils/CheckMemoryType.cpp deleted file mode 100644 index 3cf89cec5730..000000000000 --- a/packages/tpetra/core/test/Utils/CheckMemoryType.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* -// @HEADER -// *********************************************************************** -// -// Tpetra: Templated Linear Algebra Services Package -// Copyright (2008) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Michael A. Heroux (maherou@sandia.gov) -// -// ************************************************************************ -// @HEADER -*/ - -#include "Tpetra_TestingUtilities.hpp" -#include "Tpetra_Details_checkPointer.hpp" -#include "Tpetra_Details_Behavior.hpp" -#include "Kokkos_Core.hpp" - -namespace { // (anonymous) - - void - testHostPointer (bool& success, - Teuchos::FancyOStream& out, - const void* ptr) - { - using Tpetra::Details::pointerAccessibleFromExecutionSpace; - using Tpetra::Details::memorySpaceName; - using std::endl; - - Kokkos::DefaultHostExecutionSpace hostExecSpace; -#ifdef HAVE_TPETRACORE_CUDA - Kokkos::Cuda cudaExecSpace; -#endif // HAVE_TPETRACORE_CUDA - - const std::string name = memorySpaceName (ptr); - out << "memorySpaceName returned \"" << name << "\"" << endl; - TEST_EQUALITY( name, "HostSpace" ); - const bool canAccessFromHost = - pointerAccessibleFromExecutionSpace (ptr, hostExecSpace); - TEST_ASSERT( canAccessFromHost ); - -#ifdef HAVE_TPETRACORE_CUDA - const bool canAccessFromCuda = - pointerAccessibleFromExecutionSpace (ptr, cudaExecSpace); - out << "pointerAccessibleFromExecutionSpace(ptr, Cuda()): " - << (canAccessFromCuda ? "true" : "false") << endl; - TEST_ASSERT( ! canAccessFromCuda ); -#endif // HAVE_TPETRACORE_CUDA - } - - void - testCheckMemoryType (bool& success, - Teuchos::FancyOStream& out) - { - using Tpetra::Details::pointerAccessibleFromExecutionSpace; - using Tpetra::Details::memorySpaceName; - using std::endl; - - out << "Starting CheckMemoryType test" << endl; - Teuchos::OSTab tab1 (out); - -#ifdef HAVE_TPETRACORE_CUDA - Kokkos::DefaultHostExecutionSpace hostExecSpace; - Kokkos::Cuda cudaExecSpace; -#endif // HAVE_TPETRACORE_CUDA - - { - out << "Test raw integer on host" << endl; - Teuchos::OSTab tab2 (out); - - int rawInt = 666; - testHostPointer (success, out, &rawInt); - TEST_ASSERT( rawInt == 666 ); - - out << "Result: " << (success ? "true" : "false") << endl; - } - { - out << "Test host Kokkos::View" << endl; - Teuchos::OSTab tab2 (out); - - using view_type = - Kokkos::View; - view_type intView ("Host int"); - intView() = 418; - testHostPointer (success, out, intView.data ()); - TEST_ASSERT( intView() == 418 ); - - out << "Result: " << (success ? "true" : "false") << endl; - } -#ifdef HAVE_TPETRACORE_CUDA - { - out << "Test raw cudaMalloc" << endl; - Teuchos::OSTab tab2 (out); - - int* devPtr = nullptr; - cudaError_t err = cudaMalloc (&devPtr, sizeof (int)); - - if (err != cudaSuccess) { - out << "Oh no!" << endl; - } - else { - const bool canAccessFromHost = - pointerAccessibleFromExecutionSpace (devPtr, hostExecSpace); - TEST_ASSERT( ! canAccessFromHost ); - const bool canAccessFromCuda = - pointerAccessibleFromExecutionSpace (devPtr, cudaExecSpace); - TEST_ASSERT( canAccessFromCuda ); - - const std::string name = memorySpaceName (devPtr); - out << "memorySpaceName returned \"" << name << "\"" << endl; - TEST_EQUALITY( name, "CudaSpace" ); - - cudaFree (devPtr); - } - out << "Result: " << (success ? "true" : "false") << endl; - } - { - out << "Test Kokkos::View of CudaSpace" << endl; - Teuchos::OSTab tab2 (out); - - Kokkos::View cudaView ("Cuda"); - Kokkos::deep_copy (cudaView, 777); - const bool canAccessFromHost = - pointerAccessibleFromExecutionSpace (cudaView.data (), hostExecSpace); - TEST_ASSERT( ! canAccessFromHost ); - const bool canAccessFromCuda = - pointerAccessibleFromExecutionSpace (cudaView.data (), cudaExecSpace); - TEST_ASSERT( canAccessFromCuda ); - - const std::string name = memorySpaceName (cudaView.data ()); - out << "memorySpaceName returned \"" << name << "\"" << endl; - TEST_EQUALITY( name, "CudaSpace" ); - - int finalValue = 0; - Kokkos::deep_copy (finalValue, cudaView); - TEST_ASSERT( finalValue == 777 ); - - out << "Result: " << (success ? "true" : "false") << endl; - } - { - out << "Test Kokkos::View of CudaUVMSpace" << endl; - Teuchos::OSTab tab2 (out); - - using view_type = Kokkos::View; - view_type view ("CudaUVMSpace"); - Kokkos::deep_copy (view, 31); - const bool canAccessFromHost = - pointerAccessibleFromExecutionSpace (view.data (), hostExecSpace); - TEST_ASSERT( canAccessFromHost ); - const bool canAccessFromCuda = - pointerAccessibleFromExecutionSpace (view.data (), cudaExecSpace); - TEST_ASSERT( canAccessFromCuda ); - - const std::string name = memorySpaceName (view.data ()); - out << "memorySpaceName returned \"" << name << "\"" << endl; - TEST_EQUALITY( name, "CudaUVMSpace" ); - - int finalValue = 0; - Kokkos::deep_copy (finalValue, view); - TEST_ASSERT( finalValue == 31 ); - - out << "Result: " << (success ? "true" : "false") << endl; - } - { - out << "Test Kokkos::View of CudaHostPinnedSpace" << endl; - Teuchos::OSTab tab2 (out); - - using view_type = Kokkos::View; - view_type view ("CudaHostPinned"); - Kokkos::deep_copy (view, 93); - - const bool canAccessFromHost = - pointerAccessibleFromExecutionSpace (view.data (), hostExecSpace); - TEST_ASSERT( canAccessFromHost ); - const bool canAccessFromCuda = - pointerAccessibleFromExecutionSpace (view.data (), cudaExecSpace); - TEST_ASSERT( canAccessFromCuda ); - - // The CUDA API should be able to distinguish between - // host-pinned memory and plain old host memory. - const std::string name = memorySpaceName (view.data ()); - out << "memorySpaceName returned \"" << name << "\"" << endl; - TEST_EQUALITY( name, "CudaHostPinnedSpace" ); - - int finalValue = 0; - Kokkos::deep_copy (finalValue, view); - TEST_ASSERT( finalValue == 93 ); - - out << "Result: " << (success ? "true" : "false") << endl; - } -#endif // HAVE_TPETRACORE_CUDA - } - - TEUCHOS_UNIT_TEST( Utils, CheckMemoryType ) - { - // Replace 'out' with cerr in verbose mode. This lets us diagnose - // crashes, since the Teuchos unit test framework normally - // captures output until the test finishes. - using Teuchos::FancyOStream; - using Teuchos::RCP; - using Teuchos::rcpFromRef; - RCP myOutPtr; - const bool verbose = Tpetra::Details::Behavior::verbose (); - if (verbose) { - myOutPtr = Teuchos::getFancyOStream (rcpFromRef (std::cerr)); - } - else { - myOutPtr = rcpFromRef (out); - } - FancyOStream& myOut = *myOutPtr; - testCheckMemoryType (success, myOut); - } -} // namespace (anonymous) - -int -main (int argc, char* argv[]) -{ - // Initialize MPI (if enabled) before initializing Kokkos. This - // lets MPI control things like pinning processes to sockets. - Teuchos::GlobalMPISession mpiSession (&argc, &argv); - Kokkos::initialize (argc, argv); - const int errCode = - Teuchos::UnitTestRepository::runUnitTestsFromMain (argc, argv); - Kokkos::finalize (); - return errCode; -} From 065fbd4f6ad4a7bea23b4b2e15a1bb4673b48117 Mon Sep 17 00:00:00 2001 From: Christian Glusa Date: Fri, 18 Sep 2020 10:05:08 -0600 Subject: [PATCH 3/3] Geminga: Fix issue finding CUDA and use newer boost --- .../geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake | 1 + cmake/ctest/drivers/geminga/cron_driver.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake b/cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake index 61856d3f36af..87081fbcad3b 100644 --- a/cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake +++ b/cmake/ctest/drivers/geminga/TrilinosCTestDriverCore.geminga.gcc-cuda.cmake @@ -101,6 +101,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER) ### TPLS ### "-DTPL_ENABLE_CUDA:BOOL=ON" + "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" "-DTPL_ENABLE_CUSPARSE:BOOL=ON" "-DTPL_ENABLE_HWLOC:BOOL=OFF" diff --git a/cmake/ctest/drivers/geminga/cron_driver.sh b/cmake/ctest/drivers/geminga/cron_driver.sh index 0eabaa2c70da..db08667fb5e2 100755 --- a/cmake/ctest/drivers/geminga/cron_driver.sh +++ b/cmake/ctest/drivers/geminga/cron_driver.sh @@ -76,7 +76,7 @@ export CTEST_CONFIGURATION="nvcc_wrapper" module load sems-env module load sems-cmake/3.12.2 module load sems-gcc/8.3.0 -module load sems-boost/1.63.0/base +module load sems-boost/1.69.0/base module load sems-python/2.7.9 module load sems-zlib/1.2.8/base module load sems-openmpi/4.0.2 @@ -115,7 +115,7 @@ module unload sems-cuda/10.1 module unload sems-openmpi/4.0.2 module unload sems-zlib/1.2.8/base module unload sems-python/2.7.9 -module unload sems-boost/1.63.0/base +module unload sems-boost/1.69.0/base module unload sems-gcc/8.3.0 module unload sems-cmake/3.12.2 module unload sems-env