Tpetra: Allow enabling SYCL backend in Tpetra

packages/tpetra/CMakeLists.txt

@@ -33,7 +33,12 @@ ENDIF ()
 # objects for CUDA.  See Trilinos GitHub issue #1939.
 
 ASSERT_DEFINED(TPL_ENABLE_CUDA)
+
+ASSERT_DEFINED(Kokkos_ENABLE_OPENMP)
+ASSERT_DEFINED(Kokkos_ENABLE_PTHREAD)
+ASSERT_DEFINED(Kokkos_ENABLE_SERIAL)
 ASSERT_DEFINED(Kokkos_ENABLE_CUDA)
+ASSERT_DEFINED(Kokkos_ENABLE_SYCL)
 
 IF (TPL_ENABLE_CUDA AND Kokkos_ENABLE_CUDA)
   SET (Tpetra_ENABLE_CUDA_DEFAULT ON)
@@ -362,10 +367,7 @@ GLOBAL_SET(HAVE_TPETRA_INST_SERIAL_DEFAULT OFF)
 GLOBAL_SET(HAVE_TPETRA_INST_PTHREAD_DEFAULT OFF)
 GLOBAL_SET(HAVE_TPETRA_INST_OPENMP_DEFAULT OFF)
 GLOBAL_SET(HAVE_TPETRA_INST_CUDA_DEFAULT OFF)
-
-ASSERT_DEFINED(Kokkos_ENABLE_OPENMP)
-ASSERT_DEFINED(Kokkos_ENABLE_PTHREAD)
-ASSERT_DEFINED(Kokkos_ENABLE_SERIAL)
+GLOBAL_SET(HAVE_TPETRA_INST_SYCL_DEFAULT OFF)
 
 # We already have a separate Tpetra_ENABLE_CUDA option that we defined
 # above, so we don't have to check Kokkos_ENABLE_CUDA here.
@@ -509,7 +511,7 @@ ELSE () # NOT Tpetra_INST_CUDA
 ENDIF () # Tpetra_INST_CUDA
 
 # Kokkos::HIP (Kokkos::Compat::KokkosHIPWrapperNode)
-GLOBAL_SET(HAVE_TPETRA_INST_HIP_DEFAULT off)
+GLOBAL_SET(HAVE_TPETRA_INST_HIP_DEFAULT OFF)
 TRIBITS_ADD_OPTION_AND_DEFINE(
   Tpetra_INST_HIP
   HAVE_TPETRA_INST_HIP
@@ -529,6 +531,28 @@ ELSE () # NOT Tpetra_INST_HIP
     MESSAGE(STATUS "NOTE: Kokkos::HIP is ON (the CMake option Kokkos_ENABLE_HIP is ON), but the corresponding Tpetra Node type is disabled.  If you want to enable instantiation and use of Kokkos::HIP in Tpetra, please also set the CMake option Tpetra_INST_HIP:BOOL=ON.  If you use the Kokkos::HIP version of Tpetra without doing this, you will get link errors!")
   ENDIF ()
 ENDIF () # Tpetra_INST_HIP
+
+# Kokkos::SYCL (Kokkos::Compat::KokkosSYCLWrapperNode)
+TRIBITS_ADD_OPTION_AND_DEFINE(
+  Tpetra_INST_SYCL
+  HAVE_TPETRA_INST_SYCL
+  "Instantiate and/or test Tpetra classes over Node = Kokkos::Compat::KokkosSYCLWrapperNode.  If ETI is OFF, enable tests for that Node type.  This option is ${HAVE_TPETRA_INST_SYCL_DEFAULT} by default."
+  ${HAVE_TPETRA_INST_SYCL_DEFAULT}
+  )
+GLOBAL_SET(HAVE_TPETRA_SYCL ${Tpetra_INST_SYCL})
+IF (Tpetra_INST_SYCL)
+  IF (NOT Kokkos_ENABLE_SYCL)
+    MESSAGE(FATAL_ERROR "Tpetra: The Kokkos::SYCL execution space is disabled, but you enabled the corresponding Tpetra Node type by setting Tpetra_INST_SYCL=ON.  If you want to enable instantiation and use of Kokkos::SYCL in Tpetra, you must enable the Kokkos::SYCL execution space by setting Kokkos_ENABLE_SYCL=ON.")
+  ENDIF ()
+ELSE () # NOT Tpetra_INST_SYCL
+  IF (HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE)
+    MESSAGE(FATAL_ERROR "Tpetra: Node = Kokkos::Compat::KokkosSYCLWrapperNode is disabled (since Tpetra_INST_SYCL=OFF), but you set it as the default Node type.  Try setting the CMake options Kokkos_ENABLE_SYCL:BOOL=ON and Tpetra_INST_SYCL:BOOL=ON.  If you are building with a SYCL-capable compiler and Kokkos can detect that, then you are unlikely to see this message, since both Kokkos and Tpetra enable SYCL support by default in that case.")
+  ENDIF()
+  IF (Kokkos_ENABLE_SYCL)
+    MESSAGE(STATUS "NOTE: Kokkos::SYCL is ON (the CMake option Kokkos_ENABLE_SYCL is ON), but the corresponding Tpetra Node type is disabled.  If you want to enable instantiation and use of Kokkos::SYCL in Tpetra, please also set the CMake option Tpetra_INST_SYCL:BOOL=ON.  If you use the Kokkos::SYCL version of Tpetra without doing this, you will get link errors!")
+  ENDIF ()
+ENDIF () # Tpetra_INST_SYCL
+
 #
 # Check that users did not attempt to enable both the OpenMP and
 # Pthreads back-ends.
@@ -567,6 +591,13 @@ IF(Tpetra_INST_HIP)
   LIST(APPEND ${PACKAGE_NAME}_ETI_DEVICES "Kokkos::Device<Kokkos::Experimental::HIP, Kokkos::Experimental::HIPHostPinnedSpace>")
   LIST(APPEND ${PACKAGE_NAME}_ETI_NODES "Kokkos::Compat::KokkosHIPWrapperNode")
 ENDIF()
+IF(Tpetra_INST_SYCL)
+  # TpetraCore currently requires UVM, so it doesn't make sense to
+  # instantiate over Kokkos::SYCLDeviceUSMSpace as well as
+  # Kokkos::SYCLSharedUSMSpace.
+  LIST(APPEND ${PACKAGE_NAME}_ETI_DEVICES "Kokkos::Device<Kokkos::Experimental::SYCL, Kokkos::Experimental::SYCLSharedUSMSpace>")
+  LIST(APPEND ${PACKAGE_NAME}_ETI_NODES "Kokkos::Compat::KokkosSYCLWrapperNode")
+ENDIF()
 
 # Tell users what Nodes are enabled.
 MESSAGE(STATUS "Tpetra execution space availability (ON means available): ")
@@ -575,11 +606,12 @@ MESSAGE(STATUS "  - Threads: ${HAVE_TPETRA_PTHREAD}")
 MESSAGE(STATUS "  - OpenMP:  ${HAVE_TPETRA_OPENMP}")
 MESSAGE(STATUS "  - Cuda:    ${HAVE_TPETRA_CUDA}")
 MESSAGE(STATUS "  - HIP:     ${HAVE_TPETRA_HIP}")
+MESSAGE(STATUS "  - SYCL:    ${HAVE_TPETRA_SYCL}")
 
 # Fix Github Issue #190 by making sure that users enabled at least one
 # Node type.
-IF(NOT Tpetra_INST_SERIAL AND NOT Tpetra_INST_PTHREAD AND NOT Tpetra_INST_OPENMP AND NOT Tpetra_INST_CUDA AND NOT Tpetra_INST_HIP)
-  MESSAGE(FATAL_ERROR "It appears that you have disabled all of Tpetra's Node types.  This is a bad idea; please don't do this.  This may have happened either if all Kokkos execution spaces got disabled, or if you explicitly disabled some Tpetra Node types.  To fix this, look in your CMake configuration to see if you set any of the following variables to OFF explicitly:  Kokkos_ENABLE_CUDA, Kokkos_ENABLE_OPENMP, Kokkos_ENABLE_SERIAL, Kokkos_ENABLE_PTHREAD, Tpetra_INST_SERIAL, Tpetra_INST_PTHREAD, Tpetra_INST_OPENMP, or Tpetra_INST_CUDA.")
+IF(NOT Tpetra_INST_SERIAL AND NOT Tpetra_INST_PTHREAD AND NOT Tpetra_INST_OPENMP AND NOT Tpetra_INST_CUDA AND NOT Tpetra_INST_HIP AND NOT Tpetra_INST_SYCL)
+  MESSAGE(FATAL_ERROR "It appears that you have disabled all of Tpetra's Node types.  This is a bad idea; please don't do this.  This may have happened either if all Kokkos execution spaces got disabled, or if you explicitly disabled some Tpetra Node types.  To fix this, look in your CMake configuration to see if you set any of the following variables to OFF explicitly:  Kokkos_ENABLE_SYCL, Kokkos_ENABLE_HIP, Kokkos_ENABLE_CUDA, Kokkos_ENABLE_OPENMP, Kokkos_ENABLE_SERIAL, Kokkos_ENABLE_PTHREAD, Tpetra_INST_SERIAL, Tpetra_INST_PTHREAD, Tpetra_INST_OPENMP, Tpetra_INST_HIP, Tpetra_INST_SYCL or Tpetra_INST_CUDA.")
 ENDIF()
 
 # ============================================================

packages/tpetra/classic/NodeAPI/CMakeLists.txt

@@ -71,7 +71,15 @@ ENDIF()
 # performance of users' own threads -- OpenMP doesn't have
 # that problem).
 
-IF (Tpetra_DefaultNode STREQUAL "Kokkos::Compat::KokkosHIPWrapperNode")
+IF (Tpetra_DefaultNode STREQUAL "Kokkos::Compat::KokkosSYCLWrapperNode")
+  IF (NOT Kokkos_ENABLE_SYCL)
+    MESSAGE (FATAL_ERROR "Kokkos_ENABLE_SYCL is OFF, but you set Tpetra's default Node type to Kokkos::Compat::KokkosSYCLWrapperNode.")
+  ENDIF ()
+  IF (NOT Tpetra_INST_SYCL)
+    MESSAGE (FATAL_ERROR "Tpetra_INST_SYCL is OFF, but you set Tpetra's default Node type to Kokkos::Compat::KokkosSYCLWrapperNode.")
+  ENDIF ()
+  SET (HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE ON CACHE BOOL "")
+ELSEIF (Tpetra_DefaultNode STREQUAL "Kokkos::Compat::KokkosHIPWrapperNode")
   IF (NOT Kokkos_ENABLE_HIP)
     MESSAGE (FATAL_ERROR "Kokkos_ENABLE_HIP is OFF, but you set Tpetra's default Node type to Kokkos::Compat::KokkosHIPWrapperNode.")
   ENDIF ()
@@ -115,7 +123,14 @@ ELSEIF (Tpetra_DefaultNode STREQUAL "")
   #
   # The user didn't set Tpetra_DefaultNode, so we need to pick it.
   #
-  IF (Tpetra_INST_HIP)
+  IF (Tpetra_INST_SYCL)
+    IF(NOT Kokkos_ENABLE_SYCL)
+      MESSAGE(FATAL_ERROR "Tpetra_INST_SYCL is ON, but Kokkos_ENABLE_SYCL is OFF.")
+    ENDIF()
+    MESSAGE(STATUS "Setting default Node to Kokkos::Compat::KokkosSYCLWrapperNode.")
+    SET(Tpetra_DefaultNode "Kokkos::Compat::KokkosSYCLWrapperNode")
+    SET(HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE ON CACHE BOOL "")
+  ELSEIF (Tpetra_INST_HIP)
     IF(NOT Kokkos_ENABLE_HIP)
       MESSAGE(FATAL_ERROR "Tpetra_INST_HIP is ON, but Kokkos_ENABLE_HIP is OFF.")
     ENDIF()

packages/tpetra/classic/NodeAPI/KokkosCompat_ClassicNodeAPI_Wrapper.cpp

@@ -40,6 +40,14 @@ namespace Kokkos {
     }
 #endif // KOKKOS_ENABLE_HIP
 
+#ifdef KOKKOS_ENABLE_SYCL
+    template<>
+    std::string KokkosDeviceWrapperNode<Kokkos::Experimental::SYCL, Kokkos::Experimental::SYCLSharedUSMSpace>::name() {
+      return std::string("SYCL/Wrapper");
+    }
+#endif // KOKKOS_ENABLE_SYCL
+
+
   } // namespace Compat
 } // namespace Kokkos
 

packages/tpetra/classic/NodeAPI/KokkosCompat_ClassicNodeAPI_Wrapper.hpp

@@ -49,6 +49,10 @@ class KokkosDeviceWrapperNode {
   static std::string name ();
 };
 
+#ifdef KOKKOS_ENABLE_SYCL
+  typedef KokkosDeviceWrapperNode<Kokkos::Experimental::SYCL, Kokkos::Experimental::SYCLSharedUSMSpace> KokkosSYCLWrapperNode;
+#endif
+
 #ifdef KOKKOS_ENABLE_HIP
   typedef KokkosDeviceWrapperNode<Kokkos::Experimental::HIP, Kokkos::Experimental::HIPHostPinnedSpace> KokkosHIPWrapperNode;
 #endif

packages/tpetra/classic/NodeAPI/Kokkos_DefaultNode.hpp

@@ -68,7 +68,9 @@ namespace KokkosClassic {
   /// \endcode
   class DefaultNode {
   public:
-#if defined(HAVE_TPETRA_DEFAULTNODE_HIPWRAPPERNODE)
+#if defined(HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE)
+    typedef ::Kokkos::Compat::KokkosSYCLWrapperNode DefaultNodeType;
+#elif defined(HAVE_TPETRA_DEFAULTNODE_HIPWRAPPERNODE)
     typedef ::Kokkos::Compat::KokkosHIPWrapperNode DefaultNodeType;
 #elif defined(HAVE_TPETRA_DEFAULTNODE_CUDAWRAPPERNODE)
     typedef ::Kokkos::Compat::KokkosCudaWrapperNode DefaultNodeType;

packages/tpetra/classic/NodeAPI/cmake/KokkosClassic_DefaultNode_config.h.in

@@ -6,6 +6,13 @@
 // For backwards compatibility ONLY.
 #define KOKKOSCLASSIC_DEFAULTNODE TPETRA_DEFAULTNODE
 
+// Defined if and only if Tpetra's default execution space is Kokkos::SYCL.
+#cmakedefine HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE
+// For backwards compatibility ONLY.
+#ifdef HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE
+#  define HAVE_KOKKOSCLASSIC_DEFAULTNODE_SYCLWRAPPERNODE 1
+#endif // HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE
+
 // Defined if and only if Tpetra's default execution space is Kokkos::HIP.
 #cmakedefine HAVE_TPETRA_DEFAULTNODE_HIPWRAPPERNODE
 // For backwards compatibility ONLY.

packages/tpetra/core/cmake/TpetraCore_config.h.in

@@ -145,6 +145,7 @@
 #cmakedefine HAVE_TPETRA_INST_OPENMP
 #cmakedefine HAVE_TPETRA_INST_CUDA
 #cmakedefine HAVE_TPETRA_INST_HIP
+#cmakedefine HAVE_TPETRA_INST_SYCL
 
 #cmakedefine HAVE_TPETRA_INT_INT
 #cmakedefine HAVE_TPETRA_INT_LONG
@@ -165,6 +166,7 @@
 #cmakedefine HAVE_TPETRA_OPENMP
 #cmakedefine HAVE_TPETRA_CUDA
 #cmakedefine HAVE_TPETRA_HIP
+#cmakedefine HAVE_TPETRA_SYCL
 
 #cmakedefine HAVE_TPETRA_THREADED_MKL
 

packages/tpetra/core/example/BlockCrs/Tpetra_TestBlockCrs.cpp

@@ -238,7 +238,7 @@ int main (int argc, char *argv[])
     // uses a different device. An easy check is device configuration for
     // 4 mpi nodes as the typical number of GPUs per node is 4.
     if (commTest->getRank () < 4) {
-      exec_space::print_configuration (std::cout, false);
+      exec_space{}.print_configuration (std::cout, false);
     }
 
     if (debug) {
@@ -281,7 +281,8 @@ int main (int argc, char *argv[])
       // - all internal views are allocated on device; mirror as mesh database is constructed on host
       const auto mesh_gids_host = mesh.getElementGlobalIDs();
       const auto mesh_gids =
-        Kokkos::create_mirror_view_and_copy (typename exec_space::memory_space(), mesh_gids_host);
+        Kokkos::create_mirror_view_and_copy (typename map_type::node_type::memory_space{},
+                                             mesh_gids_host);
 
       // for convenience, separate the access to owned and remote gids
       const auto owned_gids =
@@ -477,7 +478,7 @@ int main (int argc, char *argv[])
       {
         TimeMonitor timerMultiVectorFill(*TimeMonitor::getNewTimer("4) MultiVectorFill"));
 
-        auto value = X->getLocalView<typename exec_space::memory_space>(Tpetra::Access::OverwriteAll);
+        auto value = X->getLocalViewDevice(Tpetra::Access::OverwriteAll);
         auto map = X->getMap()->getLocalMap();
         Kokkos::parallel_for
           (value.extent(0), KOKKOS_LAMBDA(const LO i) {

packages/tpetra/core/src/Tpetra_BlockCrsMatrix_decl.hpp

@@ -896,15 +896,16 @@ class BlockCrsMatrix :
     // to run on device then, so we should sync to device.
     static constexpr bool value =
       std::is_same<typename Device::execution_space, Kokkos::Cuda>::value;
-#else
     // Gonna badly fake this here for other execspaces
-    #if defined(KOKKOS_ENABLE_HIP)
+#elif defined(KOKKOS_ENABLE_HIP)
     static constexpr bool value =
       std::is_same<typename Device::execution_space, Kokkos::Experimental::HIP>::value;
-    #else
+#elif defined(KOKKOS_ENABLE_SYCL)
+    static constexpr bool value =
+      std::is_same<typename Device::execution_space, Kokkos::Experimental::SYCL>::value;
+#else
     static constexpr bool value = false;
-    #endif
-#endif // defined(KOKKOS_ENABLE_CUDA)
+#endif
   };
 
 public:

packages/tpetra/core/src/Tpetra_CrsMatrix_decl.hpp

@@ -437,6 +437,8 @@ namespace Tpetra {
     using device_type = typename Node::device_type;
     //! The Kokkos execution space.
     using execution_space = typename device_type::execution_space;
+    //! The Kokkos memory space.
+    using memory_space = typename device_type::memory_space;
 
     /// \brief This class' Kokkos Node type.
     ///

packages/tpetra/core/src/Tpetra_CrsMatrix_def.hpp

@@ -5991,7 +5991,7 @@ CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
     if (refill_num_row_entries) { // unpacked storage
       // We can't assume correct *this capture until C++17, and it's
       // likely more efficient just to capture what we need anyway.
-      num_row_entries_d = create_mirror_view_and_copy(execution_space(),
+      num_row_entries_d = create_mirror_view_and_copy(memory_space(),
                                                  myGraph_->k_numRowEntries_);
       Kokkos::parallel_for
         ("Fill end row pointers", range_policy(0, N),

packages/tpetra/core/src/Tpetra_Details_DefaultTypes.hpp

@@ -87,7 +87,10 @@ namespace DefaultTypes {
 
   /// \typedef execution_space
   /// \brief Default Tpetra execution space and Node type.
-#if defined(HAVE_TPETRA_DEFAULTNODE_HIPWRAPPERNODE)
+#if defined(HAVE_TPETRA_DEFAULTNODE_SYCLWRAPPERNODE)
+  using execution_space = ::Kokkos::Experimental::SYCL;
+  using node_type = ::Kokkos::Compat::KokkosSYCLWrapperNode;
+#elif defined(HAVE_TPETRA_DEFAULTNODE_HIPWRAPPERNODE)
   using execution_space = ::Kokkos::Experimental::HIP;
   using node_type = ::Kokkos::Compat::KokkosHIPWrapperNode;
 #elif defined(HAVE_TPETRA_DEFAULTNODE_CUDAWRAPPERNODE)
@@ -132,6 +135,14 @@ namespace DefaultTypes {
   };
 #endif
 
+#ifdef KOKKOS_ENABLE_SYCL
+  template<>
+  struct CommBufferMemorySpace<Kokkos::Experimental::SYCL>
+  {
+    using type = Kokkos::Experimental::SYCLDeviceUSMSpace;
+  };
+#endif
+
   template<typename Device>
   using comm_buffer_memory_space = typename CommBufferMemorySpace<typename Device::execution_space>::type;
 

packages/tpetra/core/src/Tpetra_Details_FixedHashTable_SYCL.cpp

@@ -0,0 +1,86 @@
+/*
+// @HEADER
+// ***********************************************************************
+//
+//          Tpetra: Templated Linear Algebra Services Package
+//                 Copyright (2008) Sandia Corporation
+//
+// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Michael A. Heroux ([email protected])
+//
+// ************************************************************************
+// @HEADER
+*/
+
+#include "Tpetra_Details_FixedHashTable_decl.hpp"
+
+#if defined(HAVE_TPETRA_EXPLICIT_INSTANTIATION) && defined(KOKKOS_ENABLE_SYCL)
+
+#include "TpetraCore_ETIHelperMacros.h"
+#include "Tpetra_Details_FixedHashTable_def.hpp"
+
+namespace Tpetra {
+namespace Details {
+
+  TPETRA_ETI_MANGLING_TYPEDEFS()
+
+  using sycl_device_type = typename Kokkos::Compat::KokkosSYCLWrapperNode::device_type;
+
+#define TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT_SYCL( LO, GO ) \
+  TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT( LO, GO, sycl_device_type )
+
+  TPETRA_INSTANTIATE_LG( TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT_SYCL)
+
+  // mfh 26 Sep 2015: Make sure that the {KeyType = LO, ValueType =
+  // LO} and {KeyType = int, ValueType = LO} specializations get
+  // built, since Directory needs them.
+
+  // KeyType = int doesn't get built if GO = int is disabled.
+#ifndef HAVE_TPETRA_INST_INT_INT
+#  define TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT_SYCL_INT( LO ) \
+  TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT( LO, int, sycl_device_type ) \
+
+  TPETRA_INSTANTIATE_L( TPETRA_DETAILS_FIXEDHASHTABLE_INSTANT_SYCL_INT)
+
+  // FIXME (mfh 26 Sep 2015) Once it becomes possible to disable LO =
+  // int, add an instantiation here for {KeyType = LO, ValueType =
+  // LO}.  However, this case has likely already been covered above,
+  // since disabling LO = int leaves LO = GO = long or long long as
+  // the most reasonable options.  (It would be silly to use LO = long
+  // and GO = long long if sizeof(long) = sizeof(long long).)
+
+#endif // HAVE_TPETRA_INST_INT_INT
+
+} // namespace Details
+} // namespace Tpetra
+
+#endif // defined(HAVE_TPETRA_EXPLICIT_INSTANTIATION) && defined(KOKKOS_ENABLE_SYCL)