Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Amesos2: Add Triangular Solves for SuperLU #7302

Merged
merged 19 commits into from
May 30, 2020
Merged

Amesos2: Add Triangular Solves for SuperLU #7302

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
60181b5
Amesos2: Refactor superlu to use Kokkos views
MicheldeMessieres Mar 4, 2020
b9ae885
Amesos2: Clean up loading for Kokkos adapter
MicheldeMessieres Apr 26, 2020
70e8724
Amesos2: Fix cuSOLVER refactor
MicheldeMessieres May 1, 2020
447edf5
Amesos2: Fix warning
MicheldeMessieres May 1, 2020
2f30e6e
Amesos2: Make kokkos also perturb just 1st row
MicheldeMessieres May 2, 2020
2a70520
Amesos2: Fix warnings
MicheldeMessieres May 2, 2020
51a3e59
Amesos2: Fix warning for cuSolver
MicheldeMessieres May 5, 2020
6fd4410
Amesos2: Fix Kokkos adapter to have same random as Tpetra
MicheldeMessieres May 12, 2020
e84310f
Amesos2: Add copy loops for Kokkos::complex to SuperLU native type
MicheldeMessieres May 13, 2020
ffe4364
Amesos2: Remove temporary debug file
MicheldeMessieres May 13, 2020
116cd8e
Amesos2: Cosmetic clean up indents
MicheldeMessieres May 13, 2020
dae1a2c
Amesos2: Add Triangular Solves for SuperLU
MicheldeMessieres Apr 24, 2020
025e39f
Amesos2: Turn off float tests for SuperLU TRSV
MicheldeMessieres May 17, 2020
9efa5b1
Amesos2: Change parameter name to Enable_KokkosKernels_TriangularSolves
MicheldeMessieres May 19, 2020
a9fd6b9
Amesos2: Remove HAVE_AMESOS2_TRIANGULAR_SOLVES
MicheldeMessieres May 19, 2020
5b03770
Amesos2: Remove duplication of KokkosSparse_sptrsv_superlu.hpp file
MicheldeMessieres May 19, 2020
ca29496
Amesos2: Update Superlu_UnitTests.cpp
MicheldeMessieres May 26, 2020
1100c3f
Amesos2: Fix sign compare warning for SuperLU
MicheldeMessieres May 27, 2020
1a9cefe
Amesos2: Fix Epetra crash for SuperLU
MicheldeMessieres May 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions packages/amesos2/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,26 +113,6 @@ TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_MUMPS
OFF
)

TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_TRIANGULAR_SOLVES
HAVE_AMESOS2_TRIANGULAR_SOLVES
"Enable Triangular Solves in Amesos2"
OFF
)

# If Triangular Solves is on then we must have proper settings for KokkosKernels
ASSERT_DEFINED(${PACKAGE_NAME}_ENABLE_TRIANGULAR_SOLVES)
IF(${PACKAGE_NAME}_ENABLE_TRIANGULAR_SOLVES)
ASSERT_DEFINED(TPL_ENABLE_Cholmod)
ASSERT_DEFINED(${PACKAGE_NAME}_ENABLE_Cholmod)
IF(TPL_ENABLE_Cholmod AND ${PACKAGE_NAME}_ENABLE_Cholmod)
ASSERT_DEFINED(KokkosKernels_ENABLE_SUPERNODAL_SPTRSV)
ASSERT_DEFINED(KokkosKernels_ENABLE_TPL_CHOLMOD)
IF(NOT KokkosKernels_ENABLE_SUPERNODAL_SPTRSV OR NOT KokkosKernels_ENABLE_TPL_CHOLMOD)
MESSAGE(FATAL_ERROR "You must enable -DKokkosKernels_ENABLE_SUPERNODAL_SPTRSV=ON and -DKokkosKernels_ENABLE_TPL_CHOLMOD=ON for Triangular Solves with Cholmod")
ENDIF()
ENDIF()
ENDIF()

ASSERT_DEFINED(TPL_ENABLE_METIS)
TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_METIS
HAVE_AMESOS2_METIS
Expand Down
4 changes: 2 additions & 2 deletions packages/amesos2/src/Amesos2_Cholmod_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
#include "Amesos2_SolverCore.hpp"
#include "Amesos2_Cholmod_FunctionMap.hpp"

#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
#include "KokkosKernels_Handle.hpp"
#endif

Expand Down Expand Up @@ -260,7 +260,7 @@ class Cholmod : public SolverCore<Amesos2::Cholmod, Matrix, Vector>
mutable host_solve_array_t host_bValues_;
int ldb_;

#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)

typedef Kokkos::DefaultExecutionSpace DeviceExecSpaceType;

Expand Down
24 changes: 12 additions & 12 deletions packages/amesos2/src/Amesos2_Cholmod_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@
#include "Amesos2_SolverCore_def.hpp"
#include "Amesos2_Cholmod_decl.hpp"

#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
#include "KokkosSparse_sptrsv_cholmod.hpp"
#endif // HAVE_AMESOS2_TRIANGULAR_SOLVES
#endif

namespace Amesos2 {

Expand Down Expand Up @@ -236,7 +236,7 @@ Cholmod<Matrix,Vector>::solve_impl(const Teuchos::Ptr<MultiVecAdapter<Vector> >
// In general we may want to write directly to the x space without a copy.
// So we 'get' x which may be a direct view assignment to the MV.
if(use_triangular_solves_) { // to device
#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
Util::get_1d_copy_helper_kokkos_view<MultiVecAdapter<Vector>,
device_solve_array_t>::do_get(B, device_bValues_,
Teuchos::as<size_t>(ld_rhs),
Expand Down Expand Up @@ -298,7 +298,7 @@ Cholmod<Matrix,Vector>::solve_impl(const Teuchos::Ptr<MultiVecAdapter<Vector> >
#endif

if(use_triangular_solves_) { // to device
#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
Util::put_1d_data_helper_kokkos_view<
MultiVecAdapter<Vector>,device_solve_array_t>::do_put(X, device_xValues_,
Teuchos::as<size_t>(ld_rhs),
Expand Down Expand Up @@ -338,12 +338,12 @@ Cholmod<Matrix,Vector>::setParameters_impl(const Teuchos::RCP<Teuchos::Parameter
RCP<const Teuchos::ParameterList> valid_params = getValidParameters_impl();

is_contiguous_ = parameterList->get<bool>("IsContiguous", true);
use_triangular_solves_ = parameterList->get<bool>("TriangularSolves", false);
use_triangular_solves_ = parameterList->get<bool>("Enable_KokkosKernels_TriangularSolves", false);

if(use_triangular_solves_) {
#ifndef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if not defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) || not defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error,
"Calling for triangular solves but Amesos2_ENABLE_TRIANGULAR_SOLVES was not configured." );
"Calling for triangular solves but KokkosKernels_ENABLE_SUPERNODAL_SPTRSV and KokkosKernels_ENABLE_TPL_CHOLMOD not configured." );
#endif
}

Expand Down Expand Up @@ -409,7 +409,7 @@ Cholmod<Matrix,Vector>::getValidParameters_impl() const

pl->set("useGPU", -1, "1: Use GPU is 1, 0: Do not use GPU, -1: ENV CHOLMOD_USE_GPU set GPU usage.");

pl->set("TriangularSolves", false, "Whether to use triangular solves.");
pl->set("Enable_KokkosKernels_TriangularSolves", false, "Whether to use triangular solves.");

pl->set("IsContiguous", true, "Whether GIDs contiguous");

Expand Down Expand Up @@ -489,7 +489,7 @@ template <class Matrix, class Vector>
void
Cholmod<Matrix,Vector>::triangular_solve_symbolic()
{
#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
// Create handles for U and U^T solves
device_khL_.create_sptrsv_handle(
KokkosSparse::Experimental::SPTRSVAlgorithm::SUPERNODAL_ETREE, data_.L->n, true);
Expand Down Expand Up @@ -529,7 +529,7 @@ template <class Matrix, class Vector>
void
Cholmod<Matrix,Vector>::triangular_solve_numeric()
{
#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
// Do numerical compute
KokkosSparse::Experimental::sptrsv_compute<long, kernel_handle_type>
(&device_khL_, &device_khU_, data_.L, &data_.c);
Expand All @@ -540,7 +540,7 @@ template <class Matrix, class Vector>
void
Cholmod<Matrix,Vector>::triangular_solve() const
{
#ifdef HAVE_AMESOS2_TRIANGULAR_SOLVES
#if defined(KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV) && defined(KOKKOSKERNELS_ENABLE_TPL_CHOLMOD)
size_t ld_rhs = device_xValues_.extent(0);
size_t nrhs = device_xValues_.extent(1);

Expand Down Expand Up @@ -577,7 +577,7 @@ Cholmod<Matrix,Vector>::triangular_solve() const
local_device_xValues(j,k) = local_device_trsv_rhs(local_device_trsv_perm(j),k);
}
});
#endif // HAVE_AMESOS2_TRIANGULAR_SOLVE
#endif
}

template<class Matrix, class Vector>
Expand Down
3 changes: 3 additions & 0 deletions packages/amesos2/src/Amesos2_Superlu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,7 @@ TPETRA_ETI_MANGLING_TYPEDEFS()

TPETRA_INSTANTIATE_SLGN_NO_ORDINAL_SCALAR(AMESOS2_SUPERLU_LOCAL_INSTANT)

#define AMESOS2_KOKKOS_IMPL_SOLVER_NAME Superlu
#include "Amesos2_Kokkos_Impl.hpp"

#endif // HAVE_AMESOS2_EXPLICIT_INSTANTIATION
Loading