Skip to content

Commit

Permalink
Tpetra::MultiVector: (un)pack checks for nonconst stride; dbg only
Browse files Browse the repository at this point in the history 
@trilinos/tpetra Previous commits added checks for indices in
Tpetra::MultiVector's pack and unpack (DistObject) kernels.  This
commit finishes those checks, for the case of MultiVectors with
nonconstant stride (i.e., MultiVectors that view a noncontiguous set
of columns of another MultiVector).

This commit also makes those checks happen only in a debug build
(based on the macro HAVE_TPETRA_DEBUG).
  • Loading branch information
Mark Hoemmen committed Sep 9, 2016
1 parent 7be0c06 commit 081712c
Show file tree
Hide file tree
Showing 2 changed files with 470 additions and 65 deletions.
118 changes: 78 additions & 40 deletions packages/tpetra/core/src/Tpetra_MultiVector_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1018,8 +1018,18 @@ namespace Tpetra {
typedef typename Kokkos::DualView<IST*, device_type>::t_dev::execution_space
dev_execution_space;

const bool debug = false;
if (debug) {
// TODO (mfh 09 Sep 2016): The pack and unpack functions now have
// the option to check indices. We do so in a debug build. At
// some point, it would make sense to shift this to a run-time
// option, controlled by environment variable.
#ifdef HAVE_TPETRA_DEBUG
constexpr bool debugCheckIndices = true;
#else
constexpr bool debugCheckIndices = false;
#endif // HAVE_TPETRA_DEBUG

const bool printDebugOutput = false;
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew" << std::endl;
}
// We've already called checkSizes(), so this cast must succeed.
Expand Down Expand Up @@ -1073,7 +1083,7 @@ namespace Tpetra {
// If we have no exports, there is nothing to do. Make sure this
// goes _after_ setting constantNumPackets correctly.
if (exportLIDs.dimension_0 () == 0) {
if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew DONE" << std::endl;
}
return;
Expand All @@ -1094,14 +1104,14 @@ namespace Tpetra {
// needs to know how to index into that data. Kokkos is good at
// decoupling storage intent from data layout choice.

if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew realloc" << std::endl;
}

const size_t numExportLIDs = exportLIDs.dimension_0 ();
const size_t newExportsSize = numCols * numExportLIDs;
if (static_cast<size_t> (exports.dimension_0 ()) != newExportsSize) {
if (debug) {
if (printDebugOutput) {
std::ostringstream os;
const int myRank = this->getMap ()->getComm ()->getRank ();
os << "$$$ MV::packAndPrepareNew (Proc " << myRank << ") realloc "
Expand Down Expand Up @@ -1140,83 +1150,91 @@ namespace Tpetra {
// them constant stride (by making whichVectors_ have length 0).
if (sourceMV.isConstantStride ()) {
using KokkosRefactor::Details::pack_array_single_column;
if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew pack numCols=1 const stride" << std::endl;
}
if (packOnHost) {
pack_array_single_column (exports.template view<host_memory_space> (),
create_const_view (src_host),
exportLIDs.template view<host_memory_space> (),
0);
0,
debugCheckIndices);
}
else { // pack on device
pack_array_single_column (exports.template view<dev_memory_space> (),
create_const_view (src_dev),
exportLIDs.template view<dev_memory_space> (),
0);
0,
debugCheckIndices);
}
}
else {
using KokkosRefactor::Details::pack_array_single_column;
if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew pack numCols=1 nonconst stride" << std::endl;
}
if (packOnHost) {
pack_array_single_column (exports.template view<host_memory_space> (),
create_const_view (src_host),
exportLIDs.template view<host_memory_space> (),
sourceMV.whichVectors_[0]);
sourceMV.whichVectors_[0],
debugCheckIndices);
}
else { // pack on device
pack_array_single_column (exports.template view<dev_memory_space> (),
create_const_view (src_dev),
exportLIDs.template view<dev_memory_space> (),
sourceMV.whichVectors_[0]);
sourceMV.whichVectors_[0],
debugCheckIndices);
}
}
}
else { // the source MultiVector has multiple columns
if (sourceMV.isConstantStride ()) {
using KokkosRefactor::Details::pack_array_multi_column;
if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew pack numCols>1 const stride" << std::endl;
}
if (packOnHost) {
pack_array_multi_column (exports.template view<host_memory_space> (),
create_const_view (src_host),
exportLIDs.template view<host_memory_space> (),
numCols);
numCols,
debugCheckIndices);
}
else { // pack on device
pack_array_multi_column (exports.template view<dev_memory_space> (),
create_const_view (src_dev),
exportLIDs.template view<dev_memory_space> (),
numCols);
numCols,
debugCheckIndices);
}
}
else {
using KokkosRefactor::Details::pack_array_multi_column_variable_stride;
if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew pack numCols>1 nonconst stride" << std::endl;
}
if (packOnHost) {
pack_array_multi_column_variable_stride (exports.template view<host_memory_space> (),
create_const_view (src_host),
exportLIDs.template view<host_memory_space> (),
getKokkosViewDeepCopy<host_execution_space> (sourceMV.whichVectors_ ()),
numCols);
numCols,
debugCheckIndices);
}
else { // pack on device
pack_array_multi_column_variable_stride (exports.template view<dev_memory_space> (),
create_const_view (src_dev),
exportLIDs.template view<dev_memory_space> (),
getKokkosViewDeepCopy<dev_execution_space> (sourceMV.whichVectors_ ()),
numCols);
numCols,
debugCheckIndices);
}
}
}

if (debug) {
if (printDebugOutput) {
std::cerr << "$$$ MV::packAndPrepareNew DONE" << std::endl;
}
}
Expand All @@ -1243,6 +1261,16 @@ namespace Tpetra {
const char tfecfFuncName[] = "unpackAndCombineNew: ";
const char suffix[] = " Please report this bug to the Tpetra developers.";

// TODO (mfh 09 Sep 2016): The pack and unpack functions now have
// the option to check indices. We do so in a debug build. At
// some point, it would make sense to shift this to a run-time
// option, controlled by environment variable.
#ifdef HAVE_TPETRA_DEBUG
constexpr bool debugCheckIndices = true;
#else
constexpr bool debugCheckIndices = false;
#endif // HAVE_TPETRA_DEBUG

// If we have no imports, there is nothing to do
if (importLIDs.dimension_0 () == 0) {
return;
Expand All @@ -1259,13 +1287,14 @@ namespace Tpetra {
<< " * " << importLIDs.dimension_0 () << " = "
<< numVecs * importLIDs.dimension_0 () << ".");

TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
constantNumPackets == static_cast<size_t> (0), std::runtime_error,
": constantNumPackets input argument must be nonzero.");
TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
(constantNumPackets == static_cast<size_t> (0), std::runtime_error,
"constantNumPackets input argument must be nonzero.");

TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
static_cast<size_t> (numVecs) != static_cast<size_t> (constantNumPackets),
std::runtime_error, ": constantNumPackets must equal numVecs.");
TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
(static_cast<size_t> (numVecs) !=
static_cast<size_t> (constantNumPackets),
std::runtime_error, "constantNumPackets must equal numVecs.");
#endif // HAVE_TPETRA_DEBUG

// mfh 12 Apr 2016: Decide where to unpack based on the memory
Expand Down Expand Up @@ -1340,25 +1369,28 @@ namespace Tpetra {
if (isConstantStride ()) {
if (unpackOnHost) {
unpack_array_multi_column (X_h, imports_h, importLIDs_h, op,
numVecs);
numVecs, debugCheckIndices);

}
else { // unpack on device
unpack_array_multi_column (X_d, imports_d, importLIDs_d, op,
numVecs);
numVecs, debugCheckIndices);
}
}
else { // not constant stride
if (unpackOnHost) {
unpack_array_multi_column_variable_stride (X_h, imports_h,
importLIDs_h,
whichVecs_h, op,
numVecs);
numVecs,
debugCheckIndices);
}
else { // unpack on device
unpack_array_multi_column_variable_stride (X_d, imports_d,
importLIDs_d,
whichVecs_d, op,
numVecs);
numVecs,
debugCheckIndices);
}
}
}
Expand All @@ -1367,24 +1399,28 @@ namespace Tpetra {

if (isConstantStride ()) {
if (unpackOnHost) {
unpack_array_multi_column (X_h, imports_h, importLIDs_h, op, numVecs);
unpack_array_multi_column (X_h, imports_h, importLIDs_h, op,
numVecs, debugCheckIndices);
}
else { // unpack on device
unpack_array_multi_column (X_d, imports_d, importLIDs_d, op, numVecs);
unpack_array_multi_column (X_d, imports_d, importLIDs_d, op,
numVecs, debugCheckIndices);
}
}
else { // not constant stride
if (unpackOnHost) {
unpack_array_multi_column_variable_stride (X_h, imports_h,
importLIDs_h,
whichVecs_h, op,
numVecs);
numVecs,
debugCheckIndices);
}
else { // unpack on device
unpack_array_multi_column_variable_stride (X_d, imports_d,
importLIDs_d,
whichVecs_d, op,
numVecs);
numVecs,
debugCheckIndices);
}
}
}
Expand All @@ -1394,33 +1430,35 @@ namespace Tpetra {
if (isConstantStride ()) {
if (unpackOnHost) {
unpack_array_multi_column (X_h, imports_h, importLIDs_h, op,
numVecs);
numVecs, debugCheckIndices);
}
else { // unpack on device
unpack_array_multi_column (X_d, imports_d, importLIDs_d, op,
numVecs);
numVecs, debugCheckIndices);
}
}
else {
if (unpackOnHost) {
unpack_array_multi_column_variable_stride (X_h, imports_h,
importLIDs_h,
whichVecs_h, op,
numVecs);
numVecs,
debugCheckIndices);
}
else { // unpack on device
unpack_array_multi_column_variable_stride (X_d, imports_d,
importLIDs_d,
whichVecs_d, op,
numVecs);
numVecs,
debugCheckIndices);
}
}
}
else {
TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(
CM != ADD && CM != REPLACE && CM != INSERT && CM != ABSMAX,
std::invalid_argument, ": Invalid CombineMode: " << CM << ". Valid "
"CombineMode values are ADD, REPLACE, INSERT, and ABSMAX.");
TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
(CM != ADD && CM != REPLACE && CM != INSERT && CM != ABSMAX,
std::invalid_argument, "Invalid CombineMode: " << CM << ". Valid "
"CombineMode values are ADD, REPLACE, INSERT, and ABSMAX.");
}
}
}
Expand Down
Loading

0 comments on commit 081712c

Please sign in to comment.