From ce52b64a9c20d0bcc5171ec93606f42a6082d993 Mon Sep 17 00:00:00 2001 From: Mark Hoemmen Date: Mon, 12 Sep 2016 15:09:29 -0600 Subject: [PATCH] Tpetra::{Ex,Im}port: Fix #607 (add isLocallyComplete method) @trilinos/tpetra Add the proposed isLocallyComplete method (see #607) to Tpetra::Export and Tpetra::Import. Also add a test for this method. The test passes. --- .../core/src/Tpetra_Details_Transfer_decl.hpp | 24 +- .../tpetra/core/src/Tpetra_Export_decl.hpp | 10 + .../tpetra/core/src/Tpetra_Export_def.hpp | 40 +- .../core/src/Tpetra_ImportExportData_decl.hpp | 18 + .../core/src/Tpetra_ImportExportData_def.hpp | 21 +- .../tpetra/core/src/Tpetra_Import_decl.hpp | 30 +- .../tpetra/core/src/Tpetra_Import_def.hpp | 157 ++++--- .../core/test/ImportExport/CMakeLists.txt | 11 + .../core/test/ImportExport/Issue_607.cpp | 395 ++++++++++++++++++ 9 files changed, 626 insertions(+), 80 deletions(-) create mode 100644 packages/tpetra/core/test/ImportExport/Issue_607.cpp diff --git a/packages/tpetra/core/src/Tpetra_Details_Transfer_decl.hpp b/packages/tpetra/core/src/Tpetra_Details_Transfer_decl.hpp index b4b24cf00f3c..55d225410ab7 100644 --- a/packages/tpetra/core/src/Tpetra_Details_Transfer_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_Transfer_decl.hpp @@ -114,15 +114,33 @@ class Transfer : public Teuchos::Describable { /// process getExportPiDs()[i]. virtual Teuchos::ArrayView getExportPIDs () const = 0; - //! The source Map used to construct this Export. + //! The source Map used to construct this Export or Import. virtual Teuchos::RCP getSourceMap () const = 0; - //! The target Map used to construct this Export. + //! The target Map used to construct this Export or Import. virtual Teuchos::RCP getTargetMap () const = 0; - //! The Distributor that this Export object uses to move data. + //! The Distributor that this Export or Import object uses to move data. virtual ::Tpetra::Distributor& getDistributor () const = 0; + /// \brief Is this Export or Import locally complete? + /// + /// If this is an Export, then do all source Map indices on the + /// calling process exist on at least one process (not necessarily + /// this one) in the target Map? + /// + /// If this is an Import, then do all target Map indices on the + /// calling process exist on at least one process (not necessarily + /// this one) in the source Map? + /// + /// It's not necessarily an error for an Export or Import not to be + /// locally complete on one or more processes. For example, this + /// may happen in the common use case of "restriction" -- that is, + /// taking a subset of a large object. Nevertheless, you may find + /// this predicate useful for figuring out whether you set up your + /// Maps in the way that you expect. + virtual bool isLocallyComplete () const = 0; + /// \brief Describe this object in a human-readable way to the given /// output stream. /// diff --git a/packages/tpetra/core/src/Tpetra_Export_decl.hpp b/packages/tpetra/core/src/Tpetra_Export_decl.hpp index 3b861e533869..b99bdf81d9da 100644 --- a/packages/tpetra/core/src/Tpetra_Export_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Export_decl.hpp @@ -267,6 +267,16 @@ namespace Tpetra { //! The Distributor that this Export object uses to move data. Distributor & getDistributor() const; + /// \brief Do all source Map indices on the calling process exist + /// on at least one process (not necessarily this one) in the + /// target Map? + /// + /// It's not necessarily an error for an Export not to be locally + /// complete on one or more processes. Nevertheless, you may find + /// this predicate useful for figuring out whether you set up your + /// Maps in the way that you expect. + bool isLocallyComplete () const; + //! Assignment operator Export& operator= (const Export& rhs); diff --git a/packages/tpetra/core/src/Tpetra_Export_def.hpp b/packages/tpetra/core/src/Tpetra_Export_def.hpp index 6ec807a4519d..eca2b2485f21 100644 --- a/packages/tpetra/core/src/Tpetra_Export_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Export_def.hpp @@ -374,6 +374,12 @@ namespace Tpetra { return ExportData_->distributor_; } + template + bool + Export::isLocallyComplete () const { + return ExportData_->isLocallyComplete_; + } + template Export& Export:: @@ -417,6 +423,8 @@ namespace Tpetra { typedef LocalOrdinal LO; typedef GlobalOrdinal GO; typedef typename ArrayView::size_type size_type; + const char tfecfFuncName[] = "setupExport: "; + const map_type& source = * (getSourceMap ()); const map_type& target = * (getTargetMap ()); ArrayView sourceGIDs = source.getNodeElementList (); @@ -483,12 +491,20 @@ namespace Tpetra { // overlapping, so multiple processes might send to the same LID // on a receiving process. - TPETRA_ABUSE_WARNING( - getNumExportIDs() > 0 && ! source.isDistributed(), - std::runtime_error, - "::setupSamePermuteExport(): Source has export LIDs but Source is not " - "distributed globally." << std::endl - << "Exporting to a submap of the target map."); + if (exportLIDs.size () != 0 && ! source.isDistributed ()) { + // This Export has export LIDs, meaning that the source Map has + // entries on this process that are not in the target Map on + // this process. However, the source Map is not distributed + // globally. This implies that this Import is not locally + // complete on this process. + ExportData_->isLocallyComplete_ = false; + // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be + // correct behavior, depending on the circumstances. + TPETRA_ABUSE_WARNING + (true, std::runtime_error, "::setupSamePermuteExport(): Source has " + "export LIDs but Source is not distributed globally. Exporting to " + "a submap of the target map."); + } // Compute exportPIDs_ ("outgoing" process IDs). // @@ -509,6 +525,8 @@ namespace Tpetra { const LookupStatus lookup = target.getRemoteIndexList (exportGIDs(), ExportData_->exportPIDs_ ()); + // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be + // correct behavior, depending on the circumstances. TPETRA_ABUSE_WARNING( lookup == IDNotPresent, std::runtime_error, "::setupSamePermuteExport(): The source Map has GIDs not found " "in the target Map."); @@ -517,10 +535,20 @@ namespace Tpetra { // exporting to GIDs which don't belong to any process in the // target Map. if (lookup == IDNotPresent) { + // There is at least one GID owned by the calling process in + // the source Map, which is not owned by any process in the + // target Map. + ExportData_->isLocallyComplete_ = false; + const size_type numInvalidExports = std::count_if (ExportData_->exportPIDs_().begin(), ExportData_->exportPIDs_().end(), std::bind1st (std::equal_to(), -1)); + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC + (numInvalidExports == 0, std::logic_error, "Calling getRemoteIndexList " + "on the target Map returned IDNotPresent, but none of the returned " + "\"export\" process ranks are -1. Please report this bug to the " + "Tpetra developers."); // count number of valid and total number of exports const size_type totalNumExports = ExportData_->exportPIDs_.size(); diff --git a/packages/tpetra/core/src/Tpetra_ImportExportData_decl.hpp b/packages/tpetra/core/src/Tpetra_ImportExportData_decl.hpp index cf76933ffe0e..e4df19ef1078 100644 --- a/packages/tpetra/core/src/Tpetra_ImportExportData_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_ImportExportData_decl.hpp @@ -205,6 +205,24 @@ namespace Tpetra { /// communicator; it does not complete initialization. Distributor distributor_; + /// \brief Is this Export or Import locally complete? + /// + /// If this is an Export, then do all source Map indices on the + /// calling process exist on at least one process (not necessarily + /// this one) in the target Map? + /// + /// If this is an Import, then do all target Map indices on the + /// calling process exist on at least one process (not necessarily + /// this one) in the source Map? + /// + /// It's not necessarily an error for an Export or Import not to + /// be locally complete on one or more processes. For example, + /// this may happen in the common use case of "restriction" -- + /// that is, taking a subset of a large object. Nevertheless, you + /// may find this predicate useful for figuring out whether you + /// set up your Maps in the way that you expect. + bool isLocallyComplete_; + private: //! Copy constructor (declared but not defined, do not use) ImportExportData (const ImportExportData &rhs); diff --git a/packages/tpetra/core/src/Tpetra_ImportExportData_def.hpp b/packages/tpetra/core/src/Tpetra_ImportExportData_def.hpp index 70e53cf9fdab..116da05356fb 100644 --- a/packages/tpetra/core/src/Tpetra_ImportExportData_def.hpp +++ b/packages/tpetra/core/src/Tpetra_ImportExportData_def.hpp @@ -55,7 +55,8 @@ namespace Tpetra { target_ (target), out_ (Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr))), numSameIDs_ (0), - distributor_ (source->getComm (), out_) + distributor_ (source->getComm (), out_), + isLocallyComplete_ (true) // Import/Export constructor may change this {} template @@ -67,7 +68,8 @@ namespace Tpetra { target_ (target), out_ (out), numSameIDs_ (0), - distributor_ (source->getComm (), out_) + distributor_ (source->getComm (), out_), + isLocallyComplete_ (true) // Import/Export constructor may change this {} template @@ -79,7 +81,8 @@ namespace Tpetra { target_ (target), out_ (Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr))), numSameIDs_ (0), - distributor_ (source->getComm (), out_, plist) + distributor_ (source->getComm (), out_, plist), + isLocallyComplete_ (true) // Import/Export constructor may change this {} template @@ -92,7 +95,8 @@ namespace Tpetra { target_ (target), out_ (out), numSameIDs_ (0), - distributor_ (source->getComm (), out_, plist) + distributor_ (source->getComm (), out_, plist), + isLocallyComplete_ (true) // Import/Export constructor may change this {} template @@ -120,13 +124,22 @@ namespace Tpetra { ArrayView ProcsFrom = distributor_.getProcsFrom(); ArrayView LengthsFrom = distributor_.getLengthsFrom(); + // isLocallyComplete is a local predicate. + // It could be true in one direction but false in another. + + bool isLocallyComplete = true; // by default for (size_t i = 0, j = 0; i < NumReceives; ++i) { const int pid = ProcsFrom[i]; + if (pid == -1) { + isLocallyComplete = false; + } for (size_t k = 0; k < LengthsFrom[i]; ++k) { tData->exportPIDs_[j] = pid; ++j; } } + tData->isLocallyComplete_ = isLocallyComplete; + return tData; } diff --git a/packages/tpetra/core/src/Tpetra_Import_decl.hpp b/packages/tpetra/core/src/Tpetra_Import_decl.hpp index fb30e97e1615..be1a469f317a 100644 --- a/packages/tpetra/core/src/Tpetra_Import_decl.hpp +++ b/packages/tpetra/core/src/Tpetra_Import_decl.hpp @@ -208,7 +208,7 @@ namespace Tpetra { Import (const Teuchos::RCP& source, const Teuchos::RCP& target, Teuchos::Array & remotePIDs); - + /// \brief Copy constructor. /// /// \note Currently this only makes a shallow copy of the Import's @@ -228,14 +228,14 @@ namespace Tpetra { /// Import (const Teuchos::RCP >& source, - const Teuchos::RCP >& target, - Teuchos::Array & userRemotePIDs, - Teuchos::Array& remoteGIDs, - const Teuchos::ArrayView & userExportLIDs, - const Teuchos::ArrayView & userExportPIDs, - const bool useRemotePIDs, - const Teuchos::RCP& plist = Teuchos::null, - const Teuchos::RCP& out = Teuchos::null); + const Teuchos::RCP >& target, + Teuchos::Array & userRemotePIDs, + Teuchos::Array& remoteGIDs, + const Teuchos::ArrayView & userExportLIDs, + const Teuchos::ArrayView & userExportPIDs, + const bool useRemotePIDs, + const Teuchos::RCP& plist = Teuchos::null, + const Teuchos::RCP& out = Teuchos::null); //! Destructor. @@ -298,6 +298,18 @@ namespace Tpetra { //! The Distributor that this Import object uses to move data. Distributor & getDistributor() const; + /// \brief Do all target Map indices on the calling process exist + /// on at least one process (not necessarily this one) in the + /// source Map? + /// + /// It's not necessarily an error for an Import not to be locally + /// complete on one or more processes. For example, this may + /// happen in the common use case of "restriction" -- that is, + /// taking a subset of a large object. Nevertheless, you may find + /// this predicate useful for figuring out whether you set up your + /// Maps in the way that you expect. + bool isLocallyComplete () const; + //! Assignment operator. Import& operator= (const Import& Source); diff --git a/packages/tpetra/core/src/Tpetra_Import_def.hpp b/packages/tpetra/core/src/Tpetra_Import_def.hpp index dff2cad9390b..3837d4976b78 100644 --- a/packages/tpetra/core/src/Tpetra_Import_def.hpp +++ b/packages/tpetra/core/src/Tpetra_Import_def.hpp @@ -215,16 +215,16 @@ namespace Tpetra { template Import:: Import(const Teuchos::RCP >& source, - const Teuchos::RCP >& target, - Teuchos::Array& userRemotePIDs, - Teuchos::Array& remoteGIDs, - const Teuchos::ArrayView & userExportLIDs, - const Teuchos::ArrayView & userExportPIDs, - const bool useRemotePIDGID, - const Teuchos::RCP& plist, - const Teuchos::RCP& out) : - out_ (out.is_null () ? - Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr)) : out) + const Teuchos::RCP >& target, + Teuchos::Array& userRemotePIDs, + Teuchos::Array& remoteGIDs, + const Teuchos::ArrayView & userExportLIDs, + const Teuchos::ArrayView & userExportPIDs, + const bool useRemotePIDGID, + const Teuchos::RCP& plist, + const Teuchos::RCP& out) : + out_ (out.is_null () ? + Teuchos::getFancyOStream (Teuchos::rcpFromRef (std::cerr)) : out) { using Teuchos::arcp; using Teuchos::Array; @@ -277,29 +277,29 @@ namespace Tpetra { remoteGIDs.clear(); remoteLIDs.clear(); } - + for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) { const GO curTargetGid = targetGIDs[tgtLid]; // getLocalElement() returns LINVALID if the GID isn't in the source Map. const LO srcLid = source->getLocalElement (curTargetGid); if (srcLid != LINVALID) { - permuteToLIDs.push_back (tgtLid); - permuteFromLIDs.push_back (srcLid); + permuteToLIDs.push_back (tgtLid); + permuteFromLIDs.push_back (srcLid); } else { - if(!useRemotePIDGID) { - remoteGIDs.push_back (curTargetGid); - remoteLIDs.push_back (tgtLid); - } + if(!useRemotePIDGID) { + remoteGIDs.push_back (curTargetGid); + remoteLIDs.push_back (tgtLid); + } } } - + TPETRA_ABUSE_WARNING( getNumRemoteIDs() > 0 && ! source->isDistributed(), std::runtime_error, "::constructExpert(): Target has remote LIDs but Source is not " "distributed globally." << std::endl << "Importing to a submap of the target map."); - + Array remotePIDs; remotePIDs.resize (remoteGIDs.size (),0); LookupStatus lookup = AllIDsPresent; @@ -316,14 +316,14 @@ namespace Tpetra { "means that there is at least one GID owned by some process in the target" " Map which is not owned by any process in the source Map. (That is, the" " source and target Maps do not contain the same set of GIDs globally.)"); - + // Sort remoteProcIDs in ascending order, and apply the resulting // permutation to remoteGIDs and remoteLIDs_. This ensures that // remoteProcIDs[i], remoteGIDs[i], and remoteLIDs_[i] all refer // to the same thing. - + TEUCHOS_TEST_FOR_EXCEPTION( !(remoteProcIDs.size() == remoteGIDsView.size() &&remoteGIDsView.size() == remoteLIDs.size()), std::runtime_error, - "Import::Import createExpert version: Size miss match on RemoteProcIDs, remoteGIDsView and remoteLIDs Array's to sort3. This will produce produce an error, aborting "); + "Import::Import createExpert version: Size miss match on RemoteProcIDs, remoteGIDsView and remoteLIDs Array's to sort3. This will produce produce an error, aborting "); sort3 (remoteProcIDs.begin (), remoteProcIDs.end (), @@ -335,13 +335,18 @@ namespace Tpetra { ImportData_->exportPIDs_ = Teuchos::Array(userExportPIDs.size(),0); ImportData_->exportLIDs_ = Teuchos::Array(userExportPIDs.size(),0); + bool isLocallyComplete = true; for(size_type i=0; iexportPIDs_[i] = userExportPIDs[i]; ImportData_->exportLIDs_[i] = userExportLIDs[i]; } - + ImportData_->isLocallyComplete_ = isLocallyComplete; + ImportData_->distributor_.createFromSendsAndRecvs(ImportData_->exportPIDs_,remoteProcIDs); - + } @@ -388,6 +393,14 @@ namespace Tpetra { } ImportData_ = rcp (new data_type (source, target, out_, plist)); + bool isLocallyComplete = true; + for (Teuchos::Array::size_type i = 0; i < exportPIDs.size (); ++i) { + if (exportPIDs[i] == -1) { + isLocallyComplete = false; + } + } + ImportData_->isLocallyComplete_ = isLocallyComplete; + ImportData_->numSameIDs_ = numSameIDs; ImportData_->permuteToLIDs_.swap (permuteToLIDs); ImportData_->permuteFromLIDs_.swap (permuteFromLIDs); @@ -469,6 +482,12 @@ namespace Tpetra { return ImportData_->distributor_; } + template + bool + Import::isLocallyComplete () const { + return ImportData_->isLocallyComplete_; + } + template Import& Import:: @@ -570,12 +589,20 @@ namespace Tpetra { } } - TPETRA_ABUSE_WARNING( - getNumRemoteIDs() > 0 && ! source.isDistributed(), - std::runtime_error, - "::setupSamePermuteRemote(): Target has remote LIDs but Source is not " - "distributed globally." << std::endl - << "Importing to a submap of the target map."); + if (remoteLIDs.size () != 0 && ! source.isDistributed ()) { + // This Import has remote LIDs, meaning that the target Map has + // entries on this process that are not in the source Map on + // this process. However, the source Map is not distributed + // globally. This implies that this Import is not locally + // complete on this process. + ImportData_->isLocallyComplete_ = false; + // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be + // correct behavior, depending on the circumstances. + TPETRA_ABUSE_WARNING + (true, std::runtime_error, "::setupSamePermuteRemote(): Target has " + "remote LIDs but Source is not distributed globally. Importing to a " + "submap of the target map."); + } } @@ -594,11 +621,11 @@ namespace Tpetra { typedef LocalOrdinal LO; typedef GlobalOrdinal GO; typedef typename Array::difference_type size_type; + const char tfecfFuncName[] = "setupExport: "; - TEUCHOS_TEST_FOR_EXCEPTION( - getSourceMap ().is_null (), std::logic_error, "Tpetra::Import::" - "setupExport: Source Map is null. Please report this bug to the Tpetra " - "developers."); + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC + (getSourceMap ().is_null (), std::logic_error, "Source Map is null. " + "Please report this bug to the Tpetra developers."); const map_type& source = * (getSourceMap ()); Teuchos::OSTab tab (out_); @@ -610,14 +637,13 @@ namespace Tpetra { // } // Sanity checks - TEUCHOS_TEST_FOR_EXCEPTION( - ! useRemotePIDs && (userRemotePIDs.size() > 0), std::invalid_argument, - "Tpetra::Import::setupExport: remotePIDs are non-empty but their use has " - "not been requested."); - TEUCHOS_TEST_FOR_EXCEPTION( - userRemotePIDs.size () > 0 && remoteGIDs.size () != userRemotePIDs.size (), - std::invalid_argument, "Tpetra::Import::setupExport: remotePIDs must " - "either be of size zero or match the size of remoteGIDs."); + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC + (! useRemotePIDs && (userRemotePIDs.size() > 0), std::invalid_argument, + "remotePIDs are non-empty but their use has not been requested."); + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC + (userRemotePIDs.size () > 0 && remoteGIDs.size () != userRemotePIDs.size (), + std::invalid_argument, "remotePIDs must either be of size zero or match " + "the size of remoteGIDs."); // For each entry remoteGIDs[i], remoteProcIDs[i] will contain // the process ID of the process that owns that GID. @@ -658,20 +684,35 @@ namespace Tpetra { } Array& remoteProcIDs = useRemotePIDs ? userRemotePIDs : newRemotePIDs; - TPETRA_ABUSE_WARNING( lookup == IDNotPresent, std::runtime_error, - "::setupExport(): the source Map wasn't able to figure out which process " - "owns one or more of the GIDs in the list of remote GIDs. This probably " - "means that there is at least one GID owned by some process in the target" - " Map which is not owned by any process in the source Map. (That is, the" - " source and target Maps do not contain the same set of GIDs globally.)"); - - // Ignore remote GIDs that aren't owned by any process in the - // source Map. getRemoteIndexList() gives each of these a process - // ID of -1. if (lookup == IDNotPresent) { + // There is at least one GID owned by the calling process in the + // target Map, which is not owned by any process in the source + // Map. + ImportData_->isLocallyComplete_ = false; + + // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be + // correct behavior, depending on the circumstances. + TPETRA_ABUSE_WARNING + (true, std::runtime_error, "::setupExport(): the source Map wasn't " + "able to figure out which process owns one or more of the GIDs in the " + "list of remote GIDs. This probably means that there is at least one " + "GID owned by some process in the target Map which is not owned by any" + " process in the source Map. (That is, the source and target Maps do " + "not contain the same set of GIDs globally.)"); + + // Ignore remote GIDs that aren't owned by any process in the + // source Map. getRemoteIndexList() gives each of these a + // process ID of -1. + const size_type numInvalidRemote = std::count_if (remoteProcIDs.begin (), remoteProcIDs.end (), std::bind1st (std::equal_to (), -1)); + TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC + (numInvalidRemote == 0, std::logic_error, "Calling getRemoteIndexList " + "on the source Map returned IDNotPresent, but none of the returned " + "\"remote\" process ranks are -1. Please report this bug to the " + "Tpetra developers."); + // If all of them are invalid, we can delete the whole array. const size_type totalNumRemote = getNumRemoteIDs (); if (numInvalidRemote == totalNumRemote) { @@ -1435,13 +1476,13 @@ namespace Tpetra { // Build the importer using the "expert" constructor unionImport = rcp(new Import(srcMap, - targetMapNew, - numSameIDsNew, - permuteToLIDsNew, - permuteFromLIDsNew, + targetMapNew, + numSameIDsNew, + permuteToLIDsNew, + permuteFromLIDsNew, remoteLIDsNew, - exportLIDsnew, - exportPIDsnew,D)); + exportLIDsnew, + exportPIDsnew,D)); return unionImport; } diff --git a/packages/tpetra/core/test/ImportExport/CMakeLists.txt b/packages/tpetra/core/test/ImportExport/CMakeLists.txt index ec1e54ba3fd9..6c587f155dcb 100644 --- a/packages/tpetra/core/test/ImportExport/CMakeLists.txt +++ b/packages/tpetra/core/test/ImportExport/CMakeLists.txt @@ -134,3 +134,14 @@ IF (Tpetra_INST_INT_INT) NUM_MPI_PROCS 4 ) ENDIF () + +# mfh 13 Sep 2016: Test for Github Issue #607. +TRIBITS_ADD_EXECUTABLE_AND_TEST( + Issue_607 + SOURCES + Issue_607 + ${TEUCHOS_STD_UNIT_TEST_MAIN} + COMM serial mpi + STANDARD_PASS_OUTPUT + NUM_MPI_PROCS 1-4 + ) diff --git a/packages/tpetra/core/test/ImportExport/Issue_607.cpp b/packages/tpetra/core/test/ImportExport/Issue_607.cpp new file mode 100644 index 000000000000..a6d2cf33c1df --- /dev/null +++ b/packages/tpetra/core/test/ImportExport/Issue_607.cpp @@ -0,0 +1,395 @@ +/* +// @HEADER +// *********************************************************************** +// +// Tpetra: Templated Linear Algebra Services Package +// Copyright (2008) Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +// @HEADER +*/ + +#include "Teuchos_UnitTestHarness.hpp" +#include "Tpetra_TestingUtilities.hpp" +#include "Tpetra_DefaultPlatform.hpp" +#include "Tpetra_Map.hpp" +#include "Tpetra_Import.hpp" +#include "Tpetra_Export.hpp" +#include "Teuchos_DefaultSerialComm.hpp" + +namespace { // (anonymous) + +using Teuchos::RCP; +using Teuchos::rcp; +using std::endl; + +bool +falseOnSomeProcess (const Teuchos::Comm& comm, const bool in) +{ + using Teuchos::outArg; + using Teuchos::REDUCE_MIN; + using Teuchos::reduceAll; + + const int inInt = in ? 1 : 0; + int outInt = 0; // output argument + reduceAll (comm, REDUCE_MIN, inInt, outArg (outInt)); + return (outInt == 0); +} + +bool +trueOnAllProcesses (const Teuchos::Comm& comm, const bool in) +{ + using Teuchos::outArg; + using Teuchos::REDUCE_MIN; + using Teuchos::reduceAll; + + const int inInt = in ? 1 : 0; + int outInt = 0; // output argument + reduceAll (comm, REDUCE_MIN, inInt, outArg (outInt)); + return (outInt == 1); +} + +TEUCHOS_UNIT_TEST_TEMPLATE_3_DECL( ImportExport, IsLocallyComplete, LO, GO, NT ) +{ + typedef Tpetra::Map map_type; + typedef Tpetra::Import import_type; + typedef Tpetra::Export export_type; + + out << "Test {Ex,Im}port::isLocallyComplete()" << endl; + Teuchos::OSTab tab1 (out); + + auto comm = Tpetra::DefaultPlatform::getDefaultPlatform ().getComm (); + const int myRank = comm->getRank (); + const int numProcs = comm->getSize (); + const GO indexBase = 0; + + //const Tpetra::global_size_t INVALID = + // Teuchos::OrdinalTraits::invalid (); + + out << "Test cases where both Maps live on one process" << endl; + { + Teuchos::OSTab tab2 (out); + // This is equivalent to MPI_COMM_SELF. + auto serialComm = rcp (new Teuchos::SerialComm ()); + + RCP map1 (new map_type (10, 10, indexBase, serialComm)); + RCP map2 (new map_type (10, 10, indexBase, serialComm)); + RCP map3 (new map_type (11, 11, indexBase, serialComm)); + + Teuchos::Array gids4 (10); + typedef typename Teuchos::Array::size_type size_type; + for (size_type k = 0; k < static_cast (10); ++k) { + gids4[k] = static_cast ((10 - 1) - k); // reverse order + } + RCP map4 (new map_type (10, gids4 (), indexBase, serialComm)); + + out << "Test identical Maps" << endl; + export_type exp11 (map1, map1); + TEST_ASSERT( exp11.isLocallyComplete () ); + import_type imp11 (map1, map1); + TEST_ASSERT( imp11.isLocallyComplete () ); + + out << "Test Maps which are the same, but not identical" << endl; + export_type exp12 (map1, map2); + TEST_ASSERT( exp12.isLocallyComplete () ); + import_type imp12 (map1, map2); + TEST_ASSERT( imp12.isLocallyComplete () ); + + out << "Test Maps [0, ..., 9], [0, ..., 10]" << endl; + export_type exp13 (map1, map3); + // Yes, all _source_ Map indices exist in the _target_ Map. + TEST_ASSERT( exp13.isLocallyComplete () ); + import_type imp13 (map1, map3); + TEST_ASSERT( ! imp13.isLocallyComplete () ); + + out << "Test Maps [0, ..., 10], [0, ..., 9]" << endl; + export_type exp31 (map3, map1); + TEST_ASSERT( ! exp31.isLocallyComplete () ); + import_type imp31 (map3, map1); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( imp31.isLocallyComplete () ); + + out << "Test Maps [0, ..., 9], [9, 8, ..., 0]" << endl; + export_type exp14 (map1, map4); + TEST_ASSERT( exp14.isLocallyComplete () ); + import_type imp14 (map1, map4); + TEST_ASSERT( imp14.isLocallyComplete () ); + + out << "Test Maps [9, 8, ..., 0], [0, ..., 9]" << endl; + export_type exp41 (map4, map1); + TEST_ASSERT( exp41.isLocallyComplete () ); + import_type imp41 (map4, map1); + TEST_ASSERT( imp41.isLocallyComplete () ); + + out << "Test Maps [9, 8, ..., 0], [0, ..., 10]" << endl; + export_type exp43 (map4, map3); + // Yes, all _source_ Map indices exist in the _target_ Map. + TEST_ASSERT( exp43.isLocallyComplete () ); + import_type imp43 (map4, map3); + TEST_ASSERT( ! imp43.isLocallyComplete () ); + + out << "Test Maps [0, ..., 10], [9, 8, ..., 0]" << endl; + export_type exp34 (map3, map4); + TEST_ASSERT( ! exp34.isLocallyComplete () ); + import_type imp34 (map3, map4); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( imp34.isLocallyComplete () ); + + // Make sure that the implementation looks at something other than + // the global min and max indices. + out << "Test Maps [0, 2, 3, 6], [0, 1, 4, 6]" << endl; + { + Teuchos::Array gids5 (4); + gids5[0] = 0; + gids5[1] = 2; + gids5[2] = 3; + gids5[3] = 6; + Teuchos::Array gids6 (4); + gids6[0] = 0; + gids6[1] = 1; + gids6[2] = 4; + gids6[3] = 6; + RCP map5 (new map_type (4, gids5 (), indexBase, serialComm)); + RCP map6 (new map_type (4, gids6 (), indexBase, serialComm)); + + export_type exp56 (map5, map6); + TEST_ASSERT( ! exp56.isLocallyComplete () ); + import_type imp56 (map5, map6); + TEST_ASSERT( ! imp56.isLocallyComplete () ); + } + } // test Maps with MPI_COMM_SELF / SerialComm + + if (numProcs > 1) { + out << "Test cases where both Maps share the same communicator, " + "with multiple processes" << endl; + Teuchos::OSTab tab2 (out); + + out << "Create first three Maps (all contiguous)" << endl; + + RCP map1 (new map_type (10 * numProcs, 10, indexBase, comm)); + RCP map2 (new map_type (10 * numProcs, 10, indexBase, comm)); + RCP map3 (new map_type (11 * numProcs, 11, indexBase, comm)); + + out << "Create map4 (noncontiguous, compatible with map1)" << endl; + RCP map4; + { + const LO map1LclSize = static_cast (map1->getNodeNumElements ()); + TEST_ASSERT( trueOnAllProcesses (*comm, map1LclSize == static_cast (10)) ); + Teuchos::Array gids4 (map1LclSize); + for (LO map1_lid = 0; map1_lid < map1LclSize; ++map1_lid) { + //const GO map1_gid = map1->getGlobalElement (map1_lid); + const GO map4_gid = map1->getMaxGlobalIndex () - static_cast (map1_lid); + gids4[map1_lid] = map4_gid; + } + map4 = rcp (new map_type (map1LclSize * numProcs, gids4 (), indexBase, comm)); + } + + out << "Create map5 (noncontiguous, compatible with map3)" << endl; + RCP map5; + { + const LO map3LclSize = static_cast (map3->getNodeNumElements ()); + TEST_ASSERT( trueOnAllProcesses (*comm, map3LclSize == static_cast (11)) ); + Teuchos::Array gids5 (map3LclSize); + for (LO map3_lid = 0; map3_lid < map3LclSize; ++map3_lid) { + //const GO map3_gid = map3->getGlobalElement (map3_lid); + const GO map5_gid = map3->getMaxGlobalIndex () - static_cast (map3_lid); + gids5[map3_lid] = map5_gid; + } + map5 = rcp (new map_type (map3LclSize * numProcs, gids5 (), indexBase, comm)); + } + + out << "Create map6 (one entry per process)" << endl; + Teuchos::Array gids6 (1); + gids6[0] = static_cast (myRank); + RCP map6 (new map_type (numProcs, gids6 (), 0, comm)); + + out << "Create map7 (one entry per process, reverse order of map6)" << endl; + Teuchos::Array gids7 (1); + gids7[0] = static_cast ((numProcs - 1) - myRank); // reverse order + RCP map7 (new map_type (numProcs, gids7 (), 0, comm)); + + out << "Create map8 (like map6, but with one process (ideally in the " + "middle, if numProcs > 2) with no indices)" << endl; + RCP map8; + { + Teuchos::ArrayRCP gids8; + if (myRank == 0) { // Proc 0 always gets indexBase + gids8 = Teuchos::arcp (1); + gids8[0] = static_cast (0); + } + else if (myRank != 1) { // Proc 1 has no GIDs, always + gids8 = Teuchos::arcp (1); + gids8[0] = static_cast (myRank); + } + map8 = rcp (new map_type ((numProcs - 1), gids8 (), indexBase, comm)); + } + + { + TEST_ASSERT( map1->isCompatible (*map2) ); + TEST_ASSERT( map1->isSameAs (*map2) ); + + export_type exp11 (map1, map1); + TEST_ASSERT( trueOnAllProcesses (*comm, exp11.isLocallyComplete ()) ); + import_type imp11 (map1, map1); + TEST_ASSERT( trueOnAllProcesses (*comm, imp11.isLocallyComplete ()) ); + + export_type exp12 (map1, map2); + TEST_ASSERT( trueOnAllProcesses (*comm, exp12.isLocallyComplete ()) ); + import_type imp12 (map1, map2); + TEST_ASSERT( trueOnAllProcesses (*comm, imp12.isLocallyComplete ()) ); + } + + { + TEST_ASSERT( ! map1->isCompatible (*map3) ); + TEST_ASSERT( ! map1->isSameAs (*map3) ); + + export_type exp13 (map1, map3); + // Yes, all _source_ Map indices exist in the _target_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, exp13.isLocallyComplete ()) ); + import_type imp13 (map1, map3); + TEST_ASSERT( falseOnSomeProcess (*comm, imp13.isLocallyComplete ()) ); + + export_type exp31 (map3, map1); + TEST_ASSERT( falseOnSomeProcess (*comm, exp31.isLocallyComplete ()) ); + import_type imp31 (map3, map1); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, imp31.isLocallyComplete ()) ); + } + + { + TEST_ASSERT( map1->isCompatible (*map4) ); + TEST_ASSERT( ! map1->isSameAs (*map4) ); + + export_type exp14 (map1, map4); + TEST_ASSERT( trueOnAllProcesses (*comm, exp14.getNumRemoteIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, exp14.getNumExportIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, exp14.isLocallyComplete ()) ); + + import_type imp14 (map1, map4); + TEST_ASSERT( trueOnAllProcesses (*comm, imp14.getNumRemoteIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, imp14.getNumExportIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, imp14.isLocallyComplete ()) ); + + export_type exp41 (map4, map1); + TEST_ASSERT( trueOnAllProcesses (*comm, exp41.getNumRemoteIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, exp41.getNumExportIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, exp41.isLocallyComplete ()) ); + + import_type imp41 (map4, map1); + TEST_ASSERT( trueOnAllProcesses (*comm, imp41.getNumRemoteIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, imp41.getNumExportIDs () == 0) ); + TEST_ASSERT( trueOnAllProcesses (*comm, imp41.isLocallyComplete ()) ); + + // isOneToOne() may cache communicated information for later + // use, so if there's a bug in isLocallyComplete(), calling + // isOneToOne() first may influence the above results. + TEST_ASSERT( map4->isOneToOne () ); + } + + { + TEST_ASSERT( ! map1->isCompatible (*map5) ); + TEST_ASSERT( ! map1->isSameAs (*map5) ); + + TEST_ASSERT( map3->isCompatible (*map5) ); + TEST_ASSERT( ! map3->isSameAs (*map5) ); + + export_type exp15 (map1, map5); + // Yes, all _source_ Map indices exist in the _target_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, exp15.isLocallyComplete ()) ); + import_type imp15 (map1, map5); + TEST_ASSERT( falseOnSomeProcess (*comm, imp15.isLocallyComplete ()) ); + + export_type exp51 (map5, map1); + TEST_ASSERT( falseOnSomeProcess (*comm, exp51.isLocallyComplete ()) ); + import_type imp51 (map5, map1); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, imp51.isLocallyComplete ()) ); + } + + { + export_type exp45 (map4, map5); + // Yes, all _source_ Map indices exist in the _target_ Map. + TEST_ASSERT( exp45.isLocallyComplete () ); + import_type imp45 (map4, map5); + TEST_ASSERT( falseOnSomeProcess (*comm, imp45.isLocallyComplete ()) ); + + export_type exp54 (map5, map4); + // The source Map has some indices not in the target Map on any process. + TEST_ASSERT( falseOnSomeProcess (*comm, exp54.isLocallyComplete ()) ); + import_type imp54 (map5, map4); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, imp54.isLocallyComplete ()) ); + } + + { + export_type exp67 (map6, map7); + TEST_ASSERT( trueOnAllProcesses (*comm, exp67.isLocallyComplete ()) ); + import_type imp67 (map6, map7); + TEST_ASSERT( trueOnAllProcesses (*comm, imp67.isLocallyComplete ()) ); + + export_type exp76 (map7, map6); + TEST_ASSERT( trueOnAllProcesses (*comm, exp76.isLocallyComplete ()) ); + import_type imp76 (map7, map6); + TEST_ASSERT( trueOnAllProcesses (*comm, imp76.isLocallyComplete ()) ); + + export_type exp68 (map6, map8); + // The source Map has some indices not in the target Map on any process. + TEST_ASSERT( falseOnSomeProcess (*comm, exp68.isLocallyComplete ()) ); + import_type imp68 (map6, map8); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, imp68.isLocallyComplete ()) ); + + export_type exp86 (map8, map6); + // Yes, all _target_ Map indices exist in the _source_ Map. + TEST_ASSERT( trueOnAllProcesses (*comm, exp86.isLocallyComplete ()) ); + import_type imp86 (map8, map6); + // The source Map has some indices not in the target Map on any process. + TEST_ASSERT( falseOnSomeProcess (*comm, imp86.isLocallyComplete ()) ); + } + } +} + +// +// INSTANTIATIONS +// + +#define UNIT_TEST_GROUP( LO, GO, NT ) \ + TEUCHOS_UNIT_TEST_TEMPLATE_3_INSTANT( ImportExport, IsLocallyComplete, LO, GO, NT ) + +TPETRA_ETI_MANGLING_TYPEDEFS() + +TPETRA_INSTANTIATE_LGN( UNIT_TEST_GROUP ) + +} // namespace (anonymous)