From a256f16332db08620155dd9b75f8edd5f9598cdf Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Fri, 24 Jun 2022 18:49:30 -0600 Subject: [PATCH 1/8] Tpetra: Add a variable to store the MPI tag for DistributorActor --- .../tpetra/core/src/Tpetra_Details_DistributorActor.cpp | 7 +++++-- .../tpetra/core/src/Tpetra_Details_DistributorActor.hpp | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp index 84182597af15..99c05f0821f4 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp @@ -42,14 +42,17 @@ namespace Tpetra { namespace Details { - DistributorActor::DistributorActor() { + DistributorActor::DistributorActor() + : mpiTag_(getNewMpiTag()) + { #ifdef HAVE_TPETRA_DISTRIBUTOR_TIMINGS makeTimers(); #endif // HAVE_TPETRA_DISTRIBUTOR_TIMINGS } DistributorActor::DistributorActor(const DistributorActor& otherActor) - : requests_(otherActor.requests_) + : mpiTag_(otherActor.mpiTag_), + requests_(otherActor.requests_) { #ifdef HAVE_TPETRA_DISTRIBUTOR_TIMINGS makeTimers(); diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index 2578a97e3574..b878b1a89cd5 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -91,6 +91,12 @@ class DistributorActor { bool isReady() const; private: + int getNewMpiTag() { + return 0; + } + + int mpiTag_; + Teuchos::Array>> requests_; #ifdef HAVE_TPETRA_DISTRIBUTOR_TIMINGS From 9dd541864ff4db6e434f5e83b4fb4dc63ddab50b Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Thu, 31 Mar 2022 14:10:55 -0600 Subject: [PATCH 2/8] Tpetra: add missing include in DistributorActor --- packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index b878b1a89cd5..df4659b49028 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -48,6 +48,8 @@ #include "Teuchos_RCP.hpp" #include "Teuchos_Time.hpp" +#include "Kokkos_TeuchosCommAdapters.hpp" + namespace Tpetra { namespace Details { From cc89b5ac2e23b20e158ff5364f9f74f1735cb153 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 25 Apr 2022 07:44:50 -0600 Subject: [PATCH 3/8] Tpetra: change pathTags so Plan's is 0 --- .../tpetra/core/src/Tpetra_Details_DistributorActor.hpp | 6 +++--- packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index df4659b49028..ca3df1c3acd5 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -94,7 +94,7 @@ class DistributorActor { private: int getNewMpiTag() { - return 0; + return 1; } int mpiTag_; @@ -204,7 +204,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, // (getIndicesTo().is_null()) and others might take the "slow" path for // the same doPosts() call, so the path tag must be the same for // both. - const int pathTag = 0; + const int pathTag = 1; const int tag = plan.getTag(pathTag); #ifdef HAVE_TPETRA_DEBUG @@ -482,7 +482,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, // (plan.getIndicesTo().is_null()) and others might take the "slow" path for // the same doPosts() call, so the path tag must be the same for // both. - const int pathTag = 1; + const int pathTag = 2; const int tag = plan.getTag(pathTag); #ifdef HAVE_TPETRA_DEBUG diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp index aad64a9bacb2..cc2f778bfe9d 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp @@ -644,7 +644,7 @@ void DistributorPlan::computeReceives() const int numProcs = comm_->getSize(); // MPI tag for nonblocking receives and blocking sends in this method. - const int pathTag = 2; + const int pathTag = 0; const int tag = getTag(pathTag); // toProcsFromMe[i] == the number of messages sent by this process From f15fdf5332ed0404b65e9d025efaa3e66c05d248 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 25 Apr 2022 07:51:10 -0600 Subject: [PATCH 4/8] Tpetra: use DistributorActor's tag for all communication in the actor --- .../src/Tpetra_Details_DistributorActor.hpp | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index ca3df1c3acd5..c27dfed31458 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -199,14 +199,6 @@ void DistributorActor::doPosts(const DistributorPlan& plan, size_t selfReceiveOffset = 0; - // MPI tag for nonblocking receives and blocking sends in this - // method. Some processes might take the "fast" path - // (getIndicesTo().is_null()) and others might take the "slow" path for - // the same doPosts() call, so the path tag must be the same for - // both. - const int pathTag = 1; - const int tag = plan.getTag(pathTag); - #ifdef HAVE_TPETRA_DEBUG TEUCHOS_TEST_FOR_EXCEPTION (requests_.size () != 0, @@ -263,7 +255,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, imports_view_type recvBuf = subview_offset (imports, curBufferOffset, curBufLen); requests_.push_back (ireceive (recvBuf, plan.getProcsFrom()[i], - tag, *plan.getComm())); + mpiTag_, *plan.getComm())); } else { // Receiving from myself selfReceiveOffset = curBufferOffset; // Remember the self-recv offset @@ -316,12 +308,12 @@ void DistributorActor::doPosts(const DistributorPlan& plan, subview_offset (exports, plan.getStartsTo()[p] * numPackets, plan.getLengthsTo()[p] * numPackets); requests_.push_back (isend (tmpSendBuf, plan.getProcsTo()[p], - tag, *plan.getComm())); + mpiTag_, *plan.getComm())); } else { // DISTRIBUTOR_SEND send (tmpSend, as (tmpSend.size ()), - plan.getProcsTo()[p], tag, *plan.getComm()); + plan.getProcsTo()[p], mpiTag_, *plan.getComm()); } } else { // "Sending" the message to myself @@ -395,7 +387,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, send (tmpSend, as (tmpSend.size ()), - plan.getProcsTo()[p], tag, *plan.getComm()); + plan.getProcsTo()[p], mpiTag_, *plan.getComm()); } else { // "Sending" the message to myself selfNum = p; @@ -475,17 +467,6 @@ void DistributorActor::doPosts(const DistributorPlan& plan, "enough entries to hold the expected number of import packets. " "imports.extent(0) = " << imports.extent (0) << " < " "totalNumImportPackets = " << totalNumImportPackets << "."); -#endif // HAVE_TPETRA_DEBUG - - // MPI tag for nonblocking receives and blocking sends in this - // method. Some processes might take the "fast" path - // (plan.getIndicesTo().is_null()) and others might take the "slow" path for - // the same doPosts() call, so the path tag must be the same for - // both. - const int pathTag = 2; - const int tag = plan.getTag(pathTag); - -#ifdef HAVE_TPETRA_DEBUG TEUCHOS_TEST_FOR_EXCEPTION (requests_.size () != 0, std::logic_error, "Tpetra::Distributor::" "doPosts(4 args, Kokkos): Process " << myProcID << ": requests_.size () = " @@ -538,7 +519,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, imports_view_type recvBuf = subview_offset (imports, curBufferOffset, totalPacketsFrom_i); requests_.push_back (ireceive (recvBuf, plan.getProcsFrom()[i], - tag, *plan.getComm())); + mpiTag_, *plan.getComm())); } else { // Receiving these packet(s) from myself selfReceiveOffset = curBufferOffset; // Remember the offset @@ -602,12 +583,12 @@ void DistributorActor::doPosts(const DistributorPlan& plan, exports_view_type tmpSendBuf = subview_offset (exports, sendPacketOffsets[p], packetsPerSend[p]); requests_.push_back (isend (tmpSendBuf, plan.getProcsTo()[p], - tag, *plan.getComm())); + mpiTag_, *plan.getComm())); } else { // DISTRIBUTOR_SEND send (tmpSend, as (tmpSend.size ()), - plan.getProcsTo()[p], tag, *plan.getComm()); + plan.getProcsTo()[p], mpiTag_, *plan.getComm()); } } else { // "Sending" the message to myself @@ -684,7 +665,7 @@ void DistributorActor::doPosts(const DistributorPlan& plan, send (tmpSend, as (tmpSend.size ()), - plan.getProcsTo()[p], tag, *plan.getComm()); + plan.getProcsTo()[p], mpiTag_, *plan.getComm()); } } else { // "Sending" the message to myself From d4e9f00e0a891dc63be102b60b3c3fae44d87965 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 25 Apr 2022 07:55:08 -0600 Subject: [PATCH 5/8] Tpetra: remove the no-longer-necessary DistributorPlan::getTag --- .../core/src/Tpetra_Details_DistributorPlan.cpp | 11 +++++------ .../core/src/Tpetra_Details_DistributorPlan.hpp | 6 ------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp index cc2f778bfe9d..de235b5a9afe 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp @@ -115,10 +115,6 @@ DistributorPlan::DistributorPlan(const DistributorPlan& otherPlan) indicesFrom_(otherPlan.indicesFrom_) { } -int DistributorPlan::getTag(const int pathTag) const { - return useDistinctTags_ ? pathTag : comm_->getTag(); -} - size_t DistributorPlan::createFromSends(const Teuchos::ArrayView& exportProcIDs) { using Teuchos::outArg; using Teuchos::REDUCE_MAX; @@ -644,8 +640,11 @@ void DistributorPlan::computeReceives() const int numProcs = comm_->getSize(); // MPI tag for nonblocking receives and blocking sends in this method. - const int pathTag = 0; - const int tag = getTag(pathTag); + // Note that useDistinctTags_ is deprecated. After deprecation cycle is complete, we should just + // use 0 here (although it's highly unlikely the Actor's communication will conflict with this + // communication, we still avoid the use of tag 0 in Actor, so the hardcoded 0 is in fact still + // unique with respect to the Actor code paths) + const int tag = useDistinctTags_ ? 0 : comm_->getTag(); // toProcsFromMe[i] == the number of messages sent by this process // to process i. The data in numSendsToOtherProcs_, procIdsToSendTo_, and lengthsTo_ diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp index 1534e5bf9868..c3654e63e9f2 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp @@ -104,12 +104,6 @@ class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase { DistributorPlan(Teuchos::RCP> comm); DistributorPlan(const DistributorPlan& otherPlan); - //! Get the tag to use for receives and sends. - /// - /// See useDistinctTags_. This is called in doPosts() (both - /// variants) and computeReceives(). - int getTag(const int pathTag) const; - size_t createFromSends(const Teuchos::ArrayView& exportProcIDs); void createFromRecvs(const Teuchos::ArrayView& remoteProcIDs); void createFromSendsAndRecvs(const Teuchos::ArrayView& exportProcIDs, From 5591d0466327aea426b8bd8e9b2aa917d47209c4 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Fri, 24 Jun 2022 18:53:02 -0600 Subject: [PATCH 6/8] Tpetra: Strip out remainder of deprecated useDistinctTags usage --- .../src/Tpetra_Details_DistributorPlan.cpp | 14 +++----------- .../src/Tpetra_Details_DistributorPlan.hpp | 18 ------------------ 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp index de235b5a9afe..c40e92ee7695 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp @@ -86,7 +86,6 @@ DistributorPlan::DistributorPlan(Teuchos::RCP> comm) howInitialized_(DISTRIBUTOR_NOT_INITIALIZED), reversePlan_(Teuchos::null), sendType_(DISTRIBUTOR_SEND), - useDistinctTags_(useDistinctTags_default), sendMessageToSelf_(false), numSendsToOtherProcs_(0), maxSendLength_(0), @@ -99,7 +98,6 @@ DistributorPlan::DistributorPlan(const DistributorPlan& otherPlan) howInitialized_(DISTRIBUTOR_INITIALIZED_BY_COPY), reversePlan_(otherPlan.reversePlan_), sendType_(otherPlan.sendType_), - useDistinctTags_(otherPlan.useDistinctTags_), sendMessageToSelf_(otherPlan.sendMessageToSelf_), numSendsToOtherProcs_(otherPlan.numSendsToOtherProcs_), procIdsToSendTo_(otherPlan.procIdsToSendTo_), @@ -616,7 +614,6 @@ void DistributorPlan::createReversePlan() const reversePlan_->procsFrom_ = procIdsToSendTo_; reversePlan_->startsFrom_ = startsTo_; reversePlan_->indicesFrom_ = indicesTo_; - reversePlan_->useDistinctTags_ = useDistinctTags_; } void DistributorPlan::computeReceives() @@ -639,12 +636,7 @@ void DistributorPlan::computeReceives() const int myRank = comm_->getRank(); const int numProcs = comm_->getSize(); - // MPI tag for nonblocking receives and blocking sends in this method. - // Note that useDistinctTags_ is deprecated. After deprecation cycle is complete, we should just - // use 0 here (although it's highly unlikely the Actor's communication will conflict with this - // communication, we still avoid the use of tag 0 in Actor, so the hardcoded 0 is in fact still - // unique with respect to the Actor code paths) - const int tag = useDistinctTags_ ? 0 : comm_->getTag(); + const int mpiTag = 0; // toProcsFromMe[i] == the number of messages sent by this process // to process i. The data in numSendsToOtherProcs_, procIdsToSendTo_, and lengthsTo_ @@ -786,7 +778,7 @@ void DistributorPlan::computeReceives() lengthsFromBuffers[i].resize (1); lengthsFromBuffers[i][0] = as (0); requests[i] = ireceive (lengthsFromBuffers[i], anySourceProc, - tag, *comm_); + mpiTag, *comm_); } // Post the sends: Tell each process to which we are sending how @@ -803,7 +795,7 @@ void DistributorPlan::computeReceives() // this communication pattern will send that process // lengthsTo_[i] blocks of packets. const size_t* const lengthsTo_i = &lengthsTo_[i]; - send (lengthsTo_i, 1, as (procIdsToSendTo_[i]), tag, *comm_); + send (lengthsTo_i, 1, as (procIdsToSendTo_[i]), mpiTag, *comm_); } else { // We don't need a send in the self-message case. If this diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp index c3654e63e9f2..97d5620840c1 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp @@ -49,10 +49,6 @@ namespace Tpetra { namespace Details { -namespace { - const bool useDistinctTags_default = true; -} - /// \brief The type of MPI send that Distributor should use. /// /// This is an implementation detail of Distributor. Please do @@ -115,7 +111,6 @@ class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase { Teuchos::RCP> getComm() const { return comm_; } EDistributorSendType getSendType() const { return sendType_; } - bool useDistinctTags() const { return useDistinctTags_; } size_t getNumReceives() const { return numReceives_; } size_t getNumSends() const { return numSendsToOtherProcs_; } bool hasSelfMessage() const { return sendMessageToSelf_; } @@ -153,19 +148,6 @@ class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase { //! @name Parameters read in from the Teuchos::ParameterList //@{ EDistributorSendType sendType_; - - /// \brief Whether to use different tags for different code paths. - /// - /// There are currently three code paths in Distributor that post - /// receives and sends: - /// - /// 1. Three-argument variant of doPosts() - /// 2. Four-argument variant of doPosts() - /// 3. computeReceives() - /// - /// If this option is true, Distributor will use a distinct - /// message tag for each of these paths. - bool useDistinctTags_; //@} bool sendMessageToSelf_; From bedc7d4a257586db1a7b771f2d3d5fb1cfea7882 Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Fri, 24 Jun 2022 18:54:11 -0600 Subject: [PATCH 7/8] Tpetra: Get rid of unnecessary function on DistributorActor --- packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp | 2 +- packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp index 99c05f0821f4..ece9f57fc0c3 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp @@ -43,7 +43,7 @@ namespace Tpetra { namespace Details { DistributorActor::DistributorActor() - : mpiTag_(getNewMpiTag()) + : mpiTag_(1) { #ifdef HAVE_TPETRA_DISTRIBUTOR_TIMINGS makeTimers(); diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index c27dfed31458..91be83fbdc01 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -93,10 +93,6 @@ class DistributorActor { bool isReady() const; private: - int getNewMpiTag() { - return 1; - } - int mpiTag_; Teuchos::Array>> requests_; From dcec44417a8bad3bfe5f6945476e09ce7730da1f Mon Sep 17 00:00:00 2001 From: "Timothy A. Smith" Date: Mon, 11 Jul 2022 14:31:06 -0600 Subject: [PATCH 8/8] name default mpi tag constants --- packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp | 2 +- packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp | 1 + packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp | 2 +- packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp index ece9f57fc0c3..306d40e79426 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.cpp @@ -43,7 +43,7 @@ namespace Tpetra { namespace Details { DistributorActor::DistributorActor() - : mpiTag_(1) + : mpiTag_(DEFAULT_MPI_TAG) { #ifdef HAVE_TPETRA_DISTRIBUTOR_TIMINGS makeTimers(); diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp index 91be83fbdc01..d40ff1429310 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorActor.hpp @@ -57,6 +57,7 @@ template constexpr bool areKokkosViews = Kokkos::is_view::value && Kokkos::is_view::value; class DistributorActor { + static constexpr int DEFAULT_MPI_TAG = 1; public: DistributorActor(); diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp index c40e92ee7695..a39f8e1455bc 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.cpp @@ -636,7 +636,7 @@ void DistributorPlan::computeReceives() const int myRank = comm_->getRank(); const int numProcs = comm_->getSize(); - const int mpiTag = 0; + const int mpiTag = DEFAULT_MPI_TAG; // toProcsFromMe[i] == the number of messages sent by this process // to process i. The data in numSendsToOtherProcs_, procIdsToSendTo_, and lengthsTo_ diff --git a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp index 97d5620840c1..6e9120295d19 100644 --- a/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp +++ b/packages/tpetra/core/src/Tpetra_Details_DistributorPlan.hpp @@ -96,6 +96,8 @@ DistributorHowInitializedEnumToString (EDistributorHowInitialized how); /// sends, this prevents deadlock, even if MPI_Send blocks and /// does not buffer.) class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase { + static constexpr int DEFAULT_MPI_TAG = 0; + public: DistributorPlan(Teuchos::RCP> comm); DistributorPlan(const DistributorPlan& otherPlan);