Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Teuchos: SyncTimeMonitor #6452

Merged
merged 3 commits into from
Jan 9, 2020
Merged

Teuchos: SyncTimeMonitor #6452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
99298c4
Teuchos: Add SyncTimeMonitor
cgcgcg Dec 13, 2019
89e24c3
MueLu RefMaxwell: Refactor timers, use SyncTimeMonitor
cgcgcg Dec 13, 2019
a7421a9
Teuchos: Add unit test for SyncTimeMonitor
cgcgcg Jan 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion packages/muelu/adapters/xpetra/MueLu_RefMaxwell_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,8 @@ namespace MueLu {
//! dump out real-valued multivector
void dumpCoords(const RealValuedMultiVector& X, std::string name) const;

Teuchos::RCP<Teuchos::TimeMonitor> getTimer(std::string name, RCP<const Teuchos::Comm<int> > comm=Teuchos::null) const;

//! set parameters
void setMatvecParams(Matrix& A, RCP<ParameterList> matvecParams) {
RCP<const Import> xpImporter = A.getCrsGraph()->getImporter();
Expand Down Expand Up @@ -428,7 +430,7 @@ namespace MueLu {
Teuchos::RCP<Teuchos::ParameterList> AH_AP_reuse_data_, AH_RAP_reuse_data_;
Teuchos::RCP<Teuchos::ParameterList> A22_AP_reuse_data_, A22_RAP_reuse_data_;
//! Some options
bool disable_addon_, dump_matrices_,useKokkos_,use_as_preconditioner_,implicitTranspose_,fuseProlongationAndUpdate_;
bool disable_addon_, dump_matrices_, useKokkos_, use_as_preconditioner_, implicitTranspose_, fuseProlongationAndUpdate_, syncTimers_;
int numItersH_, numIters22_;
std::string mode_;
//! Temporary memory
Expand Down
84 changes: 49 additions & 35 deletions packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ namespace MueLu {
dump_matrices_ = list.get("refmaxwell: dump matrices",MasterList::getDefault<bool>("refmaxwell: dump matrices"));
implicitTranspose_ = list.get("transpose: use implicit",MasterList::getDefault<bool>("transpose: use implicit"));
fuseProlongationAndUpdate_ = list.get("fuse prolongation and update",MasterList::getDefault<bool>("fuse prolongation and update"));
syncTimers_ = list.get("sync timers",false);
numItersH_ = list.get("refmaxwell: num iters H",1);
numIters22_ = list.get("refmaxwell: num iters 22",1);

Expand Down Expand Up @@ -191,7 +192,7 @@ namespace MueLu {
timerLabel = "MueLu RefMaxwell: compute (reuse)";
else
timerLabel = "MueLu RefMaxwell: compute";
Teuchos::TimeMonitor tmCompute(*Teuchos::TimeMonitor::getNewTimer(timerLabel));
RCP<Teuchos::TimeMonitor> tmCompute = getTimer(timerLabel);

std::map<std::string, MsgType> verbMap;
verbMap["none"] = None;
Expand Down Expand Up @@ -588,7 +589,7 @@ namespace MueLu {

if (doRebalancing) { // rebalance AH
if (!reuse) {
Teuchos::TimeMonitor tm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: Rebalance AH"));
RCP<Teuchos::TimeMonitor> tm = getTimer("MueLu RefMaxwell: Rebalance AH");

Level fineLevel, coarseLevel;
fineLevel.SetFactoryManager(null);
Expand Down Expand Up @@ -771,7 +772,7 @@ namespace MueLu {
GetOStream(Runtime0) << "RefMaxwell::compute(): building MG for (2,2)-block" << std::endl;

{ // build fine grid operator for (2,2)-block, D0* SM D0 (aka TMT)
Teuchos::TimeMonitor tm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: Build A22"));
RCP<Teuchos::TimeMonitor> tm = getTimer("MueLu RefMaxwell: Build A22");

Level fineLevel, coarseLevel;
fineLevel.SetFactoryManager(null);
Expand Down Expand Up @@ -1224,6 +1225,19 @@ namespace MueLu {
}


template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
Teuchos::RCP<Teuchos::TimeMonitor> RefMaxwell<Scalar,LocalOrdinal,GlobalOrdinal,Node>::getTimer(std::string name, RCP<const Teuchos::Comm<int> > comm) const {
if (!syncTimers_)
return Teuchos::rcp(new Teuchos::TimeMonitor(*Teuchos::TimeMonitor::getNewTimer(name)));
else {
if (comm.is_null())
return Teuchos::rcp(new Teuchos::SyncTimeMonitor(*Teuchos::TimeMonitor::getNewTimer(name), SM_Matrix_->getRowMap()->getComm().ptr()));
else
return Teuchos::rcp(new Teuchos::SyncTimeMonitor(*Teuchos::TimeMonitor::getNewTimer(name), comm.ptr()));
}
}


template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
void RefMaxwell<Scalar,LocalOrdinal,GlobalOrdinal,Node>::buildProlongator() {
// The P11 matrix maps node based aggregrates { A_j } to edges { e_i }.
Expand Down Expand Up @@ -1758,7 +1772,7 @@ namespace MueLu {

template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
void RefMaxwell<Scalar,LocalOrdinal,GlobalOrdinal,Node>::formCoarseMatrix() {
Teuchos::TimeMonitor tm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: Build coarse (1,1) matrix"));
RCP<Teuchos::TimeMonitor> tm = getTimer("MueLu RefMaxwell: Build coarse (1,1) matrix");

// coarse matrix for P11* (M1 + D1* M2 D1) P11
RCP<Matrix> Matrix1;
Expand Down Expand Up @@ -1820,7 +1834,7 @@ namespace MueLu {
}
else {
if (Addon_Matrix_.is_null()) {
Teuchos::TimeMonitor tmAddon(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: Build coarse addon matrix"));
RCP<Teuchos::TimeMonitor> tmAddon = getTimer("MueLu RefMaxwell: Build coarse addon matrix");
// catch a failure
TEUCHOS_TEST_FOR_EXCEPTION(M0inv_Matrix_==Teuchos::null,std::invalid_argument,
"MueLu::RefMaxwell::formCoarseMatrix(): Inverse of "
Expand Down Expand Up @@ -1910,52 +1924,52 @@ namespace MueLu {

{ // compute residual

Teuchos::TimeMonitor tmRes(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: residual calculation"));
RCP<Teuchos::TimeMonitor> tmRes = getTimer("MueLu RefMaxwell: residual calculation");
Utilities::Residual(*SM_Matrix_, X, RHS, *residual_);
}

{ // restrict residual to sub-hierarchies

if (implicitTranspose_) {
{
Teuchos::TimeMonitor tmP11(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: restriction coarse (1,1) (implicit)"));
RCP<Teuchos::TimeMonitor> tmRes = getTimer("MueLu RefMaxwell: restriction coarse (1,1) (implicit)");
P11_->apply(*residual_,*P11res_,Teuchos::TRANS);
}
{
Teuchos::TimeMonitor tmD0(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: restriction (2,2) (implicit)"));
RCP<Teuchos::TimeMonitor> tmD0 = getTimer("MueLu RefMaxwell: restriction (2,2) (implicit)");
D0_Matrix_->apply(*residual_,*D0res_,Teuchos::TRANS);
}
} else {
{
Teuchos::TimeMonitor tmP11(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: restriction coarse (1,1) (explicit)"));
RCP<Teuchos::TimeMonitor> tmP11 = getTimer("MueLu RefMaxwell: restriction coarse (1,1) (explicit)");
R11_->apply(*residual_,*P11res_,Teuchos::NO_TRANS);
}
{
Teuchos::TimeMonitor tmD0(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: restriction (2,2) (explicit)"));
RCP<Teuchos::TimeMonitor> tmD0 = getTimer("MueLu RefMaxwell: restriction (2,2) (explicit)");
D0_T_Matrix_->apply(*residual_,*D0res_,Teuchos::NO_TRANS);
}
}
}

{
Teuchos::TimeMonitor tmSubSolves(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: subsolves"));
RCP<Teuchos::TimeMonitor> tmSubSolves = getTimer("MueLu RefMaxwell: subsolves");

// block diagonal preconditioner on 2x2 (V-cycle for diagonal blocks)

if (!ImporterH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: import coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: import coarse (1,1)");
P11resTmp_->doImport(*P11res_, *ImporterH_, Xpetra::INSERT);
P11res_.swap(P11resTmp_);
}
if (!Importer22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: import (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: import (2,2)");
D0resTmp_->doImport(*D0res_, *Importer22_, Xpetra::INSERT);
D0res_.swap(D0resTmp_);
}

// iterate on coarse (1, 1) block
if (!AH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: solve coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: solve coarse (1,1)", AH_->getRowMap()->getComm());

RCP<const Map> origXMap = P11x_->getMap();
RCP<const Map> origRhsMap = P11res_->getMap();
Expand All @@ -1970,7 +1984,7 @@ namespace MueLu {

// iterate on (2, 2) block
if (!A22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: solve (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: solve (2,2)", A22_->getRowMap()->getComm());

RCP<const Map> origXMap = D0x_->getMap();
RCP<const Map> origRhsMap = D0res_->getMap();
Expand All @@ -1984,13 +1998,13 @@ namespace MueLu {
}

if (!Importer22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: export (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: export (2,2)");
D0xTmp_->doExport(*D0x_, *Importer22_, Xpetra::INSERT);
D0x_.swap(D0xTmp_);
}

if (!ImporterH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: export coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: export coarse (1,1)");
P11xTmp_->doExport(*P11x_, *ImporterH_, Xpetra::INSERT);
P11x_.swap(P11xTmp_);
}
Expand All @@ -1999,27 +2013,27 @@ namespace MueLu {

if (fuseProlongationAndUpdate_) {
{ // prolongate (1,1) block
Teuchos::TimeMonitor tmP11(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: prolongation coarse (1,1) (fused)"));
RCP<Teuchos::TimeMonitor> tmP11 = getTimer("MueLu RefMaxwell: prolongation coarse (1,1) (fused)");
P11_->apply(*P11x_,X,Teuchos::NO_TRANS,one,one);
}

{ // prolongate (2,2) block
Teuchos::TimeMonitor tmD0(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: prolongation (2,2) (fused)"));
RCP<Teuchos::TimeMonitor> tmD0 = getTimer("MueLu RefMaxwell: prolongation (2,2) (fused)");
D0_Matrix_->apply(*D0x_,X,Teuchos::NO_TRANS,one,one);
}
} else {
{ // prolongate (1,1) block
Teuchos::TimeMonitor tmP11(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: prolongation coarse (1,1) (unfused)"));
RCP<Teuchos::TimeMonitor> tmP11 = getTimer("MueLu RefMaxwell: prolongation coarse (1,1) (unfused)");
P11_->apply(*P11x_,*residual_,Teuchos::NO_TRANS);
}

{ // prolongate (2,2) block
Teuchos::TimeMonitor tmD0(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: prolongation (2,2) (unfused)"));
RCP<Teuchos::TimeMonitor> tmD0 = getTimer("MueLu RefMaxwell: prolongation (2,2) (unfused)");
D0_Matrix_->apply(*D0x_,*residual_,Teuchos::NO_TRANS,one,one);
}

{ // update current solution
Teuchos::TimeMonitor tmUpdate(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: update"));
RCP<Teuchos::TimeMonitor> tmUpdate = getTimer("MueLu RefMaxwell: update");
X.update(one, *residual_, one);
}
}
Expand Down Expand Up @@ -2067,7 +2081,7 @@ namespace MueLu {
Scalar one = Teuchos::ScalarTraits<Scalar>::one();

{ // compute residual
Teuchos::TimeMonitor tmRes(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: residual calculation"));
RCP<Teuchos::TimeMonitor> tmRes = getTimer("MueLu RefMaxwell: residual calculation");
Utilities::Residual(*SM_Matrix_, X, RHS,*residual_);
if (implicitTranspose_)
P11_->apply(*residual_,*P11res_,Teuchos::TRANS);
Expand All @@ -2077,12 +2091,12 @@ namespace MueLu {

{ // solve coarse (1,1) block
if (!ImporterH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: import coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: import coarse (1,1)");
P11resTmp_->doImport(*P11res_, *ImporterH_, Xpetra::INSERT);
P11res_.swap(P11resTmp_);
}
if (!AH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: solve coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: solve coarse (1,1)", AH_->getRowMap()->getComm());

RCP<const Map> origXMap = P11x_->getMap();
RCP<const Map> origRhsMap = P11res_->getMap();
Expand All @@ -2095,14 +2109,14 @@ namespace MueLu {
P11res_->replaceMap(origRhsMap);
}
if (!ImporterH_.is_null()) {
Teuchos::TimeMonitor tmH(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: export coarse (1,1)"));
RCP<Teuchos::TimeMonitor> tmH = getTimer("MueLu RefMaxwell: export coarse (1,1)");
P11xTmp_->doExport(*P11x_, *ImporterH_, Xpetra::INSERT);
P11x_.swap(P11xTmp_);
}
}

{ // update current solution
Teuchos::TimeMonitor tmUp(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: update"));
RCP<Teuchos::TimeMonitor> tmUp = getTimer("MueLu RefMaxwell: update");
P11_->apply(*P11x_,*residual_,Teuchos::NO_TRANS);
X.update(one, *residual_, one);
}
Expand All @@ -2116,7 +2130,7 @@ namespace MueLu {
Scalar one = Teuchos::ScalarTraits<Scalar>::one();

{ // compute residual
Teuchos::TimeMonitor tmRes(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: residual calculation"));
RCP<Teuchos::TimeMonitor> tmRes = getTimer("MueLu RefMaxwell: residual calculation");
Utilities::Residual(*SM_Matrix_, X, RHS, *residual_);
if (implicitTranspose_)
D0_Matrix_->apply(*residual_,*D0res_,Teuchos::TRANS);
Expand All @@ -2126,12 +2140,12 @@ namespace MueLu {

{ // solve (2,2) block
if (!Importer22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: import (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: import (2,2)");
D0resTmp_->doImport(*D0res_, *Importer22_, Xpetra::INSERT);
D0res_.swap(D0resTmp_);
}
if (!A22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: solve (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: solve (2,2)", A22_->getRowMap()->getComm());

RCP<const Map> origXMap = D0x_->getMap();
RCP<const Map> origRhsMap = D0res_->getMap();
Expand All @@ -2144,14 +2158,14 @@ namespace MueLu {
D0res_->replaceMap(origRhsMap);
}
if (!Importer22_.is_null()) {
Teuchos::TimeMonitor tm22(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: export (2,2)"));
RCP<Teuchos::TimeMonitor> tm22 = getTimer("MueLu RefMaxwell: export (2,2)");
D0xTmp_->doExport(*D0x_, *Importer22_, Xpetra::INSERT);
D0x_.swap(D0xTmp_);
}
}

{ //update current solution
Teuchos::TimeMonitor tmUp(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: update"));
RCP<Teuchos::TimeMonitor> tmUp = getTimer("MueLu RefMaxwell: update");
D0_Matrix_->apply(*D0x_,*residual_,Teuchos::NO_TRANS);
X.update(one, *residual_, one);
}
Expand All @@ -2165,15 +2179,15 @@ namespace MueLu {
Scalar /* alpha */,
Scalar /* beta */) const {

Teuchos::TimeMonitor tm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: solve"));
RCP<Teuchos::TimeMonitor> tm = getTimer("MueLu RefMaxwell: solve");

// make sure that we have enough temporary memory
if (X.getNumVectors() != P11res_->getNumVectors())
allocateMemory(X.getNumVectors());

{ // apply pre-smoothing

Teuchos::TimeMonitor tmSm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: smoothing"));
RCP<Teuchos::TimeMonitor> tmSm = getTimer("MueLu RefMaxwell: smoothing");

#if defined(MUELU_REFMAXWELL_CAN_USE_HIPTMAIR)
if (useHiptmairSmoothing_) {
Expand Down Expand Up @@ -2205,7 +2219,7 @@ namespace MueLu {

{ // apply post-smoothing

Teuchos::TimeMonitor tmSm(*Teuchos::TimeMonitor::getNewTimer("MueLu RefMaxwell: smoothing"));
RCP<Teuchos::TimeMonitor> tmSm = getTimer("MueLu RefMaxwell: smoothing");

#if defined(MUELU_REFMAXWELL_CAN_USE_HIPTMAIR)
if (useHiptmairSmoothing_)
Expand Down
8 changes: 8 additions & 0 deletions packages/teuchos/comm/src/Teuchos_TimeMonitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,6 +1113,14 @@ namespace Teuchos {
computeGlobalTimerStatistics (statData, statNames, comm.ptr(), setOp, filter);
}

SyncTimeMonitor::SyncTimeMonitor(Time& timer, Ptr<const Comm<int> > comm, bool reset)
: TimeMonitor(timer, reset), comm_(comm)
{ }

SyncTimeMonitor::~SyncTimeMonitor() {
comm_->barrier();
}


namespace {
/// \brief Quote the given string for YAML output.
Expand Down
33 changes: 33 additions & 0 deletions packages/teuchos/comm/src/Teuchos_TimeMonitor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -757,12 +757,45 @@ class TEUCHOSCOMM_LIB_DLL_EXPORT TimeMonitor :
/// ParameterList around.
static bool setParams_;

protected:
/// \brief Stacked timer for optional injection of timing from
/// TimeMonitor-enabled objects.
static Teuchos::RCP<Teuchos::StackedTimer> stackedTimer_;
};


/// \class SyncTimeMonitor
/// \brief A TimeMonitor that waits at a MPI barrier before destruction.
class SyncTimeMonitor :
public TimeMonitor {
public:

/** \name Constructor/Destructor */
//@{

/// \brief Constructor: starts the timer.
///
/// \param timer [in/out] Reference to the timer to be wrapped.
/// This constructor starts the timer, and the destructor stops
/// the timer.
///
/// \param reset [in] If true, reset the timer before starting it.
/// Default behavior is not to reset the timer.
SyncTimeMonitor(Time& timer, Ptr<const Comm<int> > comm, bool reset=false);

//! Default constructor is deleted, since it would be unsafe.
SyncTimeMonitor () = delete;

//! Destructor: stops the timer.
~SyncTimeMonitor() override;
//@}

private:
// \brief Communicator on which barrier will be called.
Ptr<const Comm<int> > comm_;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you use a Ptr and not const Comm<int>&, then it should be possible for comm_ to be null. However, it looks like you always assume that comm_ is nonnull.

};


} // namespace Teuchos


Expand Down
9 changes: 9 additions & 0 deletions packages/teuchos/comm/test/TimeMonitor/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,12 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST(
COMM mpi
NUM_MPI_PROCS 4
)

TRIBITS_ADD_EXECUTABLE_AND_TEST(
SyncTimeMonitor
SOURCES
SyncTimeMonitor.cpp
${TEUCHOS_STD_UNIT_TEST_MAIN}
COMM mpi
NUM_MPI_PROCS 4
)
Loading