diff --git a/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp b/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp index 0d08c20a438a..82b3cd19afe6 100644 --- a/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp +++ b/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp @@ -927,6 +927,15 @@ namespace MueLu { D0x_ = MultiVectorFactory::Build(D0_Matrix_->getDomainMap(), 1); residual_ = MultiVectorFactory::Build(SM_Matrix_->getDomainMap(), 1); + if (!ImporterH_.is_null() && parameterList_.isSublist("refmaxwell: ImporterH params")){ + RCP importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: ImporterH params")); + ImporterH_->setDistributorParameters(importerParams); + } + if (!Importer22_.is_null() && parameterList_.isSublist("refmaxwell: Importer22 params")){ + RCP importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: Importer22 params")); + Importer22_->setDistributorParameters(importerParams); + } + #ifdef HAVE_MUELU_CUDA if (parameterList_.get("refmaxwell: cuda profile setup", false)) cudaProfilerStop(); #endif @@ -976,6 +985,57 @@ namespace MueLu { Xpetra::IO::Write(std::string("A22.mat"), *A22_); } + if (parameterList_.isSublist("matvec params")) + { + RCP matvecParams = rcpFromRef(parameterList_.sublist("matvec params")); + + { + RCP xpImporter = SM_Matrix_->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = SM_Matrix_->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + { + RCP xpImporter = D0_Matrix_->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = D0_Matrix_->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + { + RCP xpImporter = D0_T_Matrix_->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = D0_T_Matrix_->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + { + RCP xpImporter = R11_->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = R11_->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + { + RCP xpImporter = P11_->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = P11_->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + if (!ImporterH_.is_null()) + ImporterH_->setDistributorParameters(matvecParams); + if (!Importer22_.is_null()) + Importer22_->setDistributorParameters(matvecParams); + } + + VerboseObject::SetDefaultVerbLevel(oldVerbLevel); } diff --git a/packages/muelu/doc/UsersGuide/masterList.xml b/packages/muelu/doc/UsersGuide/masterList.xml index 6222c0d1f08f..028e848c77ff 100644 --- a/packages/muelu/doc/UsersGuide/masterList.xml +++ b/packages/muelu/doc/UsersGuide/masterList.xml @@ -117,6 +117,14 @@ true + + matvec params + \parameterlist + Pass parameters to the underlying linear algebra + not supported by ML + false + + diff --git a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex index bfffcd0a61b1..e362ef44f830 100644 --- a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex +++ b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex @@ -24,6 +24,8 @@ \cbb{hierarchy label}{string}{""}{Label for the hierarchy. Is applied to timer labels.} +\cba{matvec params}{\parameterlist}{Pass parameters to the underlying linear algebra} + \cbb{smoother: pre or post}{string}{"both"}{Pre- and post-smoother combination. Possible values: "pre" (only pre-smoother), "post" (only post-smoother), "both" (both pre-and post-smoothers), "none" (no smoothing).} \cbb{smoother: type}{string}{"RELAXATION"}{Smoother type. Possible values: see Table~\ref{tab:smoothers}.} diff --git a/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp b/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp index 7a9d77be69b9..5efc6bdc5ad4 100644 --- a/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp +++ b/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp @@ -246,6 +246,8 @@ namespace MueLu { isLastLevel = r || (levelID == lastLevelID); levelID++; } + if (!matvecParams_.is_null()) + H.SetMatvecParams(matvecParams_); // FIXME: Should allow specification of NumVectors on parameterlist H.AllocateLevelMultiVectors(1); H.describe(H.GetOStream(Runtime0), verbosity_); @@ -321,6 +323,7 @@ namespace MueLu { Teuchos::Array nullspaceToPrint_; Teuchos::Array coordinatesToPrint_; Teuchos::Array elementToNodeMapsToPrint_; + Teuchos::RCP matvecParams_; std::map > keep_; diff --git a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp index bca1933f233e..3c4b9e934881 100644 --- a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp +++ b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp @@ -413,6 +413,9 @@ namespace MueLu { // Detect if we use implicit transpose changedImplicitTranspose_ = MUELU_TEST_AND_SET_VAR(paramList, "transpose: use implicit", bool, this->implicitTranspose_); + if (paramList.isSublist("matvec params")) + this->matvecParams_ = Teuchos::parameterList(paramList.sublist("matvec params")); + // Create default manager // FIXME: should it be here, or higher up RCP defaultManager = rcp(new FactoryManager()); @@ -1849,6 +1852,10 @@ namespace MueLu { hieraList.remove("transpose: use implicit"); } + if (hieraList.isSublist("matvec params")) + this->matvecParams_ = Teuchos::parameterList(hieraList.sublist("matvec params")); + + if (hieraList.isParameter("coarse grid correction scaling factor")) { this->scalingFactor_ = hieraList.get("coarse grid correction scaling factor"); hieraList.remove("coarse grid correction scaling factor"); diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp index 711e194fbfd9..007f210cd735 100644 --- a/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp +++ b/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp @@ -196,6 +196,8 @@ namespace MueLu { //! Helper function void CheckLevel(Level& level, int levelID); + void SetMatvecParams(RCP matvecParams); + //! Multi-level setup phase: build a new level of the hierarchy. /*! This method is aimed to be used in a loop building the hierarchy level by level. See Hierarchy::Setup(manager, startLevel, numDesiredLevels) for an example of usage. * diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp index 7b461a86e8ff..d7dbb0c0d833 100644 --- a/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp +++ b/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp @@ -242,6 +242,48 @@ namespace MueLu { "MueLu::Hierarchy::Setup(): wrong level parent"); } + template + void Hierarchy::SetMatvecParams(RCP matvecParams) { + for (int i = 0; i < GetNumLevels(); ++i) { + RCP level = Levels_[i]; + if (level->IsAvailable("A")) { + RCP Aop = level->Get >("A"); + RCP A = rcp_dynamic_cast(Aop); + if (!A.is_null()) { + RCP xpImporter = A->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = A->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + } + if (level->IsAvailable("P")) { + RCP P = level->Get >("P"); + RCP xpImporter = P->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = P->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + if (level->IsAvailable("R")) { + RCP R = level->Get >("R"); + RCP xpImporter = R->getCrsGraph()->getImporter(); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + RCP xpExporter = R->getCrsGraph()->getExporter(); + if (!xpExporter.is_null()) + xpExporter->setDistributorParameters(matvecParams); + } + if (level->IsAvailable("Importer")) { + RCP xpImporter = level->Get< RCP >("Importer"); + if (!xpImporter.is_null()) + xpImporter->setDistributorParameters(matvecParams); + } + } + } + // The function uses three managers: fine, coarse and next coarse // We construct the data for the coarse level, and do requests for the next coarse template diff --git a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp index e5bd4bbd8775..809507023369 100644 --- a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp +++ b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp @@ -168,6 +168,7 @@ namespace MueLu { "" "" "" + "" "" "" "" @@ -501,6 +502,8 @@ namespace MueLu { ("hierarchy label","hierarchy label") + ("matvec params","matvec params") + ("smoother: pre or post","smoother: pre or post") ("smoother: type","smoother: type") diff --git a/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp b/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp index a61a398bff51..0e7ca01917f0 100644 --- a/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp +++ b/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp @@ -150,6 +150,9 @@ namespace Xpetra { //! The target Map used to construct this Export. Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraExportT::getTargetMap"); return toXpetra(export_->TargetMap()); } + //! Set parameters on the underlying object + void setDistributorParameters(const Teuchos::RCP params) const { XPETRA_MONITOR("EpetraExportT::setDistributorParameters"); } + //@} //! @name I/O Methods diff --git a/packages/xpetra/src/Export/Xpetra_Export.hpp b/packages/xpetra/src/Export/Xpetra_Export.hpp index 0539a048edd4..ab157b3c4db9 100644 --- a/packages/xpetra/src/Export/Xpetra_Export.hpp +++ b/packages/xpetra/src/Export/Xpetra_Export.hpp @@ -111,6 +111,9 @@ namespace Xpetra { //! The target Map used to construct this Export. virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0; + //! Set parameters on the underlying object + virtual void setDistributorParameters(const Teuchos::RCP params) const = 0; + //@} //! @name I/O Methods diff --git a/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp b/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp index 9fda71258db2..c81a7a82f0f7 100644 --- a/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp +++ b/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp @@ -154,6 +154,9 @@ class TpetraExport //! The target Map used to construct this Export. Teuchos::RCP> getTargetMap() const; + //! Set distributor parameters. + void setDistributorParameters(const Teuchos::RCP params) const; + //@} diff --git a/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp b/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp index 31b70ee86e59..d754d661277a 100644 --- a/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp +++ b/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp @@ -48,7 +48,7 @@ #include "Xpetra_TpetraExport_decl.hpp" - +#include "Tpetra_Distributor.hpp" namespace Xpetra { @@ -195,6 +195,14 @@ getTargetMap() const return toXpetra(export_->getTargetMap()); } +template +void +TpetraExport:: +setDistributorParameters(const Teuchos::RCP params) const { + XPETRA_MONITOR("TpetraExport::setDistributorParameters"); + export_->getDistributor().setParameterList(params); +} + template void @@ -332,6 +340,9 @@ class TpetraExport //! The target Map used to construct this Export. Teuchos::RCP> getTargetMap() const { return Teuchos::null; } + //! Set parameters on the underlying object + void setDistributorParameters(const Teuchos::RCP params) const { }; + //@} @@ -477,6 +488,8 @@ class TpetraExport //! The target Map used to construct this Export. Teuchos::RCP> getTargetMap() const { return Teuchos::null; } + //! Set parameters on the underlying object + void setDistributorParameters(const Teuchos::RCP params) const { }; //@} diff --git a/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp b/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp index 23d50ba5f380..5ccdf67dca47 100644 --- a/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp +++ b/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp @@ -151,6 +151,8 @@ namespace Xpetra { //! The Target Map used to construct this Import object. Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraImportT::getTargetMap"); return toXpetra(import_->TargetMap()); } + void setDistributorParameters(const Teuchos::RCP params) const { XPETRA_MONITOR("EpetraImportT::setDistributorParameters"); } + //@} //! @name I/O Methods diff --git a/packages/xpetra/src/Import/Xpetra_Import.hpp b/packages/xpetra/src/Import/Xpetra_Import.hpp index 46c368867aa5..32d67c3b76ca 100644 --- a/packages/xpetra/src/Import/Xpetra_Import.hpp +++ b/packages/xpetra/src/Import/Xpetra_Import.hpp @@ -111,6 +111,9 @@ namespace Xpetra { //! The Target Map used to construct this Import object. virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0; + //! Set parameters on the underlying object + virtual void setDistributorParameters(const Teuchos::RCP params) const = 0; + //@} //! @name I/O Methods diff --git a/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp b/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp index 08788cd7fdc5..df7f1ea5a3c3 100644 --- a/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp +++ b/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp @@ -72,14 +72,16 @@ namespace Xpetra { public: //! Constructor specifying the number of non-zeros for all rows. - static RCP > Build(const RCP > &source, const RCP > &target) { + static RCP > Build(const RCP > &source, + const RCP > &target, + const Teuchos::RCP& plist = Teuchos::null) { XPETRA_MONITOR("ImportFactory::Build"); TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, ""); #ifdef HAVE_XPETRA_TPETRA if (source->lib() == UseTpetra) - return rcp( new TpetraImport(source, target)); + return rcp( new TpetraImport(source, target, plist)); #endif XPETRA_FACTORY_ERROR_IF_EPETRA(source->lib()); @@ -106,13 +108,15 @@ namespace Xpetra { public: - static RCP > Build(const RCP > &source, const RCP > &target) { + static RCP > Build(const RCP > &source, + const RCP > &target, + const Teuchos::RCP& plist = Teuchos::null) { XPETRA_MONITOR("ImportFactory::Build"); TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, ""); #ifdef HAVE_XPETRA_TPETRA if (source->lib() == UseTpetra) - return rcp( new TpetraImport(source, target)); + return rcp( new TpetraImport(source, target,plist)); #endif if (source->lib() == UseEpetra) @@ -139,13 +143,15 @@ namespace Xpetra { public: - static RCP > Build(const RCP > &source, const RCP > &target) { + static RCP > Build(const RCP > &source, + const RCP > &target, + const Teuchos::RCP& plist = Teuchos::null) { XPETRA_MONITOR("ImportFactory::Build"); TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, ""); #ifdef HAVE_XPETRA_TPETRA if (source->lib() == UseTpetra) - return rcp( new TpetraImport(source, target)); + return rcp( new TpetraImport(source, target, plist)); #endif if (source->lib() == UseEpetra) diff --git a/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp b/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp index 93b614bfb2f1..dc4cb8fd446c 100644 --- a/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp +++ b/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp @@ -108,6 +108,9 @@ namespace Xpetra { //! List of entries in the target Map to receive from other processes. ArrayView< const LocalOrdinal > getRemoteLIDs() const; + //! Set parameters on distributor + void setDistributorParameters(const Teuchos::RCP params) const; + //! Number of entries that must be sent by the calling process to other processes. size_t getNumExportIDs() const; diff --git a/packages/xpetra/src/Import/Xpetra_TpetraImport_def.hpp b/packages/xpetra/src/Import/Xpetra_TpetraImport_def.hpp index 62a2b0a4f44e..8b4567a4f95e 100644 --- a/packages/xpetra/src/Import/Xpetra_TpetraImport_def.hpp +++ b/packages/xpetra/src/Import/Xpetra_TpetraImport_def.hpp @@ -53,6 +53,7 @@ #include "Xpetra_TpetraMap.hpp" #include "Tpetra_Import.hpp" +#include "Tpetra_Distributor.hpp" namespace Xpetra { @@ -92,6 +93,10 @@ template size_t TpetraImport::getNumRemoteIDs() const { XPETRA_MONITOR("TpetraImport::getNumRemoteIDs"); return import_->getNumRemoteIDs(); } +template +void TpetraImport::setDistributorParameters(const Teuchos::RCP params) const +{ XPETRA_MONITOR("TpetraImport::setDistributorParameters"); import_->getDistributor().setParameterList(params); } + template ArrayView< const LocalOrdinal > TpetraImport::getRemoteLIDs() const { XPETRA_MONITOR("TpetraImport::getRemoteLIDs"); return import_->getRemoteLIDs(); } @@ -206,6 +211,10 @@ RCP< const Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > > TpetraImport > getTargetMap() const { return Teuchos::null; } + //! Set parameters on the underlying object + void setDistributorParameters(const Teuchos::RCP params) const { } + } + //@} //! @name I/O Methods @@ -306,6 +315,9 @@ RCP< const Tpetra::Import< LocalOrdinal, GlobalOrdinal, Node > > TpetraImport > getTargetMap() const { return Teuchos::null; } + //! Set parameters on the underlying object + void setDistributorParameters(const Teuchos::RCP params) const { } + //@} //! @name I/O Methods