Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Muelu: Allow to change distributor options #4781

Merged
merged 4 commits into from
Aug 20, 2019
Merged

Muelu: Allow to change distributor options #4781

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1f4a7d5
Xpetra::ImportFactory: Add optional parameter list
cgcgcg Mar 15, 2019
1cd9b10
Xpetra: Allow setting parameters on Distributor of Import/Export
cgcgcg Mar 20, 2019
a2c23dd
MueLu: Allow setting distributor parameters at end of setup
cgcgcg Mar 20, 2019
c84839f
MueLu RefMaxwell: Allow setting importer parameters
cgcgcg Aug 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -927,6 +927,15 @@ namespace MueLu {
D0x_ = MultiVectorFactory::Build(D0_Matrix_->getDomainMap(), 1);
residual_ = MultiVectorFactory::Build(SM_Matrix_->getDomainMap(), 1);

if (!ImporterH_.is_null() && parameterList_.isSublist("refmaxwell: ImporterH params")){
RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: ImporterH params"));
ImporterH_->setDistributorParameters(importerParams);
}
if (!Importer22_.is_null() && parameterList_.isSublist("refmaxwell: Importer22 params")){
RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: Importer22 params"));
Importer22_->setDistributorParameters(importerParams);
}

#ifdef HAVE_MUELU_CUDA
if (parameterList_.get<bool>("refmaxwell: cuda profile setup", false)) cudaProfilerStop();
#endif
Expand Down Expand Up @@ -976,6 +985,57 @@ namespace MueLu {
Xpetra::IO<SC, LO, GlobalOrdinal, Node>::Write(std::string("A22.mat"), *A22_);
}

if (parameterList_.isSublist("matvec params"))
{
RCP<ParameterList> matvecParams = rcpFromRef(parameterList_.sublist("matvec params"));

{
RCP<const Import> xpImporter = SM_Matrix_->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = SM_Matrix_->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
{
RCP<const Import> xpImporter = D0_Matrix_->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = D0_Matrix_->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
{
RCP<const Import> xpImporter = D0_T_Matrix_->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = D0_T_Matrix_->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
{
RCP<const Import> xpImporter = R11_->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = R11_->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
{
RCP<const Import> xpImporter = P11_->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = P11_->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
if (!ImporterH_.is_null())
ImporterH_->setDistributorParameters(matvecParams);
if (!Importer22_.is_null())
Importer22_->setDistributorParameters(matvecParams);
}


VerboseObject::SetDefaultVerbLevel(oldVerbLevel);
}

Expand Down
8 changes: 8 additions & 0 deletions packages/muelu/doc/UsersGuide/masterList.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,14 @@
<visible>true</visible>
</parameter>

<parameter>
<name>matvec params</name>
<type>\parameterlist</type>
<description>Pass parameters to the underlying linear algebra</description>
<comment-ML>not supported by ML</comment-ML>
<visible>false</visible>
</parameter>

</general>

<smoothing_and_coarse>
Expand Down
2 changes: 2 additions & 0 deletions packages/muelu/doc/UsersGuide/paramlist_hidden.tex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

\cbb{hierarchy label}{string}{""}{Label for the hierarchy. Is applied to timer labels.}

\cba{matvec params}{\parameterlist}{Pass parameters to the underlying linear algebra}

\cbb{smoother: pre or post}{string}{"both"}{Pre- and post-smoother combination. Possible values: "pre" (only pre-smoother), "post" (only post-smoother), "both" (both pre-and post-smoothers), "none" (no smoothing).}

\cbb{smoother: type}{string}{"RELAXATION"}{Smoother type. Possible values: see Table~\ref{tab:smoothers}.}
Expand Down
3 changes: 3 additions & 0 deletions packages/muelu/src/Interface/MueLu_HierarchyManager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,8 @@ namespace MueLu {
isLastLevel = r || (levelID == lastLevelID);
levelID++;
}
if (!matvecParams_.is_null())
H.SetMatvecParams(matvecParams_);
// FIXME: Should allow specification of NumVectors on parameterlist
H.AllocateLevelMultiVectors(1);
H.describe(H.GetOStream(Runtime0), verbosity_);
Expand Down Expand Up @@ -321,6 +323,7 @@ namespace MueLu {
Teuchos::Array<int> nullspaceToPrint_;
Teuchos::Array<int> coordinatesToPrint_;
Teuchos::Array<int> elementToNodeMapsToPrint_;
Teuchos::RCP<Teuchos::ParameterList> matvecParams_;

std::map<int, std::vector<keep_pair> > keep_;

Expand Down
7 changes: 7 additions & 0 deletions packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,9 @@ namespace MueLu {
// Detect if we use implicit transpose
changedImplicitTranspose_ = MUELU_TEST_AND_SET_VAR(paramList, "transpose: use implicit", bool, this->implicitTranspose_);

if (paramList.isSublist("matvec params"))
this->matvecParams_ = Teuchos::parameterList(paramList.sublist("matvec params"));

// Create default manager
// FIXME: should it be here, or higher up
RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
Expand Down Expand Up @@ -1849,6 +1852,10 @@ namespace MueLu {
hieraList.remove("transpose: use implicit");
}

if (hieraList.isSublist("matvec params"))
this->matvecParams_ = Teuchos::parameterList(hieraList.sublist("matvec params"));


if (hieraList.isParameter("coarse grid correction scaling factor")) {
this->scalingFactor_ = hieraList.get<double>("coarse grid correction scaling factor");
hieraList.remove("coarse grid correction scaling factor");
Expand Down
2 changes: 2 additions & 0 deletions packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ namespace MueLu {
//! Helper function
void CheckLevel(Level& level, int levelID);

void SetMatvecParams(RCP<ParameterList> matvecParams);

//! Multi-level setup phase: build a new level of the hierarchy.
/*! This method is aimed to be used in a loop building the hierarchy level by level. See Hierarchy::Setup(manager, startLevel, numDesiredLevels) for an example of usage.
*
Expand Down
42 changes: 42 additions & 0 deletions packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,48 @@ namespace MueLu {
"MueLu::Hierarchy::Setup(): wrong level parent");
}

template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
void Hierarchy<Scalar, LocalOrdinal, GlobalOrdinal, Node>::SetMatvecParams(RCP<ParameterList> matvecParams) {
for (int i = 0; i < GetNumLevels(); ++i) {
RCP<Level> level = Levels_[i];
if (level->IsAvailable("A")) {
RCP<Operator> Aop = level->Get<RCP<Operator> >("A");
RCP<Matrix> A = rcp_dynamic_cast<Matrix>(Aop);
if (!A.is_null()) {
RCP<const Import> xpImporter = A->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = A->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
}
if (level->IsAvailable("P")) {
RCP<Matrix> P = level->Get<RCP<Matrix> >("P");
RCP<const Import> xpImporter = P->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = P->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
if (level->IsAvailable("R")) {
RCP<Matrix> R = level->Get<RCP<Matrix> >("R");
RCP<const Import> xpImporter = R->getCrsGraph()->getImporter();
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
RCP<const Export> xpExporter = R->getCrsGraph()->getExporter();
if (!xpExporter.is_null())
xpExporter->setDistributorParameters(matvecParams);
}
if (level->IsAvailable("Importer")) {
RCP<const Import> xpImporter = level->Get< RCP<const Import> >("Importer");
if (!xpImporter.is_null())
xpImporter->setDistributorParameters(matvecParams);
}
}
}

// The function uses three managers: fine, coarse and next coarse
// We construct the data for the coarse level, and do requests for the next coarse
template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
Expand Down
3 changes: 3 additions & 0 deletions packages/muelu/src/MueCentral/MueLu_MasterList.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ namespace MueLu {
"<Parameter name=\"xml parameter file\" type=\"string\" value=\"\"/>"
"<Parameter name=\"parameterlist: syntax\" type=\"string\" value=\"muelu\"/>"
"<Parameter name=\"hierarchy label\" type=\"string\" value=\"\"/>"
"<ParameterList name=\"matvec params\"/>"
"<Parameter name=\"smoother: pre or post\" type=\"string\" value=\"both\"/>"
"<Parameter name=\"smoother: type\" type=\"string\" value=\"RELAXATION\"/>"
"<Parameter name=\"smoother: pre type\" type=\"string\" value=\"RELAXATION\"/>"
Expand Down Expand Up @@ -501,6 +502,8 @@ namespace MueLu {

("hierarchy label","hierarchy label")

("matvec params","matvec params")

("smoother: pre or post","smoother: pre or post")

("smoother: type","smoother: type")
Expand Down
3 changes: 3 additions & 0 deletions packages/xpetra/src/Export/Xpetra_EpetraExport.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ namespace Xpetra {
//! The target Map used to construct this Export.
Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraExportT::getTargetMap"); return toXpetra<GlobalOrdinal, Node>(export_->TargetMap()); }

//! Set parameters on the underlying object
void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { XPETRA_MONITOR("EpetraExportT::setDistributorParameters"); }

//@}

//! @name I/O Methods
Expand Down
3 changes: 3 additions & 0 deletions packages/xpetra/src/Export/Xpetra_Export.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ namespace Xpetra {
//! The target Map used to construct this Export.
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0;

//! Set parameters on the underlying object
virtual void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const = 0;

//@}

//! @name I/O Methods
Expand Down
3 changes: 3 additions & 0 deletions packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ class TpetraExport
//! The target Map used to construct this Export.
Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const;

//! Set distributor parameters.
void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const;


//@}

Expand Down
15 changes: 14 additions & 1 deletion packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@


#include "Xpetra_TpetraExport_decl.hpp"

#include "Tpetra_Distributor.hpp"

namespace Xpetra {

Expand Down Expand Up @@ -195,6 +195,14 @@ getTargetMap() const
return toXpetra(export_->getTargetMap());
}

template<class LocalOrdinal, class GlobalOrdinal, class Node>
void
TpetraExport<LocalOrdinal, GlobalOrdinal, Node>::
setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const {
XPETRA_MONITOR("TpetraExport::setDistributorParameters");
export_->getDistributor().setParameterList(params);
}


template<class LocalOrdinal, class GlobalOrdinal, class Node>
void
Expand Down Expand Up @@ -332,6 +340,9 @@ class TpetraExport<int, int, EpetraNode>
//! The target Map used to construct this Export.
Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const { return Teuchos::null; }

//! Set parameters on the underlying object
void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { };


//@}

Expand Down Expand Up @@ -477,6 +488,8 @@ class TpetraExport<int, long long, EpetraNode>
//! The target Map used to construct this Export.
Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const { return Teuchos::null; }

//! Set parameters on the underlying object
void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { };

//@}

Expand Down
2 changes: 2 additions & 0 deletions packages/xpetra/src/Import/Xpetra_EpetraImport.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ namespace Xpetra {
//! The Target Map used to construct this Import object.
Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraImportT::getTargetMap"); return toXpetra<GlobalOrdinal, Node>(import_->TargetMap()); }

void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { XPETRA_MONITOR("EpetraImportT::setDistributorParameters"); }

//@}

//! @name I/O Methods
Expand Down
3 changes: 3 additions & 0 deletions packages/xpetra/src/Import/Xpetra_Import.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ namespace Xpetra {
//! The Target Map used to construct this Import object.
virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0;

//! Set parameters on the underlying object
virtual void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const = 0;

//@}

//! @name I/O Methods
Expand Down
18 changes: 12 additions & 6 deletions packages/xpetra/src/Import/Xpetra_ImportFactory.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,16 @@ namespace Xpetra {
public:

//! Constructor specifying the number of non-zeros for all rows.
static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
XPETRA_MONITOR("ImportFactory::Build");

TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");

#ifdef HAVE_XPETRA_TPETRA
if (source->lib() == UseTpetra)
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target, plist));
#endif

XPETRA_FACTORY_ERROR_IF_EPETRA(source->lib());
Expand All @@ -106,13 +108,15 @@ namespace Xpetra {

public:

static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
XPETRA_MONITOR("ImportFactory::Build");
TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");

#ifdef HAVE_XPETRA_TPETRA
if (source->lib() == UseTpetra)
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target,plist));
#endif

if (source->lib() == UseEpetra)
Expand All @@ -139,13 +143,15 @@ namespace Xpetra {

public:

static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
XPETRA_MONITOR("ImportFactory::Build");
TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");

#ifdef HAVE_XPETRA_TPETRA
if (source->lib() == UseTpetra)
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target, plist));
#endif

if (source->lib() == UseEpetra)
Expand Down
3 changes: 3 additions & 0 deletions packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ namespace Xpetra {
//! List of entries in the target Map to receive from other processes.
ArrayView< const LocalOrdinal > getRemoteLIDs() const;

//! Set parameters on distributor
void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const;

//! Number of entries that must be sent by the calling process to other processes.
size_t getNumExportIDs() const;

Expand Down
Loading