MueLu: Allow setting distributor parameters at end of setup

I particular this allows to set the type of MPI communication that is used in the solve phase.
trilinos · Apr 1, 2019 · 7ea9b99 · 7ea9b99
1 parent 0621e89
commit 7ea9b99
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 1 deletion.
diff --git a/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp b/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp
@@ -824,6 +824,57 @@ namespace MueLu {
         Xpetra::IO<SC, LO, GlobalOrdinal, Node>::Write(std::string("A22.mat"), *A22_);
     }
 
+    if (parameterList_.isSublist("matvec params"))
+    {
+      RCP<ParameterList> matvecParams = rcpFromRef(parameterList_.sublist("matvec params"));
+
+      {
+        RCP<const Import> xpImporter = SM_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = SM_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_T_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_T_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = R11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = R11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = P11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = P11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (!ImporterH_.is_null())
+        ImporterH_->setDistributorParameters(matvecParams);
+      if (!Importer22_.is_null())
+        Importer22_->setDistributorParameters(matvecParams);
+    }
+
+
     VerboseObject::SetDefaultVerbLevel(oldVerbLevel);
   }
 

diff --git a/packages/muelu/doc/UsersGuide/masterList.xml b/packages/muelu/doc/UsersGuide/masterList.xml
@@ -109,6 +109,14 @@
        <visible>true</visible>
      </parameter>
 
+     <parameter>
+      <name>matvec params</name>
+      <type>\parameterlist</type>
+      <description>Pass parameters to the underlying linear algebra</description>
+      <comment-ML>not supported by ML</comment-ML>
+      <visible>false</visible>
+    </parameter>
+
   </general>
 
   <smoothing_and_coarse>
@@ -1044,7 +1052,7 @@
       <description>Level on which to run node-level repartitioning (negative means don't).</description>
       <visible>false</visible>
     </parameter>
-    
+
     <parameter>
       <name>repartition: node id</name>
       <type>int</type>

diff --git a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex
@@ -22,6 +22,8 @@
 
 \cbb{hierarchy label}{string}{""}{Label for the hierarchy. Is applied to timer labels.}
 
+\cba{matvec params}{\parameterlist}{Pass parameters to the underlying linear algebra}
+
 \cbb{smoother: pre or post}{string}{"both"}{Pre- and post-smoother combination. Possible values: "pre" (only pre-smoother), "post" (only post-smoother), "both" (both pre-and post-smoothers), "none" (no smoothing).}
 
 \cbb{smoother: type}{string}{"RELAXATION"}{Smoother type. Possible values: see Table~\ref{tab:smoothers}.}

diff --git a/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp b/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp
@@ -241,6 +241,8 @@ namespace MueLu {
         isLastLevel = r || (levelID == lastLevelID);
         levelID++;
       }
+      if (!matvecParams_.is_null())
+        H.SetMatvecParams(matvecParams_);
       // FIXME: Should allow specification of NumVectors on parameterlist
       H.AllocateLevelMultiVectors(1);
 
@@ -316,6 +318,7 @@ namespace MueLu {
     Teuchos::Array<int>   nullspaceToPrint_;
     Teuchos::Array<int>   coordinatesToPrint_;
     Teuchos::Array<int>   elementToNodeMapsToPrint_;
+    Teuchos::RCP<Teuchos::ParameterList> matvecParams_;
 
     std::map<int, std::vector<keep_pair> > keep_;
 

diff --git a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp
@@ -414,6 +414,9 @@ namespace MueLu {
     // Detect if we use implicit transpose
     changedImplicitTranspose_ = MUELU_TEST_AND_SET_VAR(paramList, "transpose: use implicit", bool, this->implicitTranspose_);
 
+    if (paramList.isSublist("matvec params"))
+      this->matvecParams_ = Teuchos::parameterList(paramList.sublist("matvec params"));
+
     // Create default manager
     // FIXME: should it be here, or higher up
     RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
@@ -1837,6 +1840,10 @@ namespace MueLu {
         hieraList.remove("transpose: use implicit");
       }
 
+      if (hieraList.isSublist("matvec params"))
+        this->matvecParams_ = Teuchos::parameterList(hieraList.sublist("matvec params"));
+
+
       if (hieraList.isParameter("coarse grid correction scaling factor")) {
         this->scalingFactor_ = hieraList.get<double>("coarse grid correction scaling factor");
         hieraList.remove("coarse grid correction scaling factor");

diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp
@@ -193,6 +193,8 @@ namespace MueLu {
     //! Helper function
     void CheckLevel(Level& level, int levelID);
 
+    void SetMatvecParams(RCP<ParameterList> matvecParams);
+
     //! Multi-level setup phase: build a new level of the hierarchy.
     /*!  This method is aimed to be used in a loop building the hierarchy level by level. See Hierarchy::Setup(manager, startLevel, numDesiredLevels) for an example of usage.
      *

diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp
@@ -240,6 +240,48 @@ namespace MueLu {
                                "MueLu::Hierarchy::Setup(): wrong level parent");
   }
 
+  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
+  void Hierarchy<Scalar, LocalOrdinal, GlobalOrdinal, Node>::SetMatvecParams(RCP<ParameterList> matvecParams) {
+    for (int i = 0; i < GetNumLevels(); ++i) {
+      RCP<Level> level = Levels_[i];
+      if (level->IsAvailable("A")) {
+        RCP<Operator> Aop = level->Get<RCP<Operator> >("A");
+        RCP<Matrix> A = rcp_dynamic_cast<Matrix>(Aop);
+        if (!A.is_null()) {
+          RCP<const Import> xpImporter = A->getCrsGraph()->getImporter();
+          if (!xpImporter.is_null())
+            xpImporter->setDistributorParameters(matvecParams);
+          RCP<const Export> xpExporter = A->getCrsGraph()->getExporter();
+          if (!xpExporter.is_null())
+            xpExporter->setDistributorParameters(matvecParams);
+        }
+      }
+      if (level->IsAvailable("P")) {
+        RCP<Matrix> P = level->Get<RCP<Matrix> >("P");
+        RCP<const Import> xpImporter = P->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = P->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (level->IsAvailable("R")) {
+        RCP<Matrix> R = level->Get<RCP<Matrix> >("R");
+        RCP<const Import> xpImporter = R->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = R->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (level->IsAvailable("Importer")) {
+        RCP<const Import> xpImporter = level->Get< RCP<const Import> >("Importer");
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+      }
+    }
+  }
+
   // The function uses three managers: fine, coarse and next coarse
   // We construct the data for the coarse level, and do requests for the next coarse
   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>

diff --git a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp
@@ -160,6 +160,7 @@ namespace MueLu {
   "<Parameter name=\"xml parameter file\" type=\"string\" value=\"\"/>"
   "<Parameter name=\"parameterlist: syntax\" type=\"string\" value=\"muelu\"/>"
   "<Parameter name=\"hierarchy label\" type=\"string\" value=\"\"/>"
+  "<ParameterList name=\"matvec params\"/>"
   "<Parameter name=\"smoother: pre or post\" type=\"string\" value=\"both\"/>"
   "<Parameter name=\"smoother: type\" type=\"string\" value=\"RELAXATION\"/>"
   "<Parameter name=\"smoother: pre type\" type=\"string\" value=\"RELAXATION\"/>"
@@ -481,6 +482,8 @@ namespace MueLu {
 
          ("hierarchy label","hierarchy label")
 
+         ("matvec params","matvec params")
+
          ("smoother: pre or post","smoother: pre or post")
 
          ("smoother: type","smoother: type")