Merge Pull Request #4781 from cgcgcg/Trilinos/mueluTpetraMatvec

Automatically Merged using Trilinos Pull Request AutoTester PR Title: Muelu: Allow to change distributor options PR Author: cgcgcg
trilinos · Aug 20, 2019 · 3e201d5 · 3e201d5
2 parents 0787d69 + c84839f
commit 3e201d5
Show file tree

Hide file tree

Showing 17 changed files with 182 additions and 7 deletions.
diff --git a/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp b/packages/muelu/adapters/xpetra/MueLu_RefMaxwell_def.hpp
@@ -927,6 +927,15 @@ namespace MueLu {
       D0x_      = MultiVectorFactory::Build(D0_Matrix_->getDomainMap(), 1);
     residual_  = MultiVectorFactory::Build(SM_Matrix_->getDomainMap(), 1);
 
+    if (!ImporterH_.is_null() && parameterList_.isSublist("refmaxwell: ImporterH params")){
+      RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: ImporterH params"));
+      ImporterH_->setDistributorParameters(importerParams);
+    }
+    if (!Importer22_.is_null() && parameterList_.isSublist("refmaxwell: Importer22 params")){
+      RCP<ParameterList> importerParams = rcpFromRef(parameterList_.sublist("refmaxwell: Importer22 params"));
+      Importer22_->setDistributorParameters(importerParams);
+    }
+
 #ifdef HAVE_MUELU_CUDA
     if (parameterList_.get<bool>("refmaxwell: cuda profile setup", false)) cudaProfilerStop();
 #endif
@@ -976,6 +985,57 @@ namespace MueLu {
         Xpetra::IO<SC, LO, GlobalOrdinal, Node>::Write(std::string("A22.mat"), *A22_);
     }
 
+    if (parameterList_.isSublist("matvec params"))
+    {
+      RCP<ParameterList> matvecParams = rcpFromRef(parameterList_.sublist("matvec params"));
+
+      {
+        RCP<const Import> xpImporter = SM_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = SM_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = D0_T_Matrix_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = D0_T_Matrix_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = R11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = R11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      {
+        RCP<const Import> xpImporter = P11_->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = P11_->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (!ImporterH_.is_null())
+        ImporterH_->setDistributorParameters(matvecParams);
+      if (!Importer22_.is_null())
+        Importer22_->setDistributorParameters(matvecParams);
+    }
+
+
     VerboseObject::SetDefaultVerbLevel(oldVerbLevel);
   }
 

diff --git a/packages/muelu/doc/UsersGuide/masterList.xml b/packages/muelu/doc/UsersGuide/masterList.xml
@@ -117,6 +117,14 @@
        <visible>true</visible>
      </parameter>
 
+     <parameter>
+      <name>matvec params</name>
+      <type>\parameterlist</type>
+      <description>Pass parameters to the underlying linear algebra</description>
+      <comment-ML>not supported by ML</comment-ML>
+      <visible>false</visible>
+    </parameter>
+
   </general>
 
   <smoothing_and_coarse>

diff --git a/packages/muelu/doc/UsersGuide/paramlist_hidden.tex b/packages/muelu/doc/UsersGuide/paramlist_hidden.tex
@@ -24,6 +24,8 @@
 
 \cbb{hierarchy label}{string}{""}{Label for the hierarchy. Is applied to timer labels.}
 
+\cba{matvec params}{\parameterlist}{Pass parameters to the underlying linear algebra}
+
 \cbb{smoother: pre or post}{string}{"both"}{Pre- and post-smoother combination. Possible values: "pre" (only pre-smoother), "post" (only post-smoother), "both" (both pre-and post-smoothers), "none" (no smoothing).}
 
 \cbb{smoother: type}{string}{"RELAXATION"}{Smoother type. Possible values: see Table~\ref{tab:smoothers}.}

diff --git a/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp b/packages/muelu/src/Interface/MueLu_HierarchyManager.hpp
@@ -246,6 +246,8 @@ namespace MueLu {
         isLastLevel = r || (levelID == lastLevelID);
         levelID++;
       }
+      if (!matvecParams_.is_null())
+        H.SetMatvecParams(matvecParams_);
       // FIXME: Should allow specification of NumVectors on parameterlist
       H.AllocateLevelMultiVectors(1);
       H.describe(H.GetOStream(Runtime0), verbosity_);
@@ -321,6 +323,7 @@ namespace MueLu {
     Teuchos::Array<int>   nullspaceToPrint_;
     Teuchos::Array<int>   coordinatesToPrint_;
     Teuchos::Array<int>   elementToNodeMapsToPrint_;
+    Teuchos::RCP<Teuchos::ParameterList> matvecParams_;
 
     std::map<int, std::vector<keep_pair> > keep_;
 

diff --git a/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp b/packages/muelu/src/Interface/MueLu_ParameterListInterpreter_def.hpp
@@ -413,6 +413,9 @@ namespace MueLu {
     // Detect if we use implicit transpose
     changedImplicitTranspose_ = MUELU_TEST_AND_SET_VAR(paramList, "transpose: use implicit", bool, this->implicitTranspose_);
 
+    if (paramList.isSublist("matvec params"))
+      this->matvecParams_ = Teuchos::parameterList(paramList.sublist("matvec params"));
+
     // Create default manager
     // FIXME: should it be here, or higher up
     RCP<FactoryManager> defaultManager = rcp(new FactoryManager());
@@ -1849,6 +1852,10 @@ namespace MueLu {
         hieraList.remove("transpose: use implicit");
       }
 
+      if (hieraList.isSublist("matvec params"))
+        this->matvecParams_ = Teuchos::parameterList(hieraList.sublist("matvec params"));
+
+
       if (hieraList.isParameter("coarse grid correction scaling factor")) {
         this->scalingFactor_ = hieraList.get<double>("coarse grid correction scaling factor");
         hieraList.remove("coarse grid correction scaling factor");

diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_decl.hpp
@@ -196,6 +196,8 @@ namespace MueLu {
     //! Helper function
     void CheckLevel(Level& level, int levelID);
 
+    void SetMatvecParams(RCP<ParameterList> matvecParams);
+
     //! Multi-level setup phase: build a new level of the hierarchy.
     /*!  This method is aimed to be used in a loop building the hierarchy level by level. See Hierarchy::Setup(manager, startLevel, numDesiredLevels) for an example of usage.
      *

diff --git a/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp b/packages/muelu/src/MueCentral/MueLu_Hierarchy_def.hpp
@@ -242,6 +242,48 @@ namespace MueLu {
                                "MueLu::Hierarchy::Setup(): wrong level parent");
   }
 
+  template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
+  void Hierarchy<Scalar, LocalOrdinal, GlobalOrdinal, Node>::SetMatvecParams(RCP<ParameterList> matvecParams) {
+    for (int i = 0; i < GetNumLevels(); ++i) {
+      RCP<Level> level = Levels_[i];
+      if (level->IsAvailable("A")) {
+        RCP<Operator> Aop = level->Get<RCP<Operator> >("A");
+        RCP<Matrix> A = rcp_dynamic_cast<Matrix>(Aop);
+        if (!A.is_null()) {
+          RCP<const Import> xpImporter = A->getCrsGraph()->getImporter();
+          if (!xpImporter.is_null())
+            xpImporter->setDistributorParameters(matvecParams);
+          RCP<const Export> xpExporter = A->getCrsGraph()->getExporter();
+          if (!xpExporter.is_null())
+            xpExporter->setDistributorParameters(matvecParams);
+        }
+      }
+      if (level->IsAvailable("P")) {
+        RCP<Matrix> P = level->Get<RCP<Matrix> >("P");
+        RCP<const Import> xpImporter = P->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = P->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (level->IsAvailable("R")) {
+        RCP<Matrix> R = level->Get<RCP<Matrix> >("R");
+        RCP<const Import> xpImporter = R->getCrsGraph()->getImporter();
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+        RCP<const Export> xpExporter = R->getCrsGraph()->getExporter();
+        if (!xpExporter.is_null())
+          xpExporter->setDistributorParameters(matvecParams);
+      }
+      if (level->IsAvailable("Importer")) {
+        RCP<const Import> xpImporter = level->Get< RCP<const Import> >("Importer");
+        if (!xpImporter.is_null())
+          xpImporter->setDistributorParameters(matvecParams);
+      }
+    }
+  }
+
   // The function uses three managers: fine, coarse and next coarse
   // We construct the data for the coarse level, and do requests for the next coarse
   template <class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>

diff --git a/packages/muelu/src/MueCentral/MueLu_MasterList.cpp b/packages/muelu/src/MueCentral/MueLu_MasterList.cpp
@@ -168,6 +168,7 @@ namespace MueLu {
   "<Parameter name=\"xml parameter file\" type=\"string\" value=\"\"/>"
   "<Parameter name=\"parameterlist: syntax\" type=\"string\" value=\"muelu\"/>"
   "<Parameter name=\"hierarchy label\" type=\"string\" value=\"\"/>"
+  "<ParameterList name=\"matvec params\"/>"
   "<Parameter name=\"smoother: pre or post\" type=\"string\" value=\"both\"/>"
   "<Parameter name=\"smoother: type\" type=\"string\" value=\"RELAXATION\"/>"
   "<Parameter name=\"smoother: pre type\" type=\"string\" value=\"RELAXATION\"/>"
@@ -501,6 +502,8 @@ namespace MueLu {
 
          ("hierarchy label","hierarchy label")
 
+         ("matvec params","matvec params")
+
          ("smoother: pre or post","smoother: pre or post")
 
          ("smoother: type","smoother: type")

diff --git a/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp b/packages/xpetra/src/Export/Xpetra_EpetraExport.hpp
@@ -150,6 +150,9 @@ namespace Xpetra {
     //! The target Map used to construct this Export.
     Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraExportT::getTargetMap"); return toXpetra<GlobalOrdinal, Node>(export_->TargetMap()); }
 
+    //! Set parameters on the underlying object
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { XPETRA_MONITOR("EpetraExportT::setDistributorParameters"); }
+
     //@}
 
     //! @name I/O Methods

diff --git a/packages/xpetra/src/Export/Xpetra_Export.hpp b/packages/xpetra/src/Export/Xpetra_Export.hpp
@@ -111,6 +111,9 @@ namespace Xpetra {
     //! The target Map used to construct this Export.
     virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0;
 
+    //! Set parameters on the underlying object
+    virtual void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const = 0;
+
     //@}
 
     //! @name I/O Methods

diff --git a/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp b/packages/xpetra/src/Export/Xpetra_TpetraExport_decl.hpp
@@ -154,6 +154,9 @@ class TpetraExport
     //! The target Map used to construct this Export.
     Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const;
 
+    //! Set distributor parameters.
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const;
+
 
     //@}
 

diff --git a/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp b/packages/xpetra/src/Export/Xpetra_TpetraExport_def.hpp
@@ -48,7 +48,7 @@
 
 
 #include "Xpetra_TpetraExport_decl.hpp"
-
+#include "Tpetra_Distributor.hpp"
 
 namespace Xpetra {
 
@@ -195,6 +195,14 @@ getTargetMap() const
     return toXpetra(export_->getTargetMap());
 }
 
+template<class LocalOrdinal, class GlobalOrdinal, class Node>
+void
+TpetraExport<LocalOrdinal, GlobalOrdinal, Node>::
+setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const {
+  XPETRA_MONITOR("TpetraExport::setDistributorParameters");
+  export_->getDistributor().setParameterList(params);
+}
+
 
 template<class LocalOrdinal, class GlobalOrdinal, class Node>
 void
@@ -332,6 +340,9 @@ class TpetraExport<int, int, EpetraNode>
     //! The target Map used to construct this Export.
     Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const { return Teuchos::null; }
 
+    //! Set parameters on the underlying object
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { };
+
 
     //@}
 
@@ -477,6 +488,8 @@ class TpetraExport<int, long long, EpetraNode>
     //! The target Map used to construct this Export.
     Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node>> getTargetMap() const { return Teuchos::null; }
 
+    //! Set parameters on the underlying object
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { };
 
     //@}
 

diff --git a/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp b/packages/xpetra/src/Import/Xpetra_EpetraImport.hpp
@@ -151,6 +151,8 @@ namespace Xpetra {
     //! The Target Map used to construct this Import object.
     Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const { XPETRA_MONITOR("EpetraImportT::getTargetMap"); return toXpetra<GlobalOrdinal, Node>(import_->TargetMap()); }
 
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const { XPETRA_MONITOR("EpetraImportT::setDistributorParameters");  }
+
     //@}
 
     //! @name I/O Methods

diff --git a/packages/xpetra/src/Import/Xpetra_Import.hpp b/packages/xpetra/src/Import/Xpetra_Import.hpp
@@ -111,6 +111,9 @@ namespace Xpetra {
     //! The Target Map used to construct this Import object.
     virtual Teuchos::RCP< const Map< LocalOrdinal, GlobalOrdinal, Node > > getTargetMap() const = 0;
 
+    //! Set parameters on the underlying object
+    virtual void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const = 0;
+
     //@}
 
     //! @name I/O Methods

diff --git a/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp b/packages/xpetra/src/Import/Xpetra_ImportFactory.hpp
@@ -72,14 +72,16 @@ namespace Xpetra {
   public:
 
     //! Constructor specifying the number of non-zeros for all rows.
-    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
+    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
+                                                                 const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
+                                                                 const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
       XPETRA_MONITOR("ImportFactory::Build");
 
       TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");
 
 #ifdef HAVE_XPETRA_TPETRA
       if (source->lib() == UseTpetra)
-        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
+        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target, plist));
 #endif
 
       XPETRA_FACTORY_ERROR_IF_EPETRA(source->lib());
@@ -106,13 +108,15 @@ namespace Xpetra {
 
   public:
 
-    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
+    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
+                                                                 const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
+                                                                 const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
       XPETRA_MONITOR("ImportFactory::Build");
       TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");
 
 #ifdef HAVE_XPETRA_TPETRA
       if (source->lib() == UseTpetra)
-        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
+        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target,plist));
 #endif
 
       if (source->lib() == UseEpetra)
@@ -139,13 +143,15 @@ namespace Xpetra {
 
   public:
 
-    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source, const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target) {
+    static RCP<Import<LocalOrdinal, GlobalOrdinal, Node> > Build(const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &source,
+                                                                 const RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> > &target,
+                                                                 const Teuchos::RCP<Teuchos::ParameterList>& plist = Teuchos::null) {
       XPETRA_MONITOR("ImportFactory::Build");
       TEUCHOS_TEST_FOR_EXCEPTION(source->lib() != target->lib(), Xpetra::Exceptions::RuntimeError, "");
 
 #ifdef HAVE_XPETRA_TPETRA
       if (source->lib() == UseTpetra)
-        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target));
+        return rcp( new TpetraImport<LocalOrdinal, GlobalOrdinal, Node>(source, target, plist));
 #endif
 
       if (source->lib() == UseEpetra)

diff --git a/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp b/packages/xpetra/src/Import/Xpetra_TpetraImport_decl.hpp
@@ -108,6 +108,9 @@ namespace Xpetra {
     //! List of entries in the target Map to receive from other processes.
     ArrayView< const LocalOrdinal > getRemoteLIDs() const;
 
+    //! Set parameters on distributor
+    void setDistributorParameters(const Teuchos::RCP<Teuchos::ParameterList> params) const;
+
     //! Number of entries that must be sent by the calling process to other processes.
     size_t getNumExportIDs() const;