Merge Pull Request #12196 from jgfouca/Trilinos/jgfouca/measure_fastilu

Automatically Merged using Trilinos Pull Request AutoTester PR Title: b'Shylu: FastILU fixes to handle large matrices' PR Author: jgfouca
trilinos · Sep 8, 2023 · 3268124 · 3268124
2 parents 7057688 + 9dfa45f
commit 3268124
Show file tree

Hide file tree

Showing 6 changed files with 264 additions and 73 deletions.
diff --git a/packages/ifpack2/src/Ifpack2_Details_FastILU_Base_decl.hpp b/packages/ifpack2/src/Ifpack2_Details_FastILU_Base_decl.hpp
@@ -228,6 +228,7 @@ template<class Scalar, class LocalOrdinal, class GlobalOrdinal, class Node>
       int blockSize;
       bool blockCrs;
       int blockCrsSize;
+      bool fillBlocks;
       static Params getDefaults();
     };
 

diff --git a/packages/ifpack2/src/Ifpack2_Details_FastILU_Base_def.hpp b/packages/ifpack2/src/Ifpack2_Details_FastILU_Base_def.hpp
@@ -179,24 +179,27 @@ initialize()
     throw std::runtime_error(std::string("Called ") + getName() + "::initialize() but matrix was null (call setMatrix() with a non-null matrix first)");
   }
 
-  Kokkos::Timer copyTimer;
-  CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getStructure(mat_.get(), localRowPtrsHost_, localRowPtrs_, localColInds_);
-  CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getValues(mat_.get(), localValues_, localRowPtrsHost_);
-  crsCopyTime_ = copyTimer.seconds();
-
   if (isBlockCrs()) {
-    // Create new TCrsMatrix with the new filled data
     auto crs_matrix = Ifpack2::Details::getCrsMatrix(this->mat_);
-    auto crs_matrix_block_filled = Tpetra::fillLogicalBlocks(*crs_matrix, params_.blockCrsSize);
 
-    // Convert to block crs matrix
-    auto bcrs_matrix = Tpetra::convertToBlockCrsMatrix(*crs_matrix_block_filled, params_.blockCrsSize);
-    mat_ = bcrs_matrix;
-
-    CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getStructure(mat_.get(), localRowPtrsHost_, localRowPtrs_, localColInds_);
-    CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getValues(mat_.get(), localValues_, localRowPtrsHost_);
+    if (params_.fillBlocks) {
+      // Create new TCrsMatrix with the new filled data and conver to Bcrs
+      auto crs_matrix_block_filled = Tpetra::fillLogicalBlocks(*crs_matrix, params_.blockCrsSize);
+      auto bcrs_matrix = Tpetra::convertToBlockCrsMatrix(*crs_matrix_block_filled, params_.blockCrsSize);
+      mat_ = bcrs_matrix;
+    }
+    else {
+      // Assume input is already filled, just convert to Bcrs
+      auto bcrs_matrix = Tpetra::convertToBlockCrsMatrix(*crs_matrix, params_.blockCrsSize);
+      mat_ = bcrs_matrix;
+    }
   }
 
+  Kokkos::Timer copyTimer;
+  CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getStructure(mat_.get(), localRowPtrsHost_, localRowPtrs_, localColInds_);
+  CrsArrayReader<Scalar, ImplScalar, LocalOrdinal, GlobalOrdinal, Node>::getValues(mat_.get(), localValues_, localRowPtrsHost_);
+  crsCopyTime_ = copyTimer.seconds();
+
   if (params_.use_metis)
   {
     assert(!params_.blockCrs);
@@ -448,6 +451,7 @@ Params::getDefaults()
   p.blockSize = 1;      // # of rows / thread, for SpTRSV
   p.blockCrs = false;   // whether to use block CRS for fastILU
   p.blockCrsSize = 1;   // block size for block CRS
+  p.fillBlocks = false; // whether input matrix needs to be filled
   return p;
 }
 
@@ -583,6 +587,15 @@ Params::Params(const Teuchos::ParameterList& pL, std::string precType)
     else
       TYPE_ERROR("block crs block size", "int");
   }
+  //"fill blocks for input" aka fillBlocks
+  if(pL.isParameter("fill blocks for input"))
+  {
+    if(pL.isType<bool>("fill blocks for input"))
+      blockCrsSize = pL.get<bool>("fill blocks for input");
+    else
+      TYPE_ERROR("fill blocks for input", "bool");
+  }
+
   #undef CHECK_VALUE
   #undef TYPE_ERROR
 }

diff --git a/packages/ifpack2/src/Ifpack2_Details_Filu_def.hpp b/packages/ifpack2/src/Ifpack2_Details_Filu_def.hpp
@@ -117,12 +117,7 @@ initLocalPrec()
 
   #ifdef HAVE_IFPACK2_METIS
   if (p.use_metis) {
-    if (this->isBlockCrs()) {
-      localPrec_->setMetisPerm(this->metis_perm_, this->metis_iperm_);
-    }
-    else {
-      localPrecBlockCrs_->setMetisPerm(this->metis_perm_, this->metis_iperm_);
-    }
+    localPrec_->setMetisPerm(this->metis_perm_, this->metis_iperm_);
   }
   #endif
 
@@ -144,8 +139,9 @@ template<typename Scalar, typename LocalOrdinal, typename GlobalOrdinal, typenam
 void Filu<Scalar, LocalOrdinal, GlobalOrdinal, Node, BlockCrsEnabled>::
 applyLocalPrec(ImplScalarArray x, ImplScalarArray y) const
 {
-  //since this may be applied to multiple vectors, add to applyTime_ instead of setting it
   localPrec_->apply(x, y);
+
+  //since this may be applied to multiple vectors, add to applyTime_ instead of setting it
   this->applyTime_ += localPrec_->getApplyTime();
 }
 

diff --git a/packages/ifpack2/test/belos/6w_bel_tif_FILU_blockcrs.xml b/packages/ifpack2/test/belos/6w_bel_tif_FILU_blockcrs.xml
@@ -19,6 +19,7 @@
     <Parameter name="triangular solve iterations" type="int" value="2"/>
     <Parameter name="block crs" type="bool" value="true"/>
     <Parameter name="block crs block size" type="int" value="3"/>
+    <Parameter name="fill blocks for input" type="bool" value="true"/>
   </ParameterList>
 
   <Parameter name="expectNumIters" type="int" value="5"/>

diff --git a/packages/shylu/shylu_node/fastilu/src/shylu_fastilu.hpp b/packages/shylu/shylu_node/fastilu/src/shylu_fastilu.hpp
@@ -1413,11 +1413,11 @@ class FastILUPrec
             Kokkos::deep_copy(aRowMapHost, aRowMapIn_);
             Kokkos::deep_copy(aColIdxHost, aColIdxIn_);
             Kokkos::deep_copy(aValHost,    aValIn_);
-            if (blockCrsSize > 1) {
 #ifdef FASTILU_ONE_TO_ONE_UNBLOCKED
+            if (blockCrsSize > 1) {
               unblock(aRowMapHost, aColIdxHost, aValHost, blockCrsSize);
-#endif
             }
+#endif
 
             if ((level > 0) && (guessFlag != 0))
             {