Panzer MiniEM: Add option to run with barriers

Signed-off-by: Christian Glusa <[email protected]>
trilinos · Feb 18, 2025 · a9a8557 · a9a8557
1 parent cf89eab
commit a9a8557
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 3 deletions.
diff --git a/packages/panzer/mini-em/example/BlockPrec/CMakeLists.txt b/packages/panzer/mini-em/example/BlockPrec/CMakeLists.txt
@@ -22,7 +22,7 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(CopyBlockPrecFiles
   darcyTet.xml darcyHex.xml
   darcyTetAnalytic.xml darcyHexAnalytic.xml
   solverCG.xml solverGMRES.xml
-  solverMueLu.xml solverMueLu2D.xml solverMueLuEpetra.xml
+  solverMueLu.xml solverMueLu2D.xml solverMueLuEpetra.xml solverMueLuBarrier.xml
   solverMueLuOpenMP.xml solverMueLuCuda.xml solverMueLuTruncated.xml solverMueLuTPL.xml
   solverMueLuHO.xml solverMueLuHOCuda.xml
   solverAugmentationUseILU.xml
@@ -90,6 +90,7 @@ TRIBITS_ADD_TEST(
    POSTFIX_AND_ARGS_8 "2D"                  --solver=MueLu --numTimeSteps=1 --linAlgebra=Tpetra --inputFile=maxwell2D.xml
    POSTFIX_AND_ARGS_9 "order1_analytic"     --solver=MueLu --linAlgebra=Tpetra --inputFile=maxwell-analyticSolution.xml
    POSTFIX_AND_ARGS_10 "order1_truncated"    --solver=MueLu --numTimeSteps=1 --linAlgebra=Tpetra --truncateMueLuHierarchy
+   POSTFIX_AND_ARGS_11 "order1_barriers"     --solver=MueLu --numTimeSteps=1 --linAlgebra=Tpetra --barriers
    NUM_MPI_PROCS 4
    )
 
@@ -126,6 +127,16 @@ TRIBITS_ADD_TEST(
    RUN_SERIAL
    )
 
+TRIBITS_ADD_TEST(
+   BlockPrec
+   NAME "MiniEM-BlockPrec_RefMaxwell_Performance_16_barriers"
+   ARGS "--stacked-timer --solver=MueLu --numTimeSteps=3 --linAlgebra=Tpetra --inputFile=maxwell-large.xml --barriers"
+   COMM mpi
+   NUM_MPI_PROCS 16
+   CATEGORIES PERFORMANCE
+   RUN_SERIAL
+   )
+
 IF (${PACKAGE_NAME}_ENABLE_ShyLU_NodeTacho)
 
   TRIBITS_ADD_TEST(

diff --git a/packages/panzer/mini-em/example/BlockPrec/MiniEM_helpers.cpp b/packages/panzer/mini-em/example/BlockPrec/MiniEM_helpers.cpp
@@ -111,6 +111,7 @@ namespace mini_em {
                       Teuchos::RCP<Teuchos::FancyOStream> &out,
                       std::string &xml, int basis_order,
                       const bool preferTPLs,
+                      const bool useBarriers,
                       const bool truncateMueLuHierarchy) {
     using Teuchos::RCP;
     using Teuchos::rcp;
@@ -175,6 +176,8 @@ namespace mini_em {
             updateParams("solverMueLuTruncated.xml", lin_solver_pl, comm, out);
           if (preferTPLs)
             updateParams("solverMueLuTPL.xml", lin_solver_pl, comm, out);
+          if (useBarriers)
+            updateParams("solverMueLuBarrier.xml", lin_solver_pl, comm, out);
           if (basis_order > 1) {
             RCP<Teuchos::ParameterList> lin_solver_pl_lo = lin_solver_pl;
             lin_solver_pl = rcp(new Teuchos::ParameterList("Linear Solver"));

diff --git a/packages/panzer/mini-em/example/BlockPrec/MiniEM_helpers.hpp b/packages/panzer/mini-em/example/BlockPrec/MiniEM_helpers.hpp
@@ -70,6 +70,7 @@ namespace mini_em {
                                                            std::string &xml,
                                                            int basis_order,
                                                            const bool preferTPLs = false,
+                                                           const bool useBarriers = false,
                                                            const bool truncateMueLuHierarchy = false);
 
   void setClosureParameters(physicsType physics,

diff --git a/packages/panzer/mini-em/example/BlockPrec/main.cpp b/packages/panzer/mini-em/example/BlockPrec/main.cpp
@@ -131,6 +131,7 @@ int main_(Teuchos::CommandLineProcessor &clp, int argc,char * argv[])
     solverType solverValues[5] = {AUGMENTATION, MUELU, ML, CG, GMRES};
     const char * solverNames[5] = {"Augmentation", "MueLu", "ML", "CG", "GMRES"};
     bool preferTPLs = false;
+    bool useBarriers = false;
     bool truncateMueLuHierarchy = false;
     solverType solver = MUELU;
     int numTimeSteps = 1;
@@ -159,6 +160,7 @@ int main_(Teuchos::CommandLineProcessor &clp, int argc,char * argv[])
     clp.setOption("solverFile",&xml,"XML file with the solver params");
     clp.setOption<solverType>("solver",&solver,5,solverValues,solverNames,"Solver that is used");
     clp.setOption("tpl", "no-tpl", &preferTPLs, "Prefer TPL usage over fused kernels");
+    clp.setOption("barriers", "no-barriers", &useBarriers, "Use barriers in the solver");
     clp.setOption("truncateMueLuHierarchy", "no-truncateMueLuHierarchy", &truncateMueLuHierarchy, "Truncate the MueLu hierarchy");
     clp.setOption("numTimeSteps",&numTimeSteps);
     clp.setOption("finalTime",&finalTime);
@@ -304,7 +306,7 @@ int main_(Teuchos::CommandLineProcessor &clp, int argc,char * argv[])
         throw;
     }
 
-    RCP<Teuchos::ParameterList> lin_solver_pl = mini_em::getSolverParameters(linAlgebra, physics, solver, dim, comm, out, xml, basis_order, preferTPLs, truncateMueLuHierarchy);
+    RCP<Teuchos::ParameterList> lin_solver_pl = mini_em::getSolverParameters(linAlgebra, physics, solver, dim, comm, out, xml, basis_order, preferTPLs, useBarriers, truncateMueLuHierarchy);
 
     if (lin_solver_pl->sublist("Preconditioner Types").isSublist("Teko") &&
         lin_solver_pl->sublist("Preconditioner Types").sublist("Teko").isSublist("Inverse Factory Library")) {
@@ -678,11 +680,20 @@ int main_(Teuchos::CommandLineProcessor &clp, int argc,char * argv[])
           // solve
           if (doSolveTimings)
             for (int rep = 0; rep < numReps; rep++) {
+              if (useBarriers) {
+                Teuchos::TimeMonitor solve_barrier_timer(*Teuchos::TimeMonitor::getNewTimer(std::string("Mini-EM: Solve barrier")));
+                comm->barrier();
+              }
               Thyra::assign(correction_vec.ptr(), zero);
               jacobian->solve(Thyra::NOTRANS, *residual, correction_vec.ptr());
             }
-          else
+          else {
+            if (useBarriers) {
+              Teuchos::TimeMonitor solve_barrier_timer(*Teuchos::TimeMonitor::getNewTimer(std::string("Mini-EM: Solve barrier")));
+              comm->barrier();
+            }
             jacobian->solve(Thyra::NOTRANS, *residual, correction_vec.ptr());
+          }
           Thyra::V_StVpStV(solution_vec.ptr(), one, *solution_vec, -one, *correction_vec);
 
           // compute responses

diff --git a/packages/panzer/mini-em/example/BlockPrec/solverMueLuBarrier.xml b/packages/panzer/mini-em/example/BlockPrec/solverMueLuBarrier.xml
@@ -0,0 +1,22 @@
+<ParameterList name="Linear Solver">
+  <ParameterList name="Preconditioner Types">
+    <ParameterList name="Teko">
+      <ParameterList name="Inverse Factory Library">
+        <ParameterList name="Maxwell">
+
+          <ParameterList name="S_E Preconditioner">
+            <ParameterList name="Preconditioner Types">
+              <ParameterList name="MueLuRefMaxwell">
+
+                <Parameter name="sync timers" type="bool" value="true"/>
+
+              </ParameterList>
+
+            </ParameterList>
+          </ParameterList>
+
+        </ParameterList>
+      </ParameterList>
+    </ParameterList>
+  </ParameterList>
+</ParameterList>