Skip to content

Commit

Permalink
Merge Pull Request #10900 from trilinos/Trilinos/rstumin-be3d516
Browse files Browse the repository at this point in the history 
Automatically Merged using Trilinos Pull Request AutoTester
PR Title: Ifpack2: add AVG Combine Mode option to Schwarz DD
PR Author: rstumin
  • Loading branch information
@trilinos-autotester
trilinos-autotester authored Aug 22, 2022
2 parents 127be59 + b3eb839 commit ff408f6
Show file tree
Hide file tree
Showing 6 changed files with 335 additions and 2 deletions.
7 changes: 7 additions & 0 deletions packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -809,7 +809,12 @@ class AdditiveSchwarz :
mutable Teuchos::RCP<const Teuchos::ParameterList> validParams_;

//! Combine mode for off-process elements (only if overlap is used)
//! To average values in overlap region, set CombineMode_
//! to ADD and AvgOverlap_ to true (can be done via
//! param list by setting "schwarz: combine mode" to "AVG")
//! Don't average with CG as preconditioner is nonsymmetric.
Tpetra::CombineMode CombineMode_ = Tpetra::ZERO;
bool AvgOverlap_ = false;
//! If \c true, reorder the local matrix.
bool UseReordering_ = false;
//! Record reordering for output purposes.
Expand Down Expand Up @@ -851,6 +856,8 @@ class AdditiveSchwarz :
mutable std::unique_ptr<MV> overlapping_B_;
//! Cached local (possibly) overlapping output (multi)vector.
mutable std::unique_ptr<MV> overlapping_Y_;
//! Cached local (possibly) vector that indicates how many copies of a dof exist due to overlap
mutable std::unique_ptr<MV> num_overlap_copies_;
//! Cached residual (multi)vector.
mutable std::unique_ptr<MV> R_;
//! Cached intermediate result (multi)vector.
Expand Down
32 changes: 31 additions & 1 deletion packages/ifpack2/src/Ifpack2_AdditiveSchwarz_def.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,19 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
if (C_.get () == nullptr || C_->getNumVectors () != numVectors) {
C_.reset (new MV (Y.getMap (), numVectors, false));
}
// If taking averages in overlap region, we need to compute
// the number of procs who have a copy of each overlap dof
Teuchos::ArrayRCP<scalar_type> dataNumOverlapCopies;
if (IsOverlapping_ && AvgOverlap_) {
if (num_overlap_copies_.get() == nullptr) {
num_overlap_copies_.reset (new MV (Y.getMap (), 1, false));
RCP<MV> onesVec( new MV(OverlappingMatrix_->getRowMap(), 1, false) );
onesVec->putScalar(Teuchos::ScalarTraits<scalar_type>::one());
rcp_dynamic_cast<OverlappingRowMatrix<row_matrix_type>> (OverlappingMatrix_)->exportMultiVector (*onesVec, *(num_overlap_copies_.get ()), CombineMode_);
}
dataNumOverlapCopies = num_overlap_copies_.get ()->getDataNonConst(0);
}

MV* R = R_.get ();
MV* C = C_.get ();

Expand Down Expand Up @@ -550,6 +563,14 @@ apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_t
<< "OverlappingMatrix_ is null when it shouldn't be. "
"Please report this bug to the Ifpack2 developers.");
OverlappingMatrix_->exportMultiVector (*OverlappingY, *C, CombineMode_);

// average solution in overlap regions if requested via "schwarz: combine mode" "AVG"
if (AvgOverlap_) {
Teuchos::ArrayRCP<scalar_type> dataC = C->getDataNonConst(0);
for (int i = 0; i < (int) C->getMap()->getLocalNumElements(); i++) {
dataC[i] = dataC[i]/dataNumOverlapCopies[i];
}
}
}
else {
// mfh 16 Apr 2014: Make a view of Y with the same Map as
Expand Down Expand Up @@ -785,7 +806,15 @@ setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist)
using vs2e_type = StringToIntegralParameterEntryValidator<CombineMode>;
RCP<const vs2e_type> vs2e = rcp_dynamic_cast<const vs2e_type> (v, true);

const ParameterEntry& inputEntry = plist->getEntry (cmParamName);
ParameterEntry& inputEntry = plist->getEntry (cmParamName);
// As AVG is only a Schwarz option and does not exist in Tpetra's
// version of CombineMode, we use a separate boolean local to
// Schwarz in conjunction with CombineMode_ == ADD to handle
// averaging. Here, we change input entry to ADD and set the boolean.
if (strncmp(Teuchos::getValue<std::string>(inputEntry).c_str(),"AVG",3) == 0) {
inputEntry.template setValue<std::string>("ADD");
AvgOverlap_ = true;
}
CombineMode_ = vs2e->getIntegralValue (inputEntry, cmParamName);
}
}
Expand All @@ -802,6 +831,7 @@ setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist)
if (plist->sublist("subdomain solver parameters").get<std::string>("partitioner: type") == "user") {
if (CombineMode_ == Tpetra::ADD) plist->sublist("subdomain solver parameters").set("partitioner: combine mode","ADD");
if (CombineMode_ == Tpetra::ZERO) plist->sublist("subdomain solver parameters").set("partitioner: combine mode","ZERO");
AvgOverlap_ = false; // averaging already taken care of by the partitioner: nonsymmetric overlap combine option
}
}
}
Expand Down
18 changes: 17 additions & 1 deletion packages/muelu/test/scaling/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK OR ${PACKAGE_NAME}_HAVE_EPETRA_SOLV
)

TRIBITS_COPY_FILES_TO_BINARY_DIR(Driver_cp
SOURCE_FILES scaling.xml scaling.yaml scaling-complex.xml scaling-withglobalconstants.xml scaling-complex-withglobalconstants.xml circ_nsp_dependency.xml isorropia.xml iso_poisson.xml conchas_milestone_zoltan.xml conchas_milestone_zoltan2.xml conchas_milestone_zoltan2_complex.xml sa_with_ilu.xml sa_with_Ifpack2_line_detection.xml rap.xml smoother.xml smoother_complex.xml tripleMatrixProduct.xml scaling-ml.xml elasticity3D.xml amgx.json amgx.xml scaling-with-rerun.xml scaling_distance2_agg.xml smooVec.mm smooVecCoalesce.xml pairwise.xml sa_enforce_constraints.xml recurMG.xml anisotropic.xml comp_rotations.xml generalBlkSmoothing.xml GblkMap.dat GblkAmat.dat GblkRhs.dat Gblks.dat
SOURCE_FILES scaling.xml scaling.yaml scaling-complex.xml scaling-withglobalconstants.xml scaling-complex-withglobalconstants.xml circ_nsp_dependency.xml isorropia.xml iso_poisson.xml conchas_milestone_zoltan.xml conchas_milestone_zoltan2.xml conchas_milestone_zoltan2_complex.xml sa_with_ilu.xml sa_with_Ifpack2_line_detection.xml rap.xml smoother.xml smoother_complex.xml tripleMatrixProduct.xml scaling-ml.xml elasticity3D.xml amgx.json amgx.xml scaling-with-rerun.xml scaling_distance2_agg.xml smooVec.mm smooVecCoalesce.xml pairwise.xml sa_enforce_constraints.xml recurMG.xml anisotropic.xml comp_rotations.xml generalBlkSmoothing.xml GblkMap.dat GblkAmat.dat GblkRhs.dat Gblks.dat blkSmooEquivOlapSchwarz.xml oLapSchwarzEquivBlkSmoo.xml regularOverLap.dat
CATEGORIES BASIC PERFORMANCE
)

Expand Down Expand Up @@ -289,6 +289,22 @@ IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK)
COMM mpi # HAVE_MPI required
)

TRIBITS_ADD_TEST(
Driver
NAME "BlockSmoothingWithAverages"
ARGS "--linAlgebra=Tpetra --xml=blkSmooEquivOlapSchwarz.xml --belosType=\"Fixed\ Point\" --rowmap=GblkMap.dat --matrix=GblkAmat.dat --rhs=GblkRhs.dat --tol=.1 --userBlks=regularOverLap.dat"
NUM_MPI_PROCS 4
COMM mpi # HAVE_MPI required
)

TRIBITS_ADD_TEST(
Driver
NAME "SchwarzSmoothingWithAverages"
ARGS "--linAlgebra=Tpetra --xml=oLapSchwarzEquivBlkSmoo.xml --belosType=\"Fixed\ Point\" --rowmap=GblkMap.dat --matrix=GblkAmat.dat --rhs=GblkRhs.dat --tol=.1"
NUM_MPI_PROCS 4
COMM mpi # HAVE_MPI required
)

ENDIF()

IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK)
Expand Down
50 changes: 50 additions & 0 deletions packages/muelu/test/scaling/blkSmooEquivOlapSchwarz.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
<!-- Running
mpirun -np 4 MueLu_Driver.exe __linAlgebra=Tpetra __xml=blkSmooEquivOlapSchwarz.xml __belosType=\"Fixed\ Point\" __rowmap=GblkMap.dat __matrix=GblkAmat.dat __rhs=GblkRhs.dat __tol=.1 __userBlks=regularOverLap.dat
and mpirun -np 4 MueLu_Driver.exe __linAlgebra=Tpetra __xml=oLapSchwarzEquivBlkSmoo.xml __belosType=\"Fixed\ Point\" __rowmap=GblkMap.dat __matrix=GblkAmat.dat __rhs=GblkRhs.dat __tol=.1
should give identical convergence histories as both do overlapping block Jacobi with the same blocks. Here, __ should be replaced by double-hyphen (double-hyphen not allowed in xml comments).
-->
<ParameterList name="MueLu">
<Parameter name="verbosity" type="string" value="high"/>

<Parameter name="number of equations" type="int" value="1"/>

<Parameter name="max levels" type="int" value="1"/>
<Parameter name="coarse: max size" type="int" value="1"/>

<Parameter name="multigrid algorithm" type="string" value="sa"/>

<!-- =========== AGGREGATION =========== -->
<Parameter name="aggregation: type" type="string" value="uncoupled"/>
<Parameter name="aggregation: drop scheme" type="string" value="distance laplacian"/>

<!-- =========== SMOOTHING =========== -->
<Parameter name="coarse: type" type="string" value="SCHWARZ"/>
<ParameterList name="coarse: params">
<Parameter name="schwarz: overlap level" type="int" value="1"/>
<Parameter name="schwarz: combine mode" type="string" value="ADD"/>
<Parameter name="schwarz: use reordering" type="bool" value="false"/>
<ParameterList name="schwarz: reordering list">
<Parameter name="order_method" type="string" value="rcm"/>
</ParameterList>

<Parameter name="subdomain solver name" type="string" value="BLOCK_RELAXATION"/>
<ParameterList name="subdomain solver parameters">
<Parameter name= "relaxation: type" type="string" value="Jacobi"/>
<Parameter name= "relaxation: sweeps" type="int" value="1"/>
<Parameter name= "relaxation: zero starting solution" type="bool" value="false"/>
<Parameter name= "partitioner: type" type="string" value="user"/>
<Parameter name= "partitioner: nonsymmetric overlap combine" type="bool" value="true"/>
<Parameter name= "relaxation: damping factor" type="double" value="1.0"/>
</ParameterList>
</ParameterList>

<!-- =========== REPARTITIONING =========== -->
<Parameter name="repartition: enable" type="bool" value="true"/>
<Parameter name="repartition: partitioner" type="string" value="zoltan"/>
<Parameter name="repartition: start level" type="int" value="2"/>
<Parameter name="repartition: min rows per proc" type="int" value="800"/>
<Parameter name="repartition: max imbalance" type="double" value="1.1"/>
<Parameter name="repartition: remap parts" type="bool" value="false"/>

</ParameterList>
42 changes: 42 additions & 0 deletions packages/muelu/test/scaling/oLapSchwarzEquivBlkSmoo.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<!-- Running
mpirun -np 4 MueLu_Driver.exe __linAlgebra=Tpetra __xml=blkSmooEquivOlapSchwarz.xml __belosType=\"Fixed\ Point\" __rowmap=GblkMap.dat __matrix=GblkAmat.dat __rhs=GblkRhs.dat __tol=.1 __userBlks=regularOverLap.dat
and mpirun -np 4 MueLu_Driver.exe __linAlgebra=Tpetra __xml=oLapSchwarzEquivBlkSmoo.xml __belosType=\"Fixed\ Point\" __rowmap=GblkMap.dat __matrix=GblkAmat.dat __rhs=GblkRhs.dat __tol=.1
should give identical convergence histories as both do overlapping block Jacobi with the same blocks. Here, __ should be replaced by double-hyphen (double-hyphen not allowed in xml comments).
-->
<ParameterList name="MueLu">
<Parameter name="verbosity" type="string" value="high"/>

<Parameter name="number of equations" type="int" value="1"/>

<Parameter name="max levels" type="int" value="1"/>
<Parameter name="coarse: max size" type="int" value="1"/>

<Parameter name="multigrid algorithm" type="string" value="sa"/>

<!-- =========== AGGREGATION =========== -->
<Parameter name="aggregation: type" type="string" value="uncoupled"/>
<Parameter name="aggregation: drop scheme" type="string" value="distance laplacian"/>

<!-- =========== SMOOTHING =========== -->
<Parameter name="coarse: type" type="string" value="SCHWARZ"/>
<ParameterList name="coarse: params">
<Parameter name="subdomain solver name" type="string" value="AMESOS2"/>
<Parameter name="schwarz: overlap level" type="int" value="1"/>
<Parameter name="schwarz: combine mode" type="string" value="AVG"/>
<Parameter name="schwarz: use reordering" type="bool" value="false"/>
<ParameterList name="schwarz: reordering list">
<Parameter name="order_method" type="string" value="rcm"/>
</ParameterList>

</ParameterList>

<!-- =========== REPARTITIONING =========== -->
<Parameter name="repartition: enable" type="bool" value="true"/>
<Parameter name="repartition: partitioner" type="string" value="zoltan"/>
<Parameter name="repartition: start level" type="int" value="2"/>
<Parameter name="repartition: min rows per proc" type="int" value="800"/>
<Parameter name="repartition: max imbalance" type="double" value="1.1"/>
<Parameter name="repartition: remap parts" type="bool" value="false"/>

</ParameterList>
188 changes: 188 additions & 0 deletions packages/muelu/test/scaling/regularOverLap.dat
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
%%MatrixMarket matrix coordinate real general
4 137 186
1 1 1.0
1 5 1.0
1 6 1.0
1 7 1.0
1 8 1.0
1 9 1.0
1 10 1.0
1 25 1.0
1 26 1.0
1 27 1.0
1 28 1.0
1 29 1.0
1 30 1.0
1 46 1.0
1 47 1.0
1 51 1.0
1 52 1.0
1 56 1.0
1 58 1.0
1 59 1.0
1 63 1.0
1 64 1.0
1 69 1.0
1 71 1.0
1 72 1.0
1 81 1.0
1 88 1.0
1 89 1.0
1 92 1.0
1 98 1.0
1 99 1.0
1 103 1.0
1 104 1.0
1 105 1.0
1 107 1.0
1 109 1.0
1 113 1.0
1 114 1.0
1 115 1.0
1 118 1.0
1 119 1.0
1 123 1.0
1 125 1.0
1 126 1.0
1 130 1.0
1 132 1.0
2 2 1.0
2 9 1.0
2 10 1.0
2 11 1.0
2 12 1.0
2 13 1.0
2 14 1.0
2 35 1.0
2 36 1.0
2 37 1.0
2 38 1.0
2 39 1.0
2 40 1.0
2 41 1.0
2 48 1.0
2 49 1.0
2 52 1.0
2 54 1.0
2 55 1.0
2 56 1.0
2 57 1.0
2 62 1.0
2 63 1.0
2 65 1.0
2 68 1.0
2 76 1.0
2 77 1.0
2 79 1.0
2 80 1.0
2 81 1.0
2 86 1.0
2 93 1.0
2 96 1.0
2 99 1.0
2 102 1.0
2 104 1.0
2 105 1.0
2 106 1.0
2 108 1.0
2 118 1.0
2 127 1.0
2 128 1.0
2 131 1.0
2 132 1.0
2 134 1.0
2 137 1.0
3 4 1.0
3 15 1.0
3 16 1.0
3 17 1.0
3 18 1.0
3 19 1.0
3 20 1.0
3 29 1.0
3 30 1.0
3 31 1.0
3 32 1.0
3 33 1.0
3 34 1.0
3 47 1.0
3 49 1.0
3 50 1.0
3 51 1.0
3 53 1.0
3 58 1.0
3 63 1.0
3 64 1.0
3 66 1.0
3 70 1.0
3 74 1.0
3 77 1.0
3 79 1.0
3 82 1.0
3 83 1.0
3 87 1.0
3 88 1.0
3 90 1.0
3 91 1.0
3 94 1.0
3 95 1.0
3 97 1.0
3 104 1.0
3 107 1.0
3 110 1.0
3 111 1.0
3 112 1.0
3 114 1.0
3 117 1.0
3 121 1.0
3 122 1.0
3 125 1.0
3 126 1.0
3 130 1.0
3 132 1.0
3 135 1.0
4 3 1.0
4 19 1.0
4 20 1.0
4 21 1.0
4 22 1.0
4 23 1.0
4 24 1.0
4 39 1.0
4 40 1.0
4 41 1.0
4 42 1.0
4 43 1.0
4 44 1.0
4 45 1.0
4 49 1.0
4 53 1.0
4 60 1.0
4 61 1.0
4 62 1.0
4 67 1.0
4 73 1.0
4 75 1.0
4 76 1.0
4 77 1.0
4 78 1.0
4 79 1.0
4 84 1.0
4 85 1.0
4 97 1.0
4 100 1.0
4 101 1.0
4 102 1.0
4 108 1.0
4 110 1.0
4 111 1.0
4 116 1.0
4 117 1.0
4 120 1.0
4 124 1.0
4 127 1.0
4 129 1.0
4 131 1.0
4 133 1.0
4 136 1.0
4 137 1.0

0 comments on commit ff408f6

Please sign in to comment.