diff --git a/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp b/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp index 8c0a135e5451..310469f44008 100644 --- a/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp +++ b/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_decl.hpp @@ -809,7 +809,12 @@ class AdditiveSchwarz : mutable Teuchos::RCP validParams_; //! Combine mode for off-process elements (only if overlap is used) + //! To average values in overlap region, set CombineMode_ + //! to ADD and AvgOverlap_ to true (can be done via + //! param list by setting "schwarz: combine mode" to "AVG") + //! Don't average with CG as preconditioner is nonsymmetric. Tpetra::CombineMode CombineMode_ = Tpetra::ZERO; + bool AvgOverlap_ = false; //! If \c true, reorder the local matrix. bool UseReordering_ = false; //! Record reordering for output purposes. @@ -851,6 +856,8 @@ class AdditiveSchwarz : mutable std::unique_ptr overlapping_B_; //! Cached local (possibly) overlapping output (multi)vector. mutable std::unique_ptr overlapping_Y_; + //! Cached local (possibly) vector that indicates how many copies of a dof exist due to overlap + mutable std::unique_ptr num_overlap_copies_; //! Cached residual (multi)vector. mutable std::unique_ptr R_; //! Cached intermediate result (multi)vector. diff --git a/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_def.hpp b/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_def.hpp index c7d00563a275..4c1987d2ba3d 100644 --- a/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_def.hpp +++ b/packages/ifpack2/src/Ifpack2_AdditiveSchwarz_def.hpp @@ -425,6 +425,19 @@ apply (const Tpetra::MultiVectorgetNumVectors () != numVectors) { C_.reset (new MV (Y.getMap (), numVectors, false)); } + // If taking averages in overlap region, we need to compute + // the number of procs who have a copy of each overlap dof + Teuchos::ArrayRCP dataNumOverlapCopies; + if (IsOverlapping_ && AvgOverlap_) { + if (num_overlap_copies_.get() == nullptr) { + num_overlap_copies_.reset (new MV (Y.getMap (), 1, false)); + RCP onesVec( new MV(OverlappingMatrix_->getRowMap(), 1, false) ); + onesVec->putScalar(Teuchos::ScalarTraits::one()); + rcp_dynamic_cast> (OverlappingMatrix_)->exportMultiVector (*onesVec, *(num_overlap_copies_.get ()), CombineMode_); + } + dataNumOverlapCopies = num_overlap_copies_.get ()->getDataNonConst(0); + } + MV* R = R_.get (); MV* C = C_.get (); @@ -550,6 +563,14 @@ apply (const Tpetra::MultiVectorexportMultiVector (*OverlappingY, *C, CombineMode_); + + // average solution in overlap regions if requested via "schwarz: combine mode" "AVG" + if (AvgOverlap_) { + Teuchos::ArrayRCP dataC = C->getDataNonConst(0); + for (int i = 0; i < (int) C->getMap()->getLocalNumElements(); i++) { + dataC[i] = dataC[i]/dataNumOverlapCopies[i]; + } + } } else { // mfh 16 Apr 2014: Make a view of Y with the same Map as @@ -785,7 +806,15 @@ setParameterList (const Teuchos::RCP& plist) using vs2e_type = StringToIntegralParameterEntryValidator; RCP vs2e = rcp_dynamic_cast (v, true); - const ParameterEntry& inputEntry = plist->getEntry (cmParamName); + ParameterEntry& inputEntry = plist->getEntry (cmParamName); + // As AVG is only a Schwarz option and does not exist in Tpetra's + // version of CombineMode, we use a separate boolean local to + // Schwarz in conjunction with CombineMode_ == ADD to handle + // averaging. Here, we change input entry to ADD and set the boolean. + if (strncmp(Teuchos::getValue(inputEntry).c_str(),"AVG",3) == 0) { + inputEntry.template setValue("ADD"); + AvgOverlap_ = true; + } CombineMode_ = vs2e->getIntegralValue (inputEntry, cmParamName); } } @@ -802,6 +831,7 @@ setParameterList (const Teuchos::RCP& plist) if (plist->sublist("subdomain solver parameters").get("partitioner: type") == "user") { if (CombineMode_ == Tpetra::ADD) plist->sublist("subdomain solver parameters").set("partitioner: combine mode","ADD"); if (CombineMode_ == Tpetra::ZERO) plist->sublist("subdomain solver parameters").set("partitioner: combine mode","ZERO"); + AvgOverlap_ = false; // averaging already taken care of by the partitioner: nonsymmetric overlap combine option } } } diff --git a/packages/muelu/test/scaling/CMakeLists.txt b/packages/muelu/test/scaling/CMakeLists.txt index 67b36efd4e5f..e6f49b53bfab 100644 --- a/packages/muelu/test/scaling/CMakeLists.txt +++ b/packages/muelu/test/scaling/CMakeLists.txt @@ -63,7 +63,7 @@ IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK OR ${PACKAGE_NAME}_HAVE_EPETRA_SOLV ) TRIBITS_COPY_FILES_TO_BINARY_DIR(Driver_cp - SOURCE_FILES scaling.xml scaling.yaml scaling-complex.xml scaling-withglobalconstants.xml scaling-complex-withglobalconstants.xml circ_nsp_dependency.xml isorropia.xml iso_poisson.xml conchas_milestone_zoltan.xml conchas_milestone_zoltan2.xml conchas_milestone_zoltan2_complex.xml sa_with_ilu.xml sa_with_Ifpack2_line_detection.xml rap.xml smoother.xml smoother_complex.xml tripleMatrixProduct.xml scaling-ml.xml elasticity3D.xml amgx.json amgx.xml scaling-with-rerun.xml scaling_distance2_agg.xml smooVec.mm smooVecCoalesce.xml pairwise.xml sa_enforce_constraints.xml recurMG.xml anisotropic.xml comp_rotations.xml generalBlkSmoothing.xml GblkMap.dat GblkAmat.dat GblkRhs.dat Gblks.dat + SOURCE_FILES scaling.xml scaling.yaml scaling-complex.xml scaling-withglobalconstants.xml scaling-complex-withglobalconstants.xml circ_nsp_dependency.xml isorropia.xml iso_poisson.xml conchas_milestone_zoltan.xml conchas_milestone_zoltan2.xml conchas_milestone_zoltan2_complex.xml sa_with_ilu.xml sa_with_Ifpack2_line_detection.xml rap.xml smoother.xml smoother_complex.xml tripleMatrixProduct.xml scaling-ml.xml elasticity3D.xml amgx.json amgx.xml scaling-with-rerun.xml scaling_distance2_agg.xml smooVec.mm smooVecCoalesce.xml pairwise.xml sa_enforce_constraints.xml recurMG.xml anisotropic.xml comp_rotations.xml generalBlkSmoothing.xml GblkMap.dat GblkAmat.dat GblkRhs.dat Gblks.dat blkSmooEquivOlapSchwarz.xml oLapSchwarzEquivBlkSmoo.xml regularOverLap.dat CATEGORIES BASIC PERFORMANCE ) @@ -289,6 +289,22 @@ IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK) COMM mpi # HAVE_MPI required ) + TRIBITS_ADD_TEST( + Driver + NAME "BlockSmoothingWithAverages" + ARGS "--linAlgebra=Tpetra --xml=blkSmooEquivOlapSchwarz.xml --belosType=\"Fixed\ Point\" --rowmap=GblkMap.dat --matrix=GblkAmat.dat --rhs=GblkRhs.dat --tol=.1 --userBlks=regularOverLap.dat" + NUM_MPI_PROCS 4 + COMM mpi # HAVE_MPI required + ) + + TRIBITS_ADD_TEST( + Driver + NAME "SchwarzSmoothingWithAverages" + ARGS "--linAlgebra=Tpetra --xml=oLapSchwarzEquivBlkSmoo.xml --belosType=\"Fixed\ Point\" --rowmap=GblkMap.dat --matrix=GblkAmat.dat --rhs=GblkRhs.dat --tol=.1" + NUM_MPI_PROCS 4 + COMM mpi # HAVE_MPI required + ) + ENDIF() IF (${PACKAGE_NAME}_HAVE_TPETRA_SOLVER_STACK) diff --git a/packages/muelu/test/scaling/blkSmooEquivOlapSchwarz.xml b/packages/muelu/test/scaling/blkSmooEquivOlapSchwarz.xml new file mode 100644 index 000000000000..c9a3911fbc9d --- /dev/null +++ b/packages/muelu/test/scaling/blkSmooEquivOlapSchwarz.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/muelu/test/scaling/oLapSchwarzEquivBlkSmoo.xml b/packages/muelu/test/scaling/oLapSchwarzEquivBlkSmoo.xml new file mode 100644 index 000000000000..caa33197936f --- /dev/null +++ b/packages/muelu/test/scaling/oLapSchwarzEquivBlkSmoo.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/muelu/test/scaling/regularOverLap.dat b/packages/muelu/test/scaling/regularOverLap.dat new file mode 100644 index 000000000000..20768d39b94f --- /dev/null +++ b/packages/muelu/test/scaling/regularOverLap.dat @@ -0,0 +1,188 @@ +%%MatrixMarket matrix coordinate real general +4 137 186 +1 1 1.0 +1 5 1.0 +1 6 1.0 +1 7 1.0 +1 8 1.0 +1 9 1.0 +1 10 1.0 +1 25 1.0 +1 26 1.0 +1 27 1.0 +1 28 1.0 +1 29 1.0 +1 30 1.0 +1 46 1.0 +1 47 1.0 +1 51 1.0 +1 52 1.0 +1 56 1.0 +1 58 1.0 +1 59 1.0 +1 63 1.0 +1 64 1.0 +1 69 1.0 +1 71 1.0 +1 72 1.0 +1 81 1.0 +1 88 1.0 +1 89 1.0 +1 92 1.0 +1 98 1.0 +1 99 1.0 +1 103 1.0 +1 104 1.0 +1 105 1.0 +1 107 1.0 +1 109 1.0 +1 113 1.0 +1 114 1.0 +1 115 1.0 +1 118 1.0 +1 119 1.0 +1 123 1.0 +1 125 1.0 +1 126 1.0 +1 130 1.0 +1 132 1.0 +2 2 1.0 +2 9 1.0 +2 10 1.0 +2 11 1.0 +2 12 1.0 +2 13 1.0 +2 14 1.0 +2 35 1.0 +2 36 1.0 +2 37 1.0 +2 38 1.0 +2 39 1.0 +2 40 1.0 +2 41 1.0 +2 48 1.0 +2 49 1.0 +2 52 1.0 +2 54 1.0 +2 55 1.0 +2 56 1.0 +2 57 1.0 +2 62 1.0 +2 63 1.0 +2 65 1.0 +2 68 1.0 +2 76 1.0 +2 77 1.0 +2 79 1.0 +2 80 1.0 +2 81 1.0 +2 86 1.0 +2 93 1.0 +2 96 1.0 +2 99 1.0 +2 102 1.0 +2 104 1.0 +2 105 1.0 +2 106 1.0 +2 108 1.0 +2 118 1.0 +2 127 1.0 +2 128 1.0 +2 131 1.0 +2 132 1.0 +2 134 1.0 +2 137 1.0 +3 4 1.0 +3 15 1.0 +3 16 1.0 +3 17 1.0 +3 18 1.0 +3 19 1.0 +3 20 1.0 +3 29 1.0 +3 30 1.0 +3 31 1.0 +3 32 1.0 +3 33 1.0 +3 34 1.0 +3 47 1.0 +3 49 1.0 +3 50 1.0 +3 51 1.0 +3 53 1.0 +3 58 1.0 +3 63 1.0 +3 64 1.0 +3 66 1.0 +3 70 1.0 +3 74 1.0 +3 77 1.0 +3 79 1.0 +3 82 1.0 +3 83 1.0 +3 87 1.0 +3 88 1.0 +3 90 1.0 +3 91 1.0 +3 94 1.0 +3 95 1.0 +3 97 1.0 +3 104 1.0 +3 107 1.0 +3 110 1.0 +3 111 1.0 +3 112 1.0 +3 114 1.0 +3 117 1.0 +3 121 1.0 +3 122 1.0 +3 125 1.0 +3 126 1.0 +3 130 1.0 +3 132 1.0 +3 135 1.0 +4 3 1.0 +4 19 1.0 +4 20 1.0 +4 21 1.0 +4 22 1.0 +4 23 1.0 +4 24 1.0 +4 39 1.0 +4 40 1.0 +4 41 1.0 +4 42 1.0 +4 43 1.0 +4 44 1.0 +4 45 1.0 +4 49 1.0 +4 53 1.0 +4 60 1.0 +4 61 1.0 +4 62 1.0 +4 67 1.0 +4 73 1.0 +4 75 1.0 +4 76 1.0 +4 77 1.0 +4 78 1.0 +4 79 1.0 +4 84 1.0 +4 85 1.0 +4 97 1.0 +4 100 1.0 +4 101 1.0 +4 102 1.0 +4 108 1.0 +4 110 1.0 +4 111 1.0 +4 116 1.0 +4 117 1.0 +4 120 1.0 +4 124 1.0 +4 127 1.0 +4 129 1.0 +4 131 1.0 +4 133 1.0 +4 136 1.0 +4 137 1.0