Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Coarse level options for AMG preconditioner #359

Merged
merged 8 commits into from
Aug 2, 2024
Merged

Coarse level options for AMG preconditioner #359

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c286c15
added AMG coarse level options
sseraj May 22, 2024
1bbd2a0
added options docs
sseraj May 23, 2024
7c7a647
added nondefault option test
sseraj May 23, 2024
e84e0d0
use cheapest coarse grid options by default
sseraj May 27, 2024
0a08581
Merge branch 'main' into amg-coarse
eirikurj Jun 28, 2024
8491661
added orthog type to amg
sseraj Jul 1, 2024
51b3c76
fixed typo
sseraj Jul 25, 2024
beb06fc
Merge branch 'main' into amg-coarse
sseraj Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions adflow/pyADflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5779,10 +5779,13 @@ def _getDefaultOptions():
"NKViscPC": [bool, False],
"NKGlobalPreconditioner": [str, ["additive Schwarz", "multigrid"]],
"NKASMOverlap": [int, 1],
"NKASMOverlapCoarse": [int, 0],
"NKPCILUFill": [int, 2],
"NKPCILUFillCoarse": [int, 0],
"NKJacobianLag": [int, 20],
"applyPCSubspaceSize": [int, 10],
"NKInnerPreconIts": [int, 1],
"NKInnerPreconItsCoarse": [int, 1],
"NKOuterPreconIts": [int, 1],
"NKAMGLevels": [int, 2],
"NKAMGNSmooth": [int, 1],
Expand All @@ -5802,9 +5805,12 @@ def _getDefaultOptions():
"ANKLinResMax": [float, 0.1],
"ANKGlobalPreconditioner": [str, ["additive Schwarz", "multigrid"]],
"ANKASMOverlap": [int, 1],
"ANKASMOverlapCoarse": [int, 0],
"ANKPCILUFill": [int, 2],
"ANKPCILUFillCoarse": [int, 0],
"ANKJacobianLag": [int, 10],
"ANKInnerPreconIts": [int, 1],
"ANKInnerPreconItsCoarse": [int, 1],
"ANKOuterPreconIts": [int, 1],
"ANKAMGLevels": [int, 2],
"ANKAMGNSmooth": [int, 1],
Expand Down Expand Up @@ -5884,8 +5890,11 @@ def _getDefaultOptions():
"globalPreconditioner": [str, ["additive Schwarz", "multigrid"]],
"localPreconditioner": [str, ["ILU"]],
"ILUFill": [int, 2],
"ILUFillCoarse": [int, 0],
"ASMOverlap": [int, 1],
"ASMOverlapCoarse": [int, 0],
"innerPreconIts": [int, 1],
"innerPreconItsCoarse": [int, 1],
"outerPreconIts": [int, 3],
"adjointAMGLevels": [int, 2],
"adjointAMGNSmooth": [int, 1],
Expand Down Expand Up @@ -6176,12 +6185,15 @@ def _getOptionMap(self):
"location": ["nk", "nk_precondtype"],
},
"nkasmoverlap": ["nk", "nk_asmoverlap"],
"nkasmoverlapcoarse": ["nk", "nk_asmoverlapcoarse"],
"nkpcilufill": ["nk", "nk_ilufill"],
"nkpcilufillcoarse": ["nk", "nk_ilufillcoarse"],
"nkjacobianlag": ["nk", "nk_jacobianlag"],
"nkadpc": ["nk", "nk_adpc"],
"nkviscpc": ["nk", "nk_viscpc"],
"applypcsubspacesize": ["nk", "applypcsubspacesize"],
"nkinnerpreconits": ["nk", "nk_innerpreconits"],
"nkinnerpreconitscoarse": ["nk", "nk_innerpreconitscoarse"],
"nkouterpreconits": ["nk", "nk_outerpreconits"],
"nkamglevels": ["nk", "nk_amglevels"],
"nkamgnsmooth": ["nk", "nk_amgnsmooth"],
Expand Down Expand Up @@ -6210,9 +6222,12 @@ def _getOptionMap(self):
"location": ["ank", "ank_precondtype"],
},
"ankasmoverlap": ["ank", "ank_asmoverlap"],
"ankasmoverlapcoarse": ["ank", "ank_asmoverlapcoarse"],
"ankpcilufill": ["ank", "ank_ilufill"],
"ankpcilufillcoarse": ["ank", "ank_ilufillcoarse"],
"ankjacobianlag": ["ank", "ank_jacobianlag"],
"ankinnerpreconits": ["ank", "ank_innerpreconits"],
"ankinnerpreconitscoarse": ["ank", "ank_innerpreconitscoarse"],
"ankouterpreconits": ["ank", "ank_outerpreconits"],
"ankamglevels": ["ank", "ank_amglevels"],
"ankamgnsmooth": ["ank", "ank_amgnsmooth"],
Expand Down Expand Up @@ -6305,9 +6320,12 @@ def _getOptionMap(self):
},
"localpreconditioner": {"ilu": "ilu", "location": ["adjoint", "localpctype"]},
"ilufill": ["adjoint", "filllevel"],
"ilufillcoarse": ["adjoint", "filllevelcoarse"],
"applyadjointpcsubspacesize": ["adjoint", "applyadjointpcsubspacesize"],
"asmoverlap": ["adjoint", "overlap"],
"asmoverlapcoarse": ["adjoint", "overlapcoarse"],
"innerpreconits": ["adjoint", "innerpreconits"],
"innerpreconitscoarse": ["adjoint", "innerpreconitscoarse"],
"outerpreconits": ["adjoint", "outerpreconits"],
"adjointamglevels": ["adjoint", "adjamglevels"],
"adjointamgnsmooth": ["adjoint", "adjamgnsmooth"],
Expand Down
36 changes: 36 additions & 0 deletions doc/options.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -903,12 +903,20 @@ NKASMOverlap:
More overlap levels result in a stronger preconditioner, at the expense of more expensive iterations and more memory.
Typical values range from 1 for easy cases and up to 2 or 3 for more difficult cases.

NKASMOverlapCoarse:
desc: >
Same as :py:data:`NKASMOverlap` but for the coarse levels when using ``multigrid`` for :py:data:`NKGlobalPreconditioner`.

NKPCILUFill:
desc: >
The number of levels of fill to use on the local Incomplete LU (ILU) factorization in the NK solver.
Typical values are 1 for easy cases and up to 3 for more difficult cases.
More levels of fill result in a stronger preconditioner which will result in fewer (linear) iterations, but individual iterations will be more costly and consume more memory.

NKPCILUFillCoarse:
desc: >
Same as :py:data:`NKPCILUFill` but for the coarse levels when using ``multigrid`` for :py:data:`NKGlobalPreconditioner`.

NKJacobianLag:
desc: >
The option determines the frequency at which the NK preconditioner is reformed.
Expand All @@ -928,6 +936,10 @@ NKInnerPreconIts:
More iterations may help converge the linear system faster.
This should be left at 1 unless a very difficult problem is encountered.

NKInnerPreconItsCoarse:
desc: >
Same as :py:data:`NKInnerPreconIts` but for the coarse levels when using ``multigrid`` for :py:data:`NKGlobalPreconditioner`.

NKOuterPreconIts:
desc: >
Number of global preconditioning iterations for the NK solver adjoint solution.
Expand Down Expand Up @@ -1025,10 +1037,18 @@ ANKASMOverlap:
desc: >
Similar to the :py:data:`NKASMOverlap` option but for the ANK solver.

ANKASMOverlapCoarse:
desc: >
Same as :py:data:`ANKASMOverlap` but for the coarse levels when using ``multigrid`` for :py:data:`ANKGlobalPreconditioner`.

ANKPCILUFill:
desc: >
Similar to the :py:data:`NKPCILUFill` option but for the ANK solver.

ANKPCILUFillCoarse:
desc: >
Same as :py:data:`ANKPCILUFill` but for the coarse levels when using ``multigrid`` for :py:data:`ANKGlobalPreconditioner`.

ANKJacobianLag:
desc: >
Number of nonlinear iterations between every preconditioner update.
Expand All @@ -1040,6 +1060,10 @@ ANKInnerPreconIts:
desc: >
Similar to the :py:data:`NKInnerPreconIts` option but for the ANK solver.

ANKInnerPreconItsCoarse:
desc: >
Same as :py:data:`ANKInnerPreconIts` but for the coarse levels when using ``multigrid`` for :py:data:`ANKGlobalPreconditioner`.

ANKOuterPreconIts:
desc: >
Similar to the :py:data:`NKOuterPreconIts` option but for the ANK solver.
Expand Down Expand Up @@ -1464,18 +1488,30 @@ ILUFill:
Typical values are 1 for easy cases and up to 3 for more difficult cases.
More levels of fill result in a stronger preconditioner which will result in fewer (linear) iterations, but individual iterations will be more costly and consume more memory.

ILUFillCoarse:
desc: >
Same as :py:data:`ILUFill` but for the coarse levels when using ``multigrid`` for :py:data:`globalPreconditioner`.

ASMOverlap:
desc: >
The number of overlap levels in the additive Schwarz preconditioner for the adjoint solution.
More overlap levels result in a stronger preconditioner, at the expense of more expensive iterations and more memory.
Typical values range from 1 for easy cases and up to 2 or 3 for more difficult cases.

ASMOverlapCoarse:
desc: >
Same as :py:data:`ASMOverlap` but for the coarse levels when using ``multigrid`` for :py:data:`globalPreconditioner`.

innerPreconIts:
desc: >
Number of local preconditioning iterations for the adjoint solution.
Increasing this number may help with difficult problems.
However, each iteration will take more time.

innerPreconItsCoarse:
desc: >
Same as :py:data:`innerPreconItsCoarse` but for the coarse levels when using ``multigrid`` for :py:data:`globalPreconditioner`.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo innerPreconItsCoarse

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed, thanks


outerPreconIts:
desc: >
Number of global preconditioning iterations for the adjoint solution.
Expand Down
12 changes: 10 additions & 2 deletions src/NKSolver/NKSolvers.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,11 @@ module NKSolver
integer(kind=intType) :: NK_jacobianLag
integer(kind=intType) :: NK_subspace
integer(kind=intType) :: NK_asmOverlap
integer(kind=intType) :: NK_asmOverlapCoarse
integer(kind=intType) :: NK_iluFill
integer(kind=intType) :: NK_iluFillCoarse
integer(kind=intType) :: NK_innerPreConIts
integer(kind=intType) :: NK_innerPreConItsCoarse
integer(kind=intType) :: NK_outerPreConIts
integer(kind=intType) :: NK_AMGLevels
integer(kind=intType) :: NK_AMGNSmooth
Expand Down Expand Up @@ -421,7 +424,8 @@ subroutine FormJacobianNK
else
call setupStandardMultigrid(NK_KSP, kspObjectType, NK_subSpace, &
preConSide, NK_asmOverlap, NK_outerPreConIts, &
localOrdering, NK_iluFill, NK_innerPreConIts)
localOrdering, NK_iluFill, NK_innerPreConIts, &
NK_asmOverlapCoarse, NK_iluFillCoarse, NK_innerPreConItsCoarse)
end if

! Don't do iterative refinement
Expand Down Expand Up @@ -1660,8 +1664,11 @@ module ANKSolver
integer(kind=intType) :: ANK_subSpace
integer(kind=intType) :: ANK_maxIter
integer(kind=intType) :: ANK_asmOverlap
integer(kind=intType) :: ANK_asmOverlapCoarse
integer(kind=intType) :: ANK_iluFill
integer(kind=intType) :: ANK_iluFillCoarse
integer(kind=intType) :: ANK_innerPreConIts
integer(kind=intType) :: ANK_innerPreConItsCoarse
integer(kind=intType) :: ANK_outerPreConIts
integer(kind=intType) :: ANK_AMGLevels
integer(kind=intType) :: ANK_AMGNSmooth
Expand Down Expand Up @@ -2021,7 +2028,8 @@ subroutine FormJacobianANK
else if (ANK_precondType == 'mg') then
call setupStandardMultigrid(ANK_KSP, kspObjectType, subSpace, &
preConSide, ANK_asmOverlap, outerPreConIts, &
localOrdering, ANK_iluFill, ANK_innerPreConIts)
localOrdering, ANK_iluFill, ANK_innerPreConIts, &
ANK_asmOverlapCoarse, ANK_iluFillCoarse, ANK_innerPreConItsCoarse)
end if

! Don't do iterative refinement
Expand Down
3 changes: 2 additions & 1 deletion src/adjoint/adjointAPI.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,7 +918,8 @@ subroutine setupPETScKsp
else if (PreCondType == 'mg') then

call setupStandardMultigrid(adjointKSP, ADjointSolverType, adjRestart, adjointPCSide, &
overlap, outerPreconIts, matrixOrdering, fillLevel, innerPreConIts)
overlap, outerPreconIts, matrixOrdering, fillLevel, innerPreConIts, &
overlapCoarse, fillLevelCoarse, innerPreConItsCoarse)
end if

! Setup monitor if necessary:
Expand Down
36 changes: 30 additions & 6 deletions src/adjoint/adjointUtils.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1562,12 +1562,15 @@ subroutine setupStandardKSP(kspObject, kspObjectType, gmresRestart, preConSide,
end subroutine setupStandardKSP

subroutine setupStandardMultigrid(kspObject, kspObjectType, gmresRestart, preConSide, &
ASMoverlap, outerPreconIts, localMatrixOrdering, fillLevel, localPreConIts)
ASMOverlap, outerPreconIts, localMatrixOrdering, fillLevel, localPreConIts, &
ASMOverlapCoarse, fillLevelCoarse, localPreConItsCoarse)

use constants
use utils, only: ECHk
use amg, only: amgOuterIts, amgASMOverlap, amgFillLevel, amgMatrixOrdering, amgLocalPreConIts, &
setupShellPC, destroyShellPC, applyShellPC
use inputADjoint, only: GMRESOrthogType
use amg, only: amgOuterIts, amgASMOverlapFine, amgASMOverlapCoarse, amgMatrixOrdering, &
setupShellPC, destroyShellPC, applyShellPC, &
amgFillLevelFine, amgFillLevelCoarse, amgLocalPreConItsFine, amgLocalPreConItsCoarse
#include <petsc/finclude/petsc.h>
use petsc
implicit none
Expand All @@ -1576,6 +1579,7 @@ subroutine setupStandardMultigrid(kspObject, kspObjectType, gmresRestart, preCon
KSP kspObject
character(len=*), intent(in) :: kspObjectType, preConSide, localMatrixOrdering
integer(kind=intType), intent(in) :: ASMOverlap, fillLevel, gmresRestart, outerPreconIts, localPreConIts
integer(kind=intType), intent(in) :: ASMOverlapCoarse, fillLevelCoarse, localPreConItsCoarse

! Working Variables
PC shellPC
Expand All @@ -1595,6 +1599,23 @@ subroutine setupStandardMultigrid(kspObject, kspObjectType, gmresRestart, preCon
call KSPGMRESSetRestart(kspObject, gmresRestart, ierr)
call EChk(ierr, __FILE__, __LINE__)

! Set the orthogonalization method for GMRES
select case (GMRESOrthogType)
case ('modified_gram_schmidt')
! Use modified Gram-Schmidt
call KSPGMRESSetOrthogonalization(kspObject, KSPGMRESModifiedGramSchmidtOrthogonalization, ierr)
case ('cgs_never_refine')
! Use classical Gram-Schmidt with no refinement
call KSPGMRESSetCGSRefinementType(kspObject, KSP_GMRES_CGS_REFINE_NEVER, ierr)
case ('cgs_refine_if_needed')
! Use classical Gram-Schmidt with refinement if needed
call KSPGMRESSetCGSRefinementType(kspObject, KSP_GMRES_CGS_REFINE_IFNEEDED, ierr)
case ('cgs_always_refine')
! Use classical Gram-Schmidt with refinement at every iteration
call KSPGMRESSetCGSRefinementType(kspObject, KSP_GMRES_CGS_REFINE_ALWAYS, ierr)
end select
call EChk(ierr, __FILE__, __LINE__)

call KSPGetPC(kspObject, shellPC, ierr)
call EChk(ierr, __FILE__, __LINE__)

Expand All @@ -1612,10 +1633,13 @@ subroutine setupStandardMultigrid(kspObject, kspObjectType, gmresRestart, preCon

! Save the remaining variables in the AMG module
amgOuterIts = outerPreConIts
amgASMOverlap = asmOverlap
amgFillLevel = fillLevel
amgMatrixOrdering = localMatrixOrdering
amgLocalPreConIts = localPreConIts
amgASMOverlapFine = ASMOverlap
amgFillLevelFine = fillLevel
amgLocalPreConItsFine = localPreConIts
amgASMOverlapCoarse = ASMOverlapCoarse
amgFillLevelCoarse = fillLevelCoarse
amgLocalPreConItsCoarse = localPreConItsCoarse

end subroutine setupStandardMultigrid

Expand Down
6 changes: 6 additions & 0 deletions src/f2py/adflow.pyf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ python module libadflow

character(len=maxstringlen) :: nk_precondtype
integer(kind=inttype) :: nk_innerpreconits
integer(kind=inttype) :: nk_innerpreconitscoarse
integer(kind=inttype) :: nk_outerpreconits
integer(kind=inttype) :: nk_amglevels
integer(kind=inttype) :: nk_amgnsmooth
Expand All @@ -378,8 +379,10 @@ python module libadflow
integer(kind=inttype) :: nk_ls
integer(kind=inttype) :: nk_subspace
integer(kind=inttype) :: nk_asmoverlap
integer(kind=inttype) :: nk_asmoverlapcoarse
real(kind=realtype) :: nk_rtolinit
integer(kind=inttype) :: nk_ilufill
integer(kind=inttype) :: nk_ilufillcoarse
real(kind=realtype) :: nk_switchtol
real(kind=realtype) :: nk_cfl0
real(kind=realtype) :: nk_fixedstep
Expand Down Expand Up @@ -442,6 +445,7 @@ python module libadflow

character(len=maxstringlen) :: ank_precondtype
integer(kind=inttype) :: ank_innerpreconits
integer(kind=inttype) :: ank_innerpreconitscoarse
integer(kind=inttype) :: ank_outerpreconits
integer(kind=inttype) :: ank_amglevels
integer(kind=inttype) :: ank_amgnsmooth
Expand All @@ -452,10 +456,12 @@ python module libadflow
integer(kind=inttype) :: ank_subspace
integer(kind=inttype) :: ank_maxiter
integer(kind=inttype) :: ank_asmoverlap
integer(kind=inttype) :: ank_asmoverlapcoarse
real(kind=realtype) :: ank_rtol
real(kind=realtype) :: ank_atol_buffer
real(kind=realtype) :: ank_linresmax
integer(kind=inttype) :: ank_ilufill
integer(kind=inttype) :: ank_ilufillcoarse
real(kind=realtype) :: ank_switchtol
real(kind=realtype) :: ank_cfl
real(kind=realtype) :: ank_cfl0
Expand Down
5 changes: 3 additions & 2 deletions src/modules/inputParam.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,8 @@ module inputADjoint

! FillLevel : Number of levels of fill for the ILU local PC
! Overlap : Amount of overlap in the ASM PC
integer(kind=intType) :: FillLevel, Overlap
integer(kind=intType) :: fillLevel, overlap
integer(kind=intType) :: fillLevelCoarse, overlapCoarse

! adjRelTol : Relative tolerance
! adjAbsTol : Absolute tolerance
Expand All @@ -818,7 +819,7 @@ module inputADjoint
! outerPCIts : Number of iterations to run for on (global) preconditioner
! intterPCIts : Number of iterations to run on local preconditioner
integer(kind=intType) :: outerPreConIts
integer(kind=intType) :: innerPreConIts
integer(kind=intType) :: innerPreConIts, innerPreConItsCoarse
integer(kind=intType) :: adjAMGLevels, adjAMGNSmooth

logical :: printTiming
Expand Down
18 changes: 16 additions & 2 deletions src/solver/amg.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,18 @@ module amg

! The number of local ILU iterations
integer(kind=intType) amgLocalPreConIts
integer(kind=intType) amgLocalPreConItsFine
integer(kind=intType) amgLocalPreConItsCoarse

! ASM overlap for the solver/smoother
integer(kind=intType) :: amgASMOverlap
integer(kind=intType) amgASMOverlap
integer(kind=intType) amgASMOverlapFine
integer(kind=intType) amgASMOverlapCoarse

! ILU fill for the solver/smoother
integer(kind=intType) :: amgFillLevel
integer(kind=intType) amgFillLevel
integer(kind=intType) amgFillLevelFine
integer(kind=intType) amgFillLevelCoarse

! Ordering
character(len=maxStringLen) :: amgMatrixOrdering
Expand Down Expand Up @@ -529,9 +535,17 @@ subroutine setupShellPC(pc, ierr)
if (lvl == 1) then
call KSPSetOperators(kspLevels(lvl), fineMat, fineMat, ierr)
call EChk(ierr, __FILE__, __LINE__)

amgFillLevel = amgFillLevelFine
amgLocalPreConIts = amgLocalPreConItsFine
amgASMOverlap = amgASMOverlapFine
else
call KSPSetOperators(kspLevels(lvl), A(lvl), A(lvl), ierr)
call EChk(ierr, __FILE__, __LINE__)

amgFillLevel = amgFillLevelCoarse
amgLocalPreConIts = amgLocalPreConItsCoarse
amgASMOverlap = amgASMOverlapCoarse
end if

call kspsetnormtype(kspLevels(lvl), KSP_NORM_NONE, ierr)
Expand Down
Loading