Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

do_2D mode with surface coverage restart fixes. #184

Merged
merged 3 commits into from
Feb 11, 2025
Merged

do_2D mode with surface coverage restart fixes. #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d917bf8
do_2D mode with surface coverage restart fixes.
ajnonaka Feb 11, 2025
06bf248
formatting
ajnonaka Feb 11, 2025
1212ec0
fix surfcov restart with random numbers
ajnonaka Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 59 additions & 5 deletions src_compressible_stag/Checkpoint.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ void WriteCheckPoint2D(int step,
const std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
const std::array<MultiFab, AMREX_SPACEDIM>& velVars,
const amrex::MultiFab& coVars,
const amrex::MultiFab& surfcov,
const amrex::MultiFab& surfcovMeans,
const amrex::MultiFab& surfcovVars,
const amrex::MultiFab& surfcovcoVars,
const amrex::MultiFab& spatialCross, int /*ncross*/)
{
// timer for profiling
Expand Down Expand Up @@ -410,6 +414,18 @@ void WriteCheckPoint2D(int step,
VisMF::Write(coVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "coVars"));

if (n_ads_spec>0) {
// surfcov
VisMF::Write(surfcov,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcov"));
VisMF::Write(surfcovMeans,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovMeans"));
VisMF::Write(surfcovVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovVars"));
VisMF::Write(surfcovcoVars,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "surfcovcoVars"));
}

// spatialCross
VisMF::Write(spatialCross,
amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "spatialCross")); // (do later)
Expand Down Expand Up @@ -751,7 +767,7 @@ void ReadCheckPoint3D(int& step,
MPI_Comm_size(MPI_COMM_WORLD, &n_ranks);

// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
if (seed < 0) {
if (seed == -1) {

#ifdef AMREX_USE_CUDA
Abort("Restart with negative seed not supported on GPU");
Expand Down Expand Up @@ -902,6 +918,10 @@ void ReadCheckPoint2D(int& step,
std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
std::array<MultiFab, AMREX_SPACEDIM>& velVars,
amrex::MultiFab& coVars,
amrex::MultiFab& surfcov,
amrex::MultiFab& surfcovMeans,
amrex::MultiFab& surfcovVars,
amrex::MultiFab& surfcovcoVars,
amrex::MultiFab& spatialCross,
int ncross,
BoxArray& ba, DistributionMapping& dmap)
Expand Down Expand Up @@ -981,6 +1001,14 @@ void ReadCheckPoint2D(int& step,

// coVars
coVars.define(ba,dmap,26,0);

if (n_ads_spec>0) {
// surfcov
surfcov.define(ba,dmap,n_ads_spec,0);
surfcovMeans.define(ba,dmap,n_ads_spec,0);
surfcovVars.define(ba,dmap,n_ads_spec,0);
surfcovcoVars.define(ba,dmap,n_ads_spec*6,0);
}

// spatialCross
spatialCross.define(ba,dmap,ncross,0); // (do later)
Expand All @@ -995,10 +1023,16 @@ void ReadCheckPoint2D(int& step,
int n_ranks;
MPI_Comm_size(MPI_COMM_WORLD, &n_ranks);

// don't read in all the rng states at once (overload filesystem)
// one at a time write out the rng states to different files, one for each MPI rank
// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
if (seed == -1) {

#ifdef AMREX_USE_CUDA
Abort("Restart with negative seed not supported on GPU");
#endif

// don't read in all the rng states at once (overload filesystem)
// one at a time write out the rng states to different files, one for each MPI rank
// Need to add guard for restarting with more MPI ranks than the previous checkpointing run
for (int rank=0; rank<n_ranks; ++rank) {

if (comm_rank == rank) {
Expand Down Expand Up @@ -1053,6 +1087,10 @@ void ReadCheckPoint2D(int& step,
Read_Copy_MF_Checkpoint(cumom[1],"cumomy",checkpointname,ba_old,dmap_old,1,1,1);
Read_Copy_MF_Checkpoint(cumom[2],"cumomz",checkpointname,ba_old,dmap_old,1,1,2);

if (n_ads_spec>0) {
Read_Copy_MF_Checkpoint(surfcov,"surfcov",checkpointname,ba_old,dmap_old,n_ads_spec,0);
}

// Set all stats to zero if reset stats, else read
if (reset_stats == 1) {
cuMeans.setVal(0.0);
Expand All @@ -1066,6 +1104,13 @@ void ReadCheckPoint2D(int& step,
cumomVars[d].setVal(0.);
}
coVars.setVal(0.0);
if (n_ads_spec>0) {
for (int m=0;m<n_ads_spec;m++) {
surfcovMeans.setVal(0.0);
surfcovVars.setVal(0.0);
surfcovcoVars.setVal(0.0);
}
}
spatialCross.setVal(0.0);
}
else {
Expand All @@ -1092,6 +1137,12 @@ void ReadCheckPoint2D(int& step,
Read_Copy_MF_Checkpoint(cumomVars[0],"cumomvarx",checkpointname,ba_old,dmap_old,1,0,0);
Read_Copy_MF_Checkpoint(cumomVars[1],"cumomvary",checkpointname,ba_old,dmap_old,1,0,1);
Read_Copy_MF_Checkpoint(cumomVars[2],"cumomvarz",checkpointname,ba_old,dmap_old,1,0,2);

if (n_ads_spec>0) {
Read_Copy_MF_Checkpoint(surfcovMeans,"surfcovMeans",checkpointname,ba_old,dmap_old,n_ads_spec,0);
Read_Copy_MF_Checkpoint(surfcovVars,"surfcovVars",checkpointname,ba_old,dmap_old,n_ads_spec,0);
Read_Copy_MF_Checkpoint(surfcovcoVars,"surfcovcoVars",checkpointname,ba_old,dmap_old,n_ads_spec*6,0);
}
}

// FillBoundaries
Expand Down Expand Up @@ -1209,8 +1260,11 @@ void ReadCheckPoint1D(int& step,
coVars.define(ba,dmap,26,0);

// spatialCross
if (all_correl) spatialCross.define(ba,dmap,ncross*5,0); // for five x*: [0, fl(n_cells[0]/4), fl(n_cells[0]/2), fl(n_cells[0]*3/4), n_cells[0]-1]
else spatialCross.define(ba,dmap,ncross,0);
if (all_correl) {
spatialCross.define(ba,dmap,ncross*5,0); // for five x*: [0, fl(n_cells[0]/4), fl(n_cells[0]/2), fl(n_cells[0]*3/4), n_cells[0]-1]
} else {
spatialCross.define(ba,dmap,ncross,0);
}

}

Expand Down
8 changes: 8 additions & 0 deletions src_compressible_stag/compressible_functions_stag.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ void WriteCheckPoint2D(int step,
const std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
const std::array<MultiFab, AMREX_SPACEDIM>& velVars,
const amrex::MultiFab& coVars,
const amrex::MultiFab& surfcov,
const amrex::MultiFab& surfcovMeans,
const amrex::MultiFab& surfcovVars,
const amrex::MultiFab& surfcovcoVars,
const amrex::MultiFab& spatialCross, int ncross);

void ReadCheckPoint2D(int& step,
Expand All @@ -270,6 +274,10 @@ void ReadCheckPoint2D(int& step,
std::array<MultiFab, AMREX_SPACEDIM>& velMeans,
std::array<MultiFab, AMREX_SPACEDIM>& velVars,
amrex::MultiFab& coVars,
amrex::MultiFab& surfcov,
amrex::MultiFab& surfcovMeans,
amrex::MultiFab& surfcovVars,
amrex::MultiFab& surfcovcoVars,
amrex::MultiFab& spatialCross, int ncross,
BoxArray& ba, DistributionMapping& dmap);

Expand Down
29 changes: 20 additions & 9 deletions src_compressible_stag/main_driver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,17 +508,22 @@ void main_driver(const char* argv)
if (do_1D) {
ReadCheckPoint1D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross1D, ncross, ba, dmap);
vel, velMeans, velVars, coVars,
spatialCross1D, ncross, ba, dmap);
}
else if (do_2D) {
ReadCheckPoint2D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross2D, ncross, ba, dmap);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross2D, ncross, ba, dmap);
}
else {
ReadCheckPoint3D(step_start, time, statsCount, geom, domain, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, surfcov, surfcovMeans, surfcovVars, surfcovcoVars, spatialCross3D, ncross, turbforce, ba, dmap);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross3D, ncross, turbforce, ba, dmap);
}

if (reset_stats == 1) statsCount = 1;
Expand Down Expand Up @@ -1025,16 +1030,17 @@ void main_driver(const char* argv)
} else {
Abort("splitting_MFsurfchem can be 0 or 1");
}

VisMF::Write(surfcov,"a_surfcov");
}

// FHD
if (turbRestartRun) {
RK3stepStag(cu, cumom, prim, vel, source, eta, zeta, kappa, chi, D,
faceflux, edgeflux_x, edgeflux_y, edgeflux_z, cenflux, ranchem, geom, dt, step, turbforce);
} else {
calculateTransportCoeffs(prim, eta, zeta, kappa, chi, D);
}
else {
calculateTransportCoeffs(prim, eta, zeta, kappa, chi, D);
}

if (n_ads_spec>0 && splitting_MFsurfchem == 1) {
sample_MFsurfchem(cu, prim, surfcov, dNadsdes, dNads, dNdes, geom, dt/2.0);
Expand Down Expand Up @@ -1576,17 +1582,22 @@ void main_driver(const char* argv)
if (do_1D) {
WriteCheckPoint1D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross1D, ncross);
vel, velMeans, velVars, coVars,
spatialCross1D, ncross);
}
else if (do_2D) {
WriteCheckPoint2D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, spatialCross2D, ncross);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross2D, ncross);
}
else {
WriteCheckPoint3D(step, time, statsCount, geom, cu, cuMeans, cuVars, prim,
primMeans, primVars, cumom, cumomMeans, cumomVars,
vel, velMeans, velVars, coVars, surfcov, surfcovMeans, surfcovVars, surfcovcoVars, spatialCross3D, ncross, turbforce);
vel, velMeans, velVars, coVars,
surfcov, surfcovMeans, surfcovVars, surfcovcoVars,
spatialCross3D, ncross, turbforce);
}
}

Expand Down