Skip to content

Commit

Permalink
Write restart files using the write grid component. Combined with WW3…
Browse files Browse the repository at this point in the history 
… update #1673 (#1633)

* Add 4 new quilting restart tests

* Remove tests/qr.conf

* Fix fv3_conf/regional_run.IN script to correctly copy restart files from dependent directory

* Clean up Thompson table files (#38)

* update control_qr_p8 control_restart_qr_p8

* Disable cpld_control_ciceC_p8 and cpld_control_c192_p8/cpld_restart_c192_p8 on Gaea

* Update FV3 and WW3

Co-authored-by: Dusan Jovic <[email protected]>
Co-authored-by: Jessica Meixner <[email protected]>
  • Loading branch information
@DusanJovic-NOAA @JessicaMeixner-NOAA
3 people authored Mar 28, 2023
1 parent 889254a commit 85dc0fd
Show file tree
Hide file tree
Showing 28 changed files with 12,136 additions and 9,869 deletions.
2 changes: 1 addition & 1 deletion FV3
Submodule FV3 updated 12 files
+2 −0 CMakeLists.txt
+1 −1 atmos_cubed_sphere
+22 −18 atmos_model.F90
+89 −42 fv3_cap.F90
+920 −0 io/FV3GFS_restart_io.F90
+0 −3 io/module_write_internal_state.F90
+31 −19 io/module_write_netcdf.F90
+587 −0 io/module_write_restart_netcdf.F90
+392 −118 io/module_wrt_grid_comp.F90
+19 −4 io/post_fv3.F90
+180 −27 module_fcst_grid_comp.F90
+1 −1 module_fv3_config.F90
2 changes: 1 addition & 1 deletion WW3
Submodule WW3 updated 105 files
2,844 changes: 1,554 additions & 1,290 deletions tests/RegressionTests_acorn.intel.log
View file

Large diffs are not rendered by default.

750 changes: 375 additions & 375 deletions tests/RegressionTests_cheyenne.gnu.log
View file

Large diffs are not rendered by default.

2,882 changes: 1,551 additions & 1,331 deletions tests/RegressionTests_cheyenne.intel.log
View file

Large diffs are not rendered by default.

2,972 changes: 1,522 additions & 1,450 deletions tests/RegressionTests_gaea.intel.log
View file

Large diffs are not rendered by default.

754 changes: 377 additions & 377 deletions tests/RegressionTests_hera.gnu.log
View file

Large diffs are not rendered by default.

2,900 changes: 1,582 additions & 1,318 deletions tests/RegressionTests_hera.intel.log
View file

Large diffs are not rendered by default.

2,642 changes: 1,453 additions & 1,189 deletions tests/RegressionTests_jet.intel.log
View file

Large diffs are not rendered by default.

2,888 changes: 1,576 additions & 1,312 deletions tests/RegressionTests_orion.intel.log
View file

Large diffs are not rendered by default.

2,667 changes: 1,471 additions & 1,196 deletions tests/RegressionTests_wcoss2.intel.log
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions tests/default_vars.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ export INPES=$INPES_dflt
export JNPES=$JNPES_dflt
export RESTART_INTERVAL=0
export QUILTING=.true.
export QUILTING_RESTART=.false.
export WRITE_GROUP=1
export WRTTASK_PER_GROUP=6
export ITASKS=1
Expand Down
8 changes: 4 additions & 4 deletions tests/fv3_conf/control_run.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,10 @@ fi

#Thompson
if [ $IMP_PHYSICS = 8 ]; then
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
fi

#prognostic aerosols
Expand Down
8 changes: 4 additions & 4 deletions tests/fv3_conf/cpld_control_run.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ cp @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127/oro_data_ls* ./INPUT
cp @[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127/oro_data_ss* ./INPUT

if [ $IMP_PHYSICS = 8 ]; then
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN CCN_ACTIVATE.BIN
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat freezeH2O.dat
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qg.dat qr_acr_qg.dat
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qs.dat qr_acr_qs.dat
cp @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN CCN_ACTIVATE.BIN
cp @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat freezeH2O.dat
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat qr_acr_qgV2.dat
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat qr_acr_qsV2.dat
fi

#prognostic aerosols
Expand Down
8 changes: 4 additions & 4 deletions tests/fv3_conf/hafs_fv3_run.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ else
fi

if [ ${CNTL_DIR} = "hafs_regional_atm_thompson_gfdlsf" ]; then
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
fi

if [[ $POSTAPP = 'hafs' ]]; then
Expand Down
14 changes: 7 additions & 7 deletions tests/fv3_conf/regional_run.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ cp @[INPUTDATA_ROOT]/FV3_input_data_INCCN_aeroclim/aer_data/LUTS/optics_SS.v3_3.
cp @[INPUTDATA_ROOT]/FV3_input_data_INCCN_aeroclim/aer_data/LUTS/optics_SU.v1_3.dat optics_SU.dat

if [ $WARM_START = .true. ]; then
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.coupler.res INPUT/coupler.res
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.nc INPUT/fv_core.res.nc
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.tile1.nc INPUT/fv_core.res.tile1.nc
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_srf_wnd.res.tile1.nc INPUT/fv_srf_wnd.res.tile1.nc
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.fv_tracer.res.tile1.nc INPUT/fv_tracer.res.tile1.nc
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.phy_data.nc INPUT/phy_data.nc
cp ../regional_control${RT_SUFFIX}/RESTART/20220824.030000.sfc_data.nc INPUT/sfc_data.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.coupler.res INPUT/coupler.res
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.nc INPUT/fv_core.res.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_core.res.tile1.nc INPUT/fv_core.res.tile1.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_srf_wnd.res.tile1.nc INPUT/fv_srf_wnd.res.tile1.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.fv_tracer.res.tile1.nc INPUT/fv_tracer.res.tile1.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.phy_data.nc INPUT/phy_data.nc
cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/20220824.030000.sfc_data.nc INPUT/sfc_data.nc
fi
if [ $WRITE_RESTART_WITH_BCS = .true. ]; then
cp @[INPUTDATA_ROOT]/FV3_regional/RESTART/fv_core.res.tile1_new.nc RESTART/fv_core.res.tile1_new.nc
Expand Down
8 changes: 4 additions & 4 deletions tests/fv3_conf/rrfs_warm_run.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ touch data_table

# Thompson MP lookup tables - copy standard and SIONlib tables of precomputed tables
if [ $IMP_PHYSICS = 8 ]; then
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_input_data_gsd/CCN_ACTIVATE.BIN .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qsV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/qr_acr_qgV2.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/freezeH2O.dat .
cp @[INPUTDATA_ROOT]/FV3_fix/CCN_ACTIVATE.BIN .
fi

cp ${PATHRT}/parm/postxconfig-NT-fv3lam.txt postxconfig-NT.txt
Expand Down
1 change: 1 addition & 0 deletions tests/parm/model_configure.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ restart_interval: @[RESTART_INTERVAL]
output_1st_tstep_rst: .false.

quilting: @[QUILTING]
quilting_restart: @[QUILTING_RESTART]
write_groups: @[WRITE_GROUP]
write_tasks_per_group: @[WRTTASK_PER_GROUP]
itasks: @[ITASKS]
Expand Down
1 change: 1 addition & 0 deletions tests/parm/model_configure_regional.IN
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ cpl: .false.
write_dopost: @[WRITE_DOPOST]
output_history: .true.
quilting: @[QUILTING]
quilting_restat: @[QUILTING_RESTART]
write_groups: 1
write_tasks_per_group: 10
num_files: 2
Expand Down
14 changes: 10 additions & 4 deletions tests/rt.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,16 @@ COMPILE | -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8,FV3_GFS_cpld_rasmgsho
RUN | cpld_control_gfsv17 | - wcoss2.intel | fv3 |
RUN | cpld_control_p8 | | fv3 |
RUN | cpld_restart_p8 | | | cpld_control_p8
RUN | cpld_control_qr_p8 | | |
RUN | cpld_restart_qr_p8 | | | cpld_control_qr_p8
RUN | cpld_2threads_p8 | | |
RUN | cpld_decomp_p8 | | |
RUN | cpld_mpi_p8 | | |
#RUN | cpld_multigrid_p8 | | |
RUN | cpld_control_ciceC_p8 | | fv3 |
RUN | cpld_control_ciceC_p8 | - gaea.intel | fv3 |

RUN | cpld_control_c192_p8 | - wcoss2.intel jet.intel acorn.intel | fv3 |
RUN | cpld_restart_c192_p8 | - wcoss2.intel jet.intel acorn.intel | | cpld_control_c192_p8
RUN | cpld_control_c192_p8 | - wcoss2.intel jet.intel acorn.intel gaea.intel | fv3 |
RUN | cpld_restart_c192_p8 | - wcoss2.intel jet.intel acorn.intel gaea.intel | | cpld_control_c192_p8

RUN | cpld_bmark_p8 | - jet.intel cheyenne.intel acorn.intel | fv3 |
RUN | cpld_restart_bmark_p8 | - jet.intel cheyenne.intel acorn.intel | | cpld_bmark_p8
Expand Down Expand Up @@ -62,10 +64,12 @@ RUN | control_iovr4
RUN | control_iovr5 | | fv3 |

RUN | control_p8 | | fv3 |
RUN | control_p8_lndp | | fv3 |
RUN | control_restart_p8 | | | control_p8
RUN | control_qr_p8 | | |
RUN | control_restart_qr_p8 | | | control_qr_p8
RUN | control_decomp_p8 | | |
RUN | control_2threads_p8 | | |
RUN | control_p8_lndp | | fv3 |

RUN | control_p8_rrtmgp | | fv3 |
RUN | control_p8_mynn | | fv3 |
Expand All @@ -74,6 +78,8 @@ RUN | merra2_thompson

RUN | regional_control | | fv3 |
RUN | regional_restart | | | regional_control
RUN | regional_control_qr | | |
RUN | regional_restart_qr | | | regional_control_qr
RUN | regional_decomp | | |
RUN | regional_2threads | | |
RUN | regional_noquilt | | fv3 |
Expand Down
4 changes: 2 additions & 2 deletions tests/rt_utils.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ function compute_petbounds_and_tasks() {
# ATM
ATM_io_tasks=${ATM_io_tasks:-0}
if [[ $((ATM_compute_tasks + ATM_io_tasks)) -gt 0 ]]; then
atm_petlist_bounds="${n} $((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks - 1))"
n=$((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks))
atm_petlist_bounds="${n} $((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks*atm_omp_num_threads - 1))"
n=$((n + ATM_compute_tasks*atm_omp_num_threads + ATM_io_tasks*atm_omp_num_threads))
fi

# OCN
Expand Down
5 changes: 5 additions & 0 deletions tests/run_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,11 @@ cp ${PATHTR}/modulefiles/ufs_common* ./modulefiles/.
# Get the shell file that loads the "module" command and purges modules:
cp ${PATHRT}/module-setup.sh module-setup.sh

if [[ $MACHINE_ID == wcoss2.* ]] || [[ $MACHINE_ID == acorn.* ]] ; then
# for compare_ncfile.py
module load gcc/10.3.0 python/3.8.6
fi

SRCD="${PATHTR}"
RUND="${RUNDIR}"

Expand Down
176 changes: 176 additions & 0 deletions tests/tests/control_qr_p8
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
###############################################################################
#
# Global control test GFSv16 atmosphere only at C96L127, P8 configuration
#
###############################################################################

export TEST_DESCR="Compare global control results with previous trunk version using quilting restart in atm"

export CNTL_DIR=control_p8

export LIST_FILES="sfcf000.nc \
sfcf021.nc \
sfcf024.nc \
atmf000.nc \
atmf021.nc \
atmf024.nc \
GFSFLX.GrbF00 \
GFSFLX.GrbF21 \
GFSFLX.GrbF24 \
GFSPRS.GrbF00 \
GFSPRS.GrbF21 \
GFSPRS.GrbF24 \
RESTART/20210323.060000.coupler.res \
RESTART/20210323.060000.fv_core.res.nc \
RESTART/20210323.060000.fv_core.res.tile1.nc \
RESTART/20210323.060000.fv_core.res.tile2.nc \
RESTART/20210323.060000.fv_core.res.tile3.nc \
RESTART/20210323.060000.fv_core.res.tile4.nc \
RESTART/20210323.060000.fv_core.res.tile5.nc \
RESTART/20210323.060000.fv_core.res.tile6.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile1.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile2.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile3.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile4.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile5.nc \
RESTART/20210323.060000.fv_srf_wnd.res.tile6.nc \
RESTART/20210323.060000.fv_tracer.res.tile1.nc \
RESTART/20210323.060000.fv_tracer.res.tile2.nc \
RESTART/20210323.060000.fv_tracer.res.tile3.nc \
RESTART/20210323.060000.fv_tracer.res.tile4.nc \
RESTART/20210323.060000.fv_tracer.res.tile5.nc \
RESTART/20210323.060000.fv_tracer.res.tile6.nc \
RESTART/20210323.060000.phy_data.tile1.nc \
RESTART/20210323.060000.phy_data.tile2.nc \
RESTART/20210323.060000.phy_data.tile3.nc \
RESTART/20210323.060000.phy_data.tile4.nc \
RESTART/20210323.060000.phy_data.tile5.nc \
RESTART/20210323.060000.phy_data.tile6.nc \
RESTART/20210323.060000.sfc_data.tile1.nc \
RESTART/20210323.060000.sfc_data.tile2.nc \
RESTART/20210323.060000.sfc_data.tile3.nc \
RESTART/20210323.060000.sfc_data.tile4.nc \
RESTART/20210323.060000.sfc_data.tile5.nc \
RESTART/20210323.060000.sfc_data.tile6.nc"

export_fv3
export NPZ=127
export NPZP=128
export DT_ATMOS=720
export DT_INNER=${DT_ATMOS}
export SYEAR=2021
export SMONTH=03
export SDAY=22
export SHOUR=06
export QUILTING_RESTART=.true.
export RESTART_INTERVAL="12 -1"
export OUTPUT_GRID='gaussian_grid'
export WRITE_DOPOST=.true.
export OUTPUT_FH='0 21 24'

# P7 default
export IALB=2
export IEMS=2
export LSM=2
export IOPT_DVEG=4
export IOPT_CRS=2
export IOPT_RAD=3
export IOPT_ALB=1
export IOPT_STC=3
# P8
export IOPT_SFC=3
export IOPT_TRS=2

# FV3 P7 settings
export D2_BG_K1=0.20
export D2_BG_K2=0.04
export PSM_BC=1
# P8
export DDDMP=0.1

# P7 Merra2 Aerosols & NSST
export USE_MERRA2=.true.
export IAER=1011
export NSTF_NAME=2,1,0,0,0

export LHEATSTRG=.false.
export LSEASPRAY=.true.

# P7 UGWP1
export GWD_OPT=2
export DO_UGWP_V1=.false.
export KNOB_UGWP_VERSION=0
export KNOB_UGWP_NSLOPE=1
export DO_UGWP_V0=.true.
export DO_GSL_DRAG_LS_BL=.false.
export DO_GSL_DRAG_SS=.true.
export DO_GSL_DRAG_TOFD=.false.
export DO_UGWP_V1_OROG_ONLY=.false.
export DO_UGWP_V0_NST_ONLY=.false.
export LDIAG_UGWP=.false.

# P7 CA
export DO_CA=.true.
export CA_SGS=.true.
export CA_GLOBAL=.false.
export NCA=1
export NCELLS=5
export NLIVES=12
export NTHRESH=18
export NSEED=1
export NFRACSEED=0.5
export CA_TRIGGER=.true.
export NSPINUP=1
export ISEED_CA=12345

# P7 settings
export TILEDFIX=.true.
export FNALBC="'C96.snowfree_albedo.tileX.nc'"
export FNALBC2="'C96.facsf.tileX.nc'"
export FNTG3C="'C96.substrate_temperature.tileX.nc'"
export FNVEGC="'C96.vegetation_greenness.tileX.nc'"
export FNVETC="'C96.vegetation_type.tileX.nc'"
export FNSOTC="'C96.soil_type.tileX.nc'"
export FNSMCC=${FNSMCC_control}
export FNMSKH=${FNMSKH_control}
export FNVMNC="'C96.vegetation_greenness.tileX.nc'"
export FNVMXC="'C96.vegetation_greenness.tileX.nc'"
export FNSLPC="'C96.slope_type.tileX.nc'"
export FNABSC="'C96.maximum_snow_albedo.tileX.nc'"
export LANDICE=".false."
export FSICL=0
export FSICS=0

export IMP_PHYSICS=8
export LGFDLMPRAD=.false.
export DO_SAT_ADJ=.false.
export DNATS=0
export DZ_MIN=6

#required for NML.IN sharing
export MIN_SEAICE=0.15
export FRAC_GRID=.true.
export MOM6_RESTART_SETTING=n
# P8 (not used for standalone)
export USE_CICE_ALB=.false.

export WRITE_NSFLIP=.true.

export FV3_RUN=control_run.IN
export CCPP_SUITE=FV3_GFS_v17_p8
export FIELD_TABLE=field_table_thompson_noaero_tke
export DIAG_TABLE=diag_table_p8_template
# use same namelist for standalone,coupled P7
export INPUT_NML=cpld_control.nml.IN

# RRTMGP
export DO_RRTMGP=.false.
export DOGP_CLDOPTICS_LUT=.true.
export DOGP_LWSCAT=.true.
export DOGP_SGS_CNV=.true.

export SATMEDMF=.true.

if [[ $MACHINE_ID = cheyenne.* ]]; then
TPN=18
fi
Loading

0 comments on commit 85dc0fd

Please sign in to comment.