From a1c0e7d108f803e5b7859d403d87d7f4f3fa9649 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Thu, 20 Feb 2025 10:00:39 -0500 Subject: [PATCH] Update high resolution tests and fix minor wave issues (#3289) # Description This PR updates the wave fix which was unintentionally left out of PR https://github.com/NOAA-EMC/global-workflow/pull/3190 Additionally, this updates what ICs the highres ci tests are pointing to (IC update issues: https://github.com/NOAA-EMC/global-workflow/issues/3287 https://github.com/NOAA-EMC/global-workflow/issues/3288) Resolves #3260 Refs #3326 - small changes in comments Additionally, some smaller bug fixes and changes were made: * Removes logic that tries to copy files to fix directory in ush/wave_grid_interp_sbs.sh -- Fixes #3162 * Adds MPI variables that were found needed when investigating the HR4 hang https://github.com/ufs-community/ufs-weather-model/issues/2486 -- Fixes #3259 * Changes the log file output directory corresponding to an update requested by NCO in v16.3.22 in scripts/exgfs_wave_post_pnt.sh * Through debugging HR5 (#3260) it was noticed the wave ICs were not used. While there is a code change required (https://github.com/NOAA-EMC/WW3/pull/1359), this PR fixes the scripting level issues & updates UFS to fix WW3 code issues. * Removes COMOUT_WAVE_PREP from wave post job that did not need it, plus ensured temporary files were not copied back to COM and then over-written (NCO bug) --- Fixes #297 * Updates ufs-weather-model hash. Details in table below Notes on ufs-weather-model updates: | Commit date | Commit hash/ PR | Notes for g-w changes | Baseline Changes | | :------------- | :------------- | :------------- | :------------- | | Jan 7, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/5324d642e2257ee659ab75c0fc404d3127b2d9f7 https://github.com/ufs-community/ufs-weather-model/pull/2531 | n/a | changes in regional tests, no global changes | | Jan 13, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/c7412fa00d6995246279b80c4984d47b5d40352a | n/a | changes in some global coupled tests | | Jan 17, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/3a5e52ee1fb055e7c88a76aa1797450f01281a53 | n/a | no changes | | Jan 21, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/33cde4b2fbd69a001e5fcde8c132287201bc073c | extra variable needed for waves | no changes | | Jan 22, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/b9d2bec52bf9eb1948c6c5a83aeda5c93b1dca43 | n/a | no changes | Jan 28, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/0682f90cdc855def515dd1eac122abbc904973bc | n/a | | Feb 3, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/a498735b8b03a357f2f3757e00e479a52bb8ab03 | add fv3atm_output_dir to model_configure | answers change | | Feb 5, 2025 | https://github.com/ufs-community/ufs-weather-model/commit/ee3d52d3620565dbf7bc4f47bd039fe6423fa5c1 | changes in this PR for initialization for wave fixes | if using WW3 binary restart, answers will change | --- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- env/HERCULES.env | 1 + env/ORION.env | 2 ++ jobs/JGLOBAL_WAVE_POST_SBS | 5 +---- parm/wave/bull_awips_gfswave | 2 +- parm/wave/wave_gfs.buoys.dat | 2 +- parm/wave/wave_gfs.buoys.full | 4 ++-- scripts/exgfs_wave_post_pnt.sh | 20 ++++++++--------- sorc/ufs_model.fd | 2 +- ush/forecast_postdet.sh | 21 +++++++----------- ush/parsing_model_configure_FV3.sh | 3 +++ ush/parsing_namelists_WW3.sh | 1 + ush/wave_grid_interp_sbs.sh | 32 ++------------------------- versions/fix.ver | 2 +- versions/ic.ver | 4 ++-- 15 files changed, 37 insertions(+), 66 deletions(-) diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index 11a13f0bc3..b9c47c0463 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20241120 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250218 idate: 2021032418 edate: 2021032500 nens: 0 diff --git a/env/HERCULES.env b/env/HERCULES.env index acfbe438ef..b565881a73 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -214,6 +214,7 @@ case ${step} in "fcst" | "efcs") export OMP_STACKSIZE=512M + export FI_MLX_INJECT_LIMIT=0 (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) diff --git a/env/ORION.env b/env/ORION.env index fbe00c153c..64f3f3d555 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -208,6 +208,8 @@ elif [[ "${step}" = "eupd" ]]; then elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + export FI_MLX_INJECT_LIMIT=0 + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) # With ESMF threading, the model wants to use the full node diff --git a/jobs/JGLOBAL_WAVE_POST_SBS b/jobs/JGLOBAL_WAVE_POST_SBS index 171d160515..acd654d6fa 100755 --- a/jobs/JGLOBAL_WAVE_POST_SBS +++ b/jobs/JGLOBAL_WAVE_POST_SBS @@ -13,12 +13,9 @@ export MP_PULSE=0 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_WAVE_PREP:COM_WAVE_PREP_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ - COMOUT_WAVE_PREP:COM_WAVE_PREP_TMPL \ COMOUT_WAVE_GRID:COM_WAVE_GRID_TMPL -for out_dir in "${COMOUT_WAVE_PREP}" "${COMOUT_WAVE_GRID}"; do - if [[ ! -d "${out_dir}" ]]; then mkdir -p "${out_dir}"; fi -done +if [[ ! -d "${COMOUT_WAVE_GRID}" ]]; then mkdir -p "${COMOUT_WAVE_GRID}"; fi if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then for grdID in ${wavepostGRD} ${waveinterpGRD}; do diff --git a/parm/wave/bull_awips_gfswave b/parm/wave/bull_awips_gfswave index 87aa19fe48..b28a380371 100644 --- a/parm/wave/bull_awips_gfswave +++ b/parm/wave/bull_awips_gfswave @@ -181,7 +181,7 @@ export b55020=AGPS40_KWBJ_OSBM01 export b55033=AGPS40_KWBJ_OSBM02 export b55035=AGPS40_KWBJ_OSBM03 export b55039=AGPS40_KWBJ_OSBM04 -# Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2) +# Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2) export b42001=AGGX42_KWBJ_OSBM01 export b42002=AGGX42_KWBJ_OSBM02 export b42003=AGGX42_KWBJ_OSBM03 diff --git a/parm/wave/wave_gfs.buoys.dat b/parm/wave/wave_gfs.buoys.dat index 0d2f4ab11e..1b0595f78e 100755 --- a/parm/wave/wave_gfs.buoys.dat +++ b/parm/wave/wave_gfs.buoys.dat @@ -254,7 +254,7 @@ $ South Pacific (PS) in Pacific Ocean and Pacific Isles (0) $AGPS40 $ $AGGX42 -$ Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2) +$ Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2) -89.67 25.90 '42001 ' 10.0 DAT NDBC 360 -94.42 25.17 '42002 ' 10.0 DAT NDBC 360 -85.94 26.07 '42003 ' 10.0 DAT NDBC 360 diff --git a/parm/wave/wave_gfs.buoys.full b/parm/wave/wave_gfs.buoys.full index 086de45582..d85506b8f5 100755 --- a/parm/wave/wave_gfs.buoys.full +++ b/parm/wave/wave_gfs.buoys.full @@ -339,7 +339,7 @@ $ South Pacific (PS) in Pacific Ocean and Pacific Isles (0) $AGPS40 $ $AGGX42 -$ Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2) +$ Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2) -89.67 25.90 '42001 ' 10.0 DAT NDBC 360 -94.42 25.17 '42002 ' 10.0 DAT NDBC 360 -85.94 26.07 '42003 ' 10.0 DAT NDBC 360 @@ -1190,7 +1190,7 @@ $ -65.50 32.00 'BER73 ' 999. IBP NCEP 45 -65.50 31.75 'BER74 ' 999. IBP NCEP 45 $ -$ Gulf of Mexico boundary data +$ Gulf of America boundary data $ -97. 23. 'SRH51 ' 999. IBP NCEP 45 -96. 23. 'SRH52 ' 999. IBP NCEP 45 diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index 96c60fc96c..bad848c588 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -618,13 +618,13 @@ source "${USHgfs}/preamble.sh" if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi else @@ -633,28 +633,28 @@ source "${USHgfs}/preamble.sh" nm=$(( nm + 1 )) fi if [ "$DOBLL_WAV" = YES ]; then - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) - echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile nm=$(( nm + 1 )) fi fi else if [ "$DOBNDPNT_WAV" = YES ]; then if [ "$DOSPC_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 " >> cmdtarfile fi else if [ "$DOSPC_WAV" = YES ]; then echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile fi if [ "$DOBLL_WAV" = YES ]; then - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile - echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 " >> cmdtarfile + echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 " >> cmdtarfile fi fi fi diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 76471dc6b7..ee3d52d362 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 76471dc6b7bfc3342416d1a3402f360724f7c0fa +Subproject commit ee3d52d3620565dbf7bc4f47bd039fe6423fa5c1 diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index f47f57145a..70120b9a5f 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -373,24 +373,19 @@ WW3_postdet() { elif [[ -s "${ww3_binary_restart_file}" ]]; then # found binary ww3 restart file export WW3_restart_from_binary=true - if [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then - #if this is a cmeps continue then the wave restart name is different - seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds - local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}" - ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \ - || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) - else - ${NCP} "${ww3_binary_restart_file}" "${DATA}/restart.ww3" \ - || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) - fi + seconds=$(to_seconds "${restart_date:8:2}0000") # convert HHMMSS to seconds + local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}" + ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \ + || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 ) else - if [[ "${RERUN}" == "YES" ]]; then + if [[ "${RERUN}" == "YES" ]] || [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then # In the case of a RERUN, the WW3 restart file is required - echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}', ABORT!" + # In the case of runtype=continue, if no wave restart when using PIO, the model will fail + echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}' or runtype=continue, ABORT!" exit 1 else + export WW3_restart_from_binary=false echo "WARNING: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for warm_start='${warm_start}', will start from rest!" - export WW3_restart_from_binary=true fi fi diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index d28048f098..a6adbeaf8a 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -56,6 +56,9 @@ local JMO=${LATB_JMO} local OUTPUT_FH=${FV3_OUTPUT_FH_NML} local IAU_OFFSET=${IAU_OFFSET:-0} +#set FV3 output directory: +local FV3ATM_OUTPUT_DIR="./" + # Ensure the template exists if [[ "${DO_NEST:-NO}" == "YES" ]] ; then local NEST_IMO=${npx_nest} diff --git a/ush/parsing_namelists_WW3.sh b/ush/parsing_namelists_WW3.sh index 67bffb1967..7428c00651 100755 --- a/ush/parsing_namelists_WW3.sh +++ b/ush/parsing_namelists_WW3.sh @@ -45,6 +45,7 @@ WW3_namelists(){ # Variables used in atparse of shel template export WW3_IC1="F" export WW3_IC5="F" + export WW3_WLEV="F" export WW3_ICE export WW3_CUR export WW3_OUTPARS="${OUTPARS_WAV}" diff --git a/ush/wave_grid_interp_sbs.sh b/ush/wave_grid_interp_sbs.sh index c046afb2e9..fd7180add6 100755 --- a/ush/wave_grid_interp_sbs.sh +++ b/ush/wave_grid_interp_sbs.sh @@ -66,8 +66,7 @@ source "${USHgfs}/preamble.sh" set_trace if [[ -z "${PDY}" ]] || [[ -z "${cyc}" ]] || [[ -z "${cycle}" ]] || [[ -z "${EXECgfs}" ]] || \ - [[ -z "${COMOUT_WAVE_PREP}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] || \ - [ -z "${waveGRD}" ] + [[ -z "${waveGRD}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] then set +x echo ' ' @@ -75,7 +74,7 @@ source "${USHgfs}/preamble.sh" echo '*** EXPORTED VARIABLES IN postprocessor NOT SET ***' echo '***************************************************' echo ' ' - echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${COMOUT_WAVE_PREP} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}" + echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}" set_trace exit 1 fi @@ -142,14 +141,6 @@ source "${USHgfs}/preamble.sh" "${EXECgfs}/${pgm}" 1> gint.${grdID}.out 2>&1 export err=$?;err_chk -# Write interpolation file to main TEMP dir area if not there yet - if [ "wht_OK" = 'no' ] # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}. With ${wht_OK}, the next line is trying to copy into ${FIXgfs} space. This leads to a Permission denied error. The logic here needs to be evaluated and recoded. #TODO - then - cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID} - cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID} - fi - - if [ "$err" != '0' ] then set +x @@ -168,25 +159,6 @@ source "${USHgfs}/preamble.sh" rm -f mod_def.* mv out_grd.$grdID ${DATA}/output_${ymdh}0000/out_grd.$grdID -# 1.c Save in /com - - set +x - echo " Saving GRID file as ${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}" - set_trace - cp "${DATA}/output_${ymdh}0000/out_grd.${grdID}" "${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}" - -# if [ "$SENDDBN" = 'YES' ] -# then -# set +x -# echo " Alerting GRID file as $COMOUT/rundata/$WAV_MOD_TAG.out_grd.$grdID.${PDY}${cyc} -# set_trace - -# -# PUT DBNET ALERT HERE .... -# - -# fi - # --------------------------------------------------------------------------- # # 2. Clean up the directory diff --git a/versions/fix.ver b/versions/fix.ver index db54701e77..7f69e657b2 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -23,6 +23,6 @@ export reg2grb2_ver=20220805 export sfc_climo_ver=20220805 export ugwd_ver=20240624 export verif_ver=20220805 -export wave_ver=20240105 +export wave_ver=20250110 export orog_nest_ver=global-nest.20240419 export ugwd_nest_ver=global-nest.20240419 diff --git a/versions/ic.ver b/versions/ic.ver index e7bd8358ef..1994910c92 100644 --- a/versions/ic.ver +++ b/versions/ic.ver @@ -16,7 +16,7 @@ ic_versions['C384C192']=20240610 ic_versions['C384mx025']=20241120 ic_versions['C768']=20241120 -ic_versions['C768mx025']=20241120 +ic_versions['C768mx025']=20250115 ic_versions['C1152']=20241120 -ic_versions['C1152mx025']=20241120 +ic_versions['C1152mx025']=20250115