Update high resolution tests and fix minor wave issues (#3289)

# Description
This PR updates the wave fix which was unintentionally left out of PR
#3190
Additionally, this updates what ICs the highres ci tests are pointing to
(IC update issues:
#3287
#3288)

Resolves #3260 
Refs #3326 - small changes in comments

Additionally, some smaller bug fixes and changes were made: 

* Removes logic that tries to copy files to fix directory in
ush/wave_grid_interp_sbs.sh
-- Fixes #3162 
* Adds MPI variables that were found needed when investigating the HR4
hang ufs-community/ufs-weather-model#2486
-- Fixes #3259 
* Changes the log file output directory corresponding to an update
requested by NCO in v16.3.22 in scripts/exgfs_wave_post_pnt.sh
* Through debugging HR5 (#3260) it was noticed the wave ICs were not
used. While there is a code change required
(NOAA-EMC/WW3#1359), this PR fixes the scripting
level issues & updates UFS to fix WW3 code issues.
* Removes COMOUT_WAVE_PREP from wave post job that did not need it, plus
ensured temporary files were not copied back to COM and then
over-written (NCO bug)
--- Fixes #297 
* Updates ufs-weather-model hash.  Details in table below 


Notes on ufs-weather-model updates: 
| Commit date | Commit hash/ PR | Notes for g-w changes | Baseline
Changes |
| :------------- | :------------- | :------------- | :------------- |
| Jan 7, 2025 |
ufs-community/ufs-weather-model@5324d64
ufs-community/ufs-weather-model#2531 | n/a |
changes in regional tests, no global changes |
| Jan 13, 2025 |
ufs-community/ufs-weather-model@c7412fa
| n/a | changes in some global coupled tests |
| Jan 17, 2025 |
ufs-community/ufs-weather-model@3a5e52e
| n/a | no changes |
| Jan 21, 2025 |
ufs-community/ufs-weather-model@33cde4b
| extra variable needed for waves | no changes |
| Jan 22, 2025 |
ufs-community/ufs-weather-model@b9d2bec
| n/a | no changes
| Jan 28, 2025 |
ufs-community/ufs-weather-model@0682f90
| n/a |
| Feb 3, 2025 |
ufs-community/ufs-weather-model@a498735
| add fv3atm_output_dir to model_configure | answers change |
| Feb 5, 2025 |
ufs-community/ufs-weather-model@ee3d52d
| changes in this PR for initialization for wave fixes | if using WW3
binary restart, answers will change |

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20241120
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250218
   idate: 2021032418
   edate: 2021032500
   nens: 0

env/HERCULES.env

@@ -214,6 +214,7 @@ case ${step} in
  "fcst" | "efcs")
 
     export OMP_STACKSIZE=512M
+    export FI_MLX_INJECT_LIMIT=0
 
     (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
     (( ufs_ntasks = nnodes*tasks_per_node ))

env/ORION.env

@@ -208,6 +208,8 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
+    export FI_MLX_INJECT_LIMIT=0
+
     (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
     (( ufs_ntasks = nnodes*tasks_per_node ))
     # With ESMF threading, the model wants to use the full node

jobs/JGLOBAL_WAVE_POST_SBS

@@ -13,12 +13,9 @@ export MP_PULSE=0
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_WAVE_PREP:COM_WAVE_PREP_TMPL \
     COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
-    COMOUT_WAVE_PREP:COM_WAVE_PREP_TMPL \
     COMOUT_WAVE_GRID:COM_WAVE_GRID_TMPL
 
-for out_dir in "${COMOUT_WAVE_PREP}" "${COMOUT_WAVE_GRID}"; do
-    if [[ ! -d "${out_dir}" ]]; then mkdir -p "${out_dir}"; fi
-done
+if [[ ! -d "${COMOUT_WAVE_GRID}" ]]; then mkdir -p "${COMOUT_WAVE_GRID}"; fi
 
 if [[ -n "${wavepostGRD}" || -n "${waveinterpGRD}" ]]; then
     for grdID in ${wavepostGRD} ${waveinterpGRD}; do

parm/wave/bull_awips_gfswave

@@ -181,7 +181,7 @@ export b55020=AGPS40_KWBJ_OSBM01
 export b55033=AGPS40_KWBJ_OSBM02
 export b55035=AGPS40_KWBJ_OSBM03
 export b55039=AGPS40_KWBJ_OSBM04
-# Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2)
+# Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2)
 export b42001=AGGX42_KWBJ_OSBM01
 export b42002=AGGX42_KWBJ_OSBM02
 export b42003=AGGX42_KWBJ_OSBM03

parm/wave/wave_gfs.buoys.dat

@@ -254,7 +254,7 @@ $ South Pacific (PS) in Pacific Ocean and Pacific Isles (0)
 $AGPS40
 $
 $AGGX42
-$ Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2)
+$ Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2)
    -89.67   25.90  '42001     '  10.0   DAT  NDBC     360
    -94.42   25.17  '42002     '  10.0   DAT  NDBC     360
    -85.94   26.07  '42003     '  10.0   DAT  NDBC     360

parm/wave/wave_gfs.buoys.full

@@ -339,7 +339,7 @@ $ South Pacific (PS) in Pacific Ocean and Pacific Isles (0)
 $AGPS40
 $
 $AGGX42
-$ Gulf of Mexico (GX) spectra (4) south from NC and Puerto Rico (2)
+$ Gulf of America (GX) spectra (4) south from NC and Puerto Rico (2)
    -89.67   25.90  '42001     '  10.0   DAT  NDBC     360
    -94.42   25.17  '42002     '  10.0   DAT  NDBC     360
    -85.94   26.07  '42003     '  10.0   DAT  NDBC     360
@@ -1190,7 +1190,7 @@ $
    -65.50   32.00  'BER73     '  999.   IBP  NCEP      45
    -65.50   31.75  'BER74     '  999.   IBP  NCEP      45
 $
-$ Gulf of Mexico boundary data
+$ Gulf of America boundary data
 $
    -97.     23.    'SRH51     '  999.   IBP  NCEP      45
    -96.     23.    'SRH52     '  999.   IBP  NCEP      45

scripts/exgfs_wave_post_pnt.sh

@@ -618,13 +618,13 @@ source "${USHgfs}/preamble.sh"
   if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then
     if [ "$DOBNDPNT_WAV" = YES ]; then
       if [ "$DOSPC_WAV" = YES ]; then
-        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 "   >> cmdtarfile
         nm=$(( nm + 1 ))
       fi
       if [ "$DOBLL_WAV" = YES ]; then
-        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 "   >> cmdtarfile
         nm=$(( nm + 1 ))
-        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 "   >> cmdtarfile
         nm=$(( nm + 1 ))
       fi
     else
@@ -633,28 +633,28 @@ source "${USHgfs}/preamble.sh"
         nm=$(( nm + 1 ))
       fi
       if [ "$DOBLL_WAV" = YES ]; then
-        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 "   >> cmdtarfile
         nm=$(( nm + 1 ))
-        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "$nm ${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 "   >> cmdtarfile
         nm=$(( nm + 1 ))
       fi
     fi
   else
     if [ "$DOBNDPNT_WAV" = YES ]; then
       if [ "$DOSPC_WAV" = YES ]; then
-        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_ibp_tar.out 2>&1 "   >> cmdtarfile
       fi
       if [ "$DOBLL_WAV" = YES ]; then
-        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
-        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpbull $Nb > ${WAV_MOD_TAG}_ibpbull_tar.out 2>&1 "   >> cmdtarfile
+        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG ibpcbull $Nb > ${WAV_MOD_TAG}_ibpcbull_tar.out 2>&1 "   >> cmdtarfile
       fi
     else
       if [ "$DOSPC_WAV" = YES ]; then
         echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG spec $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
       fi
       if [ "$DOBLL_WAV" = YES ]; then
-        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
-        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile
+        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG bull $Nb > ${WAV_MOD_TAG}_bull_tar.out 2>&1 "   >> cmdtarfile
+        echo "${USHgfs}/wave_tar.sh $WAV_MOD_TAG cbull $Nb > ${WAV_MOD_TAG}_cbull_tar.out 2>&1 "   >> cmdtarfile
       fi
     fi
   fi

ush/forecast_postdet.sh

@@ -373,24 +373,19 @@ WW3_postdet() {
   elif [[ -s "${ww3_binary_restart_file}" ]]; then 
     # found binary ww3 restart file 
     export WW3_restart_from_binary=true
-    if [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then
-      #if this is a cmeps continue then the wave restart name is different 
-      seconds=$(to_seconds "${restart_date:8:2}0000")  # convert HHMMSS to seconds
-      local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}"
-      ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \
-          || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 )
-    else 
-      ${NCP} "${ww3_binary_restart_file}" "${DATA}/restart.ww3" \
-          || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 )
-    fi 
+    seconds=$(to_seconds "${restart_date:8:2}0000")  # convert HHMMSS to seconds
+    local ww3_restart_dest_file="ufs.cpld.ww3.r.${restart_date:0:4}-${restart_date:4:2}-${restart_date:6:2}-${seconds}"
+    ${NCP} "${ww3_binary_restart_file}" "${DATA}/${ww3_restart_dest_file}" \
+             || ( echo "FATAL ERROR: Unable to copy binary WW3 IC, ABORT!"; exit 1 )
   else
-    if [[ "${RERUN}" == "YES" ]]; then
+    if [[ "${RERUN}" == "YES" ]] || [[ -f "${DATA}/ufs.cpld.cpl.r.nc" ]]; then
       # In the case of a RERUN, the WW3 restart file is required
-      echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}', ABORT!"
+      # In the case of runtype=continue, if no wave restart when using PIO, the model will fail
+      echo "FATAL ERROR: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for RERUN='${RERUN}' or runtype=continue, ABORT!"
       exit 1
     else
+      export WW3_restart_from_binary=false
       echo "WARNING: WW3 binary | netcdf restart file '${ww3_binary_restart_file}' | '${ww3_netcdf_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
-      export WW3_restart_from_binary=true
     fi
   fi
 

ush/parsing_model_configure_FV3.sh

@@ -56,6 +56,9 @@ local JMO=${LATB_JMO}
 local OUTPUT_FH=${FV3_OUTPUT_FH_NML}
 local IAU_OFFSET=${IAU_OFFSET:-0}
 
+#set FV3 output directory:
+local FV3ATM_OUTPUT_DIR="./"
+
 # Ensure the template exists
 if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
   local NEST_IMO=${npx_nest}

ush/parsing_namelists_WW3.sh

@@ -45,6 +45,7 @@ WW3_namelists(){
   # Variables used in atparse of shel template 
   export WW3_IC1="F"
   export WW3_IC5="F"
+  export WW3_WLEV="F"
   export WW3_ICE
   export WW3_CUR
   export WW3_OUTPARS="${OUTPARS_WAV}"

ush/wave_grid_interp_sbs.sh

@@ -66,16 +66,15 @@ source "${USHgfs}/preamble.sh"
   set_trace
 
   if [[ -z "${PDY}" ]] || [[ -z "${cyc}" ]] || [[ -z "${cycle}" ]] || [[ -z "${EXECgfs}" ]] || \
-	 [[ -z "${COMOUT_WAVE_PREP}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] || \
-	 [ -z "${waveGRD}" ]
+	 [[ -z "${waveGRD}" ]] || [[ -z "${WAV_MOD_TAG}" ]] || [[ -z "${SENDDBN}" ]] 
   then
     set +x
     echo ' '
     echo '***************************************************'
     echo '*** EXPORTED VARIABLES IN postprocessor NOT SET ***'
     echo '***************************************************'
     echo ' '
-    echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${COMOUT_WAVE_PREP} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}"
+    echo "${PDY}${cyc} ${cycle} ${EXECgfs} ${WAV_MOD_TAG} ${SENDDBN} ${waveGRD}"
     set_trace
     exit 1
   fi
@@ -142,14 +141,6 @@ source "${USHgfs}/preamble.sh"
   "${EXECgfs}/${pgm}" 1> gint.${grdID}.out 2>&1
   export err=$?;err_chk
 
-# Write interpolation file to main TEMP dir area if not there yet
-  if [ "wht_OK" = 'no' ]  # FIXME: This is never going to evaluate to true, wht_OK is a string and needs to be ${wht_OK}.  With ${wht_OK}, the next line is trying to copy into ${FIXgfs} space.  This leads to a Permission denied error. The logic here needs to be evaluated and recoded.  #TODO
-  then
-    cp -f ./WHTGRIDINT.bin ${DATA}/ww3_gint.WHTGRIDINT.bin.${grdID}
-    cp -f ./WHTGRIDINT.bin ${FIXgfs}/wave/ww3_gint.WHTGRIDINT.bin.${grdID}
-  fi
-
-
   if [ "$err" != '0' ]
   then
     set +x
@@ -168,25 +159,6 @@ source "${USHgfs}/preamble.sh"
   rm -f mod_def.*
   mv out_grd.$grdID ${DATA}/output_${ymdh}0000/out_grd.$grdID
 
-# 1.c Save in /com
-
-  set +x
-  echo "   Saving GRID file as ${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}"
-  set_trace
-  cp "${DATA}/output_${ymdh}0000/out_grd.${grdID}" "${COMOUT_WAVE_PREP}/${WAV_MOD_TAG}.out_grd.${grdID}.${PDY}${cyc}"
-
-#    if [ "$SENDDBN" = 'YES' ]
-#    then
-#      set +x
-#      echo "   Alerting GRID file as $COMOUT/rundata/$WAV_MOD_TAG.out_grd.$grdID.${PDY}${cyc}
-#      set_trace
-
-#
-# PUT DBNET ALERT HERE ....
-#
-
-#    fi
-
 # --------------------------------------------------------------------------- #
 # 2.  Clean up the directory
 

versions/fix.ver

@@ -23,6 +23,6 @@ export reg2grb2_ver=20220805
 export sfc_climo_ver=20220805
 export ugwd_ver=20240624
 export verif_ver=20220805
-export wave_ver=20240105
+export wave_ver=20250110
 export orog_nest_ver=global-nest.20240419
 export ugwd_nest_ver=global-nest.20240419

versions/ic.ver

@@ -16,7 +16,7 @@ ic_versions['C384C192']=20240610
 ic_versions['C384mx025']=20241120
 
 ic_versions['C768']=20241120
-ic_versions['C768mx025']=20241120
+ic_versions['C768mx025']=20250115
 
 ic_versions['C1152']=20241120
-ic_versions['C1152mx025']=20241120
+ic_versions['C1152mx025']=20250115