Replace sleep with wait_for_file (#2586)

This PR addresses issue #2444. The following is accomplished:

- All `sleep` statements are replaced with `wait_for_file` for the
relevant scripts beneath `scripts` and `ush`;
- Indentation and shell-norms are updated where applicable. 

Note: The WAFS scripts are not updated as per @aerorahul direction.

Resolves #2444

---------

Co-authored-by: henrywinterbottom-wxdev <[email protected]>
Co-authored-by: Walter Kolczynski - NOAA <[email protected]>

env/HERA.env

@@ -316,7 +316,6 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
 
-
 elif [[ "${step}" = "fit2obs" ]]; then
 
     nth_max=$((npe_node_max / npe_node_fit2obs))

scripts/exgfs_atmos_awips_20km_1p0deg.sh

@@ -43,20 +43,14 @@ source "${USHgfs}/product_functions.sh"
 ###############################################
 # Wait for the availability of the pgrb file
 ###############################################
-icnt=1
-while (( icnt < 1000 )); do
-   if [[ -s "${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx" ]]; then
-      break
-   fi
-
-   sleep 10
-   icnt=$((icnt + 1))
-   if (( icnt >= 180 )); then
-      msg="FATAL ERROR: No GFS pgrb2 file after 30 min of waiting"
-      err_exit "${msg}"
-      exit 5
-   fi
-done
+sleep_interval=10
+max_tries=180
+idxfile="${COM_ATMOS_GRIB_0p25}/${RUN}.${cycle}.pgrb2b.0p25.f${fcsthrs}.idx"
+if ! wait_for_file "${idxfile}" "${sleep_interval}" "${max_tries}"; then
+  msg="FATAL ERROR: No GFS pgrb2 file after waiting"
+  err_exit "${msg}"
+  exit 5
+fi
 
 ########################################
 

scripts/exgfs_atmos_postsnd.sh

@@ -50,6 +50,8 @@ declare -x LEVS
 
 ### Loop for the hour and wait for the sigma and surface flux file:
 export FSTART=$STARTHOUR
+sleep_interval=10
+max_tries=360
 #
 while [ $FSTART -lt $ENDHOUR ]
 do
@@ -69,28 +71,18 @@ export FINT=$NINT1
        export MAKEBUFR=YES
    fi
 
-   ic=0
-   while [ $ic -lt 1000 ]; do
-      if [[ ! -f "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}" ]]; then
-          sleep 10
-          ic=$(expr $ic + 1)
-      else
-          break
-      fi
-
-      if [ $ic -ge 360 ]
-      then
-         err_exit "COULD NOT LOCATE logf$FEND file AFTER 1 HOUR"
-      fi
-   done
+   filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}"
+   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then
+     err_exit "FATAL ERROR: logf${FEND} not found after waiting $((sleep_interval * ( max_tries - 1) )) secs"
+   fi
 
 ## 1-hourly output before $NEND1, 3-hourly output after
-   if [ $FEND -gt $NEND1 ]; then
+   if [[ $((10#$FEND)) -gt $((10#$NEND1)) ]]; then
      export FINT=$NINT3
    fi
    ${USHgfs}/gfs_bufr.sh
 
-   export FSTART=$FEND
+   export FSTART="${FEND}"
 done
 
 ##############################################################

scripts/exgfs_wave_nawips.sh

@@ -44,6 +44,7 @@ pdsext=no
 g2tbls=g2varswmo2.tbl
 NAGRIB=nagrib2
 
+sleep_interval=20
 maxtries=15
 fhcnt=${fstart}
 while [ ${fhcnt} -le ${FHMAX_WAV} ]; do
@@ -72,28 +73,11 @@ while [ ${fhcnt} -le ${FHMAX_WAV} ]; do
     esac
     GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdIDin}.f${fhr}.grib2"
     GRIBIN_chk=${GRIBIN}.idx
-
-    icnt=1
-    while [ ${icnt} -lt 1000 ]; do
-      if [ -r ${GRIBIN_chk} ] ; then
-        break
-      else
-        let "icnt=icnt+1"
-        sleep 20
-      fi
-      if [ ${icnt} -ge ${maxtries} ]; then
-        msg="ABORTING after 5 minutes of waiting for ${GRIBIN}."
-        echo ' '
-        echo '**************************** '
-        echo '*** ERROR : NO GRIB FILE *** '
-        echo '**************************** '
-        echo ' '
-        echo ${msg}
-        set_trace
-        echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> ${wavelog}
-        err=1;export err;${errchk} || exit ${err}
-      fi
-    done
+    if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${maxtries}"; then
+      echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs"
+      echo "${RUNwave} ${grdID} ${fhr} prdgen ${date} ${cycle} : GRIB file missing." >> "${wavelog}"
+      err=1;export err;"${errchk}" || exit "${err}"
+    fi
 
     #if [ "$grdIDin" = "global.0p25" && "$grid" = "glo_30m" ]; then
     if [ "${grdIDin}" = "global.0p25" ]; then

scripts/exgfs_wave_post_gridded_sbs.sh

@@ -231,6 +231,7 @@ source "${USHgfs}/preamble.sh"
     fhr=$FHMIN_WAV
   fi
   fhrg=$fhr
+  sleep_interval=10
   iwaitmax=120 # Maximum loop cycles for waiting until wave component output file is ready (fails after max)
   while [ $fhr -le $FHMAX_WAV ]; do
 
@@ -253,26 +254,21 @@ source "${USHgfs}/preamble.sh"
     export GRDIDATA=${DATA}/output_$YMDHMS
 
 # Gridded data (main part, need to be run side-by-side with forecast
-
+    
     if [ $fhr = $fhrg ]
     then
-      iwait=0
-      for wavGRD in ${waveGRD} ; do
-        gfile=${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS}
-        while [ ! -s ${gfile} ]; do sleep 10; let iwait=iwait+1; done
-        if [ $iwait -eq $iwaitmax ]; then
-          echo '*************************************************** '
-          echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.$grdID "
-          echo '*************************************************** '
-          echo ' '
-          set_trace
+
+      for wavGRD in ${waveGRD}; do
+        gfile="${COM_WAVE_HISTORY}/${WAV_MOD_TAG}.out_grd.${wavGRD}.${YMD}.${HMS}"
+        if ! wait_for_file "${gfile}" "${sleep_interval}" "${iwaitmax}"; then
+          echo " FATAL ERROR : NO RAW FIELD OUTPUT FILE out_grd.${grdID}"
           echo "${WAV_MOD_TAG} post ${grdID} ${PDY} ${cycle} : field output missing."
-          err=3; export err;${errchk}
-          exit $err
+          err=3; export err; "${errchk}"
+          exit "${err}"
         fi
-        ${NLN} ${gfile} ./out_grd.${wavGRD}
+        ${NLN} "${gfile}" "./out_grd.${wavGRD}"
       done
-
+      
       if [ "$DOGRI_WAV" = 'YES' ]
       then
         nigrd=1

scripts/exgfs_wave_prdgen_gridded.sh

@@ -96,30 +96,14 @@ grids=${grids:-ak_10m at_10m ep_10m wc_10m glo_30m}
      #
 
      GRIBIN="${COM_WAVE_GRID}/${RUNwave}.${cycle}.${grdID}.f${fhr}.grib2"
-     GRIBIN_chk=$GRIBIN.idx
-
-     icnt=1
-     while [ $icnt -lt 1000 ]; do
-       if [ -r $GRIBIN_chk ] ; then
-         break
-       else
-         echo "Waiting for input file: $GRIBIN"
-         let "icnt=icnt+1"
-         sleep 5
-       fi
-       if [ $icnt -ge $maxtries ]; then
-         msg="ABNORMAL EXIT: NO GRIB FILE FOR GRID $GRIBIN"
-         echo ' '
-         echo '**************************** '
-         echo '*** ERROR : NO GRIB FILE *** '
-         echo '**************************** '
-         echo ' '
-         echo $msg
-         set_trace
-         echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog
-         err=1;export err;${errchk} || exit ${err}
-       fi
-     done
+     GRIBIN_chk="${GRIBIN}.idx"
+     sleep_interval=5
+     max_tries=1000
+     if ! wait_for_file "${GRIBIN_chk}" "${sleep_interval}" "${max_tries}"; then
+       echo "FATAL ERROR: ${GRIBIN_chk} not found after waiting $((sleep_interval * ( max_tries - 1))) secs"
+       echo "$RUNwave $grdID ${fhr} prdgen $date $cycle : GRIB file missing." >> $wavelog
+       err=1;export err;${errchk} || exit ${err}
+     fi
 
      GRIBOUT=$RUNwave.$cycle.$grdID.f${fhr}.clipped.grib2
 

ush/gfs_bufr.sh

@@ -51,26 +51,20 @@ cat << EOF > gfsparm
 /
 EOF
 
+sleep_interval=10
+max_tries=1000
 for (( hr = 10#${FSTART}; hr <= 10#${FEND}; hr = hr + 10#${FINT} )); do
    hh2=$(printf %02i "${hr}")
    hh3=$(printf %03i "${hr}")
 
    #---------------------------------------------------------
    # Make sure all files are available:
-   ic=0
-   while (( ic < 1000 )); do
-      if [[ ! -f "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}" ]]; then
-          sleep 10
-          ic=$((ic + 1))
-      else
-          break
-      fi
-
-      if (( ic >= 360 )); then
-         echo "FATAL: COULD NOT LOCATE logf${hh3} file AFTER 1 HOUR"
-         exit 2
-      fi
-   done
+   filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}"
+   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then
+     echo "FATAL ERROR: COULD NOT LOCATE logf${hh3} file"
+     exit 2
+   fi
+
    #------------------------------------------------------------------
    ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh3}.${atmfm}" "sigf${hh2}"
    ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh3}.${atmfm}" "flxf${hh2}"
@@ -96,11 +90,11 @@ esac
 ${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
 export err=$?
 
-if [ $err -ne 0 ]; then
+if [[ "${err}" -ne 0 ]]; then
    echo "GFS postsnd job error, Please check files "
    echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh2}.${atmfm}"
    echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh2}.${atmfm}"
    err_chk
 fi
 
-exit ${err}
+exit "${err}"

ush/gfs_bufr_netcdf.sh

@@ -66,6 +66,9 @@ hh=$FSTART
       hh1=$(echo "${hh#"${hh%??}"}")
       hh=$hh1
    fi
+
+sleep_interval=10
+max_tries=360   
 while  test $hh -le $FEND
 do  
    if test $hh -lt 100
@@ -75,24 +78,11 @@ do
       hh2=$hh
    fi
 
-#---------------------------------------------------------
-# Make sure all files are available:
-   ic=0
-   while [ $ic -lt 1000 ]
-   do
-      if [ ! -f $COMIN/${RUN}.${cycle}.logf${hh2}.txt ]
-      then
-          sleep 10
-          ic=$(expr $ic + 1)
-      else
-          break
-      fi
+   filename="${COMIN}/${RUN}.${cycle}.logf${hh2}.txt"
+   if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}" ; then
+     err_exit "FATAL ERROR COULD NOT LOCATE logf${hh2} file"
+   fi
 
-      if [ $ic -ge 360 ]
-      then
-         err_exit "COULD NOT LOCATE logf${hh2} file AFTER 1 HOUR"
-      fi
-   done
 #------------------------------------------------------------------
    ${NLN} $COMIN/${RUN}.${cycle}.atmf${hh2}.nc sigf${hh}
    ${NLN} $COMIN/${RUN}.${cycle}.${SFCF}f${hh2}.nc flxf${hh}