Tune workflow and scripts for spin-up cycles. (NOAA-EMC#226)

* Tune workflow and scripts for spin-up cycles.

* Add option to decide if write out cycle log file.

scripts/exregional_run_post.sh

@@ -154,54 +154,11 @@ esac
 #
 #-----------------------------------------------------------------------
 #
-# Remove any files from previous runs and stage necessary files in fhr_dir.
+# Remove any files from previous runs.
 #
 #-----------------------------------------------------------------------
 #
 rm_vrfy -f fort.*
-cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat
-ln_vrfy -snf ${FIX_CRTM}/*bin ./
-if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then
-  post_config_fp="${CUSTOM_POST_CONFIG_FP}"
-  post_params_fp="${CUSTOM_POST_PARAMS_FP}"
-  print_info_msg "
-====================================================================
-Copying the user-defined post flat file specified by CUSTOM_POST_CONFIG_FP
-to the post forecast hour directory (fhr_dir):
-  CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\"
-  CUSTOM_POST_PARAMS_FP = \"${CUSTOM_POST_PARAMS_FP}\"
-  fhr_dir = \"${fhr_dir}\"
-===================================================================="
-else
-  post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt"
-  post_params_fp="${EMC_POST_DIR}/parm/params_grib2_tbl_new"
-  print_info_msg "
-====================================================================
-Copying the default post flat file specified by post_config_fp to the post
-forecast hour directory (fhr_dir):
-  post_config_fp = \"${post_config_fp}\"
-  post_params_fp = \"${post_params_fp}\"
-  fhr_dir = \"${fhr_dir}\"
-===================================================================="
-fi
-cp_vrfy ${post_config_fp} ./postxconfig-NT.txt
-cp_vrfy ${post_params_fp} ./params_grib2_tbl_new
-cp_vrfy ${EXECDIR}/upp.x .
-if [ -f ${FFG_DIR}/latest.FFG ] && [ ${NET} = "RRFS_CONUS" ]; then
-  cp_vrfy ${FFG_DIR}/latest.FFG .
-  grid_specs_rrfs="lambert:-97.5:38.500000 237.826355:1746:3000 21.885885:1014:3000"
-  wgrib2 latest.FFG -match "0-12 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_12h.grib2
-  wgrib2 latest.FFG -match "0-6 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_06h.grib2
-  wgrib2 latest.FFG -match "0-3 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_03h.grib2
-  wgrib2 latest.FFG -match "0-1 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_01h.grib2
-elif [ -f ${FFG_DIR}/latest.FFG ] && [ ${NET} = "RRFS_NA_3km" ]; then
-  cp_vrfy ${FFG_DIR}/latest.FFG .
-  grid_specs_rrfs="rot-ll:248.000000:-42.000000:0.000000 309.000000:4081:0.025000 -33.0000000:2641:0.025000"
-  wgrib2 latest.FFG -match "0-12 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_12h.grib2
-  wgrib2 latest.FFG -match "0-6 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_06h.grib2
-  wgrib2 latest.FFG -match "0-3 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_03h.grib2
-  wgrib2 latest.FFG -match "0-1 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_01h.grib2
-fi
 #
 #-----------------------------------------------------------------------
 #
@@ -259,7 +216,7 @@ if [ ! -r ${nwges_dir}/INPUT/gfs_ctrl.nc ]; then
 fi
 if [ -r "$run_dir/RESTART/${restart_prefix}.coupler.res" ]; then
   for file in ${filelist}; do
-    cp_vrfy $run_dir/RESTART/${restart_prefix}.${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
+    mv_vrfy $run_dir/RESTART/${restart_prefix}.${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
   done
   echo " ${fhr} forecast from ${yyyymmdd}${hh} is ready " #> ${nwges_dir}/RESTART/restart_done_f${fhr}
 else
@@ -279,14 +236,63 @@ else
 
   if [ -r "$run_dir/RESTART/coupler.res" ] && [ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] ; then
     for file in ${filelist}; do
-      cp_vrfy $run_dir/RESTART/${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
+      mv_vrfy $run_dir/RESTART/${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}
     done
     echo " ${fhr} forecast from ${yyyymmdd}${hh} is ready " #> ${nwges_dir}/RESTART/restart_done_f${fhr}
   else
     echo "This forecast hour does not need to save restart: ${yyyymmdd}${hh}f${fhr}"
   fi
 fi
 #
+#-----------------------------------------------------------------------
+#
+# stage necessary files in fhr_dir.
+#
+#-----------------------------------------------------------------------
+#
+cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat
+ln_vrfy -snf ${FIX_CRTM}/*bin ./
+if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then
+  post_config_fp="${CUSTOM_POST_CONFIG_FP}"
+  post_params_fp="${CUSTOM_POST_PARAMS_FP}"
+  print_info_msg "
+====================================================================
+Copying the user-defined post flat file specified by CUSTOM_POST_CONFIG_FP
+to the post forecast hour directory (fhr_dir):
+  CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\"
+  CUSTOM_POST_PARAMS_FP = \"${CUSTOM_POST_PARAMS_FP}\"
+  fhr_dir = \"${fhr_dir}\"
+===================================================================="
+else
+  post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt"
+  post_params_fp="${EMC_POST_DIR}/parm/params_grib2_tbl_new"
+  print_info_msg "
+====================================================================
+Copying the default post flat file specified by post_config_fp to the post
+forecast hour directory (fhr_dir):
+  post_config_fp = \"${post_config_fp}\"
+  post_params_fp = \"${post_params_fp}\"
+  fhr_dir = \"${fhr_dir}\"
+===================================================================="
+fi
+cp_vrfy ${post_config_fp} ./postxconfig-NT.txt
+cp_vrfy ${post_params_fp} ./params_grib2_tbl_new
+cp_vrfy ${EXECDIR}/upp.x .
+if [ -f ${FFG_DIR}/latest.FFG ] && [ ${NET} = "RRFS_CONUS" ]; then
+  cp_vrfy ${FFG_DIR}/latest.FFG .
+  grid_specs_rrfs="lambert:-97.5:38.500000 237.826355:1746:3000 21.885885:1014:3000"
+  wgrib2 latest.FFG -match "0-12 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_12h.grib2
+  wgrib2 latest.FFG -match "0-6 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_06h.grib2
+  wgrib2 latest.FFG -match "0-3 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_03h.grib2
+  wgrib2 latest.FFG -match "0-1 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_01h.grib2
+elif [ -f ${FFG_DIR}/latest.FFG ] && [ ${NET} = "RRFS_NA_3km" ]; then
+  cp_vrfy ${FFG_DIR}/latest.FFG .
+  grid_specs_rrfs="rot-ll:248.000000:-42.000000:0.000000 309.000000:4081:0.025000 -33.0000000:2641:0.025000"
+  wgrib2 latest.FFG -match "0-12 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_12h.grib2
+  wgrib2 latest.FFG -match "0-6 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_06h.grib2
+  wgrib2 latest.FFG -match "0-3 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_03h.grib2
+  wgrib2 latest.FFG -match "0-1 hour" -end -new_grid_interpolation bilinear -new_grid_winds grid -new_grid ${grid_specs_rrfs} ffg_01h.grib2
+fi
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_prepstart.sh

@@ -91,6 +91,8 @@ MM=${YYYYMMDDHH:4:2}
 DD=${YYYYMMDDHH:6:2}
 HH=${YYYYMMDDHH:8:2}
 YYYYMMDD=${YYYYMMDDHH:0:8}
+
+current_time=$(date "+%T")
 #
 #-----------------------------------------------------------------------
 #
@@ -112,7 +114,7 @@ YYJJJ1200=`date +"%y%j1200" -d "${START_DATE} 1 day ago"`
 
 BKTYPE=0
 if [ ${cycle_type} == "spinup" ]; then
-   echo "spin up cycle"
+  echo "spin up cycle"
   for cyc_start in "${CYCL_HRS_SPINSTART[@]}"; do
     if [ ${HH} -eq ${cyc_start} ]; then
       BKTYPE=1
@@ -141,6 +143,9 @@ if [ ${BKTYPE} -eq 1 ] ; then  # cold start, use prepare cold strat initial file
       cp_vrfy ${bkpath}/gfs_data.tile7.halo0.nc gfs_data.tile7.halo0.nc        
       cp_vrfy ${bkpath}/sfc_data.tile7.halo0.nc sfc_data.tile7.halo0.nc        
       print_info_msg "$VERBOSE" "cold start from $bkpath"
+      if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then
+        echo "${YYYYMMDDHH}(${cycle_type}): cold start at ${current_time} from $bkpath " >> ${EXPTDIR}/log.cycles
+      if
     else
       print_err_msg_exit "Error: cannot find cold start initial condition from : ${bkpath}"
     fi
@@ -193,6 +198,9 @@ if [ ${BKTYPE} -eq 1 ] ; then  # cold start, use prepare cold strat initial file
           ncks --append geolonlat.nc sfc_data.tile7.halo0.nc
           ncrename -v tslb,stc -v smois,smc -v sh2o,slc sfc_data.tile7.halo0.nc
           echo "cycle surface with ${checkfile}" > cycle_surface.done
+          if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then
+            echo "${YYYYMMDDHH}(${cycle_type}): cycle surface with ${checkfile} " >> ${EXPTDIR}/log.cycles
+          fi
         else
           print_info_msg "Warning: cannot find surface from previous cycle"
         fi
@@ -225,9 +233,8 @@ else
   restart_prefix="${YYYYMMDD}.${HH}0000."
   n=${DA_CYCLE_INTERV}
   while [[ $n -le 6 ]] ; do
-    checkfile=${bkpath}/${restart_prefix}fv_core.res.tile1.nc
-    checkfile1=${bkpath}/${restart_prefix}fv_tracer.res.tile1.nc
-    if [ -r "${checkfile}" ] && [ -r "${checkfile1}" ]; then
+    checkfile=${bkpath}/${restart_prefix}coupler.res
+    if [ -r "${checkfile}" ] ; then
       print_info_msg "$VERBOSE" "Found ${checkfile}; Use it as background for analysis "
       break
     else
@@ -238,9 +245,32 @@ else
     fi
   done
 #
-  checkfile=${bkpath}/${restart_prefix}fv_core.res.tile1.nc
-  checkfile1=${bkpath}/${restart_prefix}fv_tracer.res.tile1.nc
-  if [ -r "${checkfile}" ] && [ -r "${checkfile1}" ] ; then
+  checkfile=${bkpath}/${restart_prefix}coupler.res
+# spin-up cycle is not success, try to find background from full cycle
+  if [ ! -r "${checkfile}" ] && [ ${BKTYPE} -eq 2 ]; then
+     print_info_msg "$VERBOSE" "cannot find background from spin-up cycle, try product cycle"
+     fg_restart_dirname=fcst_fv3lam
+     YYYYMMDDHHmInterv=$( date +%Y%m%d%H -d "${START_DATE} ${DA_CYCLE_INTERV} hours ago" )
+     bkpath=${fg_root}/${YYYYMMDDHHmInterv}/${fg_restart_dirname}/RESTART  # cycling, use background from RESTART
+#
+     restart_prefix="${YYYYMMDD}.${HH}0000."
+     n=${DA_CYCLE_INTERV}
+     while [[ $n -le 6 ]] ; do
+       checkfile=${bkpath}/${restart_prefix}coupler.res
+       if [ -r "${checkfile}" ] ; then
+         print_info_msg "$VERBOSE" "Found ${checkfile}; Use it as background for analysis "
+         break
+       else
+         n=$((n + ${DA_CYCLE_INTERV}))
+         YYYYMMDDHHmInterv=$( date +%Y%m%d%H -d "${START_DATE} ${n} hours ago" )
+         bkpath=${fg_root}/${YYYYMMDDHHmInterv}/${fg_restart_dirname}/RESTART  # cycling, use background from RESTART
+         print_info_msg "$VERBOSE" "Trying this path: ${bkpath}"
+       fi
+     done
+  fi
+#
+  checkfile=${bkpath}/${restart_prefix}coupler.res
+  if [ -r "${checkfile}" ] ; then
     cp_vrfy ${bkpath}/${restart_prefix}fv_core.res.tile1.nc       fv_core.res.tile1.nc 
     cp_vrfy ${bkpath}/${restart_prefix}fv_tracer.res.tile1.nc     fv_tracer.res.tile1.nc
     cp_vrfy ${bkpath}/${restart_prefix}sfc_data.nc                sfc_data.nc 
@@ -249,6 +279,9 @@ else
     cp_vrfy ${bkpath}/${restart_prefix}fv_srf_wnd.res.tile1.nc    fv_srf_wnd.res.tile1.nc
     cp_vrfy ${bkpath}/${restart_prefix}phy_data.nc                phy_data.nc
     cp_vrfy ${fg_root}/${YYYYMMDDHHmInterv}/${fg_restart_dirname}/INPUT/gfs_ctrl.nc  gfs_ctrl.nc
+    if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then
+      echo "${YYYYMMDDHH}(${cycle_type}): warm start at ${current_time} from ${checkfile} " >> ${EXPTDIR}/log.cycles
+    fi
 
 # do SST update at ${SST_update_hour}z for the restart sfc_data.nc
     if [ ${HH} -eq ${SST_update_hour} ]; then
@@ -262,10 +295,10 @@ else
          ${ECHO} "ERROR: No SST update at ${time_str}!!!!"
        fi
        if [ -r "latest.SST" ]; then
-         cp_vrfy ${FIXgsm}/RTG_SST_landmask.dat                ./RTG_SST_landmask.dat
+         ln_vrfy ${FIXgsm}/RTG_SST_landmask.dat                ./RTG_SST_landmask.dat
          ln_vrfy ./latest.SST                                  ./SSTRTG
-         cp_vrfy ${FIX_GSI}/${PREDEF_GRID_NAME}/fv3_grid_spec  ./fv3_grid_spec
-         cp_vrfy ${FIX_GSI}/${PREDEF_GRID_NAME}/fv3_akbk       ./fv3_akbk
+         ln_vrfy ${FIX_GSI}/${PREDEF_GRID_NAME}/fv3_grid_spec  ./fv3_grid_spec
+         ln_vrfy ${FIX_GSI}/${PREDEF_GRID_NAME}/fv3_akbk       ./fv3_akbk
 
 cat << EOF > sst.namelist
 &setup
@@ -277,6 +310,10 @@ cat << EOF > sst.namelist
 /
 EOF
          ${EXECDIR}/process_updatesst > stdout_sstupdate 2>&1
+         sst_reference_time=$(wgrib2 -t latest.SST) 
+         if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then
+           echo "${YYYYMMDDHH}(${cycle_type}): update SST using ${sst_reference_time}" >> ${EXPTDIR}/log.cycles
+         fi
        else
          echo "ERROR: No latest SST file for update at ${YYYYMMDDHH}!!!!"
        fi
@@ -377,10 +414,11 @@ fi
 # Please consult Ming or Tanya first before turning on this surgery.
 #
 #-----------------------------------------------------------------------
-#
-if [ ${YYYYMMDDHH} -eq 9999999999 ]; then
+# 
+#if [ ${YYYYMMDDHH} -eq 2021100812 ] ; then
+if [ ${HH} -eq 06 ] || [ ${HH} -eq 18 ]; then
+if [ ${cycle_type} == "spinup" ]; then
    raphrrr_com=/mnt/lfs4/BMC/rtwbl/mhu/wcoss/nco/com/
-#   cp_vrfy ${FIX_GSI}/use_raphrrr_sfc.namelist                                  use_raphrrr_sfc.namelist
    ln_vrfy -sf ${FIX_GSI}/${PREDEF_GRID_NAME}/fv3_grid_spec                     fv3_grid_spec
    ln -s ${raphrrr_com}/rap/prod/rap.${YYYYMMDD}/rap.t${HH}z.wrf_inout_smoke    sfc_rap
    ln -s ${raphrrr_com}/hrrr/prod/hrrr.${YYYYMMDD}/conus/hrrr.t${HH}z.wrf_inout sfc_hrrr
@@ -403,13 +441,17 @@ EOF
 
       ./${exect} > stdout 2>&1 || print_info_msg "\
       Call to executable to run surface surgery returned with nonzero exit code."
+      if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then
+        echo "${YYYYMMDDHH}(${cycle_type}): run surface surgery" >> ${EXPTDIR}/log.cycles
+      fi
    else
       print_info_msg "\
       The executable specified in exect does not exist:
       exect = \"${EXECDIR}/$exect\"
       Build executable and rerun."
    fi
 fi
+fi
 #
 #-----------------------------------------------------------------------
 #

ush/config.sh.RRFS_NA_3km

@@ -24,12 +24,14 @@ if [[ $MACHINE == "hera" ]] ; then
 fi
 
 VERBOSE="TRUE"
+SAVE_CYCLE_LOG="TRUE"
 
 RUN_ENVIR="nco"
 PREDEF_GRID_NAME=RRFS_NA_3km
 
 DO_DACYCLE="TRUE"
-DO_SURFACE_CYCLE="TRUE"
+DO_SPINUP="TRUE"
+#DO_SURFACE_CYCLE="TRUE"
 #DO_RETRO="TRUE"
 #LBCS_ICS_ONLY="TRUE"
 
@@ -42,13 +44,13 @@ LBC_SPEC_INTVL_HRS="1"
 EXTRN_MDL_LBCS_OFFSET_HRS="6"
 EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0"
 
-DATE_FIRST_CYCL="20210907"
-DATE_LAST_CYCL="20210930"
+DATE_FIRST_CYCL="20211012"
+DATE_LAST_CYCL="20211022"
 CYCL_HRS=( "00" )
 CYCL_HRS_SPINSTART=("03" "15")
-CYCL_HRS_PRODSTART=("03" "15")
-CYCLEMONTH="8"
-CYCLEDAY="1-4"
+CYCL_HRS_PRODSTART=("09" "21")
+CYCLEMONTH="10"
+CYCLEDAY="12-22"
 #
 PREEXISTING_DIR_METHOD="upgrade"
 INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 12:00:00"
@@ -62,6 +64,7 @@ if [[ $DO_SPINUP == "TRUE" ]] ; then
 fi
 
 FCST_LEN_HRS="2"
+FCST_LEN_HRS_SPINUP="1"
 POSTPROC_LEN_HRS="2"
 POSTPROC_LONG_LEN_HRS="24"
 #FCST_LEN_HRS_CYCLES=(30 03 03 03 03 03 03 03 03 03 03 03 30 03 03 03 03 03 03 03 03 03 03 03)

ush/config_defaults.sh

@@ -1207,10 +1207,14 @@ PREEXISTING_DIR_METHOD="delete"
 # Set VERBOSE.  This is a flag that determines whether or not the experiment
 # generation and workflow task scripts tend to be print out more informational
 # messages.
+# Set SAVE_CYCLE_LOG.  This is a flag that determines whether or not save
+#    the information related to data assimilation cycling, such as background
+#    used in each cycle
 #
 #-----------------------------------------------------------------------
 #
 VERBOSE="TRUE"
+SAVE_CYCLE_LOG="FALSE"
 #
 #-----------------------------------------------------------------------
 #
@@ -1569,6 +1573,7 @@ WTIME_RUN_NONVARCLDANL="00:20:00"
 #
 MEMO_RUN_REF2TTEN="10G"
 MEMO_RUN_NONVARCLDANL="20G"
+MEMO_RUN_PREPSTART="24G"
 MEMO_RUN_WGRIB2="24G"
 #
 # Maximum number of attempts.

ush/generate_FV3LAM_wflow.sh

@@ -289,6 +289,7 @@ settings="\
 #
   'memo_run_ref2tten': ${MEMO_RUN_REF2TTEN}
   'memo_run_nonvarcldanl': ${MEMO_RUN_NONVARCLDANL}
+  'memo_run_prepstart': ${MEMO_RUN_PREPSTART}
   'memo_run_wgrib2': ${MEMO_RUN_WGRIB2}
 #
 # Maximum number of tries for each task.

ush/setup.sh

@@ -353,6 +353,7 @@ optionList[2]=DO_RETRO
 optionList[3]=LBCS_ICS_ONLY
 optionList[4]=DO_NONVAR_CLDANAL
 optionList[5]=DO_REFL2TTEN
+optionList[6]=SAVE_CYCLE_LOG
 
 obs_number=${#optionList[@]}
 for (( i=0; i<${obs_number}; i++ ));