[develop] Modify get_lbcs and make_lbcs. (#666)

Get_lbcs need to download 0th hour LBCS and make_lbcs need to be parallelized.
Note that there is no workflow so this PR can not be run independently until then.

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -98,6 +98,8 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
   export TIME_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-0}
   export EXTRN_MDL_NAME=${EXTRN_MDL_NAME_LBCS}
   GFS_FILE_FMT=${FV3GFS_FILE_FMT_LBCS}
+
+  export NEED_ALL_LBCS=${DO_RRFS_DEV}
 fi
 
 #

jobs/JREGIONAL_MAKE_ICS

@@ -57,10 +57,14 @@ for the FV3 (in NetCDF format).
 #
 if [ $RUN_ENVIR = "nco" ]; then
     export INPUT_DATA="${COMIN}"
+    export INPUT_DATA_NWGES="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/ics"
 else
     export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT"
+    export INPUT_DATA_NWGES="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/ics"
 fi
 mkdir_vrfy -p "${INPUT_DATA}"
+mkdir_vrfy -p "${INPUT_DATA_NWGES}"
+#
 #
 #-----------------------------------------------------------------------
 #

jobs/JREGIONAL_MAKE_LBCS

@@ -57,10 +57,13 @@ hour zero).
 #
 if [ $RUN_ENVIR = "nco" ]; then
     export INPUT_DATA="${COMIN}"
+    export INPUT_DATA_NWGES="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/lbcs"
 else
     export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT"
+    export INPUT_DATA_NWGES="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/lbcs"
 fi
 mkdir_vrfy -p "${INPUT_DATA}"
+mkdir_vrfy -p "${INPUT_DATA_NWGES}"
 #
 #-----------------------------------------------------------------------
 #

parm/wflow/coldstart.yaml

@@ -82,6 +82,7 @@ metatask_run_ensemble:
     envars:
       <<: *default_vars
       SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;'
+      NWGES_DIR: !cycstr '&NWGES_BASEDIR;/@Y@m@d@H'
     join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
     nnodes: 4
     ppn: 12
@@ -121,6 +122,9 @@ metatask_run_ensemble:
     envars:
       <<: *default_vars
       SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;'
+      NWGES_DIR: !cycstr '&NWGES_BASEDIR;/@Y@m@d@H'
+      bcgrp: '00'
+      bcgrpnum: '1'
     join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
     nnodes: 4
     ppn: 12

parm/wflow/default_workflow.yaml

@@ -30,6 +30,8 @@ rocoto:
     COLDSTART: '{{ workflow.COLDSTART }}'
     WARMSTART_CYCLE_DIR: '{{ workflow.WARMSTART_CYCLE_DIR }}'
     WORKFLOW_ID: '{{ workflow.WORKFLOW_ID }}'
+    # rrfs
+    NWGES_BASEDIR: '{{ nco.NWGES_BASEDIR }}' 
   attrs:
     cyclethrottle: "200"
     realtime: "F"

scripts/exregional_get_extrn_mdl_files.sh

@@ -76,7 +76,19 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
     CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} ))
     FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]}
   fi
-  last_time=$((TIME_OFFSET_HRS + FCST_LEN_HRS))
+  end_hr=$FCST_LEN_HRS
+  if [ $BOUNDARY_LEN_HRS -gt $end_hr ]; then
+     end_hr=$BOUNDARY_LEN_HRS
+  fi
+  # Download 0th hour lbcs if requested for it, mostly for DA
+  if [ ${NEED_ALL_LBCS} = "TRUE" ]; then
+    first_time=$((TIME_OFFSET_HRS))
+  else
+    first_time=$((TIME_OFFSET_HRS + LBC_SPEC_INTVL_HRS ))
+  fi
+  last_time=$((TIME_OFFSET_HRS + end_hr))
+
+
   fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}"
   file_names=${EXTRN_MDL_FILES_LBCS[@]}
   if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] ; then

scripts/exregional_make_ics.sh

@@ -124,7 +124,8 @@ case "${CCPP_PHYS_SUITE}" in
   "FV3_GFS_v15_thompson_mynn_lam3km" | \
   "FV3_GFS_v17_p8" | \
   "FV3_WoFS_v0" | \
-  "FV3_HRRR" )
+  "FV3_HRRR" | \
+  "FV3_RAP" )
     if [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ] || \
        [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ]; then
       varmap_file="GSDphys_var_map.txt"
@@ -676,6 +677,20 @@ fi
 #
 #-----------------------------------------------------------------------
 #
+# Symlink files to NWGES directory, dropping prefix
+#
+#-----------------------------------------------------------------------
+#
+for i in ${INPUT_DATA}/*.nc; do
+    file=$(basename $i)
+    prefix="${NET}.${cycle}${dot_ensmem}."
+    file=${file#"$prefix"}
+    file=${file/f000/000}
+    ln_vrfy -sf $i ${INPUT_DATA_NWGES}/${file}
+done
+#
+#-----------------------------------------------------------------------
+#
 # Print message indicating successful completion of script.
 #
 #-----------------------------------------------------------------------

scripts/exregional_make_lbcs.sh

@@ -99,16 +99,6 @@ fi
 DATA="${DATA}/tmp_LBCS"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
-
-if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then
-  cyc_mod=$(( ${cyc} - ${DATE_FIRST_CYCL:8:2} ))
-  CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} ))
-  FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]}
-fi
-LBC_SPEC_FCST_HRS=()
-for i_lbc in $(seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} $(( FCST_LEN_HRS+LBC_SPEC_INTVL_HRS )) ); do
-  LBC_SPEC_FCST_HRS+=("$i_lbc")
-done
 #
 #-----------------------------------------------------------------------
 #
@@ -132,7 +122,8 @@ case "${CCPP_PHYS_SUITE}" in
   "FV3_GFS_v15_thompson_mynn_lam3km" | \
   "FV3_GFS_v17_p8" | \
   "FV3_WoFS_v0" | \
-  "FV3_HRRR" )
+  "FV3_HRRR" | \
+  "FV3_RAP")
     if [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ] || \
        [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ]; then
       varmap_file="GSDphys_var_map.txt"
@@ -368,7 +359,12 @@ fi
 #-----------------------------------------------------------------------
 #
 num_fhrs="${#EXTRN_MDL_FHRS[@]}"
-for (( i=0; i<${num_fhrs}; i++ )); do
+bcgrp10=${bcgrp#0}
+bcgrpnum10=${bcgrpnum#0}
+for (( ii=0; ii<${num_fhrs}; ii=ii+bcgrpnum10 )); do
+  i=$(( ii + bcgrp10 ))
+  if [ ${i} -lt ${num_fhrs} ]; then
+    echo " group ${bcgrp10} processes member ${i}"
 #
 # Get the forecast hour of the external model.
 #
@@ -528,9 +524,22 @@ located in the following directory:
 # the forecast hour of the FV3-LAM (which is not necessarily the same as
 # that of the external model since their start times may be offset).
 #
-  fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FCST_HRS[$i]}" )
+  lbc_spec_fhrs=( "${EXTRN_MDL_FHRS[$i]}" )
+  fcst_hhh=$(( ${lbc_spec_fhrs} - ${EXTRN_MDL_LBCS_OFFSET_HRS} ))
+  fcst_hhh_FV3LAM=$( printf "%03d" "$fcst_hhh" )
   mv_vrfy gfs.bndy.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc
 
+#
+#-----------------------------------------------------------------------
+#
+# Symlink files to NWGES directory, dropping prefix
+#
+#-----------------------------------------------------------------------
+#
+  ln_vrfy -sf ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc \
+          ${INPUT_DATA_NWGES}/gfs_bndy.tile7.${fcst_hhh_FV3LAM}.nc
+
+  fi
 done
 #
 #-----------------------------------------------------------------------

ush/config_defaults.yaml

@@ -881,6 +881,15 @@ workflow:
   # An array containing the hours of the day at which the product cycle starts,
   # from cold start input or from spin-up cycle forcast
   #
+  # BOUNDARY_LEN_HRS
+  # The length of boundary condition for normal forecast, in integer hours.
+  #
+  # BOUNDARY_LONG_LEN_HRS
+  # The length of boundary condition for long forecast, in integer hours.
+  #
+  # BOUNDARY_PROC_GROUP_NUM
+  # The number of groups used to run make_lbcs, in integer from 1 to forecast longest hours.
+  #
   #-----------------------------------------------------------------------
   #
   DATE_FIRST_CYCL: "YYYYMMDDHH"
@@ -893,6 +902,9 @@ workflow:
 
   CYCL_HRS_SPINSTART: []
   CYCL_HRS_PRODSTART: []
+  BOUNDARY_LEN_HRS: 0
+  BOUNDARY_LONG_LEN_HRS: 0
+  BOUNDARY_PROC_GROUP_NUM: 1
   #
   #-----------------------------------------------------------------------
   #
@@ -1028,6 +1040,12 @@ nco:
   # LOGBASEDIR:
   # Directory in which the log files from the workflow tasks will be placed.
   # 
+  # NWGES:
+  # The beginning portion of the directory that will contain the output 
+  # files from the forecast for a given cycle.  For a cycle 
+  # that starts on the date specified by yyyymmdd and hour specified by hh
+  #   ${NWGES}/yyyymmddhh
+  # 
   # For more information on NCO standards
   #   
   #   https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf
@@ -1038,6 +1056,7 @@ nco:
   NET: "rrfs"
   RUN: "rrfs"
   model_ver: "v1.0.0"
+
   OPSROOT: '{{ workflow.EXPT_BASEDIR }}/../nco_dirs'
   COMROOT: '{{ OPSROOT }}/com'
   PACKAGEROOT: '{{ OPSROOT }}/packages'
@@ -1048,6 +1067,12 @@ nco:
   COMIN_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}'
   COMOUT_BASEDIR: '{{ COMROOT }}/{{ NET }}/{{ model_ver }}'
 
+  #
+  # New additions from RRFS_dev1
+  #
+  NWGES: '{{ OPSROOT }}/nwges'
+  NWGES_BASEDIR: '{{ NWGES }}'
+
   #
   #-----------------------------------------------------------------------
   #
@@ -1145,11 +1170,18 @@ rrfs:
   #
   #-----------------------------------------------------------------------
   #
+  # Set parameters associated with running retrospective experiments.  Definitions:
+  #
+  # DO_RRFS_DEV:
+  # Flag to turn on RRFS dev workflow. Should be removed
+  # in the future once SRW and RRFS dev workflows are completely integrated
+  #
   # DO_NLDN_LGHT
   #     Flag turn on processing NLDN NetCDF lightning data  
   #
   #-----------------------------------------------------------------------
   #
+  DO_RRFS_DEV: false # Turn this on to test RRFS dev workflow
   DO_NLDN_LGHT: false
 
 #----------------------------
@@ -1587,6 +1619,26 @@ task_get_extrn_lbcs:
   LBC_SPEC_INTVL_HRS: 6
   EXTRN_MDL_LBCS_OFFSET_HRS: '{{ 3 if EXTRN_MDL_NAME_LBCS == "RAP" else 0 }}'
   FV3GFS_FILE_FMT_LBCS: "nemsio"
+  #
+  #-----------------------------------------------------------------------
+  #
+  # LBCS_SEARCH_HRS:
+  #  When search boundary conditions tasks from previous cycles in prep_cyc step,
+  #  For example: 0 means search start for the same cycle lbcs task.
+  #               1 means search start for 1-h previous cycle lbcs task.
+  #               2 means search start for 2-h previous cycle lbcs task.
+  #
+  # EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS:
+  #  When search boundary conditions from previous cycles in prep_start step,
+  #  the search will start at cycle before (this parameter) of current cycle.
+  #  For example: 0 means search start at the same cycle lbcs directory.
+  #               1 means search start at 1-h previous cycle  lbcs directory.
+  #               2 means search start at 2-h previous cycle  lbcs directory.
+  #
+  #-----------------------------------------------------------------------
+  #
+  LBCS_SEARCH_HRS: 6
+  EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS: 0
   #-----------------------------------------------------------------------
   #
   # EXTRN_MDL_SYSBASEDIR_LBCS: