Merge branch 'feature/lnd_regrid_iau' into soilanal_det

* feature/lnd_regrid_iau:
  reverst hera hosts to default
  turn off land iau for gfs
  differentiate gfs and gdas for land iau
  cap max num procs for regridding at 36 for now
  separate increment regr hrs fhrs
  combine regrid multi times using nco
  update eupd to Clara's regridding branch
  revert eupd to develop
  update sfc inc regrid call
  modify for Clara's gauss-fv3 regridding

env/HERA.env

@@ -182,7 +182,16 @@ elif [[ "${step}" = "sfcanl" ]]; then
 
     export NTHREADS_CYCLE=${threads_per_task:-14}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
-    export APRUN_REGR="${launcher} -n 6"
+
+    # ensure div by 6 for n tasks
+    # TODO: TZG change default for sfcanl and esfc to multiple of 6 (42?)
+    # or change the reg program to use any procs
+    ntdiv6=$((ntasks/6))
+    ntasks_reg=$((ntdiv6*6))
+    if [[ ${ntasks_reg} -gt 36 ]]; then    #TODO: the reg fails with large num proc
+        ntasks_reg=36 
+    fi
+    export APRUN_REGR="${launcher} -n ${ntasks_reg} "
 
 elif [[ "${step}" = "eobs" ]]; then
 
@@ -248,7 +257,16 @@ elif [[ "${step}" = "esfc" ]]; then
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
-    export APRUN_REGR="${launcher} -n 6"
+
+    # ensure div by 6 for n tasks
+    # TODO: TZG change default for sfcanl and esfc to multiple of 6 (42?)
+    # or change the reg program to use any procs
+    ntdiv6=$((ntasks/6))
+    ntasks_reg=$((ntdiv6*6))
+    if [[ ${ntasks_reg} -gt 36 ]]; then    #TODO: the reg fails with large num proc
+        ntasks_reg=36
+    fi
+    export APRUN_REGR="${launcher} -n ${ntasks_reg} "
 
 elif [[ "${step}" = "epos" ]]; then
 

parm/config/gfs/config.base

@@ -436,6 +436,25 @@ fi
 
 export GSI_SOILANAL=@GSI_SOILANAL@
 
+# Land IAU for soil temp
+export DO_LAND_IAU="@DO_LAND_IAU@" 
+export ONLY_LAND_IAU_INC="NO"  #GSI increments only added by land iau
+export LAND_IAU_FHRS="3,6,9"
+export LAND_IAU_DELTHRS=6
+export LAND_IAU_INC_FILES='sfc_inc',''
+export LSOIL_INCR=3
+export LAND_IAU_FILTER_INCREMENTS=.false.
+export LAND_IAU_UPD_STC=.true.
+export LAND_IAU_UPD_SLC=.true.
+export LAND_IAU_DO_STCSMC_ADJUSTMENT=.true.
+export LAND_IAU_MIN_T_INCREMENT=0.0001
+#export LAND_IAU_MIN_SLC_INCREMENT=0.000001
+
+# no land iau if cycle is cold start/ forecast only
+if [[ "${MODE}" = "cycled" && "${SDATE}" = "${PDY}${cyc}" && ${EXP_WARM_START} = ".false." ]] || [[ "${MODE}" = "forecast-only" && ${EXP_WARM_START} = ".false." ]] ; then   
+  export DO_LAND_IAU="NO"   #TODO: turning off Land IAU
+fi
+
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst
 export DONST="YES"
 if [[ ${DONST} = "YES" ]]; then export FNTSFA="        "; fi

parm/config/gfs/config.esfc

@@ -23,8 +23,14 @@ fi
 
 # set up soil analysis
 if [[ ${GSI_SOILANAL} = "YES" ]]; then
-    export DO_LNDINC=".true."
-    export LND_SOI_FILE="lnd_incr"
+    #Land IAU no dolndinc
+    if [[ ${ONLY_LAND_IAU_INC} = "YES" ]]; then
+        export DO_LNDINC=".false."
+        export LND_SOI_FILE="NULL"
+    else
+        export DO_LNDINC=".true."
+        export LND_SOI_FILE="lnd_incr"
+    fi
 fi
 
 echo "END: config.esfc"

parm/config/gfs/config.fcst

@@ -298,6 +298,9 @@ elif [[ "${RUN}" =~ "gfs" ]] ; then # GFS cycle specific parameters
       export io_layout="1,1"
     fi
 
+    # Turn off Land IAU for GFS
+    export DO_LAND_IAU="NO"
+
 fi
 
 # Remember config.efcs will over-ride these values for ensemble forecasts

parm/config/gfs/yaml/defaults.yaml

@@ -19,7 +19,8 @@ base:
   FHMAX_HF_GFS: 0
   FCST_BREAKPOINTS: ""
   DO_VRFY_OCEANDA: "NO"
-  GSI_SOILANAL: "NO"
+  GSI_SOILANAL: "YES"
+  DO_LAND_IAU: "YES"
   EUPD_CYC: "gdas"
   FHMAX_ENKF_GFS: 12
   DOHYBVAR_OCN: "NO"

scripts/exgdas_enkf_sfc.sh

@@ -123,9 +123,32 @@ if [ $GSI_SOILANAL = "YES" ]; then
     export CASE_IN=${CASE_ENS}
     export CASE_OUT=${CASE_ENS}
     export OCNRES_OUT=${OCNRES}
-    export soilinc_fhrs="06"
     export NMEM_REGRID=${NMEM_ENS}
 
+    if [ $DO_LAND_IAU = "YES" ]; then
+        ifhrs=()
+        IFS=',' read -ra landifhrs <<< "${LAND_IAU_FHRS}"
+        for ihr in "${landifhrs[@]}"; do
+            hrstr=$(printf "%02d" $ihr);
+            ifhrs+=("$hrstr")
+        done
+        export landiau_fhrs="${ifhrs[@]}"
+    fi
+
+    # differentiating soilinc_fhrs and fhrs for regrid
+    # all GSI incr are regridded; not all inrements are used in soil update
+
+    ifhrs=()
+    IFS=',' read -ra landifhrs <<< "${IAUFHRS_ENKF}"  #$(echo $IAUFHRS_ENKF | sed 's/,/ /g')
+    for ihr in "${landifhrs[@]}"; do
+        hrstr=$(printf "%02d" $ihr);
+        ifhrs+=("$hrstr")
+    done
+
+    export landinc_reghrs="${ifhrs[@]}"
+
+    export soilinc_fhrs=("06")
+
     $REGRIDSH
 
 fi
@@ -134,6 +157,7 @@ export APRUNCY=${APRUN_CYCLE:-$APRUN_ESFC}
 export OMP_NUM_THREADS_CY=${NTHREADS_CYCLE:-$NTHREADS_ESFC}
 export MAX_TASKS_CY=$NMEM_ENS
 
+# ToDO TZG update this to sync with land_iau
 if [ $DOIAU = "YES" ]; then
     # Update surface restarts at beginning of window when IAU is ON
     # For now assume/hold dtfanl.nc is valid at beginning of window.
@@ -148,6 +172,7 @@ if [ $DOIAU = "YES" ]; then
             if (( smem > NMEM_ENS_MAX )); then
                smem=$((smem - NMEM_ENS_MAX))
             fi
+	    # CSD - what is this doing?
             gmemchar="mem"$(printf %03i "$smem")
             cmem=$(printf %03i $imem)
             memchar="mem$cmem"
@@ -180,6 +205,7 @@ if [ $DOIAU = "YES" ]; then
 
             if [[ ${GSI_SOILANAL} = "YES" ]]; then
                 FHR=6
+#if [ $DO_LAND_IAU = "YES" ] TODO: make sure iau updated vars are not updated again
                  ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" \
                    "${DATA}/soil_xainc.${cmem}" 
             fi
@@ -208,7 +234,6 @@ if [ $DOIAU = "YES" ]; then
             [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}"
             cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc"
 
-
             if [[ ${GSI_SOILANAL} = "YES" ]]; then
                 FHR=6
                 ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \

scripts/exglobal_atmos_sfcanl.sh

@@ -117,6 +117,7 @@ fi
 # Collect the dates in the window to update surface restarts
 gcycle_dates=("${PDY}${cyc}")  # Always update surface restarts at middle of window
 soilinc_fhrs=("06") # increment file at middle of window has FHR=6
+# ToDO TZG update this to sync with land_iau
 if [[ "${DOIAU:-}" == "YES" ]]; then  # Update surface restarts at beginning of window
   half_window=$(( assim_freq / 2 ))
   half_fhrs=$(printf %02d $half_window)
@@ -133,7 +134,28 @@ if [ $GSI_SOILANAL = "YES" ]; then
     export CASE_IN=${CASE_ENS}
     export CASE_OUT=$CASE
     export OCNRES_OUT=$OCNRES
-
+
+    if [ $DO_LAND_IAU = "YES" ]; then
+	ifhrs=()
+        IFS=',' read -ra landifhrs <<< "${LAND_IAU_FHRS}"
+        for ihr in "${landifhrs[@]}"; do
+            hrstr=$(printf "%02d" $ihr);
+            ifhrs+=("$hrstr")
+        done
+	export landiau_fhrs="${ifhrs[@]}"
+    fi
+
+    # differentiating soilinc_fhrs (used below) and fhrs for regrid
+    # all GSI incr are regridded; not all inrements are used in the update
+    ifhrs=()
+    IFS=',' read -ra landifhrs <<< "${IAUFHRS_ENKF}"  #$(echo $IAUFHRS_ENKF | sed 's/,/ /g')
+    for ihr in "${landifhrs[@]}"; do
+        hrstr=$(printf "%02d" $ihr);
+        ifhrs+=("$hrstr")
+    done
+
+    export landinc_reghrs="${ifhrs[@]}"
+
     $REGRIDSH
 
 fi
@@ -142,17 +164,18 @@ fi
 for hr in "${!gcycle_dates[@]}"; do
 
   gcycle_date=${gcycle_dates[hr]}
-  FHR=${FHR[hr]}
+  FHR=${soilinc_fhrs[hr]}   #${FHR[hr]} TODO check 
   echo "CSD check hours $FHR"
 
   echo "Updating surface restarts for ${gcycle_date} ..."
 
   datestr="${gcycle_date:0:8}.${gcycle_date:8:2}0000"
 
   # THIS BLOCK SPECIFIC TO NON-IAU CASE
-  if [[ ${GSI_SOILANAL} = "YES" ]]; then
+# TODO: TZG Add no land iau condition?
+  if [[ ${GSI_SOILANAL} = "YES" ]]; then   # && $DO_LAND_IAU = "NO" ]]; then
         for (( nn=1; nn <= ntiles; nn++ )); do
-        cp "${COMIN_ATMOS_ANALYSIS}/sfci00${FHR}.tile${nn}.nc" \
+        cp "${COMIN_ATMOS_ANALYSIS}/sfci0${FHR}.tile${nn}.nc" \
            "${DATA}/soil_xainc.00${nn}" 
         done
   fi

ush/forecast_det.sh

@@ -26,6 +26,8 @@ UFS_det(){
     IAU_OFFSET=0
     model_start_date_current_cycle=${current_cycle}
 
+    DO_LAND_IAU="NO"           #TODO: TZG
+
     # It is still possible that a restart is available from a previous forecast attempt
     # So we have to continue checking for restarts
   fi

ush/forecast_postdet.sh

@@ -2,6 +2,9 @@
 
 # Disable variable not used warnings
 # shellcheck disable=SC2034
+
+ntiles=${ntiles:-6}
+
 FV3_postdet() {
   echo "SUB ${FUNCNAME[0]}: Entering for RUN = ${RUN}"
 
@@ -135,7 +138,12 @@ FV3_postdet() {
         IAU_DELTHRS=0
         IAU_INC_FILES="''"
       fi
-
+      if [[ ${DO_LAND_IAU} = "YES" ]]; then
+	DO_LAND_IAU="NO"
+        LAND_IAU_FHRS=-1
+        LAND_IAU_DELTHRS=0
+        LAND_IAU_INC_FILES="''"
+      fi
     #--------------------------------------------------------------------------
     else  # "${RERUN}" == "NO"
 
@@ -199,6 +207,24 @@ EOF
         fi
       done
 
+      # TZG: SFC increments
+      # sfc_inc in FV3 grid, all timesteps in one file per tile (Land IAU or not)
+      if [[ ${DO_LAND_IAU} = "YES" ]]; then
+        local TN sfc_increment_file
+        for TN in $(seq 1 $ntiles); do
+          sfc_increment_file="${COMIN_ATMOS_ANALYSIS}/sfc_inc.tile${TN}.nc"
+          if [[ ! -f ${sfc_increment_file} ]]; then
+            echo "ERROR: DO_LAND_IAU=${DO_LAND_IAU}, but missing increment file ${sfc_increment_file}"
+            echo "Abort!"
+            exit 1
+          fi
+          ${NCP} "${sfc_increment_file}" "${DATA}/INPUT/sfc_inc.tile${TN}.nc"
+        done
+	LAND_IAU_INC_FILES=${LAND_IAU_INC_FILES:-'sfc_inc'}
+
+      #else #TODO: check to make sure correct sfc inc for non-IAU
+      fi
+
     fi  # if [[ "${RERUN}" == "YES" ]]; then
     #--------------------------------------------------------------------------
 

ush/parsing_namelists_FV3.sh

@@ -649,6 +649,30 @@ cat >> input.nml <<EOF
 /
 EOF
 
+## TZG: Land IAU
+LOG_DO_LAND_IAU=.false.
+if [[ ${DO_LAND_IAU} = "YES" ]]; then
+  LOG_DO_LAND_IAU=.true.
+fi
+#TODO: check land inc file names 
+echo "" >> input.nml
+cat >> input.nml << EOF
+&land_iau_nml
+  do_land_iau=${LOG_DO_LAND_IAU}
+  land_iau_fhrs=${LAND_IAU_FHRS}
+  land_iau_delthrs=${LAND_IAU_DELTHRS}
+  land_iau_inc_files=${LAND_IAU_INC_FILES} 
+  lsoil_incr=${LSOIL_INCR}
+  land_iau_filter_increments=${LAND_IAU_FILTER_INCREMENTS:-".false."}
+  land_iau_upd_stc=${LAND_IAU_UPD_STC}
+  land_iau_upd_slc=${LAND_IAU_UPD_SLC}
+  land_iau_do_stcsmc_adjustment=${LAND_IAU_DO_STCSMC_ADJUSTMENT}
+  land_iau_min_T_increment=${LAND_IAU_MIN_T_INCREMENT}
+
+/
+EOF
+echo "" >> input.nml
+
 # Add namelist for stochastic physics options
 echo "" >> input.nml
 #if [ $MEMBER -gt 0 ]; then