diff --git a/env/HERA.env b/env/HERA.env index c29b60d812..581974d7e9 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -182,7 +182,16 @@ elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" - export APRUN_REGR="${launcher} -n 6" + + # ensure div by 6 for n tasks + # TODO: TZG change default for sfcanl and esfc to multiple of 6 (42?) + # or change the reg program to use any procs + ntdiv6=$((ntasks/6)) + ntasks_reg=$((ntdiv6*6)) + if [[ ${ntasks_reg} -gt 36 ]]; then #TODO: the reg fails with large num proc + ntasks_reg=36 + fi + export APRUN_REGR="${launcher} -n ${ntasks_reg} " elif [[ "${step}" = "eobs" ]]; then @@ -248,7 +257,16 @@ elif [[ "${step}" = "esfc" ]]; then export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" - export APRUN_REGR="${launcher} -n 6" + + # ensure div by 6 for n tasks + # TODO: TZG change default for sfcanl and esfc to multiple of 6 (42?) + # or change the reg program to use any procs + ntdiv6=$((ntasks/6)) + ntasks_reg=$((ntdiv6*6)) + if [[ ${ntasks_reg} -gt 36 ]]; then #TODO: the reg fails with large num proc + ntasks_reg=36 + fi + export APRUN_REGR="${launcher} -n ${ntasks_reg} " elif [[ "${step}" = "epos" ]]; then diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index fbbe5c782f..48fe11376b 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -436,6 +436,25 @@ fi export GSI_SOILANAL=@GSI_SOILANAL@ +# Land IAU for soil temp +export DO_LAND_IAU="@DO_LAND_IAU@" +export ONLY_LAND_IAU_INC="NO" #GSI increments only added by land iau +export LAND_IAU_FHRS="3,6,9" +export LAND_IAU_DELTHRS=6 +export LAND_IAU_INC_FILES='sfc_inc','' +export LSOIL_INCR=3 +export LAND_IAU_FILTER_INCREMENTS=.false. +export LAND_IAU_UPD_STC=.true. +export LAND_IAU_UPD_SLC=.true. +export LAND_IAU_DO_STCSMC_ADJUSTMENT=.true. +export LAND_IAU_MIN_T_INCREMENT=0.0001 +#export LAND_IAU_MIN_SLC_INCREMENT=0.000001 + +# no land iau if cycle is cold start/ forecast only +if [[ "${MODE}" = "cycled" && "${SDATE}" = "${PDY}${cyc}" && ${EXP_WARM_START} = ".false." ]] || [[ "${MODE}" = "forecast-only" && ${EXP_WARM_START} = ".false." ]] ; then + export DO_LAND_IAU="NO" #TODO: turning off Land IAU +fi + # turned on nsst in anal and/or fcst steps, and turn off rtgsst export DONST="YES" if [[ ${DONST} = "YES" ]]; then export FNTSFA=" "; fi diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc index 684dea4ee3..fac846723d 100644 --- a/parm/config/gfs/config.esfc +++ b/parm/config/gfs/config.esfc @@ -23,8 +23,14 @@ fi # set up soil analysis if [[ ${GSI_SOILANAL} = "YES" ]]; then - export DO_LNDINC=".true." - export LND_SOI_FILE="lnd_incr" + #Land IAU no dolndinc + if [[ ${ONLY_LAND_IAU_INC} = "YES" ]]; then + export DO_LNDINC=".false." + export LND_SOI_FILE="NULL" + else + export DO_LNDINC=".true." + export LND_SOI_FILE="lnd_incr" + fi fi echo "END: config.esfc" diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 992f9eb329..bcdf780cea 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -298,6 +298,9 @@ elif [[ "${RUN}" =~ "gfs" ]] ; then # GFS cycle specific parameters export io_layout="1,1" fi + # Turn off Land IAU for GFS + export DO_LAND_IAU="NO" + fi # Remember config.efcs will over-ride these values for ensemble forecasts diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 55f4b03f50..7a36ad58eb 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -19,7 +19,8 @@ base: FHMAX_HF_GFS: 0 FCST_BREAKPOINTS: "" DO_VRFY_OCEANDA: "NO" - GSI_SOILANAL: "NO" + GSI_SOILANAL: "YES" + DO_LAND_IAU: "YES" EUPD_CYC: "gdas" FHMAX_ENKF_GFS: 12 DOHYBVAR_OCN: "NO" diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh index c69e27965b..87cd5c403a 100755 --- a/scripts/exgdas_enkf_sfc.sh +++ b/scripts/exgdas_enkf_sfc.sh @@ -123,9 +123,32 @@ if [ $GSI_SOILANAL = "YES" ]; then export CASE_IN=${CASE_ENS} export CASE_OUT=${CASE_ENS} export OCNRES_OUT=${OCNRES} - export soilinc_fhrs="06" export NMEM_REGRID=${NMEM_ENS} + if [ $DO_LAND_IAU = "YES" ]; then + ifhrs=() + IFS=',' read -ra landifhrs <<< "${LAND_IAU_FHRS}" + for ihr in "${landifhrs[@]}"; do + hrstr=$(printf "%02d" $ihr); + ifhrs+=("$hrstr") + done + export landiau_fhrs="${ifhrs[@]}" + fi + + # differentiating soilinc_fhrs and fhrs for regrid + # all GSI incr are regridded; not all inrements are used in soil update + + ifhrs=() + IFS=',' read -ra landifhrs <<< "${IAUFHRS_ENKF}" #$(echo $IAUFHRS_ENKF | sed 's/,/ /g') + for ihr in "${landifhrs[@]}"; do + hrstr=$(printf "%02d" $ihr); + ifhrs+=("$hrstr") + done + + export landinc_reghrs="${ifhrs[@]}" + + export soilinc_fhrs=("06") + $REGRIDSH fi @@ -134,6 +157,7 @@ export APRUNCY=${APRUN_CYCLE:-$APRUN_ESFC} export OMP_NUM_THREADS_CY=${NTHREADS_CYCLE:-$NTHREADS_ESFC} export MAX_TASKS_CY=$NMEM_ENS +# ToDO TZG update this to sync with land_iau if [ $DOIAU = "YES" ]; then # Update surface restarts at beginning of window when IAU is ON # For now assume/hold dtfanl.nc is valid at beginning of window. @@ -148,6 +172,7 @@ if [ $DOIAU = "YES" ]; then if (( smem > NMEM_ENS_MAX )); then smem=$((smem - NMEM_ENS_MAX)) fi + # CSD - what is this doing? gmemchar="mem"$(printf %03i "$smem") cmem=$(printf %03i $imem) memchar="mem$cmem" @@ -180,6 +205,7 @@ if [ $DOIAU = "YES" ]; then if [[ ${GSI_SOILANAL} = "YES" ]]; then FHR=6 +#if [ $DO_LAND_IAU = "YES" ] TODO: make sure iau updated vars are not updated again ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" \ "${DATA}/soil_xainc.${cmem}" fi @@ -208,7 +234,6 @@ if [ $DOIAU = "YES" ]; then [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" - if [[ ${GSI_SOILANAL} = "YES" ]]; then FHR=6 ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh index 4104f6d39f..7001d7ca3b 100755 --- a/scripts/exglobal_atmos_sfcanl.sh +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -117,6 +117,7 @@ fi # Collect the dates in the window to update surface restarts gcycle_dates=("${PDY}${cyc}") # Always update surface restarts at middle of window soilinc_fhrs=("06") # increment file at middle of window has FHR=6 +# ToDO TZG update this to sync with land_iau if [[ "${DOIAU:-}" == "YES" ]]; then # Update surface restarts at beginning of window half_window=$(( assim_freq / 2 )) half_fhrs=$(printf %02d $half_window) @@ -133,7 +134,28 @@ if [ $GSI_SOILANAL = "YES" ]; then export CASE_IN=${CASE_ENS} export CASE_OUT=$CASE export OCNRES_OUT=$OCNRES - + + if [ $DO_LAND_IAU = "YES" ]; then + ifhrs=() + IFS=',' read -ra landifhrs <<< "${LAND_IAU_FHRS}" + for ihr in "${landifhrs[@]}"; do + hrstr=$(printf "%02d" $ihr); + ifhrs+=("$hrstr") + done + export landiau_fhrs="${ifhrs[@]}" + fi + + # differentiating soilinc_fhrs (used below) and fhrs for regrid + # all GSI incr are regridded; not all inrements are used in the update + ifhrs=() + IFS=',' read -ra landifhrs <<< "${IAUFHRS_ENKF}" #$(echo $IAUFHRS_ENKF | sed 's/,/ /g') + for ihr in "${landifhrs[@]}"; do + hrstr=$(printf "%02d" $ihr); + ifhrs+=("$hrstr") + done + + export landinc_reghrs="${ifhrs[@]}" + $REGRIDSH fi @@ -142,7 +164,7 @@ fi for hr in "${!gcycle_dates[@]}"; do gcycle_date=${gcycle_dates[hr]} - FHR=${FHR[hr]} + FHR=${soilinc_fhrs[hr]} #${FHR[hr]} TODO check echo "CSD check hours $FHR" echo "Updating surface restarts for ${gcycle_date} ..." @@ -150,9 +172,10 @@ for hr in "${!gcycle_dates[@]}"; do datestr="${gcycle_date:0:8}.${gcycle_date:8:2}0000" # THIS BLOCK SPECIFIC TO NON-IAU CASE - if [[ ${GSI_SOILANAL} = "YES" ]]; then +# TODO: TZG Add no land iau condition? + if [[ ${GSI_SOILANAL} = "YES" ]]; then # && $DO_LAND_IAU = "NO" ]]; then for (( nn=1; nn <= ntiles; nn++ )); do - cp "${COMIN_ATMOS_ANALYSIS}/sfci00${FHR}.tile${nn}.nc" \ + cp "${COMIN_ATMOS_ANALYSIS}/sfci0${FHR}.tile${nn}.nc" \ "${DATA}/soil_xainc.00${nn}" done fi diff --git a/ush/forecast_det.sh b/ush/forecast_det.sh index 9854494859..dcbb24ce2e 100755 --- a/ush/forecast_det.sh +++ b/ush/forecast_det.sh @@ -26,6 +26,8 @@ UFS_det(){ IAU_OFFSET=0 model_start_date_current_cycle=${current_cycle} + DO_LAND_IAU="NO" #TODO: TZG + # It is still possible that a restart is available from a previous forecast attempt # So we have to continue checking for restarts fi diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7b9bd0ee48..6b0c4afe61 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -2,6 +2,9 @@ # Disable variable not used warnings # shellcheck disable=SC2034 + +ntiles=${ntiles:-6} + FV3_postdet() { echo "SUB ${FUNCNAME[0]}: Entering for RUN = ${RUN}" @@ -135,7 +138,12 @@ FV3_postdet() { IAU_DELTHRS=0 IAU_INC_FILES="''" fi - + if [[ ${DO_LAND_IAU} = "YES" ]]; then + DO_LAND_IAU="NO" + LAND_IAU_FHRS=-1 + LAND_IAU_DELTHRS=0 + LAND_IAU_INC_FILES="''" + fi #-------------------------------------------------------------------------- else # "${RERUN}" == "NO" @@ -199,6 +207,24 @@ EOF fi done + # TZG: SFC increments + # sfc_inc in FV3 grid, all timesteps in one file per tile (Land IAU or not) + if [[ ${DO_LAND_IAU} = "YES" ]]; then + local TN sfc_increment_file + for TN in $(seq 1 $ntiles); do + sfc_increment_file="${COMIN_ATMOS_ANALYSIS}/sfc_inc.tile${TN}.nc" + if [[ ! -f ${sfc_increment_file} ]]; then + echo "ERROR: DO_LAND_IAU=${DO_LAND_IAU}, but missing increment file ${sfc_increment_file}" + echo "Abort!" + exit 1 + fi + ${NCP} "${sfc_increment_file}" "${DATA}/INPUT/sfc_inc.tile${TN}.nc" + done + LAND_IAU_INC_FILES=${LAND_IAU_INC_FILES:-'sfc_inc'} + + #else #TODO: check to make sure correct sfc inc for non-IAU + fi + fi # if [[ "${RERUN}" == "YES" ]]; then #-------------------------------------------------------------------------- diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 2628b03594..da61d422d1 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -649,6 +649,30 @@ cat >> input.nml <> input.nml +cat >> input.nml << EOF +&land_iau_nml + do_land_iau=${LOG_DO_LAND_IAU} + land_iau_fhrs=${LAND_IAU_FHRS} + land_iau_delthrs=${LAND_IAU_DELTHRS} + land_iau_inc_files=${LAND_IAU_INC_FILES} + lsoil_incr=${LSOIL_INCR} + land_iau_filter_increments=${LAND_IAU_FILTER_INCREMENTS:-".false."} + land_iau_upd_stc=${LAND_IAU_UPD_STC} + land_iau_upd_slc=${LAND_IAU_UPD_SLC} + land_iau_do_stcsmc_adjustment=${LAND_IAU_DO_STCSMC_ADJUSTMENT} + land_iau_min_T_increment=${LAND_IAU_MIN_T_INCREMENT} + +/ +EOF +echo "" >> input.nml + # Add namelist for stochastic physics options echo "" >> input.nml #if [ $MEMBER -gt 0 ]; then diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh index 7014007269..5f18782ec6 100755 --- a/ush/regrid_gsiSfcIncr_to_tile.sh +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -13,7 +13,8 @@ export PGMERR=${PGMERR:-${pgmerr:-'&2'}} export REDOUT=${REDOUT:-'1>'} export REDERR=${REDERR:-'2>'} -REGRID_EXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x +REGRID_EXEC=${HOMEgfs}/exec/regridStates.x +#sorc/gdas.cd/build/bin/regridStates.x export PGM=$REGRID_EXEC export pgm=$PGM @@ -21,7 +22,6 @@ export pgm=$PGM NMEM_REGRID=${NMEM_REGRID:-1} CASE_IN=${CASE_IN:-$CASE_ENS} - # get resolutions CRES_IN=$(echo $CASE_IN | cut -c2-) LONB_CASE_IN=$((4*CRES_IN)) @@ -33,26 +33,60 @@ ntiles=6 APREFIX="${RUN/enkf}.t${cyc}z." APREFIX_ENS="enkfgdas.t${cyc}z." -cat << EOF > regrid.nml - &config - n_vars=4, - variable_list(1)="soilt1_inc ", - variable_list(2)="soilt2_inc ", - variable_list(3)="slc1_inc ", - variable_list(4)="slc2_inc ", - missing_value=0., - / - &input +n_vars=$((LSOIL_INCR*2)) + +# TODO: input this through config? +var_list_in="" +#var_list_in="soilt1_inc","slc1_inc","","","","","","","","", +for vi in $( seq 1 ${LSOIL_INCR} ); do + var_list_in=${var_list_in}'"soilt'${vi}'_inc"', +done +for vi in $( seq 1 ${LSOIL_INCR} ); do + var_list_in=${var_list_in}'"slc'${vi}'_inc"', +done + +# tempory fix till reg ouputs time dim +sfiprfx="sfci00[" #"sfci00[369]" ##only for single digit hours +landinc_fhr_f="" +dlmtr="" +IFS=',' read -ra landifhrs <<< "${LAND_IAU_FHRS}" +for ihr in "${landifhrs[@]}"; do + hrsf=$(printf "%.1f" "$ihr") + landinc_fhr_f="${landinc_fhr_f}${dlmtr}${hrsf}" + dlmtr="," + sfiprfx="${sfiprfx}${ihr}" +done +sfiprfx="${sfiprfx}]" + +# fixed input files +# TODO: copy this to fix dir for all res? +ln -sf /scratch2/BMC/gsienkf/Clara.Draper/regridding/inputs/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc gaussian_scrip.nc + +# fixed output files +for n in $(seq 1 $ntiles); do + ln -sf ${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc vegetation_type.tile${n}.nc +done + +rm -f regrid.nml + +cat > regrid.nml << EOF +&config + n_vars=${n_vars}, + variable_list=${var_list_in} + missing_value=0., +/ + +&input gridtype="gau_inc", ires=${LONB_CASE_IN}, jres=${LATB_CASE_IN}, - fname="enkfgdas.sfci.nc", + fname=enkfgdas.sfci.nc, dir="./", fname_coord="gaussian_scrip.nc", dir_coord="./" / - &output +&output gridtype="fv3_rst", ires=${CRES_OUT}, jres=${CRES_OUT}, @@ -61,18 +95,9 @@ cat << EOF > regrid.nml fname_mask="vegetation_type" dir_mask="./" dir_coord="$FIXorog", - / +/ EOF -# fixed input files -ln -sf /scratch2/BMC/gsienkf/Clara.Draper/regridding/inputs/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc gaussian_scrip.nc - -# fixed output files -for n in $(seq 1 $ntiles); do - ln -sf ${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc vegetation_type.tile${n}.nc -done - -#nfhrs=$(echo $IAUFHRS_ENKF | sed 's/,/ /g') for imem in $(seq 1 $NMEM_REGRID); do if [[ $NMEM_REGRID > 1 ]]; then cmem=$(printf %03i $imem) @@ -84,15 +109,28 @@ for imem in $(seq 1 $NMEM_REGRID); do COM_SOIL_ANALYSIS_MEM=$COM_ATMOS_ANALYSIS_MEM fi - for FHR in $soilinc_fhrs; do - ln -fs ${COM_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci0${FHR}.nc \ + for FHR in ${landinc_reghrs}; do + ln -fs ${COM_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci0${FHR}.nc \ ${DATA}/enkfgdas.sfci.nc - $APRUN_REGR $REGRID_EXEC $REDOUT$PGMOUT $REDERR$PGMERR + ${APRUN_REGR} $REGRID_EXEC $REDOUT$PGMOUT $REDERR$PGMERR - for n in $(seq 1 $ntiles); do - mv ${DATA}/sfci.tile${n}.nc ${COM_ATMOS_ANALYSIS_MEM}/sfci0${FHR}.tile${n}.nc - done + for n in $(seq 1 $ntiles); do + mv sfci.tile${n}.nc sfci0${FHR}.tile${n}.nc + done + done + #TODO: TZG temp fix till reg ouputs time dim + for n in $(seq 1 $ntiles); do + ncecat -u Time ${sfiprfx}.tile${n}.nc sfc_inc.tile${n}.nc #sfci00[369] + ncap2 -A -s @all="{${landinc_fhr_f[@]}}" sfc_inc.tile${n}.nc sfc_inc.tile${n}.nc + ncap2 -O -s'Time[Time]=@all' sfc_inc.tile${n}.nc sfc_inc.tile${n}.nc + + for FHR in ${landinc_reghrs}; do + yes |cp -u ${DATA}/sfci0${FHR}.tile${n}.nc ${COM_ATMOS_ANALYSIS_MEM}/sfci0${FHR}.tile${n}.nc + rm -f ${DATA}/sfci0${FHR}.tile${n}.nc + done + yes |cp -u ${DATA}/sfc_inc.tile${n}.nc ${COM_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc + rm -f ${DATA}/sfc_inc.tile${n}.nc done done