Merge remote-tracking branch 'emc/develop' into feature/gwdev_issue_2329

ci/Jenkinsfile

@@ -119,7 +119,7 @@ pipeline {
                     axis {
                         name 'Case'
                         // TODO add dynamic list of cases from env vars (needs addtional plugins)
-                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
+                        values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA'
                     }
                 }
                 stages {
@@ -155,12 +155,18 @@ pipeline {
                                         sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
                                         ws(HOME) {
                                             if (fileExists('RUNTESTS/error.logs')) {
+                                                def error_logs = sh(script: "cat RUNTESTS/error.logs", returnStdout: true).trim()
+                                                try {
+                                                    pullRequest.comment("Experiment ${Case} failed on ${Machine}\n\nError logs:\n\n${error_logs}")
+                                                } catch (Exception error) {
+                                                    echo "Failed to comment on PR: ${error.getMessage()}"
+                                                }
                                                 def fileContent = readFile 'RUNTESTS/error.logs'
                                                 def lines = fileContent.readLines()
                                                 for (line in lines) {
                                                     echo "archiving: ${line}"
                                                     archiveArtifacts artifacts: "${line}", fingerprint: true
-                                                }   
+                                                }
                                             }
                                         }
                                         error("Failed to run experiments ${Case} on ${Machine}")

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -19,5 +19,4 @@ arguments:
 
 skip_ci_on_hosts:
   - orion
-  - hera
   - hercules

ci/cases/pr/C96_atmsnowDA.yaml → ci/cases/pr/C96_atmaerosnowDA.yaml

@@ -4,7 +4,7 @@ experiment:
 
 arguments:
   pslot: {{ 'pslot' | getenv }}
-  app: ATM
+  app: ATMA
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
@@ -14,7 +14,7 @@ arguments:
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
   - orion

docs/source/init.rst

@@ -246,22 +246,23 @@ Automated Generation
 Cycled mode
 -----------
 
-Not yet supported. See :ref:`Manual Generation<manual-generation>` section below for how to create your ICs yourself (outside of workflow).
+Not yet supported.
 
 .. _forecastonly-coupled:
 
 ---------------------
 Forecast-only coupled
 ---------------------
-Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter ([email protected]).
+Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Sample copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.stage_ic``.
+Note however, that due to the rapid changes in the model configuration, some staged initial conditions may not work.
 
 .. _forecastonly-atmonly:
 
 -----------------------------
 Forecast-only mode (atm-only)
 -----------------------------
 
-The table below lists the needed initial condition files from past GFS versions to be used by the UFS_UTILS gdas_init utility. The utility will pull these files for you. See the next section (Manual Generation) for how to run the UFS_UTILS gdas_init utility and create initial conditions for your experiment.
+The table below lists for reference the needed initial condition files from past GFS versions to be used by the UFS_UTILS gdas_init utility. The utility will pull these files for you. See the next section (Manual Generation) for how to run the UFS_UTILS gdas_init utility and create initial conditions for your experiment.
 
 Note for table: yyyy=year; mm=month; dd=day; hh=cycle
 
@@ -284,11 +285,11 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh
 +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
 | v15 ops        |   gfs.t. ``hh`` z.atmanl.nemsio | gpfs_dell1_nco_ops_com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar | gfs. ``yyyymmdd`` /``hh``      |
 |                |                                 |                                                                             |                                |
-| pre-2020022600 |   gfs.t. ``hh`` z.sfcanl.nemsio |                                                                             |                                | 
+| pre-2020022600 |   gfs.t. ``hh`` z.sfcanl.nemsio |                                                                             |                                |
 +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
 | v15 ops        |   gfs.t. ``hh`` z.atmanl.nemsio | com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar                    | gfs. ``yyyymmdd`` /``hh``      |
 |                |                                 |                                                                             |                                |
-|                |   gfs.t. ``hh`` z.sfcanl.nemsio |                                                                             |                                |  
+|                |   gfs.t. ``hh`` z.sfcanl.nemsio |                                                                             |                                |
 +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+
 | v16 retro      |   gfs.t. ``hh`` z.atmanl.nc     | gfs_netcdfa.tar*                                                            | gfs. ``yyyymmdd`` /``hh``/atmos|
 |                |                                 |                                                                             |                                |
@@ -318,82 +319,14 @@ Manual Generation
 
 The following information is for users needing to generate cold-start initial conditions for a cycled experiment that will run at a different resolution or layer amount than the operational GFS (C768C384L127).
 
-The ``chgres_cube`` code is available from the `UFS_UTILS repository <https://github.com/ufs-community/UFS_UTILS>`_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users may clone the develop/HEAD branch or the same version used by global-workflow develop. The ``chgres_cube`` code/scripts currently support the following GFS inputs:
+The ``chgres_cube`` code is available from the `UFS_UTILS repository <https://github.com/ufs-community/UFS_UTILS>`_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users should see the documentation to generation initial conditions in the UFS_UTILS repository. The ``chgres_cube`` code/scripts currently support the following GFS inputs:
 
 * pre-GFSv14
 * GFSv14
 * GFSv15
 * GFSv16
 
-Users can use the copy of UFS_UTILS that is already cloned and built within their global-workflow clone or clone/build it separately:
-
-Within a built/linked global-workflow clone:
-
-::
-
-   cd sorc/ufs_utils.fd/util/gdas_init
-
-Clone and build separately:
-
-1. Clone UFS_UTILS:
-
-::
-
-   git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git
-
-Then switch to a different tag or use the default branch (develop).
-
-2. Build UFS_UTILS:
-
-::
-
-   sh build_all.sh
-   cd fix
-   sh link_fixdirs.sh emc $MACHINE
-
-where ``$MACHINE`` is ``wcoss2``, ``hera``, or ``jet``.
-
-.. note::
-   UFS-UTILS builds on Orion/Hercules but due to the lack of HPSS access on Orion/Hercules the ``gdas_init`` utility is not supported there.
-
-3. Configure your conversion:
-
-::
-
-   cd util/gdas_init
-   vi config
-
-Read the doc block at the top of the config and adjust the variables to meet you needs (e.g. ``yy, mm, dd, hh`` for ``SDATE``).
-
-Most users will want to adjust the following ``config`` settings for the current system design:
-
-#. EXTRACT_DATA=YES (to pull original ICs to convert off HPSS)
-#. RUN_CHGRES=YES (to run chgres_cube on the original ICs pulled off HPSS)
-#. LEVS=128 (for the L127 GFS)
-
-4. Submit conversion script:
-
-::
-
-   ./driver.$MACHINE.sh
-
-where ``$MACHINE`` is currently ``wcoss2``,  ``hera`` or ``jet``. Additional options will be available as support for other machines expands.
-
-.. note::
-   UFS-UTILS builds on Orion/Hercules but due to lack of HPSS access there is no ``gdas_init`` driver for Orion/Hercules nor support to pull initial conditions from HPSS for the ``gdas_init`` utility.
-
-Several small jobs will be submitted:
-
-  - 1 jobs to pull inputs off HPSS
-  - 1 or 2 jobs to run ``chgres_cube`` (1 for deterministic/hires and 1 for each EnKF ensemble member)
-
-The chgres jobs will have a dependency on the data-pull jobs and will wait to run until all data-pull jobs have completed.
-
-5. Check output:
-
-In the config you will have defined an output folder called ``$OUTDIR``. The converted output will be found there, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their ``$OUTDIR`` directly into their ``$ROTDIR``.
-
-Please report bugs to George Gayno ([email protected]) and Kate Friedman ([email protected]).
+See instructions in UFS_UTILS to clone, build and generate initial conditions.
 
 .. _warmstarts-prod:
 
@@ -489,7 +422,7 @@ Tarballs per cycle:
    com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp7.tar
    com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp8.tar
 
-Go to the top of your ``ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure.
+Go to the top of your ``ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure.  Note that the directory structure has changed, so this may not be correct.
 
 .. _warmstarts-preprod-parallels:
 
@@ -517,6 +450,7 @@ Recent pre-implementation parallel series was for GFS v16 (implemented March 202
 * **Where do I put the warm-start initial conditions?** Extraction should occur right inside your ROTDIR. You may need to rename the enkf folder (enkf.gdas.$PDY -> enkfgdas.$PDY).
 
 Due to a recent change in the dycore, you may also need an additional offline step to fix the checksum of the NetCDF files for warm start. See the :ref:`Fix netcdf checksum section <gfsv17-checksum>`.
+The current model has undergone several updates and the files generated may not be completely usable by the model.
 
 .. _retrospective:
 

parm/config/gefs/config.base

@@ -74,7 +74,7 @@ export NCP="/bin/cp -p"
 export NMV="/bin/mv"
 export NLN="/bin/ln -sf"
 export VERBOSE="YES"
-export KEEPDATA="NO"
+export KEEPDATA="@KEEPDATA@"
 export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true
 export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
 export CHGRP_CMD="@CHGRP_CMD@"
@@ -262,7 +262,7 @@ export MEMDIR="mem${ENSMEM}"
 
 # initialize ocean ensemble members with perturbations
 #   if true, only occurs for members greater than zero
-export OCN_ENS_PERTURB_FILES=false
+export USE_OCN_PERTURB_FILES=@STAGE_OCN_PERTURB_FILES@
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting

parm/config/gefs/config.efcs

@@ -60,7 +60,27 @@ export SPPT_TAU=21600.
 export SPPT_LSCALE=500000.
 export SPPT_LOGIT=".true."
 export SPPT_SFCLIMIT=".true."
+# OCN options
+export DO_OCN_SPPT="YES"
+export OCNSPPT="0.8,0.4,0.2,0.08,0.04"
+export OCNSPPT_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
+export OCNSPPT_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
+export DO_OCN_PERT_EPBL="YES"
+export EPBL="0.8,0.4,0.2,0.08,0.04"
+export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
+export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 
+if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then
+    export ODA_INCUPD="True"
+    export ODA_TEMPINC_VAR='t_pert'
+    export ODA_SALTINC_VAR='s_pert'
+    export ODA_THK_VAR='h_anl'
+    export ODA_UINC_VAR='u_pert'
+    export ODA_VINC_VAR='v_pert'
+    export ODA_INCUPD_NHOURS=0.0
+else
+    export ODA_INCUPD="False"
+fi
 export restart_interval="${restart_interval_gfs}"
 
 echo "END: config.efcs"

parm/config/gefs/yaml/defaults.yaml

@@ -4,4 +4,9 @@ base:
   DO_JEDIOCNVAR: "NO"
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
+  KEEPDATA: "NO" 
   FHMAX_GFS: 120
+
+stage_ic:
+  USE_OCN_PERTURB_FILES: "false"
+

parm/config/gfs/config.aeroanl

@@ -6,24 +6,26 @@
 echo "BEGIN: config.aeroanl"
 
 export CASE_ANL=${CASE}
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/config/
-export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/lists/gdas_aero_prototype.yaml
+export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2"
 export STATICB_TYPE='identity'
-export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/berror/staticb_${STATICB_TYPE}.yaml
-export BERROR_DATA_DIR=${FIXgfs}/gdas/bump/aero/${CASE_ANL}/
+export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2"
+export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/"
 export BERROR_DATE="20160630.000000"
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_var.x
+export JEDIEXE="${EXECgfs}/fv3jedi_var.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_fgat_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2"
 else
   export aero_bkg_times="6"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2"
 fi
 
 echo "END: config.aeroanl"

parm/config/gfs/config.aeroanlfinal

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlfinal"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlfinal
+source "${EXPDIR}/config.resources" aeroanlfinal
 echo "END: config.aeroanlfinal"

parm/config/gfs/config.aeroanlinit

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlinit"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlinit
+source "${EXPDIR}/config.resources" aeroanlinit
 echo "END: config.aeroanlinit"

parm/config/gfs/config.aeroanlrun

@@ -6,6 +6,6 @@
 echo "BEGIN: config.aeroanlrun"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlrun
+source "${EXPDIR}/config.resources" aeroanlrun
 
 echo "END: config.aeroanlrun"

parm/config/gfs/config.anal

@@ -50,6 +50,13 @@ export OZINFO=${FIXgfs}/gsi/global_ozinfo.txt
 export SATINFO=${FIXgfs}/gsi/global_satinfo.txt
 export OBERROR=${FIXgfs}/gsi/prepobs_errtable.global
 
+if [[ ${GSI_SOILANAL} = "YES" ]]; then
+    export hofx_2m_sfcfile=".true."
+    export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway
+    export paranc=".false." # temporary until sfc io coded for parance (PR being prepared by T. Gichamo)
+    export CONVINFO=${FIXgfs}/gsi/global_convinfo_2mObs.txt
+    export ANAVINFO=${FIXgfs}/gsi/global_anavinfo_soilanal.l127.txt
+fi
 
 # Use experimental dumps in EMC GFS v16 parallels
 if [[ ${RUN_ENVIR} == "emc" ]]; then

parm/config/gfs/config.atmanl

@@ -19,6 +19,9 @@ else
     export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2"
 fi
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@
 

parm/config/gfs/config.atmensanl

@@ -9,6 +9,9 @@ export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2"
 export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2"
 export INTERP_METHOD='barycentric'
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
 

parm/config/gfs/config.base

@@ -331,6 +331,8 @@ fi
 
 if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6"; fi
 
+export GSI_SOILANAL=@GSI_SOILANAL@
+
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst
 export DONST="YES"
 if [[ ${DONST} = "YES" ]]; then export FNTSFA="        "; fi

parm/config/gfs/config.esfc

@@ -12,7 +12,7 @@ echo "BEGIN: config.esfc"
 # Set DOSFCANL_ENKF=NO to prevent creation of sfcanl at 
 # center of analysis window.  
 
-if [ $DOIAU_ENKF = "YES" ]; then
+if [[ ${DOIAU_ENKF} = "YES" ]]; then
    export DOSFCANL_ENKF="NO"
 fi
 
@@ -21,4 +21,10 @@ if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+# set up soil analysis
+if [[ ${GSI_SOILANAL} = "YES" ]]; then
+    export DO_LNDINC=".true."
+    export LND_SOI_FILE="lnd_incr"
+fi
+
 echo "END: config.esfc"

parm/config/gfs/config.snowanl

@@ -22,6 +22,8 @@ export BESTDDEV="30."  # Background Error Std. Dev. for LETKFOI
 export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe"
 export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/apply_incr_nml.j2"
 
+export JEDI_FIX_YAML="${PARMgfs}/gdas/snow_jedi_fix.yaml.j2"
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@