diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 34535ed608..6f27804a3c 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -119,7 +119,7 @@ pipeline { axis { name 'Case' // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' + values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA' } } stages { @@ -155,12 +155,18 @@ pipeline { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") ws(HOME) { if (fileExists('RUNTESTS/error.logs')) { + def error_logs = sh(script: "cat RUNTESTS/error.logs", returnStdout: true).trim() + try { + pullRequest.comment("Experiment ${Case} failed on ${Machine}\n\nError logs:\n\n${error_logs}") + } catch (Exception error) { + echo "Failed to comment on PR: ${error.getMessage()}" + } def fileContent = readFile 'RUNTESTS/error.logs' def lines = fileContent.readLines() for (line in lines) { echo "archiving: ${line}" archiveArtifacts artifacts: "${line}", fingerprint: true - } + } } } error("Failed to run experiments ${Case} on ${Machine}") diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index d9156e38f3..b972d3a445 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -19,5 +19,4 @@ arguments: skip_ci_on_hosts: - orion - - hera - hercules diff --git a/ci/cases/pr/C96_atmsnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml similarity index 81% rename from ci/cases/pr/C96_atmsnowDA.yaml rename to ci/cases/pr/C96_atmaerosnowDA.yaml index 35fcc10fb2..7e22955a37 100644 --- a/ci/cases/pr/C96_atmsnowDA.yaml +++ b/ci/cases/pr/C96_atmaerosnowDA.yaml @@ -4,7 +4,7 @@ experiment: arguments: pslot: {{ 'pslot' | getenv }} - app: ATM + app: ATMA resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR @@ -14,7 +14,7 @@ arguments: nens: 0 gfs_cyc: 1 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: - orion diff --git a/ci/cases/yamls/atmsnowDA_defaults_ci.yaml b/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/atmsnowDA_defaults_ci.yaml rename to ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/docs/source/init.rst b/docs/source/init.rst index 14a0ea0d56..1b28d75584 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -246,14 +246,15 @@ Automated Generation Cycled mode ----------- -Not yet supported. See :ref:`Manual Generation` section below for how to create your ICs yourself (outside of workflow). +Not yet supported. .. _forecastonly-coupled: --------------------- Forecast-only coupled --------------------- -Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Sample copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.stage_ic``. +Note however, that due to the rapid changes in the model configuration, some staged initial conditions may not work. .. _forecastonly-atmonly: @@ -261,7 +262,7 @@ Coupled initial conditions are currently only generated offline and copied prior Forecast-only mode (atm-only) ----------------------------- -The table below lists the needed initial condition files from past GFS versions to be used by the UFS_UTILS gdas_init utility. The utility will pull these files for you. See the next section (Manual Generation) for how to run the UFS_UTILS gdas_init utility and create initial conditions for your experiment. +The table below lists for reference the needed initial condition files from past GFS versions to be used by the UFS_UTILS gdas_init utility. The utility will pull these files for you. See the next section (Manual Generation) for how to run the UFS_UTILS gdas_init utility and create initial conditions for your experiment. Note for table: yyyy=year; mm=month; dd=day; hh=cycle @@ -284,11 +285,11 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ | v15 ops | gfs.t. ``hh`` z.atmanl.nemsio | gpfs_dell1_nco_ops_com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar | gfs. ``yyyymmdd`` /``hh`` | | | | | | -| pre-2020022600 | gfs.t. ``hh`` z.sfcanl.nemsio | | | +| pre-2020022600 | gfs.t. ``hh`` z.sfcanl.nemsio | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ | v15 ops | gfs.t. ``hh`` z.atmanl.nemsio | com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar | gfs. ``yyyymmdd`` /``hh`` | | | | | | -| | gfs.t. ``hh`` z.sfcanl.nemsio | | | +| | gfs.t. ``hh`` z.sfcanl.nemsio | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ | v16 retro | gfs.t. ``hh`` z.atmanl.nc | gfs_netcdfa.tar* | gfs. ``yyyymmdd`` /``hh``/atmos| | | | | | @@ -318,82 +319,14 @@ Manual Generation The following information is for users needing to generate cold-start initial conditions for a cycled experiment that will run at a different resolution or layer amount than the operational GFS (C768C384L127). -The ``chgres_cube`` code is available from the `UFS_UTILS repository `_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users may clone the develop/HEAD branch or the same version used by global-workflow develop. The ``chgres_cube`` code/scripts currently support the following GFS inputs: +The ``chgres_cube`` code is available from the `UFS_UTILS repository `_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users should see the documentation to generation initial conditions in the UFS_UTILS repository. The ``chgres_cube`` code/scripts currently support the following GFS inputs: * pre-GFSv14 * GFSv14 * GFSv15 * GFSv16 -Users can use the copy of UFS_UTILS that is already cloned and built within their global-workflow clone or clone/build it separately: - -Within a built/linked global-workflow clone: - -:: - - cd sorc/ufs_utils.fd/util/gdas_init - -Clone and build separately: - -1. Clone UFS_UTILS: - -:: - - git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git - -Then switch to a different tag or use the default branch (develop). - -2. Build UFS_UTILS: - -:: - - sh build_all.sh - cd fix - sh link_fixdirs.sh emc $MACHINE - -where ``$MACHINE`` is ``wcoss2``, ``hera``, or ``jet``. - -.. note:: - UFS-UTILS builds on Orion/Hercules but due to the lack of HPSS access on Orion/Hercules the ``gdas_init`` utility is not supported there. - -3. Configure your conversion: - -:: - - cd util/gdas_init - vi config - -Read the doc block at the top of the config and adjust the variables to meet you needs (e.g. ``yy, mm, dd, hh`` for ``SDATE``). - -Most users will want to adjust the following ``config`` settings for the current system design: - -#. EXTRACT_DATA=YES (to pull original ICs to convert off HPSS) -#. RUN_CHGRES=YES (to run chgres_cube on the original ICs pulled off HPSS) -#. LEVS=128 (for the L127 GFS) - -4. Submit conversion script: - -:: - - ./driver.$MACHINE.sh - -where ``$MACHINE`` is currently ``wcoss2``, ``hera`` or ``jet``. Additional options will be available as support for other machines expands. - -.. note:: - UFS-UTILS builds on Orion/Hercules but due to lack of HPSS access there is no ``gdas_init`` driver for Orion/Hercules nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. - -Several small jobs will be submitted: - - - 1 jobs to pull inputs off HPSS - - 1 or 2 jobs to run ``chgres_cube`` (1 for deterministic/hires and 1 for each EnKF ensemble member) - -The chgres jobs will have a dependency on the data-pull jobs and will wait to run until all data-pull jobs have completed. - -5. Check output: - -In the config you will have defined an output folder called ``$OUTDIR``. The converted output will be found there, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their ``$OUTDIR`` directly into their ``$ROTDIR``. - -Please report bugs to George Gayno (george.gayno@noaa.gov) and Kate Friedman (kate.friedman@noaa.gov). +See instructions in UFS_UTILS to clone, build and generate initial conditions. .. _warmstarts-prod: @@ -489,7 +422,7 @@ Tarballs per cycle: com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp7.tar com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp8.tar -Go to the top of your ``ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure. +Go to the top of your ``ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure. Note that the directory structure has changed, so this may not be correct. .. _warmstarts-preprod-parallels: @@ -517,6 +450,7 @@ Recent pre-implementation parallel series was for GFS v16 (implemented March 202 * **Where do I put the warm-start initial conditions?** Extraction should occur right inside your ROTDIR. You may need to rename the enkf folder (enkf.gdas.$PDY -> enkfgdas.$PDY). Due to a recent change in the dycore, you may also need an additional offline step to fix the checksum of the NetCDF files for warm start. See the :ref:`Fix netcdf checksum section `. +The current model has undergone several updates and the files generated may not be completely usable by the model. .. _retrospective: diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index da315a73e7..7ec3993f5f 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -74,7 +74,7 @@ export NCP="/bin/cp -p" export NMV="/bin/mv" export NLN="/bin/ln -sf" export VERBOSE="YES" -export KEEPDATA="NO" +export KEEPDATA="@KEEPDATA@" export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true export CHGRP_RSTPROD="@CHGRP_RSTPROD@" export CHGRP_CMD="@CHGRP_CMD@" @@ -262,7 +262,7 @@ export MEMDIR="mem${ENSMEM}" # initialize ocean ensemble members with perturbations # if true, only occurs for members greater than zero -export OCN_ENS_PERTURB_FILES=false +export USE_OCN_PERTURB_FILES=@STAGE_OCN_PERTURB_FILES@ export DOIAU="NO" # While we are not doing IAU, we may want to warm start w/ IAU in the future # Check if cycle is cold starting diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index a6f34818d7..e57584be17 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -60,7 +60,27 @@ export SPPT_TAU=21600. export SPPT_LSCALE=500000. export SPPT_LOGIT=".true." export SPPT_SFCLIMIT=".true." +# OCN options +export DO_OCN_SPPT="YES" +export OCNSPPT="0.8,0.4,0.2,0.08,0.04" +export OCNSPPT_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7" +export OCNSPPT_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3" +export DO_OCN_PERT_EPBL="YES" +export EPBL="0.8,0.4,0.2,0.08,0.04" +export EPBL_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7" +export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3" +if [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then + export ODA_INCUPD="True" + export ODA_TEMPINC_VAR='t_pert' + export ODA_SALTINC_VAR='s_pert' + export ODA_THK_VAR='h_anl' + export ODA_UINC_VAR='u_pert' + export ODA_VINC_VAR='v_pert' + export ODA_INCUPD_NHOURS=0.0 +else + export ODA_INCUPD="False" +fi export restart_interval="${restart_interval_gfs}" echo "END: config.efcs" diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index b19eb57e55..5d5fe5963d 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -4,4 +4,9 @@ base: DO_JEDIOCNVAR: "NO" DO_JEDISNOWDA: "NO" DO_MERGENSST: "NO" + KEEPDATA: "NO" FHMAX_GFS: 120 + +stage_ic: + USE_OCN_PERTURB_FILES: "false" + diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index cf7981f807..972f393feb 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -6,24 +6,26 @@ echo "BEGIN: config.aeroanl" export CASE_ANL=${CASE} -export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/config/ -export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/lists/gdas_aero_prototype.yaml +export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" export STATICB_TYPE='identity' -export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/berror/staticb_${STATICB_TYPE}.yaml -export BERROR_DATA_DIR=${FIXgfs}/gdas/bump/aero/${CASE_ANL}/ +export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" +export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" export BERROR_DATE="20160630.000000" +export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" +export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/fv3jedi_var.x +export JEDIEXE="${EXECgfs}/fv3jedi_var.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" - export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_fgat_gfs_aero.yaml + export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2" else export aero_bkg_times="6" - export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_gfs_aero.yaml + export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2" fi echo "END: config.aeroanl" diff --git a/parm/config/gfs/config.aeroanlfinal b/parm/config/gfs/config.aeroanlfinal index 230ec5205a..34e5d8f116 100644 --- a/parm/config/gfs/config.aeroanlfinal +++ b/parm/config/gfs/config.aeroanlfinal @@ -6,5 +6,5 @@ echo "BEGIN: config.aeroanlfinal" # Get task specific resources -. $EXPDIR/config.resources aeroanlfinal +source "${EXPDIR}/config.resources" aeroanlfinal echo "END: config.aeroanlfinal" diff --git a/parm/config/gfs/config.aeroanlinit b/parm/config/gfs/config.aeroanlinit index 72175b8d0c..7036d3d27b 100644 --- a/parm/config/gfs/config.aeroanlinit +++ b/parm/config/gfs/config.aeroanlinit @@ -6,5 +6,5 @@ echo "BEGIN: config.aeroanlinit" # Get task specific resources -. $EXPDIR/config.resources aeroanlinit +source "${EXPDIR}/config.resources" aeroanlinit echo "END: config.aeroanlinit" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun index da13df2831..012e5b79f3 100644 --- a/parm/config/gfs/config.aeroanlrun +++ b/parm/config/gfs/config.aeroanlrun @@ -6,6 +6,6 @@ echo "BEGIN: config.aeroanlrun" # Get task specific resources -. $EXPDIR/config.resources aeroanlrun +source "${EXPDIR}/config.resources" aeroanlrun echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 98d0e88cc2..09aaa15a98 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -50,6 +50,13 @@ export OZINFO=${FIXgfs}/gsi/global_ozinfo.txt export SATINFO=${FIXgfs}/gsi/global_satinfo.txt export OBERROR=${FIXgfs}/gsi/prepobs_errtable.global +if [[ ${GSI_SOILANAL} = "YES" ]]; then + export hofx_2m_sfcfile=".true." + export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway + export paranc=".false." # temporary until sfc io coded for parance (PR being prepared by T. Gichamo) + export CONVINFO=${FIXgfs}/gsi/global_convinfo_2mObs.txt + export ANAVINFO=${FIXgfs}/gsi/global_anavinfo_soilanal.l127.txt +fi # Use experimental dumps in EMC GFS v16 parallels if [[ ${RUN_ENVIR} == "emc" ]]; then diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 11358de8a8..7cfd0cb47f 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -19,6 +19,9 @@ else export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2" fi +export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" +export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" + export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 49b903e4c0..8e824b22f6 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -9,6 +9,9 @@ export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2" export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2" export INTERP_METHOD='barycentric' +export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" +export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" + export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 3247c13fd8..4a026389e6 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -331,6 +331,8 @@ fi if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6"; fi +export GSI_SOILANAL=@GSI_SOILANAL@ + # turned on nsst in anal and/or fcst steps, and turn off rtgsst export DONST="YES" if [[ ${DONST} = "YES" ]]; then export FNTSFA=" "; fi diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc index 7c32313758..684dea4ee3 100644 --- a/parm/config/gfs/config.esfc +++ b/parm/config/gfs/config.esfc @@ -12,7 +12,7 @@ echo "BEGIN: config.esfc" # Set DOSFCANL_ENKF=NO to prevent creation of sfcanl at # center of analysis window. -if [ $DOIAU_ENKF = "YES" ]; then +if [[ ${DOIAU_ENKF} = "YES" ]]; then export DOSFCANL_ENKF="NO" fi @@ -21,4 +21,10 @@ if [[ "${DO_JEDIATMENS}" == "YES" ]]; then export DONST="NO" fi +# set up soil analysis +if [[ ${GSI_SOILANAL} = "YES" ]]; then + export DO_LNDINC=".true." + export LND_SOI_FILE="lnd_incr" +fi + echo "END: config.esfc" diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index 30e6d9c07b..7b3ffa47f3 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -22,6 +22,8 @@ export BESTDDEV="30." # Background Error Std. Dev. for LETKFOI export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe" export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/apply_incr_nml.j2" +export JEDI_FIX_YAML="${PARMgfs}/gdas/snow_jedi_fix.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 9c90255cdd..9f744df4db 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -8,6 +8,7 @@ base: DO_GOES: "NO" FHMAX_GFS: 120 DO_VRFY_OCEANDA: "NO" + GSI_SOILANAL: "NO" atmanl: LAYOUT_X_ATMANL: 8 @@ -32,12 +33,12 @@ snowanl: ocnanal: SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. CASE_ANL: "C48" # TODO: Check in gdasapp if used anywhere for SOCA - SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs + SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 SABER_BLOCKS_YAML: "" NICAS_RESOL: 1 NICAS_GRID_SIZE: 15000 prepoceanobs: - SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal - OBSPREP_YAML: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml" + SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal + OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs" diff --git a/parm/gdas/aero_crtm_coeff.yaml b/parm/gdas/aero_crtm_coeff.yaml deleted file mode 100644 index 75b54c3741..0000000000 --- a/parm/gdas/aero_crtm_coeff.yaml +++ /dev/null @@ -1,13 +0,0 @@ -mkdir: -- {{ DATA }}/crtm/ -copy: -- [{{ CRTM_FIX }}/AerosolCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/CloudCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/v.viirs-m_npp.SpcCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/v.viirs-m_npp.TauCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/v.viirs-m_j1.SpcCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/v.viirs-m_j1.TauCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/NPOESS.VISice.EmisCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/NPOESS.VISland.EmisCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/NPOESS.VISsnow.EmisCoeff.bin, {{ DATA }}/crtm/] -- [{{ CRTM_FIX }}/NPOESS.VISwater.EmisCoeff.bin, {{ DATA }}/crtm/] diff --git a/parm/gdas/aero_crtm_coeff.yaml.j2 b/parm/gdas/aero_crtm_coeff.yaml.j2 new file mode 100644 index 0000000000..b48d8ff231 --- /dev/null +++ b/parm/gdas/aero_crtm_coeff.yaml.j2 @@ -0,0 +1,13 @@ +mkdir: +- '{{ DATA }}/crtm/' +copy: +- ['{{ CRTM_FIX }}/AerosolCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/CloudCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/v.viirs-m_npp.SpcCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/v.viirs-m_npp.TauCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/v.viirs-m_j1.SpcCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/v.viirs-m_j1.TauCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/NPOESS.VISice.EmisCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/NPOESS.VISland.EmisCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/NPOESS.VISsnow.EmisCoeff.bin', '{{ DATA }}/crtm/'] +- ['{{ CRTM_FIX }}/NPOESS.VISwater.EmisCoeff.bin', '{{ DATA }}/crtm/'] diff --git a/parm/gdas/aero_jedi_fix.yaml b/parm/gdas/aero_jedi_fix.yaml deleted file mode 100644 index 16cbeac6e7..0000000000 --- a/parm/gdas/aero_jedi_fix.yaml +++ /dev/null @@ -1,11 +0,0 @@ -mkdir: -- !ENV ${DATA}/fv3jedi -copy: -- - !ENV ${FIXgfs}/gdas/fv3jedi/fv3files/akbk$(npz).nc4 - - !ENV ${DATA}/fv3jedi/akbk.nc4 -- - !ENV ${FIXgfs}/gdas/fv3jedi/fv3files/fmsmpp.nml - - !ENV ${DATA}/fv3jedi/fmsmpp.nml -- - !ENV ${FIXgfs}/gdas/fv3jedi/fv3files/field_table_gfdl - - !ENV ${DATA}/fv3jedi/field_table -- - !ENV $(HOMEgfs)/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml - - !ENV ${DATA}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/parm/gdas/atm_jedi_fix.yaml b/parm/gdas/aero_jedi_fix.yaml.j2 similarity index 100% rename from parm/gdas/atm_jedi_fix.yaml rename to parm/gdas/aero_jedi_fix.yaml.j2 diff --git a/parm/gdas/atm_crtm_coeff.yaml b/parm/gdas/atm_crtm_coeff.yaml.j2 similarity index 100% rename from parm/gdas/atm_crtm_coeff.yaml rename to parm/gdas/atm_crtm_coeff.yaml.j2 diff --git a/parm/gdas/atm_jedi_fix.yaml.j2 b/parm/gdas/atm_jedi_fix.yaml.j2 new file mode 100644 index 0000000000..69039baddf --- /dev/null +++ b/parm/gdas/atm_jedi_fix.yaml.j2 @@ -0,0 +1,7 @@ +mkdir: +- '{{ DATA }}/fv3jedi' +copy: +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4'] +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] diff --git a/parm/gdas/snow_jedi_fix.yaml.j2 b/parm/gdas/snow_jedi_fix.yaml.j2 index 4d820a82ba..69039baddf 100644 --- a/parm/gdas/snow_jedi_fix.yaml.j2 +++ b/parm/gdas/snow_jedi_fix.yaml.j2 @@ -1,7 +1,7 @@ mkdir: - '{{ DATA }}/fv3jedi' copy: -- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4'] -- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] -- ['{{ HOMEgfs }}/fix/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] -- ['{{ HOMEgfs }}/sorc/gdas.cd/parm/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/akbk{{ npz }}.nc4', '{{ DATA }}/fv3jedi/akbk.nc4'] +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] +- ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh index 85d0b2187d..73b6d95cbc 100755 --- a/scripts/exgdas_enkf_sfc.sh +++ b/scripts/exgdas_enkf_sfc.sh @@ -24,6 +24,7 @@ pwd=$(pwd) # Base variables DONST=${DONST:-"NO"} +GSI_SOILANAL=${GSI_SOILANAL:-"NO"} DOSFCANL_ENKF=${DOSFCANL_ENKF:-"YES"} export CASE=${CASE:-384} ntiles=${ntiles:-6} @@ -61,7 +62,6 @@ export DELTSFC=${DELTSFC:-6} APRUN_ESFC=${APRUN_ESFC:-${APRUN:-""}} NTHREADS_ESFC=${NTHREADS_ESFC:-${NTHREADS:-1}} - ################################################################################ # Preprocessing mkdata=NO @@ -142,8 +142,10 @@ if [ $DOIAU = "YES" ]; then MEMDIR=${memchar} RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} generate_com \ COM_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL - [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} generate_com \ + COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" ${NCP} "${COM_ATMOS_RESTART_MEM_PREV}/${bPDY}.${bcyc}0000.sfc_data.tile${n}.nc" \ "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" ${NLN} "${COM_ATMOS_RESTART_MEM_PREV}/${bPDY}.${bcyc}0000.sfc_data.tile${n}.nc" \ @@ -153,7 +155,12 @@ if [ $DOIAU = "YES" ]; then ${NLN} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" ${NLN} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" - done + if [[ ${GSI_SOILANAL} = "YES" ]]; then + FHR=6 + ${NLN} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ + "${DATA}/lnd_incr.${cmem}" + fi + done # ensembles CDATE="${PDY}${cyc}" ${CYCLESH} export err=$?; err_chk diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index a23a892914..dc9ef529f6 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -87,6 +87,7 @@ else DO_CALC_INCREMENT=${DO_CALC_INCREMENT:-"NO"} fi INCREMENTS_TO_ZERO=${INCREMENTS_TO_ZERO:-"'NONE'"} +GSI_SOILANAL=${GSI_SOILANAL:-"NO"} ################################################################################ @@ -203,6 +204,10 @@ for imem in $(seq 1 $NMEM_ENS); do for FHR in $nfhrs; do ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}atmf00${FHR}${ENKF_SUFFIX}.nc" \ "sfg_${PDY}${cyc}_fhr0${FHR}_${memchar}" + if [ $GSI_SOILANAL = "YES" ]; then + ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}sfcf00${FHR}${ENKF_SUFFIX}.nc" \ + "bfg_${PDY}${cyc}_fhr0${FHR}_${memchar}" + fi if [ $cnvw_option = ".true." ]; then ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}sfcf00${FHR}.nc" \ "sfgsfc_${PDY}${cyc}_fhr0${FHR}_${memchar}" @@ -224,6 +229,10 @@ for imem in $(seq 1 $NMEM_ENS); do "incr_${PDY}${cyc}_fhr0${FHR}_${memchar}" fi fi + if [ $GSI_SOILANAL = "YES" ]; then + ${NLN} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX}sfci00${FHR}.nc" \ + "sfcincr_${PDY}${cyc}_fhr0${FHR}_${memchar}" + fi done done @@ -238,10 +247,10 @@ for FHR in $nfhrs; do fi done -if [ $USE_CFP = "YES" ]; then +if [[ $USE_CFP = "YES" ]]; then chmod 755 $DATA/mp_untar.sh ncmd=$(cat $DATA/mp_untar.sh | wc -l) - if [ $ncmd -gt 0 ]; then + if [[ $ncmd -gt 0 ]]; then ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) APRUNCFP=$(eval echo $APRUNCFP) $APRUNCFP $DATA/mp_untar.sh @@ -398,8 +407,8 @@ cat stdout stderr > "${COM_ATMOS_ANALYSIS_STAT}/${ENKFSTAT}" ################################################################################ # Postprocessing -cd $pwd -[[ $mkdata = "YES" ]] && rm -rf $DATA +cd "$pwd" +[[ $mkdata = "YES" ]] && rm -rf "${DATA}" -exit $err +exit ${err} diff --git a/scripts/exglobal_atmos_analysis.sh b/scripts/exglobal_atmos_analysis.sh index 6da862eb54..575c112675 100755 --- a/scripts/exglobal_atmos_analysis.sh +++ b/scripts/exglobal_atmos_analysis.sh @@ -89,6 +89,8 @@ SENDDBN=${SENDDBN:-"NO"} RUN_GETGES=${RUN_GETGES:-"NO"} GETGESSH=${GETGESSH:-"getges.sh"} export gesenvir=${gesenvir:-${envir}} + +export hofx_2m_sfcfile=${hofx_2m_sfcfile:-".false."} # Observations OPREFIX=${OPREFIX:-""} @@ -748,6 +750,7 @@ cat > gsiparm.anl << EOF / &OBS_INPUT dmesh(1)=145.0,dmesh(2)=150.0,dmesh(3)=100.0,dmesh(4)=50.0,time_window_max=3.0, + hofx_2m_sfcfile=${hofx_2m_sfcfile}, ${OBSINPUT} / OBS_INPUT:: diff --git a/scripts/exglobal_oceanice_products.py b/scripts/exglobal_oceanice_products.py index 0f8e2e0d6d..9bb2b09596 100755 --- a/scripts/exglobal_oceanice_products.py +++ b/scripts/exglobal_oceanice_products.py @@ -19,8 +19,8 @@ def main(): # Pull out all the configuration keys needed to run the rest of steps keys = ['HOMEgfs', 'DATA', 'current_cycle', 'RUN', 'NET', - 'COM_OCEAN_HISTORY', 'COM_OCEAN_GRIB', - 'COM_ICE_HISTORY', 'COM_ICE_GRIB', + f'COM_{oceanice.task_config.component.upper()}_HISTORY', + f'COM_{oceanice.task_config.component.upper()}_GRIB', 'APRUN_OCNICEPOST', 'component', 'forecast_hour', 'valid_datetime', 'avg_period', 'model_grid', 'product_grids', 'oceanice_yaml'] diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index 6a1be22e71..e7dfc3c495 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -106,7 +106,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do # Ocean Perturbation Files # Extra zero on MEMDIR ensure we have a number even if the string is empty - if (( 0${MEMDIR:3} > 0 )) && [[ "${OCN_ENS_PERTURB_FILES:-false}" == "true" ]]; then + if (( 0${MEMDIR:3} > 0 )) && [[ "${USE_OCN_PERTURB_FILES:-false}" == "true" ]]; then src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.mom6_increment.nc" tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.mom6_increment.nc" ${NCP} "${src}" "${tgt}" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 8a6f825f6d..dd350d7e4d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 8a6f825f6d988c81fad11070de92a2744d5a53cc +Subproject commit dd350d7e4daab0977407e388711807f13b204f6f diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index a5fee9a7fd..d112271d0c 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -251,8 +251,15 @@ EOF ${NLN} "${FIXgfs}/am/global_co2historicaldata_glob.txt" "${DATA}/co2historicaldata_glob.txt" ${NLN} "${FIXgfs}/am/co2monthlycyc.txt" "${DATA}/co2monthlycyc.txt" - if [[ ${ICO2} -gt 0 ]]; then - for file in $(ls "${FIXgfs}/am/fix_co2_proj/global_co2historicaldata"*) ; do + # Set historical CO2 values based on whether this is a reforecast run or not + # Ref. issue 2403 + local co2dir + co2dir="fix_co2_proj" + if [[ ${reforecast:-"NO"} == "YES" ]]; then + co2dir="co2dat_4a" + fi + if (( ICO2 > 0 )); then + for file in $(ls "${FIXgfs}/am/${co2dir}/global_co2historicaldata"*) ; do ${NLN} "${file}" "${DATA}/$(basename "${file//global_}")" done fi @@ -695,6 +702,13 @@ MOM6_postdet() { ${NLN} "${COM_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" fi + # GEFS perturbations + # TODO if [[ $RUN} == "gefs" ]] block maybe be needed + # to ensure it does not interfere with the GFS + if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + ${NLN} "${COM_OCEAN_RESTART_PREV}/${sPDY}.${scyc}0000.mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" + fi + # Copy MOM6 fixed files ${NCP} "${FIXgfs}/mom6/${OCNRES}/"* "${DATA}/INPUT/" # TODO: These need to be explicit @@ -709,13 +723,11 @@ MOM6_postdet() { # If using stochatic parameterizations, create a seed that does not exceed the # largest signed integer - if [[ "${DO_OCN_SPPT}" = "YES" ]] || [[ "${DO_OCN_PERT_EPBL}" = "YES" ]]; then - if [[ ${SET_STP_SEED:-"YES"} = "YES" ]]; then - ISEED_OCNSPPT=$(( (current_cycle*1000 + MEMBER*10 + 6) % 2147483647 )) - ISEED_EPBL=$(( (current_cycle*1000 + MEMBER*10 + 7) % 2147483647 )) - else - ISEED=${ISEED:-0} - fi + if [[ ${DO_OCN_SPPT} = "YES" ]]; then + ISEED_OCNSPPT=$((current_cycle*10000 + ${MEMBER#0}*100 + 8)),$((current_cycle*10000 + ${MEMBER#0}*100 + 9)),$((current_cycle*10000 + ${MEMBER#0}*100 + 10)),$((current_cycle*10000 + ${MEMBER#0}*100 + 11)),$((current_cycle*10000 + ${MEMBER#0}*100 + 12)) + fi + if [[ ${DO_OCN_PERT_EPBL} = "YES" ]]; then + ISEED_EPBL=$((current_cycle*10000 + ${MEMBER#0}*100 + 13)),$((current_cycle*10000 + ${MEMBER#0}*100 + 14)),$((current_cycle*10000 + ${MEMBER#0}*100 + 15)),$((current_cycle*10000 + ${MEMBER#0}*100 + 16)),$((current_cycle*10000 + ${MEMBER#0}*100 + 17)) fi # Link output files @@ -931,7 +943,7 @@ GOCART_rc() { GOCART_postdet() { echo "SUB ${FUNCNAME[0]}: Linking output data for GOCART" - for fhr in ${FV3_OUTPUT_FH}; do + for fhr in ${GOCART_OUTPUT_FH}; do local vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) # Temporarily delete existing files due to noclobber in GOCART @@ -952,7 +964,7 @@ GOCART_out() { # TO DO: this should be linked but there were issues where gocart was crashing if it was linked local fhr local vdate - for fhr in ${FV3_OUTPUT_FH}; do + for fhr in ${GOCART_OUTPUT_FH}; do if (( fhr == 0 )); then continue; fi vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) ${NCP} "${DATA}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4" \ diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index ab02270b46..b5e1ad8e82 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -10,30 +10,33 @@ to_seconds() { # Function to convert HHMMSS to seconds since 00Z - local hhmmss=${1:?} - local hh=${hhmmss:0:2} - local mm=${hhmmss:2:2} - local ss=${hhmmss:4:2} - local seconds=$((10#${hh}*3600+10#${mm}*60+10#${ss})) - local padded_seconds=$(printf "%05d" "${seconds}") + local hhmmss hh mm ss seconds padded_seconds + hhmmss=${1:?} + hh=${hhmmss:0:2} + mm=${hhmmss:2:2} + ss=${hhmmss:4:2} + seconds=$((10#${hh}*3600+10#${mm}*60+10#${ss})) + padded_seconds=$(printf "%05d" "${seconds}") echo "${padded_seconds}" } middle_date(){ # Function to calculate mid-point date in YYYYMMDDHH between two dates also in YYYYMMDDHH - local date1=${1:?} - local date2=${2:?} - local date1s=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s) - local date2s=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s) - local dtsecsby2=$(( $((date2s - date1s)) / 2 )) - local mid_date=$(date --utc -d "${date1:0:8} ${date1:8:2} + ${dtsecsby2} seconds" +%Y%m%d%H%M%S) + local date1 date2 date1s date2s dtsecsby2 mid_date + date1=${1:?} + date2=${2:?} + date1s=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s) + date2s=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s) + dtsecsby2=$(( $((date2s - date1s)) / 2 )) + mid_date=$(date --utc -d "${date1:0:8} ${date1:8:2} + ${dtsecsby2} seconds" +%Y%m%d%H%M%S) echo "${mid_date:0:10}" } nhour(){ # Function to calculate hours between two dates (This replicates prod-util NHOUR) - local date1=${1:?} - local date2=${2:?} + local date1 date2 seconds1 seconds2 hours + date1=${1:?} + date2=${2:?} # Convert dates to UNIX timestamps seconds1=$(date --utc -d "${date1:0:8} ${date1:8:2}:00:00" +%s) seconds2=$(date --utc -d "${date2:0:8} ${date2:8:2}:00:00" +%s) @@ -41,21 +44,17 @@ nhour(){ echo "${hours}" } +# shellcheck disable=SC2034 common_predet(){ echo "SUB ${FUNCNAME[0]}: Defining variables for shared through model components" - # Ignore "not used" warning - # shellcheck disable=SC2034 pwd=$(pwd) CDUMP=${CDUMP:-gdas} - CASE=${CASE:-C96} CDATE=${CDATE:-"${PDY}${cyc}"} ENSMEM=${ENSMEM:-000} # Define significant cycles current_cycle="${PDY}${cyc}" previous_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} - ${assim_freq} hours" +%Y%m%d%H) - # ignore errors that variable isn't used - # shellcheck disable=SC2034 next_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${assim_freq} hours" +%Y%m%d%H) forecast_end_cycle=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHMAX} hours" +%Y%m%d%H) @@ -89,6 +88,7 @@ common_predet(){ cd "${DATA}" || ( echo "FATAL ERROR: Unable to 'cd ${DATA}', ABORT!"; exit 8 ) } +# shellcheck disable=SC2034 FV3_predet(){ echo "SUB ${FUNCNAME[0]}: Defining variables for FV3" @@ -105,8 +105,6 @@ FV3_predet(){ fi # Convert output settings into an explicit list for FV3 - # Ignore "not used" warning - # shellcheck disable=SC2034 FV3_OUTPUT_FH="" local fhr=${FHMIN} if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then @@ -116,8 +114,6 @@ FV3_predet(){ FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" # Other options - # ignore errors that variable isn't used - # shellcheck disable=SC2034 MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment @@ -169,7 +165,6 @@ FV3_predet(){ nstf_name=${nstf_name:-"${NST_MODEL},${NST_SPINUP},${NST_RESV},${ZSEA1},${ZSEA2}"} nst_anl=${nst_anl:-".false."} - # blocking factor used for threading and general physics performance #nyblocks=$(expr \( $npy - 1 \) \/ $layout_y ) #nxblocks=$(expr \( $npx - 1 \) \/ $layout_x \/ 32) @@ -215,6 +210,7 @@ WW3_predet(){ ${NLN} "${COM_WAVE_RESTART}" "restart_wave" } +# shellcheck disable=SC2034 CICE_predet(){ echo "SUB ${FUNCNAME[0]}: CICE before run type determination" @@ -227,12 +223,11 @@ CICE_predet(){ # CICE does not have a concept of high frequency output like FV3 # Convert output settings into an explicit list for CICE - # Ignore "not used" warning - # shellcheck disable=SC2034 CICE_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}") } +# shellcheck disable=SC2034 MOM6_predet(){ echo "SUB ${FUNCNAME[0]}: MOM6 before run type determination" @@ -245,8 +240,6 @@ MOM6_predet(){ # MOM6 does not have a concept of high frequency output like FV3 # Convert output settings into an explicit list for MOM6 - # Ignore "not used" warning - # shellcheck disable=SC2034 MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCNICE}" "${FHMAX}") } @@ -260,9 +253,12 @@ CMEPS_predet(){ } +# shellcheck disable=SC2034 GOCART_predet(){ echo "SUB ${FUNCNAME[0]}: GOCART before run type determination" if [[ ! -d "${COM_CHEM_HISTORY}" ]]; then mkdir -p "${COM_CHEM_HISTORY}"; fi + GOCART_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "6" "${FHMAX}") + # TODO: AERO_HISTORY.rc has hardwired output frequency to 6 hours } diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 14610f1201..ac18b06e71 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -631,6 +631,24 @@ EOF use_zmtnblck = ${use_zmtnblck:-".true."} EOF fi + + if [[ "${DO_OCN_SPPT:-NO}" == "YES" ]]; then + cat >> input.nml <> input.nml <> input.nml << EOF ${nam_stochy_nml:-} diff --git a/ush/parsing_namelists_MOM6.sh b/ush/parsing_namelists_MOM6.sh index 9c1378fec4..923288c76a 100755 --- a/ush/parsing_namelists_MOM6.sh +++ b/ush/parsing_namelists_MOM6.sh @@ -25,24 +25,6 @@ EOF # new_lscale=.true. #EOF -if [[ "${DO_OCN_SPPT}" == "YES" ]]; then - cat >> input.nml <> input.nml <> input.nml < None: super().initialize() # stage CRTM fix files - crtm_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'gdas', 'aero_crtm_coeff.yaml') - logger.debug(f"Staging CRTM fix files from {crtm_fix_list_path}") - crtm_fix_list = parse_j2yaml(crtm_fix_list_path, self.task_config) + logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") + crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) FileHandler(crtm_fix_list).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'gdas', 'aero_jedi_fix.yaml') - logger.debug(f"Staging JEDI fix files from {jedi_fix_list_path}") - jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config) + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() # stage berror files @@ -93,10 +91,9 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.fv3jedi_yaml}") - varda_yaml = parse_j2yaml(self.task_config['AEROVARYAML'], self.task_config) - save_as_yaml(varda_yaml, self.task_config.fv3jedi_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.fv3jedi_yaml}") + logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -114,7 +111,7 @@ def execute(self: Analysis) -> None: exec_cmd = Executable(self.task_config.APRUN_AEROANL) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.fv3jedi_yaml) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -212,7 +209,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) # get list of increment vars - incvars_list_path = os.path.join(self.task_config['HOMEgfs'], 'parm', 'gdas', 'aeroanl_inc_vars.yaml') + incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] super().add_fv3_increments(inc_template, bkg_template, incvars) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 5709bc130e..2221fb7b34 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -4,6 +4,7 @@ import glob import tarfile from logging import getLogger +from pprint import pformat from netCDF4 import Dataset from typing import List, Dict, Any, Union @@ -26,10 +27,14 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) self.config.ntiles = 6 # Store location of GDASApp jinja2 templates - self.gdasapp_j2tmpl_dir = os.path.join(self.config.HOMEgfs, 'parm/gdas') + self.gdasapp_j2tmpl_dir = os.path.join(self.config.PARMgfs, 'gdas') def initialize(self) -> None: super().initialize() + + # all JEDI analyses need a JEDI config + self.task_config.jedi_config = self.get_jedi_config() + # all analyses need to stage observations obs_dict = self.get_obs_dict() FileHandler(obs_dict).sync() @@ -41,13 +46,33 @@ def initialize(self) -> None: # link jedi executable to run directory self.link_jediexe() + @logit(logger) + def get_jedi_config(self) -> Dict[str, Any]: + """Compile a dictionary of JEDI configuration from JEDIYAML template file + + Parameters + ---------- + + Returns + ---------- + jedi_config : Dict + a dictionary containing the fully rendered JEDI yaml configuration + """ + + # generate JEDI YAML file + logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") + jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.debug(f"JEDI config:\n{pformat(jedi_config)}") + + return jedi_config + @logit(logger) def get_obs_dict(self) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method uses the OBS_LIST configuration variable to generate a dictionary - from a list of YAML files that specify what observation files are to be - copied to the run directory from the observation input directory + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation files that are to be copied to the run directory + from the observation input directory Parameters ---------- @@ -57,13 +82,13 @@ def get_obs_dict(self) -> Dict[str, Any]: obs_dict: Dict a dictionary containing the list of observation files to copy for FileHandler """ - logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}") - obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"obs_list_config: {obs_list_config}") - # get observers from master dictionary - observers = obs_list_config['observers'] + + logger.info(f"Extracting a list of observation files from {self.task_config.JEDIYAML}") + observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') + logger.debug(f"observations:\n{pformat(observations)}") + copylist = [] - for ob in observers: + for ob in observations['observers']: obfile = ob['obs space']['obsdatain']['engine']['obsfile'] basename = os.path.basename(obfile) copylist.append([os.path.join(self.task_config['COM_OBS'], basename), obfile]) @@ -77,9 +102,11 @@ def get_obs_dict(self) -> Dict[str, Any]: def get_bias_dict(self) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method uses the OBS_LIST configuration variable to generate a dictionary - from a list of YAML files that specify what observation bias correction files - are to be copied to the run directory from the observation input directory + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation bias correction files that are to be copied to the run directory + from the component directory. + TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in + `analysis.py` and should be implemented in the component where this is applicable. Parameters ---------- @@ -89,13 +116,13 @@ def get_bias_dict(self) -> Dict[str, Any]: bias_dict: Dict a dictionary containing the list of observation bias files to copy for FileHandler """ - logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}") - obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"obs_list_config: {obs_list_config}") - # get observers from master dictionary - observers = obs_list_config['observers'] + + logger.info(f"Extracting a list of bias correction files from {self.task_config.JEDIYAML}") + observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') + logger.debug(f"observations:\n{pformat(observations)}") + copylist = [] - for ob in observers: + for ob in observations['observers']: if 'obs bias' in ob.keys(): obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) @@ -104,6 +131,7 @@ def get_bias_dict(self) -> Dict[str, Any]: for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: bfile = f"{prefix}.{file}" copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + # TODO: Why is this specific to ATMOS? bias_dict = { 'mkdir': [os.path.join(self.runtime_config.DATA, 'bc')], @@ -328,3 +356,74 @@ def tgz_diags(statfile: str, diagdir: str) -> None: # Add diag files to tarball for diagfile in diags: tgz.add(diagfile, arcname=os.path.basename(diagfile)) + + +@logit(logger) +def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: + """ + Recursively search through a nested dictionary and return the value for the target key. + This returns the first target key it finds. So if a key exists in a subsequent + nested dictionary, it will not be found. + + Parameters + ---------- + nested_dict : Dict + Dictionary to search + target_key : str + Key to search for + + Returns + ------- + Any + Value of the target key + + Raises + ------ + KeyError + If key is not found in dictionary + + TODO: if this gives issues due to landing on an incorrect key in the nested + dictionary, we will have to implement a more concrete method to search for a key + given a more complete address. See resolved conversations in PR 2387 + + # Example usage: + nested_dict = { + 'a': { + 'b': { + 'c': 1, + 'd': { + 'e': 2, + 'f': 3 + } + }, + 'g': 4 + }, + 'h': { + 'i': 5 + }, + 'j': { + 'k': 6 + } + } + + user_key = input("Enter the key to search for: ") + result = find_value_in_nested_dict(nested_dict, user_key) + """ + + if not isinstance(nested_dict, dict): + raise TypeError(f"Input is not of type(dict)") + + result = nested_dict.get(target_key) + if result is not None: + return result + + for value in nested_dict.values(): + if isinstance(value, dict): + try: + result = find_value_in_nested_dict(value, target_key) + if result is not None: + return result + except KeyError: + pass + + raise KeyError(f"Key '{target_key}' not found in the nested dictionary") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5a90a89e34..6348bdf319 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -31,7 +31,7 @@ def __init__(self, config): _res = int(self.config.CASE[1:]) _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") + _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -48,7 +48,7 @@ def __init__(self, config): 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", - 'fv3jedi_yaml': _fv3jedi_yaml, + 'jedi_yaml': _jedi_yaml, } ) @@ -71,19 +71,17 @@ def initialize(self: Analysis) -> None: super().initialize() # stage CRTM fix files - crtm_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'atm_crtm_coeff.yaml') - logger.debug(f"Staging CRTM fix files from {crtm_fix_list_path}") - crtm_fix_list = parse_j2yaml(crtm_fix_list_path, self.task_config) + logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") + crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) FileHandler(crtm_fix_list).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'atm_jedi_fix.yaml') - logger.debug(f"Staging JEDI fix files from {jedi_fix_list_path}") - jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config) + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() # stage static background error files, otherwise it will assume ID matrix - logger.debug(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") + logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") FileHandler(self.get_berror_dict(self.task_config)).sync() # stage ensemble files for use in hybrid background error @@ -102,10 +100,9 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config))).sync() # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.fv3jedi_yaml}") - varda_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(varda_yaml, self.task_config.fv3jedi_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.fv3jedi_yaml}") + logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -123,7 +120,7 @@ def execute(self: Analysis) -> None: exec_cmd = Executable(self.task_config.APRUN_ATMANL) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.fv3jedi_yaml) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -170,7 +167,7 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") + logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 3e2c0a233c..1037b557c2 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -31,7 +31,7 @@ def __init__(self, config): _res = int(self.config.CASE_ENS[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmens.yaml") + _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -45,7 +45,7 @@ def __init__(self, config): 'OPREFIX': f"{self.config.EUPD_CYC}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", - 'fv3jedi_yaml': _fv3jedi_yaml, + 'jedi_yaml': _jedi_yaml, } ) @@ -96,19 +96,17 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': dirlist}).sync() # stage CRTM fix files - crtm_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'atm_crtm_coeff.yaml') - logger.debug(f"Staging CRTM fix files from {crtm_fix_list_path}") - crtm_fix_list = parse_j2yaml(crtm_fix_list_path, self.task_config) + logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") + crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) FileHandler(crtm_fix_list).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'atm_jedi_fix.yaml') - logger.debug(f"Staging JEDI fix files from {jedi_fix_list_path}") - jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config) + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() # stage backgrounds - logger.debug(f"Stage ensemble member background files") + logger.info(f"Stage ensemble member background files") localconf = AttrDict() keys = ['COM_ATMOS_RESTART_TMPL', 'previous_cycle', 'ROTDIR', 'RUN', 'NMEM_ENS', 'DATA', 'current_cycle', 'ntiles'] @@ -118,10 +116,9 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_fv3ens_dict(localconf)).sync() # generate ensemble da YAML file - logger.debug(f"Generate ensemble da YAML file: {self.task_config.fv3jedi_yaml}") - ensda_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(ensda_yaml, self.task_config.fv3jedi_yaml) - logger.info(f"Wrote ensemble da YAML to: {self.task_config.fv3jedi_yaml}") + logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -153,7 +150,7 @@ def execute(self: Analysis) -> None: exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_letkf.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.fv3jedi_yaml) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -206,7 +203,7 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.fv3jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") + logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") logger.debug(f"Copying {src} to {dest}") diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9eee8314c3..c149f140b6 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -260,20 +260,18 @@ def initialize(self) -> None: FileHandler({'mkdir': dirlist}).sync() # stage fix files - jedi_fix_list_path = os.path.join(self.task_config.HOMEgfs, 'parm', 'gdas', 'snow_jedi_fix.yaml.j2') - logger.info(f"Staging JEDI fix files from {jedi_fix_list_path}") - jedi_fix_list = parse_j2yaml(jedi_fix_list_path, self.task_config) + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() # stage backgrounds logger.info("Staging ensemble backgrounds") FileHandler(self.get_ens_bkg_dict(localconf)).sync() - # generate letkfoi YAML file - logger.info(f"Generate JEDI LETKF YAML file: {self.task_config.jedi_yaml}") - letkfoi_yaml = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(letkfoi_yaml, self.task_config.jedi_yaml) + # Write out letkfoi YAML file + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") + # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/versions/fix.ver b/versions/fix.ver index a2a9caf8e3..d2828518bc 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -12,7 +12,7 @@ export gdas_fv3jedi_ver=20220805 export gdas_gsibec_ver=20221031 export gdas_obs_ver=20240213 export glwu_ver=20220805 -export gsi_ver=20230911 +export gsi_ver=20240208 export lut_ver=20220805 export mom6_ver=20231219 export orog_ver=20231027 diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index b91e974c74..36b57bd6dc 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2661,7 +2661,7 @@ def earc(self): # Integer division is floor division, but we need ceiling division n_groups = -(self.nmem // -self._configs['earc']['NMEM_EARCGRP']) - groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups)]) + groups = ' '.join([f'{grp:02d}' for grp in range(0, n_groups + 1)]) cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '')