[develop] Update weather model, UPP, and UFS_UTILS hashes

.cicd/scripts/sbatch_srw_ftest.sh

@@ -7,7 +7,7 @@
 #SBATCH --account=${SRW_PROJECT}
 #SBATCH --qos=batch
 #SBATCH --nodes=1
-#SBATCH --tasks-per-node=24
+#SBATCH --tasks-per-node=12
 #SBATCH --cpus-per-task=1
 #SBATCH -t 00:30:00
 #SBATCH -o log_wrap.%j.log

.cicd/scripts/srw_ftest.sh

@@ -77,6 +77,9 @@ sed "s|^task_get_extrn_lbcs:|task_get_extrn_lbcs:\n  EXTRN_MDL_SOURCE_BASEDIR_LB
 # Use staged data for HPSS supported machines
 sed 's|^platform:|platform:\n  EXTRN_MDL_DATA_STORES: disk|g' -i ush/config.yaml
 
+# Set OMP_NUM_THREADS_RUN_FCST to 1 in config.yaml
+sed 's|^task_run_fcst:|task_run_fcst:\n  OMP_NUM_THREADS_RUN_FCST: 1|1' -i ush/config.yaml
+
 # Activate the workflow environment ...
 source etc/lmod-setup.sh ${platform,,}
 module use modulefiles
@@ -105,7 +108,7 @@ cp ${workspace}/ush/wrappers/*.sh .
 export JOBSdir=${workspace}/jobs
 export USHdir=${workspace}/ush
 export OMP_NUM_THREADS=1
-export nprocs=24
+export nprocs=12
 
 [[ -n ${TASKS} ]] || TASKS=(
                 run_make_grid

.cicd/scripts/wrapper_srw_ftest.sh

@@ -34,6 +34,10 @@ if [[ "${SRW_PLATFORM}" == hera ]]; then
     fi
 fi
 
+if [[ "${SRW_PLATFORM}" == jet ]]; then
+    sed -i '15i #SBATCH --partition=xjet' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+fi
+
 # Call job card and return job_id
 echo "Running: ${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh"
 job_id=$(${workflow_cmd} -A ${SRW_PROJECT} ${arg_1} ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh)

Externals.cfg

@@ -3,7 +3,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = dc0e4a6
+hash = 57bd832
 local_path = sorc/UFS_UTILS
 required = True
 
@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 020e783
+hash = 8518c2c
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = fae617b
+hash = 945cb2c
 local_path = sorc/UPP
 required = True
 

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -386,8 +386,8 @@ Set File Name Parameters
 ``MODEL_CONFIG_FN``: (Default: "model_configure")
    Name of a file that contains settings and configurations for the :term:`NUOPC`/:term:`ESMF` main component. In general, users should not set this variable in their configuration file (see :ref:`note <tmpl-fn-warning>`).
 
-``NEMS_CONFIG_FN``: (Default: "nems.configure")
-   Name of a file that contains information about the various :term:`NEMS` components and their run sequence. In general, users should not set this variable in their configuration file (see :ref:`note <tmpl-fn-warning>`).
+``UFS_CONFIG_FN``: (Default: "ufs.configure")
+   Name of a file that contains information about the various :term:`UFS` components and their run sequence. In general, users should not set this variable in their configuration file (see :ref:`note <tmpl-fn-warning>`).
 
 ``AQM_RC_FN``: (Default: "aqm.rc")
    Name of resource file for NOAA Air Quality Model (AQM). 
@@ -419,8 +419,8 @@ Set File Path Parameters
 ``MODEL_CONFIG_TMPL_FP``: (Default: ``'{{ [user.PARMdir, MODEL_CONFIG_FN]|path_join }}'``) 
    Path to the ``MODEL_CONFIG_FN`` file.
 
-``NEMS_CONFIG_TMPL_FP``: (Default: ``'{{ [user.PARMdir, NEMS_CONFIG_FN]|path_join }}'``) 
-   Path to the ``NEMS_CONFIG_FN`` file. 
+``UFS_CONFIG_TMPL_FP``: (Default: ``'{{ [user.PARMdir, UFS_CONFIG_FN]|path_join }}'``) 
+   Path to the ``UFS_CONFIG_FN`` file. 
 
 ``AQM_RC_TMPL_FP``: (Default: ``'{{ [user.PARMdir, AQM_RC_TMPL_FN]|path_join }}'``) 
    Path to the ``AQM_RC_TMPL_FN`` file. 
@@ -437,8 +437,8 @@ This section contains files and paths to files that are staged in the experiment
 ``FIELD_TABLE_FP``: (Default: ``'{{ [EXPTDIR, FIELD_TABLE_FN]|path_join }}'``)
    Path to the field table in the experiment directory. (The field table specifies tracers that the forecast model reads in.)
 
-``NEMS_CONFIG_FP``: (Default: ``'{{ [EXPTDIR, NEMS_CONFIG_FN]|path_join }}'``)
-   Path to the ``NEMS_CONFIG_FN`` file in the experiment directory. 
+``UFS_CONFIG_FP``: (Default: ``'{{ [EXPTDIR, UFS_CONFIG_FN]|path_join }}'``)
+   Path to the ``UFS_CONFIG_FN`` file in the experiment directory. 
 
 ``FV3_NML_FP``: (Default: ``'{{ [EXPTDIR, FV3_NML_FN]|path_join }}'``)
    Path to the ``FV3_NML_FN`` file in the experiment directory.
@@ -538,7 +538,7 @@ CCPP Parameter
 Field Dictionary Parameters
 ------------------------------
 
-``FIELD_DICT_FN``: (Default: "fd_nems.yaml")
+``FIELD_DICT_FN``: (Default: "fd_ufs.yaml")
    The name of the field dictionary file. This file is a community-based dictionary for shared coupling fields and is automatically generated by the :term:`NUOPC` Layer. 
 
 ``FIELD_DICT_IN_UWM_FP``: (Default: ``'{{ [user.UFS_WTHR_MDL_DIR, "tests", "parm", FIELD_DICT_FN]|path_join }}'``)
@@ -1109,7 +1109,7 @@ For each workflow task, certain parameter values must be passed to the job sched
       For more information, see the `Intel Development Reference Guide <https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-10/thread-affinity-interface.html>`__. 
 
 ``OMP_NUM_THREADS_RUN_FCST``: (Default: 1)
-   The number of OpenMP threads to use for parallel regions. Corresponds to the ``ATM_omp_num_threads`` value in ``nems.configure``.
+   The number of OpenMP threads to use for parallel regions. Corresponds to the ``atmos_nthreads`` value in ``model_configure``.
 
 ``OMP_STACKSIZE_RUN_FCST``: (Default: "512m")
    Controls the size of the stack for threads created by the OpenMP implementation.

modulefiles/build_derecho_intel.lua

@@ -6,7 +6,7 @@ the CISL machine Derecho (Cray) using [email protected]
 whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===])
 
 prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra")
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
 
 load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.10.0"))
 load(pathJoin("stack-cray-mpich", os.getenv("stack_cray_mpich_ver") or "8.1.25"))

modulefiles/build_gaea_intel.lua

@@ -5,7 +5,7 @@ the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Gaea C5 ]===])
 
-prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH","/ncrc/proj/epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 

modulefiles/build_hera_gnu.lua

@@ -5,7 +5,7 @@ the NOAA RDHPC machine Hera using GNU 9.2.0
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GNU 9.2.0 ]===])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
 load("stack-gcc/9.2.0")

modulefiles/build_hera_intel.lua

@@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===])
 prepend_path("MODULEPATH","/contrib/sutils/modulefiles")
 load("sutils")
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"

modulefiles/build_hercules_intel.lua

@@ -5,7 +5,7 @@ the MSU machine Hercules using intel-oneapi-compilers/2022.2.1
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Hercules ]===])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles")
 
 load("stack-intel/2021.9.0")

modulefiles/build_jet_intel.lua

@@ -5,7 +5,7 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===])
 
-prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core")
+prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env-rocky8/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles")
 
 load("stack-intel/2021.5.0")

modulefiles/build_noaacloud_intel.lua

@@ -5,7 +5,7 @@ the NOAA cloud using Intel-oneapi
 
 whatis([===[Loads libraries needed for building the UFS SRW App on NOAA cloud ]===])
 
-prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/apps/modules/modulefiles")
 prepend_path("PATH", "/contrib/EPIC/bin")
 load("stack-intel")

modulefiles/build_orion_intel.lua

@@ -5,7 +5,7 @@ the MSU machine Orion using Intel-2022.1.2
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles")
 
 load("stack-intel/2022.0.2")

modulefiles/srw_common.lua

@@ -5,8 +5,8 @@ load("libpng/1.6.37")
 load("netcdf-c/4.9.2")
 load("netcdf-fortran/4.6.0")
 load("parallelio/2.5.10")
-load("esmf/8.4.2")
-load("fms/2023.01")
+load("esmf/8.5.0")
+load("fms/2023.02.01")
 
 load("bacio/2.4.1")
 load("crtm/2.4.0")
@@ -16,8 +16,8 @@ load("ip/4.3.0")
 load("sp/2.3.3")
 load("w3emc/2.10.0")
 
-load("gftl-shared/1.5.0")
-load("mapl/2.35.2-esmf-8.4.2")
+load("gftl-shared/1.6.1")
+load("mapl/2.40.3-esmf-8.5.0")
 
 load("nemsio/2.5.4")
 load("sfcio/1.4.1")

parm/fixed_files_mapping.yaml

@@ -175,7 +175,7 @@ fixed_files:
   "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb",
   "sfc_emissivity_idx.txt     | global_sfc_emissivity_idx.txt",
   "solarconstant_noaa_an.txt  | global_solarconstant_noaa_an.txt",
-  "global_o3prdlos.f77        | "
+  "global_o3prdlos.f77        | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77"
   ]
 
   #

parm/model_configure

@@ -1,3 +1,5 @@
+total_member:            1
+PE_MEMBER01:             {{ PE_MEMBER01 }}
 start_year:              {{ start_year }}
 start_month:             {{ start_month }}
 start_day:               {{ start_day }}
@@ -11,6 +13,7 @@ ENS_SPS:                 .false.
 dt_atmos:                {{ dt_atmos }}
 calendar:                'julian'
 memuse_verbose:          .false.
+atmos_nthreads:          {{ atmos_nthreads }}
 restart_interval:        {{ restart_interval }}
 output_1st_tstep_rst:    .false.
 write_dopost:            {{ write_dopost }}

parm/nems.configure → parm/ufs.configure

@@ -1,5 +1,5 @@
 #############################################
-####  NEMS Run-Time Configuration File  #####
+####  UFS Run-Time Configuration File  #####
 #############################################
 
 # ESMF #
@@ -45,21 +45,8 @@ runSeq::
 {% else %}
 # EARTH #
 EARTH_component_list: ATM
-EARTH_attributes::
-  Verbosity = 0
-::
-
-# ATM #
-ATM_model:                      fv3
-ATM_petlist_bounds:             0 {{ pe_member01_m1 }}
-ATM_omp_num_threads:            {{ atm_omp_num_threads }}
-ATM_attributes::
-  Verbosity = 0
-  Diagnostic = 0
-::
-
-# Run Sequence #
-runSeq::
-  ATM
-::
+ ATM_model:            fv3
+ runSeq::
+   ATM
+ ::
 {% endif %}

scripts/exregional_make_orog.sh

@@ -186,6 +186,8 @@ echo $mtnres $lonb $latb $jcap $NR $NF1 $NF2 $efac $blat > "${input_redirect_fn}
 #
 echo "\"${grid_fp}\"" >> "${input_redirect_fn}"
 echo "\"$orogfile\"" >> "${input_redirect_fn}"
+echo ".false." >> "${input_redirect_fn}" #MASK_ONLY
+echo "none" >> "${input_redirect_fn}" #MERGE_FILE
 cat "${input_redirect_fn}"
 #
 #-----------------------------------------------------------------------

scripts/exregional_run_fcst.sh

@@ -696,7 +696,7 @@ fi
 #
 #-----------------------------------------------------------------------
 #
-python3 $USHdir/create_nems_configure_file.py \
+python3 $USHdir/create_ufs_configure_file.py \
   --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
   --run-dir "${DATA}"
 export err=$?

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.yaml

@@ -20,3 +20,5 @@ task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+task_run_fcst:
+  OMP_NUM_THREADS_RUN_FCST: 3

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.yaml

@@ -19,3 +19,5 @@ task_get_extrn_lbcs:
   EXTRN_MDL_NAME_LBCS: FV3GFS
   LBC_SPEC_INTVL_HRS: 3
   USE_USER_STAGED_EXTRN_FILES: true
+task_run_fcst:
+  OMP_NUM_THREADS_RUN_FCST: 3

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml

@@ -3,11 +3,10 @@ metadata:
     This test is to ensure that the workflow running in community mode
     completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta
     physics suite with ICs derived from the HRRR and LBCs derived from the RAP.
-    It also tests the "DOT_OR_USCORE" option and enables offline UPP 2D decomposition.
+    It also enables offline UPP 2D decomposition.
 user:
   RUN_ENVIR: community
 workflow:
-  DOT_OR_USCORE: .
   CCPP_PHYS_SUITE: FV3_RRFS_v1beta
   PREDEF_GRID_NAME: RRFS_CONUScompact_25km
   DATE_FIRST_CYCL: '2020081000'

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml

@@ -23,3 +23,5 @@ task_get_extrn_lbcs:
   USE_USER_STAGED_EXTRN_FILES: true
   EXTRN_MDL_FILES_LBCS:
     - '{yy}{jjj}{hh}00{fcst_hr:02d}00'
+task_run_fcst:
+  OMP_NUM_THREADS_RUN_FCST: 3

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml

@@ -24,3 +24,5 @@ task_get_extrn_lbcs:
   USE_USER_STAGED_EXTRN_FILES: true
   EXTRN_MDL_FILES_LBCS:
     - '{yy}{jjj}{hh}00{fcst_hr:02d}00'
+task_run_fcst:
+  OMP_NUM_THREADS_RUN_FCST: 3

tests/WE2E/test_configs/wflow_features/config.specify_template_filenames.yaml

@@ -9,7 +9,7 @@ workflow:
   DIAG_TABLE_TMPL_FN: diag_table.FV3_GFS_v15p2
   FIELD_TABLE_TMPL_FN: field_table.FV3_GFS_v15p2
   MODEL_CONFIG_FN: model_configure
-  NEMS_CONFIG_FN: nems.configure
+  UFS_CONFIG_FN: ufs.configure
   CCPP_PHYS_SUITE: FV3_GFS_v15p2
   PREDEF_GRID_NAME: RRFS_CONUS_25km
   DATE_FIRST_CYCL: '2019070100'

tests/test_python/test_create_model_configure_file.py

@@ -43,9 +43,11 @@ def setUp(self):
         set_env_var("USHdir", USHdir)
         set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN)
         set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP)
+        set_env_var("PE_MEMBER01", 24)
         set_env_var("FCST_LEN_HRS", 72)
         set_env_var("FHROT", 0)
         set_env_var("DT_ATMOS", 1)
+        set_env_var("OMP_NUM_THREADS_RUN_FCST", 1)
         set_env_var("RESTART_INTERVAL", 4)
         set_env_var("ITASKS", 1)