diff --git a/doc/UsersGuide/Makefile b/doc/UsersGuide/Makefile new file mode 100644 index 0000000000..d0c3cbf102 --- /dev/null +++ b/doc/UsersGuide/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/UsersGuide/README.md b/doc/UsersGuide/README.md new file mode 100644 index 0000000000..028675fd99 --- /dev/null +++ b/doc/UsersGuide/README.md @@ -0,0 +1,11 @@ +This directory contains the Developer's Guide for the UFS Medium-Range Weather Application. + +To build in html format (from this directory, above source and build) type: + +`make html` + +This will create an `index.html` file in `./build/html`. + +To view in a browser window: + +`open build/html/index.html` diff --git a/doc/UsersGuide/build/.gitignore b/doc/UsersGuide/build/.gitignore new file mode 100644 index 0000000000..0addfc818b --- /dev/null +++ b/doc/UsersGuide/build/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/doc/UsersGuide/make.bat b/doc/UsersGuide/make.bat new file mode 100644 index 0000000000..6247f7e231 --- /dev/null +++ b/doc/UsersGuide/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/doc/UsersGuide/requirements.txt b/doc/UsersGuide/requirements.txt new file mode 100644 index 0000000000..ef36addc62 --- /dev/null +++ b/doc/UsersGuide/requirements.txt @@ -0,0 +1 @@ +sphinxcontrib-bibtex diff --git a/doc/UsersGuide/source/Acronyms.rst b/doc/UsersGuide/source/Acronyms.rst new file mode 100644 index 0000000000..d529e7cf4f --- /dev/null +++ b/doc/UsersGuide/source/Acronyms.rst @@ -0,0 +1,98 @@ +.. _Acronyms: + +************************* +Acronyms +************************* + +.. table:: + :widths: 20 80 + + +----------------+---------------------------------------------------+ + | Abbreviation | Explanation | + +================+===================================================+ + | AOML | NOAA's Atlantic Oceanographic and Meteorological | + | | Laboratory | + +----------------+---------------------------------------------------+ + | API | Application Programming Interface | + +----------------+---------------------------------------------------+ + | b4b | Bit-for-bit | + +----------------+---------------------------------------------------+ + | CCPP | Common Community Physics Package | + +----------------+---------------------------------------------------+ + | CPP | C preprocessor | + +----------------+---------------------------------------------------+ + | dycore | Dynamical core | + +----------------+---------------------------------------------------+ + | EDMF | Eddy-Diffusivity Mass Flux | + +----------------+---------------------------------------------------+ + | EMC | Environmental Modeling Center | + +----------------+---------------------------------------------------+ + | ESMF | The Earth System Modeling Framework | + +----------------+---------------------------------------------------+ + | ESRL | NOAA Earth System Research Laboratory | + +----------------+---------------------------------------------------+ + | FMS | Flexible Modeling System | + +----------------+---------------------------------------------------+ + | FV3 | Finite-Volume Cubed Sphere | + +----------------+---------------------------------------------------+ + | GFDL | Geophysical Fluid Dynamics Laboratory | + +----------------+---------------------------------------------------+ + | GFS | Global Forecast System | + +----------------+---------------------------------------------------+ + | GSD | Global Systems Division | + +----------------+---------------------------------------------------+ + | HTML | Hypertext Markup Language | + +----------------+---------------------------------------------------+ + | LSM | Land Surface Model | + +----------------+---------------------------------------------------+ + | MPI | Message Passing Interface | + +----------------+---------------------------------------------------+ + | NCAR | National Center for Atmospheric Research | + +----------------+---------------------------------------------------+ + | NCEP | National Centers for Environmental Predicction | + +----------------+---------------------------------------------------+ + | NEMS | National Oceanic and Atmospheric Administration | + | | (NOAA) Environmental Modeling System | + +----------------+---------------------------------------------------+ + | NOAA | National Oceanic and Atmospheric Administration | + +----------------+---------------------------------------------------+ + | NSSL | National Severe Storms Laboratory | + +----------------+---------------------------------------------------+ + | OpenMP | Open Multi-Processing | + +----------------+---------------------------------------------------+ + | PBL | Planetary Boundary Layer | + +----------------+---------------------------------------------------+ + | png | Portable Network Graphic | + +----------------+---------------------------------------------------+ + | PR | Pull request | + +----------------+---------------------------------------------------+ + | PROD | Compiler optimization flags for production mode | + +----------------+---------------------------------------------------+ + | REPRO | Compiler optimization flags for reproduction mode | + | | (bit-for-bit testing) | + +----------------+---------------------------------------------------+ + | RRTMG | Rapid Radiative Transfer Model for Global | + | | Circulation Models | + +----------------+---------------------------------------------------+ + | RT | Regression test | + +----------------+---------------------------------------------------+ + | SAS | Simplified Arakawa-Schubert | + +----------------+---------------------------------------------------+ + | SDF | Suite Definition File | + +----------------+---------------------------------------------------+ + | sfc | Surface | + +----------------+---------------------------------------------------+ + | SHUM | Perturbed boundary layer specific humidity | + +----------------+---------------------------------------------------+ + | SKEB | Stochastic Kinetic Energy Backscatter | + +----------------+---------------------------------------------------+ + | SPPT | Stochastically Perturbed Physics Tendencies | + +----------------+---------------------------------------------------+ + | TKE | Turbulent Kinetic Energy | + +----------------+---------------------------------------------------+ + | UFS | Unified Forecast System | + +----------------+---------------------------------------------------+ + | VLab | Virtual Laboratory | + +----------------+---------------------------------------------------+ + | WM | Weather Model | + +----------------+---------------------------------------------------+ diff --git a/doc/UsersGuide/source/CodeOverview.rst b/doc/UsersGuide/source/CodeOverview.rst new file mode 100644 index 0000000000..9155e6b8a9 --- /dev/null +++ b/doc/UsersGuide/source/CodeOverview.rst @@ -0,0 +1,106 @@ +.. _CodeOverview: + +************************* +Code Overview +************************* + +=================================================== +UFS Weather Model Hierarchical Repository Structure +=================================================== + +The ufs-weather-model repository supports the short- and medium-range UFS applications. It contains atmosphere and wave components and some infrastructure components. Each of these components has its own repository. All the repositories are currently located in GitHub with public access to the broad community. :numref:`Table %s ` describes the list of repositories that comprises the ufs-weather-model. + +.. _Repo_Structure: + +.. list-table:: *List of Repositories that comprise the ufs-weather-model* + :widths: 50 50 + :header-rows: 1 + + * - Repository Description + - Authoritative repository URL + * - Umbrella repository for the UFS Weather Model + - https://github.com/ufs-community/ufs-weather-model + * - Infrastructure: Flexible Modeling System + - https://github.com/NOAA-GFDL/FMS + * - Infrastructure: NOAA Environmental Modeling System + - https://github.com/NOAA-EMC/NEMS + * - Infrastructure: add description + - https://github.com/NOAA-EMC/NCEPLIBS-pyprodutil + * - Framework to connect the CCPP library to a host model + - https://github.com/NCAR/ccpp-framework + * - CCPP library of physical parameterizations + - https://github.com/NCAR/ccpp-physics + * - Umbrella repository for the physics and dynamics of the atmospheric model + - https://github.com/NOAA-EMC/fv3atm + * - FV3 dynamical core + - https://github.com/NOAA-EMC/GFDL_atmos_cubed_sphere + * - Stochastic physics pattern generator + - https://github.com/noaa-psd/stochastic_physics + * - WAVEWATCH 3 wave model + - https://github.com/NOAA-EMC/WW3 + +In the table, the left column contains a description of each repository, and the right column shows the component repositories which are pointing to (or will point to) the authoritative repositories. The ufs-weather-model currently uses git submodule to manage the sub-components. + +The umbrella repository for the UFS Weather Model is named ufs-weather-model. Under this repository reside a number of submodules that are nested in specific directories under the parent repository’s working directory. When the ufs-weather-model repository is cloned, the *.gitmodules* file creates the following directories: + +.. code-block:: console + + ufs-weather-model/ + ├── FMS https://github.com/NOAA-GFDL/FMS + ├── FV3 https://github.com/NCAR/fv3atm + │ ├── atmos_cubed_sphere https://github.com/NCAR/GFDL_atmos_cubed_sphere + │ ├── ccpp + │ │ ├── framework https://github.com/NCAR/ccpp-framework + │ │ ├── physics https://github.com/NCAR/ccpp-physics + ├── NEMS https://github.com/NCAR/NEMS + │ └── tests/produtil/NCEPLIBS-pyprodutil https://github.com/NOAA-EMC/NCEPLIBS-pyprodutil + ├── stochastic_physics https://github.com/noaa-psd/stochastic_physics + └── WW3 https://github.com/NOAA-EMC/WW3 + +=================== +Directory Structure +=================== + +When the ufs-weather-model is cloned, the basic directory structure will be similar to the example below. Files and some directories have been removed for brevity. + +.. code-block:: console + + ufs-weather-model/ + ├── cmake --------- cmake configuration files + ├── compsets --------- configurations used by some regression tests + ├── conf --------- compile options for Tier 1 and 2 platforms + ├── doc --------- READMEs with build, reg-test hints + ├── FMS --------- The Flexible Modeling System (FMS),a software framework + ├── FV3 --------- FV3 atmosphere model + │ ├── atmos_cubed_sphere ---- FV3 dynamic core + │ │ ├── docs + │ │ ├── driver + │ │ ├── model + │ │ └── tools + │ ├── ccpp -------- Common Community Physics Package + │ │ ├── config + │ │ ├── driver + │ │ ├── framework -------- CCPP framework + │ │ ├── physics -------- CCPP compliant physics schemes + │ │ └── suites -------- CCPP physics suite definition files (SDFs) + │ ├── cpl -------- Coupling field data structures + │ ├── gfsphysics + │ │ ├── CCPP_layer + │ │ ├── GFS_layer + │ │ └── physics --------- unused - IPD version of physics codes + │ ├── io --------- FV3 write grid comp code + │ ├── ipd --------- unused - IPD driver/interfaces + ├── log --------- log files from NEMS compset regression tests + ├── modulefile --------- system module files for supported HPC systems + ├── NEMS --------- NOAA Earth Modeling System framework + │ ├── exe + │ ├── src + │ └── test + ├── parm --------- regression test configurations + ├── release --------- unused + ├── stochastic_physics -------- stochastic physics pattern generator + ├── tests --------- regression test scripts + └── WW3 --------- WAVEWATCH III (wave component) + +The physics subdirectory in the *gfsphysics* directory is not used or supported as part of this release (all physics is available through the :term:`CCPP`). + diff --git a/doc/UsersGuide/source/CompilingCodeWithoutApp.rst b/doc/UsersGuide/source/CompilingCodeWithoutApp.rst new file mode 100644 index 0000000000..e9303b0782 --- /dev/null +++ b/doc/UsersGuide/source/CompilingCodeWithoutApp.rst @@ -0,0 +1,92 @@ +.. _CompilingCodeWithoutApp: + +***************************************** +Compiling the Code without an Application +***************************************** + +UFS weather model uses the cmake build system. Since UFS weather model requires several external libraries to be available and appropriate environment variables pointing to those libraries to be set, a build script named ``build.sh`` is provided In the top-level directory of the repository to ensure all necessary variables are actually set. + +The required libraries and environment variables can be set as shown in :numref:`Table %s ` and :numref:`Table %s ` for the bash shell. + +1. NCEP libraries: + +.. _ReqLibEnvVar: + +.. list-table:: *List of NCEP libraries that comprise the ufs-weather-model* + :widths: 20 80 + :header-rows: 1 + + * - NCEP Library + - Environment Variables + * - nemsio + - export NEMSIO_INC= + * - + - export NEMSIO_LIB=/libnemsio.a + * - bacio + - export BACIO_LIB4=/libbacio.a + * - splib + - export SP_LIBd=/libsp_d.a + * - w3emc + - export W3EMC_LIBd=/libw3emc_d.a + * - w3nco + - export W3NCO_LIBd=/libw3nco_d.a + +2. Third party libraries: + +.. _ReqLibEnvVar2: + +.. list-table:: *List of External libraries that comprise the ufs-weather-model* + :widths: 20 80 + :header-rows: 1 + + * - Third Party Library + - Environment Variables + * - NetCDF + - export NETCDF= + * - ESMF + - export ESMFMKFILE=/esmf.mk + +You can either manually set those environment variables in your shell, or if you are on one of the supported platforms (Tier 1) the easiest way of setting those variables is by using environment module and load all required modules. Modulefiles for all supported platforms are located in ``modulefiles//fv3``. To load those modules, for example on hera, run: + +.. code-block:: console + + $ cd modulefiles/hera.intel + $ module use $(pwd) + $ module load fv3 + $ cd ../.. + +Currently the build system supports two compiler families: GNU and Intel. The COMPILE environment variable must be set to either ‘gnu’ or ‘intel’ before running build script. For example, to use Intel compilers when using the bash shell, run: + +.. code-block:: console + + $ export COMPILER=intel + +You can further customize compiler MPI wrappers by setting these four environment variables: + +.. code-block:: console + + CMAKE_Platform : if not set the default is linux.${COMPILER} + CMAKE_C_COMPILER : if not set the default is mpicc + CMAKE_CXX_COMPILER : if not set the default is mpicxx + CMAKE_Fortran_COMPILER : if not set the default is mpif90 + +In order to have one or more CCPP physics suites available at runtime, you need to select those suites at build time by setting the ``CCPP_SUITES`` environment variable. Multiple suites can be set, as shown below in an example for the bash shell + +.. code-block:: console + + $ export CCPP_SUITES=’FV3_GFS_v15p2,FV3_GFS_v16beta’ + +If ``CCPP_SUITES`` is not set, the default is ``‘FV3_GFS_v15p2’``. + +After setting all the environment variables, you can build the model using + +.. code-block:: console + + $ ./build.sh + +Once ``build.sh`` is finished, you should see the executable, named ufs_weather_model, in the top-level directory. + + + + + diff --git a/doc/UsersGuide/source/ContributingDevelopment.rst b/doc/UsersGuide/source/ContributingDevelopment.rst new file mode 100644 index 0000000000..3dfb26fa56 --- /dev/null +++ b/doc/UsersGuide/source/ContributingDevelopment.rst @@ -0,0 +1,219 @@ +.. _ContributingDevelopment: + +************************* +Contributing Development +************************* + +The ufs-weather-model repository contains the model code and external links needed to build the Unified Forecast System (UFS) atmosphere model and associated components, including the WaveWatch III model. This weather model is used in several of the UFS applications, including the medium-range weather application, the short-range weather application, and the sub-seasonal to seasonal application. + +------------------------ + Prerequisite libraries: +------------------------ + +Below is the list of libraries that are required to compile the model and run regression tests: + ++-------------+----------------------------------------------------------------------+---------+----------------+ +| Library name| source | version | env variables | ++=============+======================================================================+=========+================+ +| bacio | https://github.com/NOAA-EMC/NCEPLIBS-bacio | 2.0.3 | $BACIO_LIB4 | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| sp | https://github.com/NOAA-EMC/NCEPLIBS-sp | 2.0.3 | $SP_LIBd | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| w3nco | https://github.com/NOAA-EMC/NCEPLIBS-w3nco | 2.0.7 | $W3NCO_LIBd | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| w3emc | https://github.com/NOAA-EMC/NCEPLIBS-w3emc | 2.3.1 | $W3EMC_LIBd | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| nemsio | https://github.com/NOAA-EMC/NCEPLIBS-nemsio | 2.2.4 | $NEMSIO_INC | ++ + + + + +| | | | $NEMSIO_LIB | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| g2 | https://github.com/NOAA-EMC/NCEPLIBS-g2 | 3.1.1 | $G2_LIB4 | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| g2tmpl | https://github.com/NOAA-EMC/NCEPLIBS-g2tmpl(required for inline post)| 1.5.1 | $G2TMPL_LIB | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| crtm | https://github.com/NOAA-EMC/EMC_crtm (required for inline post) | 2.2.6 | $CRTM_LIB | ++-------------+----------------------------------------------------------------------+---------+----------------+ +| post | http://github.com/NOAA-EMC/EMC_post (required for inline post) | 8.0.0 | $POST_INC | ++ + + + + +| | | | $POST_LIB | ++-------------+----------------------------------------------------------------------+---------+----------------+ + +External libraries: + ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| Library name| source | version | env variables | ++=============+==============================================================+=========+=========================+ +| esmf | https://www.earthsystemcog.org/projects/esmf/download_800 | 8.0.0 | $ESMFMKFILE | ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| | | 4.5.0 | | ++ + + + + +| netCDF | https://www.unidata.ucar.edu/downloads/netcdf/index.jsp | 4.6.1 | $NETCDF | ++ + + + + +| | | 4.6.3 | | ++ + + + + +| | | 4.7.0 | | ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| | | 1.8.14 | | ++ + + + + +| | | 1.8.16 | | ++ + + + + +| HDF5 | https://www.hdfgroup.org/downloads/hdf5/ | 1.10.1 | $HDF5_ROOT | ++ + + + + +| | | 1.10.4 | | ++ + + + + +| | | 1.10.5 | | ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| jasper | https://github.com/mdadams/jasper | 1.900.1 | $JASPER_ROOT | ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| zlib | https://github.com/madler/zlib | 1.2.11 | $ZLIB_ROOT | ++-------------+--------------------------------------------------------------+---------+-------------------------+ +| libpng | http://www.libpng.org/pub/png/libpng.html | 1.2.44 | $LIBPNG_ROOT | ++-------------+--------------------------------------------------------------+---------+-------------------------+ + +Compiler : + ++--------------------+------------------------------------------------------------------+ +|Compiler name | Compiler version | ++====================+==================================================================+ +| Intel | 16.0.3.210, 16.3.210, 18.0.1.163, 18.0.3.222, 18.0.5.274, 19.0.2 | ++--------------------+------------------------------------------------------------------+ +| GNU | 8.3.0, 9.1.0, 9.2.0 | ++--------------------+------------------------------------------------------------------+ + +MPI library : + ++--------------------+------------------------------------------------------------------+ +| MPI Library name | MPI library version | ++====================+==================================================================+ +| Intel iMPI | 2018.0.4, 2018.4.274,18.0.1, 2019.2.187 | ++--------------------+------------------------------------------------------------------+ +| MPICH | 3.3.1, 3.3.2 | ++--------------------+------------------------------------------------------------------+ +| Open MPI | 4.0.2 | ++--------------------+------------------------------------------------------------------+ +| SGI MPT | 2.19 | ++--------------------+------------------------------------------------------------------+ + +These libraries are available on several platforms including Hera, Cheyenne and WCOSS. Installing these libraries on jet and gaea is a work in progress. Information how to build the libraries can be found at [https://github.com/NOAA-EMC/NCEPLIBS/wiki/Cloning-and-Compiling-NCEPLIBS] + +In addition to these libraries, other software is required such as a compiler (Intel or GNU), MPI library (impi), HDF5, cmake, Python, Perl and Ruby to compile the model and run regression tests. + +--------------------------------------------- +Making code changes using a forking workflow +--------------------------------------------- + +If developers would like to make code changes, they need to make a personal fork, set up upstream remote (for merging with the original ufs-weather-model), and create a branch for ufs-weather-model and each of the subcomponent repositories they want to change. They can then make code changes, perform testing and commit the changes to the branch in their personal fork. It is suggested that they update their branch by merging the develop branch with the develop branch of the original repositories periodically to get the latest updates and bug fixes. + +If developers would like to get their code committed back to the original repository, it is suggested to follow the steps below: + +1. Create an issue in the authoritative repository. For example to commit code changes to fv3atm, please go to https://github.com/NOAA-EMC/fv3atm, under NOAA-EMC/fv3atm and find the “Issues” tab next to the “Code” tab. Click on “Issues” and a new page will appear. On the right side of the page, there is a green “New issue” button. Clicking on that will lead to a new issue page. Fill out the title, comments to describe the code changes, and also please provide personal fork and branch information. Lastly, click on the “Submit new issue” button, so that the new issue is created. + +2. When the development is mature, tests have been conducted, and the developer is satisfied with the results, create a pull request to commit the code changes. + + * Merge developer’s branch to the latest ufs-weather-model develop branch in authoritative repository. If changes are made in model sub-components, developers need to merge their branches to branches with the corresponding authoritative repository (or original repository for some components). For this, code management practices of the subcomponents need to be followed. + + * Regression tests associated with the ufs-weather-model are available on Tier 1 and Tier 2 platforms as described in https://github.com/ufs-community/ufs-weather-model/wiki/Weather-Model-Platform-and-Compiler-Support. If the developer has access to these platforms, the developer should pass the regression test on at least one supported platform. If the developer does not have access to these platforms, this should be stated in the PR so the code manager(s) can conduct the tests. + + * For each component branch where developers make changes, developers need to go to their personal fork on GitHub and click on the “New pull request” button. When a new page “Compare changes” appears, developers will choose the branch in their fork with code changes to commit and the branch in upstream repository that the changes will be committed to. Also developers in the commit comment must add the github issue title and number created in 1) in the comment box. The code differences between the two branches will be displayed. Developers can review the differences and click on “submit pull request” to make the pull request. After code changes are committed to the component repository, developers will make pull requests to ufs-weather-model repository. + +3. When PRs are created, the creator must temporarily modify .gitmodules to point to his/her fork and branch if updates are required for submodules. + +4. Merging code from PRs with submodules requires coordination with the person making the PRs. From the "innermost" nested PR up to the top-level PR, the PRs need to be merged as-is. After each merge, the person creating the PRs has to update his/her local code to check out the merged version, revert the change to .gitmodules, and push this to GitHub to update the PR. And so on and so forth. + +5. Checking out the code ufs_release_1.0 should always be as follows: + +.. code-block:: console + + git clone https://github.com/ufs-community/ufs-weather-model + cd ufs-weather-model + git checkout ufs_release_1.00 + git submodule update --init --recursive + +6. Checking out a PR with id ID for testing it should always be as follows: + +.. code-block:: console + + git clone https://github.com/ufs-community/ufs-weather-model + cd ufs-weather-model + git fetch origin pull/ID/head:BRANCHNAME + git checkout BRANCHNAME + git submodule update --init --recursive + +It is suggested that the developers inform all the related code managers as the hierarchy structure of the ufs-weather-model repository may require collaboration among the code managers. + +----------------------------------- +Engaging in the code review process +----------------------------------- + +When code managers receive a pull request to commit the code changes, it is recommended that they add at least two code reviewers to review the code and at least one of the reviewers has write permission. The reviewers will write comments about the code changes and give a recommendation as to whether the code changes can be committed. What kinds of code changes will be accepted in the repository is beyond the scope of this document; future ufs-weather-model code management documents may have a detailed answer for that. + +Reviewers may suggest some code changes during the review process. Developers need to respond to these comments in order to get code changes committed. If developers make further changes to their branch, reviewers need to check the code changes again. When both reviewers give recommendation to commit the code, code managers will merge the changes into the repository. + +---------------------------- +Conducting regression tests +---------------------------- + +Only developers using Tier 1 and 2 platforms can run the ufs-weather-model regression tests. Other developers need to work with the code managers to assure completion of the regression tests. + +To run regression test using rt.sh + +rt.sh is a bash shell file to run the RT and has the following options: + +.. code-block:: console + + Usage: ./rt.sh -c | -f | -s | -l | -m | -k | -r | -e | -h + -c create new baseline results for + -f run full suite of regression tests + -s run standard suite of regression tests + -l run test specified in + -m compare against new baseline results + -k keep run directory (automatically deleted otherwise if all tests pass) + -r use Rocoto workflow manager + -e use ecFlow workflow manager + -h display this help + +.. code-block:: console + + % cd ufs-weather-model/tests + % ./rt.sh -f + +This command can only be used on platforms that have been configured for regression testing (Tier 1 and Tier 2 platforms as described in https://github.com/ufs-community/ufs-weather-model/wiki/Weather-Model-Platform-and-Compiler-Support). For information on testing the CCPP code, or using alternate computational platforms, see the following sections. + +This command and all others below produce log output in ./tests/log_machine.compiler. These log files contain information on the location of the run directories that can be used as templates for the user. Each rt*.conf contains one or more compile commands preceding a number of tests. + +Regression test log files (ufs-weather-model/tests/Compile_$(MACHINE_ID).log and ufs-weather-model/tests/RegressionTests_$(MACHINE_ID).log ) will be updated. + +If developers wish to contribute code that changes the results of the regression tests (because of updates to the physics, for example), it is useful to run rt.sh as described above to make sure that the test failures are as expected. It is then useful to establish a new personal baseline: + +./rt.sh -l rt.conf -c # create own reg. test baseline + +Once the personal baseline has been created, future runs of the RT should be compared against the personal baseline using the -m option. + +./rt.sh -l rt.conf -m # compare against own baseline + +To create new baseline: + +.. code-block:: console + + % cd ufs-weather-model/tests + % ./rt.sh -f -c + +An alternative/complementary regression test system is using NEMSCompsetRun, which focuses more on coupled model configurations than testing features of the standalone ufs-weather-model. To run regression test using NEMSCompsetRun: + +.. code-block:: console + + % cd ufs-weather-model + % ./NEMS/NEMSCompsetRun -f + +Regression test log files (ufs-weather-model/log/$MACHINE_ID/* ) will be updated. + +To create new baseline: + +.. code-block:: console + + % cd ufs-weather-model + % ./NEMS/NEMSCompsetRun --baseline fv3 --platform=${PLATFORM} + +The value of ${PLATFORM} can be found in ufs-weather-model/compsets/platforms.input. + +Developers need to commit the regression test log files to their branch before making pull request. diff --git a/doc/UsersGuide/source/FAQforModelConfiguration.rst b/doc/UsersGuide/source/FAQforModelConfiguration.rst new file mode 100644 index 0000000000..4467732fe3 --- /dev/null +++ b/doc/UsersGuide/source/FAQforModelConfiguration.rst @@ -0,0 +1,12 @@ +.. _FAQforModelConfiguration: + +*************************** +FAQ for Model Configuration +*************************** + +* How do I add a new field to the output file? + +* What is a “cold start” and a “warm start”? + +* How do I restart the model? + diff --git a/doc/UsersGuide/source/Glossary.rst b/doc/UsersGuide/source/Glossary.rst new file mode 100644 index 0000000000..b01d440a61 --- /dev/null +++ b/doc/UsersGuide/source/Glossary.rst @@ -0,0 +1,73 @@ +.. _Glossary: + +************************* +Glossary +************************* + +.. glossary:: + + CCPP + Model agnostic, vetted, collection of codes containing atmospheric physical parameterizations + and suites for use in NWP along with a framework that connects the physics to host models + + *CCPP-Framework* + The infrastructure that connects physics schemes with a host model; also refers to a software + repository of the same name + + *CCPP-Physics* + The pool of CCPP-compliant physics schemes; also refers to a software repository of the same name + + FMS + The Flexible Modeling System (FMS) is a software framework for supporting the efficient + development, construction, execution, and scientific interpretation of atmospheric, + oceanic, and climate system models. + + NEMS + The NOAA Environmental Modeling System - a software infrastructure that supports + NCEP/EMC’s forecast products. + + NUOPC + The National Unified Operational Prediction Capability is a consortium of Navy, NOAA, + and Air Force modelers and their research partners. It aims to advance the weather + modeling systems used by meteorologists, mission planners, and decision makers. NUOPC + partners are working toward a common model architecture - a standard way of building + models - in order to make it easier to collaboratively build modeling systems. + + Parameterization or physics scheme + The representation, in a dynamic model, of physical effects in terms of admittedly + oversimplified parameters, rather than realistically requiring such effects to be + consequences of the dynamics of the system (AMS Glossary) + + Physics *cap* + Autogenerated interface between an individual physics scheme and the host model. Used + only in the dynamic CCPP build + + PROD + Compiler flags used by NCEP for operational runs of the UFS Atmosphere and by EMC for + regression tests of the code + + REPRO + Compiler flags used by EMC to guarantee reproducibility of the UFS Atmosphere code + + Suite Definition File (SDF) + An external file containing information about the + construction of a physics suite. It describes the schemes that are called, in which + order they are called, whether they are subcycled, and whether they are assembled + into groups to be called together + + Suite + A collection of primary physics schemes and interstitial schemes that are known to work + well together + + UFS + A Unified Forecast System (UFS) is a community-based, coupled comprehensive Earth + system modeling system. The UFS numerical applications span local to global domains + and predictive time scales from sub-hourly analyses to seasonal predictions. It is + designed to support the Weather Enterprise and to be the source system for NOAA's + operational numerical weather prediction applications + + Weather Model + A prognostic model that can be used for short- and medium-range research and + operational forecasts. It can be an atmosphere-only model or be an atmospheric + model coupled with one or more additional components, such as a wave or ocean model. + diff --git a/doc/UsersGuide/source/InputsOutputs.rst b/doc/UsersGuide/source/InputsOutputs.rst new file mode 100644 index 0000000000..dea485a64e --- /dev/null +++ b/doc/UsersGuide/source/InputsOutputs.rst @@ -0,0 +1,735 @@ +.. _InputsOutputs: + +************************* +Inputs and Outputs +************************* + +This chapter describes the input and output files needed for executing the model in the various supported configurations. + +============= +Input files +============= + +There are three types of files needed to execute a run: static datasets (*fix* files containing climatological +information), files that depend on grid resolution and initial conditions, and model configuration files (such as namelists). + +------------------------------------ +Static datasets (i.e., *fix files*) +------------------------------------ +The static input files are listed and described in :numref:`Table %s `. + +.. _FixFiles: + +.. list-table:: *Fix files containing climatological information* + :widths: 40 50 + :header-rows: 1 + + * - Filename + - Description + * - aerosol.dat + - External aerosols data file + * - CFSR.SEAICE.1982.2012.monthly.clim.grb + - CFS reanalysis of monthly sea ice climatology + * - co2historicaldata_YYYY.txt + - Monthly CO2 in PPMV data for year YYYY + * - global_albedo4.1x1.grb + - Four albedo fields for seasonal mean climatology: 2 for strong zenith angle dependent (visible and near IR) + and 2 for weak zenith angle dependent + * - global_glacier.2x2.grb + - Glacier points, permanent/extreme features + * - global_h2oprdlos.f77 + - Coefficients for the parameterization of photochemical production and loss of water (H2O) + * - global_maxice.2x2.grb + - Maximum ice extent, permanent/extreme features + * - global_mxsnoalb.uariz.t126.384.190.rg.grb + - Climatological maximum snow albedo + * - global_o3prdlos.f77 + - Monthly mean ozone coefficients + * - global_shdmax.0.144x0.144.grb + - Climatological maximum vegetation cover + * - global_shdmin.0.144x0.144.grb + - Climatological minimum vegetation cover + * - global_slope.1x1.grb + - Climatological slope type + * - global_snoclim.1.875.grb + - Climatological snow depth + * - global_snowfree_albedo.bosu.t126.384.190.rg.grb + - Climatological snowfree albedo + * - global_soilmgldas.t126.384.190.grb + - Climatological soil moisture + * - global_soiltype.statsgo.t126.384.190.rg.grb + - Soil type from the STATSGO dataset + * - global_tg3clim.2.6x1.5.grb + - Climatological deep soil temperature + * - global_vegfrac.0.144.decpercent.grb + - Climatological vegetation fraction + * - global_vegtype.igbp.t126.384.190.rg.grb + - Climatological vegetation type + * - global_zorclim.1x1.grb + - Climatological surface roughness + * - RTGSST.1982.2012.monthly.clim.grb + - Monthly, climatological, real-time global sea surface temperature + * - seaice_newland.grb + - High resolution land mask + * - sfc_emissivity_idx.txt + - External surface emissivity data table + * - solarconstant_noaa_an.txt + - External solar constant data table + +--------------------------------------------- +Grid description and initial condition files +--------------------------------------------- +The input files containing grid information and the initial conditions are listed and described in :numref:`Table %s `. + +.. _GridICFiles: + +.. list-table:: *Input files containing grid information and initial conditions* + :widths: 35 50 15 + :header-rows: 1 + + * - Filename + - Description + - Date-dependent + * - C96_grid.tile[1-6].nc + - C96 grid information for tiles 1-6 + - + * - gfs_ctrl.nc + - NCEP NGGPS tracers, ak, and bk + - ✔ + * - gfs_data.tile[1-6].nc + - Initial condition fields (ps, u, v, u, z, t, q, O3). May include spfo3, spfo, spf02 if multiple gases are used + - ✔ + * - oro_data.tile[1-6].nc + - Model terrain (topographic/orographic information) for grid tiles 1-6 + - + * - sfc_ctrl.nc + - Control parameters for surface input: forecast hour, date, number of soil levels + - + * - sfc_data.tile[1-6].nc + - Surface properties for grid tiles 1-6 + - ✔ + +------------------------------------ +Model configuration files +------------------------------------ +The configuration files used by the UFS Weather Model are listed here and described below: + +- *diag_table* +- *field_table* +- *input.nml* +- *model_configure* +- *nems.configure* +- *suite_[suite_name].xml* (used only at build time) + + +*diag_table* file +------------------------------------ +There are three sections in file *diag_table*: Header (Global), File, and Field. These are described below. + +**Header Description** + +The Header section must reside in the first two lines of the *diag_table* file and contain the title and date +of the experiment (see example below). The title must be a Fortran character string. The base date is the +reference time used for the time units, and must be greater than or equal to the model start time. The base date +consists of six space-separated integers in the following format: ``year month day hour minute second``. Here is an example: + +.. code-block:: console + + 20161003.00Z.C96.64bit.non-mono + 2016 10 03 00 0 0 + +**File Description** + +The File Description lines are used to specify the name of the file(s) to which the output will be written. They +contain one or more sets of six required and five optional fields (optional fields are denoted by square brackets +``[ ]``). The lines containing File Descriptions can be intermixed with the lines containing Field Descriptions as +long as files are defined before fields that are to be written them. File entries have the following format: + +.. code-block:: console + + "file_name", output_freq, "output_freq_units", file_format, "time_axis_units", "time_axis_name" + [, new_file_freq, "new_file_freq_units"[, "start_time"[, file_duration, "file_duration_units"]]] + +These file line entries are described in :numref:`Table %s `. + +.. _FileDescription: + +.. list-table:: *Description of the six required and five optional fields used to define output file sampling rates.* + :widths: 20 25 55 + :header-rows: 1 + + * - File Entry + - Variable Type + - Description + * - file_name + - CHARACTER(len=128) + - Output file name without the trailing ".nc" + * - output_freq + - INTEGER + - | The period between records in the file_name: + | > 0 output frequency in output_freq_units. + | = 0 output frequency every time step (output_freq_units is ignored) + | =-1 output at end of run only (output_freq_units is ignored) + * - output_freq_units + - CHARACTER(len=10) + - The units in which output_freq is given. Valid values are “years”, “months”, “days”, “minutes”, “hours”, or “seconds”. + * - file_format + - INTEGER + - Currently only the netCDF file format is supported. = 1 netCDF + * - time_axis_units + - CHARACTER(len=10) + - The units to use for the time-axis in the file. Valid values are “years”, “months”, “days”, “minutes”, “hours”, + or “seconds”. + * - time_axis_name + - CHARACTER(len=128) + - Axis name for the output file time axis. The character string must contain the string 'time'. + (mixed upper and lowercase allowed.) + * - new_file_freq + - INTEGER, OPTIONAL + - Frequency for closing the existing file, and creating a new file in new_file_freq_units. + * - new_file_freq_units + - CHARACTER(len=10), OPTIONAL + - Time units for creating a new file: either years, months, days, minutes, hours, or seconds. + NOTE: If the new_file_freq field is present, then this field must also be present. + * - start_time + - CHARACTER(len=25), OPTIONAL + - Time to start the file for the first time. The format of this string is the same as the global date. + NOTE: The new_file_freq and the new_file_freq_units fields must be present to use this field. + * - file_duration + - INTEGER, OPTIONAL + - How long file should receive data after start time in file_duration_units. This optional field can only be + used if the start_time field is present. If this field is absent, then the file duration will be equal to the + frequency for creating new files. NOTE: The file_duration_units field must also be present if this field is present. + * - file_duration_units + - CHARACTER(len=10), OPTIONAL + - File duration units. Can be either years, months, days, minutes, hours, or seconds. NOTE: If the file_duration field + is present, then this field must also be present. + +**Field Description** + +The field section of the diag_table specifies the fields to be output at run time. Only fields registered +with ``register_diag_field()``, which is an API in the FMS ``diag_manager`` routine, can be used in the *diag_table*. + +Registration of diagnostic fields is done using the following syntax + +.. code-block:: console + + diag_id = register_diag_field(module_name, diag_name, axes, ...) + +in file ``FV3/atmos_cubed_sphere/tools/fv_diagnostics.F90``. As an example, the sea level pressure is registered as: + +.. code-block:: console + + id_slp = register_diag_field (trim(field), 'slp', axes(1:2), & Time, 'sea-level pressure', 'mb', missing_value=missing_value, range=slprange ) + +All data written out by ``diag_manager`` is controlled via the *diag_table*. A line in the field section of the +*diag_table* file contains eight variables with the following format: + +.. code-block:: console + + "module_name", "field_name", "output_name", "file_name", "time_sampling", "reduction_method", "regional_section", packing + +These field section entries are described in :numref:`Table %s `. + +.. _FieldDescription: + +.. list-table:: *Description of the eight variables used to define the fields written to the output files.* + :widths: 16 24 55 + :header-rows: 1 + + * - Field Entry + - Variable Type + - Description + * - module_name + - CHARACTER(len=128) + - Module that contains the field_name variable. (e.g. dynamic, gfs_phys, gfs_sfc) + * - field_name + - CHARACTER(len=128) + - The name of the variable as registered in the model. + * - output_name + - CHARACTER(len=128) + - Name of the field as written in file_name. + * - file_name + - CHARACTER(len=128) + - Name of the file where the field is to be written. + * - time_sampling + - CHARACTER(len=50) + - Currently not used. Please use the string "all". + * - reduction_method + - CHARACTER(len=50) + - The data reduction method to perform prior to writing data to disk. Current supported option is .false.. See ``FMS/diag_manager/diag_table.F90`` for more information. + * - regional_section + - CHARACTER(len=50) + - Bounds of the regional section to capture. Current supported option is “none”. See ``FMS/diag_manager/diag_table.F90`` for more information. + * - packing + - INTEGER + - Fortran number KIND of the data written. Valid values: 1=double precision, 2=float, 4=packed 16-bit integers, 8=packed 1-byte (not tested). + +Comments can be added to the diag_table using the hash symbol (``#``). + +A brief example of the diag_table is shown below. ``“...”`` denote where lines have been removed. + +.. code-block:: console + + 20161003.00Z.C96.64bit.non-mono + 2016 10 03 00 0 0 + + "grid_spec", -1, "months", 1, "days", "time" + "atmos_4xdaily", 6, "hours", 1, "days", "time" + "atmos_static" -1, "hours", 1, "hours", "time" + "fv3_history", 0, "hours", 1, "hours", "time" + "fv3_history2d", 0, "hours", 1, "hours", "time" + + # + #======================= + # ATMOSPHERE DIAGNOSTICS + #======================= + ### + # grid_spec + ### + "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, + "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, + ### + # 4x daily output + ### + "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 + "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 + + ... + ### + # gfs static data + ### + "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 + "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 + "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 + "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 + "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 + ### + # FV3 variables needed for NGGPS evaluation + ### + "gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 + "gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 + "gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 + "gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 + ... + "gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 + "gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 + "gfs_phys", "cnvprcpb_ave", "cpratb_ave","fv3_history2d", "all", .false., "none", 2 + "gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 + ... + "gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 + "gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 + "gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 + "gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 + "gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 + ... + +More information on the content of this file can be found in ``FMS/diag_manager/diag_table.F90``. + +.. note:: None of the lines in the *diag_table* can span multiple lines. + +*field_table* file +------------------------------------ +The FMS field and tracer managers are used to manage tracers and specify tracer options. All tracers +advected by the model must be registered in an ASCII table called *field_table*. The field table consists +of entries in the following format: + +The first line of an entry should consist of three quoted strings: + - The first quoted string will tell the field manager what type of field it is. The string ``“TRACER”`` is used to + declare a field entry. + - The second quoted string will tell the field manager which model the field is being applied to. The supported + type at present is ``“atmos_mod”`` for the atmosphere model. + - The third quoted string should be a unique tracer name that the model will recognize. + +The second and following lines are called ``methods``. These lines can consist of two or three quoted strings. +The first string will be an identifier that the querying module will ask for. The second string will be a name +that the querying module can use to set up values for the module. The third string, if present, can supply +parameters to the calling module that can be parsed and used to further modify values. + +An entry is ended with a forward slash (/) as the final character in a row. Comments can be inserted in the field table by having a hash symbol (#) as the first character in the line. + +Below is an example of a field table entry for the tracer called ``“sphum”``: + +.. code-block:: console + + # added by FRE: sphum must be present in atmos + # specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=3.e-6" / + +In this case, methods applied to this tracer include setting the long name to "specific humidity", the units +to "kg/kg". Finally a field named "profile_type" will be given a child field called "fixed", and that field +will be given a field called "surface_value" with a real value of 3.E-6. The “profile_type” options are listed +in :numref:`Table %s `. If the profile type is “fixed” then the tracer field values are set equal +to the surface value. If the profile type is “profile” then the top/bottom of model and surface values are read +and an exponential profile is calculated, with the profile being dependent on the number of levels in the component model. + +.. _TracerTable: + +.. list-table:: *Tracer profile setup from FMS/tracer_manager/tracer_manager.F90.* + :widths: 20 25 55 + :header-rows: 1 + + * - Method Type + - Method Name + - Method Control + * - profile_type + - fixed + - surface_value = X + * - profile_type + - profile + - surface_value = X, top_value = Y (atmosphere) + +For the case of + +.. code-block:: console + + "profile_type","profile","surface_value = 1e-12, top_value = 1e-15" + +in a 15 layer model this would return values of surf_value = 1e-12 and multiplier = 0.6309573, i.e 1e-15 = 1e-12*(0.6309573^15). + +A ``method`` is a way to allow a component module to alter the parameters it needs for various tracers. In essence, +this is a way to modify a default value. A namelist can supply default parameters for all tracers and a method, as +supplied through the field table, will allow the user to modify the default parameters on an individual tracer basis. +The lines in this file can be coded quite flexibly. Due to this flexibility, a number of restrictions are required. +See ``FMS/field_manager/field_manager.F90`` for more information. + +*input.nml* file +------------------------------------ + +The atmosphere model reads many parameters from a Fortran namelist file, named *input.nml*. This file contains +several Fortran namelist records, some of which are always required, others of which are only used when selected +physics options are chosen. + +The following link describes the various physics-related namelist records: +https://dtcenter.org/GMTB/v3.0/sci_doc/GFSsuite_nml.html + +The following link describes the stochastic physics namelist records +https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html + +The following link describes some of the other namelist records (dynamics, grid, etc): +https://www.gfdl.noaa.gov/wp-content/uploads/2017/09/fv3_namelist_Feb2017.pdf + +The namelist section relating to the FMS diagnostic manager is described in the last section of this chapter. + +*model_configure* file +------------------------------------ + +This file contains settings and configurations for the NUOPC/ESMF main component, including the simulation +start time, the processor layout/configuration, and the I/O selections. :numref:`Table %s ` +shows the following parameters that can be set in *model_configure* at run-time. + +.. _ModelConfigParams: + +.. list-table:: *Parameters that can be set in model_configure at run-time.* + :widths: 20 30 15 20 + :header-rows: 1 + + * - Parameter + - Meaning + - Type + - Default Value + * - print_esmf + - flag for ESMF PET files + - logical + - .true. + * - PE_MEMBER01 + - total number of tasks for ensemble number 1 + - integer + - 150 (for c96 with quilt) + * - start_year + - start year of model integration + - integer + - 2019 + * - start_month + - start month of model integration + - integer + - 09 + * - start_day + - start day of model integration + - integer + - 12 + * - start_hour + - start hour of model integration + - integer + - 00 + * - start_minute + - start minute of model integration + - integer + - 0 + * - start_second + - start second of model integration + - integer + - 0 + * - fhrot + - forecast hour at restart for nems/earth grid component clock in coupled model + - integer + - 0 + * - nhours_fcst + - total forecast length + - integer + - 48 + * - dt_atmos + - atmosphere time step in second + - integer + - 1800 (for C96) + * - output_1st_tstep_rst + - output first time step history file after restart + - logical + - .false. + * - memuse_verbose + - flag for printing out memory usage + - logical + - .false. + * - atmos_nthreads + - number of threads for atmosphere + - integer + - 4 + * - use_hyper_thread + - flag to use hyper threads + - logical + - .false. + * - ncores_per_node + - number of cores per node + - integer + - 24 + * - restart_interval + - frequency to output restart file + - integer + - 0 (write restart file at the end of integration) + * - quilting + - flag to turn on quilt + - logical + - .true. + * - write_groups + - total number of groups + - integer + - 2 + * - write_tasks_per_group + - total number of write tasks in each write group + - integer + - 6 + * - output_history + - flag to output history files + - logical + - .true. + * - num_files + - number of output files + - integer + - 2 + * - filename_base + - file name base for the output files + - character(255) + - 'atm' 'sfc' + * - output_grid + - output grid + - character(255) + - gaussian_grid + * - output_file + - output file format + - character(255) + - nemsio + * - imo + - i-dimension for output grid + - integer + - 384 + * - jmo + - j-dimension for output grid + - integer + - 190 + * - nfhout + - history file output frequency + - integer + - 3 + * - nfhmax_hf + - forecast length of high history file + - integer + - 0 (0:no high frequency output) + * - nfhout_hf + - high history file output frequency + - integer + - 1 + * - nsout + - output frequency of number of time step + - integer + - -1 (negative: turn off the option, 1: output history file at every time step) + +:numref:`Table %s ` shows the following parameters in *model_configure* that +are not usually changed. + +.. _ModelConfigParamsNotChanged: + +.. list-table:: *Parameters that are not usually changed in model_configure at run-time.* + :widths: 20 30 15 20 + :header-rows: 1 + + * - Parameter + - Meaning + - Type + - Default Value + * - total_member + - total number of ensemble member + - integer + - 1 + * - RUN_CONTINUE + - Flag for more than one NEMS run + - logical + - .false. + * - ENS_SPS + - flag for the ensemble stochastic coupling flag + - logical + - .false. + * - calendar + - type of calendar year + - character(*) + - 'julian' + * - cpl + - flag for coupling with MOM6/CICE5 + - logical + - .false. + * - write_dopost + - flag to do post on write grid component + - logical + - .false. + * - ideflate + - lossless compression level + - integer + - 1 (0:no compression, range 1-9) + * - nbits + - lossy compression level + - integer + - 14 (0: lossless, range 1-32) + * - write_nemsioflip + - flag to flip the vertical level for nemsio file + - logical + - .true. + * - write_fsyncflag + - flag to check if a file is synced to disk + - logical + - .true. + * - iau_offset + - IAU offset lengdth + - integer + - 0 + +*nems.configure* file +------------------------------------ +This file contains information about the various NEMS components and their run sequence. In the current release, +this is an atmosphere-only model, so this file is simple and does not need to be changed. A sample of the file contents is below: + +.. code-block:: console + + EARTH_component_list: ATM + ATM_model: fv3 + runSeq:: + ATM + :: + +*The SDF (Suite Definition File) file* +--------------------------------------- +There are two SDFs currently supported: *suite_FV3_GFS_v15p2.xml* and *suite_FV3_GFS_v16beta.xml*. + +============= +Output files +============= + +The following files are output when running *fv3.exe* in the default configuration (six files of each kind, +corresponding to the six tiles of the model grid): + +- *atmos_4xdaily.tile[1-6].nc* +- *atmos_static.tile[1-6].nc* +- *phyfHHH.tile[1-6].nc* +- *dynfHHH.tile[1-6].nc* +- *grid_spec.tile[1-6].nc* + +The specifications of the output files (type, projection, etc) may be overridden in the *model_configure* input file. + +Standard output files are *logf???*, and out and err as specified by the job submission. ESMF may also produce log +files (controlled by variable print_esmf in the *model_configure* file), called *PET???.ESMF_LogFile*. + +============================================================== +Additional Information about the FMS Diagnostic Manager +============================================================== + +The UFS Weather Model output is managed through the FMS (Flexible Modeling System) diagnostic manager (``FMS/diag_manager``) +and is configured using the *diag_table* file. Data can be written at any number of sampling and/or averaging intervals +specified at run-time. More information about the FMS diagnostic manager can be found at: +https://data1.gfdl.noaa.gov/summer-school/Lectures/July16/03_Seth1_DiagManager.pdf + +------------------------------ +Diagnostic Manager namelist +------------------------------ +The ``diag_manager_nml`` namelist contains values to control the behavior of the diagnostic manager. Some +of the more common namelist options are described in :numref:`Table %s `. See +``FMS/diag_manager/diag_manager.F90`` for the complete list. + +.. _DiagManager: + +.. list-table:: *Namelist variables used to control the behavior of the diagnostic manager.* + :widths: 15 10 30 10 + :header-rows: 1 + + * - Namelist variable + - Type + - Description + - Default value + * - max_files + - INTEGER + - Maximum number of files allowed in diag_table + - 31 + * - max_output_fields + - INTEGER + - Maximum number of output fields allowed in diag_table + - 300 + * - max_input_fields + - INTEGER + - Maximum number of registered fields allowed + - 300 + * - prepend_date + - LOGICAL + - Prepend the file start date to the output file. .TRUE. is only supported if the diag_manager_init routine is called with the optional time_init parameter. + - .TRUE. + * - do_diag_field_log + - LOGICAL + - Write out all registered fields to a log file + - .FALSE. + * - use_cmor + - LOGICAL + - Override the missing_value to the CMOR value of -1.0e20 + - .FALSE. + * - issue_oor_warnings + - LOGICAL + - Issue a warning if a value passed to diag_manager is outside the given range + - .TRUE. + * - oor_warnings_fatal + - LOGICAL + - Treat out-of-range errors as FATAL + - .FALSE. + * - debug_diag_manager + - LOGICAL + - Check if the diag table is set up correctly + - .FALSE. + +This release of the UFS Weather Model uses the following namelist: + +.. code-block:: console + + &diag_manager_nml + prepend_date = .false. + / + diff --git a/doc/UsersGuide/source/Introduction.rst b/doc/UsersGuide/source/Introduction.rst new file mode 100644 index 0000000000..ee22b8a0a7 --- /dev/null +++ b/doc/UsersGuide/source/Introduction.rst @@ -0,0 +1,100 @@ +.. _Introduction: + +************************* +Introduction +************************* + +The Unified Forecast System (:term:`UFS`) :term:`Weather Model` (WM) is a prognostic model that can be +used for short- and medium-range research and operational forecasts, as exemplified by +its use in the operational Global Forecast System (GFS) of the National Oceanic and +Atmospheric Administration (NOAA). The UFS WM v1.0 is the first public release of this +software and represents a snapshot of a continuously evolving system undergoing open +development. More information about the UFS can be found in its portal at https://ufscommunity.org/. + +Key architectural elements of the UFS WM, along with links to external detailed documentation +for those elements, are listed below: + +- The Finite-Volume Cubed-Sphere (FV3) dynamical core (https://noaa-emc.github.io/FV3_Dycore/html/index.html). + +- The Flexible Modeling System (:term:`FMS`), a software infrastructure used for functions such as + parallelization (https://www.gfdl.noaa.gov/fms/). + +- The Common-Community Physics Package (:term:`CCPP`) library of + physical parameterizations (:term:`Parameterization or physics scheme`) and the + framework to use it with the model + (https://dtcenter.org/community-code/common-community-physics-package-ccpp). + +- The stochastic physics capability, including the Stochastic Kinetic Backscatter Scheme (SKEBS), + the Stochastically Perturbed Parameterization Tendencies (SPPT) scheme, the perturbed boundary + layer humidity (SHUM) scheme, and the cellular automata method + (https://stochastic-physics.readthedocs.io/en/ufs_public_release/). + +- The NOAA Environmental Modeling System (:term:`NEMS`) model driver used to create the main program + (https://docs.google.com/document/d/1-kFhPBf7GBTUd5SaB5D_3OUGX_93pWKP21QgTh4y6ok/edit#heading=h.dah4y9bxn10l). + +- The libraries needed to build the system, such as: + - National Centers for Environmental Prediction (NCEP) Libraries + - Earth System Modeling Framework (ESMF; https://www.earthsystemcog.org/projects/esmf/) + - System libraries + +- The build system used to compile the code and generate the executable. + +- The regression tests used to maintain software integrity as innovations are added. + +For the UFS WM v1.0 release, the following aspects are supported: + +- Global configuration with resolutions of C96 (~100 km), C192 (~50 km), C384 (25 km), and C768 (~13 km) + +- Sixty-four vertical levels at predetermined locations. + +- Two physics suites (:term:`suite`), corresponding to GFS v15.2 (operational at the time of the release) and + GFS v16beta (October 2019 version, in preparation for operational implementation in 2021). + +- Ability to run with or without SKEBS, SPPT, and SHUM. + +- Ability to initialize from GFS files in Gridded Binary v2 (GRIB2) format for past dates, + starting January 1, 2018, when the preprocessing utility chgres is employed. Dates before + that may work, but are not guaranteed. + +- Output files in Network Common Data Form (NetCDF) format. + +The GFS v15.2 physics suite uses the following physical parameterization: the Simplified Arakawa Schubert shallow and deep convective schemes, the Geophysical Fluid Dynamics Laboratory (GFDL) microphysics scheme, the Noah Land Surface Model (LSM), the Rapid Radiative Transfer Model for Global Circulation Models (RRTMG) radiation scheme, the hybrid eddy-diffusivity mass-flux (EDMF) planetary boundary layer (PBL) scheme based on the Smagorinsky K theory, and an orographic gravity wave drag (GWD) parameterization. In the GFS v16beta suite, a moist TKE-based EDMF scheme replaces the K-based one and a non-stationary GWD parameterization is added. It should be noted that the public release code reflects the state of development as of the fall of 2019, and therefore the parameterizations contains innovations beyond what is currently in operations. In other words, the GFDL microphysics distributed for use in GFS v15.2 and GFS v16beta is the same scheme and contains development beyond what was transitioned to operations for GFS v15 in June 2019. It should also be noted that further changes are expected to the GFS v16 suite before it is implemented in operations in 2021. + +The UFS WM v1 code is portable and can be used with Linux and Mac operating systems with Intel and GNU compilers. It has been tested in a variety of platforms widely used by atmospheric scientists, such as the NOAA research Hera system, the National Center for Atmospheric Research (NCAR) Cheyenne system, the National Science Foundation Stampede system, and Mac laptops. + +.. note:: + + At this time, the following aspects are unsupported: standalone regional domains, configurations in which a mediator is used to couple the atmospheric model to models of other earth domains (such as ocean, ice, and waves), horizontal resolutions other than the supported ones, different number or placement of vertical levels, physics suites other than GFS v15.2 and GFS v16beta the *cellular automata* stochastic scheme, initialization from sources other than GFS, the use of different file formats for input and output, and the use of the model in different computational platforms. It is expected that the UFS WM supported capabilities will be expanded in future releases. + +It should be noted that the UFS WM is a component of the UFS Medium-Range Weather Application, which also contains pre- and post-processing components, a comprehensive build system, and workflows for configuration and execution of the application. At this time, the UFS WM is only supported to the general community for use as part of the UFS Medium-Range Weather Application. However, those wishing to contribute development to the UFS WM should become familiar with the procedures for running the model as a standalone component and for executing the regression tests to make sure no inadvertent changes to the results have been introduced during the development process. + +Support for the UFS WM is provided through the UFS Forum by the Developmental Testbed Center (DTC) and other groups involved in UFS development, such as NOAA’s Environmental Modeling Center (EMC), NOAA research laboratories (GFDL, NSSL, ESRL, and AOML), and NCAR. UFS users and developers are encouraged not only to post questions, but also to help address questions posted by other members of the community. + +This WM User’s Guide is organized as follows: + +- :numref:`Chapter %s ` (Code Overview) provides a description of the various + code repositories from which source code is pulled and an overview of the directory structure. + +- :numref:`Chapter %s ` (Inputs and Outputs) lists the model inputs and outputs + and has a description of the key files. + +- :numref:`Chapter %s ` (SDF and namelist samples and best practices) + contains a description of the :term:`Suite Definition File (SDF)` and namelists needed to configure the model + for running with the GFS v15.2 and GFS v16beta physics suites. + +- :numref:`Chapter %s ` (FAQ on model configuration) contains information on + miscellaneous topics pertaining to using the model in configurations that differ from the default. + +The next three chapters: + +- :numref:`Chapter %s ` (Contributing development) +- :numref:`Chapter %s ` (Compiling the WM code without an application) +- :numref:`Chapter %s ` (Regression tests for development) + +go beyond the capabilities supported in the public release to cover code management for conducting +development and proposing contributions back to the authoritative code repositories. It should be noted that the regression tests described here are mandatory for committing code back to the ufs-weather-model authoritative code repository. These regressions tests differ from those distributed with the workflows for UFS applications, which are intended for application users and developers to assess the quality of their installations and the impact of their code changes. Finally, +:numref:`Chapters %s ` and :numref:`%s ` contain a list of acronyms and a glossary, respectively. + +.. This is how you cite a reference :cite:`Bernardet2018`. + +.. bibliography:: references.bib diff --git a/doc/UsersGuide/source/RegTests.rst b/doc/UsersGuide/source/RegTests.rst new file mode 100644 index 0000000000..6f607abb1d --- /dev/null +++ b/doc/UsersGuide/source/RegTests.rst @@ -0,0 +1,245 @@ +.. _RegTests: + +****************************************** +Regression Tests for Development +****************************************** + +* Motivation + +* Description of the default tests + +* Directory structure for inputs and outputs + +* How to run a subset of tests + +* Baselines: existing baselines, creating new baselines, comparing against a new baseline + +* Running regression tests + +* Checking the results + +* Adding a regression test + + +System Requirements, Libraries, and Compilers +--------------------------------------------- +The build system for the UFS with CCPP relies on the use of the Python scripting language, along with ``cmake``. + +The basic requirements for building and running the UFS with CCPP are listed below. The versions listed reflect successful tests and there is no guarantee that the code will work with different versions. + + * FORTRAN 90+ compiler versions + * ifort 15.1.133, 18.0.1.163 and 19.0.2 + * gfortran 6.2, 8.1, and 9.1 + * C compiler versions + * icc v18.0.1.163 and 19.0.2 + * gcc 6.2.0 and 8.1 + * AppleClang 10.0 + * MPI job scheduler versions + * mpt 2.19 + * impi 5.1.1.109 and 5.1.2.150 + * mpich 3.2.1 + * cmake versions 2.8.12.1, 2.8.12.2, and 3.6.2 + * netCDF with HDF5, ZLIB and SZIP versions 4.3.0, 4.4.0, 4.4.1.1, 4.5.0, 4.6.1, and 4.6.3 (not 3.x) + * Python versions 2.7.5, 2.7.9, and 2.7.13 (not 3.x) + +A number of NCEP libraries are required to build and run FV3 and are listed in :numref:`Table %s `. + +.. _NCEP_lib_FV3: + +.. table:: *NCEP libraries required to build the UFS Atmosphere* + + +---------------------------+-------------+----------------------------------------------------+ + | Library | Version | Description | + +===========================+=============+====================================================+ + | bacio | 2.0.1 | NCEP binary I/O library | + +---------------------------+-------------+----------------------------------------------------+ + | ip | 2.0.0/3.0.0 | NCEP general interpolation library | + +---------------------------+-------------+----------------------------------------------------+ + | nemsio | 2.2.3 | NEMS I/O routines | + +---------------------------+-------------+----------------------------------------------------+ + | sp | 2.0.2 | NCEP spectral grid transforms | + +---------------------------+-------------+----------------------------------------------------+ + | w3emc | 2.2.0 | NCEP/EMC library for decoding data in GRIB1 format | + +---------------------------+-------------+----------------------------------------------------+ + | w3nco/v2.0.6 | 2.0.6 | NCEP/NCO library for decoding data in GRIB1 format | + +---------------------------+-------------+----------------------------------------------------+ + +These libraries are prebuilt on most NOAA machines using the Intel compiler. For those needing to build the libraries themselves, GMTB recommends using the source code from GitHub at https://github.com/NCAR/NCEPlibs.git, which includes build files for various compilers and machines using OpenMP flags and which are thread-safe. + +In addition to the NCEP libraries, some additional external libraries are needed (:numref:`Table %s `). + +.. _ext_lib_FV3: + +.. table:: *External libraries necessary to build the UFS Atmosphere* + + +--------------------+-------------------------+---------------------------------------------------------------------------------------------+ + | Library | Version | Description | + +====================+=========================+=============================================================================================+ + | ESMF | V7.1.0r and v8.0.0_bs21 | Earth System Modeling Framework for coupling applications | + +--------------------+-------------------------+---------------------------------------------------------------------------------------------+ + | netCDF | 4.3.0 and 4.6.1 | Interface to data access functions for storing and retrieving data arrays | + +--------------------+-------------------------+---------------------------------------------------------------------------------------------+ + | SIONlib (optional) | v1.7.2 | Parallel I/O library (link) that can be used to read precomputed lookup tables instead of \ | + | | | computing them on the fly (or using traditional Fortran binary data files) | + +--------------------+-------------------------+---------------------------------------------------------------------------------------------+ + +The Earth System Modeling Framework (ESMF), the SIONlib, the NCEPlibs, and the netCDF libraries must be built with the same compiler as the other components of the UFS Atmosphere. + +Building the UFS Atmosphere +--------------------------- + +A complete listing and description of the FV3 build options were discussed elsewhere. This section will describe the commands needed to build the different options using the script ``compile.sh`` provided in the NEMSfv3gfs distribution. This script calls ``ccpp_prebuild.py``, so users do not need to run the *prebuild* step manually. All builds using ``compile.sh`` are made from the ``./tests`` directory of NEMSfv3gfs and follow the basic command: + +.. code-block:: console + + ./compile.sh $PWD/../FV3 system.compiler 'MAKEOPTS' + +Here, ``system`` stands for the machine on which the code is compiled and can be any of the following machines and compilers: *theia, jet, cheyenne, gaea, stampede, wcoss_cray, wcoss_dell_p3, supermuc_phase2, macosx*, or *linux*. + +``compiler`` stands for the compiler to use and depends on the system. For *theia* and *cheyenne*, the available options are ``intel`` and ``gnu``. For *macosx* and *linux*, the only tested compiler is ``gnu``. For all other platforms, ``intel`` is the only option at this time. + +The ``MAKEOPTS`` string, enclosed in single or double quotes, allows to specify options for compiling the code. The following options are of interest for building the CCPP version of NEMSfv3gfs: + +* **CCPP=Y** - enables :term:`CCPP` (default is ``N``) +* **STATIC=Y** - enables the CCPP static mode; requires ``CCPP=Y`` (default is ``N``) and ``SUITES=...`` (see below) +* **SUITES=XYZ, ABC, DEF, ...** - specify SDF(s) to use when compiling the code in CCPP static mode; SDFs are located in ``ccpp/suites/``, omit the path in the argument; requires ``CCPP=Y STATIC=Y`` (default is ``‘’``) +* **SION=Y** - enables support for the SIONlib I/O library (used by CCPP to read precomputed lookup tables instead of computing them on the fly); available on *Theia, Cheyenne, Jet*; also available on *Mac OS X* and *Linux* if instructions in ``doc/README_{macosx,linux}.txt`` are followed (default is ``N``) +* **32BIT=Y** - compiles FV3 dynamical core in single precision; note that physics are always compiled in double precision; this option is only available on *Theia, Cheyenne*, and *Jet* (default is ``N``) +* **REPRO=Y** - compiles code in :term:`REPRO` mode, i.e. removes certain compiler optimization flags used in the default :term:`PROD` mode to obtain bit-for-bit (b4b) identical results between CCPP and non-CCPP code (default is ``N``) +* **DEBUG=Y** - compiles code in DEBUG mode, i.e. removes all optimization of :term:`PROD` mode and add bound checks; mutually exclusive with ``REPRO=Y`` (default is ``N``) +* **TRANSITION=Y** - applies selective lowering of optimization for selected files to obtain b4b with non-CCPP code in PROD mode (only when using Intel 15 on *Theia*) + +Examples: + +* Compile non-CCPP code with 32-bit dynamics on *Theia* with the Intel compiler + + .. code-block:: console + + ./compile.sh $PWD/../FV3 theia.intel ‘32BIT=Y’ + +* Compile dynamic CCPP code in ``DEBUG`` mode on *Jet* + + .. code-block:: console + + ./compile.sh $PWD/../FV3 jet.intel ‘CCPP=Y DEBUG=Y’ + +* Compile static CCPP code for the CPT suite on *Linux* with the GNU compiler, enable support for the SIONlib I/O library (requires that the library to be installed) + + .. code-block:: console + + ./compile.sh $PWD/../FV3 linux.gnu ‘SION=Y CCPP=Y STATIC=Y SUITES=FV3_CPT_v0’ + +* *Cheyenne* static build with multiple suites: + + .. code-block:: console + + ./compile.sh $PWD/../FV3 cheyenne.intel ‘CCPP=Y STATIC=Y SUITES=FV3_GFS_v15,FV3_CPT_v0’ + + +Running the UFS Atmosphere Using the Regression Tests (RTs) +------------------------------------------------------------ + +Regression testing is the process of testing changes to the programs to make sure that the existing functionalities still work when changes are introduced. By running the RTs (or a subset of them by copying a RT configuration file and editing it), the code is compiled, the run directories are set up, and the code is executed. The results are typically compared against a pre-existing baseline, but in certain occasions it is necessary to first create a new baseline (for example, in a new platform where a baseline does not exist or when it is expected that a new development will change the answer). Because the RTs set up the run directories, this is a useful and easy way to get started, since all the model configuration files and necessary input data (initial conditions, fixed data) are copied into the right place. + +Overview of the RTs +^^^^^^^^^^^^^^^^^^^ + +The RT configuration files are located in ``./tests`` relative to the top-level directory of NEMSfv3gfs and have names ``rt*.conf``. The default RT configuration file, supplied with the NEMSfv3gfs master is called ``rt.conf`` and runs four types of configurations: IPD PROD, IPD REPRO, CCPP PROD, and CCPP REPRO. For the IPD configurations, CCPP is not used, that is, the code is compiled with ``CCPP=N``. The PROD configurations use the compiler flags used in NCEP operations for superior performance, while the REPRO configurations remove certain compiler flags to create b4b identical results between CCPP and IPD configurations. Before running the RT script ``rt.sh`` in directory ``./tests``, the user has to set some environment variables on the working shell: ``ACCNR`` (account to be charged for running the RTs), ``NEMS_COMPILER`` (optional for the ``intel`` compiler option, set to ``gnu`` to switch), and potentially ``RUNDIR_ROOT`` (location for the RT run directories), underneath which directories called ``rt_$PID`` are created (``$PID`` is the process identifier of the ``rt.sh`` invocation). This may be required on systems where the user does not have write permissions in the default run directory tree. + +.. code-block:: console + + export ACCNR=... + export NEMS_COMPILER=intel + export RUNDIR_ROOT=/full/path/under/which/rt_$PID/will/be/created + +Running the full default RT suite defined in ``rt.conf`` using the script ``rt.sh``: + +.. code-block:: console + + ./rt.sh -f + +This command can only be used on a NOAA machine using the Intel compiler, where an *official baseline* is available. For information on testing the CCPP code, or using alternate computational platforms, see the following sections. + +This command and all others below produce log output in ``./tests/log_machine.compiler``. These log files contain information on the location of the run directories that can be used as templates for the user. Each ``rt*.conf`` contains one or more compile commands preceding a number of tests. + + +Baselines +^^^^^^^^^^^^^^^^^^^ + +Regression testing is only possible on machines for which baselines exist. EMC maintains *official baselines* on *Theia* and *Wcoss* created with the Intel compiler. GMTB maintains additional baselines on *Jet*, *Cheyenne*, and *Gaea*. While GMTB is trying to keep up with changes to the official repositories, baselines maintained by GMTB are not guaranteed to be up-to-date. + +When porting the code to a new machine, it is useful to start by establishing a *personal baseline*. Future runs of the RT can then be compared against the *personal baseline* to ascertain that the results have not been inadvertently affected by code developments. The ``rt.sh -c`` option is used to create a *personal baseline*. + +.. code-block:: console + + ./rt.sh -l rt.conf -c fv3 # create own reg. test baseline + +Once the *personal baseline* has been created, future runs of the RT should be compared against the *personal baseline* using the ``-m`` option. + +.. code-block:: console + + ./rt.sh -l rt.conf -m # compare against own baseline + +The script rt.sh +^^^^^^^^^^^^^^^^^^^ + +``rt.sh`` is a bash shell file to run the RT and has the following options: + +.. code-block:: console + + Usage: $0 -c | -f | -s | -l | -m | -r | -e | -h + -c create new baseline results for + -f run full suite of regression tests + -s run standard suite of regression tests + -l run test specified in + -m compare against new baseline results + -r use Rocoto workflow manager + -e use ecFlow workflow manager + -h display this help + +The location of the run directories and *personal baseline* directories is controlled in ``rt.sh`` on a per-machine basis. The user is strongly advised to NOT modify the path to the *official baseline* directories. + +The *official baseline* directory is defined as: + +.. code-block:: console + + RTPWD=$DISKNM/trunk-yyyymmdd/${COMPILER} # on Cheyenne + RTPWD=$DISKNM/trunk-yyyymmdd # elsewhere + +Note that ``yyyymmdd`` is the year, month and day the baseline was created using top of master code. + +.. warning:: Modifying ``$DISKNM`` will break the RTs! + +*Personal baseline* results (see below) are stored in + +.. code-block:: console + + NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST + +and RTs are run in ``$RUNDIR_ROOT``. + +Example: *Theia* + +.. code-block:: console + + ... + dprefix=/scratch4/NCEPDEV + DISKNM=$dprefix/nems/noscrub/emc.nemspara/RT + STMP=$dprefix/stmp4 + PTMP=$dprefix/stmp3 + .. + +In case a user does not have write permissions to ``$STMP (/scratch4/NCEPDEV/stmp4/)``, ``$STMP`` must be modified without modifying ``$DISKNM`` (i.e. ``dprefix``). Similarly, if the user does not have write permissions to ``$PTMP``, the user can set the ``$RUNDIR_ROOT`` environment variable to change the location of the run directories as described below. + +.. code-block:: console + + # Overwrite default RUNDIR_ROOT if environment variable RUNDIR_ROOT is set + RUNDIR_ROOT=${RUNDIR_ROOT:-${PTMP}/${USER}/FV3_RT}/rt_$$ + + +Compatibility between the Code Base, the SDF, and the Namelist in the UFS Atmosphere +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The variable ``suite_name`` within the ``namelist.input`` file used in the UFS Atmosphere determines which suite will be employed at run time (e.g., ``suite_name=FV3_GFS_v15``). It is the user’s responsibility to ascertain that the other variables in ``namelist.input`` are compatible with the chosen suite. When runs are executed using the RT framework described in the preceding sections, compatibility is assured. For new experiments, users are responsible for modifying the two files (``SDF`` and ``namelist.input``) consistently, since limited checks are in place. + +Information about the UFS Atmosphere physics namelist can be found with the CCPP Scientific Documentation at https://dtcenter.org/GMTB/v3.0/sci_doc/. diff --git a/doc/UsersGuide/source/SDFandNamelistExamplePractices.rst b/doc/UsersGuide/source/SDFandNamelistExamplePractices.rst new file mode 100644 index 0000000000..902e14b991 --- /dev/null +++ b/doc/UsersGuide/source/SDFandNamelistExamplePractices.rst @@ -0,0 +1,247 @@ +.. _SDFandNamelistExamplePractices: + +******************************************** +SDF and Namelist Samples and Best Practices +******************************************** + +The public release of the medium-range weather app includes two supported physics suites: +GFSv15p2 and GFSv16beta. You will find the Suite Definition Files (SDFs) for these suites +in + +https://github.com/NOAA-EMC/fv3atm/tree/ufs_public_release/ccpp/suites + +(no other SDFs are available with this release). You will find the namelists for the C96 configuration here: + +https://github.com/ufs-community/ufs-weather-model/blob/ufs_public_release/parm/ccpp_v15p2_c96.nml.IN + +and + +https://github.com/ufs-community/ufs-weather-model/blob/ufs_public_release/parm/ccpp_v16beta_c96.nml.IN + +The two CCPP suites for the medium-range weather app release are supported in four grid resolutions: +C96, C192, C384, and C768, with 64 vertical levels. + +An in depth description of the namelist settings, SDFs, and parameterizations used in the GFSv15p2 +and GFSv16beta suites can be found in the CCPP Scientific Documentation (TODO add link). Note both suites do not +use stochastic physics by default, but the stochastic physics can be activated following the +instructions described here (TODO add link). + +Both the SDF and the *input.nml* contain information about how to specify the physics suite. +Some of this information is redundant, and the user must make sure they are compatible. The +safest practice is to use the SDF and namelist provided for each suite, since those are +supported configurations. + +Changes to the SDF must be accompanied by corresponding changes to the namelist. While there +is not a one-to-one correspondence between the namelist and the SDF, :numref:`Table %s ` +shows some variables in the namelist that must match the SDF. + +.. _PBLVarOptions: + +.. list-table:: *Variables related to PBL options* + :widths: 15 30 10 10 20 30 + :header-rows: 1 + + * - Namelist option + - Meaning + - Possible Values + - Default + - Used with CCPP scheme + - Recommentation + * - + - **PBL-related variables** + - + - + - + - + * - do_myjpbl + - Flag to activate the MYJ PBL scheme + - T + - F + - mypbl_wrapper + - Set to F for GFSv15p2 and GFSv16beta + * - do_myjsfc + - Flag to activate the MYJ PBL surface layer scheme + - T, F + - F + - myjsfc_wrapper + - Set to F for GFSv15p2 and GFSv16beta + * - do_mynnedmf + - Flag to activate the MYNN-EDMF scheme + - T, F + - F + - mynnedmf_wrapper + - Set to F for GFSv15p2 and GFSv16beta + * - do_ysu + - Flag to activate the YSU PBL scheme + - T, F + - F + - ysudif + - Set to F for GFSv15p2 and GFSv16beta + * - hybedmf + - Flag to activate the K-based PBL scheme + - T, F + - F + - hedmf + - Set to T for GFSv15p2 and F for GFSv16beta + * - isatedmf + - Flag for version of scale-aware TKE-based EDMF scheme + - 0, 1 + - 0 + - 0=satmedmfvdif, 1=satmedmfvdifq + - Set to 0 for GFSv15p2 and 1 for GFSv16beta + * - ism + - Flag to choose a land surface model to use + - 0, 1, 2 + - 1 + - 1=lsm_noah, 2=lsm_ruc + - Set to 1 for GFSv15p2 and GFSv16beta + * - satedmf + - Flag to activate the scale-aware TKE-based EDMF scheme + - T, F + - F + - satmedmfvdif or satmedmfvdifq + - Set to T for GFSv15p2 and GFSv16beta + * - shinhong + - Flag to activate the Shin-Hong PBL parameterization + - T, F + - F + - shinhongdif + - Set to F for GFSv15p2 and GFSv16beta + * - + - **Convection-releated flags** + - + - + - + - + * - cscnv + - Flag to activate the Chikira-Sugiyama deep convection scheme + - T, F + - F + - cs_conv + - Set to F for GFSv15p2 and GFSv16beta + * - do_aw + - Flag to activate the Arakawa-Wu extension to the Chikira-Sugiyama deep convection scheme + - T, F + - F + - cs_conv_aw_adj + - Set to F for GFSv15p2 and GFSv16beta + * - imfdeepcnv + - Flag to choose a mass flux deep convective scheme + - -1, 2, 3, 4 + - -1 + - -1=no deep convection*, 2=samfshalcnv, 3=cu_gf_driver, 4=cu_ntiedtke + - Set to 2 for GFSv15p2 and GFSv16beta + * - imfshalcvn + - Flag to choose a mass flux shallow convective scheme + - -1, 2, 3, 4 + - -1 + - -1=no deep convection*, 2=samfshalcnv, 3=cu_gf_driver, 4=cu_ntiedtke + - Set to 2 for GFSv15p2 and GFSv16beta + +\*Even when imfdeepcvn=-1, the Chikira-Sugiyama deep convection scheme may be specified using cscnv=T. + +Other miscellaneous changes to the SDF that must be accompanied by corresponding changes in +the namelist are listed in :numref:`Table %s `. + +.. _MiscVarOptions: + +.. list-table:: *Miscellaneous namelist variables and their relation to the SDF* + :widths: 15 30 10 10 20 30 + :header-rows: 1 + + * - Namelist option + - Meaning + - Possible Values + - Default + - Used with CCPP scheme + - Recommentation + * - + - **Miscellaneous variables** + - + - + - + - + * - do_myjsfc + - Flag to activate the MYJ PBL surface scheme + - T, F + - F + - mynnsfc_wrapper + - Set to F for GFSv15p2 and GFSv16beta + * - do_shoc + - Flag to activate the Simplified Higher-Order Closure (SHOC) parameterization + - T, F + - F + - shoc + - Set to F for GFSv15p2 and GFSv16beta + * - do_ugwp** + - Flag to activate the unified Gravity Wave Physics parameterization + - T, F + - F + - cires_ugwp + - Set to F for GFSv15p2 and GFSv16beta + * - imp_physics + - Flag to choose a microphysics scheme + - 8, 10, 11 + - 99 + - 8=mp_thompson, 10=m_micro, 11=gfdl_could_microphys + - Set to 11 for GFSv15p2 and GFSv16beta + * - lsm + - Flag to choose a land surface model to use + - 0, 1, 2 + - 1 + - 1=lsm_noah, 2=lsm_ruc + - Set to 1 for GFSv15p2 and GFSv16beta + * - lsoil + - Number of soil layers + - 4, 9 + - 4 + - 4 for lsm_noah, 9 for lsm_ruc + - Set to 4 for GFSv15p2 and GFSv16beta + * - h2o_phys + - Flag for stratosphere h2o scheme + - T, F + - + - h2ophys + - Set to T for GFSv15p2 and GFSv16beta + * - oz_phys_2015 + - Flag for new (2015) ozone physics + - T, F + - + - ozphys_2015 + - Set to T for GFSv15p2 and GFSv16beta + +\*\*The CIRES Unified Gravity Wave Physics (cires_ugwp) scheme is used in GFSv15p2 and GFSv16beta SDFs with do_ugwp=F in the namelist. In this setting, the cires_ugwp calls the operational GFS v15.2 orographic gravity wave drag (gwdps) scheme. When do_ugwp=T, the cires_ugwp scheme calls an experimental orographic gravity wave (gwdps_v0). + +**Note that some namelist variables are not available for use with CCPP.** + + * **do_deep**. In order to disable deep convection, it is necessary to remove the deep convection scheme from the SDF. + * **shal_cnv**. In order to disable shallow convection, it is necessary to remove the deep convection scheme from the SDF. + * **ldiag3d** and **ldiag_ugwp**. Must be F for CCPP runs. + * **gwd_opt**. Ignored in CCPP-supported suites. + +**When certain parameterizations are turned on, additional namelist options can be used (they are ignored otherwise). +Some examples are shown in** :numref:`Table %s `. + +.. _EnabledNMLOptions: + +.. list-table:: *Enabled namelist variables* + :widths: 10 50 + :header-rows: 1 + + * - Namelist setting + - Enabled namelist variables + * - do_ugwp=T + - All variables in namelist record &cires_ugwp_nml plus do_tofd. Additionally, if namelist variable cnvgwd=T and + the third and fourth position of namelist array cdmbgwd are both 1, then the convective gravity wave drag that + is part of cires_ugwp will be called. (Not supported with the UFS) + * - do_mynnedmf=T + - bl_mynn_tkeadvect, bl_mynn_edmf, bl_mynn_edmf_mom (Not supported with the UFS) + * - imp_physics=99 + - psautco and prautco (Not supported with the UFS) + * - imp_physics=10 + - mg_* (Not supported with UFS) + * - imp_physics=11 + - All variables in namelist record gfdl_cloud_microphysics_nml and lgfdlmprad + * - satedmf=T + - isatedmf + diff --git a/doc/UsersGuide/source/_static/README b/doc/UsersGuide/source/_static/README new file mode 100644 index 0000000000..0e27faf91a --- /dev/null +++ b/doc/UsersGuide/source/_static/README @@ -0,0 +1 @@ +This directory contains custom stylesheets and other static files. diff --git a/doc/UsersGuide/source/_templates/README b/doc/UsersGuide/source/_templates/README new file mode 100644 index 0000000000..99eef870b6 --- /dev/null +++ b/doc/UsersGuide/source/_templates/README @@ -0,0 +1 @@ +This directory contains custom HTML templates. diff --git a/doc/UsersGuide/source/conf.py b/doc/UsersGuide/source/conf.py new file mode 100644 index 0000000000..aaa1ab70d9 --- /dev/null +++ b/doc/UsersGuide/source/conf.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'UFS Weather Model Users Guide' +copyright = '2020' +author = ' ' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '' + +numfig = True + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'sphinxcontrib.bibtex' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'classic' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} +html_theme_options = {"body_max_width": "none"} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +def setup(app): + app.add_stylesheet('custom.css') # may also be an URL + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'UFS-Weather-Model' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_engine = 'pdflatex' +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', + # 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Schramm, J., L. Bernardet, L. Carson, G. Firl, D. Heinzeller, L. Pan, and M. Zhang, 2020. UFS Weather Model User's Guide Release v1.0.0. Npp. Available at https://dtcenter.org.}\sphinxmaketitle' +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'UFS_WM_Users_Guide.tex', 'UFS Weather Model Users Guide', + author,'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'UFS_WM_Users_Guide', 'UFS Weather Model Users Guide', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'UFS_WM_Users_guide', 'UFS Weather Model Users Guide', + author, 'UFS_WM_Users_Guide', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True diff --git a/doc/UsersGuide/source/index.rst b/doc/UsersGuide/source/index.rst new file mode 100644 index 0000000000..0c3b4d2604 --- /dev/null +++ b/doc/UsersGuide/source/index.rst @@ -0,0 +1,21 @@ +.. UFS Weather Model Users Guide, created by + sphinx-quickstart on Tue Oct 29 09:07:29 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to the UFS Weather Model User's Guide +======================================================== + +.. toctree:: + :numbered: + :maxdepth: 3 + + Introduction + CodeOverview + InputsOutputs + SDFandNamelistExamplePractices + FAQforModelConfiguration + ContributingDevelopment + CompilingCodeWithoutApp + Acronyms + Glossary diff --git a/doc/UsersGuide/source/references.bib b/doc/UsersGuide/source/references.bib new file mode 100644 index 0000000000..9908c88ee6 --- /dev/null +++ b/doc/UsersGuide/source/references.bib @@ -0,0 +1,14 @@ +@article{Bernardet2018, + title={Community infrastructure for facilitating and accelerating improvement and testing of physical parameterizations}, + author={L. Bernardet}, + journal={WRF-GRAPES Modeling Workshop, Oct 17, Boulder, CO}, + url = {http://www2.mmm.ucar.edu/wrf/grapes/bernardet_2018.pdf}, + year={2018}, + } +@article{BernardetEtAl2018a, + title={Community infrastructure for facilitating improvement and testing of physical parameterizations}, + author={L. Bernardet and G. Firl and L. Carson and J. Dudhia and D. Gill and G. Grell and M. Harrold and C. Harrop and I. Jankov and P. Jimenez and G. Ketefian and L. Nance and D. Stark and J. Wolff and L. Xue and M. Zhang}, + journal={Understanding and Modelling Atmospheric Processes / 2nd Pan-GASS (Global Atmospheric System Studies) meeting, Lorne, Victoria, Australia, Feb 26}, + url={https://drive.google.com/drive/folders/1XYFI08wBUxTyiLkNqn5H3PcXSUgpxWqF}, + year={2018}, + } diff --git a/tests/compile_cmake.sh b/tests/compile_cmake.sh index eead0e8183..a6b0bb6bb9 100755 --- a/tests/compile_cmake.sh +++ b/tests/compile_cmake.sh @@ -69,6 +69,14 @@ fi mkdir -p ${BUILD_DIR} +# sanity check + +# Use either DEBUG or REPRO flags, not both +if [[ "${MAKE_OPT}" == *"DEBUG=Y"* && "${MAKE_OPT}" == *"REPRO=Y"* ]]; then + echo "ERROR in compile_cmake.sh: options DEBUG=Y and REPRO=Y are mutually exclusive" + exit 1 +fi + # set CCPP_CMAKE_FLAGS based on $MAKE_OPT CCPP_CMAKE_FLAGS="" diff --git a/tests/rt.sh b/tests/rt.sh index 69af641a29..f41d47c669 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -9,9 +9,9 @@ die() { echo "$@" >&2; exit 1; } usage() { set +x echo - echo "Usage: $0 -c | -f | -s | -l | -m | -k | -r | -e | -h" + echo "Usage: $0 -c | -f | -s | -l | -m | -k | -r | -e | -h" echo - echo " -c create new baseline results for " + echo " -c create new baseline results" echo " -f run full suite of regression tests" echo " -s run standard suite of regression tests" echo " -l runs test specified in "