From 2f234daf1959e0f9370c04b8d4aa689e0380a493 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:00:20 -0600 Subject: [PATCH] Update Develop-ref after #1710 (#1714) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: Hank Fisher Co-authored-by: Minna Win Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Christina Kalb Co-authored-by: Julie Prestopnik Co-authored-by: johnhg Co-authored-by: Hank Fisher Co-authored-by: Mrinal Biswas Co-authored-by: jprestop Co-authored-by: j-opatz Co-authored-by: Kathryn Newman Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Lisa Goodrich Co-authored-by: Julie Prestopnik Co-authored-by: Christina Kalb Co-authored-by: bikegeek Co-authored-by: Hank Fisher Co-authored-by: Christina Kalb Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: Molly Smith Co-authored-by: hsoh-u --- .github/actions/run_tests/Dockerfile.run | 2 +- .../actions/run_tests/Dockerfile.run_cartopy | 2 +- .github/jobs/docker_utils.py | 3 + .github/jobs/get_requirements.py | 10 +- .github/jobs/get_use_case_commands.py | 36 +- .github/jobs/setup_and_run_diff.py | 6 +- .github/jobs/setup_and_run_use_cases.py | 4 +- docs/Contributors_Guide/add_use_case.rst | 2 +- docs/Users_Guide/getting_started.rst | 379 ++++++++++++++++++ docs/Users_Guide/index.rst | 1 + docs/Users_Guide/installation.rst | 6 +- docs/Users_Guide/systemconfiguration.rst | 149 ++----- internal_tests/use_cases/all_use_cases.txt | 1 + ...serScript_obsERA_obsOnly_PhaseDiagram.conf | 4 +- .../save_input_files_txt.py | 5 + scripts/docker/docker_env/Dockerfile | 9 +- .../docker/docker_env/Dockerfile.metplus_base | 2 +- .../docker_env/Dockerfile.py_embed_base | 2 +- scripts/docker/docker_env/README.md | 168 ++++---- .../docker/docker_env/scripts/cfgrib_env.sh | 24 +- .../docker_env/scripts/cycloneplotter_env.sh | 18 +- scripts/docker/docker_env/scripts/diff_env.sh | 16 +- .../docker/docker_env/scripts/gempak_env.sh | 4 +- scripts/docker/docker_env/scripts/h5py_env.sh | 12 +- .../docker/docker_env/scripts/icecover_env.sh | 22 +- .../docker_env/scripts/metdataio_env.sh | 14 +- .../docker_env/scripts/metplotpy_env.sh | 53 +-- .../docker_env/scripts/metplus_base_env.sh | 12 +- .../docker/docker_env/scripts/netcdf4_env.sh | 10 +- .../docker_env/scripts/py_embed_base_env.sh | 15 +- .../docker/docker_env/scripts/pygrib_env.sh | 16 +- .../docker/docker_env/scripts/pytest_env.sh | 9 +- .../docker_env/scripts/spacetime_env.sh | 34 +- .../docker_env/scripts/weatherregime_env.sh | 16 +- .../docker/docker_env/scripts/xesmf_env.sh | 14 +- 35 files changed, 720 insertions(+), 360 deletions(-) create mode 100644 docs/Users_Guide/getting_started.rst diff --git a/.github/actions/run_tests/Dockerfile.run b/.github/actions/run_tests/Dockerfile.run index 133783e8a..a78029420 100644 --- a/.github/actions/run_tests/Dockerfile.run +++ b/.github/actions/run_tests/Dockerfile.run @@ -1,4 +1,4 @@ -ARG METPLUS_ENV_TAG=metplus_base +ARG METPLUS_ENV_TAG=metplus_base.v5 ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env diff --git a/.github/actions/run_tests/Dockerfile.run_cartopy b/.github/actions/run_tests/Dockerfile.run_cartopy index 58256ff2a..196c4bd50 100644 --- a/.github/actions/run_tests/Dockerfile.run_cartopy +++ b/.github/actions/run_tests/Dockerfile.run_cartopy @@ -1,4 +1,4 @@ -ARG METPLUS_ENV_TAG=cycloneplotter +ARG METPLUS_ENV_TAG=cycloneplotter.v5 ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index a10f67f0c..cf38091ad 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -12,6 +12,9 @@ # repository used for storing input data for development branches DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev' +# extension to add to conda environments +VERSION_EXT = '.v5' + def get_data_repo(branch_name): """! Branch names that start with main_v or contain only digits and dots with out without a prefix 'v' will return diff --git a/.github/jobs/get_requirements.py b/.github/jobs/get_requirements.py index d0b91cd71..e4419e31d 100755 --- a/.github/jobs/get_requirements.py +++ b/.github/jobs/get_requirements.py @@ -8,11 +8,11 @@ import get_use_case_commands -# add internal_tests/use_cases directory to path so the test suite can be found -USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) -sys.path.insert(0, USE_CASES_DIR) +# add METplus directory to path so the test suite can be found +METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, METPLUS_TOP_DIR) from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index debf7bc6b..3d8956181 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -8,17 +8,22 @@ import os # add METplus directory to sys path so the test suite can be found -USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) -sys.path.insert(0, USE_CASES_DIR) +METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, METPLUS_TOP_DIR) from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite from metplus.util.met_util import expand_int_string_to_list +from docker_utils import VERSION_EXT -METPLUS_BASE_ENV = 'metplus_base' + +# path to METplus install location in Docker METPLUS_DOCKER_LOC = '/metplus/METplus' +# name of conda environment used for cases that don't need special env +METPLUS_BASE_ENV = 'metplus_base' + # keywords in requirements list that trigger obtaining METcalcpy and METplotpy PLOTCALC_KEYWORDS = [ 'metplotpy', @@ -27,6 +32,12 @@ 'weatherregime', ] +# Docker envs that do not use Python so they do not need print conda list +NOT_PYTHON_ENVS = [ + 'gfdl-tracker', + 'gempak', +] + def handle_automation_env(host_name, reqs, work_dir): # if no env is specified, use metplus base environment conda_env = METPLUS_BASE_ENV @@ -44,12 +55,15 @@ def handle_automation_env(host_name, reqs, work_dir): return '', 'user_env_vars.MET_PYTHON_EXE=python3' return '', '' + # add version extension to conda environment name + conda_env_w_ext = f'{conda_env}{VERSION_EXT}' + # start building commands to run before run_metplus.py in Docker setup_env = 'source /etc/bashrc;' # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'envs', - conda_env, 'bin') + conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') setup_env += f' export PATH={python_dir}:$PATH;' @@ -92,12 +106,12 @@ def handle_automation_env(host_name, reqs, work_dir): setup_env += f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH;' # list packages in python environment that will be used - if conda_env != 'gempak': + if conda_env not in NOT_PYTHON_ENVS: setup_env += ( - f'echo Using environment: dtcenter/metplus-envs:{conda_env};' - f'echo cat /usr/local/envs/{conda_env}/environments.yml;' + f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext};' + f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml;' f'echo ----------------------------------------;' - f'cat /usr/local/envs/{conda_env}/environments.yml;' + f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml;' 'echo ----------------------------------------;' ) @@ -108,7 +122,7 @@ def main(categories, subset_list, work_dir=None, all_commands = [] if work_dir is None: - work_dir = USE_CASES_DIR + work_dir = METPLUS_TOP_DIR test_suite = METplusUseCaseSuite() test_suite.add_use_case_groups(categories, subset_list) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index ff7acfcfd..9b3725c90 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -5,6 +5,8 @@ import subprocess import shlex +from docker_utils import VERSION_EXT + ci_dir = os.path.join(os.environ.get('GITHUB_WORKSPACE'), '.github') sys.path.insert(0, ci_dir) @@ -46,12 +48,12 @@ mount_args = ' '.join(volume_mounts) # command to run inside Docker -cmd = ('/usr/local/envs/diff/bin/python3 ' +cmd = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 ' f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py') # run inside diff env: mount METplus code and output dir, volumes from output volumes docker_cmd = (f'docker run -e GITHUB_WORKSPACE {VOLUMES_FROM} ' - f'{mount_args} dtcenter/metplus-envs:diff ' + f'{mount_args} dtcenter/metplus-envs:diff{VERSION_EXT} ' f'bash -c "{cmd}"') print(f'RUNNING: {docker_cmd}') try: diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index ff1fa35e9..00aa68209 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -16,7 +16,7 @@ import get_use_case_commands import get_data_volumes -from docker_utils import get_branch_name +from docker_utils import get_branch_name, VERSION_EXT runner_workspace = os.environ.get('RUNNER_WORKSPACE') github_workspace = os.environ.get('GITHUB_WORKSPACE') @@ -70,6 +70,8 @@ def main(): else: env_tag = 'metplus_base' + env_tag = f'{env_tag}{VERSION_EXT}' + # get Dockerfile to use dockerfile_name = 'Dockerfile.run' if 'gempak' in str(requirements).lower(): diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index c7ac6560c..cc884963e 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -108,7 +108,7 @@ configuration file name excluding the .conf suffix. .. figure:: figure/model_applications_example.png .. figure:: figure/model_applications_subdir.png - + Use Case Rules -------------- diff --git a/docs/Users_Guide/getting_started.rst b/docs/Users_Guide/getting_started.rst new file mode 100644 index 000000000..3aada6c85 --- /dev/null +++ b/docs/Users_Guide/getting_started.rst @@ -0,0 +1,379 @@ +*************** +Getting Started +*************** + +This chapter reviews some important things to consider before starting +a METplus project. + +Questions to Consider +===================== + +Questions to ask when applying METplus to a new testing and evaluation project +------------------------------------------------------------------------------ + +If a user is new to the concept of developing a verification or +evaluations system, there are questions that should be considered to help +determine which tools to use and how to set up METplus. + +* First and foremost, what are the questions that need to be answered + with this testing and evaluation project? + +* What type of forecasts and type of observations will be used and how + can/should they be matched? + + * What attributes of the forecast should be evaluated? + + * What is the standard for comparison that provides a reference level + of skill (e.g., persistence, climatology, reference model)? + + * What is the geographic location of the model data being evaluated? + Are there specific areas of interest for the evaluation? + + * What domain should be used to evaluate on: The model domain, + observation domain (if gridded), or some other domain? + + * What is the evaluation time period? + Retrospective with specific dates? + Ongoing, near-real-time evaluation? + Or both retrospective and real-time? + +* How should the testing and evaluation project be broken down into + METplus Use Cases? One large one or multiple smaller ones? + + * How will METplus be run? Manually? Scheduled, through cron? + Automated via a workflow manager (e.g. Rocoto, EC-Flow, Rose-Cylc)? + + * Where will METplus be run? Local machine, project machine, + HPC system, in the cloud (e.g. AWS)? Serial runs or parallelized? + +This section will provide some guidance on how to use METplus based on +the answers. + +* What type of forecasts and type of observations will be used? Will they be + gridded or point-based? The METplus tools that can be used will vary + depending on the answer. Here’s a matrix to help: + +.. role:: raw-html(raw) + :format: html + +.. list-table:: METplus Tools Decision Matrix + :widths: auto + :header-rows: 1 + + * - + - Gridded Forecast + - Point Forecast + * - **Gridded** :raw-html:`
` + **Observation/Analysis** + - Gen-Ens-Prod :raw-html:`
` + PCP-Combine :raw-html:`
` + Ensemble-Stat :raw-html:`
` + Grid-Stat :raw-html:`
` + Wavelet-Stat :raw-html:`
` + MODE :raw-html:`
` + MTD :raw-html:`
` + Series-Analysis :raw-html:`
` + Grid-Diag :raw-html:`
` + TC-Gen :raw-html:`
` + TC-RMW :raw-html:`
` + - Point-Stat (run with the Analyses :raw-html:`
` + as the “forecast” and Point Forecast :raw-html:`
` + as the “observation”) + * - **Point Observations** + - Point-Stat :raw-html:`
` + Ensemble-Stat + - Stat-Analysis (run by passing in MPR records) :raw-html:`
` + TC-Pairs :raw-html:`
` + TC-Gen :raw-html:`
` + TC-Stat + + +* What attributes of the forecast should be evaluated? + + * This refers to defining the forecast fields to be evaluated, + as well as the forecast characteristics such as bias, reliability, + resolution, and prediction of events. It also means understanding + the nature of the forecast and observations. + +Examples of the nature of fields to be evaluated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Continuous fields – the values change at the decimal level. + +* Categorical fields – the values change incrementally most + likely as integers or categories. Continuous fields can also be + turned into categorical fields via applying thresholds. + +* Probability fields – the values represent the probability or + likelihood of an event occurring, usually represented by thresholds. + +* Ensemble fields – are made up of multiple predictions either from + the same modeling system or multiple systems. + +Definitions of statistics categories associated with each type of field: + +* Continuous statistics - measures how the values of the forecasts + differ from the values of the observations. + + * METplus line types: SL1L2, SAL1L2, VL1L2, VAL1L2, CNT, VCNT. + + * METplus tools: + +* Categorical statistics - measures how well the forecast captures events. + + * METplus line types: FHO, CTC, CTS, MCTC, MCTS, ECLV, TC stats, + ExtraTC stats, TC-Gen stats. + +* Probability statistics - measures attributes such as reliability, + resolution, sharpness, and uncertainty. + + * METplus line types: PCT, PSTD, PJC, PRC. + +* Ensemble statistics - measures attributes as the relationship between + rank of observation and members, spread of ensemble member solutions + and continuous measures of skill. + +Additional verification and diagnostic approaches that can be helpful +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Geographical methods demonstrate where the error occurs geographically. + + * METplus methods: Series-Analysis tool. + + * METplus line types: Most Grid-Stat and Point-Stat line types. + +* Object Based measures the location error of the forecast and how the + total error break down into variety of descriptive attributes. + + * METplus methods: MODE, MTD, MvMODE, Grid-Stat Distance Maps. + + * METplus line types: MODE object attribute files, MODE CTS, MTD object + attribute files, MTD CTS, Grid-Stat DMAP. + +* Neighborhood relaxes the requirement for an exact match by evaluating + forecasts in the local neighborhood of the observations. + + * METplus methods: Grid-Stat Neighborhood, Point-Stat HiRA, Ensemble-Stat + HiRA. + + * METplus line types: NBRCTC, NBRCTS, NBRCNT, ECNT, ORANK, RPS. + +* Domain Decomposition and Transform applies a transform to a given field + to identify errors on different spatial scales: + + * METplus methods: Grid-Stat Fourier Decomposition; Wavelet-Stat tool, + TC-RMW tool. + + * METplus line types: Grid-Stat SL1L2, SAL1L2, VL1L2, VAL1L2, CNT, VCNT; + Wavelet Stat: ISC, RMW output file. + +* Feature Relative identifies systematic errors associated with a group + of case studies. + + * METplus methods: Feature Relative Use Cases. + +* Relationship between two fields: generates a joint PDF between two fields. + + * METplus methods: Grid-Diag tool. + +* Subseasonal-to-Seasonal Diagnostics compute indices to establish the + ability of the model to predict S2S drivers. + + * METplus methods: S2S Use Cases. + +What is the standard for comparison that provides a reference level of skill (e.g., persistence, climatology, reference model)? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Climatologies or Reference models may be passed into METplus using the +following configuration options: + +* {MET TOOL}_CLIMO_MEAN + +* {MET TOOL}_CLIMO_STDEV + +* {MET TOOL}_CLIMO_CDF + +This can be found in Grid-Stat, Point-Stat, Gen-Ens-Prod, Series-Analysis, +and Ensemble-Stat tools. + +What is the geographic location of the model data being evaluated? Are there specific areas of interest for the evaluation? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Masking regions are what METplus uses to define verification areas of +interest. These can be defined prior to running tools using the +Gen-Vx-Mask tool, or during run-time using the METPLUS_MASK_DICT options. + +What domain should be used for evaluation: The model domain, observation domain (if gridded), or some other domain? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The decision to evaluate on model or observation/analysis domain is +user-specific but the user may want to consider the following: + +* Regridding to the courser domain will smooth high resolution information + that may be important but smoother forecasts tend to score better. + +* Regridding to a finer domain essentially adds in additional information + that is not real. + +* One way to avoid the interpolation debate is to regrid both to a third + grid. + +Regridding in METplus can be completed using the Regrid-Data-Plane tool if +the fields will be used more than once. + +Regridding can also be done on the fly using the {Tool}_REGRID_TO_GRID. +All grid-to-grid verification tools have the regridding capability in it. + +What is the evaluation time period? Retrospective with specific dates? Ongoing, near-real-time evaluation? Or both retrospective and realtime? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Basically, running retrospectively means that the observations/analyses are +already available on disk and running in realtime is when the system needs +to wait for the observations to be available on the system. + +In METplus, the LOOP_BY configuration can be used. + +LOOP_BY = VALID or REALTIME to have METplus proceed through the data based +on Valid Time. + +LOOP_BY = INIT or RETRO to have METplus proceed through the data based +on Initialization Time. + +How should the testing and evaluation project be broken down into METplus Use Cases? One large one or multiple smaller ones? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A single use case is typically appropriate for a given evaluation so that +all of the information is found in one configuration file. However, users +may want to evaluate different combinations of models and observations. +For example, they may want to compare forecastA with observationA, +forecastA with observationB, forecastB with observationA, forecastB with +observationB, etc. In this case, separate METplus configuration files can +be created with information specific to each forecast or observation. +Another configuration file can be used to control settings common to each +evaluation, such as timing information and the process list. The METplus +wrappers can be called with each desired combination. + +.. code-block:: ini + + run_metplus.py forecastA.conf observationA.conf use_case_name.conf + run_metplus.py forecastA.conf observationB.conf use_case_name.conf + run_metplus.py forecastB.conf observationA.conf use_case_name.conf + run_metplus.py forecastB.conf observationB.conf use_case_name.conf + +It is also worth considering the +`Use Case Rules. `_ +A case may be +affected by the size of the data, the length of time to run and other factors. + +How will METplus be run? Manually? Scheduled through cron? Automated via a workflow manger (e.g. Rocoto, EC-Flow, Rose-Cylc)? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * If run manually, this can be done. + + * If scheduled through cron, a bash or csh script can be written to + set up environment variables to pass into METplus. + + * If automated via a workflow manager, it is recommended the user consider + configuring the use cases to run smaller amounts of data. + +Where will METplus be run? Local machine, project machine, HPC system, in the cloud (e.g. AWS)? Serial runs or parallelized? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * Running on linux or a project machine – identify where METplus is + installed by running **which run_metplus.py**; it is recommended an + additional user.conf or system.conf file is passed into the + **run_metplus.py** to direct where output should be written. + + * Running on HPC systems - check with the system admin to see if it + has been configured as a module and how to load netCDF and Python + modules. For NOAA and NCAR HPCs systems, please refer to the + `Existing Builds `_ + pages for the desired version for instructions on how to load the METplus + related modules. + + * Running on Cloud (AWS) - these instructions are coming soon. + + * Running in parallel - As of MET v10.1.0 Grid-Stat can be run in parallel. + Please reach out via + `METplus Discussions `_ + if help is needed. + +Would a flowchart help to provide clarity? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Utilizing a flowchart can assist in identifying which verification +steps can be completed by which METplus tools. + +.. _running-metplus: + +Running METplus +=============== + +Example Wrapper Use Case +------------------------ + +* Create a :ref:`user_configuration_file` + (named user_system.conf in this example). + +* Run the Example Wrapper use case. In a terminal, run:: + + run_metplus.py \ + /path/to/METplus/parm/use_cases/met_tool_wrapper/Example/Example.conf \ + /path/to/user_system.conf + +Replacing */path/to/user_system.conf* with the path to the +user configuration file and +*/path/to/METplus* with the path to the location where METplus is installed. + +The last line of the screen output should match this format:: + + 05/04 09:42:52.277 metplus (met_util.py:212) INFO: METplus has successfully finished running. + +If this log message is not shown, there is likely an issue with one or more +of the default configuration variable overrides in the +:ref:`user_configuration_file`. + +This use case does not utilize any of the MET tools, but simply demonstrates +how the :ref:`common_config_variables` control a use case run. + +If the run was successful, the line above the success message should contain +the path to the METplus log file that was generated:: + + 05/04 09:44:21.534 metplus (met_util.py:211) INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421 + +* Review the log file and compare it to the Example.conf use case + configuration file to see how the settings correspond to the result. + +* Review the :ref:`metplus_final.conf` file to see all + of the settings that were used in the use case. + +GridStat Wrapper Basic Use Case +------------------------------- + +* :ref:`obtain_sample_input_data` for the **met_tool_wrapper** use cases. + The tarfile should be in the directory that corresponds to the + major/minor release and starts with sample_data-met_tool_wrapper. + +* Create a :ref:`user_configuration_file` (named user_system.conf in this + example). Ensure that **INPUT_BASE** is set + to the directory where the sample data tarfile was uncompressed. + +* Run the GridStat Wrapper basic use case. In a terminal, run:: + + run_metplus.py \ + /path/to/METplus/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf \ + /path/to/user_system.conf + +Replacing */path/to/user_system.conf* with the path to the +user configuration file and +*/path/to/METplus* with the path to the location where METplus is installed. + +If the run was successful, the line above the success message should contain +the path to the METplus log file that was generated. + +* Review the log file and compare it to the **GridStat.conf** use case + configuration file to see how the settings correspond to the result. + +* Review the :ref:`metplus_final.conf` file to see all + of the settings that were used in the use case. + diff --git a/docs/Users_Guide/index.rst b/docs/Users_Guide/index.rst index a1ab31578..e08f2ea38 100644 --- a/docs/Users_Guide/index.rst +++ b/docs/Users_Guide/index.rst @@ -78,6 +78,7 @@ is sponsored by NSF. :numbered: 4 overview + getting_started installation systemconfiguration wrappers diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index 373181750..2a2d1ba09 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -1,8 +1,8 @@ .. _install: -********************* -Software Installation -********************* +************ +Installation +************ Introduction ============ diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index eb78d81b0..b823a8f9c 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -1,8 +1,8 @@ .. _sysconf: -******************** -System Configuration -******************** +************* +Configuration +************* This chapter is a guide on configuring METplus Wrappers. @@ -31,13 +31,13 @@ Config Best Practices / Recommendations * If configuring METplus Wrappers in a common location for multiple users: - * It is recommended that the values for **MET_INSTALL_DIR** and - **INPUT_BASE** are changed to valid values in the - :ref:`default_configuration_file`. + * It is recommended that the values for **MET_INSTALL_DIR** and + **INPUT_BASE** are changed to valid values in the + :ref:`default_configuration_file`. - * It is recommended to leave **OUTPUT_BASE** set to the default value in - the :ref:`default_configuration_file`. This prevents multiple users from - accidentally writing to the same output directory. + * It is recommended to leave **OUTPUT_BASE** set to the default value in + the :ref:`default_configuration_file`. This prevents multiple users from + accidentally writing to the same output directory. * If obtaining the METplus Wrappers with the intention of updating the same local directory as new versions become available, @@ -534,79 +534,6 @@ given use case. More information about the variables set in the use case configuration files can be found in the :ref:`common_config_variables` section. -.. _running-metplus: - -Running METplus -=============== - -Example Wrapper Use Case ------------------------- - -* Create a :ref:`user_configuration_file` - (named user_system.conf in this example) - -* Run the Example Wrapper use case. In a terminal, run:: - - run_metplus.py \ - /path/to/METplus/parm/use_cases/met_tool_wrapper/Example/Example.conf \ - /path/to/user_system.conf - -replacing **/path/to/user_system.conf** with the path to the -user configuration file and -**/path/to/METplus** with the path to the location where METplus is installed - -The last line of the screen output should match this format:: - - 05/04 09:42:52.277 metplus (met_util.py:212) INFO: METplus has successfully finished running. - -If this log message is not shown, there is likely an issue with one or more -of the default configuration variable overrides in the -:ref:`user_configuration_file`. - -This use case does not utilize any of the MET tools, but simply demonstrates -how the :ref:`common_config_variables` control a use case run. - -If the run was successful, the line above the success message should contain -the path to the METplus log file that was generated:: - - 05/04 09:44:21.534 metplus (met_util.py:211) INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421 - -* Review the log file and compare it to the Example.conf use case - configuration file to see how the settings correspond to the result. - -* Review the :ref:`metplus_final.conf` file to see all - of the settings that were used in the use case. - -GridStat Wrapper Basic Use Case -------------------------------- - -* :ref:`obtain_sample_input_data` for the **met_tool_wrapper** use cases. - The tarfile should be in the directory that corresponds to the - major/minor release and starts with sample_data-met_tool_wrapper - -* Create a :ref:`user_configuration_file` (named user_system.conf in this - example). Ensure that **INPUT_BASE** is set - to the directory where the sample data tarfile was uncompressed. - -* Run the GridStat Wrapper basic use case. In a terminal, run:: - - run_metplus.py \ - /path/to/METplus/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf \ - /path/to/user_system.conf - -replacing **/path/to/user_system.conf** with the path to the -user configuration file and -**/path/to/METplus** with the path to the location where METplus is installed - -If the run was successful, the line above the success message should contain -the path to the METplus log file that was generated. - -* Review the log file and compare it to the GridStat.conf use case - configuration file to see how the settings correspond to the result. - -* Review the :ref:`metplus_final.conf` file to see all - of the settings that were used in the use case. - .. _common_config_variables: Common Config Variables @@ -810,7 +737,7 @@ then three times will be processed: 2. Initialized on 2019-02-01 at 00Z / valid on 2019-02-01 at 06Z 3. Initialized on 2019-02-01 at 00Z / valid on 2019-02-01 at 09Z -You can also define :term:`LEAD_SEQ` using a special notation for many +The user can also define :term:`LEAD_SEQ` using a special notation for many forecast leads. The notation is **begin_end_incr(b,e,i)** where b = the first lead value, e = the last lead value (inclusive), and i = the increment between leads. For example:: @@ -828,9 +755,9 @@ the :term:`LEAD_SEQ` variable for the **SeriesByLead Wrapper Only**. If :term:`SERIES_BY_LEAD_GROUP_FCSTS` = True, then groups of forecast leads can be defined to be evaluated together. -You can define any number of these groups by setting +Any number of these groups can be defined by setting configuration variables LEAD_SEQ_1, LEAD_SEQ_2, ..., :term:`LEAD_SEQ_\`. -You can define the value with a +The value can be defined with a comma-separated list of integers (currently only hours are supported here) or using the special begin_end_incr(b,e,i) notation described just above. Each :term:`LEAD_SEQ_\` must have a corresponding @@ -941,7 +868,7 @@ Example 3:: This will skip every 30th and 31st day **and** every 3rd month. -You can use **begin_end_incr(b,e,i)** syntax to define a range of times to +**begin_end_incr(b,e,i)** syntax can be used to define a range of times to skip. b = begin value, e = end value, @@ -959,7 +886,7 @@ This is equivalent to:: [config] SKIP_TIMES = "%H:0,2,4,6,8,10,12,14,16,18,20,22" -You can also specify multiple strftime directives in a single time format. +Multiple strftime directives can be specified in a single time format. Example 5:: @@ -997,15 +924,15 @@ and the METplus Wrappers is run with:: [config] VALID_END = {now?fmt=%Y%m%d%H} -then the value of :term:`VALID_END` will be set to 2019042608. You can also -use {today} to substitute the -current YYYYMMDD, i.e. 20190426. You cannot change the formatting for -the 'today' keyword. +then the value of :term:`VALID_END` will be set to 2019042608. {today} +can also be used to substitute the +current YYYYMMDD, i.e. 20190426. The formatting for +the 'today' keyword cannot be changed.. Shift Keyword """"""""""""" -You can use the 'shift' keyword to shift the current time by any number of +The 'shift' keyword can be used to shift the current time by any number of seconds. For example, if the METplus Wrappers are run at the same clock time with:: @@ -1029,7 +956,7 @@ METplus Wrappers will process the following valid times: Truncate Keyword """""""""""""""" -You may want to configure the METplus Wrappers to process at 00Z, 06Z, 12Z, +The user may want to configure the METplus Wrappers to process at 00Z, 06Z, 12Z, and 18Z of a given day instead of 02Z, 08Z, 14Z, and 20Z. Having to adjust the shift amount differently if running at 08Z or 09Z to get the times to line up would be tedious. Instead, use the 'truncate' keyword. @@ -1705,8 +1632,8 @@ Shifting Times in Filename Templates Users can use the 'shift' keyword to adjust the time referenced in the filename template relative to the run time. For example, if the input files used contained data from 01Z on the date specified in the filename to 01Z on -the following day. In this example, for a run at 00Z you want to use the -file from the previous day and for the 01Z to 23Z runs you want to use the +the following day. In this example, for a run at 00Z use the +file from the previous day and for the 01Z to 23Z runs, use the file that corresponds to the current day. Here is an example:: [config] @@ -1724,7 +1651,7 @@ Using Windows to find Valid Files The [FCST/OBS]_FILE_WINDOW_[BEGIN/END] configuration variables can be used if the time information in the input data does not exactly line up with the -run time but you still want to process the data. The default value of the +run time but the user still wants to process the data. The default value of the file window begin and end variables are both 0 seconds. If both values are set to 0, METplus Wrappers will require that a file matching the template with the exact time requested exists. If either value is non-zero, METplus @@ -1777,7 +1704,7 @@ Wrapper Specific Windows ^^^^^^^^^^^^^^^^^^^^^^^^ A user may need to specify a different window on a wrapper-by-wrapper basis. -If this is the case, you can override the file window values for each +If this is the case, the user can override the file window values for each wrapper. Consider the following configuration:: [config] @@ -1801,7 +1728,7 @@ Runtime Frequency Some wrappers have an option to specify how frequently to process data. It can be run once to process all of the available files in the desired time range, -or it can be configured to run over different intervals. This allows you to +or it can be configured to run over different intervals. This allows the user to aggregate the output in a variety of ways. The wrappers that support this functionality (along with the configuration variable that controls the setting) include: @@ -2006,12 +1933,12 @@ supports METplus configuration variables that control the MET tool configuration file settings. **The METplus wrappers provide a special "wrapped" MET configuration file that references environment variables that are set by the wrappers based on the -values set in the METplus configuration files. YOU SHOULD NOT SET ANY OF THESE -ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT +values set in the METplus configuration files. THE USER SHOULD NOT SET ANY OF +THESE ENVIRONMENT VARIABLES! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not currently -supported by METplus you'd like to control, please refer to: +supported by METplus that the user would like to control, please refer to: :ref:`Overriding Unsupported MET config file settings`. The following section demonstrates a few examples using GridStat. @@ -2661,7 +2588,7 @@ This is the equivalent of running this bash command:: on the command line before calling run_metplus.py. -You can also reference other variables in the METplus config file. +Other variables can also be referenced in the METplus config file. For example:: [config] @@ -2686,7 +2613,7 @@ on the command line before calling run_metplus.py. Setting Config Variables with Environment Variables =================================================== -You can set METplus config variables to the value of local environment +The METplus config variables can be set to the value of local environment variables when METplus is run. To set any METplus config variable to the value of a local environment variable, use the following syntax:: @@ -2747,10 +2674,10 @@ Starting in METplus 3.0, users are required to either explicitly set both FCST_* Example:: - (met_util.py) ERROR: If FCST_VAR1_NAME is set, you must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME - (met_util.py) ERROR: If FCST_VAR2_NAME is set, you must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME - (met_util.py) ERROR: If FCST_VAR1_LEVELS is set, you must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS - (met_util.py) ERROR: If FCST_VAR2_LEVELS is set, you must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS + (met_util.py) ERROR: If FCST_VAR1_NAME is set, the user must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME + (met_util.py) ERROR: If FCST_VAR2_NAME is set, the user must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME + (met_util.py) ERROR: If FCST_VAR1_LEVELS is set, the user must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS + (met_util.py) ERROR: If FCST_VAR2_LEVELS is set, the user must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes, as they may want to update differently. @@ -2809,7 +2736,7 @@ Example log output:: (met_util.py) ERROR: Set GRID_STAT_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a METplus config file to set CLIMO_MEAN_FILE in a MET config (met_util.py) ERROR: output_prefix variable should reference ${OUTPUT_PREFIX} environment variable - (met_util.py) INFO: You will need to add GRID_STAT_OUTPUT_PREFIX to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to: + (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX will need to be added to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to: (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME} These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes and make sure they add the appropriate recommended METplus configuration variables to their files to achieve the same behavior. @@ -2843,7 +2770,7 @@ The script named validate_config.py is found in the same directory as run_metplu run_metplus.py ./my_conf.py ./another_config.py validate_config.py ./my_conf.py ./another_config.py -You must pass a valid configuration to the script, as in you must properly set :term:`MET_INSTALL_DIR`, :term:`INPUT_BASE`, and :term:`OUTPUT_BASE`, or it will not run. +A valid configuration must be passed to the script, as in the user must properly set :term:`MET_INSTALL_DIR`, :term:`INPUT_BASE`, and :term:`OUTPUT_BASE`, or it will not run. The script will evaluate all of the configuration files, including any MET configuration file that is referenced in a _CONFIG_FILE variable, such as :term:`GRID_STAT_CONFIG_FILE`. For each deprecated item that is found, the script will suggest a replacement for the file where the deprecated item was found. @@ -2912,7 +2839,7 @@ Example 5 (Another MET Configuration File):: Would you like the make this change to DeprecatedConfig? (y/n)[n] - IMPORTANT: If it is not already set, add the following in the [config] section to your METplus configuration file that sets GRID_STAT_CONFIG_FILE: + IMPORTANT: If it is not already set, add the following in the [config] section to the METplus configuration file that sets GRID_STAT_CONFIG_FILE: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME} Make this change before continuing! [OK] @@ -2921,7 +2848,7 @@ Example 5 (Another MET Configuration File):: While the METplus developers are very diligent to include deprecated variables in this functionality, some may slip through the cracks. When upgrading to a new version of METplus, it is important to test - and review your use cases to ensure they produce the same results as + and review the use cases to ensure they produce the same results as the previous version. Please create a post in the `METplus GitHub Discussions Forum `_ with any questions. diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index e37b0466e..0d20c7ce2 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -139,6 +139,7 @@ Category: s2s_mid_lat 2::UserScript_obsERA_obsOnly_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf:: weatherregime_env,cartopy,metplus 3:: UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus + Category: s2s_mjo 0:: UserScript_obsERA_obsOnly_PhaseDiagram:: model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf:: spacetime_env 1:: UserScript_fcstGFS_obsERA_OMI:: model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.conf:: spacetime_env, metdataio diff --git a/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf b/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf index bb8168e9b..e4617cf89 100644 --- a/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf +++ b/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf @@ -50,7 +50,7 @@ FCST_RUN = False # Input and Output Directories for the OBS OLR Files and output text file containing the file list OBS_PDTIME_FMT = %Y%m%d-%H%M%S OBS_PDTIME_INPUT_TEMPLATE = {valid?fmt=%Y%m%d-%H%M%S} -OBS_PDTIME_OUTPUT_DIR = {INPUT_BASE}/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/ +OBS_PDTIME_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/ OBS_PDTIME_OUTPUT_TEMPLATE = time_list_lead{lead?fmt=%HHH}.txt @@ -75,7 +75,7 @@ SCRIPT_OUTPUT_BASE = {OUTPUT_BASE} PLOT_INDEX = RMM # Input Directories -OBS_PHASE_DIAGRAM_INPUT_DIR = {OBS_PDTIME_OUTPUT_DIR} +OBS_PHASE_DIAGRAM_INPUT_DIR = {INPUT_BASE}/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram # Input filename template OBS_PHASE_DIAGRAM_INPUT_FILE = rmm.1x.txt diff --git a/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/save_input_files_txt.py b/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/save_input_files_txt.py index 69a7e65d0..b8954f956 100755 --- a/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/save_input_files_txt.py +++ b/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram/save_input_files_txt.py @@ -6,6 +6,11 @@ input_file = sys.argv[1] output_file = sys.argv[2] +output_dir = os.path.dirname(output_file) +if not os.path.exists(output_dir): + print(f'Creating output dir: {output_dir}') + os.makedirs(output_dir) + filelist = open(output_file,'a+') filelist.write(input_file + '\n') filelist.close() diff --git a/scripts/docker/docker_env/Dockerfile b/scripts/docker/docker_env/Dockerfile index a5f2549a6..21be7ae23 100644 --- a/scripts/docker/docker_env/Dockerfile +++ b/scripts/docker/docker_env/Dockerfile @@ -1,13 +1,14 @@ # Dockerfile to create conda environments used for use case tests -ARG BASE_ENV=metplus_base +ARG BASE_ENV=metplus_base.v5 FROM dtcenter/metplus-envs:${BASE_ENV} ARG ENV_NAME WORKDIR /scripts COPY scripts/${ENV_NAME}_env.sh . -ARG BASE_ENV=metplus_base -RUN ./${ENV_NAME}_env.sh ${BASE_ENV} +ARG BASE_ENV=metplus_base.v5 +RUN conda update -y -n base -c defaults conda \ + && ./${ENV_NAME}_env.sh ${BASE_ENV} -RUN conda list --name ${ENV_NAME} > /usr/local/envs/${ENV_NAME}/environments.yml +RUN conda list --name ${ENV_NAME}.v5 > /usr/local/envs/${ENV_NAME}.v5/environments.yml diff --git a/scripts/docker/docker_env/Dockerfile.metplus_base b/scripts/docker/docker_env/Dockerfile.metplus_base index cb63fc13f..87e2dfbc1 100644 --- a/scripts/docker/docker_env/Dockerfile.metplus_base +++ b/scripts/docker/docker_env/Dockerfile.metplus_base @@ -9,4 +9,4 @@ COPY scripts/metplus_base_env.sh . RUN conda update -y -n base -c defaults conda \ && ./metplus_base_env.sh -RUN conda list --name metplus_base > /usr/local/envs/metplus_base/environments.yml +RUN conda list --name metplus_base.v5 > /usr/local/envs/metplus_base.v5/environments.yml diff --git a/scripts/docker/docker_env/Dockerfile.py_embed_base b/scripts/docker/docker_env/Dockerfile.py_embed_base index 90db08b03..4499570d4 100644 --- a/scripts/docker/docker_env/Dockerfile.py_embed_base +++ b/scripts/docker/docker_env/Dockerfile.py_embed_base @@ -10,4 +10,4 @@ COPY scripts/py_embed_base_env.sh . RUN conda update -y -n base -c defaults conda \ && ./py_embed_base_env.sh -RUN conda list --name py_embed_base > /usr/local/envs/py_embed_base/environments.yml +RUN conda list --name py_embed_base.v5 > /usr/local/envs/py_embed_base.v5/environments.yml diff --git a/scripts/docker/docker_env/README.md b/scripts/docker/docker_env/README.md index e39351360..521d07184 100644 --- a/scripts/docker/docker_env/README.md +++ b/scripts/docker/docker_env/README.md @@ -5,7 +5,13 @@ Instructions include how to create Docker images in dtcenter/metplus-envs so environments are available for the automated tests. Instructions to create these Conda environments on a local machine are also provided. -## metplus_base +**IMPORTANT NOTE:** If all of the Docker Conda Environment images need to be +rebuilt again, consider adding logic to update the OS packages first to +prevent potential issues with wget commands used to download the cartopy +shapefiles. Without this, the wget commands may fail because the +certificate trust store is out of date. + +## metplus_base.v5 This environment includes the minimum requirements to run the METplus wrappers. @@ -13,8 +19,8 @@ This environment includes the minimum requirements to run the METplus wrappers. ### Docker ``` -docker build -t dtcenter/metplus-envs:metplus_base -f Dockerfile.metplus_base . -docker push dtcenter/metplus-envs:metplus_base +docker build -t dtcenter/metplus-envs:metplus_base.v5 -f Dockerfile.metplus_base . +docker push dtcenter/metplus-envs:metplus_base.v5 ``` ### Local @@ -23,15 +29,16 @@ docker push dtcenter/metplus-envs:metplus_base ./scripts/metplus_base_env.sh ``` -## py_embed_base + +## py_embed_base.v5 This environment includes all python embedding requirements (xarray and netcdf). ### Docker ``` -docker build -t dtcenter/metplus-envs:py_embed_base -f Dockerfile.py_embed_base . -docker push dtcenter/metplus-envs:py_embed_base +docker build -t dtcenter/metplus-envs:py_embed_base.v5 -f Dockerfile.py_embed_base . +docker push dtcenter/metplus-envs:py_embed_base.v5 ``` ### Local @@ -40,58 +47,62 @@ docker push dtcenter/metplus-envs:py_embed_base ./scripts/py_embed_base_env.sh ``` -## h5py (from py_embed_base) + +## h5py.v5 (from py_embed_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:h5py --build-arg BASE_ENV=py_embed_base --build-arg ENV_NAME=h5py . -docker push dtcenter/metplus-envs:h5py +docker build -t dtcenter/metplus-envs:h5py.v5 --build-arg BASE_ENV=py_embed_base.v5 --build-arg ENV_NAME=h5py . +docker push dtcenter/metplus-envs:h5py.v5 ``` ### Local ``` -./scripts/h5py_env.sh py_embed_base +./scripts/h5py_env.sh ``` -## metdataio (from metplus_base) + +## metdataio.v5 (from metplus_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:metdataio --build-arg ENV_NAME=metdataio . -docker push dtcenter/metplus-envs:metdataio +docker build -t dtcenter/metplus-envs:metdataio.v5 --build-arg ENV_NAME=metdataio . +docker push dtcenter/metplus-envs:metdataio.v5 ``` ### Local ``` -./scripts/metdatadb_env.sh metplus_base +./scripts/metdataio_env.sh ``` -## pygrib (from py_embed_base) + +## pygrib.v5 (from py_embed_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:pygrib --build-arg BASE_ENV=py_embed_base --build-arg ENV_NAME=pygrib . -docker push dtcenter/metplus-envs:pygrib +docker build -t dtcenter/metplus-envs:pygrib.v5 --build-arg BASE_ENV=py_embed_base.v5 --build-arg ENV_NAME=pygrib . +docker push dtcenter/metplus-envs:pygrib.v5 ``` ### Local ``` -./scripts/pygrib_env.sh py_embed_base +./scripts/pygrib_env.sh ``` -## cfgrib (from fresh Python 3.6.8 environment) + +## cfgrib.v5 (from fresh Python 3.8.6 environment) ### Docker ``` -docker build -t dtcenter/metplus-envs:cfgrib --build-arg ENV_NAME=cfgrib . -docker push dtcenter/metplus-envs:cfgrib +docker build -t dtcenter/metplus-envs:cfgrib.v5 --build-arg ENV_NAME=cfgrib . +docker push dtcenter/metplus-envs:cfgrib.v5 ``` ### Local @@ -100,76 +111,76 @@ docker push dtcenter/metplus-envs:cfgrib ./scripts/cfgrib_env.sh ``` -## netcdf4 (from metplus_base) + +## netcdf4.v5 (from metplus_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:netcdf4 --build-arg ENV_NAME=netcdf4 . -docker push dtcenter/metplus-envs:netcdf4 +docker build -t dtcenter/metplus-envs:netcdf4.v5 --build-arg ENV_NAME=netcdf4 . +docker push dtcenter/metplus-envs:netcdf4.v5 ``` ### Local ``` -./scripts/netcdf4_env.sh metplus_base +./scripts/netcdf4_env.sh ``` -## xesmf (from metplus_base) + +## xesmf.v5 (from metplus_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:xesmf --build-arg ENV_NAME=xesmf . -docker push dtcenter/metplus-envs:xesmf +docker build -t dtcenter/metplus-envs:xesmf.v5 --build-arg ENV_NAME=xesmf . +docker push dtcenter/metplus-envs:xesmf.v5 ``` ### Local ``` -./scripts/xesmf_env.sh metplus_base +./scripts/xesmf_env.sh ``` -## spacetime (from metplus_base) + +## spacetime.v5 ### Docker ``` -docker build -t dtcenter/metplus-envs:spacetime --build-arg ENV_NAME=spacetime . -docker push dtcenter/metplus-envs:spacetime +docker build -t dtcenter/metplus-envs:spacetime.v5 --build-arg ENV_NAME=spacetime . +docker push dtcenter/metplus-envs:spacetime.v5 ``` ### Local ``` -./scripts/spacetime_env.sh metplus_base +./scripts/spacetime_env.sh ``` -## metplotpy (from metplus_base) + +## metplotpy.v5 (from metplus_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:metplotpy --build-arg ENV_NAME=metplotpy . -docker push dtcenter/metplus-envs:metplotpy +docker build -t dtcenter/metplus-envs:metplotpy.v5 --build-arg ENV_NAME=metplotpy . +docker push dtcenter/metplus-envs:metplotpy.v5 ``` ### Local -NOTE: Environment cannot be created fully using conda commands because there is a conflict -with the python-kaleido package and the version of glibc. The kaleido package installation -will fail through the script, but it should be installed using pip3 afterwards. - ``` -./scripts/metplotpy_env.sh metplus_base -/home/met_test/.conda/envs/metplotpy/bin/pip3 install kaleido==0.2.1 -/home/met_test/.conda/envs/metplotpy/bin/python3 cartopy_feature_download.py cultural physical +./scripts/metplotpy_env.sh +/home/met_test/.conda/envs/metplotpy.v5/bin/python3 cartopy_feature_download.py cultural physical rm cartopy_feature_download.py ``` #### To install METplotpy and METcalcpy packages in environment ``` +runas met_test cd /home/met_test # git clone not necessary if repo is already available @@ -178,33 +189,37 @@ git clone https://github.com/dtcenter/METcalcpy cd /home/met_test/METplotpy git checkout develop -/home/met_test/.conda/envs/metplotpy/bin/pip3 install . +/home/met_test/.conda/envs/metplotpy.v5/bin/pip3 install . cd /home/met_test/METcalcpy git checkout develop -/home/met_test/.conda/envs/metplotpy/bin/pip3 install . +/home/met_test/.conda/envs/metplotpy.v5/bin/pip3 install . + +exit ``` -## weatherregime (from metplotpy_env) + +## weatherregime.v5 (from metplotpy.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:weatherregime --build-arg BASE_ENV=metplotpy --build-arg ENV_NAME=weatherregime . -docker push dtcenter/metplus-envs:weatherregime +docker build -t dtcenter/metplus-envs:weatherregime.v5 --build-arg BASE_ENV=metplotpy.v5 --build-arg ENV_NAME=weatherregime . +docker push dtcenter/metplus-envs:weatherregime.v5 ``` ### Local ``` -./scripts/weatherregime_env.sh metplotpy -/home/met_test/.conda/envs/weatherregime/bin/python3 cartopy_feature_download.py cultural physical +./scripts/weatherregime_env.sh +/home/met_test/.conda/envs/weatherregime.v5/bin/python3 cartopy_feature_download.py cultural physical rm cartopy_feature_download.py ``` #### To install METplotpy and METcalcpy packages in environment ``` +runas met_test cd /home/met_test # git clone not necessary if repo is already available @@ -213,50 +228,57 @@ git clone https://github.com/dtcenter/METcalcpy cd /home/met_test/METplotpy git checkout develop -/home/met_test/.conda/envs/weatherregime/bin/pip3 install . +/home/met_test/.conda/envs/weatherregime.v5/bin/pip3 install . cd /home/met_test/METcalcpy git checkout develop -/home/met_test/.conda/envs/weatherregime/bin/pip3 install . +/home/met_test/.conda/envs/weatherregime.v5/bin/pip3 install . + +exit ``` -## cycloneplotter (from metplus_base) + +## cycloneplotter.v5 (from metplus_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:cycloneplotter --build-arg ENV_NAME=cycloneplotter . -docker push dtcenter/metplus-envs:cycloneplotter +docker build -t dtcenter/metplus-envs:cycloneplotter.v5 --build-arg ENV_NAME=cycloneplotter . +docker push dtcenter/metplus-envs:cycloneplotter.v5 ``` ### Local ``` -./scripts/cycloneplotter_env.sh metplus_base +./scripts/cycloneplotter_env.sh +/home/met_test/.conda/envs/cycloneplotter.v5/bin/python3 cartopy_feature_download.py cultural physical +rm cartopy_feature_download.py ``` -## icecover (from py_embed_base) + +## icecover.v5 (from py_embed_base.v5) ### Docker ``` -docker build -t dtcenter/metplus-envs:icecover --build-arg BASE_ENV=py_embed_base --build-arg ENV_NAME=icecover . -docker push dtcenter/metplus-envs:icecover +docker build -t dtcenter/metplus-envs:icecover.v5 --build-arg BASE_ENV=py_embed_base.v5 --build-arg ENV_NAME=icecover . +docker push dtcenter/metplus-envs:icecover.v5 ``` ### Local ``` -./scripts/icecover_env.sh py_embed_base +./scripts/icecover_env.sh ``` -## gempak (from metplus_base using Dockerfile.gempak_env) + +## gempak.v5 (from metplus_base.v5 using Dockerfile.gempak_env) ### Docker ``` -docker build -t dtcenter/metplus-envs:gempak --build-arg ENV_NAME=gempak -f ./Dockerfile.gempak_env . -docker push dtcenter/metplus-envs:gempak +docker build -t dtcenter/metplus-envs:gempak.v5 --build-arg ENV_NAME=gempak -f ./Dockerfile.gempak_env . +docker push dtcenter/metplus-envs:gempak.v5 ``` ### Local @@ -266,13 +288,14 @@ GempakToCF.jar file that is required to run use cases that read GEMPAK data. If this environment locally, Java should be available on the system you are running and the JAR file should be downloaded from the DTC website. -## gfdl-tracker (using Dockerfile.gfdl-tracker) + +## gfdl-tracker.v5 (using Dockerfile.gfdl-tracker) ### Docker ``` -docker build -t dtcenter/metplus-envs:gfdl-tracker -f ./Dockerfile.gfdl-tracker . -docker push dtcenter/metplus-envs:gfdl-tracker +docker build -t dtcenter/metplus-envs:gfdl-tracker.v5 -f ./Dockerfile.gfdl-tracker . +docker push dtcenter/metplus-envs:gfdl-tracker.v5 ``` ### Local @@ -284,7 +307,7 @@ to install the GFDL Tracker using these steps. Please consult the documentation for the tool for installation instructions. -## pytest (from metplus_base) +## pytest.v5 (from metplus_base.v5) This environment is used in automation to run the pytests. It requires all of the packages needed to run all of the METplus wrappers, the pytest package and the pytest @@ -293,11 +316,12 @@ code coverage package. ### Docker ``` -docker build -t dtcenter/metplus-envs:pytest --build-arg ENV_NAME=pytest . -docker push dtcenter/metplus-envs:pytest +docker build -t dtcenter/metplus-envs:pytest.v5 --build-arg ENV_NAME=pytest . +docker push dtcenter/metplus-envs:pytest.v5 ``` -## diff (from netcdf4) + +## diff.v5 (from netcdf4.v5) This environment is used to run the difference tests to compare output data to output generated in previous runs to ensure that changes to the code base do not break or change @@ -306,6 +330,6 @@ the results. ### Docker ``` -docker build -t dtcenter/metplus-envs:diff --build-arg BASE_ENV=netcdf4 --build-arg ENV_NAME=diff . -docker push dtcenter/metplus-envs:diff +docker build -t dtcenter/metplus-envs:diff.v5 --build-arg BASE_ENV=netcdf4.v5 --build-arg ENV_NAME=diff . +docker push dtcenter/metplus-envs:diff.v5 ``` diff --git a/scripts/docker/docker_env/scripts/cfgrib_env.sh b/scripts/docker/docker_env/scripts/cfgrib_env.sh index e8d836e9b..7d8687ba2 100755 --- a/scripts/docker/docker_env/scripts/cfgrib_env.sh +++ b/scripts/docker/docker_env/scripts/cfgrib_env.sh @@ -1,24 +1,24 @@ #! /bin/sh ################################################################################ -# Environment: cfgrib -# Last Updated: 2021-07-20 (mccabe@ucar.edu) +# Environment: cfgrib.v5 +# Last Updated: 2022-06-16 (mccabe@ucar.edu) # Notes: Adds Python packages needed to read GRIB data into Xarray and # so it can easily be processed with MetPy # Python Packages: -# metpy==1.0.1 -# netcdf4==1.5.6 -# cfgrib==0.9.9.0 -# pygrib==2.1.3 +# metpy==1.3.0 +# netcdf4==1.5.8 +# cfgrib==0.9.10.1 +# pygrib==2.1.4 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=cfgrib +ENV_NAME=cfgrib.v5 -conda create -y --name ${ENV_NAME} python=3.8.8 -conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.0.1 -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.6 -conda install -y --name ${ENV_NAME} -c conda-forge cfgrib==0.9.9.0 -conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.3 +conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6 +conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8 +conda install -y --name ${ENV_NAME} -c conda-forge cfgrib==0.9.10.1 +conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 diff --git a/scripts/docker/docker_env/scripts/cycloneplotter_env.sh b/scripts/docker/docker_env/scripts/cycloneplotter_env.sh index 17706c39d..01ee3e7d4 100755 --- a/scripts/docker/docker_env/scripts/cycloneplotter_env.sh +++ b/scripts/docker/docker_env/scripts/cycloneplotter_env.sh @@ -2,27 +2,27 @@ ################################################################################ # Environment: cycloneplotter -# Last Updated: 2021-06-09 (mccabe@ucar.edu) +# Last Updated: 2022-07-08 (mccabe@ucar.edu) # Notes: Adds packages needed to run CyclonePlotter wrapper # Added pandas because it is used by tc_and_extra_tc use case # Python Packages: -# cartopy==0.17.0 -# matplotlib==3.3.0 -# pandas==? +# cartopy==0.20.3 +# matplotlib==3.5.2 +# pandas==1.4.3 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=cycloneplotter +ENV_NAME=cycloneplotter.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.17.0 -conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.3.0 -conda install -y --name ${ENV_NAME} -c conda-forge pandas +conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.20.3 +conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2 +conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.4.3 yum -y install wget wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py diff --git a/scripts/docker/docker_env/scripts/diff_env.sh b/scripts/docker/docker_env/scripts/diff_env.sh index f74f43087..bc019f39d 100755 --- a/scripts/docker/docker_env/scripts/diff_env.sh +++ b/scripts/docker/docker_env/scripts/diff_env.sh @@ -1,27 +1,27 @@ #! /bin/sh ################################################################################ -# Environment: diff -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Environment: diff.v5 +# Last Updated: 2022-07-08 (mccabe@ucar.edu) # Notes: Adds packages needed to run differences tests to compare output to # truth data. # Python Packages: -# pillow==? -# pdf2image==? +# pillow==9.2.0 +# pdf2image==1.16.0 # # Other Content: # poppler-utils ################################################################################ # Conda environment to create -ENV_NAME=diff +ENV_NAME=diff.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=netcdf4.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge pillow +conda install -y --name ${ENV_NAME} -c conda-forge pillow==9.2.0 yum -y install poppler-utils -conda install -y --name ${ENV_NAME} -c conda-forge pdf2image +conda install -y --name ${ENV_NAME} -c conda-forge pdf2image==1.16.0 diff --git a/scripts/docker/docker_env/scripts/gempak_env.sh b/scripts/docker/docker_env/scripts/gempak_env.sh index ceaa6e20a..cd75f53e3 100755 --- a/scripts/docker/docker_env/scripts/gempak_env.sh +++ b/scripts/docker/docker_env/scripts/gempak_env.sh @@ -1,8 +1,8 @@ #! /bin/sh ################################################################################ -# Environment: gempak -# Last Updated: 2021-06-22 (mccabe@ucar.edu) +# Environment: gempak.v5 +# Last Updated: 2021-07-08 (mccabe@ucar.edu) # Notes: Installs Java and obtains GempakToCF.jar to convert GEMPAK # files to NetCDF format. # Python Packages: None diff --git a/scripts/docker/docker_env/scripts/h5py_env.sh b/scripts/docker/docker_env/scripts/h5py_env.sh index 54c220c9b..15f67b994 100755 --- a/scripts/docker/docker_env/scripts/h5py_env.sh +++ b/scripts/docker/docker_env/scripts/h5py_env.sh @@ -1,20 +1,20 @@ #! /bin/sh ################################################################################ -# Environment: h5py -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Environment: h5py.v5 +# Last Updated: 2022-06-15 (mccabe@ucar.edu) # Notes: Adds Python interface to the HDF5 binary format # Python Packages: -# h5py==2.10.0 +# h5py==3.6.0 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=h5py +ENV_NAME=h5py.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=py_embed_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge h5py==2.10.0 +conda install -y --name ${ENV_NAME} -c conda-forge h5py==3.6.0 diff --git a/scripts/docker/docker_env/scripts/icecover_env.sh b/scripts/docker/docker_env/scripts/icecover_env.sh index eeeaff193..60c48faf4 100755 --- a/scripts/docker/docker_env/scripts/icecover_env.sh +++ b/scripts/docker/docker_env/scripts/icecover_env.sh @@ -2,25 +2,25 @@ ################################################################################ # Environment: icecover -# Last Updated: 2021-06-29 (mccabe@ucar.edu) +# Last Updated: 2022-07-08 (mccabe@ucar.edu) # Notes: Adds Python packages required for ice cover use case # Python Packages: -# pyproj==3.0.1 -# pyresample==1.20.0 -# scikit-learn==0.24.2 +# xarray==2022.3.0 +# pyresample==1.24.1 +# scikit-learn==1.1.1 +# pyproj==3.3.1 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=icecover +ENV_NAME=icecover.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=py_embed_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge xarray==0.18.2 -conda install -y --name ${ENV_NAME} -c conda-forge pyresample==1.16.0 -conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==0.23.2 -#conda install -y --name ${ENV_NAME} -c conda-forge pyproj==3.0.1 -conda install -y --name ${ENV_NAME} -c conda-forge pyproj +conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge pyresample==1.24.1 +conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1 +conda install -y --name ${ENV_NAME} -c conda-forge pyproj==3.3.1 diff --git a/scripts/docker/docker_env/scripts/metdataio_env.sh b/scripts/docker/docker_env/scripts/metdataio_env.sh index b0808167e..4ebd3c4b0 100755 --- a/scripts/docker/docker_env/scripts/metdataio_env.sh +++ b/scripts/docker/docker_env/scripts/metdataio_env.sh @@ -1,25 +1,25 @@ #! /bin/sh ################################################################################ -# Environment: metdataio +# Environment: metdataio.v5 # Last Updated: 2022-07-13 (mccabe@ucar.edu) # Notes: Adds Python packages needed to run METdbLoad from METdataio # Python Packages: -# lxml==3.8.0 +# lxml==4.9.1 # pymysql==1.0.2 -# pandas==1.1.4 +# pandas==1.4.3 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=metdataio +ENV_NAME=metdataio.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge lxml==3.8.0 +conda install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1 conda install -y --name ${ENV_NAME} -c conda-forge pymysql==1.0.2 -conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.1.4 +conda install -y --name ${ENV_NAME} -c conda-forge pandas==1.4.3 diff --git a/scripts/docker/docker_env/scripts/metplotpy_env.sh b/scripts/docker/docker_env/scripts/metplotpy_env.sh index f48da6c93..abe8d85b4 100755 --- a/scripts/docker/docker_env/scripts/metplotpy_env.sh +++ b/scripts/docker/docker_env/scripts/metplotpy_env.sh @@ -2,45 +2,48 @@ ################################################################################ # Environment: metplotpy -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Last Updated: 2022-06-14 (mccabe@ucar.edu) # Notes: Adds Python packages needed to run METplotpy and METcalcpy -# Uses pip to install kaleido because -# could not install via Conda (glibc conflict) # Python Packages: -# matplotlib==3.3.0 -# scipy==1.5.1 -# plotly==4.9.0 -# cartopy==0.18.0 -# eofs==1.3.0 -# cmocean==2.0 -# xarray==0.17 -# netcdf4==1.5.6 -# pyyaml==? +# matplotlib==3.5.2 +# scipy==1.8.1 +# plotly==5.9.0 +# xarray==2022.3.0 +# netcdf4==1.6.0 +# pyyaml==6.0 +# statsmodels==0.13.2 # python-kaleido==0.2.1 +# imageio==2.19.3 +# imutils==0.5.4 +# scikit-image==0.19.3 +# pint==0.19.2 +# metpy=1.3.1 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=metplotpy +ENV_NAME=metplotpy.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.3.0 -conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.5.1 -conda install -y --name ${ENV_NAME} -c conda-forge plotly==4.9.0 -conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.18.0 -conda install -y --name ${ENV_NAME} -c conda-forge eofs==1.3.0 -conda install -y --name ${ENV_NAME} -c conda-forge cmocean==2.0 -conda install -y --name ${ENV_NAME} -c conda-forge xarray==0.17 -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.6 -conda install -y --name ${ENV_NAME} -c conda-forge pyyaml -conda install -y --name ${ENV_NAME} -c conda-forge statsmodels==0.11.1 -/usr/local/envs/${ENV_NAME}/bin/pip3 install kaleido==0.2.1 +conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2 +conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.8.1 +conda install -y --name ${ENV_NAME} -c conda-forge plotly==5.9.0 +conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.6.0 +conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0 +conda install -y --name ${ENV_NAME} -c conda-forge statsmodels==0.13.2 +conda install -y --name ${ENV_NAME} -c conda-forge python-kaleido==0.2.1 +conda install -y --name ${ENV_NAME} -c conda-forge imageio==2.19.3 +conda install -y --name ${ENV_NAME} -c conda-forge imutils==0.5.4 +conda install -y --name ${ENV_NAME} -c conda-forge scikit-image==0.19.3 +conda install -y --name ${ENV_NAME} -c conda-forge pint==0.19.2 +conda install -y --name ${ENV_NAME} -c conda-forge metpy=1.3.1 yum -y install wget wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py diff --git a/scripts/docker/docker_env/scripts/metplus_base_env.sh b/scripts/docker/docker_env/scripts/metplus_base_env.sh index 3c07a20ce..ad89bc1e5 100755 --- a/scripts/docker/docker_env/scripts/metplus_base_env.sh +++ b/scripts/docker/docker_env/scripts/metplus_base_env.sh @@ -1,18 +1,18 @@ #! /bin/sh ################################################################################ -# Environment: metplus_base -# Last Updated: 2021-07-06 (mccabe@ucar.edu) +# Environment: metplus_base (v5) +# Last Updated: 2022-05-12 (mccabe@ucar.edu) # Notes: Move logic to create METplus base env to script so it can be called # on a local machine to create the environment # Python Packages: -# python-dateutil==2.8.1 +# python-dateutil==2.8.2 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=metplus_base +ENV_NAME=metplus_base.v5 -conda create -y --name ${ENV_NAME} python=3.6.8 -conda install -y --name ${ENV_NAME} -c conda-forge python-dateutil==2.8.1 +conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6 +conda install -y --name ${ENV_NAME} -c conda-forge python-dateutil==2.8.2 diff --git a/scripts/docker/docker_env/scripts/netcdf4_env.sh b/scripts/docker/docker_env/scripts/netcdf4_env.sh index 43e4a98f4..e04e783a3 100755 --- a/scripts/docker/docker_env/scripts/netcdf4_env.sh +++ b/scripts/docker/docker_env/scripts/netcdf4_env.sh @@ -2,19 +2,19 @@ ################################################################################ # Environment: netcdf4 -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Last Updated: 2022-06-16 (mccabe@ucar.edu) # Notes: Adds NetCDF4 Python package # Python Packages: -# netcdf4==1.5.6 +# netcdf4==1.5.8 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=netcdf4 +ENV_NAME=netcdf4.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.6 +conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8 diff --git a/scripts/docker/docker_env/scripts/py_embed_base_env.sh b/scripts/docker/docker_env/scripts/py_embed_base_env.sh index e415da858..10bbfc853 100755 --- a/scripts/docker/docker_env/scripts/py_embed_base_env.sh +++ b/scripts/docker/docker_env/scripts/py_embed_base_env.sh @@ -1,19 +1,20 @@ #! /bin/sh ################################################################################ -# Environment: metplus_base -# Last Updated: 2021-07-06 (mccabe@ucar.edu) +# Environment: metplus_base.v5 +# Last Updated: 2022-06-15 (mccabe@ucar.edu) # Notes: Move logic to create METplus base env to script so it can be called # on a local machine to create the environment # Python Packages: -# python-dateutil==2.8.1 +# xarray==2022.3.0 +# netcdf4==1.5.8 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=py_embed_base +ENV_NAME=py_embed_base.v5 -conda create -y --name ${ENV_NAME} python=3.6.8 -conda install -y --name ${ENV_NAME} -c conda-forge xarray==0.16.2 -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.6 +conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6 +conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8 diff --git a/scripts/docker/docker_env/scripts/pygrib_env.sh b/scripts/docker/docker_env/scripts/pygrib_env.sh index 7a74287b8..8fe55adf3 100755 --- a/scripts/docker/docker_env/scripts/pygrib_env.sh +++ b/scripts/docker/docker_env/scripts/pygrib_env.sh @@ -1,24 +1,24 @@ #! /bin/sh ################################################################################ -# Environment: pygrib -# Last Updated: 2021-06-18 (mccabe@ucar.edu) +# Environment: pygrib.v5 +# Last Updated: 2022-06-16 (mccabe@ucar.edu) # Notes: Adds Python packages needed to read GRIB data # Python Packages: -# pygrib==2.0.2 -# metpy==1.0.1 +# pygrib==2.1.4 +# metpy==1.3.0 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=pygrib +ENV_NAME=pygrib.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=py_embed_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.0.2 -conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.0.1 +conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 +conda install -y --name ${ENV_NAME} -c conda-forge metpy==1.3.0 diff --git a/scripts/docker/docker_env/scripts/pytest_env.sh b/scripts/docker/docker_env/scripts/pytest_env.sh index e549f9f94..fd66bfc5f 100755 --- a/scripts/docker/docker_env/scripts/pytest_env.sh +++ b/scripts/docker/docker_env/scripts/pytest_env.sh @@ -1,12 +1,13 @@ #! /bin/sh ################################################################################ -# Environment: pytest -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Environment: pytest.v5 +# Last Updated: 2022-07-08 (mccabe@ucar.edu) # Notes: Adds pytest and pytest coverage packages to run unit tests # Added pandas because plot_util test needs it # Added netcdf4 because SeriesAnalysis test needs it # Python Packages: +# TODO: update version numbers # pytest==? # pytest-cov==? # pandas==? @@ -15,10 +16,10 @@ ################################################################################ # Conda environment to create -ENV_NAME=pytest +ENV_NAME=pytest.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} conda install -y --name ${ENV_NAME} -c conda-forge pytest diff --git a/scripts/docker/docker_env/scripts/spacetime_env.sh b/scripts/docker/docker_env/scripts/spacetime_env.sh index d1da9ef93..e0ec221c3 100755 --- a/scripts/docker/docker_env/scripts/spacetime_env.sh +++ b/scripts/docker/docker_env/scripts/spacetime_env.sh @@ -2,35 +2,27 @@ ################################################################################ # Environment: spacetime -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Last Updated: 2022-06-16 (mccabe@ucar.edu) # Notes: Adds Python packages needed to generate coherence spectra (METplotpy) # Python Packages: -# netCDF4==1.5.4 -# xarray==0.18.2 -# scipy==1.5.3 -# matplotlib==3.2.2 +# netCDF4==1.5.8 +# xarray==2022.3.0 +# scipy==1.8.1 +# matplotlib==3.5.2 # pyngl==1.6.1 -# pyyaml==5.3.1 +# pyyaml==6.0 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=spacetime +ENV_NAME=spacetime.v5 -# Conda environment to use as base for new environment -# Not used in this script because Python version differs from base version -BASE_ENV=$1 +conda create -y --name ${ENV_NAME} -c conda-forge python=3.8.6 - -conda create -y --name ${ENV_NAME} python=3.8 - -conda install -y --name ${ENV_NAME} -c conda-forge netCDF4==1.5.4 -conda install -y --name ${ENV_NAME} -c conda-forge xarray==0.18.2 -conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.5.3 -conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.2.2 +conda install -y --name ${ENV_NAME} -c conda-forge netCDF4==1.5.8 +conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.8.1 +conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.5.2 conda install -y --name ${ENV_NAME} -c conda-forge pyngl==1.6.1 -conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==5.3.1 - -# tested using anaconda channel but changed to using conda-forge -#conda install -y --name ${ENV_NAME} -c anaconda pyyaml +conda install -y --name ${ENV_NAME} -c conda-forge pyyaml==6.0 diff --git a/scripts/docker/docker_env/scripts/weatherregime_env.sh b/scripts/docker/docker_env/scripts/weatherregime_env.sh index 0f3a2b842..92c70a698 100755 --- a/scripts/docker/docker_env/scripts/weatherregime_env.sh +++ b/scripts/docker/docker_env/scripts/weatherregime_env.sh @@ -1,28 +1,30 @@ #! /bin/sh ################################################################################ -# Environment: weatherregime -# Last Updated: 2021-09-16 (mccabe@ucar.edu) +# Environment: weatherregime.v5 +# Last Updated: 2022-07-19 (mccabe@ucar.edu) # Notes: Adds Python packages needed to run weather regime use case # METplotpy and METcalcpy # Python Packages: -# All packages from metplotpy_env -# scikit-learn==0.24.2 +# All packages from metplotpy.v5 +# scikit-learn==1.1.1 # eofs==1.4.0 +# cmocean==2.0 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=weatherregime +ENV_NAME=weatherregime.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplotpy.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==0.24.2 +conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==1.1.1 conda install -y --name ${ENV_NAME} -c conda-forge eofs==1.4.0 +conda install -y --name ${ENV_NAME} -c conda-forge cmocean==2.0 rm cartopy_feature_download.py yum -y install wget diff --git a/scripts/docker/docker_env/scripts/xesmf_env.sh b/scripts/docker/docker_env/scripts/xesmf_env.sh index 44e5a084f..fc1c7807c 100755 --- a/scripts/docker/docker_env/scripts/xesmf_env.sh +++ b/scripts/docker/docker_env/scripts/xesmf_env.sh @@ -2,21 +2,23 @@ ################################################################################ # Environment: xesmf -# Last Updated: 2021-06-08 (mccabe@ucar.edu) +# Last Updated: 2022-06-16 (mccabe@ucar.edu) # Notes: Adds Python package to read Tripolar grids # Python Packages: +# netcdf4==1.5.8 +# xarray==2022.3.0 # xesmf==0.3.0 # # Other Content: None ################################################################################ # Conda environment to create -ENV_NAME=xesmf +ENV_NAME=xesmf.v5 # Conda environment to use as base for new environment -BASE_ENV=$1 +BASE_ENV=metplus_base.v5 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4 -conda install -y --name ${ENV_NAME} -c conda-forge xarray -conda install -y --name ${ENV_NAME} -c conda-forge xesmf +conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.8 +conda install -y --name ${ENV_NAME} -c conda-forge xarray==2022.3.0 +conda install -y --name ${ENV_NAME} -c conda-forge xesmf==0.3.0