Update Develop-ref after #1710 (#1714)

Co-authored-by: George McCabe <[email protected]>
Co-authored-by: Mrinal Biswas <[email protected]>
Co-authored-by: Hank Fisher <[email protected]>
Co-authored-by: Minna Win <[email protected]>
Co-authored-by: bikegeek <[email protected]>
Co-authored-by: Christina Kalb <[email protected]>
Co-authored-by: Julie Prestopnik <[email protected]>
Co-authored-by: johnhg <[email protected]>
Co-authored-by: Hank Fisher <[email protected]>
Co-authored-by: Mrinal Biswas <[email protected]>
Co-authored-by: jprestop <[email protected]>
Co-authored-by: j-opatz <[email protected]>
Co-authored-by: Kathryn Newman <[email protected]>
Co-authored-by: j-opatz <[email protected]>
Co-authored-by: Lisa Goodrich <[email protected]>
Co-authored-by: Julie Prestopnik <[email protected]>
Co-authored-by: Christina Kalb <[email protected]>
Co-authored-by: bikegeek <[email protected]>
Co-authored-by: Hank Fisher <[email protected]>
Co-authored-by: Christina Kalb <[email protected]>
Co-authored-by: lisagoodrich <[email protected]>
Co-authored-by: Howard Soh <[email protected]>
Co-authored-by: Molly Smith <[email protected]>
Co-authored-by: hsoh-u <[email protected]>

.github/actions/run_tests/Dockerfile.run

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=metplus_base
+ARG METPLUS_ENV_TAG=metplus_base.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/actions/run_tests/Dockerfile.run_cartopy

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=cycloneplotter
+ARG METPLUS_ENV_TAG=cycloneplotter.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/jobs/docker_utils.py

@@ -12,6 +12,9 @@
 # repository used for storing input data for development branches
 DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev'
 
+# extension to add to conda environments
+VERSION_EXT = '.v5'
+
 def get_data_repo(branch_name):
     """! Branch names that start with main_v or contain only
        digits and dots with out without a prefix 'v' will return

.github/jobs/get_requirements.py

@@ -8,11 +8,11 @@
 
 import get_use_case_commands
 
-# add internal_tests/use_cases directory to path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+# add METplus directory to path so the test suite can be found
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 

.github/jobs/get_use_case_commands.py

@@ -8,17 +8,22 @@
 import os
 
 # add METplus directory to sys path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 from metplus.util.met_util import expand_int_string_to_list
+from docker_utils import VERSION_EXT
 
-METPLUS_BASE_ENV = 'metplus_base'
+
+# path to METplus install location in Docker
 METPLUS_DOCKER_LOC = '/metplus/METplus'
 
+# name of conda environment used for cases that don't need special env
+METPLUS_BASE_ENV = 'metplus_base'
+
 # keywords in requirements list that trigger obtaining METcalcpy and METplotpy
 PLOTCALC_KEYWORDS = [
     'metplotpy',
@@ -27,6 +32,12 @@
     'weatherregime',
 ]
 
+# Docker envs that do not use Python so they do not need print conda list
+NOT_PYTHON_ENVS = [
+    'gfdl-tracker',
+    'gempak',
+]
+
 def handle_automation_env(host_name, reqs, work_dir):
     # if no env is specified, use metplus base environment
     conda_env = METPLUS_BASE_ENV
@@ -44,12 +55,15 @@ def handle_automation_env(host_name, reqs, work_dir):
             return '', 'user_env_vars.MET_PYTHON_EXE=python3'
         return '', ''
 
+    # add version extension to conda environment name
+    conda_env_w_ext = f'{conda_env}{VERSION_EXT}'
+
     # start building commands to run before run_metplus.py in Docker
     setup_env = 'source /etc/bashrc;'
 
     # add conda bin to beginning of PATH
     python_dir = os.path.join('/usr', 'local', 'envs',
-                              conda_env, 'bin')
+                              conda_env_w_ext, 'bin')
     python_path = os.path.join(python_dir, 'python3')
     setup_env += f' export PATH={python_dir}:$PATH;'
 
@@ -92,12 +106,12 @@ def handle_automation_env(host_name, reqs, work_dir):
         setup_env += f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH;'
 
     # list packages in python environment that will be used
-    if conda_env != 'gempak':
+    if conda_env not in NOT_PYTHON_ENVS:
         setup_env += (
-            f'echo Using environment: dtcenter/metplus-envs:{conda_env};'
-            f'echo cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext};'
+            f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             f'echo ----------------------------------------;'
-            f'cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             'echo ----------------------------------------;'
         )
 
@@ -108,7 +122,7 @@ def main(categories, subset_list, work_dir=None,
     all_commands = []
 
     if work_dir is None:
-        work_dir = USE_CASES_DIR
+        work_dir = METPLUS_TOP_DIR
 
     test_suite = METplusUseCaseSuite()
     test_suite.add_use_case_groups(categories, subset_list)

.github/jobs/setup_and_run_diff.py

@@ -5,6 +5,8 @@
 import subprocess
 import shlex
 
+from docker_utils import VERSION_EXT
+
 ci_dir = os.path.join(os.environ.get('GITHUB_WORKSPACE'), '.github')
 sys.path.insert(0, ci_dir)
 
@@ -46,12 +48,12 @@
 mount_args = ' '.join(volume_mounts)
 
 # command to run inside Docker
-cmd = ('/usr/local/envs/diff/bin/python3 '
+cmd = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 '
        f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py')
 
 # run inside diff env: mount METplus code and output dir, volumes from output volumes
 docker_cmd = (f'docker run -e GITHUB_WORKSPACE {VOLUMES_FROM} '
-              f'{mount_args} dtcenter/metplus-envs:diff '
+              f'{mount_args} dtcenter/metplus-envs:diff{VERSION_EXT} '
               f'bash -c "{cmd}"')
 print(f'RUNNING: {docker_cmd}')
 try:

.github/jobs/setup_and_run_use_cases.py

@@ -16,7 +16,7 @@
 
 import get_use_case_commands
 import get_data_volumes
-from docker_utils import get_branch_name
+from docker_utils import get_branch_name, VERSION_EXT
 
 runner_workspace = os.environ.get('RUNNER_WORKSPACE')
 github_workspace = os.environ.get('GITHUB_WORKSPACE')
@@ -70,6 +70,8 @@ def main():
         else:
             env_tag = 'metplus_base'
 
+        env_tag = f'{env_tag}{VERSION_EXT}'
+
         # get Dockerfile to use
         dockerfile_name = 'Dockerfile.run'
         if 'gempak' in str(requirements).lower():

docs/Contributors_Guide/add_use_case.rst

@@ -108,7 +108,7 @@ configuration file name excluding the .conf suffix.
 .. figure:: figure/model_applications_example.png
 
 .. figure:: figure/model_applications_subdir.png
-
+	    
 Use Case Rules
 --------------