diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 7dda2d213..3bfad8834 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then for x in `cat $PYTESTS_GROUPS_FILEPATH`; do marker="${x//_or_/ or }" marker="${marker//not_/not }" - command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../metplus -m \"$marker\"" + command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../../metplus -m \"$marker\"" command+=";if [ \$? != 0 ]; then status=1; fi;" done command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" diff --git a/.github/jobs/create_output_data_volumes.sh b/.github/jobs/create_output_data_volumes.sh index 38f55b7de..abf3e7e8a 100755 --- a/.github/jobs/create_output_data_volumes.sh +++ b/.github/jobs/create_output_data_volumes.sh @@ -28,7 +28,7 @@ if [ $? != 0 ]; then exit 1 fi -docker_data_output_dir=scripts/docker/docker_data_output +docker_data_output_dir=.github/jobs/docker_data_output success=1 for vol_name in use_cases_*; do diff --git a/scripts/docker/docker_data_output/Dockerfile b/.github/jobs/docker_data_output/Dockerfile similarity index 100% rename from scripts/docker/docker_data_output/Dockerfile rename to .github/jobs/docker_data_output/Dockerfile diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 31b996c9a..ad20f64ec 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -29,9 +29,9 @@ duration=$(( SECONDS - start_seconds )) echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS)" # set DOCKERFILE_PATH that is used by docker hook script get_met_version -export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/scripts/docker/Dockerfile +export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/scripts/docker/hooks/get_met_version` +MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version` MET_DOCKER_REPO=met-dev if [ "${MET_TAG}" != "develop" ]; then diff --git a/.github/jobs/docker_update_data_volumes.py b/.github/jobs/docker_update_data_volumes.py deleted file mode 100755 index d19d731e4..000000000 --- a/.github/jobs/docker_update_data_volumes.py +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/env python3 - -# Run by GitHub Actions (in .github/workflows/testing.yml) check DTCenter web -# server for any input data tarfiles that have been updated and need to be -# regenerated as Docker data volumes to be used in use case tests. -# Push new/updated data volumes up to DockerHub - -import sys -import os -import shlex -import requests -from bs4 import BeautifulSoup -import dateutil.parser -from urllib.parse import urljoin -import subprocess - -from docker_utils import docker_get_volumes_last_updated, get_data_repo -from docker_utils import get_branch_name - -# URL containing METplus sample data tarfiles -WEB_DATA_DIR = 'https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data/' - -# path to script that builds docker data volumes -BUILD_DOCKER_IMAGES = os.path.join(os.environ.get('GITHUB_WORKSPACE', ''), - 'scripts', - 'docker', - 'docker_data', - 'build_docker_images.sh') - -def get_tarfile_last_modified(search_dir): - # get list of tarfiles from website - soup = BeautifulSoup(requests.get(search_dir).content, - 'html.parser') - tarfiles = [a_tag.get_text() for a_tag in soup.find_all('a') - if a_tag.get_text().startswith('sample_data')] - - # get last modified time of each tarfile - tarfile_last_modified = {} - for tarfile in tarfiles: - tarfile_url = urljoin(search_dir+'/', tarfile) - last_modified = requests.head(tarfile_url).headers['last-modified'] - tarfile_last_modified[tarfile] = last_modified - - return tarfile_last_modified - -def create_data_volumes(branch_name, volumes): - if not volumes: - print("No volumes to build") - return True - - data_repo = get_data_repo(branch_name) - # log into docker using encrypted credentials and - # call build_docker_images.sh script - cmd = (f'{BUILD_DOCKER_IMAGES} -pull {branch_name} ' - f'-data {",".join(volumes)} -push {data_repo}') - print(f'Running command: {cmd}') - ret = subprocess.run(shlex.split(cmd), check=True) - - if ret.returncode: - print(f'ERROR: Command failed: {cmd}') - return False - - return True - -def main(): - - if not os.environ.get('DOCKER_USERNAME'): - print("DockerHub credentials are not stored. " - "Skipping data volume handling.") - sys.exit(0) - - # check if tarfile directory exists on web - branch_name = get_branch_name() - if not branch_name: - print("Could not get current branch. Exiting.") - sys.exit(1) - - if branch_name.endswith('-ref'): - branch_name = branch_name[0:-4] - - # search dir should be develop, feature_NNN, or vX.Y - if branch_name.startswith('main_v'): - web_subdir = branch_name[5:] - else: - web_subdir = branch_name - - if not os.environ.get('GITHUB_WORKSPACE'): - print("GITHUB_WORKSPACE not set. Exiting.") - sys.exit(1) - - search_dir = f"{urljoin(WEB_DATA_DIR, web_subdir)}/" - print(f"Looking for tarfiles in {search_dir}") - - dir_request = requests.get(search_dir) - # if it does not exist, exit script - if dir_request.status_code != 200: - print(f'URL does not exist: {search_dir}') - sys.exit(0) - - # get last modified time of each tarfile - tarfile_last_modified = get_tarfile_last_modified(search_dir) - - volumes_last_updated = docker_get_volumes_last_updated(branch_name) - - # check status of each tarfile and add them to the list of volumes to create if needed - volumes_to_create = [] - for tarfile, last_modified in tarfile_last_modified.items(): - category = os.path.splitext(tarfile)[0].split('-')[1] - print(f"Checking tarfile: {category}") - - volume_name = f'{branch_name}-{category}' - # if the data volume does not exist, create it and push it to DockerHub - if not volume_name in volumes_last_updated.keys(): - print(f'{volume_name} data volume does not exist. Creating data volume.') - volumes_to_create.append(category) - continue - - # if data volume does exist, get last updated time of volume and compare to - # tarfile last modified. if any tarfile was modified after creation of - # corresponding volume, recreate those data volumes - volume_dt = dateutil.parser.parse(volumes_last_updated[volume_name]) - tarfile_dt = dateutil.parser.parse(last_modified) - - print(f"Volume time: {volume_dt.strftime('%Y%m%d %H:%M:%S')}") - print(f"Tarfile time: {tarfile_dt.strftime('%Y%m%d %H:%M:%S')}") - - # if the tarfile has been modified more recently than the data volume was created, - # recreate the data volume - if volume_dt < tarfile_dt: - print(f'{tarfile} has changed since {volume_name} was created. ' - 'Regenerating data volume.') - volumes_to_create.append(category) - - if not create_data_volumes(branch_name, volumes_to_create): - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 04920faa9..b4de2094b 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -14,7 +14,7 @@ sys.path.insert(0, METPLUS_TOP_DIR) from internal.tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite -from metplus.util.met_util import expand_int_string_to_list +from metplus.util.string_manip import expand_int_string_to_list from docker_utils import VERSION_EXT diff --git a/.github/parm/pytest_groups.txt b/.github/parm/pytest_groups.txt index 48bf9bd41..a5ca80e66 100644 --- a/.github/parm/pytest_groups.txt +++ b/.github/parm/pytest_groups.txt @@ -1,3 +1,4 @@ +run_metplus util wrapper wrapper_a diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index be6f54633..2f6814059 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -74,6 +74,11 @@ "index_list": "6", "run": false }, + { + "category": "marine_and_cryosphere", + "index_list": "7", + "run": false + }, { "category": "medium_range", "index_list": "0", @@ -134,6 +139,11 @@ "index_list": "8", "run": false }, + { + "category": "precipitation", + "index_list": "9", + "run": false + }, { "category": "s2s", "index_list": "0", diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index bed495e39..fad214a53 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -685,7 +685,7 @@ images using Conda that can be used to run use cases. These images are stored on DockerHub in *dtcenter/metplus-envs* and are named with a tag that corresponds to the keyword without the **_env** suffix. The environments were created using Docker commands via scripts that are found -in *scripts/docker/docker_env*. +in *internal/scripts/docker_env*. Existing keywords that set up Conda environments used for use cases are: * cfgrib_env @@ -793,14 +793,14 @@ packages take a very long time to install in Docker. The new approach involves creating Docker images that use Conda to create a Python environment that can run the use case. To see what is available in each of the existing Python environments, refer to the comments in the scripts found in -*scripts/docker/docker_env/scripts*. +*internal/scripts/docker_env/scripts*. New environments must be added by a METplus developer, so please create a discussion on the `METplus GitHub Discussions `_ forum if none of these environments contain the package requirements needed to run a new use case. -A **README.md** file can be found in *scripts/docker/docker_env* that +A **README.md** file can be found in *internal/scripts/docker_env* that provides commands that can be run to recreate a Docker image if the conda environment needs to be updated. Please note that Docker must be installed on the workstation used to create new Docker images and diff --git a/docs/Contributors_Guide/deprecation.rst b/docs/Contributors_Guide/deprecation.rst index 491baef9e..6c6d63e2f 100644 --- a/docs/Contributors_Guide/deprecation.rst +++ b/docs/Contributors_Guide/deprecation.rst @@ -26,7 +26,7 @@ wrong variable and it is using WGRIB2 = wgrib2. check_for_deprecated_config() ----------------------------- -In **met_util.py** there is a function called +In **metplus/util/config_metplus.py** there is a function called check_for_deprecated_config. It contains a dictionary of dictionaries called deprecated_dict that specifies the old config name, the section it was found in, and a suggested alternative (None if no alternative diff --git a/docs/Contributors_Guide/user_support.rst b/docs/Contributors_Guide/user_support.rst index 26611fa92..ecade65d1 100644 --- a/docs/Contributors_Guide/user_support.rst +++ b/docs/Contributors_Guide/user_support.rst @@ -14,6 +14,10 @@ Support Responsibilities ======================== Five staff members take turns monitoring Discussions each day of the week. + +Each day, all support members should follow up on existing Discussions in +which they are assigned (i.e. tagged). + The responsibilities for each daily assignee are described below. diff --git a/docs/Users_Guide/getting_started.rst b/docs/Users_Guide/getting_started.rst index 3aada6c85..7941a97f6 100644 --- a/docs/Users_Guide/getting_started.rst +++ b/docs/Users_Guide/getting_started.rst @@ -327,7 +327,7 @@ user configuration file and The last line of the screen output should match this format:: - 05/04 09:42:52.277 metplus (met_util.py:212) INFO: METplus has successfully finished running. + 05/04 09:42:52.277 metplus INFO: METplus has successfully finished running. If this log message is not shown, there is likely an issue with one or more of the default configuration variable overrides in the @@ -339,7 +339,7 @@ how the :ref:`common_config_variables` control a use case run. If the run was successful, the line above the success message should contain the path to the METplus log file that was generated:: - 05/04 09:44:21.534 metplus (met_util.py:211) INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421 + 05/04 09:44:21.534 metplus INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421 * Review the log file and compare it to the Example.conf use case configuration file to see how the settings correspond to the result. diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index dcde8cefa..18a248e4f 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -221,16 +221,11 @@ The METplus Wrappers source code contains the following directory structure:: METplus/ build_components/ docs/ - environment.yml internal/ manage_exernals/ metplus/ parm/ produtil/ - README.md - requirements.txt - scripts/ - setup.py ush/ The top-level METplus Wrappers directory consists of a README.md file @@ -249,7 +244,8 @@ The Doxygen documentation is useful to contributors and is not necessary for METplus end-users. The **internal/** directory contains scripts that are only -relevant to METplus developers and contributors. +relevant to METplus developers and contributors, such as tests and files +used with Docker. The **manage_externals/** directory contains scripts used to facilitate the downloading and management @@ -262,9 +258,6 @@ METplus Wrappers. The **produtil/** directory contains part of the external utility produtil. -The **scripts/** directory contains scripts that are used for creating -Docker images. - The **ush/** directory contains the run_metplus.py script that is executed to run use cases. diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index d85d9fef8..d3d398b86 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -30,6 +30,63 @@ When applicable, release notes are followed by the `GitHub issue `_) + + * Add unique identifier for each METplus run to configuration (`#1829 `_) + + * StatAnalysis - Support setting multiple jobs (`#1842 `_) + + * StatAnalysis - Set MET verbosity (`#1772 `_) + + * StatAnalysis - Support using both init/valid variables in string substitution (`#1861 `_) + + * StatAnalysis - Allow filename template tags in jobs (`#1862 `_) + + * StatAnalysis - Support looping over groups of list items (`#1870 `_) + + * StatAnalysis - Allow processing of time ranges other than daily (`#1871 `_) + + * StatAnalysis - Add support for using a custom loop list (`#1893 `_) + + * Remove MakePlots wrapper (`#1843 `_) + +* Bugfixes: + + * PCPCombine - custom loop list does not work for subtract method (`#1884 `_) + +* New Wrappers: None + +* New Use Cases: + + * Probability of Exceedence for 85th percentile temperatures (`#1808 `_) + + * FV3 Physics Tendency plotting via METplotpy (`#1852 `_) + +* Documentation: None + +* Internal: + + * Fix GitHub Actions warnings - update the version of actions and replace set-output (`#1863 `_) + + * Update diff logic to handle CSV files that have rounding differences (`#1865 `_) + + * Add unit tests for expected failure (`dtcenter/METplus-Internal#24 `_) + METplus Version 5.0.0 Beta 3 Release Notes (2022-09-21) ------------------------------------------------------- @@ -67,8 +124,7 @@ METplus Version 5.0.0 Beta 3 Release Notes (2022-09-21) * MJO-ENSO diagnostics (`#1330 `_) -* Documentation: - +* Documentation: None * Internal: diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index a52b997d3..17a164b05 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -426,7 +426,7 @@ This defines the format of the ERROR log messages. Setting the value to:: Produces a log file with ERROR lines that match this format:: - 04/29 16:03:34.858 metplus (met_util.py:218) ERROR: METplus has finished running but had 1 error. + 04/29 16:03:34.858 metplus (run_util.py:192) ERROR: METplus has finished running but had 1 error. The format of the timestamp is set by :ref:`LOG_LINE_DATE_FORMAT`. @@ -442,7 +442,7 @@ This defines the format of the DEBUG log messages. Setting the value to:: Produces a log file with DEBUG lines that match this format:: - 04/29 15:54:22.851 metplus (met_util.py:207) DEBUG: METplus took 0:00:00.850983 to run. + 04/29 15:54:22.851 metplus (run_util.py:177) DEBUG: METplus took 0:00:00.850983 to run. The format of the timestamp is set by :ref:`LOG_LINE_DATE_FORMAT`. @@ -2648,9 +2648,9 @@ In most cases, there is a simple one-to-one relationship between a deprecated co Example:: - (met_util.py) ERROR: DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES - (met_util.py) ERROR: [dir] MODEL_DATA_DIR should be replaced with EXTRACT_TILES_GRID_INPUT_DIR - (met_util.py) ERROR: [config] STAT_LIST should be replaced with SERIES_ANALYSIS_STAT_LIST + ERROR: DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES + ERROR: [dir] MODEL_DATA_DIR should be replaced with EXTRACT_TILES_GRID_INPUT_DIR + ERROR: [config] STAT_LIST should be replaced with SERIES_ANALYSIS_STAT_LIST These cases can be handled automatically by using the :ref:`validate_config`. @@ -2666,10 +2666,10 @@ Starting in METplus 3.0, users are required to either explicitly set both FCST_* Example:: - (met_util.py) ERROR: If FCST_VAR1_NAME is set, the user must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME - (met_util.py) ERROR: If FCST_VAR2_NAME is set, the user must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME - (met_util.py) ERROR: If FCST_VAR1_LEVELS is set, the user must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS - (met_util.py) ERROR: If FCST_VAR2_LEVELS is set, the user must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS + ERROR: If FCST_VAR1_NAME is set, the user must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME + ERROR: If FCST_VAR2_NAME is set, the user must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME + ERROR: If FCST_VAR1_LEVELS is set, the user must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS + ERROR: If FCST_VAR2_LEVELS is set, the user must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes, as they may want to update differently. @@ -2682,7 +2682,7 @@ Instead of only being able to specify FCST_PCP_COMBINE_INPUT_LEVEL, users can no Example:: - (met_util.py) ERROR: [config] OBS_PCP_COMBINE_INPUT_LEVEL should be replaced with OBS_PCP_COMBINE_INPUT_ACCUMS + ERROR: [config] OBS_PCP_COMBINE_INPUT_LEVEL should be replaced with OBS_PCP_COMBINE_INPUT_ACCUMS These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes, as they may want to include other available input accumulations. @@ -2719,17 +2719,17 @@ Due to these changes, MET configuration files that refer to any of these depreca Example log output:: - (met_util.py) DEBUG: Checking for deprecated environment variables in: DeprecatedConfig - (met_util.py) ERROR: Please remove deprecated environment variable ${GRID_VX} found in MET config file: DeprecatedConfig - (met_util.py) ERROR: MET to_grid variable should reference ${REGRID_TO_GRID} environment variable - (met_util.py) INFO: Be sure to set GRID_STAT_REGRID_TO_GRID to the correct value. + DEBUG: Checking for deprecated environment variables in: DeprecatedConfig + ERROR: Please remove deprecated environment variable ${GRID_VX} found in MET config file: DeprecatedConfig + ERROR: MET to_grid variable should reference ${REGRID_TO_GRID} environment variable + INFO: Be sure to set GRID_STAT_REGRID_TO_GRID to the correct value. - (met_util.py) ERROR: Please remove deprecated environment variable ${MET_VALID_HHMM} found in MET config file: DeprecatedConfig - (met_util.py) ERROR: Set GRID_STAT_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a METplus config file to set CLIMO_MEAN_FILE in a MET config + ERROR: Please remove deprecated environment variable ${MET_VALID_HHMM} found in MET config file: DeprecatedConfig + ERROR: Set GRID_STAT_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a METplus config file to set CLIMO_MEAN_FILE in a MET config - (met_util.py) ERROR: output_prefix variable should reference ${OUTPUT_PREFIX} environment variable - (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX will need to be added to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to: - (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME} + ERROR: output_prefix variable should reference ${OUTPUT_PREFIX} environment variable + INFO: GRID_STAT_OUTPUT_PREFIX will need to be added to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to: + INFO: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME} These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes and make sure they add the appropriate recommended METplus configuration variables to their files to achieve the same behavior. diff --git a/docs/_static/marine_and_cryosphere-PointStat_fcstGFS_obsNDBC_WaveHeight.png b/docs/_static/marine_and_cryosphere-PointStat_fcstGFS_obsNDBC_WaveHeight.png new file mode 100644 index 000000000..7b66007c0 Binary files /dev/null and b/docs/_static/marine_and_cryosphere-PointStat_fcstGFS_obsNDBC_WaveHeight.png differ diff --git a/docs/_static/precipitation-PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.png b/docs/_static/precipitation-PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.png new file mode 100644 index 000000000..9afa1d5ab Binary files /dev/null and b/docs/_static/precipitation-PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.png differ diff --git a/docs/use_cases/met_tool_wrapper/Example/Example.py b/docs/use_cases/met_tool_wrapper/Example/Example.py index b19ebcf78..3c4a11b43 100644 --- a/docs/use_cases/met_tool_wrapper/Example/Example.py +++ b/docs/use_cases/met_tool_wrapper/Example/Example.py @@ -174,30 +174,30 @@ # # You should also see a series of log output listing init/valid times, forecast lead times, and filenames derived from the filename templates. Here is an excerpt:: # -# 12/30 19:44:02.901 metplus (met_util.py:425) INFO: **************************************** -# 12/30 19:44:02.901 metplus (met_util.py:426) INFO: * Running METplus -# 12/30 19:44:02.902 metplus (met_util.py:432) INFO: * at valid time: 201702010000 -# 12/30 19:44:02.902 metplus (met_util.py:435) INFO: **************************************** -# 12/30 19:44:02.902 metplus.Example (example_wrapper.py:58) INFO: Running ExampleWrapper at valid time 20170201000000 -# 12/30 19:44:02.902 metplus.Example (example_wrapper.py:63) INFO: Input directory is /dir/containing/example/data -# 12/30 19:44:02.902 metplus.Example (example_wrapper.py:64) INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext -# 12/30 19:44:02.902 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 3 hours initialized at 2017-01-31 21Z and valid at 2017-02-01 00Z -# 12/30 19:44:02.903 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_21_F003.ext -# 12/30 19:44:02.903 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 6 hours initialized at 2017-01-31 18Z and valid at 2017-02-01 00Z -# 12/30 19:44:02.903 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_18_F006.ext -# 12/30 19:44:02.904 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 9 hours initialized at 2017-01-31 15Z and valid at 2017-02-01 00Z -# 12/30 19:44:02.904 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_15_F009.ext -# 12/30 19:44:02.904 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 12 hours initialized at 2017-01-31 12Z and valid at 2017-02-01 00Z -# 12/30 19:44:02.904 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_12_F012.ext -# 12/30 19:44:02.904 metplus (met_util.py:425) INFO: **************************************** -# 12/30 19:44:02.904 metplus (met_util.py:426) INFO: * Running METplus -# 12/30 19:44:02.905 metplus (met_util.py:432) INFO: * at valid time: 201702010600 -# 12/30 19:44:02.905 metplus (met_util.py:435) INFO: **************************************** -# 12/30 19:44:02.905 metplus.Example (example_wrapper.py:58) INFO: Running ExampleWrapper at valid time 20170201060000 -# 12/30 19:44:02.905 metplus.Example (example_wrapper.py:63) INFO: Input directory is /dir/containing/example/data -# 12/30 19:44:02.905 metplus.Example (example_wrapper.py:64) INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext -# 12/30 19:44:02.905 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 3 hours initialized at 2017-02-01 03Z and valid at 2017-02-01 06Z -# 12/30 19:44:02.906 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170201/file_20170201_03_F003.ext +# 12/30 19:44:02.901 metplus INFO: **************************************** +# 12/30 19:44:02.901 metplus INFO: * Running METplus +# 12/30 19:44:02.902 metplus INFO: * at valid time: 201702010000 +# 12/30 19:44:02.902 metplus INFO: **************************************** +# 12/30 19:44:02.902 metplus INFO: Running ExampleWrapper at valid time 20170201000000 +# 12/30 19:44:02.902 metplus INFO: Input directory is /dir/containing/example/data +# 12/30 19:44:02.902 metplus INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext +# 12/30 19:44:02.902 metplus INFO: Processing forecast lead 3 hours initialized at 2017-01-31 21Z and valid at 2017-02-01 00Z +# 12/30 19:44:02.903 metplus INFO: Looking in input directory for file: 20170131/file_20170131_21_F003.ext +# 12/30 19:44:02.903 metplus INFO: Processing forecast lead 6 hours initialized at 2017-01-31 18Z and valid at 2017-02-01 00Z +# 12/30 19:44:02.903 metplus INFO: Looking in input directory for file: 20170131/file_20170131_18_F006.ext +# 12/30 19:44:02.904 metplus INFO: Processing forecast lead 9 hours initialized at 2017-01-31 15Z and valid at 2017-02-01 00Z +# 12/30 19:44:02.904 metplus INFO: Looking in input directory for file: 20170131/file_20170131_15_F009.ext +# 12/30 19:44:02.904 metplus INFO: Processing forecast lead 12 hours initialized at 2017-01-31 12Z and valid at 2017-02-01 00Z +# 12/30 19:44:02.904 metplus INFO: Looking in input directory for file: 20170131/file_20170131_12_F012.ext +# 12/30 19:44:02.904 metplus INFO: **************************************** +# 12/30 19:44:02.904 metplus INFO: * Running METplus +# 12/30 19:44:02.905 metplus INFO: * at valid time: 201702010600 +# 12/30 19:44:02.905 metplus INFO: **************************************** +# 12/30 19:44:02.905 metplus INFO: Running ExampleWrapper at valid time 20170201060000 +# 12/30 19:44:02.905 metplus INFO: Input directory is /dir/containing/example/data +# 12/30 19:44:02.905 metplus INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext +# 12/30 19:44:02.905 metplus INFO: Processing forecast lead 3 hours initialized at 2017-02-01 03Z and valid at 2017-02-01 06Z +# 12/30 19:44:02.906 metplus INFO: Looking in input directory for file: 20170201/file_20170201_03_F003.ext # ############################################################################## diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py new file mode 100644 index 000000000..475d02714 --- /dev/null +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py @@ -0,0 +1,141 @@ +""" +PointStat: read in buoy ASCII files to compare to model wave heights +==================================================================== + +model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case utilizes the new ASCII2NC method to natively read in NDBC ASCII files, a common source of sea surface data +# for operational entities. These values are then compared to GFS' new wave height output, which it incorporated from Wave Watch III. + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** GFSv16 forecast data from WAVE file category +# +# | **Observations:** ASCII buoy files from NDBC +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. + +############################################################################## +# METplus Components +# ------------------ +# +# This use case calls ASCII2NC to read in ASCII buoy files and +# then PointStat for verification against GFS model data + +############################################################################## +# METplus Workflow +# ---------------- +# +# ASCII2NC is the first tool called. It pulls in all files with a .txt type, which is +# the ASCII buoy data saved format. These observations are converted into a netCDF, which is then called by PointStat +# as the observation dataset. PointStat also pulls in a 3 hour forecast from the GFS for wave heights, which is included +# in the range of available buoy observation times. A +/- 30 minute window is allowed for the observational data. +# Thresholds are set that correspond to operational usage, and the CTC and CTS line types are requested. +# It processes the following run time: +# +# | **Valid:** 2022-10-16 09Z +# | + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf + +############################################################################## +# MET Configuration +# --------------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/Ascii2NcConfig_wrapped +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/PointStatConfig_wrapped + +############################################################################## +# Running METplus +# --------------- +# +# This use case can be run two ways: +# +# 1) Passing in PointStat_fcstGFS_obsNDBC_WaveHeight.conf then a user-specific system configuration file:: +# +# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf -c /path/to/user_system.conf +# +# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstGFS_obsNDBC_WaveHeight.conf:: +# +# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf +# +# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# +# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases +# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions +# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# +# Example User Configuration File:: +# +# [dir] +# INPUT_BASE = /path/to/sample/input/data +# OUTPUT_BASE = /path/to/output/dir +# MET_INSTALL_DIR = /path/to/met-X.Y +# +# **NOTE:** All of these items must be found under the [dir] section. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in PointStat and buoy_ASCII directories (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * point_stat_030000L_20221016_090000V_ctc.txt +# * point_stat_030000L_20221016_090000V_cts.txt +# * point_stat_030000L_20221016_090000V.stat +# * buoy_2022101609.nc + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * PointStatToolUseCase +# * ASCII2NCToolUseCase +# * GRIB2FileUseCase +# * MarineAndCryosphereAppUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/marine_and_cryosphere-PointStat_fcstGFS_obsNDBC_WaveHeight.png' + diff --git a/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py b/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py new file mode 100644 index 000000000..07127dcb1 --- /dev/null +++ b/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py @@ -0,0 +1,158 @@ +""" +PointStat: Compare community observed precipitation to model forecasts +====================================================================== + +model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# This use case ingests a CoCoRaHS csv file, a new dataset that utilizes community reporting of precipitation amounts. +# Numerous studies have shown that a community approach to weather observations not only covers areas that lack traditional verification datasets, +# but is also remarkably quality controlled. +# Utilizing Python embedding, this use case taps into a new vital observation dataset and compares it to a 24 hour precipitation accumulation forecast. + +############################################################################## +# Datasets +# --------------------- +# +# | **Forecast:** 24 URMA 1 hour precipitation accumulation files +# +# | **Observations:** CoCoRaHS, the Community Collaborative Rain, Hail, and Snow Network +# +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# +# | **Data Source:** EMC + +############################################################################## +# METplus Components +# ------------------ +# +# This use case calls a Python script in ASCII2NC for the observation dataset. +# PCPCombine is called for a user-defined summation of the forecast accumulation fields. +# Finally, PointStat processes the forecast and observation fields, and outputs the requested line types. + +############################################################################## +# METplus Workflow +# ---------------- +# +# 1 csv file of multiple valid observation times is passed to ASCII2NC via Python embedding, resulting in a netCDF output. +# 24 forecast files, each composed of 1 hour precipitation accumulation forecasts, is summarized via PCPCombine. +# The following boundary times are used for the forecast summation times: +# +# | **Valid Beg:** 2022-09-14 at 00z +# | **Valid End:** 2022-09-14 at 23z +# +# The observation data point span the same times as the 24 hour forecast accumulation summation. +# Finally, PointStat is used to compare the two new fields (point data in netCDF and precipitation accumulation over 24 hours). +# Because the Valid Time used in configuration file is set to one time (2022-09-14 at 23z) and the precipitation accumulation valid time is set to this same time, +# the observation window spans across the entire 2022-09-14 24 hour timeframe. +# + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# i.e. -c parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf +# + +############################################################################## +# MET Configuration +# --------------------- +# +# METplus sets environment variables based on the values in the METplus configuration file. These variables are referenced in the MET configuration file. **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not controlled by an environment variable, you can add additional environment variables to be set only within the METplus environment using the [user_env_vars] section of the METplus configuration files. See the ‘User Defined Config’ section on the ‘System Configuration’ page of the METplus User’s Guide for more information. +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/Ascii2NcConfig_wrapped +# .. literalinclude:: ../../../../parm/met_config/PointStatConfig_wrapped +# + +############################################################################## +# Running METplus +# --------------- +# +# This use case can be run two ways: +# +# 1) Passing in PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf then a user-specific system configuration file:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf /path/to/user_system.conf +# +# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf +# +# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# +# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases +# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions +# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# +# Example User Configuration File:: +# +# [config] +# INPUT_BASE = /path/to/sample/input/data +# OUTPUT_BASE = /path/to/output/dir +# MET_INSTALL_DIR = /path/to/met-X.Y +# +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for the use case will be found in 3 folders(relative to **OUTPUT_BASE**). +# Those folders are: +# +# * ASCII2NC +# * PCPCombine +# * PointStat +# +# The ASCII2NC folder will contain one file from the ASCII2NC tool call: +# +# * precip_20220914_summary.nc +# +# The PCPCombine folder will also contain one file, from the PCPCombine call: +# +# * fcst_24hr_precip.nc +# +# The final folder, PointStat, contains all of the following output from the PointStat call: +# +# * point_stat_000000L_20220914_230000V_cnt.txt +# * point_stat_000000L_20220914_230000V_ctc.txt +# * point_stat_000000L_20220914_230000V_cts.txt +# * point_stat_000000L_20220914_230000V_mcts.txt +# * point_stat_000000L_20220914_230000V.stat +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * PointStatToolUseCase +# * ASCII2NCToolUseCase +# * PCPCombineToolUseCase +# * PythonEmbeddingFileUseCase +# * PrecipitationAppUseCase +# * NETCDFFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/precipitation-PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.png' + diff --git a/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py b/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py index 79502d2b3..0ea4d90f8 100644 --- a/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py +++ b/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py @@ -84,7 +84,7 @@ # # 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile:: # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf # # The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: # @@ -130,7 +130,7 @@ # # .. note:: # -# * GridStatUseCase +# * GridStatToolUseCase # * ProbabilityVerificationUseCase # * PythonEmbeddingFileUseCase # * S2SAppUseCase diff --git a/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile similarity index 100% rename from scripts/docker/Dockerfile rename to internal/scripts/docker/Dockerfile diff --git a/scripts/docker/hooks/build b/internal/scripts/docker/hooks/build similarity index 100% rename from scripts/docker/hooks/build rename to internal/scripts/docker/hooks/build diff --git a/scripts/docker/hooks/get_met_version b/internal/scripts/docker/hooks/get_met_version similarity index 81% rename from scripts/docker/hooks/get_met_version rename to internal/scripts/docker/hooks/get_met_version index 6cb53c9ad..f580b55cb 100755 --- a/scripts/docker/hooks/get_met_version +++ b/internal/scripts/docker/hooks/get_met_version @@ -1,7 +1,7 @@ #!/bin/bash # get version, use develop or X+6.Y for MET_BRANCH -version_file=$(dirname $DOCKERFILE_PATH)/../../metplus/VERSION +version_file=$(dirname $DOCKERFILE_PATH)/../../../metplus/VERSION if cat $version_file | egrep -q '^[0-9.]+$'; then let major=$(cut -d '.' -f1 $version_file)+6 diff --git a/scripts/docker/docker_env/Dockerfile b/internal/scripts/docker_env/Dockerfile similarity index 100% rename from scripts/docker/docker_env/Dockerfile rename to internal/scripts/docker_env/Dockerfile diff --git a/scripts/docker/docker_env/Dockerfile.gempak_env b/internal/scripts/docker_env/Dockerfile.gempak_env similarity index 100% rename from scripts/docker/docker_env/Dockerfile.gempak_env rename to internal/scripts/docker_env/Dockerfile.gempak_env diff --git a/scripts/docker/docker_env/Dockerfile.gfdl-tracker b/internal/scripts/docker_env/Dockerfile.gfdl-tracker similarity index 100% rename from scripts/docker/docker_env/Dockerfile.gfdl-tracker rename to internal/scripts/docker_env/Dockerfile.gfdl-tracker diff --git a/scripts/docker/docker_env/Dockerfile.metplus_base b/internal/scripts/docker_env/Dockerfile.metplus_base similarity index 100% rename from scripts/docker/docker_env/Dockerfile.metplus_base rename to internal/scripts/docker_env/Dockerfile.metplus_base diff --git a/scripts/docker/docker_env/Dockerfile.py_embed_base b/internal/scripts/docker_env/Dockerfile.py_embed_base similarity index 100% rename from scripts/docker/docker_env/Dockerfile.py_embed_base rename to internal/scripts/docker_env/Dockerfile.py_embed_base diff --git a/scripts/docker/docker_env/README.md b/internal/scripts/docker_env/README.md similarity index 99% rename from scripts/docker/docker_env/README.md rename to internal/scripts/docker_env/README.md index 521d07184..516a43061 100644 --- a/scripts/docker/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -1,6 +1,6 @@ # Docker Conda Environments -Run the commands from this directory (scripts/docker/docker_env). +Run the commands from this directory (internal/scripts/docker_env). Instructions include how to create Docker images in dtcenter/metplus-envs so environments are available for the automated tests. Instructions to create these Conda environments on a local machine are also provided. diff --git a/scripts/docker/docker_env/scripts/cfgrib_env.sh b/internal/scripts/docker_env/scripts/cfgrib_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/cfgrib_env.sh rename to internal/scripts/docker_env/scripts/cfgrib_env.sh diff --git a/scripts/docker/docker_env/scripts/cycloneplotter_env.sh b/internal/scripts/docker_env/scripts/cycloneplotter_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/cycloneplotter_env.sh rename to internal/scripts/docker_env/scripts/cycloneplotter_env.sh diff --git a/scripts/docker/docker_env/scripts/diff_env.sh b/internal/scripts/docker_env/scripts/diff_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/diff_env.sh rename to internal/scripts/docker_env/scripts/diff_env.sh diff --git a/scripts/docker/docker_env/scripts/gempak_env.sh b/internal/scripts/docker_env/scripts/gempak_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/gempak_env.sh rename to internal/scripts/docker_env/scripts/gempak_env.sh diff --git a/scripts/docker/docker_env/scripts/h5py_env.sh b/internal/scripts/docker_env/scripts/h5py_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/h5py_env.sh rename to internal/scripts/docker_env/scripts/h5py_env.sh diff --git a/scripts/docker/docker_env/scripts/icecover_env.sh b/internal/scripts/docker_env/scripts/icecover_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/icecover_env.sh rename to internal/scripts/docker_env/scripts/icecover_env.sh diff --git a/scripts/docker/docker_env/scripts/metdataio_env.sh b/internal/scripts/docker_env/scripts/metdataio_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metdataio_env.sh rename to internal/scripts/docker_env/scripts/metdataio_env.sh diff --git a/scripts/docker/docker_env/scripts/metplotpy_env.sh b/internal/scripts/docker_env/scripts/metplotpy_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metplotpy_env.sh rename to internal/scripts/docker_env/scripts/metplotpy_env.sh diff --git a/scripts/docker/docker_env/scripts/metplus_base_env.sh b/internal/scripts/docker_env/scripts/metplus_base_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/metplus_base_env.sh rename to internal/scripts/docker_env/scripts/metplus_base_env.sh diff --git a/scripts/docker/docker_env/scripts/netcdf4_env.sh b/internal/scripts/docker_env/scripts/netcdf4_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/netcdf4_env.sh rename to internal/scripts/docker_env/scripts/netcdf4_env.sh diff --git a/scripts/docker/docker_env/scripts/py_embed_base_env.sh b/internal/scripts/docker_env/scripts/py_embed_base_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/py_embed_base_env.sh rename to internal/scripts/docker_env/scripts/py_embed_base_env.sh diff --git a/scripts/docker/docker_env/scripts/pygrib_env.sh b/internal/scripts/docker_env/scripts/pygrib_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/pygrib_env.sh rename to internal/scripts/docker_env/scripts/pygrib_env.sh diff --git a/scripts/docker/docker_env/scripts/pytest_env.sh b/internal/scripts/docker_env/scripts/pytest_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/pytest_env.sh rename to internal/scripts/docker_env/scripts/pytest_env.sh diff --git a/scripts/docker/docker_env/scripts/spacetime_env.sh b/internal/scripts/docker_env/scripts/spacetime_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/spacetime_env.sh rename to internal/scripts/docker_env/scripts/spacetime_env.sh diff --git a/scripts/docker/docker_env/scripts/weatherregime_env.sh b/internal/scripts/docker_env/scripts/weatherregime_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/weatherregime_env.sh rename to internal/scripts/docker_env/scripts/weatherregime_env.sh diff --git a/scripts/docker/docker_env/scripts/xesmf_env.sh b/internal/scripts/docker_env/scripts/xesmf_env.sh similarity index 100% rename from scripts/docker/docker_env/scripts/xesmf_env.sh rename to internal/scripts/docker_env/scripts/xesmf_env.sh diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py index 97af76698..8056e4cfe 100644 --- a/internal/tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -4,6 +4,7 @@ import subprocess import pytest import getpass +import shutil from pathlib import Path # add METplus directory to path so the wrappers and utilities can be found @@ -19,7 +20,8 @@ if pytest_host is None: import socket pytest_host = socket.gethostname() - print(f"No hostname provided with METPLUS_PYTEST_HOST, using {pytest_host}") + print("No hostname provided with METPLUS_PYTEST_HOST, " + f"using {pytest_host}") else: print(f"METPLUS_PYTEST_HOST = {pytest_host}") @@ -33,7 +35,8 @@ # source minimum_pytest..sh script current_user = getpass.getuser() -command = shlex.split(f"env -i bash -c 'export USER={current_user} && source {minimum_pytest_file} && env'") +command = shlex.split(f"env -i bash -c 'export USER={current_user} && " + f"source {minimum_pytest_file} && env'") proc = subprocess.Popen(command, stdout=subprocess.PIPE) for line in proc.stdout: @@ -43,21 +46,70 @@ proc.communicate() +output_base = os.environ['METPLUS_TEST_OUTPUT_BASE'] +if not output_base: + print('ERROR: METPLUS_TEST_OUTPUT_BASE must be set to a path to write') + sys.exit(1) + +test_output_dir = os.path.join(output_base, 'test_output') +if os.path.exists(test_output_dir): + print(f'Removing test output dir: {test_output_dir}') + shutil.rmtree(test_output_dir) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """! This is used to capture the status of a test so the metplus_config + fixture can remove output data from tests that pass. + """ + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + +@pytest.fixture() +def metplus_config(request): + """! Create a METplus configuration object using only the minimum required + settings found in minimum_pytest.conf. This fixture checks the result of + the test it is used in and automatically removes the output that is + generated by it unless the test fails. This makes it much easier to review + the failed tests. To use this fixture, add metplus_config to the test + function arguments and set a variable called config to metplus_config, e.g. + config = metplus_config. + """ + script_dir = os.path.dirname(__file__) + args = [os.path.join(script_dir, 'minimum_pytest.conf')] + config = config_metplus.setup(args) + yield config + + # don't remove output base if test fails + if request.node.rep_call.failed: + return + config_output_base = config.getdir('OUTPUT_BASE') + if config_output_base and os.path.exists(config_output_base): + shutil.rmtree(config_output_base) + + @pytest.fixture(scope='function') -def metplus_config(): - """! Create a METplus configuration object that can be - manipulated/modified to - reflect different paths, directories, values, etc. for individual - tests. +def metplus_config_files(): + """! Create a METplus configuration object using minimum_pytest.conf + settings and any list of config files.The metplus_config fixture is + preferred because it automatically cleans up the output files generated + by the use case unless the test fails. To use this in a test, add + metplus_config_files as an argument to the test function and pass in a list + of config files to it. Example: config = metplus_config_files([my_file]) """ - def read_configs(extra_configs=[]): + def read_configs(extra_configs): # Read in minimum pytest config file and any other extra configs script_dir = os.path.dirname(__file__) minimum_conf = os.path.join(script_dir, 'minimum_pytest.conf') - args = [minimum_conf] - if extra_configs: - args.extend(extra_configs) - + args = extra_configs.copy() + args.append(minimum_conf) config = config_metplus.setup(args) return config diff --git a/internal/tests/pytests/minimum_pytest.conf b/internal/tests/pytests/minimum_pytest.conf index 5a6893495..9982acc1a 100644 --- a/internal/tests/pytests/minimum_pytest.conf +++ b/internal/tests/pytests/minimum_pytest.conf @@ -1,6 +1,6 @@ [config] INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]} -OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]} +OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID} MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]} TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]} diff --git a/internal/tests/pytests/minimum_pytest.corrinado.sh b/internal/tests/pytests/minimum_pytest.corrinado.sh deleted file mode 100644 index 0555a9f7e..000000000 --- a/internal/tests/pytests/minimum_pytest.corrinado.sh +++ /dev/null @@ -1,13 +0,0 @@ -export METPLUS_TEST_INPUT_BASE=${HOME}/data/METplus_Data -export METPLUS_TEST_OUTPUT_BASE=${HOME}/pytest -export METPLUS_TEST_MET_INSTALL_DIR=${HOME}/met/9.0-beta3 -export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=wgrib2 -export METPLUS_TEST_EXE_CUT=cut -export METPLUS_TEST_EXE_TR=tr -export METPLUS_TEST_EXE_RM=rm -export METPLUS_TEST_EXE_NCAP2=ncap2 -export METPLUS_TEST_EXE_CONVERT=convert -export METPLUS_TEST_EXE_NCDUMP=ncdump -export METPLUS_TEST_EXE_EGREP=egrep diff --git a/internal/tests/pytests/minimum_pytest.dakota.sh b/internal/tests/pytests/minimum_pytest.dakota.sh index e3c93beef..0b66555fa 100644 --- a/internal/tests/pytests/minimum_pytest.dakota.sh +++ b/internal/tests/pytests/minimum_pytest.dakota.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/d3/projects/MET/METplus_Data export METPLUS_TEST_OUTPUT_BASE=/d3/personal/${USER}/pytest export METPLUS_TEST_MET_INSTALL_DIR=/d3/projects/MET/MET_releases/met-9.1_beta3 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.eyewall.sh b/internal/tests/pytests/minimum_pytest.eyewall.sh index b2a8a9975..06a69dd65 100644 --- a/internal/tests/pytests/minimum_pytest.eyewall.sh +++ b/internal/tests/pytests/minimum_pytest.eyewall.sh @@ -3,12 +3,3 @@ export METPLUS_TEST_OUTPUT_BASE=/d1/${USER}/pytest export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met-9.0 #export METPLUS_TEST_MET_INSTALL_DIR=/d1/CODE/MET/MET_releases/met-9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.hera.sh b/internal/tests/pytests/minimum_pytest.hera.sh index 64407e59a..bfb541180 100644 --- a/internal/tests/pytests/minimum_pytest.hera.sh +++ b/internal/tests/pytests/minimum_pytest.hera.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/home/${USER}/metplus_pytests export METPLUS_TEST_OUTPUT_BASE=/home/${USER}/metplus_pytests/out export METPLUS_TEST_MET_INSTALL_DIR=/contrib/met/8.1 export METPLUS_TEST_TMP_DIR=/tmp - -export METPLUS_TEST_EXE_WGRIB2=/apps/wgrib2/2.0.8/intel/18.0.3.222/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/usr/bin/rm -export METPLUS_TEST_EXE_NCAP2=/apps/nco/4.7.0/intel/18.0.3.051/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/apps/netcdf/4.7.0/intel/18.0.5.274/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/usr/bin/grep diff --git a/internal/tests/pytests/minimum_pytest.kiowa.sh b/internal/tests/pytests/minimum_pytest.kiowa.sh index 655f80f2d..33cb80aa9 100644 --- a/internal/tests/pytests/minimum_pytest.kiowa.sh +++ b/internal/tests/pytests/minimum_pytest.kiowa.sh @@ -1,14 +1,5 @@ export METPLUS_TEST_INPUT_BASE=/d1/projects/METplus/METplus_Data export METPLUS_TEST_OUTPUT_BASE=/d1/personal/${USER}/pytest -export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met-9.0 +export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met #export METPLUS_TEST_MET_INSTALL_DIR=/d1/projects/MET/MET_releases/met-9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp -#export METPLUS_TEST_TMP_DIR=/tmp -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.venus.sh b/internal/tests/pytests/minimum_pytest.venus.sh index 493f861ff..2c4774e34 100644 --- a/internal/tests/pytests/minimum_pytest.venus.sh +++ b/internal/tests/pytests/minimum_pytest.venus.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/gpfs/dell2/emc/verification/noscrub/$USER/METplu export METPLUS_TEST_OUTPUT_BASE=/gpfs/dell2/emc/verification/noscrub/$USER/metplus_test export METPLUS_TEST_MET_INSTALL_DIR=/gpfs/dell2/emc/verification/noscrub/$USER/met/9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=$WGRIB2 -export METPLUS_TEST_EXE_CUT=cut -export METPLUS_TEST_EXE_TR=tr -export METPLUS_TEST_EXE_RM=rm -export METPLUS_TEST_EXE_NCAP2=ncap2 -export METPLUS_TEST_EXE_CONVERT=convert -export METPLUS_TEST_EXE_NCDUMP=ncdump -export METPLUS_TEST_EXE_EGREP=egrep diff --git a/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py index 23d3a2715..0b7dca4e6 100644 --- a/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py +++ b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py @@ -99,7 +99,7 @@ def set_minimum_config_settings(config): ) @pytest.mark.plotting def test_read_loop_info(metplus_config, config_overrides, expected_loop_args): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -181,7 +181,7 @@ def test_read_loop_info(metplus_config, config_overrides, expected_loop_args): @pytest.mark.plotting def test_tcmpr_plotter_loop(metplus_config, config_overrides, expected_strings): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -278,7 +278,7 @@ def test_tcmpr_plotter(metplus_config, config_overrides, expected_string): expected_string = f' {expected_string}' for single_file in [True, False]: - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) diff --git a/internal/tests/pytests/pytest.ini b/internal/tests/pytests/pytest.ini index e9f3dd09e..1a9aa7a97 100644 --- a/internal/tests/pytests/pytest.ini +++ b/internal/tests/pytests/pytest.ini @@ -1,5 +1,6 @@ [pytest] markers = + run_metplus: custom marker for testing run_metplus.py script util: custom marker for testing metplus/util logic wrapper_a: custom marker for testing metplus/wrapper logic - A group wrapper_b: custom marker for testing metplus/wrapper logic - B group diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py new file mode 100644 index 000000000..6567e2d8e --- /dev/null +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import pytest + +from pathlib import Path +import os +import shutil +from subprocess import run + +# get METplus directory relative to this file +# from this script's directory, go up 4 directories +METPLUS_DIR = str(Path(__file__).parents[4]) +RUN_METPLUS = os.path.join(METPLUS_DIR, 'ush', 'run_metplus.py') +EXAMPLE_CONF = os.path.join(METPLUS_DIR, 'parm', 'use_cases', + 'met_tool_wrapper', 'Example', 'Example.conf') +MINIMUM_CONF = os.path.join(METPLUS_DIR, 'internal', 'tests', 'pytests', + 'minimum_pytest.conf') +TEST_OUTPUT_DIR = os.path.join(os.environ['METPLUS_TEST_OUTPUT_BASE'], + 'test_output') +NEW_OUTPUT_BASE = os.path.join(TEST_OUTPUT_DIR, 'run_metplus') +OUTPUT_BASE_OVERRIDE = f"config.OUTPUT_BASE={NEW_OUTPUT_BASE}" + +@pytest.mark.run_metplus +def test_run_metplus_exists(): + """! Check that run_metplus.py script exists """ + assert os.path.exists(RUN_METPLUS) + + +@pytest.mark.parametrize( + 'command, expected_return_code', [ + ([RUN_METPLUS], 2), + ([RUN_METPLUS, EXAMPLE_CONF], 2), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), + ] +) +@pytest.mark.run_metplus +def test_run_metplus_check_return_code(command, expected_return_code): + """! Call run_metplus.py without various arguments and check that the + expected value is returned by the script. A successful run should return + 0 and a failed run should return a non-zero return code, typically 2. + """ + process = run(command) + assert process.returncode == expected_return_code + + if os.path.exists(NEW_OUTPUT_BASE): + shutil.rmtree(NEW_OUTPUT_BASE) + + +@pytest.mark.run_metplus +def test_output_dir_is_created(): + """! Check that the test output directory was created after running tests + """ + assert os.path.exists(TEST_OUTPUT_DIR) diff --git a/internal/tests/pytests/util/config/test_config.py b/internal/tests/pytests/util/config/test_config.py index 7c054ab3d..5edd7670c 100644 --- a/internal/tests/pytests/util/config/test_config.py +++ b/internal/tests/pytests/util/config/test_config.py @@ -4,10 +4,9 @@ import os from configparser import NoOptionError -from shutil import which - -from metplus.util import met_util as util +from shutil import which, rmtree +from metplus.util.constants import MISSING_DATA_VALUE @pytest.mark.parametrize( 'input_value, result', [ @@ -28,7 +27,7 @@ ) @pytest.mark.util def test_getseconds(metplus_config, input_value, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_SECONDS', input_value) @@ -57,7 +56,7 @@ def test_getseconds(metplus_config, input_value, result): ) @pytest.mark.util def test_getstr(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETSTR', input_value) @@ -82,7 +81,7 @@ def test_getstr(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getdir(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETDIR', input_value) @@ -110,7 +109,7 @@ def test_getdir(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getraw(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config conf.set('config', 'TEST_EXTRA', 'extra') conf.set('config', 'TEST_EXTRA2', '{TEST_EXTRA}_extra') @@ -144,7 +143,7 @@ def test_getraw(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getbool(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETBOOL', input_value) @@ -167,7 +166,7 @@ def test_getbool(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getexe(metplus_config, input_value, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETEXE', input_value) @@ -178,18 +177,18 @@ def test_getexe(metplus_config, input_value, result): 'input_value, default, result', [ ('1.1', None, 1.1), ('1.1', 2.2, 1.1), - (None, None, util.MISSING_DATA_VALUE), + (None, None, MISSING_DATA_VALUE), (None, 1.1, 1.1), ('integer', None, None), ('integer', 1.1, None), ('0', None, 0.0), ('0', 2.2, 0.0), - ('', None, util.MISSING_DATA_VALUE), - ('', 2.2, util.MISSING_DATA_VALUE), + ('', None, MISSING_DATA_VALUE), + ('', 2.2, MISSING_DATA_VALUE), ] ) def test_getfloat(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETFLOAT', input_value) @@ -205,7 +204,7 @@ def test_getfloat(metplus_config, input_value, default, result): 'input_value, default, result', [ ('1', None, 1), ('1', 2, 1), - (None, None, util.MISSING_DATA_VALUE), + (None, None, MISSING_DATA_VALUE), (None, 1, 1), ('integer', None, None), ('integer', 1, None), @@ -214,13 +213,13 @@ def test_getfloat(metplus_config, input_value, default, result): ('1.7', 2, None), ('1.0', None, None), ('1.0', 2, None), - ('', None, util.MISSING_DATA_VALUE), - ('', 2.2, util.MISSING_DATA_VALUE), + ('', None, MISSING_DATA_VALUE), + ('', 2.2, MISSING_DATA_VALUE), ] ) @pytest.mark.util def test_getint(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETINT', input_value) @@ -241,15 +240,19 @@ def test_getint(metplus_config, input_value, default, result): ] ) @pytest.mark.util -def test_move_all_to_config_section(metplus_config, config_key, expected_result): +def test_move_all_to_config_section(metplus_config_files, config_key, + expected_result): config_files = ['config_1.conf', 'config_2.conf', 'config_3.conf', ] test_dir = os.path.dirname(__file__) config_files = [os.path.join(test_dir, item) for item in config_files] - config = metplus_config(config_files) + config = metplus_config_files(config_files) assert config.getstr('config', config_key) == expected_result + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) @pytest.mark.parametrize( @@ -280,11 +283,14 @@ def test_move_all_to_config_section(metplus_config, config_key, expected_result) ] ) @pytest.mark.util -def test_move_all_to_config_section_cmd_line(metplus_config, overrides, +def test_move_all_to_config_section_cmd_line(metplus_config_files, overrides, config_key, expected_result): - config = metplus_config(overrides) + config = metplus_config_files(overrides) assert config.getstr('config', config_key, '') == expected_result + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) @pytest.mark.parametrize( 'config_name, expected_result', [ @@ -330,13 +336,17 @@ def test_move_all_to_config_section_cmd_line(metplus_config, overrides, ] ) @pytest.mark.util -def test_getraw_nested_curly_braces(metplus_config, +def test_getraw_nested_curly_braces(metplus_config_files, config_name, expected_result): config_files = ['config_1.conf', ] test_dir = os.path.dirname(__file__) config_files = [os.path.join(test_dir, item) for item in config_files] - config = metplus_config(config_files) + config = metplus_config_files(config_files) sec, name = config_name.split('.', 1) assert config.getraw(sec, name) == expected_result + + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 8332aba14..8974d69b9 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -7,7 +7,7 @@ from datetime import datetime from metplus.util import config_metplus - +from metplus.util.time_util import ti_calculate @pytest.mark.util def test_get_default_config_list(): @@ -72,7 +72,7 @@ def test_get_default_config_list(): @pytest.mark.util def test_find_indices_in_config_section(metplus_config, regex, index, id, expected_result): - config = metplus_config() + config = metplus_config config.set('config', 'FCST_VAR1_NAME', 'name1') config.set('config', 'FCST_VAR1_LEVELS', 'level1') config.set('config', 'FCST_VAR2_NAME', 'name2') @@ -118,7 +118,7 @@ def test_find_indices_in_config_section(metplus_config, regex, index, ) @pytest.mark.util def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): - config = metplus_config() + config = metplus_config for conf_key, conf_value in conf_items.items(): config.set('config', conf_key, conf_value) @@ -146,13 +146,13 @@ def test_find_var_indices_fcst(metplus_config, config_var_name, expected_indices, set_met_tool): - config = metplus_config() + config = metplus_config data_types = ['FCST'] config.set('config', config_var_name, "NAME1") met_tool = 'grid_stat' if set_met_tool else None - var_name_indices = config_metplus.find_var_name_indices(config, - data_types=data_types, - met_tool=met_tool) + var_name_indices = config_metplus._find_var_name_indices(config, + data_types=data_types, + met_tool=met_tool) assert len(var_name_indices) == len(expected_indices) for actual_index in var_name_indices: @@ -229,7 +229,7 @@ def test_get_field_search_prefixes(data_type, met_tool, expected_out): ) @pytest.mark.util def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): - conf = metplus_config() + conf = metplus_config assert config_metplus.is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid @@ -272,7 +272,7 @@ def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): ) @pytest.mark.util def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): - conf = metplus_config() + conf = metplus_config for key, value in configs_to_set.items(): conf.set('config', key, value) @@ -321,7 +321,7 @@ def test_get_field_config_variables(metplus_config, search_prefixes, config_overrides, expected_value): - config = metplus_config() + config = metplus_config index = '1' field_info_types = ['name', 'levels', 'thresh', 'options', 'output_names'] for field_info_type in field_info_types: @@ -388,7 +388,7 @@ def test_get_field_config_variables_synonyms(metplus_config, config_keys, field_key, expected_value): - config = metplus_config() + config = metplus_config index = '1' prefix = 'BOTH_REGRID_DATA_PLANE_' for key in config_keys: @@ -411,7 +411,7 @@ def test_get_field_config_variables_synonyms(metplus_config, ) @pytest.mark.util def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "NAME1") conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'FCST_VAR2_NAME', "NAME2") @@ -448,7 +448,7 @@ def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): ) @pytest.mark.util def test_parse_var_list_obs(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'OBS_VAR1_NAME', "NAME1") conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'OBS_VAR2_NAME', "NAME2") @@ -485,7 +485,7 @@ def test_parse_var_list_obs(metplus_config, data_type, list_created): ) @pytest.mark.util def test_parse_var_list_both(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'BOTH_VAR1_NAME', "NAME1") conf.set('config', 'BOTH_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'BOTH_VAR2_NAME', "NAME2") @@ -512,7 +512,7 @@ def test_parse_var_list_both(metplus_config, data_type, list_created): # field info defined in both FCST_* and OBS_* variables @pytest.mark.util def test_parse_var_list_fcst_and_obs(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "FNAME1") conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") @@ -549,7 +549,7 @@ def test_parse_var_list_fcst_and_obs(metplus_config): # VAR1 defined by FCST, VAR2 defined by OBS @pytest.mark.util def test_parse_var_list_fcst_and_obs_alternate(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "FNAME1") conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") conf.set('config', 'OBS_VAR2_NAME', "ONAME2") @@ -569,7 +569,7 @@ def test_parse_var_list_fcst_and_obs_alternate(metplus_config): ) @pytest.mark.util def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_len, name_levels): - conf = metplus_config() + conf = metplus_config conf.set('config', 'OBS_VAR1_NAME', "ONAME1") conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") @@ -619,7 +619,7 @@ def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_le ) @pytest.mark.util def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "NAME1") conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'FCST_VAR1_THRESH', ">1, >2") @@ -643,12 +643,12 @@ def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): ) @pytest.mark.util def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): - conf = metplus_config() + conf = metplus_config data_type = 'FCST' conf.set('config', f'{data_type}_VAR1_NAME', "NAME1") conf.set('config', f'{data_type}_GRID_STAT_VAR2_NAME', "GSNAME2") - var_name_indices = config_metplus.find_var_name_indices(conf, data_types=[data_type], + var_name_indices = config_metplus._find_var_name_indices(conf,data_types=[data_type], met_tool=met_tool) assert var_name_indices == indices @@ -659,7 +659,7 @@ def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): # works as expected @pytest.mark.util def test_parse_var_list_ensemble(metplus_config): - config = metplus_config() + config = metplus_config config.set('config', 'ENS_VAR1_NAME', 'APCP') config.set('config', 'ENS_VAR1_LEVELS', 'A24') config.set('config', 'ENS_VAR1_THRESH', '>0.0, >=10.0') @@ -750,7 +750,7 @@ def test_parse_var_list_ensemble(metplus_config): @pytest.mark.util def test_parse_var_list_series_by(metplus_config): - config = metplus_config() + config = metplus_config config.set('config', 'BOTH_EXTRACT_TILES_VAR1_NAME', 'RH') config.set('config', 'BOTH_EXTRACT_TILES_VAR1_LEVELS', 'P850, P700') config.set('config', 'BOTH_EXTRACT_TILES_VAR1_OUTPUT_NAMES', @@ -817,8 +817,13 @@ def test_parse_var_list_series_by(metplus_config): assert actual_sa.get(key) == value +@pytest.mark.parametrize( + 'start_index', [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + ] +) @pytest.mark.util -def test_parse_var_list_priority_fcst(metplus_config): +def test_parse_var_list_priority_fcst(metplus_config, start_index): priority_list = ['FCST_GRID_STAT_VAR1_NAME', 'FCST_GRID_STAT_VAR1_INPUT_FIELD_NAME', 'FCST_GRID_STAT_VAR1_FIELD_NAME', @@ -833,22 +838,15 @@ def test_parse_var_list_priority_fcst(metplus_config): 'BOTH_VAR1_FIELD_NAME', ] time_info = {} + config = metplus_config + for key in priority_list[start_index:]: + config.set('config', key, key.lower()) - # loop through priority list, process, then pop first value off and - # process again until all items have been popped. - # This will check that list is in priority order - while(priority_list): - config = metplus_config() - for key in priority_list: - config.set('config', key, key.lower()) - - var_list = config_metplus.parse_var_list(config, time_info=time_info, - data_type='FCST', - met_tool='grid_stat') - - assert len(var_list) == 1 - assert var_list[0].get('fcst_name') == priority_list[0].lower() - priority_list.pop(0) + var_list = config_metplus.parse_var_list(config, time_info=time_info, + data_type='FCST', + met_tool='grid_stat') + assert len(var_list) == 1 + assert var_list[0].get('fcst_name') == priority_list[start_index].lower() # test that if wrapper specific field info is specified, it only gets @@ -856,7 +854,7 @@ def test_parse_var_list_priority_fcst(metplus_config): # wrapper specific field info variables are specified @pytest.mark.util def test_parse_var_list_wrapper_specific(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "ENAME1") conf.set('config', 'FCST_VAR1_LEVELS', "ELEVELS11, ELEVELS12") conf.set('config', 'FCST_VAR2_NAME', "ENAME2") @@ -942,7 +940,7 @@ def test_parse_var_list_wrapper_specific(metplus_config): @pytest.mark.util def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, expected_results): - config = metplus_config() + config = metplus_config for key, value in config_overrides.items(): config.set('config', key, value) @@ -999,7 +997,7 @@ def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, ) @pytest.mark.util def test_get_process_list(metplus_config, input_list, expected_list): - conf = metplus_config() + conf = metplus_config conf.set('config', 'PROCESS_LIST', input_list) process_list = config_metplus.get_process_list(conf) output_list = [item[0] for item in process_list] @@ -1033,7 +1031,7 @@ def test_get_process_list(metplus_config, input_list, expected_list): ) @pytest.mark.util def test_get_process_list_instances(metplus_config, input_list, expected_list): - conf = metplus_config() + conf = metplus_config conf.set('config', 'PROCESS_LIST', input_list) output_list = config_metplus.get_process_list(conf) assert output_list == expected_list @@ -1044,7 +1042,7 @@ def test_getraw_sub_and_nosub(metplus_config): raw_string = '{MODEL}_{CURRENT_FCST_NAME}' sub_actual = 'FCST_NAME' - config = metplus_config() + config = metplus_config config.set('config', 'MODEL', 'FCST') config.set('config', 'CURRENT_FCST_NAME', 'NAME') config.set('config', 'OUTPUT_PREFIX', raw_string) @@ -1062,7 +1060,7 @@ def test_getraw_instance_with_unset_var(metplus_config): """ pytest.skip() instance = 'my_section' - config = metplus_config() + config = metplus_config config.set('config', 'MODEL', 'FCST') config.add_section(instance) @@ -1075,3 +1073,95 @@ def test_getraw_instance_with_unset_var(metplus_config): ) new_config.set('config', 'CURRENT_FCST_NAME', 'NAME') assert new_config.getraw('config', 'OUTPUT_PREFIX') == 'FCST_NAME' + + +@pytest.mark.parametrize( + 'config_value, expected_result', [ + # 2 items semi-colon at end + ('GRIB_lvl_typ = 234; desc = "HI_CLOUD";', + 'GRIB_lvl_typ = 234; desc = "HI_CLOUD";'), + # 2 items no semi-colon at end + ('GRIB_lvl_typ = 234; desc = "HI_CLOUD"', + 'GRIB_lvl_typ = 234; desc = "HI_CLOUD";'), + # 1 item semi-colon at end + ('GRIB_lvl_typ = 234;', + 'GRIB_lvl_typ = 234;'), + # 1 item no semi-colon at end + ('GRIB_lvl_typ = 234', + 'GRIB_lvl_typ = 234;'), + ] +) +@pytest.mark.util +def test_format_var_items_options_semicolon(config_value, + expected_result): + time_info = {} + + field_configs = {'name': 'FNAME', + 'levels': 'FLEVEL', + 'options': config_value} + + var_items = config_metplus._format_var_items(field_configs, time_info) + result = var_items.get('extra') + assert result == expected_result + + +@pytest.mark.parametrize( + 'input_dict, expected_list', [ + ({'init': datetime(2019, 2, 1, 6), + 'lead': 7200, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z06', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L06', + }, + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z08', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L08', + }, + ]), + ({'init': datetime(2021, 4, 13, 9), + 'lead': 10800, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z09', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L09', + }, + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z12', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L12', + }, + ]), + ] +) +@pytest.mark.util +def test_sub_var_list(metplus_config, input_dict, expected_list): + config = metplus_config + config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') + config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') + config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') + config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') + + time_info = ti_calculate(input_dict) + + actual_temp = config_metplus.parse_var_list(config) + + pp = pprint.PrettyPrinter() + print(f'Actual var list (before sub):') + pp.pprint(actual_temp) + + actual_list = config_metplus.sub_var_list(actual_temp, time_info) + print(f'Actual var list (after sub):') + pp.pprint(actual_list) + + assert len(actual_list) == len(expected_list) + for actual, expected in zip(actual_list, expected_list): + for key, value in expected.items(): + assert actual.get(key) == value diff --git a/internal/tests/pytests/util/logging/test_logging.py b/internal/tests/pytests/util/logging/test_logging.py index 68eca3262..085b33aac 100644 --- a/internal/tests/pytests/util/logging/test_logging.py +++ b/internal/tests/pytests/util/logging/test_logging.py @@ -10,7 +10,7 @@ @pytest.mark.util def test_log_level(metplus_config): # Verify that the log level is set to what we indicated in the config file. - config = metplus_config() + config = metplus_config fixture_logger = config.logger # Expecting log level = INFO as set in the test config file. level = logging.getLevelName('INFO') @@ -20,7 +20,7 @@ def test_log_level(metplus_config): @pytest.mark.util def test_log_level_key(metplus_config): # Verify that the LOG_LEVEL key is in the config file - config_instance = metplus_config() + config_instance = metplus_config section = 'config' option = 'LOG_LEVEL' assert config_instance.has_option(section, option) @@ -29,7 +29,7 @@ def test_log_level_key(metplus_config): @pytest.mark.util def test_logdir_exists(metplus_config): # Verify that the expected log dir exists. - config = metplus_config() + config = metplus_config log_dir = config.get('config', 'LOG_DIR') # Verify that a logfile exists in the log dir, with a filename # like {LOG_DIR}/metplus.YYYYMMDD.log @@ -40,7 +40,7 @@ def test_logdir_exists(metplus_config): def test_logfile_exists(metplus_config): # Verify that a logfile with format metplus.log exists # We are assuming that there can be numerous files in the log directory. - config = metplus_config() + config = metplus_config log_dir = config.get('config', 'LOG_DIR') # Only check for the log file if the log directory is present if os.path.exists(log_dir): diff --git a/internal/tests/pytests/util/met_config/test_met_config.py b/internal/tests/pytests/util/met_config/test_met_config.py index 0f3adb658..e00e5f2e9 100644 --- a/internal/tests/pytests/util/met_config/test_met_config.py +++ b/internal/tests/pytests/util/met_config/test_met_config.py @@ -36,7 +36,7 @@ def test_read_climo_field(metplus_config, config_overrides, expected_value): app_name = 'app' for climo_type in ('MEAN', 'STDEV'): expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): @@ -135,7 +135,7 @@ def test_handle_climo_dict(metplus_config, config_overrides, expected_value): app_name = 'app' for climo_type in ('MEAN', 'STDEV'): expected_var = f'METPLUS_CLIMO_{climo_type}_DICT' - config = metplus_config() + config = metplus_config output_dict = {} # set config values @@ -252,7 +252,7 @@ def test_read_climo_file_name(metplus_config, config_overrides, for climo_type in CLIMO_TYPES: prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_' - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): diff --git a/internal/tests/pytests/util/met_util/test_met_util.py b/internal/tests/pytests/util/met_util/test_met_util.py deleted file mode 100644 index 481d4f9d4..000000000 --- a/internal/tests/pytests/util/met_util/test_met_util.py +++ /dev/null @@ -1,668 +0,0 @@ -#!/usr/bin/env python3 - -import pytest - -import datetime -import os -from dateutil.relativedelta import relativedelta -import pprint - -from metplus.util import met_util as util -from metplus.util import time_util -from metplus.util.config_metplus import parse_var_list - - -@pytest.mark.parametrize( - 'key, value', [ - ({"gt2.3", "gt5.5"}, True), - ({"ge2.3", "ge5.5"}, True), - ({"eq2.3"}, True), - ({"ne2.3"}, True), - ({"lt2.3", "lt1.1"}, True), - ({"le2.3", "le1.1"}, True), - ({">2.3", ">5.5"}, True), - ({">=2.3", ">=5.5"}, True), - ({"==2.3"}, True), - ({"!=.3"}, True), - ({"<2.3", "<1."}, True), - ({"<=2.3", "<=1.1"}, True), - ({"gta"}, False), - ({"gt"}, False), - ({">=a"}, False), - ({"2.3"}, False), - ({"<=2.3", "2.4", "gt2.7"}, False), - ({"<=2.3||>=4.2", "gt2.3&<4.2"}, True), - ({"gt2.3&<4.2a"}, True), - ({"gt2sd.3&<4.2"}, True), - ({"gt2.3&a<4.2"}, True), # invalid but is accepted - ({'gt4&<5&&ne4.5'}, True), - ({"<2.3", "ge5", ">SPF90"}, True), - (["NA"], True), - (["SFP70", ">SFP80", ">SFP90", ">SFP95"], True), - ([">SFP70", ">SFP80", ">SFP90", ">SFP95"], True), - ] -) -@pytest.mark.util -def test_threshold(key, value): - assert util.validate_thresholds(key) == value - - -# parses a threshold and returns a list of tuples of -# comparison and number, i.e.: -# 'gt4' => [('gt', 4)] -# gt4&<5 => [('gt', 4), ('lt', 5)] -@pytest.mark.parametrize( - 'key, value', [ - ('gt4', [('gt', 4)]), - ('gt4&<5', [('gt', 4), ('lt', 5)]), - ('gt4&<5&&ne4.5', [('gt', 4), ('lt', 5), ('ne', 4.5)]), - (">4.545", [('>', 4.545)]), - (">=4.0", [('>=', 4.0)]), - ("<4.5", [('<', 4.5)]), - ("<=4.5", [('<=', 4.5)]), - ("!=4.5", [('!=', 4.5)]), - ("==4.5", [('==', 4.5)]), - ("gt4.5", [('gt', 4.5)]), - ("ge4.5", [('ge', 4.5)]), - ("lt4.5", [('lt', 4.5)]), - ("le4.5", [('le', 4.5)]), - ("ne10.5", [('ne', 10.5)]), - ("eq4.5", [('eq', 4.5)]), - ("eq-4.5", [('eq', -4.5)]), - ("eq+4.5", [('eq', 4.5)]), - ("eq.5", [('eq', 0.5)]), - ("eq5.", [('eq', 5)]), - ("eq5.||ne0.0", [('eq', 5), ('ne', 0.0)]), - (">SFP90", [('>', 'SFP90')]), - ("SFP90", None), - ("gtSFP90", [('gt', 'SFP90')]), - ("goSFP90", None), - ("NA", [('NA', '')]), - ("2.3", ">5.5"}, True), + ({">=2.3", ">=5.5"}, True), + ({"==2.3"}, True), + ({"!=.3"}, True), + ({"<2.3", "<1."}, True), + ({"<=2.3", "<=1.1"}, True), + ({"gta"}, False), + ({"gt"}, False), + ({">=a"}, False), + ({"2.3"}, False), + ({"<=2.3", "2.4", "gt2.7"}, False), + ({"<=2.3||>=4.2", "gt2.3&<4.2"}, True), + ({"gt2.3&<4.2a"}, True), + ({"gt2sd.3&<4.2"}, True), + ({"gt2.3&a<4.2"}, True), # invalid but is accepted + ({'gt4&<5&&ne4.5'}, True), + ({"<2.3", "ge5", ">SPF90"}, True), + (["NA"], True), + (["SFP70", ">SFP80", ">SFP90", ">SFP95"], True), + ([">SFP70", ">SFP80", ">SFP90", ">SFP95"], True), + ] +) +@pytest.mark.util +def test_threshold(key, value): + assert validate_thresholds(key) == value + + +# parses a threshold and returns a list of tuples of +# comparison and number, i.e.: +# 'gt4' => [('gt', 4)] +# gt4&<5 => [('gt', 4), ('lt', 5)] +@pytest.mark.parametrize( + 'key, value', [ + ('gt4', [('gt', 4)]), + ('gt4&<5', [('gt', 4), ('lt', 5)]), + ('gt4&<5&&ne4.5', [('gt', 4), ('lt', 5), ('ne', 4.5)]), + (">4.545", [('>', 4.545)]), + (">=4.0", [('>=', 4.0)]), + ("<4.5", [('<', 4.5)]), + ("<=4.5", [('<=', 4.5)]), + ("!=4.5", [('!=', 4.5)]), + ("==4.5", [('==', 4.5)]), + ("gt4.5", [('gt', 4.5)]), + ("ge4.5", [('ge', 4.5)]), + ("lt4.5", [('lt', 4.5)]), + ("le4.5", [('le', 4.5)]), + ("ne10.5", [('ne', 10.5)]), + ("eq4.5", [('eq', 4.5)]), + ("eq-4.5", [('eq', -4.5)]), + ("eq+4.5", [('eq', 4.5)]), + ("eq.5", [('eq', 0.5)]), + ("eq5.", [('eq', 5)]), + ("eq5.||ne0.0", [('eq', 5), ('ne', 0.0)]), + (">SFP90", [('>', 'SFP90')]), + ("SFP90", None), + ("gtSFP90", [('gt', 'SFP90')]), + ("goSFP90", None), + ("NA", [('NA', '')]), + (" initializes to an empty string '' or list [], indicating all vars are to be considered -EXTRACT_TILES_VAR_LIST = - -# -# FILENAME TEMPLATES -# -[filename_templates] -# Define the format of the filenames -FCST_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2 -OBS_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2 - -[dir] -# Location of your model data of interest -EXTRACT_TILES_GRID_INPUT_DIR = {INPUT_BASE}/cyclone_track_feature/reduced_model_data - -EXTRACT_TILES_PAIRS_INPUT_DIR = {OUTPUT_BASE}/tc_pairs - -# Use this setting to separate the filtered track files from -# the series analysis directory. -EXTRACT_TILES_OUTPUT_DIR = {OUTPUT_BASE}/extract_tiles diff --git a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py index 41d54886f..aa71f0eb5 100644 --- a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py +++ b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py @@ -8,17 +8,32 @@ from metplus.wrappers.extract_tiles_wrapper import ExtractTilesWrapper -def get_config(metplus_config): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'extract_tiles_test.conf')) - return metplus_config(extra_configs) - - def extract_tiles_wrapper(metplus_config): - config = get_config(metplus_config) + config = metplus_config + config.set('config', 'PROCESS_LIST', 'ExtractTiles') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d') + config.set('config', 'INIT_BEG', '20141214') + config.set('config', 'INIT_END', '20141214') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'EXTRACT_TILES_NLAT', '60') + config.set('config', 'EXTRACT_TILES_NLON', '60') + config.set('config', 'EXTRACT_TILES_DLAT', '0.5') + config.set('config', 'EXTRACT_TILES_DLON', '0.5') + config.set('config', 'EXTRACT_TILES_LAT_ADJ', '15') + config.set('config', 'EXTRACT_TILES_LON_ADJ', '15') + config.set('config', 'EXTRACT_TILES_FILTER_OPTS', '-basin ML') + config.set('config', 'FCST_EXTRACT_TILES_INPUT_TEMPLATE', + 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2') + config.set('config', 'OBS_EXTRACT_TILES_INPUT_TEMPLATE', + 'gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2') + config.set('config', 'EXTRACT_TILES_GRID_INPUT_DIR', + '{INPUT_BASE}/cyclone_track_feature/reduced_model_data') + config.set('config', 'EXTRACT_TILES_PAIRS_INPUT_DIR', + '{OUTPUT_BASE}/tc_pairs') + config.set('config', 'EXTRACT_TILES_OUTPUT_DIR', + '{OUTPUT_BASE}/extract_tiles') - config.set('config', 'LOOP_ORDER', 'processes') wrapper = ExtractTilesWrapper(config) return wrapper diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index f8ed2b463..49990fc52 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -362,7 +362,7 @@ def handle_input_dir(config): def test_gen_ens_prod_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -440,7 +440,7 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides, ) @pytest.mark.wrapper def test_get_config_file(metplus_config, use_default_config_file): - config = metplus_config() + config = metplus_config if use_default_config_file: config_file = os.path.join(config.getdir('PARM_BASE'), @@ -463,7 +463,7 @@ def test_get_config_file(metplus_config, use_default_config_file): @pytest.mark.wrapper def test_gen_ens_prod_fill_missing(metplus_config, config_overrides, expected_num_files): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) handle_input_dir(config) diff --git a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py index b584f7549..fa1d660f9 100644 --- a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py +++ b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py @@ -16,7 +16,7 @@ def gen_vx_mask_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return GenVxMaskWrapper(config) diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 61fd59915..72e3a878d 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -68,7 +68,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): """! Test to ensure that get_all_files only gets the files that are relevant to the runtime settings and not every file in the directory """ - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', 'INIT') config.set('config', 'GRID_DIAG_RUNTIME_FREQ', 'RUN_ONCE') config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H%M%S') @@ -169,7 +169,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): ) @pytest.mark.wrapper def test_get_list_file_name(metplus_config, time_info, expected_filename): - wrapper = GridDiagWrapper(metplus_config()) + wrapper = GridDiagWrapper(metplus_config) assert(wrapper.get_list_file_name(time_info, 'input0') == expected_filename) @@ -177,7 +177,7 @@ def test_get_list_file_name(metplus_config, time_info, expected_filename): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'GridDiagConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index c10d91487..9a0caed0b 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -88,7 +88,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_b def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -132,7 +132,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, """ old_env_vars = ['CLIMO_MEAN_FILE', 'CLIMO_STDEV_FILE'] - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -695,7 +695,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, def test_grid_stat_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -750,7 +750,7 @@ def test_grid_stat_single_field(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'GridStatConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index 1e06c7715..99a98db0f 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -186,7 +186,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper def test_ioda2nc_wrapper(metplus_config, config_overrides, env_var_values, extra_args): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -238,7 +238,7 @@ def test_ioda2nc_wrapper(metplus_config, config_overrides, @pytest.mark.wrapper def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INPUT_MUST_EXIST', False) wrapper = IODA2NCWrapper(config) diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index 6f3171418..1ca480b49 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -318,7 +318,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_a def test_mode_single_field(metplus_config, config_overrides, expected_output): - config = metplus_config() + config = metplus_config # set config variables needed to run set_minimum_config_settings(config) @@ -401,7 +401,7 @@ def test_mode_single_field(metplus_config, config_overrides, @pytest.mark.wrapper_a def test_mode_multi_variate(metplus_config, config_overrides, expected_output): - config = metplus_config() + config = metplus_config # set config variables needed to run set_minimum_config_settings(config) @@ -518,7 +518,7 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, elif var_type == 'float': in_value = out_value = 4.0 - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) config.set('config', config_name, in_value) wrapper = MODEWrapper(config) @@ -533,7 +533,7 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'MODEConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 135c62031..420e6d40f 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -19,7 +19,7 @@ def mtd_wrapper(metplus_config, lead_seq=None): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'BOTH_VAR1_NAME', 'APCP') config.set('config', 'BOTH_VAR1_LEVELS', 'A06') @@ -195,7 +195,7 @@ def test_mtd_single(metplus_config): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'MTDConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/pb2nc/conf1 b/internal/tests/pytests/wrappers/pb2nc/conf1 deleted file mode 100644 index a1c694fcd..000000000 --- a/internal/tests/pytests/wrappers/pb2nc/conf1 +++ /dev/null @@ -1,72 +0,0 @@ -[config] -## Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PB2NC - -## LOOP_ORDER -## Options are: processes, times -## Looping by time- runs all items in the PROCESS_LIST for each -## initialization time and repeats until all times have been evaluated. -## Looping by processes- run each item in the PROCESS_LIST for all -## specified initialization times then repeat for the next item in the -## PROCESS_LIST. -#LOOP_ORDER = processes - -# Logging levels: DEBUG, INFO, WARN, ERROR (most verbose is DEBUG) -#LOG_LEVEL = DEBUG - -## MET Configuration files for pb2nc -PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped - -PB2NC_SKIP_IF_OUTPUT_EXISTS = True - -#LOOP_BY = VALID -#VALID_TIME_FMT = %Y%m%d -#VALID_BEG = 20170601 -#VALID_END = 20170603 -#VALID_INCREMENT = 86400 - -#LEAD_SEQ = 0 - - -# For both pb2nc and point_stat, the obs_window dictionary: -#OBS_WINDOW_BEGIN = -2700 -#OBS_WINDOW_END = 2700 - -# Either conus_sfc or upper_air -PB2NC_VERTICAL_LOCATION = conus_sfc - -# -# PB2NC -# -# These are appended with PB2NC to differentiate the GRID, POLY, and MESSAGE_TYPE for point_stat. -PB2NC_GRID = -PB2NC_POLY = -PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = - -# Leave empty to process all -PB2NC_OBS_BUFR_VAR_LIST = PMO, TOB, TDO, UOB, VOB, PWO, TOCC, D_RH - -#*********** -# ***NOTE*** -#*********** -# SET TIME_SUMMARY_FLAG to False. There is a bug in met-6.1. -## For defining the time periods for summarization -# False for no time summary, True otherwise -PB2NC_TIME_SUMMARY_FLAG = False -PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format -PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format -PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC -PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -#MODEL_NAME = gfs -#OBS_NAME = nam - -[dir] -PB2NC_INPUT_DIR = {INPUT_BASE}/grid_to_obs/prepbufr/nam - -[filename_templates] -PB2NC_INPUT_TEMPLATE = t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H} - -PB2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/nam.{valid?fmt=%Y%m%d%H}.nc \ No newline at end of file diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 9f98f0d80..8096cc1e4 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -15,19 +15,16 @@ def pb2nc_wrapper(metplus_config): """! Returns a default PB2NCWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration - to over-ride these /path/to values.""" - - # PB2NCWrapper with configuration values determined by what is set in - # the pb2nc_test.conf file. - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), 'conf1')) - config = metplus_config(extra_configs) + to over-ride these /path/to values. + """ + config = metplus_config + config.set('config', 'PB2NC_INPUT_TEMPLATE', + 't{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}') return PB2NCWrapper(config) @pytest.mark.parametrize( - # key = grid_id, value = expected reformatted grid id - 'exists, skip, run', [ + 'exists, skip, run', [ (True, True, False), (True, False, True), (False, True, True), @@ -67,12 +64,12 @@ def test_find_and_check_output_file_skip(metplus_config, exists, skip, run): # --------------------- @pytest.mark.parametrize( # list of input files - 'infiles', [ - [], - ['file1'], - ['file1', 'file2'], - ['file1', 'file2', 'file3'], - ] + 'infiles', [ + [], + ['file1'], + ['file1', 'file2'], + ['file1', 'file2', 'file3'], + ] ) @pytest.mark.wrapper def test_get_command(metplus_config, infiles): @@ -101,12 +98,12 @@ def test_get_command(metplus_config, infiles): @pytest.mark.parametrize( # offset = list of offsets to search # offset_to_find = expected offset file to find, None if no files should be found - 'offsets, offset_to_find', [ - ([6, 5, 4, 3], 5), - ([6, 4, 3], 3), - ([2, 3, 4, 5, 6], 3), - ([2, 4, 6], None), - ] + 'offsets, offset_to_find', [ + ([6, 5, 4, 3], 5), + ([6, 4, 3], 3), + ([2, 3, 4, 5, 6], 3), + ([2, 4, 6], None), + ] ) @pytest.mark.wrapper def test_find_input_files(metplus_config, offsets, offset_to_find): @@ -271,7 +268,7 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): def test_pb2nc_all_fields(metplus_config, config_overrides, env_var_values): input_dir = '/some/input/dir' - config = metplus_config() + config = metplus_config # set config variables to prevent command from running and bypass check # if input files actually exist @@ -343,7 +340,7 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PB2NCConfig_wrapped') @@ -361,7 +358,7 @@ def test_pb2nc_file_window(metplus_config): begin_value = -3600 end_value = 3600 - config = metplus_config() + config = metplus_config config.set('config', 'PB2NC_FILE_WINDOW_BEGIN', begin_value) config.set('config', 'PB2NC_FILE_WINDOW_END', end_value) wrapper = PB2NCWrapper(config) diff --git a/internal/tests/pytests/wrappers/pcp_combine/test1.conf b/internal/tests/pytests/wrappers/pcp_combine/test1.conf deleted file mode 100644 index 0d5028099..000000000 --- a/internal/tests/pytests/wrappers/pcp_combine/test1.conf +++ /dev/null @@ -1,35 +0,0 @@ -[config] -FCST_PCP_COMBINE_INPUT_ACCUMS = 6 -FCST_PCP_COMBINE_INPUT_NAMES = P06M_NONE -FCST_PCP_COMBINE_INPUT_LEVELS = "(*,*)" - -OBS_PCP_COMBINE_INPUT_ACCUMS = 1 -OBS_PCP_COMBINE_INPUT_NAMES = P01M_NONE - -OBS_PCP_COMBINE_DATA_INTERVAL = 1 -OBS_PCP_COMBINE_TIMES_PER_FILE = 4 - -FCST_PCP_COMBINE_INPUT_DATATYPE = NETCDF -OBS_PCP_COMBINE_INPUT_DATATYPE = NETCDF - -FCST_PCP_COMBINE_RUN = True - -FCST_PCP_COMBINE_METHOD = ADD - -OBS_PCP_COMBINE_RUN = True - -OBS_PCP_COMBINE_METHOD = ADD - -[dir] -OBS_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/accum -OBS_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout - -FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/fcst -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout - -[filename_templates] -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/outfile.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HH}h -FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST2_PCP_COMBINE_INPUT_TEMPLATE = file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HHH}.nc \ No newline at end of file diff --git a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index 19be7d9b2..3cfe0e676 100644 --- a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -16,17 +16,41 @@ def get_test_data_dir(config, subdir=None): top_dir = os.path.join(top_dir, subdir) return top_dir + def pcp_combine_wrapper(metplus_config, d_type): """! Returns a default PCPCombineWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" + config = metplus_config + config.set('config', 'FCST_PCP_COMBINE_INPUT_ACCUMS', '6') + config.set('config', 'FCST_PCP_COMBINE_INPUT_NAMES', 'P06M_NONE') + config.set('config', 'FCST_PCP_COMBINE_INPUT_LEVELS', '"(*,*)"') + config.set('config', 'OBS_PCP_COMBINE_INPUT_ACCUMS', '1') + config.set('config', 'OBS_PCP_COMBINE_INPUT_NAMES', 'P01M_NONE') + config.set('config', 'OBS_PCP_COMBINE_DATA_INTERVAL', '1') + config.set('config', 'OBS_PCP_COMBINE_TIMES_PER_FILE', '4') + config.set('config', 'FCST_PCP_COMBINE_METHOD', 'ADD') + config.set('config', 'OBS_PCP_COMBINE_METHOD', 'ADD') + config.set('config', 'OBS_PCP_COMBINE_INPUT_DIR', + '{METPLUS_BASE}/internal/tests/data/accum') + config.set('config', 'OBS_PCP_COMBINE_OUTPUT_DIR', + '{OUTPUT_BASE}/internal/tests/data/fakeout') + config.set('config', 'FCST_PCP_COMBINE_INPUT_DIR', + '{METPLUS_BASE}/internal/tests/data/fcst') + config.set('config', 'FCST_PCP_COMBINE_OUTPUT_DIR', + '{OUTPUT_BASE}/internal/tests/data/fakeout') + config.set('config', 'OBS_PCP_COMBINE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h') + config.set('config', 'OBS_PCP_COMBINE_OUTPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/outfile.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HH}h') + config.set('config', 'FCST_PCP_COMBINE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc') + config.set('config', 'FCST2_PCP_COMBINE_INPUT_TEMPLATE', + 'file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc') + config.set('config', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HHH}.nc') - # PCPCombineWrapper with configuration values determined by what is set in - # the test1.conf file. - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), 'test1.conf')) - config = metplus_config(extra_configs) if d_type == "FCST": config.set('config', 'FCST_PCP_COMBINE_RUN', True) elif d_type == "OBS": @@ -220,7 +244,7 @@ def test_pcp_combine_add_subhourly(metplus_config): fcst_level = 'Surface' fcst_output_name = 'A001500' fcst_fmt = f'\'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -285,7 +309,7 @@ def test_pcp_combine_add_subhourly(metplus_config): @pytest.mark.wrapper def test_pcp_combine_bucket(metplus_config): fcst_output_name = 'APCP' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -365,7 +389,7 @@ def test_pcp_combine_derive(metplus_config, config_overrides, extra_fields): fcst_name = 'APCP' fcst_level = 'A03' fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -438,7 +462,7 @@ def test_pcp_combine_derive(metplus_config, config_overrides, extra_fields): def test_pcp_combine_loop_custom(metplus_config): fcst_name = 'APCP' ens_list = ['ens1', 'ens2', 'ens3', 'ens4', 'ens5', 'ens6'] - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -500,7 +524,7 @@ def test_pcp_combine_loop_custom(metplus_config): @pytest.mark.wrapper def test_pcp_combine_subtract(metplus_config): - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -563,7 +587,7 @@ def test_pcp_combine_sum_subhourly(metplus_config): fcst_level = 'Surface' fcst_output_name = 'A001500' fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -641,7 +665,7 @@ def test_pcp_combine_sum_subhourly(metplus_config): def test_handle_name_argument(metplus_config, output_name, extra_output, expected_results): data_src = 'FCST' - config = metplus_config() + config = metplus_config wrapper = PCPCombineWrapper(config) wrapper.c_dict[data_src + '_EXTRA_OUTPUT_NAMES'] = extra_output wrapper._handle_name_argument(output_name, data_src) @@ -680,7 +704,7 @@ def test_handle_name_argument(metplus_config, output_name, extra_output, @pytest.mark.wrapper def test_get_extra_fields(metplus_config, names, levels, expected_args): data_src = 'FCST' - config = metplus_config() + config = metplus_config config.set('config', 'FCST_PCP_COMBINE_RUN', True) config.set('config', 'FCST_PCP_COMBINE_METHOD', 'ADD') config.set('config', 'FCST_PCP_COMBINE_EXTRA_NAMES', names) @@ -697,7 +721,7 @@ def test_get_extra_fields(metplus_config, names, levels, expected_args): @pytest.mark.wrapper def test_add_method_single_file(metplus_config): data_src = 'FCST' - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'INPUT_MUST_EXIST', False) @@ -767,7 +791,7 @@ def test_subtract_method_zero_accum(metplus_config): input_level = '"(*,*)"' in_dir = '/some/input/dir' out_dir = '/some/output/dir' - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'INPUT_MUST_EXIST', False) diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index 3e779d81a..172b4a93d 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -217,7 +217,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_c def test_plot_point_obs(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -235,19 +235,19 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): out_dir = wrapper.c_dict.get('OUTPUT_DIR') expected_cmds = [ (f"{app_path} {verbosity} " - f"{input_dir}/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc " + f'"{input_dir}/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc" ' f"{out_dir}/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps"), (f"{app_path} {verbosity} " - f"{input_dir}/pb2nc/ndas.20120410.t00z.prepbufr.tm00.nc " + f'"{input_dir}/pb2nc/ndas.20120410.t00z.prepbufr.tm00.nc" ' f"{out_dir}/nam_and_ndas.20120410.t00z.prepbufr_CONFIG.ps"), ] # add -point_obs argument if template has 2 items if ('PLOT_POINT_OBS_INPUT_TEMPLATE' in config_overrides and len(config_overrides['PLOT_POINT_OBS_INPUT_TEMPLATE'].split(',')) > 1): - common_str = f' -point_obs {input_dir}/ascii2nc/trmm_' - expected_cmds[0] += f'{common_str}2012040912_3hr.nc' - expected_cmds[1] += f'{common_str}2012041000_3hr.nc' + common_str = f' -point_obs "{input_dir}/ascii2nc/trmm_' + expected_cmds[0] += f'{common_str}2012040912_3hr.nc"' + expected_cmds[1] += f'{common_str}2012041000_3hr.nc"' # add -plot_grid argument if provided if 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE' in config_overrides: @@ -288,7 +288,7 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PlotPointObsConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index e4b27f5d6..c0f51d47f 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -13,7 +13,7 @@ def p2g_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return Point2GridWrapper(config) diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 84ddf4e98..2c574d247 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -41,7 +41,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_a def test_met_dictionary_in_var_options(metplus_config): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) config.set('config', 'BOTH_VAR1_NAME', 'name') @@ -514,7 +514,7 @@ def test_point_stat_all_fields(metplus_config, config_overrides, fcst_fmts.append(fcst_fmt) obs_fmts.append(obs_fmt) - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) for index, (fcst, obs) in enumerate(zip(fcsts, obss)): @@ -583,7 +583,7 @@ def test_point_stat_all_fields(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PointStatConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py index 9e6e436bf..bce689fb2 100644 --- a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py +++ b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py @@ -15,7 +15,7 @@ def rdp_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return RegridDataPlaneWrapper(config) @@ -61,7 +61,7 @@ def rdp_wrapper(metplus_config): def test_set_field_command_line_arguments(metplus_config, field_info, expected_arg): data_type = 'FCST' - config = metplus_config() + config = metplus_config rdp = RegridDataPlaneWrapper(config) @@ -128,7 +128,7 @@ def test_set_field_command_line_arguments(metplus_config, field_info, expected_a def test_get_output_names(metplus_config, var_list, expected_names): data_type = 'FCST' - rdp = RegridDataPlaneWrapper(metplus_config()) + rdp = RegridDataPlaneWrapper(metplus_config) assert rdp.get_output_names(var_list, data_type) == expected_names diff --git a/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py b/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py index 0c589349c..0d0f218cf 100644 --- a/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py +++ b/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py @@ -102,7 +102,7 @@ ) @pytest.mark.wrapper def test_compare_time_info(metplus_config, runtime, filetime, expected_result): - config = metplus_config() + config = metplus_config wrapper = RuntimeFreqWrapper(config) actual_result = wrapper.compare_time_info(runtime, filetime) diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index d588d2425..228e89114 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -37,11 +37,27 @@ def get_input_dirs(config): def series_analysis_wrapper(metplus_config, config_overrides=None): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'series_test.conf')) - config = metplus_config(extra_configs) - config.set('config', 'LOOP_ORDER', 'processes') + config = metplus_config + config.set('config', 'SERIES_ANALYSIS_STAT_LIST', 'TOTAL, FBAR, OBAR, ME') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d') + config.set('config', 'INIT_BEG', '20141214') + config.set('config', 'INIT_END', '20141214') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'SERIES_ANALYSIS_BACKGROUND_MAP', 'no') + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE', + ('{init?fmt=%Y%m%d_%H}/{storm_id}/FCST_TILE_F{lead?fmt=%3H}_' + 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%3H}.nc')) + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE', + ('{init?fmt=%Y%m%d_%H}/{storm_id}/OBS_TILE_F{lead?fmt=%3H}_gfs' + '_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%3H}.nc')) + config.set('config', 'EXTRACT_TILES_OUTPUT_DIR', + '{OUTPUT_BASE}/extract_tiles') + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_DIR', + '{EXTRACT_TILES_OUTPUT_DIR}') + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_DIR', + '{EXTRACT_TILES_OUTPUT_DIR}') + config.set('config', 'SERIES_ANALYSIS_OUTPUT_DIR', + '{OUTPUT_BASE}/series_analysis_init') if config_overrides: for key, value in config_overrides.items(): config.set('config', key, value) @@ -297,7 +313,7 @@ def set_minimum_config_settings(config): def test_series_analysis_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -851,7 +867,7 @@ def test_get_netcdf_min_max(metplus_config): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'SeriesAnalysisConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index b5a9552da..76c277fd9 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -16,17 +16,43 @@ pp = pprint.PrettyPrinter() + def stat_analysis_wrapper(metplus_config): """! Returns a default StatAnalysisWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" + config = metplus_config - # Default, empty StatAnalysisWrapper with some configuration values set - # to /path/to: - extra_configs = [TEST_CONF] - config = metplus_config(extra_configs) handle_tmp_dir(config) + config.set('config', 'PROCESS_LIST', 'StatAnalysis') + config.set('config', 'STAT_ANALYSIS_OUTPUT_DIR', + '{OUTPUT_BASE}/stat_analysis') + config.set('config', 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR', + '{METPLUS_BASE}/internal/tests/data/stat_data') + config.set('config', 'LOOP_BY', 'VALID') + config.set('config', 'VALID_TIME_FMT', '%Y%m%d') + config.set('config', 'VALID_BEG', '20190101') + config.set('config', 'VALID_END', '20190101') + config.set('config', 'VALID_INCREMENT', '86400') + config.set('config', 'MODEL1', 'MODEL_TEST') + config.set('config', 'MODEL1_REFERENCE_NAME', 'MODELTEST') + config.set('config', 'MODEL1_OBTYPE', 'MODEL_TEST_ANL') + config.set('config', 'STAT_ANALYSIS_CONFIG_FILE', + '{PARM_BASE}/met_config/STATAnalysisConfig_wrapped') + config.set('config', 'STAT_ANALYSIS_JOB_NAME', 'filter') + config.set('config', 'STAT_ANALYSIS_JOB_ARGS', '-dump_row [dump_row_file]') + config.set('config', 'MODEL_LIST', '{MODEL1}') + config.set('config', 'FCST_VALID_HOUR_LIST', '00') + config.set('config', 'FCST_INIT_HOUR_LIST', '00, 06, 12, 18') + config.set('config', 'GROUP_LIST_ITEMS', 'FCST_INIT_HOUR_LIST') + config.set('config', 'LOOP_LIST_ITEMS', 'FCST_VALID_HOUR_LIST, MODEL_LIST') + config.set('config', 'MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE', + ('{fcst_valid_hour?fmt=%H}Z/{MODEL1}/' + '{MODEL1}_{valid?fmt=%Y%m%d}.stat')) + config.set('config', 'MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE', + ('{model?fmt=%s}_{obtype?fmt=%s}_valid{valid?fmt=%Y%m%d}' + '{valid_hour?fmt=%H}_init{fcst_init_hour?fmt=%s}.stat')) return StatAnalysisWrapper(config) @@ -148,9 +174,9 @@ def set_minimum_config_settings(config): ] ) @pytest.mark.wrapper_d -def test_stat_analysis_env_vars(metplus_config, config_overrides, - expected_env_vars): - config = metplus_config() +def test_valid_init_env_vars(metplus_config, config_overrides, + expected_env_vars): + config = metplus_config set_minimum_config_settings(config) config.set('config', 'INIT_END', '20221015') for key, value in config_overrides.items(): @@ -207,7 +233,7 @@ def test_stat_analysis_env_vars(metplus_config, config_overrides, @pytest.mark.wrapper_d def test_check_required_job_template(metplus_config, config_overrides, expected_result): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) for key, value in config_overrides.items(): config.set('config', key, value) @@ -292,10 +318,11 @@ def test_check_required_job_template(metplus_config, config_overrides, ) @pytest.mark.wrapper_d def test_get_runtime_settings(metplus_config, c_dict, expected_result): - config = metplus_config() + config = metplus_config wrapper = StatAnalysisWrapper(config) runtime_settings = wrapper._get_runtime_settings(c_dict) + pp.pprint(runtime_settings) assert runtime_settings == expected_result @@ -315,7 +342,7 @@ def test_get_runtime_settings(metplus_config, c_dict, expected_result): @pytest.mark.wrapper_d def test_format_conf_list(metplus_config, list_name, config_overrides, expected_value): - config = metplus_config() + config = metplus_config for key, value in config_overrides.items(): config.set('config', key, value) @@ -587,6 +614,7 @@ def test_build_stringsub_dict(metplus_config, lists_to_loop, c_dict_overrides, if item not in lists_to_loop] config_dict['LISTS_TO_GROUP'] = lists_to_group config_dict['LISTS_TO_LOOP'] = lists_to_loop + test_stringsub_dict = st._build_stringsub_dict(config_dict) print(test_stringsub_dict) @@ -869,7 +897,7 @@ def test_run_stat_analysis(metplus_config): ) @pytest.mark.wrapper_d def test_get_level_list(metplus_config, data_type, config_list, expected_list): - config = metplus_config() + config = metplus_config config.set('config', f'{data_type}_LEVEL_LIST', config_list) saw = StatAnalysisWrapper(config) @@ -880,7 +908,7 @@ def test_get_level_list(metplus_config, data_type, config_list, expected_list): @pytest.mark.wrapper_d def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INPUT_MUST_EXIST', False) wrapper = StatAnalysisWrapper(config) diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index 248228fa9..b3cb191fa 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -293,7 +293,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): expected_edeck_count = 6 expected_shape_count = 5 - config = metplus_config() + config = metplus_config test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), 'internal', 'tests', @@ -423,7 +423,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'TCGenConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf b/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf deleted file mode 100644 index 8c574d2e5..000000000 --- a/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf +++ /dev/null @@ -1,105 +0,0 @@ -# -# CONFIGURATION -# -[config] -LOOP_METHOD = processes -# Configuration files -TC_PAIRS_CONFIG_FILE = {PARM_BASE}/met_config/TCPairsConfig_wrapped - -PROCESS_LIST = TCPairs - -# The init time begin and end times, increment, and last init hour. -INIT_TIME_FMT = %Y%m%d -INIT_BEG = 20141201 -INIT_END = 20141231 -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds -TC_PAIRS_INIT_INCLUDE = -TC_PAIRS_INIT_EXCLUDE = - -TC_PAIRS_VALID_BEG = -TC_PAIRS_VALID_END = - -TC_PAIRS_READ_ALL_FILES = no - -# set to true or yes to reformat track data into ATCF format expected by tc_pairs -TC_PAIRS_REFORMAT_DECK = yes -TC_PAIRS_REFORMAT_TYPE = SBU - - -# TC PAIRS filtering options -TC_PAIRS_MISSING_VAL_TO_REPLACE = -99 -TC_PAIRS_MISSING_VAL = -9999 - - -# OVERWRITE OPTIONS -# Don't overwrite filter files if they already exist. -# Set to no if you do NOT want to override existing files -# Set to yes if you do want to override existing files -#OVERWRITE_TRACK = yes -TC_PAIRS_SKIP_IF_REFORMAT_EXISTS = no -TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = no - -# List of models to be used (white space or comma separated) eg: DSHP, LGEM, HWRF -# If no models are listed, then process all models in the input file(s). -MODEL = - -# List of storm ids of interest (space or comma separated) e.g.: AL112012, AL122012 -# If no storm ids are listed, then process all storm ids in the input file(s). -TC_PAIRS_STORM_ID = - -# Basins (of origin/region). Indicate with space or comma-separated list of regions, eg. AL: for North Atlantic, -# WP: Western North Pacific, CP: Central North Pacific, SH: Southern Hemisphere, IO: North Indian Ocean, LS: Southern -# Hemisphere -TC_PAIRS_BASIN = - -# Cyclone, a space or comma-separated list of cyclone numbers. If left empty, all cyclones will be used. -TC_PAIRS_CYCLONE = - -# Storm name, a space or comma-separated list of storm names to evaluate. If left empty, all storms will be used. -TC_PAIRS_STORM_NAME = - -# DLAND file, the full path of the file that contains the gridded representation of the -# minimum distance from land. -TC_PAIRS_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc - - -# -# FILENAME TEMPLATES -# -[filename_templates] -# We DO NOT want to interpret time info or expand{} these values. -# Use, getraw('filename_templates','FCST_EXTRACT_TILES_INPUT_TEMPLATE') to get -# 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2' -# FCST_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2 -# GFS_FCST_NC_FILE_TMPL = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.nc -# OBS_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2 -# GFS_ANLY_NC_FILE_TMPL = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.nc - -TC_PAIRS_ADECK_TEMPLATE = {date?fmt=%Y%m}/a{basin?fmt=%s}q{date?fmt=%Y%m}*.gfso.{cyclone?fmt=%s} -TC_PAIRS_BDECK_TEMPLATE = {date?fmt=%Y%m}/b{basin?fmt=%s}q{date?fmt=%Y%m}*.gfso.{cyclone?fmt=%s} -TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gfso.{cyclone?fmt=%s} - -# -# DIRECTORIES -# -[dir] - -# Location of your model data of interest -#EXTRACT_TILES_GRID_INPUT_DIR = {METPLUS_BASE}/sample_data/GFS/reduced_model_data -#EXTRACT_TILES_GRID_INPUT_DIR = /d1/SBU/GFS/reduced_model_data -# Commonly used base METplus variables - -# track data, set to your data source -TC_PAIRS_ADECK_INPUT_DIR = {INPUT_BASE}/met_test/new/track_data -TC_PAIRS_BDECK_INPUT_DIR = {INPUT_BASE}/met_test/new/track_data - - -#TRACK_DATA_DIR = {METPLUS_BASE}/sample_data/GFS/track_data -#TC_PAIRS_ADECK_INPUT_DIR = /d1/SBU/GFS/track_data -#TC_PAIRS_ADECK_INPUT_DIR = /d1/METplus_TC/adeck -#TC_PAIRS_BDECK_INPUT_DIR = /d1/SBU/GFS/track_data -#TC_PAIRS_BDECK_INPUT_DIR = /d1/METplus_TC/bdeck -TC_PAIRS_REFORMAT_DIR = {OUTPUT_BASE}/track_data_atcf -#TRACK_DATA_SUBDIR_MOD = {PROJ_DIR}/track_data_atcf -TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs - diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 8f9de6d6f..b7ad438f2 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -68,7 +68,7 @@ def set_minimum_config_settings(config, loop_by='INIT'): ) def test_read_storm_info(metplus_config, config_overrides, isOK): """! Check if error is thrown if storm_id and basin or cyclone are set """ - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) # set config variable overrides @@ -94,7 +94,7 @@ def test_parse_storm_id(metplus_config, storm_id, basin, cyclone): Check that it returns wildcard expressions basin and cyclone cannot be parsed from storm ID """ - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -141,7 +141,7 @@ def test_get_bdeck(metplus_config, basin, cyclone, expected_files, combinations of basin/cyclone inputs """ time_info = {'date': datetime(2014, 12, 31, 18)} - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -194,7 +194,7 @@ def test_get_basin_cyclone_from_bdeck(metplus_config, template, filename, expected_basin = other_basin if other_basin else 'al' expected_cyclone = other_cyclone if other_cyclone else '1009' time_info = {'date': datetime(2014, 12, 31, 18)} - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) wrapper = TCPairsWrapper(config) @@ -245,7 +245,7 @@ def test_get_basin_cyclone_from_bdeck(metplus_config, template, filename, @pytest.mark.wrapper def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, storm_type, values_to_check): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -297,47 +297,48 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, @pytest.mark.parametrize( - 'config_overrides, env_var_values', [ + 'loop_by, config_overrides, env_var_values', [ + # LOOP_BY = INIT # 0: no config overrides that set env vars - ({}, {}), + ('INIT', {}, {}), # 1: description - ({'TC_PAIRS_DESC': 'my_desc'}, + ('INIT', {'TC_PAIRS_DESC': 'my_desc'}, {'METPLUS_DESC': 'desc = "my_desc";'}), # 2: only basin that corresponds to existing test file is used - ({'TC_PAIRS_BASIN': 'AL, ML'}, + ('INIT', {'TC_PAIRS_BASIN': 'AL, ML'}, {'METPLUS_BASIN': 'basin = ["ML"];'}), # 3: only cyclone that corresponds to existing test file is used - ({'TC_PAIRS_CYCLONE': '1005, 0104'}, + ('INIT', {'TC_PAIRS_CYCLONE': '1005, 0104'}, {'METPLUS_CYCLONE': 'cyclone = ["0104"];'}), # 4: model list - ({'MODEL': 'MOD1, MOD2'}, + ('INIT', {'MODEL': 'MOD1, MOD2'}, {'METPLUS_MODEL': 'model = ["MOD1", "MOD2"];'}), # 5: init begin - ({'TC_PAIRS_INIT_BEG': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_BEG': '20141031_14'}, {'METPLUS_INIT_BEG': 'init_beg = "20141031_14";'}), # 6: init end - ({'TC_PAIRS_INIT_END': '20151031_14'}, + ('INIT', {'TC_PAIRS_INIT_END': '20151031_14'}, {'METPLUS_INIT_END': 'init_end = "20151031_14";'}), # 7: dland file - ({'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, + ('INIT', {'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, {'METPLUS_DLAND_FILE': 'dland_file = "my_dland.nc";'}), # 8: init_exc - ({'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, {'METPLUS_INIT_EXC': 'init_exc = ["20141031_14"];'}), # 9: init_inc - ({'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, {'METPLUS_INIT_INC': 'init_inc = ["20141031_14"];'}), # 10: storm name - ({'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, + ('INIT', {'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, {'METPLUS_STORM_NAME': 'storm_name = ["KATRINA", "OTHER"];'}), # 11: valid begin - ({'TC_PAIRS_VALID_BEG': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_BEG': '20141031_14'}, {'METPLUS_VALID_BEG': 'valid_beg = "20141031_14";'}), # 12: valid end - ({'TC_PAIRS_VALID_END': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_END': '20141031_14'}, {'METPLUS_VALID_END': 'valid_end = "20141031_14";'}), # 13: consensus 1 dictionary - ({'TC_PAIRS_CONSENSUS1_NAME': 'name1', + ('INIT', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1'}, @@ -346,7 +347,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'required = [true, false];min_req = 1;}];' )}), # 14: consensus 2 dictionaries - ({'TC_PAIRS_CONSENSUS1_NAME': 'name1', + ('INIT', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1', @@ -363,199 +364,283 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'required = [false, true];min_req = 2;}];' )}), # 15: valid_exc - ({'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, {'METPLUS_VALID_EXC': 'valid_exc = ["20141031_14"];'}), # 16: valid_inc - ({'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, {'METPLUS_VALID_INC': 'valid_inc = ["20141031_14"];'}), # 17: write_valid - ({'TC_PAIRS_WRITE_VALID': '20141031_14'}, + ('INIT', {'TC_PAIRS_WRITE_VALID': '20141031_14'}, {'METPLUS_WRITE_VALID': 'write_valid = ["20141031_14"];'}), # 18: check_dup - ({'TC_PAIRS_CHECK_DUP': 'False', }, + ('INIT', {'TC_PAIRS_CHECK_DUP': 'False', }, {'METPLUS_CHECK_DUP': 'check_dup = FALSE;'}), # 19: interp12 - ({'TC_PAIRS_INTERP12': 'replace', }, + ('INIT', {'TC_PAIRS_INTERP12': 'replace', }, {'METPLUS_INTERP12': 'interp12 = REPLACE;'}), # 20 match_points - ({'TC_PAIRS_MATCH_POINTS': 'False', }, + ('INIT', {'TC_PAIRS_MATCH_POINTS': 'False', }, + {'METPLUS_MATCH_POINTS': 'match_points = FALSE;'}), + # LOOP_BY = VALID + # 21: no config overrides that set env vars + ('VALID', {}, {}), + # 22: description + ('VALID', {'TC_PAIRS_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + # 23: only basin that corresponds to existing test file is used + ('VALID', {'TC_PAIRS_BASIN': 'AL, ML'}, + {'METPLUS_BASIN': 'basin = ["ML"];'}), + # 24: only cyclone that corresponds to existing test file is used + ('VALID', {'TC_PAIRS_CYCLONE': '1005, 0104'}, + {'METPLUS_CYCLONE': 'cyclone = ["0104"];'}), + # 25: model list + ('VALID', {'MODEL': 'MOD1, MOD2'}, + {'METPLUS_MODEL': 'model = ["MOD1", "MOD2"];'}), + # 26: init begin + ('VALID', {'TC_PAIRS_INIT_BEG': '20141031_14'}, + {'METPLUS_INIT_BEG': 'init_beg = "20141031_14";'}), + # 27: init end + ('VALID', {'TC_PAIRS_INIT_END': '20151031_14'}, + {'METPLUS_INIT_END': 'init_end = "20151031_14";'}), + # 28: dland file + ('VALID', {'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, + {'METPLUS_DLAND_FILE': 'dland_file = "my_dland.nc";'}), + # 29: init_exc + ('VALID', {'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, + {'METPLUS_INIT_EXC': 'init_exc = ["20141031_14"];'}), + # 30: init_inc + ('VALID', {'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, + {'METPLUS_INIT_INC': 'init_inc = ["20141031_14"];'}), + # 31: storm name + ('VALID', {'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, + {'METPLUS_STORM_NAME': 'storm_name = ["KATRINA", "OTHER"];'}), + # 32: valid begin + ('VALID', {'TC_PAIRS_VALID_BEG': '20141031_14'}, + {'METPLUS_VALID_BEG': 'valid_beg = "20141031_14";'}), + # 33: valid end + ('VALID', {'TC_PAIRS_VALID_END': '20141031_14'}, + {'METPLUS_VALID_END': 'valid_end = "20141031_14";'}), + # 34: consensus 1 dictionary + ('VALID', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', + 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', + 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', + 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1'}, + {'METPLUS_CONSENSUS_LIST': ( + 'consensus = [{name = "name1";members = ["member1a", "member1b"];' + 'required = [true, false];min_req = 1;}];' + )}), + # 35: consensus 2 dictionaries + ('VALID', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', + 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', + 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', + 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1', + 'TC_PAIRS_CONSENSUS2_NAME': 'name2', + 'TC_PAIRS_CONSENSUS2_MEMBERS': 'member2a, member2b', + 'TC_PAIRS_CONSENSUS2_REQUIRED': 'false, true', + 'TC_PAIRS_CONSENSUS2_MIN_REQ': '2' + }, + {'METPLUS_CONSENSUS_LIST': ( + 'consensus = [' + '{name = "name1";members = ["member1a", "member1b"];' + 'required = [true, false];min_req = 1;}' + '{name = "name2";members = ["member2a", "member2b"];' + 'required = [false, true];min_req = 2;}];' + )}), + # 36: valid_exc + ('VALID', {'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, + {'METPLUS_VALID_EXC': 'valid_exc = ["20141031_14"];'}), + # 37: valid_inc + ('VALID', {'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, + {'METPLUS_VALID_INC': 'valid_inc = ["20141031_14"];'}), + # 38: write_valid + ('VALID', {'TC_PAIRS_WRITE_VALID': '20141031_14'}, + {'METPLUS_WRITE_VALID': 'write_valid = ["20141031_14"];'}), + # 39: check_dup + ('VALID', {'TC_PAIRS_CHECK_DUP': 'False', }, + {'METPLUS_CHECK_DUP': 'check_dup = FALSE;'}), + # 40: interp12 + ('VALID', {'TC_PAIRS_INTERP12': 'replace', }, + {'METPLUS_INTERP12': 'interp12 = REPLACE;'}), + # 41 match_points + ('VALID', {'TC_PAIRS_MATCH_POINTS': 'False', }, {'METPLUS_MATCH_POINTS': 'match_points = FALSE;'}), - ] ) @pytest.mark.wrapper -def test_tc_pairs_loop_order_processes(metplus_config, config_overrides, - env_var_values): - # run using init and valid time variables - for loop_by in ['INIT', 'VALID']: - remove_beg = remove_end = remove_match_points = False - config = metplus_config() - - set_minimum_config_settings(config, loop_by) - - test_data_dir = get_data_dir(config) - bdeck_dir = os.path.join(test_data_dir, 'bdeck') - adeck_dir = os.path.join(test_data_dir, 'adeck') - - config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) - config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') - - # set config variable overrides - for key, value in config_overrides.items(): - config.set('config', key, value) - - if f'METPLUS_{loop_by}_BEG' not in env_var_values: - env_var_values[f'METPLUS_{loop_by}_BEG'] = ( - f'{loop_by.lower()}_beg = "{run_times[0]}";' - ) - remove_beg = True - - if f'METPLUS_{loop_by}_END' not in env_var_values: - env_var_values[f'METPLUS_{loop_by}_END'] = ( - f'{loop_by.lower()}_end = "{run_times[-1]}";' - ) - remove_end = True - - if f'METPLUS_MATCH_POINTS' not in env_var_values: - env_var_values[f'METPLUS_MATCH_POINTS'] = ( - 'match_points = TRUE;' - ) - remove_match_points = True - - wrapper = TCPairsWrapper(config) - assert wrapper.isOK - - app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) - verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" - config_file = wrapper.c_dict.get('CONFIG_FILE') - out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} {verbosity} " - f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " - f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 " - f"-config {config_file} " - f"-out {out_dir}/mlq2014121318.gfso.0104"), - ] - - - all_cmds = wrapper.run_all_times() - print(f"ALL COMMANDS: {all_cmds}") - assert len(all_cmds) == len(expected_cmds) - - for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): - # ensure commands are generated as expected - assert cmd == expected_cmd - - # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: - match = next((item for item in env_vars if - item.startswith(env_var_key)), None) - assert match is not None - print(f'Checking env var: {env_var_key}') - actual_value = match.split('=', 1)[1] - assert env_var_values.get(env_var_key, '') == actual_value - - if remove_beg: - del env_var_values[f'METPLUS_{loop_by}_BEG'] - if remove_end: - del env_var_values[f'METPLUS_{loop_by}_END'] - if remove_match_points: - del env_var_values['METPLUS_MATCH_POINTS'] +def test_tc_pairs_loop_order_processes(metplus_config, loop_by, + config_overrides, env_var_values): + config = metplus_config + remove_beg = remove_end = remove_match_points = False + + set_minimum_config_settings(config, loop_by) + + test_data_dir = get_data_dir(config) + bdeck_dir = os.path.join(test_data_dir, 'bdeck') + adeck_dir = os.path.join(test_data_dir, 'adeck') + + config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) + config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) + + # LOOP_ORDER processes runs once, times runs once per time + config.set('config', 'LOOP_ORDER', 'processes') + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + if f'METPLUS_{loop_by}_BEG' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_BEG'] = ( + f'{loop_by.lower()}_beg = "{run_times[0]}";' + ) + remove_beg = True + + if f'METPLUS_{loop_by}_END' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_END'] = ( + f'{loop_by.lower()}_end = "{run_times[-1]}";' + ) + remove_end = True + + if f'METPLUS_MATCH_POINTS' not in env_var_values: + env_var_values[f'METPLUS_MATCH_POINTS'] = ( + 'match_points = TRUE;' + ) + remove_match_points = True + + wrapper = TCPairsWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " + f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 " + f"-config {config_file} " + f"-out {out_dir}/mlq2014121318.gfso.0104"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == actual_value + + if remove_beg: + del env_var_values[f'METPLUS_{loop_by}_BEG'] + if remove_end: + del env_var_values[f'METPLUS_{loop_by}_END'] + if remove_match_points: + del env_var_values['METPLUS_MATCH_POINTS'] @pytest.mark.parametrize( - 'config_overrides, env_var_values', [ - # 0: no config overrides that set env vars - ({}, {}), + 'loop_by, config_overrides, env_var_values', [ + # 0: no config overrides that set env vars loop by = INIT + ('INIT', {}, {}), # 1: storm_id list - ({'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, + ('INIT', {'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, {'METPLUS_STORM_ID': 'storm_id = ["AL092014", "ML082015"];'}), # 2: basin list - ({'TC_PAIRS_BASIN': 'AL, ML'}, + ('INIT', {'TC_PAIRS_BASIN': 'AL, ML'}, {'METPLUS_BASIN': 'basin = ["AL", "ML"];'}), # 3: cyclone list - ({'TC_PAIRS_CYCLONE': '1005, 0104'}, + ('INIT', {'TC_PAIRS_CYCLONE': '1005, 0104'}, + {'METPLUS_CYCLONE': 'cyclone = ["1005", "0104"];'}), + # 4: no config overrides that set env vars loop by = VALID + ('VALID', {}, {}), + # 5: storm_id list + ('VALID', {'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, + {'METPLUS_STORM_ID': 'storm_id = ["AL092014", "ML082015"];'}), + # 6: basin list + ('VALID', {'TC_PAIRS_BASIN': 'AL, ML'}, + {'METPLUS_BASIN': 'basin = ["AL", "ML"];'}), + # 7: cyclone list + ('VALID', {'TC_PAIRS_CYCLONE': '1005, 0104'}, {'METPLUS_CYCLONE': 'cyclone = ["1005", "0104"];'}), ] ) @pytest.mark.wrapper -def test_tc_pairs_read_all_files(metplus_config, config_overrides, +def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, env_var_values): - # run using init and valid time variables - for loop_by in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config + + set_minimum_config_settings(config, loop_by) - set_minimum_config_settings(config, loop_by) + test_data_dir = get_data_dir(config) + bdeck_dir = os.path.join(test_data_dir, 'bdeck') + adeck_dir = os.path.join(test_data_dir, 'adeck') - test_data_dir = get_data_dir(config) - bdeck_dir = os.path.join(test_data_dir, 'bdeck') - adeck_dir = os.path.join(test_data_dir, 'adeck') + config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) + config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) - config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) + # LOOP_ORDER processes runs once, times runs once per time + config.set('config', 'LOOP_ORDER', 'processes') - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') + config.set('config', 'TC_PAIRS_READ_ALL_FILES', True) + config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '') - config.set('config', 'TC_PAIRS_READ_ALL_FILES', True) - config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '') + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) - # set config variable overrides - for key, value in config_overrides.items(): - config.set('config', key, value) + env_var_values[f'METPLUS_{loop_by}_BEG'] = ( + f'{loop_by.lower()}_beg = "{run_times[0]}";' + ) - env_var_values[f'METPLUS_{loop_by}_BEG'] = ( - f'{loop_by.lower()}_beg = "{run_times[0]}";' - ) + env_var_values[f'METPLUS_{loop_by}_END'] = ( + f'{loop_by.lower()}_end = "{run_times[-1]}";' + ) - env_var_values[f'METPLUS_{loop_by}_END'] = ( - f'{loop_by.lower()}_end = "{run_times[-1]}";' - ) + env_var_values['METPLUS_MATCH_POINTS'] = ( + 'match_points = TRUE;' + ) - env_var_values['METPLUS_MATCH_POINTS'] = ( - 'match_points = TRUE;' - ) + wrapper = TCPairsWrapper(config) + assert wrapper.isOK - wrapper = TCPairsWrapper(config) - assert wrapper.isOK - - app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) - verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" - config_file = wrapper.c_dict.get('CONFIG_FILE') - out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} {verbosity} " - f"-bdeck {bdeck_dir} " - f"-adeck {adeck_dir} " - f"-config {config_file} " - f"-out {out_dir}/tc_pairs"), - ] - - all_cmds = wrapper.run_all_times() - print(f"ALL COMMANDS: {all_cmds}") - assert len(all_cmds) == len(expected_cmds) - - for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): - # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: - match = next((item for item in env_vars if - item.startswith(env_var_key)), None) - assert match is not None - print(f'Checking env var: {env_var_key}') - actual_value = match.split('=', 1)[1] - assert env_var_values.get(env_var_key, '') == actual_value - - # unset begin and end for next loop - del env_var_values[f'METPLUS_{loop_by}_BEG'] - del env_var_values[f'METPLUS_{loop_by}_END'] + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-bdeck {bdeck_dir} " + f"-adeck {adeck_dir} " + f"-config {config_file} " + f"-out {out_dir}/tc_pairs"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == actual_value @pytest.mark.wrapper def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INIT_TIME_FMT', time_fmt) config.set('config', 'INIT_BEG', run_times[0]) default_config_file = os.path.join(config.getdir('PARM_BASE'), diff --git a/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf b/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf deleted file mode 100755 index ea35a5b14..000000000 --- a/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf +++ /dev/null @@ -1,173 +0,0 @@ -# -# PRECONDITION: REQUIRES INSTALLATION OF R on user system -# - -# -# CONFIGURATION -# -[config] -# set looping method to processes-each 'task' in the process list runs to -# completion (for all init times) before the next 'task' is run -LOOP_METHOD = processes - -# List of 'tasks' to run -PROCESS_LIST = TcStat - -# The init time begin and end times, increment, and last init hour. -INIT_BEG = 20150301 -INIT_END = 20150304 -# This is the step-size. Increment in seconds from the begin time to the end time -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds - -# This is the last hour in your initialization time that you want to include in your time window -#INIT_HOUR_END = 18 - -# A list of times to include, in format YYYYMMDD_hh -#INIT_INCLUDE = - -# A list of times to exclude, in format YYYYMMDD_hh -#INIT_EXCLUDE = - -# -# Specify model valid time window in format YYYYMM[DD[_hh]]. Only tracks that fall within the valid time window will -# be used. -# -#VALID_BEG = -#VALID_END = - -# Run tc_stat using a config file or as command line -# if running via MET tc_stat config file, set to CONFIG. Leave blank or -# anything other than CONFIG if running via command line. -TC_STAT_CONFIG_FILE = {PARM_BASE}/met_config/TCStatConfig_wrapped - - -# !!!!!!!IMPORTANT!!!!!! -# Please refer to the README_TC located in ${MET_INSTALL_DIR}/share/met/config -# for details on setting up your analysis jobs. - -# For arithmetic expressions such as: -# -column 'ABS(AMSLP-BMSLP)', enclose the expression in ''. Notice that there are no -# whitespaces within the arithmetic expression. White spaces are to be used to -# separate options from values (e.g. -job summary -by AMODEL,LEAD,AMSLP -init_hour 00 -column 'AMSLP-BMSLP'). -# eg. -lookin {OUTPUT_BASE}/tc_pairs -job filter -dump_row {OUTPUT_BASE}/tc_stat_filter.out -basin ML -init_hr 00 -# or -lookin {OUTPUT_BASE}/tc_pairs -job summary -by AMODEL,LEAD -column AMSLP -column AMAX_WIND -column 'ABS(AMAX_WIND-BMAX_WIND)' -out {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst - -# Only if TC_STAT_RUN_VIA = CLI -# TC_STAT_CMD_LINE_JOB = -job filter -dump_row {OUTPUT_BASE}/tc_stat/tc_stat_filter.out -basin ML -init_hour 00 - -#TC_STAT_RUN_VIA=COMMAND so no need to define this, but you MUST define -# TC_STAT_JOBS_LIST - -# -# FILL in the following values if running multiple jobs which -# requires a MET tc_stat config file. -# -# These all map to the options in the default TC-Stat config file, except these -# are pre-pended with TC_STAT to avoid clashing with any other similarly -# named options from other MET tools (eg TC_STAT_AMODEL corresponds to the -# amodel option in the default MET tc-stat config file, whereas AMODEL -# corresponds to the amodel option in the MET tc-pairs config file). - -# Stratify by these columns: -TC_STAT_AMODEL = -TC_STAT_BMODEL = -TC_STAT_DESC = -TC_STAT_STORM_ID = -TC_STAT_BASIN = -TC_STAT_CYCLONE = -TC_STAT_STORM_NAME = - -# Stratify by init times via a comma-separate list of init times to -# include or exclude. Time format defined as YYYYMMDD_HH or YYYYMMDD_HHmmss -TC_STAT_INIT_BEG = 20170705 -TC_STAT_INIT_END = 20170901 -TC_STAT_INIT_INCLUDE = -TC_STAT_INIT_EXCLUDE = -TC_STAT_INIT_HOUR = 00 -TC_STAT_INIT_MASK = -TC_STAT_VALID_MASK = -TC_STAT_VALID_BEG = -TC_STAT_VALID_END = -TC_STAT_VALID_INCLUDE = -TC_STAT_VALID_EXCLUDE = -TC_STAT_LEAD_REQ = - -# Stratify by the valid time and lead time via comma-separated list of -# times in format HH[MMSS] -TC_STAT_VALID_HOUR = -TC_STAT_LEAD = - -# Stratify over the watch_warn column in the tcst file. Setting this to -# 'ALL' will match HUWARN, HUWATCH, TSWARN, TSWATCH -TC_STAT_TRACK_WATCH_WARN = - -# Stratify by applying thresholds to numeric data columns. Specify with -# comma-separated list of column names and thresholds to be applied. -# The length of TC_STAT_COLUMN_THRESH_NAME should be the same as -# TC_STAT_COLUMN_THRESH_VAL. -TC_STAT_COLUMN_THRESH_NAME = -TC_STAT_COLUMN_THRESH_VAL = - -# Stratify by a list of comma-separated columns names and values corresponding -# to non-numeric data columns of the values of interest. -TC_STAT_COLUMN_STR_NAME = -TC_STAT_COLUMN_STR_VAL = - -# Stratify by applying thresholds to numeric data columns only when lead=0. -# If lead=0 and the value does not meet the threshold, discard the entire -# track. The length of TC_STAT_INIT_THRESH_NAME must equal the length of -# TC_STAT_INIT_THRESH_VAL. -TC_STAT_INIT_THRESH_NAME = -TC_STAT_INIT_THRESH_VAL = - -# Stratify by applying thresholds to numeric data columns only when lead = 0. -# If lead = 0 but the value doesn't meet the threshold, discard the entire -# track. -TC_STAT_INIT_STR_NAME = -TC_STAT_INIT_STR_VAL = - -# Excludes any points where distance to land is <=0. When set to TRUE, once land -# is encountered, the remainder of the forecast track is NOT used for the -# verification, even if the track moves back over water. -TC_STAT_WATER_ONLY = false - -# TRUE or FALSE. To specify whether only those track points occurring near -# landfall should be retained. Landfall is the last bmodel track point before -# the distance to land switches from water to land. -TC_STAT_LANDFALL = false - - -# Define the landfall retention window, which is defined as the hours offset -# from the time of landfall. Format is in HH[MMSS]. Default TC_STAT_LANDFALL_BEG -# is set to -24, and TC_STAT_LANDFALL_END is set to 00 -TC_STAT_LANDFALL_BEG = -24 -TC_STAT_LANDFALL_END = 00 - -# Specify whether only those track points common to both the ADECK and BDECK -# tracks should be written out -TC_STAT_MATCH_POINTS = false - -# IMPORTANT Refer to the README_TC for details on setting up analysis -# jobs (located in {MET_INSTALL_DIR}/share/met/config - -# Separate each option and value with whitespace, and each job with a whitespace. -# No whitespace within arithmetic expressions or lists of items -# (e.g. -by AMSLP,AMODEL,LEAD -column '(AMAX_WIND-BMAX_WIND)') -# Enclose your arithmetic expressions with '' and separate each job -# by whitespace: -# -job filter -dump_row /path/to, -job summary -line_type TCMPR -column 'ABS(AMAX_WIND-BMAX_WIND)' -out {OUTPUT_BASE}/tc_stat/file.tcst - -TC_STAT_JOB_ARGS = -job summary -line_type TCMPR -column 'ABS(AMAX_WIND-BMAX_WIND)' -dump_row {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst - - -# -# DIRECTORIES -# -[dir] - -# TC-Stat input data (uses output from tc-pairs) -TC_STAT_LOOKIN_DIR = {INPUT_BASE}/met_test/tc_pairs - -# TC-Stat output data (creates .tcst ASCII files which can be read or used as -# input to TCMPR_Plotter_wrapper (the Python wrapper to plot_tcmpr.R) to create plots. -TC_STAT_OUTPUT_DIR = {OUTPUT_BASE}/tc_stat diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index fe66cff3f..ac7001898 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -11,10 +11,25 @@ def get_config(metplus_config): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'tc_stat_conf.conf')) - return metplus_config(extra_configs) + # extra_configs = [] + # extra_configs.append(os.path.join(os.path.dirname(__file__), + # 'tc_stat_conf.conf')) + config = metplus_config + config.set('config', 'PROCESS_LIST', 'TCStat') + config.set('config', 'INIT_BEG', '20150301') + config.set('config', 'INIT_END', '20150304') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'TC_STAT_INIT_BEG', '20170705') + config.set('config', 'TC_STAT_INIT_END', '20170901') + config.set('config', 'TC_STAT_INIT_HOUR', '00') + config.set('config', 'TC_STAT_JOB_ARGS', + ("-job summary -line_type TCMPR -column " + "'ABS(AMAX_WIND-BMAX_WIND)' " + "-dump_row {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst")) + config.set('config', 'TC_STAT_LOOKIN_DIR', + '{INPUT_BASE}/met_test/tc_pairs') + config.set('config', 'TC_STAT_OUTPUT_DIR', '{OUTPUT_BASE}/tc_stat') + return config def tc_stat_wrapper(metplus_config): @@ -265,7 +280,7 @@ def test_handle_jobs_create_parent_dir(metplus_config, jobs, init_dt, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', diff --git a/internal/tests/pytests/wrappers/user_script/test_user_script.py b/internal/tests/pytests/wrappers/user_script/test_user_script.py index b45fe8810..060f8f2a3 100644 --- a/internal/tests/pytests/wrappers/user_script/test_user_script.py +++ b/internal/tests/pytests/wrappers/user_script/test_user_script.py @@ -347,7 +347,7 @@ def set_run_type_info(config, run_type): @pytest.mark.wrapper def test_run_user_script_all_times(metplus_config, input_configs, run_types, expected_cmds): - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) for key, value in input_configs.items(): diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 1471e60e0..ef633503a 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -83,6 +83,7 @@ Category: marine_and_cryosphere 4::GridStat_fcstRTOFS_obsSMAP_climWOA_sss::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf:: icecover_env, py_embed 5::GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf:: icecover_env, py_embed 6::UserScript_fcstRTOFS_obsAOML_calcTransport::model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf:: icecover_env, py_embed +7::PointStat_fcstGFS_obsNDBC_WaveHeight::model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf #X::GridStat_fcstRTOFS_obsGHRSST_climWOA_sst::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf, model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst/ci_overrides.conf:: icecover_env, py_embed @@ -109,6 +110,7 @@ Category: precipitation 6::MTD_fcstHRRR-TLE_obsMRMS:: model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf 7::EnsembleStat_fcstWOFS_obsWOFS:: model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf:: 8::PointStat_fcstMULTI_obsMETAR_PtypeComparisons:: model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf +9::PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip:: model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf Category: s2s diff --git a/internal/tests/use_cases/metplus_use_case_suite.py b/internal/tests/use_cases/metplus_use_case_suite.py index d194b4c3a..b26accc59 100644 --- a/internal/tests/use_cases/metplus_use_case_suite.py +++ b/internal/tests/use_cases/metplus_use_case_suite.py @@ -11,7 +11,7 @@ sys.path.insert(0, os.path.join(os.path.abspath(dirname(__file__)), os.pardir, os.pardir)) -from metplus.util.met_util import subset_list +from metplus.util.string_manip import subset_list class METplusUseCase: """! Contains name of use case and a list of configuration command line diff --git a/metplus/VERSION b/metplus/VERSION index 008caf304..19a5d1e08 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.0.0-beta4-dev +5.0.0-beta5-dev diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index b4e143d39..58df7821c 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -1,11 +1,12 @@ +from .metplus_check import * from .constants import * from .string_manip import * -from .metplus_check import * -from .doc_util import * -from .config_metplus import * +from .system_util import * from .time_util import * -from .met_util import * from .string_template_substitution import * +from .doc_util import * +from .config_metplus import * +from .run_util import * from .met_config import * from .time_looping import * from .field_util import * diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index a42324cdb..5e9b02c07 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -22,11 +22,11 @@ from produtil.config import ProdConfig -from .constants import RUNTIME_CONFS -from . import met_util as util +from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE from .string_template_substitution import get_tags, do_string_sub -from .met_util import is_python_script, format_var_items -from .string_manip import getlist, remove_quotes +from .string_manip import getlist, remove_quotes, is_python_script +from .string_manip import validate_thresholds +from .system_util import mkdir_p from .doc_util import get_wrapper_name """!Creates the initial METplus directory structure, @@ -51,8 +51,12 @@ 'get_custom_string_list', 'find_indices_in_config_section', 'parse_var_list', + 'sub_var_list', 'get_process_list', 'validate_configuration_variables', + 'is_loop_by_init', + 'handle_tmp_dir', + 'log_runtime_banner', ] '''!@var METPLUS_BASE @@ -251,7 +255,7 @@ def launch(config_list): # that is logged relates to OUTPUT_BASE, not LOG_DIR, which is likely # only set incorrectly because OUTPUT_BASE is set incorrectly # Initialize the output directories - util.mkdir_p(config.getdir('OUTPUT_BASE')) + mkdir_p(config.getdir('OUTPUT_BASE')) # set and log variables to the config object get_logger(config) @@ -260,7 +264,7 @@ def launch(config_list): # create final conf directory if it doesn't already exist final_conf_dir = os.path.dirname(final_conf) - util.mkdir_p(final_conf_dir) + mkdir_p(final_conf_dir) # set METPLUS_BASE/PARM_BASE conf so they can be referenced in other confs config.set('config', 'METPLUS_BASE', METPLUS_BASE) @@ -288,7 +292,7 @@ def _set_logvars(config, logger=None): log_timestamp_template = config.getstr('config', 'LOG_TIMESTAMP_TEMPLATE', '') if config.getbool('config', 'LOG_TIMESTAMP_USE_DATATIME', False): - loop_by = 'INIT' if util.is_loop_by_init(config) else 'VALID' + loop_by = 'INIT' if is_loop_by_init(config) else 'VALID' time_str = config.getraw('config', f'{loop_by}_BEG') time_fmt = config.getraw('config', f'{loop_by}_TIME_FMT') date_t = datetime.strptime(time_str, time_fmt) @@ -353,7 +357,7 @@ def get_logger(config, sublog=None): log_level = config.getstr('config', 'LOG_LEVEL') # Create the log directory if it does not exist - util.mkdir_p(log_dir) + mkdir_p(log_dir) if sublog is not None: logger = config.log(sublog) @@ -387,7 +391,7 @@ def get_logger(config, sublog=None): # So lets check and make more directory if needed. dir_name = os.path.dirname(metpluslog) if not os.path.exists(dir_name): - util.mkdir_p(dir_name) + mkdir_p(dir_name) # do not send logs up to root logger handlers logger.propagate = False @@ -796,7 +800,7 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, # if config variable is not set except NoOptionError: if default is None: - default = util.MISSING_DATA_VALUE + default = MISSING_DATA_VALUE self.check_default(sec, name, default) return default @@ -805,7 +809,7 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, except ValueError: # check if it was an empty string and return MISSING_DATA_VALUE if super().getstr(sec, name) == '': - return util.MISSING_DATA_VALUE + return MISSING_DATA_VALUE # if value is not correct type, log error and return None self.logger.error(f"[{sec}] {name} must be an integer.") @@ -830,7 +834,7 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, # if config variable is not set except NoOptionError: if default is None: - default = float(util.MISSING_DATA_VALUE) + default = float(MISSING_DATA_VALUE) self.check_default(sec, name, default) return default @@ -839,7 +843,7 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, except ValueError: # check if it was an empty string and return MISSING_DATA_VALUE if super().getstr(sec, name) == '': - return util.MISSING_DATA_VALUE + return MISSING_DATA_VALUE # if value is not correct type, log error and return None self.logger.error(f"[{sec}] {name} must be a float.") @@ -1585,9 +1589,9 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None, # get indices of VAR items for data type and/or met tool indices = [] if met_tool: - indices = find_var_name_indices(config, data_types, met_tool).keys() + indices = _find_var_name_indices(config, data_types, met_tool).keys() if not indices: - indices = find_var_name_indices(config, data_types).keys() + indices = _find_var_name_indices(config, data_types).keys() # get config name prefixes for each data type to find dt_search_prefixes = {} @@ -1606,7 +1610,7 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None, index, search_prefixes) - field_info = format_var_items(field_configs, time_info) + field_info = _format_var_items(field_configs, time_info) if not isinstance(field_info, dict): config.logger.error(f'Could not process {current_type}_' f'VAR{index} variables: {field_info}') @@ -1707,7 +1711,7 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None, ''' return sorted(var_list, key=lambda x: x['index']) -def find_var_name_indices(config, data_types, met_tool=None): +def _find_var_name_indices(config, data_types, met_tool=None): data_type_regex = f"{'|'.join(data_types)}" # if data_types includes FCST or OBS, also search for BOTH @@ -1728,6 +1732,94 @@ def find_var_name_indices(config, data_types, met_tool=None): index_index=2, id_index=1) + +def _format_var_items(field_configs, time_info=None): + """! Substitute time information into field information and format values. + + @param field_configs dictionary with config variable names to read + @param time_info dictionary containing time info for current run + @returns dictionary containing name, levels, and output_names, as + well as thresholds and extra options if found. If not enough + information was set in the METplusConfig object, an empty + dictionary is returned. + """ + # dictionary to hold field (var) item info + var_items = {} + + # set defaults for optional items + var_items['levels'] = [] + var_items['thresh'] = [] + var_items['extra'] = '' + var_items['output_names'] = [] + + # get name, return error string if not found + search_name = field_configs.get('name') + if not search_name: + return 'Name not found' + + # perform string substitution on name + if time_info: + search_name = do_string_sub(search_name, + skip_missing_tags=True, + **time_info) + var_items['name'] = search_name + + # get levels, performing string substitution on each item of list + for level in getlist(field_configs.get('levels')): + if time_info: + level = do_string_sub(level, + **time_info) + var_items['levels'].append(level) + + # if no levels are found, add an empty string + if not var_items['levels']: + var_items['levels'].append('') + + # get threshold list if it is set + # return error string if any thresholds not formatted properly + search_thresh = field_configs.get('thresh') + if search_thresh: + thresh = getlist(search_thresh) + if not validate_thresholds(thresh): + return 'Invalid threshold supplied' + + var_items['thresh'] = thresh + + # get extra options if it is set, format with semi-colons between items + search_extra = field_configs.get('options') + if search_extra: + if time_info: + search_extra = do_string_sub(search_extra, + **time_info) + + # strip off empty space around each value + extra_list = [item.strip() for item in search_extra.split(';')] + + # split up each item by semicolon, then add a semicolon to the end + # use list(filter(None to remove empty strings from list + extra_list = list(filter(None, extra_list)) + var_items['extra'] = f"{'; '.join(extra_list)};" + + # get output names if they are set + out_name_str = field_configs.get('output_names') + + # use input name for each level if not set + if not out_name_str: + for _ in var_items['levels']: + var_items['output_names'].append(var_items['name']) + else: + for out_name in getlist(out_name_str): + if time_info: + out_name = do_string_sub(out_name, + **time_info) + var_items['output_names'].append(out_name) + + if len(var_items['levels']) != len(var_items['output_names']): + return 'Number of levels does not match number of output names' + + return var_items + + def skip_field_info_validation(config): """!Check config to see if having corresponding FCST/OBS variables is necessary. If process list only contains reformatter wrappers, don't validate field info. Also, if MTD is in the process list and @@ -1934,3 +2026,170 @@ def get_field_config_variables(config, index, search_prefixes): break return field_configs + + +def is_loop_by_init(config): + """!Check config variables to determine if looping by valid or init time""" + if config.has_option('config', 'LOOP_BY'): + loop_by = config.getstr('config', 'LOOP_BY').lower() + if loop_by in ['init', 'retro']: + return True + elif loop_by in ['valid', 'realtime']: + return False + + if config.has_option('config', 'LOOP_BY_INIT'): + return config.getbool('config', 'LOOP_BY_INIT') + + msg = 'MUST SET LOOP_BY to VALID, INIT, RETRO, or REALTIME' + if config.logger is None: + print(msg) + else: + config.logger.error(msg) + + return None + + +def handle_tmp_dir(config): + """! if env var MET_TMP_DIR is set, override config TMP_DIR with value + if it differs from what is set + get config temp dir using getdir_nocheck to bypass check for /path/to + this is done so the user can set env MET_TMP_DIR instead of config TMP_DIR + and config TMP_DIR will be set automatically""" + handle_env_var_config(config, 'MET_TMP_DIR', 'TMP_DIR') + + # create temp dir if it doesn't exist already + # this will fail if TMP_DIR is not set correctly and + # env MET_TMP_DIR was not set + mkdir_p(config.getdir('TMP_DIR')) + + +def handle_env_var_config(config, env_var_name, config_name): + """! If environment variable is set, use that value + for the config variable and warn if the previous config value differs + + @param config METplusConfig object to read + @param env_var_name name of environment variable to read + @param config_name name of METplus config variable to check + """ + env_var_value = os.environ.get(env_var_name, '') + config_value = config.getdir_nocheck(config_name, '') + + # do nothing if environment variable is not set + if not env_var_value: + return + + # override config config variable to environment variable value + config.set('config', config_name, env_var_value) + + # if config config value differed from environment variable value, warn + if config_value == env_var_value: + return + + config.logger.warning(f'Config variable {config_name} ({config_value}) ' + 'will be overridden by the environment variable ' + f'{env_var_name} ({env_var_value})') + + +def write_all_commands(all_commands, config): + """! Write all commands that were run to a file in the log + directory. This includes the environment variables that + were set before each command. + + @param all_commands list of tuples with command run and + list of environment variables that were set + @param config METplusConfig object used to write log output + and get the log timestamp to name the output file + @returns False if no commands were provided, True otherwise + """ + if not all_commands: + config.logger.error("No commands were run. " + "Skip writing all_commands file") + return False + + log_timestamp = config.getstr('config', 'LOG_TIMESTAMP') + filename = os.path.join(config.getdir('LOG_DIR'), + f'.all_commands.{log_timestamp}') + config.logger.debug(f"Writing all commands and environment to {filename}") + with open(filename, 'w') as file_handle: + for command, envs in all_commands: + for env in envs: + file_handle.write(f"{env}\n") + + file_handle.write("COMMAND:\n") + file_handle.write(f"{command}\n\n") + + return True + + +def write_final_conf(config): + """! Write final conf file including default values that were set during + run. Move variables that are specific to the user's run to the [runtime] + section to avoid issues such as overwriting existing log files. + + @param config METplusConfig object to write to file + """ + final_conf = config.getstr('config', 'METPLUS_CONF') + + # remove variables that start with CURRENT + config.remove_current_vars() + + # move runtime variables to [runtime] section + config.move_runtime_configs() + + config.logger.info('Overwrite final conf here: %s' % (final_conf,)) + with open(final_conf, 'wt') as conf_file: + config.write(conf_file) + + +def log_runtime_banner(config, time_input, process): + loop_by = time_input['loop_by'] + run_time = time_input[loop_by].strftime("%Y-%m-%d %H:%M") + + process_name = process.__class__.__name__ + if process.instance: + process_name = f"{process_name}({process.instance})" + + config.logger.info("****************************************") + config.logger.info(f"* Running METplus {process_name}") + config.logger.info(f"* at {loop_by} time: {run_time}") + config.logger.info("****************************************") + + +def sub_var_list(var_list, time_info): + """! Perform string substitution on var list values with time info + + @param var_list list of field info to substitute values into + @param time_info dictionary containing time information + @returns var_list with values substituted + """ + if not var_list: + return [] + + out_var_list = [] + for var_info in var_list: + out_var_info = _sub_var_info(var_info, time_info) + out_var_list.append(out_var_info) + + return out_var_list + + +def _sub_var_info(var_info, time_info): + if not var_info: + return {} + + out_var_info = {} + for key, value in var_info.items(): + if isinstance(value, list): + out_value = [] + for item in value: + out_value.append(do_string_sub(item, + skip_missing_tags=True, + **time_info)) + else: + out_value = do_string_sub(value, + skip_missing_tags=True, + **time_info) + + out_var_info[key] = out_value + + return out_var_info diff --git a/metplus/util/constants.py b/metplus/util/constants.py index e56f9def5..5e6f3dcb9 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -108,3 +108,8 @@ # datetime year month day hour minute second (YYYYMMDD_HHMMSS) notation YMD_HMS = '%Y%m%d_%H%M%S' + +# missing data value used to check if integer values are not set +# we often check for None if a variable is not set, but 0 and None +# have the same result in a test. 0 may be a valid integer value +MISSING_DATA_VALUE = -9999 diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 643e0db13..ddb4ef71c 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -6,9 +6,8 @@ import os import re -from .constants import PYTHON_EMBEDDING_TYPES, CLIMO_TYPES -from .string_manip import getlist -from .met_util import get_threshold_via_regex, MISSING_DATA_VALUE +from .constants import PYTHON_EMBEDDING_TYPES, CLIMO_TYPES, MISSING_DATA_VALUE +from .string_manip import getlist, get_threshold_via_regex from .string_manip import remove_quotes as util_remove_quotes from .config_metplus import find_indices_in_config_section, parse_var_list from .field_util import format_all_field_info diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py deleted file mode 100644 index d9fb9b6c5..000000000 --- a/metplus/util/met_util.py +++ /dev/null @@ -1,1476 +0,0 @@ -import os -import shutil -import sys -from datetime import datetime, timedelta, timezone -import re -import gzip -import bz2 -import zipfile -import struct -import getpass -from dateutil.relativedelta import relativedelta -from pathlib import Path -from importlib import import_module - -from .string_manip import getlist, getlistint -from .string_template_substitution import do_string_sub -from .string_template_substitution import parse_template -from . import time_util as time_util -from .time_looping import time_generator -from .. import get_metplus_version - -"""!@namespace met_util - @brief Provides Utility functions for METplus. -""" - -from .constants import * - -# missing data value used to check if integer values are not set -# we often check for None if a variable is not set, but 0 and None -# have the same result in a test. 0 may be a valid integer value -MISSING_DATA_VALUE = -9999 - -def pre_run_setup(config_inputs): - from . import config_metplus - version_number = get_metplus_version() - print(f'Starting METplus v{version_number}') - - # Read config inputs and return a config instance - config = config_metplus.setup(config_inputs) - - logger = config.logger - - user_info = get_user_info() - user_string = f' as user {user_info} ' if user_info else ' ' - - config.set('config', 'METPLUS_VERSION', version_number) - logger.info('Running METplus v%s%swith command: %s', - version_number, user_string, ' '.join(sys.argv)) - - logger.info(f"Log file: {config.getstr('config', 'LOG_METPLUS')}") - logger.info(f"METplus Base: {config.getdir('METPLUS_BASE')}") - logger.info(f"Final Conf: {config.getstr('config', 'METPLUS_CONF')}") - config_list = config.getstr('config', 'CONFIG_INPUT').split(',') - for config_item in config_list: - logger.info(f"Config Input: {config_item}") - - # validate configuration variables - isOK_A, isOK_B, isOK_C, isOK_D, all_sed_cmds = config_metplus.validate_configuration_variables(config) - if not (isOK_A and isOK_B and isOK_C and isOK_D): - # if any sed commands were generated, write them to the sed file - if all_sed_cmds: - sed_file = os.path.join(config.getdir('OUTPUT_BASE'), 'sed_commands.txt') - # remove if sed file exists - if os.path.exists(sed_file): - os.remove(sed_file) - - write_list_to_file(sed_file, all_sed_cmds) - config.logger.error(f"Find/Replace commands have been generated in {sed_file}") - - logger.error("Correct configuration variables and rerun. Exiting.") - sys.exit(1) - - if not config.getdir('MET_INSTALL_DIR', must_exist=True): - logger.error('MET_INSTALL_DIR must be set correctly to run METplus') - sys.exit(1) - - # set staging dir to OUTPUT_BASE/stage if not set - if not config.has_option('config', 'STAGING_DIR'): - config.set('config', 'STAGING_DIR', - os.path.join(config.getdir('OUTPUT_BASE'), "stage")) - - # handle dir to write temporary files - handle_tmp_dir(config) - - # handle OMP_NUM_THREADS environment variable - handle_env_var_config(config, - env_var_name='OMP_NUM_THREADS', - config_name='OMP_NUM_THREADS') - - config.env = os.environ.copy() - - return config - -def run_metplus(config, process_list): - total_errors = 0 - - try: - processes = [] - for process, instance in process_list: - try: - logname = f"{process}.{instance}" if instance else process - logger = config.log(logname) - package_name = ('metplus.wrappers.' - f'{camel_to_underscore(process)}_wrapper') - module = import_module(package_name) - command_builder = ( - getattr(module, f"{process}Wrapper")(config, - instance=instance) - ) - - # if Usage specified in PROCESS_LIST, print usage and exit - if process == 'Usage': - command_builder.run_all_times() - return 0 - except AttributeError: - logger.error("There was a problem loading " - f"{process} wrapper.") - return 1 - except ModuleNotFoundError: - logger.error(f"Could not load {process} wrapper. " - "Wrapper may have been disabled.") - return 1 - - processes.append(command_builder) - - # check if all processes initialized correctly - allOK = True - for process in processes: - if not process.isOK: - allOK = False - class_name = process.__class__.__name__.replace('Wrapper', '') - logger.error("{} was not initialized properly".format(class_name)) - - # exit if any wrappers did not initialized properly - if not allOK: - logger.info("Refer to ERROR messages above to resolve issues.") - return 1 - - all_commands = [] - for process in processes: - new_commands = process.run_all_times() - if new_commands: - all_commands.extend(new_commands) - - # if process list contains any wrapper that should run commands - if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]): - # write out all commands and environment variables to file - if not write_all_commands(all_commands, config): - # report an error if no commands were generated - total_errors += 1 - - # compute total number of errors that occurred and output results - for process in processes: - if process.errors != 0: - process_name = process.__class__.__name__.replace('Wrapper', '') - error_msg = '{} had {} error'.format(process_name, process.errors) - if process.errors > 1: - error_msg += 's' - error_msg += '.' - logger.error(error_msg) - total_errors += process.errors - - return total_errors - except: - logger.exception("Fatal error occurred") - logger.info(f"Check the log file for more information: {config.getstr('config', 'LOG_METPLUS')}") - return 1 - -def post_run_cleanup(config, app_name, total_errors): - logger = config.logger - # scrub staging directory if requested - if (config.getbool('config', 'SCRUB_STAGING_DIR') and - os.path.exists(config.getdir('STAGING_DIR'))): - staging_dir = config.getdir('STAGING_DIR') - logger.info("Scrubbing staging dir: %s", staging_dir) - logger.info('Set SCRUB_STAGING_DIR to False to preserve ' - 'intermediate files.') - shutil.rmtree(staging_dir) - - # save log file path and clock time before writing final conf file - log_message = (f"Check the log file for more information: " - f"{config.getstr('config', 'LOG_METPLUS')}") - - start_clock_time = datetime.strptime(config.getstr('config', 'CLOCK_TIME'), - '%Y%m%d%H%M%S') - - # rewrite final conf so it contains all of the default values used - write_final_conf(config) - - # compute time it took to run - end_clock_time = datetime.now() - total_run_time = end_clock_time - start_clock_time - logger.debug(f"{app_name} took {total_run_time} to run.") - - user_info = get_user_info() - user_string = f' as user {user_info}' if user_info else '' - if not total_errors: - logger.info(log_message) - logger.info('%s has successfully finished running%s.', - app_name, user_string) - return - - error_msg = (f'{app_name} has finished running{user_string} ' - f'but had {total_errors} error') - if total_errors > 1: - error_msg += 's' - error_msg += '.' - logger.error(error_msg) - logger.info(log_message) - sys.exit(1) - -def get_user_info(): - """! Get user information from OS. Note that some OS cannot obtain user ID - and some cannot obtain username. - @returns username(uid) if both username and user ID can be read, - username if only username can be read, uid if only user ID can be read, - or an empty string if neither can be read. - """ - try: - username = getpass.getuser() - except OSError: - username = None - - try: - uid = os.getuid() - except AttributeError: - uid = None - - if username and uid: - return f'{username}({uid})' - - if username: - return username - - if uid: - return uid - - return '' - -def write_all_commands(all_commands, config): - """! Write all commands that were run to a file in the log - directory. This includes the environment variables that - were set before each command. - - @param all_commands list of tuples with command run and - list of environment variables that were set - @param config METplusConfig object used to write log output - and get the log timestamp to name the output file - @returns False if no commands were provided, True otherwise - """ - if not all_commands: - config.logger.error("No commands were run. " - "Skip writing all_commands file") - return False - - log_timestamp = config.getstr('config', 'LOG_TIMESTAMP') - filename = os.path.join(config.getdir('LOG_DIR'), - f'.all_commands.{log_timestamp}') - config.logger.debug(f"Writing all commands and environment to {filename}") - with open(filename, 'w') as file_handle: - for command, envs in all_commands: - for env in envs: - file_handle.write(f"{env}\n") - - file_handle.write("COMMAND:\n") - file_handle.write(f"{command}\n\n") - - return True - -def handle_tmp_dir(config): - """! if env var MET_TMP_DIR is set, override config TMP_DIR with value - if it differs from what is set - get config temp dir using getdir_nocheck to bypass check for /path/to - this is done so the user can set env MET_TMP_DIR instead of config TMP_DIR - and config TMP_DIR will be set automatically""" - handle_env_var_config(config, 'MET_TMP_DIR', 'TMP_DIR') - - # create temp dir if it doesn't exist already - # this will fail if TMP_DIR is not set correctly and - # env MET_TMP_DIR was not set - mkdir_p(config.getdir('TMP_DIR')) - -def handle_env_var_config(config, env_var_name, config_name): - """! If environment variable is set, use that value - for the config variable and warn if the previous config value differs - - @param config METplusConfig object to read - @param env_var_name name of environment variable to read - @param config_name name of METplus config variable to check - """ - env_var_value = os.environ.get(env_var_name, '') - config_value = config.getdir_nocheck(config_name, '') - - # do nothing if environment variable is not set - if not env_var_value: - return - - # override config config variable to environment variable value - config.set('config', config_name, env_var_value) - - # if config config value differed from environment variable value, warn - if config_value != env_var_value: - config.logger.warning(f'Config variable {config_name} ({config_value}) ' - 'will be overridden by the environment variable ' - f'{env_var_name} ({env_var_value})') - -def get_skip_times(config, wrapper_name=None): - """! Read SKIP_TIMES config variable and populate dictionary of times that should be skipped. - SKIP_TIMES should be in the format: "%m:begin_end_incr(3,11,1)", "%d:30,31", "%Y%m%d:20201031" - where each item inside quotes is a datetime format, colon, then a list of times in that format - to skip. - Args: - @param config configuration object to pull SKIP_TIMES - @param wrapper_name name of wrapper if supporting - skipping times only for certain wrappers, i.e. grid_stat - @returns dictionary containing times to skip - """ - skip_times_dict = {} - skip_times_string = None - - # if wrapper name is set, look for wrapper-specific _SKIP_TIMES variable - if wrapper_name: - skip_times_string = config.getstr('config', - f'{wrapper_name.upper()}_SKIP_TIMES', '') - - # if skip times string has not been found, check for generic SKIP_TIMES - if not skip_times_string: - skip_times_string = config.getstr('config', 'SKIP_TIMES', '') - - # if no generic SKIP_TIMES, return empty dictionary - if not skip_times_string: - return {} - - # get list of skip items, but don't expand begin_end_incr yet - skip_list = getlist(skip_times_string, expand_begin_end_incr=False) - - for skip_item in skip_list: - try: - time_format, skip_times = skip_item.split(':') - - # get list of skip times for the time format, expanding begin_end_incr - skip_times_list = getlist(skip_times) - - # if time format is already in skip times dictionary, extend list - if time_format in skip_times_dict: - skip_times_dict[time_format].extend(skip_times_list) - else: - skip_times_dict[time_format] = skip_times_list - - except ValueError: - config.logger.error(f"SKIP_TIMES item does not match format: {skip_item}") - return None - - return skip_times_dict - -def skip_time(time_info, skip_times): - """!Used to check the valid time of the current run time against list of times to skip. - Args: - @param time_info dictionary with time information to check - @param skip_times dictionary of times to skip, i.e. {'%d': [31]} means skip 31st day - @returns True if run time should be skipped, False if not - """ - if not skip_times: - return False - - for time_format, skip_time_list in skip_times.items(): - # extract time information from valid time based on skip time format - run_time_value = time_info.get('valid') - if not run_time_value: - return False - - run_time_value = run_time_value.strftime(time_format) - - # loop over times to skip for this format and check if it matches - for skip_time in skip_time_list: - if int(run_time_value) == int(skip_time): - return True - - # if skip time never matches, return False - return False - -def write_final_conf(config): - """! Write final conf file including default values that were set during - run. Move variables that are specific to the user's run to the [runtime] - section to avoid issues such as overwriting existing log files. - - @param config METplusConfig object to write to file - """ - final_conf = config.getstr('config', 'METPLUS_CONF') - - # remove variables that start with CURRENT - config.remove_current_vars() - - # move runtime variables to [runtime] section - config.move_runtime_configs() - - config.logger.info('Overwrite final conf here: %s' % (final_conf,)) - with open(final_conf, 'wt') as conf_file: - config.write(conf_file) - -def is_loop_by_init(config): - """!Check config variables to determine if looping by valid or init time""" - if config.has_option('config', 'LOOP_BY'): - loop_by = config.getstr('config', 'LOOP_BY').lower() - if loop_by in ['init', 'retro']: - return True - elif loop_by in ['valid', 'realtime']: - return False - - if config.has_option('config', 'LOOP_BY_INIT'): - return config.getbool('config', 'LOOP_BY_INIT') - - msg = 'MUST SET LOOP_BY to VALID, INIT, RETRO, or REALTIME' - if config.logger is None: - print(msg) - else: - config.logger.error(msg) - - return None - -def loop_over_times_and_call(config, processes, custom=None): - """! Loop over all run times and call wrappers listed in config - - @param config METplusConfig object - @param processes list of CommandBuilder subclass objects (Wrappers) to call - @param custom (optional) custom loop string value - @returns list of tuples with all commands run and the environment variables - that were set for each - """ - # keep track of commands that were run - all_commands = [] - for time_input in time_generator(config): - if not isinstance(processes, list): - processes = [processes] - - for process in processes: - # if time could not be read, increment errors for each process - if time_input is None: - process.errors += 1 - continue - - log_runtime_banner(config, time_input, process) - add_to_time_input(time_input, - instance=process.instance, - custom=custom) - - process.clear() - process.run_at_time(time_input) - if process.all_commands: - all_commands.extend(process.all_commands) - process.all_commands.clear() - - return all_commands - -def log_runtime_banner(config, time_input, process): - loop_by = time_input['loop_by'] - run_time = time_input[loop_by].strftime("%Y-%m-%d %H:%M") - - process_name = process.__class__.__name__ - if process.instance: - process_name = f"{process_name}({process.instance})" - - config.logger.info("****************************************") - config.logger.info(f"* Running METplus {process_name}") - config.logger.info(f"* at {loop_by} time: {run_time}") - config.logger.info("****************************************") - -def add_to_time_input(time_input, clock_time=None, instance=None, custom=None): - if clock_time: - clock_dt = datetime.strptime(clock_time, '%Y%m%d%H%M%S') - time_input['now'] = clock_dt - - # if instance is set, use that value, otherwise use empty string - time_input['instance'] = instance if instance else '' - - # if custom is specified, set it - # otherwise leave it unset so it can be set within the wrapper - if custom: - time_input['custom'] = custom - -def get_lead_sequence(config, input_dict=None, wildcard_if_empty=False): - """!Get forecast lead list from LEAD_SEQ or compute it from INIT_SEQ. - Restrict list by LEAD_SEQ_[MIN/MAX] if set. Now returns list of relativedelta objects - Args: - @param config METplusConfig object to query config variable values - @param input_dict time dictionary needed to handle using INIT_SEQ. Must contain - valid key if processing INIT_SEQ - @param wildcard_if_empty if no lead sequence was set, return a - list with '*' if this is True, otherwise return a list with 0 - @returns list of relativedelta objects or a list containing 0 if none are found - """ - - out_leads = [] - lead_min, lead_max, no_max = get_lead_min_max(config) - - # check if LEAD_SEQ, INIT_SEQ, or LEAD_SEQ_ are set - # if more than one is set, report an error and exit - lead_seq = getlist(config.getstr('config', 'LEAD_SEQ', '')) - init_seq = getlistint(config.getstr('config', 'INIT_SEQ', '')) - lead_groups = get_lead_sequence_groups(config) - - if not are_lead_configs_ok(lead_seq, - init_seq, - lead_groups, - config, - input_dict, - no_max): - return None - - if lead_seq: - # return lead sequence if wildcard characters are used - if lead_seq == ['*']: - return lead_seq - - out_leads = handle_lead_seq(config, - lead_seq, - lead_min, - lead_max) - - # use INIT_SEQ to build lead list based on the valid time - elif init_seq: - out_leads = handle_init_seq(init_seq, - input_dict, - lead_min, - lead_max) - elif lead_groups: - out_leads = handle_lead_groups(lead_groups) - - if not out_leads: - if wildcard_if_empty: - return ['*'] - - return [0] - - return out_leads - -def are_lead_configs_ok(lead_seq, init_seq, lead_groups, - config, input_dict, no_max): - if lead_groups is None: - return False - - error_message = ('are both listed in the configuration. ' - 'Only one may be used at a time.') - if lead_seq: - if init_seq: - config.logger.error(f'LEAD_SEQ and INIT_SEQ {error_message}') - return False - - if lead_groups: - config.logger.error(f'LEAD_SEQ and LEAD_SEQ_ {error_message}') - return False - - if init_seq and lead_groups: - config.logger.error(f'INIT_SEQ and LEAD_SEQ_ {error_message}') - return False - - if init_seq: - # if input dictionary not passed in, - # cannot compute lead sequence from it, so exit - if input_dict is None: - config.logger.error('Cannot run using INIT_SEQ for this wrapper') - return False - - # if looping by init, fail and exit - if 'valid' not in input_dict.keys(): - log_msg = ('INIT_SEQ specified while looping by init time.' - ' Use LEAD_SEQ or change to loop by valid time') - config.logger.error(log_msg) - return False - - # maximum lead must be specified to run with INIT_SEQ - if no_max: - config.logger.error('LEAD_SEQ_MAX must be set to use INIT_SEQ') - return False - - return True - -def get_lead_min_max(config): - # remove any items that are outside of the range specified - # by LEAD_SEQ_MIN and LEAD_SEQ_MAX - # convert min and max to relativedelta objects, then use current time - # to compare them to each forecast lead - # this is an approximation because relative time offsets depend on - # each runtime - huge_max = '4000Y' - lead_min_str = config.getstr_nocheck('config', 'LEAD_SEQ_MIN', '0') - lead_max_str = config.getstr_nocheck('config', 'LEAD_SEQ_MAX', huge_max) - no_max = lead_max_str == huge_max - lead_min = time_util.get_relativedelta(lead_min_str, 'H') - lead_max = time_util.get_relativedelta(lead_max_str, 'H') - return lead_min, lead_max, no_max - -def handle_lead_seq(config, lead_strings, lead_min=None, lead_max=None): - out_leads = [] - leads = [] - for lead in lead_strings: - relative_delta = time_util.get_relativedelta(lead, 'H') - if relative_delta is not None: - leads.append(relative_delta) - else: - config.logger.error(f'Invalid item {lead} in LEAD_SEQ. Exiting.') - return None - - if lead_min is None and lead_max is None: - return leads - - # add current time to leads to approximate month and year length - now_time = datetime.now() - lead_min_approx = now_time + lead_min - lead_max_approx = now_time + lead_max - for lead in leads: - lead_approx = now_time + lead - if lead_approx >= lead_min_approx and lead_approx <= lead_max_approx: - out_leads.append(lead) - - return out_leads - -def handle_init_seq(init_seq, input_dict, lead_min, lead_max): - out_leads = [] - lead_min_hours = time_util.ti_get_hours_from_relativedelta(lead_min) - lead_max_hours = time_util.ti_get_hours_from_relativedelta(lead_max) - - valid_hr = int(input_dict['valid'].strftime('%H')) - for init in init_seq: - if valid_hr >= init: - current_lead = valid_hr - init - else: - current_lead = valid_hr + (24 - init) - - while current_lead <= lead_max_hours: - if current_lead >= lead_min_hours: - out_leads.append(relativedelta(hours=current_lead)) - current_lead += 24 - - out_leads = sorted(out_leads, key=lambda - rd: time_util.ti_get_seconds_from_relativedelta(rd, - input_dict['valid'])) - return out_leads - -def handle_lead_groups(lead_groups): - """! Read groups of forecast leads and create a list with all unique items - - @param lead_group dictionary where the values are lists of forecast - leads stored as relativedelta objects - @returns list of forecast leads stored as relativedelta objects - """ - out_leads = [] - for _, lead_seq in lead_groups.items(): - for lead in lead_seq: - if lead not in out_leads: - out_leads.append(lead) - - return out_leads - -def get_lead_sequence_groups(config): - # output will be a dictionary where the key will be the - # label specified and the value will be the list of forecast leads - lead_seq_dict = {} - # used in plotting - all_conf = config.keys('config') - indices = [] - regex = re.compile(r"LEAD_SEQ_(\d+)") - for conf in all_conf: - result = regex.match(conf) - if result is not None: - indices.append(result.group(1)) - - # loop over all possible variables and add them to list - for index in indices: - if config.has_option('config', f"LEAD_SEQ_{index}_LABEL"): - label = config.getstr('config', f"LEAD_SEQ_{index}_LABEL") - else: - log_msg = (f'Need to set LEAD_SEQ_{index}_LABEL to describe ' - f'LEAD_SEQ_{index}') - config.logger.error(log_msg) - return None - - # get forecast list for n - lead_string_list = getlist(config.getstr('config', f'LEAD_SEQ_{index}')) - lead_seq = handle_lead_seq(config, - lead_string_list, - lead_min=None, - lead_max=None) - # add to output dictionary - lead_seq_dict[label] = lead_seq - - return lead_seq_dict - -def round_0p5(val): - """! Round to the nearest point five (ie 3.3 rounds to 3.5, 3.1 - rounds to 3.0) Take the input value, multiply by two, round to integer - (no decimal places) then divide by two. Expect any input value of n.0, - n.1, or n.2 to round down to n.0, and any input value of n.5, n.6 or - n.7 to round to n.5. Finally, any input value of n.8 or n.9 will - round to (n+1).0 - Args: - @param val : The number to be rounded to the nearest .5 - Returns: - pt_five: The n.0, n.5, or (n+1).0 value as - a result of rounding the input value, val. - """ - - return round(val * 2) / 2 - -def mkdir_p(path): - """! - From stackoverflow.com/questions/600268/mkdir-p-functionality-in-python - Creates the entire directory path if it doesn't exist (including any - required intermediate directories). - Args: - @param path : The full directory path to be created - Returns - None: Creates the full directory path if it doesn't exist, - does nothing otherwise. - """ - Path(path).mkdir(parents=True, exist_ok=True) - -def get_storms(filter_filename, id_only=False, sort_column='STORM_ID'): - """! Get each storm as identified by a column in the input file. - Create dictionary storm ID as the key and a list of lines for that - storm as the value. - - @param filter_filename name of tcst file to read and extract storm id - @param sort_column column to use to sort and group storms. Default - value is STORM_ID - @returns 2 item tuple - 1)dictionary where key is storm ID and value - is list of relevant lines from tcst file, 2) header line from tcst - file. Item with key 'header' contains the header of the tcst file - """ - # Initialize a set because we want unique storm ids. - storm_id_list = set() - - try: - with open(filter_filename, "r") as file_handle: - header, *lines = file_handle.readlines() - - storm_id_column = header.split().index(sort_column) - for line in lines: - storm_id_list.add(line.split()[storm_id_column]) - except (ValueError, FileNotFoundError): - if id_only: - return [] - return {} - - # sort the unique storm ids, copy the original - # set by using sorted rather than sort. - sorted_storms = sorted(storm_id_list) - if id_only: - return sorted_storms - - if not sorted_storms: - return {} - - storm_dict = {'header': header} - # for each storm, get all lines for that storm - for storm in sorted_storms: - storm_dict[storm] = [line for line in lines if storm in line] - - return storm_dict - -def get_files(filedir, filename_regex, logger=None): - """! Get all the files (with a particular - naming format) by walking - through the directories. - Args: - @param filedir: The topmost directory from which the - search begins. - @param filename_regex: The regular expression that - defines the naming format - of the files of interest. - Returns: - file_paths (string): a list of filenames (with full filepath) - """ - file_paths = [] - - # Walk the tree - for root, _, files in os.walk(filedir): - for filename in files: - # add it to the list only if it is a match - # to the specified format - match = re.match(filename_regex, filename) - if match: - # Join the two strings to form the full - # filepath. - filepath = os.path.join(root, filename) - file_paths.append(filepath) - else: - continue - return sorted(file_paths) - -def prune_empty(output_dir, logger): - """! Start from the output_dir, and recursively check - all directories and files. If there are any empty - files or directories, delete/remove them so they - don't cause performance degradation or errors - when performing subsequent tasks. - Input: - @param output_dir: The directory from which searching - should begin. - @param logger: The logger to which all logging is - directed. - """ - - # Check for empty files. - for root, dirs, files in os.walk(output_dir): - # Create a full file path by joining the path - # and filename. - for a_file in files: - a_file = os.path.join(root, a_file) - if os.stat(a_file).st_size == 0: - logger.debug("Empty file: " + a_file + - "...removing") - os.remove(a_file) - - # Now check for any empty directories, some - # may have been created when removing - # empty files. - for root, dirs, files in os.walk(output_dir): - for direc in dirs: - full_dir = os.path.join(root, direc) - if not os.listdir(full_dir): - logger.debug("Empty directory: " + full_dir + - "...removing") - os.rmdir(full_dir) - -def camel_to_underscore(camel): - """! Change camel case notation to underscore notation, i.e. GridStatWrapper to grid_stat_wrapper - Multiple capital letters are excluded, i.e. PCPCombineWrapper to pcp_combine_wrapper - Numerals are also skipped, i.e. ASCII2NCWrapper to ascii2nc_wrapper - Args: - @param camel string to convert - @returns string in underscore notation - """ - s1 = re.sub(r'([^\d])([A-Z][a-z]+)', r'\1_\2', camel) - return re.sub(r'([a-z])([A-Z])', r'\1_\2', s1).lower() - -def shift_time_seconds(time_str, shift): - """ Adjust time by shift seconds. Format is %Y%m%d%H%M%S - Args: - @param time_str: Start time in %Y%m%d%H%M%S - @param shift: Amount to adjust time in seconds - Returns: - New time in format %Y%m%d%H%M%S - """ - return (datetime.strptime(time_str, "%Y%m%d%H%M%S") + - timedelta(seconds=shift)).strftime("%Y%m%d%H%M%S") - -def get_threshold_via_regex(thresh_string): - """!Ensure thresh values start with >,>=,==,!=,<,<=,gt,ge,eq,ne,lt,le and then a number - Optionally can have multiple comparison/number pairs separated with && or ||. - Args: - @param thresh_string: String to examine, i.e. <=3.4 - Returns: - None if string does not match any valid comparison operators or does - not contain a number afterwards - regex match object with comparison operator in group 1 and - number in group 2 if valid - """ - - comparison_number_list = [] - # split thresh string by || or && - thresh_split = re.split(r'\|\||&&', thresh_string) - # check each threshold for validity - for thresh in thresh_split: - found_match = False - for comp in list(VALID_COMPARISONS)+list(VALID_COMPARISONS.values()): - # if valid, add to list of tuples - # must be one of the valid comparison operators followed by - # at least 1 digit or NA - if thresh == 'NA': - comparison_number_list.append((thresh, '')) - found_match = True - break - - match = re.match(r'^('+comp+r')(.*\d.*)$', thresh) - if match: - comparison = match.group(1) - number = match.group(2) - # try to convert to float if it can, but allow string - try: - number = float(number) - except ValueError: - pass - - comparison_number_list.append((comparison, number)) - found_match = True - break - - # if no match was found for the item, return None - if not found_match: - return None - - if not comparison_number_list: - return None - - return comparison_number_list - - -def validate_thresholds(thresh_list): - """ Checks list of thresholds to ensure all of them have the correct format - Should be a comparison operator with number pair combined with || or && - i.e. gt4 or >3&&<5 or gt3||lt1 - Args: - @param thresh_list list of strings to check - Returns: - True if all items in the list are valid format, False if not - """ - valid = True - for thresh in thresh_list: - match = get_threshold_via_regex(thresh) - if match is None: - valid = False - - if valid is False: - print("ERROR: Threshold values must use >,>=,==,!=,<,<=,gt,ge,eq,ne,lt, or le with a number, " - "optionally combined with && or ||") - return False - return True - -def write_list_to_file(filename, output_list): - with open(filename, 'w+') as f: - for line in output_list: - f.write(f"{line}\n") - -def format_var_items(field_configs, time_info=None): - """! Substitute time information into field information and format values. - - @param field_configs dictionary with config variable names to read - @param time_info dictionary containing time info for current run - @returns dictionary containing name, levels, and output_names, as - well as thresholds and extra options if found. If not enough - information was set in the METplusConfig object, an empty - dictionary is returned. - """ - # dictionary to hold field (var) item info - var_items = {} - - # set defaults for optional items - var_items['levels'] = [] - var_items['thresh'] = [] - var_items['extra'] = '' - var_items['output_names'] = [] - - # get name, return error string if not found - search_name = field_configs.get('name') - if not search_name: - return 'Name not found' - - # perform string substitution on name - if time_info: - search_name = do_string_sub(search_name, - skip_missing_tags=True, - **time_info) - var_items['name'] = search_name - - # get levels, performing string substitution on each item of list - for level in getlist(field_configs.get('levels')): - if time_info: - level = do_string_sub(level, - **time_info) - var_items['levels'].append(level) - - # if no levels are found, add an empty string - if not var_items['levels']: - var_items['levels'].append('') - - # get threshold list if it is set - # return error string if any thresholds not formatted properly - search_thresh = field_configs.get('thresh') - if search_thresh: - thresh = getlist(search_thresh) - if not validate_thresholds(thresh): - return 'Invalid threshold supplied' - - var_items['thresh'] = thresh - - # get extra options if it is set, format with semi-colons between items - search_extra = field_configs.get('options') - if search_extra: - if time_info: - search_extra = do_string_sub(search_extra, - **time_info) - - # strip off empty space around each value - extra_list = [item.strip() for item in search_extra.split(';')] - - # split up each item by semicolon, then add a semicolon to the end - # use list(filter(None to remove empty strings from list - extra_list = list(filter(None, extra_list)) - var_items['extra'] = f"{'; '.join(extra_list)};" - - # get output names if they are set - out_name_str = field_configs.get('output_names') - - # use input name for each level if not set - if not out_name_str: - for _ in var_items['levels']: - var_items['output_names'].append(var_items['name']) - else: - for out_name in getlist(out_name_str): - if time_info: - out_name = do_string_sub(out_name, - **time_info) - var_items['output_names'].append(out_name) - - if len(var_items['levels']) != len(var_items['output_names']): - return 'Number of levels does not match number of output names' - - return var_items - -def sub_var_info(var_info, time_info): - if not var_info: - return {} - - out_var_info = {} - for key, value in var_info.items(): - if isinstance(value, list): - out_value = [] - for item in value: - out_value.append(do_string_sub(item, - skip_missing_tags=True, - **time_info)) - else: - out_value = do_string_sub(value, - skip_missing_tags=True, - **time_info) - - out_var_info[key] = out_value - - return out_var_info - -def sub_var_list(var_list, time_info): - """! Perform string substitution on var list values with time info - - @param var_list list of field info to substitute values into - @param time_info dictionary containing time information - @returns var_list with values substituted - """ - if not var_list: - return [] - - out_var_list = [] - for var_info in var_list: - out_var_info = sub_var_info(var_info, time_info) - out_var_list.append(out_var_info) - - return out_var_list - -def split_level(level): - """! If level value starts with a letter, then separate that letter from - the rest of the string. i.e. 'A03' will be returned as 'A', '03'. If no - level type letter is found and the level value consists of alpha-numeric - characters, return an empty string as the level type and the full level - string as the level value - - @param level input string to parse/split - @returns tuple of level type and level value - """ - if not level: - return '', '' - - match = re.match(r'^([a-zA-Z])(\w+)$', level) - if match: - level_type = match.group(1) - level = match.group(2) - return level_type, level - - match = re.match(r'^[\w]+$', level) - if match: - return '', level - - return '', '' - -def get_filetype(filepath, logger=None): - """!This function determines if the filepath is a NETCDF or GRIB file - based on the first eight bytes of the file. - It returns the string GRIB, NETCDF, or a None object. - - Note: If it is NOT determined to ba a NETCDF file, - it returns GRIB, regardless. - Unless there is an IOError exception, such as filepath refers - to a non-existent file or filepath is only a directory, than - None is returned, without a system exit. - - Args: - @param filepath: path/to/filename - @param logger the logger, optional - Returns: - @returns The string GRIB, NETCDF or a None object - """ - # Developer Note - # Since we have the impending code-freeze, keeping the behavior the same, - # just changing the implementation. - # The previous logic did not test for GRIB it would just return 'GRIB' - # if you couldn't run ncdump on the file. - # Also note: - # As John indicated ... there is the case when a grib file - # may not start with GRIB ... and if you pass the MET command filtetype=GRIB - # MET will handle it ok ... - - # Notes on file format and determining type. - # https://www.wmo.int/pages/prog/www/WDM/Guides/Guide-binary-2.html - # https://www.unidata.ucar.edu/software/netcdf/docs/faq.html - # http: // www.hdfgroup.org / HDF5 / doc / H5.format.html - - # Interpreting single byte by byte - so ok to ignore endianess - # od command: - # od -An -c -N8 foo.nc - # od -tx1 -N8 foo.nc - # GRIB - # Octet no. IS Content - # 1-4 'GRIB' (Coded CCITT-ITA No. 5) (ASCII); - # 5-7 Total length, in octets, of GRIB message(including Sections 0 & 5); - # 8 Edition number - currently 1 - # NETCDF .. ie. od -An -c -N4 foo.nc which will output - # C D F 001 - # C D F 002 - # 211 H D F - # HDF5 - # Magic numbers Hex: 89 48 44 46 0d 0a 1a 0a - # ASCII: \211 HDF \r \n \032 \n - - # Below is a reference that may be used in the future to - # determine grib version. - # import struct - # with open ("foo.grb2","rb")as binary_file: - # binary_file.seek(7) - # one_byte = binary_file.read(1) - # - # This would return an integer with value 1 or 2, - # B option is an unsigned char. - # struct.unpack('B',one_byte)[0] - - # if filepath is set to None, return None to avoid crash - if filepath == None: - return None - - try: - # read will return up to 8 bytes, if file is 0 bytes in length, - # than first_eight_bytes will be the empty string ''. - # Don't test the file length, just adds more time overhead. - with open(filepath, "rb") as binary_file: - binary_file.seek(0) - first_eight_bytes = binary_file.read(8) - - # From the first eight bytes of the file, unpack the bytes - # of the known identifier byte locations, in to a string. - # Example, if this was a netcdf file than ONLY name_cdf would - # equal 'CDF' the other variables, name_hdf would be 'DF ' - # name_grid 'CDF ' - name_cdf, name_hdf, name_grib = [None] * 3 - if len(first_eight_bytes) == 8: - name_cdf = struct.unpack('3s', first_eight_bytes[:3])[0] - name_hdf = struct.unpack('3s', first_eight_bytes[1:4])[0] - name_grib = struct.unpack('4s', first_eight_bytes[:4])[0] - - # Why not just use a else, instead of elif else if we are going to - # return GRIB ? It allows for expansion, ie. Maybe we pass in a - # logger and log the cases we can't determine the type. - if name_cdf == 'CDF' or name_hdf == 'HDF': - return "NETCDF" - elif name_grib == 'GRIB': - return "GRIB" - else: - # This mimicks previous behavoir, were we at least will always return GRIB. - # It also handles the case where GRIB was not in the first 4 bytes - # of a legitimate grib file, see John. - # logger.info('Can't determine type, returning GRIB - # as default %s'%filepath) - return "GRIB" - - except IOError: - # Skip the IOError, and keep processing data. - # ie. filepath references a file that does not exist - # or filepath is a directory. - return None - - # Previous Logic - # ncdump_exe = config.getexe('NCDUMP') - #try: - # result = subprocess.check_output([ncdump_exe, filepath]) - - #except subprocess.CalledProcessError: - # return "GRIB" - - #regex = re.search("netcdf", result) - #if regex is not None: - # return "NETCDF" - #else: - # return None - -def preprocess_file(filename, data_type, config, allow_dir=False): - """ Decompress gzip, bzip, or zip files or convert Gempak files to NetCDF - Args: - @param filename: Path to file without zip extensions - @param config: Config object - Returns: - Path to staged unzipped file or original file if already unzipped - """ - if not filename: - return None - - if allow_dir and os.path.isdir(filename): - return filename - - # if using python embedding for input, return the keyword - if os.path.basename(filename) in PYTHON_EMBEDDING_TYPES: - return os.path.basename(filename) - - # if filename starts with a python embedding type, return the full value - for py_embed_type in PYTHON_EMBEDDING_TYPES: - if filename.startswith(py_embed_type): - return filename - - # if _INPUT_DATATYPE value contains PYTHON, return the full value - if data_type is not None and 'PYTHON' in data_type: - return filename - - stage_dir = config.getdir('STAGING_DIR') - - if os.path.isfile(filename): - # if filename provided ends with a valid compression extension, - # remove the extension and call function again so the - # file will be uncompressed properly. This is done so that - # the function will handle files passed to it with an - # extension the same way as files passed - # without an extension but the compressed equivalent exists - for ext in COMPRESSION_EXTENSIONS: - if filename.endswith(ext): - return preprocess_file(filename[:-len(ext)], data_type, config) - # if extension is grd (Gempak), then look in staging dir for nc file - if filename.endswith('.grd') or data_type == "GEMPAK": - if filename.endswith('.grd'): - stagefile = stage_dir + filename[:-3]+"nc" - else: - stagefile = stage_dir + filename+".nc" - if os.path.isfile(stagefile): - return stagefile - # if it does not exist, run GempakToCF and return staged nc file - # Create staging area if it does not exist - mkdir_p(os.path.dirname(stagefile)) - - # only import GempakToCF if needed - from ..wrappers import GempakToCFWrapper - - run_g2c = GempakToCFWrapper(config) - run_g2c.infiles.append(filename) - run_g2c.set_output_path(stagefile) - cmd = run_g2c.get_command() - if cmd is None: - config.logger.error("GempakToCF could not generate command") - return None - if config.logger: - config.logger.debug("Converting Gempak file into {}".format(stagefile)) - run_g2c.build() - return stagefile - - return filename - - # nc file requested and the Gempak equivalent exists - if os.path.isfile(filename[:-2]+'grd'): - return preprocess_file(filename[:-2]+'grd', data_type, config) - - # if file exists in the staging area, return that path - outpath = stage_dir + filename - if os.path.isfile(outpath): - return outpath - - # Create staging area directory only if file has compression extension - if any([os.path.isfile(f'{filename}{ext}') - for ext in COMPRESSION_EXTENSIONS]): - mkdir_p(os.path.dirname(outpath)) - - # uncompress gz, bz2, or zip file - if os.path.isfile(filename+".gz"): - if config.logger: - config.logger.debug("Uncompressing gz file to {}".format(outpath)) - with gzip.open(filename+".gz", 'rb') as infile: - with open(outpath, 'wb') as outfile: - outfile.write(infile.read()) - infile.close() - outfile.close() - return outpath - elif os.path.isfile(filename+".bz2"): - if config.logger: - config.logger.debug("Uncompressing bz2 file to {}".format(outpath)) - with open(filename+".bz2", 'rb') as infile: - with open(outpath, 'wb') as outfile: - outfile.write(bz2.decompress(infile.read())) - infile.close() - outfile.close() - return outpath - elif os.path.isfile(filename+".zip"): - if config.logger: - config.logger.debug("Uncompressing zip file to {}".format(outpath)) - with zipfile.ZipFile(filename+".zip") as z: - with open(outpath, 'wb') as f: - f.write(z.read(os.path.basename(filename))) - return outpath - - # if input doesn't need to exist, return filename - if not config.getbool('config', 'INPUT_MUST_EXIST', True): - return filename - - return None - -def template_to_regex(template, time_info): - in_template = re.sub(r'\.', '\\.', template) - in_template = re.sub(r'{lead.*?}', '.*', in_template) - return do_string_sub(in_template, - **time_info) - -def is_python_script(name): - """ Check if field name is a python script by checking if any of the words - in the string end with .py - - @param name string to check - @returns True if the name is determined to be a python script command - """ - if not name: - return False - - all_items = name.split(' ') - if any(item.endswith('.py') for item in all_items): - return True - - return False - -def expand_int_string_to_list(int_string): - """! Expand string into a list of integer values. Items are separated by - commas. Items that are formatted X-Y will be expanded into each number - from X to Y inclusive. If the string ends with +, then add a str '+' - to the end of the list. Used in .github/jobs/get_use_case_commands.py - - @param int_string String containing a comma-separated list of integers - @returns List of integers and potentially '+' as the last item - """ - subset_list = [] - - # if string ends with +, remove it and add it back at the end - if int_string.strip().endswith('+'): - int_string = int_string.strip(' +') - hasPlus = True - else: - hasPlus = False - - # separate into list by comma - comma_list = int_string.split(',') - for comma_item in comma_list: - dash_list = comma_item.split('-') - # if item contains X-Y, expand it - if len(dash_list) == 2: - for i in range(int(dash_list[0].strip()), - int(dash_list[1].strip())+1, - 1): - subset_list.append(i) - else: - subset_list.append(int(comma_item.strip())) - - if hasPlus: - subset_list.append('+') - - return subset_list - -def subset_list(full_list, subset_definition): - """! Extract subset of items from full_list based on subset_definition - Used in internal/tests/use_cases/metplus_use_case_suite.py - - @param full_list List of all use cases that were requested - @param subset_definition Defines how to subset the full list. If None, - no subsetting occurs. If an integer value, select that index only. - If a slice object, i.e. slice(2,4,1), pass slice object into list. - If list, subset full list by integer index values in list. If - last item in list is '+' then subset list up to 2nd last index, then - get all items from 2nd last item and above - """ - if subset_definition is not None: - subset_list = [] - - # if case slice is a list, use only the indices in the list - if isinstance(subset_definition, list): - # if last slice value is a plus sign, get rest of items - # after 2nd last slice value - if subset_definition[-1] == '+': - plus_value = subset_definition[-2] - # add all values before last index before plus - subset_list.extend([full_list[i] - for i in subset_definition[:-2]]) - # add last index listed + all items above - subset_list.extend(full_list[plus_value:]) - else: - # list of integers, so get items based on indices - subset_list = [full_list[i] for i in subset_definition] - else: - subset_list = full_list[subset_definition] - else: - subset_list = full_list - - # if only 1 item is left, make it a list before returning - if not isinstance(subset_list, list): - subset_list = [subset_list] - - return subset_list - -def is_met_netcdf(file_path): - """! Check if a file is a MET-generated NetCDF file. - If the file is not a NetCDF file, OSError occurs. - If the MET_version attribute doesn't exist, AttributeError occurs. - If the netCDF4 package is not available, ImportError should occur. - All of these situations result in the file being considered not - a MET-generated NetCDF file - Args: - @param file_path full path to file to check - @returns True if file is a MET-generated NetCDF file and False if - it is not or it can't be determined. - """ - try: - from netCDF4 import Dataset - nc_file = Dataset(file_path, 'r') - getattr(nc_file, 'MET_version') - except (AttributeError, OSError, ImportError): - return False - - return True - -def netcdf_has_var(file_path, name, level): - """! Check if name is a variable in the NetCDF file. If not, check if - {name}_{level} (with level prefix letter removed, i.e. 06 from A06) - If the file is not a NetCDF file, OSError occurs. - If the MET_version attribute doesn't exist, AttributeError occurs. - If the netCDF4 package is not available, ImportError should occur. - All of these situations result in the file being considered not - a MET-generated NetCDF file - Args: - @param file_path full path to file to check - @returns True if file is a MET-generated NetCDF file and False if - it is not or it can't be determined. - """ - try: - from netCDF4 import Dataset - - nc_file = Dataset(file_path, 'r') - variables = nc_file.variables.keys() - - # if name is a variable, return that name - if name in variables: - return name - - - # if name_level is a variable, return that - name_underscore_level = f"{name}_{split_level(level)[1]}" - if name_underscore_level in variables: - return name_underscore_level - - # requested variable name is not found in file - return None - - except (AttributeError, OSError, ImportError): - return False - -def generate_tmp_filename(): - import random - import string - random_string = ''.join(random.choice(string.ascii_letters) - for i in range(10)) - return f"metplus_tmp_{random_string}" - -def format_level(level): - """! Format level string to prevent NetCDF level values from creating - filenames and field names with bad characters. Replaces '*' with 'all' - and ',' with '_' - - @param level string of level to format - @returns formatted string - """ - return level.replace('*', 'all').replace(',', '_') diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py new file mode 100644 index 000000000..fb7b743b3 --- /dev/null +++ b/metplus/util/run_util.py @@ -0,0 +1,194 @@ +import sys +import os +import shutil +from datetime import datetime +from importlib import import_module + +from .constants import NO_COMMAND_WRAPPERS +from .system_util import get_user_info, write_list_to_file +from .. import get_metplus_version +from . import config_metplus +from . import camel_to_underscore + +def pre_run_setup(config_inputs): + + version_number = get_metplus_version() + print(f'Starting METplus v{version_number}') + + # Read config inputs and return a config instance + config = config_metplus.setup(config_inputs) + + logger = config.logger + + user_info = get_user_info() + user_string = f' as user {user_info} ' if user_info else ' ' + + config.set('config', 'METPLUS_VERSION', version_number) + logger.info('Running METplus v%s%swith command: %s', + version_number, user_string, ' '.join(sys.argv)) + + logger.info(f"Log file: {config.getstr('config', 'LOG_METPLUS')}") + logger.info(f"METplus Base: {config.getdir('METPLUS_BASE')}") + logger.info(f"Final Conf: {config.getstr('config', 'METPLUS_CONF')}") + config_list = config.getstr('config', 'CONFIG_INPUT').split(',') + for config_item in config_list: + logger.info(f"Config Input: {config_item}") + + # validate configuration variables + isOK_A, isOK_B, isOK_C, isOK_D, all_sed_cmds = config_metplus.validate_configuration_variables(config) + if not (isOK_A and isOK_B and isOK_C and isOK_D): + # if any sed commands were generated, write them to the sed file + if all_sed_cmds: + sed_file = os.path.join(config.getdir('OUTPUT_BASE'), 'sed_commands.txt') + # remove if sed file exists + if os.path.exists(sed_file): + os.remove(sed_file) + + write_list_to_file(sed_file, all_sed_cmds) + config.logger.error(f"Find/Replace commands have been generated in {sed_file}") + + logger.error("Correct configuration variables and rerun. Exiting.") + sys.exit(1) + + if not config.getdir('MET_INSTALL_DIR', must_exist=True): + logger.error('MET_INSTALL_DIR must be set correctly to run METplus') + sys.exit(1) + + # set staging dir to OUTPUT_BASE/stage if not set + if not config.has_option('config', 'STAGING_DIR'): + config.set('config', 'STAGING_DIR', + os.path.join(config.getdir('OUTPUT_BASE'), "stage")) + + # handle dir to write temporary files + config_metplus.handle_tmp_dir(config) + + # handle OMP_NUM_THREADS environment variable + config_metplus.handle_env_var_config(config, + env_var_name='OMP_NUM_THREADS', + config_name='OMP_NUM_THREADS') + + config.env = os.environ.copy() + + return config + + +def run_metplus(config): + total_errors = 0 + + # Use config object to get the list of processes to call + process_list = config_metplus.get_process_list(config) + + try: + processes = [] + for process, instance in process_list: + try: + logname = f"{process}.{instance}" if instance else process + logger = config.log(logname) + package_name = ('metplus.wrappers.' + f'{camel_to_underscore(process)}_wrapper') + module = import_module(package_name) + command_builder = ( + getattr(module, f"{process}Wrapper")(config, + instance=instance) + ) + + # if Usage specified in PROCESS_LIST, print usage and exit + if process == 'Usage': + command_builder.run_all_times() + return 0 + except AttributeError: + logger.error("There was a problem loading " + f"{process} wrapper.") + return 1 + except ModuleNotFoundError: + logger.error(f"Could not load {process} wrapper. " + "Wrapper may have been disabled.") + return 1 + + processes.append(command_builder) + + # check if all processes initialized correctly + allOK = True + for process in processes: + if not process.isOK: + allOK = False + class_name = process.__class__.__name__.replace('Wrapper', '') + logger.error("{} was not initialized properly".format(class_name)) + + # exit if any wrappers did not initialized properly + if not allOK: + logger.info("Refer to ERROR messages above to resolve issues.") + return 1 + + all_commands = [] + for process in processes: + new_commands = process.run_all_times() + if new_commands: + all_commands.extend(new_commands) + + # if process list contains any wrapper that should run commands + if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]): + # write out all commands and environment variables to file + if not config_metplus.write_all_commands(all_commands, config): + # report an error if no commands were generated + total_errors += 1 + + # compute total number of errors that occurred and output results + for process in processes: + if process.errors != 0: + process_name = process.__class__.__name__.replace('Wrapper', '') + error_msg = '{} had {} error'.format(process_name, process.errors) + if process.errors > 1: + error_msg += 's' + error_msg += '.' + logger.error(error_msg) + total_errors += process.errors + + return total_errors + except: + logger.exception("Fatal error occurred") + logger.info(f"Check the log file for more information: {config.getstr('config', 'LOG_METPLUS')}") + return 1 + +def post_run_cleanup(config, app_name, total_errors): + logger = config.logger + # scrub staging directory if requested + if (config.getbool('config', 'SCRUB_STAGING_DIR') and + os.path.exists(config.getdir('STAGING_DIR'))): + staging_dir = config.getdir('STAGING_DIR') + logger.info("Scrubbing staging dir: %s", staging_dir) + logger.info('Set SCRUB_STAGING_DIR to False to preserve ' + 'intermediate files.') + shutil.rmtree(staging_dir) + + # save log file path and clock time before writing final conf file + log_message = (f"Check the log file for more information: " + f"{config.getstr('config', 'LOG_METPLUS')}") + + start_clock_time = datetime.strptime(config.getstr('config', 'CLOCK_TIME'), + '%Y%m%d%H%M%S') + + # rewrite final conf so it contains all of the default values used + config_metplus.write_final_conf(config) + + # compute time it took to run + end_clock_time = datetime.now() + total_run_time = end_clock_time - start_clock_time + logger.debug(f"{app_name} took {total_run_time} to run.") + + user_info = get_user_info() + user_string = f' as user {user_info}' if user_info else '' + if not total_errors: + logger.info(log_message) + logger.info('%s has successfully finished running%s.', + app_name, user_string) + return + + error_msg = (f'{app_name} has finished running{user_string} ' + f'but had {total_errors} error') + if total_errors > 1: + error_msg += 's' + error_msg += '.' + logger.error(error_msg) + logger.info(log_message) + sys.exit(1) diff --git a/metplus/util/string_manip.py b/metplus/util/string_manip.py index 2779fa7eb..5ddb62e86 100644 --- a/metplus/util/string_manip.py +++ b/metplus/util/string_manip.py @@ -6,6 +6,8 @@ import re from csv import reader +import random +import string from .constants import VALID_COMPARISONS @@ -249,3 +251,245 @@ def format_thresh(thresh_str): formatted_thresh_list.append(thresh_letter) return ','.join(formatted_thresh_list) + + +def is_python_script(name): + """ Check if field name is a python script by checking if any of the words + in the string end with .py + + @param name string to check + @returns True if the name is determined to be a python script command + """ + if not name: + return False + + all_items = name.split(' ') + if any(item.endswith('.py') for item in all_items): + return True + + return False + + +def camel_to_underscore(camel): + """! Change camel case notation to underscore notation, i.e. GridStatWrapper to grid_stat_wrapper + Multiple capital letters are excluded, i.e. PCPCombineWrapper to pcp_combine_wrapper + Numerals are also skipped, i.e. ASCII2NCWrapper to ascii2nc_wrapper + Args: + @param camel string to convert + @returns string in underscore notation + """ + s1 = re.sub(r'([^\d])([A-Z][a-z]+)', r'\1_\2', camel) + return re.sub(r'([a-z])([A-Z])', r'\1_\2', s1).lower() + + +def get_threshold_via_regex(thresh_string): + """!Ensure thresh values start with >,>=,==,!=,<,<=,gt,ge,eq,ne,lt,le and then a number + Optionally can have multiple comparison/number pairs separated with && or ||. + Args: + @param thresh_string: String to examine, i.e. <=3.4 + Returns: + None if string does not match any valid comparison operators or does + not contain a number afterwards + regex match object with comparison operator in group 1 and + number in group 2 if valid + """ + + comparison_number_list = [] + # split thresh string by || or && + thresh_split = re.split(r'\|\||&&', thresh_string) + # check each threshold for validity + for thresh in thresh_split: + found_match = False + for comp in list(VALID_COMPARISONS)+list(VALID_COMPARISONS.values()): + # if valid, add to list of tuples + # must be one of the valid comparison operators followed by + # at least 1 digit or NA + if thresh == 'NA': + comparison_number_list.append((thresh, '')) + found_match = True + break + + match = re.match(r'^('+comp+r')(.*\d.*)$', thresh) + if match: + comparison = match.group(1) + number = match.group(2) + # try to convert to float if it can, but allow string + try: + number = float(number) + except ValueError: + pass + + comparison_number_list.append((comparison, number)) + found_match = True + break + + # if no match was found for the item, return None + if not found_match: + return None + + if not comparison_number_list: + return None + + return comparison_number_list + + +def validate_thresholds(thresh_list): + """ Checks list of thresholds to ensure all of them have the correct format + Should be a comparison operator with number pair combined with || or && + i.e. gt4 or >3&&<5 or gt3||lt1 + Args: + @param thresh_list list of strings to check + Returns: + True if all items in the list are valid format, False if not + """ + valid = True + for thresh in thresh_list: + match = get_threshold_via_regex(thresh) + if match is None: + valid = False + + if valid is False: + print("ERROR: Threshold values must use >,>=,==,!=,<,<=,gt,ge,eq,ne,lt, or le with a number, " + "optionally combined with && or ||") + return False + return True + + +def round_0p5(val): + """! Round to the nearest point five (ie 3.3 rounds to 3.5, 3.1 + rounds to 3.0) Take the input value, multiply by two, round to integer + (no decimal places) then divide by two. Expect any input value of n.0, + n.1, or n.2 to round down to n.0, and any input value of n.5, n.6 or + n.7 to round to n.5. Finally, any input value of n.8 or n.9 will + round to (n+1).0 + + @param val : The number to be rounded to the nearest .5 + @returns n.0, n.5, or (n+1).0 value as a result of rounding + """ + return round(val * 2) / 2 + + +def generate_tmp_filename(): + random_string = ''.join(random.choice(string.ascii_letters) + for i in range(10)) + return f"metplus_tmp_{random_string}" + + +def template_to_regex(template): + in_template = re.sub(r'\.', '\\.', template) + return re.sub(r'{lead.*?}', '.*', in_template) + + +def split_level(level): + """! If level value starts with a letter, then separate that letter from + the rest of the string. i.e. 'A03' will be returned as 'A', '03'. If no + level type letter is found and the level value consists of alpha-numeric + characters, return an empty string as the level type and the full level + string as the level value + + @param level input string to parse/split + @returns tuple of level type and level value + """ + if not level: + return '', '' + + match = re.match(r'^([a-zA-Z])(\w+)$', level) + if match: + level_type = match.group(1) + level = match.group(2) + return level_type, level + + match = re.match(r'^[\w]+$', level) + if match: + return '', level + + return '', '' + + +def format_level(level): + """! Format level string to prevent NetCDF level values from creating + filenames and field names with bad characters. Replaces '*' with 'all' + and ',' with '_' + + @param level string of level to format + @returns formatted string + """ + return level.replace('*', 'all').replace(',', '_') + + +def expand_int_string_to_list(int_string): + """! Expand string into a list of integer values. Items are separated by + commas. Items that are formatted X-Y will be expanded into each number + from X to Y inclusive. If the string ends with +, then add a str '+' + to the end of the list. Used in .github/jobs/get_use_case_commands.py + + @param int_string String containing a comma-separated list of integers + @returns List of integers and potentially '+' as the last item + """ + subset_list = [] + + # if string ends with +, remove it and add it back at the end + if int_string.strip().endswith('+'): + int_string = int_string.strip(' +') + hasPlus = True + else: + hasPlus = False + + # separate into list by comma + comma_list = int_string.split(',') + for comma_item in comma_list: + dash_list = comma_item.split('-') + # if item contains X-Y, expand it + if len(dash_list) == 2: + for i in range(int(dash_list[0].strip()), + int(dash_list[1].strip())+1, + 1): + subset_list.append(i) + else: + subset_list.append(int(comma_item.strip())) + + if hasPlus: + subset_list.append('+') + + return subset_list + + +def subset_list(full_list, subset_definition): + """! Extract subset of items from full_list based on subset_definition + Used in internal/tests/use_cases/metplus_use_case_suite.py + + @param full_list List of all use cases that were requested + @param subset_definition Defines how to subset the full list. If None, + no subsetting occurs. If an integer value, select that index only. + If a slice object, i.e. slice(2,4,1), pass slice object into list. + If list, subset full list by integer index values in list. If + last item in list is '+' then subset list up to 2nd last index, then + get all items from 2nd last item and above + """ + if subset_definition is not None: + subset_list = [] + + # if case slice is a list, use only the indices in the list + if isinstance(subset_definition, list): + # if last slice value is a plus sign, get rest of items + # after 2nd last slice value + if subset_definition[-1] == '+': + plus_value = subset_definition[-2] + # add all values before last index before plus + subset_list.extend([full_list[i] + for i in subset_definition[:-2]]) + # add last index listed + all items above + subset_list.extend(full_list[plus_value:]) + else: + # list of integers, so get items based on indices + subset_list = [full_list[i] for i in subset_definition] + else: + subset_list = full_list[subset_definition] + else: + subset_list = full_list + + # if only 1 item is left, make it a list before returning + if not isinstance(subset_list, list): + subset_list = [subset_list] + + return subset_list diff --git a/metplus/util/system_util.py b/metplus/util/system_util.py new file mode 100644 index 000000000..f47d9a893 --- /dev/null +++ b/metplus/util/system_util.py @@ -0,0 +1,460 @@ +""" +Program Name: system_manip.py +Contact(s): George McCabe +Description: METplus utility to handle OS/system calls +""" + +import os +import re +from pathlib import Path +import getpass +import gzip +import bz2 +import zipfile +import struct + +from .constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS + + +def mkdir_p(path): + """! + From stackoverflow.com/questions/600268/mkdir-p-functionality-in-python + Creates the entire directory path if it doesn't exist (including any + required intermediate directories). + Args: + @param path : The full directory path to be created + Returns + None: Creates the full directory path if it doesn't exist, + does nothing otherwise. + """ + Path(path).mkdir(parents=True, exist_ok=True) + + +def get_user_info(): + """! Get user information from OS. Note that some OS cannot obtain user ID + and some cannot obtain username. + @returns username(uid) if both username and user ID can be read, + username if only username can be read, uid if only user ID can be read, + or an empty string if neither can be read. + """ + try: + username = getpass.getuser() + except OSError: + username = None + + try: + uid = os.getuid() + except AttributeError: + uid = None + + if username and uid: + return f'{username}({uid})' + + if username: + return username + + if uid: + return uid + + return '' + + +def write_list_to_file(filename, output_list): + with open(filename, 'w+') as f: + for line in output_list: + f.write(f"{line}\n") + + +def get_storms(filter_filename, id_only=False, sort_column='STORM_ID'): + """! Get each storm as identified by a column in the input file. + Create dictionary storm ID as the key and a list of lines for that + storm as the value. + + @param filter_filename name of tcst file to read and extract storm id + @param sort_column column to use to sort and group storms. Default + value is STORM_ID + @returns 2 item tuple - 1)dictionary where key is storm ID and value + is list of relevant lines from tcst file, 2) header line from tcst + file. Item with key 'header' contains the header of the tcst file + """ + # Initialize a set because we want unique storm ids. + storm_id_list = set() + + try: + with open(filter_filename, "r") as file_handle: + header, *lines = file_handle.readlines() + + storm_id_column = header.split().index(sort_column) + for line in lines: + storm_id_list.add(line.split()[storm_id_column]) + except (ValueError, FileNotFoundError): + if id_only: + return [] + return {} + + # sort the unique storm ids, copy the original + # set by using sorted rather than sort. + sorted_storms = sorted(storm_id_list) + if id_only: + return sorted_storms + + if not sorted_storms: + return {} + + storm_dict = {'header': header} + # for each storm, get all lines for that storm + for storm in sorted_storms: + storm_dict[storm] = [line for line in lines if storm in line] + + return storm_dict + + +def prune_empty(output_dir, logger): + """! Start from the output_dir, and recursively check + all directories and files. If there are any empty + files or directories, delete/remove them so they + don't cause performance degradation or errors + when performing subsequent tasks. + + @param output_dir The directory from which searching should begin. + @param logger The logger to which all logging is directed. + """ + + # Check for empty files. + for root, dirs, files in os.walk(output_dir): + # Create a full file path by joining the path + # and filename. + for a_file in files: + a_file = os.path.join(root, a_file) + if os.stat(a_file).st_size == 0: + logger.debug("Empty file: " + a_file + + "...removing") + os.remove(a_file) + + # Now check for any empty directories, some + # may have been created when removing + # empty files. + for root, dirs, files in os.walk(output_dir): + for direc in dirs: + full_dir = os.path.join(root, direc) + if not os.listdir(full_dir): + logger.debug("Empty directory: " + full_dir + + "...removing") + os.rmdir(full_dir) + + +def get_files(filedir, filename_regex, logger=None): + """! Get all the files (with a particular naming format) by walking + through the directories. + + @param filedir The topmost directory from which the search begins. + @param filename_regex The regular expression that defines the naming + format of the files of interest. + @returns list of filenames (with full filepath) + """ + file_paths = [] + + # Walk the tree + for root, _, files in os.walk(filedir): + for filename in files: + # add it to the list only if it is a match + # to the specified format + match = re.match(filename_regex, filename) + if match: + # Join the two strings to form the full + # filepath. + filepath = os.path.join(root, filename) + file_paths.append(filepath) + else: + continue + return sorted(file_paths) + + +def preprocess_file(filename, data_type, config, allow_dir=False): + """ Decompress gzip, bzip, or zip files or convert Gempak files to NetCDF + Args: + @param filename: Path to file without zip extensions + @param config: Config object + Returns: + Path to staged unzipped file or original file if already unzipped + """ + if not filename: + return None + + if allow_dir and os.path.isdir(filename): + return filename + + # if using python embedding for input, return the keyword + if os.path.basename(filename) in PYTHON_EMBEDDING_TYPES: + return os.path.basename(filename) + + # if filename starts with a python embedding type, return the full value + for py_embed_type in PYTHON_EMBEDDING_TYPES: + if filename.startswith(py_embed_type): + return filename + + # if _INPUT_DATATYPE value contains PYTHON, return the full value + if data_type is not None and 'PYTHON' in data_type: + return filename + + stage_dir = config.getdir('STAGING_DIR') + + if os.path.isfile(filename): + # if filename provided ends with a valid compression extension, + # remove the extension and call function again so the + # file will be uncompressed properly. This is done so that + # the function will handle files passed to it with an + # extension the same way as files passed + # without an extension but the compressed equivalent exists + for ext in COMPRESSION_EXTENSIONS: + if filename.endswith(ext): + return preprocess_file(filename[:-len(ext)], data_type, config) + # if extension is grd (Gempak), then look in staging dir for nc file + if filename.endswith('.grd') or data_type == "GEMPAK": + if filename.endswith('.grd'): + stagefile = stage_dir + filename[:-3]+"nc" + else: + stagefile = stage_dir + filename+".nc" + if os.path.isfile(stagefile): + return stagefile + # if it does not exist, run GempakToCF and return staged nc file + # Create staging area if it does not exist + mkdir_p(os.path.dirname(stagefile)) + + # only import GempakToCF if needed + from ..wrappers import GempakToCFWrapper + + run_g2c = GempakToCFWrapper(config) + run_g2c.infiles.append(filename) + run_g2c.set_output_path(stagefile) + cmd = run_g2c.get_command() + if cmd is None: + config.logger.error("GempakToCF could not generate command") + return None + if config.logger: + config.logger.debug("Converting Gempak file into {}".format(stagefile)) + run_g2c.build() + return stagefile + + return filename + + # nc file requested and the Gempak equivalent exists + if os.path.isfile(filename[:-2]+'grd'): + return preprocess_file(filename[:-2]+'grd', data_type, config) + + # if file exists in the staging area, return that path + outpath = stage_dir + filename + if os.path.isfile(outpath): + return outpath + + # Create staging area directory only if file has compression extension + if any([os.path.isfile(f'{filename}{ext}') + for ext in COMPRESSION_EXTENSIONS]): + mkdir_p(os.path.dirname(outpath)) + + # uncompress gz, bz2, or zip file + if os.path.isfile(filename+".gz"): + if config.logger: + config.logger.debug("Uncompressing gz file to {}".format(outpath)) + with gzip.open(filename+".gz", 'rb') as infile: + with open(outpath, 'wb') as outfile: + outfile.write(infile.read()) + infile.close() + outfile.close() + return outpath + elif os.path.isfile(filename+".bz2"): + if config.logger: + config.logger.debug("Uncompressing bz2 file to {}".format(outpath)) + with open(filename+".bz2", 'rb') as infile: + with open(outpath, 'wb') as outfile: + outfile.write(bz2.decompress(infile.read())) + infile.close() + outfile.close() + return outpath + elif os.path.isfile(filename+".zip"): + if config.logger: + config.logger.debug("Uncompressing zip file to {}".format(outpath)) + with zipfile.ZipFile(filename+".zip") as z: + with open(outpath, 'wb') as f: + f.write(z.read(os.path.basename(filename))) + return outpath + + # if input doesn't need to exist, return filename + if not config.getbool('config', 'INPUT_MUST_EXIST', True): + return filename + + return None + + +def netcdf_has_var(file_path, name, level): + """! Check if name is a variable in the NetCDF file. If not, check if + {name}_{level} (with level prefix letter removed, i.e. 06 from A06) + If the file is not a NetCDF file, OSError occurs. + If the MET_version attribute doesn't exist, AttributeError occurs. + If the netCDF4 package is not available, ImportError should occur. + All of these situations result in the file being considered not + a MET-generated NetCDF file. (CURRENTLY UNUSED) + + @param file_path full path to file to check + @returns True if file is a MET-generated NetCDF file and False if + it is not or it can't be determined. + """ + try: + from netCDF4 import Dataset + + nc_file = Dataset(file_path, 'r') + variables = nc_file.variables.keys() + + # if name is a variable, return that name + if name in variables: + return name + + # if name_level is a variable, return that + name_underscore_level = f"{name}_{split_level(level)[1]}" + if name_underscore_level in variables: + return name_underscore_level + + # requested variable name is not found in file + return None + + except (AttributeError, OSError, ImportError): + return False + + +def is_met_netcdf(file_path): + """! Check if a file is a MET-generated NetCDF file. + If the file is not a NetCDF file, OSError occurs. + If the MET_version attribute doesn't exist, AttributeError occurs. + If the netCDF4 package is not available, ImportError should occur. + All of these situations result in the file being considered not + a MET-generated NetCDF file (CURRENTLY NOT USED) + + @param file_path full path to file to check + @returns True if file is a MET-generated NetCDF file and False if + it is not or it can't be determined. + """ + try: + from netCDF4 import Dataset + nc_file = Dataset(file_path, 'r') + getattr(nc_file, 'MET_version') + except (AttributeError, OSError, ImportError): + return False + + return True + + +def get_filetype(filepath, logger=None): + """!This function determines if the filepath is a NETCDF or GRIB file + based on the first eight bytes of the file. + It returns the string GRIB, NETCDF, or a None object. + + Note: If it is NOT determined to ba a NETCDF file, + it returns GRIB, regardless. + Unless there is an IOError exception, such as filepath refers + to a non-existent file or filepath is only a directory, than + None is returned, without a system exit. (CURRENTLY NOT USED) + + @param filepath: path/to/filename + @param logger the logger, optional + @returns The string GRIB, NETCDF or a None object + """ + # Developer Note + # Since we have the impending code-freeze, keeping the behavior the same, + # just changing the implementation. + # The previous logic did not test for GRIB it would just return 'GRIB' + # if you couldn't run ncdump on the file. + # Also note: + # As John indicated ... there is the case when a grib file + # may not start with GRIB ... and if you pass the MET command filtetype=GRIB + # MET will handle it ok ... + + # Notes on file format and determining type. + # https://www.wmo.int/pages/prog/www/WDM/Guides/Guide-binary-2.html + # https://www.unidata.ucar.edu/software/netcdf/docs/faq.html + # http: // www.hdfgroup.org / HDF5 / doc / H5.format.html + + # Interpreting single byte by byte - so ok to ignore endianess + # od command: + # od -An -c -N8 foo.nc + # od -tx1 -N8 foo.nc + # GRIB + # Octet no. IS Content + # 1-4 'GRIB' (Coded CCITT-ITA No. 5) (ASCII); + # 5-7 Total length, in octets, of GRIB message(including Sections 0 & 5); + # 8 Edition number - currently 1 + # NETCDF .. ie. od -An -c -N4 foo.nc which will output + # C D F 001 + # C D F 002 + # 211 H D F + # HDF5 + # Magic numbers Hex: 89 48 44 46 0d 0a 1a 0a + # ASCII: \211 HDF \r \n \032 \n + + # Below is a reference that may be used in the future to + # determine grib version. + # import struct + # with open ("foo.grb2","rb")as binary_file: + # binary_file.seek(7) + # one_byte = binary_file.read(1) + # + # This would return an integer with value 1 or 2, + # B option is an unsigned char. + # struct.unpack('B',one_byte)[0] + + # if filepath is set to None, return None to avoid crash + if filepath == None: + return None + + try: + # read will return up to 8 bytes, if file is 0 bytes in length, + # than first_eight_bytes will be the empty string ''. + # Don't test the file length, just adds more time overhead. + with open(filepath, "rb") as binary_file: + binary_file.seek(0) + first_eight_bytes = binary_file.read(8) + + # From the first eight bytes of the file, unpack the bytes + # of the known identifier byte locations, in to a string. + # Example, if this was a netcdf file than ONLY name_cdf would + # equal 'CDF' the other variables, name_hdf would be 'DF ' + # name_grid 'CDF ' + name_cdf, name_hdf, name_grib = [None] * 3 + if len(first_eight_bytes) == 8: + name_cdf = struct.unpack('3s', first_eight_bytes[:3])[0] + name_hdf = struct.unpack('3s', first_eight_bytes[1:4])[0] + name_grib = struct.unpack('4s', first_eight_bytes[:4])[0] + + # Why not just use a else, instead of elif else if we are going to + # return GRIB ? It allows for expansion, ie. Maybe we pass in a + # logger and log the cases we can't determine the type. + if name_cdf == 'CDF' or name_hdf == 'HDF': + return "NETCDF" + elif name_grib == 'GRIB': + return "GRIB" + else: + # This mimicks previous behavoir, were we at least will always return GRIB. + # It also handles the case where GRIB was not in the first 4 bytes + # of a legitimate grib file, see John. + # logger.info('Can't determine type, returning GRIB + # as default %s'%filepath) + return "GRIB" + + except IOError: + # Skip the IOError, and keep processing data. + # ie. filepath references a file that does not exist + # or filepath is a directory. + return None + + # Previous Logic + # ncdump_exe = config.getexe('NCDUMP') + #try: + # result = subprocess.check_output([ncdump_exe, filepath]) + + #except subprocess.CalledProcessError: + # return "GRIB" + + #regex = re.search("netcdf", result) + #if regex is not None: + # return "NETCDF" + #else: + # return None diff --git a/metplus/util/time_looping.py b/metplus/util/time_looping.py index 2b4cdb2cd..2cd124ff8 100644 --- a/metplus/util/time_looping.py +++ b/metplus/util/time_looping.py @@ -1,8 +1,12 @@ +import re from datetime import datetime, timedelta -from .string_manip import getlist -from .time_util import get_relativedelta +from .string_manip import getlist, getlistint +from .time_util import get_relativedelta, add_to_time_input +from .time_util import ti_get_hours_from_relativedelta +from .time_util import ti_get_seconds_from_relativedelta from .string_template_substitution import do_string_sub +from .config_metplus import log_runtime_banner def time_generator(config): """! Generator used to read METplusConfig variables for time looping @@ -82,6 +86,7 @@ def time_generator(config): current_dt += time_interval + def get_start_and_end_times(config): prefix = get_time_prefix(config) if not prefix: @@ -120,6 +125,42 @@ def get_start_and_end_times(config): return start_dt, end_dt + +def loop_over_times_and_call(config, processes, custom=None): + """! Loop over all run times and call wrappers listed in config + + @param config METplusConfig object + @param processes list of CommandBuilder subclass objects (Wrappers) to call + @param custom (optional) custom loop string value + @returns list of tuples with all commands run and the environment variables + that were set for each + """ + # keep track of commands that were run + all_commands = [] + for time_input in time_generator(config): + if not isinstance(processes, list): + processes = [processes] + + for process in processes: + # if time could not be read, increment errors for each process + if time_input is None: + process.errors += 1 + continue + + log_runtime_banner(config, time_input, process) + add_to_time_input(time_input, + instance=process.instance, + custom=custom) + + process.clear() + process.run_at_time(time_input) + if process.all_commands: + all_commands.extend(process.all_commands) + process.all_commands.clear() + + return all_commands + + def _validate_time_values(start_dt, end_dt, time_interval, prefix, logger): if not start_dt: logger.error(f"Could not read {prefix}_BEG") @@ -142,6 +183,7 @@ def _validate_time_values(start_dt, end_dt, time_interval, prefix, logger): return True + def _create_time_input_dict(prefix, current_dt, clock_dt): return { 'loop_by': prefix.lower(), @@ -150,6 +192,7 @@ def _create_time_input_dict(prefix, current_dt, clock_dt): 'today': clock_dt.strftime('%Y%m%d'), } + def get_time_prefix(config): """! Read the METplusConfig object and determine the prefix for the time looping variables. @@ -179,6 +222,7 @@ def get_time_prefix(config): config.logger.error('MUST SET LOOP_BY to VALID, INIT, RETRO, or REALTIME') return None + def _get_current_dt(time_string, time_format, clock_dt, logger): """! Use time format to get datetime object from time string, substituting values for today or now template tags if specified. @@ -204,3 +248,288 @@ def _get_current_dt(time_string, time_format, clock_dt, logger): return None return current_dt + + +def get_skip_times(config, wrapper_name=None): + """! Read SKIP_TIMES config variable and populate dictionary of times that should be skipped. + SKIP_TIMES should be in the format: "%m:begin_end_incr(3,11,1)", "%d:30,31", "%Y%m%d:20201031" + where each item inside quotes is a datetime format, colon, then a list of times in that format + to skip. + Args: + @param config configuration object to pull SKIP_TIMES + @param wrapper_name name of wrapper if supporting + skipping times only for certain wrappers, i.e. grid_stat + @returns dictionary containing times to skip + """ + skip_times_dict = {} + skip_times_string = None + + # if wrapper name is set, look for wrapper-specific _SKIP_TIMES variable + if wrapper_name: + skip_times_string = config.getstr('config', + f'{wrapper_name.upper()}_SKIP_TIMES', '') + + # if skip times string has not been found, check for generic SKIP_TIMES + if not skip_times_string: + skip_times_string = config.getstr('config', 'SKIP_TIMES', '') + + # if no generic SKIP_TIMES, return empty dictionary + if not skip_times_string: + return {} + + # get list of skip items, but don't expand begin_end_incr yet + skip_list = getlist(skip_times_string, expand_begin_end_incr=False) + + for skip_item in skip_list: + try: + time_format, skip_times = skip_item.split(':') + + # get list of skip times for the time format, expanding begin_end_incr + skip_times_list = getlist(skip_times) + + # if time format is already in skip times dictionary, extend list + if time_format in skip_times_dict: + skip_times_dict[time_format].extend(skip_times_list) + else: + skip_times_dict[time_format] = skip_times_list + + except ValueError: + config.logger.error(f"SKIP_TIMES item does not match format: {skip_item}") + return None + + return skip_times_dict + + +def skip_time(time_info, skip_times): + """!Used to check the valid time of the current run time against list of times to skip. + Args: + @param time_info dictionary with time information to check + @param skip_times dictionary of times to skip, i.e. {'%d': [31]} means skip 31st day + @returns True if run time should be skipped, False if not + """ + if not skip_times: + return False + + for time_format, skip_time_list in skip_times.items(): + # extract time information from valid time based on skip time format + run_time_value = time_info.get('valid') + if not run_time_value: + return False + + run_time_value = run_time_value.strftime(time_format) + + # loop over times to skip for this format and check if it matches + for skip_time in skip_time_list: + if int(run_time_value) == int(skip_time): + return True + + # if skip time never matches, return False + return False + + +def get_lead_sequence(config, input_dict=None, wildcard_if_empty=False): + """!Get forecast lead list from LEAD_SEQ or compute it from INIT_SEQ. + Restrict list by LEAD_SEQ_[MIN/MAX] if set. Now returns list of relativedelta objects + Args: + @param config METplusConfig object to query config variable values + @param input_dict time dictionary needed to handle using INIT_SEQ. Must contain + valid key if processing INIT_SEQ + @param wildcard_if_empty if no lead sequence was set, return a + list with '*' if this is True, otherwise return a list with 0 + @returns list of relativedelta objects or a list containing 0 if none are found + """ + + out_leads = [] + lead_min, lead_max, no_max = _get_lead_min_max(config) + + # check if LEAD_SEQ, INIT_SEQ, or LEAD_SEQ_ are set + # if more than one is set, report an error and exit + lead_seq = getlist(config.getstr('config', 'LEAD_SEQ', '')) + init_seq = getlistint(config.getstr('config', 'INIT_SEQ', '')) + lead_groups = get_lead_sequence_groups(config) + + if not _are_lead_configs_ok(lead_seq, + init_seq, + lead_groups, + config, + input_dict, + no_max): + return None + + if lead_seq: + # return lead sequence if wildcard characters are used + if lead_seq == ['*']: + return lead_seq + + out_leads = _handle_lead_seq(config, + lead_seq, + lead_min, + lead_max) + + # use INIT_SEQ to build lead list based on the valid time + elif init_seq: + out_leads = _handle_init_seq(init_seq, + input_dict, + lead_min, + lead_max) + elif lead_groups: + out_leads = _handle_lead_groups(lead_groups) + + if not out_leads: + if wildcard_if_empty: + return ['*'] + + return [0] + + return out_leads + +def _are_lead_configs_ok(lead_seq, init_seq, lead_groups, + config, input_dict, no_max): + if lead_groups is None: + return False + + error_message = ('are both listed in the configuration. ' + 'Only one may be used at a time.') + if lead_seq: + if init_seq: + config.logger.error(f'LEAD_SEQ and INIT_SEQ {error_message}') + return False + + if lead_groups: + config.logger.error(f'LEAD_SEQ and LEAD_SEQ_ {error_message}') + return False + + if init_seq and lead_groups: + config.logger.error(f'INIT_SEQ and LEAD_SEQ_ {error_message}') + return False + + if init_seq: + # if input dictionary not passed in, + # cannot compute lead sequence from it, so exit + if input_dict is None: + config.logger.error('Cannot run using INIT_SEQ for this wrapper') + return False + + # if looping by init, fail and exit + if 'valid' not in input_dict.keys(): + log_msg = ('INIT_SEQ specified while looping by init time.' + ' Use LEAD_SEQ or change to loop by valid time') + config.logger.error(log_msg) + return False + + # maximum lead must be specified to run with INIT_SEQ + if no_max: + config.logger.error('LEAD_SEQ_MAX must be set to use INIT_SEQ') + return False + + return True + +def _get_lead_min_max(config): + # remove any items that are outside of the range specified + # by LEAD_SEQ_MIN and LEAD_SEQ_MAX + # convert min and max to relativedelta objects, then use current time + # to compare them to each forecast lead + # this is an approximation because relative time offsets depend on + # each runtime + huge_max = '4000Y' + lead_min_str = config.getstr_nocheck('config', 'LEAD_SEQ_MIN', '0') + lead_max_str = config.getstr_nocheck('config', 'LEAD_SEQ_MAX', huge_max) + no_max = lead_max_str == huge_max + lead_min = get_relativedelta(lead_min_str, 'H') + lead_max = get_relativedelta(lead_max_str, 'H') + return lead_min, lead_max, no_max + +def _handle_lead_seq(config, lead_strings, lead_min=None, lead_max=None): + out_leads = [] + leads = [] + for lead in lead_strings: + relative_delta = get_relativedelta(lead, 'H') + if relative_delta is not None: + leads.append(relative_delta) + else: + config.logger.error(f'Invalid item {lead} in LEAD_SEQ. Exiting.') + return None + + if lead_min is None and lead_max is None: + return leads + + # add current time to leads to approximate month and year length + now_time = datetime.now() + lead_min_approx = now_time + lead_min + lead_max_approx = now_time + lead_max + for lead in leads: + lead_approx = now_time + lead + if lead_approx >= lead_min_approx and lead_approx <= lead_max_approx: + out_leads.append(lead) + + return out_leads + +def _handle_init_seq(init_seq, input_dict, lead_min, lead_max): + out_leads = [] + lead_min_hours = ti_get_hours_from_relativedelta(lead_min) + lead_max_hours = ti_get_hours_from_relativedelta(lead_max) + + valid_hr = int(input_dict['valid'].strftime('%H')) + for init in init_seq: + if valid_hr >= init: + current_lead = valid_hr - init + else: + current_lead = valid_hr + (24 - init) + + while current_lead <= lead_max_hours: + if current_lead >= lead_min_hours: + out_leads.append(get_relativedelta(current_lead, default_unit='H')) + current_lead += 24 + + out_leads = sorted(out_leads, key=lambda + rd: ti_get_seconds_from_relativedelta(rd, input_dict['valid'])) + return out_leads + +def _handle_lead_groups(lead_groups): + """! Read groups of forecast leads and create a list with all unique items + + @param lead_group dictionary where the values are lists of forecast + leads stored as relativedelta objects + @returns list of forecast leads stored as relativedelta objects + """ + out_leads = [] + for _, lead_seq in lead_groups.items(): + for lead in lead_seq: + if lead not in out_leads: + out_leads.append(lead) + + return out_leads + +def get_lead_sequence_groups(config): + # output will be a dictionary where the key will be the + # label specified and the value will be the list of forecast leads + lead_seq_dict = {} + # used in plotting + all_conf = config.keys('config') + indices = [] + regex = re.compile(r"LEAD_SEQ_(\d+)") + for conf in all_conf: + result = regex.match(conf) + if result is not None: + indices.append(result.group(1)) + + # loop over all possible variables and add them to list + for index in indices: + if config.has_option('config', f"LEAD_SEQ_{index}_LABEL"): + label = config.getstr('config', f"LEAD_SEQ_{index}_LABEL") + else: + log_msg = (f'Need to set LEAD_SEQ_{index}_LABEL to describe ' + f'LEAD_SEQ_{index}') + config.logger.error(log_msg) + return None + + # get forecast list for n + lead_string_list = getlist(config.getstr('config', f'LEAD_SEQ_{index}')) + lead_seq = _handle_lead_seq(config, + lead_string_list, + lead_min=None, + lead_max=None) + # add to output dictionary + lead_seq_dict[label] = lead_seq + + return lead_seq_dict diff --git a/metplus/util/time_util.py b/metplus/util/time_util.py index e1bd4b1f9..6e6c5cfc0 100755 --- a/metplus/util/time_util.py +++ b/metplus/util/time_util.py @@ -33,6 +33,18 @@ } +def shift_time_seconds(time_str, shift): + """ Adjust time by shift seconds. Format is %Y%m%d%H%M%S + Args: + @param time_str: Start time in %Y%m%d%H%M%S + @param shift: Amount to adjust time in seconds + Returns: + New time in format %Y%m%d%H%M%S + """ + return (datetime.datetime.strptime(time_str, "%Y%m%d%H%M%S") + + datetime.timedelta(seconds=shift)).strftime("%Y%m%d%H%M%S") + + def get_relativedelta(value, default_unit='S'): """!Converts time values ending in Y, m, d, H, M, or S to relativedelta object Args: @@ -483,3 +495,17 @@ def ti_calculate(input_dict_preserve): out_dict['lead_seconds'] = total_seconds return out_dict + + +def add_to_time_input(time_input, clock_time=None, instance=None, custom=None): + if clock_time: + clock_dt = datetime.datetime.strptime(clock_time, '%Y%m%d%H%M%S') + time_input['now'] = clock_dt + + # if instance is set, use that value, otherwise use empty string + time_input['instance'] = instance if instance else '' + + # if custom is specified, set it + # otherwise leave it unset so it can be set within the wrapper + if custom: + time_input['custom'] = custom diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 641b336f9..02a06fd65 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -12,10 +12,9 @@ import os -from ..util import met_util as util from ..util import time_util from . import CommandBuilder -from ..util import do_string_sub +from ..util import do_string_sub, skip_time, get_lead_sequence '''!@namespace ASCII2NCWrapper @brief Wraps the ASCII2NC tool to reformat ascii format to NetCDF @@ -242,14 +241,14 @@ def run_at_time(self, input_dict): Args: @param input_dict dictionary containing timing information """ - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: self.clear() input_dict['lead'] = lead time_info = time_util.ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 31cf9da94..43e424cea 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -19,19 +19,19 @@ from .command_runner import CommandRunner from ..util.constants import PYTHON_EMBEDDING_TYPES -from ..util import getlist -from ..util import met_util as util +from ..util import getlist, preprocess_file, loop_over_times_and_call from ..util import do_string_sub, ti_calculate, get_seconds_from_string -from ..util import get_time_from_file +from ..util import get_time_from_file, shift_time_seconds from ..util import config_metplus from ..util import METConfig from ..util import MISSING_DATA_VALUE from ..util import get_custom_string_list from ..util import get_wrapped_met_config_file, add_met_config_item, format_met_config -from ..util import remove_quotes +from ..util import remove_quotes, split_level from ..util import get_field_info, format_field_info -from ..util import get_wrapper_name +from ..util import get_wrapper_name, is_python_script from ..util.met_config import add_met_config_dict, handle_climo_dict +from ..util import mkdir_p, get_skip_times # pylint:disable=pointless-string-statement '''!@namespace CommandBuilder @@ -174,8 +174,7 @@ def create_c_dict(self): c_dict['CUSTOM_LOOP_LIST'] = get_custom_string_list(self.config, app_name) - c_dict['SKIP_TIMES'] = util.get_skip_times(self.config, - app_name) + c_dict['SKIP_TIMES'] = get_skip_times(self.config, app_name) c_dict['MANDATORY'] = ( self.config.getbool('config', @@ -533,7 +532,7 @@ def find_data(self, time_info, var_info=None, data_type='', mandatory=True, # separate character from beginning of numeric # level value if applicable - level = util.split_level(v_level)[1] + level = split_level(v_level)[1] # set level to 0 character if it is not a number if not level.isdigit(): @@ -660,10 +659,10 @@ def find_exact_file(self, level, data_type, time_info, mandatory=True, # check if file exists input_data_type = self.c_dict.get(data_type + 'INPUT_DATATYPE', '') - processed_path = util.preprocess_file(file_path, - input_data_type, - self.config, - allow_dir=allow_dir) + processed_path = preprocess_file(file_path, + input_data_type, + self.config, + allow_dir=allow_dir) # report error if file path could not be found if not processed_path: @@ -706,9 +705,9 @@ def find_file_in_window(self, level, data_type, time_info, mandatory=True, # get range of times that will be considered valid_range_lower = self.c_dict.get(data_type + 'FILE_WINDOW_BEGIN', 0) valid_range_upper = self.c_dict.get(data_type + 'FILE_WINDOW_END', 0) - lower_limit = int(datetime.strptime(util.shift_time_seconds(valid_time, valid_range_lower), + lower_limit = int(datetime.strptime(shift_time_seconds(valid_time, valid_range_lower), "%Y%m%d%H%M%S").strftime("%s")) - upper_limit = int(datetime.strptime(util.shift_time_seconds(valid_time, valid_range_upper), + upper_limit = int(datetime.strptime(shift_time_seconds(valid_time, valid_range_upper), "%Y%m%d%H%M%S").strftime("%s")) msg = f"Looking for {data_type}INPUT files under {data_dir} within range " +\ @@ -767,16 +766,16 @@ def find_file_in_window(self, level, data_type, time_info, mandatory=True, # check if file(s) needs to be preprocessed before returning the path # if one file was found and return_list if False, return single file if len(closest_files) == 1 and not return_list: - return util.preprocess_file(closest_files[0], - self.c_dict.get(data_type + 'INPUT_DATATYPE', ''), - self.config) + return preprocess_file(closest_files[0], + self.c_dict.get(data_type + 'INPUT_DATATYPE', ''), + self.config) # return list if multiple files are found out = [] for close_file in closest_files: - outfile = util.preprocess_file(close_file, - self.c_dict.get(data_type + 'INPUT_DATATYPE', ''), - self.config) + outfile = preprocess_file(close_file, + self.c_dict.get(data_type + 'INPUT_DATATYPE', ''), + self.config) out.append(outfile) return out @@ -909,7 +908,7 @@ def write_list_file(self, filename, file_list, output_dir=None): list_path = os.path.join(list_dir, filename) - util.mkdir_p(list_dir) + mkdir_p(list_dir) self.logger.debug("Writing list of filenames...") with open(list_path, 'w') as file_handle: @@ -1004,7 +1003,7 @@ def find_and_check_output_file(self, time_info=None, if (not os.path.exists(parent_dir) and not self.c_dict.get('DO_NOT_RUN_EXE', False)): self.logger.debug(f"Creating output directory: {parent_dir}") - util.mkdir_p(parent_dir) + mkdir_p(parent_dir) if not output_exists or not skip_if_output_exists: return True @@ -1107,7 +1106,7 @@ def check_for_python_embedding(self, input_type, var_info): # reset file type to empty string to handle if python embedding is used for one field but not for the next self.c_dict[f'{input_type}_FILE_TYPE'] = '' - if not util.is_python_script(var_info[f"{var_input_type}_name"]): + if not is_python_script(var_info[f"{var_input_type}_name"]): # if not a python script, return var name return var_info[f"{var_input_type}_name"] @@ -1218,7 +1217,7 @@ def get_command(self): self.log_error('Must specify path to output file') return None - util.mkdir_p(parent_dir) + mkdir_p(parent_dir) cmd += " " + out_path @@ -1284,7 +1283,7 @@ def run_all_times(self, custom=None): @param custom (optional) custom loop string value """ - return util.loop_over_times_and_call(self.config, self, custom=custom) + return loop_over_times_and_call(self.config, self, custom=custom) @staticmethod def format_met_config_dict(c_dict, name, keys=None): diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index a0e0c9db5..e6ceda8be 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -37,9 +37,9 @@ WRAPPER_CANNOT_RUN = True EXCEPTION_ERR = err_msg -from ..util import met_util as util from ..util import do_string_sub from ..util import time_generator, add_to_time_input +from ..util import mkdir_p, get_files from . import CommandBuilder @@ -194,8 +194,7 @@ def retrieve_data(self): self.logger.debug("Get data from all files in the directory " + self.input_data) # Get the list of all files (full file path) in this directory - all_input_files = util.get_files(self.input_data, ".*.tcst", - self.logger) + all_input_files = get_files(self.input_data, ".*.tcst", self.logger) # read each file into pandas then concatenate them together df_list = [pd.read_csv(file, delim_whitespace=True) for file in all_input_files] @@ -343,7 +342,7 @@ def retrieve_data(self): # which is used to generate the plot. if self.gen_ascii: self.logger.debug(f" output dir: {self.output_dir}") - util.mkdir_p(self.output_dir) + mkdir_p(self.output_dir) ascii_track_parts = [self.init_date, '.csv'] ascii_track_output_name = ''.join(ascii_track_parts) final_df_filename = os.path.join(self.output_dir, ascii_track_output_name) @@ -425,7 +424,7 @@ def create_plot(self): plt.text(60, -130, watermark, fontsize=5, alpha=0.25) # Make sure the output directory exists, and create it if it doesn't. - util.mkdir_p(self.output_dir) + mkdir_p(self.output_dir) # get the points for the scatter plots (and the relevant information for annotations, etc.) points_list = self.get_plot_points() diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index e31ff8267..aa392e9b5 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -13,10 +13,9 @@ import os import glob -from ..util import met_util as util +from ..util import sub_var_list +from ..util import do_string_sub, parse_var_list, PYTHON_EMBEDDING_TYPES from . import CompareGriddedWrapper -from ..util import do_string_sub -from ..util import parse_var_list """!@namespace EnsembleStatWrapper @brief Wraps the MET tool ensemble_stat to compare ensemble datasets @@ -136,8 +135,8 @@ def create_c_dict(self): # check if more than 1 obs datatype is set to python embedding, # only one can be used - if (c_dict['OBS_POINT_INPUT_DATATYPE'] in util.PYTHON_EMBEDDING_TYPES and - c_dict['OBS_GRID_INPUT_DATATYPE'] in util.PYTHON_EMBEDDING_TYPES): + if (c_dict['OBS_POINT_INPUT_DATATYPE'] in PYTHON_EMBEDDING_TYPES and + c_dict['OBS_GRID_INPUT_DATATYPE'] in PYTHON_EMBEDDING_TYPES): self.log_error("Both OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE and " "OBS_ENSEMBLE_STAT_INPUT_GRID_DATATYPE" " are set to Python Embedding types. " @@ -145,9 +144,9 @@ def create_c_dict(self): # if either are set, set OBS_INPUT_DATATYPE to that value so # it can be found by the check_for_python_embedding function - elif c_dict['OBS_POINT_INPUT_DATATYPE'] in util.PYTHON_EMBEDDING_TYPES: + elif c_dict['OBS_POINT_INPUT_DATATYPE'] in PYTHON_EMBEDDING_TYPES: c_dict['OBS_INPUT_DATATYPE'] = c_dict['OBS_POINT_INPUT_DATATYPE'] - elif c_dict['OBS_GRID_INPUT_DATATYPE'] in util.PYTHON_EMBEDDING_TYPES: + elif c_dict['OBS_GRID_INPUT_DATATYPE'] in PYTHON_EMBEDDING_TYPES: c_dict['OBS_INPUT_DATATYPE'] = c_dict['OBS_GRID_INPUT_DATATYPE'] c_dict['N_MEMBERS'] = ( @@ -424,8 +423,7 @@ def run_at_time_all_fields(self, time_info): return # parse optional var list for FCST and/or OBS fields - var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], - time_info) + var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) # if empty var list for FCST/OBS, use None as first var, # else use first var in list diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index b0e3f9982..ed11b3835 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -13,9 +13,9 @@ from datetime import datetime import re -from ..util import met_util as util -from ..util import do_string_sub, ti_calculate -from ..util import parse_var_list +from ..util import do_string_sub, ti_calculate, skip_time +from ..util import get_lead_sequence, sub_var_list +from ..util import parse_var_list, round_0p5, get_storms, prune_empty from .regrid_data_plane_wrapper import RegridDataPlaneWrapper from . import CommandBuilder @@ -206,7 +206,7 @@ def run_at_time(self, input_dict): """ # loop of forecast leads and process each - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: input_dict['lead'] = lead @@ -217,7 +217,7 @@ def run_at_time(self, input_dict): f"Processing forecast lead {time_info['lead_string']}" ) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue @@ -247,7 +247,7 @@ def run_at_time_loop_string(self, time_info): # get unique storm ids or object cats from the input file # store list of lines from tcst/mtd file for each ID as the value - storm_dict = util.get_storms( + storm_dict = get_storms( input_path, sort_column=self.SORT_COLUMN[location_input] ) @@ -267,7 +267,7 @@ def run_at_time_loop_string(self, time_info): else: self.use_tc_stat_input(storm_dict, idx_dict) - util.prune_empty(self.c_dict['OUTPUT_DIR'], self.logger) + prune_empty(self.c_dict['OUTPUT_DIR'], self.logger) def use_tc_stat_input(self, storm_dict, idx_dict): """! Find storms in TCStat input file and create tiles using the storm. @@ -383,8 +383,7 @@ def get_object_indices(object_cats): def call_regrid_data_plane(self, time_info, track_data, input_type): # set var list from config using time info - var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], - time_info) + var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) for data_type in ['FCST', 'OBS']: grid = self.get_grid(data_type, track_data[data_type], @@ -515,8 +514,8 @@ def get_grid_info(self, lat, lon, data_type): # float(lon) - lon_subtr adj_lon = float(lon) - self.c_dict['LON_ADJ'] adj_lat = float(lat) - self.c_dict['LAT_ADJ'] - lon0 = str(util.round_0p5(adj_lon)) - lat0 = str(util.round_0p5(adj_lat)) + lon0 = round_0p5(adj_lon) + lat0 = round_0p5(adj_lat) self.logger.debug(f'{data_type} ' f'lat: {lat} (track lat) => {lat0} (lat lower left), ' diff --git a/metplus/wrappers/gempak_to_cf_wrapper.py b/metplus/wrappers/gempak_to_cf_wrapper.py index 6f618427c..53a5a5cb7 100755 --- a/metplus/wrappers/gempak_to_cf_wrapper.py +++ b/metplus/wrappers/gempak_to_cf_wrapper.py @@ -12,8 +12,7 @@ import os -from ..util import met_util as util -from ..util import do_string_sub +from ..util import do_string_sub, skip_time, get_lead_sequence from ..util import time_util from . import CommandBuilder @@ -75,7 +74,7 @@ def run_at_time(self, input_dict): Args: @param input_dict dictionary containing timing information """ - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: self.clear() input_dict['lead'] = lead @@ -87,7 +86,7 @@ def run_at_time(self, input_dict): time_info = time_util.ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index d44026af2..eb1c5e98b 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -12,11 +12,9 @@ import os -from ..util import met_util as util from ..util import time_util from . import RuntimeFreqWrapper -from ..util import do_string_sub -from ..util import parse_var_list +from ..util import do_string_sub, parse_var_list, sub_var_list '''!@namespace GridDiagWrapper @brief Wraps the Grid-Diag tool @@ -187,7 +185,7 @@ def set_data_field(self, time_info): @param time_info time dictionary to use for string substitution @returns True if field list could be built, False if not. """ - field_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) + field_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) if not field_list: self.log_error("Could not get field information from config.") return False diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index ad7650c0f..379e6a628 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -12,7 +12,6 @@ import os -from ..util import met_util as util from . import CompareGriddedWrapper # pylint:disable=pointless-string-statement diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index 1ed678cb9..421d440ce 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -13,10 +13,9 @@ import os from datetime import datetime -from ..util import met_util as util -from ..util import time_util +from ..util import ti_calculate from . import RuntimeFreqWrapper -from ..util import do_string_sub, getlist +from ..util import do_string_sub, getlist, generate_tmp_filename '''!@namespace METDbLoadWrapper @brief Parent class for wrappers that run over a grouping of times @@ -118,7 +117,7 @@ def run_at_time_once(self, time_info): if time_info.get('lead') != '*': if (time_info.get('init') != '*' or time_info.get('valid') != '*'): - time_info = time_util.ti_calculate(time_info) + time_info = ti_calculate(time_info) self.set_environment_variables(time_info) @@ -234,7 +233,7 @@ def replace_values_in_xml(self, time_info): output_lines.append(output_line) # write tmp file with XML content with substituted values - out_filename = util.generate_tmp_filename() + out_filename = generate_tmp_filename() out_path = os.path.join(self.config.getdir('TMP_DIR'), out_filename) with open(out_path, 'w') as file_handle: diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index bec9f67cf..1a539ea02 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -12,7 +12,6 @@ import os -from ..util import met_util as util from . import CompareGriddedWrapper from ..util import do_string_sub diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index fdc9b9776..217427bad 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -12,9 +12,9 @@ import os -from ..util import met_util as util -from ..util import time_util -from ..util import do_string_sub +from ..util import get_lead_sequence, sub_var_list +from ..util import ti_calculate +from ..util import do_string_sub, skip_time from ..util import parse_var_list from . import CompareGriddedWrapper @@ -179,7 +179,7 @@ def run_at_time(self, input_dict): @param input_dict dictionary containing timing information """ - if util.skip_time(input_dict, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(input_dict, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') return @@ -197,8 +197,7 @@ def run_at_time_loop_string(self, input_dict): Args: @param input_dict dictionary containing timing information """ - var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], - input_dict) + var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], input_dict) # if only processing a single data set (FCST or OBS) then only read # that var list and process @@ -219,7 +218,7 @@ def run_at_time_loop_string(self, input_dict): for var_info in var_list: if self.c_dict.get('EXPLICIT_FILE_LIST', False): - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) model_list_path = do_string_sub(self.c_dict['FCST_FILE_LIST'], **time_info) self.logger.debug(f"Explicit FCST file: {model_list_path}") @@ -246,13 +245,13 @@ def run_at_time_loop_string(self, input_dict): obs_list = [] # find files for each forecast lead time - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) tasks = [] for lead in lead_seq: input_dict['lead'] = lead - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) tasks.append(time_info) for current_task in tasks: @@ -282,7 +281,7 @@ def run_at_time_loop_string(self, input_dict): # write ascii file with list of files to process input_dict['lead'] = lead_seq[0] - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) # if var name is a python embedding script, check type of python # input and name file list file accordingly @@ -313,7 +312,7 @@ def run_single_mode(self, input_dict, var_info): data_src = self.c_dict.get('SINGLE_DATA_SRC') if self.c_dict.get('EXPLICIT_FILE_LIST', False): - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) single_list_path = do_string_sub( self.c_dict[f'{data_src}_FILE_LIST'], **time_info @@ -330,10 +329,10 @@ def run_single_mode(self, input_dict, var_info): else: find_method = self.find_model - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: input_dict['lead'] = lead - current_task = time_util.ti_calculate(input_dict) + current_task = ti_calculate(input_dict) single_file = find_method(current_task, var_info) if single_file is None: @@ -346,7 +345,7 @@ def run_single_mode(self, input_dict, var_info): # write ascii file with list of files to process input_dict['lead'] = lead_seq[0] - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) file_ext = self.check_for_python_embedding(data_src, var_info) if not file_ext: return diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 82519417c..fff7783f7 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -13,9 +13,8 @@ import os import re -from ..util import getlistint -from ..util import met_util as util -from ..util import time_util +from ..util import getlistint, skip_time, get_lead_sequence +from ..util import ti_calculate from ..util import do_string_sub from . import CommandBuilder @@ -258,11 +257,11 @@ def set_valid_window_variables(self, time_info): def run_at_time(self, input_dict): """! Loop over each forecast lead and build pb2nc command """ # loop of forecast leads and process each - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: input_dict['lead'] = lead - lead_string = time_util.ti_calculate(input_dict)['lead_string'] + lead_string = ti_calculate(input_dict)['lead_string'] self.logger.info("Processing forecast lead {}".format(lead_string)) for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: @@ -287,7 +286,7 @@ def run_at_time_once(self, input_dict): if time_info is None: return - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') return diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index 8f476647b..f87b07fad 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -7,12 +7,11 @@ import os from datetime import timedelta -from ..util import met_util as util -from ..util import do_string_sub, getlist +from ..util import do_string_sub, getlist, preprocess_file from ..util import get_seconds_from_string, ti_get_lead_string, ti_calculate from ..util import get_relativedelta, ti_get_seconds_from_relativedelta from ..util import time_string_to_met_time, seconds_to_met_time -from ..util import parse_var_list +from ..util import parse_var_list, template_to_regex, split_level from . import ReformatGriddedWrapper '''!@namespace PCPCombineWrapper @@ -348,9 +347,9 @@ def setup_subtract_method(self, time_info, accum, data_src): # get first file filepath1 = do_string_sub(full_template, **time_info) - file1 = util.preprocess_file(filepath1, - self.c_dict[data_src+'_INPUT_DATATYPE'], - self.config) + file1 = preprocess_file(filepath1, + self.c_dict[data_src+'_INPUT_DATATYPE'], + self.config) if file1 is None: self.log_error(f'Could not find {data_src} file {filepath1} ' @@ -394,9 +393,9 @@ def setup_subtract_method(self, time_info, accum, data_src): time_info2['custom'] = time_info.get('custom', '') filepath2 = do_string_sub(full_template, **time_info2) - file2 = util.preprocess_file(filepath2, - self.c_dict[data_src+'_INPUT_DATATYPE'], - self.config) + file2 = preprocess_file(filepath2, + self.c_dict[data_src+'_INPUT_DATATYPE'], + self.config) if file2 is None: self.log_error(f'Could not find {data_src} file {filepath2} ' @@ -445,10 +444,10 @@ def setup_sum_method(self, time_info, lookback, data_src): out_accum = time_string_to_met_time(lookback, 'S') time_info['level'] = in_accum - pcp_regex = util.template_to_regex( - self.c_dict[f'{data_src}_INPUT_TEMPLATE'], - time_info + pcp_regex = template_to_regex( + self.c_dict[f'{data_src}_INPUT_TEMPLATE'] ) + pcp_regex = do_string_sub(pcp_regex, **time_info) pcp_regex_split = pcp_regex.split('/') pcp_dir = os.path.join(self.c_dict[f'{data_src}_INPUT_DIR'], *pcp_regex_split[0:-1]) @@ -611,7 +610,7 @@ def _get_lookback_seconds(self, time_info, var_info, data_src): else: lookback = '0' - _, lookback = util.split_level(lookback) + _, lookback = split_level(lookback) lookback_seconds = get_seconds_from_string( lookback, @@ -791,7 +790,7 @@ def get_lowest_fcst_file(self, valid_time, data_src): search_file = do_string_sub(search_file, **time_info) self.logger.debug(f"Looking for {search_file}") - search_file = util.preprocess_file( + search_file = preprocess_file( search_file, self.c_dict[data_src+'_INPUT_DATATYPE'], self.config) @@ -817,8 +816,7 @@ def get_field_string(self, time_info=None, search_accum=0, name=None, # string sub values into full field info string using search time info if time_info: - field_info = do_string_sub(field_info, - **time_info) + field_info = do_string_sub(field_info, **time_info) return field_info def find_input_file(self, init_time, valid_time, search_accum, data_src): @@ -848,9 +846,9 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src): in_template) input_path = do_string_sub(input_path, **time_info) - return util.preprocess_file(input_path, - self.c_dict[f'{data_src}_INPUT_DATATYPE'], - self.config), lead + return preprocess_file(input_path, + self.c_dict[f'{data_src}_INPUT_DATATYPE'], + self.config), lead def get_template_accum(self, accum_dict, search_time, lead, data_src): # apply string substitution to accum amount diff --git a/metplus/wrappers/plot_data_plane_wrapper.py b/metplus/wrappers/plot_data_plane_wrapper.py index 71d1b61ed..4f874c42f 100755 --- a/metplus/wrappers/plot_data_plane_wrapper.py +++ b/metplus/wrappers/plot_data_plane_wrapper.py @@ -12,10 +12,9 @@ import os -from ..util import met_util as util from ..util import time_util from . import CommandBuilder -from ..util import do_string_sub, remove_quotes +from ..util import do_string_sub, remove_quotes, skip_time, get_lead_sequence '''!@namespace PlotDataPlaneWrapper @brief Wraps the PlotDataPlane tool to plot data @@ -115,14 +114,14 @@ def run_at_time(self, input_dict): Args: @param input_dict dictionary containing timing information """ - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: self.clear() input_dict['lead'] = lead time_info = time_util.ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 2a0ec4373..071d71310 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -177,7 +177,7 @@ def create_c_dict(self): def get_command(self): return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - f" {self.infiles[0]} {self.get_output_path()}" + f' "{self.infiles[0]}" {self.get_output_path()}' f" {' '.join(self.args)}") def run_at_time_once(self, time_info): @@ -243,7 +243,7 @@ def set_command_line_arguments(self, time_info): """ # if more than 1 input file was found, add them with -point_obs for infile in self.infiles[1:]: - self.args.append(f'-point_obs {infile}') + self.args.append(f'-point_obs "{infile}"') if self.c_dict.get('GRID_INPUT_PATH'): grid_file = do_string_sub(self.c_dict['GRID_INPUT_PATH'], diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 9f5a1645c..3115b0c80 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -13,7 +13,6 @@ import os from ..util import getlistint -from ..util import met_util as util from ..util import time_util from ..util import do_string_sub from . import CompareGriddedWrapper diff --git a/metplus/wrappers/py_embed_ingest_wrapper.py b/metplus/wrappers/py_embed_ingest_wrapper.py index c59847ded..accd8ad1f 100755 --- a/metplus/wrappers/py_embed_ingest_wrapper.py +++ b/metplus/wrappers/py_embed_ingest_wrapper.py @@ -13,11 +13,10 @@ import os import re -from ..util import met_util as util from ..util import time_util from . import CommandBuilder from . import RegridDataPlaneWrapper -from ..util import do_string_sub +from ..util import do_string_sub, get_lead_sequence VALID_PYTHON_EMBED_TYPES = ['NUMPY', 'XARRAY', 'PANDAS'] @@ -132,7 +131,7 @@ def run_at_time(self, input_dict): generally contains 'now' (current) time and 'init' or 'valid' time """ # get forecast leads to loop over - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: # set forecast lead time in hours diff --git a/metplus/wrappers/reformat_gridded_wrapper.py b/metplus/wrappers/reformat_gridded_wrapper.py index da38e4f1d..9acb45859 100755 --- a/metplus/wrappers/reformat_gridded_wrapper.py +++ b/metplus/wrappers/reformat_gridded_wrapper.py @@ -12,8 +12,8 @@ import os -from ..util import met_util as util -from ..util import time_util +from ..util import get_lead_sequence, sub_var_list +from ..util import time_util, skip_time from . import CommandBuilder # pylint:disable=pointless-string-statement @@ -52,7 +52,7 @@ def run_at_time(self, input_dict): """ app_name_caps = self.app_name.upper() class_name = self.__class__.__name__[0: -7] - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) run_list = [] if self.config.getbool('config', 'FCST_'+app_name_caps+'_RUN', False): @@ -78,7 +78,7 @@ def run_at_time(self, input_dict): self.logger.info("Processing forecast lead " f"{time_info['lead_string']}") - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES')): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): self.logger.debug('Skipping run time') continue @@ -93,8 +93,8 @@ def run_at_time(self, input_dict): self.c_dict['CUSTOM_STRING'] = custom_string var_list_name = f'VAR_LIST_{to_run}' var_list = ( - util.sub_var_list(self.c_dict.get(var_list_name, ''), - time_info) + sub_var_list(self.c_dict.get(var_list_name, ''), + time_info) ) if not var_list: var_list = None diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index 1cf8de314..c58ebfc7c 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -12,12 +12,11 @@ import os -from ..util import met_util as util from ..util import time_util from ..util import do_string_sub from ..util import parse_var_list from ..util import get_process_list -from ..util import remove_quotes +from ..util import remove_quotes, split_level, format_level from . import ReformatGriddedWrapper # pylint:disable=pointless-string-statement @@ -173,7 +172,7 @@ def handle_output_file(self, time_info, field_info, data_type): @returns True if command should be run, False if it should not be run """ - _, level = util.split_level(field_info[f'{data_type.lower()}_level']) + _, level = split_level(field_info[f'{data_type.lower()}_level']) time_info['level'] = time_util.get_seconds_from_string(level, 'H') return self.find_and_check_output_file(time_info) @@ -255,7 +254,7 @@ def get_output_names(self, var_list, data_type): for field_info in var_list: input_name = field_info[f'{data_type.lower()}_name'] input_level = field_info[f'{data_type.lower()}_level'] - input_level = util.format_level(input_level) + input_level = format_level(input_level) output_name = f"{input_name}_{input_level}" output_names.append(output_name) diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index c1446c90e..503869948 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -191,7 +191,6 @@ def create_c_dict(self): # read any [FCST/OBS]_VAR_* variables if they are set c_dict['VAR_LIST'] = parse_var_list(self.config) - c_dict['MODEL_INFO_LIST'] = self._parse_model_info() # if MODEL_LIST was not set, populate it from the model info list @@ -666,6 +665,7 @@ def _build_stringsub_hours(self, list_name, config_dict, stringsub_dict): stringsub_dict[f'{generic_list}_end'] = ( stringsub_dict[f'{sub_name}_end'] ) + if (stringsub_dict[f'{generic_list}_beg'] == stringsub_dict[f'{generic_list}_end']): stringsub_dict[generic_list] = ( diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index ab8873f96..bec1e1a56 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -14,9 +14,8 @@ import datetime import re -from ..util import met_util as util from ..util import time_util -from ..util import do_string_sub +from ..util import do_string_sub, skip_time, get_lead_sequence from ..util import time_generator from . import CommandBuilder @@ -355,7 +354,7 @@ def run_at_time(self, input_dict): input_dict['custom'] = custom_string time_info = time_util.ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue @@ -426,7 +425,7 @@ def find_input_files(self, time_info): ) # set METPLUS_LEAD_LIST to list of forecast leads used - lead_seq = util.get_lead_sequence(self.config, time_info) + lead_seq = get_lead_sequence(self.config, time_info) if lead_seq != [0]: lead_list = [] for lead in lead_seq: diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index 5881f9eb3..12a4cad6a 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -12,11 +12,10 @@ import os -from ..util import met_util as util from ..util import time_util from . import CommandBuilder -from ..util import do_string_sub -from ..util import parse_var_list +from ..util import do_string_sub, skip_time, get_lead_sequence +from ..util import parse_var_list, sub_var_list '''!@namespace TCRMWWrapper @brief Wraps the TC-RMW tool @@ -212,7 +211,7 @@ def run_at_time(self, input_dict): time_info = time_util.ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue @@ -258,8 +257,7 @@ def set_data_field(self, time_info): @param time_info time dictionary to use for string substitution @returns True if field list could be built, False if not. """ - field_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], - time_info) + field_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) if not field_list: self.log_error("Could not get field information from config.") return False @@ -293,7 +291,7 @@ def find_input_files(self, time_info): self.c_dict['DECK_FILE'] = deck_file - lead_seq = util.get_lead_sequence(self.config, time_info) + lead_seq = get_lead_sequence(self.config, time_info) # get input files if self.c_dict['INPUT_FILE_LIST']: diff --git a/metplus/wrappers/usage_wrapper.py b/metplus/wrappers/usage_wrapper.py index d3fb8cf85..77c26b575 100644 --- a/metplus/wrappers/usage_wrapper.py +++ b/metplus/wrappers/usage_wrapper.py @@ -13,7 +13,7 @@ class UsageWrapper(CommandBuilder): def __init__(self, config, instance=None): self.app_name = 'Usage' super().__init__(config, instance=instance) - # get unique list of processes from met_util + # get unique list of processes self.available_processes = list(set(val for val in LOWER_TO_WRAPPER_NAME.values())) self.available_processes.sort() diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index 50384c019..32e50ac38 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -13,7 +13,6 @@ import os from datetime import datetime -from ..util import met_util as util from ..util import time_util from . import RuntimeFreqWrapper from ..util import do_string_sub diff --git a/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf b/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf new file mode 100644 index 000000000..fb9dd2054 --- /dev/null +++ b/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf @@ -0,0 +1,137 @@ +[config] + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.html +# +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = ASCII2NC, PointStat + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2022101609 +VALID_END = 2022101609 +VALID_INCREMENT = 1M + +LEAD_SEQ = 0 + +LOOP_ORDER = times + + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +ASCII2NC_INPUT_DIR = {INPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight +ASCII2NC_INPUT_TEMPLATE = *.txt + +ASCII2NC_OUTPUT_DIR = +ASCII2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/buoy_ASCII/buoy_{valid?fmt=%Y%m%d%H}.nc + +ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False + +ASCII2NC_FILE_WINDOW_BEGIN = 0 +ASCII2NC_FILE_WINDOW_END = 0 + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight +FCST_POINT_STAT_INPUT_TEMPLATE = gfswave.t06z.global.0p16.f003.grib2 + +OBS_POINT_STAT_INPUT_DIR = +OBS_POINT_STAT_INPUT_TEMPLATE = {ASCII2NC_OUTPUT_TEMPLATE} + +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/PointStat + + +### +# Field Info +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info +### + +POINT_STAT_ONCE_PER_FIELD = False + + +FCST_VAR1_NAME = WVHGT +FCST_VAR1_LEVELS = Z0 +BOTH_VAR1_THRESH = le3.0,ge4.0&&le6.0,ge8.0 +OBS_VAR1_NAME = WVHT +OBS_VAR1_LEVELS = L0 + + +### +# ASCII2NC Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc +### + +#LOG_ASCII2NC_VERBOSITY = 1 + +ASCII2NC_CONFIG_FILE = {PARM_BASE}/met_config/Ascii2NcConfig_wrapped + +ASCII2NC_INPUT_FORMAT = ndbc_standard + +ASCII2NC_MASK_GRID = +ASCII2NC_MASK_POLY = +ASCII2NC_MASK_SID = + +ASCII2NC_TIME_SUMMARY_FLAG = False +ASCII2NC_TIME_SUMMARY_RAW_DATA = False +ASCII2NC_TIME_SUMMARY_BEG = 000000 +ASCII2NC_TIME_SUMMARY_END = 235959 +ASCII2NC_TIME_SUMMARY_STEP = 300 +ASCII2NC_TIME_SUMMARY_WIDTH = 600 +ASCII2NC_TIME_SUMMARY_GRIB_CODES = 11, 204, 211 +ASCII2NC_TIME_SUMMARY_VAR_NAMES = +ASCII2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 +ASCII2NC_TIME_SUMMARY_VALID_FREQ = 0 +ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0 + +### +# PointStat Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat +### + +#LOG_POINT_STAT_VERBOSITY = 2 + +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OUTPUT_FLAG_FHO = +POINT_STAT_OUTPUT_FLAG_CTC = BOTH +POINT_STAT_OUTPUT_FLAG_CTS = BOTH +OBS_POINT_STAT_WINDOW_BEGIN = -1800 +OBS_POINT_STAT_WINDOW_END = 1800 + +POINT_STAT_OFFSETS = 0 + +MODEL = GFSv16 + +POINT_STAT_DESC = NDBC +OBTYPE = + +POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 + +POINT_STAT_MESSAGE_TYPE = NDBC_STANDARD + +POINT_STAT_MASK_GRID = FULL +POINT_STAT_MASK_POLY = +POINT_STAT_MASK_SID = + diff --git a/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf b/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf new file mode 100644 index 000000000..4ab01878a --- /dev/null +++ b/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf @@ -0,0 +1,227 @@ +[config] + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/met_tool_wrapper/PointStat/PointStat.html + +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = ASCII2NC, PCPCombine, PointStat + + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2022091423 +VALID_END = 2022091423 +VALID_INCREMENT = 1M + +LEAD_SEQ = 24H + + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + + +ASCII2NC_INPUT_DIR = +ASCII2NC_INPUT_TEMPLATE = "{PARM_BASE}/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip/read_cocorahs_point.py {INPUT_BASE}/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip/CoCoRaHS.{valid?fmt=%Y%m%d}.DailyPrecip.csv" + +ASCII2NC_OUTPUT_DIR = +ASCII2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/ASCII2NC/precip_{valid?fmt=%Y%m%d}_summary.nc + +ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False + +ASCII2NC_FILE_WINDOW_BEGIN = 0 +ASCII2NC_FILE_WINDOW_END = 0 + + +FCST_PCP_COMBINE_RUN = True + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip +FCST_PCP_COMBINE_INPUT_TEMPLATE = {lead?fmt=%HH} + +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/PCPCombine +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = fcst_24hr_precip.nc + + +FCST_POINT_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} +FCST_POINT_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +OBS_POINT_STAT_INPUT_DIR = {OUTPUT_BASE}/ASCII2NC +OBS_POINT_STAT_INPUT_TEMPLATE = precip_{valid?fmt=%Y%m%d}_summary.nc + +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/PointStat + +POINT_STAT_CLIMO_MEAN_INPUT_DIR = +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +POINT_STAT_CLIMO_STDEV_INPUT_DIR = +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +### +# ASCII2NC Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc +### + +#LOG_ASCII2NC_VERBOSITY = 1 +#ASCII2NC_CONFIG_FILE = + +ASCII2NC_WINDOW_BEGIN = 0 +ASCII2NC_WINDOW_END = 0 + +ASCII2NC_INPUT_FORMAT = python + +ASCII2NC_MASK_GRID = +ASCII2NC_MASK_POLY = +ASCII2NC_MASK_SID = + +ASCII2NC_TIME_SUMMARY_FLAG = False +ASCII2NC_TIME_SUMMARY_RAW_DATA = False +ASCII2NC_TIME_SUMMARY_BEG = 000000 +ASCII2NC_TIME_SUMMARY_END = 235959 +ASCII2NC_TIME_SUMMARY_STEP = 300 +ASCII2NC_TIME_SUMMARY_WIDTH = 600 +ASCII2NC_TIME_SUMMARY_GRIB_CODES = +ASCII2NC_TIME_SUMMARY_VAR_NAMES = APCP +ASCII2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 +ASCII2NC_TIME_SUMMARY_VALID_FREQ = 0 +ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0 + +### +# PCPCombine Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pcpcombine +### + +FCST_PCP_COMBINE_METHOD = USER_DEFINED + +FCST_PCP_COMBINE_COMMAND = -sum 00000000_000000 1 20220914_230000 24 {FCST_PCP_COMBINE_OUTPUT_DIR}/{FCST_PCP_COMBINE_OUTPUT_TEMPLATE} -pcpdir {FCST_PCP_COMBINE_INPUT_DIR} + +#LOG_PCP_COMBINE_VERBOSITY = 2 + +FCST_IS_PROB = false + +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB + +FCST_PCP_COMBINE_INPUT_ACCUMS = 1H +FCST_PCP_COMBINE_INPUT_NAMES = APCP +FCST_PCP_COMBINE_INPUT_LEVELS = L0 + +FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 + +### +# Field Info +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info +### + +POINT_STAT_ONCE_PER_FIELD = False + +#POINT_STAT_FCST_FILE_TYPE = +#POINT_STAT_OBS_FILE_TYPE = + +FCST_POINT_STAT_VAR1_NAME = APCP_24 +FCST_POINT_STAT_VAR1_LEVELS = L0 +#FCST_VAR1_THRESH = <=273, >273 +OBS_POINT_STAT_VAR1_NAME = TotalPrecipAmt +OBS_POINT_STAT_VAR1_LEVELS = L0 +#OBS_VAR1_THRESH = <=273, >273 +BOTH_POINT_STAT_VAR1_THRESH = <=6.35, <=12.7, <=25.4 + +### +# PointStat Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat +### + +#LOG_POINT_STAT_VERBOSITY = 2 + +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +#POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +#POINT_STAT_INTERP_TYPE_METHOD = BILIN +#POINT_STAT_INTERP_TYPE_WIDTH = 2 + +#POINT_STAT_OUTPUT_FLAG_FHO = +POINT_STAT_OUTPUT_FLAG_CTC = BOTH +POINT_STAT_OUTPUT_FLAG_CTS = BOTH +#POINT_STAT_OUTPUT_FLAG_MCTC = +POINT_STAT_OUTPUT_FLAG_MCTS = BOTH +POINT_STAT_OUTPUT_FLAG_CNT = BOTH +#POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VCNT = +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +#POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True +#POINT_STAT_CLIMO_CDF_DIRECT_PROB = + +#POINT_STAT_HSS_EC_VALUE = + +OBS_POINT_STAT_WINDOW_BEGIN = -82800 +OBS_POINT_STAT_WINDOW_END = 3600 + +POINT_STAT_OFFSETS = 0 + +MODEL = URMA + +POINT_STAT_DESC = CoCoRaHS +OBTYPE = + +POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 + +POINT_STAT_OUTPUT_PREFIX = + +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + +POINT_STAT_MASK_GRID = FULL +POINT_STAT_MASK_POLY = +POINT_STAT_MASK_SID = +#POINT_STAT_MASK_LLPNT = + +POINT_STAT_MESSAGE_TYPE = ADPSFC + +#POINT_STAT_HIRA_FLAG = +#POINT_STAT_HIRA_WIDTH = +#POINT_STAT_HIRA_VLD_THRESH = +#POINT_STAT_HIRA_COV_THRESH = +#POINT_STAT_HIRA_SHAPE = +#POINT_STAT_HIRA_PROB_CAT_THRESH = + +#POINT_STAT_MESSAGE_TYPE_GROUP_MAP = diff --git a/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip/read_cocorahs_point.py b/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip/read_cocorahs_point.py new file mode 100644 index 000000000..266fbf853 --- /dev/null +++ b/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip/read_cocorahs_point.py @@ -0,0 +1,101 @@ +from __future__ import print_function + +import pandas as pd +import os +import sys +import datetime +import numpy as np + +######################################################################## + +print("Python Script:\t" + repr(sys.argv[0])) + + ## + ## input file specified on the command line + ## load the data into the numpy array + ## + +if len(sys.argv) == 2: + # Read the input file as the first argument + input_file = os.path.expandvars(sys.argv[1]) + try: + print("Input File:\t" + repr(input_file)) + + # Read and format the input 11-column observations: + # (1) string: Message_Type + # (2) string: Station_ID + # (3) string: Valid_Time(YYYYMMDD_HHMMSS) + # (4) numeric: Lat(Deg North) + # (5) numeric: Lon(Deg East) + # (6) numeric: Elevation(msl) + # (7) string: Var_Name(or GRIB_Code) + # (8) numeric: Level + # (9) numeric: Height(msl or agl) + # (10) string: QC_String + # (11) numeric: Observation_Value + + holder = pd.read_csv(input_file,usecols=['ObservationDate','ObservationTime','StationNumber','Latitude','Longitude','TotalPrecipAmt']) + #convert time stamps to MET friendly timestamps + dat = holder['ObservationDate'].values.tolist() + tim = holder['ObservationTime'].values.tolist() + vld = [] + + #grab the existing values from the csv and get them into list form + sid = holder['StationNumber'].values.tolist() + lat = holder['Latitude'].values.tolist() + lon = holder['Longitude'].values.tolist() + obs = holder['TotalPrecipAmt'].values.tolist() + + + #this loop will result in an error if every date doesn't have a time (HHMM) associated with it + #its purpose is threefold: first is to construct time strings that MET can handle + #second is to strip the whitespaces that exist in the sid and obs list items + #and third is to convert the obs items into floats + for i in range(len(dat)): + #convert the times + mush = dat[i]+tim[i] + dt = datetime.datetime.strptime(mush, '%b %d %Y %H:%M UT') + vld.append(dt.strftime('%Y%m%d_%H%M%S')) + #strip the whitespace + sid[i] = sid[i].strip() + #strip whitespace + obs[i] = obs[i].strip() + #and convert. need to check if value is 'T' + #if it is, set the value to 0 + if not obs[i].isalpha(): + obs[i] = float(obs[i].strip()) + else: + obs[i] = float(0.0) + + + #create dummy lists for the message type, elevation, variable name, level, height, and qc string + #numpy is more efficient at creating the lists, but need to convert to Pythonic lists + typ = np.full(len(vld),'ADPSFC').tolist() + elv = np.zeros(len(vld)).tolist() + var = np.full(len(vld),'TotalPrecipAmt').tolist() + lvl = np.full(len(vld),1013.25).tolist() + hgt = np.zeros(len(vld),dtype=int).tolist() + qc = np.full(len(vld),'NA').tolist() + + + + #Now to put the lists into a list of lists + #start by creating a list of tuples + #then convert the tuples to lists + + + l_tuple = list(zip(typ,sid,vld,lat,lon,elv,var,lvl,hgt,qc,obs)) + point_data = [list(ele) for ele in l_tuple] + + + + print("Data Length:\t" + repr(len(point_data))) + print("Data Type:\t" + repr(type(point_data))) + except NameError: + print("Can't find the input file") +else: + print("ERROR: read_ascii_point.py -> Must specify exactly one input file.") + sys.exit(1) + +######################################################################## + diff --git a/scripts/docker/docker_data/Dockerfile b/scripts/docker/docker_data/Dockerfile deleted file mode 100644 index b5889ec1f..000000000 --- a/scripts/docker/docker_data/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM centos:7 -MAINTAINER George McCabe - -# Required arguments -ARG TARFILE_URL -ARG MOUNTPT - -# Check that TARFILE_URL is defined. -RUN if [ "x${TARFILE_URL}" == "x" ]; then \ - echo "ERROR: TARFILE_URL undefined! Rebuild with \"--build-arg TARFILE_URL={One or more URL's}\""; \ - exit 1; \ - fi - -# Check that MOUNTPT is defined. -RUN if [ "x${MOUNTPT}" == "x" ]; then \ - echo "ERROR: MOUNTPT undefined! Rebuild with \"--build-arg MOUNTPT={VOLUME mount directory}\""; \ - exit 1; \ - fi - -ARG DATA_DIR=/data/input/METplus_Data -ENV CASE_DIR=${DATA_DIR} -RUN mkdir -p ${CASE_DIR} - -RUN for URL in `echo ${TARFILE_URL} | tr "," " "`; do \ - echo "Downloading: ${URL}"; \ - curl -SL ${URL} | tar -xzC ${CASE_DIR}; \ - done - -# Define the volume mount point -VOLUME ${MOUNTPT} - -USER root -CMD ["true"] diff --git a/scripts/docker/docker_data/build_docker_images.sh b/scripts/docker/docker_data/build_docker_images.sh deleted file mode 100755 index 5f751e796..000000000 --- a/scripts/docker/docker_data/build_docker_images.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash -# -# Build METplus-Data Docker images for sample data tarballs -#======================================================================= -# -# This script pulls sample data tarfiles from: -# https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data/ -# -# Where is specified using the required "-pull" command line -# option. It searches for tarfiles in that directory named -# "sample_data-.tgz". When the "-data" option is used, -# it only processes the specified list of datasets. Otherwise, it -# processes all datasets in that directory. For each dataset, it builds -# a Docker data volume. -# -# Each directory must contain a file named -# "volume_mount_directories". Each line of that file is formatted as: -# : -# For example, "s2s:model_applications/s2s" indicates the directory -# that should be mounted for the s2s dataset. -# -# When "-union" is used, it also builds a Docker data volume for all -# datasets in that directory. When "-push" is used, it pushes the -# resulting images to the specified DockerHub repository. -# -# See Usage statement below. -# -#======================================================================= - -# Constants -SCRIPT_DIR=$(dirname $0) -PULL_URL="https://dtcenter.ucar.edu/dfiles/code/METplus/METplus_Data" -MOUNTPT_FILE="volume_mount_directories" -MOUNTPT_BASE="/data/input/METplus_Data" - -# -# Usage statement for this script -# -function usage { - cat << EOF - -Usage: build_docker_images.sh - -pull version - [-data list] - [-union] - [-all] - [-push repo] - [-help] - - where: - "-pull version" defines the version of the datasets to be pulled (required). - "-data list" overrides the use of all datasets for this version with a comma-separated list (optional). - "-union" also creates one data volume with all datasets for this version (optional). - "-all" create data volumes from all available datasets for this version (optional). - "-push repo" pushes the images to the specified DockerHub repository (optional). - "-help" prints the usage statement. - - e.g. Pull from ${PULL_URL}/ - Push to DockerHub :- - -EOF - - exit 1 -} - -# -# Command runner utility function -# -function run_command { - echo "RUNNING: $*" - $* - error=$? - if [ ${error} -ne 0 ]; then - echo "ERROR:" - echo "ERROR: '$*' exited with status = ${error}" - echo "ERROR:" - exit ${error} - fi -} - -# Defaults for command line options -DO_UNION=0 -DO_PUSH=0 -DO_ALL=0 - -# Default for checking if using tagged version -TAGGED_VERSION=0 - -# Parse command line options -while true; do - case "$1" in - - pull | -pull | --pull ) - VERSION=$2 - echo "Will pull data from ${PULL_URL}/${VERSION}" - shift 2;; - - data | -data | --data ) - if [ -z ${PULL_DATA+x} ]; then - PULL_DATA=$2 - else - PULL_DATA="${PULL_DATA},$2" - fi - shift 2;; - - union | -union | --union ) - DO_UNION=1 - echo "Will create a data volume containing all input datasets." - shift;; - - all | -all | --all ) - DO_ALL=1 - echo "Will create a data volume for each available input dataset." - shift;; - - push | -push | --push ) - DO_PUSH=1 - PUSH_REPO=$2 - if [ -z ${PUSH_REPO} ]; then - echo "ERROR: Must provide push repository after -push" - usage - fi - echo "Will push images to DockerHub ${PUSH_REPO}." - shift 2;; - - help | -help | --help ) - usage - shift;; - - -* ) - echo "ERROR:" - echo "ERROR: Unsupported option: $1" - echo "ERROR:" - usage;; - - * ) - break;; - - esac -done - -# Check that the version has been specified -if [ -z ${VERSION+x} ]; then - echo "ERROR:" - echo "ERROR: The '-pull' option is required!" - echo "ERROR:" - usage -fi - -# use VERSION in the Docker image tag unless using a tagged version -DOCKER_VERSION=${VERSION} - -# check if using a tagged version (e.g v4.0) -# remove v from version if tagged version -if [[ ${VERSION} =~ ^v[0-9.]+$ ]]; then - TAGGED_VERSION=1 - DOCKER_VERSION=${VERSION:1} -fi - - -# Define the target repository if necessary -if [ -z ${PUSH_REPO+x} ]; then - - # Push tagged versions (e.g. v4.0) to metplus-data - # and all others to metplus-data-dev - if [ ${TAGGED_VERSION} == 1 ]; then - PUSH_REPO="dtcenter/metplus-data" - else - PUSH_REPO="dtcenter/metplus-data-dev" - fi -fi - -# Print the datasets to be processed -if [ -z ${PULL_DATA+x} ]; then - echo "Will process all available datasets." -else - echo "Will process the following datasets: ${PULL_DATA}" -fi - -# Get list of available tarfiles -TARFILE_LIST=`curl -s ${PULL_URL}/${VERSION}/ | tr "<>" "\n" | egrep sample_data | egrep -v href` - -if [[ ${TARFILE_LIST} == "" ]]; then - echo "ERROR:" - echo "ERROR: No tarfiles found in ${PULL_URL}/${VERSION}" - echo "ERROR:" - exit 1 -fi - -# Build separate image for each tarfile -for TARFILE in $TARFILE_LIST; do - - # Build a list of all URL's - CUR_URL="${PULL_URL}/${VERSION}/${TARFILE}" - - if [ -z ${URL_LIST+x} ]; then - URL_LIST=${CUR_URL} - else - URL_LIST="${URL_LIST},${CUR_URL}" - fi - - # Parse the current dataset name - CUR_DATA=`echo $TARFILE | cut -d'-' -f2 | sed 's/.tgz//g'` - - if [ -z ${PULL_DATA+x} ] || [ `echo ${PULL_DATA} | grep ${CUR_DATA}` ] || [ ${DO_ALL} == 1 ]; then - echo "Processing \"${TARFILE}\" ..." - else - echo "Skipping \"${TARFILE}\" since \"${CUR_DATA}\" was not requested in \"-data\"." - continue - fi - - # Define the docker image name - IMGNAME="${PUSH_REPO}:${DOCKER_VERSION}-${CUR_DATA}" - - # Determine the mount point - MOUNTPT_URL="${PULL_URL}/${VERSION}/${MOUNTPT_FILE}" - MOUNTPT=${MOUNTPT_BASE}/`curl -s ${MOUNTPT_URL} | grep "${CUR_DATA}:" | cut -d':' -f2` - - if [[ ${MOUNTPT} == "" ]]; then - echo "ERROR:" - echo "ERROR: No entry found for \"${CUR_DATA}\" found in ${MOUNTPT_URL}!" - echo "ERROR:" - exit 1 - fi - - echo - echo "Building image ... ${IMGNAME}" - echo - - run_command docker build -t ${IMGNAME} ${SCRIPT_DIR} \ - --build-arg TARFILE_URL=${CUR_URL} \ - --build-arg MOUNTPT=${MOUNTPT} - - if [ ${DO_PUSH} == 1 ]; then - echo - echo "Pushing image ... ${IMGNAME}" - echo - # if DOCKER_USERNAME is set, then run docker login - if [ ! -z ${DOCKER_USERNAME+x} ]; then - echo "Logging into Docker ..." - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - fi - run_command docker push ${IMGNAME} - - fi - -done - -# -# Build one image for all tarfiles -# - -if [ ${DO_UNION} == 1 ]; then - - IMGNAME="${PUSH_REPO}:${DOCKER_VERSION}" - MOUNTPT="${MOUNTPT_BASE}" - - run_command docker build -t ${IMGNAME} ${SCRIPT_DIR} \ - --build-arg TARFILE_URL=${URL_LIST} \ - --build-arg MOUNTPT=${MOUNTPT} - - if [ ${DO_PUSH} == 1 ]; then - echo - echo "Pushing image ... ${IMGNAME}" - echo - - run_command docker push ${IMGNAME} - - fi -fi - diff --git a/scripts/docker/docker_data/hooks/build b/scripts/docker/docker_data/hooks/build deleted file mode 100644 index 5adac2111..000000000 --- a/scripts/docker/docker_data/hooks/build +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./build_docker_images.sh -pull v${DOCKER_TAG} -union -push dtcenter/metplus-data diff --git a/ush/run_metplus.py b/ush/run_metplus.py index ecb0686ed..39149b7e4 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -27,13 +27,13 @@ from metplus.util import metplus_check from metplus.util import pre_run_setup, run_metplus, post_run_cleanup -from metplus.util import get_process_list from metplus import __version__ as metplus_version '''!@namespace run_metplus Main script the processes all the tasks in the PROCESS_LIST ''' + def main(): """!Main program. METplus script that invokes the necessary Python scripts @@ -49,13 +49,11 @@ def main(): "This script name will be removed in a future version.") config.logger.warning(msg) - # Use config object to get the list of processes to call - process_list = get_process_list(config) - - total_errors = run_metplus(config, process_list) + total_errors = run_metplus(config) post_run_cleanup(config, 'METplus', total_errors) + def usage(): """! How to call this script. """ @@ -73,6 +71,7 @@ def usage(): '''%(filename)) sys.exit(2) + def get_config_inputs_from_command_line(): """! Read command line arguments. Pull out configuration files and configuration variable overrides. Display @@ -121,6 +120,7 @@ def get_config_inputs_from_command_line(): return config_inputs + if __name__ == "__main__": try: produtil.setup.setup(send_dbn=False, jobname='run-METplus')