From f7c3ea9c5d33cba7d9eaff87163010acda49ac5a Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 9 Nov 2021 09:28:30 -0700 Subject: [PATCH 01/26] per #1203, created files for ioda2nc wrapper -- needs to be updated --- metplus/util/doc_util.py | 1 + metplus/wrappers/ioda2nc_wrapper.py | 256 ++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100755 metplus/wrappers/ioda2nc_wrapper.py diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index e72337ad6..5bf834d86 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -17,6 +17,7 @@ 'gfdltracker': 'GFDLTracker', 'griddiag': 'GridDiag', 'gridstat': 'GridStat', + 'ioda2nc': 'IODA2NC', 'makeplots': 'MakePlots', 'metdbload': 'METDbLoad', 'mode': 'MODE', diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py new file mode 100755 index 000000000..dbcf89fbc --- /dev/null +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -0,0 +1,256 @@ +""" +Program Name: ioda2nc_wrapper.py +Contact(s): George McCabe +Abstract: Builds commands to run ioda2nc +""" + +import os + +from ..util import met_util as util +from ..util import time_util +from . import CommandBuilder +from ..util import do_string_sub + +'''!@namespace IODA2NCWrapper +@brief Wraps the IODA2NC tool to reformat ascii format to NetCDF +@endcode +''' + + +class IODA2NCWrapper(CommandBuilder): + + WRAPPER_ENV_VAR_KEYS = [ + 'METPLUS_TIME_SUMMARY_DICT', + ] + + def __init__(self, config, instance=None, config_overrides=None): + self.app_name = "ioda2nc" + self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), + self.app_name) + super().__init__(config, + instance=instance, + config_overrides=config_overrides) + + def create_c_dict(self): + c_dict = super().create_c_dict() + c_dict['VERBOSITY'] = self.config.getstr('config', + 'LOG_IODA2NC_VERBOSITY', + c_dict['VERBOSITY']) + c_dict['ALLOW_MULTIPLE_FILES'] = True + + # IODA2NC config file is optional, so + # don't provide wrapped config file name as default value + c_dict['CONFIG_FILE'] = self.get_config_file() + + c_dict['ASCII_FORMAT'] = self.config.getstr('config', + 'IODA2NC_INPUT_FORMAT', + '') + c_dict['MASK_GRID'] = self.config.getstr('config', + 'IODA2NC_MASK_GRID', + '') + c_dict['MASK_POLY'] = self.config.getstr('config', + 'IODA2NC_MASK_POLY', + '') + c_dict['MASK_SID'] = self.config.getstr('config', + 'IODA2NC_MASK_SID', + '') + c_dict['OBS_INPUT_DIR'] = self.config.getdir('IODA2NC_INPUT_DIR', + '') + c_dict['OBS_INPUT_TEMPLATE'] = ( + self.config.getraw('filename_templates', + 'IODA2NC_INPUT_TEMPLATE') + ) + if not c_dict['OBS_INPUT_TEMPLATE']: + self.log_error("IODA2NC_INPUT_TEMPLATE required to run") + + c_dict['OUTPUT_DIR'] = self.config.getdir('IODA2NC_OUTPUT_DIR', '') + c_dict['OUTPUT_TEMPLATE'] = ( + self.config.getraw('filename_templates', + 'IODA2NC_OUTPUT_TEMPLATE') + ) + + # MET config variables + self.handle_time_summary_dict(c_dict, + ['TIME_SUMMARY_GRIB_CODES', + 'TIME_SUMMARY_VAR_NAMES', + 'TIME_SUMMARY_TYPES'] + ) + + # handle file window variables + for edge in ['BEGIN', 'END']: + file_window = ( + self.config.getseconds('config', + f'IODA2NC_FILE_WINDOW_{edge}', + '') + ) + if file_window == '': + file_window = ( + self.config.getseconds('config', + f'OBS_FILE_WINDOW_{edge}', + 0) + ) + + c_dict[f'OBS_FILE_WINDOW_{edge}'] = file_window + + return c_dict + + def set_environment_variables(self, time_info): + """!Set environment variables that will be read by the MET config file. + Reformat as needed. Print list of variables that were set and their values. + Args: + @param time_info dictionary containing timing info from current run""" + # set environment variables needed for MET application + self.add_env_var('TIME_SUMMARY_FLAG', + self.c_dict['TIME_SUMMARY_FLAG']) + self.add_env_var('TIME_SUMMARY_RAW_DATA', + self.c_dict['TIME_SUMMARY_RAW_DATA']) + self.add_env_var('TIME_SUMMARY_BEG', + self.c_dict['TIME_SUMMARY_BEG']) + self.add_env_var('TIME_SUMMARY_END', + self.c_dict['TIME_SUMMARY_END']) + self.add_env_var('TIME_SUMMARY_STEP', + self.c_dict['TIME_SUMMARY_STEP']) + self.add_env_var('TIME_SUMMARY_WIDTH', + self.c_dict['TIME_SUMMARY_WIDTH']) + self.add_env_var('TIME_SUMMARY_GRIB_CODES', + self.c_dict['TIME_SUMMARY_GRIB_CODES']) + self.add_env_var('TIME_SUMMARY_VAR_NAMES', + self.c_dict['TIME_SUMMARY_VAR_NAMES']) + self.add_env_var('TIME_SUMMARY_TYPES', + self.c_dict['TIME_SUMMARY_TYPES']) + self.add_env_var('TIME_SUMMARY_VALID_FREQ', + self.c_dict['TIME_SUMMARY_VALID_FREQ']) + self.add_env_var('TIME_SUMMARY_VALID_THRESH', + self.c_dict['TIME_SUMMARY_VALID_THRESH']) + + # set user environment variables + super().set_environment_variables(time_info) + + def get_command(self): + cmd = self.app_path + + # don't run if no input or output files were found + if not self.infiles: + self.log_error("No input files were found") + return + + if self.outfile == "": + self.log_error("No output file specified") + return + + # add input files + for infile in self.infiles: + cmd += ' ' + infile + + # add output path + out_path = self.get_output_path() + cmd += ' ' + out_path + + parent_dir = os.path.dirname(out_path) + if parent_dir == '': + self.log_error('Must specify path to output file') + return None + + # create full output dir if it doesn't already exist + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + + # add arguments + cmd += ''.join(self.args) + + # add verbosity + cmd += ' -v ' + self.c_dict['VERBOSITY'] + return cmd + + def run_at_time(self, input_dict): + """! Runs the MET application for a given run time. This function + loops over the list of forecast leads and runs the application for + each. + Args: + @param input_dict dictionary containing timing information + """ + lead_seq = util.get_lead_sequence(self.config, input_dict) + for lead in lead_seq: + self.clear() + input_dict['lead'] = lead + + time_info = time_util.ti_calculate(input_dict) + + if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + self.logger.debug('Skipping run time') + continue + + for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: + if custom_string: + self.logger.info(f"Processing custom string: {custom_string}") + + time_info['custom'] = custom_string + + self.run_at_time_once(time_info) + + def run_at_time_once(self, time_info): + """! Process runtime and try to build command to run ioda2nc + Args: + @param time_info dictionary containing timing information + """ + # get input files + if self.find_input_files(time_info) is None: + return + + # get output path + if not self.find_and_check_output_file(time_info): + return + + # get other configurations for command + self.set_command_line_arguments(time_info) + + # set environment variables if using config file + self.set_environment_variables(time_info) + + # build command and run + cmd = self.get_command() + if cmd is None: + self.log_error("Could not generate command") + return + + self.build() + + def find_input_files(self, time_info): + # if using python embedding input, don't check if file exists, + # just substitute time info and add to input file list + if self.c_dict['ASCII_FORMAT'] == 'python': + filename = do_string_sub(self.c_dict['OBS_INPUT_TEMPLATE'], + **time_info) + self.infiles.append(filename) + return self.infiles + + # get list of files even if only one is found (return_list=True) + obs_path = self.find_obs(time_info, var_info=None, return_list=True) + if obs_path is None: + return None + + self.infiles.extend(obs_path) + return self.infiles + + def set_command_line_arguments(self, time_info): + # add input data format if set + if self.c_dict['ASCII_FORMAT']: + self.args.append(" -format {}".format(self.c_dict['ASCII_FORMAT'])) + + # add config file - passing through do_string_sub to get custom string if set + if self.c_dict['CONFIG_FILE']: + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], + **time_info) + self.args.append(f" -config {config_file}") + + # add mask grid if set + if self.c_dict['MASK_GRID']: + self.args.append(" -mask_grid {}".format(self.c_dict['MASK_GRID'])) + + # add mask poly if set + if self.c_dict['MASK_POLY']: + self.args.append(" -mask_poly {}".format(self.c_dict['MASK_POLY'])) + + # add mask SID if set + if self.c_dict['MASK_SID']: + self.args.append(" -mask_sid {}".format(self.c_dict['MASK_SID'])) From 33345531bf7e8da57ea7465d6a4346e4b448d6f6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 10 Nov 2021 14:18:38 -0700 Subject: [PATCH 02/26] added info to contrib guide instructions for creating a new wrapper --- docs/Contributors_Guide/create_wrapper.rst | 64 ++++++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/Contributors_Guide/create_wrapper.rst b/docs/Contributors_Guide/create_wrapper.rst index 63ffb66c0..e051a0421 100644 --- a/docs/Contributors_Guide/create_wrapper.rst +++ b/docs/Contributors_Guide/create_wrapper.rst @@ -26,11 +26,13 @@ In metplus/util/doc_util.py, add entries to the LOWER_TO_WRAPPER_NAME dictionary so that the wrapper can be found in the PROCESS_LIST even if it is formatted differently. The key should be the wrapper name in all lower-case letters without any underscores. The value should be the class name -of the wrapper without the "Wrapper" suffix. Examples:: +of the wrapper without the "Wrapper" suffix. Add the new entry in the location +to preserve alphabetical order so it is easier for other developers to find +it. Examples:: - 'newtool': 'NewTool', 'ascii2nc': 'ASCII2NC', 'ensemblestat': 'EnsembleStat', + 'newtool': 'NewTool', The name of a tool can be formatted in different ways depending on the context. For example, the MET tool PCPCombine is written as Pcp-Combine in the MET @@ -59,6 +61,9 @@ Wrapper Components Open the wrapper file for editing the new class. +Naming +^^^^^^ + Rename the class to match the wrapper's class from the above sections. Most wrappers should be a sub-class of the CommandBuilder wrapper:: @@ -67,19 +72,28 @@ Most wrappers should be a sub-class of the CommandBuilder wrapper:: The text 'CommandBuilder' in parenthesis makes NewToolWrapper a subclass of CommandBuilder. +Find and replace can be used to rename all instances of the wrapper name in +the file. For example, to create IODA2NC wrapper from ASCII2NC, replace +**ascii2nc** with **ioda2nc** and **ASCII2NC** with **IODA2NC**. +To create EnsembleStat wrapper from GridStat, replace +**grid_stat** with **ensemble_stat** and +**GridStat** with **EnsembleStat**. + +Parent Class +^^^^^^^^^^^^ + If the new tool falls under one of the existing tool categories, then you can make the tool a subclass of one of those classes. This should only be done if the functions in the parent class are needed by the new wrapper. If you are unsure, then use CommandBuilder. -Refer to the :ref:`basic_components_of_wrappers` section of the Contributor's -Guide for more information on what should be added. - Init Function ^^^^^^^^^^^^^ Modify the init function to initialize NewTool from its base class to set the self.app_name variable to name of the application. +If the application is a MET tool, then set self.app_path to the full path +of the tool under **MET_BIN_DIR**. See the Basic Components :ref:`bc_init_function` section for more information:: def __init__(self, config, instance=None, config_overrides=None): @@ -90,6 +104,43 @@ See the Basic Components :ref:`bc_init_function` section for more information:: instance=instance, config_overrides=config_overrides) +Read Configuration Variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The create_c_dict function is called during the initialization step of each +wrapper. It is where values from the self.config object are read. +The values are stored in the **c_dict** variable that is referenced +throughout the wrapper execution via self.c_dict. + +The function should always start with a call to the parent class' +implementation of the function to read/set any variables that are common to +all wrappers:: + + c_dict = super().create_c_dict() + +The function should also always return the c_dict variable:: + + return c_dict + +File Input/Output +""""""""""""""""" + +METplus configuration variables that end with _DIR and _TEMPLATE are used +to define the criteria to search for input files. + +Allow Multiple Files +"""""""""""""""""""" + +If the application can take more than one file as input for a given category +(i.e. FCST, OBS, ENS, etc.) then ALLOW_MULTIPLE_FILES must be set to True:: + + c_dict['ALLOW_MULTIPLE_FILES'] = True + +This is set to False by default in CommandBuilder's create_c_dict function. +If it is set to False and a list of files are found for an input +(using wildcards or a list of files in the METplus config template variable) +then the wrapper will produce an error and not build the command. + Run Functions ^^^^^^^^^^^^^ @@ -182,6 +233,9 @@ Your use case/example configuration file is located in a directory structure lik Note the documentation file is in METplus/docs while the use case conf file is in METplus/parm +Refer to the :ref:`basic_components_of_wrappers` section of the Contributor's +Guide for more information on what should be added. + Documentation ------------- From ec155357950f4c6f6b8b801ce474b07d2db41943 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:43:28 -0700 Subject: [PATCH 03/26] Renamed handle_time_summary_dict function to handle_time_summary_legacy to handle pre v4.0 variables used in wrapped MET config files for pb2nc and ascii2nc. Implemented new handle_time_summary_dict that is much more simple --- metplus/wrappers/ascii2nc_wrapper.py | 10 +++---- metplus/wrappers/command_builder.py | 40 ++++++++++++++++++++++++++-- metplus/wrappers/pb2nc_wrapper.py | 2 +- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 23d7c53e9..01e675833 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -76,11 +76,11 @@ def create_c_dict(self): ) # MET config variables - self.handle_time_summary_dict(c_dict, - ['TIME_SUMMARY_GRIB_CODES', - 'TIME_SUMMARY_VAR_NAMES', - 'TIME_SUMMARY_TYPES'] - ) + self.handle_time_summary_legacy(c_dict, + ['TIME_SUMMARY_GRIB_CODES', + 'TIME_SUMMARY_VAR_NAMES', + 'TIME_SUMMARY_TYPES'] + ) # handle file window variables for edge in ['BEGIN', 'END']: diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 64bdc9c25..e1df9ef2d 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1984,7 +1984,43 @@ def get_env_var_value(self, env_var_name, read_dict=None, item_type=None): return mask_value.split('=', 1)[1].rstrip(';').strip() - def handle_time_summary_dict(self, c_dict, remove_bracket_list=None): + def handle_time_summary_dict(self): + """! Read METplusConfig variables for the MET config time_summary + dictionary and format values into an environment variable + METPLUS_TIME_SUMMARY_DICT that is referenced in the wrapped MET + config files. + """ + self.handle_met_config_dict('time_summary', { + 'flag': 'bool', + 'raw_data': 'bool', + 'beg': 'string', + 'end': 'string', + 'step': 'int', + 'width': 'string', + 'grib_code': ('list', 'remove_quotes,allow_empty', None, + ['TIME_SUMMARY_GRIB_CODES']), + 'obs_var': ('list', 'allow_empty', None, + ['TIME_SUMMARY_VAR_NAMES']), + 'type': ('list', 'allow_empty', None, ['TIME_SUMMARY_TYPES']), + 'vld_freq': ('int', None, None, ['TIME_SUMMARY_VALID_FREQ']), + 'vld_thresh': ('float', None, None, ['TIME_SUMMARY_VALID_THRESH']), + }) + + def handle_time_summary_legacy(self, c_dict, remove_bracket_list=None): + """! Read METplusConfig variables for the MET config time_summary + dictionary and format values into environment variable + METPLUS_TIME_SUMMARY_DICT as well as other environment variables + that contain individuals items of the time_summary dictionary + that were referenced in wrapped MET config files prior to METplus 4.0. + Developer note: If we discontinue support for legacy wrapped MET + config files + + @param c_dict dictionary to store time_summary item values + @param remove_bracket_list (optional) list of items that need the + square brackets around the value removed because the legacy (pre 4.0) + wrapped MET config includes square braces around the environment + variable. + """ tmp_dict = {} app = self.app_name.upper() self.set_met_config_bool(tmp_dict, @@ -2053,7 +2089,7 @@ def handle_time_summary_dict(self, c_dict, remove_bracket_list=None): time_summary = self.format_met_config_dict(tmp_dict, 'time_summary', - keys=None) + keys=None) self.env_var_dict['METPLUS_TIME_SUMMARY_DICT'] = time_summary # set c_dict values to support old method of setting env vars diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 2e8241c11..891fc367b 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -111,7 +111,7 @@ def create_c_dict(self): 'METPLUS_OBS_BUFR_VAR', allow_empty=True) - self.handle_time_summary_dict(c_dict) + self.handle_time_summary_legacy(c_dict) self.handle_file_window_variables(c_dict, dtypes=['OBS']) From fea44b967082a488c0d43cbd0282df39d24a33fa Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:16:31 -0700 Subject: [PATCH 04/26] if metplus_configs argument is not set, use _ as a default config variable to check --- metplus/wrappers/command_builder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index e1df9ef2d..08290c6ca 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -2342,6 +2342,11 @@ def add_met_config(self, **kwargs): in order of precedence (first variable is used if it is set, otherwise 2nd variable is used if set, etc.) """ + # if metplus_configs is not provided, use _ + if not kwargs.get('metplus_configs'): + kwargs['metplus_configs'] = [ + f"{self.app_name}_{kwargs.get('name')}".upper() + ] item = met_config(**kwargs) output_dict = kwargs.get('output_dict') self.handle_met_config_item(item, output_dict) From 5017a7d445dd3eb1ecae8b724d7e083a87e04ed2 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:24:14 -0700 Subject: [PATCH 05/26] don't read verbosity because it is already read in LoopTimesWrapper --- metplus/wrappers/gen_ens_prod_wrapper.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 3c2a4367d..00c7c58e7 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -59,10 +59,6 @@ def __init__(self, config, instance=None, config_overrides=None): def create_c_dict(self): c_dict = super().create_c_dict() - c_dict['VERBOSITY'] = self.config.getstr('config', - 'LOG_GEN_ENS_PROD_VERBOSITY', - c_dict['VERBOSITY']) - # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file( 'GenEnsProdConfig_wrapped' From 67377671ddf335a8e5d8cf6b39e74b77aa706be6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:27:54 -0700 Subject: [PATCH 06/26] read verbosity as integer instead of string --- metplus/wrappers/runtime_freq_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 8e0288529..eccc0604a 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -46,7 +46,7 @@ def create_c_dict(self): app_name_upper = self.app_name.upper() c_dict['VERBOSITY'] = ( - self.config.getstr('config', + self.config.getint('config', f'LOG_{app_name_upper}_VERBOSITY', c_dict['VERBOSITY']) ) From 736aaa72832797f63694874b93453f4f35afbfe1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:28:12 -0700 Subject: [PATCH 07/26] added wrapper MET config file file ioda2nc --- parm/met_config/IODA2NCConfig_wrapped | 116 ++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 parm/met_config/IODA2NCConfig_wrapped diff --git a/parm/met_config/IODA2NCConfig_wrapped b/parm/met_config/IODA2NCConfig_wrapped new file mode 100644 index 000000000..964b6c9d3 --- /dev/null +++ b/parm/met_config/IODA2NCConfig_wrapped @@ -0,0 +1,116 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA2NC configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// IODA message type +// +// message_type = [ +${METPLUS_MESSAGE_TYPE} + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +// message_type_group_map = [ +${METPLUS_MESSAGE_TYPE_GROUP_MAP} + +// +// Mapping of input IODA message types to output message types +// +// message_type_map = [ +${METPLUS_MESSAGE_TYPE_MAP} + +// +// IODA station ID +// +// station_id = [ +${METPLUS_STATION_ID} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +// obs_window = { +${METPLUS_OBS_WINDOW_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +// mask = { +${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +// elevation_range = { +${METPLUS_ELEVATION_RANGE_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Vertical levels to retain +// +// level_range = { +${METPLUS_LEVEL_RANGE_DICT} + +/////////////////////////////////////////////////////////////////////////////// + +// +// IODA variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty or 'all', process all available variables. +// +// obs_var = [ +${METPLUS_OBS_VAR} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +// obs_name_map = [ +${METPLUS_OBS_NAME_MAP} + +// +// Default mapping for Metadata. +// +// metadata_map = [ +${METPLUS_METADATA_MAP} + +// missing_thresh = [ +${METPLUS_MISSING_THRESH} + +//////////////////////////////////////////////////////////////////////////////// + +// quality_mark_thresh = +${METPLUS_QUALITY_MARK_THRESH} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +// time_summary = { +${METPLUS_TIME_SUMMARY_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +//tmp_dir = "/tmp"; +//version = "V10.0"; + +//////////////////////////////////////////////////////////////////////////////// + +${METPLUS_MET_CONFIG_OVERRIDES} From 9afb47fec9d793371be1a71f5260e8f7bf3fb717 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:28:32 -0700 Subject: [PATCH 08/26] per #1203, implement logic for wrapper --- metplus/wrappers/ioda2nc_wrapper.py | 255 +++++++++------------------- 1 file changed, 78 insertions(+), 177 deletions(-) diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index dbcf89fbc..567a870c4 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -6,20 +6,31 @@ import os -from ..util import met_util as util -from ..util import time_util -from . import CommandBuilder from ..util import do_string_sub +from . import LoopTimesWrapper '''!@namespace IODA2NCWrapper -@brief Wraps the IODA2NC tool to reformat ascii format to NetCDF +@brief Wraps the IODA2NC tool to reformat IODA NetCDF data to MET NetCDF @endcode ''' -class IODA2NCWrapper(CommandBuilder): +class IODA2NCWrapper(LoopTimesWrapper): WRAPPER_ENV_VAR_KEYS = [ + 'METPLUS_MESSAGE_TYPE', + 'METPLUS_MESSAGE_TYPE_GROUP_MAP', + 'METPLUS_MESSAGE_TYPE_MAP', + 'METPLUS_STATION_ID', + 'METPLUS_OBS_WINDOW_DICT', + 'METPLUS_MASK_DICT', + 'METPLUS_ELEVATION_RANGE_DICT', + 'METPLUS_LEVEL_RANGE_DICT', + 'METPLUS_OBS_VAR', + 'METPLUS_OBS_NAME_MAP', + 'METPLUS_METADATA_MAP', + 'METPLUS_MISSING_THRESH', + 'METPLUS_QUALITY_MARK_THRESH', 'METPLUS_TIME_SUMMARY_DICT', ] @@ -33,168 +44,66 @@ def __init__(self, config, instance=None, config_overrides=None): def create_c_dict(self): c_dict = super().create_c_dict() - c_dict['VERBOSITY'] = self.config.getstr('config', - 'LOG_IODA2NC_VERBOSITY', - c_dict['VERBOSITY']) - c_dict['ALLOW_MULTIPLE_FILES'] = True - - # IODA2NC config file is optional, so - # don't provide wrapped config file name as default value - c_dict['CONFIG_FILE'] = self.get_config_file() - c_dict['ASCII_FORMAT'] = self.config.getstr('config', - 'IODA2NC_INPUT_FORMAT', - '') - c_dict['MASK_GRID'] = self.config.getstr('config', - 'IODA2NC_MASK_GRID', - '') - c_dict['MASK_POLY'] = self.config.getstr('config', - 'IODA2NC_MASK_POLY', - '') - c_dict['MASK_SID'] = self.config.getstr('config', - 'IODA2NC_MASK_SID', - '') - c_dict['OBS_INPUT_DIR'] = self.config.getdir('IODA2NC_INPUT_DIR', - '') + # file I/O + c_dict['ALLOW_MULTIPLE_FILES'] = True + c_dict['OBS_INPUT_DIR'] = self.config.getdir('IODA2NC_INPUT_DIR', '') c_dict['OBS_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'IODA2NC_INPUT_TEMPLATE') + self.config.getraw('config', 'IODA2NC_INPUT_TEMPLATE') ) if not c_dict['OBS_INPUT_TEMPLATE']: self.log_error("IODA2NC_INPUT_TEMPLATE required to run") + # handle input file window variables + self.handle_file_window_variables(c_dict, dtypes=['OBS']) + c_dict['OUTPUT_DIR'] = self.config.getdir('IODA2NC_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'IODA2NC_OUTPUT_TEMPLATE') + self.config.getraw('config', 'IODA2NC_OUTPUT_TEMPLATE') ) - # MET config variables - self.handle_time_summary_dict(c_dict, - ['TIME_SUMMARY_GRIB_CODES', - 'TIME_SUMMARY_VAR_NAMES', - 'TIME_SUMMARY_TYPES'] - ) + # optional command line arguments + c_dict['VALID_BEG'] = self.config.getraw('config', 'IODA2NC_VALID_BEG') + c_dict['VALID_END'] = self.config.getraw('config', 'IODA2NC_VALID_END') + c_dict['NMSG'] = self.config.getint('config', 'IODA2NC_NMSG') - # handle file window variables - for edge in ['BEGIN', 'END']: - file_window = ( - self.config.getseconds('config', - f'IODA2NC_FILE_WINDOW_{edge}', - '') - ) - if file_window == '': - file_window = ( - self.config.getseconds('config', - f'OBS_FILE_WINDOW_{edge}', - 0) - ) - - c_dict[f'OBS_FILE_WINDOW_{edge}'] = file_window + # MET config variables + c_dict['CONFIG_FILE'] = self.get_config_file('IODA2NCConfig_wrapped') + + self.add_met_config(name='message_type', data_type='list') + self.add_met_config(name='message_type_map', data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='message_type_group_map', data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='station_id', data_type='list') + self.handle_met_config_window('obs_window') + self.handle_mask(single_value=True) + self.handle_met_config_window('elevation_range') + self.handle_met_config_window('level_range') + self.add_met_config(name='obs_var', data_type='list') + self.add_met_config(name='obs_name_map', data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='metadata_map', data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='missing_thresh', data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='quality_mark_thresh', data_type='float') + self.handle_time_summary_dict() return c_dict - def set_environment_variables(self, time_info): - """!Set environment variables that will be read by the MET config file. - Reformat as needed. Print list of variables that were set and their values. - Args: - @param time_info dictionary containing timing info from current run""" - # set environment variables needed for MET application - self.add_env_var('TIME_SUMMARY_FLAG', - self.c_dict['TIME_SUMMARY_FLAG']) - self.add_env_var('TIME_SUMMARY_RAW_DATA', - self.c_dict['TIME_SUMMARY_RAW_DATA']) - self.add_env_var('TIME_SUMMARY_BEG', - self.c_dict['TIME_SUMMARY_BEG']) - self.add_env_var('TIME_SUMMARY_END', - self.c_dict['TIME_SUMMARY_END']) - self.add_env_var('TIME_SUMMARY_STEP', - self.c_dict['TIME_SUMMARY_STEP']) - self.add_env_var('TIME_SUMMARY_WIDTH', - self.c_dict['TIME_SUMMARY_WIDTH']) - self.add_env_var('TIME_SUMMARY_GRIB_CODES', - self.c_dict['TIME_SUMMARY_GRIB_CODES']) - self.add_env_var('TIME_SUMMARY_VAR_NAMES', - self.c_dict['TIME_SUMMARY_VAR_NAMES']) - self.add_env_var('TIME_SUMMARY_TYPES', - self.c_dict['TIME_SUMMARY_TYPES']) - self.add_env_var('TIME_SUMMARY_VALID_FREQ', - self.c_dict['TIME_SUMMARY_VALID_FREQ']) - self.add_env_var('TIME_SUMMARY_VALID_THRESH', - self.c_dict['TIME_SUMMARY_VALID_THRESH']) - - # set user environment variables - super().set_environment_variables(time_info) - def get_command(self): - cmd = self.app_path - - # don't run if no input or output files were found - if not self.infiles: - self.log_error("No input files were found") - return - - if self.outfile == "": - self.log_error("No output file specified") - return - - # add input files - for infile in self.infiles: - cmd += ' ' + infile - - # add output path - out_path = self.get_output_path() - cmd += ' ' + out_path - - parent_dir = os.path.dirname(out_path) - if parent_dir == '': - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - - # add arguments - cmd += ''.join(self.args) - - # add verbosity - cmd += ' -v ' + self.c_dict['VERBOSITY'] - return cmd - - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - """ - lead_seq = util.get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = time_util.ti_calculate(input_dict) - - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_once(time_info) + return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" + f" {self.infiles[0]} {self.get_output_path()}" + f" {' '.join(self.args)}") def run_at_time_once(self, time_info): """! Process runtime and try to build command to run ioda2nc - Args: - @param time_info dictionary containing timing information + + @param time_info dictionary containing timing information """ # get input files - if self.find_input_files(time_info) is None: + if not self.find_input_files(time_info): return # get output path @@ -208,22 +117,13 @@ def run_at_time_once(self, time_info): self.set_environment_variables(time_info) # build command and run - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - - self.build() + return self.build() def find_input_files(self, time_info): - # if using python embedding input, don't check if file exists, - # just substitute time info and add to input file list - if self.c_dict['ASCII_FORMAT'] == 'python': - filename = do_string_sub(self.c_dict['OBS_INPUT_TEMPLATE'], - **time_info) - self.infiles.append(filename) - return self.infiles + """! Get all input files for ioda2nc + @param time_info dictionary containing timing information + """ # get list of files even if only one is found (return_list=True) obs_path = self.find_obs(time_info, var_info=None, return_list=True) if obs_path is None: @@ -233,24 +133,25 @@ def find_input_files(self, time_info): return self.infiles def set_command_line_arguments(self, time_info): - # add input data format if set - if self.c_dict['ASCII_FORMAT']: - self.args.append(" -format {}".format(self.c_dict['ASCII_FORMAT'])) + """! Set all arguments for ioda2nc command. + Note: -obs_var will be set in wrapped MET config file, not command line + + @param time_info dictionary containing timing information + """ + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(f"-config {config_file}") - # add config file - passing through do_string_sub to get custom string if set - if self.c_dict['CONFIG_FILE']: - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) - self.args.append(f" -config {config_file}") + # if more than 1 input file was found, add them with -iodafile + for infile in self.infiles[1:]: + self.args.append(f"-iodafile {infile}") - # add mask grid if set - if self.c_dict['MASK_GRID']: - self.args.append(" -mask_grid {}".format(self.c_dict['MASK_GRID'])) + if self.c_dict['VALID_BEG']: + valid_beg = do_string_sub(self.c_dict['VALID_BEG'], **time_info) + self.args.append(f"-valid_beg {valid_beg}") - # add mask poly if set - if self.c_dict['MASK_POLY']: - self.args.append(" -mask_poly {}".format(self.c_dict['MASK_POLY'])) + if self.c_dict['VALID_END']: + valid_end = do_string_sub(self.c_dict['VALID_END'], **time_info) + self.args.append(f"-valid_end {valid_end}") - # add mask SID if set - if self.c_dict['MASK_SID']: - self.args.append(" -mask_sid {}".format(self.c_dict['MASK_SID'])) + if self.c_dict['NMSG']: + self.args.append(f"-nmsg {self.c_dict['NMSG']}") From 10f17e77aad297da287682473fc1c26308c62dce Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 11 Nov 2021 16:49:07 -0700 Subject: [PATCH 09/26] Per #1101, set tmp_dir to /d1/personal/mccabe/out2/tmp in all wrapped MET config files to prevent writing tmp files to /tmp when running MET commands outside of METplus wrappers. This forces the user to set the MET_TMP_DIR environment variable in their environment when running MET commands using a wrapped MET config file --- parm/met_config/Ascii2NcConfig_wrapped | 2 ++ parm/met_config/EnsembleStatConfig_wrapped | 2 ++ parm/met_config/GenEnsProdConfig_wrapped | 2 ++ parm/met_config/GridDiagConfig_wrapped | 2 ++ parm/met_config/GridStatConfig_wrapped | 4 +++- parm/met_config/IODA2NCConfig_wrapped | 3 ++- parm/met_config/MODEConfig_wrapped | 2 ++ parm/met_config/MTDConfig_wrapped | 2 ++ parm/met_config/PB2NCConfig_wrapped | 3 ++- parm/met_config/PointStatConfig_wrapped | 3 ++- parm/met_config/STATAnalysisConfig_wrapped | 4 +++- parm/met_config/SeriesAnalysisConfig_wrapped | 4 +++- parm/met_config/TCGenConfig_wrapped | 2 ++ parm/met_config/TCPairsConfig_wrapped | 2 ++ parm/met_config/TCRMWConfig_wrapped | 2 ++ parm/met_config/TCStatConfig_wrapped | 2 ++ 16 files changed, 35 insertions(+), 6 deletions(-) diff --git a/parm/met_config/Ascii2NcConfig_wrapped b/parm/met_config/Ascii2NcConfig_wrapped index 6efa3e967..423345061 100644 --- a/parm/met_config/Ascii2NcConfig_wrapped +++ b/parm/met_config/Ascii2NcConfig_wrapped @@ -36,4 +36,6 @@ message_type_map = [ // //version = "V10.0"; +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/EnsembleStatConfig_wrapped b/parm/met_config/EnsembleStatConfig_wrapped index 5331c5583..ac7130ddc 100644 --- a/parm/met_config/EnsembleStatConfig_wrapped +++ b/parm/met_config/EnsembleStatConfig_wrapped @@ -217,4 +217,6 @@ ${METPLUS_OUTPUT_PREFIX} //////////////////////////////////////////////////////////////////////////////// +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GenEnsProdConfig_wrapped b/parm/met_config/GenEnsProdConfig_wrapped index df51805eb..59c794310 100644 --- a/parm/met_config/GenEnsProdConfig_wrapped +++ b/parm/met_config/GenEnsProdConfig_wrapped @@ -103,4 +103,6 @@ ${METPLUS_ENSEMBLE_FLAG_DICT} //////////////////////////////////////////////////////////////////////////////// +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GridDiagConfig_wrapped b/parm/met_config/GridDiagConfig_wrapped index 41592f244..06b95662b 100644 --- a/parm/met_config/GridDiagConfig_wrapped +++ b/parm/met_config/GridDiagConfig_wrapped @@ -33,4 +33,6 @@ ${METPLUS_DATA_DICT} ${METPLUS_MASK_DICT} +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GridStatConfig_wrapped b/parm/met_config/GridStatConfig_wrapped index 9152297a1..fdaf8cf20 100644 --- a/parm/met_config/GridStatConfig_wrapped +++ b/parm/met_config/GridStatConfig_wrapped @@ -178,7 +178,9 @@ ${METPLUS_NC_PAIRS_FLAG_DICT} //grid_weight_flag = ${METPLUS_GRID_WEIGHT_FLAG} -tmp_dir = "/tmp"; + +tmp_dir = "${MET_TMP_DIR}"; + // output_prefix = ${METPLUS_OUTPUT_PREFIX} diff --git a/parm/met_config/IODA2NCConfig_wrapped b/parm/met_config/IODA2NCConfig_wrapped index 964b6c9d3..09330f011 100644 --- a/parm/met_config/IODA2NCConfig_wrapped +++ b/parm/met_config/IODA2NCConfig_wrapped @@ -108,7 +108,8 @@ ${METPLUS_TIME_SUMMARY_DICT} //////////////////////////////////////////////////////////////////////////////// -//tmp_dir = "/tmp"; +tmp_dir = "${MET_TMP_DIR}"; + //version = "V10.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index a08a76164..5f0442add 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -226,6 +226,8 @@ shift_right = 0; // grid squares ${METPLUS_OUTPUT_PREFIX} //version = "V10.0"; +tmp_dir = "${MET_TMP_DIR}"; + //////////////////////////////////////////////////////////////////////////////// ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/MTDConfig_wrapped b/parm/met_config/MTDConfig_wrapped index f027e4436..f8310334a 100644 --- a/parm/met_config/MTDConfig_wrapped +++ b/parm/met_config/MTDConfig_wrapped @@ -239,6 +239,8 @@ txt_output = { ${METPLUS_OUTPUT_PREFIX} //version = "V9.0"; +tmp_dir = "${MET_TMP_DIR}"; + //////////////////////////////////////////////////////////////////////////////// ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PB2NCConfig_wrapped b/parm/met_config/PB2NCConfig_wrapped index 29d981e4a..25cab0375 100644 --- a/parm/met_config/PB2NCConfig_wrapped +++ b/parm/met_config/PB2NCConfig_wrapped @@ -131,7 +131,8 @@ ${METPLUS_TIME_SUMMARY_DICT} //////////////////////////////////////////////////////////////////////////////// -tmp_dir = "/tmp"; +tmp_dir = "${MET_TMP_DIR}"; + //version = "V9.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 2a0d8d190..cde1ba581 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -165,7 +165,8 @@ ${METPLUS_OUTPUT_FLAG_DICT} //////////////////////////////////////////////////////////////////////////////// -tmp_dir = "/tmp"; +tmp_dir = "${MET_TMP_DIR}"; + // output_prefix = ${METPLUS_OUTPUT_PREFIX} //version = "V10.0.0"; diff --git a/parm/met_config/STATAnalysisConfig_wrapped b/parm/met_config/STATAnalysisConfig_wrapped index 2fac673f6..f317e2aa9 100644 --- a/parm/met_config/STATAnalysisConfig_wrapped +++ b/parm/met_config/STATAnalysisConfig_wrapped @@ -101,7 +101,9 @@ wmo_fisher_stats = [ "CNT:PR_CORR", "CNT:SP_CORR", ${METPLUS_HSS_EC_VALUE} rank_corr_flag = FALSE; vif_flag = FALSE; -tmp_dir = "/tmp"; + +tmp_dir = "${MET_TMP_DIR}"; + //version = "V10.0"; ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/SeriesAnalysisConfig_wrapped b/parm/met_config/SeriesAnalysisConfig_wrapped index dfa6f5a4c..94fd1b262 100644 --- a/parm/met_config/SeriesAnalysisConfig_wrapped +++ b/parm/met_config/SeriesAnalysisConfig_wrapped @@ -122,7 +122,9 @@ output_stats = { //hss_ec_value = ${METPLUS_HSS_EC_VALUE} rank_corr_flag = FALSE; -tmp_dir = "/tmp"; + +tmp_dir = "${MET_TMP_DIR}"; + //version = "V10.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/met_config/TCGenConfig_wrapped b/parm/met_config/TCGenConfig_wrapped index 65655c469..c18f895f8 100644 --- a/parm/met_config/TCGenConfig_wrapped +++ b/parm/met_config/TCGenConfig_wrapped @@ -292,4 +292,6 @@ ${METPLUS_NC_PAIRS_GRID} // //version = "V10.0.0"; +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCPairsConfig_wrapped b/parm/met_config/TCPairsConfig_wrapped index c780a3d48..ce13c1db8 100644 --- a/parm/met_config/TCPairsConfig_wrapped +++ b/parm/met_config/TCPairsConfig_wrapped @@ -145,4 +145,6 @@ watch_warn = { // //version = "V9.0"; +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCRMWConfig_wrapped b/parm/met_config/TCRMWConfig_wrapped index 3b5cb2d7d..66dbc2cdf 100644 --- a/parm/met_config/TCRMWConfig_wrapped +++ b/parm/met_config/TCRMWConfig_wrapped @@ -68,4 +68,6 @@ ${METPLUS_RMW_SCALE} //////////////////////////////////////////////////////////////////////////////// +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCStatConfig_wrapped b/parm/met_config/TCStatConfig_wrapped index 03911f051..cef47ecc5 100644 --- a/parm/met_config/TCStatConfig_wrapped +++ b/parm/met_config/TCStatConfig_wrapped @@ -161,4 +161,6 @@ ${METPLUS_MATCH_POINTS} // ${METPLUS_JOBS} +tmp_dir = "${MET_TMP_DIR}"; + ${METPLUS_MET_CONFIG_OVERRIDES} From 19393ebbc2a6e1ccf1ce00fbc4490537006c867b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 10:52:06 -0700 Subject: [PATCH 10/26] per #1212, properly parse list values that contain square braces [] or semi-colons ; --- metplus/util/met_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 4e5118b6a..414b9795a 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -1694,7 +1694,7 @@ def getlist(list_str, expand_begin_end_incr=True): return [] # FIRST remove surrounding comma, and spaces, form the string. - list_str = list_str.strip().strip(',').strip() + list_str = list_str.strip(';[] ').strip().strip(',').strip() # remove space around commas list_str = re.sub(r'\s*,\s*', ',', list_str) From ddad551c6aea2a29fc75597f7fe26020a3ba331a Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 11:20:54 -0700 Subject: [PATCH 11/26] per #1203, add documentation for ioda2nc config variables --- docs/Users_Guide/glossary.rst | 258 +++++++++++++++++++++++++++++++- docs/Users_Guide/wrappers.rst | 269 ++++++++++++++++++++++++++++++++++ 2 files changed, 525 insertions(+), 2 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index c924aded3..58c5ff3d6 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -497,7 +497,7 @@ METplus Configuration Glossary ASCII2NC_CONFIG_FILE Path to optional configuration file read by ascii2nc. To utilize a configuration file, set this to - {PARM_BASE}/parm/met_config/Ascii2NcConfig_wrapped. + {PARM_BASE}/met_config/Ascii2NcConfig_wrapped. If unset, no config file will be used. | *Used by:* ASCII2NC @@ -640,7 +640,7 @@ METplus Configuration Glossary | *Used by:* ASCII2NC ASCII2NC_FILE_WINDOW_END - Used to control the upper bound of the window around the valid time to determine if an ASCII2NC input file should be used for processing. Overrides :term:`OBS_FILE_WINDOW_BEGIN`. See 'Use Windows to Find Valid Files' section for more information. + Used to control the upper bound of the window around the valid time to determine if an ASCII2NC input file should be used for processing. Overrides :term:`OBS_FILE_WINDOW_END`. See 'Use Windows to Find Valid Files' section for more information. | *Used by:* ASCII2NC @@ -8257,3 +8257,257 @@ METplus Configuration Glossary Specify the value for 'ens.file_type' in the MET configuration file for GenEnsProd. | *Used by:* GenEnsProd + + LOG_IODA2NC_VERBOSITY + Overrides the log verbosity for IODA2NC only. + If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. + + | *Used by:* IODA2NC + + IODA2NC_CUSTOM_LOOP_LIST + Sets custom string loop list for a specific wrapper. + See :term:`CUSTOM_LOOP_LIST`. + + | *Used by:* IODA2NC + + IODA2NC_FILE_WINDOW_BEG + Used to control the lower bound of the window around the valid time to + determine if an IODA2NC input file should be used for processing. + Overrides :term:`OBS_FILE_WINDOW_BEGIN`. + See 'Use Windows to Find Valid Files' section for more information. + + | *Used by:* IODA2NC + + IODA2NC_FILE_WINDOW_END + Used to control the upper bound of the window around the valid time to + determine if an IODA2NC input file should be used for processing. + Overrides :term:`OBS_FILE_WINDOW_END`. + See 'Use Windows to Find Valid Files' section for more information. + + | *Used by:* IODA2NC + + IODA2NC_SKIP_IF_OUTPUT_EXISTS + If True, do not run IODA2NC if output file already exists. Set to False to overwrite files. + + | *Used by:* IODA2NC + + IODA2NC_INPUT_DIR + Directory containing input data to IODA2NC. + This variable is optional because you can specify the full path to the + input files using :term:`IODA2NC_INPUT_TEMPLATE`. + + | *Used by:* IODA2NC + + IODA2NC_INPUT_TEMPLATE + Filename template of the input file used by IODA2NC. + See also :term:`IODA2NC_INPUT_DIR`. + + | *Used by:* IODA2NC + + IODA2NC_OUTPUT_DIR + Directory to write output data generated by IODA2NC. + This variable is optional because you can specify the full path to the + output files using :term:`IODA2NC_OUTPUT_TEMPLATE`. + + | *Used by:* IODA2NC + + IODA2NC_OUTPUT_TEMPLATE + Filename template of the output file generated by IODA2NC. + See also :term:`IODA2NC_OUTPUT_DIR`. + + | *Used by:* IODA2NC + + IODA2NC_VALID_BEG + Used to set the command line argument -valid_beg that controls the + lower bound of valid times of data to use. + Filename template notation can be used, i.e. {valid?fmt=%Y%m%d_%H%M%S} + + | *Used by:* IODA2NC + + IODA2NC_VALID_END + Used to set the command line argument -valid_end that controls the + upper bound of valid times of data to use. + Filename template notation can be used, i.e. + {valid?fmt=%Y%m%d_%H%M%S?shift=1d} (valid time shifted forward one day) + + | *Used by:* IODA2NC + + IODA2NC_NMSG + Used to set the command line argument -nmsg for ioda2nc. + + | *Used by:* IODA2NC + + IODA2NC_CONFIG_FILE + Path to wrapped MET configuration file read by ioda2nc. + If unset, {PARM_BASE}/met_config/IODA2NCConfig_wrapped will be used. + + | *Used by:* IODA2NC + + IODA2NC_MESSAGE_TYPE + Specify the value for 'message_type' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MESSAGE_TYPE_MAP + Specify the value for 'message_type_map' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MESSAGE_TYPE_GROUP_MAP + Specify the value for 'message_type_group_map' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_STATION_ID + Specify the value for 'station_id' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_OBS_WINDOW_BEG + Specify the value for 'obs_window.beg' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_OBS_WINDOW_END + Specify the value for 'obs_window.end' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MASK_GRID + Specify the value for 'mask.grid' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MASK_POLY + Specify the value for 'mask.poly' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_ELEVATION_RANGE_BEG + Specify the value for 'elevation_range.beg' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_ELEVATION_RANGE_END + Specify the value for 'elevation_range.end' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_LEVEL_RANGE_BEG + Specify the value for 'level_range.beg' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_LEVEL_RANGE_END + Specify the value for 'level_range.end' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_OBS_VAR + Specify the value for 'obs_var' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_OBS_NAME_MAP + Specify the value for 'obs_name_map' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_METADATA_MAP + Specify the value for 'metadata_map' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MISSING_THRESH + Specify the value for 'missing_thresh' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_QUALITY_MARK_THRESH + Specify the value for 'quality_mark_thresh' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_FLAG + Specify the value for 'time_summary.flag' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_RAW_DATA + Specify the value for 'time_summary.raw_data' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_BEG + Specify the value for 'time_summary.beg' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_END + Specify the value for 'time_summary.end' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_STEP + Specify the value for 'time_summary.step' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_WIDTH + Specify the value for 'time_summary.width' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_GRIB_CODE + Specify the value for 'time_summary.grib_code' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_OBS_VAR + Specify the value for 'time_summary.obs_var' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_TYPE + Specify the value for 'time_summary.type' in the MET configuration file + for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_VLD_FREQ + Specify the value for 'time_summary.vld_freq' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_TIME_SUMMARY_VLD_THRESH + Specify the value for 'time_summary.vld_thresh' in the MET configuration + file for IODA2NC. + + | *Used by:* IODA2NC + + IODA2NC_MET_CONFIG_OVERRIDES + Override any variables in the MET configuration file that are not + supported by the wrapper. This should be set to the full variable name + and value that you want to override, including the equal sign and the + ending semi-colon. The value is directly appended to the end of the + wrapped MET config file. + + Example: + IODA2NC_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; + + See :ref:`Overriding Unsupported MET config file settings` for more information + + | *Used by:* IODA2NC diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 8b07448f8..5b33cbaff 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3242,6 +3242,275 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`GRID_STAT_DISTANCE_MAP_BETA_VALUE_N` - distance_map.beta_value(n) +.. _ioda2nc_wrapper: + +IODA2NC +======== + +Description +----------- + +Used to configure the MET tool ioda2nc + +METplus Configuration +--------------------- + +| :term:`IODA2NC_INPUT_DIR` +| :term:`IODA2NC_INPUT_TEMPLATE` +| :term:`IODA2NC_OUTPUT_DIR` +| :term:`IODA2NC_OUTPUT_TEMPLATE` +| :term:`LOG_IODA2NC_VERBOSITY` +| :term:`IODA2NC_SKIP_IF_OUTPUT_EXISTS` +| :term:`IODA2NC_CONFIG_FILE` +| :term:`IODA2NC_FILE_WINDOW_BEGIN` +| :term:`IODA2NC_FILE_WINDOW_END` +| :term:`IODA2NC_VALID_BEG` +| :term:`IODA2NC_VALID_END` +| :term:`IODA2NC_NMSG` +| :term:`IODA2NC_MESSAGE_TYPE` +| :term:`IODA2NC_MESSAGE_TYPE_MAP` +| :term:`IODA2NC_MESSAGE_TYPE_GROUP_MAP` +| :term:`IODA2NC_STATION_ID` +| :term:`IODA2NC_OBS_WINDOW_BEG` +| :term:`IODA2NC_OBS_WINDOW_END` +| :term:`IODA2NC_MASK_GRID` +| :term:`IODA2NC_MASK_POLY` +| :term:`IODA2NC_ELEVATION_RANGE_BEG` +| :term:`IODA2NC_ELEVATION_RANGE_END` +| :term:`IODA2NC_LEVEL_RANGE_BEG` +| :term:`IODA2NC_LEVEL_RANGE_END` +| :term:`IODA2NC_OBS_VAR` +| :term:`IODA2NC_OBS_NAME_MAP` +| :term:`IODA2NC_METADATA_MAP` +| :term:`IODA2NC_MISSING_THRESH` +| :term:`IODA2NC_QUALITY_MARK_THRESH` +| :term:`IODA2NC_TIME_SUMMARY_FLAG` +| :term:`IODA2NC_TIME_SUMMARY_RAW_DATA` +| :term:`IODA2NC_TIME_SUMMARY_BEG` +| :term:`IODA2NC_TIME_SUMMARY_END` +| :term:`IODA2NC_TIME_SUMMARY_STEP` +| :term:`IODA2NC_TIME_SUMMARY_WIDTH` +| :term:`IODA2NC_TIME_SUMMARY_GRIB_CODE` +| :term:`IODA2NC_TIME_SUMMARY_OBS_VAR` +| :term:`IODA2NC_TIME_SUMMARY_TYPE` +| :term:`IODA2NC_TIME_SUMMARY_VLD_FREQ` +| :term:`IODA2NC_TIME_SUMMARY_VLD_THRESH` +| :term:`IODA2NC_CUSTOM_LOOP_LIST` +| :term:`IODA2NC_MET_CONFIG_OVERRIDES` + +.. _ioda2nc-met-conf: + +MET Configuration +----------------- + +Below is the wrapped MET configuration file used for this wrapper. +Environment variables are used to control entries in this configuration file. +The default value for each environment variable is obtained from +(except where noted below): + +`MET_INSTALL_DIR/share/met/config/IODA2NCConfig_default `_ + +Below the file contents are descriptions of each environment variable +referenced in this file and the corresponding METplus configuration item used +to set the value of the environment variable. For detailed examples showing +how METplus sets the values of these environment variables, +see :ref:`How METplus controls MET config file settings`. + +.. literalinclude:: ../../parm/met_config/IODA2NCConfig_wrapped + +**${METPLUS_MESSAGE_TYPE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MESSAGE_TYPE` + - message_type + +**${METPLUS_MESSAGE_TYPE_MAP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MESSAGE_TYPE_MAP` + - message_type_map + +**${METPLUS_MESSAGE_TYPE_GROUP_MAP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MESSAGE_TYPE_GROUP_MAP` + - message_type_group_map + +**${METPLUS_STATION_ID}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_STATION_ID` + - station_id + +**${METPLUS_OBS_WINDOW_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_OBS_WINDOW_BEG` + - obs_window.beg + * - :term:`IODA2NC_OBS_WINDOW_END` + - obs_window.end + +**${METPLUS_MASK_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MASK_GRID` + - mask.grid + * - :term:`IODA2NC_MASK_POLY` + - mask.poly + +**${METPLUS_ELEVATION_RANGE_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_ELEVATION_RANGE_BEG` + - elevation_range.beg + * - :term:`IODA2NC_ELEVATION_RANGE_END` + - elevation_range.end + +**${METPLUS_LEVEL_RANGE_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_LEVEL_RANGE_BEG` + - level_range.beg + * - :term:`IODA2NC_LEVEL_RANGE_END` + - level_range.end + +**${METPLUS_OBS_VAR}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_OBS_VAR` + - obs_var + +**${METPLUS_OBS_NAME_MAP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_OBS_NAME_MAP` + - obs_name_map + +**${METPLUS_METADATA_MAP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_METADATA_MAP` + - metadata_map + +**${METPLUS_MISSING_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MISSING_THRESH` + - missing_thresh + +**${METPLUS_QUALITY_MARK_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_QUALITY_MARK_THRESH` + - quality_mark_thresh + +**${METPLUS_TIME_SUMMARY_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_TIME_SUMMARY_FLAG` + - time_summary.flag + * - :term:`IODA2NC_TIME_SUMMARY_RAW_DATA` + - time_summary.raw_data + * - :term:`IODA2NC_TIME_SUMMARY_BEG` + - time_summary.beg + * - :term:`IODA2NC_TIME_SUMMARY_END` + - time_summary.end + * - :term:`IODA2NC_TIME_SUMMARY_STEP` + - time_summary.step + * - :term:`IODA2NC_TIME_SUMMARY_WIDTH` + - time_summary.width + * - :term:`IODA2NC_TIME_SUMMARY_GRIB_CODE` + - time_summary.grib_code + * - :term:`IODA2NC_TIME_SUMMARY_OBS_VAR` + - time_summary.obs_var + * - :term:`IODA2NC_TIME_SUMMARY_TYPE` + - time_summary.type + * - :term:`IODA2NC_TIME_SUMMARY_VLD_FREQ` + - time_summary.vld_freq + * - :term:`IODA2NC_TIME_SUMMARY_VLD_THRESH` + - time_summary.vld_thresh + +**${METPLUS_MET_CONFIG_OVERRIDES}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_MET_CONFIG_OVERRIDES` + - n/a + .. _make_plots_wrapper: MakePlots From e1da1ce2c91262e90710662bc3c282dd93cbf349 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 11:22:12 -0700 Subject: [PATCH 12/26] fix formatting for width to remove quotes --- metplus/wrappers/command_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 00ba14a0d..4384f4530 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1996,7 +1996,7 @@ def handle_time_summary_dict(self): 'beg': 'string', 'end': 'string', 'step': 'int', - 'width': 'string', + 'width': ('string', 'remove_quotes'), 'grib_code': ('list', 'remove_quotes,allow_empty', None, ['TIME_SUMMARY_GRIB_CODES']), 'obs_var': ('list', 'allow_empty', None, From 55dc20f4f6e9eb4baa16e65acdd670776ddcc767 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 11:23:24 -0700 Subject: [PATCH 13/26] set default -nmsg value to 0 to easily check if it should be let unset, change quality_mark_thresh date type to integer --- metplus/wrappers/ioda2nc_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index 567a870c4..1bba4af90 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -65,7 +65,7 @@ def create_c_dict(self): # optional command line arguments c_dict['VALID_BEG'] = self.config.getraw('config', 'IODA2NC_VALID_BEG') c_dict['VALID_END'] = self.config.getraw('config', 'IODA2NC_VALID_END') - c_dict['NMSG'] = self.config.getint('config', 'IODA2NC_NMSG') + c_dict['NMSG'] = self.config.getint('config', 'IODA2NC_NMSG', 0) # MET config variables c_dict['CONFIG_FILE'] = self.get_config_file('IODA2NCConfig_wrapped') @@ -87,7 +87,7 @@ def create_c_dict(self): extra_args={'remove_quotes': True}) self.add_met_config(name='missing_thresh', data_type='list', extra_args={'remove_quotes': True}) - self.add_met_config(name='quality_mark_thresh', data_type='float') + self.add_met_config(name='quality_mark_thresh', data_type='int') self.handle_time_summary_dict() return c_dict From 3b711a29f84622e9076598671e226d978599a75f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 12:18:31 -0700 Subject: [PATCH 14/26] per #1204, add documentation for IODA2NC use case example --- .../met_tool_wrapper/IODA2NC/IODA2NC.py | 110 ++++++++++++++++++ .../met_tool_wrapper/IODA2NC/README.rst | 2 + 2 files changed, 112 insertions(+) create mode 100644 docs/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.py create mode 100644 docs/use_cases/met_tool_wrapper/IODA2NC/README.rst diff --git a/docs/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.py b/docs/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.py new file mode 100644 index 000000000..6e997293f --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.py @@ -0,0 +1,110 @@ +""" +IODA2NC: Basic Use Case +======================= + +met_tool_wrapper/IODA2NC/IODA2NC.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# Convert IODA NetCDF files to MET NetCDF format. + +############################################################################## +# Datasets +# -------- +# +# **Input:** IODA NetCDF observation +# +# **Location:** All of the input data required for this use case can be found +# in the met_test sample data tarball. Click here to the METplus releases +# page and download sample data for the appropriate release: +# https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will set the +# value of INPUT_BASE. See the `Running METplus`_ section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus IODA2NC wrapper to generate a command +# to run the MET tool ioda2nc if all required files are found. + +############################################################################## +# METplus Workflow +# ---------------- +# +# IODA2NC is the only tool called in this example. +# It processes the following run time(s): +# +# | **Valid:** 2020-03-10 12Z +# + +############################################################################## +# METplus Configuration +# --------------------- +# +# parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf + +############################################################################## +# MET Configuration +# --------------------- +# +# .. note:: +# See the :ref:`IODA2NC MET Configuration` +# section of the User's Guide for more information on the environment +# variables used in the file below. +# +# parm/met_config/IODA2NCConfig_wrapped +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/IODA2NCConfig_wrapped + +############################################################################## +# Running METplus +# --------------- +# +# Provide the use case .conf configuration file to the run_metplus.py script. +# +# /path/to/METplus/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf +# +# See the :ref:`running-metplus` section of the System Configuration chapter +# for more details. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following to the screen and the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data +# was generated. Output for this use case will be found in +# met_tool_wrapper/ioda2nc +# (relative to **OUTPUT_BASE**) +# and will contain the following file(s): +# +# * ioda.NC001007.2020031012.summary.nc +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * IODA2NCToolUseCase +# +# Navigate to :ref:`quick-search` to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-IODA2NC.png' +# diff --git a/docs/use_cases/met_tool_wrapper/IODA2NC/README.rst b/docs/use_cases/met_tool_wrapper/IODA2NC/README.rst new file mode 100644 index 000000000..84d389d33 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/IODA2NC/README.rst @@ -0,0 +1,2 @@ +IODA2NC +------- From 0eaf2be6d3c30bb3c2d3c3ba8c0683eb6005009f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 12:19:04 -0700 Subject: [PATCH 15/26] per #1204, add IODA2NCToolUseCase to quick search --- docs/Users_Guide/quicksearch.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index b758b1fd8..4249e3b38 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -20,6 +20,7 @@ Use Cases by MET Tool: | `GenEnsProd <../search.html?q=GenEnsProdToolUseCase&check_keywords=yes&area=default>`_ | `GridStat <../search.html?q=GridStatToolUseCase&check_keywords=yes&area=default>`_ | `GridDiag <../search.html?q=GridDiagToolUseCase&check_keywords=yes&area=default>`_ + | `IODA2NC <../search.html?q=IODA2NCToolUseCase&check_keywords=yes&area=default>`_ | `MODE <../search.html?q=MODEToolUseCase&check_keywords=yes&area=default>`_ | `MTD <../search.html?q=MTDToolUseCase&check_keywords=yes&area=default>`_ | `PB2NC <../search.html?q=PB2NCToolUseCase&check_keywords=yes&area=default>`_ @@ -45,6 +46,7 @@ Use Cases by MET Tool: | **GenEnsProd**: *GenEnsProdToolUseCase* | **GridStat**: *GridStatToolUseCase* | **GridDiag**: *GridDiagToolUseCase* + | **IODA2NC**: *IODA2NCToolUseCase* | **MODE**: *MODEToolUseCase* | **MTD**: *MTDToolUseCase* | **PB2NC**: *PB2NCToolUseCase* From 85b92712612c314675a8e80cd1e3ec8fb5686bf8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:50:43 -0700 Subject: [PATCH 16/26] per #1203, added unit tests for ioda2nc wrapper to ensure commands are generated properly and environment variables referenced in the wrapped MET config file are set correctly --- .../pytests/ioda2nc/test_ioda2nc_wrapper.py | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py diff --git a/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py b/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py new file mode 100644 index 000000000..47030515d --- /dev/null +++ b/internal_tests/pytests/ioda2nc/test_ioda2nc_wrapper.py @@ -0,0 +1,246 @@ +import os + +import pytest + +from metplus.wrappers.ioda2nc_wrapper import IODA2NCWrapper + + +time_fmt = '%Y%m%d%H' +run_times = ['2020031012', '2020031100'] + +def set_minimum_config_settings(config): + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'IODA2NC') + config.set('config', 'LOOP_BY', 'VALID') + config.set('config', 'VALID_TIME_FMT', time_fmt) + config.set('config', 'VALID_BEG', run_times[0]) + config.set('config', 'VALID_END', run_times[-1]) + config.set('config', 'VALID_INCREMENT', '12H') + config.set('config', 'LOOP_ORDER', 'times') + config.set('config', 'IODA2NC_INPUT_DIR', + '{INPUT_BASE}/met_test/new/ioda') + config.set('config', 'IODA2NC_INPUT_TEMPLATE', + 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc') + config.set('config', 'IODA2NC_OUTPUT_DIR', + '{OUTPUT_BASE}/ioda2nc') + config.set('config', 'IODA2NC_OUTPUT_TEMPLATE', + 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.summary.nc') + +@pytest.mark.parametrize( + 'config_overrides, env_var_values, extra_args', [ + # 0 + ({'IODA2NC_MESSAGE_TYPE': 'ADPUPA, ADPSFC', }, + {'METPLUS_MESSAGE_TYPE': 'message_type = ["ADPUPA", "ADPSFC"];'}, ''), + # 1 + ({'IODA2NC_MESSAGE_TYPE_MAP': '{ key = “AIRCAR”; val = “AIRCAR_PROFILES”; }', }, + {'METPLUS_MESSAGE_TYPE_MAP': 'message_type_map = [{ key = “AIRCAR”; val = “AIRCAR_PROFILES”; }];'}, ''), + # 2 + ({'IODA2NC_MESSAGE_TYPE_GROUP_MAP': '{ key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET";},{ key = "ANYAIR"; val = "AIRCAR,AIRCFT";}', }, + {'METPLUS_MESSAGE_TYPE_GROUP_MAP': 'message_type_group_map = [{ key = "SURFACE"; val = "ADPSFC, SFCSHP, MSONET";}, { key = "ANYAIR"; val = "AIRCAR, AIRCFT";}];'}, ''), + # 3 + ({'IODA2NC_STATION_ID': 'value1, value2', }, + {'METPLUS_STATION_ID': 'station_id = ["value1", "value2"];'}, ''), + # 4 + ({'IODA2NC_OBS_WINDOW_BEG': '-5400', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -5400;}'}, ''), + # 5 + ({'IODA2NC_OBS_WINDOW_END': '5400', }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {end = 5400;}'}, ''), + # 6 + ({ + 'IODA2NC_OBS_WINDOW_BEG': '-5400', + 'IODA2NC_OBS_WINDOW_END': '5400', + }, + {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -5400;end = 5400;}'} + , ''), + # 7 + ({'IODA2NC_MASK_GRID': 'FULL', }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}, ''), + # 8 + ({'IODA2NC_MASK_POLY': '/some/polyfile.nc', }, + {'METPLUS_MASK_DICT': 'mask = {poly = "/some/polyfile.nc";}'}, ''), + # 9 + ({ + 'IODA2NC_MASK_GRID': 'FULL', + 'IODA2NC_MASK_POLY': '/some/polyfile.nc', + }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";poly = "/some/polyfile.nc";}'}, ''), + # 10 + ({'IODA2NC_ELEVATION_RANGE_BEG': '-1000', }, + {'METPLUS_ELEVATION_RANGE_DICT': 'elevation_range = {beg = -1000;}'}, ''), + # 11 + ({'IODA2NC_ELEVATION_RANGE_END': '100000', }, + {'METPLUS_ELEVATION_RANGE_DICT': 'elevation_range = {end = 100000;}'}, ''), + # 12 + ({ + 'IODA2NC_ELEVATION_RANGE_BEG': '-1000', + 'IODA2NC_ELEVATION_RANGE_END': '100000', + }, + {'METPLUS_ELEVATION_RANGE_DICT': 'elevation_range = {beg = -1000;end = 100000;}'}, ''), + # 13 + ({'IODA2NC_LEVEL_RANGE_BEG': '1', }, + {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {beg = 1;}'}, ''), + # 14 + ({'IODA2NC_LEVEL_RANGE_END': '255', }, + {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {end = 255;}'}, ''), + # 15 + ({ + 'IODA2NC_LEVEL_RANGE_BEG': '1', + 'IODA2NC_LEVEL_RANGE_END': '255', + }, + {'METPLUS_LEVEL_RANGE_DICT': 'level_range = {beg = 1;end = 255;}'}, ''), + # 16 + ({'IODA2NC_OBS_VAR': 'TMP,WDIR,RH', }, + {'METPLUS_OBS_VAR': 'obs_var = ["TMP", "WDIR", "RH"];'}, ''), + # 17 + ({'IODA2NC_OBS_NAME_MAP': '{ key = "message_type"; val = "msg_type"; },{ key = "station_id"; val = "report_identifier"; }', }, + {'METPLUS_OBS_NAME_MAP': 'obs_name_map = [{ key = "message_type"; val = "msg_type"; }, { key = "station_id"; val = "report_identifier"; }];'}, ''), + # 18 + ({'IODA2NC_METADATA_MAP': '{ key = "message_type"; val = "msg_type"; },{ key = "station_id"; val = "report_identifier"; }', }, + {'METPLUS_METADATA_MAP': 'metadata_map = [{ key = "message_type"; val = "msg_type"; }, { key = "station_id"; val = "report_identifier"; }];'}, ''), + # 19 + ({'IODA2NC_MISSING_THRESH': '<=-1e9, >=1e9, ==-9999', }, + {'METPLUS_MISSING_THRESH': 'missing_thresh = [<=-1e9, >=1e9, ==-9999];'}, ''), + # 20 + ({'IODA2NC_QUALITY_MARK_THRESH': '2', }, + {'METPLUS_QUALITY_MARK_THRESH': 'quality_mark_thresh = 2;'}, ''), + # 21 + ({}, + {'METPLUS_TIME_SUMMARY_DICT': ''}, ''), + # 22 + ({'IODA2NC_TIME_SUMMARY_FLAG': 'True'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {flag = TRUE;}'}, ''), + # 23 + ({'IODA2NC_TIME_SUMMARY_RAW_DATA': 'true'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {raw_data = TRUE;}'}, ''), + # 24 + ({'IODA2NC_TIME_SUMMARY_BEG': '123456'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {beg = "123456";}'}, ''), + # 25 + ({'IODA2NC_TIME_SUMMARY_END': '123456'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {end = "123456";}'}, ''), + # 26 + ({'IODA2NC_TIME_SUMMARY_STEP': '500'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {step = 500;}'}, ''), + # 27 + ({'IODA2NC_TIME_SUMMARY_WIDTH': '900'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {width = 900;}'}, ''), + # 28 width as dictionary + ({'IODA2NC_TIME_SUMMARY_WIDTH': '{ beg = -21600; end = 0; }'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {width = { beg = -21600; end = 0; };}'}, ''), + # 29 + ({'IODA2NC_TIME_SUMMARY_GRIB_CODE': '12, 203, 212'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {grib_code = [12, 203, 212];}'}, ''), + # 30 + ({'IODA2NC_TIME_SUMMARY_OBS_VAR': 'TMP, HGT, PRES'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {obs_var = ["TMP", "HGT", "PRES"];}'}, ''), + # 31 + ({'IODA2NC_TIME_SUMMARY_TYPE': 'min, range, max'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {type = ["min", "range", "max"];}'}, ''), + # 32 + ({'IODA2NC_TIME_SUMMARY_VALID_FREQ': '2'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {vld_freq = 2;}'}, ''), + # 33 + ({'IODA2NC_TIME_SUMMARY_VALID_THRESH': '0.5'}, + {'METPLUS_TIME_SUMMARY_DICT': + 'time_summary = {vld_thresh = 0.5;}'}, ''), + # 34 additional input file with full path + ({'IODA2NC_INPUT_TEMPLATE': 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc, /other/file.nc'}, + {}, ' -iodafile /other/file.nc'), + # 35 additional input file with relative path + ({'IODA2NC_INPUT_TEMPLATE': 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc, other/file.nc'}, + {}, ' -iodafile *INPUT_DIR*/other/file.nc'), + # 36 + ({'IODA2NC_VALID_BEG': '20200309_12'}, + {}, ' -valid_beg 20200309_12'), + # 37 + ({'IODA2NC_VALID_END': '20200310_12'}, + {}, ' -valid_end 20200310_12'), + # 38 + ({'IODA2NC_NMSG': '10'}, + {}, ' -nmsg 10'), + # 39 all optional command line args + ({'IODA2NC_INPUT_TEMPLATE': 'ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc, other/file.nc', + 'IODA2NC_VALID_BEG': '20200309_12', + 'IODA2NC_VALID_END': '20200310_12', + 'IODA2NC_NMSG': '10', + }, + {}, ' -iodafile *INPUT_DIR*/other/file.nc -valid_beg 20200309_12 -valid_end 20200310_12 -nmsg 10'), + ] +) +def test_ioda2nc_wrapper(metplus_config, config_overrides, + env_var_values, extra_args): + config = metplus_config() + + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = IODA2NCWrapper(config) + assert wrapper.isOK + + input_dir = wrapper.c_dict.get('OBS_INPUT_DIR') + output_dir = wrapper.c_dict.get('OUTPUT_DIR') + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + + extra_args = extra_args.replace('*INPUT_DIR*', input_dir) + expected_cmds = [ + (f"{app_path} {verbosity} {input_dir}/ioda.NC001007.2020031012.nc" + f" {output_dir}/ioda.NC001007.2020031012.summary.nc" + f" -config {config_file}{extra_args}"), + (f"{app_path} {verbosity} {input_dir}/ioda.NC001007.2020031100.nc" + f" {output_dir}/ioda.NC001007.2020031100.summary.nc" + f" -config {config_file}{extra_args}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + actual_value = match.split('=', 1)[1] + assert(env_var_values.get(env_var_key, '') == actual_value) + +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + config = metplus_config() + config.set('config', 'INPUT_MUST_EXIST', False) + + wrapper = IODA2NCWrapper(config) + + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'IODA2NCConfig_wrapped') + + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'IODA2NC_CONFIG_FILE', fake_config_name) + wrapper = IODA2NCWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name From 7e0c30fc751025ab036cd5ccbe103a2eef6692a8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:51:22 -0700 Subject: [PATCH 17/26] added assert to ensure number of commands generated matches the expected number --- internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py index 48da2ba78..871826c6d 100644 --- a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -92,6 +92,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, assert wrapper.isOK all_cmds = wrapper.run_all_times() + assert len(all_cmds) == len(run_times) for (_, actual_env_vars), run_time in zip(all_cmds, run_times): run_dt = datetime.strptime(run_time, time_fmt) ymdh = run_dt.strftime('%Y%m%d%H') From 28c5c83a481b36aefcd8e7629b6d364d7ad75017 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:52:12 -0700 Subject: [PATCH 18/26] per #1203, added documentation for all functions and corrected return value for run_at_time_once --- metplus/wrappers/ioda2nc_wrapper.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index 1bba4af90..3fecb4a4b 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -43,6 +43,16 @@ def __init__(self, config, instance=None, config_overrides=None): config_overrides=config_overrides) def create_c_dict(self): + """! Read METplusConfig object and sets values in dictionary to be + used by the wrapper to generate commands. Gets information regarding + input/output files, optional command line arguments, and values to + set in the wrapped MET config file. Calls self.log_error if any + required METplusConfig variables were not set properly which logs the + error and sets self.isOK to False which causes wrapper initialization + to fail. + + @returns dictionary containing configurations for this wrapper + """ c_dict = super().create_c_dict() # file I/O @@ -93,6 +103,10 @@ def create_c_dict(self): return c_dict def get_command(self): + """! Build the command to call ioda2nc + + @returns string containing command to run + """ return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" f" {self.infiles[0]} {self.get_output_path()}" f" {' '.join(self.args)}") @@ -101,14 +115,16 @@ def run_at_time_once(self, time_info): """! Process runtime and try to build command to run ioda2nc @param time_info dictionary containing timing information + @returns True if command was built/run successfully or + False if something went wrong """ # get input files if not self.find_input_files(time_info): - return + return False # get output path if not self.find_and_check_output_file(time_info): - return + return False # get other configurations for command self.set_command_line_arguments(time_info) @@ -120,9 +136,10 @@ def run_at_time_once(self, time_info): return self.build() def find_input_files(self, time_info): - """! Get all input files for ioda2nc + """! Get all input files for ioda2nc. Sets self.infiles list. @param time_info dictionary containing timing information + @returns List of files that were found or None if no files were found """ # get list of files even if only one is found (return_list=True) obs_path = self.find_obs(time_info, var_info=None, return_list=True) From 2ab26035dad2edf3fe727df85886a0645b4db19b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:53:28 -0700 Subject: [PATCH 19/26] per #1204, created use case configuration file for basic example of IODA2NC wrapper -- settings are based on a MET unit test for the tool: https://github.com/dtcenter/MET/blob/4ff28d192d07fb61c3271cea4d62b8b59b84ef6b/test/xml/unit_ioda2nc.xml#L90-L101 --- .../met_tool_wrapper/IODA2NC/IODA2NC.conf | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf diff --git a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf new file mode 100644 index 000000000..2d184ac18 --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf @@ -0,0 +1,83 @@ +[config] + +PROCESS_LIST = IODA2NC + +### +# Time Info +### + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2020031012 +VALID_END = 2020031012 +VALID_INCREMENT = 6H + +### +# File I/O Info +### + +IODA2NC_INPUT_DIR = {INPUT_BASE}/met_test/new/ioda +IODA2NC_INPUT_TEMPLATE = ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc + +IODA2NC_OUTPUT_DIR = {OUTPUT_BASE}/ioda2nc +IODA2NC_OUTPUT_TEMPLATE = ioda.NC001007.{valid?fmt=%Y%m%d%H}.summary.nc + + +# OPTIONAL CONFIGURATIONS + +### +# ioda2nc command line arguments +### + +#IODA2NC_VALID_BEG = {valid?fmt=%Y%m%d_%H?shift=-24H} +#IODA2NC_VALID_END = {valid?fmt=%Y%m%d_%H} +#IODA2NC_NMSG = 10 + + +### +# ioda2nc configuration variables +### + +#IODA2NC_MESSAGE_TYPE = + +#IODA2NC_MESSAGE_TYPE_MAP = + +#IODA2NC_MESSAGE_TYPE_GROUP_MAP = + +#IODA2NC_STATION_ID = + +IODA2NC_OBS_WINDOW_BEG = -5400 +IODA2NC_OBS_WINDOW_END = 5400 + +#IODA2NC_MASK_GRID = +#IODA2NC_MASK_POLY = + +IODA2NC_ELEVATION_RANGE_BEG = -1000 +IODA2NC_ELEVATION_RANGE_END = 100000 + +#IODA2NC_LEVEL_RANGE_BEG = 1 +#IODA2NC_LEVEL_RANGE_END = 255 + +#IODA2NC_OBS_VAR = + +IODA2NC_OBS_NAME_MAP = + { key = "wind_direction"; val = "WDIR"; }, + { key = "wind_speed"; val = "WIND"; } + +#IODA2NC_METADATA_MAP = + +#IODA2NC_MISSING_THRESH = <=-1e9, >=1e9, ==-9999 + +IODA2NC_QUALITY_MARK_THRESH = 0 + +IODA2NC_TIME_SUMMARY_FLAG = True +IODA2NC_TIME_SUMMARY_RAW_DATA = True +IODA2NC_TIME_SUMMARY_BEG = 000000 +IODA2NC_TIME_SUMMARY_END = 235959 +IODA2NC_TIME_SUMMARY_STEP = 300 +IODA2NC_TIME_SUMMARY_WIDTH = 600 +IODA2NC_TIME_SUMMARY_GRIB_CODE = +IODA2NC_TIME_SUMMARY_OBS_VAR = "WIND" +IODA2NC_TIME_SUMMARY_TYPE = "min", "max", "range", "mean", "stdev", "median", "p80" +IODA2NC_TIME_SUMMARY_VLD_FREQ = 0 +IODA2NC_TIME_SUMMARY_VLD_THRESH = 0.0 From ec2f088be0cee3e5d6a0acf9ffa422131920e48a Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 14:06:03 -0700 Subject: [PATCH 20/26] per #1204, add new use case to automated tests --- .github/parm/use_case_groups.json | 5 +++++ internal_tests/use_cases/all_use_cases.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 536774aac..ad7269338 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,6 +9,11 @@ "index_list": "30-58", "run": false }, + { + "category": "met_tool_wrapper", + "index_list": "59", + "run": true + }, { "category": "air_quality_and_comp", "index_list": "0", diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index a9ea0ea51..2d196b9b6 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -58,6 +58,7 @@ Category: met_tool_wrapper 56::GFDLTracker_ETC::met_tool_wrapper/GFDLTracker/GFDLTracker_ETC.conf::gfdl-tracker_env 57::GFDLTracker_Genesis::met_tool_wrapper/GFDLTracker/GFDLTracker_Genesis.conf::gfdl-tracker_env 58::GenEnsProd::met_tool_wrapper/GenEnsProd/GenEnsProd.conf +59::IODA2NC::met_tool_wrapper/IODA2NC/IODA2NC.conf Category: air_quality_and_comp 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf From c57676d20a4d8ac6b54a6ff47f4384fa87415f45 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 14:18:45 -0700 Subject: [PATCH 21/26] removed apostrophe that messes up documentation rendering --- parm/met_config/IODA2NCConfig_wrapped | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/met_config/IODA2NCConfig_wrapped b/parm/met_config/IODA2NCConfig_wrapped index 09330f011..ba1faa169 100644 --- a/parm/met_config/IODA2NCConfig_wrapped +++ b/parm/met_config/IODA2NCConfig_wrapped @@ -2,7 +2,7 @@ // // IODA2NC configuration file. // -// For additional information, please see the MET User's Guide. +// For additional information, please see the MET Users Guide. // //////////////////////////////////////////////////////////////////////////////// From e5ae6ce18f29024eae2878e31ea5e4ba90eb4533 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 14:19:45 -0700 Subject: [PATCH 22/26] fixed variable name mismatch --- docs/Users_Guide/wrappers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 5b33cbaff..cfc8c44fe 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3262,7 +3262,7 @@ METplus Configuration | :term:`LOG_IODA2NC_VERBOSITY` | :term:`IODA2NC_SKIP_IF_OUTPUT_EXISTS` | :term:`IODA2NC_CONFIG_FILE` -| :term:`IODA2NC_FILE_WINDOW_BEGIN` +| :term:`IODA2NC_FILE_WINDOW_BEG` | :term:`IODA2NC_FILE_WINDOW_END` | :term:`IODA2NC_VALID_BEG` | :term:`IODA2NC_VALID_END` From d2c15de899af804b71346074770bf74168e945ec Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 15:27:02 -0700 Subject: [PATCH 23/26] minor change to trigger automation --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index ad7269338..491bc2b57 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,7 +9,7 @@ "index_list": "30-58", "run": false }, - { + { "category": "met_tool_wrapper", "index_list": "59", "run": true From e1182262192dc3904c51e569eac3522e3553c84c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 15:41:42 -0700 Subject: [PATCH 24/26] change new use case to not run every time to prepare for PR --- .github/parm/use_case_groups.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 491bc2b57..4cb2de818 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,10 +9,10 @@ "index_list": "30-58", "run": false }, - { + { "category": "met_tool_wrapper", "index_list": "59", - "run": true + "run": false }, { "category": "air_quality_and_comp", From ee5e8b133a4c31de3a53d36a134a0faf2bf045f8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 16:12:01 -0700 Subject: [PATCH 25/26] Removed unit tests that were copied from another file that aren't actually applicable to this tool. The tests involve checking that legacy environment variables are set properly, but GenEnsProd is a new wrapper that does not need to support any legacy settings --- .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py index 871826c6d..ed27e0784 100644 --- a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -51,61 +51,6 @@ def set_minimum_config_settings(config): config.set('config', 'ENS_VAR1_NAME', ens_name) config.set('config', 'ENS_VAR1_LEVELS', ens_level) -@pytest.mark.parametrize( - 'config_overrides, env_var_values', [ - # 0 no climo settings - ({}, {}), - # 1 mean template only - ({'GEN_ENS_PROD_CLIMO_MEAN_INPUT_TEMPLATE': 'gs_mean_{init?fmt=%Y%m%d%H}.tmpl'}, - {'CLIMO_MEAN_FILE': '"gs_mean_YMDH.tmpl"', - 'CLIMO_STDEV_FILE': '', }), - # 2 mean template and dir - ({'GEN_ENS_PROD_CLIMO_MEAN_INPUT_TEMPLATE': 'gs_mean_{init?fmt=%Y%m%d%H}.tmpl', - 'GEN_ENS_PROD_CLIMO_MEAN_INPUT_DIR': '/climo/mean/dir'}, - {'CLIMO_MEAN_FILE': '"/climo/mean/dir/gs_mean_YMDH.tmpl"', - 'CLIMO_STDEV_FILE': '', }), - # 3 stdev template only - ({'GEN_ENS_PROD_CLIMO_STDEV_INPUT_TEMPLATE': 'gs_stdev_{init?fmt=%Y%m%d%H}.tmpl'}, - {'CLIMO_STDEV_FILE': '"gs_stdev_YMDH.tmpl"', }), - # 4 stdev template and dir - ({'GEN_ENS_PROD_CLIMO_STDEV_INPUT_TEMPLATE': 'gs_stdev_{init?fmt=%Y%m%d%H}.tmpl', - 'GEN_ENS_PROD_CLIMO_STDEV_INPUT_DIR': '/climo/stdev/dir'}, - {'CLIMO_STDEV_FILE': '"/climo/stdev/dir/gs_stdev_YMDH.tmpl"', }), - ] -) -def test_handle_climo_file_variables(metplus_config, config_overrides, - env_var_values): - """! Ensure that old and new variables for setting climo_mean and - climo_stdev are set to the correct values - """ - old_env_vars = ['CLIMO_MEAN_FILE', - 'CLIMO_STDEV_FILE'] - config = metplus_config() - - set_minimum_config_settings(config) - - # set config variable overrides - for key, value in config_overrides.items(): - config.set('config', key, value) - - wrapper = GenEnsProdWrapper(config) - assert wrapper.isOK - - all_cmds = wrapper.run_all_times() - assert len(all_cmds) == len(run_times) - for (_, actual_env_vars), run_time in zip(all_cmds, run_times): - run_dt = datetime.strptime(run_time, time_fmt) - ymdh = run_dt.strftime('%Y%m%d%H') - print(f"ACTUAL ENV VARS: {actual_env_vars}") - for old_env in old_env_vars: - match = next((item for item in actual_env_vars if - item.startswith(old_env)), None) - assert(match is not None) - actual_value = match.split('=', 1)[1] - expected_value = env_var_values.get(old_env, '') - expected_value = expected_value.replace('YMDH', ymdh) - assert(expected_value == actual_value) - @pytest.mark.parametrize( 'config_overrides, env_var_values', [ ({'MODEL': 'my_model'}, From dd5c3fc66a9115e670c22bbe6900c715453ded2d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:44:50 -0700 Subject: [PATCH 26/26] Removed missing line break --- docs/Users_Guide/glossary.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 62e284157..3b5cee3f5 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -8519,6 +8519,7 @@ METplus Configuration Glossary See :ref:`Overriding Unsupported MET config file settings` for more information | *Used by:* IODA2NC + ENSEMBLE_STAT_OBS_QUALITY_INC Specify the value for 'obs_quality_inc' in the MET configuration file for EnsembleStat.