feature 880 improve field info handling (#881)

internal_tests/pytests/met_util/test_met_util.py

@@ -319,6 +319,66 @@ def test_parse_var_list_series_by(metplus_config):
         for key, value in expected_sa.items():
             assert(actual_sa.get(key) == value)
 
+@pytest.mark.parametrize(
+    'input_dict, expected_list', [
+        ({'init': datetime.datetime(2019, 2, 1, 6),
+          'lead': 7200, },
+         [
+             {'index': '1',
+              'fcst_name': 'FNAME_2019',
+              'fcst_level': 'Z06',
+              'obs_name': 'ONAME_2019',
+              'obs_level': 'L06',
+             },
+             {'index': '1',
+              'fcst_name': 'FNAME_2019',
+              'fcst_level': 'Z08',
+              'obs_name': 'ONAME_2019',
+              'obs_level': 'L08',
+             },
+         ]),
+        ({'init': datetime.datetime(2021, 4, 13, 9),
+          'lead': 10800, },
+         [
+             {'index': '1',
+              'fcst_name': 'FNAME_2021',
+              'fcst_level': 'Z09',
+              'obs_name': 'ONAME_2021',
+              'obs_level': 'L09',
+              },
+             {'index': '1',
+              'fcst_name': 'FNAME_2021',
+              'fcst_level': 'Z12',
+              'obs_name': 'ONAME_2021',
+              'obs_level': 'L12',
+              },
+         ]),
+    ]
+)
+def test_sub_var_list(metplus_config, input_dict, expected_list):
+    config = metplus_config()
+    config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}')
+    config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}')
+    config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}')
+    config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}')
+
+    time_info = time_util.ti_calculate(input_dict)
+
+    actual_temp = util.parse_var_list(config)
+
+    pp = pprint.PrettyPrinter()
+    print(f'Actual var list (before sub):')
+    pp.pprint(actual_temp)
+
+    actual_list = util.sub_var_list(actual_temp, time_info)
+    print(f'Actual var list (after sub):')
+    pp.pprint(actual_list)
+
+    assert(len(actual_list) == len(expected_list))
+    for actual, expected in zip(actual_list, expected_list):
+        for key, value in expected.items():
+            assert(actual.get(key) == value)
+
 @pytest.mark.parametrize(
     'config_var_name, expected_indices, set_met_tool', [
         ('FCST_GRID_STAT_VAR1_NAME', ['1'], True),

internal_tests/pytests/series_analysis/test_series_analysis.py

@@ -7,7 +7,7 @@
 
 import produtil
 
-from metplus.util import ti_get_seconds_from_lead
+from metplus.util import ti_get_seconds_from_lead, sub_var_list
 from metplus.wrappers.series_analysis_wrapper import SeriesAnalysisWrapper
 
 def series_init_wrapper(metplus_config, config_overrides=None):
@@ -465,6 +465,12 @@ def test_create_ascii_storm_files_list(metplus_config, config_overrides,
     if os.path.exists(obs_file_path):
         os.remove(obs_file_path)
 
+    # perform string substitution on var list
+    wrapper.c_dict['VAR_LIST'] = (
+        sub_var_list(wrapper.c_dict['VAR_LIST_TEMP'],
+                     time_info)
+    )
+
     fcst_path, obs_path = wrapper.create_ascii_storm_files_list(time_info,
                                                                 storm_id,
                                                                 lead_group)

metplus/util/met_util.py

@@ -2147,7 +2147,7 @@ def get_field_config_variables(config, index, search_prefixes):
 
     return field_configs
 
-def format_var_items(field_configs, time_info):
+def format_var_items(field_configs, time_info=None):
     """! Substitute time information into field information and format values.
 
         @param field_configs dictionary with config variable names to read
@@ -2172,15 +2172,18 @@ def format_var_items(field_configs, time_info):
         return 'Name not found'
 
     # perform string substitution on name
-    var_items['name'] = do_string_sub(search_name,
-                                      skip_missing_tags=True,
-                                      **time_info)
+    if time_info:
+        search_name = do_string_sub(search_name,
+                                    skip_missing_tags=True,
+                                    **time_info)
+    var_items['name'] = search_name
 
     # get levels, performing string substitution on each item of list
     for level in getlist(field_configs.get('levels')):
-        subbed_level = do_string_sub(level,
-                                     **time_info)
-        var_items['levels'].append(subbed_level)
+        if time_info:
+            level = do_string_sub(level,
+                                  **time_info)
+        var_items['levels'].append(level)
 
     # if no levels are found, add an empty string
     if not var_items['levels']:
@@ -2199,11 +2202,12 @@ def format_var_items(field_configs, time_info):
     # get extra options if it is set, format with semi-colons between items
     search_extra = field_configs.get('options')
     if search_extra:
-        extra = do_string_sub(search_extra,
-                              **time_info)
+        if time_info:
+            search_extra = do_string_sub(search_extra,
+                                         **time_info)
 
         # strip off empty space around each value
-        extra_list = [item.strip() for item in extra.split(';')]
+        extra_list = [item.strip() for item in search_extra.split(';')]
 
         # split up each item by semicolon, then add a semicolon to the end
         # use list(filter(None to remove empty strings from list
@@ -2219,8 +2223,9 @@ def format_var_items(field_configs, time_info):
             var_items['output_names'].append(var_items['name'])
     else:
         for out_name in getlist(out_name_str):
-            out_name = do_string_sub(out_name,
-                                     **time_info)
+            if time_info:
+                out_name = do_string_sub(out_name,
+                                         **time_info)
             var_items['output_names'].append(out_name)
 
     if len(var_items['levels']) != len(var_items['output_names']):
@@ -2271,11 +2276,11 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None):
     # if time_info is not passed in, set 'now' to CLOCK_TIME
     # NOTE: any attempt to use string template substitution with an item other
     # than 'now' will fail if time_info is not passed into parse_var_list
-    if time_info is None:
-        time_info = {'now': datetime.datetime.strptime(
-            config.getstr('config', 'CLOCK_TIME'),
-            '%Y%m%d%H%M%S')
-        }
+#    if time_info is None:
+#        time_info = {'now': datetime.datetime.strptime(
+#            config.getstr('config', 'CLOCK_TIME'),
+#            '%Y%m%d%H%M%S')
+#        }
 
     # var_list is a list containing an list of dictionaries
     var_list = []
@@ -2350,7 +2355,9 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None):
                 output_name = field_info.get('output_names')[level_index]
 
                 # substitute level in name if filename template is specified
-                subbed_name = do_string_sub(name, **sub_info)
+                subbed_name = do_string_sub(name,
+                                            skip_missing_tags=True,
+                                            **sub_info)
 
                 var_dict[f"{current_type}_name"] = subbed_name
                 var_dict[f"{current_type}_level"] = level
@@ -2395,6 +2402,41 @@ def parse_var_list(config, time_info=None, data_type=None, met_tool=None):
     '''
     return sorted(var_list, key=lambda x: x['index'])
 
+def sub_var_info(var_info, time_info):
+    if not var_info:
+        return {}
+
+    out_var_info = {}
+    for key, value in var_info.items():
+        if isinstance(value, list):
+            out_value = []
+            for item in value:
+                out_value.append(do_string_sub(item, **time_info))
+        else:
+            out_value = do_string_sub(value,
+                                      **time_info)
+
+        out_var_info[key] = out_value
+
+    return out_var_info
+
+def sub_var_list(var_list, time_info):
+    """! Perform string substitution on var list values with time info
+
+        @param var_list list of field info to substitute values into
+        @param time_info dictionary containing time information
+        @returns var_list with values substituted
+    """
+    if not var_list:
+        return []
+
+    out_var_list = []
+    for var_info in var_list:
+        out_var_info = sub_var_info(var_info, time_info)
+        out_var_list.append(out_var_info)
+
+    return out_var_list
+
 def split_level(level):
     level_type = ""
     if not level:

metplus/wrappers/compare_gridded_wrapper.py

@@ -99,6 +99,9 @@ def create_c_dict(self):
                                    'output_prefix',
                                    'METPLUS_OUTPUT_PREFIX')
 
+        c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config,
+                                                      met_tool=self.app_name)
+
         return c_dict
 
     def set_environment_variables(self, time_info):
@@ -176,9 +179,8 @@ def run_at_time_once(self, time_info):
         # get verification mask if available
         self.get_verification_mask(time_info)
 
-        var_list = util.parse_var_list(self.config,
-                                       time_info,
-                                       met_tool=self.app_name)
+        var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'],
+                                     time_info)
 
         if not var_list and not self.c_dict.get('VAR_LIST_OPTIONAL', False):
             self.log_error('No input fields were specified. You must set '
@@ -258,9 +260,8 @@ def run_at_time_all_fields(self, time_info):
               Args:
                 @param time_info dictionary containing timing information
         """
-        var_list = util.parse_var_list(self.config,
-                                       time_info,
-                                       met_tool=self.app_name)
+        var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'],
+                                     time_info)
 
         # get model from first var to compare
         model_path = self.find_model(time_info,

metplus/wrappers/ensemble_stat_wrapper.py

@@ -315,6 +315,19 @@ def create_c_dict(self):
         # field information for fcst and obs
         c_dict['VAR_LIST_OPTIONAL'] = True
 
+        # parse var list for ENS fields
+        c_dict['ENS_VAR_LIST_TEMP'] = util.parse_var_list(
+            self.config,
+            data_type='ENS',
+            met_tool=self.app_name
+        )
+
+        # parse optional var list for FCST and/or OBS fields
+        c_dict['VAR_LIST_TEMP'] = util.parse_var_list(
+            self.config,
+            met_tool=self.app_name
+        )
+
         return c_dict
 
     def handle_nmep_smooth_dict(self):
@@ -461,13 +474,12 @@ def run_at_time_all_fields(self, time_info):
         self.infiles.append(fcst_file_list)
 
         # parse var list for ENS fields
-        ensemble_var_list = util.parse_var_list(self.config,
-                                                time_info,
-                                                data_type='ENS',
-                                                met_tool=self.app_name)
+        ensemble_var_list = util.sub_var_list(self.c_dict['ENS_VAR_LIST_TEMP'],
+                                              time_info)
 
         # parse optional var list for FCST and/or OBS fields
-        var_list = util.parse_var_list(self.config, time_info, met_tool=self.app_name)
+        var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'],
+                                     time_info)
 
         # if empty var list for FCST/OBS, use None as first var, else use first var in list
         if not var_list:

metplus/wrappers/extract_tiles_wrapper.py

@@ -92,6 +92,8 @@ def create_c_dict(self):
         c_dict['LON_ADJ'] = self.config.getfloat('config',
                                                  'EXTRACT_TILES_LON_ADJ')
 
+        c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config,
+                                                      met_tool=self.app_name)
         return c_dict
 
     def regrid_data_plane_init(self):
@@ -215,9 +217,7 @@ def create_tiles_from_storm(self, storm_id, storm_lines, idx_dict):
                                                            storm_data)
 
             # set var list from config using time info
-            var_list = util.parse_var_list(self.config,
-                                           time_info,
-                                           met_tool=self.app_name)
+            var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info)
 
             # set output grid and run for the forecast and observation data
             for dtype in ['FCST', 'OBS']:

metplus/wrappers/grid_diag_wrapper.py

@@ -81,6 +81,10 @@ def create_c_dict(self):
 
         self.handle_censor_val_and_thresh()
 
+        c_dict['VAR_LIST_TEMP'] = util.parse_var_list(self.config,
+                                                      data_type='FCST',
+                                                      met_tool=self.app_name)
+
         c_dict['MASK_POLY_TEMPLATE'] = self.read_mask_poly()
 
         return c_dict
@@ -194,11 +198,7 @@ def set_data_field(self, time_info):
                 @param time_info time dictionary to use for string substitution
                 @returns True if field list could be built, False if not.
         """
-
-        field_list = util.parse_var_list(self.config,
-                                         time_info,
-                                         data_type='FCST',
-                                         met_tool=self.app_name)
+        field_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info)
         if not field_list:
             self.log_error("Could not get field information from config.")
             return False