fixed missing frequency for cmip6 cmor checks

esmvaltool/cmor/check.py

@@ -66,7 +66,7 @@ def __init__(self,
         self._errors = list()
         self._warnings = list()
         self._cmor_var = var_info
-        if frequency is None:
+        if frequency is None or frequency == '':
             frequency = self._cmor_var.frequency
         self.frequency = frequency
         self.automatic_fixes = automatic_fixes

esmvaltool/cmor/table.py

@@ -76,6 +76,7 @@ def __init__(self, cmor_tables_path, default=None):
         self.default = default
 
         self.tables = {}
+        self.var_to_freq = {}
 
         self._load_coordinates()
         for json_file in glob.glob(os.path.join(self._cmor_folder, '*.json')):
@@ -98,22 +99,19 @@ def _load_table(self, json_file):
                 return
             table = TableInfo()
             header = raw_data['Header']
-            table.name = header['table_id'][6:].split('_')[-1]
-            self.tables[table.name] = table
-
+            table.name = header['table_id'].split(' ')[-1]
             generic_levels = header['generic_levels'].split()
-            table.frequency = header.get('frequency', '')
-            table.realm = header.get('realm', '')
+            table.realm = header['realm'].split()
+            frequency = header.get('frequency', '')
+            self.var_to_freq[table.name] = {}
 
             for var_name, var_data in raw_data['variable_entry'].items():
                 var = VariableInfo('CMIP6', var_name)
-                if 'frequency' in var_data:
-                    var.frequency = var_data['frequency']
-                else:
-                    var.frequency = table.frequency
-                var.read_json(var_data)
+                var.read_json(var_data, frequency)
                 self._assign_dimensions(var, generic_levels)
                 table[var_name] = var
+                self.var_to_freq[table.name][var_name] = var.frequency
+            self.tables[table.name] = table
 
     def _assign_dimensions(self, var, generic_levels):
         for dimension in var.dimensions:
@@ -178,6 +176,8 @@ def get_variable(self, table, short_name):
 
         """
         try:
+            self.tables[table][short_name].frequency = \
+                self.var_to_freq[table][short_name]
             return self.tables[table][short_name]
         except KeyError:
             if short_name in CMIP6Info._CMIP_5to6_varname:
@@ -217,7 +217,7 @@ class JsonInfo(object):
     def __init__(self):
         self._json_data = {}
 
-    def _read_json_variable(self, parameter):
+    def _read_json_variable(self, parameter, default=''):
         """
         Read a json parameter in json_data.
 
@@ -233,7 +233,7 @@ def _read_json_variable(self, parameter):
 
         """
         if parameter not in self._json_data:
-            return ''
+            return default
         return str(self._json_data[parameter])
 
     def _read_json_list_variable(self, parameter):
@@ -297,7 +297,7 @@ def __init__(self, table_type, short_name):
 
         self._json_data = None
 
-    def read_json(self, json_data):
+    def read_json(self, json_data, default_freq=''):
         """
         Read variable information from json.
 
@@ -309,6 +309,9 @@ def read_json(self, json_data):
             dictionary created by the json reader containing
             variable information
 
+        default_freq: str
+            Default frequency to use if it is not defined at variable level
+
         """
         self._json_data = json_data
 
@@ -320,6 +323,7 @@ def read_json(self, json_data):
         self.positive = self._read_json_variable('positive')
         self.modeling_realm = \
             self._read_json_variable('modeling_realm').split()
+        self.frequency = self._read_json_variable('frequency', default_freq)
 
         self.dimensions = self._read_json_variable('dimensions').split()
 
@@ -576,6 +580,7 @@ def __init__(self, cmor_tables_path=None):
         cwd = os.path.dirname(os.path.realpath(__file__))
         self._cmor_folder = os.path.join(cwd, 'tables', 'custom')
         self.tables = {}
+        self.var_to_freq = {}
         table = TableInfo()
         table.name = 'custom'
         self.tables[table.name] = table
@@ -626,9 +631,20 @@ def get_variable(self, table, short_name):
             found, returns None if not
 
         """
-        return self.tables['custom'].get(short_name, None)
+        var_info = self.tables['custom'].get(short_name, None)
+        if var_info is not None:
+            var_info.frequency = self.var_to_freq[short_name]
+        return var_info
 
     def _read_table_file(self, table_file, table=None):
+        with open(table_file, 'r') as cmor_table:
+            all_lines = cmor_table.readlines()
+        all_lines = [x.strip() for x in all_lines]
+        keys_vals = [x.split(':') for x in all_lines]
+        var_frequency = None
+        for key in keys_vals:
+            if key[0] == 'frequency':
+                var_frequency = key[1].strip()
         with open(table_file) as self._current_table:
             self._read_line()
             while True:
@@ -643,7 +659,8 @@ def _read_table_file(self, table_file, table=None):
                     self.coords[value] = self._read_coordinate(value)
                     continue
                 elif key == 'variable_entry':
-                    table[value] = self._read_variable(value, None)
+                    self.var_to_freq[value] = var_frequency
+                    table[value] = self._read_variable(value, var_frequency)
                     continue
                 if not self._read_line():
                     return