fixed missing frequency for cmip6 cmor checks

esmvaltool/cmor/check.py

@@ -64,7 +64,7 @@ def __init__(self,
         self._errors = list()
         self._warnings = list()
         self._cmor_var = var_info
-        if frequency is None:
+        if not frequency:
             frequency = self._cmor_var.frequency
         self.frequency = frequency
         self.automatic_fixes = automatic_fixes

esmvaltool/cmor/table.py

@@ -76,6 +76,7 @@ def __init__(self, cmor_tables_path, default=None):
         self.default = default
 
         self.tables = {}
+        self.var_to_freq = {}
 
         self._load_coordinates()
         for json_file in glob.glob(os.path.join(self._cmor_folder, '*.json')):
@@ -98,22 +99,25 @@ def _load_table(self, json_file):
                 return
             table = TableInfo()
             header = raw_data['Header']
-            table.name = header['table_id'][6:].split('_')[-1]
-            self.tables[table.name] = table
-
+            table.name = header['table_id'].split(' ')[-1]
             generic_levels = header['generic_levels'].split()
+            table.realm = header['realm'].split()
             table.frequency = header.get('frequency', '')
-            table.realm = header.get('realm', '')
+            self.var_to_freq[table.name] = {}
 
             for var_name, var_data in raw_data['variable_entry'].items():
                 var = VariableInfo('CMIP6', var_name)
-                if 'frequency' in var_data:
-                    var.frequency = var_data['frequency']
-                else:
-                    var.frequency = table.frequency
-                var.read_json(var_data)
+                var.read_json(var_data, table.frequency)
                 self._assign_dimensions(var, generic_levels)
                 table[var_name] = var
+                self.var_to_freq[table.name][var_name] = var.frequency
+
+            if not table.frequency:
+                from collections import Counter
+                var_freqs = (var.frequency for var in table.values())
+                table_freq, _ = Counter(var_freqs).most_common(1)[0]
+                table.frequency = table_freq
+            self.tables[table.name] = table
 
     def _assign_dimensions(self, var, generic_levels):
         for dimension in var.dimensions:
@@ -184,7 +188,12 @@ def get_variable(self, table, short_name):
                 new_short_name = CMIP6Info._CMIP_5to6_varname[short_name]
                 return self.get_variable(table, new_short_name)
             if self.default:
-                return self.default.get_variable(table, short_name)
+                var_info = self.default.get_variable(table, short_name)
+                if var_info is None:
+                    return None
+                var_info = var_info.copy()
+                var_info.frequency = self.tables[table].frequency
+                return var_info
             return None
 
     @staticmethod
@@ -217,7 +226,7 @@ class JsonInfo(object):
     def __init__(self):
         self._json_data = {}
 
-    def _read_json_variable(self, parameter):
+    def _read_json_variable(self, parameter, default=''):
         """
         Read a json parameter in json_data.
 
@@ -233,7 +242,7 @@ def _read_json_variable(self, parameter):
 
         """
         if parameter not in self._json_data:
-            return ''
+            return default
         return str(self._json_data[parameter])
 
     def _read_json_list_variable(self, parameter):
@@ -297,7 +306,20 @@ def __init__(self, table_type, short_name):
 
         self._json_data = None
 
-    def read_json(self, json_data):
+    def copy(self):
+        """
+        Returns a shalow copy of VariableInfo
+
+        Returns
+        -------
+        VariableInfo
+           Shallow copy of this object
+
+        """
+        from copy import copy
+        return copy(self)
+
+    def read_json(self, json_data, default_freq=''):
         """
         Read variable information from json.
 
@@ -309,6 +331,9 @@ def read_json(self, json_data):
             dictionary created by the json reader containing
             variable information
 
+        default_freq: str
+            Default frequency to use if it is not defined at variable level
+
         """
         self._json_data = json_data
 
@@ -320,6 +345,7 @@ def read_json(self, json_data):
         self.positive = self._read_json_variable('positive')
         self.modeling_realm = \
             self._read_json_variable('modeling_realm').split()
+        self.frequency = self._read_json_variable('frequency', default_freq)
 
         self.dimensions = self._read_json_variable('dimensions').split()
 
@@ -556,7 +582,12 @@ def get_variable(self, table, short_name):
         """
         var_info = self.tables.get(table, {}).get(short_name, None)
         if not var_info and self.default:
-            return self.default.get_variable(table, short_name)
+            var_info = self.default.get_variable(table, short_name)
+            if var_info is None:
+                return None
+            var_info = var_info.copy()
+            var_info.frequency = self.tables[table].frequency
+            return var_info
         return var_info
 
 
@@ -576,6 +607,7 @@ def __init__(self, cmor_tables_path=None):
         cwd = os.path.dirname(os.path.realpath(__file__))
         self._cmor_folder = os.path.join(cwd, 'tables', 'custom')
         self.tables = {}
+        self.var_to_freq = {}
         table = TableInfo()
         table.name = 'custom'
         self.tables[table.name] = table
@@ -626,9 +658,18 @@ def get_variable(self, table, short_name):
             found, returns None if not
 
         """
-        return self.tables['custom'].get(short_name, None)
+        var_info = self.tables['custom'].get(short_name, None)
+        return var_info
 
     def _read_table_file(self, table_file, table=None):
+        with open(table_file, 'r') as cmor_table:
+            all_lines = cmor_table.readlines()
+        all_lines = [x.strip() for x in all_lines]
+        keys_vals = [x.split(':') for x in all_lines]
+        var_frequency = None
+        for key in keys_vals:
+            if key[0] == 'frequency':
+                var_frequency = key[1].strip()
         with open(table_file) as self._current_table:
             self._read_line()
             while True:
@@ -643,7 +684,8 @@ def _read_table_file(self, table_file, table=None):
                     self.coords[value] = self._read_coordinate(value)
                     continue
                 elif key == 'variable_entry':
-                    table[value] = self._read_variable(value, None)
+                    self.var_to_freq[value] = var_frequency
+                    table[value] = self._read_variable(value, var_frequency)
                     continue
                 if not self._read_line():
                     return