First cut of hybrid-pressure for GRIB.

lib/iris/etc/grib_cross_reference_rules.txt

@@ -15,4 +15,13 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with Iris.  If not, see <http://www.gnu.org/licenses/>.
 
-# PLACEHOLDER
+# Equivalent to grib.paramId == 152
+IF
+grib.edition == 2
+grib.centre == 'ecmf'
+grib.discipline == 0
+grib.parameterCategory == 3
+grib.parameterNumber == 25
+grib.typeOfFirstFixedSurface == 105
+THEN
+ReferenceTarget('surface_pressure', lambda cube: {'standard_name': 'surface_air_pressure', 'units': 'Pa', 'data': numpy.exp(cube.data)})

lib/iris/etc/grib_rules.txt

@@ -582,16 +582,18 @@ CoordAndDims(DimCoord(points=0.5*(grib.scaledValueOfFirstFixedSurface/(10.0**gri
 #ExplicitCoord("model_level", "1", 'z', points=grib.scaledValueOfFirstFixedSurface)
 #ExplicitCoord('level_height', 'm', 'z', points=grib.pv[grib.scaledValueOfFirstFixedSurface], definitive=True, coord_system=HybridHeightCS(Reference('orography')))
 #ExplicitCoord('sigma', '1', 'z', points=grib.pv[grib.numberOfVerticalCoordinateValues/2 + grib.scaledValueOfFirstFixedSurface], coord_system=HybridHeightCS(Reference('orography')))
-#
-# hybrid pressure.105 deprecated for 119.
-#IF
-#grib.edition == 2
-#grib.typeOfFirstFixedSurface in [105, 119]
-#grib.numberOfCoordinatesValues > 0
-#THEN
-#ExplicitCoord("model_level", "1", 'z', points=grib.scaledValueOfFirstFixedSurface)
-#ExplicitCoord('level_height', 'm', 'z', points=grib.pv[grib.scaledValueOfFirstFixedSurface], definitive=True, coord_system=HybridPressureCS(Reference('surface_pressure')))
-#ExplicitCoord('sigma', '1', 'z', points=grib.pv[grib.numberOfPoints/2 + grib.scaledValueOfFirstFixedSurface], coord_system=HybridPressureCS(Reference('surface_pressure')))
+
+
+# hybrid pressure. 105 deprecated for 119.
+IF
+grib.edition == 2
+grib.typeOfFirstFixedSurface in [105, 119]
+grib.numberOfCoordinatesValues > 0
+THEN
+CoordAndDims(AuxCoord(grib.scaledValueOfFirstFixedSurface, standard_name='model_level_number', attributes={'positive': 'up'}))
+CoordAndDims(DimCoord(grib.pv[grib.scaledValueOfFirstFixedSurface], long_name='level_pressure', units='Pa'))
+CoordAndDims(AuxCoord(grib.pv[grib.numberOfCoordinatesValues/2 + grib.scaledValueOfFirstFixedSurface], long_name='sigma'))
+Factory(HybridPressureFactory, [{'long_name': 'level_pressure'}, {'long_name': 'sigma'}, Reference('surface_pressure')])
 
 
 

lib/iris/etc/pp_cross_reference_rules.txt

@@ -19,4 +19,4 @@
 IF
 f.lbuser[3] == 33
 THEN
-Reference('orography',)
+ReferenceTarget('orography', None)

lib/iris/fileformats/rules.py

@@ -42,6 +42,46 @@
 
 RuleResult = collections.namedtuple('RuleResult', ['cube', 'matching_rules', 'factories'])
 Factory = collections.namedtuple('Factory', ['factory_class', 'args'])
+ReferenceTarget = collections.namedtuple('ReferenceTarget',
+                                         ('name', 'transform'))
+
+
+class ConcreteReferenceTarget(object):
+    """Everything you need to make a real Cube for a named reference."""
+
+    def __init__(self, name, transform=None):
+        self.name = name
+        """The name used to connect references with referencees."""
+        self.transform = transform
+        """An optional transformation to apply to the cubes."""
+        self._src_cubes = iris.cube.CubeList()
+        self._final_cube = None
+
+    def add_cube(self, cube):
+        self._src_cubes.append(cube)
+
+    def as_cube(self):
+        if self._final_cube is None:
+            src_cubes = self._src_cubes
+            if len(src_cubes) > 1:
+                # Merge the reference cubes to allow for
+                # time-varying surface pressure in hybrid-presure.
+                src_cubes = src_cubes.merge()
+                if len(src_cubes) > 1:
+                    warnings.warn('Multiple reference cubes for {}'
+                                  .format(self.name))
+            src_cube = src_cubes[-1]
+
+            if self.transform is None:
+                self._final_cube = src_cube
+            else:
+                final_cube = src_cube.copy()
+                attributes = self.transform(final_cube)
+                for name, value in attributes.iteritems():
+                    setattr(final_cube, name, value)
+                self._final_cube = final_cube
+
+        return self._final_cube
 
 
 # Controls the deferred import of all the symbols from iris.coords.
@@ -343,7 +383,7 @@ def run_actions(self, cube, field):
             except AttributeError, err:
                 print >> sys.stderr, 'Failed to get value (%(error)s) to execute: %(command)s' % {'command':action, 'error': err}
             except Exception, err:
-                print >> sys.stderr, 'Failed (msg:%(error)s) to run: %(command)s\nFrom the rule:%(me)r' % {'me':self, 'command':action, 'error': err}
+                print >> sys.stderr, 'Failed (msg:%(error)s) to run:\n    %(command)s\nFrom the rule:\n%(me)r' % {'me':self, 'command':action, 'error': err}
                 raise err
         return factories
 
@@ -591,22 +631,13 @@ class _ReferenceError(Exception):
     pass
 
 
-def _dereference_args(factory, reference_cubes, regrid_cache, cube):
+def _dereference_args(factory, reference_targets, regrid_cache, cube):
     """Converts all the arguments for a factory into concrete coordinates."""
     args = []
     for arg in factory.args:
-        if isinstance(arg, iris.fileformats.rules.Reference):
-            if arg.name in reference_cubes:
-                # Merge the reference cubes to allow for
-                # time-varying surface pressure in hybrid-presure.
-                if len(reference_cubes[arg.name]) > 1:
-                    ref_cubes = iris.cube.CubeList(reference_cubes[arg.name])
-                    merged = ref_cubes.merge()
-                    if len(merged) > 1:
-                        warnings.warn('Multiple reference cubes for {}'
-                                      .format(arg.name))
-                    reference_cubes[arg.name] = merged[-1:]
-                src = reference_cubes[arg.name][0]
+        if isinstance(arg, Reference):
+            if arg.name in reference_targets:
+                src = reference_targets[arg.name].as_cube()
                 # If necessary, regrid the reference cube to
                 # match the grid of this cube.
                 src = _ensure_aligned(regrid_cache, src, cube)
@@ -708,7 +739,7 @@ def _ensure_aligned(regrid_cache, src_cube, target_cube):
 
 
 def load_cubes(filenames, user_callback, loader):
-    reference_cubes = {}
+    concrete_reference_targets = {}
     results_needing_reference = []
 
     if isinstance(filenames, basestring):
@@ -730,7 +761,14 @@ def load_cubes(filenames, user_callback, loader):
             rules = loader.cross_ref_rules.matching_rules(field)
             for rule in rules:
                 reference, = rule.run_actions(cube, field)
-                reference_cubes.setdefault(reference.name, []).append(cube)
+                name = reference.name
+                # Register this cube as a source cube for the named
+                # reference.
+                target = concrete_reference_targets.get(name)
+                if target is None:
+                    target = ConcreteReferenceTarget(name, reference.transform)
+                    concrete_reference_targets[name] = target
+                target.add_cube(cube)
 
             if rules_result.factories:
                 results_needing_reference.append(rules_result)
@@ -742,7 +780,7 @@ def load_cubes(filenames, user_callback, loader):
         cube = result.cube
         for factory in result.factories:
             try:
-                args = _dereference_args(factory, reference_cubes,
+                args = _dereference_args(factory, concrete_reference_targets,
                                          regrid_cache, cube)
             except _ReferenceError as e:
                 msg = 'Unable to create instance of {factory}. ' + e.message

lib/iris/tests/test_rules.py

@@ -0,0 +1,187 @@
+# (C) British Crown Copyright 2010 - 2012, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Test metadata translation rules.
+
+"""
+# import iris tests first so that some things can be initialised before importing anything else
+import iris.tests as tests
+
+import os
+import types
+
+from iris.aux_factory import HybridHeightFactory
+from iris.fileformats.rules import ConcreteReferenceTarget, Factory, Loader, \
+                                   Reference, ReferenceTarget, RuleResult, \
+                                   load_cubes
+import iris.tests.stock as stock
+
+
+class Mock(object):
+    def __repr__(self):
+        return '<Mock {!r}>'.format(self.__dict__)
+
+
+class TestConcreteReferenceTarget(tests.IrisTest):
+    def test_attributes(self):
+        with self.assertRaises(TypeError):
+            target = ConcreteReferenceTarget()
+
+        target = ConcreteReferenceTarget('foo')
+        self.assertEqual(target.name, 'foo')
+        self.assertIsNone(target.transform)
+
+        transform = lambda _: _
+        target = ConcreteReferenceTarget('foo', transform)
+        self.assertEqual(target.name, 'foo')
+        self.assertIs(target.transform, transform)
+
+    def test_single_cube_no_transform(self):
+        target = ConcreteReferenceTarget('foo')
+        src = stock.simple_2d()
+        target.add_cube(src)
+        self.assertIs(target.as_cube(), src)
+
+    def test_single_cube_with_transform(self):
+        transform = lambda cube: {'long_name': 'wibble'}
+        target = ConcreteReferenceTarget('foo', transform)
+        src = stock.simple_2d()
+        target.add_cube(src)
+        dest = target.as_cube()
+        self.assertEqual(dest.long_name, 'wibble')
+        self.assertNotEqual(dest, src)
+        dest.long_name = src.long_name
+        self.assertEqual(dest, src)
+
+    def test_multiple_cubes_no_transform(self):
+        target = ConcreteReferenceTarget('foo')
+        src = stock.realistic_4d()
+        for i in range(src.shape[0]):
+            target.add_cube(src[i])
+        dest = target.as_cube()
+        self.assertIsNot(dest, src)
+        self.assertEqual(dest, src)
+
+    def test_multiple_cubes_with_transform(self):
+        transform = lambda cube: {'long_name': 'wibble'}
+        target = ConcreteReferenceTarget('foo', transform)
+        src = stock.realistic_4d()
+        for i in range(src.shape[0]):
+            target.add_cube(src[i])
+        dest = target.as_cube()
+        self.assertEqual(dest.long_name, 'wibble')
+        self.assertNotEqual(dest, src)
+        dest.long_name = src.long_name
+        self.assertEqual(dest, src)
+
+
+class TestLoadCubes(tests.IrisTest):
+    def test_simple_factory(self):
+        # Test the creation process for a factory definition which only
+        # uses simple dict arguments.
+        field = Mock()
+        field_generator = lambda filename: [field]
+        # A fake rule set returning:
+        #   1) A parameter cube needing a simple factory construction.
+        src_cube = Mock()
+        src_cube.coord = lambda **args: args
+        src_cube.add_aux_factory = lambda aux_factory: \
+            setattr(src_cube, 'fake_aux_factory', aux_factory)
+        aux_factory = Mock()
+        factory = Mock()
+        factory.args = [{'name': 'foo'}]
+        factory.factory_class = lambda *args: \
+            setattr(aux_factory, 'fake_args', args) or aux_factory
+        rule_result = RuleResult(src_cube, Mock(), [factory])
+        rules = Mock()
+        rules.result = lambda field: rule_result
+        # A fake cross-reference rule set
+        xref_rules = Mock()
+        xref_rules.matching_rules = lambda field: []
+        # Finish by making a fake Loader
+        name = 'FAKE_PP'
+        fake_loader = Loader(field_generator, rules, xref_rules, name)
+        cubes = load_cubes(['fake_filename'], None, fake_loader)
+        # Check the result is a generator with our "cube" as the only
+        # entry.
+        self.assertIsInstance(cubes, types.GeneratorType)
+        cubes = list(cubes)
+        self.assertEqual(len(cubes), 1)
+        self.assertIs(cubes[0], src_cube)
+        # Check the "cube" has an "aux_factory" added, which itself
+        # must have been created with the correct arguments.
+        self.assertTrue(hasattr(src_cube, 'fake_aux_factory'))
+        self.assertIs(src_cube.fake_aux_factory, aux_factory)
+        self.assertTrue(hasattr(aux_factory, 'fake_args'))
+        self.assertEqual(aux_factory.fake_args, ({'name': 'foo'},))
+
+    def test_cross_reference(self):
+        # Test the creation process for a factory definition which uses
+        # a cross-reference.
+
+        param_cube = stock.realistic_4d_no_derived()
+        orog_coord = param_cube.coord('surface_altitude')
+        param_cube.remove_coord(orog_coord)
+
+        orog_cube = param_cube[0, 0, :, :]
+        orog_cube.data = orog_coord.points
+        orog_cube.rename('surface_altitude')
+        orog_cube.units = orog_coord.units
+        orog_cube.attributes = orog_coord.attributes
+
+        # We're going to test for the presence of the hybrid height
+        # stuff later, so let's make sure it's not already there!
+        assert len(param_cube.aux_factories) == 0
+        assert not param_cube.coords('surface_altitude')
+
+        press_field = Mock()
+        orog_field = Mock()
+        field_generator = lambda filename: [press_field, orog_field]
+        # A fake rule set returning:
+        #   1) A parameter cube needing an "orography" reference
+        #   2) An "orography" cube
+        factory = Factory(HybridHeightFactory, [Reference('orography')])
+        press_rule_result = RuleResult(param_cube, Mock(), [factory])
+        orog_rule_result= RuleResult(orog_cube, Mock(), [])
+        rules = Mock()
+        rules.result = lambda field: \
+            press_rule_result if field is press_field else orog_rule_result
+        # A fake cross-reference rule set
+        ref = ReferenceTarget('orography', None)
+        orog_xref_rule = Mock()
+        orog_xref_rule.run_actions = lambda cube, field: (ref,)
+        xref_rules = Mock()
+        xref_rules.matching_rules = lambda field: \
+            [orog_xref_rule] if field is orog_field else []
+        # Finish by making a fake Loader
+        name = 'FAKE_PP'
+        fake_loader = Loader(field_generator, rules, xref_rules, name)
+        cubes = load_cubes(['fake_filename'], None, fake_loader)
+        # Check the result is a generator containing both of our cubes.
+        self.assertIsInstance(cubes, types.GeneratorType)
+        cubes = list(cubes)
+        self.assertEqual(len(cubes), 2)
+        self.assertIs(cubes[0], orog_cube)
+        self.assertIs(cubes[1], param_cube)
+        # Check the "cube" has an "aux_factory" added, which itself
+        # must have been created with the correct arguments.
+        self.assertEqual(len(param_cube.aux_factories), 1)
+        self.assertEqual(len(param_cube.coords('surface_altitude')), 1)
+
+
+if __name__ == "__main__":
+    tests.main()