adopt eccodes, use gribapi.grib_is_missing

SciTools · Jul 25, 2017 · 1d8bf52 · 1d8bf52
1 parent 077b6ee
commit 1d8bf52
Show file tree

Hide file tree

Showing 20 changed files with 72 additions and 84 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -59,7 +59,7 @@ install:
       conda install --quiet --file minimal-conda-requirements.txt;
     else
       if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
-        sed -e '/ecmwf_grib/d' -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
+        sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
       else
         conda install --quiet --file conda-requirements.txt;
       fi

diff --git a/INSTALL b/INSTALL
@@ -122,11 +122,12 @@ gdal 1.9.1 or later (https://pypi.python.org/pypi/GDAL/)
 graphviz 2.18 or later (http://www.graphviz.org/)
     Graph visualisation software.
 
-grib-api 1.9.16 or later
-    (https://software.ecmwf.int/wiki/display/GRIB/Releases)
+eccodes
+    (https://software.ecmwf.int/wiki/display/ECC/ecCodes+Home)
     API for the encoding and decoding WMO FM-92 GRIB edition 1 and 
     edition 2 messages. A compression library such as Jasper is required 
     to read JPEG2000 compressed GRIB2 files.
+    Successor to GRIBAPI.
 
 matplotlib 1.2.0 (http://matplotlib.sourceforge.net/)
     Python package for 2D plotting.  

diff --git a/conda-requirements.txt b/conda-requirements.txt
@@ -25,7 +25,7 @@ imagehash
 requests
 
 # Optional iris dependencies
-python-ecmwf_grib
+python-eccodes
 esmpy>=7.0
 gdal
 libmo_unpack

diff --git a/lib/iris/fileformats/grib/_load_convert.py b/lib/iris/fileformats/grib/_load_convert.py
@@ -158,7 +158,9 @@ def _unscale(v, f):
 
     if isinstance(value, Iterable) or isinstance(factor, Iterable):
         def _masker(item):
-            result = ma.masked_equal(item, _MDI)
+            numerical_mdi = 2 ** 32 - 1
+            item = [numerical_mdi if i is None else i for i in item]
+            result = ma.masked_equal(item, numerical_mdi)
             if ma.count_masked(result):
                 # Circumvent downstream NumPy "RuntimeWarning"
                 # of "overflow encountered in power" in _unscale
@@ -177,30 +179,8 @@ def _masker(item):
     return result
 
 
-# Regulations 92.1.4 and 92.1.5.
-_MDI = 2 ** 32 - 1
-# Note:
-#   1. Integer "on-disk" values (aka. coded keys) in GRIB messages:
-#       - Are 8-, 16-, or 32-bit.
-#       - Are either signed or unsigned, with signed values stored as
-#         sign-and-magnitude (*not* twos-complement).
-#       - Use all bits set to indicate a missing value (MDI).
-#   2. Irrespective of the on-disk form, the ECMWF GRIB API *always*:
-#       - Returns values as 64-bit signed integers, either as native
-#         Python 'int' or numpy 'int64'.
-#       - Returns missing values as 2**32 - 1, but not all keys are
-#         defined as supporting missing values.
-#   NB. For keys which support missing values, the MDI value is reliably
-#   distinct from the valid range of either signed or unsigned 8-, 16-,
-#   or 32-bit values. For example:
-#       unsigned 32-bit:
-#           min = 0b000...000 = 0
-#           max = 0b111...110 = 2**32 - 2
-#           MDI = 0b111...111 = 2**32 - 1
-#       signed 32-bit:
-#           MDI = 0b111...111 = 2**32 - 1
-#           min = 0b111...110 = -(2**31 - 2)
-#           max = 0b011...111 = 2**31 - 1
+# Use ECCodes gribapi to recognise missing value
+_MDI = None
 
 
 # Non-standardised usage for negative forecast times.

diff --git a/lib/iris/fileformats/grib/_save_rules.py b/lib/iris/fileformats/grib/_save_rules.py
@@ -257,8 +257,21 @@ def latlon_first_last(x_coord, y_coord, grib):
 def dx_dy(x_coord, y_coord, grib):
     x_step = regular_step(x_coord)
     y_step = regular_step(y_coord)
-    gribapi.grib_set(grib, "DxInDegrees", float(abs(x_step)))
-    gribapi.grib_set(grib, "DyInDegrees", float(abs(y_step)))
+    # Set x and y step.  For degrees, this is encoded as an integer:
+    # 1 * 10^6 * floating point value.
+    # WMO Manual on Codes regulation 92.1.6
+    if x_coord.units == 'degrees':
+        gribapi.grib_set(grib, "iDirectionIncrement",
+                         round(1e6 * float(abs(x_step))))
+    else:
+        raise ValueError('X coordinate must be in degrees, not {}'
+                         '.'.format(x_coord.units))
+    if y_coord.units == 'degrees':
+        gribapi.grib_set(grib, "jDirectionIncrement",
+                         round(1e6 * float(abs(y_step))))
+    else:
+        raise ValueError('Y coordinate must be in degrees, not {}'
+                         '.'.format(y_coord.units))
 
 
 def scanning_mode_flags(x_coord, y_coord, grib):

diff --git a/lib/iris/fileformats/grib/message.py b/lib/iris/fileformats/grib/message.py
@@ -459,8 +459,12 @@ def _get_key_value(self, key):
             # By default these values are returned as unhelpful strings but
             # we can use int representation to compare against instead.
             res = gribapi.grib_get(self._message_id, key, int)
+            if gribapi.grib_is_missing(self._message_id, key) == 1:
+                res = None
         else:
             res = gribapi.grib_get(self._message_id, key)
+            if gribapi.grib_is_missing(self._message_id, key) == 1:
+                res = None
         return res
 
     def get_computed_key(self, key):
@@ -482,6 +486,8 @@ def get_computed_key(self, key):
             res = gribapi.grib_get_array(self._message_id, key)
         else:
             res = gribapi.grib_get(self._message_id, key)
+            if gribapi.grib_is_missing(self._message_id, key) == 1:
+                res = None
         return res
 
     def keys(self):

diff --git a/lib/iris/tests/integration/format_interop/test_name_grib.py b/lib/iris/tests/integration/format_interop/test_name_grib.py
@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2013 - 2016, Met Office
+# (C) British Crown Copyright 2013 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -72,21 +72,19 @@ def check_common(self, name_cube, grib_cube):
     def test_name2_field(self):
         filepath = tests.get_data_path(('NAME', 'NAMEII_field.txt'))
         name_cubes = iris.load(filepath)
-        # Check gribapi version, because we currently have a known load/save
-        # problem with gribapi 1v14 (at least).
-        gribapi_ver = gribapi.grib_get_api_version()
-        gribapi_fully_supported_version = \
-            (StrictVersion(gribapi.grib_get_api_version()) <
-             StrictVersion('1.13'))
+
+        # There is a known load/save problem with numerous
+        # gribapi/eccodes versions and
+        # zero only data, where min == max.
+        # This may be a problem with data scaling.
         for i, name_cube in enumerate(name_cubes):
-            if not gribapi_fully_supported_version:
-                data = name_cube.data
-                if np.min(data) == np.max(data):
-                    msg = ('NAMEII cube #{}, "{}" has empty data : '
-                           'SKIPPING test for this cube, as save/load will '
-                           'not currently work with gribabi > 1v12.')
-                    warnings.warn(msg.format(i, name_cube.name()))
-                    continue
+            data = name_cube.data
+            if np.min(data) == np.max(data):
+                msg = ('NAMEII cube #{}, "{}" has empty data : '
+                       'SKIPPING test for this cube, as save/load will '
+                       'not currently work.')
+                warnings.warn(msg.format(i, name_cube.name()))
+                continue
 
             with self.temp_filename('.grib2') as temp_filename:
                 iris.save(name_cube, temp_filename)

diff --git a/lib/iris/tests/test_grib_save.py b/lib/iris/tests/test_grib_save.py
@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2016, Met Office
+# (C) British Crown Copyright 2010 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -34,6 +34,7 @@
 
 if tests.GRIB_AVAILABLE:
     import gribapi
+    from iris.fileformats.grib._load_convert import _MDI as MDI
 
 
 @tests.skip_data
@@ -50,10 +51,10 @@ def test_latlon_forecast_plev(self):
             iris.save(cubes, temp_file_path)
             expect_diffs = {'totalLength': (4837, 4832),
                             'productionStatusOfProcessedData': (0, 255),
-                            'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                            'scaleFactorOfRadiusOfSphericalEarth': (MDI,
                                                                     0),
                             'shapeOfTheEarth': (0, 1),
-                            'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                            'scaledValueOfRadiusOfSphericalEarth': (MDI,
                                                                     6367470),
                             'typeOfGeneratingProcess': (0, 255),
                             'generatingProcessIdentifier': (128, 255),
@@ -70,10 +71,10 @@ def test_rotated_latlon(self):
             iris.save(cubes, temp_file_path)
             expect_diffs = {'totalLength': (648196, 648191),
                             'productionStatusOfProcessedData': (0, 255),
-                            'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                            'scaleFactorOfRadiusOfSphericalEarth': (MDI,
                                                                     0),
                             'shapeOfTheEarth': (0, 1),
-                            'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                            'scaledValueOfRadiusOfSphericalEarth': (MDI,
                                                                     6367470),
                             'longitudeOfLastGridPoint': (392109982, 32106370),
                             'latitudeOfLastGridPoint': (19419996, 19419285),
@@ -91,10 +92,10 @@ def test_time_mean(self):
         cubes = iris.load(source_grib)
         expect_diffs = {'totalLength': (21232, 21227),
                         'productionStatusOfProcessedData': (0, 255),
-                        'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                        'scaleFactorOfRadiusOfSphericalEarth': (MDI,
                                                                 0),
                         'shapeOfTheEarth': (0, 1),
-                        'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                        'scaledValueOfRadiusOfSphericalEarth': (MDI,
                                                                 6367470),
                         'longitudeOfLastGridPoint': (356249908, 356249809),
                         'latitudeOfLastGridPoint': (-89999938, -89999944),

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py
@@ -33,12 +33,10 @@
 import iris.coords
 import iris.exceptions
 from iris.fileformats.grib._load_convert import grid_definition_template_12
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 2 ** 32 - 1
-
-
 class Test(tests.IrisTest):
     def section_3(self):
         section = {

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py
@@ -32,12 +32,10 @@
 import iris.coord_systems
 import iris.coords
 from iris.fileformats.grib._load_convert import grid_definition_template_20
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 2 ** 32 - 1
-
-
 class Test(tests.IrisTest):
 
     def section_3(self):

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py
@@ -32,12 +32,10 @@
 import iris.coord_systems
 import iris.coords
 from iris.fileformats.grib._load_convert import grid_definition_template_30
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 2 ** 32 - 1
-
-
 class Test(tests.IrisTest):
 
     def section_3(self):

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py
@@ -32,12 +32,10 @@
 import iris.coord_systems
 import iris.coords
 from iris.fileformats.grib._load_convert import grid_definition_template_40
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 2 ** 32 - 1
-
-
 class _Section(dict):
     def get_computed_key(self, key):
         return self.get(key)

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py
@@ -34,12 +34,10 @@
 import iris.coords
 import iris.exceptions
 from iris.fileformats.grib._load_convert import grid_definition_template_90
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 2 ** 32 - 1
-
-
 class Test(tests.IrisTest):
     def uk(self):
         section = {

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py
@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2014 - 2015, Met Office
+# (C) British Crown Copyright 2014 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -30,13 +30,11 @@
 import iris.coords
 from iris.tests.unit.fileformats.grib.load_convert import (LoadConvertTest,
                                                            empty_metadata)
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.fileformats.grib._load_convert import product_definition_template_0
 from iris.tests import mock
 
 
-MDI = 0xffffffff
-
-
 def section_4():
     return {'hoursAfterDataCutoff': MDI,
             'minutesAfterDataCutoff': MDI,

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py
@@ -33,13 +33,11 @@
 from iris.coords import CellMethod, DimCoord
 from iris.exceptions import TranslationError
 from iris.fileformats.grib._load_convert import product_definition_template_15
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests.unit.fileformats.grib.load_convert import (LoadConvertTest,
                                                            empty_metadata)
 
 
-MDI = 0xffffffff
-
-
 def section_4():
     return {'productDefinitionTemplateNumber': 15,
             'hoursAfterDataCutoff': MDI,

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py
@@ -27,13 +27,11 @@
 import iris.tests as tests
 
 from iris.fileformats.grib._load_convert import product_definition_template_32
+from iris.fileformats.grib._load_convert import _MDI as MDI
 from iris.tests import mock
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
 
-MDI = 0xffffffff
-
-
 class Test(tests.IrisTest):
     def setUp(self):
         self.patch('warnings.warn')

diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py
@@ -59,7 +59,7 @@ def test_array(self):
     def test_array_mdi(self):
         result = unscale([1, MDI, 100, 1000], [1, 1, 1, MDI])
         self.assertTrue(ma.isMaskedArray(result))
-        expected = ma.masked_values([0.1, MDI, 10.0, MDI], MDI)
+        expected = ma.masked_values([0.1, 0, 10.0, 0], 0)
         np.testing.assert_array_almost_equal(result, expected)
 
 

diff --git a/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py b/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py
@@ -54,7 +54,12 @@ def test_release_file(self):
         filename = tests.get_data_path(('GRIB', '3_layer_viz',
                                         '3_layer.grib2'))
         my_file = open(filename)
-        self.patch('__builtin__.open', mock.Mock(return_value=my_file))
+        if six.PY2:
+            self.patch('__builtin__.open', mock.Mock(return_value=my_file))
+        else:
+            import builtins
+            self.patch('builtins.open', mock.Mock(return_value=my_file))
+
         messages = list(GribMessage.messages_from_filename(filename))
         self.assertFalse(my_file.closed)
         del messages

diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py
@@ -76,8 +76,8 @@ def test__grid_points(self):
         self._check_key("longitudeOfLastGridPoint", 7000000)
         self._check_key("latitudeOfFirstGridPoint", 4000000)
         self._check_key("latitudeOfLastGridPoint", 9000000)
-        self._check_key("DxInDegrees", 2.0)
-        self._check_key("DyInDegrees", 5.0)
+        self._check_key("iDirectionIncrement", 2000000)
+        self._check_key("jDirectionIncrement", 5000000)
 
     def test__scanmode(self):
         grid_definition_template_0(self.test_cube, self.mock_grib)

diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py
@@ -91,8 +91,8 @@ def test__grid_points(self):
         self._check_key("longitudeOfLastGridPoint", 7000000)
         self._check_key("latitudeOfFirstGridPoint", 4000000)
         self._check_key("latitudeOfLastGridPoint", 9000000)
-        self._check_key("DxInDegrees", 2.0)
-        self._check_key("DyInDegrees", 5.0)
+        self._check_key("iDirectionIncrement", 2000000)
+        self._check_key("jDirectionIncrement", 5000000)
 
     def test__scanmode(self):
         grid_definition_template_1(self.test_cube, self.mock_grib)