diff --git a/.travis.yml b/.travis.yml index bf618a27af..7f579c6f5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ install: conda install --quiet --file minimal-conda-requirements.txt; else if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then - sed -e '/ecmwf_grib/d' -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet; + sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet; else conda install --quiet --file conda-requirements.txt; fi diff --git a/INSTALL b/INSTALL index 36c2b8d964..3a1fe2fcb6 100644 --- a/INSTALL +++ b/INSTALL @@ -122,11 +122,12 @@ gdal 1.9.1 or later (https://pypi.python.org/pypi/GDAL/) graphviz 2.18 or later (http://www.graphviz.org/) Graph visualisation software. -grib-api 1.9.16 or later - (https://software.ecmwf.int/wiki/display/GRIB/Releases) +eccodes + (https://software.ecmwf.int/wiki/display/ECC/ecCodes+Home) API for the encoding and decoding WMO FM-92 GRIB edition 1 and edition 2 messages. A compression library such as Jasper is required to read JPEG2000 compressed GRIB2 files. + Successor to GRIBAPI. matplotlib 1.2.0 (http://matplotlib.sourceforge.net/) Python package for 2D plotting. diff --git a/conda-requirements.txt b/conda-requirements.txt index e0206a539d..7884dbe7c7 100644 --- a/conda-requirements.txt +++ b/conda-requirements.txt @@ -25,7 +25,7 @@ imagehash requests # Optional iris dependencies -python-ecmwf_grib +python-eccodes esmpy>=7.0 gdal libmo_unpack diff --git a/lib/iris/fileformats/grib/_load_convert.py b/lib/iris/fileformats/grib/_load_convert.py index 9ce3df9137..26bef5fb80 100644 --- a/lib/iris/fileformats/grib/_load_convert.py +++ b/lib/iris/fileformats/grib/_load_convert.py @@ -158,7 +158,9 @@ def _unscale(v, f): if isinstance(value, Iterable) or isinstance(factor, Iterable): def _masker(item): - result = ma.masked_equal(item, _MDI) + numerical_mdi = 2 ** 32 - 1 + item = [numerical_mdi if i is None else i for i in item] + result = ma.masked_equal(item, numerical_mdi) if ma.count_masked(result): # Circumvent downstream NumPy "RuntimeWarning" # of "overflow encountered in power" in _unscale @@ -177,30 +179,8 @@ def _masker(item): return result -# Regulations 92.1.4 and 92.1.5. -_MDI = 2 ** 32 - 1 -# Note: -# 1. Integer "on-disk" values (aka. coded keys) in GRIB messages: -# - Are 8-, 16-, or 32-bit. -# - Are either signed or unsigned, with signed values stored as -# sign-and-magnitude (*not* twos-complement). -# - Use all bits set to indicate a missing value (MDI). -# 2. Irrespective of the on-disk form, the ECMWF GRIB API *always*: -# - Returns values as 64-bit signed integers, either as native -# Python 'int' or numpy 'int64'. -# - Returns missing values as 2**32 - 1, but not all keys are -# defined as supporting missing values. -# NB. For keys which support missing values, the MDI value is reliably -# distinct from the valid range of either signed or unsigned 8-, 16-, -# or 32-bit values. For example: -# unsigned 32-bit: -# min = 0b000...000 = 0 -# max = 0b111...110 = 2**32 - 2 -# MDI = 0b111...111 = 2**32 - 1 -# signed 32-bit: -# MDI = 0b111...111 = 2**32 - 1 -# min = 0b111...110 = -(2**31 - 2) -# max = 0b011...111 = 2**31 - 1 +# Use ECCodes gribapi to recognise missing value +_MDI = None # Non-standardised usage for negative forecast times. diff --git a/lib/iris/fileformats/grib/_save_rules.py b/lib/iris/fileformats/grib/_save_rules.py index 7a5a8fce49..59356b633f 100644 --- a/lib/iris/fileformats/grib/_save_rules.py +++ b/lib/iris/fileformats/grib/_save_rules.py @@ -257,8 +257,21 @@ def latlon_first_last(x_coord, y_coord, grib): def dx_dy(x_coord, y_coord, grib): x_step = regular_step(x_coord) y_step = regular_step(y_coord) - gribapi.grib_set(grib, "DxInDegrees", float(abs(x_step))) - gribapi.grib_set(grib, "DyInDegrees", float(abs(y_step))) + # Set x and y step. For degrees, this is encoded as an integer: + # 1 * 10^6 * floating point value. + # WMO Manual on Codes regulation 92.1.6 + if x_coord.units == 'degrees': + gribapi.grib_set(grib, "iDirectionIncrement", + round(1e6 * float(abs(x_step)))) + else: + raise ValueError('X coordinate must be in degrees, not {}' + '.'.format(x_coord.units)) + if y_coord.units == 'degrees': + gribapi.grib_set(grib, "jDirectionIncrement", + round(1e6 * float(abs(y_step)))) + else: + raise ValueError('Y coordinate must be in degrees, not {}' + '.'.format(y_coord.units)) def scanning_mode_flags(x_coord, y_coord, grib): diff --git a/lib/iris/fileformats/grib/message.py b/lib/iris/fileformats/grib/message.py index c936fe6e8b..777d696fb5 100644 --- a/lib/iris/fileformats/grib/message.py +++ b/lib/iris/fileformats/grib/message.py @@ -459,8 +459,12 @@ def _get_key_value(self, key): # By default these values are returned as unhelpful strings but # we can use int representation to compare against instead. res = gribapi.grib_get(self._message_id, key, int) + if gribapi.grib_is_missing(self._message_id, key) == 1: + res = None else: res = gribapi.grib_get(self._message_id, key) + if gribapi.grib_is_missing(self._message_id, key) == 1: + res = None return res def get_computed_key(self, key): @@ -482,6 +486,8 @@ def get_computed_key(self, key): res = gribapi.grib_get_array(self._message_id, key) else: res = gribapi.grib_get(self._message_id, key) + if gribapi.grib_is_missing(self._message_id, key) == 1: + res = None return res def keys(self): diff --git a/lib/iris/tests/integration/format_interop/test_name_grib.py b/lib/iris/tests/integration/format_interop/test_name_grib.py index 192f0082b0..6fd4ae4b41 100644 --- a/lib/iris/tests/integration/format_interop/test_name_grib.py +++ b/lib/iris/tests/integration/format_interop/test_name_grib.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2016, Met Office +# (C) British Crown Copyright 2013 - 2017, Met Office # # This file is part of Iris. # @@ -72,21 +72,19 @@ def check_common(self, name_cube, grib_cube): def test_name2_field(self): filepath = tests.get_data_path(('NAME', 'NAMEII_field.txt')) name_cubes = iris.load(filepath) - # Check gribapi version, because we currently have a known load/save - # problem with gribapi 1v14 (at least). - gribapi_ver = gribapi.grib_get_api_version() - gribapi_fully_supported_version = \ - (StrictVersion(gribapi.grib_get_api_version()) < - StrictVersion('1.13')) + + # There is a known load/save problem with numerous + # gribapi/eccodes versions and + # zero only data, where min == max. + # This may be a problem with data scaling. for i, name_cube in enumerate(name_cubes): - if not gribapi_fully_supported_version: - data = name_cube.data - if np.min(data) == np.max(data): - msg = ('NAMEII cube #{}, "{}" has empty data : ' - 'SKIPPING test for this cube, as save/load will ' - 'not currently work with gribabi > 1v12.') - warnings.warn(msg.format(i, name_cube.name())) - continue + data = name_cube.data + if np.min(data) == np.max(data): + msg = ('NAMEII cube #{}, "{}" has empty data : ' + 'SKIPPING test for this cube, as save/load will ' + 'not currently work.') + warnings.warn(msg.format(i, name_cube.name())) + continue with self.temp_filename('.grib2') as temp_filename: iris.save(name_cube, temp_filename) diff --git a/lib/iris/tests/test_grib_save.py b/lib/iris/tests/test_grib_save.py index 53c14a169c..3d1e0461f6 100644 --- a/lib/iris/tests/test_grib_save.py +++ b/lib/iris/tests/test_grib_save.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2016, Met Office +# (C) British Crown Copyright 2010 - 2017, Met Office # # This file is part of Iris. # @@ -34,6 +34,7 @@ if tests.GRIB_AVAILABLE: import gribapi + from iris.fileformats.grib._load_convert import _MDI as MDI @tests.skip_data @@ -50,10 +51,10 @@ def test_latlon_forecast_plev(self): iris.save(cubes, temp_file_path) expect_diffs = {'totalLength': (4837, 4832), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (MDI, 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (MDI, 6367470), 'typeOfGeneratingProcess': (0, 255), 'generatingProcessIdentifier': (128, 255), @@ -70,10 +71,10 @@ def test_rotated_latlon(self): iris.save(cubes, temp_file_path) expect_diffs = {'totalLength': (648196, 648191), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (MDI, 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (MDI, 6367470), 'longitudeOfLastGridPoint': (392109982, 32106370), 'latitudeOfLastGridPoint': (19419996, 19419285), @@ -91,10 +92,10 @@ def test_time_mean(self): cubes = iris.load(source_grib) expect_diffs = {'totalLength': (21232, 21227), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (MDI, 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (MDI, 6367470), 'longitudeOfLastGridPoint': (356249908, 356249809), 'latitudeOfLastGridPoint': (-89999938, -89999944), diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py index ac59195a3b..2ec52f2e5c 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_12.py @@ -33,12 +33,10 @@ import iris.coords import iris.exceptions from iris.fileformats.grib._load_convert import grid_definition_template_12 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 2 ** 32 - 1 - - class Test(tests.IrisTest): def section_3(self): section = { diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py index 82c5195dfd..45a74dbdb9 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_20.py @@ -32,12 +32,10 @@ import iris.coord_systems import iris.coords from iris.fileformats.grib._load_convert import grid_definition_template_20 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 2 ** 32 - 1 - - class Test(tests.IrisTest): def section_3(self): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py index a524eff9e5..b9381279a8 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_30.py @@ -32,12 +32,10 @@ import iris.coord_systems import iris.coords from iris.fileformats.grib._load_convert import grid_definition_template_30 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 2 ** 32 - 1 - - class Test(tests.IrisTest): def section_3(self): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py index eec5513d1d..2eece12a92 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_40.py @@ -32,12 +32,10 @@ import iris.coord_systems import iris.coords from iris.fileformats.grib._load_convert import grid_definition_template_40 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 2 ** 32 - 1 - - class _Section(dict): def get_computed_key(self, key): return self.get(key) diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py index 9597852575..f004e8a7de 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_90.py @@ -34,12 +34,10 @@ import iris.coords import iris.exceptions from iris.fileformats.grib._load_convert import grid_definition_template_90 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 2 ** 32 - 1 - - class Test(tests.IrisTest): def uk(self): section = { diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py index 649aa3c2b6..9e833667f1 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_0.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -30,13 +30,11 @@ import iris.coords from iris.tests.unit.fileformats.grib.load_convert import (LoadConvertTest, empty_metadata) +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.fileformats.grib._load_convert import product_definition_template_0 from iris.tests import mock -MDI = 0xffffffff - - def section_4(): return {'hoursAfterDataCutoff': MDI, 'minutesAfterDataCutoff': MDI, diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py index 1158c86e7f..a9d5a54346 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_15.py @@ -33,13 +33,11 @@ from iris.coords import CellMethod, DimCoord from iris.exceptions import TranslationError from iris.fileformats.grib._load_convert import product_definition_template_15 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests.unit.fileformats.grib.load_convert import (LoadConvertTest, empty_metadata) -MDI = 0xffffffff - - def section_4(): return {'productDefinitionTemplateNumber': 15, 'hoursAfterDataCutoff': MDI, diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py index 89f1c2673a..b02305a536 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_32.py @@ -27,13 +27,11 @@ import iris.tests as tests from iris.fileformats.grib._load_convert import product_definition_template_32 +from iris.fileformats.grib._load_convert import _MDI as MDI from iris.tests import mock from iris.tests.unit.fileformats.grib.load_convert import empty_metadata -MDI = 0xffffffff - - class Test(tests.IrisTest): def setUp(self): self.patch('warnings.warn') diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py index 1f891177cc..3a3c3b8997 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py @@ -59,7 +59,7 @@ def test_array(self): def test_array_mdi(self): result = unscale([1, MDI, 100, 1000], [1, 1, 1, MDI]) self.assertTrue(ma.isMaskedArray(result)) - expected = ma.masked_values([0.1, MDI, 10.0, MDI], MDI) + expected = ma.masked_values([0.1, 0, 10.0, 0], 0) np.testing.assert_array_almost_equal(result, expected) diff --git a/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py b/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py index 3bbd9c8e04..a5693b8032 100644 --- a/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py +++ b/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py @@ -54,7 +54,12 @@ def test_release_file(self): filename = tests.get_data_path(('GRIB', '3_layer_viz', '3_layer.grib2')) my_file = open(filename) - self.patch('__builtin__.open', mock.Mock(return_value=my_file)) + if six.PY2: + self.patch('__builtin__.open', mock.Mock(return_value=my_file)) + else: + import builtins + self.patch('builtins.open', mock.Mock(return_value=my_file)) + messages = list(GribMessage.messages_from_filename(filename)) self.assertFalse(my_file.closed) del messages diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py index a4f581b78e..d5ee9f771e 100644 --- a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py +++ b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py @@ -76,8 +76,8 @@ def test__grid_points(self): self._check_key("longitudeOfLastGridPoint", 7000000) self._check_key("latitudeOfFirstGridPoint", 4000000) self._check_key("latitudeOfLastGridPoint", 9000000) - self._check_key("DxInDegrees", 2.0) - self._check_key("DyInDegrees", 5.0) + self._check_key("iDirectionIncrement", 2000000) + self._check_key("jDirectionIncrement", 5000000) def test__scanmode(self): grid_definition_template_0(self.test_cube, self.mock_grib) diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py index f8cdf181d4..1e7ce01a32 100644 --- a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py +++ b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py @@ -91,8 +91,8 @@ def test__grid_points(self): self._check_key("longitudeOfLastGridPoint", 7000000) self._check_key("latitudeOfFirstGridPoint", 4000000) self._check_key("latitudeOfLastGridPoint", 9000000) - self._check_key("DxInDegrees", 2.0) - self._check_key("DyInDegrees", 5.0) + self._check_key("iDirectionIncrement", 2000000) + self._check_key("jDirectionIncrement", 5000000) def test__scanmode(self): grid_definition_template_1(self.test_cube, self.mock_grib)