From a53a40f850ad3572be5e9fc7d321dfa6711c0545 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Fri, 13 Oct 2017 09:11:58 +0100 Subject: [PATCH] Remove iris.experimental.fieldsfiles module (#2640) * Remove iris.experimental.fieldsfiles module * Review actions: keep and move existing test modules * Changes to get tests passing * Common test result format * Update result files --- lib/iris/experimental/fieldsfile.py | 281 ------------------ lib/iris/tests/integration/um/__init__.py | 20 ++ .../um}/test_fieldsfile.py | 17 +- .../TestStructuredLoadFF/simple.cml | 2 +- .../TestStructuredLoadFF/simple_callback.cml | 2 +- .../TestStructuredLoadPP/simple.cml | 0 .../TestStructuredLoadPP/simple_callback.cml | 0 .../um/fast_load}/__init__.py | 7 +- .../um/fast_load}/test__convert_collation.py | 6 +- 9 files changed, 42 insertions(+), 293 deletions(-) delete mode 100644 lib/iris/experimental/fieldsfile.py create mode 100644 lib/iris/tests/integration/um/__init__.py rename lib/iris/tests/{experimental => integration/um}/test_fieldsfile.py (77%) rename lib/iris/tests/results/{experimental => integration/um}/fieldsfile/TestStructuredLoadFF/simple.cml (98%) rename lib/iris/tests/results/{experimental => integration/um}/fieldsfile/TestStructuredLoadFF/simple_callback.cml (98%) rename lib/iris/tests/results/{experimental => integration/um}/fieldsfile/TestStructuredLoadPP/simple.cml (100%) rename lib/iris/tests/results/{experimental => integration/um}/fieldsfile/TestStructuredLoadPP/simple_callback.cml (100%) rename lib/iris/tests/unit/{experimental/fieldsfile => fileformats/um/fast_load}/__init__.py (86%) rename lib/iris/tests/unit/{experimental/fieldsfile => fileformats/um/fast_load}/test__convert_collation.py (98%) diff --git a/lib/iris/experimental/fieldsfile.py b/lib/iris/experimental/fieldsfile.py deleted file mode 100644 index 42c99216f4..0000000000 --- a/lib/iris/experimental/fieldsfile.py +++ /dev/null @@ -1,281 +0,0 @@ -# (C) British Crown Copyright 2014 - 2017, Met Office -# -# This file is part of Iris. -# -# Iris is free software: you can redistribute it and/or modify it under -# the terms of the GNU Lesser General Public License as published by the -# Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Iris is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Iris. If not, see . -""" -High-speed loading of structured FieldsFiles. - -.. deprecated:: 1.10 - - This module has now been *deprecated*. - Please use :mod:`iris.fileformats.um.structured_um_loading` instead. - -""" -from __future__ import (absolute_import, division, print_function) -from six.moves import (filter, input, map, range, zip) # noqa - -import os - -from iris._deprecation import warn_deprecated - -# Issue a deprecation message when the module is loaded. -warn_deprecated("The module 'iris.experimental.fieldsfile' is deprecated. " - "Please use iris.fileformats.um.structured_um_loading" - "as a replacement.") - -from iris.coords import DimCoord -from iris.cube import CubeList -from iris.exceptions import TranslationError -from iris.fileformats import FORMAT_AGENT -from iris.fileformats.um import um_to_pp -from iris.fileformats.pp import load as pp_load -from iris.fileformats.pp_rules import (_convert_time_coords, - _convert_vertical_coords, - _convert_scalar_realization_coords, - _convert_scalar_pseudo_level_coords, - _all_other_rules) -from iris.fileformats.rules import ConversionMetadata, Loader, load_cubes -from iris.fileformats.um._fast_load_structured_fields import \ - group_structured_fields - - -# Seed the preferred order of candidate dimension coordinates. -_HINT_COORDS = ['time', 'forecast_reference_time', 'model_level_number'] -_HINTS = {name: i for i, name in zip(range(len(_HINT_COORDS)), _HINT_COORDS)} - -_FF_SPEC_NAME = 'UM Fieldsfile' -_PP_SPEC_NAME = 'UM Post Processing file' - - -def _structured_loader(fname): - with open(fname, 'rb') as fh: - spec = FORMAT_AGENT.get_spec(os.path.basename(fname), fh) - if spec.name.startswith(_FF_SPEC_NAME): - result = um_to_pp - elif spec.name.startswith(_PP_SPEC_NAME): - result = pp_load - else: - emsg = 'Require {!r} to be a structured FieldsFile or a PP file.' - raise ValueError(emsg.format(fname)) - return result - - -def _collations_from_filename(filename): - loader = _structured_loader(filename) - fields = iter(loader(filename)) - return group_structured_fields(fields) - - -def load(filenames, callback=None): - """ - Load structured FieldsFiles and PP files. - - Args: - - * filenames: - One or more filenames. - - - Kwargs: - - * callback: - A modifier/filter function. Please see the module documentation - for :mod:`iris`. - - .. note:: - - Unlike the standard :func:`iris.load` operation, the callback is - applied to the final result cubes, not individual input fields. - - Returns: - An :class:`iris.cube.CubeList`. - - - This is a streamlined load operation, to be used only on fieldsfiles or PP - files whose fields repeat regularly over the same vertical levels and - times. The results aim to be equivalent to those generated by - :func:`iris.load`, but the operation is substantially faster for input that - is structured. - - The structured input files should conform to the following requirements: - - * the file must contain fields for all possible combinations of the - vertical levels and time points found in the file. - - * the fields must occur in a regular repeating order within the file. - - (For example: a sequence of fields for NV vertical levels, repeated - for NP different forecast periods, repeated for NT different forecast - times). - - * all other metadata must be identical across all fields of the same - phenomenon. - - Each group of fields with the same values of LBUSER4, LBUSER7 and LBPROC - is identified as a separate phenomenon: These groups are processed - independently and returned as separate result cubes. - - .. note:: - - Each input file is loaded independently. Thus a single result cube can - not combine data from multiple input files. - - .. note:: - - The resulting time-related coordinates ('time', 'forecast_time' and - 'forecast_period') may be mapped to shared cube dimensions and in some - cases can also be multidimensional. However, the vertical level - information *must* have a simple one-dimensional structure, independent - of the time points, otherwise an error will be raised. - - .. note:: - - Where input data does *not* have a fully regular arrangement, the - corresponding result cube will have a single anonymous extra dimension - which indexes over all the input fields. - - This can happen if, for example, some fields are missing; or have - slightly different metadata; or appear out of order in the file. - - .. warning:: - - Any non-regular metadata variation in the input should be strictly - avoided, as not all irregularities are detected, which can cause - erroneous results. - - - """ - warn_deprecated( - "The module 'iris.experimental.fieldsfile' is deprecated. " - "Please use the 'iris.fileformats.um.structured_um_loading' facility " - "as a replacement." - "\nA call to 'iris.experimental.fieldsfile.load' can be replaced with " - "'iris.load_raw', within a 'structured_um_loading' context.") - loader = Loader(_collations_from_filename, {}, _convert_collation) - return CubeList(load_cubes(filenames, callback, loader, None)) - - -def _adjust_dims(coords_and_dims, n_dims): - def adjust(dims): - if dims is not None: - dims += n_dims - return dims - return [(coord, adjust(dims)) for coord, dims in coords_and_dims] - - -def _bind_coords(coords_and_dims, dim_coord_dims, dim_coords_and_dims, - aux_coords_and_dims): - def key_func(item): - return _HINTS.get(item[0].name(), len(_HINTS)) - # Target the first DimCoord for a dimension at dim_coords, - # and target everything else at aux_coords. - for coord, dims in sorted(coords_and_dims, key=key_func): - if (isinstance(coord, DimCoord) and dims is not None and - len(dims) == 1 and dims[0] not in dim_coord_dims): - dim_coords_and_dims.append((coord, dims)) - dim_coord_dims.add(dims[0]) - else: - aux_coords_and_dims.append((coord, dims)) - - -def _convert_collation(collation): - """ - Converts a FieldCollation into the corresponding items of Cube - metadata. - - Args: - - * collation: - A FieldCollation object. - - Returns: - A :class:`iris.fileformats.rules.ConversionMetadata` object. - - """ - # For all the scalar conversions all fields in the collation will - # give the same result, so the choice is arbitrary. - field = collation.fields[0] - - # All the "other" rules. - (references, standard_name, long_name, units, attributes, cell_methods, - dim_coords_and_dims, aux_coords_and_dims) = _all_other_rules(field) - - # Adjust any dimension bindings to account for the extra leading - # dimensions added by the collation. - if collation.vector_dims_shape: - n_collation_dims = len(collation.vector_dims_shape) - dim_coords_and_dims = _adjust_dims(dim_coords_and_dims, - n_collation_dims) - aux_coords_and_dims = _adjust_dims(aux_coords_and_dims, - n_collation_dims) - - # "Normal" (non-cross-sectional) time values - vector_headers = collation.element_arrays_and_dims - # If the collation doesn't define a vector of values for a - # particular header then it must be constant over all fields in the - # collation. In which case it's safe to get the value from any field. - t1, t1_dims = vector_headers.get('t1', (field.t1, ())) - t2, t2_dims = vector_headers.get('t2', (field.t2, ())) - lbft, lbft_dims = vector_headers.get('lbft', (field.lbft, ())) - coords_and_dims = _convert_time_coords(field.lbcode, field.lbtim, - field.time_unit('hours'), - t1, t2, lbft, - t1_dims, t2_dims, lbft_dims) - dim_coord_dims = set() - _bind_coords(coords_and_dims, dim_coord_dims, dim_coords_and_dims, - aux_coords_and_dims) - - # "Normal" (non-cross-sectional) vertical levels - blev, blev_dims = vector_headers.get('blev', (field.blev, ())) - lblev, lblev_dims = vector_headers.get('lblev', (field.lblev, ())) - bhlev, bhlev_dims = vector_headers.get('bhlev', (field.bhlev, ())) - bhrlev, bhrlev_dims = vector_headers.get('bhrlev', (field.bhrlev, ())) - brsvd1, brsvd1_dims = vector_headers.get('brsvd1', (field.brsvd[0], ())) - brsvd2, brsvd2_dims = vector_headers.get('brsvd2', (field.brsvd[1], ())) - brlev, brlev_dims = vector_headers.get('brlev', (field.brlev, ())) - # Find all the non-trivial dimension values - dims = set(filter(None, [blev_dims, lblev_dims, bhlev_dims, bhrlev_dims, - brsvd1_dims, brsvd2_dims, brlev_dims])) - if len(dims) > 1: - raise TranslationError('Unsupported multiple values for vertical ' - 'dimension.') - if dims: - v_dims = dims.pop() - if len(v_dims) > 1: - raise TranslationError('Unsupported multi-dimension vertical ' - 'headers.') - else: - v_dims = () - coords_and_dims, factories = _convert_vertical_coords(field.lbcode, - field.lbvc, - blev, lblev, - field.stash, - bhlev, bhrlev, - brsvd1, brsvd2, - brlev, v_dims) - _bind_coords(coords_and_dims, dim_coord_dims, dim_coords_and_dims, - aux_coords_and_dims) - - # Realization (aka ensemble) (--> scalar coordinates) - aux_coords_and_dims.extend(_convert_scalar_realization_coords( - lbrsvd4=field.lbrsvd[3])) - - # Pseudo-level coordinate (--> scalar coordinates) - aux_coords_and_dims.extend(_convert_scalar_pseudo_level_coords( - lbuser5=field.lbuser[4])) - - return ConversionMetadata(factories, references, standard_name, long_name, - units, attributes, cell_methods, - dim_coords_and_dims, aux_coords_and_dims) diff --git a/lib/iris/tests/integration/um/__init__.py b/lib/iris/tests/integration/um/__init__.py new file mode 100644 index 0000000000..e012485f2c --- /dev/null +++ b/lib/iris/tests/integration/um/__init__.py @@ -0,0 +1,20 @@ +# (C) British Crown Copyright 2016 - 2017, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . +"""Integration tests for :mod:`iris.fileformats.um` fast load functions.""" + +from __future__ import (absolute_import, division, print_function) +from six.moves import (filter, input, map, range, zip) # noqa diff --git a/lib/iris/tests/experimental/test_fieldsfile.py b/lib/iris/tests/integration/um/test_fieldsfile.py similarity index 77% rename from lib/iris/tests/experimental/test_fieldsfile.py rename to lib/iris/tests/integration/um/test_fieldsfile.py index c97eb2ef94..5f9a0c91c6 100644 --- a/lib/iris/tests/experimental/test_fieldsfile.py +++ b/lib/iris/tests/integration/um/test_fieldsfile.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2016, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -25,8 +25,9 @@ # import iris tests first so that some things can be initialised before # importing anything else import iris.tests as tests +from iris.cube import CubeList -from iris.experimental.fieldsfile import load +from iris.fileformats.um import load_cubes as load @tests.skip_data @@ -34,14 +35,20 @@ class TestStructuredLoadFF(tests.IrisTest): def setUp(self): self.fname = tests.get_data_path(('FF', 'structured', 'small')) + def _merge_cubes(self, cubes): + # Merge the 2D cubes returned by `iris.fileformats.um.load_cubes`. + return CubeList(cubes).merge_cube() + def test_simple(self): - cube, = load(self.fname) + list_of_cubes = list(load(self.fname, None)) + cube = self._merge_cubes(list_of_cubes) self.assertCML(cube) def test_simple_callback(self): def callback(cube, field, filename): cube.attributes['processing'] = 'fast-ff' - cube, = load(self.fname, callback=callback) + list_of_cubes = list(load(self.fname, callback=callback)) + cube = self._merge_cubes(list_of_cubes) self.assertCML(cube) @@ -51,7 +58,7 @@ def setUp(self): self.fname = tests.get_data_path(('PP', 'structured', 'small.pp')) def test_simple(self): - [cube] = load(self.fname) + [cube] = load(self.fname, None) self.assertCML(cube) def test_simple_callback(self): diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple.cml similarity index 98% rename from lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml rename to lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple.cml index 5cb0de933c..0c316bcfea 100644 --- a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple.cml +++ b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple.cml @@ -54,7 +54,7 @@ - diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple_callback.cml similarity index 98% rename from lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml rename to lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple_callback.cml index 62126aa03b..188c583b2a 100644 --- a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadFF/simple_callback.cml +++ b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadFF/simple_callback.cml @@ -55,7 +55,7 @@ - diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadPP/simple.cml b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadPP/simple.cml similarity index 100% rename from lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadPP/simple.cml rename to lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadPP/simple.cml diff --git a/lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadPP/simple_callback.cml b/lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadPP/simple_callback.cml similarity index 100% rename from lib/iris/tests/results/experimental/fieldsfile/TestStructuredLoadPP/simple_callback.cml rename to lib/iris/tests/results/integration/um/fieldsfile/TestStructuredLoadPP/simple_callback.cml diff --git a/lib/iris/tests/unit/experimental/fieldsfile/__init__.py b/lib/iris/tests/unit/fileformats/um/fast_load/__init__.py similarity index 86% rename from lib/iris/tests/unit/experimental/fieldsfile/__init__.py rename to lib/iris/tests/unit/fileformats/um/fast_load/__init__.py index 8513b08fdb..451e19c8d8 100644 --- a/lib/iris/tests/unit/experimental/fieldsfile/__init__.py +++ b/lib/iris/tests/unit/fileformats/um/fast_load/__init__.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -14,7 +14,10 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Iris. If not, see . -"""Unit tests for :mod:`iris.experimental.fieldsfile`.""" +""" +Unit tests for the module :mod:`iris.fileformats.um._fast_load`. + +""" from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa diff --git a/lib/iris/tests/unit/experimental/fieldsfile/test__convert_collation.py b/lib/iris/tests/unit/fileformats/um/fast_load/test__convert_collation.py similarity index 98% rename from lib/iris/tests/unit/experimental/fieldsfile/test__convert_collation.py rename to lib/iris/tests/unit/fileformats/um/fast_load/test__convert_collation.py index d8edbc3df7..78a9c5b8b5 100644 --- a/lib/iris/tests/unit/experimental/fieldsfile/test__convert_collation.py +++ b/lib/iris/tests/unit/fileformats/um/fast_load/test__convert_collation.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Iris. If not, see . -"""Unit tests for :func:`iris.experimental.fieldsfile._convert_collation`.""" +"""Unit tests for :func:`iris.fileformats.um._fast_load._convert_collation`.""" from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa @@ -27,7 +27,7 @@ import netcdftime import numpy as np -from iris.experimental.fieldsfile \ +from iris.fileformats.um._fast_load \ import _convert_collation as convert_collation import iris.aux_factory import iris.coord_systems