Handle fill value on netCDF save (#2747)

* Handle fill value on netCDF save

docs/iris/src/whatsnew/contributions_2.0/incompatiblechange_2017-Aug-29_fill_value_save_arg.txt

@@ -0,0 +1 @@
+* When saving a cube or list of cubes in NetCDF format, a fill value or list of fill values can be specified via a new `fill_value` argument. If a list is supplied, each fill value will be applied to each cube in turn. If a `fill_value` argument is not specified, the default fill value for the file format and the cube's data type will be used. Fill values are no longer taken from the cube's `data` attribute when it is a masked array.

docs/iris/src/whatsnew/contributions_2.0/incompatiblechange_2017-Aug-29_netCDF_packing_changes.txt

@@ -0,0 +1,3 @@
+* A 'fill_value' key can no longer be specified as part of the `packing` argument to `iris.save` when saving in netCDF format. Instead, a fill value or list of fill values should be specified as a separate `fill_value` argument if required.
+
+* If the `packing` argument to `iris.save` is a dictionary, an error is raised if it contains any keys other than 'dtype', 'scale_factor' and 'add_offset'.

lib/iris/_lazy_data.py

@@ -131,39 +131,6 @@ def as_concrete_data(data):
     return data
 
 
-def get_fill_value(data):
-    """
-    Return the fill value of a concrete or lazy masked array.
-
-    Args:
-
-    * data:
-        A dask array, NumPy `ndarray` or masked array.
-
-    Returns:
-        The fill value of `data` if `data` represents a masked array, or None.
-
-    """
-    # If lazy, get the smallest slice of the data from which we can retrieve
-    # the fill_value.
-    if is_lazy_data(data):
-        inds = [0] * data.ndim
-        if inds:
-            inds[-1] = slice(0, 1)
-            data = data[tuple(inds)]
-        data = as_concrete_data(data)
-    else:
-        if isinstance(data, ma.core.MaskedConstant):
-            data = ma.array(data.data, mask=data.mask)
-
-    # Now get the fill value.
-    fill_value = None
-    if ma.isMaskedArray(data):
-        fill_value = data.fill_value
-
-    return fill_value
-
-
 def multidim_lazy_stack(stack):
     """
     Recursively build a multidimensional stacked dask array.

lib/iris/tests/integration/test_netcdf.py

@@ -210,20 +210,6 @@ def test_lazy_preserved_save(self):
                 saver.write(acube)
         self.assertTrue(acube.has_lazy_data())
 
-    @tests.skip_dask_mask
-    def test_lazy_mask_preserve_fill_value(self):
-        data = ma.array([0, 1], mask=[False, True])
-        cube = iris.cube.Cube(data, fill_value=-1)
-        with self.temp_filename(suffix='.nc') as filename, \
-                self.temp_filename(suffix='.nc') as other_filename:
-            iris.save(cube, filename, unlimited_dimensions=[])
-            ncube = iris.load_cube(filename)
-            # Lazy save of the masked cube
-            iris.save(ncube, other_filename, unlimited_dimensions=[])
-            ds = nc.Dataset(other_filename, 'r')
-            avar = ds['unknown']
-            self.assertEqual(avar._FillValue, -1)
-
 
 @tests.skip_data
 class TestCellMeasures(tests.IrisTest):
@@ -427,7 +413,6 @@ def test_multi_packed_single_dtype(self):
         # Read PP input file.
         self._multi_test('multi_packed_single_dtype.cdl')
 
-    @tests.skip_dask_mask
     def test_multi_packed_multi_dtype(self):
         """Test saving multiple packed cubes with pack_dtype list."""
         # Read PP input file.

lib/iris/tests/results/integration/netcdf/TestPackedData/multi_packed_multi_dtype.cdl

@@ -46,7 +46,6 @@ variables:
 		height:standard_name = "height" ;
 		height:positive = "up" ;
 	float precipitation_flux(time, latitude, longitude) ;
-		precipitation_flux:_FillValue = -1.e+30f ;
 		precipitation_flux:standard_name = "precipitation_flux" ;
 		precipitation_flux:units = "kg m-2 s-1" ;
 		precipitation_flux:um_stash_source = "m01s05i216" ;

lib/iris/tests/results/netcdf/netcdf_monotonic.cml

@@ -1,6 +1,6 @@
 <?xml version="1.0" ?>
 <cubes xmlns="urn:x-iris:cubeml-0.2">
-  <cube core-dtype="float64" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind1">
+  <cube core-dtype="int32" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind1">
     <attributes>
       <attribute name="test" value="weak-monotonic time coordinate"/>
     </attributes>
@@ -18,7 +18,7 @@
     <cellMethods/>
     <data checksum="0xd4e7a32f" dtype="int32" shape="(3, 3, 3)"/>
   </cube>
-  <cube core-dtype="float64" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind2">
+  <cube core-dtype="int32" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind2">
     <attributes>
       <attribute name="test" value="masked monotonic time coordinate"/>
     </attributes>
@@ -36,7 +36,7 @@
     <cellMethods/>
     <data checksum="0xd4e7a32f" dtype="int32" shape="(3, 3, 3)"/>
   </cube>
-  <cube core-dtype="float64" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind3">
+  <cube core-dtype="int32" dtype="int32" standard_name="eastward_wind" units="m s-1" var_name="wind3">
     <attributes>
       <attribute name="test" value="masked non-monotonic time coordinate"/>
     </attributes>

lib/iris/tests/results/netcdf/netcdf_save_hybrid_height.cdl

@@ -6,7 +6,6 @@ dimensions:
 	model_level_number = 10 ;
 variables:
 	float air_potential_temperature(time, model_level_number, grid_latitude, grid_longitude) ;
-		air_potential_temperature:_FillValue = -1.073742e+09f ;
 		air_potential_temperature:standard_name = "air_potential_temperature" ;
 		air_potential_temperature:units = "K" ;
 		air_potential_temperature:um_stash_source = "m01s00i004" ;

lib/iris/tests/results/netcdf/netcdf_save_multi_0.cdl

@@ -5,7 +5,6 @@ dimensions:
 	longitude = 96 ;
 variables:
 	float air_temperature(time, latitude, longitude) ;
-		air_temperature:_FillValue = -1.e+30f ;
 		air_temperature:standard_name = "air_temperature" ;
 		air_temperature:units = "K" ;
 		air_temperature:um_stash_source = "m01s03i236" ;

lib/iris/tests/results/netcdf/netcdf_save_multi_1.cdl

@@ -5,7 +5,6 @@ dimensions:
 	longitude = 96 ;
 variables:
 	float air_temperature(time, latitude, longitude) ;
-		air_temperature:_FillValue = -1.e+30f ;
 		air_temperature:standard_name = "air_temperature" ;
 		air_temperature:units = "K" ;
 		air_temperature:um_stash_source = "m01s03i236" ;

lib/iris/tests/results/netcdf/netcdf_save_multi_2.cdl

@@ -5,7 +5,6 @@ dimensions:
 	longitude = 96 ;
 variables:
 	float precipitation_flux(time, latitude, longitude) ;
-		precipitation_flux:_FillValue = -1.e+30f ;
 		precipitation_flux:standard_name = "precipitation_flux" ;
 		precipitation_flux:units = "kg m-2 s-1" ;
 		precipitation_flux:um_stash_source = "m01s05i216" ;

lib/iris/tests/results/netcdf/netcdf_save_multiple.cdl

@@ -5,7 +5,6 @@ dimensions:
 	longitude = 96 ;
 variables:
 	float air_temperature(time, latitude, longitude) ;
-		air_temperature:_FillValue = -1.e+30f ;
 		air_temperature:standard_name = "air_temperature" ;
 		air_temperature:units = "K" ;
 		air_temperature:um_stash_source = "m01s03i236" ;
@@ -45,15 +44,13 @@ variables:
 		height:standard_name = "height" ;
 		height:positive = "up" ;
 	float air_temperature_0(time, latitude, longitude) ;
-		air_temperature_0:_FillValue = -1.e+30f ;
 		air_temperature_0:standard_name = "air_temperature" ;
 		air_temperature_0:units = "K" ;
 		air_temperature_0:um_stash_source = "m01s03i236" ;
 		air_temperature_0:cell_methods = "time: minimum (interval: 1 hour)" ;
 		air_temperature_0:grid_mapping = "latitude_longitude" ;
 		air_temperature_0:coordinates = "forecast_period forecast_reference_time height" ;
 	float precipitation_flux(time, latitude, longitude) ;
-		precipitation_flux:_FillValue = -1.e+30f ;
 		precipitation_flux:standard_name = "precipitation_flux" ;
 		precipitation_flux:units = "kg m-2 s-1" ;
 		precipitation_flux:um_stash_source = "m01s05i216" ;

lib/iris/tests/results/netcdf/netcdf_save_ndim_auxiliary.cdl

@@ -5,7 +5,6 @@ dimensions:
 	rlon = 174 ;
 variables:
 	float pr(time, rlat, rlon) ;
-		pr:_FillValue = 1.e+30f ;
 		pr:standard_name = "precipitation_flux" ;
 		pr:long_name = "Precipitation" ;
 		pr:units = "kg m-2 s-1" ;

lib/iris/tests/results/netcdf/netcdf_save_single.cdl

@@ -4,7 +4,6 @@ dimensions:
 	longitude = 96 ;
 variables:
 	float air_temperature(latitude, longitude) ;
-		air_temperature:_FillValue = -1.e+30f ;
 		air_temperature:standard_name = "air_temperature" ;
 		air_temperature:units = "K" ;
 		air_temperature:um_stash_source = "m01s03i236" ;

lib/iris/tests/results/netcdf/netcdf_units_1.cml

@@ -1,6 +1,6 @@
 <?xml version="1.0" ?>
 <cubes xmlns="urn:x-iris:cubeml-0.2">
-  <cube core-dtype="float64" dtype="int32" standard_name="air_temperature" units="unknown" var_name="cube_1">
+  <cube core-dtype="int32" dtype="int32" standard_name="air_temperature" units="unknown" var_name="cube_1">
     <attributes>
       <attribute name="invalid_units" value="kevin"/>
     </attributes>

lib/iris/tests/results/netcdf/uint32_data_netcdf3.cml

@@ -1,6 +1,6 @@
 <?xml version="1.0" ?>
 <cubes xmlns="urn:x-iris:cubeml-0.2">
-  <cube core-dtype="int64" dtype="int32" standard_name="surface_temperature" units="K" var_name="temp">
+  <cube core-dtype="int32" dtype="int32" standard_name="surface_temperature" units="K" var_name="temp">
     <attributes>
       <attribute name="Conventions" value="CF-1.5"/>
     </attributes>

lib/iris/tests/test_netcdf.py

@@ -51,7 +51,6 @@
 
 @tests.skip_data
 class TestNetCDFLoad(tests.IrisTest):
-    @tests.skip_dask_mask
     def test_monotonic(self):
         cubes = iris.load(tests.get_data_path(
             ('NetCDF', 'testing', 'test_monotonic_coordinate.nc')))
@@ -84,7 +83,6 @@ def test_missing_time_bounds(self):
             dataset.close()
             cube = iris.load_cube(filename, 'eastward_wind')
 
-    @tests.skip_dask_mask
     def test_load_global_xyzt_gems(self):
         # Test loading single xyzt CF-netCDF file (multi-cube).
         cubes = iris.load(tests.get_data_path(('NetCDF', 'global', 'xyz_t',
@@ -96,7 +94,7 @@ def test_load_global_xyzt_gems(self):
         # manager loading.
         lnsp = cubes[1]
         self.assertTrue(ma.isMaskedArray(lnsp.data))
-        self.assertEqual(-32767.0, lnsp.fill_value)
+        self.assertEqual(-32767.0, lnsp.data.fill_value)
 
     def test_load_global_xyzt_gems_iter(self):
         # Test loading stepped single xyzt CF-netCDF file (multi-cube).
@@ -245,7 +243,6 @@ def test_deferred_loading(self):
         self.assertCML(cube[0][(0, 2), (1, 3)],
                        ('netcdf', 'netcdf_deferred_mix_1.cml'))
 
-    @tests.skip_dask_mask
     def test_units(self):
         # Test exercising graceful cube and coordinate units loading.
         cube0, cube1 = sorted(iris.load(tests.get_data_path(('NetCDF',
@@ -426,7 +423,6 @@ def test_netcdf_save_format(self):
             with self.assertRaises(ValueError):
                 iris.save(cube, file_out, netcdf_format='WIBBLE')
 
-    @tests.skip_dask_mask
     @tests.skip_data
     def test_netcdf_save_single(self):
         # Test saving a single CF-netCDF file.
@@ -445,7 +441,6 @@ def test_netcdf_save_single(self):
 
     # TODO investigate why merge now make time an AuxCoord rather than a
     # DimCoord and why forecast_period is 'preferred'.
-    @tests.skip_dask_mask
     @tests.skip_data
     def test_netcdf_save_multi2multi(self):
         # Test saving multiple CF-netCDF files.
@@ -464,7 +459,6 @@ def test_netcdf_save_multi2multi(self):
                 self.assertCDL(file_out, ('netcdf',
                                           'netcdf_save_multi_%d.cdl' % index))
 
-    @tests.skip_dask_mask
     @tests.skip_data
     def test_netcdf_save_multi2single(self):
         # Test saving multiple cubes to a single CF-netCDF file.
@@ -565,7 +559,6 @@ def test_netcdf_multi_conflict_name_dup_coord(self):
             self.assertCDL(
                 file_out, ('netcdf', 'multi_dim_coord_slightly_different.cdl'))
 
-    @tests.skip_dask_mask
     @tests.skip_data
     def test_netcdf_hybrid_height(self):
         # Test saving a CF-netCDF file which contains a hybrid height
@@ -590,7 +583,6 @@ def test_netcdf_hybrid_height(self):
             self.assertCML(cube,
                            ('netcdf', 'netcdf_save_load_hybrid_height.cml'))
 
-    @tests.skip_dask_mask
     @tests.skip_data
     def test_netcdf_save_ndim_auxiliary(self):
         # Test saving CF-netCDF with multi-dimensional auxiliary coordinates.
@@ -864,7 +856,6 @@ def test_uint32_auxiliary_coord_netcdf3(self):
                                       'uint32_auxiliary_coord_netcdf3.cml'),
                            checksum=False)
 
-    @tests.skip_dask_mask
     def test_uint32_data_netcdf3(self):
         self.cube.data = self.cube.data.astype(np.uint32)
         with self.temp_filename(suffix='.nc') as filename: