From c70e58c73a48be1d3843971dbc4ea9c48ffdab1b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 12 May 2022 08:51:15 +0100 Subject: [PATCH 1/4] Data.Units --- cf/data/dask_utils.py | 40 ++++++++++++++++++++++++++++++++++++++-- cf/data/data.py | 18 +++++++----------- cf/test/test_Data.py | 16 ++++++++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 73dc7aed8e..b06fd317c9 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -14,8 +14,7 @@ from ..cfdatetime import dt2rt, rt2dt from ..functions import atol as cf_atol from ..functions import rtol as cf_rtol - -# from dask.utils import deepmap # Apply function inside nested lists +from ..units import Units def _da_ma_allclose(x, y, masked_equal=True, rtol=None, atol=None): @@ -594,3 +593,40 @@ def cf_dt2rt(a, units): """ return dt2rt(a, units_out=units, units_in=None) + + +def cf_units(a, from_units, to_units): + """Convert array values to have different equivalent units. + + .. versionadded:: TODODASK + + .. seealso:: `cf.Data.Units` + + :Parameters: + + a: `numpy.ndarray` + The array. + + from_units: `Units` + The existing units of the array. + + to_units: `Units` + The units that the array should be converted to. Must be + equivalent to *from_units*. + + :Returns: + + `numpy.ndarray` + An array containing values in the new units. + + **Examples** + + >>> import numpy as np + >>> a = np.array([1, 2]) + >>> print(cf.data.dask_utils.cf_units(a, cf.Units('km'), cf.Units('m'))) + [1000. 2000.] + + """ + return Units.conform( + a, from_units=from_units, to_units=to_units, inplace=False + ) diff --git a/cf/data/data.py b/cf/data/data.py index 4b1ea7e945..0a18a8e63b 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -50,6 +50,7 @@ cf_percentile, cf_rt2dt, cf_soften_mask, + cf_units, cf_where, ) from .mixin import DataClassDeprecationsMixin @@ -4699,11 +4700,8 @@ def Units(self, value): "Consider using the override_units method instead." ) - if not old_units: - self.override_units(value, inplace=True) - return - - if self.Units.equals(value): + if not old_units or self.Units.equals(value): + self._Units = value return dtype = self.dtype @@ -4713,13 +4711,11 @@ def Units(self, value): else: dtype = _dtype_float - def cf_Units(x): - return Units.conform( - x=x, from_units=old_units, to_units=value, inplace=False - ) - dx = self.to_dask_array() - dx = dx.map_blocks(cf_Units, dtype=dtype) + dx = dx.map_blocks( + partial(cf_units, from_units=old_units, to_units=value), + dtype=dtype, + ) self._set_dask(dx, reset_mask_hardness=False) self._Units = value diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index db61c729eb..fc008c0d21 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -3965,6 +3965,22 @@ def test_Data_get_data(self): d = cf.Data(9) self.assertIs(d, d.get_data()) + def test_Data_Units(self): + d = cf.Data(100, "m") + self.assertEqual(d.Units, cf.Units("m")) + + d.Units = cf.Units("km") + self.assertEqual(d.Units, cf.Units("km")) + self.assertEqual(d.array, 0.1) + + # Assign non-equivalent units + with self.assertRaises(ValueError): + d.Units = cf.Units("watt") + + # Delete units + with self.assertRaises(ValueError): + del d.Units + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From cd7903f759d8cef278a15d97a36c876d559f8891 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 12 May 2022 10:53:57 +0100 Subject: [PATCH 2/4] dev --- cf/data/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cf/data/data.py b/cf/data/data.py index 4b1ea7e945..4b96cd96a5 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -72,6 +72,7 @@ daskified_log_level = 0 + def daskified(apply_temp_log_level=None): def decorator(method): """Temporary decorator to mark and log methods migrated to Dask. From e6e4465096487aca654b7e2f4328bd5100281e9d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 12 May 2022 10:57:06 +0100 Subject: [PATCH 3/4] dev --- cf/data/dask_utils.py | 40 ++-------------------------------------- cf/data/data.py | 18 +++++++++++------- cf/test/test_Data.py | 16 ---------------- 3 files changed, 13 insertions(+), 61 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index b06fd317c9..73dc7aed8e 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -14,7 +14,8 @@ from ..cfdatetime import dt2rt, rt2dt from ..functions import atol as cf_atol from ..functions import rtol as cf_rtol -from ..units import Units + +# from dask.utils import deepmap # Apply function inside nested lists def _da_ma_allclose(x, y, masked_equal=True, rtol=None, atol=None): @@ -593,40 +594,3 @@ def cf_dt2rt(a, units): """ return dt2rt(a, units_out=units, units_in=None) - - -def cf_units(a, from_units, to_units): - """Convert array values to have different equivalent units. - - .. versionadded:: TODODASK - - .. seealso:: `cf.Data.Units` - - :Parameters: - - a: `numpy.ndarray` - The array. - - from_units: `Units` - The existing units of the array. - - to_units: `Units` - The units that the array should be converted to. Must be - equivalent to *from_units*. - - :Returns: - - `numpy.ndarray` - An array containing values in the new units. - - **Examples** - - >>> import numpy as np - >>> a = np.array([1, 2]) - >>> print(cf.data.dask_utils.cf_units(a, cf.Units('km'), cf.Units('m'))) - [1000. 2000.] - - """ - return Units.conform( - a, from_units=from_units, to_units=to_units, inplace=False - ) diff --git a/cf/data/data.py b/cf/data/data.py index a626e3bf7c..4b96cd96a5 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -50,7 +50,6 @@ cf_percentile, cf_rt2dt, cf_soften_mask, - cf_units, cf_where, ) from .mixin import DataClassDeprecationsMixin @@ -4701,8 +4700,11 @@ def Units(self, value): "Consider using the override_units method instead." ) - if not old_units or self.Units.equals(value): - self._Units = value + if not old_units: + self.override_units(value, inplace=True) + return + + if self.Units.equals(value): return dtype = self.dtype @@ -4712,11 +4714,13 @@ def Units(self, value): else: dtype = _dtype_float + def cf_Units(x): + return Units.conform( + x=x, from_units=old_units, to_units=value, inplace=False + ) + dx = self.to_dask_array() - dx = dx.map_blocks( - partial(cf_units, from_units=old_units, to_units=value), - dtype=dtype, - ) + dx = dx.map_blocks(cf_Units, dtype=dtype) self._set_dask(dx, reset_mask_hardness=False) self._Units = value diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index fc008c0d21..db61c729eb 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -3965,22 +3965,6 @@ def test_Data_get_data(self): d = cf.Data(9) self.assertIs(d, d.get_data()) - def test_Data_Units(self): - d = cf.Data(100, "m") - self.assertEqual(d.Units, cf.Units("m")) - - d.Units = cf.Units("km") - self.assertEqual(d.Units, cf.Units("km")) - self.assertEqual(d.array, 0.1) - - # Assign non-equivalent units - with self.assertRaises(ValueError): - d.Units = cf.Units("watt") - - # Delete units - with self.assertRaises(ValueError): - del d.Units - if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 54d6f70a83e24aaa6a3f588e8921c4fc42a0c18c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 12 May 2022 11:16:51 +0100 Subject: [PATCH 4/4] Data.get_[count|index|list] --- cf/data/data.py | 1 - cf/test/test_Data.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/cf/data/data.py b/cf/data/data.py index 4b96cd96a5..4b1ea7e945 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -72,7 +72,6 @@ daskified_log_level = 0 - def daskified(apply_temp_log_level=None): def decorator(method): """Temporary decorator to mark and log methods migrated to Dask. diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index db61c729eb..484b3f91a4 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -3965,6 +3965,42 @@ def test_Data_get_data(self): d = cf.Data(9) self.assertIs(d, d.get_data()) + def test_Data_get_count(self): + import cfdm + + f = cfdm.read("DSG_timeSeries_contiguous.nc")[0] + f = f.data + d = cf.Data(cf.RaggedContiguousArray(source=f.source())) + self.assertIsInstance(d.get_count(), cfdm.Count) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_count() + + def test_Data_get_index(self): + import cfdm + + f = cfdm.read("DSG_timeSeries_indexed.nc")[0] + f = f.data + d = cf.Data(cf.RaggedIndexedArray(source=f.source())) + self.assertIsInstance(d.get_index(), cfdm.Index) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_index() + + def test_Data_get_list(self): + import cfdm + + f = cfdm.read("gathered.nc")[0] + f = f.data + d = cf.Data(cf.GatheredArray(source=f.source())) + self.assertIsInstance(d.get_list(), cfdm.List) + + d = cf.Data(9, "m") + with self.assertRaises(ValueError): + d.get_list() + if __name__ == "__main__": print("Run date:", datetime.datetime.now())