Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dask: Data.__init__ #262

Merged
merged 5 commits into from
Oct 6, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions cf/data/abstract/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,3 @@ def __repr__(self):
def dask_asarray(self):
"""TODODASK."""
return False


# --- End: class
23 changes: 18 additions & 5 deletions cf/data/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,14 +470,25 @@ def __init__(
)

if source is not None:
super().__init__(source=source, _use_array=_use_array)
try:
array = source._get_Array(None)
except AttributeError:
array = None

super().__init__(
source=source, _use_array=_use_array and array is not None
)

if _use_array:
try:
array = source._get_dask()
except (AttributeError, TypeError):
pass
else:
self._set_dask(array, copy=copy, delete_source=False)
# TODODASK: When PR #257 (dask: Data.__getitem__,
# Data.__setitem__) is merged, add in a
# "reset_mask_hardness=False".
Comment on lines +489 to +491
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#257 is now in, and so I can get this merged now to clear the reviewing backlog I will address this TODO and add in reset_mask_hardness=False myself as a follow-up commit. (I hope and assume that's OK.)

else:
self._del_dask(None)

Expand All @@ -488,9 +499,7 @@ def __init__(

return

super().__init__(
array=array, fill_value=fill_value, _use_array=_use_array
)
super().__init__(array=array, fill_value=fill_value, _use_array=False)

# Create the _HDF_chunks attribute: defines HDF chunking when
# writing to disk.
Expand Down Expand Up @@ -566,6 +575,10 @@ def __init__(
"compressed input arrays"
)

# Save the input compressed array, as this will contain
# extra information, such as a count or index variable.
self._set_Array(array)

array = compressed_to_dask(array)

elif not is_dask_collection(array):
Expand All @@ -580,7 +593,7 @@ def __init__(
dt = hasattr(first_value, "timetuple")

# Convert string or object date-times to floating point
# reference times
# reference times, if appropriate.
if array.dtype.kind in "USO" and (dt or units.isreftime):
array, units = convert_to_reftime(array, units, first_value)
# Reset the units
Expand Down
7 changes: 1 addition & 6 deletions cf/data/gatheredarray.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import cfdm

from . import mixin


class GatheredArray(mixin.CompressedArray, cfdm.GatheredArray):
class GatheredArray(cfdm.GatheredArray):
"""An underlying gathered array.

Compression by gathering combines axes of a multidimensional array
Expand All @@ -16,6 +14,3 @@ class GatheredArray(mixin.CompressedArray, cfdm.GatheredArray):
.. versionadded:: 3.0.0

"""


# --- End: class
11 changes: 6 additions & 5 deletions cf/read_write/read.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,9 +503,9 @@ def read(

.. versionadded:: 1.5

chunks: TODO
chunks: TODODASK

.. versionadded:: 4.0.0
.. versionadded:: TODODASK

umversion: deprecated at version 3.0.0
Use the *um* parameter instead.
Expand All @@ -522,7 +522,7 @@ def read(
select_options: deprecated at version 3.0.0
Use methods on the returned `FieldList` instead.

chunk: deprecated at version 4.0.0
chunk: deprecated at version TODODASK
Use the *chunks* parameter instead.

:Returns:
Expand Down Expand Up @@ -600,7 +600,7 @@ def read(
"cf.read",
{"chunk": chunk},
"Use keyword 'chunks' instead.",
version="4.0.0",
version="TODODASK",
) # pragma: no cover

# Parse select
Expand Down Expand Up @@ -889,6 +889,7 @@ def _read_a_file(
mask=True,
warn_valid=False,
chunks="auto",
select=None,
):
"""Read the contents of a single file into a field list.

Expand Down Expand Up @@ -991,7 +992,7 @@ def _read_a_file(
# return FieldList()

extra_read_vars = {
"chunk": chunk,
"chunks": chunks,
"fmt": selected_fmt,
"ignore_read_error": ignore_read_error,
# 'cfa' defaults to False. If the file has
Expand Down