From 4a2acbc5aabb2e7955b3f405d20275f9d4b51a57 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 10:51:52 -0400 Subject: [PATCH 01/15] Switch enable_cftimeindex to True by default --- doc/time-series.rst | 86 ++++++++++++++++++++++++------------------ doc/whats-new.rst | 6 +++ xarray/core/options.py | 4 +- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index c1a686b409f..adf15c8a402 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -71,10 +71,11 @@ One unfortunate limitation of using ``datetime64[ns]`` is that it limits the native representation of dates to those that fall between the years 1678 and 2262. When a netCDF file contains dates outside of these bounds, dates will be returned as arrays of :py:class:`cftime.datetime` objects and a :py:class:`~xarray.CFTimeIndex` -can be used for indexing. The :py:class:`~xarray.CFTimeIndex` enables only a subset of -the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only enabled -when using the standalone version of ``cftime`` (not the version packaged with -earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more information. +will be used for indexing. :py:class:`~xarray.CFTimeIndex` enables a subset of +the indexing functionality of a :py:class:`pandas.DatetimeIndex` and is only +fully compatible with the standalone version of ``cftime`` (not the version +packaged with earlier versions ``netCDF4``). See :ref:`CFTimeIndex` for more +information. Datetime indexing ----------------- @@ -223,16 +224,26 @@ Through the standalone ``cftime`` library and a custom subclass of functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for dates from non-standard calendars or dates using a standard calendar, but outside the `Timestamp-valid range`_ (approximately between years 1678 and -2262). This behavior has not yet been turned on by default; to take advantage -of this functionality, you must have the ``enable_cftimeindex`` option set to -``True`` within your context (see :py:func:`~xarray.set_options` for more -information). It is expected that this will become the default behavior in -xarray version 0.11. +2262). -For instance, you can create a DataArray indexed by a time -coordinate with a no-leap calendar within a context manager setting the -``enable_cftimeindex`` option, and the time index will be cast to a -:py:class:`~xarray.CFTimeIndex`: +.. note:: + + As of xarray version 0.11, by default, :py:class:`cftime.datetime` objects + will be used to represent times (either in indexes, as a + :py:class:`~xarray.CFTimeIndex`, or in data arrays with dtype object) if + any of the following are true: + + - The dates are from a non-standard calendar + - Any dates are outside the Timestamp-valid range. + + Otherwise pandas-compatible dates from a standard calendar will be + represented with the ``np.datetime64[ns]`` data type, enabling the use of a + :py:class:`pandas.DatetimeIndex` or arrays with dtype ``np.datetime64[ns]`` + and their full set of associated features. + +For example, you can create a DataArray indexed by a time +coordinate with dates from a no-leap calendar and a +:py:class:`~xarray.CFTimeIndex` will automatically be used: .. ipython:: python @@ -241,27 +252,11 @@ coordinate with a no-leap calendar within a context manager setting the dates = [DatetimeNoLeap(year, month, 1) for year, month in product(range(1, 3), range(1, 13))] - with xr.set_options(enable_cftimeindex=True): - da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], - name='foo') + da = xr.DataArray(np.arange(24), coords=[dates], dims=['time'], name='foo') -.. note:: - - With the ``enable_cftimeindex`` option activated, a :py:class:`~xarray.CFTimeIndex` - will be used for time indexing if any of the following are true: - - - The dates are from a non-standard calendar - - Any dates are outside the Timestamp-valid range - - Otherwise a :py:class:`pandas.DatetimeIndex` will be used. In addition, if any - variable (not just an index variable) is encoded using a non-standard - calendar, its times will be decoded into :py:class:`cftime.datetime` objects, - regardless of whether or not they can be represented using - ``np.datetime64[ns]`` objects. - xarray also includes a :py:func:`~xarray.cftime_range` function, which enables -creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For instance, we can -create the same dates and DataArray we created above using: +creating a :py:class:`~xarray.CFTimeIndex` with regularly-spaced dates. For +instance, we can create the same dates and DataArray we created above using: .. ipython:: python @@ -317,13 +312,32 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: .. ipython:: python - da.to_netcdf('example.nc') - xr.open_dataset('example.nc') + da.to_netcdf('example-no-leap.nc') + xr.open_dataset('example-no-leap.nc') .. note:: - Currently resampling along the time dimension for data indexed by a - :py:class:`~xarray.CFTimeIndex` is not supported. + While much of the time series functionality that is possible for standard + dates has been implemented for dates from non-standard calendars, there are + still some remaining important features that have yet to be implemented, + for example: + + - Resampling along the time dimension for data indexed by a + :py:class:`~xarray.CFTimeIndex` + - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes. + + If at any time you would like to restore the old default behavior, which was + to attempt to decode datetimes into ``np.datetime64[ns]`` objects whenever + possible (regardless of calendar type), you can set ``enable_cftimeindex`` to + ``False`` within your context when opening a file (see + :py:func:`~xarray.set_options` for more information). For some use-cases + this behavior may still be useful (e.g. to allow the use of some forms + resample with non-standard calendars); however in this case one should use + caution to only perform operations which do not depend on differences + between dates (e.g. differentiation, interpolation, or upsampling with + resample), as these could introduce subtle and silent errors due to the + difference in calendar types between the dates encoded in your data and the + dates stored in memory. .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e1744e28077..c91e3e049ae 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,6 +33,12 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ +- The option ``enable_cftimeindex`` has now been set to ``True`` by default. + This means that by default any dates encoded using a non-standard calendar + will be decoded into objects of type :py:class:`cftime.datetime`, regardless + of whether or not it might be possible to coerce them into + ``np.datetime64[ns]`` objects. One can explicitly set the option to + ``False`` to restore the old behavior. - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. - Iterating over a ``Dataset`` now includes only data variables, not coordinates. diff --git a/xarray/core/options.py b/xarray/core/options.py index 04ea0be7172..fe67e0b4035 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -10,7 +10,7 @@ OPTIONS = { DISPLAY_WIDTH: 80, ARITHMETIC_JOIN: 'inner', - ENABLE_CFTIMEINDEX: False, + ENABLE_CFTIMEINDEX: True, FILE_CACHE_MAXSIZE: 128, CMAP_SEQUENTIAL: 'viridis', CMAP_DIVERGENT: 'RdBu_r', @@ -52,7 +52,7 @@ class set_options(object): Default: ``'inner'``. - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` for time indexes with non-standard calendars or dates outside the - Timestamp-valid range. Default: ``False``. + Timestamp-valid range. Default: ``True``. - ``file_cache_maxsize``: maximum number of open files to hold in xarray's global least-recently-usage cached. This should be smaller than your system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux. From 2c70d3217d845996639f1e7da372c31ff4816d21 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 13:50:11 -0400 Subject: [PATCH 02/15] Add a friendlier error message when plotting cftime objects --- xarray/plot/plot.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 7129157ec7f..8d21e084946 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -145,8 +145,13 @@ def plot(darray, row=None, col=None, col_wrap=None, ax=None, hue=None, darray = darray.squeeze() if contains_cftime_datetimes(darray): - raise NotImplementedError('Plotting arrays of cftime.datetime objects ' - 'is currently not possible.') + raise NotImplementedError( + 'Built-in plotting of arrays of cftime.datetime objects or arrays ' + 'indexed by cftime.datetime objects is currently not implemented ' + 'within xarray. A possible workaround is to use the ' + 'nc-time-axis package ' + '(https://github.com/SciTools/nc-time-axis) to convert the dates ' + 'to a plottable type and plot your data directly with matplotlib.') plot_dims = set(darray.dims) plot_dims.discard(row) From 69f6dc71c14b7d361a833302439bd7d6a9d261d4 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 14:05:29 -0400 Subject: [PATCH 03/15] Mention that the non-standard calendars are used in climate science --- doc/time-series.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index adf15c8a402..e8083a0ca0f 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -222,9 +222,9 @@ Non-standard calendars and dates outside the Timestamp-valid range Through the standalone ``cftime`` library and a custom subclass of :py:class:`pandas.Index`, xarray supports a subset of the indexing functionality enabled through the standard :py:class:`pandas.DatetimeIndex` for -dates from non-standard calendars or dates using a standard calendar, but -outside the `Timestamp-valid range`_ (approximately between years 1678 and -2262). +dates from non-standard calendars commonly used in climate science or dates +using a standard calendar, but outside the `Timestamp-valid range`_ +(approximately between years 1678 and 2262). .. note:: From d203491912719a51ecfbd9df6f0186f1b95a3c62 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 14:40:11 -0400 Subject: [PATCH 04/15] Add GH issue references to docs --- doc/time-series.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index e8083a0ca0f..fcaac83ed42 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -323,8 +323,9 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: for example: - Resampling along the time dimension for data indexed by a - :py:class:`~xarray.CFTimeIndex` - - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes. + :py:class:`~xarray.CFTimeIndex` (:issue:`2191`, :issue:`2458`) + - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes + (:issue:`2164`). If at any time you would like to restore the old default behavior, which was to attempt to decode datetimes into ``np.datetime64[ns]`` objects whenever From 93599cac7c826d7db67b9dc5e821ba667836db67 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Fri, 26 Oct 2018 21:54:46 -0400 Subject: [PATCH 05/15] Deprecate enable_cftimeindex option --- doc/time-series.rst | 20 +- doc/whats-new.rst | 14 +- xarray/coding/times.py | 49 ++-- xarray/core/options.py | 7 + xarray/core/utils.py | 16 +- xarray/tests/test_backends.py | 64 ++---- xarray/tests/test_cftimeindex.py | 22 +- xarray/tests/test_coding_times.py | 359 ++++++++++++------------------ xarray/tests/test_dataarray.py | 3 +- xarray/tests/test_options.py | 3 + xarray/tests/test_utils.py | 14 +- 11 files changed, 227 insertions(+), 344 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index fcaac83ed42..0f32c0250e9 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -325,16 +325,16 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: - Resampling along the time dimension for data indexed by a :py:class:`~xarray.CFTimeIndex` (:issue:`2191`, :issue:`2458`) - Built-in plotting of data with :py:class:`cftime.datetime` coordinate axes - (:issue:`2164`). - - If at any time you would like to restore the old default behavior, which was - to attempt to decode datetimes into ``np.datetime64[ns]`` objects whenever - possible (regardless of calendar type), you can set ``enable_cftimeindex`` to - ``False`` within your context when opening a file (see - :py:func:`~xarray.set_options` for more information). For some use-cases - this behavior may still be useful (e.g. to allow the use of some forms - resample with non-standard calendars); however in this case one should use - caution to only perform operations which do not depend on differences + (:issue:`2164`). + + For some use-cases it may still be useful to convert from + :py:class:`cftime.datetime` objects to ``np.datetime64[ns]`` objects, + despite the difference in calendar types (e.g. to allow the use of some + forms resample with non-standard calendars). The recommended way of doing + this is through... + + However in this case one should + use caution to only perform operations which do not depend on differences between dates (e.g. differentiation, interpolation, or upsampling with resample), as these could introduce subtle and silent errors due to the difference in calendar types between the dates encoded in your data and the diff --git a/doc/whats-new.rst b/doc/whats-new.rst index c91e3e049ae..e85c43b7a89 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -33,12 +33,14 @@ v0.11.0 (unreleased) Breaking changes ~~~~~~~~~~~~~~~~ -- The option ``enable_cftimeindex`` has now been set to ``True`` by default. - This means that by default any dates encoded using a non-standard calendar - will be decoded into objects of type :py:class:`cftime.datetime`, regardless - of whether or not it might be possible to coerce them into - ``np.datetime64[ns]`` objects. One can explicitly set the option to - ``False`` to restore the old behavior. +- For non-standard calendars commonly used in climate science, xarray will now + always use :py:class:`cftime.datetime` objects, rather than by default try to + coerce them to ``np.datetime64[ns]`` objects. A + :py:class:`~xarray.CFTimeIndex` will be used for indexing in these cases. + New public methods for converting :py:class:`cftime.datetime` dates to + ``np.datetime64[ns]`` objects have been added for use-cases where standard + datetimes are currently required. Setting the ``enable_cftimeindex`` option + is now a no-op and emits a FutureWarning. - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. - Iterating over a ``Dataset`` now includes only data variables, not coordinates. diff --git a/xarray/coding/times.py b/xarray/coding/times.py index dff7e75bdcf..d64e95b4e78 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -12,7 +12,6 @@ from ..core import indexing from ..core.common import contains_cftime_datetimes from ..core.formatting import first_n_items, format_timestamp, last_item -from ..core.options import OPTIONS from ..core.pycompat import PY3 from ..core.variable import Variable from .variables import ( @@ -61,8 +60,9 @@ def _require_standalone_cftime(): try: import cftime # noqa: F401 except ImportError: - raise ImportError('Using a CFTimeIndex requires the standalone ' - 'version of the cftime library.') + raise ImportError('Decoding times with non-standard calendars ' + 'or outside the pandas.Timestamp-valid range ' + 'requires the standalone cftime package.') def _netcdf_to_numpy_timeunit(units): @@ -84,41 +84,32 @@ def _unpack_netcdf_time_units(units): return delta_units, ref_date -def _decode_datetime_with_cftime(num_dates, units, calendar, - enable_cftimeindex): +def _decode_datetime_with_cftime(num_dates, units, calendar): cftime = _import_cftime() - if enable_cftimeindex: - _require_standalone_cftime() + + if cftime.__name__ == 'cftime': dates = np.asarray(cftime.num2date(num_dates, units, calendar, only_use_cftime_datetimes=True)) else: + # Must be using num2date from an old version of netCDF4 which + # does not have the only_use_cftime_datetimes option. dates = np.asarray(cftime.num2date(num_dates, units, calendar)) if (dates[np.nanargmin(num_dates)].year < 1678 or dates[np.nanargmax(num_dates)].year >= 2262): - if not enable_cftimeindex or calendar in _STANDARD_CALENDARS: + if calendar in _STANDARD_CALENDARS: warnings.warn( 'Unable to decode time axis into full ' 'numpy.datetime64 objects, continuing using dummy ' 'cftime.datetime objects instead, reason: dates out ' 'of range', SerializationWarning, stacklevel=3) else: - if enable_cftimeindex: - if calendar in _STANDARD_CALENDARS: - dates = cftime_to_nptime(dates) - else: - try: - dates = cftime_to_nptime(dates) - except ValueError as e: - warnings.warn( - 'Unable to decode time axis into full ' - 'numpy.datetime64 objects, continuing using ' - 'dummy cftime.datetime objects instead, reason:' - '{0}'.format(e), SerializationWarning, stacklevel=3) + if calendar in _STANDARD_CALENDARS: + dates = cftime_to_nptime(dates) return dates -def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): +def _decode_cf_datetime_dtype(data, units, calendar): # Verify that at least the first and last date can be decoded # successfully. Otherwise, tracebacks end up swallowed by # Dataset.__repr__ when users try to view their lazily decoded array. @@ -128,8 +119,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): last_item(values) or [0]]) try: - result = decode_cf_datetime(example_value, units, calendar, - enable_cftimeindex) + result = decode_cf_datetime(example_value, units, calendar) except Exception: calendar_msg = ('the default calendar' if calendar is None else 'calendar %r' % calendar) @@ -145,8 +135,7 @@ def _decode_cf_datetime_dtype(data, units, calendar, enable_cftimeindex): return dtype -def decode_cf_datetime(num_dates, units, calendar=None, - enable_cftimeindex=False): +def decode_cf_datetime(num_dates, units, calendar=None): """Given an array of numeric dates in netCDF format, convert it into a numpy array of date time objects. @@ -200,8 +189,7 @@ def decode_cf_datetime(num_dates, units, calendar=None, except (OutOfBoundsDatetime, OverflowError): dates = _decode_datetime_with_cftime( - flat_num_dates.astype(np.float), units, calendar, - enable_cftimeindex) + flat_num_dates.astype(np.float), units, calendar) return dates.reshape(num_dates.shape) @@ -399,15 +387,12 @@ def encode(self, variable, name=None): def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) - enable_cftimeindex = OPTIONS['enable_cftimeindex'] if 'units' in attrs and 'since' in attrs['units']: units = pop_to(attrs, encoding, 'units') calendar = pop_to(attrs, encoding, 'calendar') - dtype = _decode_cf_datetime_dtype( - data, units, calendar, enable_cftimeindex) + dtype = _decode_cf_datetime_dtype(data, units, calendar) transform = partial( - decode_cf_datetime, units=units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + decode_cf_datetime, units=units, calendar=calendar) data = lazy_elemwise_func(data, transform, dtype) return Variable(dims, data, attrs, encoding) diff --git a/xarray/core/options.py b/xarray/core/options.py index fe67e0b4035..f76f06032fc 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function +import warnings + DISPLAY_WIDTH = 'display_width' ARITHMETIC_JOIN = 'arithmetic_join' ENABLE_CFTIMEINDEX = 'enable_cftimeindex' @@ -83,6 +85,11 @@ class set_options(object): def __init__(self, **kwargs): self.old = OPTIONS.copy() + if ENABLE_CFTIMEINDEX in kwargs: + warnings.warn( + 'The enable_cftimeindex option is now a no-op ' + 'and will be removed in a future version of xarray.', + FutureWarning) for k, v in kwargs.items(): if k not in OPTIONS: raise ValueError( diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 5c9d8bfbf77..015916d668e 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -13,7 +13,6 @@ import numpy as np import pandas as pd -from .options import OPTIONS from .pycompat import ( OrderedDict, basestring, bytes_type, dask_array_type, iteritems) @@ -41,16 +40,13 @@ def wrapper(*args, **kwargs): def _maybe_cast_to_cftimeindex(index): from ..coding.cftimeindex import CFTimeIndex - if not OPTIONS['enable_cftimeindex']: - return index - else: - if index.dtype == 'O': - try: - return CFTimeIndex(index) - except (ImportError, TypeError): - return index - else: + if index.dtype == 'O': + try: + return CFTimeIndex(index) + except (ImportError, TypeError): return index + else: + return index def safe_cast_to_index(array): diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index c6a2df733fa..80d3a9d526e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -356,7 +356,7 @@ def test_roundtrip_numpy_datetime_data(self): assert actual.t0.encoding['units'] == 'days since 1950-01-01' @requires_cftime - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): from .test_coding_times import _all_cftime_date_types date_types = _all_cftime_date_types() @@ -373,21 +373,20 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): warnings.filterwarnings( 'ignore', 'Unable to decode time axis') - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected, save_kwargs=kwds) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t.encoding['units'] == - 'days since 0001-01-01 00:00:00.000000') - assert (actual.t.encoding['calendar'] == - expected_calendar) - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - assert (actual.t0.encoding['units'] == - 'days since 0001-01-01') - assert (actual.t.encoding['calendar'] == - expected_calendar) + with self.roundtrip(expected, save_kwargs=kwds) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t.encoding['units'] == + 'days since 0001-01-01 00:00:00.000000') + assert (actual.t.encoding['calendar'] == + expected_calendar) + + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() + assert (actual.t0.encoding['units'] == + 'days since 0001-01-01') + assert (actual.t.encoding['calendar'] == + expected_calendar) def test_roundtrip_timedelta_data(self): time_deltas = pd.to_timedelta(['1h', '2h', 'NaT']) @@ -2087,7 +2086,7 @@ def test_roundtrip_numpy_datetime_data(self): with self.roundtrip(expected) as actual: assert_identical(expected, actual) - def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): + def test_roundtrip_cftime_datetime_data(self): # Override method in DatasetIOBase - remove not applicable # save_kwds from .test_coding_times import _all_cftime_date_types @@ -2099,33 +2098,12 @@ def test_roundtrip_cftime_datetime_data_enable_cftimeindex(self): expected_decoded_t = np.array(times) expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - with xr.set_options(enable_cftimeindex=True): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() - - def test_roundtrip_cftime_datetime_data_disable_cftimeindex(self): - # Override method in DatasetIOBase - remove not applicable - # save_kwds - from .test_coding_times import _all_cftime_date_types - - date_types = _all_cftime_date_types() - for date_type in date_types.values(): - times = [date_type(1, 1, 1), date_type(1, 1, 2)] - expected = Dataset({'t': ('t', times), 't0': times[0]}) - expected_decoded_t = np.array(times) - expected_decoded_t0 = np.array([date_type(1, 1, 1)]) - - with xr.set_options(enable_cftimeindex=False): - with self.roundtrip(expected) as actual: - abs_diff = abs(actual.t.values - expected_decoded_t) - assert (abs_diff <= np.timedelta64(1, 's')).all() + with self.roundtrip(expected) as actual: + abs_diff = abs(actual.t.values - expected_decoded_t) + assert (abs_diff <= np.timedelta64(1, 's')).all() - abs_diff = abs(actual.t0.values - expected_decoded_t0) - assert (abs_diff <= np.timedelta64(1, 's')).all() + abs_diff = abs(actual.t0.values - expected_decoded_t0) + assert (abs_diff <= np.timedelta64(1, 's')).all() def test_write_store(self): # Override method in DatasetIOBase - not applicable to dask diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index e18c55d2fae..61cf3968fe0 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -594,18 +594,16 @@ def test_indexing_in_dataframe_iloc(df, index): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_concat_cftimeindex(date_type, enable_cftimeindex): - with xr.set_options(enable_cftimeindex=enable_cftimeindex): - da1 = xr.DataArray( - [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], - dims=['time']) - da2 = xr.DataArray( - [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], - dims=['time']) - da = xr.concat([da1, da2], dim='time') - - if enable_cftimeindex and has_cftime: +def test_concat_cftimeindex(date_type): + da1 = xr.DataArray( + [1., 2.], coords=[[date_type(1, 1, 1), date_type(1, 2, 1)]], + dims=['time']) + da2 = xr.DataArray( + [3., 4.], coords=[[date_type(1, 3, 1), date_type(1, 4, 1)]], + dims=['time']) + da = xr.concat([da1, da2], dim='time') + + if has_cftime: assert isinstance(da.indexes['time'], CFTimeIndex) else: assert isinstance(da.indexes['time'], pd.Index) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 10a1a956b27..3b515aa6871 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -7,9 +7,8 @@ import pandas as pd import pytest -from xarray import DataArray, Variable, coding, decode_cf, set_options -from xarray.coding.times import _import_cftime -from xarray.coding.variables import SerializationWarning +from xarray import DataArray, Variable, coding, decode_cf +from xarray.coding.times import _import_cftime, cftime_to_nptime from xarray.core.common import contains_cftime_datetimes from . import ( @@ -54,14 +53,6 @@ _STANDARD_CALENDARS)] -@np.vectorize -def _ensure_naive_tz(dt): - if hasattr(dt, 'tzinfo'): - return dt.replace(tzinfo=None) - else: - return dt - - def _all_cftime_date_types(): try: import cftime @@ -82,24 +73,23 @@ def _all_cftime_date_types(): _CF_DATETIME_TESTS) def test_cf_datetime(num_dates, units, calendar): cftime = _import_cftime() - expected = _ensure_naive_tz( - cftime.num2date(num_dates, units, calendar)) + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_dates, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_dates, units, calendar) + min_y = np.ravel(np.atleast_1d(expected))[np.nanargmin(num_dates)].year + max_y = np.ravel(np.atleast_1d(expected))[np.nanargmax(num_dates)].year + if min_y >= 1678 and max_y < 2262: + expected = cftime_to_nptime(expected) + with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime(num_dates, units, calendar) - if (isinstance(actual, np.ndarray) and - np.issubdtype(actual.dtype, np.datetime64)): - # self.assertEqual(actual.dtype.kind, 'M') - # For some reason, numpy 1.8 does not compare ns precision - # datetime64 arrays as equal to arrays of datetime objects, - # but it works for us precision. Thus, convert to us - # precision for the actual array equal comparison... - actual_cmp = actual.astype('M8[us]') - else: - actual_cmp = actual - assert_array_equal(expected, actual_cmp) + + assert_array_equal(expected, actual) encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar) if '1-1-1' not in units: @@ -123,8 +113,12 @@ def test_decode_cf_datetime_overflow(): # checks for # https://github.com/pydata/pandas/issues/14068 # https://github.com/pydata/xarray/issues/975 + try: + from cftime import DatetimeGregorian + except ImportError: + from netcdftime import DatetimeGregorian - from datetime import datetime + datetime = DatetimeGregorian units = 'days since 2000-01-01 00:00:00' # date after 2262 and before 1678 @@ -150,7 +144,7 @@ def test_decode_cf_datetime_non_standard_units(): @requires_cftime_or_netCDF4 def test_decode_cf_datetime_non_iso_strings(): # datetime strings that are _almost_ ISO compliant but not quite, - # but which netCDF4.num2date can still parse correctly + # but which cftime.num2date can still parse correctly expected = pd.date_range(periods=100, start='2000-01-01', freq='h') cases = [(np.arange(100), 'hours since 2000-01-01 0'), (np.arange(100), 'hours since 2000-1-1 0'), @@ -161,28 +155,17 @@ def test_decode_cf_datetime_non_iso_strings(): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) -def test_decode_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) +def test_decode_standard_calendar_inside_timestamp_range(calendar): cftime = _import_cftime() + units = 'days since 0001-01-01' - times = pd.date_range('2001-04-01-00', end='2001-04-30-23', - freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) + times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') + time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) expected = times.values expected_dtype = np.dtype('M8[ns]') - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -192,32 +175,28 @@ def test_decode_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') - noleap_time = cftime.date2num(times.to_pydatetime(), units, - calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar) - expected_dtype = np.dtype('O') + non_standard_time = cftime.date2num( + times.to_pydatetime(), units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date( + non_standard_time, units, calendar=calendar, + only_use_cftime_datetimes=True) else: - expected = times.values - expected_dtype = np.dtype('M8[ns]') + expected = cftime.num2date(non_standard_time, units, + calendar=calendar) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'Unable to decode time axis') - actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + expected_dtype = np.dtype('O') + + actual = coding.times.decode_cf_datetime( + non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -227,33 +206,27 @@ def test_decode_non_standard_calendar_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_dates_outside_timestamp_range( - calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_dates_outside_timestamp_range(calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times = [datetime(1, 4, 1, h) for h in range(1, 5)] - noleap_time = cftime.date2num(times, units, calendar=calendar) - if enable_cftimeindex: - expected = cftime.num2date(noleap_time, units, calendar=calendar, + time = cftime.date2num(times, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(time, units, calendar=calendar, only_use_cftime_datetimes=True) else: - expected = cftime.num2date(noleap_time, units, calendar=calendar) + expected = cftime.num2date(time, units, calendar=calendar) + expected_date_type = type(expected[0]) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - noleap_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + time, units, calendar=calendar) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, @@ -263,57 +236,37 @@ def test_decode_dates_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + num_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_non_standard_calendar_single_element_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): units = 'days since 0001-01-01' for num_time in [735368, [735368], [[735368]]]: with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert actual.dtype == np.dtype('O') - else: - assert actual.dtype == np.dtype('M8[ns]') + num_time, units, calendar=calendar) + assert actual.dtype == np.dtype('O') @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_single_element_outside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' for days in [1, 1470376]: @@ -322,40 +275,39 @@ def test_decode_single_element_outside_timestamp_range( warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - expected = cftime.num2date(days, units, calendar) + num_time, units, calendar=calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(days, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(days, units, calendar) + assert isinstance(actual.item(), type(expected)) @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _STANDARD_CALENDARS) def test_decode_standard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 expected1 = times1.values expected2 = times2.values actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('M8[ns]') abs_diff1 = abs(actual[:, 0] - expected1) @@ -368,39 +320,35 @@ def test_decode_standard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_NON_STANDARD_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _NON_STANDARD_CALENDARS) def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - + calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') - noleap_time1 = cftime.date2num(times1.to_pydatetime(), - units, calendar=calendar) - noleap_time2 = cftime.date2num(times2.to_pydatetime(), - units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) - expected_dtype = np.dtype('O') + time1 = cftime.date2num(times1.to_pydatetime(), + units, calendar=calendar) + time2 = cftime.date2num(times2.to_pydatetime(), + units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, + only_use_cftime_datetimes=True) + expected2 = cftime.num2date(time2, units, calendar, + only_use_cftime_datetimes=True) else: - expected1 = times1.values - expected2 = times2.values - expected_dtype = np.dtype('M8[ns]') + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) + + expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff1 = abs(actual[:, 0] - expected1) @@ -413,41 +361,34 @@ def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) def test_decode_multidim_time_outside_timestamp_range( - calendar, enable_cftimeindex): + calendar): from datetime import datetime - - if enable_cftimeindex: - pytest.importorskip('cftime') - cftime = _import_cftime() units = 'days since 0001-01-01' times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] - noleap_time1 = cftime.date2num(times1, units, calendar=calendar) - noleap_time2 = cftime.date2num(times2, units, calendar=calendar) - mdim_time = np.empty((len(noleap_time1), 2), ) - mdim_time[:, 0] = noleap_time1 - mdim_time[:, 1] = noleap_time2 - - if enable_cftimeindex: - expected1 = cftime.num2date(noleap_time1, units, calendar, + time1 = cftime.date2num(times1, units, calendar=calendar) + time2 = cftime.date2num(times2, units, calendar=calendar) + mdim_time = np.empty((len(time1), 2), ) + mdim_time[:, 0] = time1 + mdim_time[:, 1] = time2 + + if cftime.__name__ == 'cftime': + expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True) - expected2 = cftime.num2date(noleap_time2, units, calendar, + expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True) else: - expected1 = cftime.num2date(noleap_time1, units, calendar) - expected2 = cftime.num2date(noleap_time2, units, calendar) + expected1 = cftime.num2date(time1, units, calendar) + expected2 = cftime.num2date(time2, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( - mdim_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype('O') @@ -461,66 +402,51 @@ def test_decode_multidim_time_outside_timestamp_range( @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day', 'all_leap', '366_day'], [False, True])) -def test_decode_non_standard_calendar_single_element_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +@pytest.mark.parametrize('calendar', ['360_day', 'all_leap', '366_day']) +def test_decode_non_standard_calendar_single_element( + calendar): cftime = _import_cftime() - units = 'days since 0001-01-01' + try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: - # Must be using standalone netcdftime library + # Must be using the standalone cftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) - if enable_cftimeindex: - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - else: - with pytest.warns(SerializationWarning, - match='Unable to decode time axis'): - actual = coding.times.decode_cf_datetime( - num_time, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) + actual = coding.times.decode_cf_datetime( + num_time, units, calendar=calendar) - expected = np.asarray(cftime.num2date(num_time, units, calendar)) + if cftime.__name__ == 'cftime': + expected = np.asarray(cftime.num2date( + num_time, units, calendar, only_use_cftime_datetimes=True)) + else: + expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(['360_day'], [False, True])) -def test_decode_non_standard_calendar_fallback( - calendar, enable_cftimeindex): - if enable_cftimeindex: - pytest.importorskip('cftime') - +def test_decode_360_day_calendar(): cftime = _import_cftime() + calendar = '360_day' # ensure leap year doesn't matter for year in [2010, 2011, 2012, 2013, 2014]: units = 'days since {0}-01-01'.format(year) num_times = np.arange(100) - expected = cftime.num2date(num_times, units, calendar) + + if cftime.__name__ == 'cftime': + expected = cftime.num2date(num_times, units, calendar, + only_use_cftime_datetimes=True) + else: + expected = cftime.num2date(num_times, units, calendar) with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') actual = coding.times.decode_cf_datetime( - num_times, units, calendar=calendar, - enable_cftimeindex=enable_cftimeindex) - if enable_cftimeindex: - assert len(w) == 0 - else: - assert len(w) == 1 - assert 'Unable to decode time axis' in str(w[0].message) + num_times, units, calendar=calendar) + assert len(w) == 0 assert actual.dtype == np.dtype('O') assert_array_equal(actual, expected) @@ -670,11 +596,8 @@ def test_format_cftime_datetime(date_args, expected): assert result == expected -@pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize( - ['calendar', 'enable_cftimeindex'], - product(_ALL_CALENDARS, [False, True])) -def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_decode_cf(calendar): days = [1., 2., 3.] da = DataArray(days, coords=[days], dims=['time'], name='test') ds = da.to_dataset() @@ -683,17 +606,13 @@ def test_decode_cf_enable_cftimeindex(calendar, enable_cftimeindex): ds[v].attrs['units'] = 'days since 2001-01-01' ds[v].attrs['calendar'] = calendar - if (not has_cftime and enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if not has_cftime_or_netCDF4 and calendar not in _STANDARD_CALENDARS: with pytest.raises(ValueError): - with set_options(enable_cftimeindex=enable_cftimeindex): - ds = decode_cf(ds) - else: - with set_options(enable_cftimeindex=enable_cftimeindex): ds = decode_cf(ds) + else: + ds = decode_cf(ds) - if (enable_cftimeindex and - calendar not in _STANDARD_CALENDARS): + if calendar not in _STANDARD_CALENDARS: assert ds.test.dtype == np.dtype('O') else: assert ds.test.dtype == np.dtype('M8[ns]') diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index e49b6cdf517..4bea7a55e99 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2279,8 +2279,7 @@ def test_resample_cftimeindex(self): cftime = _import_cftime() times = cftime.num2date(np.arange(12), units='hours since 0001-01-01', calendar='noleap') - with set_options(enable_cftimeindex=True): - array = DataArray(np.arange(12), [('time', times)]) + array = DataArray(np.arange(12), [('time', times)]) with raises_regex(TypeError, 'Only valid with DatetimeIndex, ' diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 4441375a1b1..4a8cac9ac62 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -33,6 +33,9 @@ def test_enable_cftimeindex(): xarray.set_options(enable_cftimeindex=None) with xarray.set_options(enable_cftimeindex=True): assert OPTIONS['enable_cftimeindex'] + with pytest.warns(FutureWarning): + with xarray.set_options(enable_cftimeindex=True): + pass def test_file_cache_maxsize(): diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index 33021fc5ef4..ed07af0d7bb 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -46,19 +46,17 @@ def test_safe_cast_to_index(): @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): +def test_safe_cast_to_index_cftimeindex(): date_types = _all_cftime_date_types() for date_type in date_types.values(): dates = [date_type(1, 1, day) for day in range(1, 20)] - if enable_cftimeindex and has_cftime: + if has_cftime: expected = CFTimeIndex(dates) else: expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert expected.dtype == actual.dtype assert isinstance(actual, type(expected)) @@ -66,13 +64,11 @@ def test_safe_cast_to_index_cftimeindex(enable_cftimeindex): # Test that datetime.datetime objects are never used in a CFTimeIndex @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') -@pytest.mark.parametrize('enable_cftimeindex', [False, True]) -def test_safe_cast_to_index_datetime_datetime(enable_cftimeindex): +def test_safe_cast_to_index_datetime_datetime(): dates = [datetime(1, 1, day) for day in range(1, 20)] expected = pd.Index(dates) - with set_options(enable_cftimeindex=enable_cftimeindex): - actual = utils.safe_cast_to_index(np.array(dates)) + actual = utils.safe_cast_to_index(np.array(dates)) assert_array_equal(expected, actual) assert isinstance(actual, pd.Index) From fa729ec664d74d4f2f473de469d1e69f45758d34 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sat, 27 Oct 2018 12:42:58 -0400 Subject: [PATCH 06/15] Add CFTimeIndex.to_datetimeindex method --- doc/api-hidden.rst | 1 + doc/time-series.rst | 14 ++++++++-- doc/whats-new.rst | 13 +++++---- xarray/coding/cftimeindex.py | 47 ++++++++++++++++++++++++++++++++ xarray/coding/times.py | 7 ++++- xarray/tests/test_cftimeindex.py | 36 +++++++++++++++++++++++- 6 files changed, 109 insertions(+), 9 deletions(-) diff --git a/doc/api-hidden.rst b/doc/api-hidden.rst index 0e8143c72ea..f668006f84b 100644 --- a/doc/api-hidden.rst +++ b/doc/api-hidden.rst @@ -153,3 +153,4 @@ plot.FacetGrid.map CFTimeIndex.shift + CFTimeIndex.to_datetimeindex diff --git a/doc/time-series.rst b/doc/time-series.rst index 0f32c0250e9..003c2daf95e 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -328,11 +328,21 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: (:issue:`2164`). For some use-cases it may still be useful to convert from - :py:class:`cftime.datetime` objects to ``np.datetime64[ns]`` objects, + a :py:class:`~xarray.CFTimeIndex` to a :py:class:`pandas.DatetimeIndex`, despite the difference in calendar types (e.g. to allow the use of some forms resample with non-standard calendars). The recommended way of doing - this is through... + this is to use the built-in :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` + method: + .. ipython:: python + + modern_times = xr.cftime_range('2000', periods=24, freq='MS', calendar='noleap') + da = xr.DataArray(range(24), [('time', modern_times)]) + da + datetimeindex = da.indexes['time'].to_datetimeindex() + da['time'] = datetimeindex + da.resample(time='Y').mean('time') + However in this case one should use caution to only perform operations which do not depend on differences between dates (e.g. differentiation, interpolation, or upsampling with diff --git a/doc/whats-new.rst b/doc/whats-new.rst index e85c43b7a89..d1e97a9c030 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,11 +36,14 @@ Breaking changes - For non-standard calendars commonly used in climate science, xarray will now always use :py:class:`cftime.datetime` objects, rather than by default try to coerce them to ``np.datetime64[ns]`` objects. A - :py:class:`~xarray.CFTimeIndex` will be used for indexing in these cases. - New public methods for converting :py:class:`cftime.datetime` dates to - ``np.datetime64[ns]`` objects have been added for use-cases where standard - datetimes are currently required. Setting the ``enable_cftimeindex`` option - is now a no-op and emits a FutureWarning. + :py:class:`~xarray.CFTimeIndex` will be used for indexing along time + coordinates in these cases. A new method, + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex`, has been added + to aid in converting from a :py:class:`~xarray.CFTimeIndex` to a + :py:class:`pandas.DatetimeIndex` for the remaining use-cases where + using a :py:class:`~xarray.CFTimeIndex` is still a limitation (e.g. for + resample or plotting). Setting the ``enable_cftimeindex`` option is now a + no-op and emits a ``FutureWarning``. - ``Dataset.T`` has been removed as a shortcut for :py:meth:`Dataset.transpose`. Call :py:meth:`Dataset.transpose` directly instead. - Iterating over a ``Dataset`` now includes only data variables, not coordinates. diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 5de055c1b9a..8d2abca194f 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -42,6 +42,7 @@ from __future__ import absolute_import import re +import warnings from datetime import timedelta import numpy as np @@ -50,6 +51,8 @@ from xarray.core import pycompat from xarray.core.utils import is_scalar +from .times import cftime_to_nptime, infer_calendar_name, _STANDARD_CALENDARS + def named(name, pattern): return '(?P<' + name + '>' + pattern + ')' @@ -381,6 +384,50 @@ def _add_delta(self, deltas): # pandas. No longer used as of pandas 0.23. return self + deltas + def to_datetimeindex(self): + """If possible, convert this index to a pandas.DatetimeIndex. + + Returns + ------- + pandas.DatetimeIndex + + Raises + ------ + ValueError + If the CFTimeIndex contains dates that are not possible in the + standard calendar or outside the pandas.Timestamp-valid range. + + Warns + ----- + RuntimeWarning + If converting from a non-standard calendar to a DatetimeIndex. + + Warnings + -------- + Note that for non-standard calendars, this will change the calendar + type of the index. In that case the result of this method should be + used with caution. + + Examples + -------- + >>> import xarray as xr + >>> times = xr.cftime_range('2000', periods=2, calendar='gregorian') + >>> times + CFTimeIndex([2000-01-01 00:00:00, 2000-01-02 00:00:00], dtype='object') + >>> times.to_datetimeindex() + DatetimeIndex(['2000-01-01', '2000-01-02'], dtype='datetime64[ns]', freq=None) + """ # noqa: E501 + nptimes = cftime_to_nptime(self) + calendar = infer_calendar_name(self) + if calendar not in _STANDARD_CALENDARS: + warnings.warn( + 'Converting a CFTimeIndex with dates from a non-standard ' + 'calendar, {!r}, to a pandas.DatetimeIndex, which uses dates ' + 'from the standard calendar. This may lead to subtle errors ' + 'in operations that depend on the length of time between ' + 'dates.'.format(calendar), RuntimeWarning) + return pd.DatetimeIndex(nptimes) + def _parse_iso8601_without_reso(date_type, datetime_str): date, _ = _parse_iso8601_with_reso(date_type, datetime_str) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index d64e95b4e78..e6f6dc5bdf1 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -279,7 +279,12 @@ def cftime_to_nptime(times): times = np.asarray(times) new = np.empty(times.shape, dtype='M8[ns]') for i, t in np.ndenumerate(times): - dt = datetime(t.year, t.month, t.day, t.hour, t.minute, t.second) + try: + dt = pd.Timestamp(t.year, t.month, t.day, t.hour, t.minute, + t.second) + except ValueError as e: + raise ValueError('Cannot convert date {} to a date in the ' + 'standard calendar. Reason: {}.'.format(t, e)) new[i] = np.datetime64(dt) return new diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index 61cf3968fe0..496fe22ecf8 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -13,7 +13,8 @@ from xarray.tests import assert_array_equal, assert_identical from . import has_cftime, has_cftime_or_netCDF4, requires_cftime -from .test_coding_times import _all_cftime_date_types +from .test_coding_times import (_all_cftime_date_types, _ALL_CALENDARS, + _NON_STANDARD_CALENDARS) def date_dict(year=None, month=None, day=None, @@ -744,3 +745,36 @@ def test_parse_array_of_cftime_strings(): expected = np.array(DatetimeNoLeap(2000, 1, 1)) result = _parse_array_of_cftime_strings(strings, DatetimeNoLeap) np.testing.assert_array_equal(result, expected) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_to_datetimeindex(calendar): + index = xr.cftime_range('2000', periods=5, calendar=calendar) + expected = pd.date_range('2000', periods=5) + + if calendar in _NON_STANDARD_CALENDARS: + with pytest.warns(RuntimeWarning, match='non-standard'): + result = index.to_datetimeindex() + else: + result = index.to_datetimeindex() + + assert result.equals(expected) + np.testing.assert_array_equal(result, expected) + assert isinstance(result, pd.DatetimeIndex) + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', _ALL_CALENDARS) +def test_to_datetimeindex_out_of_range(calendar): + index = xr.cftime_range('0001', periods=5, calendar=calendar) + with pytest.raises(ValueError, match='0001'): + index.to_datetimeindex() + + +@pytest.mark.skipif(not has_cftime, reason='cftime not installed') +@pytest.mark.parametrize('calendar', ['all_leap', '360_day']) +def test_to_datetimeindex_feb_29(calendar): + index = xr.cftime_range('2001-02-28', periods=2, calendar=calendar) + with pytest.raises(ValueError, match='29'): + index.to_datetimeindex() From 939b95395aeac3fce2d50953935047d3c0c4c3f8 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sat, 27 Oct 2018 14:03:47 -0400 Subject: [PATCH 07/15] Add friendlier error message for resample --- xarray/core/common.py | 33 ++++++++++++++++++++++++++++++++- xarray/tests/test_dataarray.py | 4 +--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 6c03775a5dd..52b82804aca 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -658,6 +658,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, from .dataarray import DataArray from .resample import RESAMPLE_DIM + from ..coding.cftimeindex import CFTimeIndex if dim is not None: if how is None: @@ -676,13 +677,27 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, "Resampling only supported along single dimensions." ) dim, freq = indexer.popitem() - + if isinstance(dim, basestring): dim_name = dim dim = self[dim] else: raise TypeError("Dimension name should be a string; " "was passed %r" % dim) + + if isinstance(self.indexes[dim_name], CFTimeIndex): + raise TypeError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base) resampler = self._resample_cls(self, group=group, dim=dim_name, @@ -696,6 +711,8 @@ def _resample_immediately(self, freq, dim, how, skipna, """Implement the original version of .resample() which immediately executes the desired resampling operation. """ from .dataarray import DataArray + from ..coding.cftimeindex import CFTimeIndex + RESAMPLE_DIM = '__resample_dim__' warnings.warn("\n.resample() has been modified to defer " @@ -705,8 +722,22 @@ def _resample_immediately(self, freq, dim, how, skipna, dim=dim, freq=freq, how=how), FutureWarning, stacklevel=3) + if isinstance(self.indexes[dim], CFTimeIndex): + raise TypeError( + 'Resample is currently not supported along a dimension ' + 'indexed by a CFTimeIndex. For certain kinds of downsampling ' + 'it may be possible to work around this by converting your ' + 'time index to a DatetimeIndex using ' + 'CFTimeIndex.to_datetimeindex. Use caution when doing this ' + 'however, because switching to a DatetimeIndex from a ' + 'CFTimeIndex with a non-standard calendar entails a change ' + 'in the calendar type, which could lead to subtle and silent ' + 'errors.' + ) + if isinstance(dim, basestring): dim = self[dim] + group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM) grouper = pd.Grouper(freq=freq, how=how, closed=closed, label=label, base=base) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 4bea7a55e99..8ed3b45f384 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2281,9 +2281,7 @@ def test_resample_cftimeindex(self): calendar='noleap') array = DataArray(np.arange(12), [('time', times)]) - with raises_regex(TypeError, - 'Only valid with DatetimeIndex, ' - 'TimedeltaIndex or PeriodIndex'): + with raises_regex(TypeError, 'to_datetimeindex'): array.resample(time='6H').mean() def test_resample_first(self): From 1a6841df309a680b0315c1d6b1923bf8a683d8ee Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sat, 27 Oct 2018 14:13:13 -0400 Subject: [PATCH 08/15] lint --- xarray/core/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/common.py b/xarray/core/common.py index 52b82804aca..c0adacf88e4 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -677,7 +677,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, "Resampling only supported along single dimensions." ) dim, freq = indexer.popitem() - + if isinstance(dim, basestring): dim_name = dim dim = self[dim] @@ -734,7 +734,7 @@ def _resample_immediately(self, freq, dim, how, skipna, 'in the calendar type, which could lead to subtle and silent ' 'errors.' ) - + if isinstance(dim, basestring): dim = self[dim] From f65b764561c7cd95c3c3f613ea27a0ff6eb940da Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 28 Oct 2018 09:23:34 -0400 Subject: [PATCH 09/15] Address review comments --- xarray/coding/cftimeindex.py | 10 ++++++++-- xarray/coding/times.py | 4 ++++ xarray/core/common.py | 4 ++-- xarray/core/options.py | 16 ++++++++-------- xarray/tests/test_cftimeindex.py | 7 ++++--- xarray/tests/test_dataarray.py | 2 +- xarray/tests/test_options.py | 6 ++---- 7 files changed, 29 insertions(+), 20 deletions(-) diff --git a/xarray/coding/cftimeindex.py b/xarray/coding/cftimeindex.py index 8d2abca194f..2ce996b2bd2 100644 --- a/xarray/coding/cftimeindex.py +++ b/xarray/coding/cftimeindex.py @@ -384,9 +384,15 @@ def _add_delta(self, deltas): # pandas. No longer used as of pandas 0.23. return self + deltas - def to_datetimeindex(self): + def to_datetimeindex(self, unsafe=False): """If possible, convert this index to a pandas.DatetimeIndex. + Parameters + ---------- + unsafe : bool + Flag to turn off warning when converting from a CFTimeIndex with + a non-standard calendar to a DatetimeIndex (default ``False``). + Returns ------- pandas.DatetimeIndex @@ -419,7 +425,7 @@ def to_datetimeindex(self): """ # noqa: E501 nptimes = cftime_to_nptime(self) calendar = infer_calendar_name(self) - if calendar not in _STANDARD_CALENDARS: + if calendar not in _STANDARD_CALENDARS and not unsafe: warnings.warn( 'Converting a CFTimeIndex with dates from a non-standard ' 'calendar, {!r}, to a pandas.DatetimeIndex, which uses dates ' diff --git a/xarray/coding/times.py b/xarray/coding/times.py index e6f6dc5bdf1..d4cd88431a0 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -280,6 +280,10 @@ def cftime_to_nptime(times): new = np.empty(times.shape, dtype='M8[ns]') for i, t in np.ndenumerate(times): try: + # Use pandas.Timestamp in place of datetime.datetime, because + # NumPy casts it safely it np.datetime64[ns] for dates outside + # 1678 to 2262 (this is not currently the case for + # datetime.datetime). dt = pd.Timestamp(t.year, t.month, t.day, t.hour, t.minute, t.second) except ValueError as e: diff --git a/xarray/core/common.py b/xarray/core/common.py index c0adacf88e4..f0fc85deb34 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -686,7 +686,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None, "was passed %r" % dim) if isinstance(self.indexes[dim_name], CFTimeIndex): - raise TypeError( + raise NotImplementedError( 'Resample is currently not supported along a dimension ' 'indexed by a CFTimeIndex. For certain kinds of downsampling ' 'it may be possible to work around this by converting your ' @@ -723,7 +723,7 @@ def _resample_immediately(self, freq, dim, how, skipna, FutureWarning, stacklevel=3) if isinstance(self.indexes[dim], CFTimeIndex): - raise TypeError( + raise NotImplementedError( 'Resample is currently not supported along a dimension ' 'indexed by a CFTimeIndex. For certain kinds of downsampling ' 'it may be possible to work around this by converting your ' diff --git a/xarray/core/options.py b/xarray/core/options.py index f76f06032fc..6998c6c1659 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -38,8 +38,16 @@ def _set_file_cache_maxsize(value): FILE_CACHE.maxsize = value +def _warn_on_setting_enable_cftimeindex(enable_cftimeindex): + warnings.warn( + 'The enable_cftimeindex option is now a no-op ' + 'and will be removed in a future version of xarray.', + FutureWarning) + + _SETTERS = { FILE_CACHE_MAXSIZE: _set_file_cache_maxsize, + ENABLE_CFTIMEINDEX: _warn_on_setting_enable_cftimeindex } @@ -52,9 +60,6 @@ class set_options(object): Default: ``80``. - ``arithmetic_join``: DataArray/Dataset alignment in binary operations. Default: ``'inner'``. - - ``enable_cftimeindex``: flag to enable using a ``CFTimeIndex`` - for time indexes with non-standard calendars or dates outside the - Timestamp-valid range. Default: ``True``. - ``file_cache_maxsize``: maximum number of open files to hold in xarray's global least-recently-usage cached. This should be smaller than your system's per-process file descriptor limit, e.g., ``ulimit -n`` on Linux. @@ -85,11 +90,6 @@ class set_options(object): def __init__(self, **kwargs): self.old = OPTIONS.copy() - if ENABLE_CFTIMEINDEX in kwargs: - warnings.warn( - 'The enable_cftimeindex option is now a no-op ' - 'and will be removed in a future version of xarray.', - FutureWarning) for k, v in kwargs.items(): if k not in OPTIONS: raise ValueError( diff --git a/xarray/tests/test_cftimeindex.py b/xarray/tests/test_cftimeindex.py index 496fe22ecf8..5e710827ff8 100644 --- a/xarray/tests/test_cftimeindex.py +++ b/xarray/tests/test_cftimeindex.py @@ -361,7 +361,7 @@ def test_groupby(da): @pytest.mark.skipif(not has_cftime, reason='cftime not installed') def test_resample_error(da): - with pytest.raises(TypeError): + with pytest.raises(NotImplementedError, match='to_datetimeindex'): da.resample(time='Y') @@ -749,11 +749,12 @@ def test_parse_array_of_cftime_strings(): @pytest.mark.skipif(not has_cftime, reason='cftime not installed') @pytest.mark.parametrize('calendar', _ALL_CALENDARS) -def test_to_datetimeindex(calendar): +@pytest.mark.parametrize('unsafe', [False, True]) +def test_to_datetimeindex(calendar, unsafe): index = xr.cftime_range('2000', periods=5, calendar=calendar) expected = pd.date_range('2000', periods=5) - if calendar in _NON_STANDARD_CALENDARS: + if calendar in _NON_STANDARD_CALENDARS and not unsafe: with pytest.warns(RuntimeWarning, match='non-standard'): result = index.to_datetimeindex() else: diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 8ed3b45f384..32057309f84 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2281,7 +2281,7 @@ def test_resample_cftimeindex(self): calendar='noleap') array = DataArray(np.arange(12), [('time', times)]) - with raises_regex(TypeError, 'to_datetimeindex'): + with raises_regex(NotImplementedError, 'to_datetimeindex'): array.resample(time='6H').mean() def test_resample_first(self): diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 4a8cac9ac62..6f967a85fa1 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -31,11 +31,9 @@ def test_arithmetic_join(): def test_enable_cftimeindex(): with pytest.raises(ValueError): xarray.set_options(enable_cftimeindex=None) - with xarray.set_options(enable_cftimeindex=True): - assert OPTIONS['enable_cftimeindex'] - with pytest.warns(FutureWarning): + with pytest.warns(FutureWarning, match='no-op'): with xarray.set_options(enable_cftimeindex=True): - pass + assert OPTIONS['enable_cftimeindex'] def test_file_cache_maxsize(): From eaa4a44cace084a43d59bf1e3aa3c7b57d39bbe2 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 28 Oct 2018 09:29:54 -0400 Subject: [PATCH 10/15] Take into account microsecond attribute in cftime_to_nptime --- xarray/coding/times.py | 2 +- xarray/tests/test_coding_times.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/xarray/coding/times.py b/xarray/coding/times.py index d4cd88431a0..aa825b88fe3 100644 --- a/xarray/coding/times.py +++ b/xarray/coding/times.py @@ -285,7 +285,7 @@ def cftime_to_nptime(times): # 1678 to 2262 (this is not currently the case for # datetime.datetime). dt = pd.Timestamp(t.year, t.month, t.day, t.hour, t.minute, - t.second) + t.second, t.microsecond) except ValueError as e: raise ValueError('Cannot convert date {} to a date in the ' 'standard calendar. Reason: {}.'.format(t, e)) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 3b515aa6871..cf7d63a30f3 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -89,7 +89,11 @@ def test_cf_datetime(num_dates, units, calendar): actual = coding.times.decode_cf_datetime(num_dates, units, calendar) - assert_array_equal(expected, actual) + abs_diff = np.atleast_1d(abs(actual - expected)).astype(np.timedelta64) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() encoded, _, _ = coding.times.encode_cf_datetime(actual, units, calendar) if '1-1-1' not in units: @@ -151,7 +155,11 @@ def test_decode_cf_datetime_non_iso_strings(): (np.arange(100), 'hours since 2000-01-01 0:00')] for num_dates, units in cases: actual = coding.times.decode_cf_datetime(num_dates, units) - assert_array_equal(actual, expected) + abs_diff = abs(actual - expected) + # once we no longer support versions of netCDF4 older than 1.1.5, + # we could do this check with near microsecond accuracy: + # https://github.com/Unidata/netcdf4-python/issues/355 + assert (abs_diff <= np.timedelta64(1, 's')).all() @pytest.mark.skipif(not has_cftime_or_netCDF4, reason='cftime not installed') From 42406e9dd253cf85fc2a7e4980188783be5e36dc Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 28 Oct 2018 09:50:03 -0400 Subject: [PATCH 11/15] Add test for decoding dates with microsecond-resolution units This would have failed before including the microsecond attribute of each date in cftime_to_nptime in eaa4a44. --- xarray/tests/test_coding_times.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index cf1e88f2f08..5110256e9f5 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -47,7 +47,8 @@ ([0.5, 1.5], 'hours since 1900-01-01T00:00:00'), (0, 'milliseconds since 2000-01-01T00:00:00'), (0, 'microseconds since 2000-01-01T00:00:00'), - (np.int32(788961600), 'seconds since 1981-01-01') # GH2002 + (np.int32(788961600), 'seconds since 1981-01-01'), # GH2002 + (12300 + np.arange(5), 'hour since 1680-01-01 00:00:00.500000') ] _CF_DATETIME_TESTS = [num_dates_units + (calendar,) for num_dates_units, calendar in product(_CF_DATETIME_NUM_DATES_UNITS, From 359016d69ac5556f7548a69a3aa8a76f8bebe803 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 28 Oct 2018 09:58:49 -0400 Subject: [PATCH 12/15] Fix typo in time-series.rst --- doc/time-series.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index 003c2daf95e..d3abdd9266b 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -330,9 +330,9 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: For some use-cases it may still be useful to convert from a :py:class:`~xarray.CFTimeIndex` to a :py:class:`pandas.DatetimeIndex`, despite the difference in calendar types (e.g. to allow the use of some - forms resample with non-standard calendars). The recommended way of doing - this is to use the built-in :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` - method: + forms of resample with non-standard calendars). The recommended way of + doing this is to use the built-in + :py:meth:`~xarray.CFTimeIndex.to_datetimeindex` method: .. ipython:: python From a72d41e599d45fc9c6910f4b55ea596001fc6ca2 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Sun, 28 Oct 2018 09:59:19 -0400 Subject: [PATCH 13/15] Formatting --- doc/time-series.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/time-series.rst b/doc/time-series.rst index d3abdd9266b..7befd954f35 100644 --- a/doc/time-series.rst +++ b/doc/time-series.rst @@ -343,12 +343,11 @@ For data indexed by a :py:class:`~xarray.CFTimeIndex` xarray currently supports: da['time'] = datetimeindex da.resample(time='Y').mean('time') - However in this case one should - use caution to only perform operations which do not depend on differences - between dates (e.g. differentiation, interpolation, or upsampling with - resample), as these could introduce subtle and silent errors due to the - difference in calendar types between the dates encoded in your data and the - dates stored in memory. + However in this case one should use caution to only perform operations which + do not depend on differences between dates (e.g. differentiation, + interpolation, or upsampling with resample), as these could introduce subtle + and silent errors due to the difference in calendar types between the dates + encoded in your data and the dates stored in memory. .. _Timestamp-valid range: https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timestamp-limitations .. _ISO 8601-format: https://en.wikipedia.org/wiki/ISO_8601 From 079cfdff5907e825502d119ec8a64518ea628858 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Wed, 31 Oct 2018 06:58:41 -0400 Subject: [PATCH 14/15] Fix test_decode_cf_datetime_non_iso_strings --- xarray/tests/test_coding_times.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_coding_times.py b/xarray/tests/test_coding_times.py index 694494314d5..0ca57f98a6d 100644 --- a/xarray/tests/test_coding_times.py +++ b/xarray/tests/test_coding_times.py @@ -157,7 +157,7 @@ def test_decode_cf_datetime_non_iso_strings(): (np.arange(100), 'hours since 2000-01-01 0:00')] for num_dates, units in cases: actual = coding.times.decode_cf_datetime(num_dates, units) - abs_diff = abs(actual - expected) + abs_diff = abs(actual - expected.values) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 From 6d08d3b5fa5337bc1080b518544bfc1306b232ac Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Wed, 31 Oct 2018 16:03:19 -0400 Subject: [PATCH 15/15] Prevent warning emitted from set_options.__exit__ --- xarray/core/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/options.py b/xarray/core/options.py index 03b4f69d91c..ab461ca86bc 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -109,7 +109,7 @@ class set_options(object): """ def __init__(self, **kwargs): - self.old = OPTIONS.copy() + self.old = {} for k, v in kwargs.items(): if k not in OPTIONS: raise ValueError( @@ -118,6 +118,7 @@ def __init__(self, **kwargs): if k in _VALIDATORS and not _VALIDATORS[k](v): raise ValueError( 'option %r given an invalid value: %r' % (k, v)) + self.old[k] = OPTIONS[k] self._apply_update(kwargs) def _apply_update(self, options_dict): @@ -130,5 +131,4 @@ def __enter__(self): return def __exit__(self, type, value, traceback): - OPTIONS.clear() self._apply_update(self.old)