From fb81e3126c2fb512697320c294de8f88232bbc04 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 11 Jan 2023 10:48:49 +0000 Subject: [PATCH 1/4] stub out pandas-roundtrip --- pyleoclim/core/series.py | 44 ++++++++++++++++++++++++++++++++++++++ pyleoclim/utils/tsutils.py | 16 ++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/pyleoclim/core/series.py b/pyleoclim/core/series.py index 21976d27..d0f67664 100644 --- a/pyleoclim/core/series.py +++ b/pyleoclim/core/series.py @@ -7,6 +7,8 @@ How to create and manipulate such objects is described in a short example below, while `this notebook `_ demonstrates how to apply various Pyleoclim methods to Series objects. """ +import operator + from ..utils import tsutils, plotting, tsmodel, tsbase, mapping, lipdutils from ..utils import wavelet as waveutils from ..utils import spectral as specutils @@ -163,6 +165,48 @@ def __init__(self, time, value, time_name=None, time_unit=None, value_name=None, self.mean=np.mean(self.value) else: self.mean = mean + + @property + def datetime_index(self): + datum, exponent, direction = tsutils.time_unit_to_datum_exp_dir(self.time_unit) + op = operator.add if direction == 'forward' else operator.sub + + timedelta = self.time * 10**exponent * tsutils.SECONDS_PER_YEAR + years = timedelta.astype('int').astype('timedelta64[Y]') + seconds = ((timedelta % 1) * tsutils.SECONDS_PER_YEAR).astype('timedelta64[s]') + + np_times = op(np.datetime64(datum, 's'), years + seconds) + return pd.DatetimeIndex(np_times, name=self.time_name) + + @property + def metadata(self): + return dict( + time_unit = self.time_unit, + value_unit = self.value_unit, + label = self.label, + ) + + @classmethod + def from_pandas(cls, ser, metadata): + time = tsutils.convert_datetime_index_to_time(ser.index, metadata['time_unit']) + return cls( + time=time, + value=ser.to_numpy(), + time_name=ser.index.name, + value_name=ser.name, + **metadata, + ) + + def to_pandas(self): + ser = pd.Series(self.value, index=self.datetime_index, name=self.value_name) + # Could be a dataclass instead? + return (ser, self.metadata) + + def pandas_method(self, method): + ser, metadata = self.to_pandas() + result = method(ser) + return self.from_pandas(result, **metadata) + def convert_time_unit(self, time_unit='years', keep_log=False): ''' Convert the time unit of the Series object diff --git a/pyleoclim/utils/tsutils.py b/pyleoclim/utils/tsutils.py index 9826f869..225d6bce 100644 --- a/pyleoclim/utils/tsutils.py +++ b/pyleoclim/utils/tsutils.py @@ -46,6 +46,22 @@ clean_ts ) +SECONDS_PER_YEAR = 365.25 * 60 * 60 * 24 + +def time_unit_to_datum_exp_dir(time_unit): + datum = ... + exponent = ... + direction = ... + return (datum, exponent, direction) + +def datum_exp_dir_to_time_unit(datum, exponent, direction): + time_unit = ... + return time_unit + +def convert_datetime_index_to_time(datetime_index, time_unit): + time = ... + return time + def simple_stats(y, axis=None): """ Computes simple statistics From 7f7f00577162391bf789f55264176e94926706cc Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 11 Jan 2023 15:17:23 +0000 Subject: [PATCH 2/4] populate tsutil functions --- pyleoclim/core/series.py | 16 +++++++----- pyleoclim/utils/tsutils.py | 53 ++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/pyleoclim/core/series.py b/pyleoclim/core/series.py index d0f67664..5d1960bf 100644 --- a/pyleoclim/core/series.py +++ b/pyleoclim/core/series.py @@ -169,13 +169,17 @@ def __init__(self, time, value, time_name=None, time_unit=None, value_name=None, @property def datetime_index(self): datum, exponent, direction = tsutils.time_unit_to_datum_exp_dir(self.time_unit) - op = operator.add if direction == 'forward' else operator.sub - - timedelta = self.time * 10**exponent * tsutils.SECONDS_PER_YEAR - years = timedelta.astype('int').astype('timedelta64[Y]') + if direction == 'prograde': + op = operator.add + elif direction == 'retrograde': + op = operator.sub + else: + raise ValueError(f'Expected one of {"prograde", "retrograde"}, got {direction}') + timedelta = self.time * 10**exponent + years = timedelta.astype('int') seconds = ((timedelta % 1) * tsutils.SECONDS_PER_YEAR).astype('timedelta64[s]') - np_times = op(np.datetime64(datum, 's'), years + seconds) + np_times = op(op(int(datum), years).astype(str).astype('datetime64[s]'), seconds) return pd.DatetimeIndex(np_times, name=self.time_name) @property @@ -205,7 +209,7 @@ def to_pandas(self): def pandas_method(self, method): ser, metadata = self.to_pandas() result = method(ser) - return self.from_pandas(result, **metadata) + return self.from_pandas(result, metadata) def convert_time_unit(self, time_unit='years', keep_log=False): diff --git a/pyleoclim/utils/tsutils.py b/pyleoclim/utils/tsutils.py index 225d6bce..d8bcc05a 100644 --- a/pyleoclim/utils/tsutils.py +++ b/pyleoclim/utils/tsutils.py @@ -49,19 +49,56 @@ SECONDS_PER_YEAR = 365.25 * 60 * 60 * 24 def time_unit_to_datum_exp_dir(time_unit): - datum = ... - exponent = ... - direction = ... - return (datum, exponent, direction) + if time_unit is None: + datum = '0' + exponent = 0 + direction = 'prograde' + elif any(val in time_unit for val in ('ky', 'kyr', 'kyrs', 'kiloyear', 'ka')): + datum = '1950' + exponent = 3 + direction = 'retrograde' + elif any(val in time_unit for val in ('y', 'yr', 'yrs', 'year', 'year CE', 'years CE', 'year(s) AD')): + datum = '0' + exponent = 0 + direction = 'prograde' + elif time_unit == 'BP': + datum = '1950' + exponent = 0 + direction = 'retrograde' + elif any(val in time_unit for val in ('yr BP', 'yrs BP', 'years BP')): + datum ='1950' + exponent = 0 + direction = 'retrograde' + elif any(val in time_unit for val in ('Ma', 'My')): + datum ='1950' + exponent = 6 + direction = 'retrograde' + elif any(val in time_unit for val in ('Ga', 'Gy')): + datum ='1950' + exponent = 3 + direction = 'retrograde' + else: + raise ValueError(f'Time unit {time_unit} not supported') -def datum_exp_dir_to_time_unit(datum, exponent, direction): - time_unit = ... - return time_unit + return (datum, exponent, direction) def convert_datetime_index_to_time(datetime_index, time_unit): - time = ... + datum, exponent, direction = time_unit_to_datum_exp_dir(time_unit) + import operator + if direction == 'prograde': + multiplier = 1 + elif direction == 'retrograde': + multiplier = -1 + else: + raise ValueError('invalid direction') + year_diff = (datetime_index.year - int(datum)) + seconds_diff = (datetime_index.to_numpy() - datetime_index.year.astype(str).astype('datetime64[s]').to_numpy()).astype('int') + diff = year_diff + seconds_diff / SECONDS_PER_YEAR + time = multiplier * diff / 10**exponent + return time + def simple_stats(y, axis=None): """ Computes simple statistics From 4845888c7b58e55fef637250d6bc0663ebfb9414 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 11 Jan 2023 16:10:03 +0000 Subject: [PATCH 3/4] fixup --- pyleoclim/core/series.py | 3 ++- pyleoclim/utils/tsutils.py | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pyleoclim/core/series.py b/pyleoclim/core/series.py index 5d1960bf..51c3199e 100644 --- a/pyleoclim/core/series.py +++ b/pyleoclim/core/series.py @@ -175,9 +175,10 @@ def datetime_index(self): op = operator.sub else: raise ValueError(f'Expected one of {"prograde", "retrograde"}, got {direction}') + timedelta = self.time * 10**exponent years = timedelta.astype('int') - seconds = ((timedelta % 1) * tsutils.SECONDS_PER_YEAR).astype('timedelta64[s]') + seconds = ((timedelta - timedelta.astype('int')) * tsutils.SECONDS_PER_YEAR).astype('timedelta64[s]') np_times = op(op(int(datum), years).astype(str).astype('datetime64[s]'), seconds) return pd.DatetimeIndex(np_times, name=self.time_name) diff --git a/pyleoclim/utils/tsutils.py b/pyleoclim/utils/tsutils.py index d8bcc05a..7be17214 100644 --- a/pyleoclim/utils/tsutils.py +++ b/pyleoclim/utils/tsutils.py @@ -49,15 +49,15 @@ SECONDS_PER_YEAR = 365.25 * 60 * 60 * 24 def time_unit_to_datum_exp_dir(time_unit): - if time_unit is None: + if time_unit == 'years': datum = '0' exponent = 0 direction = 'prograde' - elif any(val in time_unit for val in ('ky', 'kyr', 'kyrs', 'kiloyear', 'ka')): + elif time_unit in ('ky', 'kyr', 'kyrs', 'kiloyear', 'ka'): datum = '1950' exponent = 3 direction = 'retrograde' - elif any(val in time_unit for val in ('y', 'yr', 'yrs', 'year', 'year CE', 'years CE', 'year(s) AD')): + elif time_unit in ('y', 'yr', 'yrs', 'year', 'year CE', 'years CE', 'year(s) AD'): datum = '0' exponent = 0 direction = 'prograde' @@ -65,15 +65,15 @@ def time_unit_to_datum_exp_dir(time_unit): datum = '1950' exponent = 0 direction = 'retrograde' - elif any(val in time_unit for val in ('yr BP', 'yrs BP', 'years BP')): + elif time_unit in ('yr BP', 'yrs BP', 'years BP'): datum ='1950' exponent = 0 direction = 'retrograde' - elif any(val in time_unit for val in ('Ma', 'My')): + elif time_unit in ('Ma', 'My'): datum ='1950' exponent = 6 direction = 'retrograde' - elif any(val in time_unit for val in ('Ga', 'Gy')): + elif time_unit in ('Ga', 'Gy'): datum ='1950' exponent = 3 direction = 'retrograde' @@ -90,7 +90,7 @@ def convert_datetime_index_to_time(datetime_index, time_unit): elif direction == 'retrograde': multiplier = -1 else: - raise ValueError('invalid direction') + raise ValueError(f'Expected one of {"prograde", "retrograde"}, got {direction}') year_diff = (datetime_index.year - int(datum)) seconds_diff = (datetime_index.to_numpy() - datetime_index.year.astype(str).astype('datetime64[s]').to_numpy()).astype('int') diff = year_diff + seconds_diff / SECONDS_PER_YEAR From debbd5ed8e9a24a77b7edb505c858bfd9d8a6875 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Wed, 11 Jan 2023 16:18:38 +0000 Subject: [PATCH 4/4] validate pandas method result --- pyleoclim/core/series.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyleoclim/core/series.py b/pyleoclim/core/series.py index 51c3199e..4b8e6e84 100644 --- a/pyleoclim/core/series.py +++ b/pyleoclim/core/series.py @@ -210,6 +210,8 @@ def to_pandas(self): def pandas_method(self, method): ser, metadata = self.to_pandas() result = method(ser) + if not isinstance(result, pd.Series): + raise ValueError('Given method does not return a pandas Series and cannot be applied') return self.from_pandas(result, metadata)