From f213bfe180c594ae2ac6254a80d40763d3f14518 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 15 Feb 2024 19:59:49 -0500 Subject: [PATCH 01/47] Adding signal_processing file --- pynapple/process/signal_processing.py | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 pynapple/process/signal_processing.py diff --git a/pynapple/process/signal_processing.py b/pynapple/process/signal_processing.py new file mode 100644 index 00000000..038f0394 --- /dev/null +++ b/pynapple/process/signal_processing.py @@ -0,0 +1,67 @@ +""" + Signal processing module +""" + +import numpy as np +from .. import core as nap +from scipy.signal import butter, lfilter, filtfilt + + +def _butter_bandpass(lowcut, highcut, fs, order=5): + nyq = 0.5 * fs + low = lowcut / nyq + high = highcut / nyq + b, a = butter(order, [low, high], btype='band') + return b, a + +def _butter_bandpass_filter(data, lowcut, highcut, fs, order=4): + b, a = _butter_bandpass(lowcut, highcut, fs, order=order) + y = lfilter(b, a, data) + return y + +def compute_bandpass_filter(data, freq_band, sampling_frequency=None, order=4): + """ + Bandpass filtering the LFP. + + Parameters + ---------- + data : Tsd/TsdFrame + Description + lowcut : TYPE + Description + highcut : TYPE + Description + fs : TYPE + Description + order : int, optional + Description + + Raises + ------ + RuntimeError + Description + """ + time_support = data.time_support + time_index = data.as_units('s').index.values + if type(data) is nap.TsdFrame: + tmp = np.zeros(data.shape) + for i in np.arange(data.shape[1]): + tmp[:,i] = bandpass_filter(data[:,i], lowcut, highcut, fs, order) + + return nap.TsdFrame( + t = time_index, + d = tmp, + time_support = time_support, + time_units = 's', + columns = data.columns) + + elif type(data) is nap.Tsd: + flfp = _butter_bandpass_filter(data.values, lowcut, highcut, fs, order) + return nap.Tsd( + t=time_index, + d=flfp, + time_support=time_support, + time_units='s') + + else: + raise RuntimeError("Unknow format. Should be Tsd/TsdFrame") \ No newline at end of file From 5afc35228990a0b2ba121f3a91e3a0c2bdefdf9f Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 9 Apr 2024 13:20:08 +0200 Subject: [PATCH 02/47] Make tsd, tsd-tensot, and tsd-frame lazy --- pynapple/io/interface_nwb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index 1cfc76f4..aba73946 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -142,9 +142,9 @@ def _make_tsd(obj): """ - d = obj.data[:] + d = obj.data if obj.timestamps is not None: - t = obj.timestamps[:] + t = obj.timestamps else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate @@ -167,9 +167,9 @@ def _make_tsd_tensor(obj): """ - d = obj.data[:] + d = obj.data if obj.timestamps is not None: - t = obj.timestamps[:] + t = obj.timestamps else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate @@ -192,9 +192,9 @@ def _make_tsd_frame(obj): """ - d = obj.data[:] + d = obj.data if obj.timestamps is not None: - t = obj.timestamps[:] + t = obj.timestamps else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate From ec0ffe89381ef8df39bd6908237d4ed0fe7939db Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 9 Apr 2024 13:35:54 +0200 Subject: [PATCH 03/47] Add lazy flag and set it to True for NWB interface --- pynapple/core/time_series.py | 19 +++++++++++-------- pynapple/io/interface_nwb.py | 6 +++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index ffaa33d5..78fd7f83 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -75,7 +75,7 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): Implement most of the shared functions across concrete classes `Tsd`, `TsdFrame`, `TsdTensor` """ - def __init__(self, t, d, time_units="s", time_support=None): + def __init__(self, t, d, time_units="s", time_support=None, lazy=False): super().__init__(t, time_units, time_support) # Converting d to numpy array @@ -86,7 +86,10 @@ def __init__(self, t, d, time_units="s", time_support=None): elif isinstance(d, np.ndarray): self.values = d elif is_array_like(d): - self.values = convert_to_numpy(d, "d") + if lazy: + self.values = d + else: + self.values = convert_to_numpy(d, "d") else: raise RuntimeError( "Unknown format for d. Accepted formats are numpy.ndarray, list, tuple or any array-like objects." @@ -592,7 +595,7 @@ class TsdTensor(BaseTsd): The time support of the time series """ - def __init__(self, t, d, time_units="s", time_support=None, **kwargs): + def __init__(self, t, d, time_units="s", time_support=None, lazy=False, **kwargs): """ TsdTensor initializer @@ -607,7 +610,7 @@ def __init__(self, t, d, time_units="s", time_support=None, **kwargs): time_support : IntervalSet, optional The time support of the TsdFrame object """ - super().__init__(t, d, time_units, time_support) + super().__init__(t, d, time_units, time_support, lazy) assert ( self.values.ndim >= 3 @@ -764,7 +767,7 @@ class TsdFrame(BaseTsd): The time support of the time series """ - def __init__(self, t, d=None, time_units="s", time_support=None, columns=None): + def __init__(self, t, d=None, time_units="s", time_support=None, columns=None, lazy=False): """ TsdFrame initializer A pandas.DataFrame can be passed directly @@ -792,7 +795,7 @@ def __init__(self, t, d=None, time_units="s", time_support=None, columns=None): else: assert d is not None, "Missing argument d when initializing TsdFrame" - super().__init__(t, d, time_units, time_support) + super().__init__(t, d, time_units, time_support, lazy) assert self.values.ndim <= 2, "Data should be 1 or 2 dimensional." @@ -1021,7 +1024,7 @@ class Tsd(BaseTsd): The time support of the time series """ - def __init__(self, t, d=None, time_units="s", time_support=None, **kwargs): + def __init__(self, t, d=None, time_units="s", time_support=None, lazy=False, **kwargs): """ Tsd Initializer. @@ -1042,7 +1045,7 @@ def __init__(self, t, d=None, time_units="s", time_support=None, **kwargs): else: assert d is not None, "Missing argument d when initializing Tsd" - super().__init__(t, d, time_units, time_support) + super().__init__(t, d, time_units, time_support, lazy) assert self.values.ndim == 1, "Data should be 1 dimensional" diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index aba73946..d8140f2d 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -148,7 +148,7 @@ def _make_tsd(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.Tsd(t=t, d=d) + data = nap.Tsd(t=t, d=d, lazy=True) return data @@ -173,7 +173,7 @@ def _make_tsd_tensor(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.TsdTensor(t=t, d=d) + data = nap.TsdTensor(t=t, d=d, lazy=True) return data @@ -232,7 +232,7 @@ def _make_tsd_frame(obj): else: columns = np.arange(obj.data.shape[1]) - data = nap.TsdFrame(t=t, d=d, columns=columns) + data = nap.TsdFrame(t=t, d=d, columns=columns, lazy=True) return data From 58bfbd30e9f228079542b4098c3cb13037435f3b Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Wed, 10 Apr 2024 09:07:29 +0200 Subject: [PATCH 04/47] black --- pynapple/core/time_series.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 78fd7f83..968ea6b0 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -767,7 +767,9 @@ class TsdFrame(BaseTsd): The time support of the time series """ - def __init__(self, t, d=None, time_units="s", time_support=None, columns=None, lazy=False): + def __init__( + self, t, d=None, time_units="s", time_support=None, columns=None, lazy=False + ): """ TsdFrame initializer A pandas.DataFrame can be passed directly @@ -1024,7 +1026,9 @@ class Tsd(BaseTsd): The time support of the time series """ - def __init__(self, t, d=None, time_units="s", time_support=None, lazy=False, **kwargs): + def __init__( + self, t, d=None, time_units="s", time_support=None, lazy=False, **kwargs + ): """ Tsd Initializer. From 3da0d4683924464d3ceab140ed4b3f0c7b7b4915 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Tue, 7 May 2024 01:38:03 +0200 Subject: [PATCH 05/47] add TsGroup.merge_group method --- pynapple/core/ts_group.py | 153 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 971fd560..43df2a72 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1030,6 +1030,159 @@ def getby_category(self, key): sliced = {k: self[list(groups[k])] for k in groups.keys()} return sliced + @staticmethod + def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_metadata=False): + """ + Merge multiple TsGroup objects into a single TsGroup object + + Parameters + ---------- + *tsgroups : TsGroup + The TsGroup objects to merge + reset_index : bool, optional + If True, the keys will be reset to range(len(data)) + If False, the keys of the TsGroup objects should be non-overlapping and will be preserved + reset_time_support : bool, optional + If True, the merged TsGroup will merge time supports from all the Ts/Tsd objects in data + If False, the time support of the TsGroup objects should be the same and will be preserved + ignore_metadata : bool, optional + If True, the merged TsGroup will not have any metadata columns other than 'rate' + If False, all metadata columns should be the same and all metadata will be concatenated + + Returns + ------- + TsGroup + TsGroup of merged objects + + Raises + ------ + TypeError + If the input objects are not TsGroup objects + ValueError + If ignore_metadata=False and metadata columns are not the same + If reset_index=False and keys overlap + If reset_time_support=False and time supports are not the same + + Examples + -------- + + >>> import pynapple as nap + >>> time_support_a = nap.IntervalSet(start=-1, end=1, time_units='s') + >>> time_support_b = nap.IntervalSet(start=-5, end=5, time_units='s') + + >>> dict1 = {0: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup1 = nap.TsGroup(dict1, time_support=time_support_a) + + >>> dict2 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup2 = nap.TsGroup(dict2, time_support=time_support_a) + + >>> dict3 = {0: nap.Ts(t=[-.1, 0, .1], time_units='s')} + >>> tsgroup3 = nap.TsGroup(dict3, time_support=time_support_a) + + >>> dict4 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup4 = nap.TsGroup(dict2, time_support=time_support_b) + + Merge with default options if have same time_support and non-overlapping indexes: + + >>> tsgroup_12 = nap.TsGroup.merge_group(tsgroup1, tsgroup2) + >>> tsgroup_12 + Index rate + ------- ------ + 0 1.5 + 10 1.5 + + Pass reset_index=True if indexes are overlapping: + + >>> tsgroup_13 = nap.TsGroup.merge_group(tsgroup1, tsgroup3, reset_index=True) + >>> tsgroup_13 + + Index rate + ------- ------ + 0 1.5 + 1 1.5 + + Pass reset_time_support=True if time_supports are different: + + >>> tsgroup_14 = nap.TsGroup.merge_group(tsgroup1, tsgroup4, reset_time_support=True) + >>> tsgroup_14 + >>> tsgroup_14.time_support + + Index rate + ------- ------ + 0 0.3 + 10 0.3 + start end + 0 -5 5 + shape: (1, 2), time unit: sec. + + """ + is_tsgroup = [isinstance(tsg, TsGroup) for tsg in tsgroups] + if not all(is_tsgroup): + not_tsgroup_index = [i+1 for i, boo in enumerate(is_tsgroup) if not boo] + raise TypeError(f"Passed variables at positions {not_tsgroup_index} are not TsGroup") + + if len(tsgroups) == 1: + print('Only one TsGroup object provided, no merge needed') + return tsgroups[0] + + tsg1 = tsgroups[0] + items = tsg1.items() + keys = set(tsg1.keys()) + metadata = tsg1._metadata + + for i, tsg in enumerate(tsgroups[1:]): + if not ignore_metadata: + if tsg1.metadata_columns != tsg.metadata_columns: + raise ValueError(f"TsGroup at position {i+2} has different metadata columns from previous TsGroup objects. " + "Pass ignore_metadata=True to bypass") + metadata = pd.concat([metadata, tsg._metadata], axis=0) + + if not reset_index: + key_overlap = keys.intersection(tsg.keys()) + if key_overlap: + raise ValueError(f"TsGroup at position {i+2} has overlapping keys {key_overlap} with previous TsGroup objects. " + "Pass reset_index=True to bypass") + keys.update(tsg.keys()) + + if reset_time_support: + time_support = None + else: + if not np.array_equal( + tsg1.time_support.as_units('s').to_numpy(), + tsg.time_support.as_units('s').to_numpy() + ): + raise ValueError(f"TsGroup at position {i+2} has different time support from previous TsGroup objects. " + "Pass reset_time_support=True to bypass") + time_support = tsg1.time_support + + items.extend(tsg.items()) + + if reset_index: + metadata.index = range(len(metadata)) + data = {i: ts[1] for i, ts in enumerate(items)} + else: + data = dict(items) + + if ignore_metadata: + return TsGroup( + data, time_support=time_support, bypass_check=False + ) + else: + cols = metadata.columns.drop("rate") + return TsGroup( + data, time_support=time_support, bypass_check=False, **metadata[cols] + ) + + def merge(self, *tsgroups, reset_index=False, reset_time_support=False, ignore_metadata=False): + """ + Merge the TsGroup object with other TsGroup objects + See `TsGroup.merge_group` for more details + """ + tsgroups = list(tsgroups) + tsgroups.insert(0, self) + return TsGroup.merge_group( + *tsgroups, reset_index=reset_index, reset_time_support=reset_time_support, ignore_metadata=ignore_metadata) + def save(self, filename): """ Save TsGroup object in npz format. The file will contain the timestamps, From 60719f51f6010fbbd5c26bb7194fbe4af2b01917 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Tue, 7 May 2024 01:49:08 +0200 Subject: [PATCH 06/47] doc aesthetics --- pynapple/core/ts_group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 43df2a72..d105bde4 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1111,8 +1111,8 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m ------- ------ 0 0.3 10 0.3 - start end - 0 -5 5 + start end + 0 -5 5 shape: (1, 2), time unit: sec. """ From b3874a15299e2c2af5d7f3959cfd5091d27581c6 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Tue, 7 May 2024 01:49:29 +0200 Subject: [PATCH 07/47] Update ts_group.py --- pynapple/core/ts_group.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index d105bde4..e04762fd 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1111,6 +1111,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m ------- ------ 0 0.3 10 0.3 + start end 0 -5 5 shape: (1, 2), time unit: sec. From d24f3312aeab9393809a2219337b083522686084 Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Tue, 7 May 2024 18:21:30 +0200 Subject: [PATCH 08/47] Simplify tsgroup.merge() Co-authored-by: Edoardo Balzani --- pynapple/core/ts_group.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index e04762fd..395b5e31 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1179,10 +1179,8 @@ def merge(self, *tsgroups, reset_index=False, reset_time_support=False, ignore_m Merge the TsGroup object with other TsGroup objects See `TsGroup.merge_group` for more details """ - tsgroups = list(tsgroups) - tsgroups.insert(0, self) return TsGroup.merge_group( - *tsgroups, reset_index=reset_index, reset_time_support=reset_time_support, ignore_metadata=ignore_metadata) + self, *tsgroups, reset_index=reset_index, reset_time_support=reset_time_support, ignore_metadata=ignore_metadata) def save(self, filename): """ From 04a0053fa616fa6aeea6a825d8e9507bcfdf5109 Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Tue, 7 May 2024 18:22:05 +0200 Subject: [PATCH 09/47] Update pynapple/core/ts_group.py Co-authored-by: Edoardo Balzani --- pynapple/core/ts_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 395b5e31..614eb600 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1120,7 +1120,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m is_tsgroup = [isinstance(tsg, TsGroup) for tsg in tsgroups] if not all(is_tsgroup): not_tsgroup_index = [i+1 for i, boo in enumerate(is_tsgroup) if not boo] - raise TypeError(f"Passed variables at positions {not_tsgroup_index} are not TsGroup") + raise TypeError(f"Input at positions {not_tsgroup_index} are not TsGroup!") if len(tsgroups) == 1: print('Only one TsGroup object provided, no merge needed') From c642068cb5faac287c64e8c8ccbf16f1815f499f Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Tue, 7 May 2024 18:22:21 +0200 Subject: [PATCH 10/47] Update pynapple/core/ts_group.py Co-authored-by: Edoardo Balzani --- pynapple/core/ts_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 614eb600..fe6fbed3 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1135,7 +1135,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m if not ignore_metadata: if tsg1.metadata_columns != tsg.metadata_columns: raise ValueError(f"TsGroup at position {i+2} has different metadata columns from previous TsGroup objects. " - "Pass ignore_metadata=True to bypass") + "Set `ignore_metadata=True` to bypass the check.") metadata = pd.concat([metadata, tsg._metadata], axis=0) if not reset_index: From 2d08c338a25a7e901d3a8c9a4b02075280205199 Mon Sep 17 00:00:00 2001 From: qian-chu Date: Fri, 17 May 2024 22:56:02 +0200 Subject: [PATCH 11/47] Update ts_group.py --- pynapple/core/ts_group.py | 90 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index fe6fbed3..635796e4 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -21,6 +21,7 @@ jitunion_isets, ) from .base_class import Base +from .config import time_index_precision from .interval_set import IntervalSet from .time_index import TsIndex from .time_series import BaseTsd, Ts, Tsd, TsdFrame, is_array_like @@ -1031,9 +1032,11 @@ def getby_category(self, key): return sliced @staticmethod - def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_metadata=False): + def merge_group( + *tsgroups, reset_index=False, reset_time_support=False, ignore_metadata=False + ): """ - Merge multiple TsGroup objects into a single TsGroup object + Merge multiple TsGroup objects into a single TsGroup object. Parameters ---------- @@ -1052,16 +1055,16 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m Returns ------- TsGroup - TsGroup of merged objects - + A TsGroup of merged objects + Raises ------ TypeError If the input objects are not TsGroup objects ValueError - If ignore_metadata=False and metadata columns are not the same - If reset_index=False and keys overlap - If reset_time_support=False and time supports are not the same + If `ignore_metadata=False` but metadata columns are not the same + If `reset_index=False` but keys overlap + If `reset_time_support=False` but time supports are not the same Examples -------- @@ -1082,7 +1085,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m >>> dict4 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} >>> tsgroup4 = nap.TsGroup(dict2, time_support=time_support_b) - Merge with default options if have same time_support and non-overlapping indexes: + Merge with default options if have the same time support and non-overlapping indexes: >>> tsgroup_12 = nap.TsGroup.merge_group(tsgroup1, tsgroup2) >>> tsgroup_12 @@ -1091,7 +1094,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m 0 1.5 10 1.5 - Pass reset_index=True if indexes are overlapping: + Set `reset_index=True` if indexes are overlapping: >>> tsgroup_13 = nap.TsGroup.merge_group(tsgroup1, tsgroup3, reset_index=True) >>> tsgroup_13 @@ -1101,7 +1104,7 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m 0 1.5 1 1.5 - Pass reset_time_support=True if time_supports are different: + Set `reset_time_support=True` if time supports are different: >>> tsgroup_14 = nap.TsGroup.merge_group(tsgroup1, tsgroup4, reset_time_support=True) >>> tsgroup_14 @@ -1119,43 +1122,51 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m """ is_tsgroup = [isinstance(tsg, TsGroup) for tsg in tsgroups] if not all(is_tsgroup): - not_tsgroup_index = [i+1 for i, boo in enumerate(is_tsgroup) if not boo] + not_tsgroup_index = [i + 1 for i, boo in enumerate(is_tsgroup) if not boo] raise TypeError(f"Input at positions {not_tsgroup_index} are not TsGroup!") if len(tsgroups) == 1: - print('Only one TsGroup object provided, no merge needed') + print("Only one TsGroup object provided, no merge needed.") return tsgroups[0] - tsg1 = tsgroups[0] - items = tsg1.items() - keys = set(tsg1.keys()) + tsg1 = tsgroups[0] + items = tsg1.items() + keys = set(tsg1.keys()) metadata = tsg1._metadata for i, tsg in enumerate(tsgroups[1:]): if not ignore_metadata: if tsg1.metadata_columns != tsg.metadata_columns: - raise ValueError(f"TsGroup at position {i+2} has different metadata columns from previous TsGroup objects. " - "Set `ignore_metadata=True` to bypass the check.") + raise ValueError( + f"TsGroup at position {i+2} has different metadata columns from previous TsGroup objects. " + "Set `ignore_metadata=True` to bypass the check." + ) metadata = pd.concat([metadata, tsg._metadata], axis=0) - + if not reset_index: key_overlap = keys.intersection(tsg.keys()) if key_overlap: - raise ValueError(f"TsGroup at position {i+2} has overlapping keys {key_overlap} with previous TsGroup objects. " - "Pass reset_index=True to bypass") + raise ValueError( + f"TsGroup at position {i+2} has overlapping keys {key_overlap} with previous TsGroup objects. " + "Set `reset_index=True` to bypass the check." + ) keys.update(tsg.keys()) - + if reset_time_support: time_support = None else: - if not np.array_equal( - tsg1.time_support.as_units('s').to_numpy(), - tsg.time_support.as_units('s').to_numpy() - ): - raise ValueError(f"TsGroup at position {i+2} has different time support from previous TsGroup objects. " - "Pass reset_time_support=True to bypass") + if not np.allclose( + tsg1.time_support.as_units("s").to_numpy(), + tsg.time_support.as_units("s").to_numpy(), + atol=10 ** (-time_index_precision), + rtol=0, + ): + raise ValueError( + f"TsGroup at position {i+2} has different time support from previous TsGroup objects. " + "Set `reset_time_support=True` to bypass the check." + ) time_support = tsg1.time_support - + items.extend(tsg.items()) if reset_index: @@ -1165,22 +1176,31 @@ def merge_group(*tsgroups, reset_index=False, reset_time_support=False, ignore_m data = dict(items) if ignore_metadata: - return TsGroup( - data, time_support=time_support, bypass_check=False - ) + return TsGroup(data, time_support=time_support, bypass_check=False) else: cols = metadata.columns.drop("rate") return TsGroup( data, time_support=time_support, bypass_check=False, **metadata[cols] - ) - - def merge(self, *tsgroups, reset_index=False, reset_time_support=False, ignore_metadata=False): + ) + + def merge( + self, + *tsgroups, + reset_index=False, + reset_time_support=False, + ignore_metadata=False, + ): """ Merge the TsGroup object with other TsGroup objects See `TsGroup.merge_group` for more details """ return TsGroup.merge_group( - self, *tsgroups, reset_index=reset_index, reset_time_support=reset_time_support, ignore_metadata=ignore_metadata) + self, + *tsgroups, + reset_index=reset_index, + reset_time_support=reset_time_support, + ignore_metadata=ignore_metadata, + ) def save(self, filename): """ From 26f633388b1cb81d863b5f77c581a8021ff0f5cb Mon Sep 17 00:00:00 2001 From: qian-chu Date: Sun, 19 May 2024 14:46:01 +0200 Subject: [PATCH 12/47] correctly access nap_config --- pynapple/core/ts_group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 635796e4..d8deddd5 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -21,7 +21,7 @@ jitunion_isets, ) from .base_class import Base -from .config import time_index_precision +from .config import nap_config from .interval_set import IntervalSet from .time_index import TsIndex from .time_series import BaseTsd, Ts, Tsd, TsdFrame, is_array_like @@ -1158,7 +1158,7 @@ def merge_group( if not np.allclose( tsg1.time_support.as_units("s").to_numpy(), tsg.time_support.as_units("s").to_numpy(), - atol=10 ** (-time_index_precision), + atol=10 ** (-nap_config.time_index_precision), rtol=0, ): raise ValueError( From 5c49a0ea381cd3a5584c976fff9e2bb27f7fff4d Mon Sep 17 00:00:00 2001 From: qian-chu Date: Mon, 20 May 2024 00:50:49 +0200 Subject: [PATCH 13/47] add test for merging groups --- tests/test_ts_group.py | 113 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 4 deletions(-) diff --git a/tests/test_ts_group.py b/tests/test_ts_group.py index 65d23af4..062240ff 100644 --- a/tests/test_ts_group.py +++ b/tests/test_ts_group.py @@ -13,6 +13,7 @@ from collections import UserDict import warnings from contextlib import nullcontext as does_not_raise +import itertools @pytest.fixture @@ -575,15 +576,16 @@ def test_save_npz(self, group): np.testing.assert_array_almost_equal(file['index'], index) np.testing.assert_array_almost_equal(file['meta'], np.arange(len(group), dtype=np.int64)) assert np.all(file['meta2']==np.array(['a', 'b', 'c'])) + file.close() tsgroup3 = nap.TsGroup({ 0: nap.Ts(t=np.arange(0, 20)), }) tsgroup3.save("tsgroup3") - file = np.load("tsgroup3.npz") - assert 'd' not in list(file.keys()) - np.testing.assert_array_almost_equal(file['t'], tsgroup3[0].index) + with np.load("tsgroup3.npz") as file: + assert 'd' not in list(file.keys()) + np.testing.assert_array_almost_equal(file['t'], tsgroup3[0].index) os.remove("tsgroup.npz") os.remove("tsgroup2.npz") @@ -752,4 +754,107 @@ def test_getitem_attribute_error(self, ts_group): ) def test_getitem_boolean_fail(self, ts_group, bool_idx, expectation): with expectation: - out = ts_group[bool_idx] \ No newline at end of file + out = ts_group[bool_idx] + + def test_merge_complete(self, ts_group): + with pytest.raises(TypeError) as e_info: + nap.TsGroup.merge_group(ts_group, str, dict) + assert str(e_info.value) == f"Input at positions {[2, 3]} are not TsGroup!" + + ts_group2 = nap.TsGroup( + { + 3: nap.Ts(t=np.arange(15)), + 4: nap.Ts(t=np.arange(20)), + }, + time_support=ts_group.time_support, + meta=np.array([12, 13]) + ) + merged = ts_group.merge(ts_group2) + assert len(merged) == 4 + assert np.all(merged.keys() == np.array([1, 2, 3, 4])) + assert np.all(merged.meta == np.array([10, 11, 12, 13])) + np.testing.assert_equal(merged.metadata_columns, ts_group.metadata_columns) + + @pytest.mark.parametrize( + 'col_name, ignore_metadata, expectation', + [ + ('meta', False, does_not_raise()), + ('meta', True, does_not_raise()), + ('wrong_name', False, pytest.raises(ValueError, match="TsGroup at position 2 has different metadata columns.*")), + ('wrong_name', True, does_not_raise()) + ] + ) + def test_merge_metadata(self, ts_group, col_name, ignore_metadata, expectation): + metadata = pd.DataFrame([12, 13], index=[3, 4], columns=[col_name]) + ts_group2 = nap.TsGroup( + { + 3: nap.Ts(t=np.arange(15)), + 4: nap.Ts(t=np.arange(20)), + }, + time_support=ts_group.time_support, + **metadata + ) + + with expectation: + merged = ts_group.merge(ts_group2, ignore_metadata=ignore_metadata) + + if ignore_metadata: + assert merged.metadata_columns[0] == 'rate' + elif col_name == 'meta': + np.testing.assert_equal(merged.metadata_columns, ts_group.metadata_columns) + + @pytest.mark.parametrize( + 'index, reset_index, expectation', + [ + (np.array([1, 2]), False, pytest.raises(ValueError, match="TsGroup at position 2 has overlapping keys.*")), + (np.array([1, 2]), True, does_not_raise()), + (np.array([3, 4]), False, does_not_raise()), + (np.array([3, 4]), True, does_not_raise()) + ] + ) + def test_merge_index(self, ts_group, index, reset_index, expectation): + ts_group2 = nap.TsGroup( + dict(zip(index, [nap.Ts(t=np.arange(15)), nap.Ts(t=np.arange(20))])), + time_support=ts_group.time_support, + meta=np.array([12, 13]) + ) + + with expectation: + merged = ts_group.merge(ts_group2, reset_index=reset_index) + + if reset_index: + assert np.all(merged.keys() == np.arange(4)) + elif np.all(index == np.array([3, 4])): + assert np.all(merged.keys() == np.array([1, 2, 3, 4])) + + @pytest.mark.parametrize( + 'time_support, reset_time_support, expectation', + [ + (None, False, does_not_raise()), + (None, True, does_not_raise()), + (nap.IntervalSet(start=0, end=1), False, + pytest.raises(ValueError, match="TsGroup at position 2 has different time support.*")), + (nap.IntervalSet(start=0, end=1), True, does_not_raise()) + ] + ) + def test_merge_time_support(self, ts_group, time_support, reset_time_support, expectation): + if time_support is None: + time_support = ts_group.time_support + + ts_group2 = nap.TsGroup( + { + 3: nap.Ts(t=np.arange(15)), + 4: nap.Ts(t=np.arange(20)), + }, + time_support=time_support, + meta=np.array([12, 13]) + ) + + with expectation: + merged = ts_group.merge(ts_group2, reset_time_support=reset_time_support) + + if reset_time_support: + np.testing.assert_array_almost_equal( + ts_group.time_support.as_units("s").to_numpy(), + merged.time_support.as_units("s").to_numpy() + ) \ No newline at end of file From a65f5f5e6c82f4fdf124962c149ee0111ffe0877 Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Mon, 20 May 2024 15:41:28 +0200 Subject: [PATCH 14/47] Update tests/test_ts_group.py Co-authored-by: Edoardo Balzani --- tests/test_ts_group.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_ts_group.py b/tests/test_ts_group.py index 062240ff..1688e741 100644 --- a/tests/test_ts_group.py +++ b/tests/test_ts_group.py @@ -757,9 +757,8 @@ def test_getitem_boolean_fail(self, ts_group, bool_idx, expectation): out = ts_group[bool_idx] def test_merge_complete(self, ts_group): - with pytest.raises(TypeError) as e_info: + with pytest.raises(TypeError, match=f"Input at positions {[2, 3]} are not TsGroup!"): nap.TsGroup.merge_group(ts_group, str, dict) - assert str(e_info.value) == f"Input at positions {[2, 3]} are not TsGroup!" ts_group2 = nap.TsGroup( { From 6bf20a7a849ee6d09ae1fcdcfee52a4d663c0f4c Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Mon, 20 May 2024 16:06:48 +0200 Subject: [PATCH 15/47] Update pynapple/core/ts_group.py Co-authored-by: Edoardo Balzani --- pynapple/core/ts_group.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 9be60cc5..38fcb767 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1106,10 +1106,10 @@ def merge_group( >>> tsgroup_14 >>> tsgroup_14.time_support - Index rate - ------- ------ - 0 0.3 - 10 0.3 + Index rate + ------- ------ + 0 0.3 + 10 0.3 start end 0 -5 5 From 6988758f1de88ecdaa5571c06266895081fcb833 Mon Sep 17 00:00:00 2001 From: Qian Chu <97355086+qian-chu@users.noreply.github.com> Date: Mon, 20 May 2024 16:07:06 +0200 Subject: [PATCH 16/47] Update pynapple/core/ts_group.py Co-authored-by: Edoardo Balzani --- pynapple/core/ts_group.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 38fcb767..26c5f383 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1095,10 +1095,10 @@ def merge_group( >>> tsgroup_13 = nap.TsGroup.merge_group(tsgroup1, tsgroup3, reset_index=True) >>> tsgroup_13 - Index rate - ------- ------ - 0 1.5 - 1 1.5 + Index rate + ------- ------ + 0 1.5 + 1 1.5 Set `reset_time_support=True` if time supports are different: From 41e0dfc5d89cdbdad5393dbe93f2d975ab893a44 Mon Sep 17 00:00:00 2001 From: qian-chu <97355086+qian-chu@users.noreply.github.com> Date: Mon, 20 May 2024 16:39:23 +0200 Subject: [PATCH 17/47] Move docstring to merge() --- pynapple/core/ts_group.py | 140 +++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 26c5f383..221c0436 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1062,59 +1062,6 @@ def merge_group( If `reset_index=False` but keys overlap If `reset_time_support=False` but time supports are not the same - Examples - -------- - - >>> import pynapple as nap - >>> time_support_a = nap.IntervalSet(start=-1, end=1, time_units='s') - >>> time_support_b = nap.IntervalSet(start=-5, end=5, time_units='s') - - >>> dict1 = {0: nap.Ts(t=[-1, 0, 1], time_units='s')} - >>> tsgroup1 = nap.TsGroup(dict1, time_support=time_support_a) - - >>> dict2 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} - >>> tsgroup2 = nap.TsGroup(dict2, time_support=time_support_a) - - >>> dict3 = {0: nap.Ts(t=[-.1, 0, .1], time_units='s')} - >>> tsgroup3 = nap.TsGroup(dict3, time_support=time_support_a) - - >>> dict4 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} - >>> tsgroup4 = nap.TsGroup(dict2, time_support=time_support_b) - - Merge with default options if have the same time support and non-overlapping indexes: - - >>> tsgroup_12 = nap.TsGroup.merge_group(tsgroup1, tsgroup2) - >>> tsgroup_12 - Index rate - ------- ------ - 0 1.5 - 10 1.5 - - Set `reset_index=True` if indexes are overlapping: - - >>> tsgroup_13 = nap.TsGroup.merge_group(tsgroup1, tsgroup3, reset_index=True) - >>> tsgroup_13 - - Index rate - ------- ------ - 0 1.5 - 1 1.5 - - Set `reset_time_support=True` if time supports are different: - - >>> tsgroup_14 = nap.TsGroup.merge_group(tsgroup1, tsgroup4, reset_time_support=True) - >>> tsgroup_14 - >>> tsgroup_14.time_support - - Index rate - ------- ------ - 0 0.3 - 10 0.3 - - start end - 0 -5 5 - shape: (1, 2), time unit: sec. - """ is_tsgroup = [isinstance(tsg, TsGroup) for tsg in tsgroups] if not all(is_tsgroup): @@ -1187,8 +1134,91 @@ def merge( ignore_metadata=False, ): """ - Merge the TsGroup object with other TsGroup objects - See `TsGroup.merge_group` for more details + Merge the TsGroup object with other TsGroup objects. + Common uses include adding more neurons/channels (supposing each Ts/Tsd corresponds to data from a neuron/channel) or adding more trials (supposing each Ts/Tsd corresponds to data from a trial). + + Parameters + ---------- + *tsgroups : TsGroup + The TsGroup objects to merge with + reset_index : bool, optional + If True, the keys will be reset to range(len(data)) + If False, the keys of the TsGroup objects should be non-overlapping and will be preserved + reset_time_support : bool, optional + If True, the merged TsGroup will merge time supports from all the Ts/Tsd objects in data + If False, the time support of the TsGroup objects should be the same and will be preserved + ignore_metadata : bool, optional + If True, the merged TsGroup will not have any metadata columns other than 'rate' + If False, all metadata columns should be the same and all metadata will be concatenated + + Returns + ------- + TsGroup + A TsGroup of merged objects + + Raises + ------ + TypeError + If the input objects are not TsGroup objects + ValueError + If `ignore_metadata=False` but metadata columns are not the same + If `reset_index=False` but keys overlap + If `reset_time_support=False` but time supports are not the same + + Examples + -------- + + >>> import pynapple as nap + >>> time_support_a = nap.IntervalSet(start=-1, end=1, time_units='s') + >>> time_support_b = nap.IntervalSet(start=-5, end=5, time_units='s') + + >>> dict1 = {0: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup1 = nap.TsGroup(dict1, time_support=time_support_a) + + >>> dict2 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup2 = nap.TsGroup(dict2, time_support=time_support_a) + + >>> dict3 = {0: nap.Ts(t=[-.1, 0, .1], time_units='s')} + >>> tsgroup3 = nap.TsGroup(dict3, time_support=time_support_a) + + >>> dict4 = {10: nap.Ts(t=[-1, 0, 1], time_units='s')} + >>> tsgroup4 = nap.TsGroup(dict2, time_support=time_support_b) + + Merge with default options if have the same time support and non-overlapping indexes: + + >>> tsgroup_12 = tsgroup1.merge(tsgroup2) + >>> tsgroup_12 + Index rate + ------- ------ + 0 1.5 + 10 1.5 + + Set `reset_index=True` if indexes are overlapping: + + >>> tsgroup_13 = tsgroup1.merge(tsgroup3, reset_index=True) + >>> tsgroup_13 + + Index rate + ------- ------ + 0 1.5 + 1 1.5 + + Set `reset_time_support=True` if time supports are different: + + >>> tsgroup_14 = tsgroup1.merge(tsgroup4, reset_time_support=True) + >>> tsgroup_14 + >>> tsgroup_14.time_support + + Index rate + ------- ------ + 0 0.3 + 10 0.3 + + start end + 0 -5 5 + shape: (1, 2), time unit: sec. + + See Also `TsGroup.merge_group` """ return TsGroup.merge_group( self, From e08c0e4820fa505711b55c2fdbe7f9b8c6de1900 Mon Sep 17 00:00:00 2001 From: qian-chu <97355086+qian-chu@users.noreply.github.com> Date: Mon, 20 May 2024 17:05:55 +0200 Subject: [PATCH 18/47] Fix regex error --- tests/test_ts_group.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_ts_group.py b/tests/test_ts_group.py index 1688e741..fc221f60 100644 --- a/tests/test_ts_group.py +++ b/tests/test_ts_group.py @@ -13,8 +13,6 @@ from collections import UserDict import warnings from contextlib import nullcontext as does_not_raise -import itertools - @pytest.fixture def group(): @@ -757,7 +755,7 @@ def test_getitem_boolean_fail(self, ts_group, bool_idx, expectation): out = ts_group[bool_idx] def test_merge_complete(self, ts_group): - with pytest.raises(TypeError, match=f"Input at positions {[2, 3]} are not TsGroup!"): + with pytest.raises(TypeError, match="Input at positions(.*)are not TsGroup!"): nap.TsGroup.merge_group(ts_group, str, dict) ts_group2 = nap.TsGroup( From 2f9ded5235083e5f5f37624dfcff8e5b2bc1f0de Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 12:33:28 -0400 Subject: [PATCH 19/47] changed label name --- pynapple/core/time_series.py | 24 +++++++++++++----------- pynapple/io/interface_nwb.py | 8 ++++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index bf9f7e7b..6fc82596 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -68,14 +68,16 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): Implement most of the shared functions across concrete classes `Tsd`, `TsdFrame`, `TsdTensor` """ - def __init__(self, t, d, time_units="s", time_support=None, lazy=False): + def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=True): super().__init__(t, time_units, time_support) - if lazy: - self.values = d - else: + if conv_to_array: self.values = convert_to_array(d, "d") - + else: + if not is_array_like(d): + raise TypeError("Data should be array-like, i.e. be indexable, iterable and, have attributes " + "`shape`, `ndim` and, `dtype`).") + self.values = d assert len(self.index) == len( self.values @@ -666,7 +668,7 @@ class TsdTensor(BaseTsd): The time support of the time series """ - def __init__(self, t, d, time_units="s", time_support=None, lazy=False, **kwargs): + def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=False, **kwargs): """ TsdTensor initializer @@ -681,7 +683,7 @@ def __init__(self, t, d, time_units="s", time_support=None, lazy=False, **kwargs time_support : IntervalSet, optional The time support of the TsdFrame object """ - super().__init__(t, d, time_units, time_support, lazy) + super().__init__(t, d, time_units, time_support, conv_to_array) assert ( self.values.ndim >= 3 @@ -836,7 +838,7 @@ class TsdFrame(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, columns=None, lazy=False + self, t, d=None, time_units="s", time_support=None, columns=None, conv_to_array=False ): """ TsdFrame initializer @@ -865,7 +867,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing TsdFrame" - super().__init__(t, d, time_units, time_support, lazy) + super().__init__(t, d, time_units, time_support, conv_to_array) assert self.values.ndim <= 2, "Data should be 1 or 2 dimensional." @@ -1107,7 +1109,7 @@ class Tsd(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, lazy=False, **kwargs + self, t, d=None, time_units="s", time_support=None, conv_to_array=False, **kwargs ): """ Tsd Initializer. @@ -1129,7 +1131,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing Tsd" - super().__init__(t, d, time_units, time_support, lazy) + super().__init__(t, d, time_units, time_support, conv_to_array) assert self.values.ndim == 1, "Data should be 1 dimensional" diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index d8140f2d..271dc76a 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -144,11 +144,11 @@ def _make_tsd(obj): d = obj.data if obj.timestamps is not None: - t = obj.timestamps + t = obj.timestamps[:] else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.Tsd(t=t, d=d, lazy=True) + data = nap.Tsd(t=t, d=d, conv_to_array=False) return data @@ -173,7 +173,7 @@ def _make_tsd_tensor(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.TsdTensor(t=t, d=d, lazy=True) + data = nap.TsdTensor(t=t, d=d, conv_to_array=False) return data @@ -232,7 +232,7 @@ def _make_tsd_frame(obj): else: columns = np.arange(obj.data.shape[1]) - data = nap.TsdFrame(t=t, d=d, columns=columns, lazy=True) + data = nap.TsdFrame(t=t, d=d, columns=columns, conv_to_array=False) return data From 28094487cc3939e9204ed5cfca98733b4319f36d Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:03:39 -0400 Subject: [PATCH 20/47] added a couple of tests --- pynapple/core/time_series.py | 2 +- pynapple/core/utils.py | 4 +-- tests/test_lazy_loading.py | 53 ++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/test_lazy_loading.py diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 6fc82596..1abbeaab 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -76,7 +76,7 @@ def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=True): else: if not is_array_like(d): raise TypeError("Data should be array-like, i.e. be indexable, iterable and, have attributes " - "`shape`, `ndim` and, `dtype`).") + "`shape`, `ndim` and, `dtype`).") self.values = d assert len(self.index) == len( diff --git a/pynapple/core/utils.py b/pynapple/core/utils.py index b6ea1e24..17d90546 100644 --- a/pynapple/core/utils.py +++ b/pynapple/core/utils.py @@ -125,14 +125,14 @@ def is_array_like(obj): try: obj[0] is_indexable = True - except (TypeError, IndexError): + except Exception: is_indexable = False # Check for iterable property try: iter(obj) is_iterable = True - except TypeError: + except Exception: is_iterable = False # not_tsd_type = not isinstance(obj, _AbstractTsd) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py new file mode 100644 index 00000000..86d1679f --- /dev/null +++ b/tests/test_lazy_loading.py @@ -0,0 +1,53 @@ +import h5py +import pynapple as nap +import numpy as np +import pytest +from contextlib import nullcontext as does_not_raise +from pathlib import Path + + +@pytest.mark.parametrize( + "time, data, expectation", + [ + (np.arange(12), np.arange(12), does_not_raise()), + (np.arange(12), "not_an_array", pytest.raises(TypeError, match="Data should be array-like")) + ] +) +def test_lazy_load_hdf5_is_array(time, data, expectation): + file_path = Path('data.h5') + try: + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + h5_data = h5py.File(file_path, 'r')["data"] + with expectation: + nap.Tsd(t=time, d=h5_data, conv_to_array=False) + finally: + # delete file + if file_path.exists(): + file_path.unlink() + + +@pytest.mark.parametrize( + "time, data", + [ + (np.arange(12), np.arange(12)), + ] +) +@pytest.mark.parametrize("convert_flag", [True, False]) +def test_lazy_load_hdf5_is_array(time, data, convert_flag): + file_path = Path('data.h5') + try: + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + tsd = nap.Tsd(t=time, d=h5_data, conv_to_array=convert_flag) + if convert_flag: + assert isinstance(tsd.d, np.ndarray) + else: + assert isinstance(tsd.d, h5py.Dataset) + finally: + # delete file + if file_path.exists(): + file_path.unlink() + From e02a423498b494a3f7ca0549efd0d09240506db5 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:26:15 -0400 Subject: [PATCH 21/47] tested methods --- pynapple/core/_jitted_functions.py | 9 +++-- pynapple/core/time_series.py | 2 +- tests/test_lazy_loading.py | 58 ++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/pynapple/core/_jitted_functions.py b/pynapple/core/_jitted_functions.py index a30d143a..55f30f8d 100644 --- a/pynapple/core/_jitted_functions.py +++ b/pynapple/core/_jitted_functions.py @@ -312,11 +312,14 @@ def jitthreshold(time_array, data_array, starts, ends, thr, method="above"): return (new_time_array, new_data_array, new_starts, new_ends) -@jit(nopython=True) def jitbin_array(time_array, data_array, starts, ends, bin_size): + """Slice first for compatibility with lazy loading.""" idx, countin = jitrestrict_with_count(time_array, starts, ends) - time_array = time_array[idx] - data_array = data_array[idx] + return _jitbin_array(countin, time_array[idx], data_array[idx], starts, ends, bin_size) + + +@jit(nopython=True) +def _jitbin_array(countin, time_array, data_array, starts, ends, bin_size): m = starts.shape[0] f = data_array.shape[1:] diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 1930c371..6e6cb006 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -250,7 +250,7 @@ def to_numpy(self): """ Return the data as a numpy.ndarray. Mostly useful for matplotlib plotting when calling `plot(tsd)` """ - return self.values + return np.asarray(self.values) def copy(self): """Copy the data, index and time support""" diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 86d1679f..923e941e 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -51,3 +51,61 @@ def test_lazy_load_hdf5_is_array(time, data, convert_flag): if file_path.exists(): file_path.unlink() + +@pytest.mark.parametrize("time, data", [(np.arange(12), np.arange(12))]) +@pytest.mark.parametrize("cls", [nap.Tsd, nap.TsdFrame, nap.TsdTensor]) +@pytest.mark.parametrize("func", [np.exp, lambda x: x*2]) +def test_lazy_load_hdf5_apply_func(time, data, func,cls): + """Apply a unary function to a lazy loaded array.""" + file_path = Path('data.h5') + try: + if cls is nap.TsdFrame: + data = data[:, None] + elif cls is nap.TsdTensor: + data = data[:, None, None] + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + # lazy load and apply function + res = func(cls(t=time, d=h5_data, conv_to_array=False)) + assert isinstance(res, cls) + assert isinstance(res.d, np.ndarray) + finally: + # delete file + if file_path.exists(): + file_path.unlink() + + +@pytest.mark.parametrize("time, data", [(np.arange(12), np.arange(12))]) +@pytest.mark.parametrize("cls", [nap.Tsd, nap.TsdFrame, nap.TsdTensor]) +@pytest.mark.parametrize( + "method_name, args", + [ + ("bin_average", 0.1), + ("count", 0.1), + ("interpolate", nap.Ts(t=np.linspace(0, 12, 50))), + ("convolve", np.ones(3)), + ("smooth", 2), + ("dropna", True) + ] +) +def test_lazy_load_hdf5_apply_func(time, data, method_name, args, cls): + file_path = Path('data.h5') + try: + if cls is nap.TsdFrame: + data = data[:, None] + elif cls is nap.TsdTensor: + data = data[:, None, None] + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + # lazy load and apply function + tsd = cls(t=time, d=h5_data, conv_to_array=False) + func = getattr(tsd, method_name) + res = func(args) + finally: + # delete file + if file_path.exists(): + file_path.unlink() \ No newline at end of file From 3651421565e2d115905575f2770399dcf2596b3c Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:28:58 -0400 Subject: [PATCH 22/47] added test for value from --- tests/test_lazy_loading.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 923e941e..3cf5ebc5 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -87,7 +87,8 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): ("interpolate", nap.Ts(t=np.linspace(0, 12, 50))), ("convolve", np.ones(3)), ("smooth", 2), - ("dropna", True) + ("dropna", True), + ("value_from", nap.Tsd(t=np.linspace(0, 12, 20), d=np.random.normal(size=20))) ] ) def test_lazy_load_hdf5_apply_func(time, data, method_name, args, cls): @@ -104,8 +105,9 @@ def test_lazy_load_hdf5_apply_func(time, data, method_name, args, cls): # lazy load and apply function tsd = cls(t=time, d=h5_data, conv_to_array=False) func = getattr(tsd, method_name) - res = func(args) + out = func(args) + assert isinstance(out.d, np.ndarray) finally: # delete file if file_path.exists(): - file_path.unlink() \ No newline at end of file + file_path.unlink() From 91d14f1a84eeb92dec90f8287aae0e962ec8a70c Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:44:49 -0400 Subject: [PATCH 23/47] added tsd specific methods --- pynapple/core/_core_functions.py | 4 +-- pynapple/core/time_series.py | 2 +- tests/test_lazy_loading.py | 48 ++++++++++++++++++++++++++------ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pynapple/core/_core_functions.py b/pynapple/core/_core_functions.py index b969e787..6a47fdb4 100644 --- a/pynapple/core/_core_functions.py +++ b/pynapple/core/_core_functions.py @@ -144,6 +144,6 @@ def _threshold(time_array, data_array, starts, ends, thr, method): if get_backend() == "jax": from pynajax.jax_core_threshold import threshold - return threshold(time_array, data_array, starts, ends, thr, method) + return threshold(time_array, data_array[:], starts, ends, thr, method) else: - return jitthreshold(time_array, data_array, starts, ends, thr, method) + return jitthreshold(time_array, data_array[:], starts, ends, thr, method) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 6e6cb006..15deb3fd 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -255,7 +255,7 @@ def to_numpy(self): def copy(self): """Copy the data, index and time support""" return self.__class__( - t=self.index.copy(), d=self.values.copy(), time_support=self.time_support + t=self.index.copy(), d=self.values[:].copy(), time_support=self.time_support ) def value_from(self, data, ep=None): diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 3cf5ebc5..bab6cc2a 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -1,4 +1,6 @@ import h5py +import pandas as pd + import pynapple as nap import numpy as np import pytest @@ -82,16 +84,17 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): @pytest.mark.parametrize( "method_name, args", [ - ("bin_average", 0.1), - ("count", 0.1), - ("interpolate", nap.Ts(t=np.linspace(0, 12, 50))), - ("convolve", np.ones(3)), - ("smooth", 2), - ("dropna", True), - ("value_from", nap.Tsd(t=np.linspace(0, 12, 20), d=np.random.normal(size=20))) + ("bin_average", [0.1]), + ("count", [0.1]), + ("interpolate", [nap.Ts(t=np.linspace(0, 12, 50))]), + ("convolve", [np.ones(3)]), + ("smooth", [2]), + ("dropna", [True]), + ("value_from", [nap.Tsd(t=np.linspace(0, 12, 20), d=np.random.normal(size=20))]), + ("copy", []) ] ) -def test_lazy_load_hdf5_apply_func(time, data, method_name, args, cls): +def test_lazy_load_hdf5_apply_method(time, data, method_name, args, cls): file_path = Path('data.h5') try: if cls is nap.TsdFrame: @@ -105,9 +108,36 @@ def test_lazy_load_hdf5_apply_func(time, data, method_name, args, cls): # lazy load and apply function tsd = cls(t=time, d=h5_data, conv_to_array=False) func = getattr(tsd, method_name) - out = func(args) + out = func(*args) assert isinstance(out.d, np.ndarray) finally: # delete file if file_path.exists(): file_path.unlink() + + +@pytest.mark.parametrize("time, data", [(np.arange(12), np.arange(12))]) +@pytest.mark.parametrize( + "method_name, args, expected_out_type", + [ + ("threshold", [3], nap.Tsd), + ("as_series", [], pd.Series), + ("as_units", ['ms'], pd.Series), + ("to_tsgroup", [], nap.TsGroup) + ] +) +def test_lazy_load_hdf5_apply_method_tsd_specific(time, data, method_name, args, expected_out_type): + file_path = Path('data.h5') + try: + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + # lazy load and apply function + tsd = nap.Tsd(t=time, d=h5_data, conv_to_array=False) + func = getattr(tsd, method_name) + assert isinstance(func(*args), expected_out_type) + finally: + # delete file + if file_path.exists(): + file_path.unlink() \ No newline at end of file From 6995aea61c20c54cf5d94984affe71dcb6b0e4b1 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:54:51 -0400 Subject: [PATCH 24/47] save lazy --- pynapple/core/time_series.py | 2 +- tests/test_lazy_loading.py | 45 +++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 15deb3fd..d8c1e967 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -1092,7 +1092,7 @@ def save(self, filename): np.savez( filename, t=self.index.values, - d=self.values, + d=self.values[:], start=self.time_support.start, end=self.time_support.end, columns=cols_name, diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index bab6cc2a..23f7ecb9 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -140,4 +140,47 @@ def test_lazy_load_hdf5_apply_method_tsd_specific(time, data, method_name, args, finally: # delete file if file_path.exists(): - file_path.unlink() \ No newline at end of file + file_path.unlink() + + +@pytest.mark.parametrize("time, data", [(np.arange(12), np.arange(12))]) +@pytest.mark.parametrize( + "method_name, args, expected_out_type", + [ + ("as_dataframe", [], pd.DataFrame), + ] +) +def test_lazy_load_hdf5_apply_method_tsdframe_specific(time, data, method_name, args, expected_out_type): + file_path = Path('data.h5') + try: + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data[:, None]) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + # lazy load and apply function + tsd = nap.TsdFrame(t=time, d=h5_data, conv_to_array=False) + func = getattr(tsd, method_name) + assert isinstance(func(*args), expected_out_type) + finally: + # delete file + if file_path.exists(): + file_path.unlink() + + +def test_lazy_load_hdf5_tsdframe_loc(): + file_path = Path('data.h5') + data = np.arange(10).reshape(5, 2) + try: + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + # get the tsd + h5_data = h5py.File(file_path, 'r')["data"] + # lazy load and apply function + tsd = nap.TsdFrame(t=np.arange(data.shape[0]), d=h5_data, conv_to_array=False).loc[1] + assert isinstance(tsd, nap.Tsd) + assert all(tsd.d == np.array([1, 3, 5, 7, 9])) + + finally: + # delete file + if file_path.exists(): + file_path.unlink() From 9487728543b1df349f75cccc8814571f4f050c14 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 14:59:42 -0400 Subject: [PATCH 25/47] added test for nwb lazy --- tests/test_lazy_loading.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 23f7ecb9..ea99a67c 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -184,3 +184,9 @@ def test_lazy_load_hdf5_tsdframe_loc(): # delete file if file_path.exists(): file_path.unlink() + + +def test_lazy_load_nwb(): + file_path = 'nwbfilestest/basic/pynapplenwb/A2929-200711.nwb' + tsd = nap.load_file(file_path)["z"] + assert isinstance(tsd.d, h5py.Dataset) From a985a72b56f8485857cf788a4d67629794945ca9 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Mon, 20 May 2024 15:09:12 -0400 Subject: [PATCH 26/47] Revert "Signal" --- pynapple/process/signal_processing.py | 67 --------------------------- 1 file changed, 67 deletions(-) delete mode 100644 pynapple/process/signal_processing.py diff --git a/pynapple/process/signal_processing.py b/pynapple/process/signal_processing.py deleted file mode 100644 index 038f0394..00000000 --- a/pynapple/process/signal_processing.py +++ /dev/null @@ -1,67 +0,0 @@ -""" - Signal processing module -""" - -import numpy as np -from .. import core as nap -from scipy.signal import butter, lfilter, filtfilt - - -def _butter_bandpass(lowcut, highcut, fs, order=5): - nyq = 0.5 * fs - low = lowcut / nyq - high = highcut / nyq - b, a = butter(order, [low, high], btype='band') - return b, a - -def _butter_bandpass_filter(data, lowcut, highcut, fs, order=4): - b, a = _butter_bandpass(lowcut, highcut, fs, order=order) - y = lfilter(b, a, data) - return y - -def compute_bandpass_filter(data, freq_band, sampling_frequency=None, order=4): - """ - Bandpass filtering the LFP. - - Parameters - ---------- - data : Tsd/TsdFrame - Description - lowcut : TYPE - Description - highcut : TYPE - Description - fs : TYPE - Description - order : int, optional - Description - - Raises - ------ - RuntimeError - Description - """ - time_support = data.time_support - time_index = data.as_units('s').index.values - if type(data) is nap.TsdFrame: - tmp = np.zeros(data.shape) - for i in np.arange(data.shape[1]): - tmp[:,i] = bandpass_filter(data[:,i], lowcut, highcut, fs, order) - - return nap.TsdFrame( - t = time_index, - d = tmp, - time_support = time_support, - time_units = 's', - columns = data.columns) - - elif type(data) is nap.Tsd: - flfp = _butter_bandpass_filter(data.values, lowcut, highcut, fs, order) - return nap.Tsd( - t=time_index, - d=flfp, - time_support=time_support, - time_units='s') - - else: - raise RuntimeError("Unknow format. Should be Tsd/TsdFrame") \ No newline at end of file From af6b53f6a9d05e9fb0ef5b9ef65c95eaea7ac987 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 15:09:46 -0400 Subject: [PATCH 27/47] compatibility --- pynapple/core/_jitted_functions.py | 4 +++- pynapple/core/time_series.py | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pynapple/core/_jitted_functions.py b/pynapple/core/_jitted_functions.py index 55f30f8d..4de37635 100644 --- a/pynapple/core/_jitted_functions.py +++ b/pynapple/core/_jitted_functions.py @@ -315,7 +315,9 @@ def jitthreshold(time_array, data_array, starts, ends, thr, method="above"): def jitbin_array(time_array, data_array, starts, ends, bin_size): """Slice first for compatibility with lazy loading.""" idx, countin = jitrestrict_with_count(time_array, starts, ends) - return _jitbin_array(countin, time_array[idx], data_array[idx], starts, ends, bin_size) + return _jitbin_array( + countin, time_array[idx], data_array[idx], starts, ends, bin_size + ) @jit(nopython=True) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index d8c1e967..34551e7b 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -75,8 +75,10 @@ def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=True): self.values = convert_to_array(d, "d") else: if not is_array_like(d): - raise TypeError("Data should be array-like, i.e. be indexable, iterable and, have attributes " - "`shape`, `ndim` and, `dtype`).") + raise TypeError( + "Data should be array-like, i.e. be indexable, iterable and, have attributes " + "`shape`, `ndim` and, `dtype`)." + ) self.values = d assert len(self.index) == len( @@ -668,7 +670,9 @@ class TsdTensor(BaseTsd): The time support of the time series """ - def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=False, **kwargs): + def __init__( + self, t, d, time_units="s", time_support=None, conv_to_array=True, **kwargs + ): """ TsdTensor initializer @@ -838,7 +842,13 @@ class TsdFrame(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, columns=None, conv_to_array=False + self, + t, + d=None, + time_units="s", + time_support=None, + columns=None, + conv_to_array=True, ): """ TsdFrame initializer @@ -1117,7 +1127,7 @@ class Tsd(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, conv_to_array=False, **kwargs + self, t, d=None, time_units="s", time_support=None, conv_to_array=True, **kwargs ): """ Tsd Initializer. From 468fb18b5ae161bcab472b407a86d38921bddaa9 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 15:16:31 -0400 Subject: [PATCH 28/47] test fixed --- tests/test_lazy_loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index ea99a67c..4165d201 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -187,6 +187,6 @@ def test_lazy_load_hdf5_tsdframe_loc(): def test_lazy_load_nwb(): - file_path = 'nwbfilestest/basic/pynapplenwb/A2929-200711.nwb' + file_path = 'tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb' tsd = nap.load_file(file_path)["z"] assert isinstance(tsd.d, h5py.Dataset) From bd01724c6b9cb264677cd3573d0f028c9da7a9de Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 15:23:40 -0400 Subject: [PATCH 29/47] add test for get --- tests/test_lazy_loading.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 4165d201..3a1c9c7f 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -91,7 +91,8 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): ("smooth", [2]), ("dropna", [True]), ("value_from", [nap.Tsd(t=np.linspace(0, 12, 20), d=np.random.normal(size=20))]), - ("copy", []) + ("copy", []), + ("get", [2, 7]) ] ) def test_lazy_load_hdf5_apply_method(time, data, method_name, args, cls): From 809c140bde6e92cca6438112d1470de6533df245 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 15:35:48 -0400 Subject: [PATCH 30/47] close hdf5 after opening --- tests/test_lazy_loading.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 3a1c9c7f..72f10e63 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -188,6 +188,10 @@ def test_lazy_load_hdf5_tsdframe_loc(): def test_lazy_load_nwb(): - file_path = 'tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb' - tsd = nap.load_file(file_path)["z"] + try: + nwb = nap.NWBFile("tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb") + except: + nwb = nap.NWBFile("nwbfilestest/basic/pynapplenwb/A2929-200711.nwb") + + tsd = nwb["z"] assert isinstance(tsd.d, h5py.Dataset) From 2225a2aefbdcdda59dfd8cd977b4c881aa3ccbdb Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 15:56:37 -0400 Subject: [PATCH 31/47] fixed docstrings --- pynapple/core/ts_group.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index 221c0436..b4865d6d 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -1188,37 +1188,37 @@ def merge( >>> tsgroup_12 = tsgroup1.merge(tsgroup2) >>> tsgroup_12 - Index rate - ------- ------ - 0 1.5 - 10 1.5 + Index rate + ------- ------ + 0 1.5 + 10 1.5 Set `reset_index=True` if indexes are overlapping: >>> tsgroup_13 = tsgroup1.merge(tsgroup3, reset_index=True) >>> tsgroup_13 - - Index rate - ------- ------ - 0 1.5 - 1 1.5 + Index rate + ------- ------ + 0 1.5 + 1 1.5 Set `reset_time_support=True` if time supports are different: >>> tsgroup_14 = tsgroup1.merge(tsgroup4, reset_time_support=True) >>> tsgroup_14 >>> tsgroup_14.time_support - - Index rate - ------- ------ - 0 0.3 - 10 0.3 + Index rate + ------- ------ + 0 0.3 + 10 0.3 start end 0 -5 5 shape: (1, 2), time unit: sec. - See Also `TsGroup.merge_group` + See Also + -------- + [`TsGroup.merge_group`](./#pynapple.core.ts_group.TsGroup.merge_group) """ return TsGroup.merge_group( self, From b40940960657e25522d5580d24a243e8b27d2510 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 16:02:10 -0400 Subject: [PATCH 32/47] close the nwb --- tests/test_lazy_loading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 72f10e63..0788c857 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -195,3 +195,4 @@ def test_lazy_load_nwb(): tsd = nwb["z"] assert isinstance(tsd.d, h5py.Dataset) + nwb.io.close() From 235d06e29c9162e4e1c8c64b0b9f7ecba0213645 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Mon, 20 May 2024 16:10:17 -0400 Subject: [PATCH 33/47] add checks for pr to dev --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 349e7157..7c3a45de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,7 +5,7 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: [ main, dev ] jobs: lint: From 92e0df4c900418b4a04c84c7509327b3b916cbe0 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 12:09:31 -0400 Subject: [PATCH 34/47] change to codecov --- .github/workflows/main.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c3a45de..6cc876b7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,14 +52,12 @@ jobs: - name: Test run: | coverage run --source=pynapple --branch -m pytest tests/ - coverage report -m + coverage report -m - - name: Coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls - coveralls --service=github + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} check: if: always() needs: From e6c9bc58bff3f19c5f5d26a7cfb4931fd222f563 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 12:11:42 -0400 Subject: [PATCH 35/47] updated actions --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6cc876b7..0c779db4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,9 +40,9 @@ jobs: # - os: windows-latest # python-version: 3.7 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 2904cbab98dff3124d48c072d0b451b879a5f99a Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:17:31 -0400 Subject: [PATCH 36/47] changed var name --- pynapple/core/time_series.py | 16 ++++++++-------- pynapple/io/interface_nwb.py | 6 +++--- tests/test_lazy_loading.py | 14 +++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 34551e7b..7fe0023a 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -68,10 +68,10 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): Implement most of the shared functions across concrete classes `Tsd`, `TsdFrame`, `TsdTensor` """ - def __init__(self, t, d, time_units="s", time_support=None, conv_to_array=True): + def __init__(self, t, d, time_units="s", time_support=None, to_numpy_array=True): super().__init__(t, time_units, time_support) - if conv_to_array: + if to_numpy_array: self.values = convert_to_array(d, "d") else: if not is_array_like(d): @@ -671,7 +671,7 @@ class TsdTensor(BaseTsd): """ def __init__( - self, t, d, time_units="s", time_support=None, conv_to_array=True, **kwargs + self, t, d, time_units="s", time_support=None, to_numpy_array=True, **kwargs ): """ TsdTensor initializer @@ -687,7 +687,7 @@ def __init__( time_support : IntervalSet, optional The time support of the TsdFrame object """ - super().__init__(t, d, time_units, time_support, conv_to_array) + super().__init__(t, d, time_units, time_support, to_numpy_array) assert ( self.values.ndim >= 3 @@ -848,7 +848,7 @@ def __init__( time_units="s", time_support=None, columns=None, - conv_to_array=True, + to_numpy_array=True, ): """ TsdFrame initializer @@ -877,7 +877,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing TsdFrame" - super().__init__(t, d, time_units, time_support, conv_to_array) + super().__init__(t, d, time_units, time_support, to_numpy_array) assert self.values.ndim <= 2, "Data should be 1 or 2 dimensional." @@ -1127,7 +1127,7 @@ class Tsd(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, conv_to_array=True, **kwargs + self, t, d=None, time_units="s", time_support=None, to_numpy_array=True, **kwargs ): """ Tsd Initializer. @@ -1149,7 +1149,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing Tsd" - super().__init__(t, d, time_units, time_support, conv_to_array) + super().__init__(t, d, time_units, time_support, to_numpy_array) assert self.values.ndim == 1, "Data should be 1 dimensional" diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index 271dc76a..d1f75208 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -148,7 +148,7 @@ def _make_tsd(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.Tsd(t=t, d=d, conv_to_array=False) + data = nap.Tsd(t=t, d=d, to_numpy_array=False) return data @@ -173,7 +173,7 @@ def _make_tsd_tensor(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.TsdTensor(t=t, d=d, conv_to_array=False) + data = nap.TsdTensor(t=t, d=d, to_numpy_array=False) return data @@ -232,7 +232,7 @@ def _make_tsd_frame(obj): else: columns = np.arange(obj.data.shape[1]) - data = nap.TsdFrame(t=t, d=d, columns=columns, conv_to_array=False) + data = nap.TsdFrame(t=t, d=d, columns=columns, to_numpy_array=False) return data diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 0788c857..8f90f3da 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -22,7 +22,7 @@ def test_lazy_load_hdf5_is_array(time, data, expectation): f.create_dataset('data', data=data) h5_data = h5py.File(file_path, 'r')["data"] with expectation: - nap.Tsd(t=time, d=h5_data, conv_to_array=False) + nap.Tsd(t=time, d=h5_data, to_numpy_array=False) finally: # delete file if file_path.exists(): @@ -43,7 +43,7 @@ def test_lazy_load_hdf5_is_array(time, data, convert_flag): f.create_dataset('data', data=data) # get the tsd h5_data = h5py.File(file_path, 'r')["data"] - tsd = nap.Tsd(t=time, d=h5_data, conv_to_array=convert_flag) + tsd = nap.Tsd(t=time, d=h5_data, to_numpy_array=convert_flag) if convert_flag: assert isinstance(tsd.d, np.ndarray) else: @@ -70,7 +70,7 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - res = func(cls(t=time, d=h5_data, conv_to_array=False)) + res = func(cls(t=time, d=h5_data, to_numpy_array=False)) assert isinstance(res, cls) assert isinstance(res.d, np.ndarray) finally: @@ -107,7 +107,7 @@ def test_lazy_load_hdf5_apply_method(time, data, method_name, args, cls): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = cls(t=time, d=h5_data, conv_to_array=False) + tsd = cls(t=time, d=h5_data, to_numpy_array=False) func = getattr(tsd, method_name) out = func(*args) assert isinstance(out.d, np.ndarray) @@ -135,7 +135,7 @@ def test_lazy_load_hdf5_apply_method_tsd_specific(time, data, method_name, args, # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.Tsd(t=time, d=h5_data, conv_to_array=False) + tsd = nap.Tsd(t=time, d=h5_data, to_numpy_array=False) func = getattr(tsd, method_name) assert isinstance(func(*args), expected_out_type) finally: @@ -159,7 +159,7 @@ def test_lazy_load_hdf5_apply_method_tsdframe_specific(time, data, method_name, # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.TsdFrame(t=time, d=h5_data, conv_to_array=False) + tsd = nap.TsdFrame(t=time, d=h5_data, to_numpy_array=False) func = getattr(tsd, method_name) assert isinstance(func(*args), expected_out_type) finally: @@ -177,7 +177,7 @@ def test_lazy_load_hdf5_tsdframe_loc(): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.TsdFrame(t=np.arange(data.shape[0]), d=h5_data, conv_to_array=False).loc[1] + tsd = nap.TsdFrame(t=np.arange(data.shape[0]), d=h5_data, to_numpy_array=False).loc[1] assert isinstance(tsd, nap.Tsd) assert all(tsd.d == np.array([1, 3, 5, 7, 9])) From 183a95102d140ba17d5586a5b7444baf30cb60bc Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:18:49 -0400 Subject: [PATCH 37/47] improved docstrings --- pynapple/core/time_series.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 7fe0023a..62d551f0 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -228,29 +228,31 @@ def __array_function__(self, func, types, args, kwargs): def as_array(self): """ - Return the data as a numpy.ndarray + Return the data. Returns ------- - out: numpy.ndarray + out: array-like _ """ return self.values def data(self): """ - Return the data as a numpy.ndarray + Return the data. Returns ------- - out: numpy.ndarray + out: array-like _ """ return self.values def to_numpy(self): """ - Return the data as a numpy.ndarray. Mostly useful for matplotlib plotting when calling `plot(tsd)` + Return the data as a numpy.ndarray. + + Mostly useful for matplotlib plotting when calling `plot(tsd)`. """ return np.asarray(self.values) From 57b9a1edac749bfcbe7d495d6ad2f52972d2f5df Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:21:38 -0400 Subject: [PATCH 38/47] renamed flag --- pynapple/core/time_series.py | 16 ++++++++-------- pynapple/io/interface_nwb.py | 6 +++--- tests/test_lazy_loading.py | 14 +++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 62d551f0..41b2f77b 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -68,10 +68,10 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): Implement most of the shared functions across concrete classes `Tsd`, `TsdFrame`, `TsdTensor` """ - def __init__(self, t, d, time_units="s", time_support=None, to_numpy_array=True): + def __init__(self, t, d, time_units="s", time_support=None, load_array=True): super().__init__(t, time_units, time_support) - if to_numpy_array: + if load_array: self.values = convert_to_array(d, "d") else: if not is_array_like(d): @@ -673,7 +673,7 @@ class TsdTensor(BaseTsd): """ def __init__( - self, t, d, time_units="s", time_support=None, to_numpy_array=True, **kwargs + self, t, d, time_units="s", time_support=None, load_array=True, **kwargs ): """ TsdTensor initializer @@ -689,7 +689,7 @@ def __init__( time_support : IntervalSet, optional The time support of the TsdFrame object """ - super().__init__(t, d, time_units, time_support, to_numpy_array) + super().__init__(t, d, time_units, time_support, load_array) assert ( self.values.ndim >= 3 @@ -850,7 +850,7 @@ def __init__( time_units="s", time_support=None, columns=None, - to_numpy_array=True, + load_array=True, ): """ TsdFrame initializer @@ -879,7 +879,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing TsdFrame" - super().__init__(t, d, time_units, time_support, to_numpy_array) + super().__init__(t, d, time_units, time_support, load_array) assert self.values.ndim <= 2, "Data should be 1 or 2 dimensional." @@ -1129,7 +1129,7 @@ class Tsd(BaseTsd): """ def __init__( - self, t, d=None, time_units="s", time_support=None, to_numpy_array=True, **kwargs + self, t, d=None, time_units="s", time_support=None, load_array=True, **kwargs ): """ Tsd Initializer. @@ -1151,7 +1151,7 @@ def __init__( else: assert d is not None, "Missing argument d when initializing Tsd" - super().__init__(t, d, time_units, time_support, to_numpy_array) + super().__init__(t, d, time_units, time_support, load_array) assert self.values.ndim == 1, "Data should be 1 dimensional" diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index d1f75208..496f7368 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -148,7 +148,7 @@ def _make_tsd(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.Tsd(t=t, d=d, to_numpy_array=False) + data = nap.Tsd(t=t, d=d, load_array=False) return data @@ -173,7 +173,7 @@ def _make_tsd_tensor(obj): else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.TsdTensor(t=t, d=d, to_numpy_array=False) + data = nap.TsdTensor(t=t, d=d, load_array=False) return data @@ -232,7 +232,7 @@ def _make_tsd_frame(obj): else: columns = np.arange(obj.data.shape[1]) - data = nap.TsdFrame(t=t, d=d, columns=columns, to_numpy_array=False) + data = nap.TsdFrame(t=t, d=d, columns=columns, load_array=False) return data diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 8f90f3da..723a6035 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -22,7 +22,7 @@ def test_lazy_load_hdf5_is_array(time, data, expectation): f.create_dataset('data', data=data) h5_data = h5py.File(file_path, 'r')["data"] with expectation: - nap.Tsd(t=time, d=h5_data, to_numpy_array=False) + nap.Tsd(t=time, d=h5_data, load_array=False) finally: # delete file if file_path.exists(): @@ -43,7 +43,7 @@ def test_lazy_load_hdf5_is_array(time, data, convert_flag): f.create_dataset('data', data=data) # get the tsd h5_data = h5py.File(file_path, 'r')["data"] - tsd = nap.Tsd(t=time, d=h5_data, to_numpy_array=convert_flag) + tsd = nap.Tsd(t=time, d=h5_data, load_array=convert_flag) if convert_flag: assert isinstance(tsd.d, np.ndarray) else: @@ -70,7 +70,7 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - res = func(cls(t=time, d=h5_data, to_numpy_array=False)) + res = func(cls(t=time, d=h5_data, load_array=False)) assert isinstance(res, cls) assert isinstance(res.d, np.ndarray) finally: @@ -107,7 +107,7 @@ def test_lazy_load_hdf5_apply_method(time, data, method_name, args, cls): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = cls(t=time, d=h5_data, to_numpy_array=False) + tsd = cls(t=time, d=h5_data, load_array=False) func = getattr(tsd, method_name) out = func(*args) assert isinstance(out.d, np.ndarray) @@ -135,7 +135,7 @@ def test_lazy_load_hdf5_apply_method_tsd_specific(time, data, method_name, args, # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.Tsd(t=time, d=h5_data, to_numpy_array=False) + tsd = nap.Tsd(t=time, d=h5_data, load_array=False) func = getattr(tsd, method_name) assert isinstance(func(*args), expected_out_type) finally: @@ -159,7 +159,7 @@ def test_lazy_load_hdf5_apply_method_tsdframe_specific(time, data, method_name, # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.TsdFrame(t=time, d=h5_data, to_numpy_array=False) + tsd = nap.TsdFrame(t=time, d=h5_data, load_array=False) func = getattr(tsd, method_name) assert isinstance(func(*args), expected_out_type) finally: @@ -177,7 +177,7 @@ def test_lazy_load_hdf5_tsdframe_loc(): # get the tsd h5_data = h5py.File(file_path, 'r')["data"] # lazy load and apply function - tsd = nap.TsdFrame(t=np.arange(data.shape[0]), d=h5_data, to_numpy_array=False).loc[1] + tsd = nap.TsdFrame(t=np.arange(data.shape[0]), d=h5_data, load_array=False).loc[1] assert isinstance(tsd, nap.Tsd) assert all(tsd.d == np.array([1, 3, 5, 7, 9])) From 3b02d2708c59f5216efabcc50b6f412043df70c8 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:39:21 -0400 Subject: [PATCH 39/47] added a lazy load flag to load nwb --- pynapple/io/interface_nwb.py | 43 +++++++++++++++++++++++++++--------- tests/test_lazy_loading.py | 16 +++++++++----- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index 496f7368..a450369c 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -72,13 +72,15 @@ def _extract_compatible_data_from_nwbfile(nwbfile): return data -def _make_interval_set(obj): +def _make_interval_set(obj, lazy_loading=True): """Helper function to make IntervalSet Parameters ---------- obj : pynwb.epoch.TimeIntervals NWB object + lazy_loading: bool + If True return a memory-view of the data, load otherwise. Returns ------- @@ -128,13 +130,15 @@ def _make_interval_set(obj): return obj -def _make_tsd(obj): +def _make_tsd(obj, lazy_loading=True): """Helper function to make Tsd Parameters ---------- obj : pynwb.misc.TimeSeries NWB object + lazy_loading: bool + If True return a memory-view of the data, load otherwise. Returns ------- @@ -143,23 +147,28 @@ def _make_tsd(obj): """ d = obj.data + if not lazy_loading: + d = d[:] + if obj.timestamps is not None: t = obj.timestamps[:] else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.Tsd(t=t, d=d, load_array=False) + data = nap.Tsd(t=t, d=d, load_array=not lazy_loading) return data -def _make_tsd_tensor(obj): +def _make_tsd_tensor(obj, lazy_loading=True): """Helper function to make TsdTensor Parameters ---------- obj : pynwb.misc.TimeSeries NWB object + lazy_loading: bool + If True return a memory-view of the data, load otherwise. Returns ------- @@ -168,23 +177,28 @@ def _make_tsd_tensor(obj): """ d = obj.data + if not lazy_loading: + d = d[:] + if obj.timestamps is not None: t = obj.timestamps else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate - data = nap.TsdTensor(t=t, d=d, load_array=False) + data = nap.TsdTensor(t=t, d=d, load_array=not lazy_loading) return data -def _make_tsd_frame(obj): +def _make_tsd_frame(obj, lazy_loading=True): """Helper function to make TsdFrame Parameters ---------- obj : pynwb.misc.TimeSeries NWB object + lazy_loading: bool + If True return a memory-view of the data, load otherwise. Returns ------- @@ -193,6 +207,9 @@ def _make_tsd_frame(obj): """ d = obj.data + if not lazy_loading: + d = d[:] + if obj.timestamps is not None: t = obj.timestamps else: @@ -232,12 +249,12 @@ def _make_tsd_frame(obj): else: columns = np.arange(obj.data.shape[1]) - data = nap.TsdFrame(t=t, d=d, columns=columns, load_array=False) + data = nap.TsdFrame(t=t, d=d, columns=columns, load_array=not lazy_loading) return data -def _make_tsgroup(obj): +def _make_tsgroup(obj, **kwargs): """Helper function to make TsGroup Parameters @@ -301,7 +318,7 @@ def _make_tsgroup(obj): return tsgroup -def _make_ts(obj): +def _make_ts(obj, **kwargs): """Helper function to make Ts Parameters @@ -355,12 +372,14 @@ class NWBFile(UserDict): "TsGroup": _make_tsgroup, } - def __init__(self, file): + def __init__(self, file, lazy_loading=True): """ Parameters ---------- file : str or pynwb.file.NWBFile Valid file to a NWB file + lazy_loading: bool + If True return a memory-view of the data, load otherwise. Raises ------ @@ -391,6 +410,8 @@ def __init__(self, file): self._view = [[k, self.data[k]["type"]] for k in self.data.keys()] + self._lazy_loading = lazy_loading + UserDict.__init__(self, self.data) def __str__(self): @@ -443,7 +464,7 @@ def __getitem__(self, key): if isinstance(self.data[key], dict) and "id" in self.data[key]: obj = self.nwb.objects[self.data[key]["id"]] try: - data = self._f_eval[self.data[key]["type"]](obj) + data = self._f_eval[self.data[key]["type"]](obj, lazy_loading=self._lazy_loading) except Exception: warnings.warn( "Failed to build {}.\n Returning the NWB object for manual inspection".format( diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 723a6035..44cf00c0 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -186,13 +186,19 @@ def test_lazy_load_hdf5_tsdframe_loc(): if file_path.exists(): file_path.unlink() - -def test_lazy_load_nwb(): +@pytest.mark.parametrize( + "lazy, expected_type", + [ + (True, h5py.Dataset), + (False, np.ndarray), + ] +) +def test_lazy_load_nwb(lazy, expected_type): try: - nwb = nap.NWBFile("tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb") + nwb = nap.NWBFile("tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb", lazy_loading=lazy) except: - nwb = nap.NWBFile("nwbfilestest/basic/pynapplenwb/A2929-200711.nwb") + nwb = nap.NWBFile("nwbfilestest/basic/pynapplenwb/A2929-200711.nwb", lazy_loading=lazy) tsd = nwb["z"] - assert isinstance(tsd.d, h5py.Dataset) + assert isinstance(tsd.d, expected_type) nwb.io.close() From acf8f449151bc2f2b734ca24645d20e1e631ef79 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:42:59 -0400 Subject: [PATCH 40/47] fix on interval set --- pynapple/io/interface_nwb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index a450369c..623c7f78 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -72,15 +72,13 @@ def _extract_compatible_data_from_nwbfile(nwbfile): return data -def _make_interval_set(obj, lazy_loading=True): +def _make_interval_set(obj, **kwargs): """Helper function to make IntervalSet Parameters ---------- obj : pynwb.epoch.TimeIntervals NWB object - lazy_loading: bool - If True return a memory-view of the data, load otherwise. Returns ------- From 4100806c9e358deddede055ff7e09d3ce5162d95 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 14:46:07 -0400 Subject: [PATCH 41/47] linted --- pynapple/io/interface_nwb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index 623c7f78..a98f9106 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -462,7 +462,9 @@ def __getitem__(self, key): if isinstance(self.data[key], dict) and "id" in self.data[key]: obj = self.nwb.objects[self.data[key]["id"]] try: - data = self._f_eval[self.data[key]["type"]](obj, lazy_loading=self._lazy_loading) + data = self._f_eval[self.data[key]["type"]]( + obj, lazy_loading=self._lazy_loading + ) except Exception: warnings.warn( "Failed to build {}.\n Returning the NWB object for manual inspection".format( From f05f574ec4d3660342e7e1887f37ab4d95e0941a Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Tue, 21 May 2024 15:36:37 -0400 Subject: [PATCH 42/47] Fixing t loading --- pynapple/io/interface_nwb.py | 8 ++++---- pynapple/process/_process_functions.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pynapple/io/interface_nwb.py b/pynapple/io/interface_nwb.py index a98f9106..70f33956 100644 --- a/pynapple/io/interface_nwb.py +++ b/pynapple/io/interface_nwb.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # @Author: Guillaume Viejo # @Date: 2023-08-01 11:54:45 -# @Last Modified by: gviejo -# @Last Modified time: 2023-10-19 12:16:55 +# @Last Modified by: Guillaume Viejo +# @Last Modified time: 2024-05-21 15:28:27 """ Pynapple class to interface with NWB files. @@ -179,7 +179,7 @@ def _make_tsd_tensor(obj, lazy_loading=True): d = d[:] if obj.timestamps is not None: - t = obj.timestamps + t = obj.timestamps[:] else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate @@ -209,7 +209,7 @@ def _make_tsd_frame(obj, lazy_loading=True): d = d[:] if obj.timestamps is not None: - t = obj.timestamps + t = obj.timestamps[:] else: t = obj.starting_time + np.arange(obj.num_samples) / obj.rate diff --git a/pynapple/process/_process_functions.py b/pynapple/process/_process_functions.py index 0713d26c..9d9ea5ff 100644 --- a/pynapple/process/_process_functions.py +++ b/pynapple/process/_process_functions.py @@ -190,7 +190,7 @@ def _perievent_trigger_average( time_target_array, count_array, time_array, - data_array, + data_array[:], starts, ends, windows, @@ -204,7 +204,7 @@ def _perievent_trigger_average( time_target_array, count_array, time_array, - np.expand_dims(data_array, -1), + np.expand_dims(data_array[:], -1), starts, ends, windows, @@ -216,7 +216,7 @@ def _perievent_trigger_average( time_target_array, count_array, time_array, - data_array, + data_array[:], starts, ends, windows, From b82858138e7392df4c82c08cf9fd04f5e3644bfe Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 15:47:52 -0400 Subject: [PATCH 43/47] added a test for catching warnings in different lazy-loadings --- tests/test_lazy_loading.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 44cf00c0..c56798cf 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -1,3 +1,5 @@ +import os.path + import h5py import pandas as pd @@ -202,3 +204,26 @@ def test_lazy_load_nwb(lazy, expected_type): tsd = nwb["z"] assert isinstance(tsd.d, expected_type) nwb.io.close() + + +@pytest.mark.parametrize( + "path, var_name", + [ + ("phy/pynapplenwb/A8604-211122.nwb", "units"), # TsGroup + ("basic/pynapplenwb/A2929-200711.nwb", "z"), # Tsd + ("suite2p/pynapplenwb/2022_08_08.nwb", "Neuropil"), # TsdFrame + ("suite2p/pynapplenwb/2022_08_08.nwb", "TwoPhotonSeries") # TsdTensor + ] +) +def test_lazy_load_nwb_no_warnings(path, var_name): + try: + nwb = nap.NWBFile(os.path.join("tests/nwbfilestest", path), lazy_loading=True) + except: + nwb = nap.NWBFile(os.path.join("nwbfilestest", path), lazy_loading=True) + + with does_not_raise(): + tsd = nwb[var_name] + if isinstance(tsd, (nap.Tsd, nap.TsdFrame, nap.TsdTensor)): + tsd * 2 + + nwb.io.close() From 6db739071409193a27bdd566072f411ca406698f Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 15:53:15 -0400 Subject: [PATCH 44/47] test for wanrns --- tests/test_lazy_loading.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index c56798cf..6ba7e55e 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -8,7 +8,7 @@ import pytest from contextlib import nullcontext as does_not_raise from pathlib import Path - +import warnings @pytest.mark.parametrize( "time, data, expectation", @@ -212,7 +212,6 @@ def test_lazy_load_nwb(lazy, expected_type): ("phy/pynapplenwb/A8604-211122.nwb", "units"), # TsGroup ("basic/pynapplenwb/A2929-200711.nwb", "z"), # Tsd ("suite2p/pynapplenwb/2022_08_08.nwb", "Neuropil"), # TsdFrame - ("suite2p/pynapplenwb/2022_08_08.nwb", "TwoPhotonSeries") # TsdTensor ] ) def test_lazy_load_nwb_no_warnings(path, var_name): @@ -221,9 +220,11 @@ def test_lazy_load_nwb_no_warnings(path, var_name): except: nwb = nap.NWBFile(os.path.join("nwbfilestest", path), lazy_loading=True) - with does_not_raise(): + with warnings.catch_warnings(): + warnings.simplefilter("error") tsd = nwb[var_name] if isinstance(tsd, (nap.Tsd, nap.TsdFrame, nap.TsdTensor)): tsd * 2 + nwb.io.close() From efaad745eec950100231ef6294bde7df47cbaff8 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 15:54:36 -0400 Subject: [PATCH 45/47] linted --- tests/test_lazy_loading.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 6ba7e55e..a5eb97f3 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -1,14 +1,15 @@ import os.path +import warnings +from contextlib import nullcontext as does_not_raise +from pathlib import Path import h5py +import numpy as np import pandas as pd +import pytest import pynapple as nap -import numpy as np -import pytest -from contextlib import nullcontext as does_not_raise -from pathlib import Path -import warnings + @pytest.mark.parametrize( "time, data, expectation", @@ -226,5 +227,4 @@ def test_lazy_load_nwb_no_warnings(path, var_name): if isinstance(tsd, (nap.Tsd, nap.TsdFrame, nap.TsdTensor)): tsd * 2 - nwb.io.close() From 7aa66aa825f7a975a7f5db63234e4b8aca638dbc Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Wed, 22 May 2024 12:10:54 -0400 Subject: [PATCH 46/47] added test tsgroup --- tests/test_lazy_loading.py | 63 +++++++++++++++++++++++++++----------- tests/test_nwb.py | 8 ++--- 2 files changed, 49 insertions(+), 22 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index a5eb97f3..edd16e30 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -7,6 +7,8 @@ import numpy as np import pandas as pd import pytest +from pynwb.testing.mock.base import mock_TimeSeries +from pynwb.testing.mock.file import mock_NWBFile import pynapple as nap @@ -207,24 +209,49 @@ def test_lazy_load_nwb(lazy, expected_type): nwb.io.close() -@pytest.mark.parametrize( - "path, var_name", - [ - ("phy/pynapplenwb/A8604-211122.nwb", "units"), # TsGroup - ("basic/pynapplenwb/A2929-200711.nwb", "z"), # Tsd - ("suite2p/pynapplenwb/2022_08_08.nwb", "Neuropil"), # TsdFrame - ] -) -def test_lazy_load_nwb_no_warnings(path, var_name): +@pytest.mark.parametrize("data", [np.ones(10), np.ones((10, 2)), np.ones((10, 2, 2))]) +def test_lazy_load_nwb_no_warnings(data): + file_path = Path('data.h5') + try: - nwb = nap.NWBFile(os.path.join("tests/nwbfilestest", path), lazy_loading=True) - except: - nwb = nap.NWBFile(os.path.join("nwbfilestest", path), lazy_loading=True) + with h5py.File(file_path, 'w') as f: + f.create_dataset('data', data=data) + time_series = mock_TimeSeries(name="TimeSeries", data=f["data"]) + nwbfile = mock_NWBFile() + nwbfile.add_acquisition(time_series) + nwb = nap.NWBFile(nwbfile) - with warnings.catch_warnings(): - warnings.simplefilter("error") - tsd = nwb[var_name] - if isinstance(tsd, (nap.Tsd, nap.TsdFrame, nap.TsdTensor)): - tsd * 2 + with warnings.catch_warnings(): + warnings.simplefilter("error") + tsd = nwb["TimeSeries"] + tsd.count(0.1) + assert isinstance(tsd.d, h5py.Dataset) - nwb.io.close() + finally: + if file_path.exists(): + file_path.unlink() + + +def test_tsgroup_no_warinings(): + n_units = 2 + try: + for k in range(n_units): + file_path = Path(f'data_{k}.h5') + with h5py.File(file_path, 'w') as f: + f.create_dataset('spks', data=np.sort(np.random.uniform(0, 10, size=20))) + with warnings.catch_warnings(): + nwbfile = mock_NWBFile() + + for k in range(n_units): + file_path = Path(f'data_{k}.h5') + spike_times = h5py.File(file_path, "r")['spks'] + nwbfile.add_unit(spike_times=spike_times) + nwb = nap.NWBFile(nwbfile) + warnings.simplefilter("error") + tsgroup = nwb["units"] + tsgroup.count(0.1) + finally: + for k in range(n_units): + file_path = Path(f'data_{k}.h5') + if file_path.exists(): + file_path.unlink() diff --git a/tests/test_nwb.py b/tests/test_nwb.py index bf3558e8..943726e9 100644 --- a/tests/test_nwb.py +++ b/tests/test_nwb.py @@ -519,11 +519,11 @@ def test_add_Units(): nwbfile.add_unit(spike_times=spike_times, quality="good", alpha=alpha[n_units_per_shank]) spks[n_units_per_shank] = spike_times - nwb = nap.NWBFile(nwbfile) - assert len(nwb) == 1 - assert "units" in nwb.keys() + nwb_tsgroup = nap.NWBFile(nwbfile) + assert len(nwb_tsgroup) == 1 + assert "units" in nwb_tsgroup.keys() - data = nwb['units'] + data = nwb_tsgroup['units'] assert isinstance(data, nap.TsGroup) assert len(data) == n_units for n in data.keys(): From 1e82780103f1f1e73f13cd8e02d698b8fb9b9d43 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Wed, 22 May 2024 12:19:04 -0400 Subject: [PATCH 47/47] Testings test --- tests/test_lazy_loading.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index edd16e30..9618e52e 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -232,7 +232,7 @@ def test_lazy_load_nwb_no_warnings(data): file_path.unlink() -def test_tsgroup_no_warinings(): +def test_tsgroup_no_warnings(): n_units = 2 try: for k in range(n_units): @@ -240,16 +240,19 @@ def test_tsgroup_no_warinings(): with h5py.File(file_path, 'w') as f: f.create_dataset('spks', data=np.sort(np.random.uniform(0, 10, size=20))) with warnings.catch_warnings(): + warnings.simplefilter("error") + nwbfile = mock_NWBFile() for k in range(n_units): file_path = Path(f'data_{k}.h5') spike_times = h5py.File(file_path, "r")['spks'] nwbfile.add_unit(spike_times=spike_times) - nwb = nap.NWBFile(nwbfile) - warnings.simplefilter("error") + + nwb = nap.NWBFile(nwbfile) tsgroup = nwb["units"] tsgroup.count(0.1) + finally: for k in range(n_units): file_path = Path(f'data_{k}.h5')