From e83737cd2b6762ffdf16e654518b537fa153511b Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 09:50:02 -0400 Subject: [PATCH 01/17] overview --- README.md | 35 +++++++++++++++++++++++++---------- docs/index.md | 34 +++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 75bd0809..e547c7d1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,28 @@ pynapple is a light-weight python library for neurophysiological data analysis. New release :fire: ------------------ +### pynapple >= 0.6.3 + +Starting from 0.6.3 you can access, modify, and filter the `TsGroup` metadata columns more flexibly. + +1. **Accessing Metadata Columns:** +You can access a metadata column with the following syntax, +```python +metadata = tsgroup.column_name +``` + +2. **Adding New Metadata:** +To add a new metadata column, use the following syntax, +```python +tsgroup["new_column"] = metadata +``` + +3. **Filtering Metadata:** +You can filter TsGroup instances based on metadata values using boolean indexing:, +```python +tsgroup[tsgroup.column == value] +``` + ### pynapple >= 0.6 Starting with 0.6, [`IntervalSet`](https://pynapple-org.github.io/pynapple/reference/core/interval_set/) objects are behaving as immutable numpy ndarray. Before 0.6, you could select an interval within an `IntervalSet` object with: @@ -47,18 +69,11 @@ new_intervalset = intervalset[0] See the [documentation](https://pynapple-org.github.io/pynapple/reference/core/interval_set/) for more details. - ### pynapple >= 0.4 -Starting with 0.4, pynapple rely on the [numpy array container](https://numpy.org/doc/stable/user/basics.dispatch.html) approach instead of Pandas for the time series. Pynapple builtin functions will remain the same except for functions inherited from Pandas. Typically this line of code in `pynapple<=0.3.6` : -```python -meantsd = tsdframe.mean(1) -``` -is now : -```python -meantsd = np.mean(tsdframe, 1) -``` -in `pynapple>=0.4.0`. This allows for a better handling of returned objects. +Starting with 0.4, pynapple rely on the [numpy array container](https://numpy.org/doc/stable/user/basics.dispatch.html) approach instead of Pandas for the time series. Pynapple builtin functions will remain the same except for functions inherited from Pandas. + +This allows for a better handling of returned objects. Additionaly, it is now possible to define time series objects with more than 2 dimensions with `TsdTensor`. You can also look at this [notebook](https://pynapple-org.github.io/pynapple/generated/gallery/tutorial_pynapple_numpy/) for a demonstration of numpy compatibilities. diff --git a/docs/index.md b/docs/index.md index 769461fb..82ea1150 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,6 +30,28 @@ To ask any questions or get support for using pynapple, please consider joining New releases :fire: ------------------ +### pynapple >= 0.6.3 + +Starting from 0.6.3 you can access, modify, and filter the `TsGroup` metadata columns more flexibly. + +1. **Accessing Metadata Columns:** +You can access a metadata column with the following syntax, +```python +metadata = tsgroup.column_name +``` + +2. **Adding New Metadata:** +To add a new metadata column, use the following syntax, +```python +tsgroup["new_column"] = metadata +``` + +3. **Filtering Metadata:** +You can filter TsGroup instances based on metadata values using boolean indexing:, +```python +tsgroup[tsgroup.column == value] +``` + ### pynapple >= 0.6 Starting with 0.6, [`IntervalSet`](https://pynapple-org.github.io/pynapple/reference/core/interval_set/) objects are behaving as immutable numpy ndarray. Before 0.6, you could select an interval within an `IntervalSet` object with: @@ -48,15 +70,9 @@ See the [documentation](https://pynapple-org.github.io/pynapple/reference/core/i ### pynapple >= 0.4 -Starting with 0.4, pynapple rely on the [numpy array container](https://numpy.org/doc/stable/user/basics.dispatch.html) approach instead of Pandas for the time series. Pynapple builtin functions will remain the same except for functions inherited from Pandas. Typically this line of code in `pynapple<=0.3.6` : -```python -meantsd = tsdframe.mean(1) -``` -is now : -```python -meantsd = np.mean(tsdframe, 1) -``` -in `pynapple>=0.4.0`. This allows for a better handling of returned objects. +Starting with 0.4, pynapple rely on the [numpy array container](https://numpy.org/doc/stable/user/basics.dispatch.html) approach instead of Pandas for the time series. Pynapple builtin functions will remain the same except for functions inherited from Pandas. + +This allows for a better handling of returned objects. Additionaly, it is now possible to define time series objects with more than 2 dimensions with `TsdTensor`. You can also look at this [notebook](https://pynapple-org.github.io/pynapple/generated/gallery/tutorial_pynapple_numpy/) for a demonstration of numpy compatibilities. From a9edbcaa50240f2b08bf217c22e32574621aa827 Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Tue, 21 May 2024 09:53:06 -0400 Subject: [PATCH 02/17] fix naming --- README.md | 2 +- docs/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e547c7d1..32d82533 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ tsgroup["new_column"] = metadata 3. **Filtering Metadata:** You can filter TsGroup instances based on metadata values using boolean indexing:, ```python -tsgroup[tsgroup.column == value] +tsgroup[tsgroup.column_name == value] ``` ### pynapple >= 0.6 diff --git a/docs/index.md b/docs/index.md index 82ea1150..3716a1df 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,7 +49,7 @@ tsgroup["new_column"] = metadata 3. **Filtering Metadata:** You can filter TsGroup instances based on metadata values using boolean indexing:, ```python -tsgroup[tsgroup.column == value] +tsgroup[tsgroup.column_name == value] ``` ### pynapple >= 0.6 From e0f2b97e294ac2d1b71672483efc99f415dd691a Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Wed, 22 May 2024 17:39:15 -0400 Subject: [PATCH 03/17] Update --- README.md | 22 ---- docs/api_guide/README.md | 3 + .../tutorial_pynapple_core.py | 94 +++++++++++-- .../tutorial_pynapple_io.py | 0 .../tutorial_pynapple_numpy.py | 0 docs/api_guide/tutorial_pynapple_nwb.py | 124 ++++++++++++++++++ .../tutorial_pynapple_process.py | 0 .../tutorial_pynapple_quick_start.py | 0 docs/examples/tutorial_human_dataset.py | 2 +- docs/index.md | 22 ---- mkdocs.yml | 11 +- 11 files changed, 219 insertions(+), 59 deletions(-) create mode 100644 docs/api_guide/README.md rename docs/{examples => api_guide}/tutorial_pynapple_core.py (68%) rename docs/{examples => api_guide}/tutorial_pynapple_io.py (100%) rename docs/{examples => api_guide}/tutorial_pynapple_numpy.py (100%) create mode 100644 docs/api_guide/tutorial_pynapple_nwb.py rename docs/{examples => api_guide}/tutorial_pynapple_process.py (100%) rename docs/{examples => api_guide}/tutorial_pynapple_quick_start.py (100%) diff --git a/README.md b/README.md index 32d82533..9f7a984b 100644 --- a/README.md +++ b/README.md @@ -31,28 +31,6 @@ pynapple is a light-weight python library for neurophysiological data analysis. New release :fire: ------------------ -### pynapple >= 0.6.3 - -Starting from 0.6.3 you can access, modify, and filter the `TsGroup` metadata columns more flexibly. - -1. **Accessing Metadata Columns:** -You can access a metadata column with the following syntax, -```python -metadata = tsgroup.column_name -``` - -2. **Adding New Metadata:** -To add a new metadata column, use the following syntax, -```python -tsgroup["new_column"] = metadata -``` - -3. **Filtering Metadata:** -You can filter TsGroup instances based on metadata values using boolean indexing:, -```python -tsgroup[tsgroup.column_name == value] -``` - ### pynapple >= 0.6 Starting with 0.6, [`IntervalSet`](https://pynapple-org.github.io/pynapple/reference/core/interval_set/) objects are behaving as immutable numpy ndarray. Before 0.6, you could select an interval within an `IntervalSet` object with: diff --git a/docs/api_guide/README.md b/docs/api_guide/README.md new file mode 100644 index 00000000..3655f7be --- /dev/null +++ b/docs/api_guide/README.md @@ -0,0 +1,3 @@ +# API guide + +Guide to the `pynapple` API. \ No newline at end of file diff --git a/docs/examples/tutorial_pynapple_core.py b/docs/api_guide/tutorial_pynapple_core.py similarity index 68% rename from docs/examples/tutorial_pynapple_core.py rename to docs/api_guide/tutorial_pynapple_core.py index c9a842c3..6920a0f0 100644 --- a/docs/examples/tutorial_pynapple_core.py +++ b/docs/api_guide/tutorial_pynapple_core.py @@ -111,8 +111,14 @@ tsgroup = nap.TsGroup(my_ts) print(tsgroup, "\n") -print(tsgroup[0], "\n") # dictionary like indexing returns directly the Ts object -print(tsgroup[[0, 2]]) # list like indexing + +# %% +# Dictionary like indexing returns directly the Ts object +print(tsgroup[0], "\n") + +# %% +# List like indexing +print(tsgroup[[0, 2]]) # %% # Operations such as restrict can thus be directly applied to the TsGroup as well as other operations. @@ -126,21 +132,83 @@ print(count) # %% -# One advantage of grouping time series is that metainformation can be appended directly on an element-wise basis. In this case, we add labels to each Ts object when instantiating the group and after. We can then use this label to split the group. See the [TsGroup](https://peyrachelab.github.io/pynapple/core.ts_group/) documentation for a complete methodology for splitting TsGroup objects. - +# One advantage of grouping time series is that metainformation can be added directly on an element-wise basis. In this case, we add labels to each Ts object when instantiating the group and after. We can then use this label to split the group. See the [TsGroup](https://peyrachelab.github.io/pynapple/core.ts_group/) documentation for a complete methodology for splitting TsGroup objects. +# +# First we create a pandas Series for the label. label1 = pd.Series(index=list(my_ts.keys()), data=[0, 1, 0]) -tsgroup = nap.TsGroup(my_ts, time_units="s", label1=label1) -tsgroup.set_info(label2=np.array(["a", "a", "b"])) +print(label1) -print(tsgroup, "\n") +# %% +# We can pass `label1` at the initialization step. + +tsgroup = nap.TsGroup(my_ts, time_units="s", my_label1=label1) + +print(tsgroup) + +# %% +# Notice how the label has been added as one column when printing `tsgroup`. +# +# We can also add a label for each items in 2 different ways after initializing the `TsGroup` object. +# First with `set_info` : +tsgroup.set_info(my_label2=np.array(["a", "a", "b"])) + +print(tsgroup) + +# %% +# Notice that you can pass directly a numpy array as long as it is the same size as the `TsGroup`. +# +# We can also add new metadata by passing it as an item of the dictionnary. +tsgroup["my_label3"] = np.random.randn(len(tsgroup)) + +print(tsgroup) + +# %% +# Metadata columns can be viewed as attributes of `TsGroup`. + +tsgroup.my_label1 + +# %% +# or with the `get_info` method. -newtsgroup = tsgroup.getby_category("label1") -print(newtsgroup[0], "\n") -print(newtsgroup[1]) +tsgroup.get_info("my_label3") +# %% +# Finally you can use the metadata to slice through the `TsGroup` object. +# +# There are multiple methods for it. You can use the `TsGroup` getter functions : +# +# - `getby_category(col_name)` : categorized the metadata column and return a `TsGroup` for each category. +# +# - `getby_threshold(col_name, value)` : threshold the metadata column and return a single `TsGroup`. +# +# - `getby_intervals(col_name, bins)` : digitize the metadata column and return a `TsGroup` for each bin. +# +# In this example we categorized `tsgroup` with `my_label2`. + +dict_of_tsgroup = tsgroup.getby_category("my_label2") + +print(dict_of_tsgroup["a"], "\n") +print(dict_of_tsgroup["b"]) + +# %% +# Notice that `getby_threshold` return directly a TsGroup. + +tsgroup.getby_threshold("my_label1", 0.5) + +# %% +# Similar operations can be performed using directly the attributes of `TsGroup`. +# For example, the previous line is equivalent to : + +tsgroup[tsgroup.my_label1>0.5] + +# %% +# You can also chain queries with attributes. + +tsgroup[(tsgroup.my_label1==0) & (tsgroup.my_label2=="a")] + # %% # *** # Time support @@ -165,11 +233,15 @@ tsgroup_with_time_support = nap.TsGroup(my_ts, time_support=time_support) +# %% print(tsgroup, "\n") +# %% print(tsgroup_with_time_support, "\n") -print(tsgroup_with_time_support.time_support) # acceding the time support +# %% +# acceding the time support is an important feature of pynapple +print(tsgroup_with_time_support.time_support) # %% # We can use value_from which as it indicates assign to every timestamps the closed value in time from another time series. diff --git a/docs/examples/tutorial_pynapple_io.py b/docs/api_guide/tutorial_pynapple_io.py similarity index 100% rename from docs/examples/tutorial_pynapple_io.py rename to docs/api_guide/tutorial_pynapple_io.py diff --git a/docs/examples/tutorial_pynapple_numpy.py b/docs/api_guide/tutorial_pynapple_numpy.py similarity index 100% rename from docs/examples/tutorial_pynapple_numpy.py rename to docs/api_guide/tutorial_pynapple_numpy.py diff --git a/docs/api_guide/tutorial_pynapple_nwb.py b/docs/api_guide/tutorial_pynapple_nwb.py new file mode 100644 index 00000000..f60e959e --- /dev/null +++ b/docs/api_guide/tutorial_pynapple_nwb.py @@ -0,0 +1,124 @@ +# coding: utf-8 +""" +# NWB & Lazy-loading + +Pynapple currently provides loaders for two data formats : + + - `npz` with a special structure. You can check this [notebook](../tutorial_pynapple_io) for a descrition of the methods for saving/loading `npz` files. + + - [NWB format](https://pynwb.readthedocs.io/en/stable/index.html#) + +This notebook focuses on the NWB format. Additionaly it demonstrates the capabilities of pynapple for lazy-loading different formats. + + +The dataset in this example can be found [here](https://www.dropbox.com/s/pr1ze1nuiwk8kw9/MyProject.zip?dl=1). +""" +# %% +# !!! warning +# This tutorial uses seaborn and matplotlib for displaying the figure. +# +# You can install both with `pip install matplotlib seaborn` + +import numpy as np +import pynapple as nap + +# %% +# # NWB +# +# When loading a NWB file, pynapple will walk through it and test the compatibility of each data structure with a pynapple objects. If the data structure is incompatible, pynapple will ignore it. The class that deals with reading NWB file is [`nap.NWBFile`](../../../reference/io/interface_nwb/). You can pass the path to a NWB file or directly an opened NWB file. Alternatively you can use the function [`nap.load_file`](../../../reference/io/misc/#pynapple.io.misc.load_file). +# +# +# !!! warning +# Creating the NWB file is outside the scope of pynapple. The NWB file used here has already been created before. +# Multiple tools exists to create NWB file automatically. You can check [neuroconv](https://neuroconv.readthedocs.io/en/main/), [NWBGuide](https://nwb-guide.readthedocs.io/en/latest/) or even [NWBmatic](https://github.com/pynapple-org/nwbmatic). + + +data = nap.load_file("../../your/path/to/MyProject/sub-A2929/A2929-200711/pynapplenwb/A2929-200711.nwb") + +print(data) + +# %% +# Pynapple will give you a table with all the entries of the NWB file that are compatible with a pynapple object. +# When parsing the NWB file, nothing is loaded. The `NWBFile` keeps track of the position of the data whithin the NWB file with a key. You can see it with the attributes `key_to_id`. + +data.key_to_id + + +# %% +#Loading an entry will get pynapple to read the data. + +z = data['z'] + +print(z) + +# %% +# Internally, the `NWBClass` has replaced the pointer to the data with the actual data. +# +# While it looks like pynapple has loaded the data, in fact it did not. By default, calling the NWB object will return an HDF5 dataset. + +print(type(z.values)) + +# %% +# Notice that the time array is always loaded. + +print(type(z.index.values)) + +# %% +# This is very useful in the case of large dataset that do not fit in memory. You can then get a chunk of the data that will actually be loaded. + +z_chunk = z.get(670, 680) # getting 10s of data. + +print(z_chunk) + +# %% +# Data are now loaded. + +print(type(z_chunk.values)) + + +# %% +# To change this behavior, you can pass `lazy_loading=False` when instantiating the `NWBClass`. +path = "../../your/path/to/MyProject/sub-A2929/A2929-200711/pynapplenwb/A2929-200711.nwb" +data = nap.NWBFile(path, lazy_loading=False) + +z = data['z'] + +print(type(z.d)) + + +# %% +# # Numpy memory map +# In fact, pynapple can work with any type of memory map. Here we read a binary file with `np.memmap`. + +eeg_path = "../../your/path/to/MyProject/sub-A2929/A2929-200711/A2929-200711.eeg" +frequency = 1250 # Hz +n_channels = 16 +f = open(eeg_path, 'rb') +startoffile = f.seek(0, 0) +endoffile = f.seek(0, 2) +f.close() +bytes_size = 2 +n_samples = int((endoffile-startoffile)/n_channels/bytes_size) +duration = n_samples/frequency +interval = 1/frequency + +fp = np.memmap(eeg_path, np.int16, 'r', shape = (n_samples, n_channels)) +timestep = np.arange(0, n_samples)/frequency + +print(type(fp)) + +# %% +# Instantiating a pynapple `TsdFrame` with `load_array=False` will not load the data into memory + +eeg = nap.TsdFrame(t=timestep, d=fp) + +print(eeg) + +# %% +# We can check the type of `eeg.values`. + +print(type(eeg.values)) + + +# %% +# # Zarr / h5py diff --git a/docs/examples/tutorial_pynapple_process.py b/docs/api_guide/tutorial_pynapple_process.py similarity index 100% rename from docs/examples/tutorial_pynapple_process.py rename to docs/api_guide/tutorial_pynapple_process.py diff --git a/docs/examples/tutorial_pynapple_quick_start.py b/docs/api_guide/tutorial_pynapple_quick_start.py similarity index 100% rename from docs/examples/tutorial_pynapple_quick_start.py rename to docs/api_guide/tutorial_pynapple_quick_start.py diff --git a/docs/examples/tutorial_human_dataset.py b/docs/examples/tutorial_human_dataset.py index c9b0a720..f84cbef6 100644 --- a/docs/examples/tutorial_human_dataset.py +++ b/docs/examples/tutorial_human_dataset.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -Zheng et al (2022) Dataset Tutorial +Zheng et al (2022) Tutorial ============ This tutorial demonstrates how we use Pynapple on various publicly available datasets in systems neuroscience to streamline analysis. In this tutorial, we will examine the dataset from [Zheng et al (2022)](https://www.nature.com/articles/s41593-022-01020-w), which was used to generate Figure 4c in the [publication](https://elifesciences.org/reviewed-preprints/85786). diff --git a/docs/index.md b/docs/index.md index 3716a1df..0df91139 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,28 +30,6 @@ To ask any questions or get support for using pynapple, please consider joining New releases :fire: ------------------ -### pynapple >= 0.6.3 - -Starting from 0.6.3 you can access, modify, and filter the `TsGroup` metadata columns more flexibly. - -1. **Accessing Metadata Columns:** -You can access a metadata column with the following syntax, -```python -metadata = tsgroup.column_name -``` - -2. **Adding New Metadata:** -To add a new metadata column, use the following syntax, -```python -tsgroup["new_column"] = metadata -``` - -3. **Filtering Metadata:** -You can filter TsGroup instances based on metadata values using boolean indexing:, -```python -tsgroup[tsgroup.column_name == value] -``` - ### pynapple >= 0.6 Starting with 0.6, [`IntervalSet`](https://pynapple-org.github.io/pynapple/reference/core/interval_set/) objects are behaving as immutable numpy ndarray. Before 0.6, you could select an interval within an `IntervalSet` object with: diff --git a/mkdocs.yml b/mkdocs.yml index 2660731c..063d2549 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,8 +13,12 @@ theme: plugins: - search - gallery: - examples_dirs: docs/examples - gallery_dirs: docs/generated/gallery + examples_dirs: + - docs/api_guide + - docs/examples + gallery_dirs: + - docs/generated/api_guide + - docs/generated/examples conf_script: docs/gallery_conf.py - gen-files: scripts: @@ -35,7 +39,8 @@ plugins: nav: - Overview: index.md - - Usage: generated/gallery + - Application guide: generated/api_guide + - Analysis examples: generated/examples - External projects: external.md - Pynajax - GPU acceleration: pynajax.md - Modules : reference/ From 65b6aca99abdcdf2b9e0b07cdca4ee71864b6a06 Mon Sep 17 00:00:00 2001 From: gviejo Date: Wed, 22 May 2024 23:15:03 -0400 Subject: [PATCH 04/17] Update --- docs/api_guide/tutorial_pynapple_nwb.py | 77 +++++++++++++++++++++---- pynapple/core/_jitted_functions.py | 10 ++-- pynapple/core/time_series.py | 11 ++-- pyproject.toml | 1 + 4 files changed, 77 insertions(+), 22 deletions(-) diff --git a/docs/api_guide/tutorial_pynapple_nwb.py b/docs/api_guide/tutorial_pynapple_nwb.py index f60e959e..4c4de2f9 100644 --- a/docs/api_guide/tutorial_pynapple_nwb.py +++ b/docs/api_guide/tutorial_pynapple_nwb.py @@ -14,21 +14,18 @@ The dataset in this example can be found [here](https://www.dropbox.com/s/pr1ze1nuiwk8kw9/MyProject.zip?dl=1). """ # %% -# !!! warning -# This tutorial uses seaborn and matplotlib for displaying the figure. # -# You can install both with `pip install matplotlib seaborn` import numpy as np import pynapple as nap # %% -# # NWB -# +# NWB +# -------------- # When loading a NWB file, pynapple will walk through it and test the compatibility of each data structure with a pynapple objects. If the data structure is incompatible, pynapple will ignore it. The class that deals with reading NWB file is [`nap.NWBFile`](../../../reference/io/interface_nwb/). You can pass the path to a NWB file or directly an opened NWB file. Alternatively you can use the function [`nap.load_file`](../../../reference/io/misc/#pynapple.io.misc.load_file). # # -# !!! warning +# !!! note # Creating the NWB file is outside the scope of pynapple. The NWB file used here has already been created before. # Multiple tools exists to create NWB file automatically. You can check [neuroconv](https://neuroconv.readthedocs.io/en/main/), [NWBGuide](https://nwb-guide.readthedocs.io/en/latest/) or even [NWBmatic](https://github.com/pynapple-org/nwbmatic). @@ -45,11 +42,11 @@ # %% -#Loading an entry will get pynapple to read the data. +# Loading an entry will get pynapple to read the data. z = data['z'] -print(z) +print(data['z']) # %% # Internally, the `NWBClass` has replaced the pointer to the data with the actual data. @@ -75,6 +72,13 @@ print(type(z_chunk.values)) +# %% +# You can still apply any high level function of pynapple. For example here, we compute some tuning curves without preloading the dataset. + +tc = nap.compute_1d_tuning_curves(data['units'], data['y'], 10) + +print(tc) + # %% # To change this behavior, you can pass `lazy_loading=False` when instantiating the `NWBClass`. @@ -87,8 +91,10 @@ # %% -# # Numpy memory map -# In fact, pynapple can work with any type of memory map. Here we read a binary file with `np.memmap`. +# Numpy memory map +# ---------------- +# +# In fact, pynapple can work with any type of memory map. Here we read a binary file with [`np.memmap`](https://numpy.org/doc/stable/reference/generated/numpy.memmap.html). eeg_path = "../../your/path/to/MyProject/sub-A2929/A2929-200711/A2929-200711.eeg" frequency = 1250 # Hz @@ -108,7 +114,7 @@ print(type(fp)) # %% -# Instantiating a pynapple `TsdFrame` with `load_array=False` will not load the data into memory +# Instantiating a pynapple `TsdFrame` will keep the data as a memory map. eeg = nap.TsdFrame(t=timestep, d=fp) @@ -121,4 +127,51 @@ # %% -# # Zarr / h5py +# Zarr +# -------------- +# +# It is also possible to use Higher level library like [zarr](https://zarr.readthedocs.io/en/stable/index.html) also not directly. + +import zarr +data = zarr.zeros((10000, 5), chunks=(1000, 5), dtype='i4') +timestep = np.arange(len(data)) + +tsdframe = nap.TsdFrame(t=timestep, d=data) + +# %% +# As the warning suggest, `data` is converted to numpy array. + +print(type(tsdframe.d)) + +# %% +# To maintain a zarr array, you can change the argument `load_array` to False. + +tsdframe = nap.TsdFrame(t=timestep, d=data, load_array=False) + +print(type(tsdframe.d)) + +# %% +# Within pynapple, numpy memory map are recognized as numpy array while zarr array are not. + +print(type(fp), "Is np.ndarray? ", isinstance(fp, np.ndarray)) +print(type(data), "Is np.ndarray? ", isinstance(data, np.ndarray)) + + +# %% +# Similar to numpy memory map, you can use pynapple functions directly. + +ep = nap.IntervalSet(0, 10) +tsdframe.restrict(ep) + +# %% +group = nap.TsGroup({0:nap.Ts(t=[10, 20, 30])}) + +sta = nap.compute_event_trigger_average(group, tsdframe, 1, (-2, 3)) + +print(type(tsdframe.values)) +print("\n") +print(sta) + +# %% +# !!! warning +# Carefulness should still apply when calling any pynapple function on a memory map. Pynapple does not implement any batching function internally. Calling a high level function of pynapple on a dataset that do not fit in memory will likely cause a memory error. \ No newline at end of file diff --git a/pynapple/core/_jitted_functions.py b/pynapple/core/_jitted_functions.py index 4de37635..c0849ab5 100644 --- a/pynapple/core/_jitted_functions.py +++ b/pynapple/core/_jitted_functions.py @@ -9,10 +9,11 @@ def jitrestrict(time_array, starts, ends): n = len(time_array) m = len(starts) - ix = np.zeros(n, dtype=np.bool_) + ix = np.zeros(n, dtype="int") k = 0 t = 0 + x = 0 while ends[k] < time_array[t]: k += 1 @@ -21,8 +22,6 @@ def jitrestrict(time_array, starts, ends): # Outside while t < n: if time_array[t] >= starts[k]: - # ix[t] = True - # t += 1 break t += 1 @@ -32,7 +31,8 @@ def jitrestrict(time_array, starts, ends): k += 1 break else: - ix[t] = True + ix[x] = t + x += 1 t += 1 if k == m: @@ -40,7 +40,7 @@ def jitrestrict(time_array, starts, ends): if t == n: break - return ix + return ix[0:x] @jit(nopython=True) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 41b2f77b..0f7ae38c 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -71,14 +71,15 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): def __init__(self, t, d, time_units="s", time_support=None, load_array=True): super().__init__(t, time_units, time_support) + if not is_array_like(d): + raise TypeError( + "Data should be array-like, i.e. be indexable, iterable and, have attributes " + "`shape`, `ndim` and, `dtype`)." + ) + if load_array: self.values = convert_to_array(d, "d") else: - if not is_array_like(d): - raise TypeError( - "Data should be array-like, i.e. be indexable, iterable and, have attributes " - "`shape`, `ndim` and, `dtype`)." - ) self.values = d assert len(self.index) == len( diff --git a/pyproject.toml b/pyproject.toml index ad0d99f1..1eada622 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ docs = [ "mkdocs-material", "matplotlib", "seaborn", + "zarr" ] dandi = [ "dandi", # Dandi package From b3aa862456b1d6a0e86ceb0df8d6d1a34b188605 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 10:36:28 -0400 Subject: [PATCH 05/17] fix tests --- pynapple/core/time_series.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 0f7ae38c..41b2f77b 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -71,15 +71,14 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): def __init__(self, t, d, time_units="s", time_support=None, load_array=True): super().__init__(t, time_units, time_support) - if not is_array_like(d): - raise TypeError( - "Data should be array-like, i.e. be indexable, iterable and, have attributes " - "`shape`, `ndim` and, `dtype`)." - ) - if load_array: self.values = convert_to_array(d, "d") else: + if not is_array_like(d): + raise TypeError( + "Data should be array-like, i.e. be indexable, iterable and, have attributes " + "`shape`, `ndim` and, `dtype`)." + ) self.values = d assert len(self.index) == len( From 8f328f45ad220d9937c7f40ed4c08a163e9b7ca0 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 11:06:23 -0400 Subject: [PATCH 06/17] fix --- pynapple/core/time_series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 41b2f77b..8ecfe78f 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -71,7 +71,7 @@ class BaseTsd(Base, NDArrayOperatorsMixin, abc.ABC): def __init__(self, t, d, time_units="s", time_support=None, load_array=True): super().__init__(t, time_units, time_support) - if load_array: + if load_array or isinstance(d, np.ndarray): self.values = convert_to_array(d, "d") else: if not is_array_like(d): From 730aa31eb17da954e424adf35b006ccd585b025a Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 12:42:39 -0400 Subject: [PATCH 07/17] Update --- pynapple/core/_jitted_functions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pynapple/core/_jitted_functions.py b/pynapple/core/_jitted_functions.py index c0849ab5..f8487733 100644 --- a/pynapple/core/_jitted_functions.py +++ b/pynapple/core/_jitted_functions.py @@ -9,7 +9,7 @@ def jitrestrict(time_array, starts, ends): n = len(time_array) m = len(starts) - ix = np.zeros(n, dtype="int") + ix = np.zeros(n, dtype=np.int64) k = 0 t = 0 @@ -47,11 +47,12 @@ def jitrestrict(time_array, starts, ends): def jitrestrict_with_count(time_array, starts, ends): n = len(time_array) m = len(starts) - ix = np.zeros(n, dtype=np.bool_) + ix = np.zeros(n, dtype=np.int64) count = np.zeros(m, dtype=np.int64) k = 0 t = 0 + x = 0 while ends[k] < time_array[t]: k += 1 @@ -60,9 +61,6 @@ def jitrestrict_with_count(time_array, starts, ends): # Outside while t < n: if time_array[t] >= starts[k]: - # ix[t] = True - # count[k] += 1 - # t += 1 break t += 1 @@ -72,8 +70,9 @@ def jitrestrict_with_count(time_array, starts, ends): k += 1 break else: - ix[t] = True + ix[x] = t count[k] += 1 + x += 1 t += 1 if k == m: @@ -81,7 +80,7 @@ def jitrestrict_with_count(time_array, starts, ends): if t == n: break - return ix, count + return ix[0:x], count @jit(nopython=True) @@ -160,6 +159,7 @@ def jitcount(time_array, starts, ends, bin_size): break lbound += bin_size + lbound = np.round(lbound, 9) b += 1 t = maxt k += 1 From 53670e8c924ba4dd1b6469001f3c261f3113e45d Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 14:06:46 -0400 Subject: [PATCH 08/17] Update --- pynapple/core/_core_functions.py | 6 +++--- pynapple/core/_jitted_functions.py | 1 + pynapple/core/time_series.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pynapple/core/_core_functions.py b/pynapple/core/_core_functions.py index 6a47fdb4..a7f67d4d 100644 --- a/pynapple/core/_core_functions.py +++ b/pynapple/core/_core_functions.py @@ -74,16 +74,16 @@ def _dropna(time_array, data_array, starts, ends, update_time_support, ndim): ends, ) elif np.any(index_nan): + tokeep = np.where(~index_nan)[0] if update_time_support: starts, ends = jitremove_nan(time_array, index_nan) to_fix = starts == ends if np.any(to_fix): ends[to_fix] += 1e-6 # adding 1 millisecond in case of a single point - - return (time_array[~index_nan], data_array[~index_nan], starts, ends) + return (time_array[tokeep], data_array[tokeep], starts, ends) else: - return (time_array[~index_nan], data_array[~index_nan], starts, ends) + return (time_array[tokeep], data_array[tokeep], starts, ends) else: return (time_array, data_array, starts, ends) diff --git a/pynapple/core/_jitted_functions.py b/pynapple/core/_jitted_functions.py index f8487733..669343c6 100644 --- a/pynapple/core/_jitted_functions.py +++ b/pynapple/core/_jitted_functions.py @@ -363,6 +363,7 @@ def _jitbin_array(countin, time_array, data_array, starts, ends, bin_size): break lbound += bin_size + lbound = np.round(lbound, 9) b += 1 t = maxt k += 1 diff --git a/pynapple/core/time_series.py b/pynapple/core/time_series.py index 8ecfe78f..95ae2c37 100644 --- a/pynapple/core/time_series.py +++ b/pynapple/core/time_series.py @@ -688,6 +688,10 @@ def __init__( The time units in which times are specified ('us', 'ms', 's' [default]). time_support : IntervalSet, optional The time support of the TsdFrame object + load_array : bool, optional + Whether the data should be converted to a numpy (or jax) array. Useful when passing a memory map object like zarr. + Default is True. Does not apply if `d` is already a numpy array. + """ super().__init__(t, d, time_units, time_support, load_array) @@ -868,6 +872,9 @@ def __init__( The time support of the TsdFrame object columns : iterables Column names + load_array : bool, optional + Whether the data should be converted to a numpy (or jax) array. Useful when passing a memory map object like zarr. + Default is True. Does not apply if `d` is already a numpy array. """ c = columns @@ -1144,6 +1151,9 @@ def __init__( The time units in which times are specified ('us', 'ms', 's' [default]) time_support : IntervalSet, optional The time support of the tsd object + load_array : bool, optional + Whether the data should be converted to a numpy (or jax) array. Useful when passing a memory map object like zarr. + Default is True. Does not apply if `d` is already a numpy array. """ if isinstance(t, pd.Series): d = t.values From 9d479884593a4bf1ac321dfb951befa167e540e2 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 14:12:50 -0400 Subject: [PATCH 09/17] update --- docs/AUTHORS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/AUTHORS.md b/docs/AUTHORS.md index abf78fae..6d9ff76d 100644 --- a/docs/AUTHORS.md +++ b/docs/AUTHORS.md @@ -4,15 +4,15 @@ Credits Development Lead ---------------- -- Guillaume Viejo +- Guillaume Viejo + Contributors ------------ +- Edoardo Balzani - Adrien Peyrache - Dan Levenstein - Sofia Skromne Carrasco -- Sara Mahallati -- Gilberto Vite - Davide Spalla - Luigi Petrucco \ No newline at end of file From dc13316e02ce89e012f583abecc86d56af439c8d Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 14:22:30 -0400 Subject: [PATCH 10/17] history --- docs/HISTORY.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/HISTORY.md b/docs/HISTORY.md index 34a575c8..aa672c3b 100644 --- a/docs/HISTORY.md +++ b/docs/HISTORY.md @@ -8,6 +8,12 @@ Around 2016-2017, Luke Sjulson started *TSToolbox2*, still in Matlab and which i In 2018, Francesco started neuroseries, a Python package built on Pandas. It was quickly adopted in Adrien's lab, especially by Guillaume Viejo, a postdoc in the lab. Gradually, the majority of the lab was using it and new functions were constantly added. In 2021, Guillaume and other trainees in Adrien's lab decided to fork from neuroseries and started *pynapple*. The core of pynapple is largely built upon neuroseries. Some of the original changes to TSToolbox made by Luke were included in this package, especially the *time_support* property of all ts/tsd objects. +0.6.6 (Soon) +------------------ + +- Full lazy-loading for NWB file. +- Parameter `load_array` for time series can prevent loading zarr array +- Function to merge a list of `TsGroup` 0.6.5 (2024-05-14) ------------------ From 608cd07a13d021acb94ce38fc59d1850e6c1cbeb Mon Sep 17 00:00:00 2001 From: BalzaniEdoardo Date: Thu, 23 May 2024 15:56:34 -0400 Subject: [PATCH 11/17] grammar fix --- docs/api_guide/tutorial_pynapple_core.py | 4 ++-- docs/api_guide/tutorial_pynapple_io.py | 2 +- docs/api_guide/tutorial_pynapple_quick_start.py | 4 ++-- pynapple/core/ts_group.py | 4 ++-- pynapple/io/folder.py | 6 +++--- pynapple/io/interface_npz.py | 2 +- pynapple/io/misc.py | 2 +- pynapple/process/decoding.py | 2 +- pynapple/process/perievent.py | 4 ++-- pynapple/process/tuning_curves.py | 12 ++++++------ 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/api_guide/tutorial_pynapple_core.py b/docs/api_guide/tutorial_pynapple_core.py index 6920a0f0..1cc63c7a 100644 --- a/docs/api_guide/tutorial_pynapple_core.py +++ b/docs/api_guide/tutorial_pynapple_core.py @@ -159,7 +159,7 @@ # %% # Notice that you can pass directly a numpy array as long as it is the same size as the `TsGroup`. # -# We can also add new metadata by passing it as an item of the dictionnary. +# We can also add new metadata by passing it as an item of the dictionary. tsgroup["my_label3"] = np.random.randn(len(tsgroup)) print(tsgroup) @@ -224,7 +224,7 @@ my_ts = { 0: nap.Ts( t=np.sort(np.random.uniform(0, 100, 10)), time_units="s" - ), # here a simple dictionnary + ), # here a simple dictionary 1: nap.Ts(t=np.sort(np.random.uniform(0, 100, 20)), time_units="s"), 2: nap.Ts(t=np.sort(np.random.uniform(0, 100, 30)), time_units="s"), } diff --git a/docs/api_guide/tutorial_pynapple_io.py b/docs/api_guide/tutorial_pynapple_io.py index 17f4dc72..3b0a65ae 100644 --- a/docs/api_guide/tutorial_pynapple_io.py +++ b/docs/api_guide/tutorial_pynapple_io.py @@ -47,7 +47,7 @@ # %% # Here it shows all the subjects (in this case only A2929), all the sessions and all of the derivatives folders. It shows as well all the NPZ files that contains a pynapple object and the NWB files. # -# The object project behaves like a nested dictionnary. It is then easy to loop and navigate through a hierarchy of folders when doing analyses. In this case, we are gonna take only the session A2929-200711. +# The object project behaves like a nested dictionary. It is then easy to loop and navigate through a hierarchy of folders when doing analyses. In this case, we are gonna take only the session A2929-200711. session = project["sub-A2929"]["A2929-200711"] diff --git a/docs/api_guide/tutorial_pynapple_quick_start.py b/docs/api_guide/tutorial_pynapple_quick_start.py index 2d2ecb4f..f4f664c0 100644 --- a/docs/api_guide/tutorial_pynapple_quick_start.py +++ b/docs/api_guide/tutorial_pynapple_quick_start.py @@ -67,7 +67,7 @@ print(spikes) # %% -# In this case, the TsGroup holds 15 neurons and it is possible to access, similar to a dictionnary, the spike times of a single neuron: +# In this case, the TsGroup holds 15 neurons and it is possible to access, similar to a dictionary, the spike times of a single neuron: neuron_0 = spikes[0] print(neuron_0) @@ -75,7 +75,7 @@ # `neuron_0` is a [Ts](https://pynapple-org.github.io/pynapple/core.time_series/#pynapple.core.time_series.Ts) object containing the times of the spikes. # %% -# The other information about the session is contained in `nwb["epochs"]`. In this case, the start and end of the sleep and wake epochs. If the NWB time intervals contains tags of the epochs, pynapple will try to group them together and return a dictionnary of IntervalSet instead of IntervalSet. +# The other information about the session is contained in `nwb["epochs"]`. In this case, the start and end of the sleep and wake epochs. If the NWB time intervals contains tags of the epochs, pynapple will try to group them together and return a dictionary of IntervalSet instead of IntervalSet. epochs = nwb["epochs"] print(epochs) diff --git a/pynapple/core/ts_group.py b/pynapple/core/ts_group.py index b4865d6d..6c052733 100644 --- a/pynapple/core/ts_group.py +++ b/pynapple/core/ts_group.py @@ -58,7 +58,7 @@ def _union_intervals(i_sets): class TsGroup(UserDict): """ - The TsGroup is a dictionnary-like object to hold multiple [`Ts`][pynapple.core.time_series.Ts] or [`Tsd`][pynapple.core.time_series.Tsd] objects with different time index. + The TsGroup is a dictionary-like object to hold multiple [`Ts`][pynapple.core.time_series.Ts] or [`Tsd`][pynapple.core.time_series.Tsd] objects with different time index. Attributes ---------- @@ -995,7 +995,7 @@ def getby_category(self, key): Returns ------- dict - A dictionnary of TsGroup + A dictionary of TsGroup Examples -------- diff --git a/pynapple/io/folder.py b/pynapple/io/folder.py index c3b7972f..cead9f8c 100644 --- a/pynapple/io/folder.py +++ b/pynapple/io/folder.py @@ -86,7 +86,7 @@ class Folder(UserDict): Attributes ---------- data : dict - Dictionnary holidng all the pynapple objects found in the folder. + dictionary holidng all the pynapple objects found in the folder. name : str Name of the folder npz_files : list @@ -96,7 +96,7 @@ class Folder(UserDict): path : str Absolute path of the folder subfolds : dict - Dictionnary of all the subfolders + dictionary of all the subfolders """ @@ -170,7 +170,7 @@ def __getitem__(self, key): Raises ------ KeyError - If key is not in the dictionnary + If key is not in the dictionary """ if key.__hash__: if self.__contains__(key): diff --git a/pynapple/io/interface_npz.py b/pynapple/io/interface_npz.py index 05b070f2..4795e0b2 100644 --- a/pynapple/io/interface_npz.py +++ b/pynapple/io/interface_npz.py @@ -9,7 +9,7 @@ """ File classes help to validate and load pynapple objects or NWB files. Data are always lazy-loaded. -Both classes behaves like dictionnary. +Both classes behaves like dictionary. """ import os diff --git a/pynapple/io/misc.py b/pynapple/io/misc.py index 1bec674f..36c9614f 100644 --- a/pynapple/io/misc.py +++ b/pynapple/io/misc.py @@ -69,7 +69,7 @@ def load_folder(path): Returns ------- Folder - A dictionnary-like class containing all the sub-folders and compatible files (i.e. npz, nwb) + A dictionary-like class containing all the sub-folders and compatible files (i.e. npz, nwb) Raises ------ diff --git a/pynapple/process/decoding.py b/pynapple/process/decoding.py index e05c6257..de031455 100644 --- a/pynapple/process/decoding.py +++ b/pynapple/process/decoding.py @@ -128,7 +128,7 @@ def decode_2d(tuning_curves, group, ep, bin_size, xy, time_units="s", features=N tuning_curves : dict Dictionnay of 2d tuning curves (one for each neuron). group : TsGroup or dict of Ts/Tsd object. - A group of neurons with the same keys as tuning_curves dictionnary. + A group of neurons with the same keys as tuning_curves dictionary. ep : IntervalSet The epoch on which decoding is computed bin_size : float diff --git a/pynapple/process/perievent.py b/pynapple/process/perievent.py index f6210c6a..a3dbb5d1 100644 --- a/pynapple/process/perievent.py +++ b/pynapple/process/perievent.py @@ -63,7 +63,7 @@ def compute_perievent(data, tref, minmax, time_unit="s"): data : Ts, Tsd or TsGroup The data to align to tref. If Ts/Tsd, returns a TsGroup. - If TsGroup, returns a dictionnary of TsGroup + If TsGroup, returns a dictionary of TsGroup tref : Ts or Tsd The timestamps of the event to align to minmax : tuple, int or float @@ -75,7 +75,7 @@ def compute_perievent(data, tref, minmax, time_unit="s"): ------- dict A TsGroup if data is a Ts/Tsd or - a dictionnary of TsGroup if data is a TsGroup. + a dictionary of TsGroup if data is a TsGroup. Raises ------ diff --git a/pynapple/process/tuning_curves.py b/pynapple/process/tuning_curves.py index af9153af..56b5d7fc 100644 --- a/pynapple/process/tuning_curves.py +++ b/pynapple/process/tuning_curves.py @@ -16,12 +16,12 @@ def compute_discrete_tuning_curves(group, dict_ep): """ - Compute discrete tuning curves of a TsGroup using a dictionnary of epochs. - The function returns a pandas DataFrame with each row being a key of the dictionnary of epochs + Compute discrete tuning curves of a TsGroup using a dictionary of epochs. + The function returns a pandas DataFrame with each row being a key of the dictionary of epochs and each column being a neurons. This function can typically being used for a set of stimulus being presented for multiple epochs. - An example of the dictionnary is : + An example of the dictionary is : >>> dict_ep = { "stim0": nap.IntervalSet(start=0, end=1), @@ -53,7 +53,7 @@ def compute_discrete_tuning_curves(group, dict_ep): If group is not a TsGroup object. """ assert isinstance(group, nap.TsGroup), "group should be a TsGroup." - assert isinstance(dict_ep, dict), "dict_ep should be a dictionnary of IntervalSet" + assert isinstance(dict_ep, dict), "dict_ep should be a dictionary of IntervalSet" idx = np.sort(list(dict_ep.keys())) for k in idx: assert isinstance( @@ -169,7 +169,7 @@ def compute_2d_tuning_curves(group, features, nb_bins, ep=None, minmax=None): ------- tuple A tuple containing: \n - tc (dict): Dictionnary of the tuning curves with dimensions (nb_bins, nb_bins).\n + tc (dict): dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n xy (list): List of bins center in the two dimensions Raises @@ -487,7 +487,7 @@ def compute_2d_tuning_curves_continuous( ------- tuple A tuple containing: \n - tc (dict): Dictionnary of the tuning curves with dimensions (nb_bins, nb_bins).\n + tc (dict): dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n xy (list): List of bins center in the two dimensions Raises From 62d06363bb5df0f1b41991605ca7507319d8f581 Mon Sep 17 00:00:00 2001 From: Edoardo Balzani Date: Thu, 23 May 2024 15:59:06 -0400 Subject: [PATCH 12/17] Update pynapple/process/tuning_curves.py --- pynapple/process/tuning_curves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/process/tuning_curves.py b/pynapple/process/tuning_curves.py index 56b5d7fc..16b25762 100644 --- a/pynapple/process/tuning_curves.py +++ b/pynapple/process/tuning_curves.py @@ -487,7 +487,7 @@ def compute_2d_tuning_curves_continuous( ------- tuple A tuple containing: \n - tc (dict): dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n + tc (dict): Dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n xy (list): List of bins center in the two dimensions Raises From be789191b4dc19f63f247eefc7d83aea40f05149 Mon Sep 17 00:00:00 2001 From: Edoardo Balzani Date: Thu, 23 May 2024 15:59:12 -0400 Subject: [PATCH 13/17] Update pynapple/io/folder.py --- pynapple/io/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/io/folder.py b/pynapple/io/folder.py index cead9f8c..d054be82 100644 --- a/pynapple/io/folder.py +++ b/pynapple/io/folder.py @@ -96,7 +96,7 @@ class Folder(UserDict): path : str Absolute path of the folder subfolds : dict - dictionary of all the subfolders + Dictionary of all the subfolders """ From 684f8cdbfc21b984e3c3e24f28faf5e099c303e4 Mon Sep 17 00:00:00 2001 From: Edoardo Balzani Date: Thu, 23 May 2024 15:59:19 -0400 Subject: [PATCH 14/17] Update pynapple/io/folder.py --- pynapple/io/folder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/io/folder.py b/pynapple/io/folder.py index d054be82..60042bd9 100644 --- a/pynapple/io/folder.py +++ b/pynapple/io/folder.py @@ -86,7 +86,7 @@ class Folder(UserDict): Attributes ---------- data : dict - dictionary holidng all the pynapple objects found in the folder. + Dictionary holidng all the pynapple objects found in the folder. name : str Name of the folder npz_files : list From fc38f6ae573030783ea8542dbb4b4fd58db67f6f Mon Sep 17 00:00:00 2001 From: Edoardo Balzani Date: Thu, 23 May 2024 15:59:25 -0400 Subject: [PATCH 15/17] Update pynapple/process/tuning_curves.py --- pynapple/process/tuning_curves.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynapple/process/tuning_curves.py b/pynapple/process/tuning_curves.py index 16b25762..21132387 100644 --- a/pynapple/process/tuning_curves.py +++ b/pynapple/process/tuning_curves.py @@ -169,7 +169,7 @@ def compute_2d_tuning_curves(group, features, nb_bins, ep=None, minmax=None): ------- tuple A tuple containing: \n - tc (dict): dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n + tc (dict): Dictionary of the tuning curves with dimensions (nb_bins, nb_bins).\n xy (list): List of bins center in the two dimensions Raises From a164ef30613b2238a3de4397b1b9290748decb4f Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Thu, 23 May 2024 16:42:35 -0400 Subject: [PATCH 16/17] Update --- docs/api_guide/tutorial_pynapple_core.py | 2 +- docs/api_guide/tutorial_pynapple_nwb.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/api_guide/tutorial_pynapple_core.py b/docs/api_guide/tutorial_pynapple_core.py index 1cc63c7a..c91b52ec 100644 --- a/docs/api_guide/tutorial_pynapple_core.py +++ b/docs/api_guide/tutorial_pynapple_core.py @@ -159,7 +159,7 @@ # %% # Notice that you can pass directly a numpy array as long as it is the same size as the `TsGroup`. # -# We can also add new metadata by passing it as an item of the dictionary. +# We can also add new metadata by passing it as an item of the dictionary with a string key. tsgroup["my_label3"] = np.random.randn(len(tsgroup)) print(tsgroup) diff --git a/docs/api_guide/tutorial_pynapple_nwb.py b/docs/api_guide/tutorial_pynapple_nwb.py index 4c4de2f9..1e749d7c 100644 --- a/docs/api_guide/tutorial_pynapple_nwb.py +++ b/docs/api_guide/tutorial_pynapple_nwb.py @@ -79,6 +79,9 @@ print(tc) +# %% +# !!! warning +# Carefulness should still apply when calling any pynapple function on a memory map. Pynapple does not implement any batching function internally. Calling a high level function of pynapple on a dataset that do not fit in memory will likely cause a memory error. # %% # To change this behavior, you can pass `lazy_loading=False` when instantiating the `NWBClass`. @@ -171,7 +174,3 @@ print(type(tsdframe.values)) print("\n") print(sta) - -# %% -# !!! warning -# Carefulness should still apply when calling any pynapple function on a memory map. Pynapple does not implement any batching function internally. Calling a high level function of pynapple on a dataset that do not fit in memory will likely cause a memory error. \ No newline at end of file From d28cc62740a6b6b21447417045bdd55bfe05c935 Mon Sep 17 00:00:00 2001 From: Guillaume Viejo Date: Tue, 28 May 2024 12:24:54 -0400 Subject: [PATCH 17/17] Fixing lazy loading for pynajax --- tests/test_lazy_loading.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_lazy_loading.py b/tests/test_lazy_loading.py index 9618e52e..cff585d4 100644 --- a/tests/test_lazy_loading.py +++ b/tests/test_lazy_loading.py @@ -50,7 +50,7 @@ def test_lazy_load_hdf5_is_array(time, data, convert_flag): h5_data = h5py.File(file_path, 'r')["data"] tsd = nap.Tsd(t=time, d=h5_data, load_array=convert_flag) if convert_flag: - assert isinstance(tsd.d, np.ndarray) + assert not isinstance(tsd.d, h5py.Dataset) else: assert isinstance(tsd.d, h5py.Dataset) finally: @@ -77,7 +77,7 @@ def test_lazy_load_hdf5_apply_func(time, data, func,cls): # lazy load and apply function res = func(cls(t=time, d=h5_data, load_array=False)) assert isinstance(res, cls) - assert isinstance(res.d, np.ndarray) + assert not isinstance(res.d, h5py.Dataset) finally: # delete file if file_path.exists(): @@ -115,7 +115,7 @@ def test_lazy_load_hdf5_apply_method(time, data, method_name, args, cls): tsd = cls(t=time, d=h5_data, load_array=False) func = getattr(tsd, method_name) out = func(*args) - assert isinstance(out.d, np.ndarray) + assert not isinstance(out.d, h5py.Dataset) finally: # delete file if file_path.exists(): @@ -192,20 +192,23 @@ def test_lazy_load_hdf5_tsdframe_loc(): file_path.unlink() @pytest.mark.parametrize( - "lazy, expected_type", + "lazy", [ - (True, h5py.Dataset), - (False, np.ndarray), + (True), + (False), ] ) -def test_lazy_load_nwb(lazy, expected_type): +def test_lazy_load_nwb(lazy): try: nwb = nap.NWBFile("tests/nwbfilestest/basic/pynapplenwb/A2929-200711.nwb", lazy_loading=lazy) except: nwb = nap.NWBFile("nwbfilestest/basic/pynapplenwb/A2929-200711.nwb", lazy_loading=lazy) tsd = nwb["z"] - assert isinstance(tsd.d, expected_type) + if lazy: + assert isinstance(tsd.d, h5py.Dataset) + else: + assert not isinstance(tsd.d, h5py.Dataset) nwb.io.close()