From 77d1cee321c29106ca32acb5229d55526319c76d Mon Sep 17 00:00:00 2001 From: William Patton Date: Tue, 10 Dec 2024 09:29:55 -0800 Subject: [PATCH 1/4] Add regular installation from pypi instructions. Add pypi badge --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f42dd4c..5743529 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ [![tests](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/tests.yaml) [![black](https://github.com/funkelab/funlib.persistence/actions/workflows/black.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/black.yaml) [![mypy](https://github.com/funkelab/funlib.persistence/actions/workflows/mypy.yaml/badge.svg)](https://github.com/funkelab/funlib.persistence/actions/workflows/mypy.yaml) +[![pypi](https://github.com/funkelab/funlib.persistence/actions/workflows/publish.yaml/badge.svg)](https://pypi.org/project/funlib.persistence/) # funlib.persistence Interfaces for data (arrays and graphs) and storage formats (databases and file formats) # installation regular installation for usage: -`pip install .` +`pip install funlib.persistence` developer installation including pytest etc.: -`pip install ".[dev]"` +`pip install funlib.persistence[dev]` From 4b59da51017c09bf219c067eafaa28f29d4d306f Mon Sep 17 00:00:00 2001 From: William Patton Date: Tue, 10 Dec 2024 09:39:09 -0800 Subject: [PATCH 2/4] rename actions to make the badges on the readme look better --- .github/workflows/black.yaml | 2 +- .github/workflows/mypy.yaml | 2 +- .github/workflows/publish.yaml | 2 +- .github/workflows/tests.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 48a5970..c139fd0 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -1,4 +1,4 @@ -name: Python Black +name: black on: [push, pull_request] diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index 9535ed0..3f993c0 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -1,4 +1,4 @@ -name: Python mypy +name: mypy on: [push, pull_request] diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 05ba0a8..01586bc 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -1,4 +1,4 @@ -name: Publish +name: pypi on: push: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 09262d0..628620f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,4 +1,4 @@ -name: Test +name: tests on: push: From 158ecbc445a34f7e6bca4f3cd72c46c042f48c4e Mon Sep 17 00:00:00 2001 From: William Patton Date: Tue, 10 Dec 2024 09:39:58 -0800 Subject: [PATCH 3/4] fix spacing on README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5743529..1a6dc24 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ Interfaces for data (arrays and graphs) and storage formats (databases and file formats) # installation -regular installation for usage: +Regular installation for usage: `pip install funlib.persistence` -developer installation including pytest etc.: + +Developer installation including pytest etc.: `pip install funlib.persistence[dev]` From 0b670939bd1a1574c338b519ebd9495d14fa655f Mon Sep 17 00:00:00 2001 From: William Patton Date: Mon, 30 Dec 2024 12:56:16 -0800 Subject: [PATCH 4/4] fix setter failing in github actions dask arrays are not supposed to cache results from reads, so if you change the data that the dask array wraps you should get the updated data on the next read. This does not seem to be happening for dask arrays wrapping in-memory numpy arrays on some hardware. It is consistent behavior in the github actions but I cannot recreate it locally. --- funlib/persistence/arrays/array.py | 7 +++++++ tests/test_array.py | 17 +++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/funlib/persistence/arrays/array.py b/funlib/persistence/arrays/array.py index 2317e63..11761c7 100644 --- a/funlib/persistence/arrays/array.py +++ b/funlib/persistence/arrays/array.py @@ -311,6 +311,13 @@ def __setitem__(self, key, value: np.ndarray): self._source_data[region_slices] = value + # If the source data is an in-memory numpy array, writing to the numpy + # array does not always result in the dask array reading the new data. + # It seems to be a caching issue. To work around this, we create a new + # dask array from the source data. + if isinstance(self._source_data, np.ndarray): + self.data = da.from_array(self._source_data) + else: raise RuntimeError( "This array is not writeable since you have applied a custom callable " diff --git a/tests/test_array.py b/tests/test_array.py index dab2907..132b752 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -1,6 +1,6 @@ +import dask.array as da import numpy as np import pytest -import dask.array as da from funlib.geometry import Coordinate, Roi from funlib.persistence.arrays import Array @@ -73,13 +73,14 @@ def test_setitem(): a = Array(np.zeros((2, 5)), (0, 0), (1, 1)) - a[Roi((0, 0), (2, 5))] = np.arange(0, 10).reshape(2, 5) - assert a[Coordinate((0, 0))] == 0 - assert a[Coordinate((0, 1))] == 1 - assert a[Coordinate((0, 2))] == 2 - assert a[Coordinate((1, 0))] == 5 - assert a[Coordinate((1, 1))] == 6 - assert a[Coordinate((1, 4))] == 9 + data = np.arange(0, 10).reshape(2, 5) + a[Roi((0, 0), (2, 5))] = data + assert a[Coordinate((0, 0))] == a._source_data[0, 0] == 0 + assert a[Coordinate((0, 1))] == a._source_data[0, 1] == 1 + assert a[Coordinate((0, 2))] == a._source_data[0, 2] == 2 + assert a[Coordinate((1, 0))] == a._source_data[1, 0] == 5 + assert a[Coordinate((1, 1))] == a._source_data[1, 1] == 6 + assert a[Coordinate((1, 4))] == a._source_data[1, 4] == 9 # set entirely with numpy array and channels