From 7bf1eb8cae011a7534a069021289506eadcd62f3 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Fri, 31 Jan 2025 18:01:54 -0500 Subject: [PATCH] Use zarr.empty not np.empty when creating large zarr sinks np.empty actually allocates memory, zarr.empty does not. --- CHANGELOG.md | 1 + sources/zarr/large_image_source_zarr/__init__.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7784b656c..2181f042b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Bug Fixes - Fix an issue with lazy tiles that have non power of two scaling ([#1797](../../pull/1797)) +- Use zarr.empty not np.empty when creating large zarr sinks ([#1801](../../pull/1801)) ## 1.31.0 diff --git a/sources/zarr/large_image_source_zarr/__init__.py b/sources/zarr/large_image_source_zarr/__init__.py index b79ae351c..62e6c03b4 100644 --- a/sources/zarr/large_image_source_zarr/__init__.py +++ b/sources/zarr/large_image_source_zarr/__init__.py @@ -689,7 +689,9 @@ def _resizeImage(self, arr, new_shape, new_axes, chunking): arr, [(0, s - arr.shape[i]) for i, s in enumerate(new_shape)], ) - new_arr = zarr.empty(new_shape, chunks=chunking, dtype=arr.dtype) + new_arr = zarr.empty( + new_shape, chunks=chunking, dtype=arr.dtype, + write_empty_chunks=False) new_arr[:] = arr[:] arr = new_arr else: @@ -765,12 +767,22 @@ def addTile(self, tile, x=0, y=0, mask=None, axes=None, **kwargs): self._zarr_store.rmdir(path) chunking = None if store_path not in current_arrays: - arr = np.empty(tuple(new_dims.values()), dtype=tile.dtype) chunking = tuple([ self._tileSize if a in ['x', 'y'] else new_dims.get('s') if a == 's' else 1 for a in axes ]) + # If we have to create the array, do so with the desired store + # and chunking so we don't have to immediately rechunk it + arr = zarr.empty( + tuple(new_dims.values()), + dtype=tile.dtype, + chunks=chunking, + store=self._zarr_store, + path=store_path, + write_empty_chunks=False, + ) + chunking = None else: arr = current_arrays[store_path] new_shape = tuple(