From 3af3b962ae66b504a9e06b70fc3c876e3a985cd2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 01:45:55 -0600 Subject: [PATCH 1/7] Remove zarr.Array monkey-patches, use ndim-based collection detection Replace the __len__ and __getitem__ monkey-patches on zarr.Array with proper fixes in hdmf-zarr's own code: - Add _is_collection() using ndim for array-like objects, falling back to __len__ for plain containers (lists, tuples) - Add _get_length() using shape[0] for array-like objects, falling back to len() for plain containers - Handle 0-d ndarrays in get_type() via data.item() extraction - Replace hasattr(data, "__len__") checks with _is_collection() calls This avoids globally modifying zarr.Array behavior, which would leak as side effects to other libraries in the same process. Related: - zarr-python #3740 (__len__ removal) - zarr-python #3741 (scalar indexing returns 0-d ndarray) - hdmf #1414 (ndim-based collection detection) - hdmf #1415 (0-d ndarray handling) Co-Authored-By: Claude Opus 4.6 --- src/hdmf_zarr/backend.py | 52 +++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/src/hdmf_zarr/backend.py b/src/hdmf_zarr/backend.py index a5b1aa5f..0db9e139 100644 --- a/src/hdmf_zarr/backend.py +++ b/src/hdmf_zarr/backend.py @@ -36,24 +36,6 @@ from pathlib import Path -# zarr v3 Array does not implement __len__; add it for compatibility with array-like interfaces -if not hasattr(Array, "__len__"): - Array.__len__ = lambda self: self.shape[0] - -# zarr v3 Array scalar indexing returns 0-d ndarrays instead of numpy scalars; -# patch to match zarr v2 / numpy behavior expected by hdmf type checks -_zarr_array_original_getitem = Array.__getitem__ - - -def _zarr_array_getitem_scalar_fix(self, key): - result = _zarr_array_original_getitem(self, key) - if isinstance(result, np.ndarray) and result.ndim == 0: - return result[()] - return result - - -Array.__getitem__ = _zarr_array_getitem_scalar_fix - # Module variables ROOT_NAME = "root" @@ -1400,7 +1382,7 @@ def write_dataset(self, **kwargs): # noqa: C901 ) self._written_builders.set_written(builder) # record that the builder has been written dset.attrs["zarr_dtype"] = type_str - if hasattr(refs, "__len__") and not isinstance(refs, dict): + if self._is_collection(refs) and not isinstance(refs, dict): json_refs = [json.dumps(dict(r)) for r in refs] for i, jr in enumerate(json_refs): dset[i] = jr @@ -1414,7 +1396,7 @@ def write_dataset(self, **kwargs): # noqa: C901 elif isinstance(data, AbstractDataChunkIterator): dset = self.__setup_chunked_dataset__(parent, name, data, options) self.__dci_queue.append(dataset=dset, data=data) - elif hasattr(data, "__len__"): + elif self._is_collection(data): dset = self.__list_fill__(parent, name, data, options) else: dset = self.__scalar_fill__(parent, name, data, options) @@ -1506,13 +1488,39 @@ def get_type(cls, data): return cls.__dtypes.get("str") elif isinstance(data, bytes): return cls.__dtypes.get("bytes") - elif not hasattr(data, "__len__"): + elif isinstance(data, np.ndarray) and data.ndim == 0: + return type(data.item()) + elif not cls._is_collection(data): return type(data) else: - if len(data) == 0: + if cls._get_length(data) == 0: raise ValueError("cannot determine type for empty data") return cls.get_type(data[0]) + @staticmethod + def _is_collection(data): + """Check if data is a collection (array-like with elements) vs a scalar. + + Uses ndim for array-like objects (numpy, zarr, h5py, dask) and falls back + to __len__ for plain Python containers (list, tuple). Strings and bytes + are treated as scalars. + """ + if isinstance(data, (str, bytes)): + return False + if hasattr(data, "ndim"): + return data.ndim > 0 + return hasattr(data, "__len__") + + @staticmethod + def _get_length(data): + """Get the length of the first dimension of a collection. + + Uses shape[0] for array-like objects and len() for plain containers. + """ + if hasattr(data, "shape") and data.shape is not None: + return data.shape[0] + return len(data) + __reserve_attribute = ("zarr_dtype", "zarr_link", SPEC_LOC_ATTR) def __list_fill__(self, parent, name, data, options=None): # noqa: C901 From 44133fd4890a58d97e69be9294a330fbdb63ad5a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 02:01:58 -0600 Subject: [PATCH 2/7] Use hdmf branch with ndim-based detection and 0-d ndarray fixes Point hdmf dependency to remove_duck_typing_for_type branch which includes both PRs #1414 and #1415 needed to avoid monkey-patching. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 31f6b2a2..a2825cfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Medical Science Apps." ] dependencies = [ - "hdmf>=4.2.0", + "hdmf @ git+https://github.com/hdmf-dev/hdmf.git@remove_duck_typing_for_type", "zarr>=3.1.3", "numpy>=1.26.0", "numcodecs>=0.14.0", From 0aa91ccfe4eb163ad161670606abeccd6b70a804 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 02:05:20 -0600 Subject: [PATCH 3/7] Allow direct references in hatchling for git dependency Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a2825cfb..c5dd4139 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,9 @@ docs = [ "Homepage" = "https://github.com/hdmf-dev/hdmf-zarr" "Bug Tracker" = "https://github.com/hdmf-dev/hdmf-zarr/issues" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] source = "vcs" From 5588d67c65396c92da149f7079e6475b6683be6a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 02:20:58 -0600 Subject: [PATCH 4/7] Re-trigger CI after hdmf _unwrap_scalar fix in table.py From d39c509f56876c21f6eb415a2c9cdff7b8d26d5e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 02:41:52 -0600 Subject: [PATCH 5/7] Restore git dep for CI after hdmf fixes for len() and 0-d array iteration Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c5dd4139..d308f7d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,9 @@ docs = [ "Homepage" = "https://github.com/hdmf-dev/hdmf-zarr" "Bug Tracker" = "https://github.com/hdmf-dev/hdmf-zarr/issues" +[tool.uv.sources] +hdmf = { path = "../hdmf", editable = true } + [tool.hatch.metadata] allow-direct-references = true From 1b3f624a630dad0b86e020e8259efc2731d69c5e Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 3 Mar 2026 03:00:25 -0600 Subject: [PATCH 6/7] Point uv source to remote hdmf branch instead of local path Fixes minimum CI jobs that use uv --resolution lowest-direct, which failed because ../hdmf doesn't exist on the CI runner. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d308f7d1..6c51e865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ docs = [ "Bug Tracker" = "https://github.com/hdmf-dev/hdmf-zarr/issues" [tool.uv.sources] -hdmf = { path = "../hdmf", editable = true } +hdmf = { git = "https://github.com/hdmf-dev/hdmf.git", branch = "remove_duck_typing_for_type" } [tool.hatch.metadata] allow-direct-references = true From fde2ec488b776cc12647d0ea487c177371f799a4 Mon Sep 17 00:00:00 2001 From: rly <310197+rly@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:47:42 -0700 Subject: [PATCH 7/7] Pin hdmf to released >=6.1.0 instead of the git branch The remove_duck_typing_for_type work (ndim-based collection detection, 0-d ndarray handling, and registration of zarr.Array in the array_data docval macro) is released in hdmf 6.1.0, so depend on the release and drop the now-deleted git branch reference and the uv source / hatch direct-reference overrides that only existed to support it. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6c51e865..b4b222a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Medical Science Apps." ] dependencies = [ - "hdmf @ git+https://github.com/hdmf-dev/hdmf.git@remove_duck_typing_for_type", + "hdmf>=6.1.0", "zarr>=3.1.3", "numpy>=1.26.0", "numcodecs>=0.14.0", @@ -80,12 +80,6 @@ docs = [ "Homepage" = "https://github.com/hdmf-dev/hdmf-zarr" "Bug Tracker" = "https://github.com/hdmf-dev/hdmf-zarr/issues" -[tool.uv.sources] -hdmf = { git = "https://github.com/hdmf-dev/hdmf.git", branch = "remove_duck_typing_for_type" } - -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.version] source = "vcs"