From 81264128e89ae5823957204f295d953f8d34cdfa Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 4 Jul 2026 21:04:36 +0200 Subject: [PATCH 1/3] refactor: make embedded-spec read/write backend-agnostic io.spec.writeEmbeddedSpecifications, io.spec.validateEmbeddedSpecifications, and io.spec.readEmbeddedSpecifications were hardcoded to raw HDF5 file IDs (H5G.open, H5L.delete, h5info, H5F.open, H5D.open/read), even though export/read already go through the io.backend.base.Writer/Reader abstraction everywhere else. This blocked any non-HDF5 storage backend (e.g. a Zarr v3 writer) from ever completing nwbExport/nwbRead, since NwbFile.embedSpecifications passed a raw H5-typed writer.FileId into these functions regardless of the actual backend. Adds three methods to the abstract io.backend.base.Writer interface, implemented for HDF5Writer by delegating to the existing HDF5 utility functions it already wraps: - getEmbeddedSpecLocation() (mirrors the existing Reader-side method) - listChildGroupNames(groupPath) - deleteNode(nodePath) writeEmbeddedSpecifications/validateEmbeddedSpecifications now take the writer object itself and only call through this interface. readEmbeddedSpecifications now takes the reader object and is reimplemented using the already-abstract reader.readNodeInfo/ readDatasetValue (no new Reader methods needed). NwbFile.embedSpecifications and nwbRead's generateEmbeddedSpec are updated accordingly. Verified against the full existing HDF5 test suite (517/521 passing; the 4 failures are pre-existing HDF5 dynamic-filter-availability issues on this machine, unrelated to this change) plus manual round-trip checks: fresh export, edit-mode re-export (exercises the namespace-version overwrite path), missing-namespace warning, and unused-namespace deletion. Note: this does not itself add a Zarr v3 writer/reader -- it only removes the H5-specific blocker so a future non-HDF5 backend can implement getEmbeddedSpecLocation/listChildGroupNames/deleteNode on the Writer side (the Reader side already only needs readNodeInfo/readDatasetValue, which every existing Reader implementation already provides) and get full nwbExport/nwbRead support for free. Co-Authored-By: Claude Sonnet 5 --- +io/+backend/+base/Writer.m | 21 +++++++++ +io/+backend/+hdf5/HDF5Writer.m | 12 +++++ +io/+spec/readEmbeddedSpecifications.m | 45 ++++++++++--------- +io/+spec/validateEmbeddedSpecifications.m | 26 ++++++----- +io/+spec/writeEmbeddedSpecifications.m | 22 +++------ .../+unit/+io/+backend/+base/BaseWriterTest.m | 3 ++ NwbFile.m | 2 +- nwbRead.m | 8 ++-- 8 files changed, 86 insertions(+), 53 deletions(-) diff --git a/+io/+backend/+base/Writer.m b/+io/+backend/+base/Writer.m index 313546a8f..b64ee4ec7 100644 --- a/+io/+backend/+base/Writer.m +++ b/+io/+backend/+base/Writer.m @@ -33,6 +33,27 @@ function writeAttribute(obj, attributePath, value, varargin) %#ok io.backend.base.Writer.throwNotImplemented("writeAttribute") end + function specLocation = getEmbeddedSpecLocation(obj) %#ok + % getEmbeddedSpecLocation - Return the location of embedded schema + % specifications, or '' if none are embedded yet. Mirrors + % io.backend.base.Reader.getEmbeddedSpecLocation for the write + % side (needed when editing a file that may already embed specs). + specLocation = ''; + io.backend.base.Writer.throwNotImplemented("getEmbeddedSpecLocation") + end + + function groupNames = listChildGroupNames(obj, groupPath) %#ok + % listChildGroupNames - Return the names of immediate child + % groups (not datasets) under groupPath. + groupNames = {}; + io.backend.base.Writer.throwNotImplemented("listChildGroupNames") + end + + function deleteNode(obj, nodePath) %#ok + % deleteNode - Delete the group or dataset at nodePath. + io.backend.base.Writer.throwNotImplemented("deleteNode") + end + function close(obj) %#ok % Default no-op. Concrete backends can override when they own % resources that should be released explicitly. diff --git a/+io/+backend/+hdf5/HDF5Writer.m b/+io/+backend/+hdf5/HDF5Writer.m index 02a7f0db0..19c56b938 100644 --- a/+io/+backend/+hdf5/HDF5Writer.m +++ b/+io/+backend/+hdf5/HDF5Writer.m @@ -82,6 +82,18 @@ function writeValue(obj, datasetPath, value, varargin) function writeAttribute(obj, attributePath, value, varargin) io.writeAttribute(obj.H5FileId, attributePath, value, varargin{:}); end + + function specLocation = getEmbeddedSpecLocation(obj) + specLocation = io.spec.internal.readEmbeddedSpecLocation(obj.H5FileId); + end + + function groupNames = listChildGroupNames(obj, groupPath) + groupNames = io.internal.h5.listGroupNames(obj.H5FileId, groupPath); + end + + function deleteNode(obj, nodePath) + io.internal.h5.deleteGroup(obj.H5FileId, nodePath); + end end methods (Access = protected) diff --git a/+io/+spec/readEmbeddedSpecifications.m b/+io/+spec/readEmbeddedSpecifications.m index 919e2fc9e..062ec8427 100644 --- a/+io/+spec/readEmbeddedSpecifications.m +++ b/+io/+spec/readEmbeddedSpecifications.m @@ -1,12 +1,15 @@ -function specs = readEmbeddedSpecifications(filename, specLocation) +function specs = readEmbeddedSpecifications(reader, specLocation) % readEmbeddedSpecifications - Read embedded specs from an NWB file % -% specs = io.spec.readEmbeddedSpecifications(filename, specLocation) read -% embedded specs from the specLocation in an NWB file +% specs = io.spec.readEmbeddedSpecifications(reader, specLocation) reads +% embedded specs from the specLocation in an NWB file, using reader +% (an io.backend.base.Reader) to access the file. Backend-agnostic: +% only uses the io.backend.base.Reader interface, so this works for +% any registered storage backend, not just HDF5. % % Inputs: -% filename (string) : Absolute path of an nwb file -% specLocation (string) : h5 path for the location of specs inside the NWB file +% reader (io.backend.base.Reader) : Reader for the NWB file +% specLocation (string) : Path for the location of specs inside the NWB file % % Outputs % specs cell: A cell array of structs with one element for each embedded @@ -17,40 +20,38 @@ % - schemaMap (containers.Map): A set of schema specifications for the namespace arguments - filename (1,1) string {matnwb.common.mustBeNwbFile} + reader (1,1) io.backend.base.Reader specLocation (1,1) string end - specInfo = h5info(filename, specLocation); + specInfo = reader.readNodeInfo(specLocation); specs = deal( cell(size(specInfo.Groups)) ); - - fid = H5F.open(filename); - fileCleanup = onCleanup(@(id) H5F.close(fid) ); for iGroup = 1:length(specInfo.Groups) - location = specInfo.Groups(iGroup).Groups(1); + namespaceGroupInfo = specInfo.Groups(iGroup); + location = namespaceGroupInfo.Groups(1); - namespaceName = split(specInfo.Groups(iGroup).Name, '/'); + namespaceName = split(namespaceGroupInfo.Name, '/'); namespaceName = namespaceName{end}; - filenames = {location.Datasets.Name}; - if ~any(strcmp('namespace', filenames)) + datasetNames = {location.Datasets.Name}; + if ~any(strcmp('namespace', datasetNames)) warning('NWB:Read:GenerateSpec:CacheInvalid',... 'Couldn''t find a `namespace` in namespace `%s`. Skipping cache generation.',... namespaceName); return; end - sourceNames = {location.Datasets.Name}; - fileLocation = strcat(location.Name, '/', sourceNames); + schemaMap = containers.Map; - for iFileLocation = 1:length(fileLocation) - did = H5D.open(fid, fileLocation{iFileLocation}); - if strcmp('namespace', sourceNames{iFileLocation}) - namespaceText = H5D.read(did); + for iDataset = 1:length(datasetNames) + datasetName = datasetNames{iDataset}; + datasetPath = strcat(location.Name, '/', datasetName); + datasetValue = reader.readDatasetValue(location.Datasets(iDataset), datasetPath); + if strcmp('namespace', datasetName) + namespaceText = datasetValue; else - schemaMap(sourceNames{iFileLocation}) = H5D.read(did); + schemaMap(datasetName) = datasetValue; end - H5D.close(did); end specs{iGroup}.namespaceName = namespaceName; diff --git a/+io/+spec/validateEmbeddedSpecifications.m b/+io/+spec/validateEmbeddedSpecifications.m index 9862278f5..272f67f05 100644 --- a/+io/+spec/validateEmbeddedSpecifications.m +++ b/+io/+spec/validateEmbeddedSpecifications.m @@ -1,19 +1,25 @@ -function validateEmbeddedSpecifications(h5_file_id, expectedNamespaceNames) +function validateEmbeddedSpecifications(writer, expectedNamespaceNames) % validateEmbeddedSpecifications - Validate the embedded specifications % % This function does two things: % 1) Displays a warning if specifications of expected namespaces -% are not embedded in the file. +% are not embedded in the file. % E.g if cached namespaces were cleared prior to export. -% -% 2) Deletes specifications for unused namespaces that are embedded. +% +% 2) Deletes specifications for unused namespaces that are embedded. % - E.g. If neurodata type from an embedded namespace was removed and the % file was re-exported +% +% Backend-agnostic: only uses the io.backend.base.Writer interface, so +% this works for any registered storage backend, not just HDF5. -% NB: Input h5_file_id must point to a file opened with write access + arguments + writer (1,1) io.backend.base.Writer + expectedNamespaceNames + end - specLocation = io.spec.internal.readEmbeddedSpecLocation(h5_file_id); - embeddedNamespaceNames = io.internal.h5.listGroupNames(h5_file_id, specLocation); + specLocation = writer.getEmbeddedSpecLocation(); + embeddedNamespaceNames = writer.listChildGroupNames(specLocation); checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames) @@ -21,7 +27,7 @@ function validateEmbeddedSpecifications(h5_file_id, expectedNamespaceNames) expectedNamespaceNames, embeddedNamespaceNames); if ~isempty(unusedNamespaces) - deleteUnusedNamespaces(h5_file_id, unusedNamespaces, specLocation) + deleteUnusedNamespaces(writer, unusedNamespaces, specLocation) end end @@ -45,10 +51,10 @@ function checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames) unusedNamespaces = setdiff(embeddedNamespaceNames, expectedNamespaceNames); end -function deleteUnusedNamespaces(fileId, unusedNamespaces, specRootLocation) +function deleteUnusedNamespaces(writer, unusedNamespaces, specRootLocation) for i = 1:numel(unusedNamespaces) thisName = unusedNamespaces{i}; namespaceSpecLocation = strjoin( {specRootLocation, thisName}, '/'); - io.internal.h5.deleteGroup(fileId, namespaceSpecLocation) + writer.deleteNode(namespaceSpecLocation) end end diff --git a/+io/+spec/writeEmbeddedSpecifications.m b/+io/+spec/writeEmbeddedSpecifications.m index 4468d5fd0..9dbf5f599 100644 --- a/+io/+spec/writeEmbeddedSpecifications.m +++ b/+io/+spec/writeEmbeddedSpecifications.m @@ -1,12 +1,15 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) % writeEmbeddedSpecifications - Write schema specifications to an NWB file +% +% Backend-agnostic: only uses the io.backend.base.Writer interface, so +% this works for any registered storage backend, not just HDF5. arguments writer (1,1) io.backend.base.Writer jsonSpecs % String representation of schema specifications in json format end - specLocation = io.spec.internal.readEmbeddedSpecLocation(writer.FileId); + specLocation = writer.getEmbeddedSpecLocation(); if isempty(specLocation) specLocation = '/specifications'; @@ -20,12 +23,9 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) schemaNamespaceLocation = strjoin({specLocation, JsonDatum.name}, '/'); namespaceExists = writer.writeGroup(schemaNamespaceLocation); if namespaceExists - namespaceGroupId = H5G.open(writer.FileId, schemaNamespaceLocation); - names = getVersionNames(namespaceGroupId); - H5G.close(namespaceGroupId); + names = writer.listChildGroupNames(schemaNamespaceLocation); for iNames = 1:length(names) - H5L.delete(writer.FileId, [schemaNamespaceLocation '/' names{iNames}],... - 'H5P_DEFAULT'); + writer.deleteNode([schemaNamespaceLocation '/' names{iNames}]); end end schemaLocation = ... @@ -40,13 +40,3 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) end end end - -function versionNames = getVersionNames(namespaceGroupId) - [~, ~, versionNames] = H5L.iterate(namespaceGroupId,... - 'H5_INDEX_NAME', 'H5_ITER_NATIVE',... - 0, @appendName, {}); - function [status, versionNames] = appendName(~, name, versionNames) - versionNames{end+1} = name; - status = 0; - end -end diff --git a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m index aead0e9c9..b80797962 100644 --- a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m +++ b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m @@ -8,6 +8,9 @@ "writeGroup", ... "writeValue", ... "writeAttribute", ... + "getEmbeddedSpecLocation", ... + "listChildGroupNames", ... + "deleteNode", ... "abort", ... "close", ... "ensure" ... diff --git a/NwbFile.m b/NwbFile.m index a98441443..4a5bfbdf4 100644 --- a/NwbFile.m +++ b/NwbFile.m @@ -392,7 +392,7 @@ function embedSpecifications(obj, writer) jsonSpecs); io.spec.validateEmbeddedSpecifications(... - writer.FileId, ... + writer, ... strrep(namespaceNames, '_', '-')) end diff --git a/nwbRead.m b/nwbRead.m index 36d030109..605da5302 100644 --- a/nwbRead.m +++ b/nwbRead.m @@ -87,7 +87,7 @@ end end else - generateEmbeddedSpec(filename, specLocation, 'savedir', options.savedir); + generateEmbeddedSpec(reader, specLocation, 'savedir', options.savedir); end else warnIfSchemaVersionsMismatch(schemaVersionOfFile, schemaVersionActive) @@ -119,15 +119,15 @@ nwb.resolveSoftLinks() end -function generateEmbeddedSpec(filename, specLocation, options) +function generateEmbeddedSpec(reader, specLocation, options) % generateEmbeddedSpec - Generate embedded specifications / namespaces arguments - filename (1,1) string {matnwb.common.compatibility.mustBeFile} + reader (1,1) io.backend.base.Reader specLocation (1,1) string options.savedir (1,1) string = misc.getMatnwbDir(); % {matnwb.common.compatibility.mustBeFolder} ? end - specs = io.spec.readEmbeddedSpecifications(filename, specLocation); + specs = io.spec.readEmbeddedSpecifications(reader, specLocation); specNames = cell(size(specs)); for iSpec = 1:numel(specs) From be43734f4be9044d8af29b0c65e2223ea1b745cc Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sun, 19 Jul 2026 12:45:33 +0200 Subject: [PATCH 2/3] Rename deleteNode to deleteGroup and add test --- +io/+backend/+base/Writer.m | 6 ++--- +io/+backend/+hdf5/HDF5Writer.m | 4 ++-- +io/+spec/validateEmbeddedSpecifications.m | 2 +- +io/+spec/writeEmbeddedSpecifications.m | 2 +- .../+unit/+io/+backend/+base/BaseWriterTest.m | 2 +- +tests/+unit/+io/+backend/HDF5WriterTest.m | 23 +++++++++++++++++++ 6 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 +tests/+unit/+io/+backend/HDF5WriterTest.m diff --git a/+io/+backend/+base/Writer.m b/+io/+backend/+base/Writer.m index b64ee4ec7..61894ff70 100644 --- a/+io/+backend/+base/Writer.m +++ b/+io/+backend/+base/Writer.m @@ -49,9 +49,9 @@ function writeAttribute(obj, attributePath, value, varargin) %#ok io.backend.base.Writer.throwNotImplemented("listChildGroupNames") end - function deleteNode(obj, nodePath) %#ok - % deleteNode - Delete the group or dataset at nodePath. - io.backend.base.Writer.throwNotImplemented("deleteNode") + function deleteGroup(obj, groupPath) %#ok + % deleteGroup - Delete the group at groupPath and its contents. + io.backend.base.Writer.throwNotImplemented("deleteGroup") end function close(obj) %#ok diff --git a/+io/+backend/+hdf5/HDF5Writer.m b/+io/+backend/+hdf5/HDF5Writer.m index 19c56b938..d2b2f410e 100644 --- a/+io/+backend/+hdf5/HDF5Writer.m +++ b/+io/+backend/+hdf5/HDF5Writer.m @@ -91,8 +91,8 @@ function writeAttribute(obj, attributePath, value, varargin) groupNames = io.internal.h5.listGroupNames(obj.H5FileId, groupPath); end - function deleteNode(obj, nodePath) - io.internal.h5.deleteGroup(obj.H5FileId, nodePath); + function deleteGroup(obj, groupPath) + io.internal.h5.deleteGroup(obj.H5FileId, groupPath); end end diff --git a/+io/+spec/validateEmbeddedSpecifications.m b/+io/+spec/validateEmbeddedSpecifications.m index 272f67f05..b97f254b1 100644 --- a/+io/+spec/validateEmbeddedSpecifications.m +++ b/+io/+spec/validateEmbeddedSpecifications.m @@ -55,6 +55,6 @@ function deleteUnusedNamespaces(writer, unusedNamespaces, specRootLocation) for i = 1:numel(unusedNamespaces) thisName = unusedNamespaces{i}; namespaceSpecLocation = strjoin( {specRootLocation, thisName}, '/'); - writer.deleteNode(namespaceSpecLocation) + writer.deleteGroup(namespaceSpecLocation) end end diff --git a/+io/+spec/writeEmbeddedSpecifications.m b/+io/+spec/writeEmbeddedSpecifications.m index 9dbf5f599..33eea4ec3 100644 --- a/+io/+spec/writeEmbeddedSpecifications.m +++ b/+io/+spec/writeEmbeddedSpecifications.m @@ -25,7 +25,7 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) if namespaceExists names = writer.listChildGroupNames(schemaNamespaceLocation); for iNames = 1:length(names) - writer.deleteNode([schemaNamespaceLocation '/' names{iNames}]); + writer.deleteGroup([schemaNamespaceLocation '/' names{iNames}]); end end schemaLocation = ... diff --git a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m index b80797962..32f83200f 100644 --- a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m +++ b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m @@ -10,7 +10,7 @@ "writeAttribute", ... "getEmbeddedSpecLocation", ... "listChildGroupNames", ... - "deleteNode", ... + "deleteGroup", ... "abort", ... "close", ... "ensure" ... diff --git a/+tests/+unit/+io/+backend/HDF5WriterTest.m b/+tests/+unit/+io/+backend/HDF5WriterTest.m new file mode 100644 index 000000000..b22aa8d28 --- /dev/null +++ b/+tests/+unit/+io/+backend/HDF5WriterTest.m @@ -0,0 +1,23 @@ +classdef HDF5WriterTest < matlab.unittest.TestCase + + methods (TestMethodSetup) + function setup(testCase) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + end + end + + methods (Test) + function deleteGroupDeletesPopulatedGroup(testCase) + writer = io.backend.hdf5.HDF5Writer("writer-test.nwb", "overwrite"); + testCase.addTeardown(@() writer.close()); + writer.writeGroup('/specifications/core'); + writer.writeValue('/specifications/core/namespace', 'schema'); + + writer.deleteGroup('/specifications/core'); + + groupExists = H5L.exists(writer.FileId, ... + '/specifications/core', 'H5P_DEFAULT'); + testCase.verifyFalse(logical(groupExists)); + end + end +end From f7dc2221370ce17f7d270822167c68ef67ffcc09 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sun, 19 Jul 2026 17:23:54 +0200 Subject: [PATCH 3/3] feat(zarr): add Zarr v3 reader --- .../+zarr3/+internal/ensureAvailable.m | 25 ++ .../+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m | 311 ++++++++++++++++++ +io/+backend/+zarr3/Zarr3Reader.m | 244 ++++++++++++++ +io/+backend/BackendFactory.m | 43 +++ +io/+internal/+zarr3/buildNodeInfo.m | 130 ++++++++ +io/+internal/+zarr3/convertAttributes.m | 100 ++++++ .../+zarr3/getCompoundFieldSemantics.m | 32 ++ .../+zarr3/getCompoundTypeDescriptor.m | 33 ++ +io/+internal/+zarr3/getMatlabDataType.m | 11 + .../+zarr3/normalizeDatasetDimensions.m | 27 ++ +io/+internal/+zarr3/stripLeadingSlash.m | 5 + +matnwb/+common/+compatibility/mustBeFile.m | 13 +- +matnwb/+common/mustBeNwbFile.m | 7 +- +tests/+fixtures/createZarr3TestFile.m | 92 ++++++ .../+unit/+io/+backend/Zarr3LazyArrayTest.m | 112 +++++++ +tests/+unit/+io/+backend/Zarr3ReaderTest.m | 140 ++++++++ +tests/+util/assumeZarr3Support.m | 15 + +tests/+util/getZarr3MatlabPath.m | 46 +++ .github/workflows/run_tests.yml | 6 +- .gitignore | 4 + nwbRead.m | 2 +- requirements.txt | 1 + 22 files changed, 1394 insertions(+), 5 deletions(-) create mode 100644 +io/+backend/+zarr3/+internal/ensureAvailable.m create mode 100644 +io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m create mode 100644 +io/+backend/+zarr3/Zarr3Reader.m create mode 100644 +io/+internal/+zarr3/buildNodeInfo.m create mode 100644 +io/+internal/+zarr3/convertAttributes.m create mode 100644 +io/+internal/+zarr3/getCompoundFieldSemantics.m create mode 100644 +io/+internal/+zarr3/getCompoundTypeDescriptor.m create mode 100644 +io/+internal/+zarr3/getMatlabDataType.m create mode 100644 +io/+internal/+zarr3/normalizeDatasetDimensions.m create mode 100644 +io/+internal/+zarr3/stripLeadingSlash.m create mode 100644 +tests/+fixtures/createZarr3TestFile.m create mode 100644 +tests/+unit/+io/+backend/Zarr3LazyArrayTest.m create mode 100644 +tests/+unit/+io/+backend/Zarr3ReaderTest.m create mode 100644 +tests/+util/assumeZarr3Support.m create mode 100644 +tests/+util/getZarr3MatlabPath.m create mode 100644 requirements.txt diff --git a/+io/+backend/+zarr3/+internal/ensureAvailable.m b/+io/+backend/+zarr3/+internal/ensureAvailable.m new file mode 100644 index 000000000..6242adcb0 --- /dev/null +++ b/+io/+backend/+zarr3/+internal/ensureAvailable.m @@ -0,0 +1,25 @@ +function ensureAvailable() +% ensureAvailable - Validate that the zarr-matlab package is on path. +% +% See also: https://github.com/catalystneuro/zarr-matlab + + persistent isValidated + + if isequal(isValidated, true) + return + end + + % `exist(name, "file")` does not reliably resolve dotted package-function + % names (it returns 0 even when the function is on the path), so + % availability is checked with `which` instead. + requiredFunctions = ["zarr.open", "zarr.create", "zarr.create_group"]; + isMissing = arrayfun(@(name) isempty(which(name)), requiredFunctions); + + if any(isMissing) + error("NWB:Zarr3:DependencyMissing", ... + "The `zarr-matlab` package is required on the MATLAB path. Missing function(s): %s", ... + strjoin(requiredFunctions(isMissing), ", ")) + end + + isValidated = true; +end diff --git a/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m b/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m new file mode 100644 index 000000000..b2afaea1c --- /dev/null +++ b/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m @@ -0,0 +1,311 @@ +classdef Zarr3LazyArray < io.backend.base.LazyArray +% Zarr3LazyArray - Zarr v3-backed lazy dataset access implementation. +% +% Zarr v3 stores may be written by Python NWB tools using numpy/row-major +% shape order. For rank >= 2 arrays, dims and data are reversed to match +% MatNWB's H5-style convention (see io.internal.zarr3.normalizeDatasetDimensions). +% A rank-1 array needs no correction: zarr-matlab already returns it as a +% MATLAB column vector. + + properties (Access = private) + ArrayNode = [] + FieldSemantics = [] + end + + methods + function obj = Zarr3LazyArray(filename, datasetPath, dims, dataType, fieldSemantics) + arguments + filename (1,1) string + datasetPath (1,1) string + dims double = [] + dataType = [] + fieldSemantics = [] + end + obj@io.backend.base.LazyArray(filename, datasetPath, dims, dataType); + obj.FieldSemantics = fieldSemantics; + end + + function refreshSizeInfo(obj) + arrayNode = obj.resolveArray(); + dims = double(arrayNode.shape); + if numel(dims) >= 2 + dims = fliplr(dims); + end + obj.setSizeInfo(dims, dims); + end + + function dataType = resolveDataType(obj) + % For a "structured" (compound) array, dataType is a compound + % type descriptor struct (see + % io.internal.zarr3.getCompoundTypeDescriptor), not a plain + % class name -- required by + % types.util.checkDtype/types.untyped.DataStub.isCompoundType. + % io.backend.zarr3.Zarr3Reader normally passes this in at + % construction (avoiding this lazy path entirely); it is + % reproduced here only as a fallback for a Zarr3LazyArray + % constructed directly without one. + arrayNode = obj.resolveArray(); + info = zarr.internal.dtype_info(arrayNode.meta.dataType, arrayNode.meta.dataTypeConfig); + if info.zarrType == "structured" + dataType = io.internal.zarr3.getCompoundTypeDescriptor(info, obj.getFieldSemantics()); + else + dataType = char(info.matlabClass); + end + end + + function data = load_h5_style(obj, varargin) + if isempty(varargin) + data = obj.readAllData(); + return + end + + assert(length(varargin) ~= 1, 'NWB:DataStub:InvalidNumArguments',... + 'calling load_h5_style with a single space id is no longer supported.'); + + start = varargin{1}; + count = varargin{2}; + if length(varargin) >= 3 + stride = varargin{3}; + else + stride = ones(size(start)); + end + data = obj.readPartialData(start, count, stride); + end + + function data = load_mat_style(obj, varargin) + if isempty(varargin) + data = obj.readAllData(); + if isstruct(data) + data = struct2table(data); + end + return + end + + [isSupported, fullSelection] = obj.tryBuildRegularSelection(varargin); + if isSupported + [start, count, stride] = obj.selectionToReadParameters(fullSelection); + data = obj.readPartialData(start, count, stride); + if isstruct(data) + % Record selection already happened during the partial + % read; matching io.backend.hdf5.@HDF5LazyArray's + % compound convention, the selected records are + % returned as a table rather than reshaped further. + data = struct2table(data); + else + data = obj.applySelectionShape(data, varargin); + end + else + data = obj.readAllData(); + if isstruct(data) + data = struct2table(data); + data = data(varargin{:}, :); + else + data = data(varargin{:}); + end + end + end + end + + methods (Access = private) + function arrayNode = resolveArray(obj) + if isempty(obj.ArrayNode) + relativePath = io.internal.zarr3.stripLeadingSlash(obj.DatasetPath); + obj.ArrayNode = zarr.open(obj.Filename, Path=relativePath); + end + arrayNode = obj.ArrayNode; + end + + function fieldSemantics = getFieldSemantics(obj) + if ~isempty(obj.FieldSemantics) && obj.FieldSemantics.Count > 0 + fieldSemantics = obj.FieldSemantics; + else + fieldSemantics = io.internal.zarr3.getCompoundFieldSemantics(obj.resolveArray().attrs); + end + end + + function data = postProcessCompound(obj, data) + % postProcessCompound - Convert zarr-matlab's array-of-records + % (one struct per element) into the "struct of arrays" shape + % (one scalar struct, each field an Nx1 array) that + % io.backend.hdf5.@HDF5LazyArray/load_h5_style.m produces via + % io.parseCompound, decoding any field tagged as an object + % reference (see io.internal.zarr3.getCompoundFieldSemantics) + % into a types.untyped.ObjectView array along the way. + if ~isstruct(data) + return + end + + fieldSemantics = obj.getFieldSemantics(); + fieldNames = fieldnames(data); + n = numel(data); + converted = struct(); + for iField = 1:numel(fieldNames) + name = fieldNames{iField}; + rawValues = {data.(name)}; + if isKey(fieldSemantics, name) && fieldSemantics(name) == "object" + values = types.untyped.ObjectView.empty(0, 0); + for iValue = 1:n + decoded = jsondecode(char(rawValues{iValue})); + values(iValue) = types.untyped.ObjectView(decoded.path); + end + converted.(name) = reshape(values, n, 1); + else + converted.(name) = reshape([rawValues{:}], n, 1); + end + end + data = converted; + end + + function data = readAllData(obj) + arrayNode = obj.resolveArray(); + data = arrayNode.read(); + data = io.internal.zarr3.normalizeDatasetDimensions(data, numel(arrayNode.shape)); + data = obj.postProcessCompound(data); + end + + function data = readPartialData(obj, start, count, stride) + arrayNode = obj.resolveArray(); + if any(isinf(count)) + count(isinf(count)) = obj.dims(isinf(count)) - start(isinf(count)) + 1; + end + + % start/count/stride arrive in MatNWB's H5-style dims order + % (obj.dims); reverse to raw Zarr/numpy order for rank >= 2 + % before calling zarr.Array.read (see refreshSizeInfo). + rank = numel(start); + if rank >= 2 + rawStart = fliplr(start); + rawCount = fliplr(count); + rawStride = fliplr(stride); + else + rawStart = start; + rawCount = count; + rawStride = stride; + end + + if all(rawStride == 1) + data = arrayNode.read(rawStart, rawCount); + else + % zarr.Array.read has no native stride support: read the + % contiguous bounding box spanning the strided selection, + % then subselect the stride in MATLAB. + boxedSpan = (rawCount - 1) .* rawStride + 1; + boxed = arrayNode.read(rawStart, boxedSpan); + selection = cell(1, numel(rawStart)); + for iDimension = 1:numel(rawStart) + selection{iDimension} = 1:rawStride(iDimension):boxedSpan(iDimension); + end + data = boxed(selection{:}); + end + data = io.internal.zarr3.normalizeDatasetDimensions(data, rank); + data = obj.postProcessCompound(data); + end + + function [isSupported, fullSelection] = tryBuildRegularSelection(obj, userSelection) + dataDimensions = obj.dims; + isSupported = true; + fullSelection = cell(1, length(dataDimensions)); + + if isscalar(userSelection) && isempty(userSelection{1}) + isSupported = false; + return + end + + if isscalar(userSelection) && ~ischar(userSelection{1}) + isSupported = false; + return + end + + isDanglingGroup = ischar(userSelection{end}); + for iDimension = 1:length(dataDimensions) + if iDimension > length(userSelection) && ~isDanglingGroup + fullSelection{iDimension} = 1; + elseif (iDimension > length(userSelection) && isDanglingGroup) ... + || ischar(userSelection{iDimension}) + fullSelection{iDimension} = 1:dataDimensions(iDimension); + else + selection = userSelection{iDimension}; + if ~obj.isRegularAscendingSelection(selection) + isSupported = false; + return + end + fullSelection{iDimension} = selection; + end + end + end + + function tf = isRegularAscendingSelection(~, selection) + tf = isnumeric(selection) ... + && isreal(selection) ... + && all(isfinite(selection)) ... + && all(selection > 0) ... + && all(selection == floor(selection)); + if ~tf + return + end + if isscalar(selection) + return + end + + stepSizes = diff(selection); + tf = all(stepSizes > 0) && numel(unique(stepSizes)) == 1; + end + + function [start, count, stride] = selectionToReadParameters(~, selection) + start = zeros(1, numel(selection)); + count = zeros(1, numel(selection)); + stride = ones(1, numel(selection)); + + for iDimension = 1:numel(selection) + currentSelection = selection{iDimension}; + start(iDimension) = currentSelection(1); + count(iDimension) = numel(currentSelection); + if numel(currentSelection) > 1 + stride(iDimension) = currentSelection(2) - currentSelection(1); + end + end + end + + function data = applySelectionShape(obj, data, userSelection) + expectedSize = obj.getExpectedSize(userSelection); + if isequal(size(data), expectedSize) + return + end + data = reshape(data, expectedSize); + end + + function expectedSize = getExpectedSize(obj, userSelection) + dataDimensions = obj.dims; + expectedSize = dataDimensions; + for iSelection = 1:length(userSelection) + if ~ischar(userSelection{iSelection}) + expectedSize(iSelection) = length(userSelection{iSelection}); + end + end + + if ischar(userSelection{end}) + selectedDimensionIndex = length(userSelection); + expectedSize = [expectedSize(1:(selectedDimensionIndex-1)), ... + prod(dataDimensions(selectedDimensionIndex:end))]; + else + expectedSize = expectedSize(1:length(userSelection)); + end + + if isscalar(userSelection) && isscalar(expectedSize) + if 1 < sum(1 < dataDimensions) + if ~ischar(userSelection{1}) && isrow(userSelection{1}) + expectedSize = [1 expectedSize]; + else + expectedSize = [expectedSize 1]; + end + else + if dataDimensions(1) == 1 + expectedSize = [1 expectedSize]; + else + expectedSize = [expectedSize 1]; + end + end + end + end + end +end diff --git a/+io/+backend/+zarr3/Zarr3Reader.m b/+io/+backend/+zarr3/Zarr3Reader.m new file mode 100644 index 000000000..8b1788570 --- /dev/null +++ b/+io/+backend/+zarr3/Zarr3Reader.m @@ -0,0 +1,244 @@ +classdef Zarr3Reader < io.backend.base.Reader + % Zarr3Reader - Reader implementation for local Zarr v3 stores. + % + % This reader is backed by the zarr-matlab package + % (https://github.com/catalystneuro/zarr-matlab), which must be on the + % MATLAB path, and reads Zarr v3 stores natively in MATLAB (no Python + % dependency). It pairs with io.backend.zarr3.Zarr3Writer for writing. + % + % Object references are represented as an attribute value struct + % `struct('zarr_dtype', "object", 'value', struct('path', targetPath))`, + % matching the convention used by hdmf-zarr for Zarr v2 stores. There is no published Zarr v3 convention for NWB, but + % this matches the "zarr_dtype" reference convention used by real + % hdmf-zarr-style Zarr v3 NWB exports (verified against example files), + % as well as the reader/writer pair's own round-trips. + % + % A dataset whose elements are themselves object references (e.g. a + % DynamicTable column of ElectrodeGroup references) is represented on + % disk as a plain Zarr "string" array whose elements are the same + % reference JSON, tagged via a "zarr_dtype":"object" attribute on the + % array itself (see io.internal.zarr3.buildNodeInfo); each element is + % decoded into a types.untyped.ObjectView object array. + % + % A compound (struct/table) dataset -- a Zarr v3 "structured" data_type, + % e.g. PlaneSegmentation's pixel_mask/voxel_mask, or + % TimeSeriesReferenceVectorData's response/stimulus columns -- is backed + % by io.backend.zarr3.Zarr3LazyArray; a field tagged "object" via the + % array's "zarr_dtype" attribute (see + % io.internal.zarr3.getCompoundFieldSemantics) is decoded into a + % types.untyped.ObjectView, matching the plain object-reference-array + % convention above. Requires zarr-matlab to support the Zarr v3 + % "structured" and "fixed_length_utf32" data types, which are unstable, + % unspecified zarr-python extensions -- see + % zarr.internal.dtype_info in zarr-matlab. + + properties (Access = private) + RootGroup = [] + RootInfoCache = [] + NodeInfoMap = containers.Map('KeyType', 'char', 'ValueType', 'any') + end + + methods + function obj = Zarr3Reader(filename) + obj@io.backend.base.Reader(filename); + end + + function version = getSchemaVersion(obj) + obj.ensureMetadataCache(); + attributes = obj.RootGroup.attrs; + if isfield(attributes, "nwb_version") + version = string(attributes.nwb_version); + else + error("NWB:Zarr3Reader:MissingSchemaVersion", ... + "The Zarr store `%s` does not define `nwb_version` in the root attributes.", ... + obj.Filename) + end + end + + function specLocation = getEmbeddedSpecLocation(obj) + obj.ensureMetadataCache(); + attributes = obj.RootGroup.attrs; + if isfield(attributes, "x_specloc") + specLocation = string(attributes.x_specloc); + elseif obj.RootGroup.isKey("specifications") + specLocation = "/specifications"; + else + specLocation = ""; + end + + if specLocation ~= "" && ~startsWith(specLocation, "/") + specLocation = "/" + specLocation; + end + end + + function node = readRootInfo(obj) + obj.ensureMetadataCache(); + node = obj.RootInfoCache; + end + + function node = readNodeInfo(obj, nodePath) + arguments + obj + nodePath (1,1) string + end + + obj.ensureMetadataCache(); + normalizedPath = obj.normalizeNodePath(nodePath); + if ~isKey(obj.NodeInfoMap, normalizedPath) + error("NWB:Zarr3Reader:NodeNotFound", ... + "Node `%s` was not found in `%s`.", normalizedPath, obj.Filename) + end + node = obj.NodeInfoMap(normalizedPath); + end + + function attributeValue = readAttributeValue(~, attributeInfo, ~) + if (ischar(attributeInfo.Datatype) || isstring(attributeInfo.Datatype)) ... + && strcmp(attributeInfo.Datatype, "object reference") + attributeValue = types.untyped.ObjectView(attributeInfo.Value.value.path); + else + attributeValue = attributeInfo.Value; + end + end + + function datasetValue = readDatasetValue(obj, datasetInfo, datasetPath) + dataDimensions = obj.getDatasetDims(datasetInfo); + isObjectReferenceArray = strcmp(datasetInfo.Datatype, "object"); + isStructuredArray = strcmp(datasetInfo.Datatype, "structured"); + % A true rank-0 array (this reader's own Zarr3Writer's scalar + % convention) or one explicitly marked "scalar" by hdmf-zarr's + % zarr_dtype hint (see io.internal.zarr3.buildNodeInfo) is read + % eagerly. Dataspace.Size == 1 alone is NOT a reliable scalar + % signal: hdmf-zarr represents a genuine NWB scalar property as + % a rank-1, length-1 array, which is indistinguishable by shape + % from a one-row VectorData column (e.g. a DynamicTable with a + % single row) -- collapsing the latter to a bare value would + % silently corrupt it (a char column's character count would + % be misread as its row count downstream). + isScalarMarked = isempty(dataDimensions) || strcmp(datasetInfo.Datatype, "scalar"); + if isObjectReferenceArray + datasetValue = obj.readObjectArrayValue(datasetPath); + elseif isStructuredArray + % Checked ahead of the scalar/eager branch below: a + % structured array can have prod(dataDimensions) == 1 (a + % single record, e.g. shape [1]) without being a "scalar" + % dataset in the ordinary sense. + datasetValue = obj.readStructuredValue(datasetPath, dataDimensions); + elseif isScalarMarked + datasetValue = obj.readEagerValue(datasetPath); + elseif any(dataDimensions == 0) + datasetValue = []; + else + matlabDataType = io.internal.zarr3.getMatlabDataType(datasetInfo.Datatype); + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + obj.Filename, datasetPath, dataDimensions, matlabDataType); + datasetValue = types.untyped.DataStub(... + obj.Filename, datasetPath, [], [], lazyArray); + end + end + end + + methods (Access = private) + function ensureMetadataCache(obj) + if isempty(obj.RootGroup) + io.backend.zarr3.internal.ensureAvailable() + obj.RootGroup = zarr.open(obj.Filename); + [obj.RootInfoCache, obj.NodeInfoMap] = io.internal.zarr3.buildNodeInfo(obj.RootGroup); + obj.RootInfoCache.Filename = char(obj.Filename); + end + end + + function normalizedPath = normalizeNodePath(~, nodePath) + normalizedPath = char(nodePath); + if isempty(normalizedPath) + normalizedPath = '/'; + elseif normalizedPath(1) ~= '/' + normalizedPath = ['/' normalizedPath]; + end + end + + function dataDimensions = getDatasetDims(~, datasetInfo) + % Dataspace.Size is the raw (Zarr/numpy-order) shape; reverse it + % for rank >= 2 to match MatNWB's H5-style dims convention (see + % io.internal.zarr3.normalizeDatasetDimensions). + if isfield(datasetInfo, "Dataspace") && isfield(datasetInfo.Dataspace, "Size") + dataDimensions = double(datasetInfo.Dataspace.Size); + else + dataDimensions = []; + end + + if numel(dataDimensions) >= 2 + dataDimensions = fliplr(dataDimensions); + end + end + + function datasetValue = readEagerValue(obj, datasetPath) + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + datasetValue = arrayNode.read(); + + if isstring(datasetValue) && isscalar(datasetValue) + datasetValue = char(datasetValue); + elseif iscell(datasetValue) && isscalar(datasetValue) + datasetValue = datasetValue{1}; + end + end + + function datasetValue = readStructuredValue(obj, datasetPath, dataDimensions) + % readStructuredValue - Wrap a "structured" (compound) array in + % a DataStub backed by io.backend.zarr3.Zarr3LazyArray. The + % DataStub's dataType is a compound type descriptor struct + % (field name -> MATLAB class name, or 'types.untyped.ObjectView' + % for a field tagged as a reference via the array's own + % "zarr_dtype" attribute; see + % io.internal.zarr3.getCompoundTypeDescriptor) rather than a + % plain class name, matching what + % types.util.checkDtype/types.untyped.DataStub.isCompoundType + % expect -- this alone is enough for schema validation to + % succeed without loading any data (see + % types.util.checkDtype>checkDtypeForCompoundDataset's + % DataStub fast path). + + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + info = zarr.internal.dtype_info(arrayNode.meta.dataType, arrayNode.meta.dataTypeConfig); + fieldSemantics = io.internal.zarr3.getCompoundFieldSemantics(arrayNode.attrs); + typeDescriptor = io.internal.zarr3.getCompoundTypeDescriptor(info, fieldSemantics); + + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + obj.Filename, datasetPath, dataDimensions, typeDescriptor, fieldSemantics); + datasetValue = types.untyped.DataStub(... + obj.Filename, datasetPath, [], [], lazyArray); + end + + function datasetValue = readObjectArrayValue(obj, datasetPath) + % readObjectArrayValue - Decode an array whose elements are + % JSON-encoded object references (Datatype "object"; see + % io.internal.zarr3.buildNodeInfo) into a types.untyped.ObjectView + % object array (matching io.parseReference's shape for HDF5 + % reference datasets, which types.util.checkDtype requires -- + % a cell array of ObjectView is not an accepted dtype). Each + % element is stored as a zarr "string" containing the same + % {"source":...,"path":...} JSON convention used for + % object-reference attributes (io.internal.zarr3.convertAttributes), + % but is not auto-decoded to a struct by zarr-matlab since the + % array's own Zarr v3 data_type is plain "string". + + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + rawValues = string(arrayNode.read()); + + datasetValue = types.untyped.ObjectView.empty(0, 0); + for iValue = 1:numel(rawValues) + datasetValue(iValue) = io.backend.zarr3.Zarr3Reader.decodeObjectReferenceElement(rawValues(iValue)); + end + datasetValue = reshape(datasetValue, size(rawValues)); + end + end + + methods (Static, Access = private) + function objectView = decodeObjectReferenceElement(rawElement) + decoded = jsondecode(char(rawElement)); + objectView = types.untyped.ObjectView(decoded.path); + end + end +end diff --git a/+io/+backend/BackendFactory.m b/+io/+backend/BackendFactory.m index 09c2c87fa..74fd23ab5 100644 --- a/+io/+backend/BackendFactory.m +++ b/+io/+backend/BackendFactory.m @@ -35,6 +35,8 @@ case "auto" if io.backend.BackendFactory.isHDF5File(filename) reader = io.backend.hdf5.HDF5Reader(filename); + elseif io.backend.BackendFactory.isZarr3Directory(filename) + reader = io.backend.zarr3.Zarr3Reader(filename); else error("NWB:BackendFactory:UnsupportedFormat", ... "No supported reader found for `%s`.", filename) @@ -45,6 +47,12 @@ "`%s` is not a valid HDF5 file.", filename) end reader = io.backend.hdf5.HDF5Reader(filename); + case "zarr3" + if ~io.backend.BackendFactory.isZarr3Directory(filename) + error("NWB:BackendFactory:InvalidZarr3", ... + "`%s` is not a supported local Zarr v3 directory store.", filename) + end + reader = io.backend.zarr3.Zarr3Reader(filename); otherwise error("NWB:BackendFactory:UnsupportedBackend", ... "Unsupported backend `%s`.", storageBackend) @@ -66,6 +74,8 @@ case "auto" if io.backend.BackendFactory.isHDF5File(filename) lazyArray = io.backend.hdf5.HDF5LazyArray(filename, datasetPath, dims, dataType); + elseif io.backend.BackendFactory.isZarr3Directory(filename) + lazyArray = io.backend.zarr3.Zarr3LazyArray(filename, datasetPath, dims, dataType); else error("NWB:BackendFactory:UnsupportedFormat", ... "No supported lazy array backend found for `%s`.", filename) @@ -76,6 +86,12 @@ "`%s` is not a valid HDF5 file.", filename) end lazyArray = io.backend.hdf5.HDF5LazyArray(filename, datasetPath, dims, dataType); + case "zarr3" + if ~io.backend.BackendFactory.isZarr3Directory(filename) + error("NWB:BackendFactory:InvalidZarr3", ... + "`%s` is not a supported local Zarr v3 directory store.", filename) + end + lazyArray = io.backend.zarr3.Zarr3LazyArray(filename, datasetPath, dims, dataType); otherwise error("NWB:BackendFactory:UnsupportedBackend", ... "Unsupported backend `%s`.", storageBackend) @@ -105,5 +121,32 @@ end end end + + function tf = isZarr3Directory(filename) + arguments + filename (1,1) string + end + + tf = false; + if startsWith(filename, "s3://", "IgnoreCase", true) || ~isfolder(filename) + return + end + + if ~endsWith(filename, ".zarr", "IgnoreCase", true) + return + end + + rootMetadataFile = fullfile(filename, "zarr.json"); + if ~isfile(rootMetadataFile) + return + end + + try + rootMetadata = jsondecode(fileread(rootMetadataFile)); + tf = isfield(rootMetadata, "zarr_format") && isequal(rootMetadata.zarr_format, 3); + catch + tf = false; + end + end end end diff --git a/+io/+internal/+zarr3/buildNodeInfo.m b/+io/+internal/+zarr3/buildNodeInfo.m new file mode 100644 index 000000000..b625e360a --- /dev/null +++ b/+io/+internal/+zarr3/buildNodeInfo.m @@ -0,0 +1,130 @@ +function [rootInfo, nodeInfoMap] = buildNodeInfo(rootGroup) +% buildNodeInfo - Build an h5info-like node tree from a zarr.Group root. +% +% [rootInfo, nodeInfoMap] = buildNodeInfo(rootGroup) walks the hierarchy +% below rootGroup (a zarr.Group opened via zarr.open, using its +% consolidated metadata when available) and returns: +% +% rootInfo - h5info-like struct for the root node (fields Name, Groups, +% Datasets, Links, Attributes), recursively populated. +% +% nodeInfoMap - containers.Map from absolute node path (char, leading +% '/') to the corresponding node info struct (group or dataset). +% +% The output shape mirrors h5info's, so io.parseGroup/io.parseDataset/ +% io.parseAttributes can consume it directly. + + nodeInfoMap = containers.Map('KeyType', 'char', 'ValueType', 'any'); + rootInfo = buildGroupInfo(rootGroup, "/", nodeInfoMap); +end + +function groupInfo = buildGroupInfo(group, groupPath, nodeInfoMap) + groupInfo = struct(... + 'Name', char(groupPath), ... + 'Groups', emptyGroupStruct(), ... + 'Datasets', emptyDatasetStruct(), ... + 'Links', emptyLinkStruct(), ... + 'Attributes', emptyAttributeStruct()); + + [attributes, links] = io.internal.zarr3.convertAttributes(group.attrs); + groupInfo.Attributes = attributes; + groupInfo.Links = links; + + [arrayNames, groupNames] = group.children(); + + for iArray = 1:numel(arrayNames) + childArray = group.item(arrayNames(iArray)); + childPath = joinPath(groupPath, arrayNames(iArray)); + datasetInfo = buildDatasetInfo(childArray, arrayNames(iArray)); + groupInfo.Datasets(end+1) = datasetInfo; %#ok + nodeInfoMap(char(childPath)) = datasetInfo; + end + + for iGroup = 1:numel(groupNames) + childGroup = group.item(groupNames(iGroup)); + childPath = joinPath(groupPath, groupNames(iGroup)); + childInfo = buildGroupInfo(childGroup, childPath, nodeInfoMap); + groupInfo.Groups(end+1) = childInfo; %#ok + end + + nodeInfoMap(char(groupPath)) = groupInfo; +end + +function datasetInfo = buildDatasetInfo(arrayNode, leafName) + shape = double(arrayNode.shape); + if isempty(shape) + dataspaceType = 'scalar'; + else + dataspaceType = 'simple'; + end + + datasetInfo = struct(... + 'Name', char(leafName), ... + 'Datatype', char(arrayNode.dtype), ... + 'Dataspace', struct('Size', shape, 'MaxSize', shape, 'Type', dataspaceType), ... + 'ChunkSize', double(arrayNode.chunkShape), ... + 'FillValue', arrayNode.meta.fillValue, ... + 'Filters', struct('Name', {}, 'Parameters', {}), ... + 'Attributes', emptyAttributeStruct()); + + % A "zarr_dtype" attribute (reserved; filtered out of Attributes below) + % is a legacy hdmf-zarr (v2) dtype hint. For plain numeric/string data + % it is normally redundant with (and less precise than) the native + % Zarr v3 `data_type`, so it is ignored -- except for two semantic + % markers with no native Zarr v3 equivalent, which override Datatype + % so io.backend.zarr3.Zarr3Reader can dispatch on them: + % "object" - an array of object references stored as JSON-encoded + % strings. + % "scalar" - hdmf-zarr represents an NWB scalar property (e.g. + % identifier, session_description) as a rank-1, length-1 Zarr + % array rather than a true rank-0 array, so shape alone cannot + % distinguish it from a genuine one-row VectorData column (e.g. a + % DynamicTable that happens to have a single row) -- both have + % Dataspace.Size == 1. Reading the latter eagerly as a bare scalar + % would silently collapse a 1-element char/string column, whose + % stray character count would then be misread as a table row + % count by DynamicTable height checks. + % For a "structured" (compound) array, this attribute is instead a + % per-field type descriptor list (a struct array), not a scalar + % string -- irrelevant here since the array's own Zarr v3 data_type + % ("structured") already identifies it unambiguously. + zarrDtypeAttr = []; + if isfield(arrayNode.attrs, 'zarr_dtype') + zarrDtypeAttr = arrayNode.attrs.zarr_dtype; + end + if ischar(zarrDtypeAttr) || isstring(zarrDtypeAttr) + zarrDtypeAttrText = string(zarrDtypeAttr); + if zarrDtypeAttrText == "object" + datasetInfo.Datatype = 'object'; + elseif zarrDtypeAttrText == "scalar" + datasetInfo.Datatype = 'scalar'; + end + end + + datasetInfo.Attributes = io.internal.zarr3.convertAttributes(arrayNode.attrs); +end + +function joinedPath = joinPath(parentPath, childName) + if parentPath == "/" + joinedPath = "/" + childName; + else + joinedPath = parentPath + "/" + childName; + end +end + +function groupStruct = emptyGroupStruct() + groupStruct = struct('Name', {}, 'Groups', {}, 'Datasets', {}, 'Links', {}, 'Attributes', {}); +end + +function datasetStruct = emptyDatasetStruct() + datasetStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, ... + 'ChunkSize', {}, 'FillValue', {}, 'Filters', {}, 'Attributes', {}); +end + +function attributeStruct = emptyAttributeStruct() + attributeStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, 'Value', {}); +end + +function linkStruct = emptyLinkStruct() + linkStruct = struct('Name', {}, 'Type', {}, 'Value', {}); +end diff --git a/+io/+internal/+zarr3/convertAttributes.m b/+io/+internal/+zarr3/convertAttributes.m new file mode 100644 index 000000000..e34aea87c --- /dev/null +++ b/+io/+internal/+zarr3/convertAttributes.m @@ -0,0 +1,100 @@ +function [attributes, links] = convertAttributes(rawAttributes) +% convertAttributes - Convert zarr-matlab attrs into h5info-like structures. +% +% [attributes, links] = convertAttributes(rawAttributes) converts the +% attrs struct exposed by a zarr.Group or zarr.Array (as returned by +% zarr-matlab) into: +% +% attributes - struct array with fields Name, Datatype, Dataspace, +% Value, matching the shape produced by h5info. Object references +% (see io.internal.zarr3.encodeObjectReference) are tagged with +% Datatype "object reference". +% +% links - struct array with fields Name, Type, Value, decoded from the +% reserved "zarr_link" attribute (soft/external link records written +% by io.backend.zarr3.Zarr3Writer). Non-group nodes never carry links, +% but the reserved attribute is filtered out regardless. +% +% This convention matches the shape produced by h5info, so that the same +% downstream parsing code (io.parseGroup, io.parseAttributes) can consume +% node info from any backend. + + attributes = emptyAttributeStruct(); + links = emptyLinkStruct(); + + if isempty(rawAttributes) || ~isstruct(rawAttributes) + return + end + + fieldNames = fieldnames(rawAttributes); + for iField = 1:numel(fieldNames) + name = fieldNames{iField}; + value = rawAttributes.(name); + + if strcmp(name, "zarr_link") + links = convertLinks(value); + continue + elseif strcmp(name, "x_specloc") + % Reserved marker for io.backend.zarr3.Zarr3Reader.getEmbeddedSpecLocation; + % not a schema attribute, so it is never promoted. + continue + elseif strcmp(name, "zarr_dtype") || strcmp(name, "x_ARRAY_DIMENSIONS") + % Reserved bookkeeping attributes written by hdmf-zarr-style + % exporters (dtype hint / xarray dimension names, the latter + % written to disk as "_ARRAY_DIMENSIONS" but renamed by + % jsondecode since a leading underscore is not a valid MATLAB + % identifier); not schema attributes. + continue + end + + if iscell(value) && ~isscalar(value) + % struct(...,'Value',value) would otherwise expand a non-scalar + % cell into a non-scalar struct array (one element per cell + % entry) instead of a single attribute whose Value is the cell. + value = {value}; + end + + attribute = struct('Name', name, 'Datatype', [], 'Dataspace', [], 'Value', value); + if isObjectReferenceValue(value) + attribute.Datatype = 'object reference'; + end + attributes(end+1) = attribute; %#ok + end +end + +function tf = isObjectReferenceValue(value) + tf = isstruct(value) && isscalar(value) ... + && isfield(value, 'zarr_dtype') ... + && strcmp(string(value.zarr_dtype), "object"); +end + +function links = convertLinks(rawLinks) + links = emptyLinkStruct(); + if isempty(rawLinks) + return + end + if ~iscell(rawLinks) + rawLinks = num2cell(rawLinks); + end + + for iLink = 1:numel(rawLinks) + rawLink = rawLinks{iLink}; + link = struct('Name', char(rawLink.name), 'Type', '', 'Value', []); + if strcmp(rawLink.source, '.') + link.Type = 'soft link'; + link.Value = {char(rawLink.path)}; + else + link.Type = 'external link'; + link.Value = {char(rawLink.source), char(rawLink.path)}; + end + links(end+1) = link; %#ok + end +end + +function attributeStruct = emptyAttributeStruct() + attributeStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, 'Value', {}); +end + +function linkStruct = emptyLinkStruct() + linkStruct = struct('Name', {}, 'Type', {}, 'Value', {}); +end diff --git a/+io/+internal/+zarr3/getCompoundFieldSemantics.m b/+io/+internal/+zarr3/getCompoundFieldSemantics.m new file mode 100644 index 000000000..12f0bca7c --- /dev/null +++ b/+io/+internal/+zarr3/getCompoundFieldSemantics.m @@ -0,0 +1,32 @@ +function fieldSemantics = getCompoundFieldSemantics(attrs) +% getCompoundFieldSemantics - Per-field semantic type hints for a +% "structured" (compound) Zarr v3 array. +% +% fieldSemantics = getCompoundFieldSemantics(attrs) returns a +% containers.Map from field name (char) to semantic dtype (string), +% read from the array's own "zarr_dtype" attribute -- for a compound +% array this is a per-field type descriptor list (matching the +% hdmf-zarr v2 convention), e.g. +% [{"name":"idx_start","dtype":"int32"}, ..., +% {"name":"timeseries","dtype":"object"}]. +% +% The only semantic value consumed downstream is "object" (the field +% holds a JSON-encoded object reference rather than literal text/data), +% matching io.backend.zarr3.Zarr3Reader.readObjectArrayValue's +% convention for top-level arrays of references. Other hints are +% redundant with the field's own Zarr v3 sub-dtype and are ignored. + + fieldSemantics = containers.Map('KeyType', 'char', 'ValueType', 'any'); + if ~isfield(attrs, 'zarr_dtype') + return + end + + fieldDescriptors = attrs.zarr_dtype; + if ~isstruct(fieldDescriptors) + return + end + + for i = 1:numel(fieldDescriptors) + fieldSemantics(char(fieldDescriptors(i).name)) = string(fieldDescriptors(i).dtype); + end +end diff --git a/+io/+internal/+zarr3/getCompoundTypeDescriptor.m b/+io/+internal/+zarr3/getCompoundTypeDescriptor.m new file mode 100644 index 000000000..4cf529dee --- /dev/null +++ b/+io/+internal/+zarr3/getCompoundTypeDescriptor.m @@ -0,0 +1,33 @@ +function typeDescriptor = getCompoundTypeDescriptor(info, fieldSemantics) +% getCompoundTypeDescriptor - Build the compound type descriptor struct +% expected by types.util.checkDtype / types.untyped.DataStub.dataType for +% a "structured" (compound) Zarr v3 array. +% +% typeDescriptor = getCompoundTypeDescriptor(info, fieldSemantics) +% returns a scalar struct with one field per info.fields entry (same +% name, same order -- checkDtype's validateCompoundTypeDescriptor +% requires exact field order), whose value is either: +% - 'types.untyped.ObjectView', when fieldSemantics marks that field +% as "object" (see io.internal.zarr3.getCompoundFieldSemantics), or +% - the field's MATLAB class name (info.fields(k).Info.matlabClass) +% otherwise, e.g. 'int32', 'double', 'single'. +% +% info is a zarr.internal.dtype_info struct for a "structured" dtype +% (info.zarrType == "structured"). + + arguments + info (1,1) struct + fieldSemantics + end + + typeDescriptor = struct(); + for k = 1:numel(info.fields) + f = info.fields(k); + name = char(f.Name); + if isKey(fieldSemantics, name) && fieldSemantics(name) == "object" + typeDescriptor.(name) = 'types.untyped.ObjectView'; + else + typeDescriptor.(name) = char(f.Info.matlabClass); + end + end +end diff --git a/+io/+internal/+zarr3/getMatlabDataType.m b/+io/+internal/+zarr3/getMatlabDataType.m new file mode 100644 index 000000000..2da03e9aa --- /dev/null +++ b/+io/+internal/+zarr3/getMatlabDataType.m @@ -0,0 +1,11 @@ +function matlabDataType = getMatlabDataType(zarrDataType) +% getMatlabDataType - Map a Zarr v3 data_type name to a MATLAB class name. +% +% matlabDataType = getMatlabDataType(zarrDataType) resolves the MATLAB +% class used to represent values of the given Zarr v3 `data_type` (e.g. +% "float64", "int64", "bool", "string"), reusing the dtype table from the +% zarr-matlab package. + + info = zarr.internal.dtype_info(zarrDataType); + matlabDataType = char(info.matlabClass); +end diff --git a/+io/+internal/+zarr3/normalizeDatasetDimensions.m b/+io/+internal/+zarr3/normalizeDatasetDimensions.m new file mode 100644 index 000000000..8bff6b73f --- /dev/null +++ b/+io/+internal/+zarr3/normalizeDatasetDimensions.m @@ -0,0 +1,27 @@ +function datasetValue = normalizeDatasetDimensions(datasetValue, rank) +% normalizeDatasetDimensions - Reverse axis order for rank >= 2 data. +% +% normalizeDatasetDimensions(datasetValue, rank) reverses all axes of +% datasetValue when rank (the array's Zarr rank, i.e. numel(shape) as +% reported by zarr-matlab -- NOT MATLAB's ndims, which cannot +% distinguish a genuine rank-1 column vector from a rank-2 array) is >= 2. +% +% Zarr v3 stores written by Python NWB tools use numpy/row-major shape +% order (matching the NWB schema's declared shape directly). MatNWB's +% type system (types.util.validateShape, checkDims) expects the reverse +% of that for rank >= 2, matching the convention already used by +% io.backend.hdf5.HDF5Reader. +% +% A rank-1 Zarr array is already read by zarr-matlab as a MATLAB column +% vector (see zarr.internal.mshape), so it is left untouched here. + + if rank < 2 + return + end + + if rank == 2 + datasetValue = datasetValue.'; + else + datasetValue = permute(datasetValue, rank:-1:1); + end +end diff --git a/+io/+internal/+zarr3/stripLeadingSlash.m b/+io/+internal/+zarr3/stripLeadingSlash.m new file mode 100644 index 000000000..b96b02eff --- /dev/null +++ b/+io/+internal/+zarr3/stripLeadingSlash.m @@ -0,0 +1,5 @@ +function strippedPath = stripLeadingSlash(nodePath) +% stripLeadingSlash - Remove a single leading '/' from a node path. + + strippedPath = regexprep(char(nodePath), '^/', ''); +end diff --git a/+matnwb/+common/+compatibility/mustBeFile.m b/+matnwb/+common/+compatibility/mustBeFile.m index 3cdbdca8c..f4f7780c4 100644 --- a/+matnwb/+common/+compatibility/mustBeFile.m +++ b/+matnwb/+common/+compatibility/mustBeFile.m @@ -14,7 +14,18 @@ function mustBeFile(filePath) if startsWith(filePath, "s3://") return end - + + % Directory-based stores (e.g. Zarr) use a ".zarr" suffix by convention. + % These are folders, not files, so validate folder existence instead. + if endsWith(filePath, ".zarr", "IgnoreCase", true) + try + matnwb.common.compatibility.mustBeFolder(filePath) + catch ME + throwAsCaller(ME) + end + return + end + if verLessThan('matlab', '9.9') %#ok % Custom implementation (MATLAB < R2020b) try diff --git a/+matnwb/+common/mustBeNwbFile.m b/+matnwb/+common/mustBeNwbFile.m index 1a01e8b0e..9f5d6bc36 100644 --- a/+matnwb/+common/mustBeNwbFile.m +++ b/+matnwb/+common/mustBeNwbFile.m @@ -1,9 +1,12 @@ function mustBeNwbFile(filePath) -% mustBeNwbFile - Check that file path points to existing file with .nwb extension +% mustBeNwbFile - Check that path points to an existing NWB file or Zarr store +% +% Accepts a file with a ".nwb" extension or a Zarr directory store with a +% ".zarr" extension. arguments filePath (1,1) string {matnwb.common.compatibility.mustBeFile} end if ~startsWith(filePath, "s3://", "IgnoreCase", true) - assert(endsWith(filePath, ".nwb", "IgnoreCase", true)) + assert(endsWith(filePath, [".nwb", ".zarr"], "IgnoreCase", true)) end end diff --git a/+tests/+fixtures/createZarr3TestFile.m b/+tests/+fixtures/createZarr3TestFile.m new file mode 100644 index 000000000..a55762a6d --- /dev/null +++ b/+tests/+fixtures/createZarr3TestFile.m @@ -0,0 +1,92 @@ +function fixturePath = createZarr3TestFile(rootFolder) +% createZarr3TestFile - Build a small Zarr v3 NWB-like fixture for tests. +% +% fixturePath = createZarr3TestFile(rootFolder) creates a Zarr v3 store +% under rootFolder directly via the zarr-matlab package (independent of +% io.backend.zarr3.Zarr3Writer, so io.backend.zarr3.Zarr3Reader tests are +% not circularly validated only against the writer), and returns the path +% to the store. +% +% The pixel_mask compound dataset is built via zarr.metadata.ArrayMetadata +% + a direct zarr.Array construction rather than zarr.create, because +% zarr.create's public API has no way to pass a "structured" data_type's +% field configuration (its dtype argument is a plain scalar string) -- +% the only way to create a "structured" array today, matching how +% zarr-matlab's own test suite does it (see +% TestStructuredDtype>structuredArrayEndToEnd in zarr-matlab). + + arguments + rootFolder (1,1) string + end + + fixturePath = fullfile(rootFolder, "fixture.zarr"); + + root = zarr.create_group(fixturePath, Attributes=struct(... + 'nwb_version', "2.7.0", ... + 'x_specloc', "/specifications")); + + root.createArray("identifier", [], "string").write("ZARR3_FIXTURE"); + root.createGroup("specifications"); + + acquisitionGroup = root.createGroup("acquisition"); + esGroup = acquisitionGroup.createGroup("es"); + % Stored in numpy/row-major order ([29 4], i.e. timepoints x channels), + % as a real Python NWB Zarr v3 writer would; io.backend.zarr3.Zarr3Reader + % reverses rank->=2 shape/data back to the MatNWB-facing [4 29] + % (channels x timepoints) dims read by Zarr3ReaderTest/Zarr3LazyArrayTest. + esGroup.createArray("data", [29 4], "single").write(reshape(single(1:116), [4 29]).'); + + unitsGroup = root.createGroup("units"); + unitsGroup.createArray("spike_times", 5, "double").write([1.1 2.2 3.3 4.4 5.5]); + spikeTimesIndexGroup = unitsGroup.createGroup("spike_times_index"); + spikeTimesIndexGroup.setAttr("target", ... + struct('zarr_dtype', "object", 'value', struct('path', "/units/spike_times"))); + + generalGroup = root.createGroup("general"); + electrophysGroup = generalGroup.createGroup("extracellular_ephys"); + electrodesGroup = electrophysGroup.createGroup("electrodes"); + electrodesGroup.createArray("location", 4, "string").write(repmat("brain", 4, 1)); + electrodesGroup.createArray("id", 4, "int64").write(int64([0; 1; 2; 3])); + + devicesGroup = generalGroup.createGroup("devices"); + devicesGroup.createGroup("array"); + + shank0Group = electrophysGroup.createGroup("shank0"); + shank0Group.setAttr("zarr_link", ... + {struct('name', "device", 'source', ".", 'path', "/general/devices/array")}); + + processingGroup = root.createGroup("processing"); + ophysGroup = processingGroup.createGroup("ophys"); + planeSegmentationGroup = ophysGroup.createGroup("PlaneSegmentation"); + createPixelMaskArray(planeSegmentationGroup); + + zarr.consolidate_metadata(root.store); +end + +function createPixelMaskArray(parentGroup) +% createPixelMaskArray - Write a 3-record "structured" (compound) pixel_mask +% array (x uint32, y uint32, weight float32) as a child of parentGroup, +% matching a real hdmf-zarr PlaneSegmentation.pixel_mask column. + + info = zarr.internal.dtype_info(struct('name', "structured", 'configuration', struct( ... + 'fields', {{{'x', 'uint32'}; {'y', 'uint32'}; {'weight', 'float32'}}}))); + + numRecords = 3; + meta = zarr.metadata.ArrayMetadata(); + meta.shape = numRecords; + meta.dataType = "structured"; + meta.dataTypeConfig = info.config; + meta.chunkShape = numRecords; + meta.fillValue = struct('x', uint32(0), 'y', uint32(0), 'weight', single(0)); + meta.codecs = {zarr.codecs.BytesCodec()}; + + arrayPath = parentGroup.path + "/pixel_mask"; + parentGroup.store.set(arrayPath + "/zarr.json", unicode2native(char(meta.toJsonText()), 'UTF-8')); + + records(1, 1) = struct('x', uint32(0), 'y', uint32(0), 'weight', single(0.5)); + records(2, 1) = struct('x', uint32(1), 'y', uint32(1), 'weight', single(0.6)); + records(3, 1) = struct('x', uint32(2), 'y', uint32(2), 'weight', single(0.7)); + + pixelMaskArray = zarr.Array(parentGroup.store, arrayPath, meta); + pixelMaskArray.write(records); +end diff --git a/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m b/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m new file mode 100644 index 000000000..82e5b6abe --- /dev/null +++ b/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m @@ -0,0 +1,112 @@ +classdef Zarr3LazyArrayTest < matlab.unittest.TestCase + + properties (Access = private) + FixturePath (1,1) string + DatasetPath = "/acquisition/es/data" + end + + methods (TestClassSetup) + function setupZarrFixture(testCase) + tests.util.assumeZarr3Support(testCase) + + import matlab.unittest.fixtures.PathFixture + import matlab.unittest.fixtures.TemporaryFolderFixture + + testCase.applyFixture(PathFixture(tests.util.getZarr3MatlabPath())); + + tempFixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.FixturePath = tests.fixtures.createZarr3TestFile(tempFixture.Folder); + end + end + + methods (Test) + function loadDataAndMetadata(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + expectedData = reshape(single(1:116), [4 29]); + + testCase.verifyEqual(lazyArray.dims, [4 29]); + testCase.verifyEqual(lazyArray.maxDims, [4 29]); + testCase.verifyEqual(lazyArray.dataType, 'single'); + testCase.verifyEqual(lazyArray.load_h5_style(), expectedData); + end + + function loadPartialDataWithH5StyleSelection(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + partialData = lazyArray.load_h5_style([1 2], [2 3], [2 4]); + + testCase.verifyEqual(partialData, fullData(1:2:3, 2:4:10)); + end + + function dataStubSupportsSimpleIndexing(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + expectedData = reshape(single(1:116), [4 29]); + dataStub = types.untyped.DataStub(... + testCase.FixturePath, testCase.DatasetPath, [], [], lazyArray); + + testCase.verifyEqual(dataStub.load(), expectedData); + testCase.verifyEqual(dataStub(1:3, 2), expectedData(1:3, 2)); + end + + function integer1dDatasetHasCorrectMatlabType(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, ... + "/general/extracellular_ephys/electrodes/id"); + testCase.verifyEqual(lazyArray.dataType, 'int64'); + end + + function loadWithInfCountReadsToEnd(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + partialData = lazyArray.load_h5_style([2 3], [Inf Inf]); + + testCase.verifyEqual(partialData, fullData(2:end, 3:end)); + end + + function loadMatStyleIrregularSelectionFallsBackToFullRead(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + result = lazyArray.load_mat_style([1 2 4], 1:29); + + testCase.verifyEqual(result, fullData([1 2 4], 1:29)); + end + + function loadMatStyleUsesPartialReadForRegularSelection(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + + testCase.verifyEqual(... + lazyArray.load_mat_style(1:2:3, 2:4:10), ... + fullData(1:2:3, 2:4:10)); + end + + function compoundDatasetHasStructTypeDescriptor(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + + testCase.verifyEqual(lazyArray.dims, 3); + testCase.verifyEqual(lazyArray.dataType, ... + struct('x', 'uint32', 'y', 'uint32', 'weight', 'single')); + end + + function compoundLoadH5StyleReturnsStructOfArrays(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + data = lazyArray.load_h5_style(); + + testCase.verifyClass(data, "struct"); + testCase.verifyEqual(data.x, uint32([0; 1; 2])); + testCase.verifyEqual(data.weight, single([0.5; 0.6; 0.7])); + end + + function compoundLoadMatStyleReturnsTable(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + selectedRecords = lazyArray.load_mat_style(2:3); + + testCase.verifyClass(selectedRecords, "table"); + testCase.verifyEqual(selectedRecords.x, uint32([1; 2])); + testCase.verifyEqual(selectedRecords.weight, single([0.6; 0.7])); + end + end +end diff --git a/+tests/+unit/+io/+backend/Zarr3ReaderTest.m b/+tests/+unit/+io/+backend/Zarr3ReaderTest.m new file mode 100644 index 000000000..4a3aaadf7 --- /dev/null +++ b/+tests/+unit/+io/+backend/Zarr3ReaderTest.m @@ -0,0 +1,140 @@ +classdef Zarr3ReaderTest < matlab.unittest.TestCase + + properties (Access = private) + FixturePath (1,1) string + end + + methods (TestClassSetup) + function setupZarrFixture(testCase) + tests.util.assumeZarr3Support(testCase) + + import matlab.unittest.fixtures.PathFixture + import matlab.unittest.fixtures.TemporaryFolderFixture + + testCase.applyFixture(PathFixture(tests.util.getZarr3MatlabPath())); + + tempFixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.FixturePath = tests.fixtures.createZarr3TestFile(tempFixture.Folder); + end + end + + methods (Test) + function readRootInfoAndSchemaVersion(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + rootInfo = reader.readRootInfo(); + + testCase.verifyEqual(rootInfo.Name, '/'); + testCase.verifyEqual(reader.getSchemaVersion(), "2.7.0"); + testCase.verifyEqual(reader.getEmbeddedSpecLocation(), "/specifications"); + testCase.verifyTrue(any(strcmp({rootInfo.Groups.Name}, '/general'))); + end + + function readNodeInfoIncludesLinks(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + nodeInfo = reader.readNodeInfo("/general/extracellular_ephys/shank0"); + + testCase.verifyEqual(nodeInfo.Name, '/general/extracellular_ephys/shank0'); + testCase.verifyEqual(numel(nodeInfo.Links), 1); + testCase.verifyEqual(nodeInfo.Links(1).Name, 'device'); + testCase.verifyEqual(nodeInfo.Links(1).Type, 'soft link'); + testCase.verifyEqual(string(nodeInfo.Links(1).Value{1}), "/general/devices/array"); + + % The reserved zarr_link attribute must not leak into Attributes. + testCase.verifyEmpty(nodeInfo.Attributes); + end + + function readAttributeValueConvertsObjectReference(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + nodeInfo = reader.readNodeInfo("/units/spike_times_index"); + attributeInfo = nodeInfo.Attributes(strcmp({nodeInfo.Attributes.Name}, 'target')); + attributeValue = reader.readAttributeValue(attributeInfo, "/units/spike_times_index"); + + testCase.verifyClass(attributeValue, "types.untyped.ObjectView"); + testCase.verifyEqual(string(attributeValue.path), "/units/spike_times"); + end + + function readDatasetValueReturnsScalarString(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + rootInfo = reader.readRootInfo(); + datasetInfo = rootInfo.Datasets(strcmp({rootInfo.Datasets.Name}, 'identifier')); + datasetValue = reader.readDatasetValue(datasetInfo, "/identifier"); + + testCase.verifyClass(datasetValue, "char"); + testCase.verifyEqual(datasetValue, 'ZARR3_FIXTURE'); + end + + function readNonScalarDatasetValueReturnsDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + datasetInfo = reader.readNodeInfo("/acquisition/es/data"); + datasetValue = reader.readDatasetValue(datasetInfo, "/acquisition/es/data"); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyEqual(datasetValue.dims, [4 29]); + testCase.verifyEqual(datasetValue.load(), reshape(single(1:116), [4 29])); + end + + function read1dDatasetReturnsDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + datasetInfo = reader.readNodeInfo("/units/spike_times"); + datasetValue = reader.readDatasetValue(datasetInfo, "/units/spike_times"); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyEqual(datasetValue.dims, 5); + testCase.verifyEqual(datasetValue.load(), [1.1 2.2 3.3 4.4 5.5]'); + end + + function readStringArrayDatasetContainsExpectedValues(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + locationPath = "/general/extracellular_ephys/electrodes/location"; + datasetInfo = reader.readNodeInfo(locationPath); + datasetValue = reader.readDatasetValue(datasetInfo, locationPath); + + loadedValue = datasetValue.load(); + if iscell(loadedValue) + loadedValue = string(loadedValue); + end + + testCase.verifyEqual(numel(loadedValue), 4); + testCase.verifyTrue(all(string(loadedValue) == "brain")); + end + + function readIntegerDatasetHasCorrectMatlabType(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + idPath = "/general/extracellular_ephys/electrodes/id"; + datasetInfo = reader.readNodeInfo(idPath); + datasetValue = reader.readDatasetValue(datasetInfo, idPath); + + testCase.verifyEqual(datasetValue.dataType, 'int64'); + testCase.verifyEqual(datasetValue.load(), int64([0; 1; 2; 3])); + end + + function readCompoundDatasetReturnsCompoundDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + pixelMaskPath = "/processing/ophys/PlaneSegmentation/pixel_mask"; + datasetInfo = reader.readNodeInfo(pixelMaskPath); + datasetValue = reader.readDatasetValue(datasetInfo, pixelMaskPath); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyTrue(datasetValue.isCompoundType()); + testCase.verifyEqual(datasetValue.dataType, ... + struct('x', 'uint32', 'y', 'uint32', 'weight', 'single')); + + loadedValue = datasetValue.load(); + testCase.verifyClass(loadedValue, "struct"); + testCase.verifyEqual(loadedValue.x, uint32([0; 1; 2])); + testCase.verifyEqual(loadedValue.y, uint32([0; 1; 2])); + testCase.verifyEqual(loadedValue.weight, single([0.5; 0.6; 0.7])); + + selectedRecords = datasetValue.load(1, 2); + testCase.verifyClass(selectedRecords, "table"); + testCase.verifyEqual(selectedRecords.x, uint32([0; 1])); + end + + function readNodeInfoThrowsForMissingNode(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + testCase.verifyError(... + @() reader.readNodeInfo("/does/not/exist"), ... + "NWB:Zarr3Reader:NodeNotFound"); + end + end +end diff --git a/+tests/+util/assumeZarr3Support.m b/+tests/+util/assumeZarr3Support.m new file mode 100644 index 000000000..30faeede3 --- /dev/null +++ b/+tests/+util/assumeZarr3Support.m @@ -0,0 +1,15 @@ +function assumeZarr3Support(testCase) +% assumeZarr3Support - Skip the calling test unless Zarr v3 support is available. +% +% assumeZarr3Support(testCase) filters the calling test (via assumeTrue) +% when the zarr-matlab package is not on a discoverable path. +% +% See also tests.util.getZarr3MatlabPath + + arguments + testCase (1,1) matlab.unittest.TestCase + end + + testCase.assumeTrue(strlength(tests.util.getZarr3MatlabPath()) > 0, ... + "zarr-matlab package not found (set ZARR3_MATLAB_PATH or run setup).") +end diff --git a/+tests/+util/getZarr3MatlabPath.m b/+tests/+util/getZarr3MatlabPath.m new file mode 100644 index 000000000..ef8c80159 --- /dev/null +++ b/+tests/+util/getZarr3MatlabPath.m @@ -0,0 +1,46 @@ +function packagePath = getZarr3MatlabPath() +% getZarr3MatlabPath - Resolve the path to the zarr-matlab package. +% +% packagePath = getZarr3MatlabPath() returns the location of the +% zarr-matlab package (providing the `zarr` namespace used by +% io.backend.zarr3.Zarr3Reader / Zarr3Writer: zarr.open, zarr.create, +% zarr.create_group, ...). Resolution order: +% +% 1. Already resolvable on the MATLAB path (e.g. installed by +% matbox.installRequirements, which places it in an add-ons folder +% outside this repo and adds it to the path directly). +% 2. The ZARR3_MATLAB_PATH environment variable, if set. +% 3. The default install location created by setup +% (external_packages/zarr-matlab). +% +% Returns "" if no candidate folder exists, allowing callers to skip +% Zarr v3 tests gracefully. +% +% See also setup + + packagePath = ""; + + % `exist(name, "file")` does not reliably resolve dotted package-function + % names (see io.backend.zarr3.internal.ensureAvailable), so `which` is + % used instead. zarr.open.m lives at /+zarr/open.m. + zarrOpenLocation = which("zarr.open"); + if ~isempty(zarrOpenLocation) + packagePath = string(fileparts(fileparts(zarrOpenLocation))); + return + end + + candidates = string.empty; + envPath = string(getenv("ZARR3_MATLAB_PATH")); + if strlength(envPath) > 0 + candidates(end+1) = envPath; + end + candidates(end+1) = fullfile(misc.getMatnwbDir(), ... + "external_packages", "zarr-matlab"); + + for candidate = candidates + if isfolder(fullfile(candidate, "+zarr")) + packagePath = candidate; + return + end + end +end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 41162e273..9ed06ab5c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -130,16 +130,20 @@ jobs: release: ${{ matrix.matlab-version }} cache: ${{ env.USE_CACHE }} + - name: Install MatBox + uses: ehennestad/matbox-actions/install-matbox@v1 + - name: Run tests uses: matlab-actions/run-command@v3 with: command: | + matbox.installRequirements('.'); setenv("SKIP_PYNWB_TESTS", ... num2str(${{ matrix.skip-pynwb-tests }})) setenv("SKIP_NWBINSPECTOR_TEST", ... num2str(${{ matrix.skip-nwbinspector-test }})) pyenv("ExecutionMode", "OutOfProcess"); - results = assertSuccess(nwbtest('ReportOutputFolder', '.', 'ProduceCodeCoverage', ~${{ env.SKIP_COVERAGE }} )); + results = assertSuccess(nwbtest('ReportOutputFolder', '.', 'ProduceCodeCoverage', ~${{ env.SKIP_COVERAGE }} )); assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results diff --git a/.gitignore b/.gitignore index 4dd70e8cd..ae0808cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ nwbtest.env docs/build docs/reports + +# External dependencies installed by matbox.installRequirements +# (e.g. zarr-matlab, see requirements.txt) +external_packages/zarr-matlab diff --git a/nwbRead.m b/nwbRead.m index 605da5302..8d727fd8e 100644 --- a/nwbRead.m +++ b/nwbRead.m @@ -97,7 +97,7 @@ 'attributes', {{'.specloc', 'object_id'}},... 'groups', {{}}); if ~isempty(specLocation) - blackList.groups{end+1} = specLocation; + blackList.groups{end+1} = char(specLocation); end softLinkWarningResetObj = types.untyped.SoftLink.disablePathDeprecationWarning(); %#ok diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..88520e4fe --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +https://github.com/catalystneuro/zarr-matlab