diff --git a/+io/+backend/+base/Writer.m b/+io/+backend/+base/Writer.m index 313546a8f..61894ff70 100644 --- a/+io/+backend/+base/Writer.m +++ b/+io/+backend/+base/Writer.m @@ -33,6 +33,27 @@ function writeAttribute(obj, attributePath, value, varargin) %#ok io.backend.base.Writer.throwNotImplemented("writeAttribute") end + function specLocation = getEmbeddedSpecLocation(obj) %#ok + % getEmbeddedSpecLocation - Return the location of embedded schema + % specifications, or '' if none are embedded yet. Mirrors + % io.backend.base.Reader.getEmbeddedSpecLocation for the write + % side (needed when editing a file that may already embed specs). + specLocation = ''; + io.backend.base.Writer.throwNotImplemented("getEmbeddedSpecLocation") + end + + function groupNames = listChildGroupNames(obj, groupPath) %#ok + % listChildGroupNames - Return the names of immediate child + % groups (not datasets) under groupPath. + groupNames = {}; + io.backend.base.Writer.throwNotImplemented("listChildGroupNames") + end + + function deleteGroup(obj, groupPath) %#ok + % deleteGroup - Delete the group at groupPath and its contents. + io.backend.base.Writer.throwNotImplemented("deleteGroup") + end + function close(obj) %#ok % Default no-op. Concrete backends can override when they own % resources that should be released explicitly. diff --git a/+io/+backend/+hdf5/HDF5Writer.m b/+io/+backend/+hdf5/HDF5Writer.m index 02a7f0db0..d2b2f410e 100644 --- a/+io/+backend/+hdf5/HDF5Writer.m +++ b/+io/+backend/+hdf5/HDF5Writer.m @@ -82,6 +82,18 @@ function writeValue(obj, datasetPath, value, varargin) function writeAttribute(obj, attributePath, value, varargin) io.writeAttribute(obj.H5FileId, attributePath, value, varargin{:}); end + + function specLocation = getEmbeddedSpecLocation(obj) + specLocation = io.spec.internal.readEmbeddedSpecLocation(obj.H5FileId); + end + + function groupNames = listChildGroupNames(obj, groupPath) + groupNames = io.internal.h5.listGroupNames(obj.H5FileId, groupPath); + end + + function deleteGroup(obj, groupPath) + io.internal.h5.deleteGroup(obj.H5FileId, groupPath); + end end methods (Access = protected) diff --git a/+io/+backend/+zarr3/+internal/ensureAvailable.m b/+io/+backend/+zarr3/+internal/ensureAvailable.m new file mode 100644 index 000000000..6242adcb0 --- /dev/null +++ b/+io/+backend/+zarr3/+internal/ensureAvailable.m @@ -0,0 +1,25 @@ +function ensureAvailable() +% ensureAvailable - Validate that the zarr-matlab package is on path. +% +% See also: https://github.com/catalystneuro/zarr-matlab + + persistent isValidated + + if isequal(isValidated, true) + return + end + + % `exist(name, "file")` does not reliably resolve dotted package-function + % names (it returns 0 even when the function is on the path), so + % availability is checked with `which` instead. + requiredFunctions = ["zarr.open", "zarr.create", "zarr.create_group"]; + isMissing = arrayfun(@(name) isempty(which(name)), requiredFunctions); + + if any(isMissing) + error("NWB:Zarr3:DependencyMissing", ... + "The `zarr-matlab` package is required on the MATLAB path. Missing function(s): %s", ... + strjoin(requiredFunctions(isMissing), ", ")) + end + + isValidated = true; +end diff --git a/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m b/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m new file mode 100644 index 000000000..b2afaea1c --- /dev/null +++ b/+io/+backend/+zarr3/@Zarr3LazyArray/Zarr3LazyArray.m @@ -0,0 +1,311 @@ +classdef Zarr3LazyArray < io.backend.base.LazyArray +% Zarr3LazyArray - Zarr v3-backed lazy dataset access implementation. +% +% Zarr v3 stores may be written by Python NWB tools using numpy/row-major +% shape order. For rank >= 2 arrays, dims and data are reversed to match +% MatNWB's H5-style convention (see io.internal.zarr3.normalizeDatasetDimensions). +% A rank-1 array needs no correction: zarr-matlab already returns it as a +% MATLAB column vector. + + properties (Access = private) + ArrayNode = [] + FieldSemantics = [] + end + + methods + function obj = Zarr3LazyArray(filename, datasetPath, dims, dataType, fieldSemantics) + arguments + filename (1,1) string + datasetPath (1,1) string + dims double = [] + dataType = [] + fieldSemantics = [] + end + obj@io.backend.base.LazyArray(filename, datasetPath, dims, dataType); + obj.FieldSemantics = fieldSemantics; + end + + function refreshSizeInfo(obj) + arrayNode = obj.resolveArray(); + dims = double(arrayNode.shape); + if numel(dims) >= 2 + dims = fliplr(dims); + end + obj.setSizeInfo(dims, dims); + end + + function dataType = resolveDataType(obj) + % For a "structured" (compound) array, dataType is a compound + % type descriptor struct (see + % io.internal.zarr3.getCompoundTypeDescriptor), not a plain + % class name -- required by + % types.util.checkDtype/types.untyped.DataStub.isCompoundType. + % io.backend.zarr3.Zarr3Reader normally passes this in at + % construction (avoiding this lazy path entirely); it is + % reproduced here only as a fallback for a Zarr3LazyArray + % constructed directly without one. + arrayNode = obj.resolveArray(); + info = zarr.internal.dtype_info(arrayNode.meta.dataType, arrayNode.meta.dataTypeConfig); + if info.zarrType == "structured" + dataType = io.internal.zarr3.getCompoundTypeDescriptor(info, obj.getFieldSemantics()); + else + dataType = char(info.matlabClass); + end + end + + function data = load_h5_style(obj, varargin) + if isempty(varargin) + data = obj.readAllData(); + return + end + + assert(length(varargin) ~= 1, 'NWB:DataStub:InvalidNumArguments',... + 'calling load_h5_style with a single space id is no longer supported.'); + + start = varargin{1}; + count = varargin{2}; + if length(varargin) >= 3 + stride = varargin{3}; + else + stride = ones(size(start)); + end + data = obj.readPartialData(start, count, stride); + end + + function data = load_mat_style(obj, varargin) + if isempty(varargin) + data = obj.readAllData(); + if isstruct(data) + data = struct2table(data); + end + return + end + + [isSupported, fullSelection] = obj.tryBuildRegularSelection(varargin); + if isSupported + [start, count, stride] = obj.selectionToReadParameters(fullSelection); + data = obj.readPartialData(start, count, stride); + if isstruct(data) + % Record selection already happened during the partial + % read; matching io.backend.hdf5.@HDF5LazyArray's + % compound convention, the selected records are + % returned as a table rather than reshaped further. + data = struct2table(data); + else + data = obj.applySelectionShape(data, varargin); + end + else + data = obj.readAllData(); + if isstruct(data) + data = struct2table(data); + data = data(varargin{:}, :); + else + data = data(varargin{:}); + end + end + end + end + + methods (Access = private) + function arrayNode = resolveArray(obj) + if isempty(obj.ArrayNode) + relativePath = io.internal.zarr3.stripLeadingSlash(obj.DatasetPath); + obj.ArrayNode = zarr.open(obj.Filename, Path=relativePath); + end + arrayNode = obj.ArrayNode; + end + + function fieldSemantics = getFieldSemantics(obj) + if ~isempty(obj.FieldSemantics) && obj.FieldSemantics.Count > 0 + fieldSemantics = obj.FieldSemantics; + else + fieldSemantics = io.internal.zarr3.getCompoundFieldSemantics(obj.resolveArray().attrs); + end + end + + function data = postProcessCompound(obj, data) + % postProcessCompound - Convert zarr-matlab's array-of-records + % (one struct per element) into the "struct of arrays" shape + % (one scalar struct, each field an Nx1 array) that + % io.backend.hdf5.@HDF5LazyArray/load_h5_style.m produces via + % io.parseCompound, decoding any field tagged as an object + % reference (see io.internal.zarr3.getCompoundFieldSemantics) + % into a types.untyped.ObjectView array along the way. + if ~isstruct(data) + return + end + + fieldSemantics = obj.getFieldSemantics(); + fieldNames = fieldnames(data); + n = numel(data); + converted = struct(); + for iField = 1:numel(fieldNames) + name = fieldNames{iField}; + rawValues = {data.(name)}; + if isKey(fieldSemantics, name) && fieldSemantics(name) == "object" + values = types.untyped.ObjectView.empty(0, 0); + for iValue = 1:n + decoded = jsondecode(char(rawValues{iValue})); + values(iValue) = types.untyped.ObjectView(decoded.path); + end + converted.(name) = reshape(values, n, 1); + else + converted.(name) = reshape([rawValues{:}], n, 1); + end + end + data = converted; + end + + function data = readAllData(obj) + arrayNode = obj.resolveArray(); + data = arrayNode.read(); + data = io.internal.zarr3.normalizeDatasetDimensions(data, numel(arrayNode.shape)); + data = obj.postProcessCompound(data); + end + + function data = readPartialData(obj, start, count, stride) + arrayNode = obj.resolveArray(); + if any(isinf(count)) + count(isinf(count)) = obj.dims(isinf(count)) - start(isinf(count)) + 1; + end + + % start/count/stride arrive in MatNWB's H5-style dims order + % (obj.dims); reverse to raw Zarr/numpy order for rank >= 2 + % before calling zarr.Array.read (see refreshSizeInfo). + rank = numel(start); + if rank >= 2 + rawStart = fliplr(start); + rawCount = fliplr(count); + rawStride = fliplr(stride); + else + rawStart = start; + rawCount = count; + rawStride = stride; + end + + if all(rawStride == 1) + data = arrayNode.read(rawStart, rawCount); + else + % zarr.Array.read has no native stride support: read the + % contiguous bounding box spanning the strided selection, + % then subselect the stride in MATLAB. + boxedSpan = (rawCount - 1) .* rawStride + 1; + boxed = arrayNode.read(rawStart, boxedSpan); + selection = cell(1, numel(rawStart)); + for iDimension = 1:numel(rawStart) + selection{iDimension} = 1:rawStride(iDimension):boxedSpan(iDimension); + end + data = boxed(selection{:}); + end + data = io.internal.zarr3.normalizeDatasetDimensions(data, rank); + data = obj.postProcessCompound(data); + end + + function [isSupported, fullSelection] = tryBuildRegularSelection(obj, userSelection) + dataDimensions = obj.dims; + isSupported = true; + fullSelection = cell(1, length(dataDimensions)); + + if isscalar(userSelection) && isempty(userSelection{1}) + isSupported = false; + return + end + + if isscalar(userSelection) && ~ischar(userSelection{1}) + isSupported = false; + return + end + + isDanglingGroup = ischar(userSelection{end}); + for iDimension = 1:length(dataDimensions) + if iDimension > length(userSelection) && ~isDanglingGroup + fullSelection{iDimension} = 1; + elseif (iDimension > length(userSelection) && isDanglingGroup) ... + || ischar(userSelection{iDimension}) + fullSelection{iDimension} = 1:dataDimensions(iDimension); + else + selection = userSelection{iDimension}; + if ~obj.isRegularAscendingSelection(selection) + isSupported = false; + return + end + fullSelection{iDimension} = selection; + end + end + end + + function tf = isRegularAscendingSelection(~, selection) + tf = isnumeric(selection) ... + && isreal(selection) ... + && all(isfinite(selection)) ... + && all(selection > 0) ... + && all(selection == floor(selection)); + if ~tf + return + end + if isscalar(selection) + return + end + + stepSizes = diff(selection); + tf = all(stepSizes > 0) && numel(unique(stepSizes)) == 1; + end + + function [start, count, stride] = selectionToReadParameters(~, selection) + start = zeros(1, numel(selection)); + count = zeros(1, numel(selection)); + stride = ones(1, numel(selection)); + + for iDimension = 1:numel(selection) + currentSelection = selection{iDimension}; + start(iDimension) = currentSelection(1); + count(iDimension) = numel(currentSelection); + if numel(currentSelection) > 1 + stride(iDimension) = currentSelection(2) - currentSelection(1); + end + end + end + + function data = applySelectionShape(obj, data, userSelection) + expectedSize = obj.getExpectedSize(userSelection); + if isequal(size(data), expectedSize) + return + end + data = reshape(data, expectedSize); + end + + function expectedSize = getExpectedSize(obj, userSelection) + dataDimensions = obj.dims; + expectedSize = dataDimensions; + for iSelection = 1:length(userSelection) + if ~ischar(userSelection{iSelection}) + expectedSize(iSelection) = length(userSelection{iSelection}); + end + end + + if ischar(userSelection{end}) + selectedDimensionIndex = length(userSelection); + expectedSize = [expectedSize(1:(selectedDimensionIndex-1)), ... + prod(dataDimensions(selectedDimensionIndex:end))]; + else + expectedSize = expectedSize(1:length(userSelection)); + end + + if isscalar(userSelection) && isscalar(expectedSize) + if 1 < sum(1 < dataDimensions) + if ~ischar(userSelection{1}) && isrow(userSelection{1}) + expectedSize = [1 expectedSize]; + else + expectedSize = [expectedSize 1]; + end + else + if dataDimensions(1) == 1 + expectedSize = [1 expectedSize]; + else + expectedSize = [expectedSize 1]; + end + end + end + end + end +end diff --git a/+io/+backend/+zarr3/Zarr3Reader.m b/+io/+backend/+zarr3/Zarr3Reader.m new file mode 100644 index 000000000..8b1788570 --- /dev/null +++ b/+io/+backend/+zarr3/Zarr3Reader.m @@ -0,0 +1,244 @@ +classdef Zarr3Reader < io.backend.base.Reader + % Zarr3Reader - Reader implementation for local Zarr v3 stores. + % + % This reader is backed by the zarr-matlab package + % (https://github.com/catalystneuro/zarr-matlab), which must be on the + % MATLAB path, and reads Zarr v3 stores natively in MATLAB (no Python + % dependency). It pairs with io.backend.zarr3.Zarr3Writer for writing. + % + % Object references are represented as an attribute value struct + % `struct('zarr_dtype', "object", 'value', struct('path', targetPath))`, + % matching the convention used by hdmf-zarr for Zarr v2 stores. There is no published Zarr v3 convention for NWB, but + % this matches the "zarr_dtype" reference convention used by real + % hdmf-zarr-style Zarr v3 NWB exports (verified against example files), + % as well as the reader/writer pair's own round-trips. + % + % A dataset whose elements are themselves object references (e.g. a + % DynamicTable column of ElectrodeGroup references) is represented on + % disk as a plain Zarr "string" array whose elements are the same + % reference JSON, tagged via a "zarr_dtype":"object" attribute on the + % array itself (see io.internal.zarr3.buildNodeInfo); each element is + % decoded into a types.untyped.ObjectView object array. + % + % A compound (struct/table) dataset -- a Zarr v3 "structured" data_type, + % e.g. PlaneSegmentation's pixel_mask/voxel_mask, or + % TimeSeriesReferenceVectorData's response/stimulus columns -- is backed + % by io.backend.zarr3.Zarr3LazyArray; a field tagged "object" via the + % array's "zarr_dtype" attribute (see + % io.internal.zarr3.getCompoundFieldSemantics) is decoded into a + % types.untyped.ObjectView, matching the plain object-reference-array + % convention above. Requires zarr-matlab to support the Zarr v3 + % "structured" and "fixed_length_utf32" data types, which are unstable, + % unspecified zarr-python extensions -- see + % zarr.internal.dtype_info in zarr-matlab. + + properties (Access = private) + RootGroup = [] + RootInfoCache = [] + NodeInfoMap = containers.Map('KeyType', 'char', 'ValueType', 'any') + end + + methods + function obj = Zarr3Reader(filename) + obj@io.backend.base.Reader(filename); + end + + function version = getSchemaVersion(obj) + obj.ensureMetadataCache(); + attributes = obj.RootGroup.attrs; + if isfield(attributes, "nwb_version") + version = string(attributes.nwb_version); + else + error("NWB:Zarr3Reader:MissingSchemaVersion", ... + "The Zarr store `%s` does not define `nwb_version` in the root attributes.", ... + obj.Filename) + end + end + + function specLocation = getEmbeddedSpecLocation(obj) + obj.ensureMetadataCache(); + attributes = obj.RootGroup.attrs; + if isfield(attributes, "x_specloc") + specLocation = string(attributes.x_specloc); + elseif obj.RootGroup.isKey("specifications") + specLocation = "/specifications"; + else + specLocation = ""; + end + + if specLocation ~= "" && ~startsWith(specLocation, "/") + specLocation = "/" + specLocation; + end + end + + function node = readRootInfo(obj) + obj.ensureMetadataCache(); + node = obj.RootInfoCache; + end + + function node = readNodeInfo(obj, nodePath) + arguments + obj + nodePath (1,1) string + end + + obj.ensureMetadataCache(); + normalizedPath = obj.normalizeNodePath(nodePath); + if ~isKey(obj.NodeInfoMap, normalizedPath) + error("NWB:Zarr3Reader:NodeNotFound", ... + "Node `%s` was not found in `%s`.", normalizedPath, obj.Filename) + end + node = obj.NodeInfoMap(normalizedPath); + end + + function attributeValue = readAttributeValue(~, attributeInfo, ~) + if (ischar(attributeInfo.Datatype) || isstring(attributeInfo.Datatype)) ... + && strcmp(attributeInfo.Datatype, "object reference") + attributeValue = types.untyped.ObjectView(attributeInfo.Value.value.path); + else + attributeValue = attributeInfo.Value; + end + end + + function datasetValue = readDatasetValue(obj, datasetInfo, datasetPath) + dataDimensions = obj.getDatasetDims(datasetInfo); + isObjectReferenceArray = strcmp(datasetInfo.Datatype, "object"); + isStructuredArray = strcmp(datasetInfo.Datatype, "structured"); + % A true rank-0 array (this reader's own Zarr3Writer's scalar + % convention) or one explicitly marked "scalar" by hdmf-zarr's + % zarr_dtype hint (see io.internal.zarr3.buildNodeInfo) is read + % eagerly. Dataspace.Size == 1 alone is NOT a reliable scalar + % signal: hdmf-zarr represents a genuine NWB scalar property as + % a rank-1, length-1 array, which is indistinguishable by shape + % from a one-row VectorData column (e.g. a DynamicTable with a + % single row) -- collapsing the latter to a bare value would + % silently corrupt it (a char column's character count would + % be misread as its row count downstream). + isScalarMarked = isempty(dataDimensions) || strcmp(datasetInfo.Datatype, "scalar"); + if isObjectReferenceArray + datasetValue = obj.readObjectArrayValue(datasetPath); + elseif isStructuredArray + % Checked ahead of the scalar/eager branch below: a + % structured array can have prod(dataDimensions) == 1 (a + % single record, e.g. shape [1]) without being a "scalar" + % dataset in the ordinary sense. + datasetValue = obj.readStructuredValue(datasetPath, dataDimensions); + elseif isScalarMarked + datasetValue = obj.readEagerValue(datasetPath); + elseif any(dataDimensions == 0) + datasetValue = []; + else + matlabDataType = io.internal.zarr3.getMatlabDataType(datasetInfo.Datatype); + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + obj.Filename, datasetPath, dataDimensions, matlabDataType); + datasetValue = types.untyped.DataStub(... + obj.Filename, datasetPath, [], [], lazyArray); + end + end + end + + methods (Access = private) + function ensureMetadataCache(obj) + if isempty(obj.RootGroup) + io.backend.zarr3.internal.ensureAvailable() + obj.RootGroup = zarr.open(obj.Filename); + [obj.RootInfoCache, obj.NodeInfoMap] = io.internal.zarr3.buildNodeInfo(obj.RootGroup); + obj.RootInfoCache.Filename = char(obj.Filename); + end + end + + function normalizedPath = normalizeNodePath(~, nodePath) + normalizedPath = char(nodePath); + if isempty(normalizedPath) + normalizedPath = '/'; + elseif normalizedPath(1) ~= '/' + normalizedPath = ['/' normalizedPath]; + end + end + + function dataDimensions = getDatasetDims(~, datasetInfo) + % Dataspace.Size is the raw (Zarr/numpy-order) shape; reverse it + % for rank >= 2 to match MatNWB's H5-style dims convention (see + % io.internal.zarr3.normalizeDatasetDimensions). + if isfield(datasetInfo, "Dataspace") && isfield(datasetInfo.Dataspace, "Size") + dataDimensions = double(datasetInfo.Dataspace.Size); + else + dataDimensions = []; + end + + if numel(dataDimensions) >= 2 + dataDimensions = fliplr(dataDimensions); + end + end + + function datasetValue = readEagerValue(obj, datasetPath) + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + datasetValue = arrayNode.read(); + + if isstring(datasetValue) && isscalar(datasetValue) + datasetValue = char(datasetValue); + elseif iscell(datasetValue) && isscalar(datasetValue) + datasetValue = datasetValue{1}; + end + end + + function datasetValue = readStructuredValue(obj, datasetPath, dataDimensions) + % readStructuredValue - Wrap a "structured" (compound) array in + % a DataStub backed by io.backend.zarr3.Zarr3LazyArray. The + % DataStub's dataType is a compound type descriptor struct + % (field name -> MATLAB class name, or 'types.untyped.ObjectView' + % for a field tagged as a reference via the array's own + % "zarr_dtype" attribute; see + % io.internal.zarr3.getCompoundTypeDescriptor) rather than a + % plain class name, matching what + % types.util.checkDtype/types.untyped.DataStub.isCompoundType + % expect -- this alone is enough for schema validation to + % succeed without loading any data (see + % types.util.checkDtype>checkDtypeForCompoundDataset's + % DataStub fast path). + + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + info = zarr.internal.dtype_info(arrayNode.meta.dataType, arrayNode.meta.dataTypeConfig); + fieldSemantics = io.internal.zarr3.getCompoundFieldSemantics(arrayNode.attrs); + typeDescriptor = io.internal.zarr3.getCompoundTypeDescriptor(info, fieldSemantics); + + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + obj.Filename, datasetPath, dataDimensions, typeDescriptor, fieldSemantics); + datasetValue = types.untyped.DataStub(... + obj.Filename, datasetPath, [], [], lazyArray); + end + + function datasetValue = readObjectArrayValue(obj, datasetPath) + % readObjectArrayValue - Decode an array whose elements are + % JSON-encoded object references (Datatype "object"; see + % io.internal.zarr3.buildNodeInfo) into a types.untyped.ObjectView + % object array (matching io.parseReference's shape for HDF5 + % reference datasets, which types.util.checkDtype requires -- + % a cell array of ObjectView is not an accepted dtype). Each + % element is stored as a zarr "string" containing the same + % {"source":...,"path":...} JSON convention used for + % object-reference attributes (io.internal.zarr3.convertAttributes), + % but is not auto-decoded to a struct by zarr-matlab since the + % array's own Zarr v3 data_type is plain "string". + + relativePath = io.internal.zarr3.stripLeadingSlash(datasetPath); + arrayNode = zarr.open(obj.Filename, Path=relativePath); + rawValues = string(arrayNode.read()); + + datasetValue = types.untyped.ObjectView.empty(0, 0); + for iValue = 1:numel(rawValues) + datasetValue(iValue) = io.backend.zarr3.Zarr3Reader.decodeObjectReferenceElement(rawValues(iValue)); + end + datasetValue = reshape(datasetValue, size(rawValues)); + end + end + + methods (Static, Access = private) + function objectView = decodeObjectReferenceElement(rawElement) + decoded = jsondecode(char(rawElement)); + objectView = types.untyped.ObjectView(decoded.path); + end + end +end diff --git a/+io/+backend/BackendFactory.m b/+io/+backend/BackendFactory.m index 09c2c87fa..74fd23ab5 100644 --- a/+io/+backend/BackendFactory.m +++ b/+io/+backend/BackendFactory.m @@ -35,6 +35,8 @@ case "auto" if io.backend.BackendFactory.isHDF5File(filename) reader = io.backend.hdf5.HDF5Reader(filename); + elseif io.backend.BackendFactory.isZarr3Directory(filename) + reader = io.backend.zarr3.Zarr3Reader(filename); else error("NWB:BackendFactory:UnsupportedFormat", ... "No supported reader found for `%s`.", filename) @@ -45,6 +47,12 @@ "`%s` is not a valid HDF5 file.", filename) end reader = io.backend.hdf5.HDF5Reader(filename); + case "zarr3" + if ~io.backend.BackendFactory.isZarr3Directory(filename) + error("NWB:BackendFactory:InvalidZarr3", ... + "`%s` is not a supported local Zarr v3 directory store.", filename) + end + reader = io.backend.zarr3.Zarr3Reader(filename); otherwise error("NWB:BackendFactory:UnsupportedBackend", ... "Unsupported backend `%s`.", storageBackend) @@ -66,6 +74,8 @@ case "auto" if io.backend.BackendFactory.isHDF5File(filename) lazyArray = io.backend.hdf5.HDF5LazyArray(filename, datasetPath, dims, dataType); + elseif io.backend.BackendFactory.isZarr3Directory(filename) + lazyArray = io.backend.zarr3.Zarr3LazyArray(filename, datasetPath, dims, dataType); else error("NWB:BackendFactory:UnsupportedFormat", ... "No supported lazy array backend found for `%s`.", filename) @@ -76,6 +86,12 @@ "`%s` is not a valid HDF5 file.", filename) end lazyArray = io.backend.hdf5.HDF5LazyArray(filename, datasetPath, dims, dataType); + case "zarr3" + if ~io.backend.BackendFactory.isZarr3Directory(filename) + error("NWB:BackendFactory:InvalidZarr3", ... + "`%s` is not a supported local Zarr v3 directory store.", filename) + end + lazyArray = io.backend.zarr3.Zarr3LazyArray(filename, datasetPath, dims, dataType); otherwise error("NWB:BackendFactory:UnsupportedBackend", ... "Unsupported backend `%s`.", storageBackend) @@ -105,5 +121,32 @@ end end end + + function tf = isZarr3Directory(filename) + arguments + filename (1,1) string + end + + tf = false; + if startsWith(filename, "s3://", "IgnoreCase", true) || ~isfolder(filename) + return + end + + if ~endsWith(filename, ".zarr", "IgnoreCase", true) + return + end + + rootMetadataFile = fullfile(filename, "zarr.json"); + if ~isfile(rootMetadataFile) + return + end + + try + rootMetadata = jsondecode(fileread(rootMetadataFile)); + tf = isfield(rootMetadata, "zarr_format") && isequal(rootMetadata.zarr_format, 3); + catch + tf = false; + end + end end end diff --git a/+io/+internal/+zarr3/buildNodeInfo.m b/+io/+internal/+zarr3/buildNodeInfo.m new file mode 100644 index 000000000..b625e360a --- /dev/null +++ b/+io/+internal/+zarr3/buildNodeInfo.m @@ -0,0 +1,130 @@ +function [rootInfo, nodeInfoMap] = buildNodeInfo(rootGroup) +% buildNodeInfo - Build an h5info-like node tree from a zarr.Group root. +% +% [rootInfo, nodeInfoMap] = buildNodeInfo(rootGroup) walks the hierarchy +% below rootGroup (a zarr.Group opened via zarr.open, using its +% consolidated metadata when available) and returns: +% +% rootInfo - h5info-like struct for the root node (fields Name, Groups, +% Datasets, Links, Attributes), recursively populated. +% +% nodeInfoMap - containers.Map from absolute node path (char, leading +% '/') to the corresponding node info struct (group or dataset). +% +% The output shape mirrors h5info's, so io.parseGroup/io.parseDataset/ +% io.parseAttributes can consume it directly. + + nodeInfoMap = containers.Map('KeyType', 'char', 'ValueType', 'any'); + rootInfo = buildGroupInfo(rootGroup, "/", nodeInfoMap); +end + +function groupInfo = buildGroupInfo(group, groupPath, nodeInfoMap) + groupInfo = struct(... + 'Name', char(groupPath), ... + 'Groups', emptyGroupStruct(), ... + 'Datasets', emptyDatasetStruct(), ... + 'Links', emptyLinkStruct(), ... + 'Attributes', emptyAttributeStruct()); + + [attributes, links] = io.internal.zarr3.convertAttributes(group.attrs); + groupInfo.Attributes = attributes; + groupInfo.Links = links; + + [arrayNames, groupNames] = group.children(); + + for iArray = 1:numel(arrayNames) + childArray = group.item(arrayNames(iArray)); + childPath = joinPath(groupPath, arrayNames(iArray)); + datasetInfo = buildDatasetInfo(childArray, arrayNames(iArray)); + groupInfo.Datasets(end+1) = datasetInfo; %#ok + nodeInfoMap(char(childPath)) = datasetInfo; + end + + for iGroup = 1:numel(groupNames) + childGroup = group.item(groupNames(iGroup)); + childPath = joinPath(groupPath, groupNames(iGroup)); + childInfo = buildGroupInfo(childGroup, childPath, nodeInfoMap); + groupInfo.Groups(end+1) = childInfo; %#ok + end + + nodeInfoMap(char(groupPath)) = groupInfo; +end + +function datasetInfo = buildDatasetInfo(arrayNode, leafName) + shape = double(arrayNode.shape); + if isempty(shape) + dataspaceType = 'scalar'; + else + dataspaceType = 'simple'; + end + + datasetInfo = struct(... + 'Name', char(leafName), ... + 'Datatype', char(arrayNode.dtype), ... + 'Dataspace', struct('Size', shape, 'MaxSize', shape, 'Type', dataspaceType), ... + 'ChunkSize', double(arrayNode.chunkShape), ... + 'FillValue', arrayNode.meta.fillValue, ... + 'Filters', struct('Name', {}, 'Parameters', {}), ... + 'Attributes', emptyAttributeStruct()); + + % A "zarr_dtype" attribute (reserved; filtered out of Attributes below) + % is a legacy hdmf-zarr (v2) dtype hint. For plain numeric/string data + % it is normally redundant with (and less precise than) the native + % Zarr v3 `data_type`, so it is ignored -- except for two semantic + % markers with no native Zarr v3 equivalent, which override Datatype + % so io.backend.zarr3.Zarr3Reader can dispatch on them: + % "object" - an array of object references stored as JSON-encoded + % strings. + % "scalar" - hdmf-zarr represents an NWB scalar property (e.g. + % identifier, session_description) as a rank-1, length-1 Zarr + % array rather than a true rank-0 array, so shape alone cannot + % distinguish it from a genuine one-row VectorData column (e.g. a + % DynamicTable that happens to have a single row) -- both have + % Dataspace.Size == 1. Reading the latter eagerly as a bare scalar + % would silently collapse a 1-element char/string column, whose + % stray character count would then be misread as a table row + % count by DynamicTable height checks. + % For a "structured" (compound) array, this attribute is instead a + % per-field type descriptor list (a struct array), not a scalar + % string -- irrelevant here since the array's own Zarr v3 data_type + % ("structured") already identifies it unambiguously. + zarrDtypeAttr = []; + if isfield(arrayNode.attrs, 'zarr_dtype') + zarrDtypeAttr = arrayNode.attrs.zarr_dtype; + end + if ischar(zarrDtypeAttr) || isstring(zarrDtypeAttr) + zarrDtypeAttrText = string(zarrDtypeAttr); + if zarrDtypeAttrText == "object" + datasetInfo.Datatype = 'object'; + elseif zarrDtypeAttrText == "scalar" + datasetInfo.Datatype = 'scalar'; + end + end + + datasetInfo.Attributes = io.internal.zarr3.convertAttributes(arrayNode.attrs); +end + +function joinedPath = joinPath(parentPath, childName) + if parentPath == "/" + joinedPath = "/" + childName; + else + joinedPath = parentPath + "/" + childName; + end +end + +function groupStruct = emptyGroupStruct() + groupStruct = struct('Name', {}, 'Groups', {}, 'Datasets', {}, 'Links', {}, 'Attributes', {}); +end + +function datasetStruct = emptyDatasetStruct() + datasetStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, ... + 'ChunkSize', {}, 'FillValue', {}, 'Filters', {}, 'Attributes', {}); +end + +function attributeStruct = emptyAttributeStruct() + attributeStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, 'Value', {}); +end + +function linkStruct = emptyLinkStruct() + linkStruct = struct('Name', {}, 'Type', {}, 'Value', {}); +end diff --git a/+io/+internal/+zarr3/convertAttributes.m b/+io/+internal/+zarr3/convertAttributes.m new file mode 100644 index 000000000..e34aea87c --- /dev/null +++ b/+io/+internal/+zarr3/convertAttributes.m @@ -0,0 +1,100 @@ +function [attributes, links] = convertAttributes(rawAttributes) +% convertAttributes - Convert zarr-matlab attrs into h5info-like structures. +% +% [attributes, links] = convertAttributes(rawAttributes) converts the +% attrs struct exposed by a zarr.Group or zarr.Array (as returned by +% zarr-matlab) into: +% +% attributes - struct array with fields Name, Datatype, Dataspace, +% Value, matching the shape produced by h5info. Object references +% (see io.internal.zarr3.encodeObjectReference) are tagged with +% Datatype "object reference". +% +% links - struct array with fields Name, Type, Value, decoded from the +% reserved "zarr_link" attribute (soft/external link records written +% by io.backend.zarr3.Zarr3Writer). Non-group nodes never carry links, +% but the reserved attribute is filtered out regardless. +% +% This convention matches the shape produced by h5info, so that the same +% downstream parsing code (io.parseGroup, io.parseAttributes) can consume +% node info from any backend. + + attributes = emptyAttributeStruct(); + links = emptyLinkStruct(); + + if isempty(rawAttributes) || ~isstruct(rawAttributes) + return + end + + fieldNames = fieldnames(rawAttributes); + for iField = 1:numel(fieldNames) + name = fieldNames{iField}; + value = rawAttributes.(name); + + if strcmp(name, "zarr_link") + links = convertLinks(value); + continue + elseif strcmp(name, "x_specloc") + % Reserved marker for io.backend.zarr3.Zarr3Reader.getEmbeddedSpecLocation; + % not a schema attribute, so it is never promoted. + continue + elseif strcmp(name, "zarr_dtype") || strcmp(name, "x_ARRAY_DIMENSIONS") + % Reserved bookkeeping attributes written by hdmf-zarr-style + % exporters (dtype hint / xarray dimension names, the latter + % written to disk as "_ARRAY_DIMENSIONS" but renamed by + % jsondecode since a leading underscore is not a valid MATLAB + % identifier); not schema attributes. + continue + end + + if iscell(value) && ~isscalar(value) + % struct(...,'Value',value) would otherwise expand a non-scalar + % cell into a non-scalar struct array (one element per cell + % entry) instead of a single attribute whose Value is the cell. + value = {value}; + end + + attribute = struct('Name', name, 'Datatype', [], 'Dataspace', [], 'Value', value); + if isObjectReferenceValue(value) + attribute.Datatype = 'object reference'; + end + attributes(end+1) = attribute; %#ok + end +end + +function tf = isObjectReferenceValue(value) + tf = isstruct(value) && isscalar(value) ... + && isfield(value, 'zarr_dtype') ... + && strcmp(string(value.zarr_dtype), "object"); +end + +function links = convertLinks(rawLinks) + links = emptyLinkStruct(); + if isempty(rawLinks) + return + end + if ~iscell(rawLinks) + rawLinks = num2cell(rawLinks); + end + + for iLink = 1:numel(rawLinks) + rawLink = rawLinks{iLink}; + link = struct('Name', char(rawLink.name), 'Type', '', 'Value', []); + if strcmp(rawLink.source, '.') + link.Type = 'soft link'; + link.Value = {char(rawLink.path)}; + else + link.Type = 'external link'; + link.Value = {char(rawLink.source), char(rawLink.path)}; + end + links(end+1) = link; %#ok + end +end + +function attributeStruct = emptyAttributeStruct() + attributeStruct = struct('Name', {}, 'Datatype', {}, 'Dataspace', {}, 'Value', {}); +end + +function linkStruct = emptyLinkStruct() + linkStruct = struct('Name', {}, 'Type', {}, 'Value', {}); +end diff --git a/+io/+internal/+zarr3/getCompoundFieldSemantics.m b/+io/+internal/+zarr3/getCompoundFieldSemantics.m new file mode 100644 index 000000000..12f0bca7c --- /dev/null +++ b/+io/+internal/+zarr3/getCompoundFieldSemantics.m @@ -0,0 +1,32 @@ +function fieldSemantics = getCompoundFieldSemantics(attrs) +% getCompoundFieldSemantics - Per-field semantic type hints for a +% "structured" (compound) Zarr v3 array. +% +% fieldSemantics = getCompoundFieldSemantics(attrs) returns a +% containers.Map from field name (char) to semantic dtype (string), +% read from the array's own "zarr_dtype" attribute -- for a compound +% array this is a per-field type descriptor list (matching the +% hdmf-zarr v2 convention), e.g. +% [{"name":"idx_start","dtype":"int32"}, ..., +% {"name":"timeseries","dtype":"object"}]. +% +% The only semantic value consumed downstream is "object" (the field +% holds a JSON-encoded object reference rather than literal text/data), +% matching io.backend.zarr3.Zarr3Reader.readObjectArrayValue's +% convention for top-level arrays of references. Other hints are +% redundant with the field's own Zarr v3 sub-dtype and are ignored. + + fieldSemantics = containers.Map('KeyType', 'char', 'ValueType', 'any'); + if ~isfield(attrs, 'zarr_dtype') + return + end + + fieldDescriptors = attrs.zarr_dtype; + if ~isstruct(fieldDescriptors) + return + end + + for i = 1:numel(fieldDescriptors) + fieldSemantics(char(fieldDescriptors(i).name)) = string(fieldDescriptors(i).dtype); + end +end diff --git a/+io/+internal/+zarr3/getCompoundTypeDescriptor.m b/+io/+internal/+zarr3/getCompoundTypeDescriptor.m new file mode 100644 index 000000000..4cf529dee --- /dev/null +++ b/+io/+internal/+zarr3/getCompoundTypeDescriptor.m @@ -0,0 +1,33 @@ +function typeDescriptor = getCompoundTypeDescriptor(info, fieldSemantics) +% getCompoundTypeDescriptor - Build the compound type descriptor struct +% expected by types.util.checkDtype / types.untyped.DataStub.dataType for +% a "structured" (compound) Zarr v3 array. +% +% typeDescriptor = getCompoundTypeDescriptor(info, fieldSemantics) +% returns a scalar struct with one field per info.fields entry (same +% name, same order -- checkDtype's validateCompoundTypeDescriptor +% requires exact field order), whose value is either: +% - 'types.untyped.ObjectView', when fieldSemantics marks that field +% as "object" (see io.internal.zarr3.getCompoundFieldSemantics), or +% - the field's MATLAB class name (info.fields(k).Info.matlabClass) +% otherwise, e.g. 'int32', 'double', 'single'. +% +% info is a zarr.internal.dtype_info struct for a "structured" dtype +% (info.zarrType == "structured"). + + arguments + info (1,1) struct + fieldSemantics + end + + typeDescriptor = struct(); + for k = 1:numel(info.fields) + f = info.fields(k); + name = char(f.Name); + if isKey(fieldSemantics, name) && fieldSemantics(name) == "object" + typeDescriptor.(name) = 'types.untyped.ObjectView'; + else + typeDescriptor.(name) = char(f.Info.matlabClass); + end + end +end diff --git a/+io/+internal/+zarr3/getMatlabDataType.m b/+io/+internal/+zarr3/getMatlabDataType.m new file mode 100644 index 000000000..2da03e9aa --- /dev/null +++ b/+io/+internal/+zarr3/getMatlabDataType.m @@ -0,0 +1,11 @@ +function matlabDataType = getMatlabDataType(zarrDataType) +% getMatlabDataType - Map a Zarr v3 data_type name to a MATLAB class name. +% +% matlabDataType = getMatlabDataType(zarrDataType) resolves the MATLAB +% class used to represent values of the given Zarr v3 `data_type` (e.g. +% "float64", "int64", "bool", "string"), reusing the dtype table from the +% zarr-matlab package. + + info = zarr.internal.dtype_info(zarrDataType); + matlabDataType = char(info.matlabClass); +end diff --git a/+io/+internal/+zarr3/normalizeDatasetDimensions.m b/+io/+internal/+zarr3/normalizeDatasetDimensions.m new file mode 100644 index 000000000..8bff6b73f --- /dev/null +++ b/+io/+internal/+zarr3/normalizeDatasetDimensions.m @@ -0,0 +1,27 @@ +function datasetValue = normalizeDatasetDimensions(datasetValue, rank) +% normalizeDatasetDimensions - Reverse axis order for rank >= 2 data. +% +% normalizeDatasetDimensions(datasetValue, rank) reverses all axes of +% datasetValue when rank (the array's Zarr rank, i.e. numel(shape) as +% reported by zarr-matlab -- NOT MATLAB's ndims, which cannot +% distinguish a genuine rank-1 column vector from a rank-2 array) is >= 2. +% +% Zarr v3 stores written by Python NWB tools use numpy/row-major shape +% order (matching the NWB schema's declared shape directly). MatNWB's +% type system (types.util.validateShape, checkDims) expects the reverse +% of that for rank >= 2, matching the convention already used by +% io.backend.hdf5.HDF5Reader. +% +% A rank-1 Zarr array is already read by zarr-matlab as a MATLAB column +% vector (see zarr.internal.mshape), so it is left untouched here. + + if rank < 2 + return + end + + if rank == 2 + datasetValue = datasetValue.'; + else + datasetValue = permute(datasetValue, rank:-1:1); + end +end diff --git a/+io/+internal/+zarr3/stripLeadingSlash.m b/+io/+internal/+zarr3/stripLeadingSlash.m new file mode 100644 index 000000000..b96b02eff --- /dev/null +++ b/+io/+internal/+zarr3/stripLeadingSlash.m @@ -0,0 +1,5 @@ +function strippedPath = stripLeadingSlash(nodePath) +% stripLeadingSlash - Remove a single leading '/' from a node path. + + strippedPath = regexprep(char(nodePath), '^/', ''); +end diff --git a/+io/+spec/readEmbeddedSpecifications.m b/+io/+spec/readEmbeddedSpecifications.m index 919e2fc9e..062ec8427 100644 --- a/+io/+spec/readEmbeddedSpecifications.m +++ b/+io/+spec/readEmbeddedSpecifications.m @@ -1,12 +1,15 @@ -function specs = readEmbeddedSpecifications(filename, specLocation) +function specs = readEmbeddedSpecifications(reader, specLocation) % readEmbeddedSpecifications - Read embedded specs from an NWB file % -% specs = io.spec.readEmbeddedSpecifications(filename, specLocation) read -% embedded specs from the specLocation in an NWB file +% specs = io.spec.readEmbeddedSpecifications(reader, specLocation) reads +% embedded specs from the specLocation in an NWB file, using reader +% (an io.backend.base.Reader) to access the file. Backend-agnostic: +% only uses the io.backend.base.Reader interface, so this works for +% any registered storage backend, not just HDF5. % % Inputs: -% filename (string) : Absolute path of an nwb file -% specLocation (string) : h5 path for the location of specs inside the NWB file +% reader (io.backend.base.Reader) : Reader for the NWB file +% specLocation (string) : Path for the location of specs inside the NWB file % % Outputs % specs cell: A cell array of structs with one element for each embedded @@ -17,40 +20,38 @@ % - schemaMap (containers.Map): A set of schema specifications for the namespace arguments - filename (1,1) string {matnwb.common.mustBeNwbFile} + reader (1,1) io.backend.base.Reader specLocation (1,1) string end - specInfo = h5info(filename, specLocation); + specInfo = reader.readNodeInfo(specLocation); specs = deal( cell(size(specInfo.Groups)) ); - - fid = H5F.open(filename); - fileCleanup = onCleanup(@(id) H5F.close(fid) ); for iGroup = 1:length(specInfo.Groups) - location = specInfo.Groups(iGroup).Groups(1); + namespaceGroupInfo = specInfo.Groups(iGroup); + location = namespaceGroupInfo.Groups(1); - namespaceName = split(specInfo.Groups(iGroup).Name, '/'); + namespaceName = split(namespaceGroupInfo.Name, '/'); namespaceName = namespaceName{end}; - filenames = {location.Datasets.Name}; - if ~any(strcmp('namespace', filenames)) + datasetNames = {location.Datasets.Name}; + if ~any(strcmp('namespace', datasetNames)) warning('NWB:Read:GenerateSpec:CacheInvalid',... 'Couldn''t find a `namespace` in namespace `%s`. Skipping cache generation.',... namespaceName); return; end - sourceNames = {location.Datasets.Name}; - fileLocation = strcat(location.Name, '/', sourceNames); + schemaMap = containers.Map; - for iFileLocation = 1:length(fileLocation) - did = H5D.open(fid, fileLocation{iFileLocation}); - if strcmp('namespace', sourceNames{iFileLocation}) - namespaceText = H5D.read(did); + for iDataset = 1:length(datasetNames) + datasetName = datasetNames{iDataset}; + datasetPath = strcat(location.Name, '/', datasetName); + datasetValue = reader.readDatasetValue(location.Datasets(iDataset), datasetPath); + if strcmp('namespace', datasetName) + namespaceText = datasetValue; else - schemaMap(sourceNames{iFileLocation}) = H5D.read(did); + schemaMap(datasetName) = datasetValue; end - H5D.close(did); end specs{iGroup}.namespaceName = namespaceName; diff --git a/+io/+spec/validateEmbeddedSpecifications.m b/+io/+spec/validateEmbeddedSpecifications.m index 9862278f5..b97f254b1 100644 --- a/+io/+spec/validateEmbeddedSpecifications.m +++ b/+io/+spec/validateEmbeddedSpecifications.m @@ -1,19 +1,25 @@ -function validateEmbeddedSpecifications(h5_file_id, expectedNamespaceNames) +function validateEmbeddedSpecifications(writer, expectedNamespaceNames) % validateEmbeddedSpecifications - Validate the embedded specifications % % This function does two things: % 1) Displays a warning if specifications of expected namespaces -% are not embedded in the file. +% are not embedded in the file. % E.g if cached namespaces were cleared prior to export. -% -% 2) Deletes specifications for unused namespaces that are embedded. +% +% 2) Deletes specifications for unused namespaces that are embedded. % - E.g. If neurodata type from an embedded namespace was removed and the % file was re-exported +% +% Backend-agnostic: only uses the io.backend.base.Writer interface, so +% this works for any registered storage backend, not just HDF5. -% NB: Input h5_file_id must point to a file opened with write access + arguments + writer (1,1) io.backend.base.Writer + expectedNamespaceNames + end - specLocation = io.spec.internal.readEmbeddedSpecLocation(h5_file_id); - embeddedNamespaceNames = io.internal.h5.listGroupNames(h5_file_id, specLocation); + specLocation = writer.getEmbeddedSpecLocation(); + embeddedNamespaceNames = writer.listChildGroupNames(specLocation); checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames) @@ -21,7 +27,7 @@ function validateEmbeddedSpecifications(h5_file_id, expectedNamespaceNames) expectedNamespaceNames, embeddedNamespaceNames); if ~isempty(unusedNamespaces) - deleteUnusedNamespaces(h5_file_id, unusedNamespaces, specLocation) + deleteUnusedNamespaces(writer, unusedNamespaces, specLocation) end end @@ -45,10 +51,10 @@ function checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames) unusedNamespaces = setdiff(embeddedNamespaceNames, expectedNamespaceNames); end -function deleteUnusedNamespaces(fileId, unusedNamespaces, specRootLocation) +function deleteUnusedNamespaces(writer, unusedNamespaces, specRootLocation) for i = 1:numel(unusedNamespaces) thisName = unusedNamespaces{i}; namespaceSpecLocation = strjoin( {specRootLocation, thisName}, '/'); - io.internal.h5.deleteGroup(fileId, namespaceSpecLocation) + writer.deleteGroup(namespaceSpecLocation) end end diff --git a/+io/+spec/writeEmbeddedSpecifications.m b/+io/+spec/writeEmbeddedSpecifications.m index 4468d5fd0..33eea4ec3 100644 --- a/+io/+spec/writeEmbeddedSpecifications.m +++ b/+io/+spec/writeEmbeddedSpecifications.m @@ -1,12 +1,15 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) % writeEmbeddedSpecifications - Write schema specifications to an NWB file +% +% Backend-agnostic: only uses the io.backend.base.Writer interface, so +% this works for any registered storage backend, not just HDF5. arguments writer (1,1) io.backend.base.Writer jsonSpecs % String representation of schema specifications in json format end - specLocation = io.spec.internal.readEmbeddedSpecLocation(writer.FileId); + specLocation = writer.getEmbeddedSpecLocation(); if isempty(specLocation) specLocation = '/specifications'; @@ -20,12 +23,9 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) schemaNamespaceLocation = strjoin({specLocation, JsonDatum.name}, '/'); namespaceExists = writer.writeGroup(schemaNamespaceLocation); if namespaceExists - namespaceGroupId = H5G.open(writer.FileId, schemaNamespaceLocation); - names = getVersionNames(namespaceGroupId); - H5G.close(namespaceGroupId); + names = writer.listChildGroupNames(schemaNamespaceLocation); for iNames = 1:length(names) - H5L.delete(writer.FileId, [schemaNamespaceLocation '/' names{iNames}],... - 'H5P_DEFAULT'); + writer.deleteGroup([schemaNamespaceLocation '/' names{iNames}]); end end schemaLocation = ... @@ -40,13 +40,3 @@ function writeEmbeddedSpecifications(writer, jsonSpecs) end end end - -function versionNames = getVersionNames(namespaceGroupId) - [~, ~, versionNames] = H5L.iterate(namespaceGroupId,... - 'H5_INDEX_NAME', 'H5_ITER_NATIVE',... - 0, @appendName, {}); - function [status, versionNames] = appendName(~, name, versionNames) - versionNames{end+1} = name; - status = 0; - end -end diff --git a/+matnwb/+common/+compatibility/mustBeFile.m b/+matnwb/+common/+compatibility/mustBeFile.m index 3cdbdca8c..f4f7780c4 100644 --- a/+matnwb/+common/+compatibility/mustBeFile.m +++ b/+matnwb/+common/+compatibility/mustBeFile.m @@ -14,7 +14,18 @@ function mustBeFile(filePath) if startsWith(filePath, "s3://") return end - + + % Directory-based stores (e.g. Zarr) use a ".zarr" suffix by convention. + % These are folders, not files, so validate folder existence instead. + if endsWith(filePath, ".zarr", "IgnoreCase", true) + try + matnwb.common.compatibility.mustBeFolder(filePath) + catch ME + throwAsCaller(ME) + end + return + end + if verLessThan('matlab', '9.9') %#ok % Custom implementation (MATLAB < R2020b) try diff --git a/+matnwb/+common/mustBeNwbFile.m b/+matnwb/+common/mustBeNwbFile.m index 1a01e8b0e..9f5d6bc36 100644 --- a/+matnwb/+common/mustBeNwbFile.m +++ b/+matnwb/+common/mustBeNwbFile.m @@ -1,9 +1,12 @@ function mustBeNwbFile(filePath) -% mustBeNwbFile - Check that file path points to existing file with .nwb extension +% mustBeNwbFile - Check that path points to an existing NWB file or Zarr store +% +% Accepts a file with a ".nwb" extension or a Zarr directory store with a +% ".zarr" extension. arguments filePath (1,1) string {matnwb.common.compatibility.mustBeFile} end if ~startsWith(filePath, "s3://", "IgnoreCase", true) - assert(endsWith(filePath, ".nwb", "IgnoreCase", true)) + assert(endsWith(filePath, [".nwb", ".zarr"], "IgnoreCase", true)) end end diff --git a/+tests/+fixtures/createZarr3TestFile.m b/+tests/+fixtures/createZarr3TestFile.m new file mode 100644 index 000000000..a55762a6d --- /dev/null +++ b/+tests/+fixtures/createZarr3TestFile.m @@ -0,0 +1,92 @@ +function fixturePath = createZarr3TestFile(rootFolder) +% createZarr3TestFile - Build a small Zarr v3 NWB-like fixture for tests. +% +% fixturePath = createZarr3TestFile(rootFolder) creates a Zarr v3 store +% under rootFolder directly via the zarr-matlab package (independent of +% io.backend.zarr3.Zarr3Writer, so io.backend.zarr3.Zarr3Reader tests are +% not circularly validated only against the writer), and returns the path +% to the store. +% +% The pixel_mask compound dataset is built via zarr.metadata.ArrayMetadata +% + a direct zarr.Array construction rather than zarr.create, because +% zarr.create's public API has no way to pass a "structured" data_type's +% field configuration (its dtype argument is a plain scalar string) -- +% the only way to create a "structured" array today, matching how +% zarr-matlab's own test suite does it (see +% TestStructuredDtype>structuredArrayEndToEnd in zarr-matlab). + + arguments + rootFolder (1,1) string + end + + fixturePath = fullfile(rootFolder, "fixture.zarr"); + + root = zarr.create_group(fixturePath, Attributes=struct(... + 'nwb_version', "2.7.0", ... + 'x_specloc', "/specifications")); + + root.createArray("identifier", [], "string").write("ZARR3_FIXTURE"); + root.createGroup("specifications"); + + acquisitionGroup = root.createGroup("acquisition"); + esGroup = acquisitionGroup.createGroup("es"); + % Stored in numpy/row-major order ([29 4], i.e. timepoints x channels), + % as a real Python NWB Zarr v3 writer would; io.backend.zarr3.Zarr3Reader + % reverses rank->=2 shape/data back to the MatNWB-facing [4 29] + % (channels x timepoints) dims read by Zarr3ReaderTest/Zarr3LazyArrayTest. + esGroup.createArray("data", [29 4], "single").write(reshape(single(1:116), [4 29]).'); + + unitsGroup = root.createGroup("units"); + unitsGroup.createArray("spike_times", 5, "double").write([1.1 2.2 3.3 4.4 5.5]); + spikeTimesIndexGroup = unitsGroup.createGroup("spike_times_index"); + spikeTimesIndexGroup.setAttr("target", ... + struct('zarr_dtype', "object", 'value', struct('path', "/units/spike_times"))); + + generalGroup = root.createGroup("general"); + electrophysGroup = generalGroup.createGroup("extracellular_ephys"); + electrodesGroup = electrophysGroup.createGroup("electrodes"); + electrodesGroup.createArray("location", 4, "string").write(repmat("brain", 4, 1)); + electrodesGroup.createArray("id", 4, "int64").write(int64([0; 1; 2; 3])); + + devicesGroup = generalGroup.createGroup("devices"); + devicesGroup.createGroup("array"); + + shank0Group = electrophysGroup.createGroup("shank0"); + shank0Group.setAttr("zarr_link", ... + {struct('name', "device", 'source', ".", 'path', "/general/devices/array")}); + + processingGroup = root.createGroup("processing"); + ophysGroup = processingGroup.createGroup("ophys"); + planeSegmentationGroup = ophysGroup.createGroup("PlaneSegmentation"); + createPixelMaskArray(planeSegmentationGroup); + + zarr.consolidate_metadata(root.store); +end + +function createPixelMaskArray(parentGroup) +% createPixelMaskArray - Write a 3-record "structured" (compound) pixel_mask +% array (x uint32, y uint32, weight float32) as a child of parentGroup, +% matching a real hdmf-zarr PlaneSegmentation.pixel_mask column. + + info = zarr.internal.dtype_info(struct('name', "structured", 'configuration', struct( ... + 'fields', {{{'x', 'uint32'}; {'y', 'uint32'}; {'weight', 'float32'}}}))); + + numRecords = 3; + meta = zarr.metadata.ArrayMetadata(); + meta.shape = numRecords; + meta.dataType = "structured"; + meta.dataTypeConfig = info.config; + meta.chunkShape = numRecords; + meta.fillValue = struct('x', uint32(0), 'y', uint32(0), 'weight', single(0)); + meta.codecs = {zarr.codecs.BytesCodec()}; + + arrayPath = parentGroup.path + "/pixel_mask"; + parentGroup.store.set(arrayPath + "/zarr.json", unicode2native(char(meta.toJsonText()), 'UTF-8')); + + records(1, 1) = struct('x', uint32(0), 'y', uint32(0), 'weight', single(0.5)); + records(2, 1) = struct('x', uint32(1), 'y', uint32(1), 'weight', single(0.6)); + records(3, 1) = struct('x', uint32(2), 'y', uint32(2), 'weight', single(0.7)); + + pixelMaskArray = zarr.Array(parentGroup.store, arrayPath, meta); + pixelMaskArray.write(records); +end diff --git a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m index aead0e9c9..32f83200f 100644 --- a/+tests/+unit/+io/+backend/+base/BaseWriterTest.m +++ b/+tests/+unit/+io/+backend/+base/BaseWriterTest.m @@ -8,6 +8,9 @@ "writeGroup", ... "writeValue", ... "writeAttribute", ... + "getEmbeddedSpecLocation", ... + "listChildGroupNames", ... + "deleteGroup", ... "abort", ... "close", ... "ensure" ... diff --git a/+tests/+unit/+io/+backend/HDF5WriterTest.m b/+tests/+unit/+io/+backend/HDF5WriterTest.m new file mode 100644 index 000000000..b22aa8d28 --- /dev/null +++ b/+tests/+unit/+io/+backend/HDF5WriterTest.m @@ -0,0 +1,23 @@ +classdef HDF5WriterTest < matlab.unittest.TestCase + + methods (TestMethodSetup) + function setup(testCase) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + end + end + + methods (Test) + function deleteGroupDeletesPopulatedGroup(testCase) + writer = io.backend.hdf5.HDF5Writer("writer-test.nwb", "overwrite"); + testCase.addTeardown(@() writer.close()); + writer.writeGroup('/specifications/core'); + writer.writeValue('/specifications/core/namespace', 'schema'); + + writer.deleteGroup('/specifications/core'); + + groupExists = H5L.exists(writer.FileId, ... + '/specifications/core', 'H5P_DEFAULT'); + testCase.verifyFalse(logical(groupExists)); + end + end +end diff --git a/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m b/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m new file mode 100644 index 000000000..82e5b6abe --- /dev/null +++ b/+tests/+unit/+io/+backend/Zarr3LazyArrayTest.m @@ -0,0 +1,112 @@ +classdef Zarr3LazyArrayTest < matlab.unittest.TestCase + + properties (Access = private) + FixturePath (1,1) string + DatasetPath = "/acquisition/es/data" + end + + methods (TestClassSetup) + function setupZarrFixture(testCase) + tests.util.assumeZarr3Support(testCase) + + import matlab.unittest.fixtures.PathFixture + import matlab.unittest.fixtures.TemporaryFolderFixture + + testCase.applyFixture(PathFixture(tests.util.getZarr3MatlabPath())); + + tempFixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.FixturePath = tests.fixtures.createZarr3TestFile(tempFixture.Folder); + end + end + + methods (Test) + function loadDataAndMetadata(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + expectedData = reshape(single(1:116), [4 29]); + + testCase.verifyEqual(lazyArray.dims, [4 29]); + testCase.verifyEqual(lazyArray.maxDims, [4 29]); + testCase.verifyEqual(lazyArray.dataType, 'single'); + testCase.verifyEqual(lazyArray.load_h5_style(), expectedData); + end + + function loadPartialDataWithH5StyleSelection(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + partialData = lazyArray.load_h5_style([1 2], [2 3], [2 4]); + + testCase.verifyEqual(partialData, fullData(1:2:3, 2:4:10)); + end + + function dataStubSupportsSimpleIndexing(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + expectedData = reshape(single(1:116), [4 29]); + dataStub = types.untyped.DataStub(... + testCase.FixturePath, testCase.DatasetPath, [], [], lazyArray); + + testCase.verifyEqual(dataStub.load(), expectedData); + testCase.verifyEqual(dataStub(1:3, 2), expectedData(1:3, 2)); + end + + function integer1dDatasetHasCorrectMatlabType(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, ... + "/general/extracellular_ephys/electrodes/id"); + testCase.verifyEqual(lazyArray.dataType, 'int64'); + end + + function loadWithInfCountReadsToEnd(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + partialData = lazyArray.load_h5_style([2 3], [Inf Inf]); + + testCase.verifyEqual(partialData, fullData(2:end, 3:end)); + end + + function loadMatStyleIrregularSelectionFallsBackToFullRead(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + result = lazyArray.load_mat_style([1 2 4], 1:29); + + testCase.verifyEqual(result, fullData([1 2 4], 1:29)); + end + + function loadMatStyleUsesPartialReadForRegularSelection(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(testCase.FixturePath, testCase.DatasetPath); + fullData = lazyArray.load_h5_style(); + + testCase.verifyEqual(... + lazyArray.load_mat_style(1:2:3, 2:4:10), ... + fullData(1:2:3, 2:4:10)); + end + + function compoundDatasetHasStructTypeDescriptor(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + + testCase.verifyEqual(lazyArray.dims, 3); + testCase.verifyEqual(lazyArray.dataType, ... + struct('x', 'uint32', 'y', 'uint32', 'weight', 'single')); + end + + function compoundLoadH5StyleReturnsStructOfArrays(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + data = lazyArray.load_h5_style(); + + testCase.verifyClass(data, "struct"); + testCase.verifyEqual(data.x, uint32([0; 1; 2])); + testCase.verifyEqual(data.weight, single([0.5; 0.6; 0.7])); + end + + function compoundLoadMatStyleReturnsTable(testCase) + lazyArray = io.backend.zarr3.Zarr3LazyArray(... + testCase.FixturePath, "/processing/ophys/PlaneSegmentation/pixel_mask"); + selectedRecords = lazyArray.load_mat_style(2:3); + + testCase.verifyClass(selectedRecords, "table"); + testCase.verifyEqual(selectedRecords.x, uint32([1; 2])); + testCase.verifyEqual(selectedRecords.weight, single([0.6; 0.7])); + end + end +end diff --git a/+tests/+unit/+io/+backend/Zarr3ReaderTest.m b/+tests/+unit/+io/+backend/Zarr3ReaderTest.m new file mode 100644 index 000000000..4a3aaadf7 --- /dev/null +++ b/+tests/+unit/+io/+backend/Zarr3ReaderTest.m @@ -0,0 +1,140 @@ +classdef Zarr3ReaderTest < matlab.unittest.TestCase + + properties (Access = private) + FixturePath (1,1) string + end + + methods (TestClassSetup) + function setupZarrFixture(testCase) + tests.util.assumeZarr3Support(testCase) + + import matlab.unittest.fixtures.PathFixture + import matlab.unittest.fixtures.TemporaryFolderFixture + + testCase.applyFixture(PathFixture(tests.util.getZarr3MatlabPath())); + + tempFixture = testCase.applyFixture(TemporaryFolderFixture); + testCase.FixturePath = tests.fixtures.createZarr3TestFile(tempFixture.Folder); + end + end + + methods (Test) + function readRootInfoAndSchemaVersion(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + rootInfo = reader.readRootInfo(); + + testCase.verifyEqual(rootInfo.Name, '/'); + testCase.verifyEqual(reader.getSchemaVersion(), "2.7.0"); + testCase.verifyEqual(reader.getEmbeddedSpecLocation(), "/specifications"); + testCase.verifyTrue(any(strcmp({rootInfo.Groups.Name}, '/general'))); + end + + function readNodeInfoIncludesLinks(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + nodeInfo = reader.readNodeInfo("/general/extracellular_ephys/shank0"); + + testCase.verifyEqual(nodeInfo.Name, '/general/extracellular_ephys/shank0'); + testCase.verifyEqual(numel(nodeInfo.Links), 1); + testCase.verifyEqual(nodeInfo.Links(1).Name, 'device'); + testCase.verifyEqual(nodeInfo.Links(1).Type, 'soft link'); + testCase.verifyEqual(string(nodeInfo.Links(1).Value{1}), "/general/devices/array"); + + % The reserved zarr_link attribute must not leak into Attributes. + testCase.verifyEmpty(nodeInfo.Attributes); + end + + function readAttributeValueConvertsObjectReference(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + nodeInfo = reader.readNodeInfo("/units/spike_times_index"); + attributeInfo = nodeInfo.Attributes(strcmp({nodeInfo.Attributes.Name}, 'target')); + attributeValue = reader.readAttributeValue(attributeInfo, "/units/spike_times_index"); + + testCase.verifyClass(attributeValue, "types.untyped.ObjectView"); + testCase.verifyEqual(string(attributeValue.path), "/units/spike_times"); + end + + function readDatasetValueReturnsScalarString(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + rootInfo = reader.readRootInfo(); + datasetInfo = rootInfo.Datasets(strcmp({rootInfo.Datasets.Name}, 'identifier')); + datasetValue = reader.readDatasetValue(datasetInfo, "/identifier"); + + testCase.verifyClass(datasetValue, "char"); + testCase.verifyEqual(datasetValue, 'ZARR3_FIXTURE'); + end + + function readNonScalarDatasetValueReturnsDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + datasetInfo = reader.readNodeInfo("/acquisition/es/data"); + datasetValue = reader.readDatasetValue(datasetInfo, "/acquisition/es/data"); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyEqual(datasetValue.dims, [4 29]); + testCase.verifyEqual(datasetValue.load(), reshape(single(1:116), [4 29])); + end + + function read1dDatasetReturnsDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + datasetInfo = reader.readNodeInfo("/units/spike_times"); + datasetValue = reader.readDatasetValue(datasetInfo, "/units/spike_times"); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyEqual(datasetValue.dims, 5); + testCase.verifyEqual(datasetValue.load(), [1.1 2.2 3.3 4.4 5.5]'); + end + + function readStringArrayDatasetContainsExpectedValues(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + locationPath = "/general/extracellular_ephys/electrodes/location"; + datasetInfo = reader.readNodeInfo(locationPath); + datasetValue = reader.readDatasetValue(datasetInfo, locationPath); + + loadedValue = datasetValue.load(); + if iscell(loadedValue) + loadedValue = string(loadedValue); + end + + testCase.verifyEqual(numel(loadedValue), 4); + testCase.verifyTrue(all(string(loadedValue) == "brain")); + end + + function readIntegerDatasetHasCorrectMatlabType(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + idPath = "/general/extracellular_ephys/electrodes/id"; + datasetInfo = reader.readNodeInfo(idPath); + datasetValue = reader.readDatasetValue(datasetInfo, idPath); + + testCase.verifyEqual(datasetValue.dataType, 'int64'); + testCase.verifyEqual(datasetValue.load(), int64([0; 1; 2; 3])); + end + + function readCompoundDatasetReturnsCompoundDataStub(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + pixelMaskPath = "/processing/ophys/PlaneSegmentation/pixel_mask"; + datasetInfo = reader.readNodeInfo(pixelMaskPath); + datasetValue = reader.readDatasetValue(datasetInfo, pixelMaskPath); + + testCase.verifyClass(datasetValue, "types.untyped.DataStub"); + testCase.verifyTrue(datasetValue.isCompoundType()); + testCase.verifyEqual(datasetValue.dataType, ... + struct('x', 'uint32', 'y', 'uint32', 'weight', 'single')); + + loadedValue = datasetValue.load(); + testCase.verifyClass(loadedValue, "struct"); + testCase.verifyEqual(loadedValue.x, uint32([0; 1; 2])); + testCase.verifyEqual(loadedValue.y, uint32([0; 1; 2])); + testCase.verifyEqual(loadedValue.weight, single([0.5; 0.6; 0.7])); + + selectedRecords = datasetValue.load(1, 2); + testCase.verifyClass(selectedRecords, "table"); + testCase.verifyEqual(selectedRecords.x, uint32([0; 1])); + end + + function readNodeInfoThrowsForMissingNode(testCase) + reader = io.backend.zarr3.Zarr3Reader(testCase.FixturePath); + testCase.verifyError(... + @() reader.readNodeInfo("/does/not/exist"), ... + "NWB:Zarr3Reader:NodeNotFound"); + end + end +end diff --git a/+tests/+util/assumeZarr3Support.m b/+tests/+util/assumeZarr3Support.m new file mode 100644 index 000000000..30faeede3 --- /dev/null +++ b/+tests/+util/assumeZarr3Support.m @@ -0,0 +1,15 @@ +function assumeZarr3Support(testCase) +% assumeZarr3Support - Skip the calling test unless Zarr v3 support is available. +% +% assumeZarr3Support(testCase) filters the calling test (via assumeTrue) +% when the zarr-matlab package is not on a discoverable path. +% +% See also tests.util.getZarr3MatlabPath + + arguments + testCase (1,1) matlab.unittest.TestCase + end + + testCase.assumeTrue(strlength(tests.util.getZarr3MatlabPath()) > 0, ... + "zarr-matlab package not found (set ZARR3_MATLAB_PATH or run setup).") +end diff --git a/+tests/+util/getZarr3MatlabPath.m b/+tests/+util/getZarr3MatlabPath.m new file mode 100644 index 000000000..ef8c80159 --- /dev/null +++ b/+tests/+util/getZarr3MatlabPath.m @@ -0,0 +1,46 @@ +function packagePath = getZarr3MatlabPath() +% getZarr3MatlabPath - Resolve the path to the zarr-matlab package. +% +% packagePath = getZarr3MatlabPath() returns the location of the +% zarr-matlab package (providing the `zarr` namespace used by +% io.backend.zarr3.Zarr3Reader / Zarr3Writer: zarr.open, zarr.create, +% zarr.create_group, ...). Resolution order: +% +% 1. Already resolvable on the MATLAB path (e.g. installed by +% matbox.installRequirements, which places it in an add-ons folder +% outside this repo and adds it to the path directly). +% 2. The ZARR3_MATLAB_PATH environment variable, if set. +% 3. The default install location created by setup +% (external_packages/zarr-matlab). +% +% Returns "" if no candidate folder exists, allowing callers to skip +% Zarr v3 tests gracefully. +% +% See also setup + + packagePath = ""; + + % `exist(name, "file")` does not reliably resolve dotted package-function + % names (see io.backend.zarr3.internal.ensureAvailable), so `which` is + % used instead. zarr.open.m lives at /+zarr/open.m. + zarrOpenLocation = which("zarr.open"); + if ~isempty(zarrOpenLocation) + packagePath = string(fileparts(fileparts(zarrOpenLocation))); + return + end + + candidates = string.empty; + envPath = string(getenv("ZARR3_MATLAB_PATH")); + if strlength(envPath) > 0 + candidates(end+1) = envPath; + end + candidates(end+1) = fullfile(misc.getMatnwbDir(), ... + "external_packages", "zarr-matlab"); + + for candidate = candidates + if isfolder(fullfile(candidate, "+zarr")) + packagePath = candidate; + return + end + end +end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 41162e273..9ed06ab5c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -130,16 +130,20 @@ jobs: release: ${{ matrix.matlab-version }} cache: ${{ env.USE_CACHE }} + - name: Install MatBox + uses: ehennestad/matbox-actions/install-matbox@v1 + - name: Run tests uses: matlab-actions/run-command@v3 with: command: | + matbox.installRequirements('.'); setenv("SKIP_PYNWB_TESTS", ... num2str(${{ matrix.skip-pynwb-tests }})) setenv("SKIP_NWBINSPECTOR_TEST", ... num2str(${{ matrix.skip-nwbinspector-test }})) pyenv("ExecutionMode", "OutOfProcess"); - results = assertSuccess(nwbtest('ReportOutputFolder', '.', 'ProduceCodeCoverage', ~${{ env.SKIP_COVERAGE }} )); + results = assertSuccess(nwbtest('ReportOutputFolder', '.', 'ProduceCodeCoverage', ~${{ env.SKIP_COVERAGE }} )); assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results diff --git a/.gitignore b/.gitignore index 4dd70e8cd..ae0808cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ nwbtest.env docs/build docs/reports + +# External dependencies installed by matbox.installRequirements +# (e.g. zarr-matlab, see requirements.txt) +external_packages/zarr-matlab diff --git a/NwbFile.m b/NwbFile.m index a98441443..4a5bfbdf4 100644 --- a/NwbFile.m +++ b/NwbFile.m @@ -392,7 +392,7 @@ function embedSpecifications(obj, writer) jsonSpecs); io.spec.validateEmbeddedSpecifications(... - writer.FileId, ... + writer, ... strrep(namespaceNames, '_', '-')) end diff --git a/nwbRead.m b/nwbRead.m index 36d030109..8d727fd8e 100644 --- a/nwbRead.m +++ b/nwbRead.m @@ -87,7 +87,7 @@ end end else - generateEmbeddedSpec(filename, specLocation, 'savedir', options.savedir); + generateEmbeddedSpec(reader, specLocation, 'savedir', options.savedir); end else warnIfSchemaVersionsMismatch(schemaVersionOfFile, schemaVersionActive) @@ -97,7 +97,7 @@ 'attributes', {{'.specloc', 'object_id'}},... 'groups', {{}}); if ~isempty(specLocation) - blackList.groups{end+1} = specLocation; + blackList.groups{end+1} = char(specLocation); end softLinkWarningResetObj = types.untyped.SoftLink.disablePathDeprecationWarning(); %#ok @@ -119,15 +119,15 @@ nwb.resolveSoftLinks() end -function generateEmbeddedSpec(filename, specLocation, options) +function generateEmbeddedSpec(reader, specLocation, options) % generateEmbeddedSpec - Generate embedded specifications / namespaces arguments - filename (1,1) string {matnwb.common.compatibility.mustBeFile} + reader (1,1) io.backend.base.Reader specLocation (1,1) string options.savedir (1,1) string = misc.getMatnwbDir(); % {matnwb.common.compatibility.mustBeFolder} ? end - specs = io.spec.readEmbeddedSpecifications(filename, specLocation); + specs = io.spec.readEmbeddedSpecifications(reader, specLocation); specNames = cell(size(specs)); for iSpec = 1:numel(specs) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..88520e4fe --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +https://github.com/catalystneuro/zarr-matlab