refactor(dynamictable): derive schema-defined column routing from the schema

+file/+internal/isSchemaDefinedTableCategory.m

@@ -0,0 +1,9 @@
+function tf = isSchemaDefinedTableCategory(propertyInfo, namespace)
+% isSchemaDefinedTableCategory - Determine whether a property stores a 
+% schema-defined AlignedDynamicTable category.
+
+    tf = isa(propertyInfo, 'file.Group') ...
+        && ~propertyInfo.isConstrainedSet ...
+        && ~isempty(propertyInfo.type) ...
+        && file.internal.isDescendantOf(propertyInfo.type, namespace, 'DynamicTable');
+end

+file/+internal/isSchemaDefinedTableColumn.m

@@ -0,0 +1,11 @@
+function tf = isSchemaDefinedTableColumn(propertyInfo, namespace)
+% isSchemaDefinedTableColumn - Determine whether a property stores a 
+% schema-defined DynamicTable column.
+
+    tf = isscalar(propertyInfo) ...
+        && isa(propertyInfo, 'file.Dataset') ...
+        && ~propertyInfo.isConstrainedSet ...
+        && ~isempty(propertyInfo.type) ...
+        && file.internal.isDescendantOf( ...
+            propertyInfo.type, namespace, 'VectorData');
+end

+file/fillClass.m

@@ -175,6 +175,15 @@
             {fullPropertyDefinition, schemaCategoryPropertyBlock}, newline);
     end
 
+    if file.internal.isDescendantOf(name, namespace, 'DynamicTable')
+        columnNames = collectSchemaDefinedTableColumns( ...
+            classprops, nonInherited, namespace);
+        schemaColumnPropertyBlock = file.fillPrivateConstantProperty( ...
+            'DeclaredSchemaColumns', columnNames);
+        fullPropertyDefinition = strjoin(...
+            {fullPropertyDefinition, schemaColumnPropertyBlock}, newline);
+    end
+
     constructorBody = file.fillConstructor(...
         name,...
         superclassNames{1},...
@@ -265,17 +274,22 @@
     for iProperty = 1:length(propertyNames)
         propertyName = propertyNames{iProperty};
         propertyInfo = classProps(propertyName);
-        if isSchemaDefinedAlignedDynamicTableCategory(propertyInfo, namespace)
+        if file.internal.isSchemaDefinedTableCategory(propertyInfo, namespace)
             categoryNames(end+1) = string(propertyName); %#ok<AGROW>
         end
     end
 end
 
-function tf = isSchemaDefinedAlignedDynamicTableCategory(propertyInfo, namespace)
-    tf = isa(propertyInfo, 'file.Group') ...
-        && ~propertyInfo.isConstrainedSet ...
-        && ~isempty(propertyInfo.type) ...
-        && file.internal.isDescendantOf(propertyInfo.type, namespace, 'DynamicTable');
+function columnNames = collectSchemaDefinedTableColumns( ...
+        classProps, propertyNames, namespace)
+    columnNames = string.empty(1, 0);
+    for iProperty = 1:length(propertyNames)
+        propertyName = propertyNames{iProperty};
+        propertyInfo = classProps(propertyName);
+        if file.internal.isSchemaDefinedTableColumn(propertyInfo, namespace)
+            columnNames(end+1) = string(propertyName); %#ok<AGROW>
+        end
+    end
 end
 
 function propertyBlockStr = createSchemaNameMappingBlock(schemaNames)

+file/getPropertyHooks.m

@@ -7,8 +7,7 @@
 
     fullClassName = namespace.getFullClassName(typeName);
 
-    if strcmp(fullClassName, 'types.hdmf_common.DynamicTable') ...
-            && strcmp(propName, 'colnames')
+    if strcmp(typeName, 'DynamicTable') && strcmp(propName, 'colnames')
         hooks.ValidatorLines = { ...
             'val = types.util.dynamictable.validateColnames(val);' };
     end
@@ -19,29 +18,18 @@
             'val = obj.validateCategoryNames(val);' };
     end
 
-    if file.internal.isDescendantOf(typeName, namespace, 'DynamicTable') ...
-            && isSchemaDefinedDynamicTableColumn(propName, prop)
+    isNamedTableColumn = file.internal.isDescendantOf(typeName, namespace, 'DynamicTable') ...
+        && file.internal.isSchemaDefinedTableColumn(prop, namespace) ...
+        && ~endsWith(propName, '_index');
+    if isNamedTableColumn
         hooks.PostsetStatements = { ...
             sprintf('types.util.dynamictable.syncNamedColumn(obj, ''%s'');', propName) };
     end
 
-    if file.internal.isDescendantOf(typeName, namespace, 'AlignedDynamicTable') ...
-            && isSchemaDefinedAlignedDynamicTableCategory(prop, namespace)
+    isNamedTableCategory = file.internal.isDescendantOf(typeName, namespace, 'AlignedDynamicTable') ...
+        && file.internal.isSchemaDefinedTableCategory(prop, namespace);
+    if isNamedTableCategory
         hooks.PostsetStatements = [hooks.PostsetStatements, { ...
             sprintf('obj.ensureCategoryNameRegistered(''%s'');', propName) }];
     end
 end
-
-function tf = isSchemaDefinedDynamicTableColumn(propName, prop)
-    tf = isa(prop, 'file.Dataset') ...
-        && ~prop.isConstrainedSet ...
-        && ~strcmp(propName, 'id') ...
-        && ~endsWith(propName, '_index');
-end
-
-function tf = isSchemaDefinedAlignedDynamicTableCategory(prop, namespace)
-    tf = isa(prop, 'file.Group') ...
-        && ~prop.isConstrainedSet ...
-        && ~isempty(prop.type) ...
-        && file.internal.isDescendantOf(prop.type, namespace, 'DynamicTable');
-end

+matnwb/+neurodata/DynamicTableBase.m

@@ -193,6 +193,19 @@ function clear(obj)
     end
 
     methods (Hidden)
+        function columnNames = getSchemaDefinedColumns(obj)
+        % getSchemaDefinedColumns - Return schema-defined column names.
+        %
+        % Generated DynamicTable classes declare their local schema column
+        % names as private constants. Aggregate them across the generated
+        % neurodata type hierarchy so inherited columns are included.
+
+            import matnwb.neurodata.internal.collectConstantPropertiesAcrossHierarchy
+
+            columnNames = collectConstantPropertiesAcrossHierarchy( ...
+                class(obj), 'DeclaredSchemaColumns');
+        end
+
         function ensureDynamicTableConsistency(obj)
         % ensureDynamicTableConsistency - Ensure DynamicTable column consistency.
         %

+tests/+unit/+schema/DynamicTableColumnTest.m

@@ -0,0 +1,37 @@
+classdef DynamicTableColumnTest < tests.unit.abstract.SchemaTest
+% DynamicTableColumnTest - Test schema-defined DynamicTable column detection.
+
+    properties (Constant)
+        SchemaFolder = "dynamicTableColumnSchema"
+        SchemaNamespaceFileName = "dtc.namespace.yaml"
+    end
+
+    methods (Test)
+        function testVectorDataDatasetUsesGeneratedProperty(testCase)
+            dynamicTable = types.dtc.MixedDatasetTable( ...
+                'description', 'test table');
+            schemaColumn = types.hdmf_common.VectorData( ...
+                'description', 'schema column', ...
+                'data', single((1:3)'));
+
+            dynamicTable.addColumn('schema_column', schemaColumn);
+
+            testCase.verifyEqual(dynamicTable.schema_column, schemaColumn)
+            testCase.verifyFalse( ...
+                dynamicTable.vectordata.isKey('schema_column'))
+        end
+
+        function testNonColumnDatasetRemainsPropertyCollision(testCase)
+            dynamicTable = types.dtc.MixedDatasetTable( ...
+                'description', 'test table');
+            invalidColumn = types.hdmf_common.VectorData( ...
+                'description', 'invalid column', ...
+                'data', single((1:3)'));
+
+            testCase.verifyError( ...
+                @() dynamicTable.addColumn( ...
+                    'table_metadata', invalidColumn), ...
+                'NWB:DynamicTable:AddColumn:InvalidPropertyCollision')
+        end
+    end
+end

+tests/+unit/dynamicTableTest.m

@@ -391,6 +391,29 @@ function testAddColumnUsesSchemaPropertyForTimeSeriesReferenceColumn(testCase)
             testCase.verifyFalse(timeIntervals.vectordata.isKey('timeseries'));
         end
 
+        function testAddColumnUsesSchemaPropertiesForRaggedSchemaColumn(testCase)
+            timeIntervals = types.core.TimeIntervals( ...
+                'description', 'test time intervals');
+
+            tags = types.hdmf_common.VectorData( ...
+                'description', 'tags', ...
+                'data', {'a'; 'b'; 'c'});
+            tagsIndex = types.hdmf_common.VectorIndex( ...
+                'description', 'tag indices', ...
+                'data', uint64([2; 3]), ...
+                'target', types.untyped.ObjectView(tags));
+
+            timeIntervals.addColumn('tags', tags, 'tags_index', tagsIndex);
+
+            % Both halves of a ragged schema column are stored on their
+            % generated property, but only the data column is a colname.
+            testCase.verifyEqual(timeIntervals.tags, tags);
+            testCase.verifyEqual(timeIntervals.tags_index, tagsIndex);
+            testCase.verifyFalse(timeIntervals.vectordata.isKey('tags'));
+            testCase.verifyFalse(timeIntervals.vectordata.isKey('tags_index'));
+            testCase.verifyEqual(timeIntervals.colnames, {'tags'});
+        end
+
         function testAddColumnWrongTypeForSchemaPropertyKeepsPropertyRouting(testCase)
             timeIntervals = types.core.TimeIntervals( ...
                 'description', 'test time intervals');

+tests/test-schema/dynamicTableColumnSchema/dtc.namespace.yaml

@@ -0,0 +1,7 @@
+namespaces:
+- full_name: DynamicTable Column Schema Test
+  name: dtc
+  schema:
+  - namespace: core
+  - source: dtc.tables.yaml
+  version: 1.0.0

+tests/test-schema/dynamicTableColumnSchema/dtc.tables.yaml

@@ -0,0 +1,14 @@
+groups:
+- neurodata_type_def: MixedDatasetTable
+  neurodata_type_inc: DynamicTable
+  doc: DynamicTable containing a column dataset and a non-column dataset.
+  datasets:
+  - name: schema_column
+    neurodata_type_inc: VectorData
+    doc: A schema-defined table column.
+    quantity: '?'
+  - name: table_metadata
+    neurodata_type_inc: Data
+    dtype: text
+    doc: Dataset metadata that is not a table column.
+    quantity: '?'

+types/+core/ElectrodesTable.m

@@ -23,6 +23,9 @@
     y; %  (VectorData) y coordinate of the channel location in the brain (+y is inferior). Units should be specified in microns.
     z; %  (VectorData) z coordinate of the channel location in the brain (+z is right). Units should be specified in microns.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["filtering", "group", "group_name", "imp", "location", "reference", "rel_x", "rel_y", "rel_z", "x", "y", "z"];
+end
 
 methods
     function obj = ElectrodesTable(varargin)

+types/+core/EventsTable.m

@@ -15,6 +15,9 @@
     duration; %  (DurationVectorData) Optional column containing the duration of each event, in seconds. A value of NaN can be used for events without a duration or with a duration that is not yet specified.
     source_description; %  (char) Optional short text description of where the events came from, applying to every row in the table. For example, "Acquisition system" for events emitted directly by the acquisition system (e.g., TTL edges or hardware event channels); "Thresholding of analog signal ANALOG1 at 3 V" for events produced by a detection algorithm run on acquired data; or "Manual video review" for events added by a human annotator. This is a free-text label of origin only; use `description` for the longer narrative of how the event times were computed (channels used, encoding scheme, algorithm parameters, etc.).
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["annotation", "duration", "timestamp"];
+end
 
 methods
     function obj = EventsTable(varargin)

+types/+core/ExperimentalConditionsTable.m

@@ -10,6 +10,9 @@
     repetitions; % REQUIRED (DynamicTableRegion) A reference to one or more rows in the RepetitionsTable table.
     repetitions_index; % REQUIRED (VectorIndex) Index dataset for the repetitions column.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["repetitions", "repetitions_index"];
+end
 
 methods
     function obj = ExperimentalConditionsTable(varargin)

+types/+core/FrequencyBandsTable.m

@@ -15,6 +15,9 @@
     band_mean; %  (VectorData) The mean Gaussian filters, in Hz.
     band_stdev; %  (VectorData) The standard deviation of Gaussian filters, in Hz.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["band_limits", "band_mean", "band_name", "band_stdev"];
+end
 
 methods
     function obj = FrequencyBandsTable(varargin)

+types/+core/IntracellularElectrodesTable.m

@@ -9,6 +9,9 @@
 properties
     electrode; % REQUIRED (VectorData) Column for storing the reference to the intracellular electrode.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["electrode"];
+end
 
 methods
     function obj = IntracellularElectrodesTable(varargin)

+types/+core/IntracellularRecordingsTable.m

@@ -14,6 +14,9 @@
 properties (Constant, Access = private)
     DeclaredSchemaCategories = ["electrodes", "responses", "stimuli"];
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = string.empty(1, 0);
+end
 
 methods
     function obj = IntracellularRecordingsTable(varargin)

+types/+core/IntracellularResponsesTable.m

@@ -9,6 +9,9 @@
 properties
     response; % REQUIRED (TimeSeriesReferenceVectorData) Column storing the reference to the recorded response for the recording (rows)
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["response"];
+end
 
 methods
     function obj = IntracellularResponsesTable(varargin)

+types/+core/IntracellularStimuliTable.m

@@ -13,6 +13,9 @@
 properties
     stimulus_template; %  (TimeSeriesReferenceVectorData) Column storing the reference to the stimulus template for the recording (rows).
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["stimulus", "stimulus_template"];
+end
 
 methods
     function obj = IntracellularStimuliTable(varargin)

+types/+core/PlaneSegmentation.m

@@ -18,6 +18,9 @@
     voxel_mask; %  (VectorData) Voxel masks for each ROI: a list of indices and weights for the ROI. Voxel masks are concatenated and parsing of this dataset is maintained by the PlaneSegmentation. At least one of `image_mask`, `pixel_mask`, or `voxel_mask` is required.
     voxel_mask_index; %  (VectorIndex) Index into voxel_mask.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["image_mask", "pixel_mask", "pixel_mask_index", "voxel_mask", "voxel_mask_index"];
+end
 
 methods
     function obj = PlaneSegmentation(varargin)

+types/+core/RepetitionsTable.m

@@ -10,6 +10,9 @@
     sequential_recordings; % REQUIRED (DynamicTableRegion) A reference to one or more rows in the SequentialRecordingsTable table.
     sequential_recordings_index; % REQUIRED (VectorIndex) Index dataset for the sequential_recordings column.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["sequential_recordings", "sequential_recordings_index"];
+end
 
 methods
     function obj = RepetitionsTable(varargin)

+types/+core/SequentialRecordingsTable.m

@@ -11,6 +11,9 @@
     simultaneous_recordings_index; % REQUIRED (VectorIndex) Index dataset for the simultaneous_recordings column.
     stimulus_type; % REQUIRED (VectorData) The type of stimulus used for the sequential recording.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["simultaneous_recordings", "simultaneous_recordings_index", "stimulus_type"];
+end
 
 methods
     function obj = SequentialRecordingsTable(varargin)

+types/+core/SimultaneousRecordingsTable.m

@@ -10,6 +10,9 @@
     recordings; % REQUIRED (DynamicTableRegion) A reference to one or more rows in the IntracellularRecordingsTable table.
     recordings_index; % REQUIRED (VectorIndex) Index dataset for the recordings column.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["recordings", "recordings_index"];
+end
 
 methods
     function obj = SimultaneousRecordingsTable(varargin)

+types/+core/SweepTable.m

@@ -11,6 +11,9 @@
     series_index; % REQUIRED (VectorIndex) Index for series.
     sweep_number; % REQUIRED (VectorData) Sweep number of the PatchClampSeries in that row.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["series", "series_index", "sweep_number"];
+end
 
 methods
     function obj = SweepTable(varargin)

+types/+core/TimeIntervals.m

@@ -17,6 +17,9 @@
     timeseries; %  (TimeSeriesReferenceVectorData) An index into a TimeSeries object.
     timeseries_index; %  (VectorIndex) Index for timeseries.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["start_time", "stop_time", "tags", "tags_index", "timeseries", "timeseries_index"];
+end
 
 methods
     function obj = TimeIntervals(varargin)

+types/+core/Units.m

@@ -30,6 +30,9 @@
     waveforms_index_index; %  (VectorIndex) Index into the 'waveforms_index' dataset. One value for every unit (row in the table). See 'waveforms' for more detail.
     waveforms_sampling_rate; %  (single) Sampling rate, in hertz.
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = ["electrode_group", "electrodes", "electrodes_index", "obs_intervals", "obs_intervals_index", "spike_times", "spike_times_index", "waveform_mean", "waveform_sd", "waveforms", "waveforms_index", "waveforms_index_index"];
+end
 
 methods
     function obj = Units(varargin)

+types/+hdmf_common/AlignedDynamicTable.m

@@ -19,6 +19,9 @@
 properties (Constant, Access = private)
     DeclaredSchemaCategories = string.empty(1, 0);
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = string.empty(1, 0);
+end
 
 methods
     function obj = AlignedDynamicTable(varargin)

+types/+hdmf_common/DynamicTable.m

@@ -19,6 +19,9 @@
 properties (Constant, Access = private)
     GroupPropertyNames = ["vectordata"];
 end
+properties (Constant, Access = private)
+    DeclaredSchemaColumns = string.empty(1, 0);
+end
 
 methods
     function obj = DynamicTable(varargin)