Add hdmf.typing: replace docval with type hints (Phase 1-2 of #1129)

CHANGELOG.md

@@ -1,5 +1,13 @@
 # HDMF Changelog
 
+## HDMF 6.2.0 (Upcoming)
+
+### Enhancements
+- Added the `hdmf.typing` package, the first phase of replacing `@docval` with standard Python type hints ([#1129](https://github.com/hdmf-dev/hdmf/issues/1129)). Runtime validation is built entirely on the type-hint system: type checks run through beartype and array dtype/shape checks through numpydantic `NDArray[...]` hints. The package provides type aliases that are real beartype validators, enforceable by `@beartype` or `beartype.door.is_bearable` anywhere (`Int`, `UInt`, `Float`, `Bool` accepting numpy scalar types — a bare `int` hint is strict; `ArrayData`, `ScalarData`, `AnyData` backed by a live type registry; `TypeName[...]` for MRO-name forward references; `Shaped[...]` for array shape specs), the `@validated` decorator (docval-style error messages, `TermSetWrapper` unwrapping, `HDMF_TYPE_CHECKING=off` kill switch), a migration tool (`python -m hdmf.typing.migrate`), and parity-testing helpers (`hdmf.typing.testing`). @bendichter [#TBD]
+- `hdmf.utils.get_docval` now also works on functions with type-hinted signatures: docval-compatible argument specs are synthesized from the signature, type hints, and Google-style docstring. This keeps the downstream splice pattern `@docval(*get_docval(Parent.__init__, ...))` working as HDMF (and downstream libraries) migrate off `@docval` module by module. Existing `@docval`-decorated functions are unaffected. @bendichter [#TBD]
+- New required dependencies: `beartype`, `numpydantic`, and `docstring_parser`. @bendichter [#TBD]
+- Migrated `hdmf.data_utils`, `hdmf.query`, `hdmf.validate.validator`, `hdmf.common.sparse`, `hdmf.common.resources`, and `hdmf.common.hierarchicaltable` off `@docval` to type-hinted signatures with `@validated` (includes `GenericDataChunkIterator`, `DataChunkIterator`, `DataChunk`, `DataIO`, `CSRMatrix`, `HERD`, and the validator classes). Downstream code is unaffected: `get_docval` on these functions returns equivalent specs synthesized from the type hints, so patterns like `@docval(*get_docval(DataIO.__init__), ...)` (used by `H5DataIO` and extensions) keep working, positional/keyword calling conventions are unchanged, and error messages keep the docval format. @bendichter [#TBD]
+
 ## HDMF 6.1.0 (June 25, 2026)
 
 ### Enhancements

docs/source/api_docs.rst

@@ -15,6 +15,7 @@ API Documentation
    I/O layer <hdmf.backends>
    Data I/O utilities <hdmf.data_utils>
    Development utilities <hdmf.utils>
+   Type hints and runtime validation <hdmf.typing>
    Validation utilities <hdmf.validate>
    Testing utilities <hdmf.testing>
    Full list of HDMF package contents <hdmf>

docs/source/conf.py

@@ -92,6 +92,16 @@
                   ('py:class', 'unittest.case.TestCase'),
                   ('py:class', 'pandas.ExtensionArray'),
                   ]
+# numpy scalar types appear in expanded hdmf.typing alias signatures but have no
+# py:class intersphinx targets; same for pandas internal class paths
+nitpick_ignore_regex = [
+    ('py:class', r'numpy\.(u?int|float|bool|longdouble)\w*'),
+    ('py:class', r'pandas\.core\..*'),
+    # beartype validators inside hdmf.typing alias signatures render as
+    # Is[is_array_data], Is[has_shape_...], etc., which are not documented targets
+    ('py:class', r'beartype\..*'),
+    ('py:class', r'(is|has_shape)_\w*'),
+]
 
 suppress_warnings = ["config.cache"]
 

pyproject.toml

@@ -30,9 +30,12 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Medical Science Apps.",
 ]
 dependencies = [
+    "beartype>=0.19.0",
+    "docstring_parser>=0.15",
     "h5py>=3.6.0",
     "jsonschema>=3.2.0",
     "numpy>=1.22.0",
+    "numpydantic>=1.6.0",
     "pandas>=1.4.0",
     "ruamel.yaml>=0.16",
 ]

src/hdmf/common/hierarchicaltable.py

@@ -6,17 +6,12 @@
 import numpy as np
 from hdmf.common.table import DynamicTable, DynamicTableRegion, VectorIndex
 from hdmf.common.alignedtable import AlignedDynamicTable
-from hdmf.utils import docval, getargs
+from hdmf.typing import Bool, Int, validated
 
 
-@docval({'name': 'dynamic_table', 'type': DynamicTable,
-         'doc': 'DynamicTable object to be converted to a hierarchical pandas.Dataframe'},
-        returns="Hierarchical pandas.DataFrame with usually a pandas.MultiIndex on both the index and columns.",
-        rtype='pandas.DataFrame',
-        is_method=False)
-def to_hierarchical_dataframe(dynamic_table):
-    """
-    Create a hierarchical pandas.DataFrame that represents all data from a collection of linked DynamicTables.
+@validated
+def to_hierarchical_dataframe(dynamic_table: DynamicTable) -> pd.DataFrame:
+    """Create a hierarchical pandas.DataFrame that represents all data from a collection of linked DynamicTables.
 
     **LIMITATIONS:** Currently this function only supports DynamicTables with a single DynamicTableRegion column.
     If a table has more than one DynamicTableRegion column then the function will expand only the
@@ -29,6 +24,12 @@ def to_hierarchical_dataframe(dynamic_table):
     * pandas.DataFrame.reset_index to turn the data from the pandas.MultiIndex into columns
     * :py:meth:`~hdmf.common.hierarchicaltable.drop_id_columns` to remove all 'id' columns
     * :py:meth:`~hdmf.common.hierarchicaltable.flatten_column_index` to flatten the column index
+
+    Args:
+        dynamic_table: DynamicTable object to be converted to a hierarchical pandas.Dataframe
+
+    Returns:
+        Hierarchical pandas.DataFrame with usually a pandas.MultiIndex on both the index and columns.
     """
     # TODO: Need to deal with the case where we have more than one DynamicTableRegion column in a given table
     # Get the references column
@@ -194,25 +195,24 @@ def __flatten_column_name(col):
         return col
 
 
-@docval({'name': 'dataframe', 'type': pd.DataFrame,
-         'doc': 'Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)'},
-        {'name': 'inplace', 'type': 'bool', 'doc': 'Update the dataframe inplace or return a modified copy',
-         'default': False},
-        returns="pandas.DataFrame with the id columns removed",
-        rtype='pandas.DataFrame',
-        is_method=False)
-def drop_id_columns(**kwargs):
-    """
-    Drop all columns named 'id' from the table.
+@validated
+def drop_id_columns(dataframe: pd.DataFrame, inplace: Bool = False) -> pd.DataFrame:
+    """Drop all columns named 'id' from the table.
 
     In case a column name is a tuple the function will drop any column for which
     the inner-most name is 'id'. The 'id' columns of DynamicTable is in many cases
     not necessary for analysis or display. This function allow us to easily filter
     all those columns.
 
     :raises TypeError: In case that dataframe parameter is not a pandas.Dataframe.
+
+    Args:
+        dataframe: Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)
+        inplace: Update the dataframe inplace or return a modified copy
+
+    Returns:
+        pandas.DataFrame with the id columns removed
     """
-    dataframe, inplace = getargs('dataframe', 'inplace', kwargs)
     col_name = 'id'
     drop_labels = []
     for col in dataframe.columns:
@@ -222,22 +222,11 @@ def drop_id_columns(**kwargs):
     return dataframe if inplace else re
 
 
-@docval({'name': 'dataframe', 'type': pd.DataFrame,
-         'doc': 'Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)'},
-        {'name': 'max_levels', 'type': (int, np.integer),
-         'doc': 'Maximum number of levels to use in the resulting column Index. NOTE:  When '
-                'limiting the number of levels the function simply removes levels from the '
-                'beginning. As such, removing levels may result in columns with duplicate names.'
-                'Value must be >0.',
-         'default': None},
-        {'name': 'inplace', 'type': 'bool', 'doc': 'Update the dataframe inplace or return a modified copy',
-         'default': False},
-        returns="pandas.DataFrame with a regular pandas.Index columns rather and a pandas.MultiIndex",
-        rtype='pandas.DataFrame',
-        is_method=False)
-def flatten_column_index(**kwargs):
-    """
-    Flatten the column index of a pandas DataFrame.
+@validated
+def flatten_column_index(dataframe: pd.DataFrame,
+                         max_levels: Int | np.integer | None = None,
+                         inplace: Bool = False) -> pd.DataFrame:
+    """Flatten the column index of a pandas DataFrame.
 
     The functions changes the dataframe.columns from a pandas.MultiIndex to a normal Index,
     with each column usually being identified by a tuple of strings. This function is
@@ -246,8 +235,17 @@ def flatten_column_index(**kwargs):
 
     :raises ValueError: In case the num_levels is not >0
     :raises TypeError: In case that dataframe parameter is not a pandas.Dataframe.
+
+    Args:
+        dataframe: Pandas dataframe to update (usually generated by the to_hierarchical_dataframe function)
+        max_levels: Maximum number of levels to use in the resulting column Index. NOTE: When limiting
+            the number of levels the function simply removes levels from the beginning. As such, removing
+            levels may result in columns with duplicate names. Value must be >0.
+        inplace: Update the dataframe inplace or return a modified copy
+
+    Returns:
+        pandas.DataFrame with a regular pandas.Index columns rather and a pandas.MultiIndex
     """
-    dataframe, max_levels, inplace = getargs('dataframe', 'max_levels', 'inplace', kwargs)
     if max_levels is not None and max_levels <= 0:
         raise ValueError('max_levels must be greater than 0')
     # Compute the new column names