Add namespace-agnostic type resolution to NamespaceCatalog (#1528)

CHANGELOG.md

@@ -1,5 +1,10 @@
 # HDMF Changelog
 
+## HDMF 6.2.0 (Upcoming)
+
+### Enhancements
+- Added namespace-agnostic type resolution to `NamespaceCatalog`. `get_spec`, `get_hierarchy`, and `is_sub_data_type` now accept `namespace=None` to search all loaded namespaces (returning the first match by name, mirroring `TypeMap.get_dt_container_cls`), and new `type_key`/`type_keys` properties report the catalog's data type key(s). Pass an explicit namespace to disambiguate a type name that is defined differently in more than one namespace. This is groundwork for validating files that use multiple extensions across namespaces (#608). @rly [#1531](https://github.com/hdmf-dev/hdmf/pull/1531)
+
 ## HDMF 6.1.0 (June 25, 2026)
 
 ### Enhancements

src/hdmf/spec/namespace.py

@@ -298,6 +298,16 @@ def core_namespaces(self):
         """The core namespaces used in this NamespaceCatalog"""
         return self.__core_namespaces
 
+    @property
+    def type_key(self):
+        """The data type key used by this catalog's spec classes (e.g. 'data_type')."""
+        return self.__group_spec_cls.type_key()
+
+    @property
+    def type_keys(self):
+        """The set of data type keys used by this catalog's group and dataset spec classes."""
+        return {self.__group_spec_cls.type_key(), self.__dataset_spec_cls.type_key()}
+
     def get_source_types(self, ns_name):
         """Get the source types for a namespace.
 
@@ -356,40 +366,75 @@ def get_namespace(self, **kwargs):
             raise KeyError("'%s' not a namespace" % name)
         return ret
 
-    @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'},
-            {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'},
+    def _namespace_of(self, data_type):
+        """Return the name of the first loaded namespace that defines *data_type*, or None if none do."""
+        if isinstance(data_type, type):
+            data_type = data_type.__name__
+        for name, namespace in self.__namespaces.items():
+            if namespace.catalog.get_spec(data_type) is not None:
+                return name
+        return None
+
+    @docval({'name': 'namespace', 'type': str,
+             'doc': 'the name of the namespace, or None to search all loaded namespaces', 'default': None},
+            {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for', 'default': None},
             returns="the specification for writing the given object type to HDF5 ", rtype='Spec')
     def get_spec(self, **kwargs):
         '''
-        Get the Spec object for the given type from the given Namespace
+        Get the Spec object for the given type. If *namespace* is None, search all loaded namespaces
+        and return the spec from the first namespace that defines the type. Searching by name alone
+        assumes a data type name identifies the same type in every namespace; if two namespaces
+        define different types with the same name, pass an explicit *namespace* to disambiguate.
         '''
         namespace, data_type = getargs('namespace', 'data_type', kwargs)
+        if data_type is None:
+            raise ValueError("'data_type' must be provided")
+        if namespace is None:
+            namespace = self._namespace_of(data_type)
+            if namespace is None:
+                raise ValueError("No specification for '%s' in any loaded namespace" % data_type)
         if namespace not in self.__namespaces:
             raise KeyError("'%s' not a namespace" % namespace)
         return self.__namespaces[namespace].get_spec(data_type)
 
-    @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'},
-            {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for'},
+    @docval({'name': 'namespace', 'type': str,
+             'doc': 'the name of the namespace, or None to search all loaded namespaces', 'default': None},
+            {'name': 'data_type', 'type': (str, type), 'doc': 'the data_type to get the spec for', 'default': None},
             returns="a tuple with the type hierarchy", rtype=tuple)
     def get_hierarchy(self, **kwargs):
         '''
-        Get the type hierarchy for a given data_type in a given namespace
+        Get the type hierarchy for a given data_type. If *namespace* is None, search all loaded
+        namespaces; returns an empty tuple if no loaded namespace defines the type. Searching by
+        name alone assumes the name identifies the same type in every namespace; pass an explicit
+        *namespace* to disambiguate colliding names.
         '''
         namespace, data_type = getargs('namespace', 'data_type', kwargs)
+        if data_type is None:
+            raise ValueError("'data_type' must be provided")
+        if namespace is None:
+            namespace = self._namespace_of(data_type)
+            if namespace is None:
+                return tuple()
         spec_ns = self.__namespaces.get(namespace)
         if spec_ns is None:
             raise KeyError("'%s' not a namespace" % namespace)
         return spec_ns.get_hierarchy(data_type)
 
-    @docval({'name': 'namespace', 'type': str, 'doc': 'the name of the namespace containing the data_type'},
-            {'name': 'data_type', 'type': str, 'doc': 'the data_type to check'},
-            {'name': 'parent_data_type', 'type': str, 'doc': 'the potential parent data_type'},
+    @docval({'name': 'namespace', 'type': str,
+             'doc': 'the name of the namespace containing the data_type, or None to search all loaded namespaces',
+             'default': None},
+            {'name': 'data_type', 'type': str, 'doc': 'the data_type to check', 'default': None},
+            {'name': 'parent_data_type', 'type': str, 'doc': 'the potential parent data_type', 'default': None},
             returns="True if *data_type* is a sub `data_type` of *parent_data_type*, False otherwise", rtype=bool)
     def is_sub_data_type(self, **kwargs):
         '''
-        Return whether or not *data_type* is a sub `data_type` of *parent_data_type*
+        Return whether or not *data_type* is a sub `data_type` of *parent_data_type*. If *namespace*
+        is None, search all loaded namespaces (assuming names identify the same type in every
+        namespace; pass an explicit *namespace* to disambiguate colliding names).
         '''
         ns, dt, parent_dt = getargs('namespace', 'data_type', 'parent_data_type', kwargs)
+        if dt is None or parent_dt is None:
+            raise ValueError("'data_type' and 'parent_data_type' must be provided")
         hier = self.get_hierarchy(ns, dt)
         return parent_dt in hier
 

tests/unit/spec_tests/test_multi_namespace_resolution.py

@@ -0,0 +1,106 @@
+from hdmf.spec import DatasetSpec, GroupSpec, SpecCatalog, SpecNamespace, NamespaceCatalog
+from hdmf.testing import TestCase
+
+
+class TestMultiNamespaceResolution(TestCase):
+    """Tests for namespace-agnostic type resolution across all loaded namespaces.
+
+    The fixture builds a shared dependency namespace ('test-core') and two independent
+    extension namespaces ('ndx-a', 'ndx-b'), neither of which depends on the other. As the
+    real loader does, each extension namespace's catalog also holds the dependency types it
+    includes ('Base', 'MyVector'). ns-a defines a subtype 'AVector' of 'MyVector' and ns-b
+    defines an independent subtype 'MySubVector' of 'MyVector'.
+    """
+
+    def setUp(self):
+        def core_catalog():
+            cat = SpecCatalog()
+            cat.register_spec(GroupSpec(doc='a base container', data_type_def='Base'), 'core.yaml')
+            cat.register_spec(DatasetSpec(doc='a base vector', data_type_def='MyVector', dtype='int'), 'core.yaml')
+            return cat
+
+        cat_core = core_catalog()
+
+        cat_a = core_catalog()  # includes the dependency types, as the loader would
+        cat_a.register_spec(GroupSpec(doc='type A', data_type_inc='Base', data_type_def='TypeA'), 'a.yaml')
+        cat_a.register_spec(DatasetSpec(doc='an A vector', data_type_inc='MyVector', data_type_def='AVector'), 'a.yaml')
+        # a type whose name collides with a different type in ndx-b
+        cat_a.register_spec(GroupSpec(doc='an A widget', data_type_def='Widget'), 'a.yaml')
+
+        cat_b = core_catalog()  # includes the dependency types, as the loader would
+        cat_b.register_spec(GroupSpec(doc='type B', data_type_inc='Base', data_type_def='TypeB'), 'b.yaml')
+        cat_b.register_spec(
+            DatasetSpec(doc='a sub vector', data_type_inc='MyVector', data_type_def='MySubVector'), 'b.yaml'
+        )
+        # a different type that reuses the name 'Widget' (here it inherits Base)
+        cat_b.register_spec(GroupSpec(doc='a B widget', data_type_inc='Base', data_type_def='Widget'), 'b.yaml')
+
+        core_ns = SpecNamespace('a shared core', 'test-core', [{'source': 'core.yaml'}],
+                                version='0.1.0', catalog=cat_core)
+        a_ns = SpecNamespace('extension a', 'ndx-a', [{'namespace': 'test-core'}, {'source': 'a.yaml'}],
+                             version='0.1.0', catalog=cat_a)
+        b_ns = SpecNamespace('extension b', 'ndx-b', [{'namespace': 'test-core'}, {'source': 'b.yaml'}],
+                             version='0.1.0', catalog=cat_b)
+
+        self.catalog = NamespaceCatalog()
+        self.catalog.add_namespace('test-core', core_ns)
+        self.catalog.add_namespace('ndx-a', a_ns)
+        self.catalog.add_namespace('ndx-b', b_ns)
+
+    def test_get_spec_search_all_namespaces(self):
+        """get_spec with namespace=None finds a type defined in any loaded namespace."""
+        self.assertEqual(self.catalog.get_spec(data_type='TypeA').data_type_def, 'TypeA')
+        self.assertEqual(self.catalog.get_spec(data_type='TypeB').data_type_def, 'TypeB')
+        self.assertEqual(self.catalog.get_spec(data_type='MySubVector').data_type_def, 'MySubVector')
+        # a shared dependency type resolves too
+        self.assertEqual(self.catalog.get_spec(data_type='Base').data_type_def, 'Base')
+
+    def test_get_spec_two_arg_unchanged(self):
+        """The existing (namespace, data_type) behavior is unchanged."""
+        self.assertEqual(self.catalog.get_spec('ndx-a', 'TypeA').data_type_def, 'TypeA')
+        # a type not in the named namespace still raises, as before
+        with self.assertRaises(ValueError):
+            self.catalog.get_spec('ndx-a', 'TypeB')
+
+    def test_get_spec_unknown_type_raises(self):
+        with self.assertRaises(ValueError):
+            self.catalog.get_spec(data_type='NotAType')
+
+    def test_get_spec_requires_data_type(self):
+        with self.assertRaises(ValueError):
+            self.catalog.get_spec()
+
+    def test_get_hierarchy_search_all_namespaces(self):
+        self.assertTupleEqual(self.catalog.get_hierarchy(data_type='MySubVector'), ('MySubVector', 'MyVector'))
+        self.assertTupleEqual(self.catalog.get_hierarchy(data_type='TypeA'), ('TypeA', 'Base'))
+
+    def test_get_hierarchy_two_arg_unchanged(self):
+        self.assertTupleEqual(self.catalog.get_hierarchy('ndx-b', 'MySubVector'), ('MySubVector', 'MyVector'))
+
+    def test_get_hierarchy_unknown_type_returns_empty(self):
+        self.assertTupleEqual(self.catalog.get_hierarchy(data_type='NotAType'), ())
+
+    def test_colliding_name_disambiguated_by_namespace(self):
+        """The same type name in two namespaces resolves to different specs when the namespace is given."""
+        a_widget = self.catalog.get_spec('ndx-a', 'Widget')
+        b_widget = self.catalog.get_spec('ndx-b', 'Widget')
+        self.assertIsNot(a_widget, b_widget)
+        self.assertIsNone(a_widget.data_type_inc)
+        self.assertEqual(b_widget.data_type_inc, 'Base')
+
+    def test_colliding_name_search_all_returns_first(self):
+        """With namespace=None, a colliding name resolves to the first-loaded namespace (documented behavior)."""
+        # 'ndx-a' is added before 'ndx-b', so its Widget (which does not inherit Base) is returned
+        self.assertIsNone(self.catalog.get_spec(data_type='Widget').data_type_inc)
+
+    def test_is_sub_data_type_search_all_namespaces(self):
+        self.assertTrue(self.catalog.is_sub_data_type(data_type='MySubVector', parent_data_type='MyVector'))
+        self.assertTrue(self.catalog.is_sub_data_type(data_type='TypeA', parent_data_type='Base'))
+        self.assertFalse(self.catalog.is_sub_data_type(data_type='TypeA', parent_data_type='MyVector'))
+
+    def test_is_sub_data_type_two_arg_unchanged(self):
+        self.assertTrue(self.catalog.is_sub_data_type('ndx-b', 'MySubVector', 'MyVector'))
+
+    def test_type_key_properties(self):
+        self.assertEqual(self.catalog.type_key, 'data_type')
+        self.assertSetEqual(self.catalog.type_keys, {'data_type'})