fix(libvirt): record suppressed native failures on the operator side

docs/requirements/RUN-316/requirement.md

@@ -25,3 +25,7 @@ Processors and backends require their own operational observability, but that co
 - TESTS → TEST `implementations/python/tests/test_backend_manifest.py` (Backend manifest tests cover observation contract vocabulary and experiment-run-v1 support)
 - TESTS → TEST `implementations/python/tests/test_runtime_control_plane_api.py` (Operational apparatus summary API tests)
 - IMPLEMENTS → GITHUB_ISSUE `338` (Operational Apparatus Observability (RUN-316))
+- IMPLEMENTS → GITHUB_ISSUE `1173` (Safe and complete libvirt failure observability)
+- IMPLEMENTS → PULL_REQUEST `1163` (Bounded libvirt backend failure observability)
+- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_backend_libvirt/_observability.py` (Bounded operator-side failure classification)
+- TESTS → TEST `implementations/python/tests/test_libvirt_failure_observability.py` (Suppressed-failure coverage, redaction, and expected-absence tests)

implementations/python/packages/raes_backend_libvirt/_observability.py

@@ -0,0 +1,33 @@
+"""Backend-local observability for suppressed backend failures.
+
+The portable driver boundary deliberately collapses native errors into
+value-free diagnostics. This module records bounded, non-sensitive failure
+classification on the operator's side of that boundary without retaining
+exception messages or tracebacks. It is silent unless the embedding application
+configures logging for ``raes_backend_libvirt``.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+LOGGER = logging.getLogger("raes_backend_libvirt")
+_SAFE_TYPE_RE = re.compile(r"[^A-Za-z0-9_.-]+")
+
+
+def record_suppressed_failure(
+    operation: str,
+    exc: BaseException,
+    *,
+    native_code: int | None = None,
+) -> None:
+    """Record bounded failure classification without exception text or traceback."""
+
+    exception_type = _SAFE_TYPE_RE.sub("-", type(exc).__name__)[:80] or "Exception"
+    fields = [f"exception_type={exception_type}"]
+    if isinstance(exc, OSError) and type(exc.errno) is int:
+        fields.append(f"errno={exc.errno}")
+    if type(native_code) is int:
+        fields.append(f"native_code={native_code}")
+    LOGGER.debug("%s suppressed backend failure (%s)", operation, ", ".join(fields))

implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py

@@ -15,6 +15,8 @@
 from collections.abc import Callable
 from typing import Protocol, cast
 
+from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure
+
 _SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.-]+")
 # Fixed namespace for deriving a per-address libvirt UUID. The UUID proves an
 # existing host object was realized by RAES for *this* address, so convergence
@@ -56,7 +58,8 @@ def _error_code(exc: BaseException) -> int | None:
         return None
     try:
         code = getter()
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("_error_code", exc)
         return None
     return code if isinstance(code, int) else None
 
@@ -101,7 +104,8 @@ def _existing_uuid(native: object) -> str | None:
         return None
     try:
         return reader()
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("_existing_uuid", exc)
         return None
 
 
@@ -135,7 +139,10 @@ def _lookup(connection: object, method_name: str, name: str) -> object | None:
         return None
     try:
         return method(name)
-    except Exception:
+    except Exception as exc:
+        code = _error_code(exc)
+        if code not in _ABSENCE_ERROR_CODES:
+            _record_suppressed_failure("_lookup", exc, native_code=code)
         return None
 
 

implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import contextlib
 import os
 import shutil
 import tempfile
@@ -14,6 +13,7 @@
 from raes_contracts.realization_envelope import ObservationStrength, RealizationConcern
 from raes_contracts.realization_observation import RealizationObservation
 
+from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure
 from raes_backend_libvirt.driver import (
     DomainHandle,
     DomainSpec,
@@ -100,7 +100,8 @@ def realize(
         created_domains: list[str] = []
         try:
             connection = self._conn()
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("realize", exc)
             return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),))
 
         realize_network_specs(self, connection, networks, created_networks, network_handles, diagnostics)
@@ -149,7 +150,8 @@ def _compute_substrate_observation(
                 native = lookup(self._name_for(address))
                 active = getattr(native, "isActive", None)
                 owned_and_active = _existing_uuid(native) == _raes_uuid(address) and callable(active) and active() == 1
-            except Exception:
+            except Exception as exc:
+                _record_suppressed_failure("_compute_substrate_observation", exc)
                 owned_and_active = False
             if owned_and_active:
                 envelope = load_libvirt_realization_envelope(self.driver_mode)
@@ -172,7 +174,8 @@ def observe(self, *, domains: tuple[DomainSpec, ...]) -> DriverResult:
 
         try:
             connection = self._conn()
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("observe", exc)
             return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),))
         observations: list[RealizationObservation] = []
         diagnostics: list[Diagnostic] = []
@@ -221,7 +224,8 @@ def _realize_network(self, connection: object, spec: NetworkSpec, created: list[
             native.create()
         except _OwnershipConflict:
             return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT)
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("_realize_network", exc)
             return _failure(spec.address, _CODE_OPERATION_FAILED)
         self._realized.add(spec.address)
         return None
@@ -250,7 +254,8 @@ def _realize_domain(self, connection: object, spec: DomainSpec, created: list[st
             native.create()
         except _OwnershipConflict:
             return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT)
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("_realize_domain", exc)
             return _failure(spec.address, _CODE_OPERATION_FAILED)
         self._realized.add(spec.address)
         return None
@@ -264,7 +269,8 @@ def destroy(
         diagnostics: list[Diagnostic] = []
         try:
             connection = self._conn()
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("destroy", exc)
             return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),))
 
         domain_handles = self._destroy_domains(connection, domains, diagnostics)
@@ -437,31 +443,39 @@ def _undefine_nwfilter(self, connection: object, address: str) -> None:
             return
         # Best-effort cleanup of our own filter: a filter that cannot be undefined
         # (in use, missing) must not fail the destroy.
-        with contextlib.suppress(Exception):
+        try:
             cast(_NativeResource, native).undefine()
+        except Exception as exc:
+            if not _is_absence_error(exc):
+                _record_suppressed_failure("_undefine_nwfilter", exc)
 
     def _destroy_one(self, connection: object, lookup_method: str, address: str) -> bool:
+        removed = True
         try:
             native = _find_native(connection, lookup_method, self._name_for(address))
-        except _NativeLookupError:
+        except _NativeLookupError as exc:
             # Connection/permission/internal lookup failure: fail closed so the
             # snapshot is preserved for retry instead of claiming the object gone.
-            return False
-        # A None result is genuine absence — teardown is idempotently satisfied.
-        # A present object is torn down only when its UUID proves RAES ownership
-        # (the same invariant as convergence), never a foreign name collision.
-        if native is not None:
-            if _existing_uuid(native) != _raes_uuid(address):
-                raise _OwnershipConflict(address)
-            try:
-                _stop_native(native)
-                cast(_NativeResource, native).undefine()
-            except Exception as exc:
-                # An object that vanished between lookup and undefine is still torn
-                # down; a stop/undefine that failed for permission or an internal
-                # reason fails closed and preserves the snapshot for retry.
-                return _is_absence_error(exc)
-        return True
+            _record_suppressed_failure("_destroy_one", exc.__cause__ or exc)
+            removed = False
+        else:
+            # A None result is genuine absence — teardown is idempotently satisfied.
+            # A present object is torn down only when its UUID proves RAES ownership
+            # (the same invariant as convergence), never a foreign name collision.
+            if native is not None:
+                if _existing_uuid(native) != _raes_uuid(address):
+                    raise _OwnershipConflict(address)
+                try:
+                    _stop_native(native)
+                    cast(_NativeResource, native).undefine()
+                except Exception as exc:
+                    # An object that vanished between lookup and undefine is still torn
+                    # down; a stop/undefine that failed for permission or an internal
+                    # reason fails closed and preserves the snapshot for retry.
+                    removed = _is_absence_error(exc)
+                    if not removed:
+                        _record_suppressed_failure("_destroy_one", exc)
+        return removed
 
     def _rollback(self, networks: list[str], domains: list[str]) -> None:
         if networks or domains:

implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py

@@ -22,6 +22,8 @@
 
 from raes_contracts.diagnostics import Diagnostic
 
+from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure
+
 from ._techvault_native_ops import _CODE_GUEST_FRESHNESS_UNAVAILABLE, _diagnostic
 from .driver import DomainSpec, NetworkSpec, RealizationObservation
 from .drivers.libvirt import _raes_uuid
@@ -98,7 +100,8 @@ def _prepare_operation(self, matrix: Mapping[str, object]) -> list[Diagnostic]:
         diagnostics: list[Diagnostic] = []
         try:
             candidate = self.challenge_factory()
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("_prepare_operation", exc)
             candidate = None
         if (
             not isinstance(candidate, str)

implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py

@@ -5,6 +5,8 @@
 from collections.abc import Callable
 from dataclasses import dataclass
 
+from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure
+
 from .drivers.libvirt import _error_code, _existing_uuid, _raes_uuid
 from .techvault_matrix import runtime_name
 
@@ -71,8 +73,11 @@ def _resolve_by_name(
     except KeyError:
         resolved = _resolve_verified_absence(connection, list_method, address)
     except Exception as exc:
-        if _error_code(exc) in {42, 43}:
+        code = _error_code(exc)
+        if code in {42, 43}:
             resolved = _resolve_verified_absence(connection, list_method, address)
+        else:
+            _record_suppressed_failure("_resolve_by_name", exc, native_code=code)
     else:
         resolved = NativeResolution(native=native, name=name)
     return resolved
@@ -116,7 +121,11 @@ def _invoke_native_action(native: object, method_name: str, tolerated_codes: set
     try:
         method()
     except Exception as exc:
-        return _error_code(exc) in tolerated_codes
+        code = _error_code(exc)
+        tolerated = code in tolerated_codes
+        if not tolerated:
+            _record_suppressed_failure("_invoke_native_action", exc, native_code=code)
+        return tolerated
     return True
 
 
@@ -126,7 +135,8 @@ def _list_native(connection: object, method_name: str) -> tuple[object, ...] | N
         return None
     try:
         native = method()
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("_list_native", exc)
         return None
     return tuple(native) if isinstance(native, list | tuple) else None
 
@@ -137,7 +147,8 @@ def _native_name(native: object) -> str:
         return ""
     try:
         value = method()
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("_native_name", exc)
         return ""
     return value if isinstance(value, str) else ""
 

implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py

@@ -14,6 +14,8 @@
 
 from raes_contracts.diagnostics import Diagnostic
 
+from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure
+
 from .._techvault_native_ops import (
     _CODE_OPERATION_FAILED,
     _CODE_OWNERSHIP_CONFLICT,
@@ -76,7 +78,8 @@ def define_network(
     except _OwnershipConflict:
         driver._names.pop(address, None)
         diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address)
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("define_network", exc)
         if native is None:
             driver._names.pop(address, None)
         else:
@@ -88,7 +91,8 @@ def define_network(
         handle = NetworkHandle(address=address, realized=True)
         try:
             observations = network_observations(native, network)
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("define_network", exc)
             diagnostic = _diagnostic(_CODE_READBACK_FAILED, address)
     return handle, diagnostic, observations
 
@@ -152,7 +156,8 @@ def define_domain(
     except _OwnershipConflict:
         driver._names.pop(address, None)
         diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address)
-    except Exception:
+    except Exception as exc:
+        _record_suppressed_failure("define_domain", exc)
         if native is None:
             driver._cleanup_artifacts(address)
             driver._names.pop(address, None)
@@ -171,6 +176,7 @@ def define_domain(
                 kernel=kernel,
                 initrd=initrd,
             )
-        except Exception:
+        except Exception as exc:
+            _record_suppressed_failure("define_domain", exc)
             diagnostic = _diagnostic(_CODE_READBACK_FAILED, address)
     return handle, diagnostic, observations