From 342869d33e50d4b528c7a59a53756c325e26df96 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:12:48 -0700 Subject: [PATCH 1/6] fix(libvirt): record suppressed native failures on the operator side The libvirt backend deliberately collapses native errors into value-free portable diagnostics, but that left zero field observability: 25 broad except-Exception/BaseException sites discarded the native failure entirely, so a failing real-libvirt run could not be explained. Add a backend-local logger (raes_backend_libvirt._observability, named 'raes_backend_libvirt', no handlers -- silent unless the embedding application configures logging) and record every suppressed native failure at DEBUG with the enclosing operation and full exc_info before it collapses. The portable boundary is unchanged: diagnostics stay value-free and no native detail crosses it. _verify_and_finalize moves to techvault_native/_finalize.py (the existing _define.py driver-function pattern) to keep _driver.py under the 500-line cap after instrumentation. Verification: full hermetic suite green with the coverage gate; libvirt/techvault-focused tests (366) pass; ruff clean; check_repo_policy pass. Co-Authored-By: Claude Fable 5 --- .../raes_backend_libvirt/_initramfs.py | 9 +- .../raes_backend_libvirt/_observability.py | 14 ++++ .../drivers/libvirt/_native.py | 12 ++- .../drivers/libvirt/deployment.py | 20 +++-- .../guest_certified_driver.py | 6 +- .../techvault_lifecycle.py | 9 +- .../techvault_native/_define.py | 15 +++- .../techvault_native/_driver.py | 83 +++++-------------- .../techvault_native/_finalize.py | 77 +++++++++++++++++ .../techvault_native/_preflight.py | 6 +- 10 files changed, 172 insertions(+), 79 deletions(-) create mode 100644 implementations/python/packages/raes_backend_libvirt/_observability.py create mode 100644 implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py diff --git a/implementations/python/packages/raes_backend_libvirt/_initramfs.py b/implementations/python/packages/raes_backend_libvirt/_initramfs.py index 5e4bc5086..7408a3546 100644 --- a/implementations/python/packages/raes_backend_libvirt/_initramfs.py +++ b/implementations/python/packages/raes_backend_libvirt/_initramfs.py @@ -14,6 +14,9 @@ from pathlib import Path from typing import BinaryIO +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + _NEWC_MAGIC = b"070701" _NEWC_TRAILER = "TRAILER!!!" _ELF_MACHINE_X86_64 = 62 @@ -175,7 +178,8 @@ def atomic_write(path: Path, payload: bytes, *, mode: int) -> Path: os.chmod(temporary, mode) os.replace(temporary, path) _fsync_directory(path.parent) - except BaseException: + except BaseException as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "atomic_write", exc_info=exc) temporary.unlink(missing_ok=True) raise return path @@ -207,7 +211,8 @@ def atomic_copy_by_digest(source: Path, target: Path, *, mode: int) -> Path: os.chmod(temporary, mode) os.replace(temporary, target) _fsync_directory(target.parent) - except BaseException: + except BaseException as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "atomic_copy_by_digest", exc_info=exc) temporary.unlink(missing_ok=True) raise return target diff --git a/implementations/python/packages/raes_backend_libvirt/_observability.py b/implementations/python/packages/raes_backend_libvirt/_observability.py new file mode 100644 index 000000000..46b21f831 --- /dev/null +++ b/implementations/python/packages/raes_backend_libvirt/_observability.py @@ -0,0 +1,14 @@ +"""Backend-local observability for native libvirt failures. + +The portable driver boundary deliberately collapses native errors into +value-free diagnostics; this logger records the collapsed detail on the +operator's side of that boundary. It is silent unless the embedding +application configures logging for ``raes_backend_libvirt``. +""" + +from __future__ import annotations + +import logging + +LOGGER = logging.getLogger("raes_backend_libvirt") +NATIVE_FAILURE_LOG = "%s suppressed a native libvirt failure" diff --git a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py index 334368ae8..56e78d07b 100644 --- a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py +++ b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py @@ -15,6 +15,9 @@ from collections.abc import Callable from typing import Protocol, cast +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + _SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.-]+") # Fixed namespace for deriving a per-address libvirt UUID. The UUID proves an # existing host object was realized by RAES for *this* address, so convergence @@ -56,7 +59,8 @@ def _error_code(exc: BaseException) -> int | None: return None try: code = getter() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_error_code", exc_info=exc) return None return code if isinstance(code, int) else None @@ -101,7 +105,8 @@ def _existing_uuid(native: object) -> str | None: return None try: return reader() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_existing_uuid", exc_info=exc) return None @@ -135,7 +140,8 @@ def _lookup(connection: object, method_name: str, name: str) -> object | None: return None try: return method(name) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_lookup", exc_info=exc) return None diff --git a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py index 9e06c6ec7..ef2d8c11b 100644 --- a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py +++ b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py @@ -14,6 +14,8 @@ from raes_contracts.realization_envelope import ObservationStrength, RealizationConcern from raes_contracts.realization_observation import RealizationObservation +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG from raes_backend_libvirt.driver import ( DomainHandle, DomainSpec, @@ -100,7 +102,8 @@ def realize( created_domains: list[str] = [] try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "realize", exc_info=exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) realize_network_specs(self, connection, networks, created_networks, network_handles, diagnostics) @@ -149,7 +152,8 @@ def _compute_substrate_observation( native = lookup(self._name_for(address)) active = getattr(native, "isActive", None) owned_and_active = _existing_uuid(native) == _raes_uuid(address) and callable(active) and active() == 1 - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_compute_substrate_observation", exc_info=exc) owned_and_active = False if owned_and_active: envelope = load_libvirt_realization_envelope(self.driver_mode) @@ -172,7 +176,8 @@ def observe(self, *, domains: tuple[DomainSpec, ...]) -> DriverResult: try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "observe", exc_info=exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) observations: list[RealizationObservation] = [] diagnostics: list[Diagnostic] = [] @@ -221,7 +226,8 @@ def _realize_network(self, connection: object, spec: NetworkSpec, created: list[ native.create() except _OwnershipConflict: return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_realize_network", exc_info=exc) return _failure(spec.address, _CODE_OPERATION_FAILED) self._realized.add(spec.address) return None @@ -250,7 +256,8 @@ def _realize_domain(self, connection: object, spec: DomainSpec, created: list[st native.create() except _OwnershipConflict: return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_realize_domain", exc_info=exc) return _failure(spec.address, _CODE_OPERATION_FAILED) self._realized.add(spec.address) return None @@ -264,7 +271,8 @@ def destroy( diagnostics: list[Diagnostic] = [] try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "destroy", exc_info=exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) domain_handles = self._destroy_domains(connection, domains, diagnostics) diff --git a/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py b/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py index 966464919..d49db99f3 100644 --- a/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py +++ b/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py @@ -22,6 +22,9 @@ from raes_contracts.diagnostics import Diagnostic +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + from ._techvault_native_ops import _CODE_GUEST_FRESHNESS_UNAVAILABLE, _diagnostic from .driver import DomainSpec, NetworkSpec, RealizationObservation from .drivers.libvirt import _raes_uuid @@ -98,7 +101,8 @@ def _prepare_operation(self, matrix: Mapping[str, object]) -> list[Diagnostic]: diagnostics: list[Diagnostic] = [] try: candidate = self.challenge_factory() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_prepare_operation", exc_info=exc) candidate = None if ( not isinstance(candidate, str) diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py index 5d2d42b49..6fef1f429 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py @@ -5,6 +5,9 @@ from collections.abc import Callable from dataclasses import dataclass +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + from .drivers.libvirt import _error_code, _existing_uuid, _raes_uuid from .techvault_matrix import runtime_name @@ -126,7 +129,8 @@ def _list_native(connection: object, method_name: str) -> tuple[object, ...] | N return None try: native = method() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_list_native", exc_info=exc) return None return tuple(native) if isinstance(native, list | tuple) else None @@ -137,7 +141,8 @@ def _native_name(native: object) -> str: return "" try: value = method() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_native_name", exc_info=exc) return "" return value if isinstance(value, str) else "" diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py index a81e79359..fa9e9fdf3 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py @@ -14,6 +14,9 @@ from raes_contracts.diagnostics import Diagnostic +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + from .._techvault_native_ops import ( _CODE_OPERATION_FAILED, _CODE_OWNERSHIP_CONFLICT, @@ -76,7 +79,8 @@ def define_network( except _OwnershipConflict: driver._names.pop(address, None) diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_network", exc_info=exc) if native is None: driver._names.pop(address, None) else: @@ -88,7 +92,8 @@ def define_network( handle = NetworkHandle(address=address, realized=True) try: observations = network_observations(native, network) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_network", exc_info=exc) diagnostic = _diagnostic(_CODE_READBACK_FAILED, address) return handle, diagnostic, observations @@ -152,7 +157,8 @@ def define_domain( except _OwnershipConflict: driver._names.pop(address, None) diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_domain", exc_info=exc) if native is None: driver._cleanup_artifacts(address) driver._names.pop(address, None) @@ -171,6 +177,7 @@ def define_domain( kernel=kernel, initrd=initrd, ) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_domain", exc_info=exc) diagnostic = _diagnostic(_CODE_READBACK_FAILED, address) return handle, diagnostic, observations diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py index 7bc68eb23..afed75de0 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py @@ -16,8 +16,15 @@ from raes_contracts.diagnostics import Diagnostic -from .._techvault_native_helpers import default_connector as _default_connector -from .._techvault_native_helpers import default_kernel_path as _default_kernel_path +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + +from .._techvault_native_helpers import ( + default_connector as _default_connector, +) +from .._techvault_native_helpers import ( + default_kernel_path as _default_kernel_path, +) from .._techvault_native_ops import ( _CODE_OPERATION_FAILED, _CODE_OWNERSHIP_CONFLICT, @@ -27,6 +34,7 @@ _artifact_token, _diagnostic, ) +from ._finalize import _verify_and_finalize if TYPE_CHECKING: from raes_contracts.realization_envelope import BackendRealizationEnvelopeModel @@ -42,7 +50,7 @@ from ..drivers.libvirt import Connector, _existing_uuid, _raes_uuid from ..envelopes import load_libvirt_realization_envelope from ..techvault_appliance import BusyboxInitramfsBuilder, InitramfsBuilder -from ..techvault_concerns import techvault_observation_diagnostics, techvault_spec_diagnostics +from ..techvault_concerns import techvault_spec_diagnostics from ..techvault_lifecycle import ( NativeOwnershipConflict as _OwnershipConflict, ) @@ -66,7 +74,6 @@ canonical_digest, file_digest, native_active, - snapshot_from_observations, substrate_observation, ) from ._define import define_domain, define_domains, define_network, define_networks @@ -128,7 +135,8 @@ def realize( else: try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "realize", exc_info=exc) result = DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) else: result = self._realize_matrix( @@ -160,7 +168,8 @@ def _realize_matrix( domain_diagnostics.extend(self._rollback(connection, network_handles, domain_handles)) return DriverResult(diagnostics=tuple(domain_diagnostics)) observations = (*network_observations, *domain_observations) - return self._verify_and_finalize( + return _verify_and_finalize( + self, connection, matrix, specs=(networks, domains), @@ -170,56 +179,6 @@ def _realize_matrix( configuration_digest=configuration_digest, ) - def _verify_and_finalize( - self, - connection: object, - matrix: Mapping[str, object], - *, - specs: tuple[tuple[NetworkSpec, ...], tuple[DomainSpec, ...]], - handles: tuple[list[NetworkHandle], list[DomainHandle]], - observations: tuple[RealizationObservation, ...], - envelope_digest: str, - configuration_digest: str, - ) -> DriverResult: - networks, domains = specs - network_handles, domain_handles = handles - observations = tuple( - replace( - observation, - envelope_digest=envelope_digest, - configuration_digest=configuration_digest, - ) - if observation.concern.value == "compute-substrate" - else observation - for observation in observations - ) - diagnostics = techvault_observation_diagnostics( - networks=networks, - domains=domains, - result=DriverResult(observations=observations), - ) - # Staged: the guest observation runs only after the daemon gate passes and a - # later stage never repairs an earlier one. - guest_observations: tuple[RealizationObservation, ...] = () - if not diagnostics: - guest_observations, diagnostics = self._guest_stage(connection, matrix, specs, observations) - if diagnostics: - diagnostics.extend(self._rollback(connection, network_handles, domain_handles)) - return DriverResult(diagnostics=tuple(diagnostics)) - try: - binding = self._material_binding(envelope_digest, configuration_digest) - snapshot = snapshot_from_observations(matrix, observations, binding=binding) - except Exception: - binding_diagnostics = [_diagnostic(_CODE_OPERATION_FAILED, "runtime.libvirt.binding")] - binding_diagnostics.extend(self._rollback(connection, network_handles, domain_handles)) - return DriverResult(diagnostics=tuple(binding_diagnostics)) - self.last_snapshot = snapshot - return DriverResult( - networks=tuple(network_handles), - domains=tuple(domain_handles), - observations=(*observations, *guest_observations), - ) - def _admission_diagnostics( self, networks: tuple[NetworkSpec, ...], @@ -306,7 +265,8 @@ def destroy( ) -> DriverResult: try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "destroy", exc_info=exc) return DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) domain_handles: list[DomainHandle] = [] network_handles: list[NetworkHandle] = [] @@ -344,7 +304,8 @@ def observe(self, *, domains: tuple[DomainSpec, ...]) -> DriverResult: try: connection = self._conn() - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "observe", exc_info=exc) return DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) envelope = load_libvirt_realization_envelope(self.driver_mode) observations: list[RealizationObservation] = [] @@ -386,7 +347,8 @@ def _observed_domain(self, connection: object, address: str) -> _NativeResolutio native = None if resolved is None else resolved.native if native is None or _existing_uuid(native) != _raes_uuid(address) or not native_active(native): return None - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_observed_domain", exc_info=exc) return None return resolved @@ -489,5 +451,6 @@ def _rollback_handles( def _try_destroy(self, connection: object, lookup_method: str, address: str) -> bool: try: return self._destroy_one(connection, lookup_method, address) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_try_destroy", exc_info=exc) return False diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py new file mode 100644 index 000000000..651b345e5 --- /dev/null +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py @@ -0,0 +1,77 @@ +"""Post-realization verification and snapshot finalization for the native driver.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import replace +from typing import TYPE_CHECKING + +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + +from .._techvault_native_ops import _CODE_OPERATION_FAILED, _diagnostic +from ..driver import ( + DomainHandle, + DomainSpec, + DriverResult, + NetworkHandle, + NetworkSpec, + RealizationObservation, +) +from ..techvault_concerns import techvault_observation_diagnostics +from ..techvault_observation import snapshot_from_observations + +if TYPE_CHECKING: + from ._driver import TechVaultNativeLibvirtDriver + + +def _verify_and_finalize( + driver: TechVaultNativeLibvirtDriver, + connection: object, + matrix: Mapping[str, object], + *, + specs: tuple[tuple[NetworkSpec, ...], tuple[DomainSpec, ...]], + handles: tuple[list[NetworkHandle], list[DomainHandle]], + observations: tuple[RealizationObservation, ...], + envelope_digest: str, + configuration_digest: str, +) -> DriverResult: + networks, domains = specs + network_handles, domain_handles = handles + observations = tuple( + replace( + observation, + envelope_digest=envelope_digest, + configuration_digest=configuration_digest, + ) + if observation.concern.value == "compute-substrate" + else observation + for observation in observations + ) + diagnostics = techvault_observation_diagnostics( + networks=networks, + domains=domains, + result=DriverResult(observations=observations), + ) + # Staged: the guest observation runs only after the daemon gate passes and a + # later stage never repairs an earlier one. + guest_observations: tuple[RealizationObservation, ...] = () + if not diagnostics: + guest_observations, diagnostics = driver._guest_stage(connection, matrix, specs, observations) + if diagnostics: + diagnostics.extend(driver._rollback(connection, network_handles, domain_handles)) + return DriverResult(diagnostics=tuple(diagnostics)) + try: + binding = driver._material_binding(envelope_digest, configuration_digest) + snapshot = snapshot_from_observations(matrix, observations, binding=binding) + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "_verify_and_finalize", exc_info=exc) + binding_diagnostics = [_diagnostic(_CODE_OPERATION_FAILED, "runtime.libvirt.binding")] + binding_diagnostics.extend(driver._rollback(connection, network_handles, domain_handles)) + return DriverResult(diagnostics=tuple(binding_diagnostics)) + driver.last_snapshot = snapshot + return DriverResult( + networks=tuple(network_handles), + domains=tuple(domain_handles), + observations=(*observations, *guest_observations), + ) diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py index 918d6fd87..1f9a15f25 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py @@ -8,6 +8,9 @@ from raes_contracts.diagnostics import Diagnostic +from raes_backend_libvirt._observability import LOGGER as _LOGGER +from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG + from .._initramfs import builder_preflight from .._techvault_native_ops import ( _CODE_KERNEL_UNAVAILABLE, @@ -59,7 +62,8 @@ def artifact_preflight_diagnostics( else: try: toolchain = builder_preflight(initramfs_builder) - except Exception: + except Exception as exc: + _LOGGER.debug(_NATIVE_FAILURE_LOG, "artifact_preflight_diagnostics", exc_info=exc) toolchain = None if toolchain is None or not toolchain.ready: diagnostic = _diagnostic(_CODE_TOOLCHAIN_UNAVAILABLE, "runtime.libvirt.initramfs") From dbfa8d897476bb81b4c616d499e8542124bfb193 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:38:35 -0700 Subject: [PATCH 2/6] test(libvirt): exercise every suppressed-failure arm and bundle digests The PR quality gate counted the new debug lines in never-exercised exception arms as uncovered new code (74% vs the 80% floor) and flagged _verify_and_finalize's eighth parameter after the relocation. test_libvirt_failure_observability.py now forces all 25 instrumented collapse sites with raising fakes, pinning both halves of the contract per arm: the portable behavior is unchanged (value-free diagnostic, None, or empty result) and the suppressed native failure is recorded on the raes_backend_libvirt logger at DEBUG with the operation named and exc_info attached. Every instrumented line is now covered by the hermetic suite. _verify_and_finalize takes the envelope/configuration digests as one binding_digests tuple, returning the signature to seven parameters. Co-Authored-By: Claude Fable 5 --- .../techvault_native/_driver.py | 3 +- .../techvault_native/_finalize.py | 4 +- .../tests/test_behavioral_relation_claims.py | 1 + ...st_dsl_437_benign_participant_execution.py | 7 +- .../tests/test_formal_semantic_validation.py | 1 + .../python/tests/test_http_download.py | 1 + .../tests/test_identity_cutover_policy.py | 1 + ...issue_898_participant_execution_control.py | 15 +- ..._issue_899_participant_resource_budgets.py | 21 +- ...est_issue_963_participant_opacity_proof.py | 3 +- .../test_libvirt_failure_observability.py | 216 ++++++++++++++++++ .../python/tests/test_project_positioning.py | 1 + .../python/tests/test_public_docs_policy.py | 1 + .../python/tests/test_repo_policy_tools.py | 7 +- .../tests/test_reusable_asset_trust_policy.py | 1 + .../test_scientific_scenario_completeness.py | 1 + .../python/tests/test_sdl_lineage.py | 3 +- .../test_sem_230_information_flow_control.py | 1 + .../python/tests/test_semantic_coverage.py | 1 + 19 files changed, 260 insertions(+), 29 deletions(-) create mode 100644 implementations/python/tests/test_libvirt_failure_observability.py diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py index afed75de0..1f223c6ff 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py @@ -175,8 +175,7 @@ def _realize_matrix( specs=(networks, domains), handles=(network_handles, domain_handles), observations=observations, - envelope_digest=envelope_digest, - configuration_digest=configuration_digest, + binding_digests=(envelope_digest, configuration_digest), ) def _admission_diagnostics( diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py index 651b345e5..c04a606f0 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py @@ -33,9 +33,9 @@ def _verify_and_finalize( specs: tuple[tuple[NetworkSpec, ...], tuple[DomainSpec, ...]], handles: tuple[list[NetworkHandle], list[DomainHandle]], observations: tuple[RealizationObservation, ...], - envelope_digest: str, - configuration_digest: str, + binding_digests: tuple[str, str], ) -> DriverResult: + envelope_digest, configuration_digest = binding_digests networks, domains = specs network_handles, domain_handles = handles observations = tuple( diff --git a/implementations/python/tests/test_behavioral_relation_claims.py b/implementations/python/tests/test_behavioral_relation_claims.py index a91a4891d..381718a6a 100644 --- a/implementations/python/tests/test_behavioral_relation_claims.py +++ b/implementations/python/tests/test_behavioral_relation_claims.py @@ -6,6 +6,7 @@ import pytest from raes_contracts.behavioral_relations import load_behavioral_relation_catalog + from tools.check_behavioral_relation_claims import ( _should_validate_structured_bindings, _validate_claim_text, diff --git a/implementations/python/tests/test_dsl_437_benign_participant_execution.py b/implementations/python/tests/test_dsl_437_benign_participant_execution.py index bcf050cd3..1ed07207d 100644 --- a/implementations/python/tests/test_dsl_437_benign_participant_execution.py +++ b/implementations/python/tests/test_dsl_437_benign_participant_execution.py @@ -9,9 +9,6 @@ import pytest import yaml -from implementations.python.tests.participant_execution_test_backend import ( - NativeParticipantExecutionController, -) from raes._errors import SDLValidationError from raes.parser import parse_sdl from raes.participant_behavior import ParticipantFailureClass @@ -67,6 +64,10 @@ from raes_runtime.participant_scheduler import ParticipantScheduler from raes_runtime.time_coordinator import ReferenceTimeRuntime, TimeCoordinator +from implementations.python.tests.participant_execution_test_backend import ( + NativeParticipantExecutionController, +) + REPO_ROOT = Path(__file__).resolve().parents[3] EXAMPLE = REPO_ROOT / "examples" / "scenarios" / "enterprise-participant-evidence-loop.sdl.yaml" IMPLEMENTATION_REF = "participant-implementation-manifests.green-worker.v1" diff --git a/implementations/python/tests/test_formal_semantic_validation.py b/implementations/python/tests/test_formal_semantic_validation.py index d47c1611d..1cabc6871 100644 --- a/implementations/python/tests/test_formal_semantic_validation.py +++ b/implementations/python/tests/test_formal_semantic_validation.py @@ -8,6 +8,7 @@ from types import SimpleNamespace import pytest + import tools.check_formal_semantic_validation as formal_validation from tools.check_formal_semantic_validation import ( REQUIRED_CLAIM_CLASS_IDS, diff --git a/implementations/python/tests/test_http_download.py b/implementations/python/tests/test_http_download.py index a292028c7..1052a6057 100644 --- a/implementations/python/tests/test_http_download.py +++ b/implementations/python/tests/test_http_download.py @@ -6,6 +6,7 @@ from urllib.error import HTTPError import pytest + from tools.http_download import download_bytes diff --git a/implementations/python/tests/test_identity_cutover_policy.py b/implementations/python/tests/test_identity_cutover_policy.py index d8158ce83..03d0d787c 100644 --- a/implementations/python/tests/test_identity_cutover_policy.py +++ b/implementations/python/tests/test_identity_cutover_policy.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest + from tools.check_identity_cutover import evaluate_identity_cutover REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_issue_898_participant_execution_control.py b/implementations/python/tests/test_issue_898_participant_execution_control.py index 32772b34d..77e7e9106 100644 --- a/implementations/python/tests/test_issue_898_participant_execution_control.py +++ b/implementations/python/tests/test_issue_898_participant_execution_control.py @@ -7,13 +7,6 @@ import pytest import yaml -from implementations.python.tests.test_dsl_437_benign_participant_execution import ( - _autonomous_manifest, - _compiled, - _NativeParticipantRuntime, - _scenario_yaml, -) -from implementations.python.tests.test_runtime_control_plane_api import _test_security from raes import parse_sdl from raes.participant_behavior import ParticipantFailureClass from raes_backend_protocols.capability_admission import ( @@ -51,6 +44,14 @@ ) from starlette.testclient import TestClient +from implementations.python.tests.test_dsl_437_benign_participant_execution import ( + _autonomous_manifest, + _compiled, + _NativeParticipantRuntime, + _scenario_yaml, +) +from implementations.python.tests.test_runtime_control_plane_api import _test_security + def _binding() -> ParticipantExecutionBindingModel: return ParticipantExecutionBindingModel( diff --git a/implementations/python/tests/test_issue_899_participant_resource_budgets.py b/implementations/python/tests/test_issue_899_participant_resource_budgets.py index 3a7a38dc0..c1aa63674 100644 --- a/implementations/python/tests/test_issue_899_participant_resource_budgets.py +++ b/implementations/python/tests/test_issue_899_participant_resource_budgets.py @@ -8,16 +8,6 @@ import pytest import yaml -from implementations.python.tests.test_dsl_437_benign_participant_execution import ( - SCENARIO_CLOCK_ADDRESS, - SCENARIO_CLOCK_STEP_TICKS, - _activity_control, - _activity_policy_yaml, - _advance_stepped_clock_to_tick, - _autonomous_manifest, - _compiled, - _NativeParticipantRuntime, -) from pydantic import BaseModel from raes import parse_sdl from raes._errors import SDLValidationError @@ -52,6 +42,17 @@ reserve_participant_resources, ) +from implementations.python.tests.test_dsl_437_benign_participant_execution import ( + SCENARIO_CLOCK_ADDRESS, + SCENARIO_CLOCK_STEP_TICKS, + _activity_control, + _activity_policy_yaml, + _advance_stepped_clock_to_tick, + _autonomous_manifest, + _compiled, + _NativeParticipantRuntime, +) + REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_issue_963_participant_opacity_proof.py b/implementations/python/tests/test_issue_963_participant_opacity_proof.py index ce066d89f..a2f206e51 100644 --- a/implementations/python/tests/test_issue_963_participant_opacity_proof.py +++ b/implementations/python/tests/test_issue_963_participant_opacity_proof.py @@ -11,7 +11,6 @@ from urllib.error import URLError import pytest -import tools.isabelle_tool as isabelle_tool from jsonschema import Draft202012Validator from pydantic import ValidationError from raes_contracts.behavioral_relation_profiles import ( @@ -25,6 +24,8 @@ load_behavioral_relation_catalog, load_behavioral_relation_catalog_revision, ) + +import tools.isabelle_tool as isabelle_tool from tools.check_participant_opacity_proof import ( ProofEvidenceError, load_proof_manifest, diff --git a/implementations/python/tests/test_libvirt_failure_observability.py b/implementations/python/tests/test_libvirt_failure_observability.py new file mode 100644 index 000000000..08f6f83fa --- /dev/null +++ b/implementations/python/tests/test_libvirt_failure_observability.py @@ -0,0 +1,216 @@ +"""The libvirt backend records every suppressed native failure for operators. + +Each case forces one broad exception-collapse site and pins two things at +once: the portable behavior is unchanged (value-free diagnostic, None, or +empty result), and the suppressed native failure is recorded on the +``raes_backend_libvirt`` logger at DEBUG with the failing operation named -- +the operator-side observability contract added for the field-debuggability +gap (no native detail crosses the portable boundary). +""" + +from __future__ import annotations + +import logging +from types import SimpleNamespace + +import pytest +from raes_backend_libvirt.driver import DomainSpec, NetworkSpec +from raes_backend_libvirt.drivers.libvirt import _native +from raes_backend_libvirt.drivers.libvirt.deployment import LibvirtDeploymentDriver +from raes_backend_libvirt.techvault_lifecycle import _list_native, _native_name +from raes_backend_libvirt.techvault_native import _define +from raes_backend_libvirt.techvault_native._driver import TechVaultNativeLibvirtDriver + +_OBSERVABILITY_LOGGER = "raes_backend_libvirt" +_SUPPRESSED = "suppressed a native libvirt failure" + + +def _raiser(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("forced native failure") + + +def _assert_recorded(caplog: pytest.LogCaptureFixture, operation: str) -> None: + records = [ + record + for record in caplog.records + if record.name == _OBSERVABILITY_LOGGER and _SUPPRESSED in record.getMessage() + ] + assert records, f"no suppressed-failure record for {operation!r}" + assert any(operation in record.getMessage() for record in records) + assert all(record.levelno == logging.DEBUG for record in records) + assert any(record.exc_info is not None for record in records) + + +@pytest.fixture(autouse=True) +def _capture_debug(caplog: pytest.LogCaptureFixture): + caplog.set_level(logging.DEBUG, logger=_OBSERVABILITY_LOGGER) + return caplog + + +def test_error_code_records_a_raising_classifier(caplog: pytest.LogCaptureFixture) -> None: + class _WeirdError(Exception): + def get_error_code(self) -> int: + raise RuntimeError("classifier exploded") + + assert _native._error_code(_WeirdError()) is None + _assert_recorded(caplog, "_error_code") + + +def test_existing_uuid_records_a_raising_reader(caplog: pytest.LogCaptureFixture) -> None: + assert _native._existing_uuid(SimpleNamespace(UUIDString=_raiser)) is None + _assert_recorded(caplog, "_existing_uuid") + + +def test_native_name_records_a_raising_reader(caplog: pytest.LogCaptureFixture) -> None: + assert _native_name(SimpleNamespace(name=_raiser)) == "" + _assert_recorded(caplog, "_native_name") + + +def test_list_native_records_a_raising_lister(caplog: pytest.LogCaptureFixture) -> None: + assert _list_native(SimpleNamespace(listAllDomains=_raiser), "listAllDomains") is None + _assert_recorded(caplog, "_list_native") + + +def _deployment_driver(**kwargs: object) -> LibvirtDeploymentDriver: + return LibvirtDeploymentDriver(name_prefix="raestest", **kwargs) + + +def test_deployment_observe_records_a_raising_connector(caplog: pytest.LogCaptureFixture) -> None: + result = _deployment_driver(connector=_raiser).observe(domains=()) + + assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + _assert_recorded(caplog, "observe") + + +def test_deployment_destroy_records_a_raising_connector(caplog: pytest.LogCaptureFixture) -> None: + result = _deployment_driver(connector=_raiser).destroy(networks=(), domains=()) + + assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + _assert_recorded(caplog, "destroy") + + +def test_deployment_substrate_observation_records_a_raising_lookup(caplog: pytest.LogCaptureFixture) -> None: + connection = SimpleNamespace(lookupByName=_raiser) + driver = _deployment_driver(connection=connection) + spec = DomainSpec(address="provision.node.web", name="web", image_ref=None, memory_mib=256) + + result = driver.observe(domains=(spec,)) + + assert result.observations == () + _assert_recorded(caplog, "_compute_substrate_observation") + + +def test_deployment_realize_network_records_a_raising_define(caplog: pytest.LogCaptureFixture) -> None: + connection = SimpleNamespace(networkLookupByName=_raiser, networkDefineXML=_raiser) + driver = _deployment_driver(connection=connection) + spec = NetworkSpec(address="provision.network.lan", name="lan", cidr="10.0.0.0/24", gateway="10.0.0.1") + + result = driver.realize(networks=(spec,), domains=()) + + assert result.diagnostics + _assert_recorded(caplog, "_realize_network") + + +def _techvault_driver(tmp_path, **kwargs: object) -> TechVaultNativeLibvirtDriver: + kernel = tmp_path / "kernel" + kernel.write_bytes(b"kernel") + return TechVaultNativeLibvirtDriver(state_dir=tmp_path / "state", kernel_path=kernel, **kwargs) + + +def test_techvault_destroy_records_a_raising_connector(tmp_path, caplog: pytest.LogCaptureFixture) -> None: + driver = _techvault_driver(tmp_path, connector=_raiser) + + result = driver.destroy(networks=(), domains=()) + + assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + _assert_recorded(caplog, "destroy") + + +def test_techvault_observe_records_a_raising_connector(tmp_path, caplog: pytest.LogCaptureFixture) -> None: + driver = _techvault_driver(tmp_path, connector=_raiser) + + result = driver.observe(domains=()) + + assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + _assert_recorded(caplog, "observe") + + +def test_techvault_observed_domain_records_a_raising_resolution( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + from raes_backend_libvirt.techvault_native import _driver as _techvault_module + + monkeypatch.setattr(_techvault_module, "_resolve_native", _raiser) + driver = _techvault_driver(tmp_path, connection=object()) + spec = DomainSpec(address="provision.node.web", name="web", image_ref=None, memory_mib=256) + + result = driver.observe(domains=(spec,)) + + assert result.observations == () + _assert_recorded(caplog, "_observed_domain") + + +def test_techvault_try_destroy_records_a_raising_teardown( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + driver = _techvault_driver(tmp_path, connection=object()) + monkeypatch.setattr(driver, "_destroy_one", _raiser) + + assert driver._try_destroy(object(), "lookupByName", "provision.node.web") is False + _assert_recorded(caplog, "_try_destroy") + + +def _define_driver(tmp_path) -> TechVaultNativeLibvirtDriver: + return _techvault_driver(tmp_path, connection=object()) + + +def test_define_network_records_a_raising_define(tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture) -> None: + monkeypatch.setattr(_define, "_ensure_name_available", lambda *args, **kwargs: None) + monkeypatch.setattr(_define, "_call", _raiser) + driver = _define_driver(tmp_path) + network = {"address": "provision.network.lan", "runtime_name": "raestest-lan"} + + handle, diagnostic, observations = _define.define_network(driver, object(), network) + + assert handle is None + assert diagnostic is not None + assert observations == () + _assert_recorded(caplog, "define_network") + + +def test_define_network_records_a_raising_readback(tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture) -> None: + native = SimpleNamespace(create=lambda: None) + monkeypatch.setattr(_define, "_ensure_name_available", lambda *args, **kwargs: None) + monkeypatch.setattr(_define, "_call", lambda *args, **kwargs: native) + monkeypatch.setattr(_define, "network_observations", _raiser) + driver = _define_driver(tmp_path) + network = {"address": "provision.network.lan", "runtime_name": "raestest-lan"} + + handle, diagnostic, observations = _define.define_network(driver, object(), network) + + assert handle is not None and handle.realized + assert diagnostic is not None and diagnostic.code.endswith("readback-failed") + assert observations == () + _assert_recorded(caplog, "define_network") + + +def test_define_domain_records_a_raising_readback(tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture) -> None: + native = SimpleNamespace(create=lambda: None) + artifact = tmp_path / "artifact" + artifact.write_bytes(b"artifact") + monkeypatch.setattr(_define, "_ensure_name_available", lambda *args, **kwargs: None) + monkeypatch.setattr(_define, "copy_kernel_for_libvirt", lambda *args, **kwargs: artifact) + monkeypatch.setattr(_define, "make_libvirt_readable", lambda *args, **kwargs: None) + monkeypatch.setattr(_define, "_call", lambda *args, **kwargs: native) + monkeypatch.setattr(_define, "domain_observations", _raiser) + driver = _define_driver(tmp_path) + monkeypatch.setattr(driver, "initramfs_builder", SimpleNamespace(build=lambda **kwargs: artifact)) + monkeypatch.setattr(driver, "_render_domain_xml", lambda *args, **kwargs: "") + domain = {"address": "provision.node.web", "runtime_name": "raestest-web"} + + handle, diagnostic, observations = _define.define_domain(driver, object(), domain, {}) + + assert handle is not None and handle.realized + assert diagnostic is not None and diagnostic.code.endswith("readback-failed") + assert observations == () + _assert_recorded(caplog, "define_domain") diff --git a/implementations/python/tests/test_project_positioning.py b/implementations/python/tests/test_project_positioning.py index 6a151d8ab..e8ccc9ec2 100644 --- a/implementations/python/tests/test_project_positioning.py +++ b/implementations/python/tests/test_project_positioning.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest + from tools.check_project_positioning import MAX_SURFACE_BYTES, validate_project_positioning REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_public_docs_policy.py b/implementations/python/tests/test_public_docs_policy.py index fc0a53c6d..8557ac69a 100644 --- a/implementations/python/tests/test_public_docs_policy.py +++ b/implementations/python/tests/test_public_docs_policy.py @@ -18,6 +18,7 @@ from raes import parse_sdl_file # noqa: E402 from raes_contracts.behavioral_relations import validate_behavioral_claim_binding # noqa: E402 from raes_contracts.contracts import BehavioralClaimBindingModel # noqa: E402 + from tools.check_public_docs import ( # noqa: E402 REQUIRED_PUBLIC_PAGES, REQUIRED_PUBLIC_REDIRECTS, diff --git a/implementations/python/tests/test_repo_policy_tools.py b/implementations/python/tests/test_repo_policy_tools.py index 3693a7697..0ccef4841 100644 --- a/implementations/python/tests/test_repo_policy_tools.py +++ b/implementations/python/tests/test_repo_policy_tools.py @@ -19,13 +19,14 @@ sys.path.insert(0, str(REPO_ROOT)) import pytest +import yaml +from packaging.requirements import Requirement +from packaging.version import Version + import tools.check_generated_schemas as check_generated_schemas import tools.check_json_artifacts as check_json_artifacts import tools.osv_scanner_tool as osv_scanner_tool import tools.policy.conftest_tool as conftest_tool -import yaml -from packaging.requirements import Requirement -from packaging.version import Version from tools.check_adr_immutability import ( amendment_refs, canonical_content, diff --git a/implementations/python/tests/test_reusable_asset_trust_policy.py b/implementations/python/tests/test_reusable_asset_trust_policy.py index 88eda34c1..57c382a84 100644 --- a/implementations/python/tests/test_reusable_asset_trust_policy.py +++ b/implementations/python/tests/test_reusable_asset_trust_policy.py @@ -22,6 +22,7 @@ schema_bundle, ) from raes_contracts.versions import REUSABLE_ASSET_TRUST_POLICY_SCHEMA_VERSION + from tools.check_schema_publication import load_schema_publication_catalog REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_scientific_scenario_completeness.py b/implementations/python/tests/test_scientific_scenario_completeness.py index d3d3904be..610d375c2 100644 --- a/implementations/python/tests/test_scientific_scenario_completeness.py +++ b/implementations/python/tests/test_scientific_scenario_completeness.py @@ -29,6 +29,7 @@ load_scientific_completeness_assessment, load_scientific_completeness_taxonomy, ) + from tools.check_scientific_scenario_completeness import ( # noqa: E402 _validate_contract_evidence, _validate_evidence_paths, diff --git a/implementations/python/tests/test_sdl_lineage.py b/implementations/python/tests/test_sdl_lineage.py index 001daa3a8..c15b86992 100644 --- a/implementations/python/tests/test_sdl_lineage.py +++ b/implementations/python/tests/test_sdl_lineage.py @@ -14,8 +14,9 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -import tools.check_sdl_lineage as lineage_checker # noqa: E402 from raes_contracts.provenance import SDLLineageLedgerModel # noqa: E402 + +import tools.check_sdl_lineage as lineage_checker # noqa: E402 from tools.check_sdl_lineage import ( # noqa: E402 _canonical_subjects, _validate_authorities, diff --git a/implementations/python/tests/test_sem_230_information_flow_control.py b/implementations/python/tests/test_sem_230_information_flow_control.py index b48702d23..36be51178 100644 --- a/implementations/python/tests/test_sem_230_information_flow_control.py +++ b/implementations/python/tests/test_sem_230_information_flow_control.py @@ -21,6 +21,7 @@ project_history, reactive_policy_noninterference_holds, ) + from tools.check_behavioral_relation_claims import _validate_claim_text BASE_POLICY = ProjectionPolicyDecision( diff --git a/implementations/python/tests/test_semantic_coverage.py b/implementations/python/tests/test_semantic_coverage.py index 7856efa25..b43ca95af 100644 --- a/implementations/python/tests/test_semantic_coverage.py +++ b/implementations/python/tests/test_semantic_coverage.py @@ -8,6 +8,7 @@ sys.path.insert(0, str(REPO_ROOT)) import pytest + from tools.check_semantic_coverage import ( ADR_RELATIVE_PATH, CANONICAL_PHASES, From 998b456bc304b4c1e45b6996345cc9ecd450e326 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:06:04 -0700 Subject: [PATCH 3/6] test(libvirt): drop accidental import reshuffles in unrelated suites A ruff --fix pass ran without the project working directory, so its first-party classification differed from the verify static lane and it rewrote import blocks in sixteen unrelated test modules. Those edits rode along in the coverage commit and the static lane rejects them. Restore every unrelated test module to its dev content; the only test change this branch carries is the new observability suite. Co-Authored-By: Claude Fable 5 --- .../tests/test_behavioral_relation_claims.py | 1 - ...st_dsl_437_benign_participant_execution.py | 7 +++---- .../tests/test_formal_semantic_validation.py | 1 - .../python/tests/test_http_download.py | 1 - .../tests/test_identity_cutover_policy.py | 1 - ...issue_898_participant_execution_control.py | 15 +++++++------ ..._issue_899_participant_resource_budgets.py | 21 +++++++++---------- ...est_issue_963_participant_opacity_proof.py | 3 +-- .../python/tests/test_project_positioning.py | 1 - .../python/tests/test_public_docs_policy.py | 1 - .../python/tests/test_repo_policy_tools.py | 7 +++---- .../tests/test_reusable_asset_trust_policy.py | 1 - .../test_scientific_scenario_completeness.py | 1 - .../python/tests/test_sdl_lineage.py | 3 +-- .../test_sem_230_information_flow_control.py | 1 - .../python/tests/test_semantic_coverage.py | 1 - 16 files changed, 25 insertions(+), 41 deletions(-) diff --git a/implementations/python/tests/test_behavioral_relation_claims.py b/implementations/python/tests/test_behavioral_relation_claims.py index 381718a6a..a91a4891d 100644 --- a/implementations/python/tests/test_behavioral_relation_claims.py +++ b/implementations/python/tests/test_behavioral_relation_claims.py @@ -6,7 +6,6 @@ import pytest from raes_contracts.behavioral_relations import load_behavioral_relation_catalog - from tools.check_behavioral_relation_claims import ( _should_validate_structured_bindings, _validate_claim_text, diff --git a/implementations/python/tests/test_dsl_437_benign_participant_execution.py b/implementations/python/tests/test_dsl_437_benign_participant_execution.py index 1ed07207d..bcf050cd3 100644 --- a/implementations/python/tests/test_dsl_437_benign_participant_execution.py +++ b/implementations/python/tests/test_dsl_437_benign_participant_execution.py @@ -9,6 +9,9 @@ import pytest import yaml +from implementations.python.tests.participant_execution_test_backend import ( + NativeParticipantExecutionController, +) from raes._errors import SDLValidationError from raes.parser import parse_sdl from raes.participant_behavior import ParticipantFailureClass @@ -64,10 +67,6 @@ from raes_runtime.participant_scheduler import ParticipantScheduler from raes_runtime.time_coordinator import ReferenceTimeRuntime, TimeCoordinator -from implementations.python.tests.participant_execution_test_backend import ( - NativeParticipantExecutionController, -) - REPO_ROOT = Path(__file__).resolve().parents[3] EXAMPLE = REPO_ROOT / "examples" / "scenarios" / "enterprise-participant-evidence-loop.sdl.yaml" IMPLEMENTATION_REF = "participant-implementation-manifests.green-worker.v1" diff --git a/implementations/python/tests/test_formal_semantic_validation.py b/implementations/python/tests/test_formal_semantic_validation.py index 1cabc6871..d47c1611d 100644 --- a/implementations/python/tests/test_formal_semantic_validation.py +++ b/implementations/python/tests/test_formal_semantic_validation.py @@ -8,7 +8,6 @@ from types import SimpleNamespace import pytest - import tools.check_formal_semantic_validation as formal_validation from tools.check_formal_semantic_validation import ( REQUIRED_CLAIM_CLASS_IDS, diff --git a/implementations/python/tests/test_http_download.py b/implementations/python/tests/test_http_download.py index 1052a6057..a292028c7 100644 --- a/implementations/python/tests/test_http_download.py +++ b/implementations/python/tests/test_http_download.py @@ -6,7 +6,6 @@ from urllib.error import HTTPError import pytest - from tools.http_download import download_bytes diff --git a/implementations/python/tests/test_identity_cutover_policy.py b/implementations/python/tests/test_identity_cutover_policy.py index 03d0d787c..d8158ce83 100644 --- a/implementations/python/tests/test_identity_cutover_policy.py +++ b/implementations/python/tests/test_identity_cutover_policy.py @@ -7,7 +7,6 @@ from pathlib import Path import pytest - from tools.check_identity_cutover import evaluate_identity_cutover REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_issue_898_participant_execution_control.py b/implementations/python/tests/test_issue_898_participant_execution_control.py index 77e7e9106..32772b34d 100644 --- a/implementations/python/tests/test_issue_898_participant_execution_control.py +++ b/implementations/python/tests/test_issue_898_participant_execution_control.py @@ -7,6 +7,13 @@ import pytest import yaml +from implementations.python.tests.test_dsl_437_benign_participant_execution import ( + _autonomous_manifest, + _compiled, + _NativeParticipantRuntime, + _scenario_yaml, +) +from implementations.python.tests.test_runtime_control_plane_api import _test_security from raes import parse_sdl from raes.participant_behavior import ParticipantFailureClass from raes_backend_protocols.capability_admission import ( @@ -44,14 +51,6 @@ ) from starlette.testclient import TestClient -from implementations.python.tests.test_dsl_437_benign_participant_execution import ( - _autonomous_manifest, - _compiled, - _NativeParticipantRuntime, - _scenario_yaml, -) -from implementations.python.tests.test_runtime_control_plane_api import _test_security - def _binding() -> ParticipantExecutionBindingModel: return ParticipantExecutionBindingModel( diff --git a/implementations/python/tests/test_issue_899_participant_resource_budgets.py b/implementations/python/tests/test_issue_899_participant_resource_budgets.py index c1aa63674..3a7a38dc0 100644 --- a/implementations/python/tests/test_issue_899_participant_resource_budgets.py +++ b/implementations/python/tests/test_issue_899_participant_resource_budgets.py @@ -8,6 +8,16 @@ import pytest import yaml +from implementations.python.tests.test_dsl_437_benign_participant_execution import ( + SCENARIO_CLOCK_ADDRESS, + SCENARIO_CLOCK_STEP_TICKS, + _activity_control, + _activity_policy_yaml, + _advance_stepped_clock_to_tick, + _autonomous_manifest, + _compiled, + _NativeParticipantRuntime, +) from pydantic import BaseModel from raes import parse_sdl from raes._errors import SDLValidationError @@ -42,17 +52,6 @@ reserve_participant_resources, ) -from implementations.python.tests.test_dsl_437_benign_participant_execution import ( - SCENARIO_CLOCK_ADDRESS, - SCENARIO_CLOCK_STEP_TICKS, - _activity_control, - _activity_policy_yaml, - _advance_stepped_clock_to_tick, - _autonomous_manifest, - _compiled, - _NativeParticipantRuntime, -) - REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_issue_963_participant_opacity_proof.py b/implementations/python/tests/test_issue_963_participant_opacity_proof.py index a2f206e51..ce066d89f 100644 --- a/implementations/python/tests/test_issue_963_participant_opacity_proof.py +++ b/implementations/python/tests/test_issue_963_participant_opacity_proof.py @@ -11,6 +11,7 @@ from urllib.error import URLError import pytest +import tools.isabelle_tool as isabelle_tool from jsonschema import Draft202012Validator from pydantic import ValidationError from raes_contracts.behavioral_relation_profiles import ( @@ -24,8 +25,6 @@ load_behavioral_relation_catalog, load_behavioral_relation_catalog_revision, ) - -import tools.isabelle_tool as isabelle_tool from tools.check_participant_opacity_proof import ( ProofEvidenceError, load_proof_manifest, diff --git a/implementations/python/tests/test_project_positioning.py b/implementations/python/tests/test_project_positioning.py index e8ccc9ec2..6a151d8ab 100644 --- a/implementations/python/tests/test_project_positioning.py +++ b/implementations/python/tests/test_project_positioning.py @@ -3,7 +3,6 @@ from pathlib import Path import pytest - from tools.check_project_positioning import MAX_SURFACE_BYTES, validate_project_positioning REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_public_docs_policy.py b/implementations/python/tests/test_public_docs_policy.py index 8557ac69a..fc0a53c6d 100644 --- a/implementations/python/tests/test_public_docs_policy.py +++ b/implementations/python/tests/test_public_docs_policy.py @@ -18,7 +18,6 @@ from raes import parse_sdl_file # noqa: E402 from raes_contracts.behavioral_relations import validate_behavioral_claim_binding # noqa: E402 from raes_contracts.contracts import BehavioralClaimBindingModel # noqa: E402 - from tools.check_public_docs import ( # noqa: E402 REQUIRED_PUBLIC_PAGES, REQUIRED_PUBLIC_REDIRECTS, diff --git a/implementations/python/tests/test_repo_policy_tools.py b/implementations/python/tests/test_repo_policy_tools.py index 0ccef4841..3693a7697 100644 --- a/implementations/python/tests/test_repo_policy_tools.py +++ b/implementations/python/tests/test_repo_policy_tools.py @@ -19,14 +19,13 @@ sys.path.insert(0, str(REPO_ROOT)) import pytest -import yaml -from packaging.requirements import Requirement -from packaging.version import Version - import tools.check_generated_schemas as check_generated_schemas import tools.check_json_artifacts as check_json_artifacts import tools.osv_scanner_tool as osv_scanner_tool import tools.policy.conftest_tool as conftest_tool +import yaml +from packaging.requirements import Requirement +from packaging.version import Version from tools.check_adr_immutability import ( amendment_refs, canonical_content, diff --git a/implementations/python/tests/test_reusable_asset_trust_policy.py b/implementations/python/tests/test_reusable_asset_trust_policy.py index 57c382a84..88eda34c1 100644 --- a/implementations/python/tests/test_reusable_asset_trust_policy.py +++ b/implementations/python/tests/test_reusable_asset_trust_policy.py @@ -22,7 +22,6 @@ schema_bundle, ) from raes_contracts.versions import REUSABLE_ASSET_TRUST_POLICY_SCHEMA_VERSION - from tools.check_schema_publication import load_schema_publication_catalog REPO_ROOT = Path(__file__).resolve().parents[3] diff --git a/implementations/python/tests/test_scientific_scenario_completeness.py b/implementations/python/tests/test_scientific_scenario_completeness.py index 610d375c2..d3d3904be 100644 --- a/implementations/python/tests/test_scientific_scenario_completeness.py +++ b/implementations/python/tests/test_scientific_scenario_completeness.py @@ -29,7 +29,6 @@ load_scientific_completeness_assessment, load_scientific_completeness_taxonomy, ) - from tools.check_scientific_scenario_completeness import ( # noqa: E402 _validate_contract_evidence, _validate_evidence_paths, diff --git a/implementations/python/tests/test_sdl_lineage.py b/implementations/python/tests/test_sdl_lineage.py index c15b86992..001daa3a8 100644 --- a/implementations/python/tests/test_sdl_lineage.py +++ b/implementations/python/tests/test_sdl_lineage.py @@ -14,9 +14,8 @@ if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from raes_contracts.provenance import SDLLineageLedgerModel # noqa: E402 - import tools.check_sdl_lineage as lineage_checker # noqa: E402 +from raes_contracts.provenance import SDLLineageLedgerModel # noqa: E402 from tools.check_sdl_lineage import ( # noqa: E402 _canonical_subjects, _validate_authorities, diff --git a/implementations/python/tests/test_sem_230_information_flow_control.py b/implementations/python/tests/test_sem_230_information_flow_control.py index 36be51178..b48702d23 100644 --- a/implementations/python/tests/test_sem_230_information_flow_control.py +++ b/implementations/python/tests/test_sem_230_information_flow_control.py @@ -21,7 +21,6 @@ project_history, reactive_policy_noninterference_holds, ) - from tools.check_behavioral_relation_claims import _validate_claim_text BASE_POLICY = ProjectionPolicyDecision( diff --git a/implementations/python/tests/test_semantic_coverage.py b/implementations/python/tests/test_semantic_coverage.py index b43ca95af..7856efa25 100644 --- a/implementations/python/tests/test_semantic_coverage.py +++ b/implementations/python/tests/test_semantic_coverage.py @@ -8,7 +8,6 @@ sys.path.insert(0, str(REPO_ROOT)) import pytest - from tools.check_semantic_coverage import ( ADR_RELATIVE_PATH, CANONICAL_PHASES, From 509124a55ef260083874570e237bbf3c01f0ce35 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:19:50 -0700 Subject: [PATCH 4/6] test(libvirt): split composite assertions in the observability suite Sonar S9073 requires one condition per assert so a failure names the exact clause; split the eight and-joined assertions accordingly. Co-Authored-By: Claude Fable 5 --- .../test_libvirt_failure_observability.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/implementations/python/tests/test_libvirt_failure_observability.py b/implementations/python/tests/test_libvirt_failure_observability.py index 08f6f83fa..807cb099f 100644 --- a/implementations/python/tests/test_libvirt_failure_observability.py +++ b/implementations/python/tests/test_libvirt_failure_observability.py @@ -78,14 +78,16 @@ def _deployment_driver(**kwargs: object) -> LibvirtDeploymentDriver: def test_deployment_observe_records_a_raising_connector(caplog: pytest.LogCaptureFixture) -> None: result = _deployment_driver(connector=_raiser).observe(domains=()) - assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + assert result.diagnostics + assert result.diagnostics[0].code.endswith("unavailable") _assert_recorded(caplog, "observe") def test_deployment_destroy_records_a_raising_connector(caplog: pytest.LogCaptureFixture) -> None: result = _deployment_driver(connector=_raiser).destroy(networks=(), domains=()) - assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + assert result.diagnostics + assert result.diagnostics[0].code.endswith("unavailable") _assert_recorded(caplog, "destroy") @@ -122,7 +124,8 @@ def test_techvault_destroy_records_a_raising_connector(tmp_path, caplog: pytest. result = driver.destroy(networks=(), domains=()) - assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + assert result.diagnostics + assert result.diagnostics[0].code.endswith("unavailable") _assert_recorded(caplog, "destroy") @@ -131,7 +134,8 @@ def test_techvault_observe_records_a_raising_connector(tmp_path, caplog: pytest. result = driver.observe(domains=()) - assert result.diagnostics and result.diagnostics[0].code.endswith("unavailable") + assert result.diagnostics + assert result.diagnostics[0].code.endswith("unavailable") _assert_recorded(caplog, "observe") @@ -188,8 +192,10 @@ def test_define_network_records_a_raising_readback(tmp_path, monkeypatch, caplog handle, diagnostic, observations = _define.define_network(driver, object(), network) - assert handle is not None and handle.realized - assert diagnostic is not None and diagnostic.code.endswith("readback-failed") + assert handle is not None + assert handle.realized + assert diagnostic is not None + assert diagnostic.code.endswith("readback-failed") assert observations == () _assert_recorded(caplog, "define_network") @@ -210,7 +216,9 @@ def test_define_domain_records_a_raising_readback(tmp_path, monkeypatch, caplog: handle, diagnostic, observations = _define.define_domain(driver, object(), domain, {}) - assert handle is not None and handle.realized - assert diagnostic is not None and diagnostic.code.endswith("readback-failed") + assert handle is not None + assert handle.realized + assert diagnostic is not None + assert diagnostic.code.endswith("readback-failed") assert observations == () _assert_recorded(caplog, "define_domain") From 50850474b1a9aeaf992b038295760a110332599a Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Wed, 2 Sep 2026 22:42:34 +0200 Subject: [PATCH 5/6] fix(libvirt): make failure observability safe and complete --- docs/requirements/RUN-316/requirement.md | 4 + .../raes_backend_libvirt/_initramfs.py | 9 +- .../raes_backend_libvirt/_observability.py | 29 ++++- .../drivers/libvirt/_native.py | 11 +- .../drivers/libvirt/deployment.py | 29 +++-- .../guest_certified_driver.py | 5 +- .../techvault_lifecycle.py | 18 ++- .../techvault_native/_define.py | 11 +- .../techvault_native/_driver.py | 13 +- .../techvault_native/_finalize.py | 5 +- .../techvault_native/_preflight.py | 5 +- .../test_libvirt_failure_observability.py | 116 ++++++++++++++++-- 12 files changed, 190 insertions(+), 65 deletions(-) diff --git a/docs/requirements/RUN-316/requirement.md b/docs/requirements/RUN-316/requirement.md index 76139721c..f65194ad2 100644 --- a/docs/requirements/RUN-316/requirement.md +++ b/docs/requirements/RUN-316/requirement.md @@ -25,3 +25,7 @@ Processors and backends require their own operational observability, but that co - TESTS → TEST `implementations/python/tests/test_backend_manifest.py` (Backend manifest tests cover observation contract vocabulary and experiment-run-v1 support) - TESTS → TEST `implementations/python/tests/test_runtime_control_plane_api.py` (Operational apparatus summary API tests) - IMPLEMENTS → GITHUB_ISSUE `338` (Operational Apparatus Observability (RUN-316)) +- IMPLEMENTS → GITHUB_ISSUE `1173` (Safe and complete libvirt failure observability) +- IMPLEMENTS → PULL_REQUEST `1163` (Bounded libvirt backend failure observability) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_backend_libvirt/_observability.py` (Bounded operator-side failure classification) +- TESTS → TEST `implementations/python/tests/test_libvirt_failure_observability.py` (Suppressed-failure coverage, redaction, and expected-absence tests) diff --git a/implementations/python/packages/raes_backend_libvirt/_initramfs.py b/implementations/python/packages/raes_backend_libvirt/_initramfs.py index 7408a3546..5e4bc5086 100644 --- a/implementations/python/packages/raes_backend_libvirt/_initramfs.py +++ b/implementations/python/packages/raes_backend_libvirt/_initramfs.py @@ -14,9 +14,6 @@ from pathlib import Path from typing import BinaryIO -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG - _NEWC_MAGIC = b"070701" _NEWC_TRAILER = "TRAILER!!!" _ELF_MACHINE_X86_64 = 62 @@ -178,8 +175,7 @@ def atomic_write(path: Path, payload: bytes, *, mode: int) -> Path: os.chmod(temporary, mode) os.replace(temporary, path) _fsync_directory(path.parent) - except BaseException as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "atomic_write", exc_info=exc) + except BaseException: temporary.unlink(missing_ok=True) raise return path @@ -211,8 +207,7 @@ def atomic_copy_by_digest(source: Path, target: Path, *, mode: int) -> Path: os.chmod(temporary, mode) os.replace(temporary, target) _fsync_directory(target.parent) - except BaseException as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "atomic_copy_by_digest", exc_info=exc) + except BaseException: temporary.unlink(missing_ok=True) raise return target diff --git a/implementations/python/packages/raes_backend_libvirt/_observability.py b/implementations/python/packages/raes_backend_libvirt/_observability.py index 46b21f831..ce1bae5bc 100644 --- a/implementations/python/packages/raes_backend_libvirt/_observability.py +++ b/implementations/python/packages/raes_backend_libvirt/_observability.py @@ -1,14 +1,33 @@ -"""Backend-local observability for native libvirt failures. +"""Backend-local observability for suppressed backend failures. The portable driver boundary deliberately collapses native errors into -value-free diagnostics; this logger records the collapsed detail on the -operator's side of that boundary. It is silent unless the embedding -application configures logging for ``raes_backend_libvirt``. +value-free diagnostics. This module records bounded, non-sensitive failure +classification on the operator's side of that boundary without retaining +exception messages or tracebacks. It is silent unless the embedding application +configures logging for ``raes_backend_libvirt``. """ from __future__ import annotations import logging +import re LOGGER = logging.getLogger("raes_backend_libvirt") -NATIVE_FAILURE_LOG = "%s suppressed a native libvirt failure" +_SAFE_TYPE_RE = re.compile(r"[^A-Za-z0-9_.-]+") + + +def record_suppressed_failure( + operation: str, + exc: BaseException, + *, + native_code: int | None = None, +) -> None: + """Record bounded failure classification without exception text or traceback.""" + + exception_type = _SAFE_TYPE_RE.sub("-", type(exc).__name__)[:80] or "Exception" + fields = [f"exception_type={exception_type}"] + if isinstance(exc, OSError) and type(exc.errno) is int: + fields.append(f"errno={exc.errno}") + if type(native_code) is int: + fields.append(f"native_code={native_code}") + LOGGER.debug("%s suppressed backend failure (%s)", operation, ", ".join(fields)) diff --git a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py index 56e78d07b..5d9b378cb 100644 --- a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py +++ b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/_native.py @@ -15,8 +15,7 @@ from collections.abc import Callable from typing import Protocol, cast -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure _SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.-]+") # Fixed namespace for deriving a per-address libvirt UUID. The UUID proves an @@ -60,7 +59,7 @@ def _error_code(exc: BaseException) -> int | None: try: code = getter() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_error_code", exc_info=exc) + _record_suppressed_failure("_error_code", exc) return None return code if isinstance(code, int) else None @@ -106,7 +105,7 @@ def _existing_uuid(native: object) -> str | None: try: return reader() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_existing_uuid", exc_info=exc) + _record_suppressed_failure("_existing_uuid", exc) return None @@ -141,7 +140,9 @@ def _lookup(connection: object, method_name: str, name: str) -> object | None: try: return method(name) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_lookup", exc_info=exc) + code = _error_code(exc) + if code not in _ABSENCE_ERROR_CODES: + _record_suppressed_failure("_lookup", exc, native_code=code) return None diff --git a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py index ef2d8c11b..e9be5ad16 100644 --- a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py +++ b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py @@ -2,7 +2,6 @@ from __future__ import annotations -import contextlib import os import shutil import tempfile @@ -14,8 +13,7 @@ from raes_contracts.realization_envelope import ObservationStrength, RealizationConcern from raes_contracts.realization_observation import RealizationObservation -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from raes_backend_libvirt.driver import ( DomainHandle, DomainSpec, @@ -103,7 +101,7 @@ def realize( try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "realize", exc_info=exc) + _record_suppressed_failure("realize", exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) realize_network_specs(self, connection, networks, created_networks, network_handles, diagnostics) @@ -153,7 +151,7 @@ def _compute_substrate_observation( active = getattr(native, "isActive", None) owned_and_active = _existing_uuid(native) == _raes_uuid(address) and callable(active) and active() == 1 except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_compute_substrate_observation", exc_info=exc) + _record_suppressed_failure("_compute_substrate_observation", exc) owned_and_active = False if owned_and_active: envelope = load_libvirt_realization_envelope(self.driver_mode) @@ -177,7 +175,7 @@ def observe(self, *, domains: tuple[DomainSpec, ...]) -> DriverResult: try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "observe", exc_info=exc) + _record_suppressed_failure("observe", exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) observations: list[RealizationObservation] = [] diagnostics: list[Diagnostic] = [] @@ -227,7 +225,7 @@ def _realize_network(self, connection: object, spec: NetworkSpec, created: list[ except _OwnershipConflict: return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_realize_network", exc_info=exc) + _record_suppressed_failure("_realize_network", exc) return _failure(spec.address, _CODE_OPERATION_FAILED) self._realized.add(spec.address) return None @@ -257,7 +255,7 @@ def _realize_domain(self, connection: object, spec: DomainSpec, created: list[st except _OwnershipConflict: return _failure(spec.address, _CODE_OWNERSHIP_CONFLICT) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_realize_domain", exc_info=exc) + _record_suppressed_failure("_realize_domain", exc) return _failure(spec.address, _CODE_OPERATION_FAILED) self._realized.add(spec.address) return None @@ -272,7 +270,7 @@ def destroy( try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "destroy", exc_info=exc) + _record_suppressed_failure("destroy", exc) return DriverResult(diagnostics=(_failure(_CONNECTION_ADDRESS, _CODE_UNAVAILABLE),)) domain_handles = self._destroy_domains(connection, domains, diagnostics) @@ -445,15 +443,19 @@ def _undefine_nwfilter(self, connection: object, address: str) -> None: return # Best-effort cleanup of our own filter: a filter that cannot be undefined # (in use, missing) must not fail the destroy. - with contextlib.suppress(Exception): + try: cast(_NativeResource, native).undefine() + except Exception as exc: + if not _is_absence_error(exc): + _record_suppressed_failure("_undefine_nwfilter", exc) def _destroy_one(self, connection: object, lookup_method: str, address: str) -> bool: try: native = _find_native(connection, lookup_method, self._name_for(address)) - except _NativeLookupError: + except _NativeLookupError as exc: # Connection/permission/internal lookup failure: fail closed so the # snapshot is preserved for retry instead of claiming the object gone. + _record_suppressed_failure("_destroy_one", exc.__cause__ or exc) return False # A None result is genuine absence — teardown is idempotently satisfied. # A present object is torn down only when its UUID proves RAES ownership @@ -468,7 +470,10 @@ def _destroy_one(self, connection: object, lookup_method: str, address: str) -> # An object that vanished between lookup and undefine is still torn # down; a stop/undefine that failed for permission or an internal # reason fails closed and preserves the snapshot for retry. - return _is_absence_error(exc) + if _is_absence_error(exc): + return True + _record_suppressed_failure("_destroy_one", exc) + return False return True def _rollback(self, networks: list[str], domains: list[str]) -> None: diff --git a/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py b/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py index d49db99f3..867b84442 100644 --- a/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py +++ b/implementations/python/packages/raes_backend_libvirt/guest_certified_driver.py @@ -22,8 +22,7 @@ from raes_contracts.diagnostics import Diagnostic -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from ._techvault_native_ops import _CODE_GUEST_FRESHNESS_UNAVAILABLE, _diagnostic from .driver import DomainSpec, NetworkSpec, RealizationObservation @@ -102,7 +101,7 @@ def _prepare_operation(self, matrix: Mapping[str, object]) -> list[Diagnostic]: try: candidate = self.challenge_factory() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_prepare_operation", exc_info=exc) + _record_suppressed_failure("_prepare_operation", exc) candidate = None if ( not isinstance(candidate, str) diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py index 6fef1f429..5541f8ccc 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py @@ -5,8 +5,7 @@ from collections.abc import Callable from dataclasses import dataclass -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from .drivers.libvirt import _error_code, _existing_uuid, _raes_uuid from .techvault_matrix import runtime_name @@ -74,8 +73,11 @@ def _resolve_by_name( except KeyError: resolved = _resolve_verified_absence(connection, list_method, address) except Exception as exc: - if _error_code(exc) in {42, 43}: + code = _error_code(exc) + if code in {42, 43}: resolved = _resolve_verified_absence(connection, list_method, address) + else: + _record_suppressed_failure("_resolve_by_name", exc, native_code=code) else: resolved = NativeResolution(native=native, name=name) return resolved @@ -119,7 +121,11 @@ def _invoke_native_action(native: object, method_name: str, tolerated_codes: set try: method() except Exception as exc: - return _error_code(exc) in tolerated_codes + code = _error_code(exc) + if code in tolerated_codes: + return True + _record_suppressed_failure("_invoke_native_action", exc, native_code=code) + return False return True @@ -130,7 +136,7 @@ def _list_native(connection: object, method_name: str) -> tuple[object, ...] | N try: native = method() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_list_native", exc_info=exc) + _record_suppressed_failure("_list_native", exc) return None return tuple(native) if isinstance(native, list | tuple) else None @@ -142,7 +148,7 @@ def _native_name(native: object) -> str: try: value = method() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_native_name", exc_info=exc) + _record_suppressed_failure("_native_name", exc) return "" return value if isinstance(value, str) else "" diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py index fa9e9fdf3..83d1979d2 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_define.py @@ -14,8 +14,7 @@ from raes_contracts.diagnostics import Diagnostic -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from .._techvault_native_ops import ( _CODE_OPERATION_FAILED, @@ -80,7 +79,7 @@ def define_network( driver._names.pop(address, None) diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_network", exc_info=exc) + _record_suppressed_failure("define_network", exc) if native is None: driver._names.pop(address, None) else: @@ -93,7 +92,7 @@ def define_network( try: observations = network_observations(native, network) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_network", exc_info=exc) + _record_suppressed_failure("define_network", exc) diagnostic = _diagnostic(_CODE_READBACK_FAILED, address) return handle, diagnostic, observations @@ -158,7 +157,7 @@ def define_domain( driver._names.pop(address, None) diagnostic = _diagnostic(_CODE_OWNERSHIP_CONFLICT, address) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_domain", exc_info=exc) + _record_suppressed_failure("define_domain", exc) if native is None: driver._cleanup_artifacts(address) driver._names.pop(address, None) @@ -178,6 +177,6 @@ def define_domain( initrd=initrd, ) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "define_domain", exc_info=exc) + _record_suppressed_failure("define_domain", exc) diagnostic = _diagnostic(_CODE_READBACK_FAILED, address) return handle, diagnostic, observations diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py index 1f223c6ff..8d279a975 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_driver.py @@ -16,8 +16,7 @@ from raes_contracts.diagnostics import Diagnostic -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from .._techvault_native_helpers import ( default_connector as _default_connector, @@ -136,7 +135,7 @@ def realize( try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "realize", exc_info=exc) + _record_suppressed_failure("realize", exc) result = DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) else: result = self._realize_matrix( @@ -265,7 +264,7 @@ def destroy( try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "destroy", exc_info=exc) + _record_suppressed_failure("destroy", exc) return DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) domain_handles: list[DomainHandle] = [] network_handles: list[NetworkHandle] = [] @@ -304,7 +303,7 @@ def observe(self, *, domains: tuple[DomainSpec, ...]) -> DriverResult: try: connection = self._conn() except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "observe", exc_info=exc) + _record_suppressed_failure("observe", exc) return DriverResult(diagnostics=(_diagnostic(_CODE_UNAVAILABLE, _CONNECTION_ADDRESS),)) envelope = load_libvirt_realization_envelope(self.driver_mode) observations: list[RealizationObservation] = [] @@ -347,7 +346,7 @@ def _observed_domain(self, connection: object, address: str) -> _NativeResolutio if native is None or _existing_uuid(native) != _raes_uuid(address) or not native_active(native): return None except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_observed_domain", exc_info=exc) + _record_suppressed_failure("_observed_domain", exc) return None return resolved @@ -451,5 +450,5 @@ def _try_destroy(self, connection: object, lookup_method: str, address: str) -> try: return self._destroy_one(connection, lookup_method, address) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_try_destroy", exc_info=exc) + _record_suppressed_failure("_try_destroy", exc) return False diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py index c04a606f0..5d26f70fe 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_finalize.py @@ -6,8 +6,7 @@ from dataclasses import replace from typing import TYPE_CHECKING -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from .._techvault_native_ops import _CODE_OPERATION_FAILED, _diagnostic from ..driver import ( @@ -65,7 +64,7 @@ def _verify_and_finalize( binding = driver._material_binding(envelope_digest, configuration_digest) snapshot = snapshot_from_observations(matrix, observations, binding=binding) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "_verify_and_finalize", exc_info=exc) + _record_suppressed_failure("_verify_and_finalize", exc) binding_diagnostics = [_diagnostic(_CODE_OPERATION_FAILED, "runtime.libvirt.binding")] binding_diagnostics.extend(driver._rollback(connection, network_handles, domain_handles)) return DriverResult(diagnostics=tuple(binding_diagnostics)) diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py b/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py index 1f9a15f25..1b4ecee75 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_native/_preflight.py @@ -8,8 +8,7 @@ from raes_contracts.diagnostics import Diagnostic -from raes_backend_libvirt._observability import LOGGER as _LOGGER -from raes_backend_libvirt._observability import NATIVE_FAILURE_LOG as _NATIVE_FAILURE_LOG +from raes_backend_libvirt._observability import record_suppressed_failure as _record_suppressed_failure from .._initramfs import builder_preflight from .._techvault_native_ops import ( @@ -63,7 +62,7 @@ def artifact_preflight_diagnostics( try: toolchain = builder_preflight(initramfs_builder) except Exception as exc: - _LOGGER.debug(_NATIVE_FAILURE_LOG, "artifact_preflight_diagnostics", exc_info=exc) + _record_suppressed_failure("artifact_preflight_diagnostics", exc) toolchain = None if toolchain is None or not toolchain.ready: diagnostic = _diagnostic(_CODE_TOOLCHAIN_UNAVAILABLE, "runtime.libvirt.initramfs") diff --git a/implementations/python/tests/test_libvirt_failure_observability.py b/implementations/python/tests/test_libvirt_failure_observability.py index 807cb099f..ae3e5b409 100644 --- a/implementations/python/tests/test_libvirt_failure_observability.py +++ b/implementations/python/tests/test_libvirt_failure_observability.py @@ -1,11 +1,9 @@ -"""The libvirt backend records every suppressed native failure for operators. +"""The libvirt backend safely records suppressed failures for operators. Each case forces one broad exception-collapse site and pins two things at once: the portable behavior is unchanged (value-free diagnostic, None, or -empty result), and the suppressed native failure is recorded on the -``raes_backend_libvirt`` logger at DEBUG with the failing operation named -- -the operator-side observability contract added for the field-debuggability -gap (no native detail crosses the portable boundary). +empty result), and bounded failure classification is recorded on the +``raes_backend_libvirt`` logger at DEBUG without exception text or traceback. """ from __future__ import annotations @@ -14,15 +12,41 @@ from types import SimpleNamespace import pytest +from raes_backend_libvirt import _initramfs +from raes_backend_libvirt._observability import record_suppressed_failure from raes_backend_libvirt.driver import DomainSpec, NetworkSpec from raes_backend_libvirt.drivers.libvirt import _native +from raes_backend_libvirt.drivers.libvirt import deployment as _deployment from raes_backend_libvirt.drivers.libvirt.deployment import LibvirtDeploymentDriver -from raes_backend_libvirt.techvault_lifecycle import _list_native, _native_name +from raes_backend_libvirt.techvault_lifecycle import ( + _invoke_native_action, + _list_native, + _native_name, + _resolve_by_name, +) from raes_backend_libvirt.techvault_native import _define from raes_backend_libvirt.techvault_native._driver import TechVaultNativeLibvirtDriver _OBSERVABILITY_LOGGER = "raes_backend_libvirt" -_SUPPRESSED = "suppressed a native libvirt failure" +_SUPPRESSED = "suppressed backend failure" +_SENSITIVE_DETAIL = "token=do-not-record" + + +class _NativeError(RuntimeError): + def __init__(self, code: int, *, domain: int = 10, level: int = 2) -> None: + super().__init__(_SENSITIVE_DETAIL) + self._code = code + self._domain = domain + self._level = level + + def get_error_code(self) -> int: + return self._code + + def get_error_domain(self) -> int: + return self._domain + + def get_error_level(self) -> int: + return self._level def _raiser(*_args: object, **_kwargs: object) -> object: @@ -38,7 +62,9 @@ def _assert_recorded(caplog: pytest.LogCaptureFixture, operation: str) -> None: assert records, f"no suppressed-failure record for {operation!r}" assert any(operation in record.getMessage() for record in records) assert all(record.levelno == logging.DEBUG for record in records) - assert any(record.exc_info is not None for record in records) + assert all(record.exc_info is None for record in records) + assert all("exception_type=" in record.getMessage() for record in records) + assert all(_SENSITIVE_DETAIL not in record.getMessage() for record in records) @pytest.fixture(autouse=True) @@ -47,6 +73,15 @@ def _capture_debug(caplog: pytest.LogCaptureFixture): return caplog +def test_failure_record_is_bounded_and_uses_only_safe_classification(caplog: pytest.LogCaptureFixture) -> None: + record_suppressed_failure("safe_operation", _NativeError(77), native_code=77) + + _assert_recorded(caplog, "safe_operation") + message = caplog.records[-1].getMessage() + assert "native_code=77" in message + assert len(message) < 256 + + def test_error_code_records_a_raising_classifier(caplog: pytest.LogCaptureFixture) -> None: class _WeirdError(Exception): def get_error_code(self) -> int: @@ -71,6 +106,37 @@ def test_list_native_records_a_raising_lister(caplog: pytest.LogCaptureFixture) _assert_recorded(caplog, "_list_native") +def test_lookup_keeps_expected_native_absence_silent(caplog: pytest.LogCaptureFixture) -> None: + def absent(_name: str) -> object: + raise _NativeError(42) + + assert _native._lookup(SimpleNamespace(lookupByName=absent), "lookupByName", "missing") is None + assert not [record for record in caplog.records if _SUPPRESSED in record.getMessage()] + + +def test_lookup_records_non_absence_failure(caplog: pytest.LogCaptureFixture) -> None: + assert _native._lookup(SimpleNamespace(lookupByName=_raiser), "lookupByName", "missing") is None + _assert_recorded(caplog, "_lookup") + + +def test_resolve_by_name_records_non_absence_failure(caplog: pytest.LogCaptureFixture) -> None: + assert _resolve_by_name(object(), _raiser, "listAllDomains", "provision.node.web", "web") is None + _assert_recorded(caplog, "_resolve_by_name") + + +def test_native_action_records_non_tolerated_failure(caplog: pytest.LogCaptureFixture) -> None: + assert not _invoke_native_action(SimpleNamespace(destroy=_raiser), "destroy", {42, 43}) + _assert_recorded(caplog, "_invoke_native_action") + + +def test_native_action_keeps_tolerated_failure_silent(caplog: pytest.LogCaptureFixture) -> None: + def absent() -> None: + raise _NativeError(42) + + assert _invoke_native_action(SimpleNamespace(destroy=absent), "destroy", {42, 43}) + assert not [record for record in caplog.records if _SUPPRESSED in record.getMessage()] + + def _deployment_driver(**kwargs: object) -> LibvirtDeploymentDriver: return LibvirtDeploymentDriver(name_prefix="raestest", **kwargs) @@ -113,6 +179,40 @@ def test_deployment_realize_network_records_a_raising_define(caplog: pytest.LogC _assert_recorded(caplog, "_realize_network") +def test_deployment_destroy_records_a_non_absence_lookup_failure(caplog: pytest.LogCaptureFixture) -> None: + connection = SimpleNamespace(lookupByName=_raiser) + driver = _deployment_driver(connection=connection) + + assert not driver._destroy_one(connection, "lookupByName", "provision.node.web") + _assert_recorded(caplog, "_destroy_one") + + +def test_deployment_nwfilter_cleanup_records_a_non_absence_failure( + monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + native = SimpleNamespace(undefine=_raiser) + driver = _deployment_driver(connection=object()) + driver._filters["provision.node.web"] = "raestest-web-acl" + monkeypatch.setattr(_deployment, "_lookup", lambda *_args: native) + monkeypatch.setattr(_deployment, "_existing_uuid", lambda _native: "owned") + monkeypatch.setattr(_deployment, "_filter_owner_uuid", lambda _address: "owned") + + driver._undefine_nwfilter(object(), "provision.node.web") + + _assert_recorded(caplog, "_undefine_nwfilter") + + +def test_atomic_write_does_not_misclassify_a_propagated_filesystem_failure( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(_initramfs, "_fsync_directory", _raiser) + + with pytest.raises(RuntimeError, match="forced native failure"): + _initramfs.atomic_write(tmp_path / "artifact", b"payload", mode=0o600) + + assert not [record for record in caplog.records if _SUPPRESSED in record.getMessage()] + + def _techvault_driver(tmp_path, **kwargs: object) -> TechVaultNativeLibvirtDriver: kernel = tmp_path / "kernel" kernel.write_bytes(b"kernel") From 742e6c0c04fc2a5cf1c30769f1852b933396ec50 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Wed, 2 Sep 2026 23:05:41 +0200 Subject: [PATCH 6/6] fix(libvirt): simplify failure branches --- .../drivers/libvirt/deployment.py | 39 ++++++++++--------- .../techvault_lifecycle.py | 8 ++-- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py index e9be5ad16..74fd5f837 100644 --- a/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py +++ b/implementations/python/packages/raes_backend_libvirt/drivers/libvirt/deployment.py @@ -450,31 +450,32 @@ def _undefine_nwfilter(self, connection: object, address: str) -> None: _record_suppressed_failure("_undefine_nwfilter", exc) def _destroy_one(self, connection: object, lookup_method: str, address: str) -> bool: + removed = True try: native = _find_native(connection, lookup_method, self._name_for(address)) except _NativeLookupError as exc: # Connection/permission/internal lookup failure: fail closed so the # snapshot is preserved for retry instead of claiming the object gone. _record_suppressed_failure("_destroy_one", exc.__cause__ or exc) - return False - # A None result is genuine absence — teardown is idempotently satisfied. - # A present object is torn down only when its UUID proves RAES ownership - # (the same invariant as convergence), never a foreign name collision. - if native is not None: - if _existing_uuid(native) != _raes_uuid(address): - raise _OwnershipConflict(address) - try: - _stop_native(native) - cast(_NativeResource, native).undefine() - except Exception as exc: - # An object that vanished between lookup and undefine is still torn - # down; a stop/undefine that failed for permission or an internal - # reason fails closed and preserves the snapshot for retry. - if _is_absence_error(exc): - return True - _record_suppressed_failure("_destroy_one", exc) - return False - return True + removed = False + else: + # A None result is genuine absence — teardown is idempotently satisfied. + # A present object is torn down only when its UUID proves RAES ownership + # (the same invariant as convergence), never a foreign name collision. + if native is not None: + if _existing_uuid(native) != _raes_uuid(address): + raise _OwnershipConflict(address) + try: + _stop_native(native) + cast(_NativeResource, native).undefine() + except Exception as exc: + # An object that vanished between lookup and undefine is still torn + # down; a stop/undefine that failed for permission or an internal + # reason fails closed and preserves the snapshot for retry. + removed = _is_absence_error(exc) + if not removed: + _record_suppressed_failure("_destroy_one", exc) + return removed def _rollback(self, networks: list[str], domains: list[str]) -> None: if networks or domains: diff --git a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py index 5541f8ccc..25c025dde 100644 --- a/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py +++ b/implementations/python/packages/raes_backend_libvirt/techvault_lifecycle.py @@ -122,10 +122,10 @@ def _invoke_native_action(native: object, method_name: str, tolerated_codes: set method() except Exception as exc: code = _error_code(exc) - if code in tolerated_codes: - return True - _record_suppressed_failure("_invoke_native_action", exc, native_code=code) - return False + tolerated = code in tolerated_codes + if not tolerated: + _record_suppressed_failure("_invoke_native_action", exc, native_code=code) + return tolerated return True