diff --git a/.github/workflows/probity-evidence.yml b/.github/workflows/probity-evidence.yml new file mode 100644 index 00000000000..c673e4b3ee8 --- /dev/null +++ b/.github/workflows/probity-evidence.yml @@ -0,0 +1,100 @@ +name: Native signed-effect contract + +on: + workflow_dispatch: + schedule: + - cron: '29 1 * * *' + pull_request: + paths: + - 'haystack/components/agents/**' + - 'haystack/core/pipeline/**' + - 'haystack/tools/**' + - 'haystack/tracing/**' + - 'haystack/dataclasses/**' + - 'e2e/probity_evidence/**' + - 'pyproject.toml' + - 'VERSION.txt' + - '.github/workflows/probity-evidence.yml' + +permissions: + contents: read + +concurrency: + group: native-signed-effect-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + signed-effects: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + UV_LINK_MODE: copy + UV_CONCURRENT_BUILDS: '1' + CMAKE_BUILD_PARALLEL_LEVEL: '1' + OMP_NUM_THREADS: '1' + OPENBLAS_NUM_THREADS: '1' + MKL_NUM_THREADS: '1' + NUMEXPR_NUM_THREADS: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + path: host + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: probityai/agent-evidence-observer + ref: 5d9feedcfffcf441380e11e9f25a5db44d274068 + persist-credentials: false + path: selected-observer + sparse-checkout: | + src + interop/haystack-native-2026-10-03 + pyproject.toml + README.md + LICENSE + sparse-checkout-cone-mode: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.13.3' + - name: Install repository tooling + run: python -m pip install hatch==1.16.5 + - name: Check source mutation refusals + working-directory: host/e2e/probity_evidence + run: >- + hatch run python -m unittest -v test_source_selection + > '${{ runner.temp }}/haystack-source-guards.stdout' + 2> '${{ runner.temp }}/haystack-source-guards.stderr' + - name: Qualify the current framework and installed consumer + working-directory: host/e2e/probity_evidence + run: >- + hatch run python run.py + --source ../../../selected-observer + --selection observer-source-selection.json + --framework-source ../.. + --output '${{ runner.temp }}/haystack-evidence-contract' + --python 3.13.3 + - name: Retain original native records and decisions + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: native-signed-effect-contract + path: | + ${{ runner.temp }}/haystack-source-guards.stdout + ${{ runner.temp }}/haystack-source-guards.stderr + ${{ runner.temp }}/haystack-evidence-contract/packet/ + ${{ runner.temp }}/haystack-evidence-contract/changed-input-packet/ + ${{ runner.temp }}/haystack-evidence-contract/changed-input-publication/ + ${{ runner.temp }}/haystack-evidence-contract/unselected-publication/ + ${{ runner.temp }}/haystack-evidence-contract/wheels/ + ${{ runner.temp }}/haystack-evidence-contract/framework-wheels/ + ${{ runner.temp }}/haystack-evidence-contract/reference-derivation/ + ${{ runner.temp }}/haystack-evidence-contract/host-selected*.json + ${{ runner.temp }}/haystack-evidence-contract/publication/ + ${{ runner.temp }}/haystack-evidence-contract/*source-selection.json + ${{ runner.temp }}/haystack-evidence-contract/commands.json + ${{ runner.temp }}/haystack-evidence-contract/native-tests.xml + ${{ runner.temp }}/haystack-evidence-contract/COMPLETE.json + ${{ runner.temp }}/haystack-evidence-contract/*.stdout + ${{ runner.temp }}/haystack-evidence-contract/*.stderr + if-no-files-found: error + retention-days: 14 diff --git a/e2e/probity_evidence/README.md b/e2e/probity_evidence/README.md new file mode 100644 index 00000000000..10449cbf5e1 --- /dev/null +++ b/e2e/probity_evidence/README.md @@ -0,0 +1,41 @@ +# Native signed-effect contract + +This end-to-end job checks native agent effects against a separately installed reader. +It helps Haystack maintainers catch changes in tool, trace and terminal behavior. + +It uses published Observer source pinned to `5d9feedcfffcf441380e11e9f25a5db44d274068` and its hash locks. +The workflow selects that source automatically. +For a local run, put the selected Observer checkout beside the Haystack checkout. + +```sh +cd e2e/probity_evidence +hatch run python run.py \ + --source ../../../selected-observer \ + --selection observer-source-selection.json \ + --framework-source ../.. \ + --output /tmp/fresh-haystack-evidence-contract \ + --python 3.13.3 +``` + +The runner builds and installs the current Haystack checkout's wheel. +It checks installed source bytes against the selected Git tree and checks package dependencies. +It copies only selected, rechecked bytes into each build. +The reader has no Haystack installation. + +The job derives a current-host reference contract from the frozen release profile. +It binds only `SDK_VERSION` and `SDK_SOURCE` to the selected host version and commit. +It keeps all 31 original tests and all substantive guards. +It retains original and derived module bytes, hashes and the exact two-selector diff. +Derived wheel metadata carries a distinct local version, matching producer dependency and embedded receipt. +This is a declared source transformation, not the unchanged frozen release profile. + +The fixed workload runs real `Pipeline`, `Agent`, `Tool` and tracer transitions with scripted generator replies. +Its seven cases cover permission, wrong content, faults before and after durable writes, step limits and truncated replies. +Only the permitted effect reaches publication. +Two changed-input controls must refuse, including a changed invocation whose artifact digest was updated. +Original packets, decisions, refusal output, wheel hashes and process receipts remain in the job artifact. + +This is a framework contract test, not a model-quality benchmark. +The producer, initial policy and keys share one operator. +Separate environments do not establish outside custody. +The repository operator owns the job, reference-source updates, policy and artifact retention after acceptance. diff --git a/e2e/probity_evidence/observer-source-selection.json b/e2e/probity_evidence/observer-source-selection.json new file mode 100644 index 00000000000..5ba4c0f4a94 --- /dev/null +++ b/e2e/probity_evidence/observer-source-selection.json @@ -0,0 +1,374 @@ +{ + "repo": "probityai/agent-evidence-observer", + "commit": "5d9feedcfffcf441380e11e9f25a5db44d274068", + "files": [ + { + "path": "LICENSE", + "mode": "100644", + "type": "blob", + "sha": "d645695673349e3947e8e5ae42332d0ac3164cd7", + "size": 11358, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/d645695673349e3947e8e5ae42332d0ac3164cd7" + }, + { + "path": "README.md", + "mode": "100644", + "type": "blob", + "sha": "a04fb335b5c755a38648a0833195172c8e2ada1a", + "size": 3938, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/a04fb335b5c755a38648a0833195172c8e2ada1a" + }, + { + "path": "interop/haystack-native-2026-10-03/.gitignore", + "mode": "100644", + "type": "blob", + "sha": "29a4b52fc8977585234141feb934aef776f2e6c6", + "size": 47, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/29a4b52fc8977585234141feb934aef776f2e6c6" + }, + { + "path": "interop/haystack-native-2026-10-03/PROFILE.md", + "mode": "100644", + "type": "blob", + "sha": "0098e8f627f821aa70b4afd8e3eecfadcb300985", + "size": 3673, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/0098e8f627f821aa70b4afd8e3eecfadcb300985" + }, + { + "path": "interop/haystack-native-2026-10-03/README.md", + "mode": "100644", + "type": "blob", + "sha": "78f0be0983dae24b0230ffdc5e6bc89bae61e5bf", + "size": 994, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/78f0be0983dae24b0230ffdc5e6bc89bae61e5bf" + }, + { + "path": "interop/haystack-native-2026-10-03/probity_haystack/__init__.py", + "mode": "100644", + "type": "blob", + "sha": "7e8a0b012c9b1755a159492a809eb1ebb034cc9f", + "size": 77, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/7e8a0b012c9b1755a159492a809eb1ebb034cc9f" + }, + { + "path": "interop/haystack-native-2026-10-03/probity_haystack/contract.py", + "mode": "100644", + "type": "blob", + "sha": "538737ca6a15576e4563d56a2c4a5fc13cc1ac03", + "size": 1678, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/538737ca6a15576e4563d56a2c4a5fc13cc1ac03" + }, + { + "path": "interop/haystack-native-2026-10-03/probity_haystack/producer.py", + "mode": "100644", + "type": "blob", + "sha": "2c7c6809ee8dc95c1fb741c9766f0d1e1d1521e0", + "size": 9354, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/2c7c6809ee8dc95c1fb741c9766f0d1e1d1521e0" + }, + { + "path": "interop/haystack-native-2026-10-03/probity_haystack/reader.py", + "mode": "100644", + "type": "blob", + "sha": "537e3195130c1b51bc205405e5fc05b7989023d9", + "size": 8831, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/537e3195130c1b51bc205405e5fc05b7989023d9" + }, + { + "path": "interop/haystack-native-2026-10-03/pyproject.toml", + "mode": "100644", + "type": "blob", + "sha": "c180a34a31d80f0def08ccb2e0990820cbc9d6fa", + "size": 500, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/c180a34a31d80f0def08ccb2e0990820cbc9d6fa" + }, + { + "path": "interop/haystack-native-2026-10-03/requirements-reader.in", + "mode": "100644", + "type": "blob", + "sha": "694ff0284cc0873482935dde3e222a1abb6e1780", + "size": 21, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/694ff0284cc0873482935dde3e222a1abb6e1780" + }, + { + "path": "interop/haystack-native-2026-10-03/requirements-reader.lock", + "mode": "100644", + "type": "blob", + "sha": "bab63da9e6e28f7dd68f8ab5fb2c1d38ea1f011a", + "size": 13207, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/bab63da9e6e28f7dd68f8ab5fb2c1d38ea1f011a" + }, + { + "path": "interop/haystack-native-2026-10-03/requirements.in", + "mode": "100644", + "type": "blob", + "sha": "a745cf444bb228cbd4127b7ac659470fdf98d7e4", + "size": 100, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/a745cf444bb228cbd4127b7ac659470fdf98d7e4" + }, + { + "path": "interop/haystack-native-2026-10-03/requirements.lock", + "mode": "100644", + "type": "blob", + "sha": "6b8c9ea675abd24e030b20e5a567f6140803b04d", + "size": 92735, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/6b8c9ea675abd24e030b20e5a567f6140803b04d" + }, + { + "path": "interop/haystack-native-2026-10-03/test_native.py", + "mode": "100644", + "type": "blob", + "sha": "7754b35cbbf1b0496523343a32bf37c1c7e07c58", + "size": 7772, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/7754b35cbbf1b0496523343a32bf37c1c7e07c58" + }, + { + "path": "interop/haystack-native-2026-10-03/verify_install.sh", + "mode": "100755", + "type": "blob", + "sha": "327b178f79a8e65322836d055320584964e4bfa6", + "size": 2015, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/327b178f79a8e65322836d055320584964e4bfa6" + }, + { + "path": "pyproject.toml", + "mode": "100644", + "type": "blob", + "sha": "1ed3e3bcddca283971c4ede0374c2d739804c665", + "size": 1357, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/1ed3e3bcddca283971c4ede0374c2d739804c665" + }, + { + "path": "src/probity_observer/__init__.py", + "mode": "100644", + "type": "blob", + "sha": "8cc5bd8971d99c69415f3214ad5d12901ca74e9a", + "size": 723, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/8cc5bd8971d99c69415f3214ad5d12901ca74e9a" + }, + { + "path": "src/probity_observer/aae_binding.py", + "mode": "100644", + "type": "blob", + "sha": "f87f0f196dc44d319d4a7a096e82a3576c8a91a2", + "size": 10885, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/f87f0f196dc44d319d4a7a096e82a3576c8a91a2" + }, + { + "path": "src/probity_observer/aae_dispatch.py", + "mode": "100644", + "type": "blob", + "sha": "6c32395ec77f4206b93a1691c110350de033490b", + "size": 8015, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/6c32395ec77f4206b93a1691c110350de033490b" + }, + { + "path": "src/probity_observer/aae_enforce.py", + "mode": "100644", + "type": "blob", + "sha": "3aff211562c90eb59375f82469cfe14b44e3595d", + "size": 21776, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/3aff211562c90eb59375f82469cfe14b44e3595d" + }, + { + "path": "src/probity_observer/aae_ticket.py", + "mode": "100644", + "type": "blob", + "sha": "01443cd9b45eb69526870ae9260ae19ef4aed3a7", + "size": 6287, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/01443cd9b45eb69526870ae9260ae19ef4aed3a7" + }, + { + "path": "src/probity_observer/admission.py", + "mode": "100644", + "type": "blob", + "sha": "ef0162dd12516c13781cab09c08f4d965b74a01a", + "size": 20440, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/ef0162dd12516c13781cab09c08f4d965b74a01a" + }, + { + "path": "src/probity_observer/atomic_reader.py", + "mode": "100644", + "type": "blob", + "sha": "9776eef7d8314e20db820ed5292d091e7cb26bff", + "size": 15087, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/9776eef7d8314e20db820ed5292d091e7cb26bff" + }, + { + "path": "src/probity_observer/attribution.py", + "mode": "100644", + "type": "blob", + "sha": "986aa9fe179ffd3e722b22aed71875f61003918b", + "size": 7799, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/986aa9fe179ffd3e722b22aed71875f61003918b" + }, + { + "path": "src/probity_observer/authorization.py", + "mode": "100644", + "type": "blob", + "sha": "e68bd819943b83e21e8091a70308694e00f33ce1", + "size": 29467, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/e68bd819943b83e21e8091a70308694e00f33ce1" + }, + { + "path": "src/probity_observer/broker.py", + "mode": "100644", + "type": "blob", + "sha": "41761b052164dacf1622fe1b78a4398851fe4658", + "size": 20264, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/41761b052164dacf1622fe1b78a4398851fe4658" + }, + { + "path": "src/probity_observer/cli.py", + "mode": "100644", + "type": "blob", + "sha": "ee5b867d580db77fd8402e35c4bbf66e59cbf131", + "size": 4090, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/ee5b867d580db77fd8402e35c4bbf66e59cbf131" + }, + { + "path": "src/probity_observer/crypto.py", + "mode": "100644", + "type": "blob", + "sha": "41f59a7dbefc8fc9a4573dc4d1bcd7abd5908d04", + "size": 5729, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/41f59a7dbefc8fc9a4573dc4d1bcd7abd5908d04" + }, + { + "path": "src/probity_observer/declaration.py", + "mode": "100644", + "type": "blob", + "sha": "a4dc12aa2f32c56c6dfa4331723391fa9c49e124", + "size": 20694, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/a4dc12aa2f32c56c6dfa4331723391fa9c49e124" + }, + { + "path": "src/probity_observer/evaluation_history.py", + "mode": "100644", + "type": "blob", + "sha": "a221121b979ab229e87916ccb74e9a1bf399d2ae", + "size": 23576, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/a221121b979ab229e87916ccb74e9a1bf399d2ae" + }, + { + "path": "src/probity_observer/history.py", + "mode": "100644", + "type": "blob", + "sha": "20f9ba426e2ce350076b7b2bc9b8231e723d2d3c", + "size": 11134, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/20f9ba426e2ce350076b7b2bc9b8231e723d2d3c" + }, + { + "path": "src/probity_observer/inspect_history.py", + "mode": "100644", + "type": "blob", + "sha": "c8dada16a77a23b1073aa774597bdacbc8eb4054", + "size": 26903, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/c8dada16a77a23b1073aa774597bdacbc8eb4054" + }, + { + "path": "src/probity_observer/isolation.py", + "mode": "100644", + "type": "blob", + "sha": "e41db79db6f561a679e3b3150444f796f25a79f0", + "size": 13166, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/e41db79db6f561a679e3b3150444f796f25a79f0" + }, + { + "path": "src/probity_observer/ledger.py", + "mode": "100644", + "type": "blob", + "sha": "4cb0ab35c4c9cae4ada265bcb633fabb796ba5a2", + "size": 18404, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/4cb0ab35c4c9cae4ada265bcb633fabb796ba5a2" + }, + { + "path": "src/probity_observer/native_reader.py", + "mode": "100644", + "type": "blob", + "sha": "129cfbdca0e9b165fe6ae5d36dde23393f9d65a7", + "size": 12815, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/129cfbdca0e9b165fe6ae5d36dde23393f9d65a7" + }, + { + "path": "src/probity_observer/probe_agent.py", + "mode": "100644", + "type": "blob", + "sha": "315fbabe36079c8e4406fcefdafdb39496d43177", + "size": 2584, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/315fbabe36079c8e4406fcefdafdb39496d43177" + }, + { + "path": "src/probity_observer/protected_dispatch.py", + "mode": "100644", + "type": "blob", + "sha": "3b5430bb9e7c095f4c505b763a8dbc7298f009c5", + "size": 38146, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/3b5430bb9e7c095f4c505b763a8dbc7298f009c5" + }, + { + "path": "src/probity_observer/protected_isolation.py", + "mode": "100644", + "type": "blob", + "sha": "43e8405382e880654c8ea7c504d11bb9326d009f", + "size": 19183, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/43e8405382e880654c8ea7c504d11bb9326d009f" + }, + { + "path": "src/probity_observer/protected_probe.py", + "mode": "100644", + "type": "blob", + "sha": "a6d8b7635867ea36650eaa5a35214fe0c8826fd8", + "size": 5872, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/a6d8b7635867ea36650eaa5a35214fe0c8826fd8" + }, + { + "path": "src/probity_observer/replay.py", + "mode": "100644", + "type": "blob", + "sha": "556692fd9630dfec0a0b2adee8fadbb5d583cd28", + "size": 29625, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/556692fd9630dfec0a0b2adee8fadbb5d583cd28" + }, + { + "path": "src/probity_observer/run_selection.py", + "mode": "100644", + "type": "blob", + "sha": "c0d24f29aa369bf473235353cd63afc861c00a5b", + "size": 11374, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/c0d24f29aa369bf473235353cd63afc861c00a5b" + }, + { + "path": "src/probity_observer/selection_profile.py", + "mode": "100644", + "type": "blob", + "sha": "91e55d54a8ebc43b0b8fdeed9117922ef1349ef4", + "size": 15773, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/91e55d54a8ebc43b0b8fdeed9117922ef1349ef4" + }, + { + "path": "src/probity_observer/ticket_service.py", + "mode": "100644", + "type": "blob", + "sha": "0f100788bad8dea4deba6e6107c47ecb7bc0a6c3", + "size": 38767, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/0f100788bad8dea4deba6e6107c47ecb7bc0a6c3" + }, + { + "path": "src/probity_observer/verify.py", + "mode": "100644", + "type": "blob", + "sha": "9e16e5937e37ae251c0efc8058ea481b502a2fa1", + "size": 9511, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/9e16e5937e37ae251c0efc8058ea481b502a2fa1" + }, + { + "path": "src/probity_observer/witness_port.py", + "mode": "100644", + "type": "blob", + "sha": "795e8f28da07d5124c32f2e79390eb42f64f96cb", + "size": 2677, + "url": "https://api.github.com/repos/probityai/agent-evidence-observer/git/blobs/795e8f28da07d5124c32f2e79390eb42f64f96cb" + } + ] +} diff --git a/e2e/probity_evidence/pyproject.toml b/e2e/probity_evidence/pyproject.toml new file mode 100644 index 00000000000..9a929e66dc7 --- /dev/null +++ b/e2e/probity_evidence/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "haystack-signed-effect-contract" +version = "0.0.0" +requires-python = ">=3.13" + +[tool.hatch.envs.default] +skip-install = true +python = "3.13" +installer = "uv" +dependencies = ["uv==0.11.28", "hatch==1.16.5"] diff --git a/e2e/probity_evidence/run.py b/e2e/probity_evidence/run.py new file mode 100644 index 00000000000..ae12ea2645e --- /dev/null +++ b/e2e/probity_evidence/run.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: 2026 Sankalp Gilda +# +# SPDX-License-Identifier: Apache-2.0 + +"""Qualify a Haystack host publication gate with installed isolated readers.""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +import shutil +import signal +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + +_OBSERVER_PIN = "5d9feedcfffcf441380e11e9f25a5db44d274068" +_PROFILE = Path("interop/haystack-native-2026-10-03") + + +def _encoded(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _selected_bytes(source: Path, member: dict) -> tuple[Path, bytes]: + rel = Path(member["path"]) + if rel.is_absolute() or ".." in rel.parts: + raise ValueError("source path escapes selection") + path = source / rel + if path.is_symlink() or any(p.is_symlink() for p in path.parents if p.is_relative_to(source)): + raise ValueError("source must not be a symlink") + body = path.read_bytes() + blob = hashlib.sha1(b"blob " + str(len(body)).encode() + b"\0" + body).hexdigest() + if blob != member["sha"]: + raise ValueError(f"source blob differs: {rel}") + return rel, body + + +def _check_source(source: Path, selection: Path) -> dict: + data = json.loads(selection.read_text()) + if data["repo"] != "probityai/agent-evidence-observer" or data["commit"] != _OBSERVER_PIN: + raise ValueError("unexpected source selection") + for member in data["files"]: + _selected_bytes(source, member) + return data + + +def _stage_selected_source(source: Path, destination: Path, data: dict) -> None: + # Recheck the bytes that will enter the build; ignore every unselected file. + selected = [_selected_bytes(source, member) for member in data["files"]] + destination.mkdir() + for rel, body in selected: + target = destination / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(body) + + +def _clean_environment(out: Path) -> dict[str, str]: + env = dict(os.environ) + for name in list(env): + if ( + name == "VIRTUAL_ENV" + or any(word in name for word in ("TOKEN", "SECRET", "API_KEY", "PASSWORD")) + or name.startswith(("PYTHON", "PIP_", "HATCH_")) + ): + env.pop(name) + temporary = out / "temporary" + temporary.mkdir() + env.update( + PYTHONDONTWRITEBYTECODE="1", + HAYSTACK_TELEMETRY_ENABLED="false", + TMPDIR=str(temporary), + SOURCE_DATE_EPOCH="946684800", + UV_CACHE_DIR=str(out / "uv-cache"), + UV_PYTHON_INSTALL_DIR=str(out / "uv-python"), + UV_LINK_MODE="copy", + HATCH_DATA_DIR=str(out / "hatch-data"), + HATCH_CACHE_DIR=str(out / "hatch-cache"), + ) + return env + + +class _Commands: + def __init__(self, out: Path, env: dict[str, str]): + self.out, self.env, self.rows = out, env, [] + + def run( + self, name: str, command: list[str], *, expected: int | None = 0, timeout: int = 600, cwd: Path | None = None + ): + start = time.monotonic() + row = { + "name": name, + "command": command, + "cwd": str(cwd or self.out), + "started_at": datetime.now(timezone.utc).isoformat(), + } + process = None + with (self.out / (name + ".stdout")).open("wb") as stdout, (self.out / (name + ".stderr")).open("wb") as stderr: + try: + process = subprocess.Popen( + command, cwd=cwd or self.out, env=self.env, stdout=stdout, stderr=stderr, start_new_session=True + ) + row["process_group"] = process.pid + row["exit_code"] = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + row["exit_code"] = None + row["timeout"] = True + except OSError as error: + row.update(exit_code=None, launch_error=type(error).__name__, errno=error.errno) + except KeyboardInterrupt: + row.update(exit_code=None, cancelled=True) + finally: + if process is not None: + try: + row["live_group_members_before_cleanup"] = _live_group_members(process.pid) + row["process_group_clear"] = _stop_process_group(process) + except (OSError, subprocess.TimeoutExpired) as error: + row.update(process_group_clear=False, cleanup_error=type(error).__name__) + row["process_returncode"] = process.returncode + row["elapsed_seconds"] = time.monotonic() - start + self.rows.append(row) + (self.out / "commands.json").write_bytes(_encoded(self.rows)) + print(name, row["exit_code"], flush=True) + if row.get("cancelled"): + raise KeyboardInterrupt + if ( + row.get("timeout") + or row.get("launch_error") + or row.get("process_group_clear") is False + or expected is not None + and row["exit_code"] != expected + ): + raise RuntimeError(f"{name} failed; retained stdout/stderr and process receipt") + return row + + +def _live_group_members(group: int) -> list[int]: + members = [] + for entry in Path("/proc").iterdir(): + if entry.name.isdigit(): + try: + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(fields[2]) == group and fields[0] != "Z": + members.append(int(entry.name)) + except (FileNotFoundError, ProcessLookupError): + pass + return members + + +def _stop_process_group(process: subprocess.Popen) -> bool: + for action in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(process.pid, action) + except ProcessLookupError: + break + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + pass + deadline = time.monotonic() + 2 + while _live_group_members(process.pid) and time.monotonic() < deadline: + time.sleep(0.02) + if not _live_group_members(process.pid): + break + process.wait(timeout=2) + return not _live_group_members(process.pid) + + +def _reader_command(python: Path, packet: Path, policy: Path, publication: Path | None = None): + command = [ + str(python), + "-I", + "-B", + "-m", + "probity_haystack.reader", + str(packet), + "--host-policy", + str(policy), + "--policy-sha256", + _sha(policy), + ] + return command + (["--publish", str(publication)] if publication else []) + + +def _framework_selection(source: Path, captured: Path | None) -> dict: + if captured: + data = json.loads(captured.read_text()) + else: + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + tree = subprocess.check_output( + ["git", "ls-tree", "-rz", "HEAD", "--", "haystack", "pyproject.toml", "VERSION.txt", "README.md"], + cwd=source, + ) + files = [] + for row in tree.split(b"\0"): + if row: + fields, path = row.split(b"\t", 1) + mode, kind, blob = fields.decode().split() + if kind != "blob" or mode not in ("100644", "100755"): + raise ValueError("unsupported host source member") + files.append({"path": path.decode(), "sha": blob}) + data = {"repo": "deepset-ai/haystack", "commit": commit, "files": files} + if data["repo"] != "deepset-ai/haystack" or len(data["commit"]) != 40 or not data["files"]: + raise ValueError("unexpected host source selection") + if any(char not in "0123456789abcdef" for char in data["commit"]): + raise ValueError("invalid host source commit") + return data + + +def _derive_host_reference(source_copy: Path, framework: dict, version: str, out: Path) -> dict: + profile = source_copy / _PROFILE + contract = profile / "probity_haystack/contract.py" + original = contract.read_bytes() + declarations = { + "SDK_VERSION": ('SDK_VERSION = "3.3.0"\n', version), + "SDK_SOURCE": ('SDK_SOURCE = "daa2d1ffacd083dcb1fc9a455adf541360a9e09c"\n', framework["commit"]), + } + lines = original.decode().splitlines(keepends=True) + for name, (expected, value) in declarations.items(): + if lines.count(expected) != 1: + raise ValueError("original SDK identity declarations differ") + lines[lines.index(expected)] = name + " = " + json.dumps(value) + "\n" + derived = "".join(lines).encode() + metadata = profile / "pyproject.toml" + metadata_original = metadata.read_bytes() + text = metadata_original.decode() + expected_version = 'version = "0.0.1"\n' + expected_extra = 'producer = ["haystack-ai==3.3.0"]\n' + if text.count(expected_version) != 1 or text.count(expected_extra) != 1: + raise ValueError("original reference metadata differs") + local_version = "0.0.1+host." + framework["commit"][:12] + text = text.replace(expected_version, "version = " + json.dumps(local_version) + "\n", 1) + text = text.replace(expected_extra, "producer = [" + json.dumps("haystack-ai==" + version) + "]\n", 1) + text += '\n[tool.setuptools.package-data]\nprobity_haystack = ["host-reference-derivation.json"]\n' + metadata_derived = text.encode() + retained = out / "reference-derivation" + retained.mkdir() + for name, before, after in [ + ("contract.py", original, derived), + ("pyproject.toml", metadata_original, metadata_derived), + ]: + (retained / ("original-" + name)).write_bytes(before) + (retained / ("derived-" + name)).write_bytes(after) + difference = "".join( + difflib.unified_diff( + before.decode().splitlines(keepends=True), + after.decode().splitlines(keepends=True), + fromfile="original/" + name, + tofile="derived/" + name, + ) + ) + (retained / (name + ".diff")).write_text(difference) + receipt = { + "schema": "haystack-host-reference-derivation-v1", + "base_observer_commit": _OBSERVER_PIN, + "host_commit": framework["commit"], + "host_sdk_version": version, + "derived_distribution_version": local_version, + "runtime_selector_changes": ["SDK_VERSION", "SDK_SOURCE"], + "original_contract_sha256": hashlib.sha256(original).hexdigest(), + "derived_contract_sha256": hashlib.sha256(derived).hexdigest(), + "original_metadata_sha256": hashlib.sha256(metadata_original).hexdigest(), + "derived_metadata_sha256": hashlib.sha256(metadata_derived).hexdigest(), + "original_tests_sha256": _sha(profile / "test_native.py"), + "substantive_guards_changed": False, + } + contract.write_bytes(derived) + metadata.write_bytes(metadata_derived) + (retained / "receipt.json").write_bytes(_encoded(receipt)) + (profile / "probity_haystack/host-reference-derivation.json").write_bytes(_encoded(receipt)) + return receipt + + +def _qualify( + source: Path, + selection: Path, + out: Path, + python: str, + framework_source: Path | None = None, + captured_framework: Path | None = None, +) -> dict: + out.mkdir() + data = _check_source(source, selection) + (out / "source-selection.json").write_bytes(_encoded(data)) + commands = _Commands(out, _clean_environment(out)) + source_copy = out / "build-source" + _stage_selected_source(source, source_copy, data) + producer = out / "producer/bin/python" + reader = out / "reader/bin/python" + commands.run("producer-venv", ["uv", "venv", str(out / "producer"), "--python", python]) + commands.run("reader-venv", ["uv", "venv", str(out / "reader"), "--python", python]) + for label, executable, lock in [ + ("producer", producer, "requirements.lock"), + ("reader", reader, "requirements-reader.lock"), + ]: + commands.run( + label + "-dependencies", + [ + "uv", + "pip", + "install", + "--python", + str(executable), + "--require-hashes", + "-r", + str(source_copy / _PROFILE / lock), + ], + ) + framework = None + derivation = None + if framework_source: + framework = _framework_selection(framework_source, captured_framework) + (out / "framework-source-selection.json").write_bytes(_encoded(framework)) + framework_copy = out / "framework-build-source" + _stage_selected_source(framework_source, framework_copy, framework) + commands.run( + "framework-wheel", ["hatch", "build", "-t", "wheel", str(out / "framework-wheels")], cwd=framework_copy + ) + host_wheels = list((out / "framework-wheels").glob("*.whl")) + if len(host_wheels) != 1: + raise ValueError("expected one host framework wheel") + commands.run( + "framework-install", + ["uv", "pip", "install", "--python", str(producer), "--no-deps", "--reinstall", str(host_wheels[0])], + ) + commands.run("framework-dependencies-check", ["uv", "pip", "check", "--python", str(producer)]) + commands.run( + "framework-installed-source", + [ + str(producer), + "-I", + "-B", + "-c", + "import hashlib,importlib.metadata,json,pathlib,haystack; from packaging.version import Version; " + "data=json.loads(pathlib.Path(__import__('sys').argv[1]).read_text()); " + "root=pathlib.Path(haystack.__file__).parent.parent; " + "members=[m for m in data['files'] if m['path'].startswith('haystack/')]; " + "bodies=[(m,(root/m['path']).read_bytes()) for m in members]; " + "assert all(hashlib.sha1(b'blob '+str(len(b)).encode()+b'\\0'+b).hexdigest()==m['sha'] " + "for m,b in bodies),'installed host source differs'; " + "assert Version(importlib.metadata.version('haystack-ai'))==Version(" + "pathlib.Path(__import__('sys').argv[2]).read_text().strip()),'host version differs'; " + "print(json.dumps({'installed_files':len(members),'version':importlib.metadata.version('haystack-ai')}))", + str(out / "framework-source-selection.json"), + str(framework_copy / "VERSION.txt"), + ], + ) + _check_source(source_copy, selection) + installed = json.loads((out / "framework-installed-source.stdout").read_text()) + derivation = _derive_host_reference(source_copy, framework, installed["version"], out) + wheels = out / "wheels" + for label, directory in [("observer", source_copy), ("haystack-profile", source_copy / _PROFILE)]: + commands.run( + label + "-wheel", + [ + str(producer), + "-I", + "-B", + "-m", + "build", + "--wheel", + "--no-isolation", + "--outdir", + str(wheels), + str(directory), + ], + ) + selected_wheels = sorted(str(p) for p in wheels.glob("*.whl")) + if len(selected_wheels) != 2: + raise ValueError("expected exactly two wheels") + for label, executable in [("producer", producer), ("reader", reader)]: + commands.run( + label + "-install", ["uv", "pip", "install", "--python", str(executable), "--no-deps", *selected_wheels] + ) + commands.run(label + "-freeze", ["uv", "pip", "freeze", "--python", str(executable)]) + if derivation: + commands.run( + "derived-reference-receipt", + [ + str(reader), + "-I", + "-B", + "-c", + "import importlib.metadata,json,pathlib; import probity_haystack.contract as contract; " + "expected=json.loads(pathlib.Path(__import__('sys').argv[1]).read_text()); " + "actual=json.loads(pathlib.Path(contract.__file__)." + "with_name('host-reference-derivation.json').read_text()); " + "assert actual==expected,'installed derivation receipt differs'; " + "assert contract.SDK_VERSION==expected['host_sdk_version'] and " + "contract.SDK_SOURCE==expected['host_commit'],'derived selectors differ'; " + "assert importlib.metadata.version('probity-haystack-reference')==" + "expected['derived_distribution_version'], " + "'derived distribution version differs'; print(json.dumps(actual))", + str(out / "reference-derivation/receipt.json"), + ], + ) + commands.run( + "reader-framework-absence", + [str(reader), "-I", "-c", "import importlib.util; assert importlib.util.find_spec('haystack') is None"], + ) + commands.env["HAYSTACK_READER_PYTHON"] = str(reader) + commands.run( + "upstream-native-controls", + [ + str(producer), + "-I", + "-B", + "-m", + "pytest", + "-c", + "/dev/null", + str(source_copy / _PROFILE / "test_native.py"), + "-q", + "-p", + "no:cacheprovider", + "--basetemp=" + str(out / "native-test-temporary"), + "--junitxml=" + str(out / "native-tests.xml"), + ], + ) + packet = out / "packet" + commands.run("fresh-native-producer", [str(producer), "-I", "-B", "-m", "probity_haystack.producer", str(packet)]) + # Keep the selected policy outside the candidate packet; custody remains local. + policy = out / "host-selected-policy.json" + shutil.copyfile(packet / "host-policy.json", policy) + commands.run("reader-first", _reader_command(reader, packet, policy)) + commands.run("reader-second", _reader_command(reader, packet, policy)) + if (out / "reader-first.stdout").read_bytes() != (out / "reader-second.stdout").read_bytes(): + raise ValueError("repeated installed reader decisions differ") + publication = out / "publication" + commands.run("publish-original", _reader_command(reader, packet, policy, publication)) + original_publication = _sha(publication / "published.json") + # A candidate rewrites both its tool trace and digest. Signed effects still bind original bytes. + altered = out / "changed-input-packet" + shutil.copytree(packet, altered) + native_path = altered / "permit/native.json" + native = json.loads(native_path.read_text()) + native["toolCalls"][0]["content"] = "changed after signed permission" + native_path.write_bytes(_encoded(native)) + changed_policy_data = json.loads(policy.read_text()) + changed_policy_data["cases"]["permit"]["artifacts"]["native.json"] = _sha(native_path) + changed_policy = out / "host-selected-changed-policy.json" + changed_policy.write_bytes(_encoded(changed_policy_data)) + changed_publication = out / "changed-input-publication" + result = commands.run( + "refuse-rehashed-changed-input", + _reader_command(reader, altered, changed_policy, changed_publication), + expected=None, + ) + if result["exit_code"] == 0 or changed_publication.exists(): + raise ValueError("changed invocation admitted or refusal wrote a publication") + if b"invocation bytes differ" not in (out / "refuse-rehashed-changed-input.stderr").read_bytes(): + raise ValueError("changed-input check did not reach the expected semantic refusal") + commands.run( + "refuse-original-policy-change", + _reader_command(reader, altered, policy, out / "unselected-publication"), + expected=None, + ) + if commands.rows[-1]["exit_code"] == 0 or (out / "unselected-publication").exists(): + raise ValueError("changed candidate passed original host selection") + if _sha(publication / "published.json") != original_publication: + raise ValueError("refused candidate changed the original publication") + report = json.loads((out / "reader-first.stdout").read_text()) + released = json.loads((publication / "published.json").read_text()) + if len(report["records"]) != 7 or [x["case"] for x in released["records"]] != ["permit"]: + raise ValueError("unexpected publication population") + completion = { + "schema": "haystack-host-gate-qualification-v1", + "source_commit": _OBSERVER_PIN, + "source_selection_sha256": _sha(selection), + "native_records": 7, + "released_cases": ["permit"], + "changed_input_semantic_refused": True, + "changed_input_pin_refused": True, + "publication_unchanged": True, + "repeated_reader_bytes_equal": True, + "reader_has_haystack": False, + "model_inference_calls": 0, + "independent_custody": False, + "runtime_context": "unverified; see external CI or operator receipt", + "reader_report": report, + "wheels": [{"name": p.name, "sha256": _sha(p)} for p in sorted(wheels.glob("*.whl"))], + } + if framework: + completion["host_framework"] = { + "repo": framework["repo"], + "commit": framework["commit"], + "selected_source_files": len(framework["files"]), + "source_selection_sha256": _sha(out / "framework-source-selection.json"), + "wheels": [{"name": p.name, "sha256": _sha(p)} for p in host_wheels], + } + completion["reference_derivation"] = derivation + (out / "COMPLETE.json").write_bytes(_encoded(completion)) + return completion + + +def _main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--selection", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--python", default="python3.13") + parser.add_argument("--framework-source", type=Path) + parser.add_argument("--framework-selection", type=Path) + args = parser.parse_args() + if args._framework_selection and not args.framework_source: + parser.error("--framework-selection needs --framework-source") + print( + json.dumps( + _qualify( + args.source.resolve(), + args.selection.resolve(), + args.output.resolve(), + args.python, + args.framework_source.resolve() if args.framework_source else None, + args._framework_selection.resolve() if args._framework_selection else None, + ) + ) + ) + + +if __name__ == "__main__": + _main() diff --git a/e2e/probity_evidence/test_source_selection.py b/e2e/probity_evidence/test_source_selection.py new file mode 100644 index 00000000000..c7c2df66bda --- /dev/null +++ b/e2e/probity_evidence/test_source_selection.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: 2026 Sankalp Gilda +# +# SPDX-License-Identifier: Apache-2.0 + +"""Check source mutations that must not enter a native contract build.""" + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +_SPEC = importlib.util.spec_from_file_location("evidence_gate", Path(__file__).with_name("run.py")) +_GATE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_GATE) + + +class TestSourceSelection(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "source" + (self.source / "haystack").mkdir(parents=True) + for name, body in { + "README.md": "fixture\n", + "VERSION.txt": "3.4.0rc0\n", + "pyproject.toml": "[project]\nname='fixture'\n", + "haystack/__init__.py": "FIXTURE = True\n", + }.items(): + (self.source / name).write_text(body) + for args in [ + ["init", "-q"], + ["add", "."], + ["-c", "user.name=Test Author", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"], + ]: + subprocess.run(["git", *args], cwd=self.source, check=True, capture_output=True) + + def test_untracked_source_cannot_enter_selected_stage(self): + (self.source / "haystack/unselected.py").write_text("UNSELECTED = True\n") + data = _GATE._framework_selection(self.source, None) + staged = self.root / "staged" + _GATE._stage_selected_source(self.source, staged, data) + self.assertTrue((staged / "haystack/__init__.py").is_file()) + self.assertFalse((staged / "haystack/unselected.py").exists()) + self.assertEqual(len(data["files"]), 4) + + def test_changed_tracked_source_refuses_before_stage_creation(self): + data = _GATE._framework_selection(self.source, None) + (self.source / "haystack/__init__.py").write_text("CHANGED = True\n") + staged = self.root / "staged" + with self.assertRaisesRegex(ValueError, "source blob differs: haystack/__init__.py"): + _GATE._stage_selected_source(self.source, staged, data) + self.assertFalse(staged.exists()) + + def test_selected_symlink_refuses_before_stage_creation(self): + data = _GATE._framework_selection(self.source, None) + target = self.root / "outside.py" + target.write_text("OUTSIDE = True\n") + member = self.source / "haystack/__init__.py" + member.unlink() + member.symlink_to(target) + with self.assertRaisesRegex(ValueError, "source must not be a symlink"): + _GATE._stage_selected_source(self.source, self.root / "staged", data) + self.assertFalse((self.root / "staged").exists()) + + def test_selected_path_cannot_escape_source(self): + data = {"files": [{"path": "../outside.py", "sha": "0" * 40}]} + with self.assertRaisesRegex(ValueError, "source path escapes selection"): + _GATE._stage_selected_source(self.source, self.root / "staged", data) + self.assertFalse((self.root / "staged").exists()) + + def test_captured_selection_refuses_wrong_repository(self): + captured = self.root / "selection.json" + data = _GATE._framework_selection(self.source, None) + data["repo"] = "other/project" + captured.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, "unexpected host source selection"): + _GATE._framework_selection(self.source, captured) + + def test_child_environment_drops_credentials_and_python_overrides(self): + values = { + name: "not-a-secret" + for name in [ + "AUTH_TOKEN", + "PASSWORD", + "PYTHONPATH", + "PIP_INDEX_URL", + "HATCH_ENV", + "HATCH_ENV_ACTIVE", + "HATCH_PROJECT", + "VIRTUAL_ENV", + ] + } + with patch.dict(os.environ, values): + child = _GATE._clean_environment(self.root) + self.assertTrue(all(name not in child for name in values)) + self.assertEqual(child["TMPDIR"], str(self.root / "temporary")) + self.assertEqual(child["HAYSTACK_TELEMETRY_ENABLED"], "false") + self.assertEqual(child["HATCH_DATA_DIR"], str(self.root / "hatch-data")) + self.assertEqual(child["HATCH_CACHE_DIR"], str(self.root / "hatch-cache")) + + def _reference_fixture(self): + profile = self.source / _GATE._PROFILE + (profile / "probity_haystack").mkdir(parents=True) + contract = ( + b'SDK_VERSION = "3.3.0"\n' + b'SDK_SOURCE = "daa2d1ffacd083dcb1fc9a455adf541360a9e09c"\n' + b'SUBSTANTIVE_GUARD = "retain me"\n' + ) + metadata = b'[project]\nversion = "0.0.1"\n[project.optional-dependencies]\nproducer = ["haystack-ai==3.3.0"]\n' + (profile / "probity_haystack/contract.py").write_bytes(contract) + (profile / "pyproject.toml").write_bytes(metadata) + (profile / "test_native.py").write_bytes(b"ORIGINAL_TESTS = True\n") + return profile, contract, metadata + + def test_derivation_changes_only_declared_runtime_selectors(self): + profile, original, metadata = self._reference_fixture() + framework = {"commit": "a" * 40} + receipt = _GATE._derive_host_reference(self.source, framework, "3.4.0rc0", self.root) + retained = self.root / "reference-derivation" + self.assertEqual((retained / "original-contract.py").read_bytes(), original) + self.assertEqual((retained / "original-pyproject.toml").read_bytes(), metadata) + derived = (profile / "probity_haystack/contract.py").read_bytes() + changed = [ + (before, after) for before, after in zip(original.splitlines(), derived.splitlines()) if before != after + ] + self.assertEqual([before.split(b" = ")[0] for before, _ in changed], [b"SDK_VERSION", b"SDK_SOURCE"]) + self.assertIn(b'SUBSTANTIVE_GUARD = "retain me"', derived) + self.assertEqual((profile / "test_native.py").read_bytes(), b"ORIGINAL_TESTS = True\n") + self.assertEqual(receipt["derived_distribution_version"], "0.0.1+host.aaaaaaaaaaaa") + self.assertFalse(receipt["substantive_guards_changed"]) + self.assertEqual(json.loads((profile / "probity_haystack/host-reference-derivation.json").read_text()), receipt) + + def test_derivation_refuses_changed_original_selector_without_mutation(self): + profile, original, metadata = self._reference_fixture() + path = profile / "probity_haystack/contract.py" + changed = original.replace(b"3.3.0", b"3.3.1") + path.write_bytes(changed) + with self.assertRaisesRegex(ValueError, "original SDK identity declarations differ"): + _GATE._derive_host_reference(self.source, {"commit": "a" * 40}, "3.4.0rc0", self.root) + self.assertEqual(path.read_bytes(), changed) + self.assertEqual((profile / "pyproject.toml").read_bytes(), metadata) + self.assertFalse((self.root / "reference-derivation").exists()) + + def test_derivation_refuses_changed_original_metadata_without_mutation(self): + profile, original, metadata = self._reference_fixture() + path = profile / "pyproject.toml" + changed = metadata.replace(b"0.0.1", b"0.0.2") + path.write_bytes(changed) + with self.assertRaisesRegex(ValueError, "original reference metadata differs"): + _GATE._derive_host_reference(self.source, {"commit": "a" * 40}, "3.4.0rc0", self.root) + self.assertEqual(path.read_bytes(), changed) + self.assertEqual((profile / "probity_haystack/contract.py").read_bytes(), original) + self.assertFalse((self.root / "reference-derivation").exists()) + + def test_launch_failure_retains_receipt_even_without_expected_exit(self): + commands = _GATE._Commands(self.root, dict(os.environ)) + with self.assertRaisesRegex(RuntimeError, "missing failed"): + commands.run("missing", [str(self.root / "missing-program")], expected=None) + row = json.loads((self.root / "commands.json").read_text())[0] + self.assertEqual(row["launch_error"], "FileNotFoundError") + self.assertIsNone(row["exit_code"]) + + def test_successful_command_retains_output_and_clear_process_group(self): + commands = _GATE._Commands(self.root, dict(os.environ)) + row = commands.run("success", [sys.executable, "-I", "-c", "print('completed')"]) + self.assertEqual(row["exit_code"], 0) + self.assertTrue(row["process_group_clear"]) + self.assertEqual((self.root / "success.stdout").read_text(), "completed\n") + + def test_nonzero_command_retains_exit_and_clear_process_group(self): + commands = _GATE._Commands(self.root, dict(os.environ)) + with self.assertRaisesRegex(RuntimeError, "nonzero failed"): + commands.run("nonzero", [sys.executable, "-I", "-c", "raise SystemExit(7)"]) + row = json.loads((self.root / "commands.json").read_text())[0] + self.assertEqual(row["exit_code"], 7) + self.assertTrue(row["process_group_clear"]) + + def test_timeout_stops_child_and_grandchild(self): + """A real timeout needs one second to observe the child tree before refusal.""" + pid_file = self.root / "child-pids.json" + code = ( + "import json,os,pathlib,subprocess,sys,time; " + "child=subprocess.Popen([sys.executable,'-I','-c','import time;time.sleep(60)']); " + "pathlib.Path(sys.argv[1]).write_text(json.dumps([os.getpid(),child.pid])); " + "time.sleep(60)" + ) + commands = _GATE._Commands(self.root, dict(os.environ)) + with self.assertRaisesRegex(RuntimeError, "timeout failed"): + commands.run("timeout", [sys.executable, "-I", "-c", code, str(pid_file)], timeout=1) + row = json.loads((self.root / "commands.json").read_text())[0] + pids = json.loads(pid_file.read_text()) + self.assertTrue(row["timeout"]) + self.assertTrue(row["process_group_clear"]) + self.assertTrue(set(pids).issubset(row["live_group_members_before_cleanup"])) + self.assertEqual(_GATE._live_group_members(row["process_group"]), []) + + +if __name__ == "__main__": + unittest._main()