From eb5e81547f8cde28618920f0c571ff1f35292473 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 17:09:30 +0000 Subject: [PATCH 1/6] Reduce PR CI fanout and schedule full validation Keep nine baseline validation executions, add owned target coverage, and use a stable selected-check aggregate. Move full validation and Pages publication to an exact-input four-hour cycle, without granting deployment permissions to the validator. Generate selected smoke archives with the existing site archive writers. Preserve execution-receipt reuse, visible advisory failures, and honest completion checkpoints. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- .github/workflows/ci.yml | 304 ++++++++++++++++++++--------- .github/workflows/full-ci.yml | 127 ++++++++++++ AGENTS.md | 12 +- README.md | 14 +- docs/ci-reuse.md | 74 ++++++- scripts/ci-plan.py | 352 ++++++++++++++++++++++++++++++++++ src/bin/roundhouse.rs | 35 ++++ src/project.rs | 15 +- tests/ci_plan_test.py | 228 ++++++++++++++++++++++ tests/selected_archives.rs | 100 ++++++++++ tests/workflow_yaml_parses.rs | 162 ++++++++++++++-- 11 files changed, 1307 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/full-ci.yml create mode 100755 scripts/ci-plan.py create mode 100644 tests/ci_plan_test.py create mode 100644 tests/selected_archives.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe35dc99..ac9cd5a89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,21 +7,31 @@ on: branches: [main] # Drafts run fixture + unit only. Leaving draft starts the full run; # returning to draft cancels the superseded full run via concurrency. - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - # Not on `push`: every commit on main keeps a full run. - paths-ignore: ['**.md', 'docs/**'] + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, labeled, unlabeled] + workflow_call: + inputs: + full: + type: boolean + default: false + publish: + type: boolean + default: false + spinel_revision: + type: string + default: '' + outputs: + complete: + value: ${{ jobs.ci-required.outputs.complete }} + publication-ready: + value: ${{ jobs.compact-required.outputs.passed == 'true' && jobs.assemble-site.outputs.ready == 'true' }} -# Cancel superseded PR runs. On main, a started run finishes and a burst of -# pushes collapses to the newest queued one, so each run that starts is -# validated to the end. Pages keeps its separate deployment-only lock below. +# Separate from the full-ci caller's lock: workflow_call inherits its context. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + group: validation-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read - pages: write - id-token: write env: CARGO_TERM_COLOR: always @@ -38,9 +48,35 @@ env: WRITEBOOK_SHA: f3fadd21907ad9b18cb23800d971c2cc25045e2a jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + plan: ${{ steps.plan.outputs.plan }} + jobs: ${{ steps.plan.outputs.jobs }} + extra-compare: ${{ steps.plan.outputs.extra-compare }} + smoke: ${{ steps.plan.outputs.smoke }} + archives: ${{ steps.plan.outputs.archives }} + wasm: ${{ steps.plan.outputs.wasm }} + site: ${{ steps.plan.outputs.site }} + publish: ${{ steps.plan.outputs.publish }} + spinel-revision: ${{ steps.plan.outputs.spinel-revision }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + - name: Select required coverage from the event's actual tree + id: plan + env: + CI_FULL: ${{ inputs.full || false }} + CI_PUBLISH: ${{ inputs.publish || false }} + CI_SPINEL_REVISION: ${{ inputs.spinel_revision }} + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci-plan.py plan + writebook-inventory: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'writebook-inventory') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -119,6 +155,7 @@ jobs: # install cost once per CI run instead of in each of seven jobs. # ------------------------------------------------------------------- generate-fixture: + needs: plan runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -165,8 +202,8 @@ jobs: # goes red it is the analyzer's next entry, not noise. # ------------------------------------------------------------------- store-check: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'store-check') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -229,8 +266,8 @@ jobs: # need is the WASI SDK (a manual tarball, cached here). # ------------------------------------------------------------------- build-wasm: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'build-wasm') }} runs-on: ubuntu-latest # The job normally finishes in a few minutes; a hung apt install # once rode the old 45-minute ceiling and stalled the whole @@ -286,14 +323,17 @@ jobs: # # The single `spinel` binary resolves its `spinel_rbs_extract` sidecar # and `lib/` relative to its own directory, so the tree ships as a unit. - # Tracking master rather than pinning a SHA — bump to a pin if upstream - # churn turns flaky. + # Resolve current master once in the planner/caller, then build that SHA. + # A failed lookup falls back to fresh master and cannot save a checkpoint. # ------------------------------------------------------------------- build-spinel: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'build-spinel') }} + outputs: + artifact-id: ${{ steps.toolchain.outputs.artifact-id }} + execution: ${{ steps.result.outputs.status }} # Re-enabled 2026-06-17 (framework-tests-spinel 5/5 + compare-spinel 7/7 - # green again on matz/spinel master). Tracks matz/spinel master UNPINNED, + # green again on matz/spinel master). Tracks moving matz/spinel master, # so it can go red independently of any roundhouse change; continue-on-error # keeps that advisory rather than failing the whole run (the downstream # framework-tests / compare-spinel jobs are likewise advisory). If a build @@ -313,7 +353,7 @@ jobs: - uses: actions/checkout@v5 with: repository: matz/spinel - ref: master + ref: ${{ needs.plan.outputs.spinel-revision }} path: spinel-src - name: Set up sccache id: sccache @@ -395,11 +435,19 @@ jobs: cp -R spinel-src/builtins spinel-dist/builtins git -C spinel-src rev-parse HEAD > spinel-dist/revision.txt - name: Upload spinel toolchain + id: toolchain uses: actions/upload-artifact@v7 with: name: spinel-dist path: spinel-dist retention-days: 1 + - &advisory-result + name: Report actual advisory execution + id: result + if: always() + env: + STATUS: ${{ job.status }} + run: echo "status=$STATUS" >> "$GITHUB_OUTPUT" # ------------------------------------------------------------------- # Library correctness. Runs the default (non-ignored) test suite: @@ -499,8 +547,8 @@ jobs: # NOT from the published archive. Playwright config gates on $CI # (GitHub sets it) → fresh server + forbidOnly + github reporter. browser-smoke-typescript: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'browser-smoke-typescript') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -726,6 +774,9 @@ jobs: framework-tests-spinel: # Re-enabled 2026-06-17 (5/5 green on matz/spinel master). Advisory. needs: build-spinel + if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} + outputs: &advisory-outputs + execution: ${{ steps.result.outputs.status }} runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -768,6 +819,7 @@ jobs: - name: cargo test --test rails_compat_vectors_spinel -- --ignored if: always() run: cargo test --test rails_compat_vectors_spinel -- --ignored --nocapture + - *advisory-result # Campfire conformance — ONCE Campfire's OWN test suite, run against a # roundhouse emit of it. The C4 rung of the coverage ladder, and the @@ -811,8 +863,8 @@ jobs: # walk against the compiled binary and stay advisory, because spinel # is tracked unpinned. campfire-compare: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare') }} runs-on: ubuntu-latest timeout-minutes: 45 services: @@ -899,11 +951,13 @@ jobs: # scenario overlay and must not reuse this binary. build-campfire-compare-spinel: needs: build-spinel + if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true outputs: artifact-id: ${{ steps.binary.outputs.artifact-id }} + execution: ${{ steps.result.outputs.status }} steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup-rust @@ -983,6 +1037,7 @@ jobs: else echo 'No Campfire comparison binary was uploaded. The default, minor-gc and verify-gen comparisons are blocked, not passed; see the producer failure above.' >> "$GITHUB_STEP_SUMMARY" fi + - *advisory-result # campfire-compare-spinel: the same equivalence, with the compiled # BINARY as the emit lane — Rails' broadcast frames vs the binary's. @@ -1034,8 +1089,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true + outputs: + default: ${{ steps.result.outputs.default }} + minor-gc: ${{ steps.result.outputs.minor-gc }} + verify-gen: ${{ steps.result.outputs.verify-gen }} strategy: fail-fast: false + max-parallel: 1 matrix: include: - gc: default @@ -1096,12 +1156,21 @@ jobs: run: gem install websocket-driver --no-document - name: scripts/campfire-compare --spinel ${{ matrix.flag }} run: scripts/campfire-compare --reuse "$PWD/build/campfire-compare-spinel" --spinel ${{ matrix.flag }} /tmp/campfire + - name: Report each GC mode's actual execution + id: result + if: always() + env: + STATUS: ${{ job.status }} + MODE: ${{ matrix.gc }} + run: echo "$MODE=$STATUS" >> "$GITHUB_OUTPUT" # A different app (the model-scenario overlay), so it needs its own # build, not the comparison binary. Start after the toolchain is ready # instead of serializing it behind the shared build and request walk. campfire-db-differential-spinel: needs: build-spinel + if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true @@ -1143,10 +1212,11 @@ jobs: run: scripts/campfire-oracle prepare --app /tmp/campfire - name: scripts/campfire-db-differential --spinel run: scripts/campfire-db-differential --spinel /tmp/campfire + - *advisory-result campfire-conformance: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'campfire-conformance') }} runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -1593,6 +1663,8 @@ jobs: # master UNPINNED, so it surfaces regressions without gating unrelated # work; if master churns red, the signal is visible but non-blocking. needs: [generate-fixture, build-spinel] + if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -1667,6 +1739,7 @@ jobs: name: spinel-toolchain-repro path: ci-spinel-repro retention-days: 7 + - *advisory-result # ------------------------------------------------------------------- # DOM equivalence: boot Rails + the target server side-by-side and @@ -1687,8 +1760,8 @@ jobs: # below: two setup-ruby steps / an extra build-spinel dependency don't # fit this shape. compare: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare') }} runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -1696,9 +1769,10 @@ jobs: actions: read strategy: fail-fast: false + max-parallel: 2 matrix: - target: [rust, crystal, kotlin, swift, csharp, typescript, go, elixir, python] - steps: + target: [rust, typescript] + steps: &compare-steps - uses: actions/checkout@v5 - uses: ./.github/actions/setup-rust - uses: Swatinem/rust-cache@v2 @@ -1829,6 +1903,24 @@ jobs: if: ${{ !cancelled() }} run: scripts/compare ${{ matrix.target }} + # Same commands, distinct result: extra-target failures must not be confused + # with the Rust/TypeScript publication floor. Anchors preserve physical job + # names used by the existing strict PR-local execution receipts. + compare-extra: + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-extra') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + max-parallel: 2 + matrix: + target: ${{ fromJSON(needs.plan.outputs.extra-compare) }} + steps: *compare-steps + # compare-ruby: same shape as every other compare-* — Puma+Rack # server bound to :3000 via ruby_overlay's config.ru, hit over # HTTP. Two ruby setups: the fixture's Gemfile (Rails reference) @@ -1836,8 +1928,8 @@ jobs: # for tailwindcss asset compilation; no target compile step # (Ruby is interpreted). compare-ruby: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-ruby') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -1882,8 +1974,8 @@ jobs: # assets aren't built (the DOM diff compares asset tags, not files — # see scripts/compare's jruby branch). compare-jruby: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-jruby') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -1928,6 +2020,8 @@ jobs: compare-spinel: # Re-enabled 2026-06-17 (7/7 paths match Rails on matz/spinel master). Advisory. needs: [generate-fixture, build-spinel] + if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -1966,6 +2060,7 @@ jobs: node-version: '20' - name: scripts/compare spinel run: scripts/compare spinel + - *advisory-result # ------------------------------------------------------------------- # smoke-: the archive runs its own README. Each published @@ -2005,7 +2100,8 @@ jobs: # ------------------------------------------------------------------- smoke: - needs: build-site + needs: [build-site, plan] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.build-site.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs), 'smoke') }} runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -2013,8 +2109,9 @@ jobs: actions: read strategy: fail-fast: false + max-parallel: 2 matrix: - target: [rust, crystal, kotlin, swift, csharp, typescript, go, elixir, python, ruby, jruby] + target: ${{ fromJSON(needs.plan.outputs.smoke) }} steps: - uses: actions/checkout@v5 - name: Use an explicit browser payload directory @@ -2141,6 +2238,8 @@ jobs: # for the compiler and build-site for the published spinel.tgz. smoke-spinel: needs: [build-site, build-spinel] + if: ${{ !cancelled() && needs.build-site.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 20 continue-on-error: true @@ -2171,6 +2270,7 @@ jobs: "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke spinel run: scripts/smoke --tgz browse/spinel.tgz spinel + - *advisory-result # ------------------------------------------------------------------- # build-campfire-archive: ONCE Campfire for the spinel target, packaged @@ -2205,7 +2305,8 @@ jobs: # the source archive from publishing. # ------------------------------------------------------------------- build-campfire-archive: - needs: [build-spinel] + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-campfire-archive') }} runs-on: ubuntu-latest # 60, not 30: `spin pack` runs spinel's front-end over the whole # campfire tree, and with the ruby-vips package in it the job took @@ -2254,6 +2355,7 @@ jobs: # spinel job. smoke-campfire-docker: needs: [build-campfire-archive] + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -2267,7 +2369,7 @@ jobs: set -euo pipefail if [ ! -f campfire/docker.tgz ]; then echo "::warning::no campfire/docker.tgz this run (spinel toolchain missing?)" - exit 0 + exit 1 fi tar xzf campfire/docker.tgz cd campfire-docker @@ -2286,6 +2388,7 @@ jobs: docker logs campfire docker rm -f campfire >/dev/null [ "$root" = 302 ] && [ "$first" = 200 ] && [ "$logo" = 200 ] + - *advisory-result # smoke-campfire: the campfire archive runs its own README, exactly as # smoke-spinel does for the blog — extract, execute every fenced ```sh @@ -2305,6 +2408,8 @@ jobs: # mean this passed. smoke-campfire: needs: [build-campfire-archive, build-spinel] + if: ${{ !cancelled() && needs.build-campfire-archive.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -2337,6 +2442,7 @@ jobs: "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke campfire run: scripts/smoke --tgz campfire/spinel.tgz campfire + - *advisory-result # ------------------------------------------------------------------- # Build the Pages site: per-target manifests (real-blog fixture) + @@ -2346,25 +2452,22 @@ jobs: # publish *current emit* even when per-target toolchain/compare jobs # fail (broken emit is exactly what's interesting to investigate). # - # So build-site only gates on the fixture and compiler wasm. It does + # So build-site gates on the fixture, plus wasm for full site builds. It does # not gate on unit / toolchain-* / compare-* — a flaky elixir CI run # or a known-broken target shouldn't stall the archive workflow, and # the archive itself is available via the workflow-run artifact even - # when unit fails. Pages publication (the `deploy` job below) does - # gate on unit, so a broken library can't push to the live site. + # when unit fails. Pages publication in full-ci.yml gates on the compact + # floor, so a broken library can't push to the live site. # - # build-site runs on PRs too (no `if: main`). Building the archives is - # harmless off-main and catches archive-generation breakage on the PR; - # PUBLISHING is the boundary, and that lives entirely in `deploy` - # (gated `if: main`). It also uploads the `browse-archives` artifact so - # the e2e- jobs can boot the exact built bytes without - # rebuilding the site. + # Selected PR archive checks run here too, without website/WASM assembly. + # PUBLISHING is the boundary, entirely in the canonical-main full caller. + # The browse-archives artifact lets smokes boot the exact producer bytes. # ------------------------------------------------------------------- build-site: # Publish blog archives before waiting for the independent Campfire # inputs. assemble-site adds those to the same bytes before deployment. - needs: [generate-fixture, build-wasm] - if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-wasm.result == 'success' }} + needs: [generate-fixture, build-wasm, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-site') && needs.generate-fixture.result == 'success' && (needs.plan.outputs.site != 'true' || needs.build-wasm.result == 'success') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -2379,6 +2482,7 @@ jobs: # The compiler wasm, built fresh from source by the parallel build-wasm # job (no committed binary). Lands in _site/lib/ for /playground/+/studio/. - uses: actions/download-artifact@v8 + if: needs.plan.outputs.site == 'true' with: name: roundhouse-wasm path: wasm-artifact @@ -2434,10 +2538,17 @@ jobs: cp fixtures/real-blog/app/javascript/application.js build/assets/ cp fixtures/real-blog/app/javascript/controllers/*.js build/assets/controllers/ echo "baked asset graph:"; (cd build/assets && find . -type f | sort) - - name: roundhouse --site + - name: Build selected archives or the complete site env: ROUNDHOUSE_ASSETS_DIR: build/assets - run: cargo run --release --bin roundhouse -- --site + SITE: ${{ needs.plan.outputs.site }} + TARGETS: ${{ needs.plan.outputs.archives }} + run: | + if [ "$SITE" = true ]; then + cargo run --release --bin roundhouse -- --site + else + cargo run --release --bin roundhouse -- --archives "$TARGETS" + fi # Live SharedWorker blog demo → _site/blog/. Emits real-blog under # the worker profile, vite-builds it with ROUNDHOUSE_BASE set to the @@ -2450,6 +2561,7 @@ jobs: # ROUNDHOUSE_BASE here must match the base hard-coded in # site/404.html. - name: Build live SharedWorker blog demo (→ _site/blog) + if: needs.plan.outputs.site == 'true' env: ROUNDHOUSE_BASE: /roundhouse/blog/ run: | @@ -2475,6 +2587,7 @@ jobs: # and the "Publishing to Pages" section of docs/archive/browser-demo-plan.md. Pure # static files (no npm/vite, no COOP/COEP), so they serve straight off Pages. - name: Bundle the in-browser playground + studio demos (→ _site/lib, _site/playground, _site/studio) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail mkdir -p _site/lib _site/playground _site/studio @@ -2501,13 +2614,14 @@ jobs: # LICENSE + commit embedded — AGPL source redistribution). Same # bundle + cache key the browser-smoke-ide job verified. - name: Cache Mastodon source bundle + if: needs.plan.outputs.site == 'true' id: mastodon-src-site uses: actions/cache@v4 with: path: wasm/ide/app-src.json key: mastodon-app-src-${{ env.MASTODON_SHA }}-v2 - name: Fetch pinned Mastodon + bundle sources (cache miss) - if: steps.mastodon-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.mastodon-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/mastodon/mastodon/tar.gz/${MASTODON_SHA}" -o /tmp/mastodon.tar.gz @@ -2516,6 +2630,7 @@ jobs: --name mastodon --commit "${MASTODON_SHA}" \ --open app/controllers/statuses_controller.rb - name: Bundle the in-browser IDE demo (→ _site/ide) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail mkdir -p _site/ide @@ -2531,13 +2646,14 @@ jobs: # gap-tolerant. Lobsters (ruby-bench, MIT) ships to both; the blog ships # to /ide/ as a source bundle (the playground reuses lib/fixture.json). - name: Cache lobsters source bundle + if: needs.plan.outputs.site == 'true' id: lobsters-src-site uses: actions/cache@v4 with: path: /tmp/app-lobsters.json key: lobsters-app-src-${{ env.RUBY_BENCH_SHA }}-v1 - name: Fetch pinned ruby-bench + bundle lobsters (cache miss) - if: steps.lobsters-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.lobsters-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/ruby/ruby-bench/tar.gz/${RUBY_BENCH_SHA}" -o /tmp/ruby-bench.tar.gz @@ -2547,13 +2663,14 @@ jobs: --name lobsters --commit "${RUBY_BENCH_SHA}" \ --open app/controllers/stories_controller.rb - name: Cache campfire source bundle + if: needs.plan.outputs.site == 'true' id: campfire-src-site uses: actions/cache@v4 with: path: /tmp/app-campfire.json key: campfire-app-src-${{ env.CAMPFIRE_SHA }}-v1 - name: Fetch pinned once-campfire + bundle sources (cache miss) - if: steps.campfire-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.campfire-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/basecamp/once-campfire/tar.gz/${CAMPFIRE_SHA}" -o /tmp/campfire.tar.gz @@ -2562,6 +2679,7 @@ jobs: --name campfire --commit "${CAMPFIRE_SHA}" \ --open app/models/message.rb - name: Assemble app-picker bundles + manifests (→ _site/lib, _site/ide) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail # Blog source bundle for /ide/ (the playground reuses lib/fixture.json). @@ -2602,10 +2720,10 @@ jobs: # `_site/bench/` directory simply isn't created and the landing # page's link 404s until the next CI cycle picks up fresh data. # These historical reports have no PR test consumer. Fetch them only - # for publication; PRs still build the demos and all smoke archives. + # for publication; full PR validation still builds demos and archives. - name: Fetch latest bench data id: fetch-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party run: | @@ -2636,7 +2754,7 @@ jobs: # the bench box doesn't blank the page. - name: Fetch latest lobsters bench data id: fetch-lobsters-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/lobsters run: | @@ -2708,7 +2826,7 @@ jobs: # having one. - name: Fetch latest lobsters conformance data id: fetch-lobsters-specs - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/lobsters-specs run: | @@ -2733,7 +2851,7 @@ jobs: # comparison rather than two unrelated measurements. - name: Fetch latest campfire bench data id: fetch-campfire-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/campfire run: | @@ -2789,7 +2907,7 @@ jobs: # roundbench.showcase.party/campfire-suite/. - name: Fetch latest compiled campfire conformance data id: fetch-campfire-suite-spinel - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/campfire-suite run: | @@ -2815,7 +2933,7 @@ jobs: path: _site/browse/*.tgz - name: Upload Pages artifact - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' uses: actions/upload-pages-artifact@v5 with: name: site-base @@ -2832,8 +2950,10 @@ jobs: # where it is most useful would be exactly backwards. A missing # artifact degrades to a skipped render and a warning, not a broken # site build. - needs: [build-site, campfire-conformance, build-campfire-archive] - if: ${{ !cancelled() && github.ref == 'refs/heads/main' && needs.build-site.result == 'success' }} + needs: [build-site, campfire-conformance, build-campfire-archive, plan] + if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' }} + outputs: + ready: ${{ steps.ready.outputs.ready }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -2912,28 +3032,40 @@ jobs: uses: actions/upload-pages-artifact@v5 with: path: _site + - name: Report successful assembly + id: ready + run: echo 'ready=true' >> "$GITHUB_OUTPUT" - deploy: - needs: - - assemble-site - - unit - # Forks run the full build/test matrix but need no Pages configuration. - # Only the canonical repository publishes the project website. - if: github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' - # Only one Pages deployment at a time. Scoped to this job, not the - # workflow: at the workflow level the same group serialised every - # run — a pull request queued behind a push to main and the next - # push behind that PR, at 20-40 minutes a run — for a lock only the - # deployment needs. - concurrency: - group: pages - cancel-in-progress: false + compact-required: + needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare] + if: always() runs-on: ubuntu-latest - timeout-minutes: 30 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + timeout-minutes: 5 + outputs: + passed: ${{ steps.gate.outputs.passed }} + steps: + - uses: actions/checkout@v5 + - name: Require every selected compact check to succeed + id: gate + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: | + python3 scripts/ci-plan.py compact-gate + echo 'passed=true' >> "$GITHUB_OUTPUT" + + ci-required: + needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare, compact-required, compare-extra, compare-jruby, build-wasm, browser-smoke-ide, build-site, smoke, writebook-inventory, build-spinel, framework-tests-spinel, build-campfire-compare-spinel, campfire-compare-spinel, campfire-db-differential-spinel, toolchain-spinel, compare-spinel, smoke-spinel, build-campfire-archive, smoke-campfire, smoke-campfire-docker, assemble-site] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + complete: ${{ steps.gate.outputs.complete }} steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v5 + - uses: actions/checkout@v5 + - name: Reject missing, skipped or failed required coverage + id: gate + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: python3 scripts/ci-plan.py gate diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml new file mode 100644 index 000000000..b2c0e6d62 --- /dev/null +++ b/.github/workflows/full-ci.yml @@ -0,0 +1,127 @@ +name: Full validation + +on: + schedule: + - cron: '17 */4 * * *' + workflow_dispatch: + inputs: + publish: + description: 'Publish after the compact gate passes (canonical main only)' + type: boolean + default: false + +permissions: + contents: read + +# Keep a started snapshot; replace obsolete pending requests. This literal +# namespace is different from the reusable validation workflow's lock. +concurrency: + group: full-background-${{ github.ref }} + cancel-in-progress: false + +jobs: + preflight: + if: github.event_name != 'schedule' || github.repository == 'rubys/roundhouse' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run: ${{ github.event_name != 'schedule' || github.run_attempt != 1 || steps.lookup.outcome != 'success' || steps.lookup.outputs.cache-hit != 'true' }} + key: ${{ steps.inputs.outputs.key }} + spinel: ${{ steps.inputs.outputs.spinel }} + known: ${{ steps.inputs.outputs.known }} + publish: ${{ steps.inputs.outputs.publish }} + steps: + - name: Resolve the immutable inputs and publication boundary + id: inputs + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + REQUEST_PUBLISH: ${{ inputs.publish || false }} + run: | + publish=false + if [ "$EVENT" = schedule ] || [ "$REQUEST_PUBLISH" = true ]; then + test "$GITHUB_REPOSITORY" = rubys/roundhouse + test "$GITHUB_REF" = refs/heads/main + publish=true + fi + known=false + spinel=$(gh api repos/matz/spinel/commits/master --jq .sha) || spinel=master + if [[ "$spinel" =~ ^[0-9a-f]{40}$ ]]; then known=true; else spinel=master; fi + echo "spinel=$spinel" >> "$GITHUB_OUTPUT" + echo "known=$known" >> "$GITHUB_OUTPUT" + echo "publish=$publish" >> "$GITHUB_OUTPUT" + # Daily refresh also exercises floating Rails/SDK/runner inputs. + echo "key=full-ci-v1-$GITHUB_SHA-$spinel-$(date -u +%F)" >> "$GITHUB_OUTPUT" + - name: Look up a fully executed and published snapshot + id: lookup + if: github.event_name == 'schedule' && github.run_attempt == 1 && steps.inputs.outputs.known == 'true' + continue-on-error: true + uses: actions/cache/restore@v4 + with: + path: .ci-full-completed + key: ${{ steps.inputs.outputs.key }} + lookup-only: true + - name: Report the scheduling decision + env: + HIT: ${{ steps.lookup.outputs.cache-hit }} + LOOKUP: ${{ steps.lookup.outcome }} + SPINEL: ${{ steps.inputs.outputs.spinel }} + EVENT: ${{ github.event_name }} + run: | + printf 'Main/ref snapshot: `%s`\n\nSpinel: `%s`\n\nExact completion marker: `%s` (lookup `%s`). Only scheduled first attempts may skip; dispatches and reruns execute freshly.\n' "$GITHUB_SHA" "$SPINEL" "$HIT" "$LOOKUP" >> "$GITHUB_STEP_SUMMARY" + + validation: + needs: preflight + if: needs.preflight.outputs.run == 'true' + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci.yml + with: + full: true + publish: ${{ needs.preflight.outputs.publish == 'true' }} + spinel_revision: ${{ needs.preflight.outputs.spinel }} + + # No Pages/OIDC permissions are available to the reusable validator or any + # PR. Extra-target failures must not hide useful broken-emit repro archives. + deploy: + needs: [preflight, validation] + if: ${{ !cancelled() && needs.preflight.outputs.publish == 'true' && needs.validation.outputs.publication-ready == 'true' && github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' }} + permissions: + contents: read + pages: write + id-token: write + concurrency: + group: pages + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Refuse to publish a superseded main snapshot + env: + GH_TOKEN: ${{ github.token }} + VALIDATED_SHA: ${{ github.sha }} + run: test "$(gh api repos/rubys/roundhouse/git/ref/heads/main --jq .object.sha)" = "$VALIDATED_SHA" + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5 + + checkpoint: + needs: [preflight, validation, deploy] + if: ${{ !cancelled() && github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' && needs.preflight.outputs.known == 'true' && needs.preflight.outputs.publish == 'true' && needs.validation.result == 'success' && needs.validation.outputs.complete == 'true' && needs.deploy.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Mark only actual complete execution and successful publication + env: + KEY: ${{ needs.preflight.outputs.key }} + run: | + mkdir .ci-full-completed + printf '%s\n' "$KEY" > .ci-full-completed/inputs + - uses: actions/cache/save@v4 + with: + path: .ci-full-completed + key: ${{ needs.preflight.outputs.key }} diff --git a/AGENTS.md b/AGENTS.md index 6673a9cfc..341e9603a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,9 +87,11 @@ defect even if the build is green. you changed. End commit messages with the standard `Co-Authored-By` trailer. - **Outside contributors: fork, and open a pull request against `main`.** - CI runs the full matrix on a PR — every toolchain lane, the DOM compare - against live Rails, the Spinel lanes — so you do not need every - toolchain locally; CI is the oracle for the lanes you cannot run. + CI runs a compact floor plus targeted lanes; add `ci:full` for the + complete PR matrix. Full main validation/publication is scheduled every + four hours, with changed-input checks and a daily refresh. `ci-required` + gates selected blocking coverage; see [CI coverage](docs/ci-reuse.md). + You do not need every toolchain locally; CI covers the missing lanes. Before opening one: `bin/rh fixture` (the test fixtures are generated, not checked in — see below), `cargo test --lib` plus the targeted integration test for what you touched, and a test that pins the fix. @@ -120,5 +122,5 @@ defect even if the build is green. The endpoint is not "does the fixture compile." It is a **per-target ledger of how much of Rails transpiles** — the honest unsupported list, driven down over time. Don't trade that real goal for a locally reachable one. The proving lanes -today are the real-blog fixture (every target, DOM-equivalent to Rails on every -push) and lobsters/Mastodon on the inference + Spinel path (see README). +today are the real-blog fixture (every target, DOM-equivalent to Rails in full +validation) and lobsters/Mastodon on the inference + Spinel path (see README). diff --git a/README.md b/README.md index 5425b689a..cee3b2fcd 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ oracle: the same URL fetched from Rails and from each target must produce the same response — emitted tests, a differential compare against live Rails (DOM node for DOM node, JSON value for value), and browser end-to-end tests for what a static diff can't reach — and -every target on the list passes it on every push. → +the full target matrix runs in scheduled validation, with a compact +floor and targeted additions on pull requests. → [`--target`](docs/guide/transpile.md) · [targets](docs/guide/targets.md) · [what of Rails comes through](docs/guide/rails-coverage.md) · [verifying](docs/guide/verifying.md) @@ -73,7 +74,7 @@ compiled targets it has the closest behavior to Rails by a distance, and will for the foreseeable future, because it runs the framework runtime itself rather than a translation of it. Basecamp's Campfire runs this way — every page and every cable frame compared against -live Rails on every push, and a +live Rails in the full validation cycle, and a [Docker archive](https://rubys.github.io/roundhouse/apps/campfire.html) you can run in minutes. → [Spinel](docs/guide/spinel.md) @@ -168,10 +169,11 @@ its predecessors; [`WHY.md`](WHY.md) is why do it at all. ## Contributing Issues and pull requests are both welcome, and a PR does not need a -conversation first: CI runs the whole matrix on every pull request — -the unit suite, each target's toolchain, the DOM compare against live -Rails, the Spinel lanes — so a change you can only test partially on -your machine is tested fully there. The `unit` job gates; the Spinel +conversation first: CI runs a compact correctness/runtime floor plus +targeted checks. Add `ci:full` for full PR validation; the complete +matrix also runs every four hours on changed main/Spinel inputs, +with a daily refresh. See [CI coverage](docs/ci-reuse.md). The +`ci-required` status gates selected blocking coverage; the Spinel and Campfire jobs marked `continue-on-error` track moving toolchains, and red there is a signal to read, not a check to make pass. Setup, the test cycle, and what a PR should carry are in diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index 9f9bbcdf4..74f911223 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -1,10 +1,80 @@ -# Conservative PR-local CI reuse +# CI coverage and conservative PR-local reuse + +## Compact PR checks and full validation + +`.github/workflows/ci.yml` runs a compact floor on PRs and main pushes: + +- Fixture generation and `unit` (`cargo test --all-targets`, including emitted + Ruby execution tests and the debug-profile bench emission checks). +- Store analysis, Ruby/Rust/TypeScript comparisons against live Rails. +- TypeScript SharedWorker browser tests, Campfire conformance, and Campfire + comparison including its model/database differential. + +These are nine validation executions, plus three small orchestration jobs +(`plan`, `compact-required`, `ci-required`). Drafts select only fixture and +unit validation. **`ci-required` is the stable merge-gate status:** it rejects +missing, skipped, cancelled or failed selected blocking checks. Unselected +jobs may skip. Advisory failures remain visible, not merge blockers. +Repository administrators should migrate required-check settings only after +observing the new status on a real PR; the workflow does not change protection. + +The planner compares the actual PR merge tree with its base, not just the last +commit. Renames/deletions retain both ownership sets. Unknown diff identity +expands to full validation. Documentation-only PRs still get the required +status instead of being left pending by workflow-level path filters. + +| Changed inputs | Additional coverage | +|---|---| +| Target emitter file **or directory**, target runtime, toolchain/framework test | Owning comparison and archive smoke; existing embedded framework/toolchain suites stay intact | +| Ruby emitter or Spinel adapters | Ruby/JRuby/Spinel family, since interpreted targets also consume Spinel-tree files | +| Shared analyzer, lowerer, `runtime/ruby/` | Compact floor; reviewers request full validation for broad/risky changes | +| `wasm/` | WASM build and IDE/playground/studio browser verification | +| Site/guide sources | Site/archive build and WASM verification, without publishing | +| Shared compare or archive/E2E harness | Its owning comparisons or archive smokes | +| CI policy, cross-target packaging, unknown new target | Full validation | + +Add **`ci:full`** to a non-draft PR to request the full matrix for its current +merge tree. Removing the label restores automatic routing. Label/draft/head +transitions cancel superseded PR runs. A manual **Full validation** dispatch +validates the chosen ref freshly; a branch-head dispatch is not a replacement +for the PR merge-tree check. Publication is opt-in for dispatch and is accepted +only on canonical `rubys/roundhouse` main. No deployment privileges are granted +to the shared validator or PR jobs. + +Targeted archive smokes use `roundhouse --archives rust,go` (selected +`browse/.{json,tgz,zip}` outputs) and do not build WASM, demos, website +assets or unrelated archives. The same archive writers power `--site`, which +keeps the complete developer-facing build. Full publication and smoke consume +the same producer bytes, without later re-emission. + +`.github/workflows/full-ci.yml` checks canonical main at **00:17, 04:17, +08:17, 12:17, 16:17 and 20:17 UTC**, bundling full validation and publication. +Its exact completion key includes main SHA, observed Spinel master SHA and UTC +day. Spinel is resolved once and built at that revision. The daily refresh +exercises floating Rails/SDK/runner inputs even without repository changes. +Only an exact successful lookup can skip a scheduled first attempt; lookup +uncertainty, dispatches and reruns execute. Failed/unavailable advisory work, +failed required coverage or failed publication prevents saving the marker, so +the next interval retries. This can mean all six runs execute during upstream +failure; it does not pretend that failure was validated successfully. + +Publication requires the **compact** floor and assembled site, not passing all +extra-target or advisory lanes: broken-emit archives are also upstream repros. +The deployment lock rechecks that canonical main still equals the validated +SHA; lookup failure or a superseded snapshot prevents publication. A new main +commit racing the final check cannot be made atomic with Pages deployment. +Started background runs finish; only the newest pending request is retained. +Extra comparison/smoke matrices use `max-parallel: 2`, GC uses 1. These bound +fanouts, **not** total repository concurrency or a guaranteed PR runner priority. + +## Reuse within selected PR checks `scripts/ci-reuse.py` can reuse **executed, successful** checks from the same pull request. The allowlist is `store-check`, `writebook-inventory`, Rust archive smoke, SharedWorker browser smoke, and the Rust inflector framework suite. Unit tests, fixture generation, artifact producers, DOM comparisons, other -toolchain lanes and all main-branch checks still execute freshly. +selected toolchain lanes and all selected main-branch checks execute freshly. +Routing changes the required coverage, not the execution-receipt trust rules. ## What must match diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py new file mode 100755 index 000000000..84851d0c8 --- /dev/null +++ b/scripts/ci-plan.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Select coverage, not test results. Unknown inputs expand to full validation.""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +TARGETS = [ + "rust", + "crystal", + "kotlin", + "swift", + "csharp", + "typescript", + "go", + "elixir", + "python", + "ruby", + "jruby", +] +BASE = [ + "generate-fixture", + "unit", + "store-check", + "compare", + "compare-ruby", + "browser-smoke-typescript", + "campfire-conformance", + "campfire-compare", +] +SPINEL = [ + "build-spinel", + "framework-tests-spinel", + "build-campfire-compare-spinel", + "campfire-compare-spinel", + "campfire-db-differential-spinel", + "toolchain-spinel", + "compare-spinel", + "smoke-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", +] +ADVISORY = set(SPINEL) - {"build-campfire-archive"} +SHA = re.compile(r"[0-9a-f]{40}\Z") + + +def select(paths, *, draft=False, full=False, publish=False): + if draft: + return finish( + BASE[:2], [], [], False, False, False, ["draft: fixture and unit only"] + ) + targets, smoke = set(), set() + wasm = site = spinel = writebook = False + reasons = [] + for path in paths: + if path.startswith(".github/") or path in { + "scripts/ci-plan.py", + "tests/ci_plan_test.py", + "tests/workflow_yaml_parses.rs", + "src/project.rs", + "src/bin/roundhouse.rs", + }: + full = True + reasons.append(f"{path}: validation/packaging policy") + match = re.match(r"(?:src/emit/|runtime/)([^/.]+)(?:[/.]|$)", path) + test = re.match( + r"tests/(?:framework_tests_)?([a-z]+)_toolchain\.rs$|tests/framework_tests_([a-z]+)\.rs$", + path, + ) + target = ( + match[1] + if match + else next((v for v in test.groups() if v), None) + if test + else None + ) + if target in TARGETS or target == "spinel": + owners = ( + {"ruby", "jruby", "spinel"} + if target in {"ruby", "spinel"} and not path.startswith("runtime/ruby/") + else {target} + ) + if path.startswith("runtime/ruby/"): + owners = ( + set() + ) # Shared runtime: the chosen compact floor, not all targets. + targets.update(owners - {"spinel"}) + smoke.update(owners - {"spinel"}) + spinel |= "spinel" in owners + if owners: + reasons.append(f"{path}: {', '.join(sorted(owners))}") + elif target and target not in { + "ruby", + "ruby_family", + "roda", + "mod", + "shared", + "rails", + }: + full = True + reasons.append(f"{path}: unknown target ownership") + if path.startswith("wasm/"): + wasm = True + reasons.append(f"{path}: WASM/browser compiler") + if path.startswith(("site/", "docs/guide/")): + site = wasm = True + if path.startswith("e2e/") or path in { + "scripts/smoke", + "scripts/ci-playwright-install", + "scripts/create-blog", + "scripts/create-store", + "bin/rh", + }: + smoke.update(TARGETS) + spinel = True + if ( + path.startswith(("tools/compare/", "tests/framework_test_support")) + or path == "scripts/compare" + ): + targets.update(TARGETS) + spinel = True + if "spinel" in path and path.startswith(("tests/", "scripts/")): + spinel = True + if path.startswith( + ("scripts/campfire-", "scripts/build-campfire", "e2e/campfire/") + ): + spinel = True + if path in {"tests/writebook.rs", "tests/fixtures/writebook-inventory.json"}: + writebook = True + if full: + targets.update(TARGETS) + smoke.update(TARGETS) + wasm = site = spinel = writebook = True + reasons.append("full validation requested") + if site or spinel: + smoke.add("spinel") if spinel else None + jobs = list(BASE) + extra = [ + t + for t in TARGETS + if t in targets and t not in {"rust", "typescript", "ruby", "jruby"} + ] + if extra: + jobs.append("compare-extra") + if "jruby" in targets: + jobs.append("compare-jruby") + if wasm: + jobs.extend(["build-wasm", "browser-smoke-ide"]) + if smoke or site: + jobs.append("build-site") + if smoke - {"spinel"}: + jobs.append("smoke") + if spinel: + jobs.extend(SPINEL) + if writebook: + jobs.append("writebook-inventory") + if publish: + if not full: + raise ValueError("publication requires full validation mode") + jobs.append("assemble-site") + return finish( + jobs, + extra, + [t for t in TARGETS if t in smoke], + wasm, + site, + spinel, + reasons, + publish, + ) + + +def finish(jobs, extra, smoke, wasm, site, spinel, reasons, publish=False): + archives = ( + ["blog", "spinel", *TARGETS, "typescript-worker"] + if site + else [*smoke, *(["spinel"] if spinel else [])] + ) + return { + "jobs": jobs, + "required": [j for j in jobs if j not in ADVISORY], + "extra_compare": extra, + "smoke": smoke, + "archives": archives, + "wasm": wasm, + "site": site, + "spinel": spinel, + "publish": publish, + "reasons": reasons, + } + + +def git(*args): + return subprocess.check_output(["git", *args]) + + +def changed_paths(event, event_name, sha): + if not SHA.fullmatch(sha) or git("rev-parse", "HEAD").decode().strip() != sha: + raise ValueError("checkout is not the event SHA") + if event_name == "pull_request": + pr = event["pull_request"] + parents = git("show", "-s", "--format=%P", "HEAD").decode().split() + if parents != [pr["base"]["sha"], pr["head"]["sha"]]: + raise ValueError("checkout is not the event's PR merge tree") + base = parents[0] + elif event_name == "push": + base = event["before"] + if not SHA.fullmatch(base) or base == "0" * 40: + raise ValueError("no previous main tree") + try: + git("cat-file", "-e", base) + except subprocess.CalledProcessError: + git("fetch", "--no-tags", "--depth=1", "origin", base) + else: + return [] + # Renames become a deletion and addition; both ownership sets are selected. + return [ + p.decode("utf-8") + for p in git("diff", "--name-only", "--no-renames", "-z", base, sha).split( + b"\0" + ) + if p + ] + + +def check_results(plan, needs, *, compact=False): + required = ( + BASE[:2] if plan["jobs"] == BASE[:2] else BASE if compact else plan["required"] + ) + failures = [ + f"{j}: {needs.get(j, {}).get('result', 'missing')}" + for j in required + if needs.get(j, {}).get("result") != "success" + ] + if needs.get("plan", {}).get("result") != "success": + failures.append("plan: no successful routing decision") + if not compact and needs.get("compact-required", {}).get("result") != "success": + failures.append("compact-required: no successful baseline gate") + # Advisory work never blocks the gate, but failed/unavailable work is not + # a successful full-execution checkpoint for the next scheduled interval. + complete = not failures and all( + needs.get(j, {}).get("result") == "success" + and ( + j not in ADVISORY + or all( + needs[j].get("outputs", {}).get(key) == "success" + for key in ( + ["default", "minor-gc", "verify-gen"] + if j == "campfire-compare-spinel" + else ["execution"] + ) + ) + ) + for j in plan["jobs"] + ) + return failures, complete + + +def write_outputs(values): + output = os.environ.get("GITHUB_OUTPUT") + lines = "".join( + f"{key}={json.dumps(value, separators=(',', ':')) if not isinstance(value, str) else value}\n" + for key, value in values.items() + ) + if output: + with open(output, "a") as f: + f.write(lines) + print(lines, end="") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["plan", "gate", "compact-gate"]) + args = parser.parse_args() + if args.command != "plan": + plan = json.loads(os.environ["CI_PLAN"]) + failures, complete = check_results( + plan, + json.loads(os.environ["CI_NEEDS"]), + compact=args.command == "compact-gate", + ) + write_outputs({"complete": complete}) + for failure in failures: + print(f"::error::{failure}") + return bool(failures) + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + event_name = os.environ["GITHUB_EVENT_NAME"] + pr = event.get("pull_request", {}) + full = os.environ.get("CI_FULL") == "true" or any( + label["name"] == "ci:full" for label in pr.get("labels", []) + ) + reason = None + try: + paths = changed_paths(event, event_name, os.environ["GITHUB_SHA"]) + except (KeyError, ValueError, UnicodeError, subprocess.CalledProcessError) as e: + paths, full, reason = ( + [], + True, + f"Unknown changed inputs: {e}; running full validation", + ) + publish = os.environ.get("CI_PUBLISH") == "true" + if publish and ( + os.environ["GITHUB_REPOSITORY"] != "rubys/roundhouse" + or os.environ["GITHUB_REF"] != "refs/heads/main" + or event_name not in {"schedule", "workflow_dispatch"} + ): + raise ValueError("publication is only allowed by canonical main's full caller") + plan = select(paths, draft=pr.get("draft", False), full=full, publish=publish) + if reason: + plan["reasons"].append(reason) + spinel = os.environ.get("CI_SPINEL_REVISION", "") + if plan["spinel"] and not spinel: + try: + spinel = subprocess.check_output( + ["gh", "api", "repos/matz/spinel/commits/master", "--jq", ".sha"], + text=True, + ).strip() + except subprocess.CalledProcessError: + spinel = "master" + plan["reasons"].append( + "Spinel lookup unavailable: fresh master build, no scheduler checkpoint" + ) + if spinel and spinel != "master" and not SHA.fullmatch(spinel): + raise ValueError("invalid Spinel revision") + write_outputs( + { + "plan": plan, + "jobs": plan["jobs"], + "extra-compare": plan["extra_compare"], + "smoke": plan["smoke"], + "archives": ",".join(plan["archives"]), + "wasm": plan["wasm"], + "site": plan["site"], + "publish": plan["publish"], + "spinel-revision": spinel, + } + ) + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + Path(summary).write_text( + "## Selected CI coverage\n\n```json\n" + + json.dumps(plan, indent=2) + + "\n```\n" + ) + return False + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bin/roundhouse.rs b/src/bin/roundhouse.rs index 63639e066..a94cbc95c 100644 --- a/src/bin/roundhouse.rs +++ b/src/bin/roundhouse.rs @@ -37,6 +37,7 @@ fn usage() -> &'static str { "\ Usage: roundhouse --target LANG [INPUT] [-o OUT] roundhouse --site [INPUT] [-o OUT] + roundhouse --archives LANG,LANG [INPUT] [-o OUT] roundhouse check [--continue] [APP] roundhouse lsp roundhouse mcp [APP] @@ -60,6 +61,8 @@ Options: Default INPUT=. Default OUT=./out// --site Build all targets + landing-page assets. Default INPUT=fixtures/real-blog Default OUT=./_site/ + --archives LIST Build only comma-separated targets' browse archives. + Same defaults as --site; no website or browser demos. -o, --output PATH Output directory. --allow-unsupported Don't fail on unsupported-construct gaps: emit a @@ -127,6 +130,13 @@ fn cli() -> ExitCode { ExitCode::FAILURE } }, + Ok(Action::Archives { targets, input, out }) => match project::build_archives(&input, &out, &targets) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("roundhouse: {e}"); + ExitCode::FAILURE + } + }, Err(e) => { eprintln!("roundhouse: {e}"); eprintln!(); @@ -150,11 +160,17 @@ enum Action { input: PathBuf, out: PathBuf, }, + Archives { + targets: Vec, + input: PathBuf, + out: PathBuf, + }, } fn parse_args(args: Vec) -> Result { let mut target: Option = None; let mut site = false; + let mut archives = None; let mut out: Option = None; let mut allow_unsupported = false; let mut survey = false; @@ -166,6 +182,15 @@ fn parse_args(args: Vec) -> Result { "-h" | "--help" => return Ok(Action::Help), "-V" | "--version" => return Ok(Action::Version), "--site" => site = true, + "--archives" => { + let value = iter.next().ok_or("--archives requires a target list")?; + let targets = value.split(',').map(|name| { + BuildTarget::from_str(name) + .filter(|t| BuildTarget::ALL.contains(t)) + .ok_or_else(|| format!("unknown archive target '{name}'")) + }).collect::, _>>()?; + archives = Some(targets); + } "--allow-unsupported" => allow_unsupported = true, "--survey" => survey = true, "-t" | "--target" => { @@ -201,6 +226,16 @@ fn parse_args(args: Vec) -> Result { )); } + if let Some(targets) = archives { + if target.is_some() || site || allow_unsupported || survey { + return Err("--archives cannot be combined with --target, --site, --survey or --allow-unsupported".into()); + } + return Ok(Action::Archives { + targets, + input: positional.pop().map(PathBuf::from).unwrap_or_else(|| PathBuf::from("fixtures/real-blog")), + out: out.unwrap_or_else(|| PathBuf::from("_site")), + }); + } match (target, site) { (Some(_), true) => Err("--target and --site are mutually exclusive".into()), (None, false) => Err("one of --target LANG or --site is required".into()), diff --git a/src/project.rs b/src/project.rs index a35348265..47ecac84c 100644 --- a/src/project.rs +++ b/src/project.rs @@ -6613,10 +6613,23 @@ pub fn build_site(fixture: &Path, out: &Path) -> Result<(), String> { fs::create_dir_all(out.join("browse")) .map_err(|e| format!("mkdir {}: {e}", out.display()))?; + // Archive generation cleans browse/, so copy its viewer assets afterwards. + build_archives(fixture, out, BuildTarget::ALL)?; copy_site_assets(out)?; copy_create_blog(out)?; crate::guide::render_site(out)?; + Ok(()) +} + +/// Build selected browse archives without website assets or WASM demos. +/// The file sets and archive writers are shared with `build_site`. +pub fn build_archives(fixture: &Path, out: &Path, targets: &[BuildTarget]) -> Result<(), String> { + let browse = out.join("browse"); + if browse.exists() { + fs::remove_dir_all(&browse).map_err(|e| format!("clean {}: {e}", browse.display()))?; + } + fs::create_dir_all(&browse).map_err(|e| format!("mkdir {}: {e}", browse.display()))?; let mut app = ingest_app(fixture).map_err(|e| format!("ingest {}: {e}", fixture.display()))?; // Analyze + the same post-analyze shared lowerings as the @@ -6624,7 +6637,7 @@ pub fn build_site(fixture: &Path, out: &Path) -> Result<(), String> { // so the residue is dropped. let _ = crate::session::analyze_and_lower(&mut app); - for target in BuildTarget::ALL { + for target in targets { let files = target_files(&app, fixture, *target)?; let name = target.as_str(); diff --git a/tests/ci_plan_test.py b/tests/ci_plan_test.py new file mode 100644 index 000000000..d1144af50 --- /dev/null +++ b/tests/ci_plan_test.py @@ -0,0 +1,228 @@ +"""Routing boundaries and false-green checks, without GitHub or toolchains.""" + +import importlib.util +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location( + "ci_plan", Path(__file__).parents[1] / "scripts/ci-plan.py" +) +ci = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ci) + + +class Routing(unittest.TestCase): + def test_shared_compiler_and_runtime_have_only_the_nine_execution_floor(self): + plan = ci.select( + [ + "src/analyze/call.rs", + "src/lower/rails.rs", + "runtime/ruby/relation.rb", + "tests/new_regression.rs", + ] + ) + self.assertEqual(plan["jobs"], ci.BASE) + self.assertEqual(plan["archives"], []) + self.assertFalse(plan["wasm"]) + + def test_draft_overrides_full_and_target_expansion(self): + plan = ci.select( + ["src/emit/go/expressions.rs", ".github/workflows/ci.yml"], + draft=True, + full=True, + ) + self.assertEqual(plan["required"], ["generate-fixture", "unit"]) + + def test_target_partial_does_not_pull_in_wasm_or_other_archives(self): + plan = ci.select(["src/emit/go/expressions.rs"]) + self.assertEqual(plan["extra_compare"], ["go"]) + self.assertEqual(plan["archives"], ["go"]) + self.assertEqual(plan["smoke"], ["go"]) + self.assertFalse(plan["site"]) + self.assertFalse(plan["wasm"]) + self.assertNotIn("build-spinel", plan["jobs"]) + + def test_baseline_emitter_adds_its_archive_not_duplicate_compare(self): + plan = ci.select(["src/emit/rust.rs"]) + self.assertEqual(plan["extra_compare"], []) + self.assertEqual(plan["archives"], ["rust"]) + self.assertNotIn("compare-extra", plan["jobs"]) + + def test_framework_and_toolchain_tests_belong_to_compare(self): + for path in ["tests/swift_toolchain.rs", "tests/framework_tests_swift.rs"]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual(plan["extra_compare"], ["swift"]) + self.assertEqual(plan["smoke"], ["swift"]) + + def test_ruby_emit_and_spinel_adapters_cover_consuming_family(self): + for path in ["src/emit/ruby.rs", "runtime/spinel/db_jruby.rb"]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertIn("compare-jruby", plan["jobs"]) + self.assertTrue(plan["spinel"]) + self.assertEqual(plan["smoke"], ["ruby", "jruby"]) + + def test_wasm_changes_have_no_archive_or_spinel_fanout(self): + plan = ci.select(["wasm/lib/driver.mjs"]) + self.assertTrue(plan["wasm"]) + self.assertIn("browser-smoke-ide", plan["required"]) + self.assertNotIn("build-site", plan["jobs"]) + + def test_packaging_and_unknown_target_changes_expand_to_full(self): + for path in [ + "src/project.rs", + "src/emit/newlang.rs", + "tests/framework_tests_newlang.rs", + "scripts/ci-plan.py", + ]: + with self.subTest(path=path): + self.assertEqual(ci.select([path])["smoke"], ci.TARGETS) + + def test_full_manual_and_publication_are_distinct(self): + plan = ci.select([], full=True) + self.assertIn("writebook-inventory", plan["required"]) + self.assertNotIn("deploy", plan["jobs"]) + self.assertIn( + "assemble-site", ci.select([], full=True, publish=True)["required"] + ) + with self.assertRaises(ValueError): + ci.select([], publish=True) + + def test_shared_smoke_and_compare_harnesses_select_their_owners(self): + plan = ci.select(["scripts/smoke"]) + self.assertEqual(plan["smoke"], ci.TARGETS) + self.assertTrue(plan["spinel"]) + self.assertEqual(plan["extra_compare"], []) + self.assertEqual( + ci.select(["tools/compare/src/main.rs"])["extra_compare"], + ["crystal", "kotlin", "swift", "csharp", "go", "elixir", "python"], + ) + + +class Results(unittest.TestCase): + def needs(self, plan): + return { + "plan": {"result": "success"}, + "compact-required": {"result": "success"}, + **{ + j: { + "result": "success", + "outputs": { + "execution": "success", + "default": "success", + "minor-gc": "success", + "verify-gen": "success", + }, + } + for j in plan["jobs"] + }, + } + + def test_selected_skips_and_missing_results_are_not_green(self): + plan = ci.select(["src/emit/go.rs"]) + for outcome in ["skipped", "cancelled", "failure", None]: + with self.subTest(outcome=outcome): + needs = self.needs(plan) + needs["compare-extra"] = {"result": outcome} + failures, complete = ci.check_results(plan, needs) + self.assertTrue(failures) + self.assertFalse(complete) + + def test_extra_failure_does_not_block_compact_publication_floor(self): + plan = ci.select([], full=True, publish=True) + needs = self.needs(plan) + needs["compare-extra"]["result"] = "failure" + self.assertFalse(ci.check_results(plan, needs, compact=True)[0]) + self.assertTrue(ci.check_results(plan, needs)[0]) + needs["compare"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs, compact=True)[0]) + + def test_advisory_failure_is_visible_but_does_not_fail_required_gate(self): + plan = ci.select([], full=True) + needs = self.needs(plan) + needs["build-spinel"]["outputs"]["execution"] = "failure" + failures, complete = ci.check_results(plan, needs) + self.assertEqual(failures, []) + self.assertFalse(complete) + needs["smoke-spinel"]["result"] = "skipped" + self.assertFalse(ci.check_results(plan, needs)[1]) + + def test_assembly_failure_cannot_issue_checkpoint(self): + plan = ci.select([], full=True, publish=True) + needs = self.needs(plan) + needs["assemble-site"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs)[0]) + self.assertFalse(ci.check_results(plan, needs)[1]) + + def test_each_gc_mode_must_actually_pass_for_completion(self): + plan = ci.select([], full=True) + self.assertEqual(ci.check_results(plan, self.needs(plan)), ([], True)) + for mode in ["default", "minor-gc", "verify-gen"]: + needs = self.needs(plan) + needs["campfire-compare-spinel"]["outputs"][mode] = "failure" + self.assertEqual(ci.check_results(plan, needs), ([], False)) + + def test_unselected_jobs_may_skip_but_planner_must_succeed(self): + plan = ci.select([]) + needs = self.needs(plan) + needs["build-wasm"] = {"result": "skipped"} + self.assertEqual(ci.check_results(plan, needs), ([], True)) + needs["plan"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs)[0]) + + +class MergeTree(unittest.TestCase): + def test_diff_tracks_both_rename_owners_and_deletions(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + + def git(*args): + return ( + subprocess.check_output( + ["git", "-C", directory, *args], stderr=subprocess.DEVNULL + ) + .decode() + .strip() + ) + + git("init") + git("config", "user.name", "CI test") + git("config", "user.email", "test@example.invalid") + (root / "src/emit").mkdir(parents=True) + (root / "src/emit/go.rs").write_text("old owner\n") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("switch", "-c", "feature") + (root / "src/emit/go.rs").rename(root / "src/emit/swift.rs") + git("add", ".") + git("commit", "-m", "rename") + head = git("rev-parse", "HEAD") + git("switch", "-") + git("merge", "--no-ff", "feature", "-m", "merge") + sha = git("rev-parse", "HEAD") + event = {"pull_request": {"base": {"sha": base}, "head": {"sha": head}}} + previous = os.getcwd() + try: + os.chdir(root) + paths = ci.changed_paths(event, "pull_request", sha) + self.assertEqual(set(paths), {"src/emit/go.rs", "src/emit/swift.rs"}) + self.assertEqual(ci.select(paths)["extra_compare"], ["swift", "go"]) + with self.assertRaises(ValueError): + ci.changed_paths(event, "pull_request", head) + with ( + patch.dict(event["pull_request"]["base"], sha=head), + self.assertRaises(ValueError), + ): + ci.changed_paths(event, "pull_request", sha) + finally: + os.chdir(previous) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/selected_archives.rs b/tests/selected_archives.rs new file mode 100644 index 000000000..ee2d97bf1 --- /dev/null +++ b/tests/selected_archives.rs @@ -0,0 +1,100 @@ +//! A targeted smoke consumes the same bytes as full publication, without +//! requiring website assets, WASM or unselected archives. + +use std::fs; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn selected_cli_archives_match_full_site_bytes_and_clear_stale_targets() { + roundhouse::stack::run(|| { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = + std::env::temp_dir().join(format!("selected-archives-{}-{unique}", std::process::id())); + let selected = root.join("selected"); + let full = root.join("full"); + fs::create_dir_all(selected.join("browse")).unwrap(); + fs::write(selected.join("browse/go.tgz"), "stale, unselected bytes").unwrap(); + fs::write(selected.join("keep.txt"), "unrelated output").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_roundhouse")) + .args(["--archives", "ruby,rust"]) + .arg(roundhouse::fixtures::real_blog()) + .arg("-o") + .arg(&selected) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut names: Vec<_> = fs::read_dir(selected.join("browse")) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + assert_eq!( + names, + [ + "ruby.json", + "ruby.tgz", + "ruby.zip", + "rust.json", + "rust.tgz", + "rust.zip" + ] + ); + assert_eq!( + fs::read_to_string(selected.join("keep.txt")).unwrap(), + "unrelated output" + ); + assert!(!selected.join("index.html").exists()); + assert!(!selected.join("lib").exists()); + + roundhouse::project::build_site(roundhouse::fixtures::real_blog(), &full).unwrap(); + assert!(full.join("index.html").is_file()); + assert!(full.join("browse/index.html").is_file()); + assert!(full.join("browse/app.js").is_file()); + for name in names { + assert_eq!( + fs::read(selected.join("browse").join(&name)).unwrap(), + fs::read(full.join("browse").join(&name)).unwrap(), + "archive drift: {name}" + ); + } + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(selected.join("browse/rust.json")).unwrap()).unwrap(); + let files = manifest["files"].as_array().unwrap(); + for required in ["README.md", "e2e/package.json", "e2e/playwright.config.js"] { + assert!( + files.iter().any(|f| f["path"].as_str() == Some(required)), + "missing archive contract: {required}" + ); + } + fs::remove_dir_all(root).unwrap(); + }); +} + +#[test] +fn archive_cli_rejects_ambiguous_modes_and_invalid_lists() { + for args in [ + vec!["--archives"], + vec!["--archives", ""], + vec!["--archives", "rust,unknown"], + vec!["--archives", "roda"], + vec!["--archives", "rust", "--site"], + vec!["--archives", "rust", "--target", "go"], + vec!["--archives", "rust", "--allow-unsupported"], + ] { + let output = Command::new(env!("CARGO_BIN_EXE_roundhouse")) + .args(&args) + .output() + .unwrap(); + assert!(!output.status.success(), "accepted {args:?}"); + assert!(String::from_utf8_lossy(&output.stderr).contains("roundhouse:")); + } +} diff --git a/tests/workflow_yaml_parses.rs b/tests/workflow_yaml_parses.rs index b3674d27b..614720323 100644 --- a/tests/workflow_yaml_parses.rs +++ b/tests/workflow_yaml_parses.rs @@ -310,10 +310,10 @@ fn pr_archives_remain_tested_without_pages_publication_work() { assert_eq!( jobs["build-site"]["if"].as_str(), Some( - "${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-wasm.result == 'success' }}" + "${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-site') && needs.generate-fixture.result == 'success' && (needs.plan.outputs.site != 'true' || needs.build-wasm.result == 'success') }}" ) ); - assert_eq!(jobs["smoke"]["needs"].as_str(), Some("build-site")); + assert_eq!(jobs["smoke"]["needs"][0].as_str(), Some("build-site")); let steps = jobs["build-site"]["steps"].as_sequence().unwrap(); for (id, output, renderer) in [ ("fetch-bench", "bench_data", "Render bench page"), @@ -344,7 +344,7 @@ fn pr_archives_remain_tested_without_pages_publication_work() { .unwrap(); assert_eq!( fetch["if"].as_str(), - Some("github.ref == 'refs/heads/main'") + Some("needs.plan.outputs.publish == 'true'") ); let render = steps .iter() @@ -368,23 +368,17 @@ fn pr_archives_remain_tested_without_pages_publication_work() { .unwrap(); assert_eq!( pages["if"].as_str(), - Some("github.ref == 'refs/heads/main'") + Some("needs.plan.outputs.publish == 'true'") ); // Keep the status function: a failed Campfire floor must not suppress // its explanatory publication, but cancellation or a bad site must. assert_eq!( jobs["assemble-site"]["if"].as_str(), Some( - "${{ !cancelled() && github.ref == 'refs/heads/main' && needs.build-site.result == 'success' }}" + "${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' }}" ) ); - assert_eq!( - jobs["deploy"]["if"].as_str(), - Some("github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main'") - ); - assert!(jobs["deploy"].get("continue-on-error").is_none(), "production deployment failures must remain visible"); - assert_eq!(jobs["deploy"]["needs"][0].as_str(), Some("assemble-site")); - assert_eq!(jobs["deploy"]["needs"][1].as_str(), Some("unit")); + assert!(jobs.get("deploy").is_none(), "PR validation must not carry deployment privileges"); } #[test] @@ -392,13 +386,13 @@ fn draft_transitions_replace_the_previous_pr_run() { let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); let events = ci["on"]["pull_request"]["types"].as_sequence().unwrap(); - for event in ["ready_for_review", "converted_to_draft"] { + for event in ["ready_for_review", "converted_to_draft", "labeled", "unlabeled"] { assert!(events.iter().any(|value| value.as_str() == Some(event))); } assert_eq!( ci["concurrency"]["group"].as_str(), Some( - "${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}" + "validation-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}" ) ); assert_eq!( @@ -809,7 +803,7 @@ fn pr_reuse_never_masks_validation_failures_or_changes_the_job_graph() { &["build", "check"] } "writebook-inventory" => { - assert_eq!(job["needs"].as_str(), Some("unit")); + assert_eq!(job["needs"][0].as_str(), Some("unit")); &["inventory", "report"] } "browser-smoke-typescript" => { @@ -818,7 +812,7 @@ fn pr_reuse_never_masks_validation_failures_or_changes_the_job_graph() { &["browser"] } "smoke" => { - assert_eq!(job["needs"].as_str(), Some("build-site")); + assert_eq!(job["needs"][0].as_str(), Some("build-site")); assert_eq!( probe["if"].as_str(), Some("github.event_name == 'pull_request' && matrix.target == 'rust'") @@ -987,3 +981,139 @@ fn reused_checks_keep_cargo_dependencies_locked_and_upload_only_execution_receip assert_eq!(uploads[0]["with"]["retention-days"].as_u64(), Some(7)); } } + +#[test] +fn routing_and_required_results_reject_false_green() { + let result = std::process::Command::new("python3") + .args(["-B", "tests/ci_plan_test.py", "-v"]) + .output() + .expect("CI routing tests require python3"); + assert!(result.status.success(), "{}\n{}", + String::from_utf8_lossy(&result.stdout), String::from_utf8_lossy(&result.stderr)); +} + +#[test] +fn compact_and_extra_compare_share_commands_but_not_results() { + let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str( + &fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + assert_eq!(jobs["compare"]["strategy"]["matrix"]["target"], + serde_yaml_ng::from_str::("[rust, typescript]").unwrap()); + assert_eq!(jobs["compare"]["steps"], jobs["compare-extra"]["steps"]); + assert_eq!(jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), Some(2)); + assert_eq!(jobs["smoke"]["strategy"]["max-parallel"].as_u64(), Some(2)); + let smoke_guard = jobs["smoke"]["if"].as_str().unwrap(); + for condition in ["!cancelled()", "needs.plan.result == 'success'", "needs.build-site.result == 'success'"] { + assert!(smoke_guard.contains(condition), "selected smoke must run after its skipped WASM ancestor: {condition}"); + } + assert_eq!(jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), Some(1)); + assert!(ci["on"]["pull_request"].get("paths-ignore").is_none(), + "required status must run even for documentation-only PRs"); + for name in ["compact-required", "ci-required"] { + assert_eq!(jobs[name]["if"].as_str(), Some("always()")); + } + let gate = jobs["ci-required"]["needs"].as_sequence().unwrap(); + for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { + if name != "ci-required" { + assert!(gate.iter().any(|v| v.as_str() == Some(name)), "missing result: {name}"); + } + } + assert_eq!(ci["permissions"]["contents"].as_str(), Some("read")); + assert!(ci["permissions"].get("pages").is_none()); + assert!(ci["permissions"].get("id-token").is_none()); +} + +#[test] +fn full_scheduler_only_skips_completed_inputs_and_never_grants_pr_deploy_permissions() { + let full: serde_yaml_ng::Value = serde_yaml_ng::from_str( + &fs::read_to_string(".github/workflows/full-ci.yml").unwrap()).unwrap(); + assert_eq!(full["on"]["schedule"][0]["cron"].as_str(), Some("17 */4 * * *")); + assert!(full["on"].get("push").is_none()); + assert!(full["on"].get("pull_request").is_none()); + assert_eq!(full["concurrency"]["cancel-in-progress"].as_bool(), Some(false)); + let jobs = &full["jobs"]; + let preflight = &jobs["preflight"]; + let lookup = preflight["steps"].as_sequence().unwrap().iter() + .find(|s| s["id"].as_str() == Some("lookup")).unwrap(); + assert_eq!(lookup["uses"].as_str(), Some("actions/cache/restore@v4")); + assert_eq!(lookup["with"]["lookup-only"].as_bool(), Some(true)); + assert!(lookup["with"].get("restore-keys").is_none()); + let skip = preflight["outputs"]["run"].as_str().unwrap(); + for condition in ["github.event_name != 'schedule'", "github.run_attempt != 1", + "steps.lookup.outcome != 'success'", "steps.lookup.outputs.cache-hit != 'true'"] { + assert!(skip.contains(condition), "skip must fail open: {condition}"); + } + let checkpoint = jobs["checkpoint"]["if"].as_str().unwrap(); + for condition in ["!cancelled()", "github.repository == 'rubys/roundhouse'", + "github.ref == 'refs/heads/main'", "needs.preflight.outputs.known == 'true'", + "needs.validation.result == 'success'", "needs.validation.outputs.complete == 'true'", + "needs.deploy.result == 'success'"] { + assert!(checkpoint.contains(condition), "unsafe completion marker: {condition}"); + } + assert_eq!(jobs["validation"]["uses"].as_str(), Some("./.github/workflows/ci.yml")); + assert_eq!(jobs["validation"]["with"]["full"].as_bool(), Some(true)); + assert_eq!(jobs["validation"]["permissions"]["contents"].as_str(), Some("read")); + assert_eq!(jobs["validation"]["permissions"]["actions"].as_str(), Some("read")); + assert_eq!(jobs["validation"]["permissions"].as_mapping().unwrap().len(), 2); + assert_eq!(full["permissions"]["contents"].as_str(), Some("read")); + let deploy = &jobs["deploy"]; + let guard = deploy["if"].as_str().unwrap(); + assert!(guard.contains("needs.validation.outputs.publication-ready == 'true'")); + assert!(!guard.contains("needs.validation.result == 'success'"), "extra failures cannot hide repro publication"); + assert!(deploy.get("continue-on-error").is_none()); + assert_eq!(deploy["permissions"]["pages"].as_str(), Some("write")); + assert!(deploy["steps"][0]["run"].as_str().unwrap().contains("$VALIDATED_SHA")); +} + +#[cfg(unix)] +#[test] +fn scheduler_preflight_executes_publication_guards_and_unknown_input_fallback() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let full: serde_yaml_ng::Value = serde_yaml_ng::from_str( + &fs::read_to_string(".github/workflows/full-ci.yml").unwrap()).unwrap(); + let body = full["jobs"]["preflight"]["steps"][0]["run"].as_str().unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let root = std::env::temp_dir().join(format!("full-preflight-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + for (name, script) in [ + ("gh", "#!/bin/sh\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n"), + ("date", "#!/bin/sh\nprintf '2026-10-02\\n'\n"), + ] { + let path = root.join(name); + fs::write(&path, script).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + let main_sha = "1234567890123456789012345678901234567890"; + let spinel_sha = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + for (event, repo, reference, publish, revision, expected) in [ + ("schedule", "rubys/roundhouse", "refs/heads/main", "false", spinel_sha, Some(("true", "true", spinel_sha))), + ("schedule", "rubys/roundhouse", "refs/heads/main", "false", "unavailable", Some(("true", "false", "master"))), + ("schedule", "rubys/roundhouse", "refs/heads/main", "false", "malformed", Some(("true", "false", "master"))), + ("workflow_dispatch", "contributor/roundhouse", "refs/heads/topic", "false", spinel_sha, Some(("false", "true", spinel_sha))), + ("workflow_dispatch", "rubys/roundhouse", "refs/heads/main", "true", spinel_sha, Some(("true", "true", spinel_sha))), + ("schedule", "contributor/roundhouse", "refs/heads/main", "false", spinel_sha, None), + ("workflow_dispatch", "rubys/roundhouse", "refs/heads/topic", "true", spinel_sha, None), + ] { + let outputs = root.join("outputs"); + fs::write(&outputs, "").unwrap(); + let result = Command::new("bash") + .args(["-e", "-o", "pipefail", "-c", body]) + .env("PATH", format!("{}:{}", root.display(), std::env::var("PATH").unwrap())) + .env("EVENT", event).env("GITHUB_REPOSITORY", repo) + .env("GITHUB_REF", reference).env("REQUEST_PUBLISH", publish) + .env("GITHUB_SHA", main_sha).env("MOCK_SPINEL", revision) + .env("GITHUB_OUTPUT", &outputs).output().unwrap(); + assert_eq!(result.status.success(), expected.is_some(), "{event} {repo} {reference}: {result:?}"); + let actual = fs::read_to_string(outputs).unwrap(); + if let Some((published, known, resolved)) = expected { + assert_eq!(actual, format!( + "spinel={resolved}\nknown={known}\npublish={published}\nkey=full-ci-v1-{main_sha}-{resolved}-2026-10-02\n")); + } else { + assert!(actual.is_empty(), "rejected publication must not issue a cache key"); + } + } + fs::remove_dir_all(root).unwrap(); +} From 74d22651a4d4ae43a86b0d36a5d1374f76782d41 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 18:09:22 +0000 Subject: [PATCH 2/6] Refine targeted Spinel checks and archive validation evidence Keep scheduled full validation fresh on every cycle and select native Spinel checks by actual runtime, harness, and packaging ownership. Retain partial archive outputs and report byte-specific validation, compiler provenance, rerun uncertainty, and publication availability. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- .github/workflows/ci.yml | 209 ++++++++++---- .github/workflows/full-ci.yml | 43 +-- AGENTS.md | 10 +- README.md | 10 +- docs/ci-reuse.md | 61 ++-- scripts/ci-archive-evidence.py | 438 +++++++++++++++++++++++++++++ scripts/ci-plan.py | 215 ++++++++++++-- tests/ci_archive_evidence_test.py | 399 ++++++++++++++++++++++++++ tests/ci_plan_test.py | 172 +++++++++++- tests/workflow_yaml_parses.rs | 452 +++++++++++++++++++++++------- 10 files changed, 1763 insertions(+), 246 deletions(-) create mode 100755 scripts/ci-archive-evidence.py create mode 100644 tests/ci_archive_evidence_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac9cd5a89..197d3e261 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: site: ${{ steps.plan.outputs.site }} publish: ${{ steps.plan.outputs.publish }} spinel-revision: ${{ steps.plan.outputs.spinel-revision }} + spinel-tests: ${{ steps.plan.outputs.spinel-tests }} steps: - uses: actions/checkout@v5 with: @@ -773,8 +774,8 @@ jobs: # rewiring of the test_helper. framework-tests-spinel: # Re-enabled 2026-06-17 (5/5 green on matz/spinel master). Advisory. - needs: build-spinel - if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'framework-tests-spinel') && needs.build-spinel.outputs.artifact-id != '' }} outputs: &advisory-outputs execution: ${{ steps.result.outputs.status }} runs-on: ubuntu-latest @@ -792,33 +793,18 @@ jobs: run: | chmod +x spinel-dist/spinel spinel-dist/spin spinel-dist/spinel_rbs_extract echo "$PWD/spinel-dist" >> $GITHUB_PATH - - name: cargo test --test framework_tests_spinel -- --ignored - run: cargo test --test framework_tests_spinel -- --ignored --nocapture - # The web-push gem's crypto ported onto spinel's openssl package - # (runtime/spinel/web_push_crypto.rb), pinned against the gem's - # own bytes. Same staged spinel; the package links the runner's - # system libssl. - - name: cargo test --test spinel_web_push_crypto -- --ignored - if: always() - run: cargo test --test spinel_web_push_crypto -- --ignored --nocapture - # A request that raises gives its connection back - # (runtime/spinel/db.rb `with_connection`); a leaked lease used to - # wedge a pool shard for good after four 500s. - - name: cargo test --test spinel_db_lease -- --ignored - if: always() - run: cargo test --test spinel_db_lease -- --ignored --nocapture - # Rails' request-params builder, ported for the spinel dispatcher - # (runtime/spinel/param_builder.rb), held to Rails' answers for - # 2,755 query strings. The CRuby half runs in the core job. - - name: cargo test --test spinel_param_builder -- --ignored - if: always() - run: cargo test --test spinel_param_builder -- --ignored --nocapture - # Rails' signed-value vectors on the spinel-compiled verifier (the - # CRuby half runs in the core job), plus secrets whose derived keys - # hold a zero byte — the binary used to cut those keys short. - - name: cargo test --test rails_compat_vectors_spinel -- --ignored - if: always() - run: cargo test --test rails_compat_vectors_spinel -- --ignored --nocapture + # Full runs select all five suites; targeted runs select their owners. + # Continue after a failing suite without losing its failure status. + - name: Run selected native framework checks + env: + TESTS: ${{ join(fromJSON(needs.plan.outputs.spinel-tests), ' ') }} + run: | + test -n "$TESTS" + status=0 + for suite in $TESTS; do + cargo test --test "$suite" -- --ignored --nocapture || status=1 + done + exit "$status" - *advisory-result # Campfire conformance — ONCE Campfire's OWN test suite, run against a @@ -950,8 +936,8 @@ jobs: # its executable bit in a tarball. The DB differential builds its own # scenario overlay and must not reuse this binary. build-campfire-compare-spinel: - needs: build-spinel - if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-campfire-compare-spinel') && needs.build-spinel.outputs.artifact-id != '' }} runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true @@ -1082,10 +1068,10 @@ jobs: # nothing. campfire-compare-spinel: name: campfire-compare-spinel (${{ matrix.gc }}) - needs: build-campfire-compare-spinel + needs: [build-campfire-compare-spinel, plan] # An advisory producer failure must not become three artifact-download # failures. Only the upload's artifact ID proves the binary is available. - if: ${{ !cancelled() && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }} + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare-spinel') && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }} runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true @@ -1168,8 +1154,8 @@ jobs: # build, not the comparison binary. Start after the toolchain is ready # instead of serializing it behind the shared build and request walk. campfire-db-differential-spinel: - needs: build-spinel - if: ${{ !cancelled() && needs.build-spinel.outputs.artifact-id != '' }} + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-db-differential-spinel') && needs.build-spinel.outputs.artifact-id != '' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 45 @@ -1662,8 +1648,8 @@ jobs: # via continue-on-error — like every spinel job it tracks matz/spinel # master UNPINNED, so it surfaces regressions without gating unrelated # work; if master churns red, the signal is visible but non-blocking. - needs: [generate-fixture, build-spinel] - if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + needs: [generate-fixture, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'toolchain-spinel') && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 @@ -2019,8 +2005,8 @@ jobs: # drop it once the upstream cadence settles. compare-spinel: # Re-enabled 2026-06-17 (7/7 paths match Rails on matz/spinel master). Advisory. - needs: [generate-fixture, build-spinel] - if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + needs: [generate-fixture, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'compare-spinel') && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 @@ -2225,6 +2211,25 @@ jobs: name: ci-executed-smoke-rust-${{ github.run_attempt }} path: ${{ steps.reuse.outputs.bundle }} retention-days: 7 + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: browse/${{ matrix.target }}.tgz + CHECK: smoke-${{ matrix.target }} + OUTCOME: ${{ steps.smoke.outcome }} + REUSED: ${{ matrix.target == 'rust' && steps.reuse.outcome == 'success' && steps.reuse.outputs.hit == 'true' }} + run: &archive-validation | + if [ "${REUSED:-false}" = true ]; then OUTCOME=reused; fi + python3 scripts/ci-archive-evidence.py validation \ + --archive "$ARCHIVE_PATH" --path "$ARCHIVE_PATH" --check "$CHECK" \ + --outcome "$OUTCOME" --spinel-revision-file spinel-dist/revision.txt \ + --out "ci-evidence/validation-$CHECK-$GITHUB_RUN_ATTEMPT.json" + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-${{ matrix.target }}-${{ github.run_attempt }} + path: ci-evidence/*.json # spinel: the archive's README § Build AOT-compiles main.rb → build/blog # with the spinel toolchain (build-spinel artifact); § Test runs the @@ -2237,8 +2242,8 @@ jobs: # spinel job — it tracks matz/spinel master UNPINNED. needs build-spinel # for the compiler and build-site for the published spinel.tgz. smoke-spinel: - needs: [build-site, build-spinel] - if: ${{ !cancelled() && needs.build-site.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + needs: [build-site, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-spinel') && needs.build-site.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 20 @@ -2269,7 +2274,21 @@ jobs: npm install --no-audit --no-fund "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke spinel + id: smoke run: scripts/smoke --tgz browse/spinel.tgz spinel + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: browse/spinel.tgz + CHECK: smoke-spinel + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-spinel-${{ github.run_attempt }} + path: ci-evidence/*.json - *advisory-result # ------------------------------------------------------------------- @@ -2341,7 +2360,24 @@ jobs: tar -xzf /tmp/campfire.tar.gz -C /tmp/campfire --strip-components=1 - name: Build campfire/spinel.tgz + campfire/docker.tgz run: scripts/build-campfire-archive --out _site /tmp/campfire + - name: Record archive producer evidence + if: always() + env: + GROUP: campfire + OUTCOME: ${{ job.status }} + run: &archive-inventory | + python3 scripts/ci-archive-evidence.py inventory \ + --group "$GROUP" --root _site --outcome "$OUTCOME" \ + --spinel-revision-file spinel-dist/revision.txt \ + --out "ci-evidence/inventory-$GROUP-$GITHUB_RUN_ATTEMPT.json" + - name: Save archive producer evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-inventory-campfire-${{ github.run_attempt }} + path: ci-evidence/*.json - uses: actions/upload-artifact@v7 + if: always() with: name: campfire-archive path: _site/campfire/*.tgz @@ -2354,17 +2390,20 @@ jobs: # read the sign-in page triggers -- is found. Advisory like every # spinel job. smoke-campfire-docker: - needs: [build-campfire-archive] + needs: [build-campfire-archive, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-campfire-docker') && needs.build-campfire-archive.result == 'success' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true steps: + - uses: actions/checkout@v5 - uses: actions/download-artifact@v8 with: name: campfire-archive path: campfire - name: Build and run the image + id: smoke run: | set -euo pipefail if [ ! -f campfire/docker.tgz ]; then @@ -2388,6 +2427,19 @@ jobs: docker logs campfire docker rm -f campfire >/dev/null [ "$root" = 302 ] && [ "$first" = 200 ] && [ "$logo" = 200 ] + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: campfire/docker.tgz + CHECK: smoke-campfire-docker + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-campfire-docker-${{ github.run_attempt }} + path: ci-evidence/*.json - *advisory-result # smoke-campfire: the campfire archive runs its own README, exactly as @@ -2407,8 +2459,8 @@ jobs: # rather than our regression. Read it job-by-job; a green RUN does not # mean this passed. smoke-campfire: - needs: [build-campfire-archive, build-spinel] - if: ${{ !cancelled() && needs.build-campfire-archive.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + needs: [build-campfire-archive, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-campfire') && needs.build-campfire-archive.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 @@ -2441,7 +2493,21 @@ jobs: npm ci --no-audit --no-fund "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke campfire + id: smoke run: scripts/smoke --tgz campfire/spinel.tgz campfire + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: campfire/spinel.tgz + CHECK: smoke-campfire + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-campfire-${{ github.run_attempt }} + path: ci-evidence/*.json - *advisory-result # ------------------------------------------------------------------- @@ -2926,7 +2992,20 @@ jobs: # The per-target archives, for the e2e- jobs to download and # boot — the exact bytes published to rubys.github.io/roundhouse. + - name: Record archive producer evidence + if: always() + env: + GROUP: browse + OUTCOME: ${{ job.status }} + run: *archive-inventory + - name: Save archive producer evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-inventory-browse-${{ github.run_attempt }} + path: ci-evidence/*.json - name: Upload browse archives + if: always() uses: actions/upload-artifact@v7 with: name: browse-archives @@ -2939,6 +3018,33 @@ jobs: name: site-base path: _site + # Report actual per-archive outcomes, including failed/unavailable advisory + # consumers. A report is not a passing validation or an execution receipt. + archive-results: + needs: [plan, build-site, build-campfire-archive, smoke, smoke-spinel, smoke-campfire, smoke-campfire-docker] + if: ${{ !cancelled() && needs.plan.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs), 'archive-results') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + - name: Collect this run's archive evidence + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: ci-archive-* + path: ci-evidence + - name: Describe tested bytes without certifying missing evidence + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: python3 scripts/ci-archive-evidence.py report --evidence-dir ci-evidence --out archive-report/archive-results.json + - name: Save archive outcome report + if: always() + uses: actions/upload-artifact@v7 + with: + name: archive-results + path: archive-report/archive-results.json + # Publication only: PR smoke jobs consume browse-archives, not Pages. # Finish Pages assembly without keeping blog smoke jobs behind Campfire. assemble-site: @@ -2950,8 +3056,8 @@ jobs: # where it is most useful would be exactly backwards. A missing # artifact degrades to a skipped render and a warning, not a broken # site build. - needs: [build-site, campfire-conformance, build-campfire-archive, plan] - if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' }} + needs: [build-site, campfire-conformance, build-campfire-archive, archive-results, plan] + if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' && needs.archive-results.result == 'success' }} outputs: ready: ${{ steps.ready.outputs.ready }} runs-on: ubuntu-latest @@ -2967,6 +3073,11 @@ jobs: path: site-base - name: Restore the site run: mkdir -p _site && tar -xf site-base/artifact.tar -C _site + - name: Attach archive outcomes alongside the immutable downloads + uses: actions/download-artifact@v8 + with: + name: archive-results + path: _site/ci # The campfire archive, built by the parallel build-campfire-archive # job. It lands AFTER `--site` because build_site() removes and @@ -3028,6 +3139,10 @@ jobs: rmdir _site/bench 2>/dev/null || true fi + - name: Verify published archive bytes + # Verify the copied download bytes against their producer/test report. + # Failed validations may publish as repros; mismatched bytes may not. + run: python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json - name: Upload Pages artifact uses: actions/upload-pages-artifact@v5 with: @@ -3055,7 +3170,7 @@ jobs: echo 'passed=true' >> "$GITHUB_OUTPUT" ci-required: - needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare, compact-required, compare-extra, compare-jruby, build-wasm, browser-smoke-ide, build-site, smoke, writebook-inventory, build-spinel, framework-tests-spinel, build-campfire-compare-spinel, campfire-compare-spinel, campfire-db-differential-spinel, toolchain-spinel, compare-spinel, smoke-spinel, build-campfire-archive, smoke-campfire, smoke-campfire-docker, assemble-site] + needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare, compact-required, compare-extra, compare-jruby, build-wasm, browser-smoke-ide, build-site, smoke, writebook-inventory, build-spinel, framework-tests-spinel, build-campfire-compare-spinel, campfire-compare-spinel, campfire-db-differential-spinel, toolchain-spinel, compare-spinel, smoke-spinel, build-campfire-archive, smoke-campfire, smoke-campfire-docker, archive-results, assemble-site] if: always() runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index b2c0e6d62..b1b4c35e4 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -25,10 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: - run: ${{ github.event_name != 'schedule' || github.run_attempt != 1 || steps.lookup.outcome != 'success' || steps.lookup.outputs.cache-hit != 'true' }} - key: ${{ steps.inputs.outputs.key }} spinel: ${{ steps.inputs.outputs.spinel }} - known: ${{ steps.inputs.outputs.known }} publish: ${{ steps.inputs.outputs.publish }} steps: - name: Resolve the immutable inputs and publication boundary @@ -44,35 +41,18 @@ jobs: test "$GITHUB_REF" = refs/heads/main publish=true fi - known=false spinel=$(gh api repos/matz/spinel/commits/master --jq .sha) || spinel=master - if [[ "$spinel" =~ ^[0-9a-f]{40}$ ]]; then known=true; else spinel=master; fi + if ! [[ "$spinel" =~ ^[0-9a-f]{40}$ ]]; then spinel=master; fi echo "spinel=$spinel" >> "$GITHUB_OUTPUT" - echo "known=$known" >> "$GITHUB_OUTPUT" echo "publish=$publish" >> "$GITHUB_OUTPUT" - # Daily refresh also exercises floating Rails/SDK/runner inputs. - echo "key=full-ci-v1-$GITHUB_SHA-$spinel-$(date -u +%F)" >> "$GITHUB_OUTPUT" - - name: Look up a fully executed and published snapshot - id: lookup - if: github.event_name == 'schedule' && github.run_attempt == 1 && steps.inputs.outputs.known == 'true' - continue-on-error: true - uses: actions/cache/restore@v4 - with: - path: .ci-full-completed - key: ${{ steps.inputs.outputs.key }} - lookup-only: true - - name: Report the scheduling decision + - name: Report fresh validation inputs env: - HIT: ${{ steps.lookup.outputs.cache-hit }} - LOOKUP: ${{ steps.lookup.outcome }} SPINEL: ${{ steps.inputs.outputs.spinel }} - EVENT: ${{ github.event_name }} run: | - printf 'Main/ref snapshot: `%s`\n\nSpinel: `%s`\n\nExact completion marker: `%s` (lookup `%s`). Only scheduled first attempts may skip; dispatches and reruns execute freshly.\n' "$GITHUB_SHA" "$SPINEL" "$HIT" "$LOOKUP" >> "$GITHUB_STEP_SUMMARY" + printf 'Source snapshot: `%s`\n\nRequested Spinel: `%s`\n\nEvery cycle executes validation freshly, including floating Rails/package/system inputs. Actual Spinel revisions are recorded in the toolchain and archive evidence.\n' "$GITHUB_SHA" "$SPINEL" >> "$GITHUB_STEP_SUMMARY" validation: needs: preflight - if: needs.preflight.outputs.run == 'true' permissions: contents: read actions: read @@ -108,20 +88,3 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v5 - - checkpoint: - needs: [preflight, validation, deploy] - if: ${{ !cancelled() && github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' && needs.preflight.outputs.known == 'true' && needs.preflight.outputs.publish == 'true' && needs.validation.result == 'success' && needs.validation.outputs.complete == 'true' && needs.deploy.result == 'success' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Mark only actual complete execution and successful publication - env: - KEY: ${{ needs.preflight.outputs.key }} - run: | - mkdir .ci-full-completed - printf '%s\n' "$KEY" > .ci-full-completed/inputs - - uses: actions/cache/save@v4 - with: - path: .ci-full-completed - key: ${{ needs.preflight.outputs.key }} diff --git a/AGENTS.md b/AGENTS.md index 341e9603a..336315e96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,7 +89,7 @@ defect even if the build is green. - **Outside contributors: fork, and open a pull request against `main`.** CI runs a compact floor plus targeted lanes; add `ci:full` for the complete PR matrix. Full main validation/publication is scheduled every - four hours, with changed-input checks and a daily refresh. `ci-required` + four hours and executes freshly every cycle. `ci-required` gates selected blocking coverage; see [CI coverage](docs/ci-reuse.md). You do not need every toolchain locally; CI covers the missing lanes. Before opening one: `bin/rh fixture` (the test fixtures are generated, @@ -113,9 +113,11 @@ defect even if the build is green. `#[ignore]`-gated (`cargo test --test _toolchain -- --ignored`); CI runs each in its own job. - **CI is deliberately not uniformly gating.** The core `cargo test` job gates. - The ~5 `continue-on-error: true` jobs track upstream Spinel and other moving - toolchains on purpose — **red there is a signal to read, not a regression to - shim away.** Don't add workarounds just to make an advisory job green. + The ~5 `continue-on-error: true` jobs track current Spinel master and other + moving toolchains on purpose — **red there is a signal to read, not a + regression to shim away.** A Spinel red can come from Roundhouse runtime, RBS + or packaging, or from upstream; it is not by itself proof of an upstream + fault. Don't add workarounds just to make an advisory job green. ## The actual goal diff --git a/README.md b/README.md index cee3b2fcd..10e662c4d 100644 --- a/README.md +++ b/README.md @@ -171,11 +171,13 @@ its predecessors; [`WHY.md`](WHY.md) is why do it at all. Issues and pull requests are both welcome, and a PR does not need a conversation first: CI runs a compact correctness/runtime floor plus targeted checks. Add `ci:full` for full PR validation; the complete -matrix also runs every four hours on changed main/Spinel inputs, -with a daily refresh. See [CI coverage](docs/ci-reuse.md). The +matrix also runs freshly every four hours. See +[CI coverage](docs/ci-reuse.md). The `ci-required` status gates selected blocking coverage; the Spinel -and Campfire jobs marked `continue-on-error` track moving toolchains, -and red there is a signal to read, not a check to make pass. Setup, +and Campfire jobs marked `continue-on-error` track current Spinel +master, and red there is a signal to investigate Roundhouse runtime, +RBS or packaging as well as possible upstream drift—not proof of an +upstream fault. Setup, the test cycle, and what a PR should carry are in [`DEVELOPMENT.md`](DEVELOPMENT.md); the invariants not to break are in [`AGENTS.md`](AGENTS.md). diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index 74f911223..d051c6a64 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -26,12 +26,15 @@ status instead of being left pending by workflow-level path filters. | Changed inputs | Additional coverage | |---|---| | Target emitter file **or directory**, target runtime, toolchain/framework test | Owning comparison and archive smoke; existing embedded framework/toolchain suites stay intact | -| Ruby emitter or Spinel adapters | Ruby/JRuby/Spinel family, since interpreted targets also consume Spinel-tree files | -| Shared analyzer, lowerer, `runtime/ruby/` | Compact floor; reviewers request full validation for broad/risky changes | +| Ruby emitter | Ruby/JRuby owners plus advisory native Spinel core (`build-spinel`, `toolchain-spinel`, `compare-spinel`) | +| `runtime/ruby/`, native `runtime/spinel/`, or a focused Spinel test | Advisory native core plus the relevant focused Spinel framework suite | +| Interpreter-only `db_jruby` / `markly_jruby` or `scaffold/ruby_overlay` | Interpreted Ruby/JRuby owners, not native fanout | +| Spinel scaffold packaging | Advisory native core plus the native archive smoke | +| Shared analyzer or lowerer | Compact floor; no mandatory Spinel lane on an ordinary PR; reviewers can request `ci:full` for broad/risky changes | | `wasm/` | WASM build and IDE/playground/studio browser verification | | Site/guide sources | Site/archive build and WASM verification, without publishing | -| Shared compare or archive/E2E harness | Its owning comparisons or archive smokes | -| CI policy, cross-target packaging, unknown new target | Full validation | +| Shared compare, framework, archive, or E2E harness | The checks owned by that harness | +| Cross-target packaging, CI policy/workflows/planner, Cargo/build/toolchain policy, unknown new target | Full validation | Add **`ci:full`** to a non-draft PR to request the full matrix for its current merge tree. Removing the label restores automatic routing. Label/draft/head @@ -49,20 +52,33 @@ the same producer bytes, without later re-emission. `.github/workflows/full-ci.yml` checks canonical main at **00:17, 04:17, 08:17, 12:17, 16:17 and 20:17 UTC**, bundling full validation and publication. -Its exact completion key includes main SHA, observed Spinel master SHA and UTC -day. Spinel is resolved once and built at that revision. The daily refresh -exercises floating Rails/SDK/runner inputs even without repository changes. -Only an exact successful lookup can skip a scheduled first attempt; lookup -uncertainty, dispatches and reruns execute. Failed/unavailable advisory work, -failed required coverage or failed publication prevents saving the marker, so -the next interval retries. This can mean all six runs execute during upstream -failure; it does not pretend that failure was validated successfully. +Every cycle executes freshly; validation results are not cached. Spinel master +is resolved once per run, and evidence records the compiler revision actually +used. Its lanes remain advisory. Ordinary build caches and the conservative PR +execution receipts described below are unchanged. + +Archive producers upload with `always()`, preserving any files already produced +if a later producer step fails. The outcome report names the source SHA, run and +attempt, actual byte size/hash, and applicable Spinel provenance, and classifies +each archive `passed`, `reused`, `failed`, `unverified`, or `not-selected`. +Validation is byte-specific: a TGZ smoke does not certify sibling ZIP or JSON +bytes. A Spinel source archive contains source, not a built compiler; its native +smoke witness therefore records the compiler revision separately. Likewise a +Docker producer's compiler revision is distinct provenance from a native +archive consumer's compiler revision. Publication requires the **compact** floor and assembled site, not passing all -extra-target or advisory lanes: broken-emit archives are also upstream repros. -The deployment lock rechecks that canonical main still equals the validated -SHA; lookup failure or a superseded snapshot prevents publication. A new main -commit racing the final check cannot be made atomic with Pages deployment. +extra-target or advisory lanes: failed or unverified archives can still be +useful repros, but are not described as validated. Assembly waits for the same +run's outcome report, copies only that run's archives, and verifies the actual +copied bytes against the report hashes. It never rebuilds from a newer main. +The published sidecar at `ci/archive-results.json` separately records whether +each archive is present for download. Older-attempt witnesses remain visible +but are conservatively `unverified` on a rerun, not evidence that it passed. +Deployment guards are unchanged: the deployment lock rechecks that canonical +main still equals the validated SHA; lookup failure or a superseded snapshot +prevents publication. A new main commit racing the final check cannot be made +atomic with Pages deployment. Started background runs finish; only the newest pending request is retained. Extra comparison/smoke matrices use `max-parallel: 2`, GC uses 1. These bound fanouts, **not** total repository concurrency or a guaranteed PR runner priority. @@ -205,14 +221,11 @@ Stage-aware cancellation needs a separate controller to inspect the previous run; GitHub's concurrency expression cannot inspect that run's job progress. This PR does not add privileged cancellation machinery. -The next substantial opportunity is sharing **current-run** native compiler -builds, not declaring more outputs unchanged without a dependency witness. -In [run 36998514579](https://github.com/rubys/roundhouse/actions/runs/36998514579), -the unit Cargo step took 309 seconds and the two TypeScript toolchain/framework -steps took 210 seconds together. Those are whole-step timings, not measurements -of compiler cost alone. A shared build needs consistent profiles/toolchains, -native-library ABI, current commit provenance and coverage of each consumer's -actual build flags. It should be measured and proved separately. +A separate measured follow-up may evaluate sharing the `unit` job's Debug +binary with Campfire conformance/compare. This change does not copy Cargo target +directories, share images, introduce Bazel, or claim a measured queue-time +improvement. Any sharing proposal still needs compatible toolchains/build flags, +native-library ABI, current-commit provenance, and proof at each consumer. Matrix `fail-fast: false` still preserves cross-target diagnostic coverage; advisory lanes remain signals rather than reasons to cancel independent checks. diff --git a/scripts/ci-archive-evidence.py b/scripts/ci-archive-evidence.py new file mode 100755 index 000000000..10c07e349 --- /dev/null +++ b/scripts/ci-archive-evidence.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Record and summarize byte-level evidence for CI download archives.""" + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path + +SCHEMA = 1 +SHA = re.compile(r"[0-9a-f]{40}\Z") +GROUPS = {"browse": "build-site", "campfire": "build-campfire-archive"} +CHECKS = { + **{ + f"smoke-{target}": f"browse/{target}.tgz" + for target in ( + "rust", + "crystal", + "kotlin", + "swift", + "csharp", + "typescript", + "go", + "elixir", + "python", + "ruby", + "jruby", + ) + }, + "smoke-spinel": "browse/spinel.tgz", + "smoke-campfire": "campfire/spinel.tgz", + "smoke-campfire-docker": "campfire/docker.tgz", +} +CHECK_JOBS = { + check: ( + "smoke" if path.startswith("browse/") and check != "smoke-spinel" else check + ) + for check, path in CHECKS.items() +} + + +def identity(): + source = os.environ.get("GITHUB_SHA", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + attempt = os.environ.get("GITHUB_RUN_ATTEMPT", "") + if not SHA.fullmatch(source): + raise ValueError("GITHUB_SHA must be a lowercase full 40-character SHA") + if not run_id.isdigit() or int(run_id) < 1: + raise ValueError("GITHUB_RUN_ID must be a positive integer") + if not attempt.isdigit() or int(attempt) < 1: + raise ValueError("GITHUB_RUN_ATTEMPT must be a positive integer") + return source, run_id, int(attempt) + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def revision(path): + if path is None or not Path(path).is_file(): + return None + value = Path(path).read_text(encoding="utf-8").strip() + if not SHA.fullmatch(value): + raise ValueError(f"invalid Spinel revision in {path}") + return value + + +def write_json(path, value): + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n") + temporary.replace(destination) + + +def base(kind): + source, run_id, attempt = identity() + return { + "schema": SCHEMA, + "kind": kind, + "source_sha": source, + "run_id": run_id, + "attempt": attempt, + } + + +def inventory(args): + root = Path(args.root) + compiler = revision(args.spinel_revision_file) + archives = [] + group_root = root / args.group + if group_root.is_dir(): + for path in sorted(group_root.iterdir()): + if not path.is_file() or path.suffix not in {".tgz", ".zip", ".json"}: + continue + logical = path.relative_to(root).as_posix() + archives.append( + { + "path": logical, + "sha256": digest(path), + "size": path.stat().st_size, + "spinel_revision": ( + compiler if logical == "campfire/docker.tgz" else None + ), + } + ) + document = base("inventory") + document.update( + {"group": args.group, "producer_outcome": args.outcome, "archives": archives} + ) + write_json(args.out, document) + + +def validation(args): + if CHECKS[args.check] != args.path: + raise ValueError(f"{args.check} does not validate {args.path}") + archive = Path(args.archive) + document = base("validation") + document.update( + { + "check": args.check, + "path": args.path, + "sha256": digest(archive) if archive.is_file() else None, + "size": archive.stat().st_size if archive.is_file() else None, + "outcome": args.outcome, + "spinel_revision": revision(args.spinel_revision_file), + } + ) + write_json(args.out, document) + + +def load_evidence(directory, source, run_id, current_attempt): + inventories, validations = {}, {} + for path in sorted(Path(directory).rglob("*.json")): + try: + item = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"malformed evidence {path}: {error}") from error + if not isinstance(item, dict) or item.get("schema") != SCHEMA: + raise ValueError(f"malformed evidence {path}: unsupported schema") + if item.get("source_sha") != source or item.get("run_id") != run_id: + raise ValueError(f"foreign evidence {path}") + attempt = item.get("attempt") + if type(attempt) is not int or not 1 <= attempt <= current_attempt: + raise ValueError(f"malformed evidence {path}: invalid attempt") + kind = item.get("kind") + if kind == "inventory": + validate_inventory(item, path) + key = item["group"] + target = inventories + elif kind == "validation": + validate_validation(item, path) + key = item["check"] + target = validations + else: + raise ValueError(f"malformed evidence {path}: invalid kind") + previous = target.get(key) + if previous is None or item["attempt"] > previous["attempt"]: + target[key] = item + elif item["attempt"] == previous["attempt"] and item != previous: + raise ValueError(f"conflicting evidence for {key} attempt {attempt}") + return inventories, validations + + +def valid_hash(value): + return value is None or bool(re.fullmatch(r"[0-9a-f]{64}", value)) + + +def validate_inventory(item, path): + if ( + item.get("group") not in GROUPS + or not isinstance(item.get("producer_outcome"), str) + or not isinstance(item.get("archives"), list) + ): + raise ValueError(f"malformed inventory {path}") + seen = set() + for archive in item["archives"]: + if not isinstance(archive, dict): + raise TypeError(f"malformed inventory archive {path}") + logical = archive.get("path") + prefix = item["group"] + "/" + if ( + not isinstance(logical, str) + or not logical.startswith(prefix) + or "/" in logical[len(prefix) :] + or Path(logical).suffix not in {".tgz", ".zip", ".json"} + or logical in seen + or not valid_hash(archive.get("sha256")) + or archive.get("sha256") is None + or type(archive.get("size")) is not int + or archive["size"] < 0 + or ( + archive.get("spinel_revision") is not None + and not SHA.fullmatch(archive["spinel_revision"]) + ) + or ( + logical != "campfire/docker.tgz" + and archive.get("spinel_revision") is not None + ) + ): + raise ValueError(f"malformed inventory archive {path}") + seen.add(logical) + + +def validate_validation(item, path): + check = item.get("check") + if ( + check not in CHECKS + or item.get("path") != CHECKS.get(check) + or not isinstance(item.get("outcome"), str) + or not valid_hash(item.get("sha256")) + or (item.get("size") is not None and type(item.get("size")) is not int) + or (item.get("sha256") is None) != (item.get("size") is None) + or ( + item.get("spinel_revision") is not None + and not SHA.fullmatch(item["spinel_revision"]) + ) + ): + raise ValueError(f"malformed validation {path}") + + +def expected_paths(plan): + result = set() + for value in plan.get("archives", []): + if not isinstance(value, str): + raise TypeError("CI_PLAN archives must be strings") + result.add(value if "/" in value else f"browse/{value}.tgz") + jobs = plan.get("jobs", []) + smoke = plan.get("smoke", []) + if not isinstance(jobs, list) or not isinstance(smoke, list): + raise TypeError("CI_PLAN jobs and smoke must be lists") + if "build-campfire-archive" in jobs: + result.update(("campfire/spinel.tgz", "campfire/docker.tgz")) + return result + + +def need_result(needs, job): + value = needs.get(job) + return value.get("result") if isinstance(value, dict) else None + + +def archive_status(path, producer, witness, selected, needs, attempt): + if not selected: + return "not-selected", None + if producer is None: + return "unverified", "missing producer inventory" + if producer["producer_outcome"] != "success": + return "failed" if producer[ + "producer_outcome" + ] == "failure" else "unverified", ( + f"producer outcome {producer['producer_outcome']}" + ) + producer_job = GROUPS[path.split("/", 1)[0]] + producer_need = need_result(needs, producer_job) + if needs and producer_need != "success": + return "unverified", f"producer job result {producer_need or 'missing'}" + archive = next((a for a in producer["archives"] if a["path"] == path), None) + if archive is None: + return "unverified", "selected archive missing" + check = next((name for name, target in CHECKS.items() if target == path), None) + if check is None: + return "unverified", "no archive-specific smoke check" + if witness is None: + return "unverified", "selected witness missing" + if witness["attempt"] != attempt: + return ( + "unverified", + "earlier attempt witness; current execution not established", + ) + if witness["sha256"] != archive["sha256"] or witness["size"] != archive["size"]: + return "failed", "producer and witness bytes differ" + if ( + path in {"browse/spinel.tgz", "campfire/spinel.tgz"} + and not witness["spinel_revision"] + ): + return "unverified", "native Spinel revision unknown" + if path == "campfire/docker.tgz" and not archive["spinel_revision"]: + return "unverified", "packaging Spinel revision unknown" + outcome = witness["outcome"] + check_need = need_result(needs, CHECK_JOBS[check]) + if outcome == "failure": + return "failed", "harness outcome failure" + # The ordinary smoke job is a matrix. One failed sibling must not erase + # another target's actual byte-specific successful harness result. + if needs and (check_need is None or check_need in {"skipped", "cancelled"}): + return "unverified", f"consumer job result {check_need or 'missing'}" + if outcome == "success": + return "passed", None + if outcome == "reused": + return "reused", None + return "unverified", f"harness outcome {outcome or 'unknown'}" + + +def report(args): + source, run_id, attempt = identity() + try: + plan = json.loads(os.environ["CI_PLAN"]) + needs = json.loads(os.environ.get("CI_NEEDS", "{}")) + except (KeyError, json.JSONDecodeError) as error: + raise ValueError(f"invalid CI plan/needs: {error}") from error + if not isinstance(plan, dict) or not isinstance(needs, dict): + raise TypeError("CI_PLAN and CI_NEEDS must be JSON objects") + inventories, validations = load_evidence(args.evidence_dir, source, run_id, attempt) + paths = expected_paths(plan) + selected = {f"browse/{target}.tgz" for target in plan.get("smoke", [])} + selected.update(CHECKS[job] for job in plan.get("jobs", []) if job in CHECKS) + for producer in inventories.values(): + paths.update(archive["path"] for archive in producer["archives"]) + archives = [] + for path in sorted(paths): + group = path.split("/", 1)[0] + producer = inventories.get(group) + check = next((name for name, target in CHECKS.items() if target == path), None) + witness = validations.get(check) if check else None + status, reason = archive_status( + path, producer, witness, path in selected, needs, attempt + ) + archive = ( + next((a for a in producer["archives"] if a["path"] == path), None) + if producer + else None + ) + archives.append( + { + "path": path, + "selected": path in selected, + "status": status, + "reason": reason, + "sha256": archive["sha256"] if archive else None, + "size": archive["size"] if archive else None, + "check": check, + "validation_attempt": (witness or {}).get("attempt"), + "harness_outcome": (witness or {}).get("outcome"), + "spinel_revision": ( + (witness or {}).get("spinel_revision") + if path in {"browse/spinel.tgz", "campfire/spinel.tgz"} + else (archive or {}).get("spinel_revision") + ), + } + ) + document = { + "schema": SCHEMA, + "source_sha": source, + "run_id": run_id, + "attempt": attempt, + "producers": [ + { + "group": group, + "attempt": item["attempt"], + "outcome": item["producer_outcome"], + "job_result": need_result(needs, GROUPS[group]), + } + for group, item in sorted(inventories.items()) + ], + "archives": archives, + } + write_json(args.out, document) + + +def verify(args): + source, run_id, attempt = identity() + document = json.loads(Path(args.report).read_text()) + if ( + document.get("schema") != SCHEMA + or document.get("source_sha") != source + or document.get("run_id") != run_id + or document.get("attempt") != attempt + ): + raise ValueError("publication report has foreign identity") + archives = {item["path"]: item for item in document["archives"]} + root = Path(args.root) + published = set() + for group in GROUPS: + directory = root / group + if not directory.is_dir(): + continue + for path in directory.iterdir(): + if not path.is_file() or path.suffix not in {".tgz", ".zip", ".json"}: + continue + logical = path.relative_to(root).as_posix() + item = archives.get(logical) + if ( + item is None + or item.get("sha256") != digest(path) + or item.get("size") != path.stat().st_size + ): + raise ValueError(f"untracked or changed publication bytes: {logical}") + published.add(logical) + # Verification checks identity, not success: failed repro bytes remain useful. + for item in document["archives"]: + item["published"] = item["path"] in published + write_json(args.report, document) + + +def parser(): + result = argparse.ArgumentParser() + commands = result.add_subparsers(dest="command", required=True) + inv = commands.add_parser("inventory") + inv.add_argument("--group", choices=sorted(GROUPS), required=True) + inv.add_argument("--root", required=True) + inv.add_argument("--out", required=True) + inv.add_argument("--outcome", required=True) + inv.add_argument("--spinel-revision-file") + inv.set_defaults(function=inventory) + val = commands.add_parser("validation") + val.add_argument("--archive", required=True) + val.add_argument("--path", choices=sorted(CHECKS.values()), required=True) + val.add_argument("--check", choices=sorted(CHECKS), required=True) + val.add_argument("--outcome", required=True) + val.add_argument("--out", required=True) + val.add_argument("--spinel-revision-file") + val.set_defaults(function=validation) + rep = commands.add_parser("report") + rep.add_argument("--evidence-dir", required=True) + rep.add_argument("--out", required=True) + rep.set_defaults(function=report) + pub = commands.add_parser("verify") + pub.add_argument("--root", required=True) + pub.add_argument("--report", required=True) + pub.set_defaults(function=verify) + return result + + +def main(): + try: + args = parser().parse_args() + args.function(args) + except (OSError, TypeError, ValueError) as error: + print(f"ci-archive-evidence: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py index 84851d0c8..9eb8b7a1d 100755 --- a/scripts/ci-plan.py +++ b/scripts/ci-plan.py @@ -31,7 +31,15 @@ "campfire-conformance", "campfire-compare", ] -SPINEL = [ +CORE = ["build-spinel", "toolchain-spinel", "compare-spinel"] +SPINEL_TESTS = [ + "framework_tests_spinel", + "spinel_web_push_crypto", + "spinel_db_lease", + "spinel_param_builder", + "rails_compat_vectors_spinel", +] +SPINEL11 = [ "build-spinel", "framework-tests-spinel", "build-campfire-compare-spinel", @@ -44,25 +52,41 @@ "smoke-campfire", "smoke-campfire-docker", ] -ADVISORY = set(SPINEL) - {"build-campfire-archive"} +ADVISORY = set(SPINEL11) - {"build-campfire-archive"} SHA = re.compile(r"[0-9a-f]{40}\Z") def select(paths, *, draft=False, full=False, publish=False): if draft: return finish( - BASE[:2], [], [], False, False, False, ["draft: fixture and unit only"] + BASE[:2], + [], + [], + False, + False, + False, + ["draft: fixture and unit only"], + spinel_tests=[], ) targets, smoke = set(), set() + jobs_selected, spinel_tests = set(), set() wasm = site = spinel = writebook = False reasons = [] for path in paths: - if path.startswith(".github/") or path in { + if path.startswith((".github/", ".cargo/")) or path in { "scripts/ci-plan.py", + "scripts/ci-reuse.py", + "scripts/ci-archive-evidence.py", "tests/ci_plan_test.py", + "tests/ci_archive_evidence_test.py", "tests/workflow_yaml_parses.rs", "src/project.rs", "src/bin/roundhouse.rs", + "Cargo.toml", + "Cargo.lock", + "build.rs", + "rust-toolchain.toml", + ".cargo/config.toml", }: full = True reasons.append(f"{path}: validation/packaging policy") @@ -78,16 +102,45 @@ def select(paths, *, draft=False, full=False, publish=False): if test else None ) + interpreter_only = path.startswith( + "runtime/spinel/scaffold/ruby_overlay/" + ) or path in { + "runtime/spinel/db_jruby.rb", + "runtime/spinel/markly_jruby.rb", + "runtime/spinel/db_cruby.rb", + "runtime/spinel/message_digest_cruby.rb", + "runtime/spinel/module_delegate.rb", + } + native_path = ( + path.startswith(("runtime/ruby/", "runtime/spinel/", "src/emit/ruby/")) + or path == "src/emit/ruby.rs" + ) + native_path |= path.startswith("tests/spinel") and path.endswith((".rs", ".rb")) + native_path |= path in {f"tests/{name}.rs" for name in SPINEL_TESTS} + if native_path and not interpreter_only: + spinel = True + jobs_selected.update(CORE) + reasons.append(f"{path}: native Spinel core") + if path.startswith("runtime/ruby/") and path.endswith((".rb", ".rbs")): + spinel_tests.add("framework_tests_spinel") if target in TARGETS or target == "spinel": owners = ( {"ruby", "jruby", "spinel"} if target in {"ruby", "spinel"} and not path.startswith("runtime/ruby/") else {target} ) - if path.startswith("runtime/ruby/"): + if ( + path.startswith(("runtime/ruby/", "runtime/spinel/")) + or (path.startswith("tests/spinel") and path.endswith(".rs")) + or path in {f"tests/{name}.rs" for name in SPINEL_TESTS} + ): + owners = set() # Native framework coverage; no interpreted archives. + if interpreter_only: owners = ( - set() - ) # Shared runtime: the chosen compact floor, not all targets. + {"jruby"} + if path.endswith(("db_jruby.rb", "markly_jruby.rb")) + else {"ruby", "jruby"} + ) targets.update(owners - {"spinel"}) smoke.update(owners - {"spinel"}) spinel |= "spinel" in owners @@ -108,7 +161,9 @@ def select(paths, *, draft=False, full=False, publish=False): reasons.append(f"{path}: WASM/browser compiler") if path.startswith(("site/", "docs/guide/")): site = wasm = True - if path.startswith("e2e/") or path in { + if ( + path.startswith("e2e/") and not path.startswith("e2e/campfire/") + ) or path in { "scripts/smoke", "scripts/ci-playwright-install", "scripts/create-blog", @@ -123,12 +178,121 @@ def select(paths, *, draft=False, full=False, publish=False): ): targets.update(TARGETS) spinel = True - if "spinel" in path and path.startswith(("tests/", "scripts/")): + jobs_selected.update(CORE) + if path.startswith("tests/framework_test_support"): + spinel_tests.add("framework_tests_spinel") + focused = re.fullmatch( + r"tests/(framework_tests_spinel|spinel_web_push_crypto|spinel_db_lease|spinel_param_builder|rails_compat_vectors_spinel)\.(?:rs|rb)", + path, + ) + if focused: spinel = True + jobs_selected.update(CORE) + spinel_tests.add(focused[1]) + if ( + path.startswith(("runtime/spinel/", "runtime/ruby/")) + and not interpreter_only + ): + name = path.rsplit("/", 1)[-1] + owned_tests = set() + if any(word in name for word in ("web_push", "base64")): + owned_tests.add("spinel_web_push_crypto") + if any( + word in name + for word in ( + "signed_cookie", + "message_verifier", + "signed_id", + "message_digest", + "base64", + ) + ) or path.startswith("runtime/spinel/tep/url."): + owned_tests.add("rails_compat_vectors_spinel") + if any( + word in path + for word in ("/db", "sqlite", "active_support_time_parsing") + ): + owned_tests.add("spinel_db_lease") + if any(word in name for word in ("param", "multipart", "request")): + owned_tests.add("spinel_param_builder") + if ( + path.startswith("runtime/spinel/") + and not path.startswith("runtime/spinel/scaffold/") + and not owned_tests + ): + owned_tests.add("framework_tests_spinel") + spinel_tests.update(owned_tests) + if ( + path.startswith(("tests/rails_compat/", "tests/params_vectors/")) + or path == "tests/rails_compat_vectors.rb" + ): + spinel = True + jobs_selected.update(CORE) + spinel_tests.add( + "spinel_param_builder" + if path.startswith("tests/params_vectors/") + else "rails_compat_vectors_spinel" + ) + if path.startswith("runtime/spinel/scaffold/") and not interpreter_only: + spinel = True + jobs_selected.update((*CORE, "smoke-spinel", "build-site")) if path.startswith( - ("scripts/campfire-", "scripts/build-campfire", "e2e/campfire/") + ("scripts/campfire-compare", "scripts/build-campfire-compare") ): spinel = True + jobs_selected.update( + ( + "build-spinel", + "build-campfire-compare-spinel", + "campfire-compare-spinel", + ) + ) + if path.startswith("scripts/campfire-db-differential"): + spinel = True + jobs_selected.update(("build-spinel", "campfire-db-differential-spinel")) + if ( + path.startswith( + ("scripts/build-campfire-archive", "scripts/campfire-archive") + ) + or path == "scripts/campfire-docker-files" + ): + spinel = True + jobs_selected.update( + ( + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) + if path.startswith("e2e/campfire/"): + spinel = True + jobs_selected.update( + ( + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) + if path in {"scripts/smoke", "scripts/ci-playwright-install"}: + jobs_selected.update( + ( + "smoke-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) + if path.startswith("e2e/") and not path.startswith("e2e/campfire/"): + jobs_selected.update( + ( + "smoke-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) if path in {"tests/writebook.rs", "tests/fixtures/writebook-inventory.json"}: writebook = True if full: @@ -136,8 +300,10 @@ def select(paths, *, draft=False, full=False, publish=False): smoke.update(TARGETS) wasm = site = spinel = writebook = True reasons.append("full validation requested") - if site or spinel: - smoke.add("spinel") if spinel else None + jobs_selected.update(SPINEL11) + spinel_tests.update(SPINEL_TESTS) + if spinel: + jobs_selected.add("build-spinel") jobs = list(BASE) extra = [ t @@ -154,8 +320,15 @@ def select(paths, *, draft=False, full=False, publish=False): jobs.append("build-site") if smoke - {"spinel"}: jobs.append("smoke") - if spinel: - jobs.extend(SPINEL) + if spinel_tests: + jobs_selected.add("framework-tests-spinel") + if "smoke-spinel" in jobs_selected: + jobs_selected.add("build-site") + if "build-site" in jobs_selected and "build-site" not in jobs: + jobs.append("build-site") + if "build-site" in jobs or {"build-site", "build-campfire-archive"} & jobs_selected: + jobs_selected.add("archive-results") + jobs.extend(j for j in [*SPINEL11, "archive-results"] if j in jobs_selected) if writebook: jobs.append("writebook-inventory") if publish: @@ -171,14 +344,17 @@ def select(paths, *, draft=False, full=False, publish=False): spinel, reasons, publish, + [t for t in SPINEL_TESTS if t in spinel_tests], ) -def finish(jobs, extra, smoke, wasm, site, spinel, reasons, publish=False): +def finish( + jobs, extra, smoke, wasm, site, spinel, reasons, publish=False, spinel_tests=None +): archives = ( ["blog", "spinel", *TARGETS, "typescript-worker"] if site - else [*smoke, *(["spinel"] if spinel else [])] + else [*smoke, *(["spinel"] if "smoke-spinel" in jobs else [])] ) return { "jobs": jobs, @@ -189,6 +365,7 @@ def finish(jobs, extra, smoke, wasm, site, spinel, reasons, publish=False): "wasm": wasm, "site": site, "spinel": spinel, + "spinel_tests": spinel_tests or [], "publish": publish, "reasons": reasons, } @@ -240,8 +417,7 @@ def check_results(plan, needs, *, compact=False): failures.append("plan: no successful routing decision") if not compact and needs.get("compact-required", {}).get("result") != "success": failures.append("compact-required: no successful baseline gate") - # Advisory work never blocks the gate, but failed/unavailable work is not - # a successful full-execution checkpoint for the next scheduled interval. + # Advisory work never blocks the gate, but incomplete work is not complete. complete = not failures and all( needs.get(j, {}).get("result") == "success" and ( @@ -322,7 +498,7 @@ def main(): except subprocess.CalledProcessError: spinel = "master" plan["reasons"].append( - "Spinel lookup unavailable: fresh master build, no scheduler checkpoint" + "Spinel lookup unavailable: fresh master build; actual revision recorded by producer" ) if spinel and spinel != "master" and not SHA.fullmatch(spinel): raise ValueError("invalid Spinel revision") @@ -331,6 +507,7 @@ def main(): "plan": plan, "jobs": plan["jobs"], "extra-compare": plan["extra_compare"], + "spinel-tests": plan["spinel_tests"], "smoke": plan["smoke"], "archives": ",".join(plan["archives"]), "wasm": plan["wasm"], diff --git a/tests/ci_archive_evidence_test.py b/tests/ci_archive_evidence_test.py new file mode 100644 index 000000000..64fb39314 --- /dev/null +++ b/tests/ci_archive_evidence_test.py @@ -0,0 +1,399 @@ +"""Tests for bounded CI archive evidence.""" + +import hashlib +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci-archive-evidence.py" +SPEC = importlib.util.spec_from_file_location("ci_archive_evidence", SCRIPT) +evidence = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(evidence) +SHA = "1" * 40 +SPINEL = "2" * 40 + + +class ArchiveEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.env = { + "GITHUB_SHA": SHA, + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "1", + } + + def tearDown(self): + self.temp.cleanup() + + def run_cli(self, *args, ok=True, extra_env=None): + env = os.environ | self.env | (extra_env or {}) + result = subprocess.run( + ["python3", str(SCRIPT), *map(str, args)], + text=True, + capture_output=True, + env=env, + check=False, + ) + if ok and result.returncode: + self.fail(result.stderr) + return result + + def make_inventory(self, group, files, outcome="success", attempt=1, revision=None): + site = self.root / f"site-{group}-{attempt}" + for name, contents in files.items(): + path = site / group / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + rev = self.root / "revision.txt" + if revision: + rev.write_text(revision) + out = self.root / "evidence" / f"inventory-{group}-{attempt}.json" + with patch.dict(self.env, {"GITHUB_RUN_ATTEMPT": str(attempt)}): + self.run_cli( + "inventory", + "--group", + group, + "--root", + site, + "--out", + out, + "--outcome", + outcome, + *(["--spinel-revision-file", rev] if revision else []), + extra_env={"GITHUB_RUN_ATTEMPT": str(attempt)}, + ) + return out + + def make_validation( + self, check, data=None, outcome="success", attempt=1, revision=None + ): + archive = self.root / f"input-{check}-{attempt}.tgz" + if data is not None: + archive.write_bytes(data) + rev = self.root / "consumer-revision.txt" + if revision: + rev.write_text(revision) + out = self.root / "evidence" / f"validation-{check}-{attempt}.json" + self.run_cli( + "validation", + "--archive", + archive, + "--path", + evidence.CHECKS[check], + "--check", + check, + "--outcome", + outcome, + "--out", + out, + *(["--spinel-revision-file", rev] if revision else []), + extra_env={"GITHUB_RUN_ATTEMPT": str(attempt)}, + ) + return out + + def make_report(self, plan, needs=None, ok=True): + out = self.root / "report.json" + result = self.run_cli( + "report", + "--evidence-dir", + self.root / "evidence", + "--out", + out, + ok=ok, + extra_env={ + "CI_PLAN": json.dumps(plan), + "CI_NEEDS": json.dumps(needs or {}), + }, + ) + return (json.loads(out.read_text()) if result.returncode == 0 else None), result + + def status(self, report, path): + return next(item for item in report["archives"] if item["path"] == path) + + def test_inventory_hashes_bytes_independently_and_does_not_mutate(self): + original = b"archive bytes" + out = self.make_inventory("browse", {"rust.tgz": original, "map.json": b"{}"}) + item = json.loads(out.read_text()) + rust = next(a for a in item["archives"] if a["path"] == "browse/rust.tgz") + self.assertEqual(rust["sha256"], hashlib.sha256(original).hexdigest()) + self.assertEqual( + (self.root / "site-browse-1/browse/rust.tgz").read_bytes(), original + ) + self.assertIsNone(rust["spinel_revision"]) + + def test_same_name_changed_bytes_cannot_pass(self): + self.make_inventory("browse", {"rust.tgz": b"producer"}) + self.make_validation("smoke-rust", b"changed") + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_missing_file_validation_has_null_hash_and_never_passes(self): + witness = self.make_validation("smoke-rust", None, outcome="success") + self.assertIsNone(json.loads(witness.read_text())["sha256"]) + self.make_inventory("browse", {"rust.tgz": b"bytes"}) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_success_reuse_and_non_success_outcomes(self): + for outcome, expected in ( + ("success", "passed"), + ("reused", "reused"), + ("failure", "failed"), + ("skipped", "unverified"), + ("cancelled", "unverified"), + ("unknown", "unverified"), + ): + with self.subTest(outcome=outcome): + (self.root / "evidence").mkdir(exist_ok=True) + for path in (self.root / "evidence").iterdir(): + path.unlink() + self.make_inventory("browse", {"rust.tgz": b"same"}) + self.make_validation("smoke-rust", b"same", outcome=outcome) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual( + self.status(report, "browse/rust.tgz")["status"], expected + ) + + def test_missing_or_unavailable_selected_witness_is_unverified(self): + self.make_inventory("browse", {"rust.tgz": b"same"}) + plan = {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + report, _ = self.make_report(plan) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "unverified") + self.make_validation("smoke-rust", b"same") + report, _ = self.make_report(plan, {"smoke": {"result": "cancelled"}}) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "unverified") + + def test_native_spinel_requires_consumer_revision_but_source_archive_does_not_package_one( + self, + ): + self.make_inventory("browse", {"spinel.tgz": b"native"}, revision=SPINEL) + self.make_validation("smoke-spinel", b"native") + plan = {"archives": ["spinel"], "jobs": ["smoke-spinel"], "smoke": []} + report, _ = self.make_report(plan) + item = self.status(report, "browse/spinel.tgz") + self.assertEqual(item["status"], "unverified") + inventory = json.loads( + (self.root / "evidence/inventory-browse-1.json").read_text() + ) + self.assertIsNone(inventory["archives"][0]["spinel_revision"]) + + def test_native_and_docker_spinel_revision_rules_are_distinct(self): + data = b"campfire" + self.make_inventory( + "campfire", {"spinel.tgz": data, "docker.tgz": data}, revision=SPINEL + ) + self.make_validation("smoke-campfire", data, revision=SPINEL) + self.make_validation("smoke-campfire-docker", data) + plan = { + "archives": [], + "jobs": [ + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ], + "smoke": [], + } + report, _ = self.make_report(plan) + self.assertEqual(self.status(report, "campfire/spinel.tgz")["status"], "passed") + docker = self.status(report, "campfire/docker.tgz") + self.assertEqual(docker["status"], "passed") + self.assertEqual(docker["spinel_revision"], SPINEL) + + def test_docker_unknown_packaging_revision_is_unverified(self): + self.make_inventory("campfire", {"docker.tgz": b"docker"}) + self.make_validation("smoke-campfire-docker", b"docker") + plan = { + "archives": [], + "jobs": ["build-campfire-archive", "smoke-campfire-docker"], + "smoke": [], + } + report, _ = self.make_report(plan) + self.assertEqual( + self.status(report, "campfire/docker.tgz")["status"], "unverified" + ) + + def test_non_tgz_inventory_is_not_certified_by_tgz_smoke(self): + self.make_inventory( + "browse", {"rust.tgz": b"x", "rust.zip": b"x", "rust.json": b"x"} + ) + self.make_validation("smoke-rust", b"x") + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "passed") + self.assertEqual( + self.status(report, "browse/rust.zip")["status"], "not-selected" + ) + self.assertEqual( + self.status(report, "browse/rust.json")["status"], "not-selected" + ) + + def test_latest_attempt_is_selected_deterministically(self): + self.make_inventory("browse", {"rust.tgz": b"old"}, attempt=1) + self.make_validation("smoke-rust", b"old", attempt=1) + self.make_inventory("browse", {"rust.tgz": b"new"}, attempt=2) + self.make_validation("smoke-rust", b"new", attempt=2) + self.env["GITHUB_RUN_ATTEMPT"] = "2" + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual( + self.status(report, "browse/rust.tgz")["sha256"], + hashlib.sha256(b"new").hexdigest(), + ) + self.assertEqual(report["producers"][0]["attempt"], 2) + + def test_wrong_source_or_run_and_conflicts_are_errors(self): + path = self.make_inventory("browse", {"rust.tgz": b"x"}) + item = json.loads(path.read_text()) + for field, value in (("source_sha", "3" * 40), ("run_id", "999")): + with self.subTest(field=field): + changed = dict(item, **{field: value}) + path.write_text(json.dumps(changed)) + _, result = self.make_report( + {"archives": [], "jobs": [], "smoke": []}, ok=False + ) + self.assertEqual(result.returncode, 2) + path.write_text(json.dumps(item)) + duplicate = self.root / "evidence/duplicate.json" + duplicate.write_text(json.dumps(dict(item, producer_outcome="failure"))) + _, result = self.make_report( + {"archives": [], "jobs": [], "smoke": []}, ok=False + ) + self.assertEqual(result.returncode, 2) + + def test_missing_selected_archive_is_reported(self): + self.make_inventory("browse", {}) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + item = self.status(report, "browse/rust.tgz") + self.assertEqual(item["status"], "unverified") + self.assertIsNone(item["sha256"]) + + def test_producer_failure_after_partial_output_cannot_validate(self): + self.make_inventory("browse", {"rust.tgz": b"same"}, outcome="failure") + self.make_validation("smoke-rust", b"same") + report, result = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_unselected_existing_witness_and_failed_matrix_sibling(self): + self.make_inventory("browse", {"go.tgz": b"go", "rust.tgz": b"rust"}) + self.make_validation("smoke-go", b"go") + self.make_validation("smoke-rust", b"rust") + report, _ = self.make_report( + {"archives": ["go", "rust"], "jobs": ["smoke"], "smoke": ["go"]}, + {"build-site": {"result": "success"}, "smoke": {"result": "failure"}}, + ) + self.assertEqual(self.status(report, "browse/go.tgz")["status"], "passed") + self.assertEqual( + self.status(report, "browse/rust.tgz")["status"], "not-selected" + ) + + def test_future_evidence_attempt_is_rejected(self): + self.make_inventory("browse", {"rust.tgz": b"future"}, attempt=2) + _, result = self.make_report( + {"archives": ["rust"], "smoke": ["rust"]}, ok=False + ) + self.assertEqual(result.returncode, 2) + + def test_old_success_does_not_certify_failed_or_masked_rerun(self): + self.make_inventory("browse", {"spinel.tgz": b"native"}) + self.make_validation("smoke-spinel", b"native", revision=SPINEL) + self.env["GITHUB_RUN_ATTEMPT"] = "2" + for result in ["failure", "success"]: + with self.subTest(result=result): + report, _ = self.make_report( + {"archives": ["spinel"], "jobs": ["smoke-spinel"]}, + { + "build-site": {"result": "success"}, + "smoke-spinel": { + "result": result, + "outputs": {"execution": "failure"}, + }, + }, + ) + item = self.status(report, "browse/spinel.tgz") + self.assertEqual(item["status"], "unverified") + self.assertEqual(item["validation_attempt"], 1) + + def test_publication_checks_bytes_not_validation_success(self): + self.make_inventory( + "browse", {"rust.tgz": b"repro", "rust.zip": b"zip"}, outcome="failure" + ) + report, _ = self.make_report({"archives": ["rust"], "smoke": ["rust"]}) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + site = self.root / "site-browse-1" + args = ("verify", "--root", site, "--report", self.root / "report.json") + self.run_cli(*args) + self.assertEqual((site / "browse/rust.tgz").read_bytes(), b"repro") + document = json.loads((self.root / "report.json").read_text()) + self.assertTrue(self.status(document, "browse/rust.tgz")["published"]) + (site / "browse/rust.tgz").unlink() + self.run_cli(*args) + document = json.loads((self.root / "report.json").read_text()) + self.assertFalse(self.status(document, "browse/rust.tgz")["published"]) + self.assertEqual(self.status(document, "browse/rust.tgz")["status"], "failed") + (site / "browse/rust.zip").write_bytes(b"changed") + self.assertEqual(self.run_cli(*args, ok=False).returncode, 2) + (site / "browse/rust.zip").write_bytes(b"zip") + (site / "browse/unknown.tgz").write_bytes(b"new") + self.assertEqual(self.run_cli(*args, ok=False).returncode, 2) + (site / "browse/unknown.tgz").unlink() + self.assertEqual( + self.run_cli( + *args, ok=False, extra_env={"GITHUB_SHA": "3" * 40} + ).returncode, + 2, + ) + + def test_cli_rejects_mismatched_path_check_and_missing_identity(self): + result = self.run_cli( + "validation", + "--archive", + self.root / "none", + "--path", + "browse/spinel.tgz", + "--check", + "smoke-rust", + "--outcome", + "success", + "--out", + self.root / "out.json", + ok=False, + ) + self.assertEqual(result.returncode, 2) + result = self.run_cli( + "inventory", + "--group", + "browse", + "--root", + self.root, + "--out", + self.root / "out.json", + "--outcome", + "success", + ok=False, + extra_env={"GITHUB_SHA": "short"}, + ) + self.assertEqual(result.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ci_plan_test.py b/tests/ci_plan_test.py index d1144af50..19624e8c2 100644 --- a/tests/ci_plan_test.py +++ b/tests/ci_plan_test.py @@ -16,18 +16,32 @@ class Routing(unittest.TestCase): - def test_shared_compiler_and_runtime_have_only_the_nine_execution_floor(self): - plan = ci.select( - [ - "src/analyze/call.rs", - "src/lower/rails.rs", - "runtime/ruby/relation.rb", - "tests/new_regression.rs", - ] - ) + def extras(self, plan): + return set(plan["jobs"]) - set(ci.BASE) + + def test_general_analysis_and_lowering_retain_base(self): + plan = ci.select(["src/analyze/call.rs", "src/lower/rails.rs"]) self.assertEqual(plan["jobs"], ci.BASE) self.assertEqual(plan["archives"], []) - self.assertFalse(plan["wasm"]) + + def test_native_only_test_selects_core_without_archives_or_campfire(self): + plan = ci.select(["tests/spinel_toolchain.rs"]) + self.assertEqual(self.extras(plan), set(ci.CORE)) + self.assertEqual(plan["spinel_tests"], []) + self.assertEqual(plan["archives"], []) + self.assertFalse( + any("campfire" in job for job in plan["jobs"] if job not in ci.BASE) + ) + + def test_shared_runtime_rb_and_rbs_select_framework_native_coverage(self): + for path in ["runtime/ruby/active_record.rb", "runtime/ruby/test/model.rbs"]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual( + self.extras(plan), set(ci.CORE) | {"framework-tests-spinel"} + ) + self.assertEqual(plan["spinel_tests"], ["framework_tests_spinel"]) + self.assertEqual(plan["archives"], []) def test_draft_overrides_full_and_target_expansion(self): plan = ci.select( @@ -59,13 +73,94 @@ def test_framework_and_toolchain_tests_belong_to_compare(self): self.assertEqual(plan["extra_compare"], ["swift"]) self.assertEqual(plan["smoke"], ["swift"]) - def test_ruby_emit_and_spinel_adapters_cover_consuming_family(self): - for path in ["src/emit/ruby.rs", "runtime/spinel/db_jruby.rb"]: + def test_ruby_emit_preserves_interpreted_family_and_adds_native_core(self): + plan = ci.select(["src/emit/ruby.rs"]) + self.assertIn("compare-jruby", plan["jobs"]) + self.assertEqual(plan["smoke"], ["ruby", "jruby"]) + self.assertTrue(set(ci.CORE).issubset(plan["jobs"])) + + def test_focused_tests_select_only_themselves(self): + for binary in ci.SPINEL_TESTS: + with self.subTest(binary=binary): + plan = ci.select([f"tests/{binary}.rs"]) + self.assertEqual(plan["spinel_tests"], [binary]) + self.assertEqual( + self.extras(plan), set(ci.CORE) | {"framework-tests-spinel"} + ) + + def test_runtime_owners_choose_asymmetric_focused_binaries(self): + cases = { + "runtime/spinel/web_push_crypto.rb": "spinel_web_push_crypto", + "runtime/spinel/signed_cookies.rbs": "rails_compat_vectors_spinel", + "runtime/spinel/sqlite_adapter.rb": "spinel_db_lease", + "runtime/spinel/active_record_equality_spinel.rb": "framework_tests_spinel", + "runtime/spinel/param_builder.rb": "spinel_param_builder", + "runtime/spinel/multipart.rb": "spinel_param_builder", + } + for path, binary in cases.items(): + with self.subTest(path=path): + self.assertEqual(ci.select([path])["spinel_tests"], [binary]) + + def test_interpreter_only_files_do_not_start_native_work(self): + for path, targets in { + "runtime/spinel/db_jruby.rb": ["jruby"], + "runtime/spinel/markly_jruby.rb": ["jruby"], + "runtime/spinel/scaffold/ruby_overlay/main.rb": ["ruby", "jruby"], + "runtime/spinel/module_delegate.rb": ["ruby", "jruby"], + }.items(): with self.subTest(path=path): plan = ci.select([path]) + self.assertEqual(plan["smoke"], targets) self.assertIn("compare-jruby", plan["jobs"]) - self.assertTrue(plan["spinel"]) - self.assertEqual(plan["smoke"], ["ruby", "jruby"]) + self.assertNotIn("build-spinel", plan["jobs"]) + self.assertEqual(plan["spinel_tests"], []) + self.assertEqual(ci.select(["README.md"])["jobs"], ci.BASE) + + def test_shared_runtime_and_driver_inputs_select_real_harnesses(self): + cases = { + "runtime/ruby/action_controller/message_verifier.rbs": [ + "framework_tests_spinel", + "rails_compat_vectors_spinel", + ], + "runtime/ruby/params.rb": [ + "framework_tests_spinel", + "spinel_param_builder", + ], + "runtime/spinel/base64.rb": [ + "spinel_web_push_crypto", + "rails_compat_vectors_spinel", + ], + "tests/spinel_db_lease.rb": ["spinel_db_lease"], + "tests/params_vectors/canon.rb": ["spinel_param_builder"], + "tests/rails_compat_vectors.rb": ["rails_compat_vectors_spinel"], + } + for path, expected in cases.items(): + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual(plan["spinel_tests"], expected) + self.assertTrue(set(ci.CORE).issubset(plan["jobs"])) + self.assertNotIn("smoke-campfire", plan["jobs"]) + + def test_routing_union_is_order_independent(self): + paths = [ + "runtime/spinel/web_push_crypto.rb", + "runtime/spinel/fragment_cache.rb", + ] + self.assertEqual( + ci.select(paths)["spinel_tests"], ci.select(paths[::-1])["spinel_tests"] + ) + self.assertEqual( + ci.select(paths)["spinel_tests"], + ["framework_tests_spinel", "spinel_web_push_crypto"], + ) + + def test_union_and_deletion_paths_keep_each_owner(self): + plan = ci.select( + ["runtime/spinel/web_push_crypto.rb", "runtime/spinel/sqlite_adapter.rb"] + ) + self.assertEqual( + plan["spinel_tests"], ["spinel_web_push_crypto", "spinel_db_lease"] + ) def test_wasm_changes_have_no_archive_or_spinel_fanout(self): plan = ci.select(["wasm/lib/driver.mjs"]) @@ -79,12 +174,18 @@ def test_packaging_and_unknown_target_changes_expand_to_full(self): "src/emit/newlang.rs", "tests/framework_tests_newlang.rs", "scripts/ci-plan.py", + "Cargo.toml", + "scripts/ci-reuse.py", + "tests/ci_archive_evidence_test.py", ]: with self.subTest(path=path): self.assertEqual(ci.select([path])["smoke"], ci.TARGETS) def test_full_manual_and_publication_are_distinct(self): plan = ci.select([], full=True) + self.assertEqual(plan["spinel_tests"], ci.SPINEL_TESTS) + self.assertTrue(set(ci.SPINEL11).issubset(plan["jobs"])) + self.assertIn("archive-results", plan["required"]) self.assertIn("writebook-inventory", plan["required"]) self.assertNotIn("deploy", plan["jobs"]) self.assertIn( @@ -102,6 +203,49 @@ def test_shared_smoke_and_compare_harnesses_select_their_owners(self): ci.select(["tools/compare/src/main.rs"])["extra_compare"], ["crystal", "kotlin", "swift", "csharp", "go", "elixir", "python"], ) + self.assertTrue( + set(ci.CORE).issubset(ci.select(["tools/compare/src/main.rs"])["jobs"]) + ) + + def test_spinel_archive_and_campfire_paths_have_exact_heavy_owners(self): + scaffold = ci.select(["runtime/spinel/scaffold/Makefile"]) + self.assertEqual( + self.extras(scaffold), + set(ci.CORE) | {"build-site", "smoke-spinel", "archive-results"}, + ) + self.assertEqual(scaffold["archives"], ["spinel"]) + compare = ci.select(["scripts/campfire-compare-diff.rb"]) + self.assertEqual( + self.extras(compare), + { + "build-spinel", + "build-campfire-compare-spinel", + "campfire-compare-spinel", + }, + ) + db = ci.select(["scripts/campfire-db-differential"]) + self.assertEqual( + self.extras(db), {"build-spinel", "campfire-db-differential-spinel"} + ) + archive = ci.select(["e2e/campfire/assets.spec.js"]) + self.assertEqual( + self.extras(archive), + { + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + "archive-results", + }, + ) + self.assertEqual( + ci.select(["scripts/campfire-docker-files"])["jobs"], + ci.select(["scripts/build-campfire-archive"])["jobs"], + ) + self.assertNotIn( + "campfire-compare-spinel", + ci.select(["scripts/campfire-docker-files"])["jobs"], + ) class Results(unittest.TestCase): diff --git a/tests/workflow_yaml_parses.rs b/tests/workflow_yaml_parses.rs index 614720323..a0a4bafa3 100644 --- a/tests/workflow_yaml_parses.rs +++ b/tests/workflow_yaml_parses.rs @@ -90,16 +90,12 @@ fn spinel_cache_download_failure_falls_back_without_hiding_build_failures() { .unwrap(); assert!(output.status.success(), "{output:?}"); assert_eq!(fs::read_to_string(&env_file).unwrap(), "NO_CCACHE=1\n"); - assert!( - String::from_utf8(output.stdout) - .unwrap() - .contains("::warning::") - ); - assert!( - fs::read_to_string(summary) - .unwrap() - .contains("without compiler cache") - ); + assert!(String::from_utf8(output.stdout) + .unwrap() + .contains("::warning::")); + assert!(fs::read_to_string(summary) + .unwrap() + .contains("without compiler cache")); // Execute the actual build bodies with controlled make exits, both with // caching enabled and after sourcing the fallback's exported environment. @@ -360,25 +356,26 @@ fn pr_archives_remain_tested_without_pages_publication_work() { .iter() .find(|step| step["name"].as_str() == Some("Upload browse archives")) .unwrap(); - assert!(archives.get("if").is_none(), "PR smoke needs the archives"); + assert_eq!(archives["if"].as_str(), Some("always()")); assert_eq!(archives["with"]["name"].as_str(), Some("browse-archives")); - let pages = steps + let pages = jobs["assemble-site"]["steps"] + .as_sequence() + .unwrap() .iter() .find(|step| step["name"].as_str() == Some("Upload Pages artifact")) .unwrap(); - assert_eq!( - pages["if"].as_str(), - Some("needs.plan.outputs.publish == 'true'") - ); - // Keep the status function: a failed Campfire floor must not suppress - // its explanatory publication, but cancellation or a bad site must. + assert!(pages.get("if").is_none()); + // Publication waits for the report that describes the exact archive bytes. assert_eq!( jobs["assemble-site"]["if"].as_str(), Some( - "${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' }}" + "${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' && needs.archive-results.result == 'success' }}" ) ); - assert!(jobs.get("deploy").is_none(), "PR validation must not carry deployment privileges"); + assert!( + jobs.get("deploy").is_none(), + "PR validation must not carry deployment privileges" + ); } #[test] @@ -386,7 +383,12 @@ fn draft_transitions_replace_the_previous_pr_run() { let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); let events = ci["on"]["pull_request"]["types"].as_sequence().unwrap(); - for event in ["ready_for_review", "converted_to_draft", "labeled", "unlabeled"] { + for event in [ + "ready_for_review", + "converted_to_draft", + "labeled", + "unlabeled", + ] { assert!(events.iter().any(|value| value.as_str() == Some(event))); } assert_eq!( @@ -414,9 +416,10 @@ fn campfire_comparisons_require_an_uploaded_binary_and_report_blocking() { assert_eq!(producer["continue-on-error"].as_bool(), Some(true)); assert_eq!(consumer["continue-on-error"].as_bool(), Some(true)); assert_eq!( - consumer["needs"].as_str(), + consumer["needs"][0].as_str(), Some("build-campfire-compare-spinel") ); + assert_eq!(consumer["needs"][1].as_str(), Some("plan")); assert_eq!( producer["outputs"]["artifact-id"].as_str(), Some("${{ steps.binary.outputs.artifact-id }}") @@ -424,7 +427,7 @@ fn campfire_comparisons_require_an_uploaded_binary_and_report_blocking() { assert_eq!( consumer["if"].as_str(), Some( - "${{ !cancelled() && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }}" + "${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare-spinel') && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }}" ) ); let steps = producer["steps"].as_sequence().unwrap(); @@ -728,7 +731,8 @@ fn spinel_model_differential_does_not_wait_for_the_gc_comparison_build() { let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str(&src).expect("parse CI workflow"); let jobs = &ci["jobs"]; let db = &jobs["campfire-db-differential-spinel"]; - assert_eq!(db["needs"].as_str(), Some("build-spinel")); + assert_eq!(db["needs"][0].as_str(), Some("build-spinel")); + assert_eq!(db["needs"][1].as_str(), Some("plan")); assert_eq!(db["continue-on-error"].as_bool(), Some(true)); let command = "scripts/campfire-db-differential --spinel /tmp/campfire"; @@ -744,7 +748,11 @@ fn spinel_model_differential_does_not_wait_for_the_gc_comparison_build() { ); let gc = &jobs["campfire-compare-spinel"]; - assert_eq!(gc["needs"].as_str(), Some("build-campfire-compare-spinel")); + assert_eq!( + gc["needs"][0].as_str(), + Some("build-campfire-compare-spinel") + ); + assert_eq!(gc["needs"][1].as_str(), Some("plan")); let modes: Vec<_> = gc["strategy"]["matrix"]["include"] .as_sequence() .expect("GC matrix") @@ -984,38 +992,64 @@ fn reused_checks_keep_cargo_dependencies_locked_and_upload_only_execution_receip #[test] fn routing_and_required_results_reject_false_green() { - let result = std::process::Command::new("python3") - .args(["-B", "tests/ci_plan_test.py", "-v"]) - .output() - .expect("CI routing tests require python3"); - assert!(result.status.success(), "{}\n{}", - String::from_utf8_lossy(&result.stdout), String::from_utf8_lossy(&result.stderr)); + for test in ["tests/ci_plan_test.py", "tests/ci_archive_evidence_test.py"] { + let result = std::process::Command::new("python3") + .args(["-B", test, "-v"]) + .output() + .expect("CI helper tests require python3"); + assert!( + result.status.success(), + "{test}:\n{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + } } #[test] fn compact_and_extra_compare_share_commands_but_not_results() { - let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str( - &fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); let jobs = &ci["jobs"]; - assert_eq!(jobs["compare"]["strategy"]["matrix"]["target"], - serde_yaml_ng::from_str::("[rust, typescript]").unwrap()); + assert_eq!( + jobs["compare"]["strategy"]["matrix"]["target"], + serde_yaml_ng::from_str::("[rust, typescript]").unwrap() + ); assert_eq!(jobs["compare"]["steps"], jobs["compare-extra"]["steps"]); - assert_eq!(jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), Some(2)); + assert_eq!( + jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), + Some(2) + ); assert_eq!(jobs["smoke"]["strategy"]["max-parallel"].as_u64(), Some(2)); let smoke_guard = jobs["smoke"]["if"].as_str().unwrap(); - for condition in ["!cancelled()", "needs.plan.result == 'success'", "needs.build-site.result == 'success'"] { - assert!(smoke_guard.contains(condition), "selected smoke must run after its skipped WASM ancestor: {condition}"); + for condition in [ + "!cancelled()", + "needs.plan.result == 'success'", + "needs.build-site.result == 'success'", + ] { + assert!( + smoke_guard.contains(condition), + "selected smoke must run after its skipped WASM ancestor: {condition}" + ); } - assert_eq!(jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), Some(1)); - assert!(ci["on"]["pull_request"].get("paths-ignore").is_none(), - "required status must run even for documentation-only PRs"); + assert_eq!( + jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), + Some(1) + ); + assert!( + ci["on"]["pull_request"].get("paths-ignore").is_none(), + "required status must run even for documentation-only PRs" + ); for name in ["compact-required", "ci-required"] { assert_eq!(jobs[name]["if"].as_str(), Some("always()")); } let gate = jobs["ci-required"]["needs"].as_sequence().unwrap(); for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { if name != "ci-required" { - assert!(gate.iter().any(|v| v.as_str() == Some(name)), "missing result: {name}"); + assert!( + gate.iter().any(|v| v.as_str() == Some(name)), + "missing result: {name}" + ); } } assert_eq!(ci["permissions"]["contents"].as_str(), Some("read")); @@ -1023,46 +1057,211 @@ fn compact_and_extra_compare_share_commands_but_not_results() { assert!(ci["permissions"].get("id-token").is_none()); } +#[cfg(unix)] #[test] -fn full_scheduler_only_skips_completed_inputs_and_never_grants_pr_deploy_permissions() { - let full: serde_yaml_ng::Value = serde_yaml_ng::from_str( - &fs::read_to_string(".github/workflows/full-ci.yml").unwrap()).unwrap(); - assert_eq!(full["on"]["schedule"][0]["cron"].as_str(), Some("17 */4 * * *")); +fn focused_framework_loop_runs_every_selection_and_preserves_failure() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let step = ci["jobs"]["framework-tests-spinel"]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["name"].as_str() == Some("Run selected native framework checks")) + .unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("framework-loop-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + let cargo = root.join("cargo"); + fs::write( + &cargo, + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CARGO_LOG\"\n[ \"$3\" != fails ]\n", + ) + .unwrap(); + fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap(); + for (tests, expected_success, expected_log) in [ + ("first second", true, "test --test first -- --ignored --nocapture\ntest --test second -- --ignored --nocapture\n"), + ("fails survivor", false, "test --test fails -- --ignored --nocapture\ntest --test survivor -- --ignored --nocapture\n"), + ("", false, ""), + ] { + let log = root.join("cargo.log"); + fs::write(&log, "").unwrap(); + let result = Command::new("bash").args(["-e", "-o", "pipefail", "-c", step["run"].as_str().unwrap()]) + .env("TESTS", tests).env("CARGO_LOG", &log) + .env("PATH", format!("{}:{}", root.display(), std::env::var("PATH").unwrap())) + .output().unwrap(); + assert_eq!(result.status.success(), expected_success, "{result:?}"); + assert_eq!(fs::read_to_string(log).unwrap(), expected_log); + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn spinel_jobs_are_selected_explicitly_and_archive_evidence_reaches_pages() { + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + for name in [ + "framework-tests-spinel", + "campfire-db-differential-spinel", + "toolchain-spinel", + "compare-spinel", + "smoke-spinel", + "smoke-campfire", + ] { + let job = &jobs[name]; + let needs = job["needs"] + .as_sequence() + .expect("Spinel consumer needs plan and producer"); + assert!( + needs.iter().any(|need| need.as_str() == Some("plan")), + "{name}" + ); + assert!( + job["if"].as_str().unwrap().contains(&format!( + "contains(fromJSON(needs.plan.outputs.jobs), '{name}')" + )), + "{name}" + ); + } + + for (job_name, artifact_name) in [ + ("build-site", "browse-archives"), + ("build-campfire-archive", "campfire-archive"), + ] { + let upload = jobs[job_name]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["with"]["name"].as_str() == Some(artifact_name)) + .unwrap(); + assert_eq!(upload["if"].as_str(), Some("always()")); + assert!(upload["with"]["path"].as_str().unwrap().ends_with("*.tgz")); + } + let report = &jobs["archive-results"]; + for dependency in [ + "build-site", + "build-campfire-archive", + "smoke", + "smoke-spinel", + "smoke-campfire", + "smoke-campfire-docker", + ] { + assert!(report["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some(dependency))); + } + let report_steps = report["steps"].as_sequence().unwrap(); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Collect this run's archive evidence")) + .unwrap()["with"]["pattern"] + .as_str(), + Some("ci-archive-*") + ); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Save archive outcome report")) + .unwrap()["with"]["name"] + .as_str(), + Some("archive-results") + ); + + let assemble = &jobs["assemble-site"]; + assert!(assemble["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results"))); + let steps = assemble["steps"].as_sequence().unwrap(); + let verify = steps.iter().position(|step| step["run"].as_str() == Some("python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json")).expect("archive verification step"); + let pages = steps + .iter() + .position(|step| { + step["uses"] + .as_str() + .is_some_and(|uses| uses.starts_with("actions/upload-pages-artifact@")) + }) + .unwrap(); + assert!(verify < pages); + assert!(jobs["ci-required"]["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results"))); +} + +#[test] +fn full_scheduler_runs_every_preflight_success_fresh_and_never_grants_pr_deploy_permissions() { + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + assert_eq!( + full["on"]["schedule"][0]["cron"].as_str(), + Some("17 */4 * * *") + ); assert!(full["on"].get("push").is_none()); assert!(full["on"].get("pull_request").is_none()); - assert_eq!(full["concurrency"]["cancel-in-progress"].as_bool(), Some(false)); + assert_eq!( + full["concurrency"]["cancel-in-progress"].as_bool(), + Some(false) + ); let jobs = &full["jobs"]; let preflight = &jobs["preflight"]; - let lookup = preflight["steps"].as_sequence().unwrap().iter() - .find(|s| s["id"].as_str() == Some("lookup")).unwrap(); - assert_eq!(lookup["uses"].as_str(), Some("actions/cache/restore@v4")); - assert_eq!(lookup["with"]["lookup-only"].as_bool(), Some(true)); - assert!(lookup["with"].get("restore-keys").is_none()); - let skip = preflight["outputs"]["run"].as_str().unwrap(); - for condition in ["github.event_name != 'schedule'", "github.run_attempt != 1", - "steps.lookup.outcome != 'success'", "steps.lookup.outputs.cache-hit != 'true'"] { - assert!(skip.contains(condition), "skip must fail open: {condition}"); - } - let checkpoint = jobs["checkpoint"]["if"].as_str().unwrap(); - for condition in ["!cancelled()", "github.repository == 'rubys/roundhouse'", - "github.ref == 'refs/heads/main'", "needs.preflight.outputs.known == 'true'", - "needs.validation.result == 'success'", "needs.validation.outputs.complete == 'true'", - "needs.deploy.result == 'success'"] { - assert!(checkpoint.contains(condition), "unsafe completion marker: {condition}"); - } - assert_eq!(jobs["validation"]["uses"].as_str(), Some("./.github/workflows/ci.yml")); + assert!(preflight["outputs"].get("run").is_none()); + assert!(preflight["outputs"].get("known").is_none()); + assert!(jobs.get("checkpoint").is_none()); + let preflight_text = serde_yaml_ng::to_string(preflight).unwrap(); + assert!(!preflight_text.contains("actions/cache")); + assert_eq!( + preflight_text + .matches("repos/matz/spinel/commits/master") + .count(), + 1 + ); + assert_eq!( + jobs["validation"]["uses"].as_str(), + Some("./.github/workflows/ci.yml") + ); assert_eq!(jobs["validation"]["with"]["full"].as_bool(), Some(true)); - assert_eq!(jobs["validation"]["permissions"]["contents"].as_str(), Some("read")); - assert_eq!(jobs["validation"]["permissions"]["actions"].as_str(), Some("read")); - assert_eq!(jobs["validation"]["permissions"].as_mapping().unwrap().len(), 2); + assert_eq!( + jobs["validation"]["permissions"]["contents"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"]["actions"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"] + .as_mapping() + .unwrap() + .len(), + 2 + ); assert_eq!(full["permissions"]["contents"].as_str(), Some("read")); let deploy = &jobs["deploy"]; let guard = deploy["if"].as_str().unwrap(); assert!(guard.contains("needs.validation.outputs.publication-ready == 'true'")); - assert!(!guard.contains("needs.validation.result == 'success'"), "extra failures cannot hide repro publication"); + assert!( + !guard.contains("needs.validation.result == 'success'"), + "extra failures cannot hide repro publication" + ); assert!(deploy.get("continue-on-error").is_none()); assert_eq!(deploy["permissions"]["pages"].as_str(), Some("write")); - assert!(deploy["steps"][0]["run"].as_str().unwrap().contains("$VALIDATED_SHA")); + assert!(deploy["steps"][0]["run"] + .as_str() + .unwrap() + .contains("$VALIDATED_SHA")); } #[cfg(unix)] @@ -1071,49 +1270,114 @@ fn scheduler_preflight_executes_publication_guards_and_unknown_input_fallback() use std::os::unix::fs::PermissionsExt; use std::process::Command; - let full: serde_yaml_ng::Value = serde_yaml_ng::from_str( - &fs::read_to_string(".github/workflows/full-ci.yml").unwrap()).unwrap(); - let body = full["jobs"]["preflight"]["steps"][0]["run"].as_str().unwrap(); + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + let body = full["jobs"]["preflight"]["steps"][0]["run"] + .as_str() + .unwrap(); let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); let root = std::env::temp_dir().join(format!("full-preflight-{}-{unique}", std::process::id())); fs::create_dir(&root).unwrap(); for (name, script) in [ - ("gh", "#!/bin/sh\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n"), - ("date", "#!/bin/sh\nprintf '2026-10-02\\n'\n"), + ("gh", "#!/bin/sh\nprintf 'called\\n' >> \"$GH_LOG\"\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n"), ] { let path = root.join(name); fs::write(&path, script).unwrap(); fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); } - let main_sha = "1234567890123456789012345678901234567890"; let spinel_sha = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; for (event, repo, reference, publish, revision, expected) in [ - ("schedule", "rubys/roundhouse", "refs/heads/main", "false", spinel_sha, Some(("true", "true", spinel_sha))), - ("schedule", "rubys/roundhouse", "refs/heads/main", "false", "unavailable", Some(("true", "false", "master"))), - ("schedule", "rubys/roundhouse", "refs/heads/main", "false", "malformed", Some(("true", "false", "master"))), - ("workflow_dispatch", "contributor/roundhouse", "refs/heads/topic", "false", spinel_sha, Some(("false", "true", spinel_sha))), - ("workflow_dispatch", "rubys/roundhouse", "refs/heads/main", "true", spinel_sha, Some(("true", "true", spinel_sha))), - ("schedule", "contributor/roundhouse", "refs/heads/main", "false", spinel_sha, None), - ("workflow_dispatch", "rubys/roundhouse", "refs/heads/topic", "true", spinel_sha, None), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "unavailable", + Some(("true", "master")), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "malformed", + Some(("true", "master")), + ), + ( + "workflow_dispatch", + "contributor/roundhouse", + "refs/heads/topic", + "false", + spinel_sha, + Some(("false", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/main", + "true", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/topic", + "true", + spinel_sha, + None, + ), ] { let outputs = root.join("outputs"); fs::write(&outputs, "").unwrap(); let result = Command::new("bash") .args(["-e", "-o", "pipefail", "-c", body]) - .env("PATH", format!("{}:{}", root.display(), std::env::var("PATH").unwrap())) - .env("EVENT", event).env("GITHUB_REPOSITORY", repo) - .env("GITHUB_REF", reference).env("REQUEST_PUBLISH", publish) - .env("GITHUB_SHA", main_sha).env("MOCK_SPINEL", revision) - .env("GITHUB_OUTPUT", &outputs).output().unwrap(); - assert_eq!(result.status.success(), expected.is_some(), "{event} {repo} {reference}: {result:?}"); + .env( + "PATH", + format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), + ) + .env("EVENT", event) + .env("GITHUB_REPOSITORY", repo) + .env("GITHUB_REF", reference) + .env("REQUEST_PUBLISH", publish) + .env("MOCK_SPINEL", revision) + .env("GH_LOG", root.join("gh.log")) + .env("GITHUB_OUTPUT", &outputs) + .output() + .unwrap(); + assert_eq!( + result.status.success(), + expected.is_some(), + "{event} {repo} {reference}: {result:?}" + ); let actual = fs::read_to_string(outputs).unwrap(); - if let Some((published, known, resolved)) = expected { - assert_eq!(actual, format!( - "spinel={resolved}\nknown={known}\npublish={published}\nkey=full-ci-v1-{main_sha}-{resolved}-2026-10-02\n")); + if let Some((published, resolved)) = expected { + assert_eq!(actual, format!("spinel={resolved}\npublish={published}\n")); } else { - assert!(actual.is_empty(), "rejected publication must not issue a cache key"); + assert!( + actual.is_empty(), + "rejected publication must not issue outputs" + ); } } + assert_eq!( + fs::read_to_string(root.join("gh.log")) + .unwrap() + .lines() + .count(), + 5 + ); fs::remove_dir_all(root).unwrap(); } From 37c2139b63b813f99a39c04ad11c58f1a8b23d73 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 18:39:51 +0000 Subject: [PATCH 3/6] Present CI aggregation as an informational summary Keep selected-check failure reporting and publication guards unchanged, but leave merge decisions to maintainers without suggesting a mandatory status or branch-protection change. Document Python helpers briefly. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- .github/workflows/ci.yml | 8 +++++--- AGENTS.md | 5 +++-- README.md | 5 +++-- docs/ci-reuse.md | 18 +++++++++++------- tests/workflow_yaml_parses.rs | 16 +++++++++++----- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 197d3e261..2e5316291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ on: default: '' outputs: complete: - value: ${{ jobs.ci-required.outputs.complete }} + value: ${{ jobs.ci-summary.outputs.complete }} publication-ready: value: ${{ jobs.compact-required.outputs.passed == 'true' && jobs.assemble-site.outputs.ready == 'true' }} @@ -3169,7 +3169,9 @@ jobs: python3 scripts/ci-plan.py compact-gate echo 'passed=true' >> "$GITHUB_OUTPUT" - ci-required: + # Informational validation summary; no branch-protection or merge policy. + ci-summary: + name: CI summary needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare, compact-required, compare-extra, compare-jruby, build-wasm, browser-smoke-ide, build-site, smoke, writebook-inventory, build-spinel, framework-tests-spinel, build-campfire-compare-spinel, campfire-compare-spinel, campfire-db-differential-spinel, toolchain-spinel, compare-spinel, smoke-spinel, build-campfire-archive, smoke-campfire, smoke-campfire-docker, archive-results, assemble-site] if: always() runs-on: ubuntu-latest @@ -3178,7 +3180,7 @@ jobs: complete: ${{ steps.gate.outputs.complete }} steps: - uses: actions/checkout@v5 - - name: Reject missing, skipped or failed required coverage + - name: Report missing, skipped or failed selected checks id: gate env: CI_PLAN: ${{ needs.plan.outputs.plan }} diff --git a/AGENTS.md b/AGENTS.md index 336315e96..a9fe2deb2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,8 +89,9 @@ defect even if the build is green. - **Outside contributors: fork, and open a pull request against `main`.** CI runs a compact floor plus targeted lanes; add `ci:full` for the complete PR matrix. Full main validation/publication is scheduled every - four hours and executes freshly every cycle. `ci-required` - gates selected blocking coverage; see [CI coverage](docs/ci-reuse.md). + four hours and executes freshly every cycle. `CI summary` reports + selected checks, not a mandatory merge gate; maintainers decide when + to merge. See [CI coverage](docs/ci-reuse.md). You do not need every toolchain locally; CI covers the missing lanes. Before opening one: `bin/rh fixture` (the test fixtures are generated, not checked in — see below), `cargo test --lib` plus the targeted diff --git a/README.md b/README.md index 10e662c4d..035046d7f 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ the prerequisites. - [**Campfire**](https://rubys.github.io/roundhouse/apps/campfire.html) — the compiled product, as a Docker archive. - [**Browse**](https://rubys.github.io/roundhouse/browse/) — what every - emitter produces from the blog fixture, updated on each push. + emitter produces from the blog fixture, refreshed by scheduled full validation. - [**Bench**](https://rubys.github.io/roundhouse/bench/) — throughput, memory and latency across the live targets on a fixed box, against Rails as it ships. @@ -173,7 +173,8 @@ conversation first: CI runs a compact correctness/runtime floor plus targeted checks. Add `ci:full` for full PR validation; the complete matrix also runs freshly every four hours. See [CI coverage](docs/ci-reuse.md). The -`ci-required` status gates selected blocking coverage; the Spinel +`CI summary` status reports selected checks without imposing a merge +requirement; maintainers decide when to merge. The Spinel and Campfire jobs marked `continue-on-error` track current Spinel master, and red there is a signal to investigate Roundhouse runtime, RBS or packaging as well as possible upstream drift—not proof of an diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index d051c6a64..29fe0f7d3 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -11,18 +11,22 @@ comparison including its model/database differential. These are nine validation executions, plus three small orchestration jobs -(`plan`, `compact-required`, `ci-required`). Drafts select only fixture and -unit validation. **`ci-required` is the stable merge-gate status:** it rejects -missing, skipped, cancelled or failed selected blocking checks. Unselected -jobs may skip. Advisory failures remain visible, not merge blockers. -Repository administrators should migrate required-check settings only after -observing the new status on a real PR; the workflow does not change protection. +(`plan`, `compact-required`, `ci-summary`). Drafts select only fixture and +unit validation. **`CI summary` is informational:** it reports missing, +skipped, cancelled or failed selected non-advisory checks as red. Unselected +jobs may skip; advisory failures remain separate signals. This workflow does +not configure branch protection or require the summary to pass before merging; +maintainers decide when to merge. Publication still requires the compact floor +and verified assembly. The planner compares the actual PR merge tree with its base, not just the last commit. Renames/deletions retain both ownership sets. Unknown diff identity -expands to full validation. Documentation-only PRs still get the required +expands to full validation. Documentation-only PRs still get the summary status instead of being left pending by workflow-level path filters. +The CI helpers use Python's standard library, following the existing receipt +helper, without additional Python packages. + | Changed inputs | Additional coverage | |---|---| | Target emitter file **or directory**, target runtime, toolchain/framework test | Owning comparison and archive smoke; existing embedded framework/toolchain suites stay intact | diff --git a/tests/workflow_yaml_parses.rs b/tests/workflow_yaml_parses.rs index a0a4bafa3..a63199d58 100644 --- a/tests/workflow_yaml_parses.rs +++ b/tests/workflow_yaml_parses.rs @@ -1038,14 +1038,20 @@ fn compact_and_extra_compare_share_commands_but_not_results() { ); assert!( ci["on"]["pull_request"].get("paths-ignore").is_none(), - "required status must run even for documentation-only PRs" + "summary must run even for documentation-only PRs" ); - for name in ["compact-required", "ci-required"] { + assert!(jobs.get("ci-required").is_none()); + assert_eq!(jobs["ci-summary"]["name"].as_str(), Some("CI summary")); + assert_eq!( + ci["on"]["workflow_call"]["outputs"]["complete"]["value"].as_str(), + Some("${{ jobs.ci-summary.outputs.complete }}") + ); + for name in ["compact-required", "ci-summary"] { assert_eq!(jobs[name]["if"].as_str(), Some("always()")); } - let gate = jobs["ci-required"]["needs"].as_sequence().unwrap(); + let gate = jobs["ci-summary"]["needs"].as_sequence().unwrap(); for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { - if name != "ci-required" { + if name != "ci-summary" { assert!( gate.iter().any(|v| v.as_str() == Some(name)), "missing result: {name}" @@ -1193,7 +1199,7 @@ fn spinel_jobs_are_selected_explicitly_and_archive_evidence_reaches_pages() { }) .unwrap(); assert!(verify < pages); - assert!(jobs["ci-required"]["needs"] + assert!(jobs["ci-summary"]["needs"] .as_sequence() .unwrap() .iter() From f747b35779c4481d310e97195c215f4a247789ec Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 18:50:02 +0000 Subject: [PATCH 4/6] Separate native and archive CI routing policies and workflow tests Preserve routing outputs while isolating native/interpreter ownership from archive and Campfire consumers. Keep new CI policy tests in a focused suite. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- scripts/ci-plan.py | 238 +++++++++---------- tests/ci_policy_workflow.rs | 422 ++++++++++++++++++++++++++++++++++ tests/workflow_yaml_parses.rs | 398 -------------------------------- 3 files changed, 531 insertions(+), 527 deletions(-) create mode 100644 tests/ci_policy_workflow.rs diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py index 9eb8b7a1d..53d1923a8 100755 --- a/scripts/ci-plan.py +++ b/scripts/ci-plan.py @@ -56,6 +56,108 @@ SHA = re.compile(r"[0-9a-f]{40}\Z") +def native_coverage(path): + """Identify native core/focused suites and interpreter-only exceptions.""" + interpreter_only = path.startswith( + "runtime/spinel/scaffold/ruby_overlay/" + ) or path in { + "runtime/spinel/db_jruby.rb", + "runtime/spinel/markly_jruby.rb", + "runtime/spinel/db_cruby.rb", + "runtime/spinel/message_digest_cruby.rb", + "runtime/spinel/module_delegate.rb", + } + native = ( + path.startswith(("runtime/ruby/", "runtime/spinel/", "src/emit/ruby/")) + or path == "src/emit/ruby.rs" + or (path.startswith("tests/spinel") and path.endswith((".rs", ".rb"))) + or path in {f"tests/{name}.rs" for name in SPINEL_TESTS} + ) and not interpreter_only + suites = set() + if path.startswith("runtime/ruby/") and path.endswith((".rb", ".rbs")): + suites.add("framework_tests_spinel") + focused = re.fullmatch(r"tests/([^/]+)\.(?:rs|rb)", path) + if focused and focused[1] in SPINEL_TESTS: + suites.add(focused[1]) + if path.startswith(("runtime/spinel/", "runtime/ruby/")) and not interpreter_only: + name = path.rsplit("/", 1)[-1] + owned_tests = set() + if any(word in name for word in ("web_push", "base64")): + owned_tests.add("spinel_web_push_crypto") + if any( + word in name + for word in ( + "signed_cookie", + "message_verifier", + "signed_id", + "message_digest", + "base64", + ) + ) or path.startswith("runtime/spinel/tep/url."): + owned_tests.add("rails_compat_vectors_spinel") + if any( + word in path for word in ("/db", "sqlite", "active_support_time_parsing") + ): + owned_tests.add("spinel_db_lease") + if any(word in name for word in ("param", "multipart", "request")): + owned_tests.add("spinel_param_builder") + if ( + path.startswith("runtime/spinel/") + and not path.startswith("runtime/spinel/scaffold/") + and not owned_tests + ): + owned_tests.add("framework_tests_spinel") + suites.update(owned_tests) + if ( + path.startswith(("tests/rails_compat/", "tests/params_vectors/")) + or path == "tests/rails_compat_vectors.rb" + ): + suites.add( + "spinel_param_builder" + if path.startswith("tests/params_vectors/") + else "rails_compat_vectors_spinel" + ) + return native, interpreter_only, suites + + +def archive_and_campfire_jobs(path, interpreter_only): + """Select packaging and Campfire consumers, not every native runtime edit.""" + jobs = set() + if path.startswith("runtime/spinel/scaffold/") and not interpreter_only: + jobs.update((*CORE, "smoke-spinel", "build-site")) + if path.startswith(("scripts/campfire-compare", "scripts/build-campfire-compare")): + jobs.update( + ("build-spinel", "build-campfire-compare-spinel", "campfire-compare-spinel") + ) + if path.startswith("scripts/campfire-db-differential"): + jobs.update(("build-spinel", "campfire-db-differential-spinel")) + campfire_archive = ( + path.startswith( + ( + "scripts/build-campfire-archive", + "scripts/campfire-archive", + "e2e/campfire/", + ) + ) + or path == "scripts/campfire-docker-files" + ) + shared_smoke = ( + path.startswith("e2e/") and not path.startswith("e2e/campfire/") + ) or path in {"scripts/smoke", "scripts/ci-playwright-install"} + if campfire_archive or shared_smoke: + jobs.update( + ( + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) + if shared_smoke: + jobs.add("smoke-spinel") + return jobs + + def select(paths, *, draft=False, full=False, publish=False): if draft: return finish( @@ -102,27 +204,13 @@ def select(paths, *, draft=False, full=False, publish=False): if test else None ) - interpreter_only = path.startswith( - "runtime/spinel/scaffold/ruby_overlay/" - ) or path in { - "runtime/spinel/db_jruby.rb", - "runtime/spinel/markly_jruby.rb", - "runtime/spinel/db_cruby.rb", - "runtime/spinel/message_digest_cruby.rb", - "runtime/spinel/module_delegate.rb", - } - native_path = ( - path.startswith(("runtime/ruby/", "runtime/spinel/", "src/emit/ruby/")) - or path == "src/emit/ruby.rs" - ) - native_path |= path.startswith("tests/spinel") and path.endswith((".rs", ".rb")) - native_path |= path in {f"tests/{name}.rs" for name in SPINEL_TESTS} - if native_path and not interpreter_only: + native, interpreter_only, owned_tests = native_coverage(path) + spinel_tests.update(owned_tests) + if native or owned_tests: spinel = True jobs_selected.update(CORE) + if native: reasons.append(f"{path}: native Spinel core") - if path.startswith("runtime/ruby/") and path.endswith((".rb", ".rbs")): - spinel_tests.add("framework_tests_spinel") if target in TARGETS or target == "spinel": owners = ( {"ruby", "jruby", "spinel"} @@ -181,118 +269,10 @@ def select(paths, *, draft=False, full=False, publish=False): jobs_selected.update(CORE) if path.startswith("tests/framework_test_support"): spinel_tests.add("framework_tests_spinel") - focused = re.fullmatch( - r"tests/(framework_tests_spinel|spinel_web_push_crypto|spinel_db_lease|spinel_param_builder|rails_compat_vectors_spinel)\.(?:rs|rb)", - path, - ) - if focused: + archive_jobs = archive_and_campfire_jobs(path, interpreter_only) + if archive_jobs: spinel = True - jobs_selected.update(CORE) - spinel_tests.add(focused[1]) - if ( - path.startswith(("runtime/spinel/", "runtime/ruby/")) - and not interpreter_only - ): - name = path.rsplit("/", 1)[-1] - owned_tests = set() - if any(word in name for word in ("web_push", "base64")): - owned_tests.add("spinel_web_push_crypto") - if any( - word in name - for word in ( - "signed_cookie", - "message_verifier", - "signed_id", - "message_digest", - "base64", - ) - ) or path.startswith("runtime/spinel/tep/url."): - owned_tests.add("rails_compat_vectors_spinel") - if any( - word in path - for word in ("/db", "sqlite", "active_support_time_parsing") - ): - owned_tests.add("spinel_db_lease") - if any(word in name for word in ("param", "multipart", "request")): - owned_tests.add("spinel_param_builder") - if ( - path.startswith("runtime/spinel/") - and not path.startswith("runtime/spinel/scaffold/") - and not owned_tests - ): - owned_tests.add("framework_tests_spinel") - spinel_tests.update(owned_tests) - if ( - path.startswith(("tests/rails_compat/", "tests/params_vectors/")) - or path == "tests/rails_compat_vectors.rb" - ): - spinel = True - jobs_selected.update(CORE) - spinel_tests.add( - "spinel_param_builder" - if path.startswith("tests/params_vectors/") - else "rails_compat_vectors_spinel" - ) - if path.startswith("runtime/spinel/scaffold/") and not interpreter_only: - spinel = True - jobs_selected.update((*CORE, "smoke-spinel", "build-site")) - if path.startswith( - ("scripts/campfire-compare", "scripts/build-campfire-compare") - ): - spinel = True - jobs_selected.update( - ( - "build-spinel", - "build-campfire-compare-spinel", - "campfire-compare-spinel", - ) - ) - if path.startswith("scripts/campfire-db-differential"): - spinel = True - jobs_selected.update(("build-spinel", "campfire-db-differential-spinel")) - if ( - path.startswith( - ("scripts/build-campfire-archive", "scripts/campfire-archive") - ) - or path == "scripts/campfire-docker-files" - ): - spinel = True - jobs_selected.update( - ( - "build-spinel", - "build-campfire-archive", - "smoke-campfire", - "smoke-campfire-docker", - ) - ) - if path.startswith("e2e/campfire/"): - spinel = True - jobs_selected.update( - ( - "build-spinel", - "build-campfire-archive", - "smoke-campfire", - "smoke-campfire-docker", - ) - ) - if path in {"scripts/smoke", "scripts/ci-playwright-install"}: - jobs_selected.update( - ( - "smoke-spinel", - "build-campfire-archive", - "smoke-campfire", - "smoke-campfire-docker", - ) - ) - if path.startswith("e2e/") and not path.startswith("e2e/campfire/"): - jobs_selected.update( - ( - "smoke-spinel", - "build-campfire-archive", - "smoke-campfire", - "smoke-campfire-docker", - ) - ) + jobs_selected.update(archive_jobs) if path in {"tests/writebook.rs", "tests/fixtures/writebook-inventory.json"}: writebook = True if full: diff --git a/tests/ci_policy_workflow.rs b/tests/ci_policy_workflow.rs new file mode 100644 index 000000000..b8d4259a4 --- /dev/null +++ b/tests/ci_policy_workflow.rs @@ -0,0 +1,422 @@ +use std::fs; + +#[test] +fn routing_and_required_results_reject_false_green() { + for test in ["tests/ci_plan_test.py", "tests/ci_archive_evidence_test.py"] { + let result = std::process::Command::new("python3") + .args(["-B", test, "-v"]) + .output() + .expect("CI helper tests require python3"); + assert!( + result.status.success(), + "{test}:\n{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + } +} + +#[test] +fn compact_and_extra_compare_share_commands_but_not_results() { + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + assert_eq!( + jobs["compare"]["strategy"]["matrix"]["target"], + serde_yaml_ng::from_str::("[rust, typescript]").unwrap() + ); + assert_eq!(jobs["compare"]["steps"], jobs["compare-extra"]["steps"]); + assert_eq!( + jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), + Some(2) + ); + assert_eq!(jobs["smoke"]["strategy"]["max-parallel"].as_u64(), Some(2)); + let smoke_guard = jobs["smoke"]["if"].as_str().unwrap(); + for condition in [ + "!cancelled()", + "needs.plan.result == 'success'", + "needs.build-site.result == 'success'", + ] { + assert!( + smoke_guard.contains(condition), + "selected smoke must run after its skipped WASM ancestor: {condition}" + ); + } + assert_eq!( + jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), + Some(1) + ); + assert!( + ci["on"]["pull_request"].get("paths-ignore").is_none(), + "summary must run even for documentation-only PRs" + ); + assert!(jobs.get("ci-required").is_none()); + assert_eq!(jobs["ci-summary"]["name"].as_str(), Some("CI summary")); + assert_eq!( + ci["on"]["workflow_call"]["outputs"]["complete"]["value"].as_str(), + Some("${{ jobs.ci-summary.outputs.complete }}") + ); + for name in ["compact-required", "ci-summary"] { + assert_eq!(jobs[name]["if"].as_str(), Some("always()")); + } + let gate = jobs["ci-summary"]["needs"].as_sequence().unwrap(); + for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { + if name != "ci-summary" { + assert!( + gate.iter().any(|v| v.as_str() == Some(name)), + "missing result: {name}" + ); + } + } + assert_eq!(ci["permissions"]["contents"].as_str(), Some("read")); + assert!(ci["permissions"].get("pages").is_none()); + assert!(ci["permissions"].get("id-token").is_none()); +} + +#[cfg(unix)] +#[test] +fn focused_framework_loop_runs_every_selection_and_preserves_failure() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let step = ci["jobs"]["framework-tests-spinel"]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["name"].as_str() == Some("Run selected native framework checks")) + .unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("framework-loop-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + let cargo = root.join("cargo"); + fs::write( + &cargo, + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CARGO_LOG\"\n[ \"$3\" != fails ]\n", + ) + .unwrap(); + fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap(); + for (tests, expected_success, expected_log) in [ + ( + "first second", + true, + "test --test first -- --ignored --nocapture\ntest --test second -- --ignored --nocapture\n", + ), + ( + "fails survivor", + false, + "test --test fails -- --ignored --nocapture\ntest --test survivor -- --ignored --nocapture\n", + ), + ("", false, ""), + ] { + let log = root.join("cargo.log"); + fs::write(&log, "").unwrap(); + let result = Command::new("bash") + .args(["-e", "-o", "pipefail", "-c", step["run"].as_str().unwrap()]) + .env("TESTS", tests) + .env("CARGO_LOG", &log) + .env( + "PATH", + format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), + ) + .output() + .unwrap(); + assert_eq!(result.status.success(), expected_success, "{result:?}"); + assert_eq!(fs::read_to_string(log).unwrap(), expected_log); + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn spinel_jobs_are_selected_explicitly_and_archive_evidence_reaches_pages() { + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + for name in [ + "framework-tests-spinel", + "campfire-db-differential-spinel", + "toolchain-spinel", + "compare-spinel", + "smoke-spinel", + "smoke-campfire", + ] { + let job = &jobs[name]; + let needs = job["needs"] + .as_sequence() + .expect("Spinel consumer needs plan and producer"); + assert!( + needs.iter().any(|need| need.as_str() == Some("plan")), + "{name}" + ); + assert!( + job["if"].as_str().unwrap().contains(&format!( + "contains(fromJSON(needs.plan.outputs.jobs), '{name}')" + )), + "{name}" + ); + } + + for (job_name, artifact_name) in [ + ("build-site", "browse-archives"), + ("build-campfire-archive", "campfire-archive"), + ] { + let upload = jobs[job_name]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["with"]["name"].as_str() == Some(artifact_name)) + .unwrap(); + assert_eq!(upload["if"].as_str(), Some("always()")); + assert!(upload["with"]["path"].as_str().unwrap().ends_with("*.tgz")); + } + let report = &jobs["archive-results"]; + for dependency in [ + "build-site", + "build-campfire-archive", + "smoke", + "smoke-spinel", + "smoke-campfire", + "smoke-campfire-docker", + ] { + assert!( + report["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some(dependency)) + ); + } + let report_steps = report["steps"].as_sequence().unwrap(); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Collect this run's archive evidence")) + .unwrap()["with"]["pattern"] + .as_str(), + Some("ci-archive-*") + ); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Save archive outcome report")) + .unwrap()["with"]["name"] + .as_str(), + Some("archive-results") + ); + + let assemble = &jobs["assemble-site"]; + assert!( + assemble["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results")) + ); + let steps = assemble["steps"].as_sequence().unwrap(); + let verify = steps.iter().position(|step| step["run"].as_str() == Some("python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json")).expect("archive verification step"); + let pages = steps + .iter() + .position(|step| { + step["uses"] + .as_str() + .is_some_and(|uses| uses.starts_with("actions/upload-pages-artifact@")) + }) + .unwrap(); + assert!(verify < pages); + assert!( + jobs["ci-summary"]["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results")) + ); +} + +#[test] +fn full_scheduler_runs_every_preflight_success_fresh_and_never_grants_pr_deploy_permissions() { + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + assert_eq!( + full["on"]["schedule"][0]["cron"].as_str(), + Some("17 */4 * * *") + ); + assert!(full["on"].get("push").is_none()); + assert!(full["on"].get("pull_request").is_none()); + assert_eq!( + full["concurrency"]["cancel-in-progress"].as_bool(), + Some(false) + ); + let jobs = &full["jobs"]; + let preflight = &jobs["preflight"]; + assert!(preflight["outputs"].get("run").is_none()); + assert!(preflight["outputs"].get("known").is_none()); + assert!(jobs.get("checkpoint").is_none()); + let preflight_text = serde_yaml_ng::to_string(preflight).unwrap(); + assert!(!preflight_text.contains("actions/cache")); + assert_eq!( + preflight_text + .matches("repos/matz/spinel/commits/master") + .count(), + 1 + ); + assert_eq!( + jobs["validation"]["uses"].as_str(), + Some("./.github/workflows/ci.yml") + ); + assert_eq!(jobs["validation"]["with"]["full"].as_bool(), Some(true)); + assert_eq!( + jobs["validation"]["permissions"]["contents"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"]["actions"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"] + .as_mapping() + .unwrap() + .len(), + 2 + ); + assert_eq!(full["permissions"]["contents"].as_str(), Some("read")); + let deploy = &jobs["deploy"]; + let guard = deploy["if"].as_str().unwrap(); + assert!(guard.contains("needs.validation.outputs.publication-ready == 'true'")); + assert!( + !guard.contains("needs.validation.result == 'success'"), + "extra failures cannot hide repro publication" + ); + assert!(deploy.get("continue-on-error").is_none()); + assert_eq!(deploy["permissions"]["pages"].as_str(), Some("write")); + assert!( + deploy["steps"][0]["run"] + .as_str() + .unwrap() + .contains("$VALIDATED_SHA") + ); +} + +#[cfg(unix)] +#[test] +fn scheduler_preflight_executes_publication_guards_and_unknown_input_fallback() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + let body = full["jobs"]["preflight"]["steps"][0]["run"] + .as_str() + .unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("full-preflight-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + for (name, script) in [( + "gh", + "#!/bin/sh\nprintf 'called\\n' >> \"$GH_LOG\"\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n", + )] { + let path = root.join(name); + fs::write(&path, script).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + let spinel_sha = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + for (event, repo, reference, publish, revision, expected) in [ + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "unavailable", + Some(("true", "master")), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "malformed", + Some(("true", "master")), + ), + ( + "workflow_dispatch", + "contributor/roundhouse", + "refs/heads/topic", + "false", + spinel_sha, + Some(("false", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/main", + "true", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/topic", + "true", + spinel_sha, + None, + ), + ] { + let outputs = root.join("outputs"); + fs::write(&outputs, "").unwrap(); + let result = Command::new("bash") + .args(["-e", "-o", "pipefail", "-c", body]) + .env( + "PATH", + format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), + ) + .env("EVENT", event) + .env("GITHUB_REPOSITORY", repo) + .env("GITHUB_REF", reference) + .env("REQUEST_PUBLISH", publish) + .env("MOCK_SPINEL", revision) + .env("GH_LOG", root.join("gh.log")) + .env("GITHUB_OUTPUT", &outputs) + .output() + .unwrap(); + assert_eq!( + result.status.success(), + expected.is_some(), + "{event} {repo} {reference}: {result:?}" + ); + let actual = fs::read_to_string(outputs).unwrap(); + if let Some((published, resolved)) = expected { + assert_eq!(actual, format!("spinel={resolved}\npublish={published}\n")); + } else { + assert!( + actual.is_empty(), + "rejected publication must not issue outputs" + ); + } + } + assert_eq!( + fs::read_to_string(root.join("gh.log")) + .unwrap() + .lines() + .count(), + 5 + ); + fs::remove_dir_all(root).unwrap(); +} diff --git a/tests/workflow_yaml_parses.rs b/tests/workflow_yaml_parses.rs index a63199d58..95cacde06 100644 --- a/tests/workflow_yaml_parses.rs +++ b/tests/workflow_yaml_parses.rs @@ -989,401 +989,3 @@ fn reused_checks_keep_cargo_dependencies_locked_and_upload_only_execution_receip assert_eq!(uploads[0]["with"]["retention-days"].as_u64(), Some(7)); } } - -#[test] -fn routing_and_required_results_reject_false_green() { - for test in ["tests/ci_plan_test.py", "tests/ci_archive_evidence_test.py"] { - let result = std::process::Command::new("python3") - .args(["-B", test, "-v"]) - .output() - .expect("CI helper tests require python3"); - assert!( - result.status.success(), - "{test}:\n{}\n{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr) - ); - } -} - -#[test] -fn compact_and_extra_compare_share_commands_but_not_results() { - let ci: serde_yaml_ng::Value = - serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); - let jobs = &ci["jobs"]; - assert_eq!( - jobs["compare"]["strategy"]["matrix"]["target"], - serde_yaml_ng::from_str::("[rust, typescript]").unwrap() - ); - assert_eq!(jobs["compare"]["steps"], jobs["compare-extra"]["steps"]); - assert_eq!( - jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), - Some(2) - ); - assert_eq!(jobs["smoke"]["strategy"]["max-parallel"].as_u64(), Some(2)); - let smoke_guard = jobs["smoke"]["if"].as_str().unwrap(); - for condition in [ - "!cancelled()", - "needs.plan.result == 'success'", - "needs.build-site.result == 'success'", - ] { - assert!( - smoke_guard.contains(condition), - "selected smoke must run after its skipped WASM ancestor: {condition}" - ); - } - assert_eq!( - jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), - Some(1) - ); - assert!( - ci["on"]["pull_request"].get("paths-ignore").is_none(), - "summary must run even for documentation-only PRs" - ); - assert!(jobs.get("ci-required").is_none()); - assert_eq!(jobs["ci-summary"]["name"].as_str(), Some("CI summary")); - assert_eq!( - ci["on"]["workflow_call"]["outputs"]["complete"]["value"].as_str(), - Some("${{ jobs.ci-summary.outputs.complete }}") - ); - for name in ["compact-required", "ci-summary"] { - assert_eq!(jobs[name]["if"].as_str(), Some("always()")); - } - let gate = jobs["ci-summary"]["needs"].as_sequence().unwrap(); - for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { - if name != "ci-summary" { - assert!( - gate.iter().any(|v| v.as_str() == Some(name)), - "missing result: {name}" - ); - } - } - assert_eq!(ci["permissions"]["contents"].as_str(), Some("read")); - assert!(ci["permissions"].get("pages").is_none()); - assert!(ci["permissions"].get("id-token").is_none()); -} - -#[cfg(unix)] -#[test] -fn focused_framework_loop_runs_every_selection_and_preserves_failure() { - use std::os::unix::fs::PermissionsExt; - use std::process::Command; - - let ci: serde_yaml_ng::Value = - serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); - let step = ci["jobs"]["framework-tests-spinel"]["steps"] - .as_sequence() - .unwrap() - .iter() - .find(|step| step["name"].as_str() == Some("Run selected native framework checks")) - .unwrap(); - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos(); - let root = std::env::temp_dir().join(format!("framework-loop-{}-{unique}", std::process::id())); - fs::create_dir(&root).unwrap(); - let cargo = root.join("cargo"); - fs::write( - &cargo, - "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CARGO_LOG\"\n[ \"$3\" != fails ]\n", - ) - .unwrap(); - fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap(); - for (tests, expected_success, expected_log) in [ - ("first second", true, "test --test first -- --ignored --nocapture\ntest --test second -- --ignored --nocapture\n"), - ("fails survivor", false, "test --test fails -- --ignored --nocapture\ntest --test survivor -- --ignored --nocapture\n"), - ("", false, ""), - ] { - let log = root.join("cargo.log"); - fs::write(&log, "").unwrap(); - let result = Command::new("bash").args(["-e", "-o", "pipefail", "-c", step["run"].as_str().unwrap()]) - .env("TESTS", tests).env("CARGO_LOG", &log) - .env("PATH", format!("{}:{}", root.display(), std::env::var("PATH").unwrap())) - .output().unwrap(); - assert_eq!(result.status.success(), expected_success, "{result:?}"); - assert_eq!(fs::read_to_string(log).unwrap(), expected_log); - } - fs::remove_dir_all(root).unwrap(); -} - -#[test] -fn spinel_jobs_are_selected_explicitly_and_archive_evidence_reaches_pages() { - let ci: serde_yaml_ng::Value = - serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); - let jobs = &ci["jobs"]; - for name in [ - "framework-tests-spinel", - "campfire-db-differential-spinel", - "toolchain-spinel", - "compare-spinel", - "smoke-spinel", - "smoke-campfire", - ] { - let job = &jobs[name]; - let needs = job["needs"] - .as_sequence() - .expect("Spinel consumer needs plan and producer"); - assert!( - needs.iter().any(|need| need.as_str() == Some("plan")), - "{name}" - ); - assert!( - job["if"].as_str().unwrap().contains(&format!( - "contains(fromJSON(needs.plan.outputs.jobs), '{name}')" - )), - "{name}" - ); - } - - for (job_name, artifact_name) in [ - ("build-site", "browse-archives"), - ("build-campfire-archive", "campfire-archive"), - ] { - let upload = jobs[job_name]["steps"] - .as_sequence() - .unwrap() - .iter() - .find(|step| step["with"]["name"].as_str() == Some(artifact_name)) - .unwrap(); - assert_eq!(upload["if"].as_str(), Some("always()")); - assert!(upload["with"]["path"].as_str().unwrap().ends_with("*.tgz")); - } - let report = &jobs["archive-results"]; - for dependency in [ - "build-site", - "build-campfire-archive", - "smoke", - "smoke-spinel", - "smoke-campfire", - "smoke-campfire-docker", - ] { - assert!(report["needs"] - .as_sequence() - .unwrap() - .iter() - .any(|need| need.as_str() == Some(dependency))); - } - let report_steps = report["steps"].as_sequence().unwrap(); - assert_eq!( - report_steps - .iter() - .find(|step| step["name"].as_str() == Some("Collect this run's archive evidence")) - .unwrap()["with"]["pattern"] - .as_str(), - Some("ci-archive-*") - ); - assert_eq!( - report_steps - .iter() - .find(|step| step["name"].as_str() == Some("Save archive outcome report")) - .unwrap()["with"]["name"] - .as_str(), - Some("archive-results") - ); - - let assemble = &jobs["assemble-site"]; - assert!(assemble["needs"] - .as_sequence() - .unwrap() - .iter() - .any(|need| need.as_str() == Some("archive-results"))); - let steps = assemble["steps"].as_sequence().unwrap(); - let verify = steps.iter().position(|step| step["run"].as_str() == Some("python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json")).expect("archive verification step"); - let pages = steps - .iter() - .position(|step| { - step["uses"] - .as_str() - .is_some_and(|uses| uses.starts_with("actions/upload-pages-artifact@")) - }) - .unwrap(); - assert!(verify < pages); - assert!(jobs["ci-summary"]["needs"] - .as_sequence() - .unwrap() - .iter() - .any(|need| need.as_str() == Some("archive-results"))); -} - -#[test] -fn full_scheduler_runs_every_preflight_success_fresh_and_never_grants_pr_deploy_permissions() { - let full: serde_yaml_ng::Value = - serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) - .unwrap(); - assert_eq!( - full["on"]["schedule"][0]["cron"].as_str(), - Some("17 */4 * * *") - ); - assert!(full["on"].get("push").is_none()); - assert!(full["on"].get("pull_request").is_none()); - assert_eq!( - full["concurrency"]["cancel-in-progress"].as_bool(), - Some(false) - ); - let jobs = &full["jobs"]; - let preflight = &jobs["preflight"]; - assert!(preflight["outputs"].get("run").is_none()); - assert!(preflight["outputs"].get("known").is_none()); - assert!(jobs.get("checkpoint").is_none()); - let preflight_text = serde_yaml_ng::to_string(preflight).unwrap(); - assert!(!preflight_text.contains("actions/cache")); - assert_eq!( - preflight_text - .matches("repos/matz/spinel/commits/master") - .count(), - 1 - ); - assert_eq!( - jobs["validation"]["uses"].as_str(), - Some("./.github/workflows/ci.yml") - ); - assert_eq!(jobs["validation"]["with"]["full"].as_bool(), Some(true)); - assert_eq!( - jobs["validation"]["permissions"]["contents"].as_str(), - Some("read") - ); - assert_eq!( - jobs["validation"]["permissions"]["actions"].as_str(), - Some("read") - ); - assert_eq!( - jobs["validation"]["permissions"] - .as_mapping() - .unwrap() - .len(), - 2 - ); - assert_eq!(full["permissions"]["contents"].as_str(), Some("read")); - let deploy = &jobs["deploy"]; - let guard = deploy["if"].as_str().unwrap(); - assert!(guard.contains("needs.validation.outputs.publication-ready == 'true'")); - assert!( - !guard.contains("needs.validation.result == 'success'"), - "extra failures cannot hide repro publication" - ); - assert!(deploy.get("continue-on-error").is_none()); - assert_eq!(deploy["permissions"]["pages"].as_str(), Some("write")); - assert!(deploy["steps"][0]["run"] - .as_str() - .unwrap() - .contains("$VALIDATED_SHA")); -} - -#[cfg(unix)] -#[test] -fn scheduler_preflight_executes_publication_guards_and_unknown_input_fallback() { - use std::os::unix::fs::PermissionsExt; - use std::process::Command; - - let full: serde_yaml_ng::Value = - serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) - .unwrap(); - let body = full["jobs"]["preflight"]["steps"][0]["run"] - .as_str() - .unwrap(); - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos(); - let root = std::env::temp_dir().join(format!("full-preflight-{}-{unique}", std::process::id())); - fs::create_dir(&root).unwrap(); - for (name, script) in [ - ("gh", "#!/bin/sh\nprintf 'called\\n' >> \"$GH_LOG\"\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n"), - ] { - let path = root.join(name); - fs::write(&path, script).unwrap(); - fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); - } - let spinel_sha = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; - for (event, repo, reference, publish, revision, expected) in [ - ( - "schedule", - "rubys/roundhouse", - "refs/heads/main", - "false", - spinel_sha, - Some(("true", spinel_sha)), - ), - ( - "schedule", - "rubys/roundhouse", - "refs/heads/main", - "false", - "unavailable", - Some(("true", "master")), - ), - ( - "schedule", - "rubys/roundhouse", - "refs/heads/main", - "false", - "malformed", - Some(("true", "master")), - ), - ( - "workflow_dispatch", - "contributor/roundhouse", - "refs/heads/topic", - "false", - spinel_sha, - Some(("false", spinel_sha)), - ), - ( - "workflow_dispatch", - "rubys/roundhouse", - "refs/heads/main", - "true", - spinel_sha, - Some(("true", spinel_sha)), - ), - ( - "workflow_dispatch", - "rubys/roundhouse", - "refs/heads/topic", - "true", - spinel_sha, - None, - ), - ] { - let outputs = root.join("outputs"); - fs::write(&outputs, "").unwrap(); - let result = Command::new("bash") - .args(["-e", "-o", "pipefail", "-c", body]) - .env( - "PATH", - format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), - ) - .env("EVENT", event) - .env("GITHUB_REPOSITORY", repo) - .env("GITHUB_REF", reference) - .env("REQUEST_PUBLISH", publish) - .env("MOCK_SPINEL", revision) - .env("GH_LOG", root.join("gh.log")) - .env("GITHUB_OUTPUT", &outputs) - .output() - .unwrap(); - assert_eq!( - result.status.success(), - expected.is_some(), - "{event} {repo} {reference}: {result:?}" - ); - let actual = fs::read_to_string(outputs).unwrap(); - if let Some((published, resolved)) = expected { - assert_eq!(actual, format!("spinel={resolved}\npublish={published}\n")); - } else { - assert!( - actual.is_empty(), - "rejected publication must not issue outputs" - ); - } - } - assert_eq!( - fs::read_to_string(root.join("gh.log")) - .unwrap() - .lines() - .count(), - 5 - ); - fs::remove_dir_all(root).unwrap(); -} From 8a3b72c5d67f6bc200ecd29bc329918196537f38 Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 18:52:35 +0000 Subject: [PATCH 5/6] Select full validation for shared emitter changes Cross-target emitter helpers must not fall back to the compact-only floor. Pin the shared schema, operator and module boundaries and register the extracted CI policy suite with full coverage ownership. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- docs/ci-reuse.md | 1 + scripts/ci-plan.py | 5 ++++- tests/ci_plan_test.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index 29fe0f7d3..3d57e80b3 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -34,6 +34,7 @@ helper, without additional Python packages. | `runtime/ruby/`, native `runtime/spinel/`, or a focused Spinel test | Advisory native core plus the relevant focused Spinel framework suite | | Interpreter-only `db_jruby` / `markly_jruby` or `scaffold/ruby_overlay` | Interpreted Ruby/JRuby owners, not native fanout | | Spinel scaffold packaging | Advisory native core plus the native archive smoke | +| Shared emitter helpers (`src/emit/shared/`) | Full validation across target languages | | Shared analyzer or lowerer | Compact floor; no mandatory Spinel lane on an ordinary PR; reviewers can request `ci:full` for broad/risky changes | | `wasm/` | WASM build and IDE/playground/studio browser verification | | Site/guide sources | Site/archive build and WASM verification, without publishing | diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py index 53d1923a8..11ada5096 100755 --- a/scripts/ci-plan.py +++ b/scripts/ci-plan.py @@ -182,6 +182,7 @@ def select(paths, *, draft=False, full=False, publish=False): "tests/ci_plan_test.py", "tests/ci_archive_evidence_test.py", "tests/workflow_yaml_parses.rs", + "tests/ci_policy_workflow.rs", "src/project.rs", "src/bin/roundhouse.rs", "Cargo.toml", @@ -234,12 +235,14 @@ def select(paths, *, draft=False, full=False, publish=False): spinel |= "spinel" in owners if owners: reasons.append(f"{path}: {', '.join(sorted(owners))}") + elif target == "shared": + full = True + reasons.append(f"{path}: shared code generation") elif target and target not in { "ruby", "ruby_family", "roda", "mod", - "shared", "rails", }: full = True diff --git a/tests/ci_plan_test.py b/tests/ci_plan_test.py index 19624e8c2..88a9c44a0 100644 --- a/tests/ci_plan_test.py +++ b/tests/ci_plan_test.py @@ -24,6 +24,25 @@ def test_general_analysis_and_lowering_retain_base(self): self.assertEqual(plan["jobs"], ci.BASE) self.assertEqual(plan["archives"], []) + def test_shared_emitters_select_cross_target_full_coverage(self): + for path in [ + "src/emit/shared/schema_sql.rs", + "src/emit/shared/ops.rs", + "src/emit/shared/mod.rs", + ]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual( + plan["extra_compare"], + ["crystal", "kotlin", "swift", "csharp", "go", "elixir", "python"], + ) + self.assertEqual(plan["smoke"], ci.TARGETS) + self.assertTrue(plan["site"]) + self.assertTrue(plan["wasm"]) + self.assertTrue(set(ci.SPINEL11).issubset(plan["jobs"])) + self.assertIn("writebook-inventory", plan["required"]) + self.assertIn("archive-results", plan["required"]) + def test_native_only_test_selects_core_without_archives_or_campfire(self): plan = ci.select(["tests/spinel_toolchain.rs"]) self.assertEqual(self.extras(plan), set(ci.CORE)) @@ -177,6 +196,7 @@ def test_packaging_and_unknown_target_changes_expand_to_full(self): "Cargo.toml", "scripts/ci-reuse.py", "tests/ci_archive_evidence_test.py", + "tests/ci_policy_workflow.rs", ]: with self.subTest(path=path): self.assertEqual(ci.select([path])["smoke"], ci.TARGETS) From acc6006f50a21702f770be533d2a23852681c32d Mon Sep 17 00:00:00 2001 From: Thomas Klemm Date: Fri, 2 Oct 2026 19:47:46 +0000 Subject: [PATCH 6/6] Clarify full CI requests and publication boundaries Align contributor guidance, target guides and pipeline topology with the selected coverage policy. Distinguish full coverage from fresh execution and correct assembly/publication comments without changing workflow logic. Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a0fd60-52fd-76ce-811f-6461776c58f1 --- .github/workflows/ci.yml | 37 +++++++++----------- AGENTS.md | 18 +++++----- README.md | 20 +++++------ docs/ci-reuse.md | 64 ++++++++++++++++++++++++++++------- docs/guide/rails-coverage.md | 7 ++-- docs/guide/spinel.md | 11 +++--- docs/guide/targets.md | 12 ++++--- docs/pipeline/verification.md | 57 ++++++++++++++++++------------- 8 files changed, 140 insertions(+), 86 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e5316291..321e7e7f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,8 @@ on: branches: [main] pull_request: branches: [main] - # Drafts run fixture + unit only. Leaving draft starts the full run; - # returning to draft cancels the superseded full run via concurrency. + # Drafts run fixture + unit only. Ready PRs use selected coverage. + # Label/draft/head transitions cancel superseded PR runs via concurrency. types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, labeled, unlabeled] workflow_call: inputs: @@ -1044,9 +1044,9 @@ jobs: # long-lived process holding connection registries and a rendered # room page is that shape — lobsters is request-shaped, LangArena the # opposite. All legs consume the SAME comparison binary artifact of the - # SAME run, so a red `minor-gc` beside a green `default` is the - # collector alone. That is the report he wants most: reduce it, file - # it upstream, and say minor-GC-only in the title. Only the served + # SAME run. A red `minor-gc` beside a green `default` is a mode-specific + # signal, not proof of an upstream collector defect: reduce the case + # and establish its cause before filing upstream. Only the served # binary sees the variable — scripts/campfire-compare exports it # explicitly, 0 or 1, on the launch line and nowhere else. # @@ -3049,13 +3049,10 @@ jobs: # Finish Pages assembly without keeping blog smoke jobs behind Campfire. assemble-site: # campfire-conformance is here for ORDERING ONLY — its summary.json is - # this job's input. The `if:` deliberately does not require it to have - # SUCCEEDED: that job is blocking (pinned input, so a red floor is a - # real regression and must fail the run), but the page it feeds is the - # thing that says WHAT regressed, and suppressing the page on the run - # where it is most useful would be exactly backwards. A missing - # artifact degrades to a skipped render and a warning, not a broken - # site build. + # this job's input. Assembly can render a failed conformance report + # for triage, but a failed compact floor still prevents Pages deploy. + # A missing summary degrades to a skipped render and warning here; + # assembly does not override the separate publication guards. needs: [build-site, campfire-conformance, build-campfire-archive, archive-results, plan] if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' && needs.archive-results.result == 'success' }} outputs: @@ -3080,20 +3077,18 @@ jobs: path: _site/ci # The campfire archive, built by the parallel build-campfire-archive - # job. It lands AFTER `--site` because build_site() removes and - # recreates the output directory, which would take _site/campfire - # with it. + # job, is attached after restoring this run's site-base artifact. + # Assembly never re-emits archives or rebuilds from a newer main. # # The bytes go live at rubys.github.io/roundhouse/campfire/spinel.tgz # (source; needs a spinel checkout) and campfire/docker.tgz (the # `spin pack` output with a Dockerfile; needs Docker and nothing - # else). Both are linked from site/apps/campfire.html. + # else), if publication guards pass. Both are linked from + # site/apps/campfire.html. # - # Tolerant of a missing artifact ON PURPOSE, and for the same reason - # this job does not require campfire-conformance to have SUCCEEDED: - # per matz/spinel#314 the published archives are an extraction - # surface for upstream bug repros, so a broken campfire emit must not - # stall publication of the eleven blog archives beside it. + # Missing or failed archive outputs remain honest repro evidence, + # not successful validation. Same-run hashes and download availability + # are verified separately; compact-floor failure still blocks deploy. - uses: actions/download-artifact@v8 continue-on-error: true with: diff --git a/AGENTS.md b/AGENTS.md index a9fe2deb2..5fb86916f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,11 +87,13 @@ defect even if the build is green. you changed. End commit messages with the standard `Co-Authored-By` trailer. - **Outside contributors: fork, and open a pull request against `main`.** - CI runs a compact floor plus targeted lanes; add `ci:full` for the - complete PR matrix. Full main validation/publication is scheduled every - four hours and executes freshly every cycle. `CI summary` reports - selected checks, not a mandatory merge gate; maintainers decide when - to merge. See [CI coverage](docs/ci-reuse.md). + CI runs a compact floor plus targeted lanes. For broad/risky changes, + ask a maintainer to apply `ci:full` for the complete PR matrix. + This expands validation only; PR runs never publish. Drafts retain + fixture + unit coverage even with the label. Full main validation and + publication are scheduled every four hours and execute freshly every cycle. + `CI summary` reports selected checks, not a mandatory merge gate; + maintainers decide when to merge. See [CI coverage](docs/ci-reuse.md). You do not need every toolchain locally; CI covers the missing lanes. Before opening one: `bin/rh fixture` (the test fixtures are generated, not checked in — see below), `cargo test --lib` plus the targeted @@ -113,9 +115,9 @@ defect even if the build is green. `cargo test --all-targets` at milestones. Real-toolchain tests are `#[ignore]`-gated (`cargo test --test _toolchain -- --ignored`); CI runs each in its own job. -- **CI is deliberately not uniformly gating.** The core `cargo test` job gates. - The ~5 `continue-on-error: true` jobs track current Spinel master and other - moving toolchains on purpose — **red there is a signal to read, not a +- **CI is deliberately not uniformly gating.** The core `cargo test` job is + non-advisory. Jobs marked `continue-on-error: true` track current Spinel + master and other moving toolchains on purpose — **red is a signal, not a regression to shim away.** A Spinel red can come from Roundhouse runtime, RBS or packaging, or from upstream; it is not by itself proof of an upstream fault. Don't add workarounds just to make an advisory job green. diff --git a/README.md b/README.md index 035046d7f..c2679c806 100644 --- a/README.md +++ b/README.md @@ -170,16 +170,16 @@ its predecessors; [`WHY.md`](WHY.md) is why do it at all. Issues and pull requests are both welcome, and a PR does not need a conversation first: CI runs a compact correctness/runtime floor plus -targeted checks. Add `ci:full` for full PR validation; the complete -matrix also runs freshly every four hours. See -[CI coverage](docs/ci-reuse.md). The -`CI summary` status reports selected checks without imposing a merge -requirement; maintainers decide when to merge. The Spinel -and Campfire jobs marked `continue-on-error` track current Spinel -master, and red there is a signal to investigate Roundhouse runtime, -RBS or packaging as well as possible upstream drift—not proof of an -upstream fault. Setup, -the test cycle, and what a PR should carry are in +targeted checks. Ask a maintainer to apply `ci:full` for full PR validation; +the label expands tests, never publication. The complete matrix also runs +freshly every four hours. See [CI coverage](docs/ci-reuse.md) for requesting +full/fresh validation and the separate publication controls. +`CI summary` reports selected checks without imposing a merge requirement; +maintainers decide when to merge. Spinel and Campfire jobs marked +`continue-on-error` track current Spinel master, and red there is a signal +to investigate Roundhouse runtime, RBS or packaging as well as possible +upstream drift—not proof of an upstream fault. Setup, the test cycle, +and what a PR should carry are in [`DEVELOPMENT.md`](DEVELOPMENT.md); the invariants not to break are in [`AGENTS.md`](AGENTS.md). diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index 3d57e80b3..74ce6c01c 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -41,13 +41,49 @@ helper, without additional Python packages. | Shared compare, framework, archive, or E2E harness | The checks owned by that harness | | Cross-target packaging, CI policy/workflows/planner, Cargo/build/toolchain policy, unknown new target | Full validation | -Add **`ci:full`** to a non-draft PR to request the full matrix for its current -merge tree. Removing the label restores automatic routing. Label/draft/head -transitions cancel superseded PR runs. A manual **Full validation** dispatch -validates the chosen ref freshly; a branch-head dispatch is not a replacement -for the PR merge-tree check. Publication is opt-in for dispatch and is accepted -only on canonical `rubys/roundhouse` main. No deployment privileges are granted -to the shared validator or PR jobs. +### Requesting broader or fresh validation + +For broad/risky changes, or when targeted coverage is insufficient, request +**`ci:full`**. Applying labels requires upstream repository triage access or +higher; fork contributors and their agents without that access should ask a +maintainer to apply the label. A request in a PR comment alone does not trigger +CI. The label must exist in the upstream repository before it can be applied. + +- The PR must be **ready for review**: drafts retain fixture + unit only, + even with `ci:full`. +- Applying the label starts a new full-matrix run of the current **PR merge + tree**, without a new commit. Superseded PR runs are cancelled. +- Further pushes retain full coverage while the label remains set. +- Removing it starts a run with automatic coverage selection. Policy/shared + emitter changes can still select full coverage without the label. +- Full **coverage** does not disable conservative PR execution-receipt reuse. + For fresh execution, ask a maintainer to select **Re-run all jobs** on the + full-matrix run. A rerun retains that run's original SHA and event coverage; + rerunning an older compact run does not expand coverage or test a newer head. + +A manual **Actions → Full validation → Run workflow** dispatch validates the +chosen ref freshly. A branch-head dispatch is not a replacement for the PR +merge-tree check. Leave **publish** unchecked for validation only. + +### Validation is not publication + +`full` selects coverage; `publish` separately authorizes publication work. +Applying `ci:full` never enables `publish`. + +| Trigger | Publication behavior | +|---|---| +| PR, including `ci:full`, or ordinary main push | Validation/artifacts only; no deploy | +| Manual Full validation, `publish=false` (default) | Fresh validation only; no deploy | +| Manual Full validation, `publish=true` | Accepted only on canonical `rubys/roundhouse` main; guarded publication | +| Four-hour schedule on canonical main | Fresh full validation plus guarded publication | + +The shared validator and PR jobs have no Pages/OIDC deployment privileges and +no deploy job. Only the separate full-workflow deploy job receives those +permissions. Publication requests outside canonical main and schedule/manual +events are rejected. Publication still requires the compact floor, verified +assembly and a live-main SHA check; the detailed archive contract follows below. + +### Current-run artifacts and the full cycle Targeted archive smokes use `roundhouse --archives rust,go` (selected `browse/.{json,tgz,zip}` outputs) and do not build WASM, demos, website @@ -234,17 +270,18 @@ native-library ABI, current-commit provenance, and proof at each consumer. Matrix `fail-fast: false` still preserves cross-target diagnostic coverage; advisory lanes remain signals rather than reasons to cancel independent checks. -Existing `needs` gates and normal step failure handling are unchanged. The -Campfire Docker recipe no longer downloads a separate Dockerfile frontend: +Receipt reuse does not bypass selected checks' dependency or failure handling. +The Campfire Docker recipe no longer downloads a separate Dockerfile frontend: its ordinary multi-stage instructions use the bundled frontend and COPY preserves the archive's executable boot mode. Base images and apt packages still resolve freshly, and the real Docker smoke remains enabled. ## Forcing a fresh check and extending the allowlist -GitHub's **Re-run jobs** always executes these checks freshly, even if a matching -receipt exists. The fresh attempt may produce new evidence. Existing draft, -`needs` and main-branch behavior is unchanged. +Rerun checks bypass matching receipts. Choose **Re-run all jobs** to execute +the entire selected graph freshly, rather than only failed or individual jobs. +The rerun keeps the original SHA, ref and coverage; the fresh attempt may +produce new evidence. Draft, `needs` and main-branch behavior is unchanged. Before adding another job, audit its complete input contract, including generated artifacts, framework tests, executable README blocks and the actual @@ -256,3 +293,6 @@ unless current-run outputs and their provenance can be preserved honestly. Run `PYTHONDONTWRITEBYTECODE=1 python3 tests/ci_reuse_test.py -v` and `cargo test --test workflow_yaml_parses` when changing this policy. The Rust workflow tests execute the Python adversarial suite, so normal unit CI gates it. +For coverage routing, archive evidence or full-workflow changes, also run +`cargo test --test ci_policy_workflow`; it executes the planner and archive +evidence suites and checks the publication boundaries. diff --git a/docs/guide/rails-coverage.md b/docs/guide/rails-coverage.md index f6f4e358b..66649327b 100644 --- a/docs/guide/rails-coverage.md +++ b/docs/guide/rails-coverage.md @@ -23,9 +23,10 @@ tiers. **The blog** (`fixtures/real-blog`, the Rails 8 scaffold with articles, comments, nested routes, validations, Turbo Streams, Action Cable, -Tailwind, JSON endpoints) is what **every server target** passes the -DOM-equivalence gate against on every push. What the blog uses is -supported everywhere. +Tailwind, JSON endpoints) is the shared DOM-equivalence fixture for +**every server target** in full validation. Ordinary PRs/main pushes run +the compact floor plus selected target lanes; see [CI coverage](../ci-reuse.md). +A passing target's comparison proves the blog's features on that target. **Campfire** (Basecamp's chat product — file attachments with image variants, rich text, web push, bots and webhooks, full-text search, diff --git a/docs/guide/spinel.md b/docs/guide/spinel.md index 4f4beea57..002c677b8 100644 --- a/docs/guide/spinel.md +++ b/docs/guide/spinel.md @@ -149,8 +149,10 @@ A new deployment with no such volume can drop `boot` for `USER runs the whole product — sign-in, rooms, uploads, search, live updates over the socket — from `docker run -p 3000:3000`. [rubys.github.io/roundhouse/campfire/docker.tgz](https://rubys.github.io/roundhouse/campfire/docker.tgz) -is that archive, rebuilt on every push; it is the fastest way to see -the door's end state before pointing it at your own app. +is that archive, refreshed by scheduled full validation or an explicitly +publishing manual run on canonical main. PR archive checks never publish it; +see [CI coverage](../ci-reuse.md). It is the fastest way to see the door's +end state before pointing it at your own app. ## What to expect @@ -169,8 +171,9 @@ cleanly, and when it does not, the failure is in one of two places: - **It builds but behaves differently from Rails.** The [compare oracle](verifying.md) is the reproduction: `bin/rh compare spinel` for the fixture, or `roundhouse-compare` against your own - app. The Campfire lane compares every page and every cable frame - against live Rails on every push. + app. The Campfire Spinel lane compares every page and every cable frame + against live Rails in full validation and when its owning inputs select + it; ordinary native Spinel changes do not automatically run that lane. The binary's performance is measured continuously on the blog and on Campfire against Rails as Rails ships it — with its fragment caching diff --git a/docs/guide/targets.md b/docs/guide/targets.md index fe11d366a..dce9a7619 100644 --- a/docs/guide/targets.md +++ b/docs/guide/targets.md @@ -36,8 +36,11 @@ app's model and controller tests and a Playwright `e2e/` suite; the ## How far each is tested -Every claim below is a CI job on every push to `main`, all of them -against the blog fixture (`fixtures/real-blog`: articles, comments, +The lanes below run across targets in full validation, scheduled every four +hours or requested manually. Ordinary PRs/main pushes use a compact floor plus +targeted additions; maintainers can request full PR coverage with `ci:full`. +See [CI coverage](../ci-reuse.md). The lanes use the blog fixture +(`fixtures/real-blog`: articles, comments, nested routes, validations, Turbo Streams over Action Cable, Tailwind) unless another app is named. A target's row in [`RELEASES.md`](../../RELEASES.md) records where these stood at the @@ -62,8 +65,9 @@ that gap. ## The conformance bar -A target is on the list above because it passes `compare` on the -fixture on every push. A page that renders one node differently — an +The conformance bar is a passing `compare` result against the fixture, +not the assumption that every target was checked on an ordinary PR. +A page that renders one node differently — an attribute missing, a whitespace text node — is red. That is deliberately a higher bar than "the tests pass": it means the emitted app is the same app, as a browser or a scraper would see it, not a diff --git a/docs/pipeline/verification.md b/docs/pipeline/verification.md index 5cdac7d60..9c70cf636 100644 --- a/docs/pipeline/verification.md +++ b/docs/pipeline/verification.md @@ -38,7 +38,8 @@ error diagnostics fire, and the framework Ruby is itself sound. `#[ignore]`d, so it gates every `cargo test`. All of this runs in the `unit` CI job (`cargo test --all-targets`) — -the one job the Pages deploy is actually gated on (see CI topology). +part of the compact floor required for Pages publication, alongside +the other baseline checks and verified assembly (see CI topology). **Why it exists:** if the ingester silently drops a construct, the emitter has nothing to emit and downstream tests trivially pass. @@ -251,9 +252,10 @@ Both lanes **publish a worklist**, not just a floor. per-file tally, spliced-stub ledger, and the failures clustered into named causes by `bench/campfire/suite-causes.json` — which `scripts/campfire-suite-report` renders to -`/bench/campfire-suite/`. build-site consumes that artifact from the -same run WITHOUT gating on the job's conclusion: a run that trips the -floor is precisely the run whose page someone needs to read. The +`/bench/campfire-suite/`. Publication assembly consumes that artifact +from the same run and can render a failed conformance report for triage. +A failed compact floor still prevents Pages deployment; the raw CI +summary remains available for investigation. The lobsters twin (`scripts/lobsters-spec-report` → `/bench/lobsters-specs/`) keeps the same contract over data fetched from the bench box. Re-running either report against a saved tally is @@ -282,34 +284,41 @@ diffs between two IRs drop out from plain `diff` across the outputs. ## CI topology -`.github/workflows/ci.yml`, honestly stated: +The current selection and publication policy is documented in +[CI coverage](../ci-reuse.md). The ownership boundaries are: - **`generate-fixture`** builds `fixtures/real-blog` once per run and shares it as an artifact every downstream job downloads. -- **The hard gate is `unit`.** `deploy` needs exactly - `[build-site, unit]`; `build-site` needs - `[generate-fixture, build-wasm]`. Archives publish *current emit* - even when target jobs are red — broken emit is exactly what's - interesting to investigate upstream (see the build-site comment). -- **Everything else is red-as-signal:** compare, smoke, - framework-tests, and toolchain jobs turn the run red without - blocking deploy; the red is the shared worklist. -- **Seven jobs are `continue-on-error`**, all for one structural - reason — they track matz/spinel master unpinned, so upstream churn - must not gate unrelated work: `build-spinel`, - `framework-tests-spinel`, `toolchain-spinel`, `compare-spinel`, - `smoke-spinel`, `smoke-campfire`, and `campfire-compare-spinel`, - which runs as three legs. The cost: an advisory failure is invisible - in the run's conclusion. Read the jobs, not the conclusion. +- **PRs/main pushes select coverage**, rather than running every target + on every push: compact floor plus owned additions, or full coverage + for policy/shared-emitter changes and `ci:full`. Drafts run fixture + unit. +- **`CI summary` reports selected results**, including missing/skipped/ + cancelled non-advisory checks. It does not configure mandatory merge + protection; maintainers decide when to merge. +- **Spinel master remains advisory.** Its revision is resolved once per + run and the actual built revision is recorded. Failures can come from + Roundhouse runtime/RBS/packaging or upstream. Inspect raw job outcomes + and archive evidence, not just the overall run conclusion. +- **Publication is separate**, in `.github/workflows/full-ci.yml`: + four-hour scheduled validation/publication on canonical main, or fresh + manual validation with publication opt-in. PR checks never deploy. + Pages requires the compact floor, verified same-run assembly and a + live-main SHA check. Extra-target/advisory failures may remain useful + repro archives, but are not presented as successful validation. + +The native Campfire comparison's GC modes run when that lane is selected, +not automatically for every native-runtime change: + - **`campfire-compare-spinel (default)`, `(minor-gc)` and - `(verify-gen)`** serve the same binary, from the same `build-spinel` - artifact, through the same Rails-oracle walk. `minor-gc` adds + `(verify-gen)`** serve the same comparison binary, produced by + `build-campfire-compare-spinel` using the shared `build-spinel` toolchain, + through the same Rails-oracle walk. `minor-gc` adds `SPINEL_GC_MINOR=1`, spinel's generational collector, which is opt-in until it goes default-on. matz asked for that leg (matz/spinel#4260): campfire is the retained-heap shape the default-on decision lacks. A red `minor-gc` beside a green `default` - is the collector alone — reduce it, file it upstream, and say - minor-GC-only in the title. + is a mode-specific signal: reduce the failing case and establish its + cause before attributing it upstream. - **`(verify-gen)` is why the other two can be believed.** `minor-gc` only fails when a missed write barrier reaches the page; `SPINEL_GC_VERIFY_GEN=1` re-marks the whole heap after every minor