diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe35dc994..321e7e7f1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,23 +5,33 @@ on: branches: [main] pull_request: branches: [main] - # Drafts run fixture + unit only. Leaving draft starts the full run; - # returning to draft cancels the superseded full run via concurrency. - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - # Not on `push`: every commit on main keeps a full run. - paths-ignore: ['**.md', 'docs/**'] + # Drafts run fixture + unit only. Ready PRs use selected coverage. + # Label/draft/head transitions cancel superseded PR runs via concurrency. + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, labeled, unlabeled] + workflow_call: + inputs: + full: + type: boolean + default: false + publish: + type: boolean + default: false + spinel_revision: + type: string + default: '' + outputs: + complete: + value: ${{ jobs.ci-summary.outputs.complete }} + publication-ready: + value: ${{ jobs.compact-required.outputs.passed == 'true' && jobs.assemble-site.outputs.ready == 'true' }} -# Cancel superseded PR runs. On main, a started run finishes and a burst of -# pushes collapses to the newest queued one, so each run that starts is -# validated to the end. Pages keeps its separate deployment-only lock below. +# Separate from the full-ci caller's lock: workflow_call inherits its context. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + group: validation-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read - pages: write - id-token: write env: CARGO_TERM_COLOR: always @@ -38,9 +48,36 @@ env: WRITEBOOK_SHA: f3fadd21907ad9b18cb23800d971c2cc25045e2a jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + plan: ${{ steps.plan.outputs.plan }} + jobs: ${{ steps.plan.outputs.jobs }} + extra-compare: ${{ steps.plan.outputs.extra-compare }} + smoke: ${{ steps.plan.outputs.smoke }} + archives: ${{ steps.plan.outputs.archives }} + wasm: ${{ steps.plan.outputs.wasm }} + site: ${{ steps.plan.outputs.site }} + publish: ${{ steps.plan.outputs.publish }} + spinel-revision: ${{ steps.plan.outputs.spinel-revision }} + spinel-tests: ${{ steps.plan.outputs.spinel-tests }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + - name: Select required coverage from the event's actual tree + id: plan + env: + CI_FULL: ${{ inputs.full || false }} + CI_PUBLISH: ${{ inputs.publish || false }} + CI_SPINEL_REVISION: ${{ inputs.spinel_revision }} + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci-plan.py plan + writebook-inventory: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'writebook-inventory') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -119,6 +156,7 @@ jobs: # install cost once per CI run instead of in each of seven jobs. # ------------------------------------------------------------------- generate-fixture: + needs: plan runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -165,8 +203,8 @@ jobs: # goes red it is the analyzer's next entry, not noise. # ------------------------------------------------------------------- store-check: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'store-check') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -229,8 +267,8 @@ jobs: # need is the WASI SDK (a manual tarball, cached here). # ------------------------------------------------------------------- build-wasm: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'build-wasm') }} runs-on: ubuntu-latest # The job normally finishes in a few minutes; a hung apt install # once rode the old 45-minute ceiling and stalled the whole @@ -286,14 +324,17 @@ jobs: # # The single `spinel` binary resolves its `spinel_rbs_extract` sidecar # and `lib/` relative to its own directory, so the tree ships as a unit. - # Tracking master rather than pinning a SHA — bump to a pin if upstream - # churn turns flaky. + # Resolve current master once in the planner/caller, then build that SHA. + # A failed lookup falls back to fresh master and cannot save a checkpoint. # ------------------------------------------------------------------- build-spinel: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'build-spinel') }} + outputs: + artifact-id: ${{ steps.toolchain.outputs.artifact-id }} + execution: ${{ steps.result.outputs.status }} # Re-enabled 2026-06-17 (framework-tests-spinel 5/5 + compare-spinel 7/7 - # green again on matz/spinel master). Tracks matz/spinel master UNPINNED, + # green again on matz/spinel master). Tracks moving matz/spinel master, # so it can go red independently of any roundhouse change; continue-on-error # keeps that advisory rather than failing the whole run (the downstream # framework-tests / compare-spinel jobs are likewise advisory). If a build @@ -313,7 +354,7 @@ jobs: - uses: actions/checkout@v5 with: repository: matz/spinel - ref: master + ref: ${{ needs.plan.outputs.spinel-revision }} path: spinel-src - name: Set up sccache id: sccache @@ -395,11 +436,19 @@ jobs: cp -R spinel-src/builtins spinel-dist/builtins git -C spinel-src rev-parse HEAD > spinel-dist/revision.txt - name: Upload spinel toolchain + id: toolchain uses: actions/upload-artifact@v7 with: name: spinel-dist path: spinel-dist retention-days: 1 + - &advisory-result + name: Report actual advisory execution + id: result + if: always() + env: + STATUS: ${{ job.status }} + run: echo "status=$STATUS" >> "$GITHUB_OUTPUT" # ------------------------------------------------------------------- # Library correctness. Runs the default (non-ignored) test suite: @@ -499,8 +548,8 @@ jobs: # NOT from the published archive. Playwright config gates on $CI # (GitHub sets it) → fresh server + forbidOnly + github reporter. browser-smoke-typescript: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'browser-smoke-typescript') }} runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -725,7 +774,10 @@ jobs: # rewiring of the test_helper. framework-tests-spinel: # Re-enabled 2026-06-17 (5/5 green on matz/spinel master). Advisory. - needs: build-spinel + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'framework-tests-spinel') && needs.build-spinel.outputs.artifact-id != '' }} + outputs: &advisory-outputs + execution: ${{ steps.result.outputs.status }} runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -741,33 +793,19 @@ jobs: run: | chmod +x spinel-dist/spinel spinel-dist/spin spinel-dist/spinel_rbs_extract echo "$PWD/spinel-dist" >> $GITHUB_PATH - - name: cargo test --test framework_tests_spinel -- --ignored - run: cargo test --test framework_tests_spinel -- --ignored --nocapture - # The web-push gem's crypto ported onto spinel's openssl package - # (runtime/spinel/web_push_crypto.rb), pinned against the gem's - # own bytes. Same staged spinel; the package links the runner's - # system libssl. - - name: cargo test --test spinel_web_push_crypto -- --ignored - if: always() - run: cargo test --test spinel_web_push_crypto -- --ignored --nocapture - # A request that raises gives its connection back - # (runtime/spinel/db.rb `with_connection`); a leaked lease used to - # wedge a pool shard for good after four 500s. - - name: cargo test --test spinel_db_lease -- --ignored - if: always() - run: cargo test --test spinel_db_lease -- --ignored --nocapture - # Rails' request-params builder, ported for the spinel dispatcher - # (runtime/spinel/param_builder.rb), held to Rails' answers for - # 2,755 query strings. The CRuby half runs in the core job. - - name: cargo test --test spinel_param_builder -- --ignored - if: always() - run: cargo test --test spinel_param_builder -- --ignored --nocapture - # Rails' signed-value vectors on the spinel-compiled verifier (the - # CRuby half runs in the core job), plus secrets whose derived keys - # hold a zero byte — the binary used to cut those keys short. - - name: cargo test --test rails_compat_vectors_spinel -- --ignored - if: always() - run: cargo test --test rails_compat_vectors_spinel -- --ignored --nocapture + # Full runs select all five suites; targeted runs select their owners. + # Continue after a failing suite without losing its failure status. + - name: Run selected native framework checks + env: + TESTS: ${{ join(fromJSON(needs.plan.outputs.spinel-tests), ' ') }} + run: | + test -n "$TESTS" + status=0 + for suite in $TESTS; do + cargo test --test "$suite" -- --ignored --nocapture || status=1 + done + exit "$status" + - *advisory-result # Campfire conformance — ONCE Campfire's OWN test suite, run against a # roundhouse emit of it. The C4 rung of the coverage ladder, and the @@ -811,8 +849,8 @@ jobs: # walk against the compiled binary and stay advisory, because spinel # is tracked unpinned. campfire-compare: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare') }} runs-on: ubuntu-latest timeout-minutes: 45 services: @@ -898,12 +936,14 @@ jobs: # its executable bit in a tarball. The DB differential builds its own # scenario overlay and must not reuse this binary. build-campfire-compare-spinel: - needs: build-spinel + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-campfire-compare-spinel') && needs.build-spinel.outputs.artifact-id != '' }} runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true outputs: artifact-id: ${{ steps.binary.outputs.artifact-id }} + execution: ${{ steps.result.outputs.status }} steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup-rust @@ -983,6 +1023,7 @@ jobs: else echo 'No Campfire comparison binary was uploaded. The default, minor-gc and verify-gen comparisons are blocked, not passed; see the producer failure above.' >> "$GITHUB_STEP_SUMMARY" fi + - *advisory-result # campfire-compare-spinel: the same equivalence, with the compiled # BINARY as the emit lane — Rails' broadcast frames vs the binary's. @@ -1003,9 +1044,9 @@ jobs: # long-lived process holding connection registries and a rendered # room page is that shape — lobsters is request-shaped, LangArena the # opposite. All legs consume the SAME comparison binary artifact of the - # SAME run, so a red `minor-gc` beside a green `default` is the - # collector alone. That is the report he wants most: reduce it, file - # it upstream, and say minor-GC-only in the title. Only the served + # SAME run. A red `minor-gc` beside a green `default` is a mode-specific + # signal, not proof of an upstream collector defect: reduce the case + # and establish its cause before filing upstream. Only the served # binary sees the variable — scripts/campfire-compare exports it # explicitly, 0 or 1, on the launch line and nowhere else. # @@ -1027,15 +1068,20 @@ jobs: # nothing. campfire-compare-spinel: name: campfire-compare-spinel (${{ matrix.gc }}) - needs: build-campfire-compare-spinel + needs: [build-campfire-compare-spinel, plan] # An advisory producer failure must not become three artifact-download # failures. Only the upload's artifact ID proves the binary is available. - if: ${{ !cancelled() && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }} + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare-spinel') && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }} runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true + outputs: + default: ${{ steps.result.outputs.default }} + minor-gc: ${{ steps.result.outputs.minor-gc }} + verify-gen: ${{ steps.result.outputs.verify-gen }} strategy: fail-fast: false + max-parallel: 1 matrix: include: - gc: default @@ -1096,12 +1142,21 @@ jobs: run: gem install websocket-driver --no-document - name: scripts/campfire-compare --spinel ${{ matrix.flag }} run: scripts/campfire-compare --reuse "$PWD/build/campfire-compare-spinel" --spinel ${{ matrix.flag }} /tmp/campfire + - name: Report each GC mode's actual execution + id: result + if: always() + env: + STATUS: ${{ job.status }} + MODE: ${{ matrix.gc }} + run: echo "$MODE=$STATUS" >> "$GITHUB_OUTPUT" # A different app (the model-scenario overlay), so it needs its own # build, not the comparison binary. Start after the toolchain is ready # instead of serializing it behind the shared build and request walk. campfire-db-differential-spinel: - needs: build-spinel + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-db-differential-spinel') && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 45 continue-on-error: true @@ -1143,10 +1198,11 @@ jobs: run: scripts/campfire-oracle prepare --app /tmp/campfire - name: scripts/campfire-db-differential --spinel run: scripts/campfire-db-differential --spinel /tmp/campfire + - *advisory-result campfire-conformance: - needs: unit - if: ${{ !github.event.pull_request.draft }} + needs: [unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'campfire-conformance') }} runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -1592,7 +1648,9 @@ jobs: # via continue-on-error — like every spinel job it tracks matz/spinel # master UNPINNED, so it surfaces regressions without gating unrelated # work; if master churns red, the signal is visible but non-blocking. - needs: [generate-fixture, build-spinel] + needs: [generate-fixture, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'toolchain-spinel') && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -1667,6 +1725,7 @@ jobs: name: spinel-toolchain-repro path: ci-spinel-repro retention-days: 7 + - *advisory-result # ------------------------------------------------------------------- # DOM equivalence: boot Rails + the target server side-by-side and @@ -1687,8 +1746,8 @@ jobs: # below: two setup-ruby steps / an extra build-spinel dependency don't # fit this shape. compare: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare') }} runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -1696,9 +1755,10 @@ jobs: actions: read strategy: fail-fast: false + max-parallel: 2 matrix: - target: [rust, crystal, kotlin, swift, csharp, typescript, go, elixir, python] - steps: + target: [rust, typescript] + steps: &compare-steps - uses: actions/checkout@v5 - uses: ./.github/actions/setup-rust - uses: Swatinem/rust-cache@v2 @@ -1829,6 +1889,24 @@ jobs: if: ${{ !cancelled() }} run: scripts/compare ${{ matrix.target }} + # Same commands, distinct result: extra-target failures must not be confused + # with the Rust/TypeScript publication floor. Anchors preserve physical job + # names used by the existing strict PR-local execution receipts. + compare-extra: + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-extra') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + max-parallel: 2 + matrix: + target: ${{ fromJSON(needs.plan.outputs.extra-compare) }} + steps: *compare-steps + # compare-ruby: same shape as every other compare-* — Puma+Rack # server bound to :3000 via ruby_overlay's config.ru, hit over # HTTP. Two ruby setups: the fixture's Gemfile (Rails reference) @@ -1836,8 +1914,8 @@ jobs: # for tailwindcss asset compilation; no target compile step # (Ruby is interpreted). compare-ruby: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-ruby') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -1882,8 +1960,8 @@ jobs: # assets aren't built (the DOM diff compares asset tags, not files — # see scripts/compare's jruby branch). compare-jruby: - needs: [generate-fixture, unit] - if: ${{ !github.event.pull_request.draft }} + needs: [generate-fixture, unit, plan] + if: ${{ contains(fromJSON(needs.plan.outputs.jobs), 'compare-jruby') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -1927,7 +2005,9 @@ jobs: # drop it once the upstream cadence settles. compare-spinel: # Re-enabled 2026-06-17 (7/7 paths match Rails on matz/spinel master). Advisory. - needs: [generate-fixture, build-spinel] + needs: [generate-fixture, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'compare-spinel') && needs.generate-fixture.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -1966,6 +2046,7 @@ jobs: node-version: '20' - name: scripts/compare spinel run: scripts/compare spinel + - *advisory-result # ------------------------------------------------------------------- # smoke-: the archive runs its own README. Each published @@ -2005,7 +2086,8 @@ jobs: # ------------------------------------------------------------------- smoke: - needs: build-site + needs: [build-site, plan] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.build-site.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs), 'smoke') }} runs-on: ubuntu-latest timeout-minutes: 20 permissions: @@ -2013,8 +2095,9 @@ jobs: actions: read strategy: fail-fast: false + max-parallel: 2 matrix: - target: [rust, crystal, kotlin, swift, csharp, typescript, go, elixir, python, ruby, jruby] + target: ${{ fromJSON(needs.plan.outputs.smoke) }} steps: - uses: actions/checkout@v5 - name: Use an explicit browser payload directory @@ -2128,6 +2211,25 @@ jobs: name: ci-executed-smoke-rust-${{ github.run_attempt }} path: ${{ steps.reuse.outputs.bundle }} retention-days: 7 + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: browse/${{ matrix.target }}.tgz + CHECK: smoke-${{ matrix.target }} + OUTCOME: ${{ steps.smoke.outcome }} + REUSED: ${{ matrix.target == 'rust' && steps.reuse.outcome == 'success' && steps.reuse.outputs.hit == 'true' }} + run: &archive-validation | + if [ "${REUSED:-false}" = true ]; then OUTCOME=reused; fi + python3 scripts/ci-archive-evidence.py validation \ + --archive "$ARCHIVE_PATH" --path "$ARCHIVE_PATH" --check "$CHECK" \ + --outcome "$OUTCOME" --spinel-revision-file spinel-dist/revision.txt \ + --out "ci-evidence/validation-$CHECK-$GITHUB_RUN_ATTEMPT.json" + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-${{ matrix.target }}-${{ github.run_attempt }} + path: ci-evidence/*.json # spinel: the archive's README § Build AOT-compiles main.rb → build/blog # with the spinel toolchain (build-spinel artifact); § Test runs the @@ -2140,7 +2242,9 @@ jobs: # spinel job — it tracks matz/spinel master UNPINNED. needs build-spinel # for the compiler and build-site for the published spinel.tgz. smoke-spinel: - needs: [build-site, build-spinel] + needs: [build-site, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-spinel') && needs.build-site.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 20 continue-on-error: true @@ -2170,7 +2274,22 @@ jobs: npm install --no-audit --no-fund "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke spinel + id: smoke run: scripts/smoke --tgz browse/spinel.tgz spinel + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: browse/spinel.tgz + CHECK: smoke-spinel + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-spinel-${{ github.run_attempt }} + path: ci-evidence/*.json + - *advisory-result # ------------------------------------------------------------------- # build-campfire-archive: ONCE Campfire for the spinel target, packaged @@ -2205,7 +2324,8 @@ jobs: # the source archive from publishing. # ------------------------------------------------------------------- build-campfire-archive: - needs: [build-spinel] + needs: [build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-campfire-archive') }} runs-on: ubuntu-latest # 60, not 30: `spin pack` runs spinel's front-end over the whole # campfire tree, and with the ruby-vips package in it the job took @@ -2240,7 +2360,24 @@ jobs: tar -xzf /tmp/campfire.tar.gz -C /tmp/campfire --strip-components=1 - name: Build campfire/spinel.tgz + campfire/docker.tgz run: scripts/build-campfire-archive --out _site /tmp/campfire + - name: Record archive producer evidence + if: always() + env: + GROUP: campfire + OUTCOME: ${{ job.status }} + run: &archive-inventory | + python3 scripts/ci-archive-evidence.py inventory \ + --group "$GROUP" --root _site --outcome "$OUTCOME" \ + --spinel-revision-file spinel-dist/revision.txt \ + --out "ci-evidence/inventory-$GROUP-$GITHUB_RUN_ATTEMPT.json" + - name: Save archive producer evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-inventory-campfire-${{ github.run_attempt }} + path: ci-evidence/*.json - uses: actions/upload-artifact@v7 + if: always() with: name: campfire-archive path: _site/campfire/*.tgz @@ -2253,21 +2390,25 @@ jobs: # read the sign-in page triggers -- is found. Advisory like every # spinel job. smoke-campfire-docker: - needs: [build-campfire-archive] + needs: [build-campfire-archive, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-campfire-docker') && needs.build-campfire-archive.result == 'success' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true steps: + - uses: actions/checkout@v5 - uses: actions/download-artifact@v8 with: name: campfire-archive path: campfire - name: Build and run the image + id: smoke run: | set -euo pipefail if [ ! -f campfire/docker.tgz ]; then echo "::warning::no campfire/docker.tgz this run (spinel toolchain missing?)" - exit 0 + exit 1 fi tar xzf campfire/docker.tgz cd campfire-docker @@ -2286,6 +2427,20 @@ jobs: docker logs campfire docker rm -f campfire >/dev/null [ "$root" = 302 ] && [ "$first" = 200 ] && [ "$logo" = 200 ] + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: campfire/docker.tgz + CHECK: smoke-campfire-docker + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-campfire-docker-${{ github.run_attempt }} + path: ci-evidence/*.json + - *advisory-result # smoke-campfire: the campfire archive runs its own README, exactly as # smoke-spinel does for the blog — extract, execute every fenced ```sh @@ -2304,7 +2459,9 @@ jobs: # rather than our regression. Read it job-by-job; a green RUN does not # mean this passed. smoke-campfire: - needs: [build-campfire-archive, build-spinel] + needs: [build-campfire-archive, build-spinel, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'smoke-campfire') && needs.build-campfire-archive.result == 'success' && needs.build-spinel.outputs.artifact-id != '' }} + outputs: *advisory-outputs runs-on: ubuntu-latest timeout-minutes: 30 continue-on-error: true @@ -2336,7 +2493,22 @@ jobs: npm ci --no-audit --no-fund "$GITHUB_WORKSPACE/scripts/ci-playwright-install" - name: scripts/smoke campfire + id: smoke run: scripts/smoke --tgz campfire/spinel.tgz campfire + - name: Record byte-specific archive validation + if: always() + env: + ARCHIVE_PATH: campfire/spinel.tgz + CHECK: smoke-campfire + OUTCOME: ${{ steps.smoke.outcome }} + run: *archive-validation + - name: Save archive validation evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-validation-campfire-${{ github.run_attempt }} + path: ci-evidence/*.json + - *advisory-result # ------------------------------------------------------------------- # Build the Pages site: per-target manifests (real-blog fixture) + @@ -2346,25 +2518,22 @@ jobs: # publish *current emit* even when per-target toolchain/compare jobs # fail (broken emit is exactly what's interesting to investigate). # - # So build-site only gates on the fixture and compiler wasm. It does + # So build-site gates on the fixture, plus wasm for full site builds. It does # not gate on unit / toolchain-* / compare-* — a flaky elixir CI run # or a known-broken target shouldn't stall the archive workflow, and # the archive itself is available via the workflow-run artifact even - # when unit fails. Pages publication (the `deploy` job below) does - # gate on unit, so a broken library can't push to the live site. + # when unit fails. Pages publication in full-ci.yml gates on the compact + # floor, so a broken library can't push to the live site. # - # build-site runs on PRs too (no `if: main`). Building the archives is - # harmless off-main and catches archive-generation breakage on the PR; - # PUBLISHING is the boundary, and that lives entirely in `deploy` - # (gated `if: main`). It also uploads the `browse-archives` artifact so - # the e2e- jobs can boot the exact built bytes without - # rebuilding the site. + # Selected PR archive checks run here too, without website/WASM assembly. + # PUBLISHING is the boundary, entirely in the canonical-main full caller. + # The browse-archives artifact lets smokes boot the exact producer bytes. # ------------------------------------------------------------------- build-site: # Publish blog archives before waiting for the independent Campfire # inputs. assemble-site adds those to the same bytes before deployment. - needs: [generate-fixture, build-wasm] - if: ${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-wasm.result == 'success' }} + needs: [generate-fixture, build-wasm, plan] + if: ${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-site') && needs.generate-fixture.result == 'success' && (needs.plan.outputs.site != 'true' || needs.build-wasm.result == 'success') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -2379,6 +2548,7 @@ jobs: # The compiler wasm, built fresh from source by the parallel build-wasm # job (no committed binary). Lands in _site/lib/ for /playground/+/studio/. - uses: actions/download-artifact@v8 + if: needs.plan.outputs.site == 'true' with: name: roundhouse-wasm path: wasm-artifact @@ -2434,10 +2604,17 @@ jobs: cp fixtures/real-blog/app/javascript/application.js build/assets/ cp fixtures/real-blog/app/javascript/controllers/*.js build/assets/controllers/ echo "baked asset graph:"; (cd build/assets && find . -type f | sort) - - name: roundhouse --site + - name: Build selected archives or the complete site env: ROUNDHOUSE_ASSETS_DIR: build/assets - run: cargo run --release --bin roundhouse -- --site + SITE: ${{ needs.plan.outputs.site }} + TARGETS: ${{ needs.plan.outputs.archives }} + run: | + if [ "$SITE" = true ]; then + cargo run --release --bin roundhouse -- --site + else + cargo run --release --bin roundhouse -- --archives "$TARGETS" + fi # Live SharedWorker blog demo → _site/blog/. Emits real-blog under # the worker profile, vite-builds it with ROUNDHOUSE_BASE set to the @@ -2450,6 +2627,7 @@ jobs: # ROUNDHOUSE_BASE here must match the base hard-coded in # site/404.html. - name: Build live SharedWorker blog demo (→ _site/blog) + if: needs.plan.outputs.site == 'true' env: ROUNDHOUSE_BASE: /roundhouse/blog/ run: | @@ -2475,6 +2653,7 @@ jobs: # and the "Publishing to Pages" section of docs/archive/browser-demo-plan.md. Pure # static files (no npm/vite, no COOP/COEP), so they serve straight off Pages. - name: Bundle the in-browser playground + studio demos (→ _site/lib, _site/playground, _site/studio) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail mkdir -p _site/lib _site/playground _site/studio @@ -2501,13 +2680,14 @@ jobs: # LICENSE + commit embedded — AGPL source redistribution). Same # bundle + cache key the browser-smoke-ide job verified. - name: Cache Mastodon source bundle + if: needs.plan.outputs.site == 'true' id: mastodon-src-site uses: actions/cache@v4 with: path: wasm/ide/app-src.json key: mastodon-app-src-${{ env.MASTODON_SHA }}-v2 - name: Fetch pinned Mastodon + bundle sources (cache miss) - if: steps.mastodon-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.mastodon-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/mastodon/mastodon/tar.gz/${MASTODON_SHA}" -o /tmp/mastodon.tar.gz @@ -2516,6 +2696,7 @@ jobs: --name mastodon --commit "${MASTODON_SHA}" \ --open app/controllers/statuses_controller.rb - name: Bundle the in-browser IDE demo (→ _site/ide) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail mkdir -p _site/ide @@ -2531,13 +2712,14 @@ jobs: # gap-tolerant. Lobsters (ruby-bench, MIT) ships to both; the blog ships # to /ide/ as a source bundle (the playground reuses lib/fixture.json). - name: Cache lobsters source bundle + if: needs.plan.outputs.site == 'true' id: lobsters-src-site uses: actions/cache@v4 with: path: /tmp/app-lobsters.json key: lobsters-app-src-${{ env.RUBY_BENCH_SHA }}-v1 - name: Fetch pinned ruby-bench + bundle lobsters (cache miss) - if: steps.lobsters-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.lobsters-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/ruby/ruby-bench/tar.gz/${RUBY_BENCH_SHA}" -o /tmp/ruby-bench.tar.gz @@ -2547,13 +2729,14 @@ jobs: --name lobsters --commit "${RUBY_BENCH_SHA}" \ --open app/controllers/stories_controller.rb - name: Cache campfire source bundle + if: needs.plan.outputs.site == 'true' id: campfire-src-site uses: actions/cache@v4 with: path: /tmp/app-campfire.json key: campfire-app-src-${{ env.CAMPFIRE_SHA }}-v1 - name: Fetch pinned once-campfire + bundle sources (cache miss) - if: steps.campfire-src-site.outputs.cache-hit != 'true' + if: needs.plan.outputs.site == 'true' && steps.campfire-src-site.outputs.cache-hit != 'true' run: | set -euo pipefail curl -fsSL "https://codeload.github.com/basecamp/once-campfire/tar.gz/${CAMPFIRE_SHA}" -o /tmp/campfire.tar.gz @@ -2562,6 +2745,7 @@ jobs: --name campfire --commit "${CAMPFIRE_SHA}" \ --open app/models/message.rb - name: Assemble app-picker bundles + manifests (→ _site/lib, _site/ide) + if: needs.plan.outputs.site == 'true' run: | set -euo pipefail # Blog source bundle for /ide/ (the playground reuses lib/fixture.json). @@ -2602,10 +2786,10 @@ jobs: # `_site/bench/` directory simply isn't created and the landing # page's link 404s until the next CI cycle picks up fresh data. # These historical reports have no PR test consumer. Fetch them only - # for publication; PRs still build the demos and all smoke archives. + # for publication; full PR validation still builds demos and archives. - name: Fetch latest bench data id: fetch-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party run: | @@ -2636,7 +2820,7 @@ jobs: # the bench box doesn't blank the page. - name: Fetch latest lobsters bench data id: fetch-lobsters-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/lobsters run: | @@ -2708,7 +2892,7 @@ jobs: # having one. - name: Fetch latest lobsters conformance data id: fetch-lobsters-specs - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/lobsters-specs run: | @@ -2733,7 +2917,7 @@ jobs: # comparison rather than two unrelated measurements. - name: Fetch latest campfire bench data id: fetch-campfire-bench - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/campfire run: | @@ -2789,7 +2973,7 @@ jobs: # roundbench.showcase.party/campfire-suite/. - name: Fetch latest compiled campfire conformance data id: fetch-campfire-suite-spinel - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' env: BENCH_URL: https://roundbench.showcase.party/campfire-suite run: | @@ -2808,32 +2992,71 @@ jobs: # The per-target archives, for the e2e- jobs to download and # boot — the exact bytes published to rubys.github.io/roundhouse. + - name: Record archive producer evidence + if: always() + env: + GROUP: browse + OUTCOME: ${{ job.status }} + run: *archive-inventory + - name: Save archive producer evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ci-archive-inventory-browse-${{ github.run_attempt }} + path: ci-evidence/*.json - name: Upload browse archives + if: always() uses: actions/upload-artifact@v7 with: name: browse-archives path: _site/browse/*.tgz - name: Upload Pages artifact - if: github.ref == 'refs/heads/main' + if: needs.plan.outputs.publish == 'true' uses: actions/upload-pages-artifact@v5 with: name: site-base path: _site + # Report actual per-archive outcomes, including failed/unavailable advisory + # consumers. A report is not a passing validation or an execution receipt. + archive-results: + needs: [plan, build-site, build-campfire-archive, smoke, smoke-spinel, smoke-campfire, smoke-campfire-docker] + if: ${{ !cancelled() && needs.plan.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs), 'archive-results') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v5 + - name: Collect this run's archive evidence + continue-on-error: true + uses: actions/download-artifact@v8 + with: + pattern: ci-archive-* + path: ci-evidence + - name: Describe tested bytes without certifying missing evidence + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: python3 scripts/ci-archive-evidence.py report --evidence-dir ci-evidence --out archive-report/archive-results.json + - name: Save archive outcome report + if: always() + uses: actions/upload-artifact@v7 + with: + name: archive-results + path: archive-report/archive-results.json + # Publication only: PR smoke jobs consume browse-archives, not Pages. # Finish Pages assembly without keeping blog smoke jobs behind Campfire. assemble-site: # campfire-conformance is here for ORDERING ONLY — its summary.json is - # this job's input. The `if:` deliberately does not require it to have - # SUCCEEDED: that job is blocking (pinned input, so a red floor is a - # real regression and must fail the run), but the page it feeds is the - # thing that says WHAT regressed, and suppressing the page on the run - # where it is most useful would be exactly backwards. A missing - # artifact degrades to a skipped render and a warning, not a broken - # site build. - needs: [build-site, campfire-conformance, build-campfire-archive] - if: ${{ !cancelled() && github.ref == 'refs/heads/main' && needs.build-site.result == 'success' }} + # this job's input. Assembly can render a failed conformance report + # for triage, but a failed compact floor still prevents Pages deploy. + # A missing summary degrades to a skipped render and warning here; + # assembly does not override the separate publication guards. + needs: [build-site, campfire-conformance, build-campfire-archive, archive-results, plan] + if: ${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' && needs.archive-results.result == 'success' }} + outputs: + ready: ${{ steps.ready.outputs.ready }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -2847,22 +3070,25 @@ jobs: path: site-base - name: Restore the site run: mkdir -p _site && tar -xf site-base/artifact.tar -C _site + - name: Attach archive outcomes alongside the immutable downloads + uses: actions/download-artifact@v8 + with: + name: archive-results + path: _site/ci # The campfire archive, built by the parallel build-campfire-archive - # job. It lands AFTER `--site` because build_site() removes and - # recreates the output directory, which would take _site/campfire - # with it. + # job, is attached after restoring this run's site-base artifact. + # Assembly never re-emits archives or rebuilds from a newer main. # # The bytes go live at rubys.github.io/roundhouse/campfire/spinel.tgz # (source; needs a spinel checkout) and campfire/docker.tgz (the # `spin pack` output with a Dockerfile; needs Docker and nothing - # else). Both are linked from site/apps/campfire.html. + # else), if publication guards pass. Both are linked from + # site/apps/campfire.html. # - # Tolerant of a missing artifact ON PURPOSE, and for the same reason - # this job does not require campfire-conformance to have SUCCEEDED: - # per matz/spinel#314 the published archives are an extraction - # surface for upstream bug repros, so a broken campfire emit must not - # stall publication of the eleven blog archives beside it. + # Missing or failed archive outputs remain honest repro evidence, + # not successful validation. Same-run hashes and download availability + # are verified separately; compact-floor failure still blocks deploy. - uses: actions/download-artifact@v8 continue-on-error: true with: @@ -2908,32 +3134,50 @@ jobs: rmdir _site/bench 2>/dev/null || true fi + - name: Verify published archive bytes + # Verify the copied download bytes against their producer/test report. + # Failed validations may publish as repros; mismatched bytes may not. + run: python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json - name: Upload Pages artifact uses: actions/upload-pages-artifact@v5 with: path: _site + - name: Report successful assembly + id: ready + run: echo 'ready=true' >> "$GITHUB_OUTPUT" - deploy: - needs: - - assemble-site - - unit - # Forks run the full build/test matrix but need no Pages configuration. - # Only the canonical repository publishes the project website. - if: github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' - # Only one Pages deployment at a time. Scoped to this job, not the - # workflow: at the workflow level the same group serialised every - # run — a pull request queued behind a push to main and the next - # push behind that PR, at 20-40 minutes a run — for a lock only the - # deployment needs. - concurrency: - group: pages - cancel-in-progress: false + compact-required: + needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare] + if: always() runs-on: ubuntu-latest - timeout-minutes: 30 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + timeout-minutes: 5 + outputs: + passed: ${{ steps.gate.outputs.passed }} + steps: + - uses: actions/checkout@v5 + - name: Require every selected compact check to succeed + id: gate + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: | + python3 scripts/ci-plan.py compact-gate + echo 'passed=true' >> "$GITHUB_OUTPUT" + + # Informational validation summary; no branch-protection or merge policy. + ci-summary: + name: CI summary + needs: [plan, generate-fixture, unit, store-check, compare, compare-ruby, browser-smoke-typescript, campfire-conformance, campfire-compare, compact-required, compare-extra, compare-jruby, build-wasm, browser-smoke-ide, build-site, smoke, writebook-inventory, build-spinel, framework-tests-spinel, build-campfire-compare-spinel, campfire-compare-spinel, campfire-db-differential-spinel, toolchain-spinel, compare-spinel, smoke-spinel, build-campfire-archive, smoke-campfire, smoke-campfire-docker, archive-results, assemble-site] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + complete: ${{ steps.gate.outputs.complete }} steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v5 + - uses: actions/checkout@v5 + - name: Report missing, skipped or failed selected checks + id: gate + env: + CI_PLAN: ${{ needs.plan.outputs.plan }} + CI_NEEDS: ${{ toJSON(needs) }} + run: python3 scripts/ci-plan.py gate diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml new file mode 100644 index 0000000000..b1b4c35e45 --- /dev/null +++ b/.github/workflows/full-ci.yml @@ -0,0 +1,90 @@ +name: Full validation + +on: + schedule: + - cron: '17 */4 * * *' + workflow_dispatch: + inputs: + publish: + description: 'Publish after the compact gate passes (canonical main only)' + type: boolean + default: false + +permissions: + contents: read + +# Keep a started snapshot; replace obsolete pending requests. This literal +# namespace is different from the reusable validation workflow's lock. +concurrency: + group: full-background-${{ github.ref }} + cancel-in-progress: false + +jobs: + preflight: + if: github.event_name != 'schedule' || github.repository == 'rubys/roundhouse' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + spinel: ${{ steps.inputs.outputs.spinel }} + publish: ${{ steps.inputs.outputs.publish }} + steps: + - name: Resolve the immutable inputs and publication boundary + id: inputs + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + REQUEST_PUBLISH: ${{ inputs.publish || false }} + run: | + publish=false + if [ "$EVENT" = schedule ] || [ "$REQUEST_PUBLISH" = true ]; then + test "$GITHUB_REPOSITORY" = rubys/roundhouse + test "$GITHUB_REF" = refs/heads/main + publish=true + fi + spinel=$(gh api repos/matz/spinel/commits/master --jq .sha) || spinel=master + if ! [[ "$spinel" =~ ^[0-9a-f]{40}$ ]]; then spinel=master; fi + echo "spinel=$spinel" >> "$GITHUB_OUTPUT" + echo "publish=$publish" >> "$GITHUB_OUTPUT" + - name: Report fresh validation inputs + env: + SPINEL: ${{ steps.inputs.outputs.spinel }} + run: | + printf 'Source snapshot: `%s`\n\nRequested Spinel: `%s`\n\nEvery cycle executes validation freshly, including floating Rails/package/system inputs. Actual Spinel revisions are recorded in the toolchain and archive evidence.\n' "$GITHUB_SHA" "$SPINEL" >> "$GITHUB_STEP_SUMMARY" + + validation: + needs: preflight + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci.yml + with: + full: true + publish: ${{ needs.preflight.outputs.publish == 'true' }} + spinel_revision: ${{ needs.preflight.outputs.spinel }} + + # No Pages/OIDC permissions are available to the reusable validator or any + # PR. Extra-target failures must not hide useful broken-emit repro archives. + deploy: + needs: [preflight, validation] + if: ${{ !cancelled() && needs.preflight.outputs.publish == 'true' && needs.validation.outputs.publication-ready == 'true' && github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main' }} + permissions: + contents: read + pages: write + id-token: write + concurrency: + group: pages + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Refuse to publish a superseded main snapshot + env: + GH_TOKEN: ${{ github.token }} + VALIDATED_SHA: ${{ github.sha }} + run: test "$(gh api repos/rubys/roundhouse/git/ref/heads/main --jq .object.sha)" = "$VALIDATED_SHA" + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v5 diff --git a/AGENTS.md b/AGENTS.md index 6673a9cfcb..5fb86916f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,9 +87,14 @@ defect even if the build is green. you changed. End commit messages with the standard `Co-Authored-By` trailer. - **Outside contributors: fork, and open a pull request against `main`.** - CI runs the full matrix on a PR — every toolchain lane, the DOM compare - against live Rails, the Spinel lanes — so you do not need every - toolchain locally; CI is the oracle for the lanes you cannot run. + CI runs a compact floor plus targeted lanes. For broad/risky changes, + ask a maintainer to apply `ci:full` for the complete PR matrix. + This expands validation only; PR runs never publish. Drafts retain + fixture + unit coverage even with the label. Full main validation and + publication are scheduled every four hours and execute freshly every cycle. + `CI summary` reports selected checks, not a mandatory merge gate; + maintainers decide when to merge. See [CI coverage](docs/ci-reuse.md). + You do not need every toolchain locally; CI covers the missing lanes. Before opening one: `bin/rh fixture` (the test fixtures are generated, not checked in — see below), `cargo test --lib` plus the targeted integration test for what you touched, and a test that pins the fix. @@ -110,15 +115,17 @@ defect even if the build is green. `cargo test --all-targets` at milestones. Real-toolchain tests are `#[ignore]`-gated (`cargo test --test _toolchain -- --ignored`); CI runs each in its own job. -- **CI is deliberately not uniformly gating.** The core `cargo test` job gates. - The ~5 `continue-on-error: true` jobs track upstream Spinel and other moving - toolchains on purpose — **red there is a signal to read, not a regression to - shim away.** Don't add workarounds just to make an advisory job green. +- **CI is deliberately not uniformly gating.** The core `cargo test` job is + non-advisory. Jobs marked `continue-on-error: true` track current Spinel + master and other moving toolchains on purpose — **red is a signal, not a + regression to shim away.** A Spinel red can come from Roundhouse runtime, RBS + or packaging, or from upstream; it is not by itself proof of an upstream + fault. Don't add workarounds just to make an advisory job green. ## The actual goal The endpoint is not "does the fixture compile." It is a **per-target ledger of how much of Rails transpiles** — the honest unsupported list, driven down over time. Don't trade that real goal for a locally reachable one. The proving lanes -today are the real-blog fixture (every target, DOM-equivalent to Rails on every -push) and lobsters/Mastodon on the inference + Spinel path (see README). +today are the real-blog fixture (every target, DOM-equivalent to Rails in full +validation) and lobsters/Mastodon on the inference + Spinel path (see README). diff --git a/README.md b/README.md index 5425b689a8..c2679c806e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ oracle: the same URL fetched from Rails and from each target must produce the same response — emitted tests, a differential compare against live Rails (DOM node for DOM node, JSON value for value), and browser end-to-end tests for what a static diff can't reach — and -every target on the list passes it on every push. → +the full target matrix runs in scheduled validation, with a compact +floor and targeted additions on pull requests. → [`--target`](docs/guide/transpile.md) · [targets](docs/guide/targets.md) · [what of Rails comes through](docs/guide/rails-coverage.md) · [verifying](docs/guide/verifying.md) @@ -73,7 +74,7 @@ compiled targets it has the closest behavior to Rails by a distance, and will for the foreseeable future, because it runs the framework runtime itself rather than a translation of it. Basecamp's Campfire runs this way — every page and every cable frame compared against -live Rails on every push, and a +live Rails in the full validation cycle, and a [Docker archive](https://rubys.github.io/roundhouse/apps/campfire.html) you can run in minutes. → [Spinel](docs/guide/spinel.md) @@ -106,7 +107,7 @@ the prerequisites. - [**Campfire**](https://rubys.github.io/roundhouse/apps/campfire.html) — the compiled product, as a Docker archive. - [**Browse**](https://rubys.github.io/roundhouse/browse/) — what every - emitter produces from the blog fixture, updated on each push. + emitter produces from the blog fixture, refreshed by scheduled full validation. - [**Bench**](https://rubys.github.io/roundhouse/bench/) — throughput, memory and latency across the live targets on a fixed box, against Rails as it ships. @@ -168,13 +169,17 @@ its predecessors; [`WHY.md`](WHY.md) is why do it at all. ## Contributing Issues and pull requests are both welcome, and a PR does not need a -conversation first: CI runs the whole matrix on every pull request — -the unit suite, each target's toolchain, the DOM compare against live -Rails, the Spinel lanes — so a change you can only test partially on -your machine is tested fully there. The `unit` job gates; the Spinel -and Campfire jobs marked `continue-on-error` track moving toolchains, -and red there is a signal to read, not a check to make pass. Setup, -the test cycle, and what a PR should carry are in +conversation first: CI runs a compact correctness/runtime floor plus +targeted checks. Ask a maintainer to apply `ci:full` for full PR validation; +the label expands tests, never publication. The complete matrix also runs +freshly every four hours. See [CI coverage](docs/ci-reuse.md) for requesting +full/fresh validation and the separate publication controls. +`CI summary` reports selected checks without imposing a merge requirement; +maintainers decide when to merge. Spinel and Campfire jobs marked +`continue-on-error` track current Spinel master, and red there is a signal +to investigate Roundhouse runtime, RBS or packaging as well as possible +upstream drift—not proof of an upstream fault. Setup, the test cycle, +and what a PR should carry are in [`DEVELOPMENT.md`](DEVELOPMENT.md); the invariants not to break are in [`AGENTS.md`](AGENTS.md). diff --git a/docs/ci-reuse.md b/docs/ci-reuse.md index 9f9bbcdf4b..74ce6c01c6 100644 --- a/docs/ci-reuse.md +++ b/docs/ci-reuse.md @@ -1,10 +1,137 @@ -# Conservative PR-local CI reuse +# CI coverage and conservative PR-local reuse + +## Compact PR checks and full validation + +`.github/workflows/ci.yml` runs a compact floor on PRs and main pushes: + +- Fixture generation and `unit` (`cargo test --all-targets`, including emitted + Ruby execution tests and the debug-profile bench emission checks). +- Store analysis, Ruby/Rust/TypeScript comparisons against live Rails. +- TypeScript SharedWorker browser tests, Campfire conformance, and Campfire + comparison including its model/database differential. + +These are nine validation executions, plus three small orchestration jobs +(`plan`, `compact-required`, `ci-summary`). Drafts select only fixture and +unit validation. **`CI summary` is informational:** it reports missing, +skipped, cancelled or failed selected non-advisory checks as red. Unselected +jobs may skip; advisory failures remain separate signals. This workflow does +not configure branch protection or require the summary to pass before merging; +maintainers decide when to merge. Publication still requires the compact floor +and verified assembly. + +The planner compares the actual PR merge tree with its base, not just the last +commit. Renames/deletions retain both ownership sets. Unknown diff identity +expands to full validation. Documentation-only PRs still get the summary +status instead of being left pending by workflow-level path filters. + +The CI helpers use Python's standard library, following the existing receipt +helper, without additional Python packages. + +| Changed inputs | Additional coverage | +|---|---| +| Target emitter file **or directory**, target runtime, toolchain/framework test | Owning comparison and archive smoke; existing embedded framework/toolchain suites stay intact | +| Ruby emitter | Ruby/JRuby owners plus advisory native Spinel core (`build-spinel`, `toolchain-spinel`, `compare-spinel`) | +| `runtime/ruby/`, native `runtime/spinel/`, or a focused Spinel test | Advisory native core plus the relevant focused Spinel framework suite | +| Interpreter-only `db_jruby` / `markly_jruby` or `scaffold/ruby_overlay` | Interpreted Ruby/JRuby owners, not native fanout | +| Spinel scaffold packaging | Advisory native core plus the native archive smoke | +| Shared emitter helpers (`src/emit/shared/`) | Full validation across target languages | +| Shared analyzer or lowerer | Compact floor; no mandatory Spinel lane on an ordinary PR; reviewers can request `ci:full` for broad/risky changes | +| `wasm/` | WASM build and IDE/playground/studio browser verification | +| Site/guide sources | Site/archive build and WASM verification, without publishing | +| Shared compare, framework, archive, or E2E harness | The checks owned by that harness | +| Cross-target packaging, CI policy/workflows/planner, Cargo/build/toolchain policy, unknown new target | Full validation | + +### Requesting broader or fresh validation + +For broad/risky changes, or when targeted coverage is insufficient, request +**`ci:full`**. Applying labels requires upstream repository triage access or +higher; fork contributors and their agents without that access should ask a +maintainer to apply the label. A request in a PR comment alone does not trigger +CI. The label must exist in the upstream repository before it can be applied. + +- The PR must be **ready for review**: drafts retain fixture + unit only, + even with `ci:full`. +- Applying the label starts a new full-matrix run of the current **PR merge + tree**, without a new commit. Superseded PR runs are cancelled. +- Further pushes retain full coverage while the label remains set. +- Removing it starts a run with automatic coverage selection. Policy/shared + emitter changes can still select full coverage without the label. +- Full **coverage** does not disable conservative PR execution-receipt reuse. + For fresh execution, ask a maintainer to select **Re-run all jobs** on the + full-matrix run. A rerun retains that run's original SHA and event coverage; + rerunning an older compact run does not expand coverage or test a newer head. + +A manual **Actions → Full validation → Run workflow** dispatch validates the +chosen ref freshly. A branch-head dispatch is not a replacement for the PR +merge-tree check. Leave **publish** unchecked for validation only. + +### Validation is not publication + +`full` selects coverage; `publish` separately authorizes publication work. +Applying `ci:full` never enables `publish`. + +| Trigger | Publication behavior | +|---|---| +| PR, including `ci:full`, or ordinary main push | Validation/artifacts only; no deploy | +| Manual Full validation, `publish=false` (default) | Fresh validation only; no deploy | +| Manual Full validation, `publish=true` | Accepted only on canonical `rubys/roundhouse` main; guarded publication | +| Four-hour schedule on canonical main | Fresh full validation plus guarded publication | + +The shared validator and PR jobs have no Pages/OIDC deployment privileges and +no deploy job. Only the separate full-workflow deploy job receives those +permissions. Publication requests outside canonical main and schedule/manual +events are rejected. Publication still requires the compact floor, verified +assembly and a live-main SHA check; the detailed archive contract follows below. + +### Current-run artifacts and the full cycle + +Targeted archive smokes use `roundhouse --archives rust,go` (selected +`browse/.{json,tgz,zip}` outputs) and do not build WASM, demos, website +assets or unrelated archives. The same archive writers power `--site`, which +keeps the complete developer-facing build. Full publication and smoke consume +the same producer bytes, without later re-emission. + +`.github/workflows/full-ci.yml` checks canonical main at **00:17, 04:17, +08:17, 12:17, 16:17 and 20:17 UTC**, bundling full validation and publication. +Every cycle executes freshly; validation results are not cached. Spinel master +is resolved once per run, and evidence records the compiler revision actually +used. Its lanes remain advisory. Ordinary build caches and the conservative PR +execution receipts described below are unchanged. + +Archive producers upload with `always()`, preserving any files already produced +if a later producer step fails. The outcome report names the source SHA, run and +attempt, actual byte size/hash, and applicable Spinel provenance, and classifies +each archive `passed`, `reused`, `failed`, `unverified`, or `not-selected`. +Validation is byte-specific: a TGZ smoke does not certify sibling ZIP or JSON +bytes. A Spinel source archive contains source, not a built compiler; its native +smoke witness therefore records the compiler revision separately. Likewise a +Docker producer's compiler revision is distinct provenance from a native +archive consumer's compiler revision. + +Publication requires the **compact** floor and assembled site, not passing all +extra-target or advisory lanes: failed or unverified archives can still be +useful repros, but are not described as validated. Assembly waits for the same +run's outcome report, copies only that run's archives, and verifies the actual +copied bytes against the report hashes. It never rebuilds from a newer main. +The published sidecar at `ci/archive-results.json` separately records whether +each archive is present for download. Older-attempt witnesses remain visible +but are conservatively `unverified` on a rerun, not evidence that it passed. +Deployment guards are unchanged: the deployment lock rechecks that canonical +main still equals the validated SHA; lookup failure or a superseded snapshot +prevents publication. A new main commit racing the final check cannot be made +atomic with Pages deployment. +Started background runs finish; only the newest pending request is retained. +Extra comparison/smoke matrices use `max-parallel: 2`, GC uses 1. These bound +fanouts, **not** total repository concurrency or a guaranteed PR runner priority. + +## Reuse within selected PR checks `scripts/ci-reuse.py` can reuse **executed, successful** checks from the same pull request. The allowlist is `store-check`, `writebook-inventory`, Rust archive smoke, SharedWorker browser smoke, and the Rust inflector framework suite. Unit tests, fixture generation, artifact producers, DOM comparisons, other -toolchain lanes and all main-branch checks still execute freshly. +selected toolchain lanes and all selected main-branch checks execute freshly. +Routing changes the required coverage, not the execution-receipt trust rules. ## What must match @@ -135,28 +262,26 @@ Stage-aware cancellation needs a separate controller to inspect the previous run; GitHub's concurrency expression cannot inspect that run's job progress. This PR does not add privileged cancellation machinery. -The next substantial opportunity is sharing **current-run** native compiler -builds, not declaring more outputs unchanged without a dependency witness. -In [run 36998514579](https://github.com/rubys/roundhouse/actions/runs/36998514579), -the unit Cargo step took 309 seconds and the two TypeScript toolchain/framework -steps took 210 seconds together. Those are whole-step timings, not measurements -of compiler cost alone. A shared build needs consistent profiles/toolchains, -native-library ABI, current commit provenance and coverage of each consumer's -actual build flags. It should be measured and proved separately. +A separate measured follow-up may evaluate sharing the `unit` job's Debug +binary with Campfire conformance/compare. This change does not copy Cargo target +directories, share images, introduce Bazel, or claim a measured queue-time +improvement. Any sharing proposal still needs compatible toolchains/build flags, +native-library ABI, current-commit provenance, and proof at each consumer. Matrix `fail-fast: false` still preserves cross-target diagnostic coverage; advisory lanes remain signals rather than reasons to cancel independent checks. -Existing `needs` gates and normal step failure handling are unchanged. The -Campfire Docker recipe no longer downloads a separate Dockerfile frontend: +Receipt reuse does not bypass selected checks' dependency or failure handling. +The Campfire Docker recipe no longer downloads a separate Dockerfile frontend: its ordinary multi-stage instructions use the bundled frontend and COPY preserves the archive's executable boot mode. Base images and apt packages still resolve freshly, and the real Docker smoke remains enabled. ## Forcing a fresh check and extending the allowlist -GitHub's **Re-run jobs** always executes these checks freshly, even if a matching -receipt exists. The fresh attempt may produce new evidence. Existing draft, -`needs` and main-branch behavior is unchanged. +Rerun checks bypass matching receipts. Choose **Re-run all jobs** to execute +the entire selected graph freshly, rather than only failed or individual jobs. +The rerun keeps the original SHA, ref and coverage; the fresh attempt may +produce new evidence. Draft, `needs` and main-branch behavior is unchanged. Before adding another job, audit its complete input contract, including generated artifacts, framework tests, executable README blocks and the actual @@ -168,3 +293,6 @@ unless current-run outputs and their provenance can be preserved honestly. Run `PYTHONDONTWRITEBYTECODE=1 python3 tests/ci_reuse_test.py -v` and `cargo test --test workflow_yaml_parses` when changing this policy. The Rust workflow tests execute the Python adversarial suite, so normal unit CI gates it. +For coverage routing, archive evidence or full-workflow changes, also run +`cargo test --test ci_policy_workflow`; it executes the planner and archive +evidence suites and checks the publication boundaries. diff --git a/docs/guide/rails-coverage.md b/docs/guide/rails-coverage.md index f6f4e358b5..66649327b6 100644 --- a/docs/guide/rails-coverage.md +++ b/docs/guide/rails-coverage.md @@ -23,9 +23,10 @@ tiers. **The blog** (`fixtures/real-blog`, the Rails 8 scaffold with articles, comments, nested routes, validations, Turbo Streams, Action Cable, -Tailwind, JSON endpoints) is what **every server target** passes the -DOM-equivalence gate against on every push. What the blog uses is -supported everywhere. +Tailwind, JSON endpoints) is the shared DOM-equivalence fixture for +**every server target** in full validation. Ordinary PRs/main pushes run +the compact floor plus selected target lanes; see [CI coverage](../ci-reuse.md). +A passing target's comparison proves the blog's features on that target. **Campfire** (Basecamp's chat product — file attachments with image variants, rich text, web push, bots and webhooks, full-text search, diff --git a/docs/guide/spinel.md b/docs/guide/spinel.md index 4f4beea57a..002c677b8f 100644 --- a/docs/guide/spinel.md +++ b/docs/guide/spinel.md @@ -149,8 +149,10 @@ A new deployment with no such volume can drop `boot` for `USER runs the whole product — sign-in, rooms, uploads, search, live updates over the socket — from `docker run -p 3000:3000`. [rubys.github.io/roundhouse/campfire/docker.tgz](https://rubys.github.io/roundhouse/campfire/docker.tgz) -is that archive, rebuilt on every push; it is the fastest way to see -the door's end state before pointing it at your own app. +is that archive, refreshed by scheduled full validation or an explicitly +publishing manual run on canonical main. PR archive checks never publish it; +see [CI coverage](../ci-reuse.md). It is the fastest way to see the door's +end state before pointing it at your own app. ## What to expect @@ -169,8 +171,9 @@ cleanly, and when it does not, the failure is in one of two places: - **It builds but behaves differently from Rails.** The [compare oracle](verifying.md) is the reproduction: `bin/rh compare spinel` for the fixture, or `roundhouse-compare` against your own - app. The Campfire lane compares every page and every cable frame - against live Rails on every push. + app. The Campfire Spinel lane compares every page and every cable frame + against live Rails in full validation and when its owning inputs select + it; ordinary native Spinel changes do not automatically run that lane. The binary's performance is measured continuously on the blog and on Campfire against Rails as Rails ships it — with its fragment caching diff --git a/docs/guide/targets.md b/docs/guide/targets.md index fe11d366ab..dce9a76192 100644 --- a/docs/guide/targets.md +++ b/docs/guide/targets.md @@ -36,8 +36,11 @@ app's model and controller tests and a Playwright `e2e/` suite; the ## How far each is tested -Every claim below is a CI job on every push to `main`, all of them -against the blog fixture (`fixtures/real-blog`: articles, comments, +The lanes below run across targets in full validation, scheduled every four +hours or requested manually. Ordinary PRs/main pushes use a compact floor plus +targeted additions; maintainers can request full PR coverage with `ci:full`. +See [CI coverage](../ci-reuse.md). The lanes use the blog fixture +(`fixtures/real-blog`: articles, comments, nested routes, validations, Turbo Streams over Action Cable, Tailwind) unless another app is named. A target's row in [`RELEASES.md`](../../RELEASES.md) records where these stood at the @@ -62,8 +65,9 @@ that gap. ## The conformance bar -A target is on the list above because it passes `compare` on the -fixture on every push. A page that renders one node differently — an +The conformance bar is a passing `compare` result against the fixture, +not the assumption that every target was checked on an ordinary PR. +A page that renders one node differently — an attribute missing, a whitespace text node — is red. That is deliberately a higher bar than "the tests pass": it means the emitted app is the same app, as a browser or a scraper would see it, not a diff --git a/docs/pipeline/verification.md b/docs/pipeline/verification.md index 5cdac7d60d..9c70cf6368 100644 --- a/docs/pipeline/verification.md +++ b/docs/pipeline/verification.md @@ -38,7 +38,8 @@ error diagnostics fire, and the framework Ruby is itself sound. `#[ignore]`d, so it gates every `cargo test`. All of this runs in the `unit` CI job (`cargo test --all-targets`) — -the one job the Pages deploy is actually gated on (see CI topology). +part of the compact floor required for Pages publication, alongside +the other baseline checks and verified assembly (see CI topology). **Why it exists:** if the ingester silently drops a construct, the emitter has nothing to emit and downstream tests trivially pass. @@ -251,9 +252,10 @@ Both lanes **publish a worklist**, not just a floor. per-file tally, spliced-stub ledger, and the failures clustered into named causes by `bench/campfire/suite-causes.json` — which `scripts/campfire-suite-report` renders to -`/bench/campfire-suite/`. build-site consumes that artifact from the -same run WITHOUT gating on the job's conclusion: a run that trips the -floor is precisely the run whose page someone needs to read. The +`/bench/campfire-suite/`. Publication assembly consumes that artifact +from the same run and can render a failed conformance report for triage. +A failed compact floor still prevents Pages deployment; the raw CI +summary remains available for investigation. The lobsters twin (`scripts/lobsters-spec-report` → `/bench/lobsters-specs/`) keeps the same contract over data fetched from the bench box. Re-running either report against a saved tally is @@ -282,34 +284,41 @@ diffs between two IRs drop out from plain `diff` across the outputs. ## CI topology -`.github/workflows/ci.yml`, honestly stated: +The current selection and publication policy is documented in +[CI coverage](../ci-reuse.md). The ownership boundaries are: - **`generate-fixture`** builds `fixtures/real-blog` once per run and shares it as an artifact every downstream job downloads. -- **The hard gate is `unit`.** `deploy` needs exactly - `[build-site, unit]`; `build-site` needs - `[generate-fixture, build-wasm]`. Archives publish *current emit* - even when target jobs are red — broken emit is exactly what's - interesting to investigate upstream (see the build-site comment). -- **Everything else is red-as-signal:** compare, smoke, - framework-tests, and toolchain jobs turn the run red without - blocking deploy; the red is the shared worklist. -- **Seven jobs are `continue-on-error`**, all for one structural - reason — they track matz/spinel master unpinned, so upstream churn - must not gate unrelated work: `build-spinel`, - `framework-tests-spinel`, `toolchain-spinel`, `compare-spinel`, - `smoke-spinel`, `smoke-campfire`, and `campfire-compare-spinel`, - which runs as three legs. The cost: an advisory failure is invisible - in the run's conclusion. Read the jobs, not the conclusion. +- **PRs/main pushes select coverage**, rather than running every target + on every push: compact floor plus owned additions, or full coverage + for policy/shared-emitter changes and `ci:full`. Drafts run fixture + unit. +- **`CI summary` reports selected results**, including missing/skipped/ + cancelled non-advisory checks. It does not configure mandatory merge + protection; maintainers decide when to merge. +- **Spinel master remains advisory.** Its revision is resolved once per + run and the actual built revision is recorded. Failures can come from + Roundhouse runtime/RBS/packaging or upstream. Inspect raw job outcomes + and archive evidence, not just the overall run conclusion. +- **Publication is separate**, in `.github/workflows/full-ci.yml`: + four-hour scheduled validation/publication on canonical main, or fresh + manual validation with publication opt-in. PR checks never deploy. + Pages requires the compact floor, verified same-run assembly and a + live-main SHA check. Extra-target/advisory failures may remain useful + repro archives, but are not presented as successful validation. + +The native Campfire comparison's GC modes run when that lane is selected, +not automatically for every native-runtime change: + - **`campfire-compare-spinel (default)`, `(minor-gc)` and - `(verify-gen)`** serve the same binary, from the same `build-spinel` - artifact, through the same Rails-oracle walk. `minor-gc` adds + `(verify-gen)`** serve the same comparison binary, produced by + `build-campfire-compare-spinel` using the shared `build-spinel` toolchain, + through the same Rails-oracle walk. `minor-gc` adds `SPINEL_GC_MINOR=1`, spinel's generational collector, which is opt-in until it goes default-on. matz asked for that leg (matz/spinel#4260): campfire is the retained-heap shape the default-on decision lacks. A red `minor-gc` beside a green `default` - is the collector alone — reduce it, file it upstream, and say - minor-GC-only in the title. + is a mode-specific signal: reduce the failing case and establish its + cause before attributing it upstream. - **`(verify-gen)` is why the other two can be believed.** `minor-gc` only fails when a missed write barrier reaches the page; `SPINEL_GC_VERIFY_GEN=1` re-marks the whole heap after every minor diff --git a/scripts/ci-archive-evidence.py b/scripts/ci-archive-evidence.py new file mode 100755 index 0000000000..10c07e3499 --- /dev/null +++ b/scripts/ci-archive-evidence.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Record and summarize byte-level evidence for CI download archives.""" + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path + +SCHEMA = 1 +SHA = re.compile(r"[0-9a-f]{40}\Z") +GROUPS = {"browse": "build-site", "campfire": "build-campfire-archive"} +CHECKS = { + **{ + f"smoke-{target}": f"browse/{target}.tgz" + for target in ( + "rust", + "crystal", + "kotlin", + "swift", + "csharp", + "typescript", + "go", + "elixir", + "python", + "ruby", + "jruby", + ) + }, + "smoke-spinel": "browse/spinel.tgz", + "smoke-campfire": "campfire/spinel.tgz", + "smoke-campfire-docker": "campfire/docker.tgz", +} +CHECK_JOBS = { + check: ( + "smoke" if path.startswith("browse/") and check != "smoke-spinel" else check + ) + for check, path in CHECKS.items() +} + + +def identity(): + source = os.environ.get("GITHUB_SHA", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + attempt = os.environ.get("GITHUB_RUN_ATTEMPT", "") + if not SHA.fullmatch(source): + raise ValueError("GITHUB_SHA must be a lowercase full 40-character SHA") + if not run_id.isdigit() or int(run_id) < 1: + raise ValueError("GITHUB_RUN_ID must be a positive integer") + if not attempt.isdigit() or int(attempt) < 1: + raise ValueError("GITHUB_RUN_ATTEMPT must be a positive integer") + return source, run_id, int(attempt) + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def revision(path): + if path is None or not Path(path).is_file(): + return None + value = Path(path).read_text(encoding="utf-8").strip() + if not SHA.fullmatch(value): + raise ValueError(f"invalid Spinel revision in {path}") + return value + + +def write_json(path, value): + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(destination.name + ".tmp") + temporary.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n") + temporary.replace(destination) + + +def base(kind): + source, run_id, attempt = identity() + return { + "schema": SCHEMA, + "kind": kind, + "source_sha": source, + "run_id": run_id, + "attempt": attempt, + } + + +def inventory(args): + root = Path(args.root) + compiler = revision(args.spinel_revision_file) + archives = [] + group_root = root / args.group + if group_root.is_dir(): + for path in sorted(group_root.iterdir()): + if not path.is_file() or path.suffix not in {".tgz", ".zip", ".json"}: + continue + logical = path.relative_to(root).as_posix() + archives.append( + { + "path": logical, + "sha256": digest(path), + "size": path.stat().st_size, + "spinel_revision": ( + compiler if logical == "campfire/docker.tgz" else None + ), + } + ) + document = base("inventory") + document.update( + {"group": args.group, "producer_outcome": args.outcome, "archives": archives} + ) + write_json(args.out, document) + + +def validation(args): + if CHECKS[args.check] != args.path: + raise ValueError(f"{args.check} does not validate {args.path}") + archive = Path(args.archive) + document = base("validation") + document.update( + { + "check": args.check, + "path": args.path, + "sha256": digest(archive) if archive.is_file() else None, + "size": archive.stat().st_size if archive.is_file() else None, + "outcome": args.outcome, + "spinel_revision": revision(args.spinel_revision_file), + } + ) + write_json(args.out, document) + + +def load_evidence(directory, source, run_id, current_attempt): + inventories, validations = {}, {} + for path in sorted(Path(directory).rglob("*.json")): + try: + item = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError(f"malformed evidence {path}: {error}") from error + if not isinstance(item, dict) or item.get("schema") != SCHEMA: + raise ValueError(f"malformed evidence {path}: unsupported schema") + if item.get("source_sha") != source or item.get("run_id") != run_id: + raise ValueError(f"foreign evidence {path}") + attempt = item.get("attempt") + if type(attempt) is not int or not 1 <= attempt <= current_attempt: + raise ValueError(f"malformed evidence {path}: invalid attempt") + kind = item.get("kind") + if kind == "inventory": + validate_inventory(item, path) + key = item["group"] + target = inventories + elif kind == "validation": + validate_validation(item, path) + key = item["check"] + target = validations + else: + raise ValueError(f"malformed evidence {path}: invalid kind") + previous = target.get(key) + if previous is None or item["attempt"] > previous["attempt"]: + target[key] = item + elif item["attempt"] == previous["attempt"] and item != previous: + raise ValueError(f"conflicting evidence for {key} attempt {attempt}") + return inventories, validations + + +def valid_hash(value): + return value is None or bool(re.fullmatch(r"[0-9a-f]{64}", value)) + + +def validate_inventory(item, path): + if ( + item.get("group") not in GROUPS + or not isinstance(item.get("producer_outcome"), str) + or not isinstance(item.get("archives"), list) + ): + raise ValueError(f"malformed inventory {path}") + seen = set() + for archive in item["archives"]: + if not isinstance(archive, dict): + raise TypeError(f"malformed inventory archive {path}") + logical = archive.get("path") + prefix = item["group"] + "/" + if ( + not isinstance(logical, str) + or not logical.startswith(prefix) + or "/" in logical[len(prefix) :] + or Path(logical).suffix not in {".tgz", ".zip", ".json"} + or logical in seen + or not valid_hash(archive.get("sha256")) + or archive.get("sha256") is None + or type(archive.get("size")) is not int + or archive["size"] < 0 + or ( + archive.get("spinel_revision") is not None + and not SHA.fullmatch(archive["spinel_revision"]) + ) + or ( + logical != "campfire/docker.tgz" + and archive.get("spinel_revision") is not None + ) + ): + raise ValueError(f"malformed inventory archive {path}") + seen.add(logical) + + +def validate_validation(item, path): + check = item.get("check") + if ( + check not in CHECKS + or item.get("path") != CHECKS.get(check) + or not isinstance(item.get("outcome"), str) + or not valid_hash(item.get("sha256")) + or (item.get("size") is not None and type(item.get("size")) is not int) + or (item.get("sha256") is None) != (item.get("size") is None) + or ( + item.get("spinel_revision") is not None + and not SHA.fullmatch(item["spinel_revision"]) + ) + ): + raise ValueError(f"malformed validation {path}") + + +def expected_paths(plan): + result = set() + for value in plan.get("archives", []): + if not isinstance(value, str): + raise TypeError("CI_PLAN archives must be strings") + result.add(value if "/" in value else f"browse/{value}.tgz") + jobs = plan.get("jobs", []) + smoke = plan.get("smoke", []) + if not isinstance(jobs, list) or not isinstance(smoke, list): + raise TypeError("CI_PLAN jobs and smoke must be lists") + if "build-campfire-archive" in jobs: + result.update(("campfire/spinel.tgz", "campfire/docker.tgz")) + return result + + +def need_result(needs, job): + value = needs.get(job) + return value.get("result") if isinstance(value, dict) else None + + +def archive_status(path, producer, witness, selected, needs, attempt): + if not selected: + return "not-selected", None + if producer is None: + return "unverified", "missing producer inventory" + if producer["producer_outcome"] != "success": + return "failed" if producer[ + "producer_outcome" + ] == "failure" else "unverified", ( + f"producer outcome {producer['producer_outcome']}" + ) + producer_job = GROUPS[path.split("/", 1)[0]] + producer_need = need_result(needs, producer_job) + if needs and producer_need != "success": + return "unverified", f"producer job result {producer_need or 'missing'}" + archive = next((a for a in producer["archives"] if a["path"] == path), None) + if archive is None: + return "unverified", "selected archive missing" + check = next((name for name, target in CHECKS.items() if target == path), None) + if check is None: + return "unverified", "no archive-specific smoke check" + if witness is None: + return "unverified", "selected witness missing" + if witness["attempt"] != attempt: + return ( + "unverified", + "earlier attempt witness; current execution not established", + ) + if witness["sha256"] != archive["sha256"] or witness["size"] != archive["size"]: + return "failed", "producer and witness bytes differ" + if ( + path in {"browse/spinel.tgz", "campfire/spinel.tgz"} + and not witness["spinel_revision"] + ): + return "unverified", "native Spinel revision unknown" + if path == "campfire/docker.tgz" and not archive["spinel_revision"]: + return "unverified", "packaging Spinel revision unknown" + outcome = witness["outcome"] + check_need = need_result(needs, CHECK_JOBS[check]) + if outcome == "failure": + return "failed", "harness outcome failure" + # The ordinary smoke job is a matrix. One failed sibling must not erase + # another target's actual byte-specific successful harness result. + if needs and (check_need is None or check_need in {"skipped", "cancelled"}): + return "unverified", f"consumer job result {check_need or 'missing'}" + if outcome == "success": + return "passed", None + if outcome == "reused": + return "reused", None + return "unverified", f"harness outcome {outcome or 'unknown'}" + + +def report(args): + source, run_id, attempt = identity() + try: + plan = json.loads(os.environ["CI_PLAN"]) + needs = json.loads(os.environ.get("CI_NEEDS", "{}")) + except (KeyError, json.JSONDecodeError) as error: + raise ValueError(f"invalid CI plan/needs: {error}") from error + if not isinstance(plan, dict) or not isinstance(needs, dict): + raise TypeError("CI_PLAN and CI_NEEDS must be JSON objects") + inventories, validations = load_evidence(args.evidence_dir, source, run_id, attempt) + paths = expected_paths(plan) + selected = {f"browse/{target}.tgz" for target in plan.get("smoke", [])} + selected.update(CHECKS[job] for job in plan.get("jobs", []) if job in CHECKS) + for producer in inventories.values(): + paths.update(archive["path"] for archive in producer["archives"]) + archives = [] + for path in sorted(paths): + group = path.split("/", 1)[0] + producer = inventories.get(group) + check = next((name for name, target in CHECKS.items() if target == path), None) + witness = validations.get(check) if check else None + status, reason = archive_status( + path, producer, witness, path in selected, needs, attempt + ) + archive = ( + next((a for a in producer["archives"] if a["path"] == path), None) + if producer + else None + ) + archives.append( + { + "path": path, + "selected": path in selected, + "status": status, + "reason": reason, + "sha256": archive["sha256"] if archive else None, + "size": archive["size"] if archive else None, + "check": check, + "validation_attempt": (witness or {}).get("attempt"), + "harness_outcome": (witness or {}).get("outcome"), + "spinel_revision": ( + (witness or {}).get("spinel_revision") + if path in {"browse/spinel.tgz", "campfire/spinel.tgz"} + else (archive or {}).get("spinel_revision") + ), + } + ) + document = { + "schema": SCHEMA, + "source_sha": source, + "run_id": run_id, + "attempt": attempt, + "producers": [ + { + "group": group, + "attempt": item["attempt"], + "outcome": item["producer_outcome"], + "job_result": need_result(needs, GROUPS[group]), + } + for group, item in sorted(inventories.items()) + ], + "archives": archives, + } + write_json(args.out, document) + + +def verify(args): + source, run_id, attempt = identity() + document = json.loads(Path(args.report).read_text()) + if ( + document.get("schema") != SCHEMA + or document.get("source_sha") != source + or document.get("run_id") != run_id + or document.get("attempt") != attempt + ): + raise ValueError("publication report has foreign identity") + archives = {item["path"]: item for item in document["archives"]} + root = Path(args.root) + published = set() + for group in GROUPS: + directory = root / group + if not directory.is_dir(): + continue + for path in directory.iterdir(): + if not path.is_file() or path.suffix not in {".tgz", ".zip", ".json"}: + continue + logical = path.relative_to(root).as_posix() + item = archives.get(logical) + if ( + item is None + or item.get("sha256") != digest(path) + or item.get("size") != path.stat().st_size + ): + raise ValueError(f"untracked or changed publication bytes: {logical}") + published.add(logical) + # Verification checks identity, not success: failed repro bytes remain useful. + for item in document["archives"]: + item["published"] = item["path"] in published + write_json(args.report, document) + + +def parser(): + result = argparse.ArgumentParser() + commands = result.add_subparsers(dest="command", required=True) + inv = commands.add_parser("inventory") + inv.add_argument("--group", choices=sorted(GROUPS), required=True) + inv.add_argument("--root", required=True) + inv.add_argument("--out", required=True) + inv.add_argument("--outcome", required=True) + inv.add_argument("--spinel-revision-file") + inv.set_defaults(function=inventory) + val = commands.add_parser("validation") + val.add_argument("--archive", required=True) + val.add_argument("--path", choices=sorted(CHECKS.values()), required=True) + val.add_argument("--check", choices=sorted(CHECKS), required=True) + val.add_argument("--outcome", required=True) + val.add_argument("--out", required=True) + val.add_argument("--spinel-revision-file") + val.set_defaults(function=validation) + rep = commands.add_parser("report") + rep.add_argument("--evidence-dir", required=True) + rep.add_argument("--out", required=True) + rep.set_defaults(function=report) + pub = commands.add_parser("verify") + pub.add_argument("--root", required=True) + pub.add_argument("--report", required=True) + pub.set_defaults(function=verify) + return result + + +def main(): + try: + args = parser().parse_args() + args.function(args) + except (OSError, TypeError, ValueError) as error: + print(f"ci-archive-evidence: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci-plan.py b/scripts/ci-plan.py new file mode 100755 index 0000000000..11ada50967 --- /dev/null +++ b/scripts/ci-plan.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +"""Select coverage, not test results. Unknown inputs expand to full validation.""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + +TARGETS = [ + "rust", + "crystal", + "kotlin", + "swift", + "csharp", + "typescript", + "go", + "elixir", + "python", + "ruby", + "jruby", +] +BASE = [ + "generate-fixture", + "unit", + "store-check", + "compare", + "compare-ruby", + "browser-smoke-typescript", + "campfire-conformance", + "campfire-compare", +] +CORE = ["build-spinel", "toolchain-spinel", "compare-spinel"] +SPINEL_TESTS = [ + "framework_tests_spinel", + "spinel_web_push_crypto", + "spinel_db_lease", + "spinel_param_builder", + "rails_compat_vectors_spinel", +] +SPINEL11 = [ + "build-spinel", + "framework-tests-spinel", + "build-campfire-compare-spinel", + "campfire-compare-spinel", + "campfire-db-differential-spinel", + "toolchain-spinel", + "compare-spinel", + "smoke-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", +] +ADVISORY = set(SPINEL11) - {"build-campfire-archive"} +SHA = re.compile(r"[0-9a-f]{40}\Z") + + +def native_coverage(path): + """Identify native core/focused suites and interpreter-only exceptions.""" + interpreter_only = path.startswith( + "runtime/spinel/scaffold/ruby_overlay/" + ) or path in { + "runtime/spinel/db_jruby.rb", + "runtime/spinel/markly_jruby.rb", + "runtime/spinel/db_cruby.rb", + "runtime/spinel/message_digest_cruby.rb", + "runtime/spinel/module_delegate.rb", + } + native = ( + path.startswith(("runtime/ruby/", "runtime/spinel/", "src/emit/ruby/")) + or path == "src/emit/ruby.rs" + or (path.startswith("tests/spinel") and path.endswith((".rs", ".rb"))) + or path in {f"tests/{name}.rs" for name in SPINEL_TESTS} + ) and not interpreter_only + suites = set() + if path.startswith("runtime/ruby/") and path.endswith((".rb", ".rbs")): + suites.add("framework_tests_spinel") + focused = re.fullmatch(r"tests/([^/]+)\.(?:rs|rb)", path) + if focused and focused[1] in SPINEL_TESTS: + suites.add(focused[1]) + if path.startswith(("runtime/spinel/", "runtime/ruby/")) and not interpreter_only: + name = path.rsplit("/", 1)[-1] + owned_tests = set() + if any(word in name for word in ("web_push", "base64")): + owned_tests.add("spinel_web_push_crypto") + if any( + word in name + for word in ( + "signed_cookie", + "message_verifier", + "signed_id", + "message_digest", + "base64", + ) + ) or path.startswith("runtime/spinel/tep/url."): + owned_tests.add("rails_compat_vectors_spinel") + if any( + word in path for word in ("/db", "sqlite", "active_support_time_parsing") + ): + owned_tests.add("spinel_db_lease") + if any(word in name for word in ("param", "multipart", "request")): + owned_tests.add("spinel_param_builder") + if ( + path.startswith("runtime/spinel/") + and not path.startswith("runtime/spinel/scaffold/") + and not owned_tests + ): + owned_tests.add("framework_tests_spinel") + suites.update(owned_tests) + if ( + path.startswith(("tests/rails_compat/", "tests/params_vectors/")) + or path == "tests/rails_compat_vectors.rb" + ): + suites.add( + "spinel_param_builder" + if path.startswith("tests/params_vectors/") + else "rails_compat_vectors_spinel" + ) + return native, interpreter_only, suites + + +def archive_and_campfire_jobs(path, interpreter_only): + """Select packaging and Campfire consumers, not every native runtime edit.""" + jobs = set() + if path.startswith("runtime/spinel/scaffold/") and not interpreter_only: + jobs.update((*CORE, "smoke-spinel", "build-site")) + if path.startswith(("scripts/campfire-compare", "scripts/build-campfire-compare")): + jobs.update( + ("build-spinel", "build-campfire-compare-spinel", "campfire-compare-spinel") + ) + if path.startswith("scripts/campfire-db-differential"): + jobs.update(("build-spinel", "campfire-db-differential-spinel")) + campfire_archive = ( + path.startswith( + ( + "scripts/build-campfire-archive", + "scripts/campfire-archive", + "e2e/campfire/", + ) + ) + or path == "scripts/campfire-docker-files" + ) + shared_smoke = ( + path.startswith("e2e/") and not path.startswith("e2e/campfire/") + ) or path in {"scripts/smoke", "scripts/ci-playwright-install"} + if campfire_archive or shared_smoke: + jobs.update( + ( + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ) + ) + if shared_smoke: + jobs.add("smoke-spinel") + return jobs + + +def select(paths, *, draft=False, full=False, publish=False): + if draft: + return finish( + BASE[:2], + [], + [], + False, + False, + False, + ["draft: fixture and unit only"], + spinel_tests=[], + ) + targets, smoke = set(), set() + jobs_selected, spinel_tests = set(), set() + wasm = site = spinel = writebook = False + reasons = [] + for path in paths: + if path.startswith((".github/", ".cargo/")) or path in { + "scripts/ci-plan.py", + "scripts/ci-reuse.py", + "scripts/ci-archive-evidence.py", + "tests/ci_plan_test.py", + "tests/ci_archive_evidence_test.py", + "tests/workflow_yaml_parses.rs", + "tests/ci_policy_workflow.rs", + "src/project.rs", + "src/bin/roundhouse.rs", + "Cargo.toml", + "Cargo.lock", + "build.rs", + "rust-toolchain.toml", + ".cargo/config.toml", + }: + full = True + reasons.append(f"{path}: validation/packaging policy") + match = re.match(r"(?:src/emit/|runtime/)([^/.]+)(?:[/.]|$)", path) + test = re.match( + r"tests/(?:framework_tests_)?([a-z]+)_toolchain\.rs$|tests/framework_tests_([a-z]+)\.rs$", + path, + ) + target = ( + match[1] + if match + else next((v for v in test.groups() if v), None) + if test + else None + ) + native, interpreter_only, owned_tests = native_coverage(path) + spinel_tests.update(owned_tests) + if native or owned_tests: + spinel = True + jobs_selected.update(CORE) + if native: + reasons.append(f"{path}: native Spinel core") + if target in TARGETS or target == "spinel": + owners = ( + {"ruby", "jruby", "spinel"} + if target in {"ruby", "spinel"} and not path.startswith("runtime/ruby/") + else {target} + ) + if ( + path.startswith(("runtime/ruby/", "runtime/spinel/")) + or (path.startswith("tests/spinel") and path.endswith(".rs")) + or path in {f"tests/{name}.rs" for name in SPINEL_TESTS} + ): + owners = set() # Native framework coverage; no interpreted archives. + if interpreter_only: + owners = ( + {"jruby"} + if path.endswith(("db_jruby.rb", "markly_jruby.rb")) + else {"ruby", "jruby"} + ) + targets.update(owners - {"spinel"}) + smoke.update(owners - {"spinel"}) + spinel |= "spinel" in owners + if owners: + reasons.append(f"{path}: {', '.join(sorted(owners))}") + elif target == "shared": + full = True + reasons.append(f"{path}: shared code generation") + elif target and target not in { + "ruby", + "ruby_family", + "roda", + "mod", + "rails", + }: + full = True + reasons.append(f"{path}: unknown target ownership") + if path.startswith("wasm/"): + wasm = True + reasons.append(f"{path}: WASM/browser compiler") + if path.startswith(("site/", "docs/guide/")): + site = wasm = True + if ( + path.startswith("e2e/") and not path.startswith("e2e/campfire/") + ) or path in { + "scripts/smoke", + "scripts/ci-playwright-install", + "scripts/create-blog", + "scripts/create-store", + "bin/rh", + }: + smoke.update(TARGETS) + spinel = True + if ( + path.startswith(("tools/compare/", "tests/framework_test_support")) + or path == "scripts/compare" + ): + targets.update(TARGETS) + spinel = True + jobs_selected.update(CORE) + if path.startswith("tests/framework_test_support"): + spinel_tests.add("framework_tests_spinel") + archive_jobs = archive_and_campfire_jobs(path, interpreter_only) + if archive_jobs: + spinel = True + jobs_selected.update(archive_jobs) + if path in {"tests/writebook.rs", "tests/fixtures/writebook-inventory.json"}: + writebook = True + if full: + targets.update(TARGETS) + smoke.update(TARGETS) + wasm = site = spinel = writebook = True + reasons.append("full validation requested") + jobs_selected.update(SPINEL11) + spinel_tests.update(SPINEL_TESTS) + if spinel: + jobs_selected.add("build-spinel") + jobs = list(BASE) + extra = [ + t + for t in TARGETS + if t in targets and t not in {"rust", "typescript", "ruby", "jruby"} + ] + if extra: + jobs.append("compare-extra") + if "jruby" in targets: + jobs.append("compare-jruby") + if wasm: + jobs.extend(["build-wasm", "browser-smoke-ide"]) + if smoke or site: + jobs.append("build-site") + if smoke - {"spinel"}: + jobs.append("smoke") + if spinel_tests: + jobs_selected.add("framework-tests-spinel") + if "smoke-spinel" in jobs_selected: + jobs_selected.add("build-site") + if "build-site" in jobs_selected and "build-site" not in jobs: + jobs.append("build-site") + if "build-site" in jobs or {"build-site", "build-campfire-archive"} & jobs_selected: + jobs_selected.add("archive-results") + jobs.extend(j for j in [*SPINEL11, "archive-results"] if j in jobs_selected) + if writebook: + jobs.append("writebook-inventory") + if publish: + if not full: + raise ValueError("publication requires full validation mode") + jobs.append("assemble-site") + return finish( + jobs, + extra, + [t for t in TARGETS if t in smoke], + wasm, + site, + spinel, + reasons, + publish, + [t for t in SPINEL_TESTS if t in spinel_tests], + ) + + +def finish( + jobs, extra, smoke, wasm, site, spinel, reasons, publish=False, spinel_tests=None +): + archives = ( + ["blog", "spinel", *TARGETS, "typescript-worker"] + if site + else [*smoke, *(["spinel"] if "smoke-spinel" in jobs else [])] + ) + return { + "jobs": jobs, + "required": [j for j in jobs if j not in ADVISORY], + "extra_compare": extra, + "smoke": smoke, + "archives": archives, + "wasm": wasm, + "site": site, + "spinel": spinel, + "spinel_tests": spinel_tests or [], + "publish": publish, + "reasons": reasons, + } + + +def git(*args): + return subprocess.check_output(["git", *args]) + + +def changed_paths(event, event_name, sha): + if not SHA.fullmatch(sha) or git("rev-parse", "HEAD").decode().strip() != sha: + raise ValueError("checkout is not the event SHA") + if event_name == "pull_request": + pr = event["pull_request"] + parents = git("show", "-s", "--format=%P", "HEAD").decode().split() + if parents != [pr["base"]["sha"], pr["head"]["sha"]]: + raise ValueError("checkout is not the event's PR merge tree") + base = parents[0] + elif event_name == "push": + base = event["before"] + if not SHA.fullmatch(base) or base == "0" * 40: + raise ValueError("no previous main tree") + try: + git("cat-file", "-e", base) + except subprocess.CalledProcessError: + git("fetch", "--no-tags", "--depth=1", "origin", base) + else: + return [] + # Renames become a deletion and addition; both ownership sets are selected. + return [ + p.decode("utf-8") + for p in git("diff", "--name-only", "--no-renames", "-z", base, sha).split( + b"\0" + ) + if p + ] + + +def check_results(plan, needs, *, compact=False): + required = ( + BASE[:2] if plan["jobs"] == BASE[:2] else BASE if compact else plan["required"] + ) + failures = [ + f"{j}: {needs.get(j, {}).get('result', 'missing')}" + for j in required + if needs.get(j, {}).get("result") != "success" + ] + if needs.get("plan", {}).get("result") != "success": + failures.append("plan: no successful routing decision") + if not compact and needs.get("compact-required", {}).get("result") != "success": + failures.append("compact-required: no successful baseline gate") + # Advisory work never blocks the gate, but incomplete work is not complete. + complete = not failures and all( + needs.get(j, {}).get("result") == "success" + and ( + j not in ADVISORY + or all( + needs[j].get("outputs", {}).get(key) == "success" + for key in ( + ["default", "minor-gc", "verify-gen"] + if j == "campfire-compare-spinel" + else ["execution"] + ) + ) + ) + for j in plan["jobs"] + ) + return failures, complete + + +def write_outputs(values): + output = os.environ.get("GITHUB_OUTPUT") + lines = "".join( + f"{key}={json.dumps(value, separators=(',', ':')) if not isinstance(value, str) else value}\n" + for key, value in values.items() + ) + if output: + with open(output, "a") as f: + f.write(lines) + print(lines, end="") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["plan", "gate", "compact-gate"]) + args = parser.parse_args() + if args.command != "plan": + plan = json.loads(os.environ["CI_PLAN"]) + failures, complete = check_results( + plan, + json.loads(os.environ["CI_NEEDS"]), + compact=args.command == "compact-gate", + ) + write_outputs({"complete": complete}) + for failure in failures: + print(f"::error::{failure}") + return bool(failures) + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + event_name = os.environ["GITHUB_EVENT_NAME"] + pr = event.get("pull_request", {}) + full = os.environ.get("CI_FULL") == "true" or any( + label["name"] == "ci:full" for label in pr.get("labels", []) + ) + reason = None + try: + paths = changed_paths(event, event_name, os.environ["GITHUB_SHA"]) + except (KeyError, ValueError, UnicodeError, subprocess.CalledProcessError) as e: + paths, full, reason = ( + [], + True, + f"Unknown changed inputs: {e}; running full validation", + ) + publish = os.environ.get("CI_PUBLISH") == "true" + if publish and ( + os.environ["GITHUB_REPOSITORY"] != "rubys/roundhouse" + or os.environ["GITHUB_REF"] != "refs/heads/main" + or event_name not in {"schedule", "workflow_dispatch"} + ): + raise ValueError("publication is only allowed by canonical main's full caller") + plan = select(paths, draft=pr.get("draft", False), full=full, publish=publish) + if reason: + plan["reasons"].append(reason) + spinel = os.environ.get("CI_SPINEL_REVISION", "") + if plan["spinel"] and not spinel: + try: + spinel = subprocess.check_output( + ["gh", "api", "repos/matz/spinel/commits/master", "--jq", ".sha"], + text=True, + ).strip() + except subprocess.CalledProcessError: + spinel = "master" + plan["reasons"].append( + "Spinel lookup unavailable: fresh master build; actual revision recorded by producer" + ) + if spinel and spinel != "master" and not SHA.fullmatch(spinel): + raise ValueError("invalid Spinel revision") + write_outputs( + { + "plan": plan, + "jobs": plan["jobs"], + "extra-compare": plan["extra_compare"], + "spinel-tests": plan["spinel_tests"], + "smoke": plan["smoke"], + "archives": ",".join(plan["archives"]), + "wasm": plan["wasm"], + "site": plan["site"], + "publish": plan["publish"], + "spinel-revision": spinel, + } + ) + if summary := os.environ.get("GITHUB_STEP_SUMMARY"): + Path(summary).write_text( + "## Selected CI coverage\n\n```json\n" + + json.dumps(plan, indent=2) + + "\n```\n" + ) + return False + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bin/roundhouse.rs b/src/bin/roundhouse.rs index 63639e066a..a94cbc95c4 100644 --- a/src/bin/roundhouse.rs +++ b/src/bin/roundhouse.rs @@ -37,6 +37,7 @@ fn usage() -> &'static str { "\ Usage: roundhouse --target LANG [INPUT] [-o OUT] roundhouse --site [INPUT] [-o OUT] + roundhouse --archives LANG,LANG [INPUT] [-o OUT] roundhouse check [--continue] [APP] roundhouse lsp roundhouse mcp [APP] @@ -60,6 +61,8 @@ Options: Default INPUT=. Default OUT=./out// --site Build all targets + landing-page assets. Default INPUT=fixtures/real-blog Default OUT=./_site/ + --archives LIST Build only comma-separated targets' browse archives. + Same defaults as --site; no website or browser demos. -o, --output PATH Output directory. --allow-unsupported Don't fail on unsupported-construct gaps: emit a @@ -127,6 +130,13 @@ fn cli() -> ExitCode { ExitCode::FAILURE } }, + Ok(Action::Archives { targets, input, out }) => match project::build_archives(&input, &out, &targets) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("roundhouse: {e}"); + ExitCode::FAILURE + } + }, Err(e) => { eprintln!("roundhouse: {e}"); eprintln!(); @@ -150,11 +160,17 @@ enum Action { input: PathBuf, out: PathBuf, }, + Archives { + targets: Vec, + input: PathBuf, + out: PathBuf, + }, } fn parse_args(args: Vec) -> Result { let mut target: Option = None; let mut site = false; + let mut archives = None; let mut out: Option = None; let mut allow_unsupported = false; let mut survey = false; @@ -166,6 +182,15 @@ fn parse_args(args: Vec) -> Result { "-h" | "--help" => return Ok(Action::Help), "-V" | "--version" => return Ok(Action::Version), "--site" => site = true, + "--archives" => { + let value = iter.next().ok_or("--archives requires a target list")?; + let targets = value.split(',').map(|name| { + BuildTarget::from_str(name) + .filter(|t| BuildTarget::ALL.contains(t)) + .ok_or_else(|| format!("unknown archive target '{name}'")) + }).collect::, _>>()?; + archives = Some(targets); + } "--allow-unsupported" => allow_unsupported = true, "--survey" => survey = true, "-t" | "--target" => { @@ -201,6 +226,16 @@ fn parse_args(args: Vec) -> Result { )); } + if let Some(targets) = archives { + if target.is_some() || site || allow_unsupported || survey { + return Err("--archives cannot be combined with --target, --site, --survey or --allow-unsupported".into()); + } + return Ok(Action::Archives { + targets, + input: positional.pop().map(PathBuf::from).unwrap_or_else(|| PathBuf::from("fixtures/real-blog")), + out: out.unwrap_or_else(|| PathBuf::from("_site")), + }); + } match (target, site) { (Some(_), true) => Err("--target and --site are mutually exclusive".into()), (None, false) => Err("one of --target LANG or --site is required".into()), diff --git a/src/project.rs b/src/project.rs index 5fd9da40f3..325658d9f3 100644 --- a/src/project.rs +++ b/src/project.rs @@ -6623,10 +6623,23 @@ pub fn build_site(fixture: &Path, out: &Path) -> Result<(), String> { fs::create_dir_all(out.join("browse")) .map_err(|e| format!("mkdir {}: {e}", out.display()))?; + // Archive generation cleans browse/, so copy its viewer assets afterwards. + build_archives(fixture, out, BuildTarget::ALL)?; copy_site_assets(out)?; copy_create_blog(out)?; crate::guide::render_site(out)?; + Ok(()) +} + +/// Build selected browse archives without website assets or WASM demos. +/// The file sets and archive writers are shared with `build_site`. +pub fn build_archives(fixture: &Path, out: &Path, targets: &[BuildTarget]) -> Result<(), String> { + let browse = out.join("browse"); + if browse.exists() { + fs::remove_dir_all(&browse).map_err(|e| format!("clean {}: {e}", browse.display()))?; + } + fs::create_dir_all(&browse).map_err(|e| format!("mkdir {}: {e}", browse.display()))?; let mut app = ingest_app(fixture).map_err(|e| format!("ingest {}: {e}", fixture.display()))?; // Analyze + the same post-analyze shared lowerings as the @@ -6634,7 +6647,7 @@ pub fn build_site(fixture: &Path, out: &Path) -> Result<(), String> { // so the residue is dropped. let _ = crate::session::analyze_and_lower(&mut app); - for target in BuildTarget::ALL { + for target in targets { let files = target_files(&app, fixture, *target)?; let name = target.as_str(); diff --git a/tests/ci_archive_evidence_test.py b/tests/ci_archive_evidence_test.py new file mode 100644 index 0000000000..64fb39314f --- /dev/null +++ b/tests/ci_archive_evidence_test.py @@ -0,0 +1,399 @@ +"""Tests for bounded CI archive evidence.""" + +import hashlib +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci-archive-evidence.py" +SPEC = importlib.util.spec_from_file_location("ci_archive_evidence", SCRIPT) +evidence = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(evidence) +SHA = "1" * 40 +SPINEL = "2" * 40 + + +class ArchiveEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.env = { + "GITHUB_SHA": SHA, + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "1", + } + + def tearDown(self): + self.temp.cleanup() + + def run_cli(self, *args, ok=True, extra_env=None): + env = os.environ | self.env | (extra_env or {}) + result = subprocess.run( + ["python3", str(SCRIPT), *map(str, args)], + text=True, + capture_output=True, + env=env, + check=False, + ) + if ok and result.returncode: + self.fail(result.stderr) + return result + + def make_inventory(self, group, files, outcome="success", attempt=1, revision=None): + site = self.root / f"site-{group}-{attempt}" + for name, contents in files.items(): + path = site / group / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + rev = self.root / "revision.txt" + if revision: + rev.write_text(revision) + out = self.root / "evidence" / f"inventory-{group}-{attempt}.json" + with patch.dict(self.env, {"GITHUB_RUN_ATTEMPT": str(attempt)}): + self.run_cli( + "inventory", + "--group", + group, + "--root", + site, + "--out", + out, + "--outcome", + outcome, + *(["--spinel-revision-file", rev] if revision else []), + extra_env={"GITHUB_RUN_ATTEMPT": str(attempt)}, + ) + return out + + def make_validation( + self, check, data=None, outcome="success", attempt=1, revision=None + ): + archive = self.root / f"input-{check}-{attempt}.tgz" + if data is not None: + archive.write_bytes(data) + rev = self.root / "consumer-revision.txt" + if revision: + rev.write_text(revision) + out = self.root / "evidence" / f"validation-{check}-{attempt}.json" + self.run_cli( + "validation", + "--archive", + archive, + "--path", + evidence.CHECKS[check], + "--check", + check, + "--outcome", + outcome, + "--out", + out, + *(["--spinel-revision-file", rev] if revision else []), + extra_env={"GITHUB_RUN_ATTEMPT": str(attempt)}, + ) + return out + + def make_report(self, plan, needs=None, ok=True): + out = self.root / "report.json" + result = self.run_cli( + "report", + "--evidence-dir", + self.root / "evidence", + "--out", + out, + ok=ok, + extra_env={ + "CI_PLAN": json.dumps(plan), + "CI_NEEDS": json.dumps(needs or {}), + }, + ) + return (json.loads(out.read_text()) if result.returncode == 0 else None), result + + def status(self, report, path): + return next(item for item in report["archives"] if item["path"] == path) + + def test_inventory_hashes_bytes_independently_and_does_not_mutate(self): + original = b"archive bytes" + out = self.make_inventory("browse", {"rust.tgz": original, "map.json": b"{}"}) + item = json.loads(out.read_text()) + rust = next(a for a in item["archives"] if a["path"] == "browse/rust.tgz") + self.assertEqual(rust["sha256"], hashlib.sha256(original).hexdigest()) + self.assertEqual( + (self.root / "site-browse-1/browse/rust.tgz").read_bytes(), original + ) + self.assertIsNone(rust["spinel_revision"]) + + def test_same_name_changed_bytes_cannot_pass(self): + self.make_inventory("browse", {"rust.tgz": b"producer"}) + self.make_validation("smoke-rust", b"changed") + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_missing_file_validation_has_null_hash_and_never_passes(self): + witness = self.make_validation("smoke-rust", None, outcome="success") + self.assertIsNone(json.loads(witness.read_text())["sha256"]) + self.make_inventory("browse", {"rust.tgz": b"bytes"}) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_success_reuse_and_non_success_outcomes(self): + for outcome, expected in ( + ("success", "passed"), + ("reused", "reused"), + ("failure", "failed"), + ("skipped", "unverified"), + ("cancelled", "unverified"), + ("unknown", "unverified"), + ): + with self.subTest(outcome=outcome): + (self.root / "evidence").mkdir(exist_ok=True) + for path in (self.root / "evidence").iterdir(): + path.unlink() + self.make_inventory("browse", {"rust.tgz": b"same"}) + self.make_validation("smoke-rust", b"same", outcome=outcome) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual( + self.status(report, "browse/rust.tgz")["status"], expected + ) + + def test_missing_or_unavailable_selected_witness_is_unverified(self): + self.make_inventory("browse", {"rust.tgz": b"same"}) + plan = {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + report, _ = self.make_report(plan) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "unverified") + self.make_validation("smoke-rust", b"same") + report, _ = self.make_report(plan, {"smoke": {"result": "cancelled"}}) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "unverified") + + def test_native_spinel_requires_consumer_revision_but_source_archive_does_not_package_one( + self, + ): + self.make_inventory("browse", {"spinel.tgz": b"native"}, revision=SPINEL) + self.make_validation("smoke-spinel", b"native") + plan = {"archives": ["spinel"], "jobs": ["smoke-spinel"], "smoke": []} + report, _ = self.make_report(plan) + item = self.status(report, "browse/spinel.tgz") + self.assertEqual(item["status"], "unverified") + inventory = json.loads( + (self.root / "evidence/inventory-browse-1.json").read_text() + ) + self.assertIsNone(inventory["archives"][0]["spinel_revision"]) + + def test_native_and_docker_spinel_revision_rules_are_distinct(self): + data = b"campfire" + self.make_inventory( + "campfire", {"spinel.tgz": data, "docker.tgz": data}, revision=SPINEL + ) + self.make_validation("smoke-campfire", data, revision=SPINEL) + self.make_validation("smoke-campfire-docker", data) + plan = { + "archives": [], + "jobs": [ + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + ], + "smoke": [], + } + report, _ = self.make_report(plan) + self.assertEqual(self.status(report, "campfire/spinel.tgz")["status"], "passed") + docker = self.status(report, "campfire/docker.tgz") + self.assertEqual(docker["status"], "passed") + self.assertEqual(docker["spinel_revision"], SPINEL) + + def test_docker_unknown_packaging_revision_is_unverified(self): + self.make_inventory("campfire", {"docker.tgz": b"docker"}) + self.make_validation("smoke-campfire-docker", b"docker") + plan = { + "archives": [], + "jobs": ["build-campfire-archive", "smoke-campfire-docker"], + "smoke": [], + } + report, _ = self.make_report(plan) + self.assertEqual( + self.status(report, "campfire/docker.tgz")["status"], "unverified" + ) + + def test_non_tgz_inventory_is_not_certified_by_tgz_smoke(self): + self.make_inventory( + "browse", {"rust.tgz": b"x", "rust.zip": b"x", "rust.json": b"x"} + ) + self.make_validation("smoke-rust", b"x") + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "passed") + self.assertEqual( + self.status(report, "browse/rust.zip")["status"], "not-selected" + ) + self.assertEqual( + self.status(report, "browse/rust.json")["status"], "not-selected" + ) + + def test_latest_attempt_is_selected_deterministically(self): + self.make_inventory("browse", {"rust.tgz": b"old"}, attempt=1) + self.make_validation("smoke-rust", b"old", attempt=1) + self.make_inventory("browse", {"rust.tgz": b"new"}, attempt=2) + self.make_validation("smoke-rust", b"new", attempt=2) + self.env["GITHUB_RUN_ATTEMPT"] = "2" + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual( + self.status(report, "browse/rust.tgz")["sha256"], + hashlib.sha256(b"new").hexdigest(), + ) + self.assertEqual(report["producers"][0]["attempt"], 2) + + def test_wrong_source_or_run_and_conflicts_are_errors(self): + path = self.make_inventory("browse", {"rust.tgz": b"x"}) + item = json.loads(path.read_text()) + for field, value in (("source_sha", "3" * 40), ("run_id", "999")): + with self.subTest(field=field): + changed = dict(item, **{field: value}) + path.write_text(json.dumps(changed)) + _, result = self.make_report( + {"archives": [], "jobs": [], "smoke": []}, ok=False + ) + self.assertEqual(result.returncode, 2) + path.write_text(json.dumps(item)) + duplicate = self.root / "evidence/duplicate.json" + duplicate.write_text(json.dumps(dict(item, producer_outcome="failure"))) + _, result = self.make_report( + {"archives": [], "jobs": [], "smoke": []}, ok=False + ) + self.assertEqual(result.returncode, 2) + + def test_missing_selected_archive_is_reported(self): + self.make_inventory("browse", {}) + report, _ = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + item = self.status(report, "browse/rust.tgz") + self.assertEqual(item["status"], "unverified") + self.assertIsNone(item["sha256"]) + + def test_producer_failure_after_partial_output_cannot_validate(self): + self.make_inventory("browse", {"rust.tgz": b"same"}, outcome="failure") + self.make_validation("smoke-rust", b"same") + report, result = self.make_report( + {"archives": ["rust"], "jobs": [], "smoke": ["rust"]} + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + + def test_unselected_existing_witness_and_failed_matrix_sibling(self): + self.make_inventory("browse", {"go.tgz": b"go", "rust.tgz": b"rust"}) + self.make_validation("smoke-go", b"go") + self.make_validation("smoke-rust", b"rust") + report, _ = self.make_report( + {"archives": ["go", "rust"], "jobs": ["smoke"], "smoke": ["go"]}, + {"build-site": {"result": "success"}, "smoke": {"result": "failure"}}, + ) + self.assertEqual(self.status(report, "browse/go.tgz")["status"], "passed") + self.assertEqual( + self.status(report, "browse/rust.tgz")["status"], "not-selected" + ) + + def test_future_evidence_attempt_is_rejected(self): + self.make_inventory("browse", {"rust.tgz": b"future"}, attempt=2) + _, result = self.make_report( + {"archives": ["rust"], "smoke": ["rust"]}, ok=False + ) + self.assertEqual(result.returncode, 2) + + def test_old_success_does_not_certify_failed_or_masked_rerun(self): + self.make_inventory("browse", {"spinel.tgz": b"native"}) + self.make_validation("smoke-spinel", b"native", revision=SPINEL) + self.env["GITHUB_RUN_ATTEMPT"] = "2" + for result in ["failure", "success"]: + with self.subTest(result=result): + report, _ = self.make_report( + {"archives": ["spinel"], "jobs": ["smoke-spinel"]}, + { + "build-site": {"result": "success"}, + "smoke-spinel": { + "result": result, + "outputs": {"execution": "failure"}, + }, + }, + ) + item = self.status(report, "browse/spinel.tgz") + self.assertEqual(item["status"], "unverified") + self.assertEqual(item["validation_attempt"], 1) + + def test_publication_checks_bytes_not_validation_success(self): + self.make_inventory( + "browse", {"rust.tgz": b"repro", "rust.zip": b"zip"}, outcome="failure" + ) + report, _ = self.make_report({"archives": ["rust"], "smoke": ["rust"]}) + self.assertEqual(self.status(report, "browse/rust.tgz")["status"], "failed") + site = self.root / "site-browse-1" + args = ("verify", "--root", site, "--report", self.root / "report.json") + self.run_cli(*args) + self.assertEqual((site / "browse/rust.tgz").read_bytes(), b"repro") + document = json.loads((self.root / "report.json").read_text()) + self.assertTrue(self.status(document, "browse/rust.tgz")["published"]) + (site / "browse/rust.tgz").unlink() + self.run_cli(*args) + document = json.loads((self.root / "report.json").read_text()) + self.assertFalse(self.status(document, "browse/rust.tgz")["published"]) + self.assertEqual(self.status(document, "browse/rust.tgz")["status"], "failed") + (site / "browse/rust.zip").write_bytes(b"changed") + self.assertEqual(self.run_cli(*args, ok=False).returncode, 2) + (site / "browse/rust.zip").write_bytes(b"zip") + (site / "browse/unknown.tgz").write_bytes(b"new") + self.assertEqual(self.run_cli(*args, ok=False).returncode, 2) + (site / "browse/unknown.tgz").unlink() + self.assertEqual( + self.run_cli( + *args, ok=False, extra_env={"GITHUB_SHA": "3" * 40} + ).returncode, + 2, + ) + + def test_cli_rejects_mismatched_path_check_and_missing_identity(self): + result = self.run_cli( + "validation", + "--archive", + self.root / "none", + "--path", + "browse/spinel.tgz", + "--check", + "smoke-rust", + "--outcome", + "success", + "--out", + self.root / "out.json", + ok=False, + ) + self.assertEqual(result.returncode, 2) + result = self.run_cli( + "inventory", + "--group", + "browse", + "--root", + self.root, + "--out", + self.root / "out.json", + "--outcome", + "success", + ok=False, + extra_env={"GITHUB_SHA": "short"}, + ) + self.assertEqual(result.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ci_plan_test.py b/tests/ci_plan_test.py new file mode 100644 index 0000000000..88a9c44a0c --- /dev/null +++ b/tests/ci_plan_test.py @@ -0,0 +1,392 @@ +"""Routing boundaries and false-green checks, without GitHub or toolchains.""" + +import importlib.util +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location( + "ci_plan", Path(__file__).parents[1] / "scripts/ci-plan.py" +) +ci = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ci) + + +class Routing(unittest.TestCase): + def extras(self, plan): + return set(plan["jobs"]) - set(ci.BASE) + + def test_general_analysis_and_lowering_retain_base(self): + plan = ci.select(["src/analyze/call.rs", "src/lower/rails.rs"]) + self.assertEqual(plan["jobs"], ci.BASE) + self.assertEqual(plan["archives"], []) + + def test_shared_emitters_select_cross_target_full_coverage(self): + for path in [ + "src/emit/shared/schema_sql.rs", + "src/emit/shared/ops.rs", + "src/emit/shared/mod.rs", + ]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual( + plan["extra_compare"], + ["crystal", "kotlin", "swift", "csharp", "go", "elixir", "python"], + ) + self.assertEqual(plan["smoke"], ci.TARGETS) + self.assertTrue(plan["site"]) + self.assertTrue(plan["wasm"]) + self.assertTrue(set(ci.SPINEL11).issubset(plan["jobs"])) + self.assertIn("writebook-inventory", plan["required"]) + self.assertIn("archive-results", plan["required"]) + + def test_native_only_test_selects_core_without_archives_or_campfire(self): + plan = ci.select(["tests/spinel_toolchain.rs"]) + self.assertEqual(self.extras(plan), set(ci.CORE)) + self.assertEqual(plan["spinel_tests"], []) + self.assertEqual(plan["archives"], []) + self.assertFalse( + any("campfire" in job for job in plan["jobs"] if job not in ci.BASE) + ) + + def test_shared_runtime_rb_and_rbs_select_framework_native_coverage(self): + for path in ["runtime/ruby/active_record.rb", "runtime/ruby/test/model.rbs"]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual( + self.extras(plan), set(ci.CORE) | {"framework-tests-spinel"} + ) + self.assertEqual(plan["spinel_tests"], ["framework_tests_spinel"]) + self.assertEqual(plan["archives"], []) + + def test_draft_overrides_full_and_target_expansion(self): + plan = ci.select( + ["src/emit/go/expressions.rs", ".github/workflows/ci.yml"], + draft=True, + full=True, + ) + self.assertEqual(plan["required"], ["generate-fixture", "unit"]) + + def test_target_partial_does_not_pull_in_wasm_or_other_archives(self): + plan = ci.select(["src/emit/go/expressions.rs"]) + self.assertEqual(plan["extra_compare"], ["go"]) + self.assertEqual(plan["archives"], ["go"]) + self.assertEqual(plan["smoke"], ["go"]) + self.assertFalse(plan["site"]) + self.assertFalse(plan["wasm"]) + self.assertNotIn("build-spinel", plan["jobs"]) + + def test_baseline_emitter_adds_its_archive_not_duplicate_compare(self): + plan = ci.select(["src/emit/rust.rs"]) + self.assertEqual(plan["extra_compare"], []) + self.assertEqual(plan["archives"], ["rust"]) + self.assertNotIn("compare-extra", plan["jobs"]) + + def test_framework_and_toolchain_tests_belong_to_compare(self): + for path in ["tests/swift_toolchain.rs", "tests/framework_tests_swift.rs"]: + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual(plan["extra_compare"], ["swift"]) + self.assertEqual(plan["smoke"], ["swift"]) + + def test_ruby_emit_preserves_interpreted_family_and_adds_native_core(self): + plan = ci.select(["src/emit/ruby.rs"]) + self.assertIn("compare-jruby", plan["jobs"]) + self.assertEqual(plan["smoke"], ["ruby", "jruby"]) + self.assertTrue(set(ci.CORE).issubset(plan["jobs"])) + + def test_focused_tests_select_only_themselves(self): + for binary in ci.SPINEL_TESTS: + with self.subTest(binary=binary): + plan = ci.select([f"tests/{binary}.rs"]) + self.assertEqual(plan["spinel_tests"], [binary]) + self.assertEqual( + self.extras(plan), set(ci.CORE) | {"framework-tests-spinel"} + ) + + def test_runtime_owners_choose_asymmetric_focused_binaries(self): + cases = { + "runtime/spinel/web_push_crypto.rb": "spinel_web_push_crypto", + "runtime/spinel/signed_cookies.rbs": "rails_compat_vectors_spinel", + "runtime/spinel/sqlite_adapter.rb": "spinel_db_lease", + "runtime/spinel/active_record_equality_spinel.rb": "framework_tests_spinel", + "runtime/spinel/param_builder.rb": "spinel_param_builder", + "runtime/spinel/multipart.rb": "spinel_param_builder", + } + for path, binary in cases.items(): + with self.subTest(path=path): + self.assertEqual(ci.select([path])["spinel_tests"], [binary]) + + def test_interpreter_only_files_do_not_start_native_work(self): + for path, targets in { + "runtime/spinel/db_jruby.rb": ["jruby"], + "runtime/spinel/markly_jruby.rb": ["jruby"], + "runtime/spinel/scaffold/ruby_overlay/main.rb": ["ruby", "jruby"], + "runtime/spinel/module_delegate.rb": ["ruby", "jruby"], + }.items(): + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual(plan["smoke"], targets) + self.assertIn("compare-jruby", plan["jobs"]) + self.assertNotIn("build-spinel", plan["jobs"]) + self.assertEqual(plan["spinel_tests"], []) + self.assertEqual(ci.select(["README.md"])["jobs"], ci.BASE) + + def test_shared_runtime_and_driver_inputs_select_real_harnesses(self): + cases = { + "runtime/ruby/action_controller/message_verifier.rbs": [ + "framework_tests_spinel", + "rails_compat_vectors_spinel", + ], + "runtime/ruby/params.rb": [ + "framework_tests_spinel", + "spinel_param_builder", + ], + "runtime/spinel/base64.rb": [ + "spinel_web_push_crypto", + "rails_compat_vectors_spinel", + ], + "tests/spinel_db_lease.rb": ["spinel_db_lease"], + "tests/params_vectors/canon.rb": ["spinel_param_builder"], + "tests/rails_compat_vectors.rb": ["rails_compat_vectors_spinel"], + } + for path, expected in cases.items(): + with self.subTest(path=path): + plan = ci.select([path]) + self.assertEqual(plan["spinel_tests"], expected) + self.assertTrue(set(ci.CORE).issubset(plan["jobs"])) + self.assertNotIn("smoke-campfire", plan["jobs"]) + + def test_routing_union_is_order_independent(self): + paths = [ + "runtime/spinel/web_push_crypto.rb", + "runtime/spinel/fragment_cache.rb", + ] + self.assertEqual( + ci.select(paths)["spinel_tests"], ci.select(paths[::-1])["spinel_tests"] + ) + self.assertEqual( + ci.select(paths)["spinel_tests"], + ["framework_tests_spinel", "spinel_web_push_crypto"], + ) + + def test_union_and_deletion_paths_keep_each_owner(self): + plan = ci.select( + ["runtime/spinel/web_push_crypto.rb", "runtime/spinel/sqlite_adapter.rb"] + ) + self.assertEqual( + plan["spinel_tests"], ["spinel_web_push_crypto", "spinel_db_lease"] + ) + + def test_wasm_changes_have_no_archive_or_spinel_fanout(self): + plan = ci.select(["wasm/lib/driver.mjs"]) + self.assertTrue(plan["wasm"]) + self.assertIn("browser-smoke-ide", plan["required"]) + self.assertNotIn("build-site", plan["jobs"]) + + def test_packaging_and_unknown_target_changes_expand_to_full(self): + for path in [ + "src/project.rs", + "src/emit/newlang.rs", + "tests/framework_tests_newlang.rs", + "scripts/ci-plan.py", + "Cargo.toml", + "scripts/ci-reuse.py", + "tests/ci_archive_evidence_test.py", + "tests/ci_policy_workflow.rs", + ]: + with self.subTest(path=path): + self.assertEqual(ci.select([path])["smoke"], ci.TARGETS) + + def test_full_manual_and_publication_are_distinct(self): + plan = ci.select([], full=True) + self.assertEqual(plan["spinel_tests"], ci.SPINEL_TESTS) + self.assertTrue(set(ci.SPINEL11).issubset(plan["jobs"])) + self.assertIn("archive-results", plan["required"]) + self.assertIn("writebook-inventory", plan["required"]) + self.assertNotIn("deploy", plan["jobs"]) + self.assertIn( + "assemble-site", ci.select([], full=True, publish=True)["required"] + ) + with self.assertRaises(ValueError): + ci.select([], publish=True) + + def test_shared_smoke_and_compare_harnesses_select_their_owners(self): + plan = ci.select(["scripts/smoke"]) + self.assertEqual(plan["smoke"], ci.TARGETS) + self.assertTrue(plan["spinel"]) + self.assertEqual(plan["extra_compare"], []) + self.assertEqual( + ci.select(["tools/compare/src/main.rs"])["extra_compare"], + ["crystal", "kotlin", "swift", "csharp", "go", "elixir", "python"], + ) + self.assertTrue( + set(ci.CORE).issubset(ci.select(["tools/compare/src/main.rs"])["jobs"]) + ) + + def test_spinel_archive_and_campfire_paths_have_exact_heavy_owners(self): + scaffold = ci.select(["runtime/spinel/scaffold/Makefile"]) + self.assertEqual( + self.extras(scaffold), + set(ci.CORE) | {"build-site", "smoke-spinel", "archive-results"}, + ) + self.assertEqual(scaffold["archives"], ["spinel"]) + compare = ci.select(["scripts/campfire-compare-diff.rb"]) + self.assertEqual( + self.extras(compare), + { + "build-spinel", + "build-campfire-compare-spinel", + "campfire-compare-spinel", + }, + ) + db = ci.select(["scripts/campfire-db-differential"]) + self.assertEqual( + self.extras(db), {"build-spinel", "campfire-db-differential-spinel"} + ) + archive = ci.select(["e2e/campfire/assets.spec.js"]) + self.assertEqual( + self.extras(archive), + { + "build-spinel", + "build-campfire-archive", + "smoke-campfire", + "smoke-campfire-docker", + "archive-results", + }, + ) + self.assertEqual( + ci.select(["scripts/campfire-docker-files"])["jobs"], + ci.select(["scripts/build-campfire-archive"])["jobs"], + ) + self.assertNotIn( + "campfire-compare-spinel", + ci.select(["scripts/campfire-docker-files"])["jobs"], + ) + + +class Results(unittest.TestCase): + def needs(self, plan): + return { + "plan": {"result": "success"}, + "compact-required": {"result": "success"}, + **{ + j: { + "result": "success", + "outputs": { + "execution": "success", + "default": "success", + "minor-gc": "success", + "verify-gen": "success", + }, + } + for j in plan["jobs"] + }, + } + + def test_selected_skips_and_missing_results_are_not_green(self): + plan = ci.select(["src/emit/go.rs"]) + for outcome in ["skipped", "cancelled", "failure", None]: + with self.subTest(outcome=outcome): + needs = self.needs(plan) + needs["compare-extra"] = {"result": outcome} + failures, complete = ci.check_results(plan, needs) + self.assertTrue(failures) + self.assertFalse(complete) + + def test_extra_failure_does_not_block_compact_publication_floor(self): + plan = ci.select([], full=True, publish=True) + needs = self.needs(plan) + needs["compare-extra"]["result"] = "failure" + self.assertFalse(ci.check_results(plan, needs, compact=True)[0]) + self.assertTrue(ci.check_results(plan, needs)[0]) + needs["compare"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs, compact=True)[0]) + + def test_advisory_failure_is_visible_but_does_not_fail_required_gate(self): + plan = ci.select([], full=True) + needs = self.needs(plan) + needs["build-spinel"]["outputs"]["execution"] = "failure" + failures, complete = ci.check_results(plan, needs) + self.assertEqual(failures, []) + self.assertFalse(complete) + needs["smoke-spinel"]["result"] = "skipped" + self.assertFalse(ci.check_results(plan, needs)[1]) + + def test_assembly_failure_cannot_issue_checkpoint(self): + plan = ci.select([], full=True, publish=True) + needs = self.needs(plan) + needs["assemble-site"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs)[0]) + self.assertFalse(ci.check_results(plan, needs)[1]) + + def test_each_gc_mode_must_actually_pass_for_completion(self): + plan = ci.select([], full=True) + self.assertEqual(ci.check_results(plan, self.needs(plan)), ([], True)) + for mode in ["default", "minor-gc", "verify-gen"]: + needs = self.needs(plan) + needs["campfire-compare-spinel"]["outputs"][mode] = "failure" + self.assertEqual(ci.check_results(plan, needs), ([], False)) + + def test_unselected_jobs_may_skip_but_planner_must_succeed(self): + plan = ci.select([]) + needs = self.needs(plan) + needs["build-wasm"] = {"result": "skipped"} + self.assertEqual(ci.check_results(plan, needs), ([], True)) + needs["plan"]["result"] = "failure" + self.assertTrue(ci.check_results(plan, needs)[0]) + + +class MergeTree(unittest.TestCase): + def test_diff_tracks_both_rename_owners_and_deletions(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + + def git(*args): + return ( + subprocess.check_output( + ["git", "-C", directory, *args], stderr=subprocess.DEVNULL + ) + .decode() + .strip() + ) + + git("init") + git("config", "user.name", "CI test") + git("config", "user.email", "test@example.invalid") + (root / "src/emit").mkdir(parents=True) + (root / "src/emit/go.rs").write_text("old owner\n") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("switch", "-c", "feature") + (root / "src/emit/go.rs").rename(root / "src/emit/swift.rs") + git("add", ".") + git("commit", "-m", "rename") + head = git("rev-parse", "HEAD") + git("switch", "-") + git("merge", "--no-ff", "feature", "-m", "merge") + sha = git("rev-parse", "HEAD") + event = {"pull_request": {"base": {"sha": base}, "head": {"sha": head}}} + previous = os.getcwd() + try: + os.chdir(root) + paths = ci.changed_paths(event, "pull_request", sha) + self.assertEqual(set(paths), {"src/emit/go.rs", "src/emit/swift.rs"}) + self.assertEqual(ci.select(paths)["extra_compare"], ["swift", "go"]) + with self.assertRaises(ValueError): + ci.changed_paths(event, "pull_request", head) + with ( + patch.dict(event["pull_request"]["base"], sha=head), + self.assertRaises(ValueError), + ): + ci.changed_paths(event, "pull_request", sha) + finally: + os.chdir(previous) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ci_policy_workflow.rs b/tests/ci_policy_workflow.rs new file mode 100644 index 0000000000..b8d4259a4f --- /dev/null +++ b/tests/ci_policy_workflow.rs @@ -0,0 +1,422 @@ +use std::fs; + +#[test] +fn routing_and_required_results_reject_false_green() { + for test in ["tests/ci_plan_test.py", "tests/ci_archive_evidence_test.py"] { + let result = std::process::Command::new("python3") + .args(["-B", test, "-v"]) + .output() + .expect("CI helper tests require python3"); + assert!( + result.status.success(), + "{test}:\n{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + } +} + +#[test] +fn compact_and_extra_compare_share_commands_but_not_results() { + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + assert_eq!( + jobs["compare"]["strategy"]["matrix"]["target"], + serde_yaml_ng::from_str::("[rust, typescript]").unwrap() + ); + assert_eq!(jobs["compare"]["steps"], jobs["compare-extra"]["steps"]); + assert_eq!( + jobs["compare-extra"]["strategy"]["max-parallel"].as_u64(), + Some(2) + ); + assert_eq!(jobs["smoke"]["strategy"]["max-parallel"].as_u64(), Some(2)); + let smoke_guard = jobs["smoke"]["if"].as_str().unwrap(); + for condition in [ + "!cancelled()", + "needs.plan.result == 'success'", + "needs.build-site.result == 'success'", + ] { + assert!( + smoke_guard.contains(condition), + "selected smoke must run after its skipped WASM ancestor: {condition}" + ); + } + assert_eq!( + jobs["campfire-compare-spinel"]["strategy"]["max-parallel"].as_u64(), + Some(1) + ); + assert!( + ci["on"]["pull_request"].get("paths-ignore").is_none(), + "summary must run even for documentation-only PRs" + ); + assert!(jobs.get("ci-required").is_none()); + assert_eq!(jobs["ci-summary"]["name"].as_str(), Some("CI summary")); + assert_eq!( + ci["on"]["workflow_call"]["outputs"]["complete"]["value"].as_str(), + Some("${{ jobs.ci-summary.outputs.complete }}") + ); + for name in ["compact-required", "ci-summary"] { + assert_eq!(jobs[name]["if"].as_str(), Some("always()")); + } + let gate = jobs["ci-summary"]["needs"].as_sequence().unwrap(); + for name in jobs.as_mapping().unwrap().keys().filter_map(|v| v.as_str()) { + if name != "ci-summary" { + assert!( + gate.iter().any(|v| v.as_str() == Some(name)), + "missing result: {name}" + ); + } + } + assert_eq!(ci["permissions"]["contents"].as_str(), Some("read")); + assert!(ci["permissions"].get("pages").is_none()); + assert!(ci["permissions"].get("id-token").is_none()); +} + +#[cfg(unix)] +#[test] +fn focused_framework_loop_runs_every_selection_and_preserves_failure() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let step = ci["jobs"]["framework-tests-spinel"]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["name"].as_str() == Some("Run selected native framework checks")) + .unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("framework-loop-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + let cargo = root.join("cargo"); + fs::write( + &cargo, + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CARGO_LOG\"\n[ \"$3\" != fails ]\n", + ) + .unwrap(); + fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap(); + for (tests, expected_success, expected_log) in [ + ( + "first second", + true, + "test --test first -- --ignored --nocapture\ntest --test second -- --ignored --nocapture\n", + ), + ( + "fails survivor", + false, + "test --test fails -- --ignored --nocapture\ntest --test survivor -- --ignored --nocapture\n", + ), + ("", false, ""), + ] { + let log = root.join("cargo.log"); + fs::write(&log, "").unwrap(); + let result = Command::new("bash") + .args(["-e", "-o", "pipefail", "-c", step["run"].as_str().unwrap()]) + .env("TESTS", tests) + .env("CARGO_LOG", &log) + .env( + "PATH", + format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), + ) + .output() + .unwrap(); + assert_eq!(result.status.success(), expected_success, "{result:?}"); + assert_eq!(fs::read_to_string(log).unwrap(), expected_log); + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn spinel_jobs_are_selected_explicitly_and_archive_evidence_reaches_pages() { + let ci: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); + let jobs = &ci["jobs"]; + for name in [ + "framework-tests-spinel", + "campfire-db-differential-spinel", + "toolchain-spinel", + "compare-spinel", + "smoke-spinel", + "smoke-campfire", + ] { + let job = &jobs[name]; + let needs = job["needs"] + .as_sequence() + .expect("Spinel consumer needs plan and producer"); + assert!( + needs.iter().any(|need| need.as_str() == Some("plan")), + "{name}" + ); + assert!( + job["if"].as_str().unwrap().contains(&format!( + "contains(fromJSON(needs.plan.outputs.jobs), '{name}')" + )), + "{name}" + ); + } + + for (job_name, artifact_name) in [ + ("build-site", "browse-archives"), + ("build-campfire-archive", "campfire-archive"), + ] { + let upload = jobs[job_name]["steps"] + .as_sequence() + .unwrap() + .iter() + .find(|step| step["with"]["name"].as_str() == Some(artifact_name)) + .unwrap(); + assert_eq!(upload["if"].as_str(), Some("always()")); + assert!(upload["with"]["path"].as_str().unwrap().ends_with("*.tgz")); + } + let report = &jobs["archive-results"]; + for dependency in [ + "build-site", + "build-campfire-archive", + "smoke", + "smoke-spinel", + "smoke-campfire", + "smoke-campfire-docker", + ] { + assert!( + report["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some(dependency)) + ); + } + let report_steps = report["steps"].as_sequence().unwrap(); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Collect this run's archive evidence")) + .unwrap()["with"]["pattern"] + .as_str(), + Some("ci-archive-*") + ); + assert_eq!( + report_steps + .iter() + .find(|step| step["name"].as_str() == Some("Save archive outcome report")) + .unwrap()["with"]["name"] + .as_str(), + Some("archive-results") + ); + + let assemble = &jobs["assemble-site"]; + assert!( + assemble["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results")) + ); + let steps = assemble["steps"].as_sequence().unwrap(); + let verify = steps.iter().position(|step| step["run"].as_str() == Some("python3 scripts/ci-archive-evidence.py verify --root _site --report _site/ci/archive-results.json")).expect("archive verification step"); + let pages = steps + .iter() + .position(|step| { + step["uses"] + .as_str() + .is_some_and(|uses| uses.starts_with("actions/upload-pages-artifact@")) + }) + .unwrap(); + assert!(verify < pages); + assert!( + jobs["ci-summary"]["needs"] + .as_sequence() + .unwrap() + .iter() + .any(|need| need.as_str() == Some("archive-results")) + ); +} + +#[test] +fn full_scheduler_runs_every_preflight_success_fresh_and_never_grants_pr_deploy_permissions() { + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + assert_eq!( + full["on"]["schedule"][0]["cron"].as_str(), + Some("17 */4 * * *") + ); + assert!(full["on"].get("push").is_none()); + assert!(full["on"].get("pull_request").is_none()); + assert_eq!( + full["concurrency"]["cancel-in-progress"].as_bool(), + Some(false) + ); + let jobs = &full["jobs"]; + let preflight = &jobs["preflight"]; + assert!(preflight["outputs"].get("run").is_none()); + assert!(preflight["outputs"].get("known").is_none()); + assert!(jobs.get("checkpoint").is_none()); + let preflight_text = serde_yaml_ng::to_string(preflight).unwrap(); + assert!(!preflight_text.contains("actions/cache")); + assert_eq!( + preflight_text + .matches("repos/matz/spinel/commits/master") + .count(), + 1 + ); + assert_eq!( + jobs["validation"]["uses"].as_str(), + Some("./.github/workflows/ci.yml") + ); + assert_eq!(jobs["validation"]["with"]["full"].as_bool(), Some(true)); + assert_eq!( + jobs["validation"]["permissions"]["contents"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"]["actions"].as_str(), + Some("read") + ); + assert_eq!( + jobs["validation"]["permissions"] + .as_mapping() + .unwrap() + .len(), + 2 + ); + assert_eq!(full["permissions"]["contents"].as_str(), Some("read")); + let deploy = &jobs["deploy"]; + let guard = deploy["if"].as_str().unwrap(); + assert!(guard.contains("needs.validation.outputs.publication-ready == 'true'")); + assert!( + !guard.contains("needs.validation.result == 'success'"), + "extra failures cannot hide repro publication" + ); + assert!(deploy.get("continue-on-error").is_none()); + assert_eq!(deploy["permissions"]["pages"].as_str(), Some("write")); + assert!( + deploy["steps"][0]["run"] + .as_str() + .unwrap() + .contains("$VALIDATED_SHA") + ); +} + +#[cfg(unix)] +#[test] +fn scheduler_preflight_executes_publication_guards_and_unknown_input_fallback() { + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let full: serde_yaml_ng::Value = + serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/full-ci.yml").unwrap()) + .unwrap(); + let body = full["jobs"]["preflight"]["steps"][0]["run"] + .as_str() + .unwrap(); + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("full-preflight-{}-{unique}", std::process::id())); + fs::create_dir(&root).unwrap(); + for (name, script) in [( + "gh", + "#!/bin/sh\nprintf 'called\\n' >> \"$GH_LOG\"\n[ \"$MOCK_SPINEL\" != unavailable ] || exit 1\nprintf '%s\\n' \"$MOCK_SPINEL\"\n", + )] { + let path = root.join(name); + fs::write(&path, script).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + let spinel_sha = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + for (event, repo, reference, publish, revision, expected) in [ + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "unavailable", + Some(("true", "master")), + ), + ( + "schedule", + "rubys/roundhouse", + "refs/heads/main", + "false", + "malformed", + Some(("true", "master")), + ), + ( + "workflow_dispatch", + "contributor/roundhouse", + "refs/heads/topic", + "false", + spinel_sha, + Some(("false", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/main", + "true", + spinel_sha, + Some(("true", spinel_sha)), + ), + ( + "workflow_dispatch", + "rubys/roundhouse", + "refs/heads/topic", + "true", + spinel_sha, + None, + ), + ] { + let outputs = root.join("outputs"); + fs::write(&outputs, "").unwrap(); + let result = Command::new("bash") + .args(["-e", "-o", "pipefail", "-c", body]) + .env( + "PATH", + format!("{}:{}", root.display(), std::env::var("PATH").unwrap()), + ) + .env("EVENT", event) + .env("GITHUB_REPOSITORY", repo) + .env("GITHUB_REF", reference) + .env("REQUEST_PUBLISH", publish) + .env("MOCK_SPINEL", revision) + .env("GH_LOG", root.join("gh.log")) + .env("GITHUB_OUTPUT", &outputs) + .output() + .unwrap(); + assert_eq!( + result.status.success(), + expected.is_some(), + "{event} {repo} {reference}: {result:?}" + ); + let actual = fs::read_to_string(outputs).unwrap(); + if let Some((published, resolved)) = expected { + assert_eq!(actual, format!("spinel={resolved}\npublish={published}\n")); + } else { + assert!( + actual.is_empty(), + "rejected publication must not issue outputs" + ); + } + } + assert_eq!( + fs::read_to_string(root.join("gh.log")) + .unwrap() + .lines() + .count(), + 5 + ); + fs::remove_dir_all(root).unwrap(); +} diff --git a/tests/selected_archives.rs b/tests/selected_archives.rs new file mode 100644 index 0000000000..ee2d97bf1b --- /dev/null +++ b/tests/selected_archives.rs @@ -0,0 +1,100 @@ +//! A targeted smoke consumes the same bytes as full publication, without +//! requiring website assets, WASM or unselected archives. + +use std::fs; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn selected_cli_archives_match_full_site_bytes_and_clear_stale_targets() { + roundhouse::stack::run(|| { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = + std::env::temp_dir().join(format!("selected-archives-{}-{unique}", std::process::id())); + let selected = root.join("selected"); + let full = root.join("full"); + fs::create_dir_all(selected.join("browse")).unwrap(); + fs::write(selected.join("browse/go.tgz"), "stale, unselected bytes").unwrap(); + fs::write(selected.join("keep.txt"), "unrelated output").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_roundhouse")) + .args(["--archives", "ruby,rust"]) + .arg(roundhouse::fixtures::real_blog()) + .arg("-o") + .arg(&selected) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut names: Vec<_> = fs::read_dir(selected.join("browse")) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + assert_eq!( + names, + [ + "ruby.json", + "ruby.tgz", + "ruby.zip", + "rust.json", + "rust.tgz", + "rust.zip" + ] + ); + assert_eq!( + fs::read_to_string(selected.join("keep.txt")).unwrap(), + "unrelated output" + ); + assert!(!selected.join("index.html").exists()); + assert!(!selected.join("lib").exists()); + + roundhouse::project::build_site(roundhouse::fixtures::real_blog(), &full).unwrap(); + assert!(full.join("index.html").is_file()); + assert!(full.join("browse/index.html").is_file()); + assert!(full.join("browse/app.js").is_file()); + for name in names { + assert_eq!( + fs::read(selected.join("browse").join(&name)).unwrap(), + fs::read(full.join("browse").join(&name)).unwrap(), + "archive drift: {name}" + ); + } + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(selected.join("browse/rust.json")).unwrap()).unwrap(); + let files = manifest["files"].as_array().unwrap(); + for required in ["README.md", "e2e/package.json", "e2e/playwright.config.js"] { + assert!( + files.iter().any(|f| f["path"].as_str() == Some(required)), + "missing archive contract: {required}" + ); + } + fs::remove_dir_all(root).unwrap(); + }); +} + +#[test] +fn archive_cli_rejects_ambiguous_modes_and_invalid_lists() { + for args in [ + vec!["--archives"], + vec!["--archives", ""], + vec!["--archives", "rust,unknown"], + vec!["--archives", "roda"], + vec!["--archives", "rust", "--site"], + vec!["--archives", "rust", "--target", "go"], + vec!["--archives", "rust", "--allow-unsupported"], + ] { + let output = Command::new(env!("CARGO_BIN_EXE_roundhouse")) + .args(&args) + .output() + .unwrap(); + assert!(!output.status.success(), "accepted {args:?}"); + assert!(String::from_utf8_lossy(&output.stderr).contains("roundhouse:")); + } +} diff --git a/tests/workflow_yaml_parses.rs b/tests/workflow_yaml_parses.rs index b3674d27be..95cacde069 100644 --- a/tests/workflow_yaml_parses.rs +++ b/tests/workflow_yaml_parses.rs @@ -90,16 +90,12 @@ fn spinel_cache_download_failure_falls_back_without_hiding_build_failures() { .unwrap(); assert!(output.status.success(), "{output:?}"); assert_eq!(fs::read_to_string(&env_file).unwrap(), "NO_CCACHE=1\n"); - assert!( - String::from_utf8(output.stdout) - .unwrap() - .contains("::warning::") - ); - assert!( - fs::read_to_string(summary) - .unwrap() - .contains("without compiler cache") - ); + assert!(String::from_utf8(output.stdout) + .unwrap() + .contains("::warning::")); + assert!(fs::read_to_string(summary) + .unwrap() + .contains("without compiler cache")); // Execute the actual build bodies with controlled make exits, both with // caching enabled and after sourcing the fallback's exported environment. @@ -310,10 +306,10 @@ fn pr_archives_remain_tested_without_pages_publication_work() { assert_eq!( jobs["build-site"]["if"].as_str(), Some( - "${{ !cancelled() && needs.generate-fixture.result == 'success' && needs.build-wasm.result == 'success' }}" + "${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'build-site') && needs.generate-fixture.result == 'success' && (needs.plan.outputs.site != 'true' || needs.build-wasm.result == 'success') }}" ) ); - assert_eq!(jobs["smoke"]["needs"].as_str(), Some("build-site")); + assert_eq!(jobs["smoke"]["needs"][0].as_str(), Some("build-site")); let steps = jobs["build-site"]["steps"].as_sequence().unwrap(); for (id, output, renderer) in [ ("fetch-bench", "bench_data", "Render bench page"), @@ -344,7 +340,7 @@ fn pr_archives_remain_tested_without_pages_publication_work() { .unwrap(); assert_eq!( fetch["if"].as_str(), - Some("github.ref == 'refs/heads/main'") + Some("needs.plan.outputs.publish == 'true'") ); let render = steps .iter() @@ -360,31 +356,26 @@ fn pr_archives_remain_tested_without_pages_publication_work() { .iter() .find(|step| step["name"].as_str() == Some("Upload browse archives")) .unwrap(); - assert!(archives.get("if").is_none(), "PR smoke needs the archives"); + assert_eq!(archives["if"].as_str(), Some("always()")); assert_eq!(archives["with"]["name"].as_str(), Some("browse-archives")); - let pages = steps + let pages = jobs["assemble-site"]["steps"] + .as_sequence() + .unwrap() .iter() .find(|step| step["name"].as_str() == Some("Upload Pages artifact")) .unwrap(); - assert_eq!( - pages["if"].as_str(), - Some("github.ref == 'refs/heads/main'") - ); - // Keep the status function: a failed Campfire floor must not suppress - // its explanatory publication, but cancellation or a bad site must. + assert!(pages.get("if").is_none()); + // Publication waits for the report that describes the exact archive bytes. assert_eq!( jobs["assemble-site"]["if"].as_str(), Some( - "${{ !cancelled() && github.ref == 'refs/heads/main' && needs.build-site.result == 'success' }}" + "${{ !cancelled() && needs.plan.outputs.publish == 'true' && needs.build-site.result == 'success' && needs.archive-results.result == 'success' }}" ) ); - assert_eq!( - jobs["deploy"]["if"].as_str(), - Some("github.repository == 'rubys/roundhouse' && github.ref == 'refs/heads/main'") + assert!( + jobs.get("deploy").is_none(), + "PR validation must not carry deployment privileges" ); - assert!(jobs["deploy"].get("continue-on-error").is_none(), "production deployment failures must remain visible"); - assert_eq!(jobs["deploy"]["needs"][0].as_str(), Some("assemble-site")); - assert_eq!(jobs["deploy"]["needs"][1].as_str(), Some("unit")); } #[test] @@ -392,13 +383,18 @@ fn draft_transitions_replace_the_previous_pr_run() { let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str(&fs::read_to_string(".github/workflows/ci.yml").unwrap()).unwrap(); let events = ci["on"]["pull_request"]["types"].as_sequence().unwrap(); - for event in ["ready_for_review", "converted_to_draft"] { + for event in [ + "ready_for_review", + "converted_to_draft", + "labeled", + "unlabeled", + ] { assert!(events.iter().any(|value| value.as_str() == Some(event))); } assert_eq!( ci["concurrency"]["group"].as_str(), Some( - "${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}" + "validation-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}" ) ); assert_eq!( @@ -420,9 +416,10 @@ fn campfire_comparisons_require_an_uploaded_binary_and_report_blocking() { assert_eq!(producer["continue-on-error"].as_bool(), Some(true)); assert_eq!(consumer["continue-on-error"].as_bool(), Some(true)); assert_eq!( - consumer["needs"].as_str(), + consumer["needs"][0].as_str(), Some("build-campfire-compare-spinel") ); + assert_eq!(consumer["needs"][1].as_str(), Some("plan")); assert_eq!( producer["outputs"]["artifact-id"].as_str(), Some("${{ steps.binary.outputs.artifact-id }}") @@ -430,7 +427,7 @@ fn campfire_comparisons_require_an_uploaded_binary_and_report_blocking() { assert_eq!( consumer["if"].as_str(), Some( - "${{ !cancelled() && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }}" + "${{ !cancelled() && contains(fromJSON(needs.plan.outputs.jobs), 'campfire-compare-spinel') && needs.build-campfire-compare-spinel.outputs.artifact-id != '' }}" ) ); let steps = producer["steps"].as_sequence().unwrap(); @@ -734,7 +731,8 @@ fn spinel_model_differential_does_not_wait_for_the_gc_comparison_build() { let ci: serde_yaml_ng::Value = serde_yaml_ng::from_str(&src).expect("parse CI workflow"); let jobs = &ci["jobs"]; let db = &jobs["campfire-db-differential-spinel"]; - assert_eq!(db["needs"].as_str(), Some("build-spinel")); + assert_eq!(db["needs"][0].as_str(), Some("build-spinel")); + assert_eq!(db["needs"][1].as_str(), Some("plan")); assert_eq!(db["continue-on-error"].as_bool(), Some(true)); let command = "scripts/campfire-db-differential --spinel /tmp/campfire"; @@ -750,7 +748,11 @@ fn spinel_model_differential_does_not_wait_for_the_gc_comparison_build() { ); let gc = &jobs["campfire-compare-spinel"]; - assert_eq!(gc["needs"].as_str(), Some("build-campfire-compare-spinel")); + assert_eq!( + gc["needs"][0].as_str(), + Some("build-campfire-compare-spinel") + ); + assert_eq!(gc["needs"][1].as_str(), Some("plan")); let modes: Vec<_> = gc["strategy"]["matrix"]["include"] .as_sequence() .expect("GC matrix") @@ -809,7 +811,7 @@ fn pr_reuse_never_masks_validation_failures_or_changes_the_job_graph() { &["build", "check"] } "writebook-inventory" => { - assert_eq!(job["needs"].as_str(), Some("unit")); + assert_eq!(job["needs"][0].as_str(), Some("unit")); &["inventory", "report"] } "browser-smoke-typescript" => { @@ -818,7 +820,7 @@ fn pr_reuse_never_masks_validation_failures_or_changes_the_job_graph() { &["browser"] } "smoke" => { - assert_eq!(job["needs"].as_str(), Some("build-site")); + assert_eq!(job["needs"][0].as_str(), Some("build-site")); assert_eq!( probe["if"].as_str(), Some("github.event_name == 'pull_request' && matrix.target == 'rust'")