ci.yml

name: ci
permissions:
  contents: read

env:
  # The preinstalled rustup proxies use these runner-owned locations.
  CARGO_HOME: /home/runner/.cargo
  RUSTUP_HOME: /home/runner/.rustup

on:
  push:
    branches: [main]
    paths-ignore:
      - "**.md"
  pull_request:
    paths-ignore:
      - "**.md"
  schedule:
    - cron: "0 2 * * *"
  workflow_dispatch:
    inputs:
      profile:
        description: "Property profile: fast (default) or intensive"
        required: false
        default: "fast"

jobs:
  rust-lint:
    name: rust-lint
    runs-on: [self-hosted, preloop-cpane]
    timeout-minutes: 20
    env:
      CARGO_BUILD_JOBS: "6"
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          persist-credentials: false
          lfs: false
      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          # Fresh microVMs start with an empty RUSTUP_HOME, so dtolnay would
          # reinstall the toolchain on every run. Restore it from cache;
          # bump the key together with `toolchain:` below.
          path: |
            /home/runner/.rustup/toolchains
            /home/runner/.rustup/settings.toml
          key: rustup-1.97-${{ runner.os }}-${{ runner.arch }}
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          toolchain: "1.97"
          components: rustfmt,clippy
      - name: Select Rust toolchain binaries
        run: |
          toolchain_bin="$(dirname "$(rustup which rustc)")"
          echo "$toolchain_bin" >> "$GITHUB_PATH"
          echo "RUSTC=$toolchain_bin/rustc" >> "$GITHUB_ENV"
          echo "RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
          echo "CARGO_BUILD_RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
      # Content-addressed restore replaces the whole-target/ tarball this job
      # used to restore: only crates whose inputs changed are recompiled, and
      # entries are shared across branches, shards and machines. A failing run
      # still publishes what it built, because uploads happen as builds finish
      # rather than in a post-job save step.
      - uses: kunobi-ninja/kache-action@ad7317540dcdd71c904f7a60a87b620a0c0e67ed # v1
        with:
          version: 0.26.3
          s3-bucket: preloop-kache
          s3-endpoint: ${{ secrets.KACHE_R2_ENDPOINT }}
          s3-region: auto
          s3-access-key-id: ${{ secrets.KACHE_R2_ACCESS_KEY_ID }}
          s3-secret-access-key: ${{ secrets.KACHE_R2_SECRET_ACCESS_KEY }}
          # Scopes the prefetch manifest and the shard namespace: one key per
          # build variant so plans never mix clippy, test and release entries.
          manifest-key: clippy
          # Below this, recompiling is cheaper than prefetching.
          min-compile-ms: 2000
          # Stats land in the job summary; PR comments would need
          # pull-requests: write, which this workflow deliberately withholds.
          pr-comment: false
          # Kache publishes to the remote only from protected-branch pushes:
          # a read-only job would run the post-step push, list the whole
          # bucket and upload nothing. Skip that work instead.
          save-cache: ${{ github.event_name == 'push' }}
      - run: cargo fmt --all --check
      - run: |
          cargo clippy --locked -p preloop-gha-protocol -p preloop-gha-parser -p preloop-gha-expressions -p preloop-cache -p preloop-artifacts -p runner-watch -p preloop-conformance --all-targets -- -D warnings
          cargo clippy --locked -p preloop-runner -p preloop-runner-server -p preloop-runner-client -p preloop-dap --all-targets

  rust-test:
    # Sharded unit/integration suite. Each shard runs 1/4 of the tests (split
    # by test-name hash, so no manual bookkeeping) on its own runner; wall
    # clock is setup + slowest shard instead of setup + everything.
    # NOTE: these display names are load-bearing — the `main` ruleset gates
    # on them. Renaming a shard name requires updating the ruleset to match.
    name: rust shard ${{ matrix.shard }} of 4
    runs-on: [self-hosted, preloop-cpane]
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    env:
      # 4 GiB runner guests OOM (SIGKILL) when too many rustc/linker jobs run
      # at once — the preloop-runner-server lib test is the largest unit and
      # is the first to go. 6 jobs was tuned for a bigger box; 4 keeps
      # parallel peak memory inside the guest.
      CARGO_BUILD_JOBS: "4"
      PROPTEST_CASES: "8"
      RUST_TEST_THREADS: "6"
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          persist-credentials: false
          lfs: false
      - name: Materialize conformance goldens
        run: |
          # Only these nine captures are read by workspace tests:
          # eight acquirejob parser fixtures plus runner-watch's scenario-07
          # replay metadata. Fetch each flow directly instead of the complete
          # pinned corpus (which also contains 100+ MB container captures).
          version=$(python3 -c 'import tomllib; print(tomllib.load(open("versions.toml", "rb"))["runner_version"])')
          git lfs install --local --force
          includes=""
          for scenario in \
            06-multi-step 07-step-failure 08-job-outputs-needs \
            10-uses-checkout 11-cache-roundtrip 12-artifact \
            13-composite-action 14-annotations 15-oidc-id-token; do
            path=".runner-watch/golden/v${version}/${scenario}/flows.jsonl"
            includes="${includes:+$includes,}$path"
          done
          git lfs pull --include="$includes"
          for scenario in \
            06-multi-step 07-step-failure 08-job-outputs-needs \
            10-uses-checkout 11-cache-roundtrip 12-artifact \
            13-composite-action 14-annotations 15-oidc-id-token; do
            fixture=".runner-watch/golden/v${version}/${scenario}/flows.jsonl"
            test -s "$fixture"
            if grep -q '^version https://git-lfs.github.com/spec/' "$fixture"; then
              echo "LFS fixture was not materialized: $fixture" >&2
              exit 1
            fi
          done
      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          # Fresh microVMs start with an empty RUSTUP_HOME, so dtolnay would
          # reinstall the toolchain on every run. Restore it from cache;
          # bump the key together with `toolchain:` below.
          path: |
            /home/runner/.rustup/toolchains
            /home/runner/.rustup/settings.toml
          key: rustup-1.97-${{ runner.os }}-${{ runner.arch }}
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          toolchain: "1.97"
          components: rustfmt,clippy
      - name: Select Rust toolchain binaries
        run: |
          toolchain_bin="$(dirname "$(rustup which rustc)")"
          echo "$toolchain_bin" >> "$GITHUB_PATH"
          echo "RUSTC=$toolchain_bin/rustc" >> "$GITHUB_ENV"
          echo "RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
          echo "CARGO_BUILD_RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
      # All four shards build the same workspace and differ only in which tests
      # they select, so they share one manifest-key and one set of entries
      # instead of each warming separately. PR runs restore what a protected
      # branch published but never write, so this cache cannot be poisoned by
      # a pull request.
      - uses: kunobi-ninja/kache-action@ad7317540dcdd71c904f7a60a87b620a0c0e67ed # v1
        with:
          version: 0.26.3
          s3-bucket: preloop-kache
          s3-endpoint: ${{ secrets.KACHE_R2_ENDPOINT }}
          s3-region: auto
          s3-access-key-id: ${{ secrets.KACHE_R2_ACCESS_KEY_ID }}
          s3-secret-access-key: ${{ secrets.KACHE_R2_SECRET_ACCESS_KEY }}
          # Scopes the prefetch manifest and the shard namespace: one key per
          # build variant so plans never mix clippy, test and release entries.
          manifest-key: test
          # Below this, recompiling is cheaper than prefetching.
          min-compile-ms: 2000
          # Stats land in the job summary; PR comments would need
          # pull-requests: write, which this workflow deliberately withholds.
          pr-comment: false
          # Kache publishes to the remote only from protected-branch pushes:
          # a read-only job would run the post-step push, list the whole
          # bucket and upload nothing. Skip that work instead.
          save-cache: ${{ github.event_name == 'push' }}
      - name: Install cargo-nextest
        run: |
          # `cargo install` built nextest from source on every cold cache
          # (~300s per shard). Install the published binary, pinned by version
          # and verified by digest.
          set -euo pipefail
          version=0.9.144
          case "$(uname -m)" in
            x86_64)
              asset=linux
              digest=8a4f726272b0a1c499bd87ca3978bfbb1a8c20bb08ccf075b9996e2081bd1e1e
              ;;
            aarch64|arm64)
              asset=linux-arm
              digest=7fecfd431b810c05c589d800524286b8f80ce2fe9fb1fef05caf16d095cea407
              ;;
            *)
              echo "no pinned cargo-nextest build for $(uname -m)" >&2
              exit 1
              ;;
          esac
          archive="$RUNNER_TEMP/cargo-nextest.tar.gz"
          curl -fsSL --retry 3 -o "$archive" "https://get.nexte.st/${version}/${asset}"
          echo "${digest}  ${archive}" | sha256sum -c -
          mkdir -p "$HOME/.local/bin"
          tar -xzf "$archive" -C "$HOME/.local/bin" cargo-nextest
          "$HOME/.local/bin/cargo-nextest" --version
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
      - name: Run cargo nextest run --locked --workspace --partition hash:${{ matrix.shard }}/4
        run: |
          set -o pipefail
          # Keep the full log in the guest before printing it. The engine caps
          # a job's retained log, so a step that prints tens of megabytes can
          # lose its output — including the failing test names — and the run
          # becomes undiagnosable. The summary step below re-prints just the
          # verdict, which always fits.
          cargo nextest run --locked --workspace --partition hash:${{ matrix.shard }}/4 2>&1 \
            | tee "$RUNNER_TEMP/shard-${{ matrix.shard }}.log"
        env:
          RUSTFLAGS: "-A warnings"
      - name: Shard failure summary
        if: failure()
        run: |
          set -uo pipefail
          log="$RUNNER_TEMP/shard-${{ matrix.shard }}.log"
          echo "::group::shard ${{ matrix.shard }} verdict"
          if [ -f "$log" ]; then
            # A shard killed mid-run (OOM, SIGKILL) prints no FAIL block and no
            # summary line, so the marker grep alone can come back empty on the
            # exact failure worth seeing. The tail is the verdict then.
            grep -E "^ *FAIL |^ *TRY |^ *SIGSEGV|^ *SIGKILL|^failures:|^test result:|^error(\[|:)|Out of memory|oom-kill|Killed process|Cannot allocate memory" "$log" | tail -40 || true
            echo "--- last lines ---"
            tail -15 "$log" || true
          else
            echo "no shard log captured at $log"
          fi
          echo "--- guest kernel (OOM?) ---"
          sudo dmesg 2>/dev/null | tail -15 || echo "(no dmesg)"
          echo "--- guest memory ---"
          free -m 2>/dev/null | head -3 || true
          echo "::endgroup::"
      # The preloop-runner-server integration tests were split out of the lib
      # into separate tests/*.rs crates (the single large unit did not fit the
      # 4 GiB guest). They need the `test-support` feature for the deterministic
      # hooks, so they run under `cargo test` here — nextest can't scope a
      # feature to one package in a --workspace run. Partition by test-file
      # name hash so the shard split still holds.
      - name: Server integration tests (test-support)
        run: |
          set -euo pipefail
          shard=${{ matrix.shard }}
          crates="recovery runs_api logs registration broker security webhooks concurrency execution_protection_api runner_deprecations store_env"
          selected=""
          for c in $crates; do
            h=$(printf '%s' "$c" | md5sum | cut -c1-8)
            if [ $(( (16#$h) % 4 + 1 )) -eq "$shard" ]; then
              selected="$selected $c"
            fi
          done
          if [ -z "${selected# }" ]; then
            echo "no server integration crates hashed to shard $shard"
            exit 0
          fi
          for c in $selected; do
            echo "=== shard $shard: cargo test --test $c ==="
            cargo test --locked -p preloop-runner-server --features test-support --test "$c"
          done
        env:
          RUSTFLAGS: "-A warnings"

  property-tests-fast:
    name: property-tests-fast
    runs-on: [self-hosted, preloop-cpane]
    timeout-minutes: 30
    if: github.event_name != 'schedule'
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          persist-credentials: false
      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          # Fresh microVMs start with an empty RUSTUP_HOME, so dtolnay would
          # reinstall the toolchain on every run. Restore it from cache;
          # bump the key together with `toolchain:` below.
          path: |
            /home/runner/.rustup/toolchains
            /home/runner/.rustup/settings.toml
          key: rustup-1.97-${{ runner.os }}-${{ runner.arch }}
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          toolchain: "1.97"
      - name: Select Rust toolchain binaries
        run: |
          toolchain_bin="$(dirname "$(rustup which rustc)")"
          echo "$toolchain_bin" >> "$GITHUB_PATH"
          echo "RUSTC=$toolchain_bin/rustc" >> "$GITHUB_ENV"
          echo "RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
          echo "CARGO_BUILD_RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
      - uses: kunobi-ninja/kache-action@ad7317540dcdd71c904f7a60a87b620a0c0e67ed # v1
        with:
          version: 0.26.3
          s3-bucket: preloop-kache
          s3-endpoint: ${{ secrets.KACHE_R2_ENDPOINT }}
          s3-region: auto
          s3-access-key-id: ${{ secrets.KACHE_R2_ACCESS_KEY_ID }}
          s3-secret-access-key: ${{ secrets.KACHE_R2_SECRET_ACCESS_KEY }}
          # Scopes the prefetch manifest and the shard namespace: one key per
          # build variant so plans never mix clippy, test and release entries.
          manifest-key: property-tests-fast
          # Below this, recompiling is cheaper than prefetching.
          min-compile-ms: 2000
          # Stats land in the job summary; PR comments would need
          # pull-requests: write, which this workflow deliberately withholds.
          pr-comment: false
          # Kache publishes to the remote only from protected-branch pushes:
          # a read-only job would run the post-step push, list the whole
          # bucket and upload nothing. Skip that work instead.
          save-cache: ${{ github.event_name == 'push' }}

      - name: Verify property filters match tests
        run: |
          # One `--list` pass per crate. Keep each server filter separate so
          # removing a property module cannot hide behind a broad match.
          server_list=$(PROPTEST_CASES=1 cargo test -p preloop-runner-server -- --list 2>/dev/null)
          legacy=$(printf '%s\n' "$server_list" | grep -Ec '^concurrency::properties(:|$)' || true)
          http=$(printf '%s\n' "$server_list" | grep -Ec '^concurrency_http_properties(:|$)' || true)
          runner=$(cargo test -p preloop-runner -- --list 2>/dev/null | grep -c 'timespan_tests' || true)
          parser=$(cargo test -p preloop-gha-parser -- --list 2>/dev/null | grep -c 'concurrency_' || true)
          if [ "$legacy" -eq 0 ] || [ "$http" -eq 0 ] || [ "$runner" -eq 0 ] || [ "$parser" -eq 0 ]; then
            echo "ERROR: zero matching tests (legacy=$legacy http=$http runner=$runner parser=$parser)" >&2
            exit 1
          fi
          echo "server filters: legacy=$legacy http=$http; runner=$runner parser=$parser"
      - name: Check no sleep() in property test files
        run: |
          files="crates/preloop-runner-server/src/concurrency.rs crates/preloop-runner-server/src/concurrency_http_properties.rs"
          bad=0
          for f in $files; do
            if [ -f "$f" ] && grep -q 'sleep(' "$f"; then
              echo "ERROR: sleep() found in $f" >&2
              bad=1
            fi
          done
          if [ "$bad" -ne 0 ]; then exit 1; fi
          echo "Sleep guard: OK"

      - name: Pure concurrency properties
        run: |
          PROPTEST_CASES=256 cargo test -p preloop-runner-server \
            'concurrency::properties' -- --test-threads=1

      - name: Expression and parser properties
        run: |
          PROPTEST_CASES=256 cargo test -p preloop-gha-expressions -- --test-threads=1
          cargo test -p preloop-gha-parser concurrency_ -- --test-threads=1

      - name: Runner dispatcher properties
        run: |
          cargo test -p preloop-runner timespan_tests -- --test-threads=1

      - name: HTTP sequence properties
        run: |
          PROPTEST_CASES=64 cargo test -p preloop-runner-server \
            concurrency_http_properties -- --test-threads=1

      - name: Differential harness dry-run (schema + algebraic validation)
        run: |
          python3 benchmarks/real-world/run-concurrency-property-probes.py \
            --dry-run \
            --corpus benchmarks/real-world/concurrency-property-cases.json

      - name: Differential harness rejects contaminated fixture
        run: |
          set +e
          python3 benchmarks/real-world/run-concurrency-property-probes.py \
            --dry-run \
            --corpus benchmarks/real-world/fixtures/contaminated-case.json
          status=$?
          set -e
          if [ "$status" -eq 0 ]; then
            echo "ERROR: harness did not reject contaminated fixture (expected nonzero exit)" >&2
            exit 1
          fi
          echo "Contamination rejection: OK (exit $status)"

      - name: Upload proptest regressions
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: proptest-regressions-fast
          path: |
            crates/**/proptest-regressions/
          if-no-files-found: ignore

  property-tests-intensive:
    name: Property tests intensive
    runs-on: [self-hosted, preloop-cpane]
    timeout-minutes: 120
    if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.profile == 'intensive')
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          persist-credentials: false
      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          # Fresh microVMs start with an empty RUSTUP_HOME, so dtolnay would
          # reinstall the toolchain on every run. Restore it from cache;
          # bump the key together with `toolchain:` below.
          path: |
            /home/runner/.rustup/toolchains
            /home/runner/.rustup/settings.toml
          key: rustup-1.97-${{ runner.os }}-${{ runner.arch }}
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          toolchain: "1.97"
      - name: Select Rust toolchain binaries
        run: |
          toolchain_bin="$(dirname "$(rustup which rustc)")"
          echo "$toolchain_bin" >> "$GITHUB_PATH"
          echo "RUSTC=$toolchain_bin/rustc" >> "$GITHUB_ENV"
          echo "RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"
          echo "CARGO_BUILD_RUSTDOC=$toolchain_bin/rustdoc" >> "$GITHUB_ENV"

      - uses: kunobi-ninja/kache-action@ad7317540dcdd71c904f7a60a87b620a0c0e67ed # v1
        with:
          version: 0.26.3
          s3-bucket: preloop-kache
          s3-endpoint: ${{ secrets.KACHE_R2_ENDPOINT }}
          s3-region: auto
          s3-access-key-id: ${{ secrets.KACHE_R2_ACCESS_KEY_ID }}
          s3-secret-access-key: ${{ secrets.KACHE_R2_SECRET_ACCESS_KEY }}
          # Release builds key differently from the fast profile, so this
          # variant gets its own manifest and shard namespace.
          manifest-key: property-tests-intensive
          # Below this, recompiling is cheaper than prefetching.
          min-compile-ms: 2000
          # Stats land in the job summary; PR comments would need
          # pull-requests: write, which this workflow deliberately withholds.
          pr-comment: false
          # Kache publishes to the remote only from protected-branch pushes:
          # a read-only job would run the post-step push, list the whole
          # bucket and upload nothing. Skip that work instead.
          save-cache: ${{ github.event_name == 'push' }}

      - name: Verify property filters match tests
        run: |
          server_list=$(PROPTEST_CASES=1 cargo test -p preloop-runner-server -- --list 2>/dev/null)
          legacy=$(printf '%s\n' "$server_list" | grep -Ec '^concurrency::properties(:|$)' || true)
          http=$(printf '%s\n' "$server_list" | grep -Ec '^concurrency_http_properties(:|$)' || true)
          runner=$(cargo test -p preloop-runner -- --list 2>/dev/null | grep -c 'timespan_tests' || true)
          parser=$(cargo test -p preloop-gha-parser -- --list 2>/dev/null | grep -c 'concurrency_' || true)
          if [ "$legacy" -eq 0 ] || [ "$http" -eq 0 ] || [ "$runner" -eq 0 ] || [ "$parser" -eq 0 ]; then
            echo "ERROR: zero matching tests (legacy=$legacy http=$http runner=$runner parser=$parser)" >&2
            exit 1
          fi
          echo "server filters: legacy=$legacy http=$http; runner=$runner parser=$parser"
      - name: Check no sleep() in property test files
        run: |
          bad=0
          for f in \
            crates/preloop-runner-server/src/concurrency.rs \
            crates/preloop-runner-server/src/concurrency_http_properties.rs; do
            if [ -f "$f" ] && grep -q 'sleep(' "$f"; then
              echo "ERROR: sleep() found in $f" >&2
              bad=1
            fi
          done
          if [ "$bad" -ne 0 ]; then exit 1; fi

      - name: Intensive property profile (release)
        run: |
          PROPTEST_CASES=10000 PROPTEST_MAX_SHRINK_ITERS=100000 \
            cargo test -p preloop-runner-server 'concurrency::properties' \
            --release -- --test-threads=1

      - name: Intensive runner timing profile (release)
        run: cargo test -p preloop-runner timespan_tests --release -- --test-threads=1

      - name: Intensive expression / parser profile (release)
        run: |
          PROPTEST_CASES=10000 cargo test -p preloop-gha-expressions \
            --release -- --test-threads=1
          cargo test -p preloop-gha-parser concurrency_ \
            --release -- --test-threads=1

      - name: Intensive HTTP sequence profile (release)
        run: |
          PROPTEST_CASES=1000 PROPTEST_MAX_SHRINK_ITERS=100000 \
            cargo test -p preloop-runner-server concurrency_http_properties \
            --release -- --test-threads=1

      - name: Differential harness dry-run
        run: |
          python3 benchmarks/real-world/run-concurrency-property-probes.py \
            --dry-run \
            --corpus benchmarks/real-world/concurrency-property-cases.json

      - name: Differential harness rejects contaminated fixture
        run: |
          set +e
          python3 benchmarks/real-world/run-concurrency-property-probes.py \
            --dry-run \
            --corpus benchmarks/real-world/fixtures/contaminated-case.json
          status=$?
          set -e
          if [ "$status" -eq 0 ]; then
            echo "ERROR: harness did not reject contaminated fixture" >&2
            exit 1
          fi

      - name: Upload proptest regressions
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: proptest-regressions-intensive
          path: |
            crates/**/proptest-regressions/
          if-no-files-found: ignore