diff --git a/.github/workflows/apt-indices-refresh.yml b/.github/workflows/apt-indices-refresh.yml deleted file mode 100644 index d813cf925..000000000 --- a/.github/workflows/apt-indices-refresh.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: apt-indices-refresh - -# Weekly freshness reminder for golden apt indices. Each bake stamps the pack -# (marker in the image, `apt_indices` in provenance.json); this job compares -# the published pack's stamp against the versions.toml policy. When stale, it -# opens a draft PR asking for a rebuild on a host with enough disk. The -# schedule never dispatches the hosted golden bake. -on: - schedule: - # Monday 04:00 UTC: at most one reminder per week when indices exceed policy. - - cron: "0 4 * * 1" - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - check: - name: Open draft PR when golden apt indices are stale - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - sparse-checkout: versions.toml - - - name: Check published index freshness - id: freshness - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - MAX_AGE_DAYS="$(grep '^apt_indices_max_age_days' versions.toml | cut -d'"' -f2)" - test -n "$MAX_AGE_DAYS" - TAG="$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)" - echo "policy: ${MAX_AGE_DAYS}d; latest release: $TAG" - rm -rf /tmp/apt-prov - if gh release download "$TAG" \ - --pattern 'preloop-ubuntu-24.04-x86_64.provenance.json' \ - --dir /tmp/apt-prov \ - --clobber \ - --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - STAMP="$(python3 -c 'import json; print(json.load(open("/tmp/apt-prov/preloop-ubuntu-24.04-x86_64.provenance.json")).get("apt_indices", {}).get("baked_at", ""))')" - else - echo "no provenance asset; treating pack as stale" - STAMP="" - fi - if [ -z "$STAMP" ]; then - { - echo "stale=true" - echo "stamp=missing" - echo "age_days=unknown" - } >> "$GITHUB_OUTPUT" - exit 0 - fi - AGE_DAYS="$(( ($(date -u +%s) - $(date -u -d "$STAMP" +%s)) / 86400 ))" - echo "indices baked $STAMP (${AGE_DAYS}d old)" - { - echo "stamp=$STAMP" - echo "age_days=$AGE_DAYS" - } >> "$GITHUB_OUTPUT" - if [ "$AGE_DAYS" -gt "$MAX_AGE_DAYS" ]; then - echo "stale=true" >> "$GITHUB_OUTPUT" - else - echo "stale=false" >> "$GITHUB_OUTPUT" - echo "fresh; nothing to do" - fi - - - name: Open manual rebuild reminder - if: steps.freshness.outputs.stale == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - APT_INDEX_STAMP: ${{ steps.freshness.outputs.stamp }} - APT_INDEX_AGE_DAYS: ${{ steps.freshness.outputs.age_days }} - run: | - set -euo pipefail - REQUESTED_AT="$(date -u +%F)" - BRANCH="chore/golden-apt-refresh-${REQUESTED_AT}" - EXISTING_URL="$(gh pr list \ - --repo "$GITHUB_REPOSITORY" \ - --state open \ - --head "$BRANCH" \ - --json url \ - --jq '.[0].url // empty')" - if [ -n "$EXISTING_URL" ]; then - echo "rebuild reminder already open: $EXISTING_URL" - exit 0 - fi - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git switch -c "$BRANCH" - awk -v marker="# Golden rebuild requested ${REQUESTED_AT}: apt indices ${APT_INDEX_STAMP} (${APT_INDEX_AGE_DAYS}d old)." ' - /^# Golden rebuild requested / { next } - /^apt_indices_max_age_days = / { print marker } - { print } - ' versions.toml > versions.toml.tmp - mv versions.toml.tmp versions.toml - git add versions.toml - git commit -m "chore(golden): request apt index refresh [skip ci]" - gh auth setup-git - git fetch origin "$BRANCH" || true - git push --force-with-lease origin "HEAD:refs/heads/$BRANCH" - - BODY="$RUNNER_TEMP/golden-refresh-pr.md" - cat > "$BODY" <110 GiB of staging. Production bakes are host-driven: x86_64 on a Linux -# host with KVM and disk, aarch64 on an Apple Silicon host. `release.yml` seeds -# each release with the newest complete golden pair (payload + sidecars). +# keyed by the official-image pin in official-image.toml, which does not move +# with an engine release, and the runtime runner bundle is mounted from the +# engine at job time, so a per-release bake buys nothing while costing a hosted +# runner >110 GiB of staging. Production bakes are host-driven: x86_64 on an +# x86_64 host with KVM and disk, aarch64 on macstudio. `release.yml` seeds each +# release with the newest complete golden pair (payload + sidecars). on: workflow_dispatch: inputs: @@ -295,13 +294,10 @@ jobs: env: GOLDEN_NAME: preloop-ubuntu-24.04-${{ env.GOLDEN_ARCH }} run: | - MAX_AGE_DAYS="$(grep -E '^[[:space:]]*apt_indices_max_age_days[[:space:]]*=' versions.toml | cut -d'"' -f2 || true)" - test -n "$MAX_AGE_DAYS" python3 scripts/write-golden-provenance.py \ --golden "dist/$GOLDEN_NAME" \ --base-evidence "dist/$GOLDEN_NAME.base-provenance.json" \ --base-sbom "dist/$GOLDEN_NAME.base-sbom.spdx.json" \ - --apt-indices-max-age-days "$MAX_AGE_DAYS" \ --output "dist/$GOLDEN_NAME.provenance.json" - name: Install cosign diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b8decf52..ab3e8d1e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -323,12 +323,11 @@ jobs: subject-path: artifacts/* # Engines resolve the golden at their own release tag, so a release must # carry a complete, verifiable golden pair — but goldens are baked by - # their own hosts (x86_64 on a Linux KVM host, aarch64 on an Apple - # Silicon host), never by this pipeline: a golden's content is keyed by - # the official image pin, the toolchain pins and the apt-index bake - # date, and the runner bundle is mounted from the engine at job time, so - # a per-release bake would ship an identical artifact for >110 GiB of - # staging. + # their own hosts (x86_64 on an x86_64 KVM host, aarch64 on macstudio), + # never by this pipeline: a golden's content is keyed by the official + # image pin and the runner contract, and the runner bundle is mounted + # from the engine at job time, so a per-release bake would ship an + # identical artifact for >110 GiB of staging. # Seed the newest pair that exists, sidecars included, so a release is # never a guaranteed 404 (golden_url_candidates prefers the own tag) and # never claims a checksum it cannot prove. diff --git a/.github/workflows/smolvm-release-verify.yml b/.github/workflows/smolvm-release-verify.yml index 8fad0f5f4..8f4e4e34c 100644 --- a/.github/workflows/smolvm-release-verify.yml +++ b/.github/workflows/smolvm-release-verify.yml @@ -136,8 +136,16 @@ jobs: rm -f "$SOCKET" } trap cleanup EXIT - IMAGE="$(grep '^ubuntu_24_04_base' versions.toml | cut -d'"' -f2)" - [ -n "$IMAGE" ] || { echo "ubuntu_24_04_base missing from versions.toml" >&2; exit 1; } + # The base image once came from versions.toml's `ubuntu_24_04_base`; + # that pin is gone with the curated bake. Verify the runtime against + # the image the official goldens are actually built from, per arch. + case "$(uname -m)" in + x86_64) OFFICIAL_KEY=official_runner_image_base_amd64 ;; + arm64|aarch64) OFFICIAL_KEY=official_runner_image_base_arm64 ;; + *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;; + esac + IMAGE="$(awk -F'"' -v k="^${OFFICIAL_KEY} = " '$0 ~ k { print $2; exit }' official-image.toml)" + [ -n "$IMAGE" ] || { echo "$OFFICIAL_KEY missing from official-image.toml" >&2; exit 1; } python3 -c "import socket; s = socket.socket(socket.AF_UNIX); s.bind('$SOCKET'); s.close()" echo "creating machine from $IMAGE" smolvm machine create --name "$NAME" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e487e6da..65d65c86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,104 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Releases before v0.27.0 predate the changelog. ## [Unreleased] +### Changed + +- **One golden source: the official packed golden, or the image you + configured.** `runs-on: ubuntu-latest`/`ubuntu-24.04` — and any pooled + environment with no image configured — now resolve to the published packed + golden, downloaded per architecture from a digest-pinned OCI reference + (`PRELOOP_GOLDEN_OCI_REF` overrides it; `PRELOOP_GOLDEN_URL` still selects a + release-asset mirror) and installed only after its checksum (release asset) + or layer digest (OCI) matches. The transfer resumes from a `.partial` file + across retries and engine restarts, and a download that cannot complete + fails the job: there is no local bake and no stock-Ubuntu fallback behind it. + `ubuntu-22.04` now maps nowhere and keeps the configured image instead of + silently selecting a 24.04 base. + + The curated stock bake is deleted with it: no toolchain layers + (Rust/Go/Python/Node), no `base_install_script`, no apt package pins or + index-freshness marker (and no weekly `apt-indices-refresh` workflow), no + goldens baked per `runs-on` environment, no local bake from stock Ubuntu and + no direct-create fallback when the packed artifact is unavailable. + `PRELOOP_USE_PACKED_GOLDEN` is gone — a file-pack backend always uses a + packed golden — and the pool holds one golden per pool environment and forks + it per job. + + A configured image (`PRELOOP_RUNNER_BASE_IMAGE`, or the `[golden] + base_image` that `preloop init` records) is now used exactly as it is. + Preloop adds only the GitHub-runner machinery at golden build: a `runner` + account (uid 1001) with its home, `_work` and passwordless sudo unless the + image already has them; an owner-only ownership walk over the runner home + (`find … ! -user 1001 -exec chown -h 1001:1001 {} +`, which leaves a + runner-owned file's group alone); a writable `/opt/hostedtoolcache` with + `RUNNER_TOOL_CACHE`/`AGENT_TOOLSDIRECTORY` in `/etc/environment`; and the + `/etc/preloop-bake.json` build record. The only checked requirement is a + glibc dynamic loader — a missing `bash`, `git` or `docker` fails the step + that needs it, exactly as on GitHub-hosted runners. `preloop build-golden` + takes the image it bakes (`--base-image `, or the configured image when + the flag is absent); the official golden is published packed and cannot be + built locally. + + Guest `PATH`, `RUSTUP_HOME` and `CARGO_HOME` overrides are gone: the guest + runner's PATH is the system PATH, and tool versions come from the workflow's + `setup-*` actions, as they do on GitHub. + +- **A golden is keyed by everything that changes it.** The official golden's + fingerprint now covers the source this host would fetch it from + (`PRELOOP_GOLDEN_OCI_REF` or the `PRELOOP_GOLDEN_URL` mirror), so changing + the mirror re-fetches instead of serving the previous pack forever; a + configured image's fingerprint covers its effective runner account, so + changing `PRELOOP_RUNNER_USER`/`_UID` rebuilds instead of adopting a golden + baked for the account before it. A backend that cannot restore packs + (AgentENV) boots the pinned official runner image for the official golden — + the sentinel itself is not an image — and an unconfigured pool no longer + fails to start there. + +- **Leftovers of earlier releases are cleaned up at startup.** Packed-golden + payloads keyed on the retired stock Ubuntu bases + (`preloop-[mirror.gcr.io-library-]ubuntu-{24.04,22.04}[-sha256-…]--`) + live under a stem this release no longer produces, so no fingerprint + rotation would ever reach them; the startup sweep now removes them. Goldens + prepared per `runs-on` environment (`-golden-`) keep + their fingerprint record, which exempted them from the stale-machine + cleanup; one that neither the official golden nor the configured image + resolves to is now deleted with its record. + +- **Apt indices are baked into the golden.** The runner-image dump ships with + `/var/lib/apt/lists` wiped, and a workflow's `sudo apt-get install ` + with no `apt-get update` first (uv's musl cell) fails with `E: Unable to + locate package`. The golden contract now refreshes the indices at build, and + an unpacked pack that has none is refreshed once before it is frozen, so + forks inherit them. Also fixes `golden_contract_script("root", …)`, which + rendered `; ;` and did not parse as `sh`. + +- **The guest runner inherits the image's environment.** It is launched + through a shell that sources `/etc/environment`, so the PATH a step sees is + the image's own — the tools a hosted image preinstalls keep resolving — + while the pool's variables are re-asserted afterwards. An image that + already carries a `runner` account at another uid now has that account moved + to the configured uid instead of running jobs as a uid that owns nothing. + The Rosetta amd64 install is no longer fatal for a configured image without + apt: the bake warns that x86_64 binaries will not run instead of refusing an + otherwise usable image. + +- **Stopping the engine no longer waits for a golden transfer.** + `prepare_fork_base` checks the shutdown token before starting a download or + bake, so a restart during the warm returns promptly; the transfer resumes on + the next start. + +### Security + +- **The golden's runner account name is validated** before it reaches the + root-run bake scripts and the `sudoers.d` filename: anything that is not a + Linux account name (or `root`) is refused, closing a shell-injection hole + for a hostile or mistyped `PRELOOP_RUNNER_USER`. + +- **A `PRELOOP_GOLDEN_URL` mirror must be verifiable.** Its bytes become the + image every job runs, so an unverified payload is now refused: the mirror + must publish `.sha256` or the host must pin the digest with + `PRELOOP_GOLDEN_SHA256`. The engine's own release assets keep the previous + tolerated-with-a-warning behavior. ### Added - In-place reruns now persist `jobs.request_id` and copy it into per-attempt `job_history`, preserving log/step/artifact resolution for diff --git a/benchmarks/real-world/conformance-10repos.sh b/benchmarks/real-world/conformance-10repos.sh index de34d073b..065c35c0b 100755 --- a/benchmarks/real-world/conformance-10repos.sh +++ b/benchmarks/real-world/conformance-10repos.sh @@ -37,9 +37,17 @@ export PRELOOP_CLIENT_TIMEOUT_SECONDS="${PRELOOP_CLIENT_TIMEOUT_SECONDS:-3600}" SERVER_BIN="${PRELOOP_BIN:-$ROOT/target/debug/preloop}" CLIENT_BIN="${PRELOOP_CLIENT_BIN:-$ROOT/target/debug/preloop-runner-client}" GUEST_RUNNER_BUNDLE="${PRELOOP_RUNNER_BUNDLE:-$ROOT/target/aarch64-unknown-linux-gnu/debug}" -OFFICIAL_GOLDEN_BASE="ghcr.io/preloopdev/runner-images:ubuntu24-arm64-runner-large-latest@sha256:a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e" -OFFICIAL_GOLDEN_NAME="preloop-ghcr.io-preloopdev-runner-images-ubuntu24-arm64-runner-large-latest-sha256-a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e-aarch64" -OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/${OFFICIAL_GOLDEN_NAME}.smolmachine}" +# The official sentinel: what a pool with no image configured resolves to, +# and what `runs-on: ubuntu-latest`/`ubuntu-24.04` resolve to. The engine +# fetches the packed golden from the per-architecture OCI reference +# (PRELOOP_GOLDEN_OCI_REF overrides it) and keys the payload by this +# sentinel plus the environment fingerprint — ask `preloop golden-path` +# for that path, never spell it here. +OFFICIAL_GOLDEN_SENTINEL="preloop-official-golden" +# Operator cache of the same ~9GB payload, symlinked into the campaign home +# so a harness restart does not re-download it. Point PRELOOP_GOLDEN_ARTIFACT +# at a copy that lives elsewhere. +OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/${OFFICIAL_GOLDEN_SENTINEL}-aarch64.smolmachine}" SERVER_PID="" FAILED_TARGETS="" HOST_HOME="${HOME:-}" @@ -119,7 +127,7 @@ prepare_golden_home() { # environment fingerprint). A hand-rolled hash drifts whenever the bake, # toolchains, or node pins change, and a miss means a full 9GB rebuild. expected="$(PRELOOP_HOME="$CAMPAIGN_HOME" \ - PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_BASE" \ + PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_SENTINEL" \ "$SERVER_BIN" golden-path)" || fail "could not compute golden payload path" mkdir -p "$(dirname "$expected")" ln -sfn "$OFFICIAL_GOLDEN_ARTIFACT" "$expected" @@ -176,10 +184,9 @@ start_server() { SMOLVM_DATA_DIR="${SMOLVM_DATA_DIR:-$CAMPAIGN_HOME/smolvm}" \ SMOLVM_AGENT_ROOTFS="${SMOLVM_AGENT_ROOTFS:-$SMOLVM_HOME_DIR/agent-rootfs}" \ SMOLVM_LIB_DIR="${SMOLVM_LIB_DIR:-$SMOLVM_HOME_DIR/lib}" \ - PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_BASE" \ + PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_SENTINEL" \ PRELOOP_RUNNER_BUNDLE="$GUEST_RUNNER_BUNDLE" \ PRELOOP_RUNNER_NAME_PREFIX="conformance-10repos" \ - PRELOOP_USE_PACKED_GOLDEN=1 \ PRELOOP_RUNNER_POOL_ENABLED=1 \ PRELOOP_RUNNER_POOL_SIZE="$POOL_SIZE" \ PRELOOP_USE_FORK=1 \ diff --git a/benchmarks/real-world/conformance-5repos.sh b/benchmarks/real-world/conformance-5repos.sh index 0cba2c02c..5c23e9960 100755 --- a/benchmarks/real-world/conformance-5repos.sh +++ b/benchmarks/real-world/conformance-5repos.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -# Run the five-repository conformance campaign against the pinned official +# Run the five-repository conformance campaign against the official packed # GitHub-hosted runner golden. # -# The golden is the 9GB packed artifact produced from the official -# ubuntu24-arm64 runner image. It is selected by its immutable base-image -# reference and exposed to Preloop through a temporary PRELOOP_HOME symlink; -# no second 9GB copy is made. +# The golden is the ~9GB packed artifact produced from the official +# ubuntu24-arm64 runner image. A pool with no image configured resolves to the +# official sentinel, so hosted labels (ubuntu-latest, ubuntu-24.04) and the +# pool's own labels both run on it; the artifact is exposed to Preloop through +# a temporary PRELOOP_HOME symlink, so no second 9GB copy is made. # # Usage: # benchmarks/real-world/conformance-5repos.sh [all|grafana|deno|pydantic|valkey|cli] @@ -36,12 +37,17 @@ SERVER_BIN="${PRELOOP_BIN:-$ROOT/target/debug/preloop}" CLIENT_BIN="${PRELOOP_CLIENT_BIN:-$ROOT/target/debug/preloop-runner-client}" GUEST_RUNNER_BUNDLE="${PRELOOP_RUNNER_BUNDLE:-$ROOT/target/aarch64-unknown-linux-gnu/debug}" -# This is the official ubuntu24 arm64 hosted-image snapshot whose local packed -# artifact is approximately 9GB. Keep the digest and artifact name together: -# changing either silently selects a different image and invalidates results. -OFFICIAL_GOLDEN_BASE="ghcr.io/preloopdev/runner-images:ubuntu24-arm64-runner-large-latest@sha256:a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e" -OFFICIAL_GOLDEN_NAME="preloop-ghcr.io-preloopdev-runner-images-ubuntu24-arm64-runner-large-latest-sha256-a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e-aarch64" -OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/${OFFICIAL_GOLDEN_NAME}.smolmachine}" +# The official sentinel: what a pool with no image configured resolves to, and +# what `runs-on: ubuntu-latest`/`ubuntu-24.04` resolve to. The engine fetches +# the packed golden from the per-architecture OCI reference (GOLDEN_OCI_REF_* +# in preloop-orchestrator, overridable with PRELOOP_GOLDEN_OCI_REF) and keys the +# payload by this sentinel plus the environment fingerprint — ask +# `preloop golden-path` for that path, never spell it here. +OFFICIAL_GOLDEN_SENTINEL="preloop-official-golden" +# Operator cache of the same ~9GB payload, symlinked into the campaign home so a +# harness restart does not re-download it. Point PRELOOP_GOLDEN_ARTIFACT at a +# copy that lives elsewhere. +OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/preloop-official-golden-aarch64.smolmachine}" SERVER_PID="" FAILED_TARGETS="" @@ -152,27 +158,16 @@ prepare_golden_home() { # outside this home (~/.config/preloop/vms), so a clean slate is safe. rm -rf "$CAMPAIGN_HOME" if [ "${PRELOOP_GOLDEN_SOURCE:-}" = ghcr ]; then - # Fresh pull of the digest-pinned golden (GOLDEN_OCI_REF_* in - # preloop-orchestrator): no local artifact is linked, so the engine - # downloads the packed golden from ghcr into the empty campaign home. - echo "=== golden: downloading digest-pinned image from ghcr.io (no local artifact) ===" - # Only the stock ubuntu bases trigger the prebaked-golden download - # (should_download_prebaked_golden); the runner-images ref is "custom". - # It must be the exact digest pin (the engine default): the packed golden - # is registered under this base's fingerprint, and hosted labels - # (ubuntu-24.04*, ubuntu-latest) resolve to the pin. Any other spelling - # (e.g. bare `ubuntu:24.04`) misses that golden, and the pool bakes a - # second, stock-Ubuntu golden for those jobs — no cmake, no hostedtoolcache. - GOLDEN_BASE_OVERRIDE="$(sed -n 's/^ubuntu_24_04_base = "\(.*\)"$/\1/p' "$ROOT/versions.toml")" - [ -n "$GOLDEN_BASE_OVERRIDE" ] || fail "ubuntu_24_04_base pin missing from versions.toml" - # A verified ghcr download is kept in $PRELOOP_GOLDEN_CACHE (APFS clone, so - # reuse is instant) to avoid re-pulling 9.6 GB on every harness restart. - # The downloaded bytes are the same official golden for any stock base, so - # clone the cached file to the path the engine computes for this base. + # Fresh download into the empty campaign home: nothing is linked, so the + # engine fetches the packed golden from the per-arch OCI reference. + echo "=== golden: downloading the official packed golden from ghcr.io (no local artifact) ===" + # A verified download is kept in $PRELOOP_GOLDEN_CACHE (APFS clone, so reuse + # is instant) to avoid re-pulling 9.6 GB on every harness restart. Clone it + # to the payload path the engine computes for the official sentinel. local cache="${PRELOOP_GOLDEN_CACHE:-$HOME/golden-cache}" cached target cached="$(compgen -G "$cache/preloop-*" | head -1 || true)" if [ -n "$cached" ]; then - target="$("$SERVER_BIN" golden-path --home "$CAMPAIGN_HOME" --base-image "$GOLDEN_BASE_OVERRIDE")" + target="$("$SERVER_BIN" golden-path --home "$CAMPAIGN_HOME" --base-image "$OFFICIAL_GOLDEN_SENTINEL")" mkdir -p "$(dirname "$target")" cp -c "$cached" "$target" echo "=== golden: reusing verified ghcr download $cached as $target ===" @@ -185,10 +180,10 @@ prepare_golden_home() { # Reject the small launcher stub or a partial download. The packed payload # used by this campaign is the ~9GB .smolmachine sidecar. [ "$bytes" -ge 8589934592 ] || fail "golden is ${bytes} bytes, not the required ~9GB .smolmachine payload" - # Ask the engine for its own payload path — the fingerprint folds in - # base/toolchains/bake AND the node-external pins, so duplicating the hash + # Ask the engine for its own payload path — the fingerprint folds in the + # official OCI reference and the node-external pins, so duplicating the hash # here drifts. `preloop golden-path` keeps the harness in lockstep. - expected="$("$SERVER_BIN" golden-path --home "$CAMPAIGN_HOME" --base-image "$OFFICIAL_GOLDEN_BASE")" + expected="$("$SERVER_BIN" golden-path --home "$CAMPAIGN_HOME" --base-image "$OFFICIAL_GOLDEN_SENTINEL")" mkdir -p "$(dirname "$expected")" ln -sfn "$OFFICIAL_GOLDEN_ARTIFACT" "$expected" [ -f "$expected" ] || fail "failed to expose official golden at $expected" @@ -226,9 +221,9 @@ start_server() { local log="$OUTPUT_ROOT/server.log" mkdir -p "$OUTPUT_ROOT" "$CAMPAIGN_HOME" "$SMOLVM_PROCESS_HOME" rm -f "$log" - # The base reference is intentionally the same string encoded in the local - # cache filename. The CLI then consumes the symlinked 9GB artifact instead - # of downloading or rebuilding a different golden. + # No image is configured: every job resolves to the official sentinel, so the + # pool consumes the symlinked 9GB artifact instead of downloading another + # golden. Configure PRELOOP_RUNNER_BASE_IMAGE only to run a different image. # A broad `gh auth token` PAT is refused by the engine whenever a job's # declared `permissions:` are narrower; PRELOOP_SKIP_GH_TOKEN=1 runs tokenless. if [ -z "${PRELOOP_SKIP_GH_TOKEN:-}" ] && [ -z "${PRELOOP_GITHUB_TOKEN:-}" ] && command -v gh >/dev/null 2>&1; then @@ -249,10 +244,9 @@ start_server() { SMOLVM_DATA_DIR="${SMOLVM_DATA_DIR:-$CAMPAIGN_HOME/smolvm}" \ SMOLVM_AGENT_ROOTFS="${SMOLVM_AGENT_ROOTFS:-$HOST_HOME/.smolvm/agent-rootfs}" \ SMOLVM_LIB_DIR="${SMOLVM_LIB_DIR:-$HOST_HOME/.smolvm/lib}" \ - PRELOOP_RUNNER_BASE_IMAGE="${GOLDEN_BASE_OVERRIDE:-$OFFICIAL_GOLDEN_BASE}" \ + PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_SENTINEL" \ PRELOOP_RUNNER_BUNDLE="$GUEST_RUNNER_BUNDLE" \ PRELOOP_RUNNER_NAME_PREFIX="conformance-5repos" \ - PRELOOP_USE_PACKED_GOLDEN=1 \ PRELOOP_RUNNER_POOL_ENABLED=1 \ PRELOOP_RUNNER_POOL_SIZE="$POOL_SIZE" \ PRELOOP_USE_FORK=1 \ diff --git a/benchmarks/real-world/conformance-new5repos.sh b/benchmarks/real-world/conformance-new5repos.sh index 871bc7599..1dd7cead2 100755 --- a/benchmarks/real-world/conformance-new5repos.sh +++ b/benchmarks/real-world/conformance-new5repos.sh @@ -29,9 +29,17 @@ export PRELOOP_CLIENT_TIMEOUT_SECONDS="${PRELOOP_CLIENT_TIMEOUT_SECONDS:-3600}" SERVER_BIN="${PRELOOP_BIN:-$ROOT/target/debug/preloop}" CLIENT_BIN="${PRELOOP_CLIENT_BIN:-$ROOT/target/debug/preloop-runner-client}" GUEST_RUNNER_BUNDLE="${PRELOOP_RUNNER_BUNDLE:-$ROOT/target/aarch64-unknown-linux-gnu/debug}" -OFFICIAL_GOLDEN_BASE="ghcr.io/preloopdev/runner-images:ubuntu24-arm64-runner-large-latest@sha256:a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e" -OFFICIAL_GOLDEN_NAME="preloop-ghcr.io-preloopdev-runner-images-ubuntu24-arm64-runner-large-latest-sha256-a58990d6b6f8ca5861f33d77e1d3f0732d7d14261caacd6fba8c8f707c05b40e-aarch64" -OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/${OFFICIAL_GOLDEN_NAME}.smolmachine}" +# The official sentinel: what a pool with no image configured resolves to, +# and what `runs-on: ubuntu-latest`/`ubuntu-24.04` resolve to. The engine +# fetches the packed golden from the per-architecture OCI reference +# (PRELOOP_GOLDEN_OCI_REF overrides it) and keys the payload by this +# sentinel plus the environment fingerprint — ask `preloop golden-path` +# for that path, never spell it here. +OFFICIAL_GOLDEN_SENTINEL="preloop-official-golden" +# Operator cache of the same ~9GB payload, symlinked into the campaign home +# so a harness restart does not re-download it. Point PRELOOP_GOLDEN_ARTIFACT +# at a copy that lives elsewhere. +OFFICIAL_GOLDEN_ARTIFACT="${PRELOOP_GOLDEN_ARTIFACT:-$HOME/.config/preloop/vms/${OFFICIAL_GOLDEN_SENTINEL}-aarch64.smolmachine}" SERVER_PID="" FAILED_TARGETS="" if [ "$(uname -s)" = Darwin ]; then @@ -98,7 +106,7 @@ prepare_golden_home() { # Reimplementing EnvironmentSpec here drifted when runner-root and smolvm # inputs changed, silently causing a fresh multi-GiB image pull. expected="$(PRELOOP_HOME="$CAMPAIGN_HOME" \ - PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_BASE" \ + PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_SENTINEL" \ "$SERVER_BIN" golden-path)" || fail "could not compute golden payload path" mkdir -p "$(dirname "$expected")" ln -sfn "$OFFICIAL_GOLDEN_ARTIFACT" "$expected" @@ -148,10 +156,9 @@ start_server() { SMOLVM_DATA_DIR="${SMOLVM_DATA_DIR:-$CAMPAIGN_HOME/smolvm}" \ SMOLVM_AGENT_ROOTFS="${SMOLVM_AGENT_ROOTFS:-$SMOLVM_HOME_DIR/agent-rootfs}" \ SMOLVM_LIB_DIR="${SMOLVM_LIB_DIR:-$SMOLVM_HOME_DIR/lib}" \ - PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_BASE" \ + PRELOOP_RUNNER_BASE_IMAGE="$OFFICIAL_GOLDEN_SENTINEL" \ PRELOOP_RUNNER_BUNDLE="$GUEST_RUNNER_BUNDLE" \ PRELOOP_RUNNER_NAME_PREFIX="conformance-new5repos" \ - PRELOOP_USE_PACKED_GOLDEN=1 \ PRELOOP_RUNNER_POOL_ENABLED=1 \ PRELOOP_RUNNER_POOL_SIZE="$POOL_SIZE" \ PRELOOP_USE_FORK=1 \ diff --git a/benchmarks/substrates/e2e-bench.sh b/benchmarks/substrates/e2e-bench.sh index fc23c809f..4aa1f8611 100755 --- a/benchmarks/substrates/e2e-bench.sh +++ b/benchmarks/substrates/e2e-bench.sh @@ -57,7 +57,7 @@ log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$OUTDIR/driver.log case "$SUBSTRATE" in aenv) export PRELOOP_VM_BACKEND=agentenv ;; - smolvm) export PRELOOP_VM_BACKEND=smolvm; export PRELOOP_USE_PACKED_GOLDEN=1 ;; + smolvm) export PRELOOP_VM_BACKEND=smolvm ;; *) echo "unknown substrate $SUBSTRATE" >&2; exit 2 ;; esac diff --git a/benchmarks/substrates/project-bench.sh b/benchmarks/substrates/project-bench.sh index 27ddce3c0..5bfb8f5a9 100755 --- a/benchmarks/substrates/project-bench.sh +++ b/benchmarks/substrates/project-bench.sh @@ -37,7 +37,7 @@ log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" | tee -a "$OUTDIR/driver.log case "$SUBSTRATE" in aenv) export PRELOOP_VM_BACKEND=agentenv ;; - smolvm) export PRELOOP_VM_BACKEND=smolvm PRELOOP_USE_PACKED_GOLDEN=1 ;; + smolvm) export PRELOOP_VM_BACKEND=smolvm ;; *) echo "unknown substrate: $SUBSTRATE" >&2; exit 2 ;; esac diff --git a/crates/preloop-cli/src/init.rs b/crates/preloop-cli/src/init.rs index 60a605bfc..91fbc3a7c 100644 --- a/crates/preloop-cli/src/init.rs +++ b/crates/preloop-cli/src/init.rs @@ -361,8 +361,8 @@ struct GoldenAnswer { /// Dockerfile built and saved). struct GoldenDecision { kind: GoldenChoice, - /// Persisted base image; `None` = the engine's stock base, i.e. the packed - /// official golden the engine downloads itself. + /// Persisted base image; `None` = nothing configured, i.e. the packed + /// official golden the engine downloads and verifies itself. base_image: Option, dockerfile: Option, label: String, @@ -980,7 +980,8 @@ impl Session { arch_ok: None, warnings: Vec::new(), detail: json!({ - "note": "the engine downloads the packed official golden on the first serve", + "note": "the engine downloads the pinned packed official golden on the first \ + serve; a failed download fails the job (no local fallback)", "download_bytes": OFFICIAL_GOLDEN_DOWNLOAD_BYTES, "working_set_bytes": OFFICIAL_GOLDEN_WORKING_SET_BYTES, }), @@ -1008,6 +1009,10 @@ impl Session { self.record_local_path(&path) } }?; + // A configured image is baked by the pool on the next serve. The + // official golden takes the opposite path — downloaded packed by the + // engine, never baked — so this line must not be shown for it (and + // cannot be: an official decision carries no base image). if decision.base_image.is_some() { self.note( "serve builds this golden locally on the first run (a builder VM, then a \ @@ -1892,8 +1897,8 @@ async fn print_probe() -> anyhow::Result<()> { "golden": { "kind": configured_kind(&config).map(GoldenChoice::as_str), // The base image `serve` would use right now: the environment - // variable when set, else the persisted choice, else the engine's - // stock base (null). + // variable when set, else the persisted choice; null means + // nothing is configured, i.e. the packed official golden. "base_image": base_image, "env_override": std::env::var(preloop_runner_server::config::BASE_IMAGE_ENV) .ok() @@ -2462,6 +2467,34 @@ mod tests { ); } + /// The official choice is `kind` alone — no base image — and that is a + /// complete answer: a re-run keeps it without re-asking, and nothing + /// pretends a local bake is in its future (the engine downloads the + /// pinned packed golden; a failed download fails the job). + #[test] + fn the_stored_official_golden_is_a_complete_answer() { + let config = ConfigFile { + golden: GoldenConfig { + kind: Some(GoldenChoice::Official.as_str().to_owned()), + base_image: None, + dockerfile: None, + }, + ..ConfigFile::default() + }; + let (kind, label) = golden_prior(&config).expect("the stored golden must be offered"); + assert_eq!(kind, GoldenChoice::Official); + assert!(label.contains("official"), "{label}"); + + let decision = decision_from_config(&config, &GoldenAnswer::of(GoldenChoice::Official)) + .expect("the stored golden must be a complete answer"); + assert_eq!(decision.kind, GoldenChoice::Official); + assert_eq!(decision.base_image, None); + assert_eq!( + decision.artifact_bytes, None, + "the official golden is downloaded, not staged from a local artifact" + ); + } + #[test] fn dockerfile_context_defaults_to_the_dockerfile_directory() { assert_eq!( diff --git a/crates/preloop-cli/src/main.rs b/crates/preloop-cli/src/main.rs index e79d714fc..8468c4ce9 100644 --- a/crates/preloop-cli/src/main.rs +++ b/crates/preloop-cli/src/main.rs @@ -5,7 +5,7 @@ use base64::Engine as _; use clap::{Parser, Subcommand}; use futures_util::StreamExt; use preloop_gha_protocol::{ExecutionStatus, NdjsonEvent, RunAccepted, RunId, WorkflowSubmission}; -use preloop_orchestrator::environment::{DEFAULT_BASE_IMAGE, is_stock_base_image}; +use preloop_orchestrator::environment::{OFFICIAL_GOLDEN, is_official_golden}; use preloop_orchestrator::{RunnerPool, RunnerPoolConfig, artifact_payload}; use preloop_runner_server::credential_store::CredentialStore; use std::collections::BTreeMap; @@ -827,14 +827,16 @@ struct GoldenPathArgs { #[arg(long, env = "PRELOOP_HOME")] home: Option, - /// OCI base image. Explicit CLI input overrides - /// PRELOOP_RUNNER_BASE_IMAGE. + /// Base image to key the packed-golden payload on: the official sentinel + /// or a configured image. Resolved like the pool — explicit CLI input + /// overrides PRELOOP_RUNNER_BASE_IMAGE, then `[golden] base_image`; + /// nothing configured means the official golden. #[arg( - long, - env = "PRELOOP_RUNNER_BASE_IMAGE", - default_value = DEFAULT_BASE_IMAGE + long = "base-image", + value_name = "REF", + env = "PRELOOP_RUNNER_BASE_IMAGE" )] - base_image: String, + base_image: Option, } #[derive(Debug, Parser)] @@ -843,26 +845,22 @@ struct BuildGoldenArgs { #[arg(long)] runner_bundle: PathBuf, - /// Workspace to detect toolchains from (rust-toolchain.toml, .nvmrc, …). - /// Defaults to the current directory, so the release-golden workflow — - /// which runs from the repo checkout — bakes the project's toolchains - /// into the artifact. Without them every fork of the packed golden - /// reinstalls rust per job. - #[arg(long)] - workspace: Option, - /// Destination path for the packed artifact. #[arg(long)] output: PathBuf, - /// OCI base image or packed artifact to use. Explicit CLI input overrides - /// PRELOOP_RUNNER_BASE_IMAGE. + /// Image to bake into a golden: a registry reference, a docker-save tar, + /// or a `.smolmachine`/rootfs path. Explicit CLI input overrides + /// PRELOOP_RUNNER_BASE_IMAGE and `[golden] base_image`; with neither set, + /// the configured image is baked. The image is used as-is plus the runner + /// contract — no packages or toolchains are added. The official golden is + /// published packed and cannot be built here. #[arg( - long, - env = "PRELOOP_RUNNER_BASE_IMAGE", - default_value = DEFAULT_BASE_IMAGE + long = "base-image", + value_name = "REF", + env = "PRELOOP_RUNNER_BASE_IMAGE" )] - base_image: String, + base_image: Option, /// Persistent guest storage in GiB. Large official runner snapshots may /// need 80 GiB or more. @@ -1189,18 +1187,25 @@ fn systemd_socket_activation_requested() -> bool { } async fn cmd_golden_path(args: GoldenPathArgs) -> anyhow::Result<()> { - // Same stem construction as the pool config: home/vms/preloop--. + // Same resolution and stem construction as the pool config: + // home/vms/preloop--, then the environment fingerprint. let home = args.home.map(Ok).unwrap_or_else(|| { std::env::var_os("PRELOOP_HOME") .map(PathBuf::from) .context("PRELOOP_HOME is not set (pass --home)") })?; + // An exported-but-blank PRELOOP_RUNNER_BASE_IMAGE behaves like an unset + // one, exactly as `serve` resolves it. + let base_image = match args.base_image.filter(|value| !value.trim().is_empty()) { + Some(base) => base, + None => configured_base_image()?, + }; let stem = home.join("vms").join(format!( "preloop-{}-{}", - args.base_image.replace(['/', ':', '@'], "-"), + base_image.replace(['/', ':', '@'], "-"), std::env::consts::ARCH )); - println!("{}", artifact_payload(&stem, &args.base_image).display()); + println!("{}", artifact_payload(&stem, &base_image).display()); Ok(()) } @@ -1218,6 +1223,21 @@ async fn cmd_build_golden(args: BuildGoldenArgs) -> anyhow::Result<()> { } else { std::env::current_dir()?.join(args.output) }; + // `--base-image` (or PRELOOP_RUNNER_BASE_IMAGE) names the image to bake; + // with neither, bake the configured `[golden] base_image`, exactly like + // `serve` would (a blank variable behaves like an unset one). Nothing + // configured resolves to the official golden, which has no local build: + // bail instead of trying to fetch one. + let base_image = match args.base_image.filter(|value| !value.trim().is_empty()) { + Some(base) => base, + None => configured_base_image()?, + }; + if is_official_golden(&base_image) { + anyhow::bail!( + "the official golden is published packed and cannot be built here; pass the image \ + to bake (--base-image ) or set PRELOOP_RUNNER_BASE_IMAGE" + ); + } // Enterprise option: verify the base image's provenance before baking. // Dump-style base images carry a GitHub-signed SLSA attestation and a // cosign keyless signature from the publishing workflow; a golden should @@ -1226,15 +1246,14 @@ async fn cmd_build_golden(args: BuildGoldenArgs) -> anyhow::Result<()> { if std::env::var_os("PRELOOP_VERIFY_BASE_IMAGE") .is_some_and(|value| value != "0" && value != "false") { - verify_base_image(&args.base_image).await?; + verify_base_image(&base_image).await?; } if env_flag("PRELOOP_REQUIRE_BASE_DIGEST", false) { - require_digest_pinned_base(&args.base_image)?; + require_digest_pinned_base(&base_image)?; } let config = RunnerPoolConfig { size: 1, use_fork: false, - use_packed_artifact: false, // Unique per bake: smolvm keys a machine's data dir by a hash of its // name and reuses a dir left behind by a failed/interrupted run at its // old on-disk size (smolvm#956). A stale dir then boots with the @@ -1250,8 +1269,7 @@ async fn cmd_build_golden(args: BuildGoldenArgs) -> anyhow::Result<()> { // fresh-disk guarantee above. Per-invocation suffix instead. format!("preloop-release-golden-{}", std::process::id()) }), - base_image: args.base_image, - workspace: args.workspace.or_else(|| std::env::current_dir().ok()), + base_image, artifact_stem: output.clone(), release_version: env!("CARGO_PKG_VERSION").to_owned(), runner_bundle, @@ -2165,14 +2183,15 @@ fn truncate_reason(s: &str) -> String { /// Base image the runner pool boots its golden from. /// /// `PRELOOP_RUNNER_BASE_IMAGE` wins, then the `[golden]` section -/// `preloop init` writes to the config file, then the engine's stock base — -/// the digest-pinned Ubuntu pin whose packed official golden the pool -/// downloads. Reading the file here is what makes one `preloop init` enough: -/// `serve` and `server install` never needed a separate golden flag. +/// `preloop init` writes to the config file, then [`OFFICIAL_GOLDEN`] — the +/// packed official GitHub runner golden the engine downloads. Nothing is +/// baked locally unless an image is configured. Reading the file here is what +/// makes one `preloop init` enough: `serve` and `server install` never needed +/// a separate golden flag. fn configured_base_image() -> anyhow::Result { let config = preloop_runner_server::config::load_config()?; Ok(preloop_runner_server::config::golden_base_image(&config) - .unwrap_or_else(|| DEFAULT_BASE_IMAGE.to_owned())) + .unwrap_or_else(|| OFFICIAL_GOLDEN.to_owned())) } /// `serve` with no golden configured: offer the wizard when a human is @@ -2293,19 +2312,6 @@ fn local_runner_pool_config( if backend == VmBackend::Agentenv { aenv_egress::ensure_engine_egress(&server_url)?; } - // AgentENV keeps packs as server-side snapshots, so there is no artifact - // file to build or reuse: its golden is prepared straight from the base - // image and forked per job, which is its fast path anyway. - let use_packed_artifact = backend == VmBackend::Smolvm - && env_flag("PRELOOP_USE_PACKED_GOLDEN", DEFAULT_USE_PACKED_GOLDEN); - // The workspace is scanned for toolchain version files (rust-toolchain.toml, - // .nvmrc, etc.) so the golden can be built with the project's toolchains - // pre-installed instead of installing them per job. PRELOOP_WORKSPACE - // overrides the current directory for daemon-style deployments. - let workspace = std::env::var_os("PRELOOP_WORKSPACE") - .map(PathBuf::from) - .or_else(|| std::env::current_dir().ok()) - .unwrap_or_else(|| PathBuf::from(".")); // The mounted control socket only makes sense when the runner URL is // loopback (the VM reaches the host through the socket relay). With // `PRELOOP_RUNNER_URL` pointing at a host-reachable LAN address — the @@ -2324,16 +2330,15 @@ fn local_runner_pool_config( control_socket }; // The golden's base image: `PRELOOP_RUNNER_BASE_IMAGE`, else the choice - // `preloop init` persisted, else the stock pin. + // `preloop init` persisted, else the official golden. let base_image = configured_base_image()?; - // A custom base image (`.smolmachine` artifact or any non-stock OCI - // reference) serves every queued job itself, so environment-based runner - // replacement has nothing to switch to: the job's implied stock base - // (`ubuntu:24.04` from `ubuntu-latest` labels) will always differ from - // the configured image and idle runners would be replaced forever. - // Compare on the plain `repository:tag`, so the digest-pinned defaults - // (ubuntu:24.04@sha256:…) still count as stock Ubuntu images. - let custom_base = !is_stock_base_image(&base_image); + // A configured image serves every queued job itself, so environment-based + // runner replacement has nothing to switch to: a queued job's implied + // environment (`ubuntu-latest`/`ubuntu-24.04` resolve to the official + // golden) will always differ from the configured image, and idle runners + // would be replaced forever. Only the official golden takes part in that + // replacement. + let custom_base = !is_official_golden(&base_image); let cpus = runner_cpus(); Ok(RunnerPoolConfig { // Size zero is the deliberate low-memory mode: keep the local @@ -2346,14 +2351,11 @@ fn local_runner_pool_config( } else { 0 }, - // Forking is safe whenever the packed artifact is enabled. The warm - // pool switch controls whether idle runners stay registered; size-zero - // mode still prepares one golden and forks a disposable VM per job. - use_packed_artifact, - // Forking is safe whenever the packed artifact is enabled, and always - // on AgentENV, whose snapshots are immutable and re-forkable. - use_fork: (use_packed_artifact || backend == VmBackend::Agentenv) - && env_flag("PRELOOP_USE_FORK", true), + // Forking is safe on every backend now: file-pack backends always + // restore the packed golden, and AgentENV keeps immutable, + // re-forkable snapshots. Size-zero mode still prepares one golden and + // forks a disposable VM per job. + use_fork: env_flag("PRELOOP_USE_FORK", true), // Multiple engines on one host must not share a namespace: smolvm // keys machines and persistent overlays by name, and cross-engine // reuse boots a runner whose persisted state points at the other @@ -2362,6 +2364,10 @@ fn local_runner_pool_config( .ok() .filter(|value| !value.is_empty()) .unwrap_or_else(|| "preloop-runner".into()), + // The resolved base: the official sentinel when nothing is + // configured, else the configured image. `custom_base` decides + // whether idle runners may be replaced by a job's implied + // environment. base_image: base_image.clone(), // GitHub-hosted parity: guest runners run as a dedicated account // instead of root, so steps see the hosted user-session contract. @@ -2376,12 +2382,12 @@ fn local_runner_pool_config( .ok() .and_then(|value| value.parse().ok()) .or(Some(1001)), - workspace: Some(workspace), - // The packed artifact cache key includes the resolved base image - // (tag AND digest): the digest-pinned defaults are a golden's - // provenance. When the pin moves, a stale packed golden baked from - // the old digest must not be reused. Custom `.smolmachine` bases are - // filesystem paths, so separators are normalized out of the key. + // The packed artifact cache key is the resolved base image — the + // official sentinel or a configured image (tag AND digest). The + // payload filename appends the environment fingerprint, so a moved + // pin or a changed bake contract rotates the file instead of reusing + // a stale golden. Non-registry bases are filesystem paths, so + // separators are normalized out of the key. artifact_stem: home.join("vms").join(format!( "preloop-{}-{}", base_image.replace(['/', ':', '@'], "-"), @@ -2454,8 +2460,6 @@ fn runner_pool_labels() -> Vec { const RUNNER_CPUS: u16 = 8; /// Low-memory on-demand provisioning is the default; opt into idle warm VMs. const DEFAULT_RUNNER_POOL_ENABLED: bool = false; -/// Published or locally cached packed images avoid cold OCI bootstrap per job. -const DEFAULT_USE_PACKED_GOLDEN: bool = true; /// Memory given to each runner VM, in MiB. SmolVM balloons this, so an idle /// runner commits far less than its ceiling. const RUNNER_MEMORY_MIB: u32 = 4096; @@ -5207,10 +5211,22 @@ mod tests { panic!("expected build-golden command"); }; assert_eq!( - args.base_image, - "ghcr.io/acme/runner-images:ubuntu24-runner-large-latest-arm64" + args.base_image.as_deref(), + Some("ghcr.io/acme/runner-images:ubuntu24-runner-large-latest-arm64") ); assert_eq!(args.storage_gib, Some(80)); + + // `--base-image` is optional: release automation names the image + // through PRELOOP_RUNNER_BASE_IMAGE (or the recorded `[golden]` + // choice), so parsing must succeed without the flag. + parse(&[ + "build-golden", + "--runner-bundle", + "/tmp/runner", + "--output", + "/tmp/golden", + ]) + .unwrap(); } #[test] @@ -5303,10 +5319,7 @@ mod tests { "__PRELOOP_TEST_POOL_FLAG_UNSET__", DEFAULT_RUNNER_POOL_ENABLED )); - assert!(env_flag( - "__PRELOOP_TEST_GOLDEN_FLAG_UNSET__", - DEFAULT_USE_PACKED_GOLDEN - )); + assert!(env_flag("__PRELOOP_TEST_FORK_FLAG_UNSET__", true)); } #[test] @@ -5381,6 +5394,13 @@ mod tests { unsafe { std::env::set_var(preloop_runner_server::config::BASE_IMAGE_ENV, " ") }; assert_eq!(configured_base_image().unwrap(), "ghcr.io/acme/base:1"); + // Nothing recorded and no override: the pool boots the packed + // official golden — no stock Ubuntu default exists any more. + unsafe { std::env::remove_var(preloop_runner_server::config::BASE_IMAGE_ENV) }; + let unconfigured = preloop_runner_server::config::ConfigFile::default(); + preloop_runner_server::config::write_config_to(&config_path, &unconfigured).unwrap(); + assert_eq!(configured_base_image().unwrap(), OFFICIAL_GOLDEN); + unsafe { match previous_config { Some(value) => { @@ -5397,8 +5417,13 @@ mod tests { } } + /// Nothing configured boots the packed official golden, and only the + /// official golden takes part in environment-based runner replacement: a + /// configured image serves every queued job itself, so the job's implied + /// hosted environment (`ubuntu-latest`/`ubuntu-24.04`) would differ from + /// it forever and idle runners would be replaced forever. #[test] - fn custom_base_image_disables_environment_replacement() { + fn only_the_official_golden_keeps_environment_replacement() { let _env_guard = TEST_ENV_MUTEX.lock().unwrap(); let home = tempfile::tempdir().unwrap(); // The config resolves a Linux guest runner bundle; fabricate one so @@ -5409,17 +5434,27 @@ mod tests { [0x7f, b'E', b'L', b'F'], ) .unwrap(); + // Pin the config path to a file that does not exist, so an operator's + // real `[golden]` cannot leak into the unset case. + let previous_config = std::env::var_os(preloop_runner_server::config::CONFIG_PATH_ENV); unsafe { std::env::set_var("PRELOOP_RUNNER_BUNDLE", bundle.path()); std::env::set_var("PRELOOP_RUNNER_STORAGE_GB", "80"); - std::env::remove_var("PRELOOP_USE_PACKED_GOLDEN"); + std::env::set_var( + preloop_runner_server::config::CONFIG_PATH_ENV, + home.path().join("config.toml"), + ); + std::env::remove_var("PRELOOP_RUNNER_BASE_IMAGE"); } let queue_depth = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let next_job_runs_on = std::sync::Arc::new(std::sync::RwLock::new(vec!["ubuntu-latest".to_owned()])); - let config = |base: &str| { + let config = |base: Option<&str>| { unsafe { - std::env::set_var("PRELOOP_RUNNER_BASE_IMAGE", base); + match base { + Some(base) => std::env::set_var("PRELOOP_RUNNER_BASE_IMAGE", base), + None => std::env::remove_var("PRELOOP_RUNNER_BASE_IMAGE"), + } } let config = local_runner_pool_config( home.path(), @@ -5442,38 +5477,46 @@ mod tests { } config }; - // Stock base: environment-based replacement stays enabled. - let stock = config("ubuntu:24.04"); - assert!(stock.next_job_runs_on.is_some()); + // Nothing configured: the official golden, replacement stays enabled. + let official = config(None); + assert_eq!(official.base_image, OFFICIAL_GOLDEN); + assert!(is_official_golden(&official.base_image)); + assert!(official.next_job_runs_on.is_some()); assert!( - stock.use_packed_artifact, - "packed golden use must be enabled by default" + official.use_fork, + "forking must remain enabled when the warm pool is off" ); - assert!( - stock.use_fork, - "packed golden forks must remain enabled when the warm pool is off" - ); - assert_eq!(stock.storage_gib, 80); - assert!( - config("mirror.gcr.io/library/ubuntu:24.04") - .next_job_runs_on - .is_some(), - "the mirrored pinned Ubuntu image is still a stock environment" - ); - // Custom base (artifact): disabled, so idle runners are never - // replaced by the job's implied stock base. - assert!(config("/tmp/custom.smolmachine").next_job_runs_on.is_none()); - unsafe { - std::env::set_var("PRELOOP_USE_PACKED_GOLDEN", "false"); + assert_eq!(official.storage_gib, 80); + // The sentinel spelled out resolves to the same golden. + let explicit = config(Some(OFFICIAL_GOLDEN)); + assert_eq!(explicit.base_image, OFFICIAL_GOLDEN); + assert!(explicit.next_job_runs_on.is_some()); + // A configured image is used as-is and disables replacement, so idle + // runners are never replaced by the job's implied hosted environment. + for base in [ + "ubuntu:24.04", + "mirror.gcr.io/library/ubuntu:24.04", + "ghcr.io/acme/base:1", + // A `.smolmachine` artifact, the old "custom base" spelling. + "/tmp/custom.smolmachine", + ] { + assert!(!is_official_golden(base)); + let configured = config(Some(base)); + assert_eq!(configured.base_image, base, "the image must not be rebased"); + assert!( + configured.next_job_runs_on.is_none(), + "{base} must disable environment-based replacement" + ); } - assert!( - !config("ubuntu:24.04").use_packed_artifact, - "operators can still disable packed golden use explicitly" - ); unsafe { - std::env::remove_var("PRELOOP_USE_PACKED_GOLDEN"); std::env::remove_var("PRELOOP_RUNNER_BUNDLE"); std::env::remove_var("PRELOOP_RUNNER_STORAGE_GB"); + match previous_config { + Some(value) => { + std::env::set_var(preloop_runner_server::config::CONFIG_PATH_ENV, value) + } + None => std::env::remove_var(preloop_runner_server::config::CONFIG_PATH_ENV), + } } } diff --git a/crates/preloop-orchestrator/src/environment.rs b/crates/preloop-orchestrator/src/environment.rs index 2904abc41..86c11988c 100644 --- a/crates/preloop-orchestrator/src/environment.rs +++ b/crates/preloop-orchestrator/src/environment.rs @@ -1,4 +1,17 @@ -//! Toolchain and base-image resolution for disposable job environments. +//! Base-image selection for disposable job environments. +//! +//! A job VM is forked from a golden. There are exactly two golden sources: +//! +//! * the **official** packed golden ([`OFFICIAL_GOLDEN`]) — the published +//! preloop runner image, downloaded and verified per architecture, and +//! mandatory: nothing is baked locally when it cannot be fetched; +//! * a **configured image** — whatever `PRELOOP_RUNNER_BASE_IMAGE` or +//! `[golden] base_image` names, baked into a golden as-is plus the +//! GitHub-runner machinery (see `golden_contract_script`). +//! +//! There is no third source: the stock Ubuntu bake (apt baseline, language +//! toolchains, package pins) is gone, and with it the "curated" classification +//! that used to decide between it and a custom image. use preloop_gha_protocol::oci_image_ref; use serde::{Deserialize, Serialize}; @@ -68,407 +81,119 @@ pub fn scan_workflow_images(workspace: &Path) -> Vec { images.into_iter().collect() } -/// A toolchain that must be available in a job VM. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum ToolchainLayer { - /// Node.js release (for example, `22`, `20.11.0`, or `lts/*`). - Node(String), - /// Rust channel or release (for example, `stable`, `nightly`, or `1.85.1`). - Rust(String), - /// Python release (for example, `3.12` or `3.11.8`). - Python(String), - /// Go release (for example, `1.24` or `1.23.4`). - Go(String), -} - -impl std::fmt::Display for ToolchainLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Node(version) => write!(f, "node {version}"), - Self::Rust(channel) => write!(f, "rust {channel}"), - Self::Python(version) => write!(f, "python {version}"), - Self::Go(version) => write!(f, "go {version}"), - } - } -} - -impl ToolchainLayer { - /// Return shell commands to install this toolchain in a SmolVM. - /// - /// Commands are represented as argv vectors, as expected by - /// [`preloop_vm::VmProvider::exec`]. Commands requiring a pipe are run - /// through `sh -c` so that the returned vectors remain valid argv. - pub fn install_commands(&self) -> Vec> { - match self { - Self::Node(version) => { - // Exact-version tarball install, not the apt series: a workflow - // pinning `22.23.1` gets exactly that, and a major (`22`) or - // `lts/*` request resolves against the nodejs.org release - // index at bake time (GitHub's setup-node resolves the same - // way, so this matches hosted behavior instead of floating - // with the apt archive). - // The version is interpolated into a `sh -c` script, so it - // must be allowlisted (same `safe_component` the Rust and Go - // layers use): a workflow-controlled value carrying shell - // metacharacters would execute arbitrary commands in the - // provisioning VM. - let version = safe_component(version.trim().trim_start_matches('v')); - vec![ - vec![ - "sh".into(), - "-c".into(), - format!( - "set -e\n\ - WANT=v{version}\n\ - case '{version}' in lts/*) WANT='lts/*' ;; esac\n\ - VERSION=$(curl -fsSL https://nodejs.org/dist/index.json | python3 -c '\n\ - import json, sys\n\ - want = sys.argv[1]\n\ - idx = json.load(sys.stdin)\n\ - print(next((e[\"version\"] for e in idx if (want == \"lts/*\" and e[\"lts\"]) or e[\"version\"] == want or e[\"version\"].startswith(want + \".\")), \"\"))\n\ - ' \"$WANT\")\n\ - [ -n \"$VERSION\" ] || {{ echo \"no node release matching {version}\" >&2; exit 1; }}\n\ - arch=$(uname -m)\n\ - case \"$arch\" in\n\ - x86_64) NODE_ARCH=x64 ;;\n\ - aarch64|arm64) NODE_ARCH=arm64 ;;\n\ - *) echo \"unsupported arch: $arch\" >&2; exit 1 ;;\n\ - esac\n\ - curl -fsSL \"https://nodejs.org/dist/$VERSION/node-$VERSION-linux-$NODE_ARCH.tar.gz\" \\\n\ - | tar -xz --strip-components=1 -C /usr/local\n\ - node --version" - ), - ], - ] - } - Self::Rust(channel) => vec![ - vec![ - "sh".into(), - "-c".into(), - format!( - // Pin the rustup installer itself (the `sh.rustup.rs` - // wrapper floats with every rustup release). The - // channel stays workflow-driven — `stable` resolves - // at bake time exactly as GitHub resolves it at job - // time — and the resolved version is recorded in the - // golden's bake manifest. - "set -e\n\ - arch=$(uname -m)\n\ - case \"$arch\" in\n\ - x86_64) RUST_ARCH=x86_64 ;;\n\ - aarch64|arm64) RUST_ARCH=aarch64 ;;\n\ - *) echo \"unsupported arch: $arch\" >&2; exit 1 ;;\n\ - esac\n\ - export RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo\n\ - curl -fsSL \"https://static.rust-lang.org/rustup/archive/{}/$RUST_ARCH-unknown-linux-gnu/rustup-init\" -o /tmp/rustup-init\n\ - chmod +x /tmp/rustup-init\n\ - /tmp/rustup-init -y --profile minimal --default-toolchain {} --component rustfmt,clippy\n\ - rm -f /tmp/rustup-init", - crate::RUSTUP_VERSION, - safe_component(channel) - ), - ], - vec![ - "sh".into(), - "-c".into(), - // Run steps execute with `bash --noprofile --norc`, so - // profile.d PATH exports are never sourced. The toolchain - // lives at the fixed system addresses exported by - // `guest_env_prefix` (RUSTUP_HOME / CARGO_HOME above): - // the bake runs as root while job steps run as the - // unprivileged runner user, and rustup obeys those - // variables verbatim, so a $HOME-derived location is - // simply unreachable across that boundary (/root is - // 0700). Symlink the shims into /usr/local/bin so they - // resolve on the default system PATH for every step. - "ln -sf /usr/local/cargo/bin/cargo /usr/local/bin/cargo; ln -sf /usr/local/cargo/bin/cargo-fmt /usr/local/bin/cargo-fmt; ln -sf /usr/local/cargo/bin/cargo-clippy /usr/local/bin/cargo-clippy; ln -sf /usr/local/cargo/bin/rustc /usr/local/bin/rustc; ln -sf /usr/local/cargo/bin/rustdoc /usr/local/bin/rustdoc; ln -sf /usr/local/cargo/bin/rustup /usr/local/bin/rustup".into(), - ], - ], - Self::Python(version) => { - let package = format!("python{}", version.trim()); - vec![vec![ - "apt-get".into(), - "install".into(), - "-y".into(), - package, - "python3-pip".into(), - ]] - } - Self::Go(version) => vec![ - vec![ - "sh".into(), - "-c".into(), - format!( - // `go.mod` carries a minimum (`go 1.24`), not a tarball - // version — resolve it against the go.dev release index - // so `go1.24` becomes the newest 1.24.x and the install - // is exact and reproducible. - "set -e\n\ - WANT='{}'\n\ - VERSION=$(curl -fsSL 'https://go.dev/dl/?mode=json&include=all' | python3 -c '\n\ - import json, sys\n\ - want = sys.argv[1]\n\ - if not want.startswith(\"go\"):\n \ - want = \"go\" + want\n\ - idx = json.load(sys.stdin)\n\ - print(next((e[\"version\"] for e in idx if e[\"version\"] == want or e[\"version\"].startswith(want + \".\")), \"\"))\n\ - ' \"$WANT\")\n\ - [ -n \"$VERSION\" ] || {{ echo \"no go release matching $WANT\" >&2; exit 1; }}\n\ - arch=$(uname -m)\n\ - case \"$arch\" in aarch64) arch=arm64 ;; x86_64) arch=amd64 ;; esac\n\ - curl -fsSL \"https://go.dev/dl/$VERSION.linux-$arch.tar.gz\" | tar -C /usr/local -xzf -", - safe_component(version) - ), - ], - vec![ - "sh".into(), - "-c".into(), - // The Go tarball extracts to /usr/local/go/bin, which is - // not on the default system PATH. Step shells run with - // `bash --noprofile --norc`, so profile.d exports never - // apply; symlink the binaries like the Rust layer does. - "ln -sf /usr/local/go/bin/go /usr/local/bin/go; ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt".into(), - ], - ], - } - } - - /// Binary that must exist on the default PATH once this layer is - /// installed. Verified after install so a provision interrupted mid-way - /// (or a toolchain that silently failed) fails the machine instead of - /// running the job without the tool it asked for. - pub fn verify_binary(&self) -> &'static str { - match self { - Self::Node(_) => "node", - Self::Rust(_) => "cargo", - Self::Python(_) => "python3", - Self::Go(_) => "go", - } - } - - /// Shell predicate proving this exact layer is usable. - pub fn verify_command(&self) -> String { - match self { - Self::Rust(channel) => { - let channel = safe_component(channel); - format!( - "export RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo && \ - command -v cargo >/dev/null && \ - rustup run {channel} rustc --version >/dev/null && \ - rustup run {channel} cargo-fmt --version >/dev/null && \ - rustup run {channel} cargo-clippy --version >/dev/null" - ) - } - Self::Go(_) => { - "export PATH=/usr/local/go/bin:$PATH && command -v go >/dev/null".to_owned() - } - _ => format!("command -v {} >/dev/null", self.verify_binary()), - } - } -} - -/// Toolchains missing from the stock Ubuntu base and therefore baked once -/// into the reusable golden instead of installed in every ephemeral job VM. -pub fn curated_toolchains() -> Vec { - vec![ - ToolchainLayer::Rust(crate::RUST_TOOLCHAIN_VERSION.into()), - ToolchainLayer::Go("1.26".into()), - ] -} - -/// Digest-pinned Ubuntu base images. -/// -/// The floating tags (`ubuntu:24.04`) move whenever Canonical publishes a -/// point release, so two goldens baked a month apart would differ for no -/// reason anyone recorded. These pins are the provenance: bumping one is a -/// deliberate, reviewable change. Digests are the registry manifest-list -/// digests, valid for both x86_64 and arm64 guests. -pub const UBUNTU_24_04_PIN: &str = crate::UBUNTU_24_04_BASE; -pub const UBUNTU_22_04_PIN: &str = crate::UBUNTU_22_04_BASE; -pub const OFFICIAL_RUNNER_IMAGE_AMD64_PIN: &str = crate::OFFICIAL_RUNNER_IMAGE_BASE_AMD64; -pub const OFFICIAL_RUNNER_IMAGE_ARM64_PIN: &str = crate::OFFICIAL_RUNNER_IMAGE_BASE_ARM64; - -/// The default base image for GitHub-runner-labelled jobs. -pub const DEFAULT_BASE_IMAGE: &str = UBUNTU_24_04_PIN; - -/// Guest path of the apt-index freshness marker baked into golden images. -/// Written by `prepare_artifact` after the baseline install; forks inherit -/// it through the packed rootfs (unlike post-create exec writes, which -/// snapshots do not carry). -pub const APT_INDICES_MARKER_PATH: &str = "/etc/preloop/apt-indices-baked-at"; -/// Fallback freshness policy when a marker lacks its policy line. -pub const APT_INDICES_DEFAULT_MAX_AGE_DAYS: u64 = 7; - -/// Parse an apt-index marker into `(age_days, max_age_days)`. +/// The sentinel naming the official packed golden. /// -/// The marker is `YYYY-MM-DD` on line one with an optional policy age on -/// line two. `today_days` is days since the Unix epoch. Returns `None` when -/// the marker is absent or malformed — the caller then behaves exactly as -/// before markers existed (full refresh, no warning). -pub fn apt_marker_age_days(marker: &str, today_days: u64) -> Option<(u64, u64)> { - let mut lines = marker.lines().map(str::trim); - let baked = days_from_civil_date(lines.next()?)?; - let max_age = lines - .next() - .map_or(APT_INDICES_DEFAULT_MAX_AGE_DAYS, |line| { - line.parse::() - .ok() - .filter(|days| *days > 0) - .unwrap_or(APT_INDICES_DEFAULT_MAX_AGE_DAYS) - }); - Some((today_days.saturating_sub(baked), max_age)) -} - -/// Days since the Unix epoch for a civil date, or `None` if invalid. -/// (Howard Hinnant's days_from_civil; std has no civil-date type and this -/// crate has no chrono dependency for three lines of math.) -fn days_from_civil_date(text: &str) -> Option { - let mut parts = text.split('-'); - let (year, month, day) = ( - parts.next()?.parse::().ok()?, - parts.next()?.parse::().ok()?, - parts.next()?.parse::().ok()?, - ); - if parts.next().is_some() || !(1..=12).contains(&month) { - return None; - } - let leap = month == 2 && (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); - let max_day = match month { - 2 if leap => 29, - 2 => 28, - 4 | 6 | 9 | 11 => 30, - _ => 31, - }; - if day == 0 || day > max_day { - return None; - } - let year = if month <= 2 { year - 1 } else { year }; - let era = year.div_euclid(400); - let yoe = (year - era * 400) as u64; - let mp = (month + 9) % 12; - let doy = (153 * mp + 2) / 5 + day - 1; - let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - (era as u64) - .checked_mul(146097)? - .checked_add(doe)? - .checked_sub(719468) +/// `runs-on: ubuntu-latest`/`ubuntu-24.04` resolve to this, and so does a pool +/// with no image configured. It is deliberately not an OCI reference: the +/// official golden is fetched as a packed `.smolmachine` (per architecture, +/// digest-pinned), never built from a base image, and the sentinel keeps that +/// distinct from "an operator pointed us at an image". +pub const OFFICIAL_GOLDEN: &str = "preloop-official-golden"; + +/// Whether a resolved base image names the official packed golden. +pub fn is_official_golden(base: &str) -> bool { + base == OFFICIAL_GOLDEN } -/// The plain repository:tag of an image reference, ignoring any `@digest`. -pub fn base_name(image_ref: &str) -> &str { - image_ref.split('@').next().unwrap_or(image_ref) -} - -/// Whether an image reference is one of Preloop's stock Ubuntu bases. -pub fn is_stock_base_image(image_ref: &str) -> bool { - matches!( - base_name(image_ref), - "ubuntu:24.04" - | "ubuntu:22.04" - | "mirror.gcr.io/library/ubuntu:24.04" - | "mirror.gcr.io/library/ubuntu:22.04" - ) -} - -/// Resolved base image and toolchains for one job. +/// Resolved base image for one job: a golden source plus its fingerprint. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentSpec { - /// Base OCI image or VM image identifier. + /// [`OFFICIAL_GOLDEN`] or the configured image reference. pub base: String, - /// Sorted, deduplicated toolchain layers. - pub toolchains: Vec, - /// Whether Preloop's complete package bake applies to this base. - pub curated: bool, - /// SHA-256 hex digest of the normalized base and toolchain list. + /// SHA-256 hex digest of everything the golden for this base depends on. pub fingerprint: String, } impl EnvironmentSpec { - /// Build a normalized environment specification and compute its fingerprint. - pub fn new(base: String, toolchains: Vec) -> Self { - Self::from_parts(base, toolchains, true) - } - /// Resolve the environment for a base image. /// - /// Stock Ubuntu bases receive the complete Preloop package bake. Official - /// runner snapshots and other custom images are used as-is; workflow - /// `setup-*` actions select exact language-tool versions at job time. + /// The image is used exactly as it is: no packages, toolchains, PATH or + /// environment overrides. Workflow `setup-*` actions select exact language + /// versions at job time, as they do on GitHub-hosted runners. + /// + /// The runner account comes from `PRELOOP_RUNNER_USER`/`_UID` — the same + /// variables `serve` resolves it from — because a configured image bakes + /// that account into its golden: a host that changes either must not adopt + /// the artifact baked for the previous one. Reading it here (rather than + /// only at the bake) is what keeps every caller, `preloop golden-path` + /// included, naming the same payload. pub fn for_base(base: String) -> Self { - let curated = is_stock_base_image(&base); - let toolchains = if curated { - curated_toolchains() - } else { - Vec::new() - }; - Self::from_parts(base, toolchains, curated) + Self::from_base(base) } - /// Replace the base image, recomputing the fingerprint. - pub fn with_base(mut self, base: String) -> Self { - self.base = base; - Self::from_parts(self.base.clone(), self.toolchains.clone(), self.curated) + /// Resolve the environment for a base image under an explicit account, + /// for callers that already resolved one. + pub fn for_base_with_account(base: String, user: &str, uid: u32) -> Self { + Self::from_base_with_account(base, user, uid) } /// Resolve a queued job's base image from its `runs-on` labels. /// - /// Only a label that *names* a hosted Ubuntu image selects a stock pin. - /// Everything else — `[self-hosted, preloop-cpane]`, `[self-hosted, + /// Only a label that *names* a hosted Ubuntu image selects the official + /// golden. Everything else — `[self-hosted, preloop-cpane]`, `[self-hosted, /// runner-sync]`, any private label set — keeps `configured`, the image /// this pool was built around. /// - /// Falling back to the stock pin instead is what wedged production: a - /// self-hosted-only label set resolved to stock Ubuntu, which is - /// `curated`, so every slot tried to bake the full hosted apt baseline - /// against the live Ubuntu archive. That bake fails whenever the archive - /// has moved past the pinned versions (observed: `libatk1.0-0t64`, - /// clang-16/17/18, gcc-13/14 all unlocatable), so slots burned on - /// doomed bakes and the pool served one runner instead of three while - /// the configured golden sat ready and forkable. + /// The label set used to resolve to the stock `mirror.gcr.io/library/ubuntu` + /// pin, which was "curated" and therefore sent every slot into the hosted + /// apt baseline against the live Ubuntu archive. That bake fails whenever + /// the archive has moved past the pinned versions (observed: + /// `libatk1.0-0t64`, clang-16/17/18, gcc-13/14 all unlocatable), so slots + /// burned on doomed bakes and the pool served one runner instead of three + /// while the configured golden sat ready and forkable. The official golden + /// is the hosted image, so the mapping is now direct and cannot fail that + /// way. + /// + /// `ubuntu-22.04` maps nowhere: the archive baseline that used to back it + /// was part of the deleted stock bake, so the label keeps `configured` + /// instead of silently selecting a 24.04 image. pub fn base_for_labels(runs_on: &[String], configured: &str) -> String { if runs_on.iter().any(|label| { let label = label.to_ascii_lowercase(); label.contains("ubuntu-24.04") || label.contains("ubuntu-latest") }) { - return UBUNTU_24_04_PIN.into(); - } - if runs_on - .iter() - .any(|label| label.to_ascii_lowercase().contains("ubuntu-22.04")) - { - return UBUNTU_22_04_PIN.into(); + return OFFICIAL_GOLDEN.into(); } configured.to_owned() } - fn from_parts(base: String, mut toolchains: Vec, curated: bool) -> Self { - toolchains.sort(); - toolchains.dedup(); + fn from_base(base: String) -> Self { + let (user, uid) = resolve_runner_account(); + Self::from_base_with_account(base, &user, uid) + } + + fn from_base_with_account(base: String, user: &str, uid: u32) -> Self { let normalized = serde_json::json!({ "base": &base, - "toolchains": &toolchains, - "curated": curated, // The pool only rebuilds when the fingerprint-suffixed artifact // file is missing, so bake-content changes MUST invalidate the - // fingerprint or the pool silently keeps the old golden forever - // (packages, resolv.conf, nvm, tool pins all live in the bake - // script, which interpolates the versions.toml pins). Custom - // bases skip the bake entirely, so the fingerprint records that - // too. - "bake": if curated { crate::base_install_script() } else { String::new() }, + // fingerprint or the pool silently keeps the old golden forever. + // + // The official golden is built by the release pipeline from a + // published runner image, not here: its identity is whatever + // source this host would fetch it from — the digest pinned in + // `official_golden_source`, a `PRELOOP_GOLDEN_OCI_REF` override, + // or the `PRELOOP_GOLDEN_URL` mirror, which selects a different + // pack with a different name. A configured image is baked + // locally, so its fingerprint is the bake contract itself — bump + // a step of `golden_contract_script` and every golden is rebuilt. + "golden": if is_official_golden(&base) { + crate::official_golden_source() + } else { + crate::golden_contract_script(user, uid) + }, // Rosetta x86_64 translation exists only on Apple Silicon hosts. - // The packed-golden prep installs the amd64 loader + libc into - // arm64 goldens so dynamically linked x86_64 binaries run under - // it; the fingerprint records whether that shim is present so a - // host-class change re-preps the golden once instead of adopting - // a base built for the other class. + // The golden prep installs the amd64 loader + libc so dynamically + // linked x86_64 binaries run under it; the fingerprint records + // whether that shim is present so a host-class change re-preps the + // golden once instead of adopting a base built for the other class. "rosetta_libs": cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", - // The packed artifact's baked runner resolves node through the - // bundle mount at the pins it was built against. A pin bump that - // left the fingerprint untouched would keep an artifact whose - // runner demands the previous version, so every JS action step - // fails with `bundled nodeXX is missing` against a bundle that is - // itself perfectly valid at the new pin. + // The golden's runner resolves node through the bundle mount at + // the pins it was built against. A pin bump that left the + // fingerprint untouched would keep a golden whose runner demands + // the previous version, so every JS action step fails with + // `bundled nodeXX is missing` against a bundle that is itself + // perfectly valid at the new pin. "node_externals": crate::node_externals::expected_runtimes() .iter() .map(|(runtime, version)| format!("{runtime}={version}")) @@ -477,22 +202,28 @@ impl EnvironmentSpec { let bytes = serde_json::to_vec(&normalized).expect("normalized environment is serializable"); let fingerprint = hex_digest(&bytes); - Self { - base, - toolchains, - curated, - fingerprint, - } + Self { base, fingerprint } } } -fn safe_component(value: &str) -> String { - value - .chars() - .filter(|character| { - character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_' | '/' | '*') - }) - .collect() +/// The runner account this host would run jobs as. +/// +/// Mirrors the resolution `serve` performs: `PRELOOP_RUNNER_USER` (an empty +/// value disables the switch, i.e. the guest's own user runs the runner) and +/// `PRELOOP_RUNNER_UID`, defaulting to the hosted `runner`/1001. Kept here so +/// the fingerprint — and therefore `preloop golden-path` — always names the +/// account the bake will actually install. +fn resolve_runner_account() -> (String, u32) { + let user = match std::env::var("PRELOOP_RUNNER_USER") { + Ok(value) if value.is_empty() => crate::DEFAULT_RUNNER_USER.to_owned(), + Ok(value) => value, + Err(_) => crate::DEFAULT_RUNNER_USER.to_owned(), + }; + let uid = std::env::var("PRELOOP_RUNNER_UID") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(crate::DEFAULT_RUNNER_UID); + (user, uid) } fn hex_digest(bytes: &[u8]) -> String { @@ -505,227 +236,67 @@ fn hex_digest(bytes: &[u8]) -> String { #[cfg(test)] mod tests { use super::*; - use std::collections::BTreeSet; - /// Hosted labels select a stock pin; a self-hosted label set keeps the - /// pool's own image. The last case is the production wedge: resolving - /// `[self-hosted, …]` to stock Ubuntu sent every slot into a curated - /// apt bake that the rolling archive can no longer satisfy. + /// Hosted labels select the official golden; a self-hosted label set keeps + /// the pool's own image. The last case is the production wedge described + /// on `base_for_labels`: resolving `[self-hosted, …]` to a stock Ubuntu pin + /// sent every slot into a curated apt bake that the rolling archive can no + /// longer satisfy. #[test] - fn labels_select_hosted_pins_and_otherwise_keep_the_configured_image() { - const CONFIGURED: &str = "ghcr.io/preloopdev/runner-images:ubuntu24-runner-large-latest@sha256:4f7e4be438e4eb9c0f23bebdec12cf1d25520876ad2052412ba18580919b7795"; + fn labels_select_the_official_golden_and_otherwise_keep_the_configured_image() { + const CONFIGURED: &str = "ghcr.io/acme/runner-image:ubuntu24"; assert_eq!( EnvironmentSpec::base_for_labels(&["ubuntu-latest".into()], CONFIGURED), - UBUNTU_24_04_PIN + OFFICIAL_GOLDEN ); assert_eq!( EnvironmentSpec::base_for_labels(&["ubuntu-24.04".into()], CONFIGURED), - UBUNTU_24_04_PIN + OFFICIAL_GOLDEN ); + // No image configured: every label resolves to the official golden. assert_eq!( - EnvironmentSpec::base_for_labels(&["ubuntu-22.04".into()], CONFIGURED), - UBUNTU_22_04_PIN + EnvironmentSpec::base_for_labels(&["self-hosted".into()], OFFICIAL_GOLDEN), + OFFICIAL_GOLDEN ); for labels in [ vec!["self-hosted".to_owned()], vec!["self-hosted".to_owned(), "preloop-cpane".to_owned()], vec!["self-hosted".to_owned(), "runner-sync".to_owned()], + // The 22.04 archive baseline went with the stock bake: the label + // must keep the configured image rather than select a 24.04 one. + vec!["ubuntu-22.04".to_owned()], ] { assert_eq!( EnvironmentSpec::base_for_labels(&labels, CONFIGURED), CONFIGURED, - "{labels:?} must not be rebased onto a stock pin" - ); - } - } - - #[test] - fn stock_base_pins_use_explicit_mirror_and_digest() { - for pin in [UBUNTU_24_04_PIN, UBUNTU_22_04_PIN] { - let digest = pin - .strip_prefix("mirror.gcr.io/library/ubuntu:") - .and_then(|reference| reference.split_once("@sha256:")) - .map(|(_, digest)| digest) - .unwrap_or_else(|| { - panic!( - "stock image must use the explicit mirror.gcr.io registry and a sha256 digest: {pin}" - ) - }); - assert_eq!(digest.len(), 64, "sha256 digest has the wrong length"); - assert!( - digest.bytes().all(|byte| byte.is_ascii_hexdigit()), - "sha256 digest is not hexadecimal: {digest}" + "{labels:?} must not be rebased onto another image" ); } } #[test] - fn base_name_strips_digest() { - assert_eq!(base_name("ubuntu:24.04@sha256:abc"), "ubuntu:24.04"); - assert_eq!(base_name("ubuntu:24.04"), "ubuntu:24.04"); - assert_eq!(base_name(""), ""); - } - - #[test] - fn stock_base_detection_ignores_digest_but_rejects_custom_images() { + fn official_golden_is_the_only_sentinel() { + assert!(is_official_golden(OFFICIAL_GOLDEN)); for image in [ + "official", + "preloop-official-golden@sha256:abc", + "ghcr.io/preloopdev/preloop-arm64-smolvm-golden@sha256:abc", "ubuntu:24.04", - "ubuntu:24.04@sha256:abc", - "mirror.gcr.io/library/ubuntu:22.04@sha256:def", - ] { - assert!(is_stock_base_image(image), "{image}"); - } - for image in [ - "ghcr.io/acme/runner-image:ubuntu24", - "ghcr.io/christopherhx/runner-images:ubuntu24-runner-large-latest-arm64", - "/tmp/custom.smolmachine", ] { - assert!(!is_stock_base_image(image), "{image}"); + assert!(!is_official_golden(image), "{image}"); } } + /// Two configured images must not share a fingerprint, and the same image + /// must resolve to the same fingerprint on every call. #[test] - fn node_layer_install_uses_pinned_tarball() { - // The Node layer must never resolve through apt: exact versions are - // installed verbatim and major/lts requests resolve via the nodejs.org - // index, never the floating apt series. - let commands = ToolchainLayer::Node("20.11.0".into()).install_commands(); - let script = commands[0].join(" "); - assert!(script.contains("nodejs.org/dist/index.json")); - assert!(script.contains("node-$VERSION-linux-$NODE_ARCH.tar.gz")); - assert!(!script.contains("nodesource")); - assert!(!script.contains("apt-get")); - } - - #[test] - fn apt_marker_reports_age_against_baked_policy() { - // 2026-09-16 is day 20712; keep the arithmetic anchored to fixed - // values rather than the wall clock so the test never rots. - const TODAY: u64 = 20712; - assert_eq!( - apt_marker_age_days("2026-09-09\n7\n", TODAY), - Some((7, 7)), - "exactly at policy reads fresh-or-stale by strict greater-than" - ); - assert_eq!(apt_marker_age_days("2026-09-08\n7\n", TODAY), Some((8, 7))); - assert_eq!( - apt_marker_age_days("2026-09-16\n", TODAY), - Some((0, APT_INDICES_DEFAULT_MAX_AGE_DAYS)), - "missing policy line falls back to the default" - ); - assert_eq!( - apt_marker_age_days("2026-09-16\n0\n", TODAY), - Some((0, APT_INDICES_DEFAULT_MAX_AGE_DAYS)), - "zero policy is garbage, not instant-stale" - ); - } - - #[test] - fn apt_marker_rejects_impossible_dates() { - const TODAY: u64 = 20712; - for bad in [ - "", - "yesterday\n7\n", - "2026-13-01\n7\n", - "2026-02-30\n7\n", - "2023-02-29\n7\n", - ] { - assert_eq!(apt_marker_age_days(bad, TODAY), None, "{bad:?}"); - } - // Leap day parses; future bakes saturate at age zero. - assert_eq!( - apt_marker_age_days("2024-02-29\n7\n", TODAY), - Some((TODAY - 19782, 7)) - ); - assert_eq!(apt_marker_age_days("2026-09-20\n7\n", TODAY), Some((0, 7))); - } - - #[test] - fn node_layer_version_rejects_shell_metacharacters() { - // A workflow-controlled `node-version` is interpolated into the - // `sh -c` provisioning script (`WANT=v{version}`, the case pattern, - // and the error message), so shell metacharacters would execute - // arbitrary commands inside the bake VM. The version must be - // allowlisted through the same `safe_component` used by the Rust and - // Go layers. - let payload = "22; touch /tmp/pwned; #"; - let commands = ToolchainLayer::Node(payload.into()).install_commands(); - let script = commands[0].join(" "); - // The raw payload must not survive into the script... - assert!(!script.contains("touch /tmp/pwned")); - assert!(!script.contains("22;")); - // ...and the sanitized version is still interpolated everywhere the - // original was (WANT, case, error message), never silently dropped. - assert!(script.contains("WANT=v22touch/tmp/pwned")); - assert!(script.contains("no node release matching 22touch/tmp/pwned")); - } - - #[test] - fn rust_layer_install_uses_pinned_rustup() { - let commands = ToolchainLayer::Rust("stable".into()).install_commands(); - let script = commands[0].join(" "); - assert!(script.contains(&format!( - "static.rust-lang.org/rustup/archive/{}", - crate::RUSTUP_VERSION - ))); - assert!(script.contains("--profile minimal")); - assert!(script.contains("--default-toolchain stable")); - assert!(!script.contains("sh.rustup.rs")); - } - - /// `cargo test --workspace` spawns `rustdoc` by bare name for doctests, so - /// the interpreter must be on the default PATH of a `bash --noprofile - /// --norc` step shell. Without the symlink the workspace test step fails - /// with `could not execute process rustdoc` after every real test passed. - #[test] - fn rust_layer_puts_rustdoc_on_default_path() { - let commands = ToolchainLayer::Rust("stable".into()).install_commands(); - let script = commands[1].join(" "); - for binary in [ - "cargo", - "cargo-fmt", - "cargo-clippy", - "rustc", - "rustdoc", - "rustup", - ] { - assert!( - script.contains(&format!("/usr/local/bin/{binary}")), - "{binary} must be linked onto the default PATH: {script}" - ); - } - } - - #[test] - fn go_layer_install_resolves_minimum_version() { - let commands = ToolchainLayer::Go("1.24".into()).install_commands(); - let script = commands[0].join(" "); - assert!(script.contains("go.dev/dl/?mode=json")); - assert!(script.contains("$VERSION.linux")); - assert!(script.contains("tar -C /usr/local -xzf -")); - assert!(!script.contains("go1.24.linux")); // never a raw minimum - } - - #[test] - fn go_layer_puts_binary_on_default_path() { - // The Go tarball extracts to /usr/local/go/bin, which is not on the - // default PATH of `bash --noprofile --norc` step shells, so `go` and - // `gofmt` would be unresolvable in job steps without the symlinks. - let commands = ToolchainLayer::Go("1.24".into()).install_commands(); - assert_eq!(commands.len(), 2); - let script = commands[1].join(" "); - assert!(script.contains("ln -sf /usr/local/go/bin/go /usr/local/bin/go")); - assert!(script.contains("ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt")); - } - - #[test] - fn with_base_recomputes_fingerprint() { - let spec = EnvironmentSpec::new("ubuntu:24.04".into(), curated_toolchains()); - let original_fingerprint = spec.fingerprint.clone(); - let rebased = spec.clone().with_base("ubuntu:22.04".into()); - assert_eq!(rebased.base, "ubuntu:22.04"); - assert_eq!(rebased.toolchains, spec.toolchains); - assert_ne!(rebased.fingerprint, original_fingerprint); + fn configured_images_fingerprint_deterministically_per_image() { + let first = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".into()); + let second = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".into()); + let other = EnvironmentSpec::for_base("ghcr.io/acme/runner:2".into()); + assert_eq!(first, second); + assert_ne!(first.fingerprint, other.fingerprint); + assert_eq!(first.base, "ghcr.io/acme/runner:latest"); } /// The artifact filename is keyed by this fingerprint, and the artifact's @@ -734,7 +305,7 @@ mod tests { /// serving an artifact whose runner demands a version no longer shipped. #[test] fn fingerprint_covers_the_node_externals_pins() { - let spec = EnvironmentSpec::for_base("ubuntu:24.04".into()); + let spec = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".into()); let pins: Vec = crate::node_externals::expected_runtimes() .iter() .map(|(runtime, version)| format!("{runtime}={version}")) @@ -744,10 +315,11 @@ mod tests { // Recompute the digest with a bumped pin: it must not collide. let bumped: Vec = pins.iter().map(|pin| format!("{pin}-bumped")).collect(); let normalized = serde_json::json!({ - "base": "ubuntu:24.04", - "toolchains": &spec.toolchains, - "curated": spec.curated, - "bake": if spec.curated { crate::base_install_script() } else { String::new() }, + "base": "ghcr.io/acme/runner:latest", + "golden": crate::golden_contract_script( + crate::DEFAULT_RUNNER_USER, + crate::DEFAULT_RUNNER_UID + ), "rosetta_libs": cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", "node_externals": bumped, }); @@ -758,141 +330,92 @@ mod tests { ); } - /// Stock bases get Preloop's complete bake, while official runner - /// snapshots and arbitrary custom images stay untouched. + /// The runner account is part of a configured image's bake contract: the + /// contract text embeds it, so changing `PRELOOP_RUNNER_USER`/`_UID` must + /// produce a different artifact name instead of adopting the golden baked + /// for the previous account. #[test] - fn for_base_selects_only_required_layers() { - let stock = EnvironmentSpec::for_base(UBUNTU_24_04_PIN.to_owned()); - assert!(stock.curated, "stock bases must carry the curated bake"); - assert!(!stock.toolchains.is_empty()); - - let official = EnvironmentSpec::for_base(OFFICIAL_RUNNER_IMAGE_AMD64_PIN.to_owned()); - assert!( - !official.curated, - "official images already contain packages" - ); - assert!( - official.toolchains.is_empty(), - "official runner images must resolve workflow toolchains through setup actions" - ); - - let custom = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".to_owned()); - assert!(!custom.curated, "custom bases must not be curated"); - assert!( - custom.toolchains.is_empty(), - "custom bases must run without the curated toolchain layer" - ); - assert_ne!( - stock.fingerprint, custom.fingerprint, - "the bake decision must invalidate the golden fingerprint" + fn fingerprint_tracks_the_configured_runner_account() { + let base = "ghcr.io/acme/runner:latest".to_owned(); + let default = EnvironmentSpec::for_base_with_account( + base.clone(), + crate::DEFAULT_RUNNER_USER, + crate::DEFAULT_RUNNER_UID, ); + let other_user = EnvironmentSpec::for_base_with_account(base.clone(), "builder", 1001); + let other_uid = EnvironmentSpec::for_base_with_account(base, "runner", 2000); - assert!(EnvironmentSpec::for_base(UBUNTU_22_04_PIN.to_owned()).curated); + assert_ne!(default.fingerprint, other_user.fingerprint); + assert_ne!(default.fingerprint, other_uid.fingerprint); + assert_ne!(other_user.fingerprint, other_uid.fingerprint); + assert_eq!(default.base, "ghcr.io/acme/runner:latest"); } + /// The official golden is keyed by *where this host fetches it from*: the + /// pinned digest, the `PRELOOP_GOLDEN_OCI_REF` override, or the mirror URL. + /// A changed mirror must not keep serving the pack the previous one left + /// at the payload path, and it must not be keyed on the bake contract + /// either (the release pipeline bakes it, so a contract edit cannot move + /// the published artifact). #[test] - fn curated_toolchains_is_fixed_and_deduped() { - let first = curated_toolchains(); - let second = curated_toolchains(); - assert_eq!(first, second, "the curated set must be deterministic"); - assert_eq!( - first.iter().collect::>().len(), - first.len(), - "no duplicate toolchains" - ); - // The repository CI pin is compiled from versions.toml and baked into - // stock goldens; changing it therefore changes the environment - // fingerprint and forces a rebuild. - assert!(first.contains(&ToolchainLayer::Rust(crate::RUST_TOOLCHAIN_VERSION.into()))); - assert!(first.contains(&ToolchainLayer::Go("1.26".into()))); - } - - #[test] - fn install_commands_are_non_empty() { - for layer in [ - ToolchainLayer::Node("22".into()), - ToolchainLayer::Rust("stable".into()), - ToolchainLayer::Python("3.12".into()), - ToolchainLayer::Go("1.24".into()), - ] { - assert!(!layer.install_commands().is_empty()); - } - } + fn fingerprint_tracks_the_official_golden_source() { + let official = EnvironmentSpec::for_base(OFFICIAL_GOLDEN.into()); + let oci_source = format!("oci:{}", crate::official_golden_reference()); + let expected = |golden: &str| { + let normalized = serde_json::json!({ + "base": OFFICIAL_GOLDEN, + "golden": golden, + "rosetta_libs": cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", + "node_externals": crate::node_externals::expected_runtimes() + .iter() + .map(|(runtime, version)| format!("{runtime}={version}")) + .collect::>(), + }); + hex_digest(&serde_json::to_vec(&normalized).unwrap()) + }; - /// Rust is baked into fixed system homes, while bare provider `exec` - /// commands run as root with HOME=/root. Verification must select the same - /// homes as installation and job execution or it falsely reports the - /// installed toolchain missing, reinstalls it, then fails the same probe. - #[test] - fn rust_verification_uses_the_baked_system_homes() { - let command = ToolchainLayer::Rust("1.97".into()).verify_command(); - assert!( - command.starts_with( - "export RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo && " - ), - "verification must resolve the baked system homes, not $HOME: {command}" + assert_eq!(official.fingerprint, expected(&oci_source)); + assert_ne!( + official.fingerprint, + expected("https://mirror.example/new-pack.smolmachine"), + "a mirror switch must not reuse the pack the previous source left behind" ); - assert!(command.contains("rustup run 1.97 rustc --version")); + assert_ne!( + official.fingerprint, + expected(&crate::golden_contract_script( + crate::DEFAULT_RUNNER_USER, + crate::DEFAULT_RUNNER_UID + )), + "the official golden is published, not baked here" + ); + // The in-guest bake a non-pack backend performs boots this image, so + // it must be a real digest-pinned reference — the sentinel itself is + // not an image any backend can start. + match crate::official_boot_image() { + Some(image) => assert!( + image.contains("@sha256:"), + "a non-pack backend can only boot a digest-pinned image: {image}" + ), + None => assert!( + !matches!(std::env::consts::ARCH, "aarch64" | "x86_64"), + "every supported architecture must have a pinned runner image" + ), + } } - /// The Go layer emits an inline Python resolver whose body must survive - /// the Rust line-continuation string. A `\n\` continuation strips the - /// leading whitespace of the following source line, which used to flatten - /// the `if` body to column zero — the emitted script then died with - /// `IndentationError` on every provisioning run and no runner ever - /// registered. Run the actual emitted Python against a sample release - /// index so a regression fails the suite instead of the pool. #[test] - fn go_toolchain_python_resolver_is_valid_and_resolves() { - let commands = ToolchainLayer::Go("1.24".into()).install_commands(); - let shell = &commands[0][2]; - assert!( - shell.contains("python3 -c"), - "resolver must be inline python" - ); - - // Extract the python -c program: everything between the single-quoted - // `python3 -c '` and the closing `' "$WANT"`. - let start = shell.find("python3 -c '").expect("inline python") + "python3 -c '".len(); - let end = shell[start..] - .find("' \"$WANT\"") - .map(|offset| start + offset) - .expect("closing quote"); - let program = &shell[start..end]; - - // The emitted shell wraps the python in its own quoting; decode the - // `\\n` -> `\n` and `\"` -> `"` escapes the Rust string introduced. - let program = program.replace("\\n", "\n").replace("\\\"", "\""); - - // The program reads JSON from stdin and prints the matching version. - // go.dev serves the index newest-first, which is what makes `next` - // resolve a `1.24` minimum to the newest 1.24.x. - let index = r#"[{"version":"go1.25.0"},{"version":"go1.24.2"},{"version":"go1.24.1"},{"version":"go1.24.0"}]"#; - let mut child = std::process::Command::new("python3") - .arg("-c") - .arg(&program) - .arg("1.24") - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .spawn() - .expect("python3 available for the test"); - use std::io::Write; - child - .stdin - .take() - .expect("stdin") - .write_all(index.as_bytes()) - .unwrap(); - let output = child.wait_with_output().unwrap(); - assert!( - output.status.success(), - "emitted python must parse and run: {}", - String::from_utf8_lossy(&output.stderr) - ); - let resolved = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - assert_eq!( - resolved, "go1.24.2", - "a `go 1.24` minimum must resolve to the newest 1.24.x" - ); + fn workflow_scan_reads_container_images_from_yaml() { + let root = std::env::temp_dir().join(format!("env-scan-{}", std::process::id())); + let workflows = root.join(".github/workflows"); + std::fs::create_dir_all(&workflows).unwrap(); + std::fs::write( + workflows.join("ci.yml"), + "jobs:\n test:\n runs-on: ubuntu-latest\n container:\n image: node:22\n services:\n db:\n image: postgres:16\n", + ) + .unwrap(); + std::fs::write(workflows.join("broken.yml"), "jobs: [: not yaml").unwrap(); + let images = scan_workflow_images(&root); + assert_eq!(images, vec!["node:22".to_owned(), "postgres:16".to_owned()]); + let _ = std::fs::remove_dir_all(&root); } } diff --git a/crates/preloop-orchestrator/src/lib.rs b/crates/preloop-orchestrator/src/lib.rs index 24e309097..24fd8d420 100644 --- a/crates/preloop-orchestrator/src/lib.rs +++ b/crates/preloop-orchestrator/src/lib.rs @@ -7,10 +7,7 @@ mod keys; pub mod node_externals; pub mod oci; -use crate::environment::{ - APT_INDICES_MARKER_PATH, EnvironmentSpec, ToolchainLayer, curated_toolchains, - is_stock_base_image, -}; +use crate::environment::{EnvironmentSpec, is_official_golden}; use crate::keys::{KeyPool, StagedKey}; use crate::oci::{MANIFEST_ACCEPT, OciManifest, OciReference, get_manifest, is_packed_vm_layer}; use preloop_gha_protocol::RUNNER_BUSY_SENTINEL; @@ -30,7 +27,7 @@ use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use thiserror::Error; use tokio::io::AsyncWriteExt as _; @@ -41,21 +38,6 @@ use tracing::{debug, error, info, warn}; const GUEST_CONTROL_DIR: &str = "/run/preloop-control"; const GUEST_CONTROL_SOCKET: &str = "/run/preloop-control/engine.sock"; -/// Rust toolchain homes inside the guest. -/// -/// rustup obeys `RUSTUP_HOME`/`CARGO_HOME` verbatim — no fallback to `$HOME`, -/// no search for a writable candidate — so these fixed system addresses are a -/// contract, not a hint. Every party MUST agree: the bake installs here -/// (`ToolchainLayer::Rust`), `runner_account_script` chowns them to the runner -/// uid, `guest_env_prefix` exports them, `guest_runner_path` puts -/// `$CARGO_HOME/bin` on PATH, and `verify_toolchain_homes` refuses to register -/// a runner whose golden disagrees. A `$HOME`-derived location instead would -/// split root's copy from the runner's: `/root` is 0700, so the runner gets -/// EACCES statting it, and a second writable home silently shadows the baked -/// toolchain with a fresh `stable` download. -const GUEST_RUSTUP_HOME: &str = "/usr/local/rustup"; -const GUEST_CARGO_HOME: &str = "/usr/local/cargo"; - const GUEST_FAILURE_MARKER: &str = "/home/runner/.preloop-job-failed"; /// Written by the worker while a job is paused in a debug session and removed /// when the session closes. The pool probes it to release the slot's @@ -518,8 +500,7 @@ fn default_golden_url(release_version: &str) -> String { /// /// The golden is only attached to releases that baked one, so an engine whose /// own release predates the artifact (or whose release has only the `.sig` -/// sidecar) otherwise 404s and falls back to a local build even though a -/// published golden exists. +/// sidecar) otherwise 404s even though a published golden exists. fn latest_golden_url() -> String { format!( "https://github.com/preloopdev/preloop/releases/latest/download/preloop-ubuntu-24.04-{}", @@ -1012,8 +993,9 @@ async fn reconcile_orphans( /// GitHub Release assets are capped at 2 GiB; `PRELOOP_GOLDEN_URL` selects /// a custom host when one is available. /// -/// An architecture with no published packed golden returns `None`; the -/// engine then falls back to the release-asset path and a local bake. +/// An architecture with no published packed golden returns `None`, and the +/// engine then has no official golden for this host at all: the download +/// fails and the caller reports it. There is no local bake to fall back to. fn default_golden_oci_ref() -> Option<&'static str> { match std::env::consts::ARCH { "aarch64" => Some(GOLDEN_OCI_REF_ARM64), @@ -1034,10 +1016,9 @@ const GOLDEN_OCI_REF_X86_64: &str = "ghcr.io/preloopdev/preloop-x86_64-smolvm-go /// /// The packed golden runs to ~9.6 GB, so this budget is really a floor on /// link speed rather than a formality: finishing inside an hour needs ~21 -/// Mbps sustained. The original 10-minute budget demanded 128 Mbps, which -/// an ordinary connection cannot serve — it killed the transfer around -/// two-thirds through and fell back to a local bake that looked like the -/// artifact was missing. +/// Mbps sustained. The original 10-minute budget demanded 128 Mbps, which an +/// ordinary connection cannot serve — it killed the transfer around two-thirds +/// through, and the artifact then read as missing. const GOLDEN_DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(60 * 60); /// Transfer attempts for one golden source before it is abandoned. /// @@ -1176,8 +1157,55 @@ fn ensure_golden_download_space( }) } -fn should_download_prebaked_golden(base_image: &str, custom_golden_url: bool) -> bool { - is_stock_base_image(base_image) || custom_golden_url +/// The packed official golden this host will fetch, honouring +/// `PRELOOP_GOLDEN_OCI_REF`. +/// +/// Also the official environment's fingerprint input (see +/// `EnvironmentSpec::from_base`): a newly published golden moves this +/// reference, so the pool re-downloads instead of reusing the previous pack. +pub(crate) fn official_golden_reference() -> String { + std::env::var("PRELOOP_GOLDEN_OCI_REF") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| default_golden_oci_ref().map(str::to_owned)) + .unwrap_or_default() +} + +/// The source the official golden would be fetched from on this host: the +/// `PRELOOP_GOLDEN_URL` mirror when one is set, else +/// [`official_golden_reference`]. +/// +/// Also the official environment's fingerprint input (see +/// `EnvironmentSpec::from_base`): it decides both which pack lands at the +/// payload path and whether a pack is fetched at all, so a mirror flip or a +/// newly published golden must move the fingerprint — `ensure_golden_payload` +/// returns immediately for an existing file and would otherwise serve the +/// previous pack forever. +pub(crate) fn official_golden_source() -> String { + golden_url_override().unwrap_or_else(|| format!("oci:{}", official_golden_reference())) +} + +/// `PRELOOP_GOLDEN_URL` when it names a mirror. An exported-but-blank value +/// behaves like an unset one, exactly as [`download_prebaked_golden_with_space`] +/// resolves it. +fn golden_url_override() -> Option { + std::env::var("PRELOOP_GOLDEN_URL") + .ok() + .filter(|value| !value.trim().is_empty()) +} + +/// The official runner image a non-file-pack backend boots to build its +/// golden in-guest, honouring neither `PRELOOP_GOLDEN_OCI_REF` nor +/// `PRELOOP_GOLDEN_URL`: those name a *packed* artifact, which only a backend +/// that restores packs can consume. +/// +/// `None` on an architecture with no published runner image. +pub(crate) fn official_boot_image() -> Option<&'static str> { + match std::env::consts::ARCH { + "aarch64" => Some(OFFICIAL_RUNNER_IMAGE_BASE_ARM64), + "x86_64" => Some(OFFICIAL_RUNNER_IMAGE_BASE_AMD64), + _ => None, + } } async fn download_prebaked_golden( @@ -1234,13 +1262,21 @@ async fn download_prebaked_golden_with_space( } else { None }; + let mirror = forced_url.is_some(); for url in golden_url_candidates(release_version, resolved_tag.as_deref(), forced_url) { info!( url = %url, target = %payload.display(), "Downloading pre-baked golden from release asset (this may take several minutes)" ); - if download_release_asset(&client, &url, payload, available_space).await? { + // A forced mirror's bytes become the golden *without* any other gate, + // so they must be verified: an operator pointing `PRELOOP_GOLDEN_URL` + // at a mirror is trusting that host, and an unverified payload turns a + // compromised mirror into every job's image. The published + // `linux-amd64`/`linux-arm64` assets we release ourselves are the only + // source allowed to skip verification (a missing sidecar there is an + // accident, and refusing would strand every host that updates first). + if download_release_asset(&client, &url, payload, available_space, mirror).await? { return Ok(true); } } @@ -1256,7 +1292,47 @@ async fn download_release_asset( url: &str, payload: &Path, available_space: AvailableSpace, + require_checksum: bool, ) -> Result { + // Fetch the companion checksum before committing bandwidth to the body. + // A truncated or corrupted golden only fails much later, when a VM tries + // to boot it, so a mismatch must be caught here. `PRELOOP_GOLDEN_SHA256` + // pins the digest for hosts whose mirror publishes no sidecar (the + // exported-but-blank rule applies here too). Releases that do not publish + // a checksum are tolerated for our own assets with a warning — the + // download is still the best path when the alternative is a full local + // build — but a forced mirror must be verifiable, see the caller. + let pinned_sha256 = std::env::var("PRELOOP_GOLDEN_SHA256") + .ok() + .filter(|value| !value.trim().is_empty()) + .and_then(|value| parse_sha256_checksum(&value)); + let expected_sha256 = match pinned_sha256 { + Some(sha256) => Some(sha256), + None => match tokio::time::timeout( + Duration::from_secs(15), + client.get(format!("{url}.sha256")).send(), + ) + .await + { + Ok(Ok(res)) if res.status().is_success() => match res.text().await { + Ok(text) => parse_sha256_checksum(&text), + Err(_) => None, + }, + _ => None, + }, + }; + if expected_sha256.is_none() { + if require_checksum { + warn!( + url = %url, + "PRELOOP_GOLDEN_URL is set but the mirror publishes no checksum: refusing an \ + unverified payload. Publish `.sha256` beside it, or pin the digest with \ + PRELOOP_GOLDEN_SHA256" + ); + return Ok(false); + } + warn!(url = %format!("{url}.sha256"), "no golden checksum published; downloading without verification"); + } let probe = match client.get(url).send().await { Ok(response) if response.status().is_success() => response, Ok(response) => { @@ -1275,27 +1351,6 @@ async fn download_release_asset( let total_bytes = probe.content_length(); ensure_golden_download_space(payload, total_bytes, available_space)?; - // Fetch the companion checksum before committing bandwidth to the body. - // A truncated or corrupted golden only fails much later, when a VM tries - // to boot it, so a mismatch must be caught here. Releases that do not - // publish a checksum are tolerated with a warning (the download is still - // the best path when the alternative is a full local build). - let expected_sha256 = match tokio::time::timeout( - Duration::from_secs(15), - client.get(format!("{url}.sha256")).send(), - ) - .await - { - Ok(Ok(res)) if res.status().is_success() => match res.text().await { - Ok(text) => parse_sha256_checksum(&text), - Err(_) => None, - }, - _ => None, - }; - if expected_sha256.is_none() { - warn!(url = %format!("{url}.sha256"), "no golden checksum published; downloading without verification"); - } - let partial = golden_partial_path(payload); let downloaded_bytes = { let client = client.clone(); @@ -1678,76 +1733,6 @@ fn parse_sha256_checksum(text: &str) -> Option { None } } -/// Packages common to Ubuntu 22.04 and 24.04 environment goldens. -/// -/// Tracks the apt baseline of GitHub's hosted Ubuntu images. ABI-transition -/// packages are selected separately: Jammy uses the original names while -/// Noble renamed them with the `t64` suffix. -const BASE_PACKAGES: &str = "\ - git curl wget ca-certificates gnupg2 sudo openssh-client \ - libnspr4 libnss3 libcairo2 libdbus-1-3 libdrm2 libgbm1 \ - libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 \ - libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 \ - ruby ruby-rubygems perl cpanminus lsb-release fonts-noto-color-emoji \ - haveged mediainfo p7zip-rar pollinate sshpass telnet tk xvfb zsync ftp \ - sphinxsearch systemd-coredump libnss3-tools software-properties-common \ - build-essential pkg-config libssl-dev make autoconf automake libtool m4 \ - bison flex texinfo patchelf swig dpkg-dev fakeroot binutils lld \ - libicu-dev libsqlite3-dev libyaml-dev \ - python3 python3-pip python-is-python3 \ - unzip zip xz-utils zstd bzip2 brotli lz4 pigz p7zip-full tar \ - jq file tree shellcheck parallel time acl locales tzdata \ - rsync dnsutils iputils-ping net-tools iproute2 netcat-openbsd \ - sqlite3 rpm aria2 mercurial libcurl4-openssl-dev zlib1g-dev gettext \ - libexpat1-dev"; - -const BASE_PACKAGES_22_04: &str = - "libatk1.0-0 libatk-bridge2.0-0 libatspi2.0-0 libcups2 libglib2.0-0 libasound2"; -const BASE_PACKAGES_24_04: &str = "\ - libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 \ - libcups2t64 libglib2.0-0t64 libasound2t64"; - -/// Node.js baked into the base image, pinned (via `versions.toml`) to the -/// GitHub-hosted ubuntu-24.04 system Node. Ubuntu's apt `nodejs` (18.19) is -/// deliberately *not* installed: workflows written against hosted runners -/// assume a modern Node on PATH, and the apt series floats with the archive. -pub const BASE_NODE_VERSION: &str = crate::NODE_VERSION; - -/// Container engine, installed separately from [`BASE_PACKAGES`]. -/// -/// Installed from Docker's official apt repository (not Ubuntu's `docker.io` -/// package): the runner needs parity with the `ubuntu-latest` container -/// stack, and the official packages ship `dockerd`, the CLI, and the -/// buildx/compose plugins as first-class artifacts. -/// -/// Kept apart because it needs storage configuration the other packages do not -/// — see [`DOCKER_DATA_ROOT`]. -/// Docker's apt repository supplies the service unit and runtime dependencies. -/// Its retained package set floats, so the bake overlays the exact official -/// image engine/CLI and plugin binaries afterward. -fn docker_apt_packages() -> String { - "docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin".to_owned() -} - -/// Compiler families preinstalled by GitHub's Ubuntu 24.04 image. -/// -/// Clang includes the compiler, formatter, and tidy tools for each version. -/// GNU C, C++, and Fortran are all present for 12, 13, and 14. -fn compiler_apt_packages() -> String { - let mut packages = Vec::new(); - for version in CLANG_VERSIONS.split_whitespace() { - for package in ["clang", "clang-format", "clang-tidy"] { - packages.push(format!("{package}-{version}")); - } - } - for version in GCC_VERSIONS.split_whitespace() { - for package in ["gcc", "g++", "gfortran"] { - packages.push(format!("{package}-{version}")); - } - } - packages.join(" ") -} - /// Where the container engine stores images and layers. /// /// Must be a real filesystem, not the guest's overlayfs root. containerd's @@ -1764,7 +1749,6 @@ fn compiler_apt_packages() -> String { /// layer; the failure only shows up in a fork. const DOCKER_DATA_ROOT: &str = "/storage/docker"; -/// Standard loopback entries for `/etc/hosts`. /// Runner root inside the guest. Must match the `--runner-root` argument /// passed to configure at provision time. /// Lives under the runner user's home, matching the GitHub-hosted layout @@ -1847,161 +1831,6 @@ const GUEST_NOFILE_ULIMIT: &str = GOLDEN_NOFILE_ULIMIT_RAISE; /// strict form above and fail loudly. const GUEST_NOFILE_ULIMIT_BEST_EFFORT: &str = GOLDEN_NOFILE_ULIMIT_RAISE_BEST_EFFORT; -/// Standard loopback entries for `/etc/hosts`. -/// -/// The base image ships an **empty** `/etc/hosts`, and `nsswitch.conf` is -/// `hosts: files dns` so `localhost` falls through to the upstream resolver -/// and fails to resolve at all. Everything still works over `127.0.0.1`, which -/// is why this hides so well. -/// -/// It breaks a large share of real workflows: `services:` containers are -/// reached at `localhost:`, and most test suites connect to `localhost` -/// by name. GitHub's runners resolve it, so a workflow that depends on it is -/// correct — the gap is ours. -const LOOPBACK_HOSTS: &str = "127.0.0.1 localhost\\n\ - ::1 localhost ip6-localhost ip6-loopback\\n\ - fe00::0 ip6-localnet\\n\ - ff00::0 ip6-mcastprefix\\n\ - ff02::1 ip6-allnodes\\n\ - ff02::2 ip6-allrouters\\n"; - -/// The golden's apt baseline, every package version-pinned (versions.toml). -/// Versions marked EXACT there match the official ubuntu-24.04 runner image. -fn base_packages_pinned() -> String { - format!( - "git={APT_GIT} \ - curl={APT_CURL} \ - wget={APT_WGET} \ - ca-certificates={APT_CA_CERTIFICATES} \ - gnupg2={APT_GNUPG2} \ - sudo={APT_SUDO} \ - openssh-client={APT_OPENSSH_CLIENT} \ - libnspr4={APT_LIBNSPR4} \ - libnss3={APT_LIBNSS3} \ - libatk1.0-0t64={APT_LIBATK1} \ - libatk-bridge2.0-0t64={APT_LIBATK_BRIDGE} \ - libatspi2.0-0t64={APT_LIBATSPI} \ - libcairo2={APT_LIBCAIRO2} \ - libcups2t64={APT_LIBCUPS2T64} \ - libdbus-1-3={APT_LIBDBUS_1_3} \ - libdrm2={APT_LIBDRM2} \ - libgbm1={APT_LIBGBM1} \ - libglib2.0-0t64={APT_LIBGLIB2} \ - libpango-1.0-0={APT_LIBPANGO} \ - libx11-6={APT_LIBX11_6} \ - libxcb1={APT_LIBXCB1} \ - libxcomposite1={APT_LIBXCOMPOSITE1} \ - libxdamage1={APT_LIBXDAMAGE1} \ - libxext6={APT_LIBXEXT6} \ - libxfixes3={APT_LIBXFIXES3} \ - libxkbcommon0={APT_LIBXKBCOMMON0} \ - libxrandr2={APT_LIBXRANDR2} \ - libasound2t64={APT_LIBASOUND2T64} \ - ruby={APT_RUBY} \ - ruby-rubygems={APT_RUBY_RUBYGEMS} \ - perl={APT_PERL} \ - cpanminus={APT_CPANMINUS} \ - lsb-release={APT_LSB_RELEASE} \ - fonts-noto-color-emoji={APT_FONTS_NOTO_COLOR_EMOJI} \ - haveged={APT_HAVEGED} \ - mediainfo={APT_MEDIAINFO} \ - p7zip-rar={APT_P7ZIP_RAR} \ - pollinate={APT_POLLINATE} \ - sshpass={APT_SSHPASS} \ - telnet={APT_TELNET} \ - tk={APT_TK} \ - xvfb={APT_XVFB} \ - zsync={APT_ZSYNC} \ - ftp={APT_FTP} \ - sphinxsearch={APT_SPHINXSEARCH} \ - systemd-coredump={APT_SYSTEMD_COREDUMP} \ - libnss3-tools={APT_LIBNSS3_TOOLS} \ - software-properties-common={APT_SOFTWARE_PROPERTIES_COMMON} \ - build-essential={APT_BUILD_ESSENTIAL} \ - pkg-config={APT_PKG_CONFIG} \ - libssl-dev={APT_LIBSSL_DEV} \ - make={APT_MAKE} \ - autoconf={APT_AUTOCONF} \ - automake={APT_AUTOMAKE} \ - libtool={APT_LIBTOOL} \ - m4={APT_M4} \ - bison={APT_BISON} \ - flex={APT_FLEX} \ - texinfo={APT_TEXINFO} \ - patchelf={APT_PATCHELF} \ - swig={APT_SWIG} \ - dpkg-dev={APT_DPKG_DEV} \ - fakeroot={APT_FAKEROOT} \ - binutils={APT_BINUTILS} \ - lld={APT_LLD} \ - libicu-dev={APT_LIBICU_DEV} \ - libsqlite3-dev={APT_LIBSQLITE3_DEV} \ - libyaml-dev={APT_LIBYAML_DEV} \ - python3={APT_PYTHON3} \ - python3-pip={APT_PYTHON3_PIP} \ - python-is-python3={APT_PYTHON_IS_PYTHON3} \ - unzip={APT_UNZIP} \ - zip={APT_ZIP} \ - xz-utils={APT_XZ_UTILS} \ - zstd={APT_ZSTD} \ - bzip2={APT_BZIP2} \ - brotli={APT_BROTLI} \ - lz4={APT_LZ4} \ - pigz={APT_PIGZ} \ - p7zip-full={APT_P7ZIP_FULL} \ - tar={APT_TAR} \ - jq={APT_JQ} \ - file={APT_FILE} \ - tree={APT_TREE} \ - shellcheck={APT_SHELLCHECK} \ - parallel={APT_PARALLEL} \ - time={APT_TIME} \ - acl={APT_ACL} \ - locales={APT_LOCALES} \ - tzdata={APT_TZDATA} \ - rsync={APT_RSYNC} \ - dnsutils={APT_DNSUTILS} \ - iputils-ping={APT_IPUTILS_PING} \ - net-tools={APT_NET_TOOLS} \ - iproute2={APT_IPROUTE2} \ - netcat-openbsd={APT_NETCAT_OPENBSD} \ - sqlite3={APT_SQLITE3} \ - rpm={APT_RPM} \ - aria2={APT_ARIA2} \ - mercurial={APT_MERCURIAL} \ - libcurl4-openssl-dev={APT_LIBCURL4_OPENSSL_DEV} \ - zlib1g-dev={APT_ZLIB1G_DEV} \ - gettext={APT_GETTEXT} \ - libexpat1-dev={APT_LIBEXPAT1_DEV}" - ) -} - -/// The golden image's package baseline. Exposed for the fidelity tests. -pub fn base_packages() -> &'static str { - BASE_PACKAGES -} - -/// The golden image's container engine baseline. Exposed for the fidelity -/// tests. -pub fn docker_packages() -> String { - docker_apt_packages() -} - -/// The golden image's hosted compiler package baseline. -pub fn compiler_packages() -> String { - compiler_apt_packages() -} - -/// Where the container engine stores layers. Exposed for the fidelity tests. -pub fn docker_data_root() -> &'static str { - DOCKER_DATA_ROOT -} - -/// Loopback `/etc/hosts` contents. Exposed for the fidelity tests. -pub fn loopback_hosts() -> &'static str { - LOOPBACK_HOSTS -} - fn node_externals_at(runner_root: &str) -> Vec> { // Pinned SHA per runtime+platform, derived from versions.toml via build.rs. // Empty string means no pinned entry — SHASUMS verification still applies. @@ -2101,27 +1930,58 @@ pub const DEFAULT_RUNNER_USER: &str = "runner"; /// Default UID for [`DEFAULT_RUNNER_USER`], matching the hosted image. pub const DEFAULT_RUNNER_UID: u32 = 1001; +/// Whether `user` may be used as the runner account. +/// +/// Deliberately narrow: the name reaches root-run guest scripts unquoted by +/// design (it is a *name*, not data), so anything a Linux account name cannot +/// contain is refused rather than escaped. `root` is allowed as the explicit +/// "do not switch user" value; an empty name is not. +pub fn is_valid_account_name(user: &str) -> bool { + if user == "root" { + return true; + } + let mut chars = user.chars(); + match chars.next() { + Some(first) if first.is_ascii_lowercase() || first == '_' => {} + _ => return false, + } + chars.all(|character| { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || matches!(character, '_' | '-') + }) +} + /// Create the unprivileged runner account and hand it every path a job writes. /// -/// Part of [`base_install_script`] — and therefore of the environment -/// fingerprint — on purpose. Run as a separate post-bake `exec`, a change here -/// left the fingerprint untouched, so the pool adopted the previous golden and -/// silently served jobs an account the new code no longer matched. Keep every -/// step idempotent: the same script runs against an already-prepared rootfs. -/// -/// The Rust homes are the subtle ones. `ToolchainLayer::Rust` installs them as -/// root at fixed system addresses (`/usr/local/rustup`, `/usr/local/cargo`) -/// that `guest_env_prefix` exports to every user, so without this ownership -/// the runner cannot write them and `rustup toolchain install` dies with -/// `could not create home directory`. +/// Part of [`golden_contract_script`] — and therefore of the environment +/// fingerprint — on purpose. Applied per `exec` instead, a change here left the +/// fingerprint untouched, so the pool adopted the previous golden and silently +/// served jobs an account the new code no longer matched. Keep every step +/// idempotent: the same script runs against an image that already has the +/// account (the official golden's `runner`) and against one that does not. +/// +/// An account the image already carries is *reconciled* to `uid` rather than +/// assumed to have it: the runtime drops privileges to the configured uid, so +/// an image whose `runner` sits at another uid would otherwise run jobs as a +/// uid that owns nothing in its own home. `usermod -u` also re-owns the +/// account's files, and [`golden_ownership_script`] then covers the rest. +/// +/// Ownership is *not* here: the walk over the runner home happens once, in +/// [`golden_ownership_script`], because on overlayfs a recursive `chown` +/// copies every file it touches into the per-VM upper layer. pub fn runner_account_script(user: &str, uid: u32) -> String { format!( "getent passwd {user} >/dev/null 2>&1 || useradd -m -u {uid} -s /bin/bash {user} 2>/dev/null; \ + if [ \"$(id -u {user} 2>/dev/null)\" != \"{uid}\" ]; then \ + usermod -u {uid} {user} 2>/dev/null || \ + printf 'preloop: could not move %s to uid %s; jobs run as that uid and may not own their home\\n' \ + '{user}' '{uid}' >&2; \ + fi; \ printf '%s\\n' '{user} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/preloop-{user} \ && chmod 0440 /etc/sudoers.d/preloop-{user}; \ - mkdir -p /run/user/{uid} /opt/hostedtoolcache /usr/local/rustup /usr/local/cargo; \ - chown {uid}:{uid} /run/user/{uid} {root} 2>/dev/null; \ - chown -R {uid}:{uid} /usr/local/rustup /usr/local/cargo 2>/dev/null; \ + mkdir -p /run/user/{uid} /opt/hostedtoolcache {root}/_work; \ + chown {uid}:{uid} /run/user/{uid} {root} {root}/_work 2>/dev/null; \ [ \"$(stat -c %a /opt/hostedtoolcache 2>/dev/null)\" = \"777\" ] || \ chmod -R 777 /opt/hostedtoolcache 2>/dev/null; \ grep -q AGENT_TOOLSDIRECTORY /etc/environment 2>/dev/null || \ @@ -2131,84 +1991,27 @@ pub fn runner_account_script(user: &str, uid: u32) -> String { ) } -/// Guest script that hands the runner account ownership of every path a job -/// writes, without touching the image's privilege policy. +/// Hand the runner account ownership of everything a job writes. /// -/// Ownership only, deliberately: account creation, the `/etc/sudoers.d` rule, -/// and docker-group membership stay in [`runner_account_script`], which the -/// curated bake runs as part of building an image. Installing that policy on -/// an image which never had it would hand blanket root — and the container -/// daemon — to whatever a fork executes. +/// One owner-only walk, run once at golden build — a fork inherits the result, +/// so no per-VM walk exists at all. `find … ! -user uid -exec chown -h uid:uid +/// {} +` touches only the inodes whose *owner* is wrong, for two reasons: /// -/// The script first decides whether anything is actually wrong (`needs`), so -/// an already-correct machine pays exactly one exec round trip. Only when work -/// is needed does it escalate — directly when the exec landed on root, else -/// through passwordless sudo — and an escalation that cannot run is reported -/// as a failure rather than masked: a machine whose `RUSTUP_HOME` resolves to -/// an unusable directory fails every rustup step at job time. -pub fn runner_ownership_reconcile_script(user: &str, uid: u32) -> String { - let home = format!("/home/{user}"); - // The image's own toolchain homes. Official runner images install Rust - // under the runner's `$HOME` (`~/.rustup` with `settings.toml`, `~/.cargo` - // with the rustup shims) while preloop's contract exports - // `RUSTUP_HOME=/usr/local/rustup` to every user — rustup obeys that - // verbatim, so an image without those paths leaves `cargo` unable to - // resolve a toolchain at all: the shim reads an empty home and dies with - // `could not create home directory` (as the runner user) or `could not - // choose a version of cargo to run`. Point the contract at the image's - // home instead of refusing, which is what GitHub-hosted runners - // effectively have: one toolchain every user resolves identically. - let adopt_homes = format!( - "if [ ! -f /usr/local/rustup/settings.toml ] && [ -f {home}/.rustup/settings.toml ]; then \ - rm -rf /usr/local/rustup; \ - ln -s {home}/.rustup /usr/local/rustup; \ - fi; \ - if [ ! -d /usr/local/cargo/bin ] && [ -d {home}/.cargo/bin ]; then \ - rm -rf /usr/local/cargo; \ - ln -s {home}/.cargo /usr/local/cargo; \ - fi" - ); - // The privileged half, applied only after `needs=1`. `set -e` makes every - // required operation fail the script: this is the ownership jobs depend - // on, so a partial apply must not report success. - let apply = format!( - "set -e; \ - {adopt_homes}; \ - for d in /usr/local/rustup /usr/local/cargo; do \ - if [ -e \"$d\" ]; then chown -R {uid}:{uid} \"$d\"; fi; \ - done; \ - if [ -d /opt/hostedtoolcache ]; then \ - [ \"$(stat -c %a /opt/hostedtoolcache)\" = \"777\" ] || chmod -R 777 /opt/hostedtoolcache; \ - fi; \ - grep -q AGENT_TOOLSDIRECTORY /etc/environment || \ - printf 'AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache\\nRUNNER_TOOL_CACHE=/opt/hostedtoolcache\\n' >> /etc/environment; \ - mkdir -p /run/user/{uid}; \ - chown {uid}:{uid} /run/user/{uid}" - ); - use base64::Engine as _; - let apply_b64 = base64::engine::general_purpose::STANDARD.encode(&apply); - // Same predicate as `adopt_homes`, expressed as a `needs` signal so the - // fast path does not skip a machine that still needs the link. - let adopt_needs = format!( - "if [ ! -f /usr/local/rustup/settings.toml ] && [ -f {home}/.rustup/settings.toml ]; then needs=1; fi; \ - if [ ! -d /usr/local/cargo/bin ] && [ -d {home}/.cargo/bin ]; then needs=1; fi" - ); +/// * the guest root is overlayfs, where a recursive `chown` copies every file +/// it visits out of the packed lower layer into the per-VM upper even when +/// the owner already matches (measured: 304 s and 1.22 GB to re-group two +/// inodes on the production golden's runner home); +/// * a group predicate would re-group files the official images deliberately +/// leave in another group — `/home/runner/.docker` is `runner:docker`, and +/// the official runner leaves it alone. +/// +/// `-h` preserves `chown -R`'s no-symlink-traversal default, and `-xdev` keeps +/// the walk out of read-only mounts some images carry under the runner home. +pub fn golden_ownership_script(uid: u32) -> String { format!( - "needs=0; \ - {adopt_needs}; \ - for d in /usr/local/rustup /usr/local/cargo; do \ - if [ -e \"$d\" ]; then \ - if [ \"$(stat -L -c %u \"$d\" 2>/dev/null)\" != \"{uid}\" ]; then needs=1; \ - elif find \"$d/.\" ! -uid {uid} -print -quit 2>/dev/null | grep -q .; then needs=1; fi; \ - fi; \ - done; \ - if [ -d /opt/hostedtoolcache ]; then \ - [ \"$(stat -c %a /opt/hostedtoolcache 2>/dev/null)\" = \"777\" ] || needs=1; \ - fi; \ - grep -q AGENT_TOOLSDIRECTORY /etc/environment 2>/dev/null || needs=1; \ - if [ \"$needs\" = \"0\" ]; then echo 'runner ownership already reconciled'; exit 0; fi; \ - if [ \"$(id -u)\" -eq 0 ]; then printf %s '{apply_b64}' | base64 -d | sh; \ - else printf %s '{apply_b64}' | base64 -d | sudo -n sh; fi" + "find {root} -xdev ! -user {uid} -exec chown -h {uid}:{uid} {{}} + 2>/dev/null; \ + chown {uid}:{uid} /run/user/{uid} 2>/dev/null || true", + root = RUNNER_ROOT, ) } @@ -2484,174 +2287,128 @@ fn hosted_runtime_init_script_at(hosts: &str, sysctl_root: &str) -> String { } /// The guest bootstrap script, one shell round trip. +/// The one thing a golden build checks, and refuses an image over. +/// +/// smolvm boots a Linux guest and every dynamically linked binary a workflow +/// runs — the runner included — resolves through the guest's glibc loader. An +/// image without one (musl-only, scratch) cannot run a job at all, and the +/// failure at job time is a wall of `exec: no such file` from steps that have +/// nothing to do with the image. Fail the bake instead, naming what was +/// missing. +pub fn golden_glibc_check_script() -> &'static str { + "loader=0; \ + for p in /lib64/ld-linux-*.so.* /lib/ld-linux-*.so.* /lib/*-linux-gnu*/ld-linux-*.so.*; do \ + [ -e \"$p\" ] && { loader=1; break; }; \ + done; \ + [ \"$loader\" = \"1\" ] || { \ + echo 'preloop: this image carries no glibc dynamic loader (looked for /lib64/ld-linux-*.so.*, /lib/ld-linux-*.so.*, /lib/*-linux-gnu*/ld-linux-*.so.*); a golden built from it could not run any dynamically linked binary' >&2; \ + exit 1; \ + }" +} + +/// The complete golden contract for a configured image: the image is used +/// exactly as it is, and this is everything Preloop adds. +/// +/// GitHub-runner machinery only — the account jobs run as (with its home, +/// `_work` and passwordless sudo), the ownership of the paths they write, the +/// writable tool cache with `RUNNER_TOOL_CACHE`/`AGENT_TOOLSDIRECTORY` in +/// `/etc/environment`, the glibc requirement above, and the image's apt package +/// indices ([`golden_apt_lists_script`]). No packages, no +/// toolchains, no PATH or environment overrides: on a hosted runner a missing +/// `bash`, `git` or `docker` fails the step that needs it, and the same is true +/// here. +/// +/// The text is also the fingerprint input for configured images (see +/// `EnvironmentSpec::from_base`), so editing a step rebuilds every golden baked +/// from an image instead of adopting one the old contract produced. +pub fn golden_contract_script(user: &str, uid: u32) -> String { + // `PRELOOP_RUNNER_USER=root` restores the old "run as root" behavior, and + // `as_runner_user` runs the runner unchanged for it. The account steps must + // skip then: creating and chowning a `root` account (and moving its home + // to `/home/root`) would leave the image in a state nothing runs as. + // Steps are joined rather than formatted into fixed slots: an empty slot + // renders `; ;`, which `sh` rejects, and the root contract hit exactly that. + let mut steps = vec!["set -e".to_owned(), golden_glibc_check_script().to_owned()]; + if user != "root" { + steps.push(runner_account_script(user, uid)); + steps.push(golden_ownership_script(uid)); + } + steps.push(golden_apt_lists_script()); + steps.join("; ") +} + +/// Refresh apt's package indices, best-effort and bounded. +/// +/// Hosted images keep populated lists, so real workflows run +/// `sudo apt-get install ` with no `apt-get update` first (uv's musl cell +/// installs `musl-tools` that way). The runner-image dump ships with its lists +/// wiped, so a golden without this step fails those steps with +/// `E: Unable to locate package`. +/// +/// A failed refresh (offline bake host, held lock) warns and carries on: the +/// golden is still correct, and [`restore_apt_lists_script`] covers a pack +/// that shipped without lists. No `#` comments in the text: the statements are +/// joined into one physical line. +fn golden_apt_update_command() -> &'static str { + "limit=; command -v timeout >/dev/null 2>&1 && limit='timeout 300'; \ + $limit apt-get -o DPkg::Lock::Timeout=10 update -qq \ + || echo 'preloop: apt-get update failed; this golden ships without apt package indices' >&2" +} + +/// The contract step that bakes fresh apt indices into a golden. /// -/// Every `exec` is a host process spawn plus a vsock round trip, and this runs -/// on the engine's start-up critical path. Exposed for the fidelity tests. -pub fn base_install_script() -> String { +/// Unconditional, unlike [`restore_apt_lists_script`]: a rebuilt golden should +/// carry the indices of its bake day, not whatever an older image shipped. An +/// image without apt skips it. +pub fn golden_apt_lists_script() -> String { format!( - "(find /usr/bin /usr/sbin /bin /sbin /etc -type f 2>/dev/null | \ - while IFS= read -r f; do chown 0:0 \"$f\" 2>/dev/null; done) || true; \ - chown 0:0 /etc/sudo.conf /etc/sudoers 2>/dev/null; \ - for f in /etc/sudoers.d/*; do [ -f \"$f\" ] && chown 0:0 \"$f\" 2>/dev/null; done; \ - chmod 0440 /etc/sudoers /etc/sudoers.d/* 2>/dev/null; \ - (for b in sudo su mount umount passwd chsh chfn newgrp gpasswd expiry chage wall write pkexec ping fusermount fusermount3; do \ - for p in /usr/bin/$b /bin/$b /usr/sbin/$b; do \ - if [ -f \"$p\" ]; then chown 0:0 \"$p\" 2>/dev/null; chmod u+s \"$p\" 2>/dev/null; fi; \ - done; \ - done; \ - for p in /usr/lib/openssh/ssh-keysign /usr/lib/dbus-1.0/dbus-daemon-launch-helper; do \ - if [ -f \"$p\" ]; then chown 0:0 \"$p\" 2>/dev/null; chmod u+s \"$p\" 2>/dev/null; fi; \ - done) && \ - apt-get update -qq && \ - . /etc/os-release && \ - case \"$VERSION_ID\" in \ - 22.04) base_packages='{BASE_PACKAGES} {BASE_PACKAGES_22_04}' ;; \ - *) base_packages='{BASE_PACKAGES} {BASE_PACKAGES_24_04}' ;; \ - esac && \ - (echo \"### install hosted apt baseline\" >&2 && \ - if [ \"$VERSION_ID\" = 24.04 ] && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get -s install -qq --no-install-recommends {base_packages_pinned} >/dev/null 2>&1; then \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq --no-install-recommends {base_packages_pinned}; \ - else \ - echo \"WARNING: exact hosted apt pins are unavailable; falling back to archive versions\" >&2; \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq --no-install-recommends $base_packages; \ - fi) \ - && printf '{LOOPBACK_HOSTS}' > /etc/hosts && \ - printf '127.0.0.1 %s\\n' \"$(hostname)\" >> /etc/hosts && \ - printf 'APT::Get::Assume-Yes \"true\";\\n' > /etc/apt/apt.conf.d/90assumeyes && \ - rm -f /usr/lib/python3*/EXTERNALLY-MANAGED && \ - arch=$(uname -m); \ - case \"$arch\" in x86_64) NODE_ARCH=x64 ;; aarch64|arm64) NODE_ARCH=arm64 ;; *) NODE_ARCH=x64 ;; esac; \ - case \"$NODE_ARCH\" in \ - x64) LFS_ARCH=amd64; DOCKER_STATIC_ARCH=x86_64; DOCKER_PLUGIN_ARCH=amd64; COMPOSE_ARCH=x86_64 ;; \ - *) LFS_ARCH=arm64; DOCKER_STATIC_ARCH=aarch64; DOCKER_PLUGIN_ARCH=arm64; COMPOSE_ARCH=aarch64 ;; \ - esac; \ - (echo \"### install hosted compiler matrix\" >&2 && \ - available_compiler_packages=''; \ - compiler_matrix_complete=1; \ - for package in {compiler_packages}; do \ - if DEBIAN_FRONTEND=noninteractive \ - apt-get -s install -qq --no-install-recommends \"$package\" >/dev/null 2>&1; then \ - available_compiler_packages=\"$available_compiler_packages $package\"; \ - else \ - compiler_matrix_complete=0; \ - echo \"compiler package unavailable: $package\" >&2; \ - fi; \ - done; \ - if [ -n \"$available_compiler_packages\" ]; then \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq --no-install-recommends $available_compiler_packages || exit 1; \ - fi; \ - if [ \"$compiler_matrix_complete\" = 1 ]; then \ - clang-16 --version | head -1 | grep -F '{CLANG_16_VERSION}' && \ - clang-17 --version | head -1 | grep -F '{CLANG_17_VERSION}' && \ - clang-18 --version | head -1 | grep -F '{CLANG_18_VERSION}' && \ - test \"$(gcc-12 -dumpfullversion)\" = '{GCC_12_VERSION}' && \ - test \"$(gcc-13 -dumpfullversion)\" = '{GCC_13_VERSION}' && \ - test \"$(gcc-14 -dumpfullversion)\" = '{GCC_14_VERSION}' || exit 1; \ - else \ - echo \"WARNING: hosted compiler matrix is incomplete in this Ubuntu archive; adding the archive-default compiler toolchain\" >&2; \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq --no-install-recommends clang clang-format clang-tidy gcc g++ gfortran || exit 1; \ - fi; \ - for version in {CLANG_VERSIONS}; do \ - if [ -x /usr/bin/clang++-$version ]; then \ - update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$version 100 || true; \ - fi; \ - if [ -x /usr/bin/clang-$version ]; then \ - update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$version 100 || true; \ - fi; \ - if [ -x /usr/bin/clang-format-$version ]; then \ - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$version 100 || true; \ - fi; \ - if [ -x /usr/bin/clang-tidy-$version ]; then \ - update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-$version 100 || true; \ - fi; \ - if [ -x /usr/bin/run-clang-tidy-$version ]; then \ - update-alternatives --install /usr/bin/run-clang-tidy run-clang-tidy /usr/bin/run-clang-tidy-$version 100 || true; \ - fi; \ - done; \ - for tool in clang clang++ clang-format clang-tidy run-clang-tidy; do \ - if [ -x \"/usr/bin/$tool-{CLANG_DEFAULT_VERSION}\" ]; then \ - update-alternatives --set \"$tool\" \"/usr/bin/$tool-{CLANG_DEFAULT_VERSION}\" || exit 1; \ - fi; \ - done) && \ - (echo \"### fetch system node v{BASE_NODE_VERSION}\" >&2 && \ - curl -fsSL \"https://nodejs.org/dist/v{BASE_NODE_VERSION}/node-v{BASE_NODE_VERSION}-linux-$NODE_ARCH.tar.gz\" \ - | tar -xz --strip-components=1 -C /usr/local) && \ - (install -m 0755 -d /etc/apt/keyrings && \ - (echo \"### fetch docker gpg\" >&2 && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc && \ - echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable\" > /etc/apt/sources.list.d/docker.list && \ - apt-get update -qq && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq {docker_packages} && \ - (echo \"### install gh cli\" >&2 && \ - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg && \ - echo \"deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" > /etc/apt/sources.list.d/github-cli.list && \ - apt-get update -qq && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y -qq gh && \ - gh --version | head -1) && \ - echo \"### overlay docker v{DOCKER_VERSION}\" >&2 && \ - rm -rf /tmp/docker-static && mkdir -p /tmp/docker-static && \ - curl -fsSL \"https://download.docker.com/linux/static/stable/$DOCKER_STATIC_ARCH/docker-{DOCKER_VERSION}.tgz\" \ - | tar -xz -C /tmp/docker-static && \ - install -m 0755 /tmp/docker-static/docker/* /usr/local/bin/ && \ - rm -rf /tmp/docker-static && \ - install -m 0755 -d /usr/local/lib/docker/cli-plugins && \ - curl -fsSL \"https://github.com/docker/buildx/releases/download/v{DOCKER_BUILDX_VERSION}/buildx-v{DOCKER_BUILDX_VERSION}.linux-$DOCKER_PLUGIN_ARCH\" \ - -o /usr/local/lib/docker/cli-plugins/docker-buildx && \ - curl -fsSL \"https://github.com/docker/compose/releases/download/v{DOCKER_COMPOSE_VERSION}/docker-compose-linux-$COMPOSE_ARCH\" \ - -o /usr/local/lib/docker/cli-plugins/docker-compose && \ - chmod 0755 /usr/local/lib/docker/cli-plugins/docker-buildx /usr/local/lib/docker/cli-plugins/docker-compose && \ - docker --version | grep -F '{DOCKER_VERSION}' && \ - dockerd --version | grep -F '{DOCKER_VERSION}' && \ - docker buildx version | grep -F 'v{DOCKER_BUILDX_VERSION}' && \ - docker compose version --short | grep -F '{DOCKER_COMPOSE_VERSION}' && \ - mkdir -p {DOCKER_DATA_ROOT} /etc/docker && \ - printf '{{\"data-root\":\"{DOCKER_DATA_ROOT}\"}}\\n' > /etc/docker/daemon.json)) && \ - (echo \"### fetch cargo-shear\" >&2 && \ - curl -sSL https://github.com/Boshen/cargo-shear/releases/download/v{CARGO_SHEAR_VERSION}/cargo-shear-$(uname -m)-unknown-linux-musl.tar.gz 2>/dev/null | tar -xz -C /usr/local/bin 2>/dev/null || true) && \ - (echo \"### bake git v{GIT_VERSION}\" >&2 && \ - apt-get install -y -qq --no-install-recommends libcurl4-openssl-dev zlib1g-dev gettext libexpat-dev && \ - curl -fsSL https://github.com/git/git/archive/refs/tags/v{GIT_VERSION}.tar.gz | tar -xz -C /tmp && \ - (cd /tmp/git-{GIT_VERSION} && make -s prefix=/usr all && make -s prefix=/usr install) && \ - rm -rf /tmp/git-{GIT_VERSION} && \ - echo \"### bake git-lfs v{GIT_LFS_VERSION}\" >&2 && \ - curl -fsSL https://github.com/git-lfs/git-lfs/releases/download/v{GIT_LFS_VERSION}/git-lfs-linux-$LFS_ARCH-v{GIT_LFS_VERSION}.tar.gz | tar -xz -C /tmp && \ - /tmp/git-lfs-{GIT_LFS_VERSION}/install.sh && rm -rf /tmp/git-lfs-{GIT_LFS_VERSION}) && \ - (mkdir -p /usr/local/share && \ - echo \"### bake nvm v{NVM_VERSION}\" >&2 && \ - curl -fsSL https://github.com/nvm-sh/nvm/archive/refs/tags/v{NVM_VERSION}.tar.gz | tar -xz -C /usr/local/share && \ - mv /usr/local/share/nvm-{NVM_VERSION} /usr/local/share/nvm && \ - printf 'export NVM_DIR=/usr/local/share/nvm\\n[ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\"\\n' > /etc/profile.d/nvm.sh) && \ - echo \"### bake yarn v{YARN_VERSION}\" >&2 && \ - npm install -g yarn@{YARN_VERSION} && \ - install -d -m 0777 /opt/hostedtoolcache && \ - printf 'AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache\\nRUNNER_TOOL_CACHE=/opt/hostedtoolcache\\n' >> /etc/environment && \ - (useradd -m -u 1000 -s /bin/bash ubuntu 2>/dev/null || true) && \ - ({runner_account}) && \ - apt-get clean && \ - rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/*", - runner_account = runner_account_script(DEFAULT_RUNNER_USER, DEFAULT_RUNNER_UID), - docker_packages = docker_apt_packages(), - compiler_packages = compiler_apt_packages(), - base_packages_pinned = base_packages_pinned() + "if command -v apt-get >/dev/null 2>&1; then {}; fi", + golden_apt_update_command() ) } -fn base_install_commands() -> Vec> { - [vec![ - "sh".to_owned(), - "-c".to_owned(), - base_install_script(), - ]] - .into_iter() - .collect() +/// Restore apt's indices when the golden has none, a no-op when it has them. +/// +/// For a published pack baked before the contract carried +/// [`golden_apt_lists_script`]: it is unpacked, not baked here, so nothing else +/// would populate its lists. `lists` is the directory apt keeps them in. +fn restore_apt_lists_script_at(lists: &str) -> String { + format!( + "if command -v apt-get >/dev/null 2>&1 \ + && [ -z \"$(find {lists} -name '*_Packages*' -print -quit 2>/dev/null)\" ]; then {update}; fi", + lists = shell_quote(lists), + update = golden_apt_update_command(), + ) +} + +fn restore_apt_lists_script() -> String { + restore_apt_lists_script_at("/var/lib/apt/lists") +} + +/// Restore apt's indices in a freshly unpacked golden, before it is frozen. +/// +/// Done once per golden, not per fork: a fork's own writes never reach its +/// siblings, so a per-fork refresh would repay the download on every job. Never +/// fatal; a golden without indices still runs jobs that do not `apt-get +/// install` blind. +async fn restore_apt_lists(provider: &P, golden: &MachineName) { + let script = run_as_root_or_sudo(&restore_apt_lists_script()); + match provider + .exec(golden, &["sh".to_owned(), "-c".to_owned(), script]) + .await + { + Ok(output) if output.exit_code == 0 => {} + Ok(output) => warn!( + machine = golden.as_str(), + exit = output.exit_code, + "apt index restore failed; workflow `apt-get install` steps may not resolve" + ), + Err(error) => warn!( + machine = golden.as_str(), + %error, + "apt index restore could not run; workflow `apt-get install` steps may not resolve" + ), + } } /// Start the container engine, if one is installed. @@ -2845,61 +2602,20 @@ async fn await_guest_ready( } } -/// Restore apt's package indices when the image shipped without them. -/// -/// Hosted images keep populated lists, so real workflows run -/// `sudo apt-get install ` with no `apt-get update` first (uv's musl cell -/// installs `musl-tools` that way). Every pack published while the baseline -/// script ended in `rm -rf /var/lib/apt/lists/*` boots without them, and each -/// of those steps fails with `E: Unable to locate package`. Cheap to check, -/// and a no-op on an image that has them. -/// -/// Hard-bounded: a fork of a packed golden can inherit a held apt lock from the -/// frozen image, and `apt-get update` then waits forever — which would block -/// provisioning, not just the refresh. A missed refresh costs a workflow one -/// `apt-get update`; a hung one costs the whole pool. -fn apt_lists_refresh_command() -> Vec { - vec![ - "sh".to_owned(), - "-c".to_owned(), - "[ -n \"$(find /var/lib/apt/lists -name '*_Packages*' -print -quit 2>/dev/null)\" ] \ - || timeout 120 apt-get -o DPkg::Lock::Timeout=10 update -qq || true" - .to_owned(), - ] -} - -async fn install_base_dependencies( - provider: &P, - name: &MachineName, -) -> Result<(), OrchestratorError> { - for command in base_install_commands() { - let output = provider.exec(name, &command).await?; - if output.exit_code != 0 { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(OrchestratorError::Config(format!( - "base package install failed (exit {}): {}", - output.exit_code, - stderr.lines().last().unwrap_or("unknown error") - ))); - } - } - // Node externals are no longer baked: they arrive via the read-only - // host mount (`ensure_host_externals` + `runner_volumes`), which keeps - // the golden pack and every machine image lean. - Ok(()) -} - /// Where a baked golden records its provenance. Root-owned, so writing it /// needs the privileged hop — see [`write_bake_manifest`]. const BAKE_MANIFEST_PATH: &str = "/etc/preloop-bake.json"; -/// Record what a golden actually baked, so provenance is inspectable -/// instead of reconstructed. -/// -/// The resolved versions are the point: channels (`stable`, `22`, `lts/*`, -/// `go 1.24` minimums) resolve at bake time, and the manifest captures what -/// they resolved to. `/etc/preloop-bake.json` in any fork answers "what is -/// in this environment?" without re-deriving it. +/// The `@sha256:…` digest of an image reference, when it pins one. +fn source_digest(image_ref: &str) -> Option<&str> { + image_ref.split_once('@').map(|(_, digest)| digest) +} + +/// Record what a golden baked, so provenance is inspectable instead of +/// reconstructed: the image it came from (with its digest when the reference +/// pins one), what the bake resolved at build time, and which Preloop built it. +/// `/etc/preloop-bake.json` in any fork answers "what is in this environment?" +/// without re-deriving it. async fn write_bake_manifest( provider: &P, name: &MachineName, @@ -2908,10 +2624,10 @@ async fn write_bake_manifest( let probe = [ "sh".to_owned(), "-c".to_owned(), - "for cmd in node npm python3 docker git rustc cargo go cargo-shear; do \ + "for cmd in node npm python3 docker git rustc cargo go; do \ printf '%s=%s\\n' \"$cmd\" \"$($cmd --version 2>/dev/null | head -n1 || echo missing)\"; \ done; \ - printf 'packages=%s\\n' \"$(dpkg-query -W -f={{Package}} | wc -l)\"; \ + printf 'packages=%s\\n' \"$(dpkg-query -W -f={{Package}} 2>/dev/null | wc -l)\"; \ printf 'built_at=%s\\n' \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"" .to_owned(), ]; @@ -2924,12 +2640,15 @@ async fn write_bake_manifest( } let manifest = serde_json::json!({ "base": env_spec.base, - "toolchains": env_spec.toolchains, + // The digest part of the reference when it pins one; a floating tag or + // a local path has none, and saying so is honest. + "source_digest": source_digest(&env_spec.base), + "fingerprint": env_spec.fingerprint, + // What the image itself carries: a tool the image does not ship reads + // `missing`, exactly as it would at job time. "versions": versions, - "base_node": BASE_NODE_VERSION, - "cargo_shear": CARGO_SHEAR_VERSION, - // Derived from the same generated pins the install path uses, so a - // version bump can never install one version and record another. + // What the runner resolves through the bundle mount, from the same + // generated pins the externals materialization uses. "node_externals": [ format!("node20 {NODE20_EXTERNALS_VERSION}"), format!("node24 {NODE24_EXTERNALS_VERSION}"), @@ -2985,40 +2704,31 @@ async fn write_bake_manifest( Ok(()) } -/// PATH the guest runner process exports to every step. -/// Hosted images carry the toolchain bin directories on the runner's own PATH, -/// which is what makes `cargo install`-style actions work: `taiki-e/install-action` -/// drops `cargo-hack` in `$CARGO_HOME/bin` and the next step runs `cargo hack`. -/// dtolnay/rust-toolchain only appends that directory to `$GITHUB_PATH` when it -/// has to install rustup itself, so on an image that already has rustup — ours, -/// and GitHub's — the directory is on PATH or the tool is simply unreachable. -/// -/// The cargo bin dir is the fixed system address `/usr/local/cargo/bin`, -/// matching the exported `CARGO_HOME` (see `guest_env_prefix`): it is -/// identical for root and switched runners by construction. A per-user -/// `$HOME/.cargo/bin` here would reintroduce the EACCES trap the homes fix -/// removes — `/root` is 0700, so exporting `/root/.cargo/bin` to an -/// unprivileged runner makes every tool lookup fail statting it -/// (nodejs/ci: `EACCES: permission denied, stat '/root/.cargo/bin/git'`). -/// Absent directories cost nothing. +/// PATH the guest runner falls back to when the image declares none. +/// +/// GitHub's hosted images declare their PATH in `/etc/environment`, and the +/// runner must see exactly that: the image's toolchains are where *it* put +/// them, and a host-side static PATH cannot know them (the deleted curated +/// bakes put Cargo and Go at fixed `/usr/local` addresses; the official image +/// does not). [`guest_env_prefix`] therefore runs the runner through a shell +/// that sources `/etc/environment` and only falls back to this list when the +/// image carries no PATH of its own. pub fn guest_runner_path(_config: &RunnerPoolConfig) -> String { - format!( - "{GUEST_CARGO_HOME}/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:\ - /usr/sbin:/usr/bin:/sbin:/bin" - ) + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_owned() } -/// `env` prefix for guest runner invocations, empty when nothing needs setting. +/// The `KEY=value` entries the runner is launched with: the variables the pool +/// sets, with no launcher shell. [`guest_env_prefix`] adds the shell that +/// sources the image's `/etc/environment` on top of these. /// /// Control-socket routing and failure-marker debugging are independent /// features: a pool can debug failed jobs without a mounted control socket and /// vice versa, so neither may gate the other. -fn guest_env_prefix(config: &RunnerPoolConfig, name: &MachineName) -> Vec { - let mut env = Vec::new(); - env.push(format!("PATH={}", guest_runner_path(config))); - // The guest needs its own VM name so a debug session can tell a controller - // which machine to open a shell into. Nothing else in the guest knows it. - env.push(format!("PRELOOP_MACHINE_NAME={}", name.as_str())); +fn guest_env_entries(config: &RunnerPoolConfig, name: &MachineName) -> Vec { + let mut env = vec![ + format!("PATH={}", guest_runner_path(config)), + format!("PRELOOP_MACHINE_NAME={}", name.as_str()), + ]; if config.control_socket.is_some() { env.push(format!( "PRELOOP_CONTROL_ORIGIN={}", @@ -3044,23 +2754,40 @@ fn guest_env_prefix(config: &RunnerPoolConfig, name: &MachineName) -> Vec Vec { + let env = guest_env_entries(config, name); + // `env` re-asserts every variable above *after* the image's environment is + // sourced, so a malformed `/etc/environment` cannot unset `HOME` or the + // control routing. It also resolves the program through the PATH the + // source may have replaced (a running process keeps the PATH it started + // with), which is exactly the heap the image declares. + let pairs = env + .iter() + .map(|entry| shell_quote(entry)) + .collect::>() + .join(" "); + let launcher = format!( + "if [ -r /etc/environment ]; then . /etc/environment; fi; \ + exec /usr/bin/env {pairs} \"$0\" \"$@\"" + ); + let mut argv = vec!["sh".to_owned(), "-c".to_owned(), launcher]; + argv.extend(env); + argv +} + /// Local ephemeral-runner pool configuration. #[derive(Debug, Clone)] pub struct RunnerPoolConfig { @@ -3070,16 +2797,14 @@ pub struct RunnerPoolConfig { /// When enabled, a single "golden" VM boots once and each runner slot /// clones from it with CoW memory and disks. pub use_fork: bool, - /// Create runners from the prepared packed artifact instead of the base - /// OCI image. The caller must provide a SmolVM build that preserves - /// explicitly supplied socket mappings for packed-machine creation. - pub use_packed_artifact: bool, /// Prefix used for owned SmolVM names. pub name_prefix: String, - /// Base OCI image used for one-time tool installation. + /// Image the pool's golden is built from: [`OFFICIAL_GOLDEN`], or whatever + /// `PRELOOP_RUNNER_BASE_IMAGE`/`[golden] base_image` names. Configured + /// images are baked as-is plus the runner contract (see + /// [`golden_contract_script`]); the official golden is downloaded packed + /// and never built here. pub base_image: String, - /// Optional workspace path for environment detection from version files. - pub workspace: Option, /// Host path stem for the reusable packed VM artifact. pub artifact_stem: PathBuf, /// Preloop release whose architecture-specific golden asset should be used. @@ -3189,31 +2914,34 @@ pub struct RunnerPoolConfig { pub observability: Option, } -/// Cache of environment-specific golden VMs. -pub(crate) struct GoldenRegistry { +/// The pool's prepared fork bases, keyed by environment fingerprint. +/// +/// Normally one: the pool prepares the configured environment's golden at +/// startup. A slot that resolves a *different* environment — a +/// `runs-on: ubuntu-latest` job on a pool whose configured image is custom — +/// memoizes its golden here instead of rebuilding it per job. Every golden is +/// prepared the same way (`prepare_fork_base`): from the packed artifact +/// on a file-pack backend, from the image itself where the backend keeps packs +/// server-side. +pub(crate) struct GoldenCache { goldens: RwLock>, - /// Per-fingerprint construction locks. A single build_lock used to be - /// held across the whole bake, so one environment's golden build parked - /// every other slot on the mutex — silently, with no logs — freezing the - /// pool until a restart. Distinct fingerprints now build concurrently; - /// the same fingerprint is still serialized (the second caller would - /// otherwise delete the first's half-built VM, since - /// `prepare_golden_for_env` removes any existing machine of that name). + /// Per-fingerprint construction locks. A single build lock used to be held + /// across the whole bake, so one environment's golden build parked every + /// other slot on the mutex — silently, with no logs — freezing the pool + /// until a restart. Distinct fingerprints now build concurrently; the same + /// fingerprint is still serialized (the second caller would otherwise + /// delete the first's half-built VM, since `prepare_fork_base` removes + /// any existing machine of that name). build_locks: RwLock>>>, name_prefix: String, - /// Set when the startup packed-golden import failed. In that case the - /// pool must create runners directly from their requested base image - /// rather than attempting a second environment-golden bake. - packed_unavailable: AtomicBool, } -impl GoldenRegistry { +impl GoldenCache { pub fn new(name_prefix: String) -> Self { Self { goldens: RwLock::new(HashMap::new()), build_locks: RwLock::new(HashMap::new()), name_prefix, - packed_unavailable: AtomicBool::new(false), } } @@ -3222,18 +2950,6 @@ impl GoldenRegistry { &self.name_prefix } - /// Disable packed-golden and environment-golden preparation after the - /// startup packed import fails. Direct per-runner creation is the only - /// honest fallback: a second environment golden would repeat the same - /// disk-heavy unpack under a different name. - pub fn disable_packed(&self) { - self.packed_unavailable.store(true, Ordering::Release); - } - - pub fn is_packed_disabled(&self) -> bool { - self.packed_unavailable.load(Ordering::Acquire) - } - /// Get existing golden or return None if not yet prepared. pub async fn get(&self, fingerprint: &str) -> Option { self.goldens.read().await.get(fingerprint).cloned() @@ -3267,10 +2983,7 @@ impl GoldenRegistry { if let Some(golden) = self.get(fingerprint).await { return Ok(golden); } - info!( - fingerprint, - "building golden for environment; other environments proceed concurrently" - ); + info!(fingerprint, "preparing the golden for this environment"); let name = build.await?; self.insert(fingerprint.to_owned(), name.clone()).await; Ok(name) @@ -3281,12 +2994,6 @@ impl GoldenRegistry { self.goldens.write().await.insert(fingerprint, name); } - /// Remove and return a golden VM entry. - #[allow(dead_code)] - pub async fn remove(&self, fingerprint: &str) -> Option { - self.goldens.write().await.remove(fingerprint) - } - /// Return all registered golden machine names. pub async fn all_names(&self) -> Vec { self.goldens.read().await.values().cloned().collect() @@ -3307,6 +3014,20 @@ impl RunnerPoolConfig { )); } MachineName::new(format!("{}-0", self.name_prefix))?; + if let Some(user) = &self.runner_user { + // The account name is interpolated into root-run guest scripts + // (`runner_account_script`, `as_runner_user`) and into the + // `sudoers.d` filename. Restrict it to what a Linux account name + // may be, so a configured value cannot smuggle shell syntax into + // a root shell (or escape the sudoers directory). + if !is_valid_account_name(user) { + return Err(OrchestratorError::Config(format!( + "runner user `{user}` is not a valid account name: use a lowercase \ + letter or underscore followed by lowercase letters, digits, \ + underscores, or dashes (or `root`)" + ))); + } + } if self.base_image.trim().is_empty() || self.server_url.trim().is_empty() || self.release_version.trim().is_empty() @@ -3352,11 +3073,11 @@ impl RunnerPoolConfig { fn artifact_payload(&self) -> PathBuf { // The packed artifact is keyed by the resolved base image AND the - // environment fingerprint (toolchains + curated bake content). A - // stem-only key would let a golden keep the previous bake forever: - // bake-content changes (package pins, the ownership repair, new - // toolchains) must invalidate the pack or the fork base silently - // serves jobs the old toolchain. + // environment fingerprint (the published official reference, or the + // bake contract for a configured image). A stem-only key would let a + // golden keep the previous bake forever: a contract change or a newly + // published official golden must invalidate the pack, or the fork base + // silently serves jobs the old image. let fingerprint = EnvironmentSpec::for_base(self.base_image.clone()).fingerprint; let mut path = self.artifact_stem.clone().into_os_string(); path.push(format!("-{fingerprint}")); @@ -3527,12 +3248,23 @@ fn rosetta_multiarch_script() -> String { // failed apt step must surface as a nonzero exit. The lenient // `|| true` form would report success and bake a golden whose forks all // fail `test -f /lib64/ld-linux-x86-64.so.2` consumers. + // + // An image without apt cannot be given the amd64 runtime — the loader and + // its libraries only come from Debian/Ubuntu packages. That is not a bake + // failure for a *configured* image, which is used as-is: the operator opted + // into whatever it carries, and job VMs that need x86_64 translation are + // the price. Say so loudly and continue; an arm64 guest keeps the image's + // own arm64 runtime either way. run_as_root_or_sudo_strict(&format!( "set -e; \ case \"$(uname -m)\" in \ aarch64|arm64) ;; \ *) echo 'guest is not arm64; rosetta multiarch install is a no-op' >&2; exit 0 ;; \ esac; \ + if ! command -v apt-get >/dev/null 2>&1 || ! command -v dpkg >/dev/null 2>&1; then \ + echo 'preloop: this image has no dpkg/apt, so the amd64 runtime for Rosetta x86_64 translation cannot be installed; x86_64 binaries will not run in jobs' >&2; \ + exit 0; \ + fi; \ {}; \ dpkg --add-architecture amd64; \ CODENAME=$(. /etc/os-release 2>/dev/null && echo \"$VERSION_CODENAME\"); \ @@ -3564,39 +3296,49 @@ fn rosetta_multiarch_script() -> String { /// The package set covers the loader, libc, C++ runtime, zlib, and systemd /// (valkey's official x86_64 tarballs link `libsystemd.so.0`); apt pulls the /// amd64 transitive deps. +/// +/// Returns whether the golden gained the amd64 runtime. `false` means the +/// guest skipped it — no apt, or apt sources that are not Ubuntu's — which is +/// a real limitation for a configured image used as-is, not a bake failure. async fn prepare_rosetta_multiarch( provider: &P, golden: &MachineName, -) -> Result<(), OrchestratorError> { + unpacked_pack: bool, +) -> Result { if !(cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64") { - return Ok(()); + return Ok(true); } - // Ubuntu 24.04 uses deb822 `ubuntu.sources`; the pinned 22.04 image uses - // one-line `/etc/apt/sources.list` instead. Both arm64 sources point at - // ports.ubuntu.com, which has no amd64 packages. Scope native sources to - // arm64 before adding amd64, including any third-party `.list` entries - // without an architecture restriction, or every later apt-get update - // probes those mirrors for amd64 and fails. Add the explicit amd64 archive - // across all four suites: the base suite alone may be older than the - // image's security-update glibc, whose mutual Breaks pins block - // installation. One suite per deb line; the one-line format misparses - // extra suites as components. `sync` flushes before forking the golden. let script = rosetta_multiarch_script(); let output = provider .exec(golden, &["sh".to_owned(), "-c".to_owned(), script]) .await?; if output.exit_code != 0 { - return Err(OrchestratorError::Config(format!( - "rosetta multiarch install failed (exit {}): {}", - output.exit_code, - String::from_utf8_lossy(&output.stderr).trim() - ))); + // Fatal for a *packed* golden: the pack's jobs expect amd64 (it was + // built from the official multiarch image), so a half-installed base + // must not be adoptable. A configured image is the operator's own + // contract and is used as-is — refusing to bake it over a missing + // translation runtime would be the "everything must be checked" rule + // this design deleted. + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + if unpacked_pack { + return Err(OrchestratorError::Config(format!( + "rosetta multiarch install failed (exit {}): {stderr}", + output.exit_code + ))); + } + warn!( + machine = golden.as_str(), + exit = output.exit_code, + "rosetta multiarch install failed; x86_64 binaries will not run in this \ + golden's jobs: {stderr}" + ); + return Ok(false); } info!( machine = golden.as_str(), "installed amd64 multiarch libs into golden for Rosetta x86_64 translation" ); - Ok(()) + Ok(true) } /// Prepare a running forkable golden VM with the requested environment. @@ -3742,25 +3484,40 @@ async fn adopt_golden( Ok(adopted) } -/// Prepare a running forkable golden VM with the requested environment. +/// Prepare the forkable golden this pool forks every runner from, or adopt the +/// one a previous engine run left behind. /// -/// SmolVM takes the forkable RAM/disk snapshot when `start --forkable` runs. -/// Provision the guest while it is a normal machine, then restart it as the -/// fork base so package and external-runtime writes are inherited by clones. -async fn prepare_golden_for_env( +/// This is the pool's single golden: the environment is the pool's configured +/// base image, so a second call only happens when a job's labels resolve to the +/// official golden while the pool was configured with a custom image. +/// +/// Two shapes, one contract: +/// +/// * a **file-pack backend** (SmolVM) unpacks the packed artifact — the +/// published official golden, or one baked here from a configured image — +/// into a machine. A pack is the only shape whose guest writes a fork +/// reliably inherits, which is why every golden on this backend is one; +/// * a backend that keeps its packs server-side (AgentENV) has no artifact to +/// unpack: boot the image, apply the contract in the guest, and let +/// `start --forkable` capture the result, which that backend's snapshots do +/// carry into forks. +/// +/// Fatal on any failure. A pool without its golden cannot serve a job, and the +/// deleted alternative — warn, then create runners directly from the image — +/// ran jobs on a machine with none of the runner machinery. +async fn prepare_fork_base( provider: &Arc

, config: &RunnerPoolConfig, golden: &MachineName, env_spec: &EnvironmentSpec, + shutdown: &CancellationToken, ) -> Result<(), OrchestratorError> { - // The golden registry is in-memory, so without adoption every engine - // restart rebakes a golden that is still sitting there fully baked — - // apt plus rustup, five to eleven minutes, before the first job of that - // environment can run, paid again on every deploy. + // Adoption first: without it every engine restart repays the golden's full + // build or unpack (tens of GB of storage writes) before the first job. if adopt_golden(provider, config, golden, &env_spec.fingerprint, "golden").await? { return Ok(()); } - // Any record must die before the machine does: a rebake interrupted + // Any record must die before the machine does: a rebuild interrupted // halfway would otherwise leave a fingerprint claiming a golden that no // longer carries it. remove_golden_record(config, golden); @@ -3768,15 +3525,88 @@ async fn prepare_golden_for_env( provider.delete(golden).await?; vm_telemetry_deregister(config, golden); } + // Give up before the expensive part when a shutdown is already pending. + // The caller checks `is_cancelled` afterwards and reports a clean stop + // instead of a download or bake error. + if shutdown.is_cancelled() { + return Err(OrchestratorError::Pool( + "golden preparation cancelled by shutdown".into(), + )); + } + let prepared = if provider.capabilities().file_packs { + unpack_golden_pack(provider, config, golden, env_spec, shutdown).await + } else { + bake_golden_in_guest(provider, config, golden, env_spec).await + }; + if let Err(error) = prepared { + vm_telemetry_deregister(config, golden); + let _ = provider.delete(golden).await; + return Err(error); + } + write_golden_record(config, golden, &env_spec.fingerprint); + info!( + machine = golden.as_str(), + base = %env_spec.base, + "golden fork base ready" + ); + Ok(()) +} + +/// Refuse to start an expensive preparation while a shutdown is pending. +/// +/// The warm is idempotent and resumable, so abandoning it costs one restart's +/// preparation; waiting would make a restart sit through a multi-gigabyte +/// download or an eleven-minute bake. Callers treat the error as a clean stop +/// (they check [`CancellationToken::is_cancelled`]) rather than a failure. +fn abort_preparation_if_shutting_down( + shutdown: &CancellationToken, +) -> Result<(), OrchestratorError> { + if shutdown.is_cancelled() { + return Err(OrchestratorError::Pool( + "golden preparation cancelled by shutdown".into(), + )); + } + Ok(()) +} + +/// Unpack the packed artifact for `env_spec` into the forkable golden. +/// +/// Socket mappings are supplied by the local machine definition, never read +/// from the artifact. SmolVM must preserve those explicit mappings on its +/// `machine create --from` path for local control-plane routing to work. +async fn unpack_golden_pack( + provider: &Arc

, + config: &RunnerPoolConfig, + golden: &MachineName, + env_spec: &EnvironmentSpec, + shutdown: &CancellationToken, +) -> Result<(), OrchestratorError> { + let payload = artifact_payload(&config.artifact_stem, &env_spec.base); + ensure_golden_payload(provider, config, env_spec, &payload, shutdown).await?; + // Unpacking writes the golden's filesystem; its disk can grow to the + // configured storage ceiling, and job forks grow on top of it. Not a + // refusal: how much of the ceiling a given golden writes is image-specific + // and the disk is sparse, so only flag a host that could not hold one + // golden at its ceiling. + warn_if_disk_below( + &golden_disk_root(config), + u64::from(config.storage_gib) * GIB, + "golden unpack", + ); + // smolvm's `machine create --from` consumes the SMOLPACK, not the ELF + // launcher stub written at the payload stem. A downloaded release asset + // IS the pack at the stem; a locally built golden leaves the pack in the + // `.smolmachine` sidecar. Centralized in [`packed_golden_path`]. + let pack = packed_golden_path(&payload); let spec = MachineSpec { name: golden.clone(), - image: env_spec.base.clone(), + image: pack.display().to_string(), cpus: config.cpus, memory_mib: config.memory_mib, storage_gib: config.storage_gib, overlay_gib: config.overlay_gib, network: NetworkPolicy::PublicOnly, - volumes: runner_volumes(config, golden, true)?, + volumes: runner_volumes(config, golden, false)?, sockets: config .control_socket .iter() @@ -3791,121 +3621,92 @@ async fn prepare_golden_for_env( provider.create(&spec).await?; provider.start(golden).await?; vm_telemetry_register(config, golden, "golden", Some(&spec)); - if let Err(error) = await_guest_ready(provider.as_ref(), golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error); - } - if env_spec.curated { - // Same contract as the packed-golden path: on Apple Silicon the - // multiarch shim must reach the forkable base, or every dynamically - // linked x86_64 binary fails at job time. Runs before the baseline - // install so the amd64 sources are scoped before any later apt - // update. Fatal, not a warning — a half-installed golden would be - // adopted on later restarts (amd64 arch added, loader missing). - // Custom bases skip it along with the rest of the bake: the image is - // the operator's contract. - if let Err(error) = prepare_rosetta_multiarch(provider.as_ref(), golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error); - } - if let Err(error) = install_base_dependencies(provider.as_ref(), golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error); - } - } - for layer in &env_spec.toolchains { - for command in layer.install_commands() { - if let Err(error) = provider.exec(golden, &command).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error.into()); - } - } - } - if let Err(error) = write_bake_manifest(provider.as_ref(), golden, env_spec).await { - // Provenance is an audit aid, not a build gate. - warn!(machine = golden.as_str(), %error, "bake manifest not written"); - } + await_guest_ready(provider.as_ref(), golden).await?; + // Fatal for a packed golden: its jobs expect the multiarch shim, and a + // failed install leaves a reusable base that is adoptable on later + // restarts (amd64 arch added, loader missing). + prepare_rosetta_multiarch(provider.as_ref(), golden, true).await?; + // The published pack ships with apt's indices wiped (the runner-image dump + // does), and `sudo apt-get install ` with no `apt-get update` first is + // how real workflows install system packages. On Apple Silicon the + // multiarch step above already refreshed them; elsewhere nothing else + // would. Before the freeze, so every fork inherits them. + restore_apt_lists(provider.as_ref(), golden).await; if let Err(error) = preload_images(provider.as_ref(), golden, &config.preload_images).await { - // A preload miss costs a run-time pull, not a broken job. warn!( machine = golden.as_str(), %error, "image preload failed; jobs will pull at run time" ); } - if let Err(error) = provider.stop(golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error.into()); - } - if let Err(error) = provider.start_forkable(golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error.into()); + provider.stop(golden).await?; + provider.start_forkable(golden).await?; + // Issue #295: smolvm's pack export leaves ~29 GB of intermediates + // (`storage.ext4`, `layers/*.tar`) beside the finished `storage.raw`, + // and every fork copies the golden's whole data directory. Prune them + // now that the disk is written. Best-effort: never fail the golden + // over disk hygiene. + match provider.prune_pack_intermediates(golden).await { + Ok(true) => info!( + machine = golden.as_str(), + "pruned pack/ build intermediates from golden" + ), + Ok(false) => {} + Err(error) => warn!( + machine = golden.as_str(), + %error, + "failed to prune pack/ build intermediates from golden" + ), } - write_golden_record(config, golden, &env_spec.fingerprint); - info!(machine = golden.as_str(), "golden fork base ready"); Ok(()) } -/// Prepare a forkable golden from the dependency-prepared packed artifact. +/// The image a non-file-pack backend boots for `env_spec`. /// -/// Socket mappings are supplied by the local machine definition, never read -/// from the artifact. SmolVM must preserve those explicit mappings on its -/// `machine create --from` path for local control-plane routing to work. -async fn prepare_packed_golden( +/// A configured image is its own base. The official sentinel resolves to the +/// pinned runner image: the same one the release pipeline packs, so both +/// golden sources stay the same image and the in-guest contract keeps the +/// result equivalent to the published pack. +fn golden_boot_image(env_spec: &EnvironmentSpec) -> Result { + if !is_official_golden(&env_spec.base) { + return Ok(env_spec.base.clone()); + } + official_boot_image().map(str::to_owned).ok_or_else(|| { + OrchestratorError::Config(format!( + "there is no official golden for {}: no runner image is pinned for this \ + architecture and this backend cannot restore the packed artifact. Configure \ + one with PRELOOP_RUNNER_BASE_IMAGE or `[golden] base_image`", + std::env::consts::ARCH + )) + }) +} + +/// Boot the environment's image, apply the golden contract, freeze it as the +/// fork base. +/// +/// The non-file-pack path: there is no artifact to unpack, and the backend's +/// forkable snapshot is what carries the contract's guest writes into clones. +/// +/// The official sentinel is not an image the backend can boot, so it resolves +/// to the pinned runner image here — the same image the release pipeline +/// packs for the pack-restoring backends. `PRELOOP_GOLDEN_OCI_REF` and +/// `PRELOOP_GOLDEN_URL` name *packed* artifacts and are deliberately not +/// consulted: there is nothing on this path that could restore a pack. +async fn bake_golden_in_guest( provider: &Arc

, config: &RunnerPoolConfig, golden: &MachineName, + env_spec: &EnvironmentSpec, ) -> Result<(), OrchestratorError> { - // Same adoption rule as the baked-golden path: an engine restart must not - // re-unpack a multi-GiB packed golden that is still sitting there - // fingerprint-matched. Without this every `serve` restart pays the - // full unpack (tens of GB of storage writes) before the first job. - let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - if adopt_golden( - provider, - config, - golden, - &env_spec.fingerprint, - "packed golden", - ) - .await? - { - return Ok(()); - } - // Unpacking writes the golden's filesystem; its disk can grow to the - // configured storage ceiling, and job forks grow on top of it. Not a - // refusal: how much of the ceiling a given golden writes is image-specific - // and the disk is sparse, so only flag a host that could not hold one - // golden at its ceiling. - warn_if_disk_below( - &golden_disk_root(config), - u64::from(config.storage_gib) * GIB, - "packed golden unpack", - ); - remove_golden_record(config, golden); - if provider.status(golden).await? != MachineState::Missing { - provider.delete(golden).await?; - vm_telemetry_deregister(config, golden); - } - // smolvm's `machine create --from` consumes the SMOLPACK, not the ELF - // launcher stub written at the payload stem. A downloaded release asset - // IS the pack at the stem; a locally built golden leaves the pack in the - // `.smolmachine` sidecar. Centralized in [`packed_golden_path`]. - let pack = packed_golden_path(&config.artifact_payload()); + let base = golden_boot_image(env_spec)?; let spec = MachineSpec { name: golden.clone(), - image: pack.display().to_string(), + image: base, cpus: config.cpus, memory_mib: config.memory_mib, storage_gib: config.storage_gib, overlay_gib: config.overlay_gib, network: NetworkPolicy::PublicOnly, - volumes: runner_volumes(config, golden, false)?, + volumes: runner_volumes(config, golden, true)?, sockets: config .control_socket .iter() @@ -3917,63 +3718,227 @@ async fn prepare_packed_golden( dns: config.dns.clone(), rosetta: cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", }; - if let Err(error) = provider.create(&spec).await { - let _ = provider.delete(golden).await; - return Err(error.into()); - } - if let Err(error) = provider.start(golden).await { - let _ = provider.delete(golden).await; - return Err(error.into()); - } + provider.create(&spec).await?; + provider.start(golden).await?; vm_telemetry_register(config, golden, "golden", Some(&spec)); - if let Err(error) = await_guest_ready(provider.as_ref(), golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error); - } - // Fatal, not a warning: on Apple Silicon the multiarch shim is the - // golden's contract, and a failed install leaves a reusable base that is - // adoptable on later restarts (amd64 arch added, loader missing). - prepare_rosetta_multiarch(provider.as_ref(), golden).await?; + await_guest_ready(provider.as_ref(), golden).await?; + prepare_rosetta_multiarch(provider.as_ref(), golden, false).await?; + apply_golden_contract(provider.as_ref(), config, golden).await?; if let Err(error) = preload_images(provider.as_ref(), golden, &config.preload_images).await { warn!( machine = golden.as_str(), %error, "image preload failed; jobs will pull at run time" ); } - if let Err(error) = provider.stop(golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; - return Err(error.into()); + provider.stop(golden).await?; + provider.start_forkable(golden).await?; + Ok(()) +} + +/// Apply [`golden_contract_script`] to a machine booted from its image. +/// +/// Through the root-or-sudo hop: a machine created from an arbitrary image may +/// declare `USER` (the official runner image runs as `runner`), and the +/// contract's `/etc` writes, `chown`s and account creation need root either +/// way. Strict, so a refused sudo fails the bake instead of shipping a golden +/// without its runner account. +async fn apply_golden_contract( + provider: &P, + config: &RunnerPoolConfig, + name: &MachineName, +) -> Result<(), OrchestratorError> { + let user = config.runner_user.as_deref().unwrap_or(DEFAULT_RUNNER_USER); + let uid = config.runner_uid.unwrap_or(DEFAULT_RUNNER_UID); + let contract = golden_contract_script(user, uid); + let output = provider + .exec( + name, + &[ + "sh".to_owned(), + "-c".to_owned(), + run_as_root_or_sudo_strict(&contract), + ], + ) + .await?; + if output.exit_code != 0 { + return Err(OrchestratorError::Config(format!( + "golden contract failed on {} (exit {}): {}", + name.as_str(), + output.exit_code, + String::from_utf8_lossy(&output.stderr) + .lines() + .last() + .unwrap_or("unknown error") + ))); } - if let Err(error) = provider.start_forkable(golden).await { - vm_telemetry_deregister(config, golden); - let _ = provider.delete(golden).await; + Ok(()) +} + +/// Ensure the packed golden payload exists at `payload`. +/// +/// The official golden is fetched, never baked: it is published as a pack, and +/// a download that cannot complete fails the caller — the Ubuntu baseline this +/// used to fall back on is gone, and so is the class of failures that came with +/// it (unpinned packages, an apt archive that moved past the pins). A +/// configured image is baked here, as-is plus the runner contract. +/// +/// `shutdown` is only consulted before the expensive work: a warm abandoned +/// mid-transfer resumes on the next start, and an operator restart must not +/// wait out a multi-gigabyte download. +async fn ensure_golden_payload( + provider: &Arc

, + config: &RunnerPoolConfig, + env_spec: &EnvironmentSpec, + payload: &Path, + shutdown: &CancellationToken, +) -> Result<(), OrchestratorError> { + if payload.is_file() { + return Ok(()); + } + if let Some(parent) = payload.parent() { + std::fs::create_dir_all(parent)?; + } + if is_official_golden(&env_spec.base) { + abort_preparation_if_shutting_down(shutdown)?; + info!(payload = %payload.display(), "downloading the official golden"); + return match download_prebaked_golden(payload, &config.release_version).await { + Ok(true) => Ok(()), + Ok(false) => Err(OrchestratorError::Config(format!( + "the official golden could not be downloaded for {}: no OCI image at `{}` and no \ + release asset for v{}. Point PRELOOP_GOLDEN_OCI_REF or PRELOOP_GOLDEN_URL at a \ + reachable golden; Preloop does not bake a substitute image", + std::env::consts::ARCH, + official_golden_reference(), + config.release_version + ))), + Err(error) => Err(error), + }; + } + abort_preparation_if_shutting_down(shutdown)?; + info!( + base_image = %env_spec.base, + "building a golden from the configured image" + ); + build_golden_artifact(provider, config, env_spec, payload).await +} + +/// Bake a packed golden from a configured image: boot it, apply the runner +/// contract once, pack the result. +/// +/// Nothing else is installed. The image is the operator's contract, and its +/// contents are exactly what a job sees — the same rule GitHub-hosted runners +/// follow, where a missing `bash`, `git` or `docker` fails the step that needs +/// it rather than being repaired beforehand. +async fn build_golden_artifact( + provider: &Arc

, + config: &RunnerPoolConfig, + env_spec: &EnvironmentSpec, + payload: &Path, +) -> Result<(), OrchestratorError> { + let name = MachineName::new(format!("{}-builder", config.name_prefix))?; + if provider.status(&name).await? != MachineState::Missing { + provider.delete(&name).await?; + } + // Packing exports a second copy of the guest filesystem before producing + // the artifact. Give the one-shot builder headroom without increasing the + // storage allocated to job VMs. + let builder_storage_gib = golden_builder_storage_gib(config.storage_gib); + // Checked after the stale builder is gone, so its space counts as free. + ensure_disk_for_golden_build(&golden_disk_root(config), builder_storage_gib) + .map_err(OrchestratorError::Config)?; + let spec = MachineSpec { + name: name.clone(), + image: golden_boot_image(env_spec)?, + cpus: config.cpus, + memory_mib: config.memory_mib, + storage_gib: builder_storage_gib, + overlay_gib: config.overlay_gib, + network: NetworkPolicy::PublicOnly, + volumes: Vec::new(), + sockets: Vec::new(), + dns: config.dns.clone(), + rosetta: cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", + }; + provider.create(&spec).await?; + provider.start(&name).await?; + if let Err(error) = await_guest_ready(provider.as_ref(), &name).await { + let _ = provider.delete(&name).await; + return Err(error); + } + if let Err(error) = apply_golden_contract(provider.as_ref(), config, &name).await { + // A golden whose runner account is wrong cannot run a job: every step + // fails on permissions, which reads as flaky CI. A gold build that + // cannot satisfy the contract (no glibc loader, no way to escalate to + // root) must not produce an artifact. + let _ = provider.delete(&name).await; + return Err(error); + } + // Bake the externals *pointer*, not the externals: the packed rootfs + // gets `/externals -> /opt/preloop/bin/externals` so node rides + // the runner-bundle mount instead of being baked into the image or + // downloaded per machine. The symlink must live in the pack itself — + // forkable snapshots do not capture exec writes made after create — + // and it must be baked here, in the builder, where the rootfs layer + // is flattened into the artifact. The bundle's host side carries the + // real `externals/` (see `ensure_host_externals`). + let link_command = format!( + "mkdir -p {root} && rm -rf {root}/externals && \ + ln -s /opt/preloop/bin/externals {root}/externals", + root = RUNNER_ROOT + ); + let output = provider + .exec(&name, &["sh".to_owned(), "-c".to_owned(), link_command]) + .await?; + if output.exit_code != 0 { + let _ = provider.delete(&name).await; + return Err(OrchestratorError::Config(format!( + "baking externals symlink failed (exit {}): {}", + output.exit_code, + String::from_utf8_lossy(&output.stderr) + .lines() + .last() + .unwrap_or("unknown") + ))); + } + if let Err(error) = write_bake_manifest(provider.as_ref(), &name, env_spec).await { + // Provenance is an audit aid, not a build gate. + warn!(machine = name.as_str(), %error, "bake manifest not written"); + } + provider.stop(&name).await?; + let temporary = payload + .parent() + .map(|parent| parent.join(format!(".tmp-golden-{}", uuid::Uuid::new_v4()))) + .ok_or_else(|| { + OrchestratorError::Config(format!( + "golden artifact path has no parent: {}", + payload.display() + )) + })?; + // Claim the companion lock for the pack duration: the heartbeat + // keeps it fresh, and dropping the guard removes it. A staging file + // without a fresh lock reads as orphaned to a concurrent sweep. + let _staging_guard = StagingLockGuard::claim(staging_lock_path(&temporary)); + if let Err(error) = provider.pack(&name, &temporary).await { + let _ = std::fs::remove_file(&temporary); return Err(error.into()); } - // Issue #295: smolvm's pack export leaves ~29 GB of intermediates - // (`storage.ext4`, `layers/*.tar`) beside the finished `storage.raw`, - // and every fork copies the golden's whole data directory. Prune them - // now that the disk is written. Best-effort: never fail the golden - // over disk hygiene. - match provider.prune_pack_intermediates(golden).await { - Ok(true) => info!( - machine = golden.as_str(), - "pruned pack/ build intermediates from packed golden" - ), - Ok(false) => {} - Err(error) => warn!( - machine = golden.as_str(), - %error, - "failed to prune pack/ build intermediates from packed golden" - ), + // smolvm pack writes two files: `` (ELF executable stub) and + // `.smolmachine` (the packed VM data). The latter is the + // artifact consumed by `machine create --from`; the stub is only a + // launcher and is discarded. + let sidecar = PathBuf::from(format!("{}.smolmachine", temporary.display())); + let rename_res = std::fs::rename(&sidecar, payload); + let _ = std::fs::remove_file(&temporary); + rename_res.inspect_err(|_| { + let _ = std::fs::remove_file(&sidecar); + })?; + provider.delete(&name).await?; + if !payload.is_file() { + return Err(OrchestratorError::Config(format!( + "smolvm did not create expected artifact {}", + payload.display() + ))); } - write_golden_record(config, golden, &env_spec.fingerprint); - info!( - machine = golden.as_str(), - artifact = %config.artifact_payload().display(), - "packed golden fork base ready" - ); Ok(()) } @@ -4007,13 +3972,28 @@ impl RunnerPool

{ ensure_host_externals(&self.config)?; // A backend whose packs are not host files (AgentENV keeps them as // server-side snapshots) has no artifact to build, download, or - // relocate: its golden is prepared directly from the base image - // below, and building a pack first would boot and discard a whole - // extra VM for nothing. + // relocate: its golden is prepared directly from the base image by + // `prepare_fork_base` below. + // + // A shutdown that arrives before the warm starts must not begin a + // multi-gigabyte download at all; the caller only needs the pool to + // stop, and the next start resumes from the same point. + if shutdown.is_cancelled() { + self.remove_stale_machines().await?; + return Ok(()); + } if self.provider.capabilities().file_packs - && (self.config.use_packed_artifact || self.config.control_socket.is_none()) + && let Err(error) = self.prepare_artifact(&shutdown).await { - self.prepare_artifact(true).await?; + // A shutdown issued mid-download reads as a preparation error; + // report the stop, not a failure. Nothing is half-adopted: the + // payload is only renamed into place once complete. + if shutdown.is_cancelled() { + info!(%error, "shutdown during artifact preparation; stopping the pool"); + self.remove_stale_machines().await?; + return Ok(()); + } + return Err(error); } self.sweep_stale_artifacts().await; self.remove_stale_machines().await?; @@ -4034,36 +4014,43 @@ impl RunnerPool

{ }, }; - let golden_registry = Arc::new(GoldenRegistry::new(self.config.name_prefix.clone())); + let golden_cache = Arc::new(GoldenCache::new(self.config.name_prefix.clone())); - // If fork mode is enabled, prepare a golden fork base VM for the - // workspace's default environment (base image plus any toolchains - // detected from version files like rust-toolchain.toml). + // Fork mode needs its golden before any job can run, and there is no + // fallback that could serve one: without a fork base a slot would have + // to boot a raw image, which is neither the official golden nor a + // baked contract. Fail startup loudly instead — the server's + // starvation sweep and `pool_status` report the pool as unavailable, + // which is the honest signal, where a warn line plus per-runner + // creation used to look like a working pool that never serves. if self.config.use_fork { - let default_environment = EnvironmentSpec::for_base(self.config.base_image.clone()); - let golden = MachineName::new(format!("{}-golden", golden_registry.name_prefix))?; - let result = if self.config.use_packed_artifact { - prepare_packed_golden(&self.provider, &self.config, &golden).await - } else { - prepare_golden_for_env(&self.provider, &self.config, &golden, &default_environment) - .await - }; - if let Err(error) = result { - if self.config.use_packed_artifact { - golden_registry.disable_packed(); - warn!( - %error, - "packed golden fork base unavailable; using direct per-runner creation \ - instead of building a second environment golden" + let environment = EnvironmentSpec::for_base(self.config.base_image.clone()); + let golden = MachineName::new(format!("{}-golden", golden_cache.name_prefix))?; + // A shutdown arriving mid-warm must not wait for the rest of a + // multi-gigabyte download or an eleven-minute bake: an operator + // restarting the engine should not sit through either. The warm is + // idempotent (an interrupted payload is never adopted, and the + // next start resumes the download), so abandoning it is safe. + if let Err(error) = prepare_fork_base( + &self.provider, + &self.config, + &golden, + &environment, + &shutdown, + ) + .await + { + if shutdown.is_cancelled() { + info!( + machine = golden.as_str(), + "shutdown during golden preparation; leaving it for the next start" ); - } else { - warn!(%error, "golden fork base unavailable; falling back to direct creation"); + self.remove_stale_machines().await?; + return Ok(()); } - } else { - golden_registry - .insert(default_environment.fingerprint, golden) - .await; + return Err(error); } + golden_cache.insert(environment.fingerprint, golden).await; } // The warm is done: the pool can now register runners for queued @@ -4090,7 +4077,7 @@ impl RunnerPool

{ // jobs arrive, capped by the host's CPU and memory budget. if self.config.size == 0 { return self - .run_on_demand(shutdown, golden_registry, idle, keys, building) + .run_on_demand(shutdown, golden_cache, idle, keys, building) .await; } @@ -4130,7 +4117,7 @@ impl RunnerPool

{ let provider = self.provider.clone(); let config = self.config.clone(); let slot_shutdown = shutdown.child_token(); - let slot_registry = golden_registry.clone(); + let slot_registry = golden_cache.clone(); let slot_handles = PoolHandles { idle: idle.clone(), keys: keys.clone(), @@ -4178,7 +4165,7 @@ impl RunnerPool

{ // machine directory (fork snapshot) must survive so the next engine // start can adopt it instead of rebaking. `remove_stale_machines` // spares recorded goldens; the fingerprint record gates adoption. - for golden in golden_registry.all_names().await { + for golden in golden_cache.all_names().await { vm_telemetry_deregister(&self.config, &golden); if let Err(error) = self.provider.stop(&golden).await { warn!( @@ -4198,7 +4185,7 @@ impl RunnerPool

{ async fn run_on_demand( &self, shutdown: CancellationToken, - golden_registry: Arc, + golden_cache: Arc, idle: Arc>, keys: Arc, building: Arc, @@ -4268,7 +4255,7 @@ impl RunnerPool

{ let provider = self.provider.clone(); let config = self.config.clone(); let slot_shutdown = shutdown.child_token(); - let slot_registry = golden_registry.clone(); + let slot_registry = golden_cache.clone(); let slot_handles = PoolHandles { idle: idle.clone(), keys: keys.clone(), @@ -4338,7 +4325,7 @@ impl RunnerPool

{ // snapshot) must survive so the next engine start can adopt the golden // instead of rebaking it. `remove_stale_machines` spares recorded // goldens; the fingerprint record gates adoption. - for golden in golden_registry.all_names().await { + for golden in golden_cache.all_names().await { vm_telemetry_deregister(&self.config, &golden); if let Err(error) = self.provider.stop(&golden).await { warn!( @@ -4352,278 +4339,94 @@ impl RunnerPool

{ Ok(()) } - /// Build a fresh packed runner artifact without downloading or reusing an - /// existing release asset. + /// Build a fresh packed artifact from the configured image, ignoring any + /// payload that already exists. + /// + /// `preloop build-golden` builds an image into a golden with this. The + /// official golden cannot be rebuilt: it is published packed, and a local + /// build of it does not exist (that was the curated Ubuntu bake). pub async fn rebuild_artifact(&self) -> Result<(), OrchestratorError> { + if is_official_golden(&self.config.base_image) { + return Err(OrchestratorError::Config( + "the official golden is published packed and cannot be built locally; \ + pass the image to build from (`--base-image `)" + .into(), + )); + } let payload = self.config.artifact_payload(); match std::fs::remove_file(&payload) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(error.into()), } - self.prepare_artifact(false).await + self.prepare_artifact(&CancellationToken::new()).await } - async fn prepare_artifact(&self, allow_download: bool) -> Result<(), OrchestratorError> { - let payload = self.config.artifact_payload(); - if payload.is_file() { - return Ok(()); - } - if let Some(parent) = self.config.artifact_stem.parent() { - std::fs::create_dir_all(parent)?; - } - let custom_golden_url = std::env::var("PRELOOP_GOLDEN_URL") - .ok() - .is_some_and(|value| !value.trim().is_empty()); - if allow_download - && should_download_prebaked_golden(&self.config.base_image, custom_golden_url) - { - if download_prebaked_golden(&payload, &self.config.release_version).await? { - return Ok(()); - } - } else if allow_download { - info!( - base_image = %self.config.base_image, - "custom base image has no PRELOOP_GOLDEN_URL; building its golden locally" - ); - } - - let name = MachineName::new(format!("{}-builder", self.config.name_prefix))?; - if self.provider.status(&name).await? != MachineState::Missing { - self.provider.delete(&name).await?; - } - // Packing exports a second copy of the guest filesystem before - // producing the artifact. Give the one-shot builder headroom - // without increasing the storage allocated to job VMs. - let builder_storage_gib = golden_builder_storage_gib(self.config.storage_gib); - // Checked after the stale builder is gone, so its space counts as free. - ensure_disk_for_golden_build(&golden_disk_root(&self.config), builder_storage_gib) - .map_err(OrchestratorError::Config)?; - let spec = MachineSpec { - name: name.clone(), - image: self.config.base_image.clone(), - cpus: self.config.cpus, - memory_mib: self.config.memory_mib, - storage_gib: builder_storage_gib, - overlay_gib: self.config.overlay_gib, - network: NetworkPolicy::PublicOnly, - volumes: Vec::new(), - sockets: Vec::new(), - dns: self.config.dns.clone(), - rosetta: cfg!(target_os = "macos") && std::env::consts::ARCH == "aarch64", - }; - self.provider.create(&spec).await?; - self.provider.start(&name).await?; - // Plain Ubuntu needs the hosted-runner package baseline. Official - // runner snapshots already contain it and workflow setup actions own - // language-toolchain selection. - let stock_base = is_stock_base_image(&self.config.base_image); - if stock_base - && let Err(error) = install_base_dependencies(self.provider.as_ref(), &name).await - { - let _ = self.provider.delete(&name).await; - return Err(error); - } - if stock_base { - for layer in curated_toolchains() { - for command in layer.install_commands() { - let output = self.provider.exec(&name, &command).await?; - if output.exit_code != 0 { - let _ = self.provider.delete(&name).await; - return Err(OrchestratorError::Config(format!( - "toolchain install failed for {layer} (exit {}): {}", - output.exit_code, - String::from_utf8_lossy(&output.stderr) - .lines() - .last() - .unwrap_or("unknown error") - ))); - } - } - } - // Toolchain installation runs as root and recreates writable - // rustup state beneath these homes. Re-apply runner ownership - // after every layer; doing it only in the base script leaves - // `/usr/local/rustup/tmp` root-owned and job-time rustup updates - // fail with EACCES. - let output = self - .provider - .exec( - &name, - &[ - "sh".to_owned(), - "-c".to_owned(), - runner_account_script(DEFAULT_RUNNER_USER, DEFAULT_RUNNER_UID), - ], - ) - .await?; - if output.exit_code != 0 { - let _ = self.provider.delete(&name).await; - return Err(OrchestratorError::Config(format!( - "final runner-account ownership failed (exit {}): {}", - output.exit_code, - String::from_utf8_lossy(&output.stderr) - .lines() - .last() - .unwrap_or("unknown error") - ))); - } - } - // Stamp apt-index freshness into the image. Provisioning reads this - // back to log the pack's age and warn past the versions.toml policy; - // the weekly apt-indices-refresh workflow rebuilds stale packs. The - // policy travels in the marker so a running engine never needs the - // policy compiled in — only the bake-time CLI does. Never fail the - // bake for telemetry: a missing marker just means "stale", which is - // today's behavior (full refresh) everywhere. - if stock_base { - let stamp = format!( - "mkdir -p /etc/preloop && printf '%s\\n%s\\n' \"$(date -u +%F)\" \"{}\" > /etc/preloop/apt-indices-baked-at", - crate::APT_INDICES_MAX_AGE_DAYS - ); - if let Err(error) = self - .provider - .exec(&name, &["sh".to_owned(), "-c".to_owned(), stamp]) - .await - { - warn!( - machine = name.as_str(), - %error, - "apt-index freshness stamp failed; pack will read as stale" - ); - } - } - // Bake the externals *pointer*, not the externals: the packed rootfs - // gets `/externals -> /opt/preloop/bin/externals` so node rides - // the runner-bundle mount instead of being baked into the image or - // downloaded per machine. The symlink must live in the pack itself — - // forkable snapshots do not capture exec writes made after create — - // and it must be baked here, in the builder, where the rootfs layer - // is flattened into the artifact. The bundle's host side carries the - // real `externals/` (see `ensure_host_externals`). - let link_command = format!( - "mkdir -p {root} && rm -rf {root}/externals && \ - ln -s /opt/preloop/bin/externals {root}/externals", - root = RUNNER_ROOT - ); - let output = self - .provider - .exec(&name, &["sh".to_owned(), "-c".to_owned(), link_command]) - .await?; - if output.exit_code != 0 { - let _ = self.provider.delete(&name).await; - return Err(OrchestratorError::Config(format!( - "baking externals symlink failed (exit {}): {}", - output.exit_code, - String::from_utf8_lossy(&output.stderr) - .lines() - .last() - .unwrap_or("unknown") - ))); - } - // `base_install_script` already prepared the default account for - // curated bases, and provisioning reconciles it again before configure - // (custom bases get the fix there for the first time). Re-run it here - // only for a configured non-default user, whose identity the - // fingerprint cannot know: the script is idempotent either way. - let runner_user = self - .config - .runner_user - .as_deref() - .unwrap_or(DEFAULT_RUNNER_USER); - let runner_uid = self.config.runner_uid.unwrap_or(DEFAULT_RUNNER_UID); - if runner_user != DEFAULT_RUNNER_USER || runner_uid != DEFAULT_RUNNER_UID { - let output = self - .provider - .exec( - &name, - &[ - "sh".to_owned(), - "-c".to_owned(), - runner_account_script(runner_user, runner_uid), - ], - ) - .await?; - if output.exit_code != 0 { - // A golden whose runner account is wrong cannot run a job: - // every step fails on permissions, which reads as flaky CI. - let _ = self.provider.delete(&name).await; - return Err(OrchestratorError::Config(format!( - "baking runner account {runner_user} failed (exit {}): {}", - output.exit_code, - String::from_utf8_lossy(&output.stderr) - .lines() - .last() - .unwrap_or("unknown") - ))); - } - } + /// Ensure the packed artifact for the pool's configured image exists: + /// download the official golden, or bake a configured image into a pack. + async fn prepare_artifact( + &self, + shutdown: &CancellationToken, + ) -> Result<(), OrchestratorError> { let env_spec = EnvironmentSpec::for_base(self.config.base_image.clone()); - if let Err(error) = write_bake_manifest(self.provider.as_ref(), &name, &env_spec).await { - // Provenance is an audit aid, not a build gate. - warn!(machine = name.as_str(), %error, "bake manifest not written"); - } - self.provider.stop(&name).await?; - let temporary = payload - .parent() - .map(|parent| parent.join(format!(".tmp-golden-{}", uuid::Uuid::new_v4()))) - .ok_or_else(|| { - OrchestratorError::Config(format!( - "golden artifact path has no parent: {}", - payload.display() - )) - })?; - // Claim the companion lock for the pack duration: the heartbeat - // keeps it fresh, and dropping the guard removes it. A staging file - // without a fresh lock reads as orphaned to a concurrent sweep. - let _staging_guard = StagingLockGuard::claim(staging_lock_path(&temporary)); - if let Err(error) = self.provider.pack(&name, &temporary).await { - let _ = std::fs::remove_file(&temporary); - return Err(error.into()); - } - // smolvm pack writes two files: `` (ELF executable stub) and - // `.smolmachine` (the packed VM data). The latter is the - // artifact consumed by `machine create --from`; the stub is only a - // launcher and is discarded. - let sidecar = PathBuf::from(format!("{}.smolmachine", temporary.display())); - let rename_res = std::fs::rename(&sidecar, &payload); - let _ = std::fs::remove_file(&temporary); - rename_res.inspect_err(|_| { - let _ = std::fs::remove_file(&sidecar); - })?; - self.provider.delete(&name).await?; - if !payload.is_file() { - return Err(OrchestratorError::Config(format!( - "smolvm did not create expected artifact {}", - payload.display() - ))); - } - Ok(()) + let payload = self.config.artifact_payload(); + ensure_golden_payload(&self.provider, &self.config, &env_spec, &payload, shutdown).await } - async fn remove_stale_machines(&self) -> Result<(), OrchestratorError> { + /// Delete every machine this pool owns that the startup adoption path + /// cannot reuse. + /// + /// A golden fork base recorded by a previous engine run is not stale: its + /// machine directory plus the fingerprint record are exactly what adoption + /// needs. Deleting it here forced a full rebake on every restart (#293); + /// the prepare path rebuilds it anyway when the fingerprint no longer + /// matches. + /// + /// The exception is a golden no environment can resolve to any more + /// ([`retired_golden`]): the curated per-`runs-on` goldens of earlier + /// releases. Their records would otherwise protect them forever, since + /// nothing prepares them again. + async fn remove_stale_runner_machines(&self) -> Result<(), OrchestratorError> { for name in self.provider.list().await? { - if name + if !name .as_str() .starts_with(&format!("{}-", self.config.name_prefix)) { - // A golden fork base recorded by a previous engine run is not - // stale: its machine directory plus the fingerprint record are - // exactly what the startup adoption path needs. Deleting it - // here forced a full rebake on every restart (#293); the - // prepare path rebuilds it anyway when the fingerprint no - // longer matches. - if golden_record_path(&self.config, &name).is_some_and(|path| path.is_file()) { - continue; - } + continue; + } + let retired = retired_golden(&self.config, &name); + if !retired + && golden_record_path(&self.config, &name).is_some_and(|path| path.is_file()) + { + continue; + } + if !retired { notify_runner_gone(&self.config, &name).await; - vm_telemetry_deregister(&self.config, &name); - if let Err(error) = self.provider.delete(&name).await { + } + vm_telemetry_deregister(&self.config, &name); + match self.provider.delete(&name).await { + // The record goes only with the machine: a failed delete keeps + // it, so the next start retries instead of forgetting a golden + // that still holds its storage. + Ok(()) if retired => { + remove_golden_record(&self.config, &name); + info!( + machine = name.as_str(), + "removed a golden no pool environment resolves to any more" + ); + } + Ok(()) => {} + Err(error) => { record_slot_failure(&self.config, "stale_cleanup"); warn!(machine = name.as_str(), %error, "failed to delete stale Preloop runner"); } } } + Ok(()) + } + + async fn remove_stale_machines(&self) -> Result<(), OrchestratorError> { + self.remove_stale_runner_machines().await?; // A crashed server orphans its detached `_boot-vm` hypervisor // processes; when the data dir was cleaned out from under them the // smolvm DB no longer knows the machines, so the deletes above @@ -4706,7 +4509,11 @@ impl RunnerPool

{ && suffix[1..].chars().all(|c| c.is_ascii_hexdigit()) }) }) && path != current_payload; - if !stale_tmp && !stale_payload { + // Payloads under a stem this pool no longer produces: the stock + // Ubuntu bases are not golden sources any more, so no later + // fingerprint rotation would ever reach them. + let retired_payload = is_retired_stock_payload(&name) && path != current_payload; + if !stale_tmp && !stale_payload && !retired_payload { continue; } match std::fs::remove_file(&path) { @@ -4847,14 +4654,11 @@ async fn notify_runner_gone(config: &RunnerPoolConfig, name: &MachineName) { struct RunnerEnvironment { /// Fingerprint of the golden this runner was forked from. fingerprint: Option, - /// Base image this runner actually booted from. + /// Base image this runner actually booted from: the official golden, or the + /// configured image. Nothing is installed on top — the golden carries the + /// runner contract, and `runner_environment_labels` derives the labels a + /// job's `runs-on` can match from this value. base: String, - /// Toolchains this runner must carry (installed after boot when the - /// runner is created fresh rather than forked from a prepared golden). - toolchains: Vec, - /// Whether Preloop's curated bake applies to this base. Custom base - /// images are used as-is and must not receive the apt/toolchain bake. - curated: bool, } /// Handles every slot in the pool shares. @@ -5249,20 +5053,12 @@ async fn run_on_demand_slot( config: RunnerPoolConfig, slot: usize, shutdown: CancellationToken, - golden_registry: Arc, + golden_cache: Arc, handles: PoolHandles, _slot_provisioning: Arc>, semaphore: Arc, permit: Arc>>, ) -> Result<(), OrchestratorError> { - // Mirror `run_slot`: once the packed golden is known-unusable, on-demand - // slots must not attempt another environment-golden bake either — fall - // back to direct per-runner creation the same way. - let mut config = config; - if golden_registry.is_packed_disabled() { - config.use_packed_artifact = false; - config.use_fork = false; - } let provisioning = handles.provisioning.clone(); let preparing = PreparingGuard::enter( provisioning.clone(), @@ -5279,22 +5075,21 @@ async fn run_on_demand_slot( None => config.base_image.clone(), }; let env_spec = EnvironmentSpec::for_base(env_base.clone()); - let curated = env_spec.curated; let fingerprint = env_spec.fingerprint.clone(); - let toolchains = env_spec.toolchains.clone(); - let selected = golden_registry + let selected = golden_cache .get_or_prepare(&fingerprint, { let provider = provider.clone(); let config = config.clone(); - let name_prefix = golden_registry.name_prefix().to_owned(); + let name_prefix = golden_cache.name_prefix().to_owned(); let fp = fingerprint.clone(); + let shutdown = shutdown.clone(); async move { let name = MachineName::new(format!( "{}-golden-{}", name_prefix, &fp[..12.min(fp.len())] ))?; - prepare_golden_for_env(&provider, &config, &name, &env_spec).await?; + prepare_fork_base(&provider, &config, &name, &env_spec, &shutdown).await?; Ok(name) } }) @@ -5308,20 +5103,15 @@ async fn run_on_demand_slot( RunnerEnvironment { fingerprint: Some(fingerprint), base: env_base, - toolchains, - curated, }, ) } else { let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - let curated = env_spec.curated; ( None, RunnerEnvironment { fingerprint: None, - base: env_spec.base.clone(), - toolchains: env_spec.toolchains, - curated, + base: env_spec.base, }, ) }; @@ -5406,14 +5196,9 @@ async fn run_slot( config: RunnerPoolConfig, slot: usize, shutdown: CancellationToken, - golden_registry: Arc, + golden_cache: Arc, handles: PoolHandles, ) -> Result<(), OrchestratorError> { - let mut config = config; - if golden_registry.is_packed_disabled() { - config.use_packed_artifact = false; - config.use_fork = false; - } let PoolHandles { idle, keys, @@ -5448,26 +5233,23 @@ async fn run_slot( } None => config.base_image.clone(), }; - // The golden carries the curated toolchain set; base image still - // comes from the queued job's `runs-on` labels. let env_spec = EnvironmentSpec::for_base(env_base.clone()); - let curated = env_spec.curated; let fingerprint = env_spec.fingerprint.clone(); - let toolchains = env_spec.toolchains.clone(); - let selected = match golden_registry + let selected = match golden_cache .get_or_prepare(&fingerprint, { let provider = provider.clone(); let config = config.clone(); - let name_prefix = golden_registry.name_prefix().to_owned(); + let name_prefix = golden_cache.name_prefix().to_owned(); let fp = fingerprint.clone(); + let shutdown = shutdown.clone(); async move { let name = MachineName::new(format!( "{}-golden-{}", name_prefix, &fp[..12.min(fp.len())] ))?; - prepare_golden_for_env(&provider, &config, &name, &env_spec).await?; + prepare_fork_base(&provider, &config, &name, &env_spec, &shutdown).await?; Ok(name) } }) @@ -5478,15 +5260,16 @@ async fn run_slot( Some(name) } Err(error) => { - // A bake failure is usually deterministic (a stock apt - // pin the archive dropped, a registry the guest cannot - // resolve), so a fixed 500 ms retry meant this slot - // rebuilt the same doomed golden ~100 times an hour -- - // each attempt boots a VM and runs apt -- while the - // queue it was meant to drain starved. Back off - // geometrically, capped, so a slot costs one attempt per - // minute at worst and recovers immediately once the - // environment becomes buildable again. + // A golden prep failure is usually deterministic (the + // official pack unreachable, a registry the guest cannot + // resolve, an image that cannot satisfy the contract), so a + // fixed 500 ms retry meant this slot retried the same + // doomed prepare ~100 times an hour -- each attempt + // downloading or booting a VM -- while the queue it was + // meant to drain starved. Back off geometrically, capped, + // so a slot costs one attempt per minute at worst and + // recovers immediately once the golden becomes available + // again. let wait = golden_backoff.map_or(GOLDEN_RETRY_MIN, |last: Duration| { (last * 2).min(GOLDEN_RETRY_MAX) }); @@ -5509,21 +5292,16 @@ async fn run_slot( RunnerEnvironment { fingerprint: Some(fingerprint), base: env_base, - toolchains, - curated, }, ) } else { // create-per-runner path: no golden, provision fresh each time. let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - let curated = env_spec.curated; ( None, RunnerEnvironment { fingerprint: None, - base: env_spec.base.clone(), - toolchains: env_spec.toolchains, - curated, + base: env_spec.base, }, ) }; @@ -5690,7 +5468,19 @@ async fn provision_slot( } } +/// Labels a runner booted from `base` can serve beyond the pool's own. +/// +/// The official golden *is* the hosted Ubuntu 24.04 image, so a job that says +/// `runs-on: ubuntu-latest` can be matched to it. A configured image keeps the +/// same name-based mapping it had before the bake was deleted — an operator who +/// brings `ubuntu:22.04` still serves `runs-on: ubuntu-22.04` — but nothing +/// about it is *provisioned* to match: `base_for_labels` only routes labels to +/// the official golden, so a configured pool serves every label from its own +/// image. fn runner_environment_labels(base: &str) -> Vec { + if is_official_golden(base) { + return vec!["ubuntu-24.04".to_owned(), "ubuntu-latest".to_owned()]; + } let normalized = base.to_ascii_lowercase(); if normalized.contains("22.04") { vec!["ubuntu-22.04".to_owned()] @@ -6189,6 +5979,83 @@ fn plain_packed_golden_name(config: &RunnerPoolConfig) -> String { format!("{}-golden", config.name_prefix) } +/// Whether `golden` is a per-environment fork base no environment resolves to +/// any more. +/// +/// A pool resolves a job to exactly two bases: the official golden or its +/// configured image (`EnvironmentSpec::base_for_labels`). A suffixed golden +/// (`{prefix}-golden-{fingerprint12}`) whose suffix is neither base's +/// fingerprint was prepared for an environment that no longer exists — the +/// stock Ubuntu environments of earlier releases, or an image since replaced — +/// and nothing will ever prepare or adopt it again. The plain golden is never +/// retired here: `prepare_fork_base` replaces it in place. +fn retired_golden(config: &RunnerPoolConfig, golden: &MachineName) -> bool { + let prefix = plain_packed_golden_name(config); + let Some(suffix) = golden + .as_str() + .strip_prefix(&prefix) + .and_then(|rest| rest.strip_prefix('-')) + else { + return false; + }; + if suffix.len() != 12 || !suffix.bytes().all(|b| b.is_ascii_hexdigit()) { + return false; + } + ![ + crate::environment::OFFICIAL_GOLDEN, + config.base_image.as_str(), + ] + .iter() + .any(|base| { + EnvironmentSpec::for_base((*base).to_owned()) + .fingerprint + .starts_with(suffix) + }) +} + +/// Whether a file name is a packed-golden payload of a retired stock Ubuntu +/// stem: `preloop-[mirror.gcr.io-library-]ubuntu-{24.04,22.04}[-sha256-<64 +/// hex>]--<64 hex>`. +/// +/// Earlier releases keyed the payload on the stock base they baked from; the +/// stem is the base reference with `/`, `:` and `@` turned into `-` +/// (`artifact_stem`). The architecture and both digests must match, so a +/// hand-named file or another image's payload is never taken for one. +fn is_retired_stock_payload(name: &str) -> bool { + let is_hex = |text: &str| !text.is_empty() && text.bytes().all(|b| b.is_ascii_hexdigit()); + let Some(rest) = name.strip_prefix("preloop-") else { + return false; + }; + let Some(split) = rest.len().checked_sub(65) else { + return false; + }; + if !rest.is_char_boundary(split) { + return false; + } + let (stem, fingerprint) = rest.split_at(split); + if !fingerprint.starts_with('-') || fingerprint[1..].len() != 64 || !is_hex(&fingerprint[1..]) { + return false; + } + let stem = stem.strip_prefix("mirror.gcr.io-library-").unwrap_or(stem); + let Some(after_release) = stem + .strip_prefix("ubuntu-24.04") + .or_else(|| stem.strip_prefix("ubuntu-22.04")) + else { + return false; + }; + let arch = match after_release.strip_prefix("-sha256-") { + Some(tail) => match tail.split_once('-') { + Some((digest, arch)) if digest.len() == 64 && is_hex(digest) => arch, + _ => return false, + }, + None => match after_release.strip_prefix('-') { + Some(arch) => arch, + None => return false, + }, + }; + matches!(arch, "x86_64" | "aarch64") +} + /// Best-effort VM telemetry: register a just-created or forked machine. /// /// Fills what the pool knows about the machine; unknown fields stay unset. @@ -6279,28 +6146,15 @@ async fn provision_runner( keys: &Arc, environment: &RunnerEnvironment, ) -> Result, OrchestratorError> { - let mut direct_create_from_packed = config.use_packed_artifact; + // Every runner is a fork of the pool's golden. There is no backup path that + // boots a raw image: such a machine carries neither the published golden's + // contents nor the contract this pool baked, so it would register as a + // runner that cannot run the job it was created for. A fork that cannot be + // recovered here is an error, and the slot retries it with backoff. let forked_golden = match golden { Some(golden) => match provider.fork(golden, name).await { Ok(()) => Some(golden), - Err(error @ VmError::ForkBaseBusy { .. }) - if config.use_packed_artifact && managed_golden(config, golden) => - { - // A live plain-fork clone still depends on the golden's frozen - // storage. Do not touch the base and do not create another VM - // from that same packed payload: SmolVM's mixed fork/create - // path has returned ESTALE in both machines. Boot this slot - // independently from the job's OCI environment instead. - warn!( - machine = name.as_str(), - golden = golden.as_str(), - %error, - "fork base busy; creating runner independently from the OCI image" - ); - direct_create_from_packed = false; - None - } - Err(error) if config.use_packed_artifact && managed_golden(config, golden) => { + Err(error) if managed_golden(config, golden) => { if fork_base_unusable(&error) { // The base is spent. Re-arm it atomically with forking: // partial-clone cleanup, the live-clone check, and the @@ -6324,8 +6178,7 @@ async fn provision_runner( machine = name.as_str(), golden = golden.as_str(), %retry_error, - "re-armed golden still cannot fork; falling back to \ - direct creation" + "re-armed golden still cannot fork" ); let _ = provider.delete(name).await; None @@ -6336,15 +6189,12 @@ async fn provision_runner( // A live clone (another runner forked from the // golden) blocks the re-freeze; those clones are // ephemeral and exit after their job. Wait for - // them to drain, probing with exponential backoff - // so a long-running job does not force the slow - // direct-create path for every queued job in the - // meantime. The fork path is ~0.5 s vs ~8 min for - // independent creation, so waiting is worth it up - // to a generous total budget; only then fall back - // to direct creation (whose socket mount cannot - // serve the control transport, so the fallback - // usually fails registration anyway). + // them to drain, probing with exponential backoff. + // The fork path is ~0.5 s, so waiting is worth it + // up to a generous total budget; past it the + // provision fails and the slot retries, which is + // the only honest outcome — no image boot can + // stand in for a fork of this golden. let mut rearmed = false; let mut probe_delay = GOLDEN_DRAIN_PROBE_DELAY; let drain_deadline = tokio::time::Instant::now() + GOLDEN_DRAIN_BUDGET; @@ -6389,42 +6239,42 @@ async fn provision_runner( machine = name.as_str(), golden = golden.as_str(), %retry_error, - "re-armed golden still cannot fork; falling back to \ - direct creation" + "re-armed golden still cannot fork" ); let _ = provider.delete(name).await; - None + return Err(retry_error.into()); } } } else { error!( golden = golden.as_str(), "fork base spent and could not be re-armed after waiting for \ - clone drain; falling back to independent OCI creation" + clone drain; the slot will retry" ); let _ = provider.delete(name).await; - direct_create_from_packed = false; - None + return Err(OrchestratorError::Config(format!( + "golden {} is spent and could not be re-armed within {}s", + golden.as_str(), + GOLDEN_DRAIN_BUDGET.as_secs() + ))); } } Err(rearm_error) => { error!( golden = golden.as_str(), %rearm_error, - "failed to re-arm spent fork base; falling back to independent \ - OCI creation" + "failed to re-arm spent fork base" ); - direct_create_from_packed = false; - None + return Err(rearm_error.into()); } } } else { // A restored clone can wedge transiently (agent readiness // or rejuvenation timeouts) while the golden stays frozen // with its retained checkpoint, so the fork itself is safe - // to retry. A retry costs about a second; the direct - // create cold-boots the packed image, which takes minutes - // on macOS while the guest unpacks its layers. + // to retry. A retry costs about a second; rebooting the + // packed image cold takes minutes on macOS while the guest + // unpacks its layers, so the retries are worth it. let mut last_error = error; let mut retried = None; for attempt in 1..=FORK_RETRY_ATTEMPTS { @@ -6434,7 +6284,7 @@ async fn provision_runner( golden = golden.as_str(), attempt, error = %last_error, - "packed golden fork failed; retrying the fork" + "golden fork failed; retrying the fork" ); match provider.fork(golden, name).await { Ok(()) => { @@ -6444,21 +6294,19 @@ async fn provision_runner( Err(error) => last_error = error, } } - if retried.is_none() { - // A failed fork can leave a partial clone behind. - // Best-effort cleanup makes the direct create safe; if - // cleanup itself is still racing SmolVM state, create - // returns the actionable error and the slot supervisor - // retries normally. - warn!( - machine = name.as_str(), - golden = golden.as_str(), - error = %last_error, - "packed golden fork failed; creating runner directly from packed artifact" - ); + if let Some(golden) = retried { + Some(golden) + } else { + // A failed fork can leave a partial clone behind, and + // the slot retries this provision. remove_failed_fork(provider, name).await; + return Err(OrchestratorError::Config(format!( + "forking {} from {} failed after {} attempts: {last_error}", + name.as_str(), + golden.as_str(), + FORK_RETRY_ATTEMPTS + ))); } - retried } } Err(error) => return Err(error.into()), @@ -6466,144 +6314,37 @@ async fn provision_runner( None => None, }; - if let Some(golden) = forked_golden { - // The fork succeeded, so the clone exists as a live machine. + if forked_golden.is_some() { + // The fork succeeded, so the clone exists as a live machine — and it + // carries the whole golden: a pack has its rootfs flattened into the + // artifact, a snapshot backend carries the guest writes that built it. + // Nothing is installed here. The runner contract (account, ownership, + // tool cache, `/etc/environment`) was applied once, when the golden was + // built, which is also why no per-VM `chown` walk exists: on overlayfs + // a recursive one copies the tree into the per-VM upper layer. vm_telemetry_register(config, name, "runner", None); - // Fork from the already-booted golden VM instant CoW clone. - // The PACKED golden carries its bake inside the artifact's flattened - // rootfs, which forks inherit through the storage chain — so the apt - // baseline is already there. Environment goldens are different: - // `prepare_golden_for_env` - // bakes via guest `exec`, and SmolVM's forkable snapshot does NOT - // carry post-create exec writes into clones (verified empirically), - // so an env-golden fork boots the bare stock base image. Install the - // apt baseline into the fork itself — it is the job's single-use - // machine, so the writes persist for its lifetime. Language versions - // remain the workflow's setup action responsibility. - // Only the plain `{prefix}-golden` fork base is created from the - // packed artifact (`prepare_packed_golden` at pool startup), whose - // rootfs already carries the apt baseline that forks inherit. - // Fingerprint-suffixed goldens are baked by - // `prepare_golden_for_env` from the job's OCI image via guest exec, - // and SmolVM's forkable snapshot does NOT carry post-create exec - // writes into clones — so those forks must install the baseline - // themselves. Treating an env golden as packed skipped that install - // and provisioned runners without the curated baseline. - // A fork arrives already baked in two cases: it came from the packed - // artifact (the bake is inside the flattened rootfs), or the backend's - // snapshot carries the golden's post-boot writes. AgentENV does the - // latter; SmolVM does neither for an environment golden, which is why - // that case still installs the baseline per fork. Getting this wrong - // on a write-inheriting backend re-runs the whole bake (apt baseline, - // rust, go, docker tooling) inside every single-use runner. - let golden_is_baked = (config.use_packed_artifact - && golden.as_str() == plain_packed_golden_name(config)) - || provider.capabilities().fork_inherits_guest_writes; - if golden_is_baked { - // The pack carries the apt baseline, but not necessarily apt's - // indices — restore them before any workflow apt-installs. A - // custom base is used as-is: no apt assumptions. - if environment.curated - && let Err(error) = provider.exec(name, &apt_lists_refresh_command()).await - { - warn!( - machine = name.as_str(), - %error, "apt list refresh failed; workflow apt installs may not resolve" - ); - } - // Log the pack's apt-index age from the freshness marker baked - // with it. A missing marker just means "stale" (packs predating - // the stamp, or env-golden forks that boot bare) — the refresh - // above already ran, so behavior is unchanged; only the warning - // is new, and the weekly apt-indices-refresh rebuilds stale packs. - match provider - .exec( - name, - &["cat".to_owned(), APT_INDICES_MARKER_PATH.to_owned()], - ) - .await - { - Ok(output) if output.exit_code == 0 => { - let today_days = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|elapsed| elapsed.as_secs() / 86_400) - .unwrap_or(0); - let text = String::from_utf8_lossy(&output.stdout); - match crate::environment::apt_marker_age_days(&text, today_days) { - Some((age_days, max_age_days)) if age_days > max_age_days => { - warn!( - machine = name.as_str(), - age_days, - max_age_days, - "pack apt indices are stale; refresh ran but consider rebuilding the golden" - ); - } - Some((age_days, _)) => { - info!(machine = name.as_str(), age_days, "pack apt indices fresh"); - } - None => { - debug!( - machine = name.as_str(), - "pack has no parseable apt-index marker" - ); - } - } - } - _ => { - debug!( - machine = name.as_str(), - "pack has no apt-index marker; treating as stale" - ); - } - } - } else if environment.curated { - install_base_dependencies(provider.as_ref(), name).await?; - for layer in &environment.toolchains { - for command in layer.install_commands() { - if let Err(error) = provider.exec(name, &command).await { - return Err(error.into()); - } - } - verify_toolchain_installed(provider.as_ref(), name, layer).await?; - } - } else { - // Custom base image: used as-is, no apt bake, no toolchains. - debug!( - machine = name.as_str(), - base = %environment.base, - "custom base image — skipping the curated bake" - ); - } } else { - // The packed-artifact fallback is valid only for the plain packed - // golden: it boots the *default* OS image. A fingerprint-suffixed - // golden is an environment golden baked from the job's requested - // image, so every fork-failure fallback must boot that job's own - // environment instead — regardless of which branch failed the fork. - // When no golden was attempted (`golden` is None, the create-per- - // runner path), the packed artifact is the pool's normal image and - // stays as-is. - let golden_is_plain_packed = match golden { - Some(golden) => golden.as_str() == plain_packed_golden_name(config), - None => true, - }; - let uses_packed_artifact = direct_create_from_packed && golden_is_plain_packed; + // No golden at all: `use_fork` is off, so the machine is created from + // the golden's payload instead of forked from it. A file-pack backend + // boots the packed artifact — the same rootfs a fork would have + // inherited — while a backend without host-side packs boots the + // configured image and runs the contract on the machine itself, since + // there is no golden to inherit it from. + let from_packed = provider.capabilities().file_packs; let pack = packed_golden_path(&config.artifact_payload()); let spec = MachineSpec { name: name.clone(), - image: if uses_packed_artifact { + image: if from_packed { pack.display().to_string() - } else if config.use_packed_artifact { - environment.base.clone() } else { - config.base_image.clone() + environment.base.clone() }, cpus: config.cpus, memory_mib: config.memory_mib, storage_gib: config.storage_gib, overlay_gib: config.overlay_gib, network: NetworkPolicy::PublicOnly, - volumes: runner_volumes(config, name, !uses_packed_artifact)?, + volumes: runner_volumes(config, name, !from_packed)?, sockets: config .control_socket .iter() @@ -6618,59 +6359,16 @@ async fn provision_runner( provider.create(&spec).await?; provider.start(name).await?; vm_telemetry_register(config, name, "runner", Some(&spec)); - // The packed artifact is the golden's frozen image; the live golden - // receives the apt baseline and toolchain bake *after* boot, so a - // machine created from the artifact is bare and must install the - // baseline itself — otherwise node actions die with "curl: command - // not found" and rust jobs with "cargo: command not found". The - // installs are idempotent, so a fully baked artifact only pays the - // presence checks. A custom base is the operator's contract: no apt - // baseline, no toolchain curation. - if environment.curated { - install_base_dependencies(provider.as_ref(), name).await?; - for layer in &environment.toolchains { - for command in layer.install_commands() { - if let Err(error) = provider.exec(name, &command).await { - return Err(error.into()); - } - } - verify_toolchain_installed(provider.as_ref(), name, layer).await?; - } + if !from_packed { + await_guest_ready(provider.as_ref(), name).await?; + apply_golden_contract(provider.as_ref(), config, name).await?; } } - // Reconcile ownership of the paths a job writes on every machine, whatever - // its provenance: curated stock bases, custom/official images, and forked - // goldens alike. `base_install_script` runs the full account script only - // inside the curated bake, so custom bases reached jobs with root-owned - // `/usr/local/rustup`, `/usr/local/cargo`, and `/opt/hostedtoolcache` — and - // any step that writes them (`rustup component add`, `cargo fmt`, toolcache - // drops) died with EACCES. This is the ownership half only: privilege - // policy stays where it was baked. One exec round trip, and the script - // itself no-ops when the machine is already correct. - { - let runner_user = config.runner_user.as_deref().unwrap_or(DEFAULT_RUNNER_USER); - let runner_uid = config.runner_uid.unwrap_or(DEFAULT_RUNNER_UID); - let reconcile = runner_ownership_reconcile_script(runner_user, runner_uid); - let output = provider - .exec(name, &["sh".to_owned(), "-c".to_owned(), reconcile]) - .await?; - if output.exit_code != 0 { - return Err(OrchestratorError::Config(format!( - "runner-ownership reconciliation failed on {} (exit {}): {} — \ - /usr/local/rustup, /usr/local/cargo, and /opt/hostedtoolcache must be \ - writable by uid {}; the engine could not escalate (image user is not \ - root and passwordless sudo is unavailable)", - name.as_str(), - output.exit_code, - String::from_utf8_lossy(&output.stderr) - .lines() - .last() - .unwrap_or("unknown"), - runner_uid - ))); - } - } + // No per-VM ownership pass: the golden already owns every path a job + // writes, and a fork/snapshot inherits that. A `chown` here used to be the + // safety net for curated stock bases; with the contract baked once it would + // only walk (and on overlayfs, copy) a tree that is already correct. // Apply the hosted runtime init — the per-guest half of what the hosted // image's own init does for a GitHub-hosted runner. Same always-run, @@ -6940,7 +6638,7 @@ fn as_runner_user(config: &RunnerPoolConfig, argv: &[String]) -> Vec { if user == "root" { return with_guest_hosted_limits(argv); } - let uid = config.runner_uid.unwrap_or(1001); + let uid = config.runner_uid.unwrap_or(DEFAULT_RUNNER_UID); let home = format!("/home/{user}"); let program = shell_quote(&argv[0]); let args = argv[1..] @@ -6954,6 +6652,15 @@ fn as_runner_user(config: &RunnerPoolConfig, argv: &[String]) -> Vec { // the official golden declares USER runner, so `machine exec` lands on // runner and setpriv below self-drops to the same uid (no privilege // change needed). + // + // No ownership `chown` here: the golden owns every path a job writes + // (see `golden_ownership_script`), and a fork inherits that. A per-VM walk + // would be pure cost — on overlayfs a recursive one copies the tree it + // visits into the VM's upper layer (measured: 304 s and 1.22 GB on the + // production runner home). The tool-cache mode and `/etc/environment` + // lines below stay as a cheap, idempotent re-assert for a golden prepared + // by an older contract. + // // `/tmp` lives on the VM's ext4 data disk, not the overlayfs root: the // small tmpfs the guest boots with fills on real test suites, and the // overlay root it used to fall through to does not support @@ -6968,10 +6675,7 @@ fn as_runner_user(config: &RunnerPoolConfig, argv: &[String]) -> Vec { || true; \ chmod 0440 /etc/sudoers.d/preloop-{user} 2>/dev/null || true; \ mkdir -p /run/user/{uid} /opt/hostedtoolcache; \ - chown -R {uid}:{uid} /home/runner 2>/dev/null || true; \ chown {uid}:{uid} /run/user/{uid} 2>/dev/null || true; \ - if [ -d /usr/local/rustup ]; then chown -R {uid}:{uid} /usr/local/rustup; fi; \ - if [ -d /usr/local/cargo ]; then chown -R {uid}:{uid} /usr/local/cargo; fi; \ [ \"$(stat -c %a /opt/hostedtoolcache 2>/dev/null)\" = \"777\" ] || \ chmod -R 777 /opt/hostedtoolcache 2>/dev/null; \ grep -q AGENT_TOOLSDIRECTORY /etc/environment 2>/dev/null || \ @@ -7033,24 +6737,6 @@ fn shell_quote(arg: &str) -> String { format!("'{}'", arg.replace('\'', "'\\''")) } -/// Fail the provision if a toolchain layer's binary is not on the default -/// PATH after install. A provision interrupted between install commands (or -/// an install that silently succeeded without producing the binary) would -/// otherwise leave the job running without its toolchain — e.g. cargo-dist -/// failing on "you don't appear to have cargo installed" with no hint that -/// the machine itself was broken. -async fn verify_toolchain_installed( - provider: &P, - name: &MachineName, - layer: &ToolchainLayer, -) -> Result<(), OrchestratorError> { - let command = vec!["sh".to_owned(), "-c".to_owned(), layer.verify_command()]; - if let Err(error) = provider.exec(name, &command).await { - return Err(OrchestratorError::Vm(error)); - } - Ok(()) -} - /// Stage a pre-generated keypair for one `configure` call, if one is ready. /// /// Absent a staged key the guest generates its own, which is simply the @@ -7287,6 +6973,8 @@ mod lifecycle_tests { /// When set, `exec` answers exit 1 for any argv whose debug form /// contains this marker — models a guest command that fails. fail_exec_containing: Mutex>, + /// Whether this provider keeps packs as host files. AgentENV does not. + file_packs: bool, } impl TestProvider { @@ -7320,9 +7008,17 @@ mod lifecycle_tests { suspends: false, prune_pack_calls: Mutex::new(Vec::new()), fail_exec_containing: Mutex::new(None), + file_packs: true, } } + /// A backend whose packs are not host files (AgentENV): its golden is + /// prepared in the guest instead of unpacked from an artifact. + fn no_packs(mut self) -> Self { + self.file_packs = false; + self + } + fn that_suspends(mut self) -> Self { self.suspends = true; self @@ -7701,7 +7397,7 @@ esac async fn bake_manifest_is_written_as_root_and_verified() { let provider = TestProvider::new(false, false, false, false, false); let name = MachineName::new("golden-manifest".to_owned()).unwrap(); - let env_spec = EnvironmentSpec::for_base(crate::environment::DEFAULT_BASE_IMAGE.to_owned()); + let env_spec = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".to_owned()); write_bake_manifest(&provider, &name, &env_spec) .await @@ -7937,57 +7633,64 @@ chmod +x "$dest/bin/node" config.control_origin = Some("http://127.0.0.1:9090".to_owned()); let name = MachineName::new("runner").unwrap(); - let env = guest_env_prefix(&config, &name); + let env = guest_env_entries(&config, &name); assert!(env.contains(&"PRELOOP_CONTROL_ORIGIN=http://127.0.0.1:9090".to_owned())); assert!(!env.contains(&"PRELOOP_CONTROL_ORIGIN=http://192.168.1.20:9090".to_owned())); } - /// A workflow that installs a cargo subcommand and runs it in the next - /// step (`taiki-e/install-action` + `cargo hack`) only works when the - /// toolchain's bin directory is on the runner's PATH, as it is on hosted - /// images. Without it the install "succeeds" and the next step reports - /// `no such command: hack`. + /// The guest runner's environment is the image's, not the host's: the + /// launcher sources `/etc/environment` before handing over, so the PATH a + /// hosted step shell would get is the PATH a step here gets. The curated + /// Rust/Go bakes that used to justify host-side toolchain entries are + /// gone, so the pool's own list stays a system default and nothing + /// exports a Rust home. #[test] - fn runner_path_carries_toolchain_bin_directories() { + fn runner_env_comes_from_the_image_environment_file() { let config = test_config(false); let name = MachineName::new("runner").unwrap(); - let env = guest_env_prefix(&config, &name); - - let path = env - .iter() - .find_map(|entry| entry.strip_prefix("PATH=")) - .expect("the runner is launched with an explicit PATH"); - let entries: Vec<&str> = path.split(':').collect(); - assert!( - entries.contains(&"/usr/local/cargo/bin"), - "cargo-installed binaries must be reachable: {path}" - ); - // Toolchain homes are fixed system addresses, identical for root - // and switched runners: a $HOME-derived location would be invisible - // across the bake-user/step-user boundary (/root is 0700). - for expected in [ - "RUSTUP_HOME=/usr/local/rustup", - "CARGO_HOME=/usr/local/cargo", - ] { - assert!( - env.iter().any(|entry| entry == expected), - "guest env must pin {expected}: {env:?}" - ); - } + let argv = guest_env_prefix(&config, &name); + assert_eq!(argv[0], "sh"); + assert_eq!(argv[1], "-c"); + let launcher = &argv[2]; assert!( - entries.contains(&"/usr/local/go/bin"), - "the go layer untars into /usr/local/go: {path}" + launcher.contains(". /etc/environment"), + "the image's environment must be sourced: {launcher}" ); assert!( - entries.contains(&"/usr/local/bin") && entries.contains(&"/usr/bin"), - "the system PATH must survive: {path}" + launcher.contains("exec /usr/bin/env"), + "the pool's variables must survive the source: {launcher}" ); - assert!( - env.contains(&"RUSTUP_HOME=/usr/local/rustup".to_owned()), - "the shared rustup metadata must be selected: {env:?}" + // The `KEY=value` words after the launcher are the runner's + // environment, so the entries the assertions below read are exactly + // what `env` applies. + assert_eq!( + &argv[3], + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" ); + assert_eq!(argv[4], format!("PRELOOP_MACHINE_NAME={}", name.as_str())); + + let env = guest_env_entries(&config, &name); + let path = env + .iter() + .find_map(|entry| entry.strip_prefix("PATH=")) + .expect("the runner is launched with a PATH"); + assert_eq!( + path, + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ); + for absent in [ + "/usr/local/cargo/bin", + "/usr/local/go/bin", + "RUSTUP_HOME=", + "CARGO_HOME=", + ] { + assert!( + !path.contains(absent) && !env.iter().any(|entry| entry.starts_with(absent)), + "the curated toolchain contract is deleted; {absent} must not be exported: {env:?}" + ); + } } #[test] @@ -7998,77 +7701,64 @@ chmod +x "$dest/bin/node" config.control_upstream = Some("http://10.0.0.161:9090".to_owned()); let name = MachineName::new("runner").unwrap(); - let env = guest_env_prefix(&config, &name); + let env = guest_env_entries(&config, &name); assert!(env.contains(&"PRELOOP_CONTROL_ORIGIN=http://127.0.0.1:9090".to_owned())); assert!(env.contains(&"PRELOOP_CONTROL_UPSTREAM=http://10.0.0.161:9090".to_owned())); assert!(!env.iter().any(|v| v.starts_with("PRELOOP_CONTROL_SOCKET"))); } - /// The always-run ownership reconciliation must not install privilege - /// policy: custom/official images that never - /// had blanket sudo must not gain it just because a job runs there — and - /// it must not mask failures the way the account script's `|| true` tail - /// does. + /// The contract is the *whole* golden contract for a configured image, so + /// it must both refuse an image that could never run a job and hand the + /// runner account everything it writes. #[test] - fn ownership_reconcile_script_keeps_privilege_policy_out() { - let script = runner_ownership_reconcile_script(DEFAULT_RUNNER_USER, 1001); - assert!(!script.contains("NOPASSWD"), "{script}"); - assert!(!script.contains("usermod"), "{script}"); - assert!(!script.contains("sudoers"), "{script}"); - assert!(!script.contains("docker"), "{script}"); + fn golden_contract_requires_glibc_and_hands_over_the_runner_paths() { + let contract = golden_contract_script(DEFAULT_RUNNER_USER, DEFAULT_RUNNER_UID); assert!( - script.contains("needs=0"), - "an already-correct machine must skip the privileged half: {script}" + contract.contains("/lib64/ld-linux-*.so.*"), + "the glibc loader probe must be part of the contract: {contract}" ); assert!( - script.contains("sudo -n sh"), - "a machine that needs changes must escalate: {script}" + contract.contains("this image carries no glibc dynamic loader"), + "a missing loader must fail the bake with a legible message: {contract}" ); + assert!(contract.contains("NOPASSWD"), "{contract}"); assert!( - !script.contains("|| true"), - "escalation and chown failures must stay observable: {script}" + contract.contains("/home/runner/_work") && contract.contains("mkdir -p"), + "{contract}" ); - // Official runner images ship rustup under the runner's $HOME while - // the exported contract points at /usr/local; without adopting it, - // every `cargo`/`rustup` invocation resolves an empty home. The apply - // half travels base64-encoded, so assert on the decoded text. - let decoded = script - .split("| base64 -d") - .filter_map(|part| { - let close = part.rfind('\'')?; - let open = part[..close].rfind('\'')?; - Some(part[open + 1..close].to_owned()) - }) - .filter_map(|blob| { - base64::engine::general_purpose::STANDARD - .decode(blob) - .ok() - .and_then(|bytes| String::from_utf8(bytes).ok()) - }) - .collect::>() - .join("\n"); assert!( - decoded.contains("ln -s /home/runner/.rustup /usr/local/rustup"), - "the image's rustup home must be adopted: {decoded}" + contract.contains("/opt/hostedtoolcache") + && contract.contains("AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache"), + "the tool cache must be writable and advertised to the runner: {contract}" ); assert!( - decoded.contains("ln -s /home/runner/.cargo /usr/local/cargo"), - "the image's cargo home must be adopted: {decoded}" + contract.contains("RUNNER_TOOL_CACHE=/opt/hostedtoolcache"), + "{contract}" ); + } + + /// One owner-only walk at golden build, never a recursive `chown`: on + /// overlayfs a recursive one copies every visited file out of the packed + /// lower layer into the per-VM upper (measured: 304 s and 1.22 GB to + /// re-group two inodes), and a group predicate would re-group files the + /// official images deliberately leave in another group — GitHub ships + /// `/home/runner/.docker` as `runner:docker`. + #[test] + fn golden_ownership_walks_only_the_mismatched_inodes() { + let ownership = golden_ownership_script(1001); assert!( - script.contains("/home/runner/.rustup/settings.toml"), - "adoption must be gated on a usable rustup home: {script}" + ownership + .contains("find /home/runner -xdev ! -user 1001 -exec chown -h 1001:1001 {} +"), + "the walk must match on the owner alone and touch only those inodes: {ownership}" ); - // A runner-owned root with root-owned descendants passes a top-level - // `stat` and then skips the recursive chown; the probe must descend. assert!( - script.contains("find \"$d/.\" ! -uid 1001 -print -quit"), - "the ownership probe must be recursive: {script}" + !ownership.contains("chown -R"), + "no recursive chown may exist at golden build: {ownership}" ); assert!( - script.contains("stat -L -c %u"), - "the probe must dereference an adopted symlink home: {script}" + !ownership.contains("! -group"), + "a runner-owned file in another group must be left alone: {ownership}" ); } @@ -9576,6 +9266,99 @@ done assert_eq!(shell_quote("a'b"), "'a'\\''b'"); } + /// The runner account name is interpolated unquoted into root-run guest + /// scripts and into the `sudoers.d` filename, so the only safe contract is + /// to refuse anything that is not a Linux account name. + #[test] + fn runner_account_names_are_restricted_to_account_syntax() { + for valid in ["runner", "root", "_preloop", "ci-runner", "runner2"] { + assert!(is_valid_account_name(valid), "{valid} must be allowed"); + } + for invalid in [ + "", + "Runner", + "1runner", + "-runner", + "run ner", + "runner;id", + "runner$(id)", + "runner`id`", + "../root", + "runner\nroot", + "runner:root", + ] { + assert!( + !is_valid_account_name(invalid), + "{invalid:?} must be refused: it reaches a root shell unquoted" + ); + } + + // And the pool refuses it before touching machine state. + // `validate` also checks the registration-token environment; seed the + // fixture so this test stays focused on the account-name contract. + unsafe { std::env::set_var("LIFECYCLE_TEST_TOKEN", "test") }; + let mut config = test_config(false); + config.runner_user = Some("runner;id".to_owned()); + let error = config + .validate() + .expect_err("an injectable runner user must be refused"); + let message = error.to_string(); + assert!(message.contains("runner;id"), "{message}"); + assert!(message.contains("account name"), "{message}"); + // The default and the explicit root keep validating. + config.runner_user = Some(DEFAULT_RUNNER_USER.to_owned()); + config + .validate() + .expect("the default account must validate"); + config.runner_user = Some("root".to_owned()); + config.validate().expect("root must validate"); + } + + /// `PRELOOP_RUNNER_USER=root` runs the runner as the guest's own user: + /// there is no account to create, no ownership to hand over, and moving a + /// would-be `root` account's home would leave the image broken. + #[test] + fn root_runner_user_skips_the_account_and_ownership_steps() { + let contract = golden_contract_script("root", DEFAULT_RUNNER_UID); + assert!( + !contract.contains("useradd"), + "root must not be created as an account: {contract}" + ); + assert!( + !contract.contains("NOPASSWD"), + "root needs no sudoers entry: {contract}" + ); + assert!( + !contract.contains("find /home/runner"), + "root owns nothing that needs handing over: {contract}" + ); + // The glibc requirement is a property of the image, not the account. + assert!(contract.contains("no glibc dynamic loader"), "{contract}"); + } + + /// An account the image already carries must be *moved* to the configured + /// uid, not assumed to have it: the runtime drops to that uid, so an image + /// whose `runner` sits at 1000 would hand jobs a uid that owns nothing. + #[test] + fn existing_runner_accounts_are_reconciled_to_the_configured_uid() { + let script = runner_account_script("runner", 2000); + assert!( + script.contains(r#"if [ "$(id -u runner 2>/dev/null)" != "2000" ]; then"#), + "{script}" + ); + assert!(script.contains("usermod -u 2000 runner"), "{script}"); + // A refusal must not be silent, but must not fail the bake either: the + // account may be one the image manages. + assert!( + script.contains("could not move %s to uid %s"), + "a refused usermod must say so: {script}" + ); + assert!( + !script.contains("exit 1"), + "the reconciliation is best-effort: {script}" + ); + } + #[test] fn rosetta_multiarch_script_parses_as_posix_sh() { // A spliced fragment ending in a newline once put `;` at the start of @@ -9677,15 +9460,19 @@ done "run_as_root_or_sudo_strict".to_owned(), run_as_root_or_sudo_strict("true"), ), + ( + "golden_contract_script".to_owned(), + golden_contract_script(DEFAULT_RUNNER_USER, DEFAULT_RUNNER_UID), + ), + ( + "golden_contract_script (root)".to_owned(), + golden_contract_script("root", DEFAULT_RUNNER_UID), + ), + ( + "restore_apt_lists_script".to_owned(), + restore_apt_lists_script(), + ), ]; - // The curated-bake scripts this list used to carry - // (`base_install_script`, `runner_ownership_reconcile_script`, - // `apt_lists_refresh_command`, the toolchain install/verify commands) - // exist only on this branch's lineage: the main lineage replaces them - // with the golden-contract scripts, so listing them here would break - // the test build on a merged tree. Add that lineage's - // `golden_contract_script` / `golden_ownership_script` / - // `golden_glibc_check_script` here when the merge happens. for (name, script) in &scripts { let output = std::process::Command::new(shell) @@ -9941,7 +9728,7 @@ done config.clone(), 0, shutdown.clone(), - Arc::new(GoldenRegistry::new(config.name_prefix.clone())), + Arc::new(GoldenCache::new(config.name_prefix.clone())), handles, )); @@ -10143,10 +9930,8 @@ done RunnerPoolConfig { size: 1, use_fork: false, - use_packed_artifact: false, name_prefix: "lifecycle-test".to_owned(), base_image: "base-image".to_owned(), - workspace: None, artifact_stem: PathBuf::from("/tmp/lifecycle-artifact"), release_version: "9.9.9".to_owned(), runner_bundle, @@ -10199,6 +9984,7 @@ done fn capabilities(&self) -> ProviderCapabilities { ProviderCapabilities { preserves_runtime_state_on_suspend: self.suspends, + file_packs: self.file_packs, ..ProviderCapabilities::default() } } @@ -10359,7 +10145,12 @@ done } async fn list(&self) -> Result, VmError> { - Ok(Vec::new()) + self.machines + .lock() + .await + .keys() + .map(|name| MachineName::new(name.clone())) + .collect::, _>>() } async fn exec(&self, name: &MachineName, argv: &[String]) -> Result { @@ -10468,7 +10259,18 @@ done Ok(()) } - async fn pack(&self, _name: &MachineName, _output: &Path) -> Result<(), VmError> { + async fn pack(&self, _name: &MachineName, output: &Path) -> Result<(), VmError> { + // The real provider writes an ELF launcher at the output stem and + // the pack data in the `.smolmachine` sidecar, and a bake + // consumes the sidecar (`packed_golden_path`). A fake that wrote + // nothing would fail every bake with "did not create expected + // artifact". + let failed = |source: std::io::Error| VmError::Launch { + program: "smolvm".to_owned(), + source, + }; + std::fs::write(output, b"launcher").map_err(failed)?; + std::fs::write(format!("{}.smolmachine", output.display()), b"pack").map_err(failed)?; Ok(()) } } @@ -10489,8 +10291,6 @@ done RunnerEnvironment { fingerprint: None, base: config.base_image.clone(), - toolchains: Vec::new(), - curated: true, }, &CancellationToken::new(), ) @@ -10513,17 +10313,14 @@ done #[tokio::test] async fn provisioning_failures_delete_created_runner() { + // No `fail_install` case: the only install that used to run was the + // curated apt bake, and nothing is installed on a machine any more. let cases = [ ( TestProvider::new(true, false, false, false, false), false, "start-failure", ), - ( - TestProvider::new(false, true, false, false, false), - true, - "install-failure", - ), ( TestProvider::new(false, false, true, false, false), false, @@ -10571,8 +10368,6 @@ done RunnerEnvironment { fingerprint: None, base: config.base_image.clone(), - toolchains: Vec::new(), - curated: true, }, &CancellationToken::new(), ) @@ -10593,8 +10388,6 @@ done RunnerEnvironment { fingerprint: None, base: config.base_image.clone(), - toolchains: Vec::new(), - curated: true, }, &CancellationToken::new(), ) @@ -10694,7 +10487,7 @@ done 1, None, &keys, - test_runner_environment("base-image", Vec::new(), true), + test_runner_environment("base-image"), &shutdown, ); if case.starts { @@ -10782,7 +10575,7 @@ done generation, None, &Arc::new(KeyPool::new()), - test_runner_environment("base-image", Vec::new(), true), + test_runner_environment("base-image"), &shutdown, ) .await @@ -10908,29 +10701,24 @@ done fn packed_fork_config() -> RunnerPoolConfig { let mut config = test_config(false); config.use_fork = true; - config.use_packed_artifact = true; config } - fn test_runner_environment( - base: impl Into, - toolchains: Vec, - curated: bool, - ) -> RunnerEnvironment { + fn test_runner_environment(base: impl Into) -> RunnerEnvironment { RunnerEnvironment { fingerprint: None, base: base.into(), - toolchains, - curated, } } - /// A retained SmolVM checkpoint can become unusable after the packed - /// golden has been prepared. Retrying the same fork forever starves every - /// queued job, while creating a runner directly from the same packed - /// artifact remains valid. + /// A retained SmolVM checkpoint can become unusable after the golden has + /// been prepared. The pool re-arms it and retries the fork; when the base + /// still cannot fork, the provision fails. It must not create a machine + /// from the raw image: that machine carries neither the packed golden's + /// contents nor the runner contract, so it would register as a runner that + /// cannot run the job it was created for. #[tokio::test] - async fn packed_golden_fork_failure_falls_back_to_direct_creation() { + async fn unrecoverable_fork_base_failure_fails_the_provision() { // Verbatim from SmolVM 1.7.x: a spent fork base is reported through the // CLI's stderr, so the signature match is the only handle on it. const SPENT_BASE: &str = "smolvm fork failed with exit code 1: Freezing golden \ @@ -10961,41 +10749,25 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await - .expect("a broken packed-golden fork falls back to direct creation"); + .expect_err("an unrecoverable fork base must fail the provision"); let events = provider.events().await; let fork = format!("fork:{}:{}", golden.as_str(), name.as_str()); - let delete = format!("delete:{}", name.as_str()); - let create = format!("create:{}", name.as_str()); - let start = format!("start:{}", name.as_str()); - let fork_index = events - .iter() - .position(|event| event == &fork) - .expect("fork was attempted first"); - let delete_index = events - .iter() - .position(|event| event == &delete) - .expect("a partial clone was cleaned up"); - let create_index = events - .iter() - .position(|event| event == &create) - .expect("runner was created from the packed artifact"); - let start_index = events - .iter() - .position(|event| event == &start) - .expect("directly created runner was started"); assert!( - fork_index < delete_index && delete_index < create_index && create_index < start_index, - "fallback order must be fork, cleanup, create, start: {events:?}" + events.iter().any(|event| event == &fork), + "the fork must have been attempted: {events:?}" + ); + assert!( + !events.iter().any(|event| event.starts_with("create:")), + "no machine may be created from a raw image as a fork fallback: {events:?}" ); - assert!(provider.has_machine(&name).await); } #[tokio::test] - async fn transient_packed_golden_fork_failure_retries_the_fork_before_direct_creation() { + async fn transient_fork_failure_retries_the_fork() { let provider = Arc::new(TestProvider::new(false, false, false, false, false).failing_fork_once()); let config = packed_fork_config(); @@ -11008,7 +10780,7 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await .expect("a transient fork failure recovers by forking again"); @@ -11022,7 +10794,7 @@ done ); assert!( !events.contains(&format!("start:{}", name.as_str())), - "a recovered fork must not cold-boot the packed artifact: {events:?}" + "a recovered fork must not cold-boot the image: {events:?}" ); } @@ -11056,18 +10828,28 @@ done /// finished disk. A failed prune is best-effort and must not fail the /// golden — but the hook itself must fire exactly once per preparation. #[tokio::test] - async fn packed_golden_prepare_prunes_pack_intermediates() { + async fn golden_prepare_prunes_pack_intermediates() { let provider = Arc::new(TestProvider::new(false, false, false, false, false)); // Keep the golden fingerprint record off /tmp: the test only needs a // writable directory, not the hardcoded lifecycle paths. let scratch = tempfile::tempdir().expect("scratch dir"); let mut config = packed_fork_config(); config.artifact_stem = scratch.path().join("artifact"); + let payload = config.artifact_payload(); + std::fs::create_dir_all(payload.parent().unwrap()).unwrap(); + std::fs::write(&payload, b"packed-golden").unwrap(); let golden = MachineName::new("lifecycle-test-golden").unwrap(); + let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - prepare_packed_golden(&provider, &config, &golden) - .await - .expect("packed golden preparation succeeds"); + prepare_fork_base( + &provider, + &config, + &golden, + &env_spec, + &CancellationToken::new(), + ) + .await + .expect("golden preparation succeeds"); assert_eq!( *provider.prune_pack_calls.lock().await, @@ -11083,18 +10865,28 @@ done } /// The prune hook must not fire when the golden never becomes forkable: - /// `prepare_packed_golden` returns before the hook on any earlier failure. + /// `prepare_fork_base` returns before the hook on any earlier failure. #[tokio::test] - async fn packed_golden_prepare_skips_prune_on_start_failure() { + async fn golden_prepare_skips_prune_on_start_failure() { let provider = Arc::new(TestProvider::new(true, false, false, false, false)); let scratch = tempfile::tempdir().expect("scratch dir"); let mut config = packed_fork_config(); config.artifact_stem = scratch.path().join("artifact"); + let payload = config.artifact_payload(); + std::fs::create_dir_all(payload.parent().unwrap()).unwrap(); + std::fs::write(&payload, b"packed-golden").unwrap(); let golden = MachineName::new("lifecycle-test-golden").unwrap(); + let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - prepare_packed_golden(&provider, &config, &golden) - .await - .expect_err("start failure aborts golden preparation"); + prepare_fork_base( + &provider, + &config, + &golden, + &env_spec, + &CancellationToken::new(), + ) + .await + .expect_err("start failure aborts golden preparation"); assert!( provider.prune_pack_calls.lock().await.is_empty(), @@ -11122,7 +10914,7 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await .expect("the re-armed golden serves the fork"); @@ -11172,7 +10964,7 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await .expect("the re-armed fingerprint-suffixed golden serves the fork"); @@ -11200,19 +10992,18 @@ done ); } - /// A fingerprint-suffixed golden is an *environment* golden baked from - /// the job's requested image. When its fork fails and the pool falls - /// back to independent creation, the runner must boot that job's - /// environment — not the default packed artifact, which would run a - /// non-default `runs-on` job on the wrong operating system. + /// A fingerprint-suffixed golden is the memoized golden for a different + /// environment. When its fork fails unrecoverably the provision fails; the + /// pool must not boot *any* image directly as a substitute, because a + /// machine created from a raw image carries none of the golden's contents. #[tokio::test] - async fn fingerprint_golden_fork_failure_falls_back_to_the_job_environment() { + async fn fingerprint_golden_fork_failure_does_not_boot_a_substitute_image() { let provider = Arc::new(TestProvider::new(false, false, false, false, false).failing_fork()); let config = packed_fork_config(); let golden = MachineName::new("lifecycle-test-golden-3577f5d5a384").unwrap(); let name = MachineName::new("lifecycle-test-0-9").unwrap(); - let env = test_runner_environment("mirror.gcr.io/library/ubuntu:22.04", Vec::new(), true); + let env = test_runner_environment("mirror.gcr.io/library/ubuntu:22.04"); provision_runner( &provider, @@ -11223,26 +11014,23 @@ done &env, ) .await - .expect("an env-golden fork failure falls back to the job environment"); + .expect_err("an unrecoverable fork failure must propagate"); - let created = provider - .created_image(&name) - .await - .expect("the fallback created the runner machine"); - assert_eq!( - created, "mirror.gcr.io/library/ubuntu:22.04", - "an env-golden fallback must boot the job's requested image, not the default pack" + assert!( + provider.created_image(&name).await.is_none(), + "no machine may be created from a raw image as a fork fallback" ); } /// A spent base that still has live clones must NOT be re-armed: resuming - /// it would corrupt the copy-on-write clones. The pool falls back to a - /// full create instead. + /// it would corrupt the copy-on-write clones. The pool waits for them to + /// drain; when they outlive the wait, the provision fails rather than + /// booting the image — the slot retries once the clones are gone. // Paused time: the drain loop sleeps GOLDEN_DRAIN_PROBE_DELAY between // probes; without this the 12-probe worst case would stall the test for // two minutes of real time. #[tokio::test(start_paused = true)] - async fn spent_fork_base_with_live_clones_is_not_rearmed() { + async fn spent_fork_base_with_live_clones_times_out_and_fails() { let provider = Arc::new( TestProvider::new(false, false, false, false, false) .with_live_forks(true) @@ -11258,10 +11046,10 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await - .expect("falls back to direct creation"); + .expect_err("a base that cannot be re-armed within the drain budget fails the provision"); let events = provider.events().await; assert!( @@ -11269,19 +11057,14 @@ done "the golden must not be touched while clones exist: {events:?}" ); assert!( - events.contains(&format!("delete:{}", name.as_str())), - "the partial clone is cleaned up: {events:?}" - ); - assert_eq!( - provider.created_image(&name).await.as_deref(), - Some(config.base_image.as_str()), - "a live clone makes the shared packed payload unsafe; fallback must use OCI" + !events.iter().any(|event| event.starts_with("create:")), + "no machine may be created from a raw image: {events:?}" ); } /// A live clone that exits mid-drain must be re-armed once it is gone: /// the drain loop keeps probing with backoff, and the golden resumes - /// serving forks instead of falling back to slow direct creation. + /// serving forks without booting the image. /// Paused time advances the probe sleeps instantly. #[tokio::test(start_paused = true)] async fn spent_fork_base_with_clone_that_drains_is_rearmed_and_retried() { @@ -11301,7 +11084,7 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await .expect("the re-armed golden serves the fork after the clone drains"); @@ -11332,9 +11115,10 @@ done /// The provider reports a live clone before invoking SmolVM for another /// plain fork. The orchestrator must neither re-arm the shared golden nor - /// instantiate the packed payload beside that clone. + /// instantiate any machine beside that clone — the clone owns the base's + /// frozen storage and no image boot may stand in for the fork. #[tokio::test] - async fn busy_packed_fork_base_uses_independent_environment_image() { + async fn busy_packed_fork_base_fails_without_touching_the_golden() { let provider = Arc::new(TestProvider::new(false, false, false, false, false).with_busy_fork_base()); let config = packed_fork_config(); @@ -11348,20 +11132,19 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(environment_base, Vec::new(), true), + &test_runner_environment(environment_base), ) .await - .expect("a busy plain-fork base falls back to an independent OCI machine"); + .expect_err("a busy plain-fork base fails the provision"); let events = provider.events().await; assert!( !events.iter().any(|event| event.starts_with("rearm:")), "a golden with a live clone must not be re-armed: {events:?}" ); - assert_eq!( - provider.created_image(&name).await.as_deref(), - Some(environment_base), - "fallback must use the job's resolved environment, not the shared packed payload" + assert!( + provider.created_image(&name).await.is_none(), + "no machine may be created from an image as a fork fallback: {events:?}" ); } @@ -11384,10 +11167,10 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), + &test_runner_environment(config.base_image.clone()), ) .await - .expect("falls back to direct creation"); + .expect_err("a base that cannot even be cleaned up fails the provision"); let events = provider.events().await; assert!( @@ -11397,9 +11180,8 @@ done } /// An environment-specific golden may represent a different `runs-on` - /// image from the packed artifact. Falling back there would run the job - /// on the wrong operating system, so only the default packed golden may - /// take the direct-create recovery path. + /// image from the packed artifact, so no fork failure may substitute a + /// machine booted from a raw image. #[tokio::test] async fn environment_golden_fork_failure_does_not_change_the_job_image() { let provider = @@ -11414,7 +11196,7 @@ done &name, Some(&golden), &Arc::new(KeyPool::new()), - &test_runner_environment("mirror.gcr.io/library/ubuntu:22.04", Vec::new(), true), + &test_runner_environment("mirror.gcr.io/library/ubuntu:22.04"), ) .await .expect_err("an environment-golden fork failure must propagate"); @@ -11429,37 +11211,6 @@ done ); } - /// A pack published before the baseline stopped wiping `/var/lib/apt/lists` - /// boots without apt indices, and `sudo apt-get install ` — how real - /// workflows install system packages — then resolves nothing. - #[tokio::test] - async fn packed_golden_fork_restores_apt_indices() { - let provider = Arc::new(TestProvider::new(false, false, false, false, false)); - let config = packed_fork_config(); - let golden = MachineName::new("lifecycle-test-golden").unwrap(); - let name = MachineName::new("lifecycle-test-0-3").unwrap(); - - provision_runner( - &provider, - &config, - &name, - Some(&golden), - &Arc::new(KeyPool::new()), - &test_runner_environment(config.base_image.clone(), Vec::new(), true), - ) - .await - .expect("provisioning succeeds"); - - let events = provider.events().await; - assert!( - events.iter().any(|event| event.contains("_Packages") - && event.contains("apt-get") - && event.contains("update") - && event.contains("timeout 120")), - "the fork must restore apt indices when the pack has none: {events:?}" - ); - } - /// The golden registry dies with the process, so a restart would rebake a /// golden that is still sitting there fully baked. The host-side record is /// what makes adoption safe: it must match the requested fingerprint, and @@ -11541,7 +11292,7 @@ done ); } - /// An adopted golden never went through `prepare_packed_golden`, so the + /// An adopted golden never went through `prepare_fork_base`, so the /// #295 pack/ prune must fire on the adopt path too — otherwise a golden /// carried across restarts keeps its intermediates forever and every /// fork copies them. @@ -11622,6 +11373,190 @@ done assert!(goldens.is_dir(), "goldens/ record dir survives"); } + /// Earlier releases keyed the payload on the stock Ubuntu base they baked + /// from. No fingerprint rotation under the current stem reaches those + /// files, so the sweep names them itself — and only them: another image's + /// payload, a hand-named file and an unknown architecture stay. + #[tokio::test] + async fn sweep_stale_artifacts_removes_payloads_of_retired_stock_stems() { + let temp = tempfile::tempdir().unwrap(); + let vms = temp.path().join("vms"); + std::fs::create_dir_all(&vms).unwrap(); + let mut config = test_config(false); + config.artifact_stem = vms.join("preloop-preloop-official-golden-aarch64"); + // SAFETY: test-only env for config validation; the pool is never run. + unsafe { std::env::set_var("LIFECYCLE_TEST_TOKEN", "test") }; + let pool = RunnerPool::new( + Arc::new(TestProvider::new(false, false, false, false, false)), + config.clone(), + ) + .expect("pool config validates"); + + let current = config.artifact_payload(); + std::fs::write(¤t, b"current").unwrap(); + let retired = [ + format!( + "preloop-mirror.gcr.io-library-ubuntu-24.04-sha256-{:064x}-aarch64-{:064x}", + 1, 2 + ), + format!("preloop-ubuntu-24.04-aarch64-{:064x}", 3), + format!("preloop-ubuntu-22.04-sha256-{:064x}-x86_64-{:064x}", 4, 5), + ]; + let kept = [ + format!( + "preloop-ghcr.io-acme-runner-images-ubuntu24-aarch64-{:064x}", + 6 + ), + "preloop-ubuntu-24.04-aarch64-notes.txt".to_owned(), + format!("preloop-ubuntu-24.04-armv7-{:064x}", 7), + format!("preloop-ubuntu-20.04-aarch64-{:064x}", 8), + ]; + for name in retired.iter().chain(kept.iter()) { + std::fs::write(vms.join(name), b"payload").unwrap(); + } + + pool.sweep_stale_artifacts().await; + + assert!(current.is_file(), "current payload survives"); + for name in &retired { + assert!(!vms.join(name).exists(), "{name} is swept"); + } + for name in &kept { + assert!(vms.join(name).is_file(), "{name} survives"); + } + } + + /// A suffixed golden record protects its machine from the startup cleanup + /// so adoption can reuse it — but only while some environment can still + /// resolve to that golden. The per-`runs-on` goldens of earlier releases + /// are recorded too and nothing prepares them again, so they must go, with + /// their records; the plain golden and the goldens of the two current + /// environments stay. + #[tokio::test] + async fn startup_cleanup_removes_goldens_no_environment_resolves_to() { + let temp = tempfile::tempdir().unwrap(); + let vms = temp.path().join("vms"); + std::fs::create_dir_all(&vms).unwrap(); + let mut config = test_config(false); + config.artifact_stem = vms.join("preloop-image-aarch64"); + config.base_image = "ghcr.io/acme/runner-images:custom".to_owned(); + // SAFETY: test-only env for config validation; the pool is never run. + unsafe { std::env::set_var("LIFECYCLE_TEST_TOKEN", "test") }; + let provider = Arc::new(TestProvider::new(false, false, false, false, false)); + let pool = + RunnerPool::new(provider.clone(), config.clone()).expect("pool config validates"); + + let prefix = plain_packed_golden_name(&config); + let official = + EnvironmentSpec::for_base(crate::environment::OFFICIAL_GOLDEN.to_owned()).fingerprint; + let configured = EnvironmentSpec::for_base(config.base_image.clone()).fingerprint; + let plain = prefix.clone(); + let official_golden = format!("{prefix}-{}", &official[..12]); + let configured_golden = format!("{prefix}-{}", &configured[..12]); + let retired_golden = format!("{prefix}-aaaaaaaaaaaa"); + for name in [ + &plain, + &official_golden, + &configured_golden, + &retired_golden, + ] { + let name = MachineName::new(name.clone()).unwrap(); + provider + .create(&MachineSpec { + name: name.clone(), + image: "ubuntu".to_owned(), + cpus: 1, + memory_mib: 512, + storage_gib: 1, + overlay_gib: Some(1), + network: NetworkPolicy::PublicOnly, + volumes: Vec::new(), + sockets: Vec::new(), + dns: None, + rosetta: false, + }) + .await + .unwrap(); + write_golden_record(&config, &name, "fp"); + } + + pool.remove_stale_runner_machines().await.unwrap(); + + let left: Vec = provider + .list() + .await + .unwrap() + .iter() + .map(|name| name.as_str().to_owned()) + .collect(); + for kept in [&plain, &official_golden, &configured_golden] { + assert!(left.contains(kept), "{kept} is still reachable: {left:?}"); + let name = MachineName::new(kept.clone()).unwrap(); + assert!( + golden_record_path(&config, &name).unwrap().is_file(), + "{kept} keeps its record" + ); + } + assert!( + !left.contains(&retired_golden), + "an unreachable golden is deleted: {left:?}" + ); + let name = MachineName::new(retired_golden).unwrap(); + assert!( + !golden_record_path(&config, &name).unwrap().exists(), + "its record goes with it" + ); + } + + /// The restore must not touch a golden that already has indices (it would + /// repay a download for nothing) and must refresh one that has none. Run + /// against a stub `apt-get` so only the decision is under test. + #[cfg(unix)] + #[test] + fn restore_apt_lists_refreshes_only_a_golden_without_indices() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + let marker = temp.path().join("apt-called"); + let stub = bin.join("apt-get"); + std::fs::write( + &stub, + format!("#!/bin/sh\necho \"$@\" >> '{}'\n", marker.display()), + ) + .unwrap(); + std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let run = |lists: &Path| { + let script = restore_apt_lists_script_at(lists.to_str().unwrap()); + let path = format!("{}:/usr/bin:/bin", bin.display()); + let status = std::process::Command::new("sh") + .args(["-c", &script]) + .env("PATH", path) + .status() + .unwrap(); + assert!(status.success(), "the restore never fails the golden"); + }; + + let empty = temp.path().join("empty"); + std::fs::create_dir_all(&empty).unwrap(); + run(&empty); + let called = std::fs::read_to_string(&marker).expect("an empty lists dir is refreshed"); + assert!(called.contains("update"), "{called}"); + + std::fs::remove_file(&marker).unwrap(); + let populated = temp.path().join("populated"); + std::fs::create_dir_all(&populated).unwrap(); + std::fs::write( + populated.join("archive.ubuntu.com_ubuntu_dists_noble_main_binary-amd64_Packages"), + b"x", + ) + .unwrap(); + run(&populated); + assert!(!marker.exists(), "a golden with indices is left alone"); + } + /// A `.tmp-golden-*` staging file with a fresh companion lock must survive /// the startup sweep, so concurrent pools sharing the artifact directory /// cannot delete each other's in-flight bakes or downloads. A stale lock @@ -11694,8 +11629,6 @@ done RunnerEnvironment { fingerprint: None, base: config.base_image.clone(), - toolchains: Vec::new(), - curated: true, }, &CancellationToken::new(), ) @@ -11715,8 +11648,6 @@ done environment: RunnerEnvironment { fingerprint: None, base: config.base_image.clone(), - toolchains: Vec::new(), - curated: true, }, idle: &idle, keys: &Arc::new(KeyPool::new()), @@ -11753,7 +11684,7 @@ done config.clone(), 0, CancellationToken::new(), - Arc::new(GoldenRegistry::new(config.name_prefix.clone())), + Arc::new(GoldenCache::new(config.name_prefix.clone())), handles, Arc::new(std::sync::Mutex::new(0)), Arc::new(tokio::sync::Semaphore::new(1)), @@ -11775,60 +11706,113 @@ done ); } - /// A custom base image is the operator's contract: the golden must not - /// receive Preloop's curated bake. Stock bases still get it. + /// A configured image is baked as-is plus the runner contract: the + /// contract runs once, through the root-or-sudo hop (an arbitrary image may + /// declare `USER`), and nothing else is installed. + /// + /// Exercised through `apply_golden_contract` rather than the whole bake: + /// the full path gates on the host's free disk (`ensure_disk_for_golden_build` + /// wants the builder disk plus pack staging free), which a unit test must + /// not depend on. #[tokio::test] - async fn custom_base_golden_skips_the_curated_bake() { + async fn configured_image_contract_runs_once_without_installing_anything() { let provider = Arc::new(TestProvider::new(false, false, false, false, false)); let config = test_config(false); - let golden = MachineName::new("lifecycle-test-nobake-golden").unwrap(); + let name = MachineName::new("lifecycle-test-builder").unwrap(); - let custom = EnvironmentSpec::for_base("ghcr.io/acme/runner:latest".to_owned()); - assert!(!custom.curated); - prepare_golden_for_env(&provider, &config, &golden, &custom) + apply_golden_contract(provider.as_ref(), &config, &name) .await - .expect("custom base golden provision succeeds"); - let custom_events = provider.events().await; + .expect("the contract applies to the image as-is"); + + let events = provider.events().await; + assert!( + !events.iter().any(|event| event.contains("apt-get install")), + "a configured image must never receive a package install: {events:?}" + ); + // The contract travels base64-encoded inside the root-or-sudo wrapper, + // which is what makes it work whether the exec lands as root or as an + // image-declared user with passwordless sudo. + let contract = base64::engine::general_purpose::STANDARD.encode(golden_contract_script( + DEFAULT_RUNNER_USER, + DEFAULT_RUNNER_UID, + )); + assert!( + events.iter().any(|event| event.contains(&contract)), + "the golden contract must run on the builder: {events:?}" + ); assert!( - !custom_events.iter().any(|event| event.contains("apt-get")), - "a custom base golden must not run the curated apt bake: {custom_events:?}" + events.iter().any(|event| event.contains("sudo -n sh")), + "the wrapper must escalate when the exec is not root: {events:?}" ); + } + + /// The official golden is published packed — there is nothing to bake it + /// from, and the deleted curated path is not a fallback. `build-golden` + /// must refuse it instead of producing a substitute image. + #[tokio::test] + async fn official_golden_cannot_be_built_locally() { + let provider = Arc::new(TestProvider::new(false, false, false, false, false)); + let mut config = test_config(false); + config.base_image = crate::environment::OFFICIAL_GOLDEN.to_owned(); + // SAFETY: test-only env for config validation; the pool never serves. + unsafe { std::env::set_var("LIFECYCLE_TEST_TOKEN", "test") }; + let pool = RunnerPool::new(provider.clone(), config).unwrap(); - let stock = EnvironmentSpec::for_base(crate::environment::UBUNTU_24_04_PIN.to_owned()); - assert!(stock.curated); - prepare_golden_for_env(&provider, &config, &golden, &stock) + let error = pool + .rebuild_artifact() .await - .expect("stock base golden provision succeeds"); - let stock_events = provider.events().await; + .expect_err("the official golden is not buildable locally"); + + assert!(error.to_string().contains("published packed"), "{error}"); assert!( - stock_events.iter().any(|event| event.contains("apt-get")), - "a stock base golden must run the curated apt bake" + provider.created_images.lock().await.is_empty(), + "no builder VM may be booted for the official golden" ); } - /// Direct (no-golden) provisioning of a custom base must not run the - /// curated apt bake either — the image is the operator's contract. + /// A backend without host-side packs (AgentENV) has no artifact to unpack, + /// so the golden is prepared in-guest: boot the image, apply the contract, + /// and freeze the result as the fork base. #[tokio::test] - async fn custom_base_direct_provision_skips_the_curated_bake() { - let provider = Arc::new(TestProvider::new(false, false, false, false, false)); - let config = test_config(false); - let name = MachineName::new("lifecycle-test-nobake-direct").unwrap(); + async fn imageless_backend_bakes_the_golden_in_guest() { + let provider = Arc::new(TestProvider::new(false, false, false, false, false).no_packs()); + let mut config = test_config(false); + config.base_image = "ghcr.io/acme/runner:latest".to_owned(); + let golden = MachineName::new("lifecycle-test-nopack-golden").unwrap(); + let env_spec = EnvironmentSpec::for_base(config.base_image.clone()); - provision_runner( + prepare_fork_base( &provider, &config, - &name, - None, - &Arc::new(KeyPool::new()), - &test_runner_environment("ghcr.io/acme/runner:latest", Vec::new(), false), + &golden, + &env_spec, + &CancellationToken::new(), ) .await - .expect("custom base provisioning succeeds"); + .expect("the in-guest golden preparation succeeds"); + let created = provider.created_images.lock().await.clone(); + assert!( + created + .iter() + .any(|(name, image)| name == golden.as_str() + && image == "ghcr.io/acme/runner:latest"), + "the golden must boot the configured image: {created:?}" + ); let events = provider.events().await; assert!( - !events.iter().any(|event| event.contains("apt-get")), - "a custom base must not receive the curated apt bake: {events:?}" + events.iter().any(|event| event + .contains(&format!("start_forkable:{}", golden.as_str())) + || event.contains(&format!("start:{}", golden.as_str()))), + "the golden must be frozen as the fork base: {events:?}" + ); + let contract = base64::engine::general_purpose::STANDARD.encode(golden_contract_script( + DEFAULT_RUNNER_USER, + DEFAULT_RUNNER_UID, + )); + assert!( + events.iter().any(|event| event.contains(&contract)), + "the contract must be applied in the guest: {events:?}" ); } } @@ -11879,16 +11863,23 @@ mod golden_download_tests { } #[test] - fn custom_base_without_golden_url_does_not_adopt_stock_release() { - assert!(should_download_prebaked_golden("ubuntu:24.04", false)); - assert!(!should_download_prebaked_golden( - "ghcr.io/acme/runner-images:ubuntu24-runner-large-latest-arm64", - false + fn only_the_official_golden_is_downloaded() { + // The official sentinel is fetched (from the OCI reference or the + // release asset); a configured image is baked locally instead, with or + // without a PRELOOP_GOLDEN_URL mirror in the environment. + assert!(crate::environment::is_official_golden( + crate::environment::OFFICIAL_GOLDEN )); - assert!(should_download_prebaked_golden( + for configured in [ "ghcr.io/acme/runner-images:ubuntu24-runner-large-latest-arm64", - true - )); + "ubuntu:24.04", + "debian:bookworm-slim", + ] { + assert!( + !crate::environment::is_official_golden(configured), + "{configured} must not be mistaken for the official golden" + ); + } } /// Answers exactly one request with `head` followed by `body`, then closes @@ -11908,9 +11899,10 @@ mod golden_download_tests { format!("http://{address}/golden") } - /// Answers the payload request, then one more connection with - /// `checksum_body` (the `.sha256` companion). Used to exercise the - /// checksum verification path. + /// Answers the payload and checksum requests in either order. Keeping the + /// responses path-directed lets the client fetch the sidecar before or + /// after probing the payload without coupling the fixture to connection + /// reuse details. async fn serve_with_checksum( payload_head: String, payload_body: Vec, @@ -11919,21 +11911,23 @@ mod golden_download_tests { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); tokio::spawn(async move { - for (head, body) in [ - (payload_head, payload_body), - ( - format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", - checksum_body.len() - ), - checksum_body, - ), - ] { + let checksum_head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + checksum_body.len() + ); + for _ in 0..2 { let (mut socket, _) = listener.accept().await.unwrap(); let mut request = [0_u8; 1024]; - let _ = socket.read(&mut request).await; + let read = socket.read(&mut request).await.unwrap_or(0); + let is_checksum = + String::from_utf8_lossy(&request[..read]).contains("GET /golden.sha256 "); + let (head, body) = if is_checksum { + (&checksum_head, &checksum_body) + } else { + (&payload_head, &payload_body) + }; let _ = socket.write_all(head.as_bytes()).await; - let _ = socket.write_all(&body).await; + let _ = socket.write_all(body).await; let _ = socket.shutdown().await; } }); @@ -11957,9 +11951,15 @@ mod golden_download_tests { // Larger than any single chunk hyper will hand back, so the loop has to // append across iterations to reproduce the body. let body: Vec = (0..4_u32 * 1024 * 1024).map(|i| i as u8).collect(); - let url = serve_once( + use sha2::{Digest, Sha256}; + let digest: String = Sha256::digest(&body) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + let url = serve_with_checksum( format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()), body.clone(), + format!("{digest} golden\n").into_bytes(), ) .await; unsafe { std::env::set_var("PRELOOP_GOLDEN_URL", &url) }; @@ -11980,9 +11980,10 @@ mod golden_download_tests { let directory = tempfile::tempdir().unwrap(); let payload = directory.path().join("golden.smolmachine"); let expected_bytes = 512 * 1024; - let url = serve_once( + let url = serve_with_checksum( format!("HTTP/1.1 200 OK\r\nContent-Length: {expected_bytes}\r\n\r\n"), Vec::new(), + format!("{} golden\n", "00".repeat(32)).into_bytes(), ) .await; unsafe { std::env::set_var("PRELOOP_GOLDEN_URL", &url) }; @@ -12257,24 +12258,16 @@ mod golden_download_tests { } #[cfg(test)] -mod golden_registry_tests { +mod golden_cache_tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; - #[test] - fn packed_failure_disables_secondary_environment_bakes() { - let registry = GoldenRegistry::new("test".to_owned()); - assert!(!registry.is_packed_disabled()); - registry.disable_packed(); - assert!(registry.is_packed_disabled()); - } - /// Distinct environments must build concurrently: one fingerprint's bake /// must not park other slots (the pre-freeze `build_lock` behavior). #[tokio::test] async fn distinct_fingerprints_build_concurrently() { - let registry = GoldenRegistry::new("test".to_owned()); + let registry = GoldenCache::new("test".to_owned()); let started = Arc::new(AtomicUsize::new(0)); let max_active = Arc::new(AtomicUsize::new(0)); let active = Arc::new(AtomicUsize::new(0)); @@ -12312,7 +12305,7 @@ mod golden_registry_tests { /// the first caller's golden via the re-check. #[tokio::test] async fn same_fingerprint_builds_once() { - let registry = GoldenRegistry::new("test".to_owned()); + let registry = GoldenCache::new("test".to_owned()); let builds = Arc::new(AtomicUsize::new(0)); let build = |builds: Arc| async move { builds.fetch_add(1, Ordering::SeqCst); diff --git a/crates/preloop-orchestrator/tests/golden_fidelity.rs b/crates/preloop-orchestrator/tests/golden_fidelity.rs deleted file mode 100644 index 2b39ca34e..000000000 --- a/crates/preloop-orchestrator/tests/golden_fidelity.rs +++ /dev/null @@ -1,475 +0,0 @@ -//! The golden image must not drift from GitHub's `ubuntu-latest` baseline. -//! -//! Workflows are written against `ubuntu-latest`. Every tool it ships that the -//! golden lacks is a latent "works on GitHub, fails locally" bug — the exact -//! class Preloop exists to eliminate. These tests pin the baseline so a package -//! cannot quietly fall out of the list. -//! -//! Scope is deliberate: the apt baseline plus compiler families that workflows -//! invoke implicitly. Android SDK, five JDKs, .NET, browsers, and cloud CLIs -//! come to roughly 90 GB and are the job of setup actions and containers. - -use preloop_orchestrator::{ - BASE_NODE_VERSION, GITHUB_GUEST_SYSCTLS, base_install_script, base_packages, compiler_packages, - docker_data_root, docker_packages, guest_sysctl_script, loopback_hosts, -}; - -/// Commands a workflow may reasonably assume exist, because `ubuntu-latest` -/// ships them. Grouped by the failure each omission causes. -const REQUIRED: &[(&str, &str)] = &[ - // Bootstrap: nothing else works without these. - ("git", "checkout"), - ("curl", "action downloads and countless run steps"), - ("wget", "run steps"), - ("ca-certificates", "TLS verification"), - ( - "sudo", - "`sudo apt-get install` appears in a large share of workflows", - ), - ("gnupg2", "apt key management and signing"), - ("openssh-client", "git over SSH and deploy keys"), - // Archive handling: most setup-* actions extract archives. - ("unzip", "actions/setup-* extract zip archives"), - ("zip", "artifact packaging"), - ("tar", "archive extraction"), - ("xz-utils", "archive extraction"), - ("zstd", "actions/cache compresses with zstd by default"), - ("bzip2", "archive extraction"), - ("p7zip-full", "archive extraction"), - // Toolchain: native builds and build scripts. - ("build-essential", "any native build; libc, gcc, g++"), - ("make", "build scripts"), - ("pkg-config", "native dependency discovery"), - ("libssl-dev", "crates and gems that link OpenSSL"), - ("autoconf", "autotools builds"), - ("automake", "autotools builds"), - ("libtool", "autotools builds"), - // Runtimes assumed present by actions and scripts. - ("python3", "scripts and composite actions"), - ("python-is-python3", "scripts invoking bare `python`"), - // Shell-step staples. - ("jq", "JSON handling in run steps"), - ("file", "type detection"), - ("rsync", "deploy and copy actions"), - ("sqlite3", "test fixtures"), - // Network diagnostics workflows use when something breaks. - ("dnsutils", "dig/nslookup"), - ("iputils-ping", "connectivity checks"), - ("net-tools", "netstat/ifconfig"), -]; - -#[test] -fn golden_carries_every_baseline_package() { - let packages: Vec<&str> = base_packages().split_whitespace().collect(); - let missing: Vec<&(&str, &str)> = REQUIRED - .iter() - .filter(|(package, _)| !packages.contains(package)) - .collect(); - - assert!( - missing.is_empty(), - "golden image is missing packages that `ubuntu-latest` ships:\n{}", - missing - .iter() - .map(|(package, why)| format!(" {package:<20} needed for: {why}")) - .collect::>() - .join("\n") - ); -} - -/// Node.js is no longer an apt package in the golden: it is baked from the -/// official dist tarball, pinned to the version GitHub's ubuntu-24.04 image -/// ships (22.23.1). The apt series (18.19 on Ubuntu 24.04) is a fidelity bug, -/// so this test locks the pinned-tarball install in place. -#[test] -fn golden_bakes_pinned_node_from_dist_tarball() { - let script = base_install_script(); - assert!( - script.contains(&format!( - "https://nodejs.org/dist/v{BASE_NODE_VERSION}/node-v{BASE_NODE_VERSION}-linux-$NODE_ARCH.tar.gz" - )), - "base install must bake pinned node {BASE_NODE_VERSION} from the dist tarball" - ); - assert!( - !script.contains("apt-get install -y -qq --no-install-recommends nodejs"), - "apt nodejs (18.19 on 24.04) must not be installed" - ); -} - -/// First-party setup actions own their requested runtime versions. Baking a -/// matrix of Node, Python, and Go duplicates those actions' distribution -/// mechanism and pushes the flattened golden beyond smolvm's 4 GiB export -/// limit. The baked Rust layer uses rustup's minimal profile for the same -/// reason. Keep only system Node/Python for direct shell use, and leave a -/// writable toolcache for setup actions. -#[test] -fn setup_actions_own_versioned_runtime_toolcaches() { - let script = base_install_script(); - for fragment in [ - "bake node toolcache", - "actions/python-versions", - "https://go.dev/dl/go$GO_VERSION", - ] { - assert!( - !script.contains(fragment), - "setup-managed runtime must not be baked: {fragment}" - ); - } - assert!( - script.contains("install -d -m 0777 /opt/hostedtoolcache"), - "the hosted image creates /opt/hostedtoolcache root-owned, mode 777 \ - (runner-images configure-environment.sh); setup actions need a \ - world-writable toolcache" - ); - assert!( - script.contains("AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache"), - "the hosted image sets AGENT_TOOLSDIRECTORY in /etc/environment" - ); - assert!( - script.contains("RUNNER_TOOL_CACHE=/opt/hostedtoolcache"), - "the hosted image sets RUNNER_TOOL_CACHE in /etc/environment" - ); -} - -/// Docker's official repo packages (docker-ce stack), not Ubuntu's -/// `docker.io`: the CLI and the buildx/compose plugins must be the official -/// artifacts so container jobs behave exactly like they do on `ubuntu-latest`. -const REQUIRED_DOCKER: &[&str] = &[ - "docker-ce", - "docker-ce-cli", - "containerd.io", - "docker-buildx-plugin", - "docker-compose-plugin", -]; - -#[test] -fn golden_carries_container_engine_packages() { - let baseline = docker_packages(); - let packages: Vec<&str> = baseline.split_whitespace().collect(); - let missing: Vec<&str> = REQUIRED_DOCKER - .iter() - .copied() - // Packages carry apt version pins (`docker-ce=28.0.4*`) — match the - // package name prefix. - .filter(|p| !packages.iter().any(|entry| entry.starts_with(p))) - .collect(); - assert!( - missing.is_empty(), - "container engine baseline is missing packages: {missing:?}" - ); - let mut seen = std::collections::BTreeSet::new(); - let duplicates: Vec<&&str> = packages.iter().filter(|p| !seen.insert(**p)).collect(); - assert!( - duplicates.is_empty(), - "duplicate packages in the container engine baseline: {duplicates:?}" - ); -} - -#[test] -fn golden_carries_official_compiler_matrix() { - let baseline = compiler_packages(); - let packages: Vec<&str> = baseline.split_whitespace().collect(); - for version in ["16", "17", "18"] { - for tool in ["clang", "clang-format", "clang-tidy"] { - let package = format!("{tool}-{version}"); - assert!( - packages.contains(&package.as_str()), - "compiler baseline is missing {package}" - ); - } - } - for version in ["12", "13", "14"] { - for tool in ["gcc", "g++", "gfortran"] { - let package = format!("{tool}-{version}"); - assert!( - packages.contains(&package.as_str()), - "compiler baseline is missing {package}" - ); - } - } -} - -#[test] -fn install_script_pins_docker_repo_and_cargo_shear() { - // The container engine is only installable after Docker's apt repo is - // bootstrapped (keyring + sources.list), and cargo-shear is a release - // tarball, not an apt package. Pin all three so a botched merge cannot - // silently drop the repo setup or the binary. - let script = base_install_script(); - for fragment in [ - "https://download.docker.com/linux/ubuntu/gpg", - "/etc/apt/sources.list.d/docker.list", - "docker-28.0.4.tgz", - "buildx-v0.35.0.linux-$DOCKER_PLUGIN_ARCH", - "docker-compose-linux-$COMPOSE_ARCH", - "cargo-shear-$(uname -m)-unknown-linux-musl.tar.gz", - ] { - assert!( - script.contains(fragment), - "install script lost {fragment:?}" - ); - } -} - -#[test] -fn install_script_survives_archive_pin_drift() { - let script = base_install_script(); - assert!( - script.contains("apt-get -s install -qq --no-install-recommends git=") - && script.contains("exact hosted apt pins are unavailable") - && script.contains("22.04) base_packages='") - && script.contains("libatk1.0-0 libatk-bridge2.0-0") - && script.contains("libatk1.0-0t64 libatk-bridge2.0-0t64") - && script.contains("apt-get install -y -qq --no-install-recommends $base_packages",), - "the baseline must choose Jammy/Noble ABI package names before falling back" - ); - assert!( - script.contains("for package in clang-16 clang-format-16 clang-tidy-16") - && script.contains("apt-get -s install -qq --no-install-recommends \"$package\"") - && script.contains("available_compiler_packages=\"$available_compiler_packages $package\"") - && script.contains("hosted compiler matrix is incomplete") - && script.contains( - "apt-get install -y -qq --no-install-recommends clang clang-format clang-tidy gcc g++ gfortran", - ), - "the compiler fallback must retain available versioned tools and add archive defaults" - ); -} - -#[test] -fn install_script_preserves_compiler_fidelity_when_pins_resolve() { - let script = base_install_script(); - for fragment in [ - "clang-16 --version | head -1 | grep -F '16.0.6'", - "clang-17 --version | head -1 | grep -F '17.0.6'", - "clang-18 --version | head -1 | grep -F '18.1.3'", - "test \"$(gcc-12 -dumpfullversion)\" = '12.4.0'", - "test \"$(gcc-13 -dumpfullversion)\" = '13.3.0'", - "test \"$(gcc-14 -dumpfullversion)\" = '14.2.0' || exit 1", - "apt-get install -y -qq --no-install-recommends $available_compiler_packages || exit 1", - "update-alternatives --set \"$tool\" \"/usr/bin/$tool-18\" || exit 1", - ] { - assert!( - script.contains(fragment), - "install script lost compiler fidelity check/default: {fragment}" - ); - } - assert!( - script.contains("if [ \"$compiler_matrix_complete\" = 1 ]"), - "exact version checks must run only when the complete hosted matrix resolved" - ); -} - -#[test] -fn install_script_is_valid_shell() { - let status = std::process::Command::new("sh") - .args(["-n", "-c", &base_install_script()]) - .status() - .expect("run the shell parser"); - assert!( - status.success(), - "base install script must parse as POSIX shell" - ); -} - -/// The runner account lives in the bake script, not in a post-bake `exec`. -/// -/// The fingerprint hashes `base_install_script`, and nothing else about the -/// bake. Preparing the account outside it left the fingerprint unchanged, so a -/// pool adopted the previous golden and served jobs an account the new code no -/// longer matched — the `/usr/local/rustup` EACCES that failed every Rust job. -#[test] -fn install_script_prepares_the_runner_account_and_toolchain_homes() { - let script = base_install_script(); - assert!( - script.contains("useradd -m -u 1001 -s /bin/bash runner"), - "the unprivileged runner account must be baked, not created at runtime" - ); - // `ToolchainLayer::Rust` installs these as root at the addresses - // `guest_env_prefix` exports; without ownership the runner cannot write - // them and `rustup toolchain install` dies before any build starts. - assert!( - script.contains("chown -R 1001:1001 /usr/local/rustup /usr/local/cargo"), - "the exported RUSTUP_HOME/CARGO_HOME must be owned by the runner account" - ); - assert!( - script.contains("/etc/sudoers.d/preloop-runner"), - "steps run `sudo apt-get install`, so the account needs NOPASSWD sudo" - ); -} - -/// Hosted images keep their apt package lists, so real workflows install -/// system packages with a bare `sudo apt-get install ` and no preceding -/// `apt-get update` — uv's musl cell (`apt-get install musl-tools`) is one. -/// Wiping `/var/lib/apt/lists` to save image bytes turns every one of those -/// steps into `E: Unable to locate package`. -#[test] -fn golden_keeps_apt_lists_for_workflow_installs() { - let script = base_install_script(); - assert!( - !script.contains("/var/lib/apt/lists"), - "the golden must keep apt lists: workflows apt-install without updating" - ); - assert!( - script.contains("apt-get clean"), - "cached .deb archives are still dropped" - ); - assert!( - script.contains("rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/*"), - "non-runtime documentation must not push the packed image past smolvm's 4 GiB export cap" - ); -} - -/// GitHub's images ship `/etc/apt/apt.conf.d/90assumeyes`, which is why -/// `sudo apt-get install musl-tools` (uv's musl cell — no `-y`) installs three -/// new packages in the golden without ever printing a confirmation prompt. -/// Without it the same step stops at `Do you want to continue? [Y/n]`. -#[test] -fn golden_assumes_yes_for_apt_like_hosted_images() { - let script = base_install_script(); - assert!( - script.contains("APT::Get::Assume-Yes") - && script.contains("/etc/apt/apt.conf.d/90assumeyes"), - "the golden must carry the hosted images' apt assume-yes config" - ); -} - -/// `sudo` resolves its own hostname on every invocation; an unresolvable one -/// prints `sudo: unable to resolve host ` before each command, which no -/// hosted-runner log contains. -#[test] -fn golden_resolves_its_own_hostname() { - let script = base_install_script(); - assert!( - script.contains("$(hostname)") && script.contains(">> /etc/hosts"), - "the golden must add its own hostname to /etc/hosts" - ); -} - -#[test] -fn baseline_has_no_duplicate_entries() { - // A duplicate is harmless to apt but signals a botched merge, and the list - // is edited by hand often enough for that to happen. - let packages: Vec<&str> = base_packages().split_whitespace().collect(); - let mut seen = std::collections::BTreeSet::new(); - let duplicates: Vec<&&str> = packages.iter().filter(|p| !seen.insert(**p)).collect(); - assert!( - duplicates.is_empty(), - "duplicate packages in the baseline: {duplicates:?}" - ); -} - -#[test] -fn container_storage_avoids_the_overlay_root() { - // Two constraints, both verified on live VMs, and only `/storage` meets both. - // - // 1. Must not be the overlay root. containerd mounts each container rootfs - // as an overlay whose lowerdir is an image layer; when those layers are - // themselves on overlayfs the mount fails `invalid argument` and every - // `docker create` exits 1. This hides in single-VM testing, where pulls - // land in that VM's own upper layer and run fine -- it only breaks in a - // fork, where the layers arrive through a *lower* overlay. - // - // 2. Must be inherited by forks, so images preloaded into the golden are - // free for every runner. `smolvm machine fork` copy-on-writes the ext4 - // disk as well as the overlay root (verified: a 64 MiB file written to - // the golden's `/dev/vda` mount is present in the clone). - let root = docker_data_root(); - assert!( - root.starts_with("/storage/") || root.starts_with("/workspace/"), - "container data root {root} must be the forked ext4 volume, not the overlay root" - ); - assert_ne!( - root, "/var/lib/docker", - "the default path lands on the overlay root, where container rootfs mounts fail" - ); -} - -#[test] -fn baseline_excludes_toolchains_that_belong_to_setup_actions() { - // Adding these would bloat the image and, worse, let a workflow pass - // locally while depending on something it never declared — which breaks on - // real GitHub. Toolchains belong to `actions/setup-*` or `container:`. - let packages: Vec<&str> = base_packages().split_whitespace().collect(); - for forbidden in [ - "openjdk-17-jdk", - "golang-go", - "rustc", - "cargo", - "dotnet-sdk-8.0", - "php", - "ruby-full", - "google-chrome-stable", - "firefox", - "postgresql", - "mysql-server", - ] { - assert!( - !packages.contains(&forbidden), - "`{forbidden}` must not be baked into the golden — a workflow that \ - relies on it without declaring `setup-*`/`container:`/`services:` \ - passes locally and fails on GitHub" - ); - } -} - -#[test] -fn guest_sysctls_match_the_hosted_image() { - // GitHub's hosted image writes these into /etc/sysctl.conf at build time - // (actions/runner-images images/ubuntu/scripts/build/configure-environment.sh) - // and a probe job on a hosted runner reads them back from /proc/sys - // (image 20260927.320.1 x64 / 20260927.135.1 arm64, kernel - // 6.17.0-1022-azure). The golden's guest boots without an init applying - // /etc/sysctl.*, so the same values are applied per machine; if this list - // drifts, suites that depend on them (Redis/Valkey's vm.max_map_count, - // file watchers) diverge silently from ubuntu-latest. - let expected = [ - ("vm.max_map_count", "262144"), - ("fs.inotify.max_user_watches", "655360"), - ("fs.inotify.max_user_instances", "1280"), - ]; - assert_eq!( - GITHUB_GUEST_SYSCTLS, - expected.as_slice(), - "the hosted sysctl baseline changed — re-probe a real hosted runner \ - before updating this list" - ); - let script = guest_sysctl_script(); - for (key, value) in expected { - assert!( - script.contains(&format!("{key}={value}")), - "the guest script must carry {key}={value}: {script}" - ); - } - // vm.overcommit_memory is 0 on hosted runners too (probe of image - // 20260927.320.1, kernel 6.17.0-1022-azure), so Valkey's overcommit - // warning is expected on both sides; forcing it to 1 would diverge. - assert!( - !GITHUB_GUEST_SYSCTLS - .iter() - .any(|(key, _)| *key == "vm.overcommit_memory"), - "overcommit must stay at the value GitHub-hosted runners have: 0" - ); -} - -#[test] -fn loopback_resolves_by_name() { - // The base image ships an empty /etc/hosts and `nsswitch.conf` is - // `hosts: files dns`, so `localhost` falls through to the upstream resolver - // and fails. Everything still works over 127.0.0.1, which is exactly why - // this hid until a `services:` job tried to reach localhost:5432. - let hosts = loopback_hosts(); - assert!( - hosts.contains("127.0.0.1 localhost"), - "IPv4 loopback must resolve by name; `services:` are reached at localhost:" - ); - assert!( - hosts.contains("::1 localhost"), - "IPv6 loopback must resolve by name" - ); - for entry in ["ip6-localhost", "ip6-loopback", "ip6-allnodes"] { - assert!( - hosts.contains(entry), - "missing standard entry `{entry}` present on a stock Ubuntu host" - ); - } -} diff --git a/crates/preloop-orchestrator/tests/runner_pool_lifecycle.rs b/crates/preloop-orchestrator/tests/runner_pool_lifecycle.rs index 6a70dadd5..7567e150f 100644 --- a/crates/preloop-orchestrator/tests/runner_pool_lifecycle.rs +++ b/crates/preloop-orchestrator/tests/runner_pool_lifecycle.rs @@ -4,8 +4,8 @@ use preloop_orchestrator::{ artifact_payload, node_externals, }; use preloop_vm::{ - ExecOutput, MachineName, MachineSpec, MachineState, NetworkPolicy, OutputChunk, SecretSource, - SocketMount, VmError, VmProvider, VolumeMount, + ExecOutput, MachineName, MachineSpec, MachineState, NetworkPolicy, OutputChunk, + ProviderCapabilities, SecretSource, SocketMount, VmError, VmProvider, VolumeMount, }; use std::collections::HashMap; use std::fs; @@ -51,6 +51,10 @@ struct RecordingVmProvider { state: Mutex, run_actions: Mutex>, changed: Notify, + /// Whether packs are host files (SmolVM) or live server-side (AgentENV). + /// A backend without host-side packs has no artifact to unpack, so its + /// golden is baked in the guest instead. + file_packs: bool, } impl RecordingVmProvider { @@ -66,9 +70,17 @@ impl RecordingVmProvider { }), run_actions: Mutex::new(run_actions), changed: Notify::new(), + file_packs: true, } } + /// Model a backend whose packs are not host files (AgentENV): there is no + /// artifact to unpack, so the pool prepares its golden in the guest. + fn no_packs(mut self) -> Self { + self.file_packs = false; + self + } + async fn wait_until(&self, predicate: F) where F: Fn(&ProviderState) -> bool, @@ -128,6 +140,13 @@ fn provider_error(message: &'static str) -> VmError { #[async_trait] impl VmProvider for RecordingVmProvider { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + file_packs: self.file_packs, + ..ProviderCapabilities::default() + } + } + async fn create(&self, spec: &MachineSpec) -> Result<(), VmError> { let mut state = self.state.lock().await; state @@ -305,11 +324,12 @@ struct Fixture { /// restores the previous values on drop. The pool reads all of these from the /// process environment, so without the pins the tests depend on the host: /// -/// - `PRELOOP_GOLDEN_URL` -> unreachable. The pool downloads a prebaked golden -/// before building when one is reachable (`prepare_artifact` -> -/// `download_prebaked_golden`); a host with egress to the release asset -/// would skip the local build+pack these tests assert and hang -/// `wait_until(Event::Pack)` forever. +/// - `PRELOOP_GOLDEN_URL` -> unreachable. Configured-image fixtures bake +/// their golden locally and never download; only the official sentinel does +/// (`prepare_artifact` -> `download_prebaked_golden`), so the pin keeps +/// that download off the network and failing fast in +/// `official_golden_download_failure_is_a_startup_error` instead of +/// fetching a real pack from GHCR or GitHub on a host with egress. /// - `PRELOOP_SKIP_DISK_PREFLIGHT` -> on. The golden build refuses when the /// host volume lacks builder disk + pack staging (60 GiB for this fixture); /// the provider here is a recording mock that writes nothing, so a host @@ -356,6 +376,9 @@ impl Drop for HermeticEnvGuard { impl Fixture { fn new(label: &str, payload_exists: bool) -> Self { + // A configured custom image, not the official sentinel: the pool bakes + // it locally (as-is plus the runner contract), so no fixture here + // depends on the released official golden. const BASE_IMAGE: &str = "ghcr.io/preloop/base:latest"; let env_guard = TEST_ENV_LOCK .lock() @@ -388,11 +411,9 @@ impl Fixture { let config = RunnerPoolConfig { size: 1, use_fork: false, - use_packed_artifact: false, name_prefix: format!("pool-{label}-{id}"), pool_status: None, base_image: BASE_IMAGE.to_owned(), - workspace: None, artifact_stem, release_version: "9.9.9".to_owned(), runner_bundle: bundle, @@ -571,8 +592,9 @@ async fn artifact_preparation_runs_once_and_reuses_payload_on_next_run() { vec![RunAction::Wait, RunAction::Wait], )); let pool = RunnerPool::new(provider.clone(), fixture.config.clone()).unwrap(); - // The builder machine's install steps count as `exec` calls, so a - // `run_calls` wait can cancel the pool between the builder's install and + // The builder machine's bake steps (guest readiness, the runner + // contract, the externals symlink) count as `exec` calls, so a + // `run_calls` wait can cancel the pool between the builder's start and // its pack on a loaded host — the pack never runs and the assertion // below fails with `pack_calls == 0`. Wait on the pack itself, which // only happens after the artifact is fully built, then cancel. @@ -618,21 +640,64 @@ async fn artifact_preparation_runs_once_and_reuses_payload_on_next_run() { ); } +/// The official golden is downloaded packed and is never baked locally. +/// +/// The deleted stock-Ubuntu bake used to stand in when the official golden +/// could not be fetched, which silently ran jobs on a different image than the +/// one `runs-on: ubuntu-latest` names. There is no substitute now: an +/// unreachable pack fails the pool at startup, and the error names the +/// official golden so an operator knows to point +/// `PRELOOP_GOLDEN_OCI_REF`/`PRELOOP_GOLDEN_URL` at a reachable one. +#[tokio::test] +async fn official_golden_download_failure_is_a_startup_error() { + let fixture = Fixture::new("official-fatal", false); + let mut config = fixture.config.clone(); + config.base_image = preloop_orchestrator::environment::OFFICIAL_GOLDEN.to_owned(); + let provider = Arc::new(RecordingVmProvider::with_machines(&[], vec![])); + let pool = RunnerPool::new(provider.clone(), config).unwrap(); + + let error = pool + .run(CancellationToken::new()) + .await + .expect_err("an unreachable official golden must fail the pool, not bake a substitute"); + let message = error.to_string(); + assert!( + message.contains("official golden"), + "the error must name the official golden: {message}" + ); + assert!( + provider.snapshot().await.events.is_empty(), + "no substitute golden may be created for the official sentinel" + ); +} + +/// A backend without host-side packs has no artifact to unpack: its golden is +/// booted from the configured image, the runner contract is applied in the +/// guest, and the result is frozen with `start_forkable`. The contract is +/// applied once, on the golden — forks inherit it — so no runner pays for it. #[tokio::test] async fn fork_golden_is_marked_forkable_after_guest_provisioning() { - let fixture = Fixture::new("fork-golden", true); + let fixture = Fixture::new("fork-golden", false); let mut config = fixture.config.clone(); config.use_fork = true; config.control_socket = Some(fixture.root.join("engine.sock")); - let provider = Arc::new(RecordingVmProvider::with_machines( - &[], - vec![RunAction::Wait], - )); + let provider = + Arc::new(RecordingVmProvider::with_machines(&[], vec![RunAction::Wait]).no_packs()); let pool = RunnerPool::new(provider.clone(), config).unwrap(); run_until_cancelled(pool, &provider, CancellationToken::new(), 1).await; - let events = provider.snapshot().await.events; + let snapshot = provider.snapshot().await; let golden = format!("{}-golden", fixture.config.name_prefix); + // The golden boots the configured image itself: there is no packed + // artifact on this backend to unpack, and none may be built. + let spec = snapshot + .created_specs + .iter() + .find(|spec| spec.name.as_str() == golden) + .expect("golden machine specification"); + assert_eq!(spec.image, fixture.config.base_image); + + let events = &snapshot.events; let golden_starts = events .iter() .enumerate() @@ -650,7 +715,8 @@ async fn fork_golden_is_marked_forkable_after_guest_provisioning() { assert!( events[first_start + 1..stop] .iter() - .any(|event| matches!(event, Event::Exec(name, _) if name == &golden)) + .any(|event| matches!(event, Event::Exec(name, _) if name == &golden)), + "the guest contract must be applied to the golden before it freezes: {events:?}" ); assert!(stop < golden_starts[1]); } @@ -717,6 +783,11 @@ async fn runner_keeps_public_only_egress_and_wires_control_socket_and_environmen .find(|spec| spec.name.as_str() == runner) .expect("runner machine specification"); assert_eq!(spec.network, NetworkPolicy::PublicOnly); + // A packed machine reaches node through the artifact's baked symlink + // (`/externals -> /opt/preloop/bin/externals`): the externals ride + // the bundle mount instead of a third virtiofs mount, which the packed + // launcher has no IRQ budget for. Only a machine booted from a raw image + // carries the separate externals mount. assert_eq!( &spec.volumes, &vec![ @@ -725,13 +796,6 @@ async fn runner_keeps_public_only_egress_and_wires_control_socket_and_environmen guest: PathBuf::from("/opt/preloop/bin"), read_only: true, }, - VolumeMount { - // Node externals are mounted host-side, never baked into the - // machine image or downloaded per runner. - host: fixture.root.join("host-externals").join("externals"), - guest: PathBuf::from("/home/runner/externals"), - read_only: true, - }, VolumeMount { // Per-machine control-bridge directory: the guest agent's // socket node lands INSIDE this host directory, so a shared @@ -751,11 +815,26 @@ async fn runner_keeps_public_only_egress_and_wires_control_socket_and_environmen ); // The guest is always told its own machine name: a debug session needs it - // to tell a controller which VM to open a shell into. + // to tell a controller which VM to open a shell into. The environment is + // applied by the launcher shell that sources the image's + // `/etc/environment`, so the argv is `sh -c ` followed by the + // `KEY=value` entries in the same order `env` receives them. + let path = format!("PATH={}", preloop_orchestrator::guest_runner_path(&config)); + let machine = format!("PRELOOP_MACHINE_NAME={runner}"); let expected_prefix = vec![ - "/usr/bin/env".to_owned(), - format!("PATH={}", preloop_orchestrator::guest_runner_path(&config)), - format!("PRELOOP_MACHINE_NAME={runner}"), + "sh".to_owned(), + "-c".to_owned(), + [ + "if [ -r /etc/environment ]; then . /etc/environment; fi; ", + "exec /usr/bin/env ", + &format!("'{path}' '{machine}' "), + "'PRELOOP_CONTROL_ORIGIN=https://preloop.example' ", + "'PRELOOP_CONTROL_SOCKET=/run/preloop-control/engine.sock' ", + "\"$0\" \"$@\"", + ] + .concat(), + path.clone(), + machine.clone(), "PRELOOP_CONTROL_ORIGIN=https://preloop.example".to_owned(), "PRELOOP_CONTROL_SOCKET=/run/preloop-control/engine.sock".to_owned(), ]; @@ -880,18 +959,39 @@ async fn guest_environment_tracks_control_socket_and_debug_dir_independently() { .map(String::as_str) .collect(); let machine_name = format!("PRELOOP_MACHINE_NAME={runner}"); + // The runner's fallback PATH is the system directories a hosted step + // shell sees; the launcher replaces it with the image's + // `/etc/environment` PATH whenever the image declares one. The deleted + // Rust/Go language installs (and their `/usr/local/cargo/bin` and + // `/usr/local/go/bin` entries) are gone. let path = format!("PATH={}", preloop_orchestrator::guest_runner_path(&config)); - let want_base = vec!["/usr/bin/env", path.as_str(), machine_name.as_str()]; - // Unconditional toolchain homes come last in the prefix (fixed - // system addresses shared by root and switched runners). - let want_tail = [ - "RUSTUP_HOME=/usr/local/rustup", - "CARGO_HOME=/usr/local/cargo", - ]; - let mut want = want_base; - want.extend(expected); - want.extend(want_tail); - assert_eq!(prefix, want,); + let mut entries = vec![path.clone(), machine_name.clone()]; + entries.extend(expected.iter().map(|entry| (*entry).to_owned())); + let pairs = entries + .iter() + .map(|entry| format!("'{entry}'")) + .collect::>() + .join(" "); + let launcher = [ + "if [ -r /etc/environment ]; then . /etc/environment; fi; ", + "exec /usr/bin/env ", + &pairs, + " \"$0\" \"$@\"", + ] + .concat(); + let mut want = vec!["sh".to_owned(), "-c".to_owned(), launcher]; + want.extend(entries); + assert_eq!(prefix, want.iter().map(String::as_str).collect::>()); + // Nor may anything export the deleted Rust/Go install homes: a runner + // told RUSTUP_HOME/CARGO_HOME would point rustup at paths the golden + // no longer carries. + assert!( + configure + .iter() + .all(|argument| !argument.starts_with("RUSTUP_HOME=") + && !argument.starts_with("CARGO_HOME=")), + "the deleted Rust/Go install homes must not be exported: {configure:?}" + ); } } diff --git a/crates/preloop-runner-server/src/config.rs b/crates/preloop-runner-server/src/config.rs index e1dfa1c69..77b71eb4a 100644 --- a/crates/preloop-runner-server/src/config.rs +++ b/crates/preloop-runner-server/src/config.rs @@ -47,8 +47,9 @@ pub struct GoldenConfig { #[serde(skip_serializing_if = "Option::is_none")] pub kind: Option, /// Base image `serve` boots its golden from: an OCI reference, a - /// docker-save `.tar`, or a `.smolmachine`/rootfs path. Absent means the - /// engine's stock base — the packed official GitHub runner golden. + /// docker-save `.tar`, or a `.smolmachine`/rootfs path. Absent selects the + /// packed official GitHub runner golden — the image `runs-on: + /// ubuntu-latest` means, downloaded and verified, never baked locally. #[serde(skip_serializing_if = "Option::is_none")] pub base_image: Option, /// Dockerfile a `kind = "dockerfile"` tar was built from, so a re-run can @@ -58,7 +59,7 @@ pub struct GoldenConfig { } /// Effective golden base image: [`BASE_IMAGE_ENV`] when set to a non-blank -/// value, else the `[golden]` section. `None` means the engine's stock base. +/// value, else the `[golden]` section. `None` means the packed official golden. /// /// An exported-but-blank variable behaves like an unset one — the same rule /// `PRELOOP_GOLDEN_URL` follows — so a shell that exports an empty value diff --git a/docs/build-goldens.md b/docs/build-goldens.md index dbc0cd52c..8dbed81bb 100644 --- a/docs/build-goldens.md +++ b/docs/build-goldens.md @@ -111,11 +111,18 @@ export PRELOOP_GOLDEN_NAME_PREFIX=preloop-official-golden-x86-64 ./target/release/preloop build-golden \ --runner-bundle target/x86_64-unknown-linux-gnu/release \ + --base-image "$PRELOOP_RUNNER_BASE_IMAGE" \ --output dist/preloop-ubuntu-24.04-x86_64 sha256sum dist/preloop-ubuntu-24.04-x86_64 \ > dist/preloop-ubuntu-24.04-x86_64.sha256 ``` +`build-golden` bakes the image it is given: `--base-image ` (or the +configured `PRELOOP_RUNNER_BASE_IMAGE` / `[golden] base_image` when the flag is +absent). There is no default — the official golden is published packed and +cannot be built locally. The bake adds the runner contract only; see +[vm-images.md](vm-images.md#what-the-golden-contains). + `build-golden` renames the `.smolmachine` sidecar to the requested output path. The launcher stub is discarded; the output file itself is the packed payload. diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 7b03d9ca8..c73a0f13f 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -460,17 +460,21 @@ preloop build-golden \ ``` The command is hidden from the normal help because it is an operator/build -command rather than part of workflow submission. The base must currently be -Ubuntu-derived, and its architecture must match the runner bundle and host. -`--base-image` also accepts a registry snapshot of the official hosted image. -`--workspace ` selects the workspace whose toolchain version files -(rust-toolchain.toml, .nvmrc, …) are baked into the golden; it defaults to -the current directory. +command rather than part of workflow submission. `build-golden` must know the +image it bakes: pass `--base-image `, or leave it out to use the +configured image (`PRELOOP_RUNNER_BASE_IMAGE`, else `[golden] base_image`). +There is no default base — the official golden is published packed and cannot +be built locally. A registry snapshot of the official hosted image is a valid +base, and so is any other image; the only requirement the bake checks is a +glibc dynamic loader, and its architecture must match the runner bundle and +host. The image is used as it is, plus the GitHub-runner machinery (the +`runner` account, tool cache, and build record) — no packages or toolchains +are added. Set `--storage-gb` or `PRELOOP_RUNNER_STORAGE_GB` for large snapshots; the default is 80 GiB. See [VM images and version tracking](vm-images.md#building-a-golden) for the -stock build, custom OCI base, checksum, publishing, and runtime configuration -steps. +custom-OCI-base, pre-bake, and runtime configuration steps, and +[Building official goldens](build-goldens.md) for the official artifact. For the everyday case — "run my workflows on something other than the official image" — `preloop init` records the choice instead: @@ -483,8 +487,10 @@ dockerfile = "ci/Dockerfile" # only for kind = "dockerfil ``` `serve` reads `base_image` when `PRELOOP_RUNNER_BASE_IMAGE` is unset, and -prepares the golden on its normal path: the packed official artifact for the -stock base, or a local bake for a custom one. +prepares the golden on its normal path: jobs that name `ubuntu-latest` or +`ubuntu-24.04` — and a pool with no image configured at all — use the +official packed golden, downloaded per architecture; every other label set +uses the configured image, baked locally as-is. ## Environment variables @@ -499,11 +505,11 @@ stock base, or a local bake for a custom one. | `PRELOOP_RUNNER_POOL_ENABLED` | Enable the local microVM runner pool (default off) | | `PRELOOP_RUNNER_POOL_SIZE` | Pool size (warm forks/VMs) | | `PRELOOP_RUNNER_CPUS` | vCPUs allocated to each runner VM (default 8) | -| `PRELOOP_USE_FORK` | Run the pool as forked microVMs (default true with a packed golden) | -| `PRELOOP_USE_PACKED_GOLDEN` | Use a release or locally cached packed golden (default on; set `false` for cold OCI provisioning) | -| `PRELOOP_GOLDEN_URL` | Override the packed golden URL; checksum URL is this value plus `.sha256` | +| `PRELOOP_USE_FORK` | Run the pool as forked microVMs (default true) | +| `PRELOOP_GOLDEN_URL` | Mirror of the official packed golden; the checksum URL is this value plus `.sha256`. When set it replaces the OCI reference, and the mirror must be verifiable — an unverified mirror is refused | +| `PRELOOP_GOLDEN_SHA256` | Expected SHA-256 of the packed golden, for mirrors that publish no `.sha256` sidecar | | `PRELOOP_GOLDEN_OCI_REF` | Override the per-architecture packed golden OCI reference; the engine has digest-pinned defaults for both the official arm64 and x86_64 GHCR artifacts | -| `PRELOOP_RUNNER_BASE_IMAGE` | Override the digest-pinned Ubuntu base identity at serve time; set it with `PRELOOP_GOLDEN_URL` for a custom packed golden. Wins over the `[golden] base_image` that `preloop init` records | +| `PRELOOP_RUNNER_BASE_IMAGE` | Image a custom golden is baked from, used as-is plus the GitHub-runner machinery. Wins over the `[golden] base_image` that `preloop init` records; `runs-on: ubuntu-latest`/`ubuntu-24.04` and an unconfigured pool use the official packed golden regardless | | `PRELOOP_VERIFY_BASE_IMAGE` / `PRELOOP_VERIFY_BASE_IMAGE_REPO` | Require a digest-pinned OCI base's GitHub attestation and Cosign signature before `build-golden` | | `PRELOOP_REQUIRE_BASE_DIGEST` | Reject mutable registry tags during `build-golden` (used by release provenance builds) | | `PRELOOP_RUNNER_STORAGE_GB` | Persistent guest storage per runner and golden build (default 80 GiB) | @@ -513,7 +519,6 @@ stock base, or a local bake for a custom one. | `PRELOOP_RUNNER_PACK_NO_PROXY` | Proxy bypass list for golden packing; `NO_PROXY` and `no_proxy` are fallbacks | | `PRELOOP_RUNNER_LABELS` | Extra `runs-on` labels the pool's runners declare | | `PRELOOP_RUNNER_USER` / `PRELOOP_RUNNER_UID` | Guest runner account (default `runner`/1001); `root` restores root; empty disables switching | -| `PRELOOP_WORKSPACE` | Workspace whose toolchain version files (rust-toolchain.toml, .nvmrc, …) drive golden toolchain baking; overrides the current directory for daemon deployments | | `PRELOOP_URL` | Server URL for the client commands (default `http://127.0.0.1:9090`) | | `PRELOOP_SYSTEM_TOKEN` | Native API bearer token (also `PRELOOP_TOKEN`) | | `PRELOOP_GITHUB_TOKEN` | PAT fallback for GitHub API calls (check runs need the App) | diff --git a/docs/fidelity-gap.md b/docs/fidelity-gap.md index b521c4842..a92548765 100644 --- a/docs/fidelity-gap.md +++ b/docs/fidelity-gap.md @@ -443,12 +443,12 @@ workflow. All are fixed: | `runs-on` was not evaluated per matrix combination | `runs-on: ${{ matrix.os }}` — the shape tokio, caddy and uv all use — expanded to a label no runner can advertise, so every cell queued forever. This is why the campaign's clones all carried a patched `runs-on`, which defeats the point of the exercise | `expand.rs` resolves each label against the cell's matrix (and reusable-workflow inputs), like every other per-cell field | | A self-hosted runner stood in for **any** hosted image label, across operating systems | a macOS host sharing the engine claimed `ubuntu-latest` jobs and failed them inside a step (tokio's Linux-only `taskdump`, uv's `/home/runner` layout) | `job_matches_runner` matches the OS the runner declares; a runner declaring none stays eligible | | A machine took a job it only loosely matched | a 24.04 machine claimed the `ubuntu-22.04` job while the pool was still baking the 22.04 golden that job asked for | `take_matching_job` prefers a job whose labels the runner carries verbatim, and stands in only when nothing exact is claimable | -| Env goldens were rebaked on every engine restart | the in-memory registry forgot a golden that was still running and still correct, so the first `ubuntu-22.04` job after any deploy waited 5–11 minutes for apt and rustup | a host-side fingerprint record beside the packed artifact; a running golden whose record matches is adopted | +| Env goldens were rebaked on every engine restart **(obsolete — the environment-golden machinery and the curated bake are deleted; the golden is the packed official image, or a configured image plus the runner contract)** | the in-memory registry forgot a golden that was still running and still correct, so the first `ubuntu-22.04` job after any deploy waited 5–11 minutes for apt and rustup | a host-side fingerprint record beside the packed artifact; a running golden whose record matches is adopted | | A custom `shell:` command line was collapsed into one argv entry | `taiki-e/install-action` declares `shell: /usr/bin/env -u ENV … /bin/sh -eu {0}`; `env` unset one absurdly named variable, found no command, printed the environment, exited 0 — the step "succeeded" installing nothing and the next step died with `no such command: hack` | `resolve_shell` splits every whitespace token; `{0}` substitution per token | -| Toolchain bin directories were not on the runner's PATH | `dtolnay/rust-toolchain` only appends `$CARGO_HOME/bin` to `$GITHUB_PATH` when it installs rustup itself, so on an image that already has rustup (ours, and GitHub's) anything `cargo install`-ed was unreachable | the guest runner is launched with an explicit PATH covering `$HOME/.cargo/bin` and `/usr/local/go/bin` | -| The baseline wiped `/var/lib/apt/lists` | uv's musl cell runs `sudo apt-get install musl-tools` with no `apt-get update`: `E: Unable to locate package` | keep the lists; a fork of a pack that shipped without them refreshes on provision | +| Toolchain bin directories were not on the runner's PATH **(obsolete — no toolchain is baked any more, and the guest PATH is the system PATH, as on GitHub)** | `dtolnay/rust-toolchain` only appends `$CARGO_HOME/bin` to `$GITHUB_PATH` when it installs rustup itself, so on an image that already has rustup (ours, and GitHub's) anything `cargo install`-ed was unreachable | the guest runner is launched with an explicit PATH covering `$HOME/.cargo/bin` and `/usr/local/go/bin` | +| The baseline wiped `/var/lib/apt/lists` **(obsolete — there is no bake to wipe them; the golden keeps the image's own apt state)** | uv's musl cell runs `sudo apt-get install musl-tools` with no `apt-get update`: `E: Unable to locate package` | keep the lists; a fork of a pack that shipped without them refreshes on provision | | Rosetta enable failure left a half-created machine | an Apple Silicon host without Rosetta 2 (or a transient `smolvm machine update --rosetta` failure) left a broken VM the pool kept reusing; x86_64 golden jobs failed at start instead of surfacing the cause | the provider propagates the `update --rosetta` error and deletes the partial machine (`rosetta_update_failure_is_returned_and_partial_machine_is_deleted`); x86_64 guests on Apple Silicon run under Rosetta 2 translation, with arm64 native preferred (see `docs/vm-images.md`) | -| A fork of the packed golden trusted the pack's bake | a published pack predating the workspace's toolchain pin boots without cargo, and cargo-dist dies on "you don't appear to have cargo installed" in 11s | probe each toolchain layer per fork and install only what the pack lacks; and `build-golden` now bakes the workspace's toolchains (`rust-toolchain.toml` etc.) into the artifact, so forks inherit them — the per-fork install is gone once the pack is rebuilt | +| A fork of the packed golden trusted the pack's bake **(obsolete — there is no per-fork toolchain install: the golden is the image plus the runner contract, and nothing is installed on top)** | a published pack predating the workspace's toolchain pin boots without cargo, and cargo-dist dies on "you don't appear to have cargo installed" in 11s | probe each toolchain layer per fork and install only what the pack lacks; and `build-golden` now bakes the workspace's toolchains (`rust-toolchain.toml` etc.) into the artifact, so forks inherit them — the per-fork install is gone once the pack is rebuilt | | The guest hostname did not resolve | every `sudo` call printed `sudo: unable to resolve host `, which appears in no hosted log | the baseline adds the hostname to `/etc/hosts` | | `apt-get install` prompted, and a step's stdin never reached EOF | the hosted images ship `/etc/apt/apt.conf.d/90assumeyes`, so uv's `sudo apt-get install musl-tools` (no `-y`) installs three packages without a prompt; here it asked `Do you want to continue? [Y/n]` and, because the runner is a child of a guest `exec` whose stdin never closes, blocked for 45 minutes | ship the same apt config, and give every step `Stdio::null()` for stdin | @@ -525,9 +525,9 @@ Every item below broke a real workflow step and was fixed in preloop: | Job/workflow-level `env:` never reached action processes | moby's `docker buildx bake` resolved `${DESTDIR}` to `""` because the hcl default took over — the `govulncheck` target's `output = ["${DESTDIR}"]` collapsed to an empty output list, the report was never exported, and the bake action failed on the missing path. The server put job env in the message `variables` map and left the wire `environmentVariables` (the field the official runner materializes into step environments) empty | the job message builder now populates `environmentVariables` from the job env | | A server restart left every queued run permanently wedged | jobs restored from the store sat "pending" forever: the on-demand pool only forks while its shared `queue_depth` atomic is non-zero, and after a restart no runner exists to refresh it from a broker poll | `serve` re-arms the atomic with the recovered ready-queue length and re-syncs `next_job_runs_on` | | A run stuck on unclaimable jobs permanently parks its concurrency group | neovim's `test.yml` pins `concurrency:` on the workflow; a run whose remaining jobs were `windows-*` cells (held queued indefinitely by the starvation sweep, which deliberately waits for an external host) never went terminal, so its run-level concurrency holder never released and every later submission in the group parked forever ("pending") | a later submission acquiring the same concurrency key now treats a holder whose remaining jobs all need an external host — with no such host registered — as stuck and takes the slot (a registered-but-busy host keeps it); restore-time reconciliation separately clears holders whose runs are terminal or missing. The stuck run stays queued and re-acquires if a host ever appears | -| The golden lacked the Chromium/playwright runtime libraries | vite's tests died in playwright's host-requirements check (`Failed to launch browser`, missing `libnss3` etc.); GitHub's ubuntu-24.04 image ships these | 21 browser runtime libs pinned in `versions.toml` and added to the golden bake. Caveat: the *packed* artifact cache is keyed only by the base-image digest, so package-list changes reach a packed golden only after the artifact is rebuilt (`prepare_artifact` rebuilds when the payload file is missing) — the pins take effect immediately for non-packed goldens and for the next artifact build | +| The golden lacked the Chromium/playwright runtime libraries **(obsolete — browser libraries come from the image now: the official golden is the hosted image, and a configured image is used as-is)** | vite's tests died in playwright's host-requirements check (`Failed to launch browser`, missing `libnss3` etc.); GitHub's ubuntu-24.04 image ships these | 21 browser runtime libs pinned in `versions.toml` and added to the golden bake. Caveat: the *packed* artifact cache is keyed only by the base-image digest, so package-list changes reach a packed golden only after the artifact is rebuilt (`prepare_artifact` rebuilds when the payload file is missing) — the pins take effect immediately for non-packed goldens and for the next artifact build | | The golden lacked RubyGems and Perl's cpanminus | neovim's functionaltest setup runs `gem install … neovim` and `sudo cpanm -n Neovim::Ext`; `gem: command not found` killed every posix cell | `ruby`, `ruby-rubygems`, `perl`, `cpanminus` pinned in `versions.toml` and baked (artifact rebuilt for the campaign) | -| The golden lacked the rest of the hosted image's apt baseline | neovim's LLVM install script died on `lsb_release: command not found`; each missing utility (`xvfb`, `telnet`, `sshpass`, …) was a separate workflow-killing gap | the remaining packages from the official ubuntu-24.04 image's apt list (image readme 20260720.247.2) pinned and baked: `lsb-release`, `fonts-noto-color-emoji`, `haveged`, `mediainfo`, `p7zip-rar`, `pollinate`, `sshpass`, `telnet`, `tk`, `xvfb`, `zsync`, `ftp`, `sphinxsearch`, `systemd-coredump`, `libnss3-tools` | +| The golden lacked the rest of the hosted image's apt baseline **(obsolete — the official golden is the hosted image itself, and a configured image is used as-is; nothing is pinned or baked)** | neovim's LLVM install script died on `lsb_release: command not found`; each missing utility (`xvfb`, `telnet`, `sshpass`, …) was a separate workflow-killing gap | the remaining packages from the official ubuntu-24.04 image's apt list (image readme 20260720.247.2) pinned and baked: `lsb-release`, `fonts-noto-color-emoji`, `haveged`, `mediainfo`, `p7zip-rar`, `pollinate`, `sshpass`, `telnet`, `tk`, `xvfb`, `zsync`, `ftp`, `sphinxsearch`, `systemd-coredump`, `libnss3-tools` | | The runner VMs had 4 GiB of RAM against workflows that assume the hosted 7 GiB | TypeScript's `ci.yml` sets `NODE_OPTIONS=--max-old-space-size=6144` ("7 GiB by default on GitHub"); the jake test workers died with a silent exit-2 crash | local pool raised to `PRELOOP_RUNNER_MEMORY_MIB=8192` (the remote already runs 6144) | | The runner account could not `sudo` | `sudo` demanded a password — the GitHub image grants the runner user passwordless sudo | the provisioning wrapper writes `/etc/sudoers.d/preloop-runner` (`NOPASSWD: ALL`) when it creates the account. Trust boundary: this grants workflow code unrestricted root and is only safe because these are disposable, isolated runner VMs — never install it on shared or persistent hosts | | Job/workflow `env:` entries containing `${{ }}` were emitted verbatim | ruff's `sccache` step resolves `SCCACHE_GHA_ENABLED:${{ github.ref_name == 'main' }}` — the raw template string reached the step env and the sccache action died (`${{` is not valid in an env value for the official runner) | the job message builder now resolves env expressions server-side against the job context before emitting `environmentVariables`. Scope: expressions resolve where the value is known at build time; secrets resolve to their masking placeholder in that context (never shipped as plaintext); runner-only contexts (`github.workspace`, `runner.*`) are preserved for step-time evaluation by the runner | diff --git a/docs/self-hosting.md b/docs/self-hosting.md index d888735f5..a79064dfa 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -262,23 +262,24 @@ a restart cannot resurrect them. Workflows see the effective value as | `PRELOOP_SKIP_DISK_PREFLIGHT` | — | Proceed past the golden disk check with a warning. Without it, a golden download that cannot fit is refused before it starts, and a golden build is refused below builder disk (`PRELOOP_RUNNER_STORAGE_GB`, min 40) + 20 GiB pack staging | | `PRELOOP_RUNNER_OVERLAY_GB` | — | Per-VM writable overlay size | | `PRELOOP_RUNNER_USER` / `PRELOOP_RUNNER_UID` | `runner` / `1001` | Guest account steps run as, for GitHub-hosted parity. `root` restores root; empty disables switching | -| `PRELOOP_USE_FORK` | — | Fork machines from a prepared golden instead of building each | -| `PRELOOP_USE_PACKED_GOLDEN` | `true` | Use a release or locally cached packed golden for on-demand and pooled runners | -| `PRELOOP_GOLDEN_URL` | release asset | Packed golden URL; the optional checksum is fetched from the same URL plus `.sha256` | +| `PRELOOP_USE_FORK` | `true` | Run the pool as forks of the golden rather than one boot per job | +| `PRELOOP_GOLDEN_URL` | release asset | Mirror of the official packed golden. When set it replaces the OCI reference, and the mirror must be verifiable: the checksum is this URL plus `.sha256`, or `PRELOOP_GOLDEN_SHA256` names the digest. An unverified mirror is refused | +| `PRELOOP_GOLDEN_SHA256` | — | Expected SHA-256 of the packed golden, for mirrors that publish no `.sha256` sidecar | | `PRELOOP_GOLDEN_OCI_REF` | per-architecture official GHCR artifacts | OCI packed golden reference downloaded automatically on the matching host architecture | | `PRELOOP_RUNNER_BUNDLE` | — | Directory of runner binaries mounted into guests | | `PRELOOP_RUNNER_EXTERNALS` | `$PRELOOP_HOME/externals` | Host-side Node externals directory | -| `PRELOOP_RUNNER_BASE_IMAGE` | digest-pinned Ubuntu 24.04 | OCI base identity for `runs-on` resolution; set it with `PRELOOP_GOLDEN_URL` for a custom packed golden | +| `PRELOOP_RUNNER_BASE_IMAGE` | — (official golden) | Image a custom golden is baked from, used as-is plus the GitHub-runner machinery. `runs-on: ubuntu-latest`/`ubuntu-24.04` and an unconfigured pool use the official packed golden instead. Wins over the `[golden] base_image` that `preloop init` records | | `PRELOOP_RUNNER_LABELS` | — | Extra labels on every pool runner. **Jobs only dispatch to runners whose labels match `runs-on`** | | `PRELOOP_RUNNER_NAME_PREFIX` | `preloop-runner` | Machine naming prefix | | `PRELOOP_RUNNER_DNS` | `8.8.8.8` | Resolver used inside guests; override when a different public resolver is preferred | | `PRELOOP_RUNNER_PACK_PROXY` / `_NO_PROXY` | — | Proxy and no-proxy values used while downloading and packing golden artifacts | -| `PRELOOP_WORKSPACE` | — | Workspace context for daemon deployments; not a package or toolchain installation input | | `PRELOOP_REQUIRE_JOB_ASSIGNMENTS` | — | Only let a runner claim jobs explicitly assigned to it | -To add organization-wide software, derive an OCI image from one of Preloop's -digest-pinned Ubuntu bases and build a custom packed golden. For the complete -build, checksum, publishing, and runtime configuration flow, see +To add organization-wide software, publish an OCI image that carries it and +point `PRELOOP_RUNNER_BASE_IMAGE` (or `[golden] base_image`, which +`preloop init` records) at it: the image is used exactly as it is, plus the +GitHub-runner machinery a bake adds. For the complete build, pre-bake, and +runtime configuration flow, see [VM images and version tracking](vm-images.md#adding-organization-wide-software). Keep repository-specific software in workflow setup actions, install steps, or a job `container:`. diff --git a/docs/vm-images.md b/docs/vm-images.md index 796a1640f..3ff4c0e4a 100644 --- a/docs/vm-images.md +++ b/docs/vm-images.md @@ -1,71 +1,98 @@ # VM images & version tracking -Preloop executes jobs on three substrates, one of which is a packed microVM -image ("the golden"). This page covers what the image contains, how it is -built, exactly which versions are tracked where  and which versions we -match to the official GitHub runner image to avoid drift. +Preloop executes jobs on three substrates, one of which is a packed microVM +image ("the golden"). This page covers what a golden contains, how it is built, +and which versions are tracked where. + +A golden comes from exactly one of two sources. The **official packed golden** +is the published official GitHub runner image with `preloop-runner` baked in; +Preloop downloads it per architecture, digest-pinned, and verifies it before +use. A **configured image** (`PRELOOP_RUNNER_BASE_IMAGE`, or the `[golden] +base_image` that `preloop init` records) is baked as it is, plus the +GitHub-runner machinery described below. There is no third source and no local +fallback: a golden download that cannot complete fails the job that needed it. ## Execution substrates | Mode | How jobs run | Enabled by | | -------------------- | ---------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| **MicroVM** | A libkrun guest (Hypervisor.framework on macOS, KVM on Linux) boots the packed golden image and runs the job inside it | `preloop serve`; packed golden use is the default | -| **Fork pool** | CoW clones (`machine fork`) of a prepared golden microVM — same guest, no boot/provision | `PRELOOP_USE_FORK=true` (default when a packed golden is present) | +| **MicroVM** | A libkrun guest (Hypervisor.framework on macOS, KVM on Linux) boots the packed golden image and runs the job inside it | `preloop serve` | +| **Fork pool** | CoW clones (`machine fork`) of a prepared golden microVM — same guest, no boot/provision | `PRELOOP_USE_FORK=true` (the default) | | **External runners** | Any runner that registers against the server: the official `actions/runner`, `preloop-runner` on another machine, containers | `preloop-runner configure` + `run` | The VM image and the fork pool share the same artifact (the golden); fork mode just skips the boot. -## Image layers and pins +## Image sources and pins -Four different kinds of image appear in the execution path: +Three kinds of image appear in the execution path: | Image | Purpose | How it is selected | | -------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| **GitHub runner image snapshot** | Upstream parity reference for preinstalled tool and package versions | `github_runner_image_version` in `versions.toml` | -| **OCI base image** | Root filesystem from which Preloop provisions a golden | `ubuntu_24_04_base` / `ubuntu_22_04_base` in `versions.toml`, or `--base-image` / `PRELOOP_RUNNER_BASE_IMAGE` | -| **Packed golden** | Pre-provisioned, architecture-specific microVM artifact used by the pool | Release asset by default, or `PRELOOP_GOLDEN_URL` | +| **Official golden** | The packed official GitHub runner image (Ubuntu 24.04) with `preloop-runner` baked in, per architecture | `runs-on: ubuntu-latest` / `ubuntu-24.04`, and any pool with no image configured. `PRELOOP_GOLDEN_OCI_REF` overrides the per-arch reference; `PRELOOP_GOLDEN_URL` selects a release-asset mirror instead | +| **Configured image** | Root filesystem a golden is baked from, used as-is | `PRELOOP_RUNNER_BASE_IMAGE`, or `[golden] base_image` recorded by `preloop init`; `--base-image` for `build-golden` | | **Workflow images** | Job `container:` and `services:` environments | Workflow YAML | - +`ubuntu-22.04` maps nowhere: the 22.04 archive baseline went with the deleted +stock bake, so the label keeps the configured image (the official golden when +nothing is configured) instead of silently selecting a 24.04 one. `ubuntu-slim` +has no separate slim image either, and resolves the same way. ## What the golden contains -A stock golden is a pre-provisioned microVM image: the OCI base, -`preloop-runner`, and the curated toolchain baseline, provisioned once and packed by smolvm -into a single bootable, architecture-specific file. The pool boots it -directly, and the fork pool runs CoW clones (`machine fork`) of the same -golden — the same guest, no boot or provision. - -Why pack one: provisioning (pulling the base, installing packages, baking toolchains) is the expensive part, and a golden does it exactly once. A runner then starts in one or two seconds from a local artifact instead of re-pulling and re-baking the base for every new VM. Because a golden is one checksummed -file, it is also reproducible: the same artifact produces the same runner on any host, and a stale build is caught by the checksum. - -The stock golden contains: - -1. **Base OS**: the published packed golden is Ubuntu 24.04, pinned by digest; -`ubuntu-latest` currently selects that 24.04 base. `ubuntu-22.04` selects a -separate digest-pinned plain Ubuntu 22.04 OCI base and builds its environment -golden on demand; there is no published packed 22.04 golden. `ubuntu-slim` has -no separate slim image and currently falls back to the configured base. -Preloop's microVM pool runs Linux guests; the release workflow's macOS builds -use GitHub-hosted runners. -2. **The runner**: `preloop-runner` cross-built for `aarch64-unknown-linux-gnu` -(`cargo-zigbuild`), fidelity-tracked against the official `actions/runner` -(see `versions.toml`). -3. **Curated toolchains**: a fixed Rust/Go baseline is baked into **stock -Ubuntu goldens** for workflows that invoke those tools implicitly. An -official GitHub runner-image golden is used as-is; `setup-*` actions resolve -the exact language version requested by each workflow at job time. -4. **Base dependencies**: the apt set `install_base_dependencies` installs -git, curl, build-essential, python3, jq, unzip/zip, locales, …). -5. **Docker**: daemon + CLI, so `container:` / `services:` jobs work. - -Because setup actions own version selection, the same golden serves every -project without baking a repository-specific language version. +A golden is a pre-provisioned microVM image, packed by smolvm into a single +bootable, architecture-specific file. The pool boots it directly, and the fork +pool runs CoW clones (`machine fork`) of the same golden — the same guest, no +boot or provision. + +Why pack one: provisioning (pulling the base, applying the runner contract) is +the expensive part, and a golden does it exactly once. A runner then starts in +one or two seconds from a local artifact instead of re-pulling and re-baking +the base for every new VM. Because a golden is one checksummed file, it is also +reproducible: the same artifact produces the same runner on any host, and a +stale build is caught by the checksum. + +The **official golden** contains the official GitHub-hosted runner image for +its architecture, with the Linux `preloop-runner` added. It is fetched, never +baked locally: `runs-on: ubuntu-latest` and `ubuntu-24.04` name it, and so does +a pool with no image configured. + +A golden baked from a **configured image** contains that image, exactly as it +is, plus the GitHub-runner machinery — and nothing else: + +1. **The runner account**: `runner` (uid 1001) with its home, `_work`, and + passwordless sudo, created only when the image does not already have it. +2. **Ownership of the runner home**: fixed once at build time with an + owner-only walk, `find … ! -user 1001 -exec chown -h 1001:1001 {} +`. It + touches only inodes whose owner is wrong, so it leaves a runner-owned file's + group alone (GitHub's `~/.docker` stays `runner:docker`), and it is never + repeated per VM — a fork inherits the result. +3. **The tool cache**: a writable `/opt/hostedtoolcache`, with + `RUNNER_TOOL_CACHE` and `AGENT_TOOLSDIRECTORY` in `/etc/environment`, so + `setup-*` actions can install into it. +4. **The build record**: `/etc/preloop-bake.json`, naming the image, its digest + when the reference pins one, the bake fingerprint, and the versions the + guest reported at build time. +5. **A glibc check**: the one requirement a bake enforces. smolvm boots a Linux + guest and every dynamically linked binary — the runner included — resolves + through the guest's glibc loader; an image without one (musl-only, scratch) + fails the bake, naming what was missing. +6. **Apt package indices**: `apt-get update` at build time (best-effort, bounded + to five minutes, skipped on an image without apt), so a step's + `sudo apt-get install ` resolves with no `apt-get update` first, as on + a hosted runner. The runner-image dump ships with its lists wiped; the + official golden gets them from this step when it is baked, and an + already-published pack that lacks them is refreshed once when it is + unpacked, before it is frozen — never per fork. + +No packages, no toolchains, no PATH or environment overrides. A missing `bash`, +`git`, or `docker` fails the step that needs it, exactly as on a GitHub-hosted +runner. Workflows select language versions with the ecosystem's `setup-*` +actions at job time, so the same golden serves every project. ## Building a golden @@ -87,26 +114,29 @@ injection fix is in progress). | x86-64 | `x86_64-unknown-linux-gnu` | `x86_64` | -Build the Linux runner bundle, then pack the default digest-pinned Ubuntu -base: +Build the Linux runner bundle, then bake the image you want the golden to +carry: ```sh just build-preloop preloop build-golden \ --runner-bundle target/aarch64-unknown-linux-gnu/debug \ - --output dist/preloop-ubuntu-24.04-aarch64 + --base-image 'ghcr.io/acme/preloop-base@sha256:' \ + --output dist/acme-aarch64 ``` - `--runner-bundle`: directory containing the Linux `preloop-runner` binary (`just build-preloop` cross-builds it). -- `--base-image`: optional Ubuntu-derived OCI image or `.smolmachine` -artifact. It defaults to the digest-pinned Ubuntu 24.04 image compiled from -`versions.toml`. -- `--workspace`: retained as workspace context for build automation. It does -not currently change the packed artifact, install packages, or derive -toolchains from `.nvmrc`, `rust-toolchain.toml`, or similar files. +- `--base-image`: the image to bake. There is no default: the official golden +is published packed and cannot be built locally, so with no flag the command +uses the configured image (`PRELOOP_RUNNER_BASE_IMAGE`, else `[golden] +base_image`) and refuses when nothing is configured. An OCI reference (digest +pinned for reproducibility) or a local `.smolmachine`/archive path both work. - `--output`: destination for the packed golden. +- The bake applies the runner contract (see "What the golden contains") and + writes `/etc/preloop-bake.json`; it installs no packages (it refreshes apt's + indices and nothing else). - Release publication first seeds both architecture assets from the newest complete release, so a newly tagged engine never points at missing goldens. Every GitHub release then triggers `release-golden.yml`; release notes do not @@ -114,17 +144,18 @@ toolchains from `.nvmrc`, `rust-toolchain.toml`, or similar files. `ubuntu-latest` (KVM-enabled) in `release-golden.yml`. Successful refreshes replace the seeded `preloop-ubuntu-24.04-x86_64` asset; failed refreshes stay visible without removing the valid seeded pair. The pool stores the - downloaded artifact at a base-image-specific path below + downloaded artifact at a base-and-fingerprint-specific path below `/vms/` (`preloop_home` is `~/.preloop` unless - `PRELOOP_HOME` says otherwise). -- The pool fetches that asset from its own release first, then from the newest - release that actually carries it (a GitHub API lookup, cached for five - minutes, stable releases over prereleases), and finally from - `/releases/latest/download/…`, which needs no API call and covers a lookup - that failed. An engine whose release carries no golden — the seed step - failed, or the tag predates the artifact — therefore still finds the - published one instead of baking locally. `PRELOOP_GOLDEN_URL` replaces every - candidate. + `PRELOOP_HOME` says otherwise); `preloop golden-path --home + --base-image ` prints the exact payload path. +- The pool fetches the official golden from the per-architecture OCI reference + first, then from its own release asset, then from the newest release that + actually carries it (a GitHub API lookup, cached for five minutes, stable + releases over prereleases), and finally from `/releases/latest/download/…`, + which needs no API call and covers a lookup that failed. `PRELOOP_GOLDEN_URL` + replaces every release-asset candidate (and skips the OCI path). A download + that cannot complete fails the job: there is no local bake and nothing else + to fall back to. - The OCI goldens the engine downloads by default are baked out-of-band — aarch64 on an Apple Silicon host, x86_64 on a Linux KVM host — and published to GHCR as `preloop--smolvm-golden`, digest-pinned in the engine and @@ -141,17 +172,19 @@ toolchains from `.nvmrc`, `rust-toolchain.toml`, or similar files. installed only after its checksum (release asset) or layer digest (OCI) matches, and a failed transfer leaves the partial file for the next attempt, including one after an engine restart. -- When the pool warms a golden, it also pre-pulls the `container:` / +- When the pool prepares a golden, it also pre-pulls the `container:` / `services:` images declared by the current workspace's workflows. ### Adding organization-wide software `build-golden` does not accept an apt package list, Dockerfile, or -post-provisioning script. Put organization-wide software in an Ubuntu-derived -OCI base, then ask Preloop to provision and pack that image. Use workflow -steps or setup actions instead when software differs by repository. +post-provisioning script, and a configured image is not modified: put +organization-wide software in the OCI image itself and configure that image as +the golden. Use workflow steps or setup actions instead when software differs +by repository. -Example custom base: +Example custom base, derived from the official snapshot so the image keeps +hosted-runner parity: ```dockerfile ARG BASE_IMAGE @@ -166,66 +199,38 @@ RUN apt-get update \ ``` Build and publish it for the architecture on which the golden will run. Pass -the current `ubuntu_24_04_base` value from `versions.toml` as `BASE_IMAGE`: +the official snapshot pin for that architecture +(`official_runner_image_base_arm64` / `official_runner_image_base_amd64` in +`official-image.toml`) as `BASE_IMAGE`: ```sh docker buildx build \ --platform linux/arm64 \ - --build-arg BASE_IMAGE='' \ + --build-arg BASE_IMAGE='ghcr.io/preloopdev/runner-images:ubuntu24-arm64-runner-large-latest@sha256:…' \ --tag ghcr.io/acme/preloop-base:2026-08-08 \ --push \ . ``` -Use the immutable digest returned by the registry, not the mutable tag, when -building the golden: - -```sh -CUSTOM_BASE='ghcr.io/acme/preloop-base@sha256:' -GOLDEN='acme-ubuntu-24.04-aarch64' - -preloop build-golden \ - --runner-bundle target/aarch64-unknown-linux-gnu/release \ - --base-image "$CUSTOM_BASE" \ - --output "dist/$GOLDEN" - -(cd dist && shasum -a 256 "$GOLDEN" > "$GOLDEN.sha256") -``` - -On Linux, use `sha256sum "$GOLDEN" > "$GOLDEN.sha256"` instead. - -Publish both files: - -```text -https://artifacts.acme.example/acme-ubuntu-24.04-aarch64 -https://artifacts.acme.example/acme-ubuntu-24.04-aarch64.sha256 -``` - -Configure both the base identity and packed artifact URL: +Then configure it by its immutable digest — not the mutable tag — and start +the engine: ```sh PRELOOP_RUNNER_BASE_IMAGE='ghcr.io/acme/preloop-base@sha256:' \ -PRELOOP_GOLDEN_URL='https://artifacts.acme.example/acme-ubuntu-24.04-aarch64' \ preloop serve ``` -The pool fetches the checksum from exactly -`${PRELOOP_GOLDEN_URL}.sha256`. A missing checksum is tolerated with a -warning, but publishing it is strongly recommended. Setting both variables -keeps the golden's cache identity and its packed payload tied to the same OCI -base. - -To provision directly from the custom OCI base without publishing a packed -golden: - -```sh -PRELOOP_RUNNER_BASE_IMAGE='ghcr.io/acme/preloop-base@sha256:' \ -PRELOOP_USE_PACKED_GOLDEN=false \ -preloop serve -``` +The first job's pool environment bakes the image once — as it is, plus the +runner contract — and forks that golden per job. To bake where there is more +disk and move the artifact, run `preloop build-golden --base-image +--output

` on the build host, then copy the payload to the path `preloop +golden-path --home --base-image ` prints on the serving +host. -Provisioning currently assumes an Ubuntu 24.04 or 22.04 userspace and uses -`apt-get`. Use a workflow `container:` image for another distribution. +The image must carry a glibc dynamic loader, and it should carry what your +workflows invoke implicitly: a missing `bash`, `git`, or `docker` fails the +step that needs it. Nothing else is installed and no PATH or environment +override is applied at bake time. ### Verifying a base image's provenance @@ -251,13 +256,12 @@ verification then uses `cosign verify --key` rather than the identity check. ### Golden provenance -The stock base in `versions.toml` is served from `mirror.gcr.io`, Google's -cache of the Docker Official Ubuntu image. It is not a Google-built Ubuntu -image. The release workflow resolves the pinned digest, records the selected -platform manifest and its OCI attestation descriptors, and preserves the -upstream SPDX SBOM beside the golden. The cache is useful for availability and -rate-limit avoidance; the digest and the upstream image metadata are the -provenance inputs. +The official golden is baked from the digest-pinned official runner image in +`official-image.toml`, republished as OCI by the snapshot pipeline. The release +workflow resolves the pinned digest, records the selected platform manifest and +its OCI attestation descriptors, and preserves the upstream SPDX SBOM beside +the golden. The digest and the upstream image metadata are the provenance +inputs. Each release golden then receives: @@ -285,9 +289,9 @@ gh attestation verify \ Release builds require the base to be digest-pinned with `PRELOOP_REQUIRE_BASE_DIGEST=1`. This protects the golden cache and the -provenance record from mutable image tags. The stock image's attached SPDX -SBOM is evidence about the upstream input; it is not treated as a Google -signature or as a substitute for the Preloop golden attestation. +provenance record from mutable image tags. The snapshot's attached SPDX SBOM is +evidence about the upstream input; it is not treated as a substitute for the +Preloop golden attestation. ### Using a snapshot of the official hosted image @@ -298,7 +302,9 @@ dump pipeline, then scanned, signed, and SLSA-attested before the floating tag advances). The release assets (`preloop-ubuntu-24.04-x86_64` and its `.provenance.json` / `.base-sbom.spdx.json` / `.bundle` sidecars) are therefore the official runner image with the Preloop runner baked in — no extra -provisioning needed. +provisioning needed. That is also what a pool with no image configured +downloads, so this section matters only when you need a snapshot or an +architecture the published golden does not carry. GitHub's hosted runner images are not published as OCI images, but the community [runner-image-blobs](https://github.com/ChristopherHX/runner-image-blobs) @@ -312,20 +318,19 @@ and run its dump workflow, which publishes the same tags under your fork's own GHCR namespace. Public GHCR packages are free, so publishing costs nothing while the fork and its packages stay public. -You can point Preloop at the snapshot directly, with no golden: +You can configure the snapshot as the golden's image: ```sh PRELOOP_RUNNER_BASE_IMAGE='ghcr.io//runner-images:ubuntu24-runner-large-latest-arm64' \ -PRELOOP_USE_PACKED_GOLDEN=false \ PRELOOP_RUNNER_STORAGE_GB=80 \ preloop serve ``` -Cold provisioning pulls the OCI image and bakes the runner baseline into each -new VM (`PRELOOP_RUNNER_STORAGE_GB=80` covers the ~60 GB extracted snapshot). -That works, but the official snapshots are large (about 20 GB compressed, -60 GB extracted), so every new runner pays a multi-GB pull and bake before -its first job. For the official image we recommend packing a golden once -instead: + +The pool pulls the OCI image and bakes it into its golden once +(`PRELOOP_RUNNER_STORAGE_GB=80` covers the ~60 GB extracted snapshot), then +forks that golden per job. The snapshots are large (about 20 GB compressed, +60 GB extracted), so the first bake on a host is expensive. To pay it once and +reuse it — or to bake on a host with more disk — bake explicitly: ```sh OFFICIAL_IMAGE='ghcr.io//runner-images:ubuntu24-runner-large-latest-arm64' @@ -336,13 +341,13 @@ preloop build-golden \ --output dist/official-ubuntu-24.04-aarch64 ``` -The golden then serves every runner with a fast local boot (or a fork-pool -start), the pull and bake happen once per host, and the artifact can be -published and reused via `PRELOOP_GOLDEN_URL`. The tradeoffs are artifact -size (tens of GB packed) and the rebuild step: a golden is fixed at bake -time, so refresh it when the snapshot changes. Pin a digest for -reproducibility, and use the architecture matching your host (`arm64` with -an aarch64 bundle, `amd64` with x86-64). +Copy the artifact to the payload path `preloop golden-path --home --base-image "$OFFICIAL_IMAGE"` prints on the serving host, and the pool +adopts it instead of baking again. The tradeoffs are artifact size (tens of GB +packed) and the rebuild step: a golden is fixed at bake time, so refresh it +when the snapshot changes. Pin a digest for reproducibility, and use the +architecture matching your host (`arm64` with an aarch64 bundle, `amd64` with +x86-64). ### Installing repository-specific software @@ -385,27 +390,26 @@ bundle is needed. Missing on macOS, the engine logs a startup warning and jobs fail after the queue grace window; install the bundle with `preloop update` or set `PRELOOP_RUNNER_BUNDLE`. - ## Version tracking (`versions.toml`, `official-image.toml`) +## Version tracking (`versions.toml`, `official-image.toml`) - Every pinned version lives in one of two flat files — `versions.toml` for - toolchain and runtime pins, `official-image.toml` for the official runner - bases (bumping those means rebuilding goldens, so they trigger - separately) — and is consumed by the build: +Every pinned version lives in one of two flat files — `versions.toml` for the +runner, runtime, and externals pins, `official-image.toml` for the official +runner images (bumping those means rebuilding goldens, so they trigger +separately) — and is consumed by the build: - | Key | What it pins | Bump when | - | ------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Key | What it pins | Bump when | +| ------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | | `runner_version` | Official `actions/runner` protocol target (currently `2.336.0`) | Upstream runner changes protocol surface | | `smolvm_min_version` | SmolVM runtime floor `preloop update --ensure-runtime` accepts and upgrades from | A future SmolVM drops a capability preloop needs (rare, human-driven) | | `smolvm_golden_version` | SmolVM release the golden workflow builds with | Upstream ships a newer stable (Renovate opens a bump PR, `smolvm-release-verify` gates it) | -| `github_runner_image_version` | Official `actions/runner-images` Ubuntu 24.04 snapshot used as the parity source | Refreshing the hosted-image parity bake list | -| `ubuntu_24_04_base` | Base image by digest (`ubuntu:24.04@sha256:…`) | You want a newer OS snapshot — always bump the digest, never a bare tag | -| `ubuntu_22_04_base` | Second pinned base | Same | -| `official_runner_image_base_amd64` (`official-image.toml`) | Official GitHub-hosted runner image OCI reference for the x86_64 golden | Bump the digest after re-running `runner-image-blobs` attestation and verifying the new digest | -| `official_runner_image_base_arm64` (`official-image.toml`) | Official GitHub-hosted runner image OCI reference for the aarch64 golden | Same as above | -| `node_version` | System node baked into the golden | A workflow needs a newer default Node | -| `node20_externals_version` / `node24_externals_version` | Node runtimes baked as the runner's externals | Same | -| `rustup_version` | Rustup used to install baked Rust toolchains | Toolchain bootstrap changes | -| `cargo_shear_version` | Auxiliary cargo tooling | Same | +| `node20_externals_version` / `node24_externals_version` | Node runtimes the guest runner mounts for JavaScript actions (downloaded on the host, never baked into a golden) | A new runtime pin is needed | +| `runner_image_docker_version` / `runner_image_buildx_version` | Docker CLI and Buildx baked into the standalone `preloop-runner` container image (`ci/runner.Dockerfile`) | The official `actions/runner` image moves | +| `official_runner_image_base_amd64` (`official-image.toml`) | Official GitHub-hosted runner image OCI reference the x86_64 golden is baked from | Bump the digest after re-running `runner-image-blobs` attestation and verifying the new digest | +| `official_runner_image_base_arm64` (`official-image.toml`) | Official GitHub-hosted runner image OCI reference the aarch64 golden is baked from | Same as above | + +`versions.toml` no longer carries OS bases, toolchain versions, or apt package +pins: a golden's contents are its image's, and an operator-selected image is +configured at runtime rather than compiled in. The protocol target (`runner_version`) and the VM image are independent: @@ -427,17 +431,18 @@ not auto-bumped: it is a capability floor, not a tracked release. The job runs on Renovate's `smolvm_golden_version` bump PRs (head branches `renovate/**`) and manual dispatches, and targets `ubuntu-latest`. The runner must expose usable `/dev/kvm` access for the nested microVM smoke test and -provide registry access for the pinned Ubuntu base. Renovate auto-merges smolvm -golden bumps once the verify check and `ci.yml` pass — the merge gate is -Renovate's auto-merge, not branch protection. +provide registry access for the pinned official runner image. Renovate +auto-merges smolvm golden bumps once the verify check and `ci.yml` pass — the +merge gate is Renovate's auto-merge, not branch protection. `versions.toml` defines Preloop's compiled distribution defaults. It is not a -per-install user configuration file. Operators select custom OCI bases and -packed goldens with `--base-image`, `PRELOOP_RUNNER_BASE_IMAGE`, and -`PRELOOP_GOLDEN_URL`; downstream distributions can change the pins before -compiling. Editing `versions.toml` does not change an already-built or -installed `preloop` executable. Rebuild the CLI and replace the deployed -binary before expecting `preloop serve` to use a changed pin: +per-install user configuration file. Operators select the image a golden is +baked from with `--base-image` or `PRELOOP_RUNNER_BASE_IMAGE`, and the official +golden's download source with `PRELOOP_GOLDEN_OCI_REF` / `PRELOOP_GOLDEN_URL`; +downstream distributions can change the pins before compiling. Editing +`versions.toml` does not change an already-built or installed `preloop` +executable. Rebuild the CLI and replace the deployed binary before expecting +`preloop serve` to use a changed pin: ```sh just build-preloop @@ -451,74 +456,28 @@ Install or deploy `target/release/preloop` through the same mechanism used for the existing executable. Check `which preloop` when a shell still launches an older installed copy. -## GitHub-hosted parity bake list - -The official runner image (`actions/runner-images` ubuntu-24.04) preinstalls -~100 tools; our golden deliberately bakes only the ones workflows touch -*implicitly* — the hidden dependencies that cause drift when missing. The -versions below are taken directly from the official image's toolset -identified by `github_runner_image_version` in `versions.toml`. These are the -parity targets to bake (or pin) so CI results on Preloop match GitHub: - -### Tier 1 — proven drift sources (bake these) - - -| Item | Exact version (official image) | Why | -| -------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Node.js (system) | **22.23.1** | The #1 proven failure: workflows call `node`/`npm`/`npx` directly — 6/7 repos in the 2026-07-28 campaign failed on "Node 24 missing". Runner-internal node is covered by externals; system node is not | -| npm / yarn / nvm | **npm 10.9.8, yarn 1.22.22, nvm 0.40.6** | Same hidden-dependency class | -| Docker stack | **client 28.0.4, server 28.0.4, buildx 0.35.0, compose 2.38.2** | Container/service jobs are a whole workflow category; apt's older docker + missing buildx/compose changes `docker buildx` / `docker compose` behavior | -| Clang family | **clang/format/tidy 16.0.6, 17.0.6, 18.1.3** | There is no standard GitHub setup action; C/C++ workflows commonly invoke versioned binaries directly | -| GNU compiler family | **gcc/g++/gfortran 12.4.0, 13.3.0, 14.2.0** | Same implicit system-tool contract; `build-essential` supplies only the default compiler | -| Runner user contract | **`runner` (uid 1001), `HOME=/home/runner`, `/run/user/1001`** | Every `id -u` / `env_var('USER')` / `runtime_directory()` check drifts without it (implemented — see `PRELOOP_RUNNER_USER`/`PRELOOP_RUNNER_UID` in self-hosting.md §4) | - - -### Tier 2 — behavior parity (bake when size allows) - - -| Item | Exact version (official image) | Why | -| ------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| git | **2.54.0** + **Git LFS 3.7.1** | checkout-adjacent behavior: safe.directory, submodules, protocol quirks (apt ships 2.43.x) | -| `ubuntu` admin user | **uid 1000** | Workflows/actions that `chown` to 1000 or assume the admin account (a documented GitHub container-job gotcha) | - - -### Setup-action boundary - -GitHub maintains first-party setup actions for Node -(`actions/setup-node`), Python/PyPy (`actions/setup-python`), Go -(`actions/setup-go`), Java (`actions/setup-java`), and .NET -(`actions/setup-dotnet`). Those toolchains do not need every hosted-image -version baked for correctness; their actions install a requested version. -Official runner-image goldens therefore do not receive an extra preloop -Rust/Go version. They provide a runner-writable `/opt/hostedtoolcache` for -setup actions, plus the system tools already present in the image. - -The following have ecosystem-owned setup actions: Ruby (`ruby/setup-ruby`), -Julia (`julia-actions/setup-julia`), Haskell (`haskell-actions/setup`), PHP -(`shivammathur/setup-php`), Android (`android-actions/setup-android`), Rust -(`dtolnay/rust-toolchain` or `actions-rust-lang/setup-rust-toolchain`), CMake -(`jwlawson/actions-setup-cmake`), browsers (`browser-actions/setup-*`), and -Docker (`docker/setup-docker-action`). These are not GitHub-maintained, but a -workflow can declare the version instead of depending on the hosted image. -`docker/setup-buildx-action` installs Buildx, not the Docker daemon. - -There is no standard setup action for the hosted Clang and GNU compiler -matrices, so those are baked. Databases should be declared with `services:`. -Cloud authentication actions generally do not install their CLIs: -`aws-actions/configure-aws-credentials` and `azure/login` assume the AWS and -Azure CLIs are already present, while `google-github-actions/setup-gcloud` -does install gcloud. - -Browsers + drivers, Android SDK, .NET SDKs, Java, Ruby/PHP/Julia/Kotlin/ -Swift, cloud CLIs, and databases remain deliberately unbaked. Baking that -set would add tens of gigabytes; setup actions or service containers provide -the explicit version at job time. Rust is already baked through rustup, with -the workspace pin applied by normal Rust workflows. - -**Rule of thumb**: match what workflows touch implicitly (system node, the -user contract, docker, git); leave what they must declare anyway to job-time -installs. New parity targets belong in `versions.toml` with a comment -naming the official image version they were taken from. +## What a golden adds + +Nothing but the GitHub-runner machinery. The official golden is the official +hosted image with `preloop-runner` baked in; a configured image is used as it +is. Neither path installs packages or toolchains, and neither sets PATH or +environment overrides, so there is no parity bake list to maintain. + +A workflow that needs a language version declares it with the ecosystem's +`setup-*` action (`actions/setup-node`, `actions/setup-python`, +`actions/setup-go`, `actions/setup-java`, `actions/setup-dotnet`, Ruby's +`ruby/setup-ruby`, Rust's `dtolnay/rust-toolchain`, and so on), exactly as on +GitHub-hosted runners; the golden only guarantees the writable +`/opt/hostedtoolcache` those actions install into. Databases belong in +`services:`, and a fully controlled userspace belongs in a job `container:`. +Cloud CLIs, browsers, Android SDKs, and similar large toolchains are +deliberately not added by Preloop — use the action or service container that +owns the version. + +**Rule of thumb**: the image decides what is present; the workflow declares +what it needs. A tool the image lacks (`bash`, `git`, `docker`, a compiler) +fails the step that needs it, exactly as on a GitHub-hosted runner, so put +what your workflows invoke implicitly into the image you configure. ### Process limits (raised per launch, not baked) @@ -642,13 +601,13 @@ ad-hoc `docker run`, and runs the deep-recursion probe pydantic's own | Env var | Effect | | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `PRELOOP_USE_PACKED_GOLDEN` | Use a release or locally cached packed golden (default on; set `false` for cold OCI provisioning) | -| `PRELOOP_GOLDEN_URL` | Override the packed golden URL; its optional checksum is fetched from the same URL plus `.sha256` | -| `PRELOOP_USE_FORK` | Run the pool as host forks instead of booting microVMs (default true with a golden) | +| `PRELOOP_GOLDEN_OCI_REF` | Override the per-architecture packed golden OCI reference the official golden is fetched from | +| `PRELOOP_GOLDEN_URL` | Select a release-asset mirror of the official packed golden instead. A mirror **must** be verifiable: the checksum is this URL plus `.sha256`, or `PRELOOP_GOLDEN_SHA256` names the digest; an unverified mirror is refused | +| `PRELOOP_GOLDEN_SHA256` | Expected SHA-256 of the packed golden, for mirrors that publish no `.sha256` sidecar | +| `PRELOOP_USE_FORK` | Run the pool as host forks instead of booting microVMs (default true) | | `PRELOOP_RUNNER_POOL_SIZE` | Pool size (warm forks / VMs) | | `PRELOOP_RUNNER_CPUS` | vCPUs allocated to each runner VM (default 8) | -| `PRELOOP_WORKSPACE` | Workspace context for daemon deployments; it does not install packages or derive toolchains for a packed golden | -| `PRELOOP_RUNNER_BASE_IMAGE` | Override the base image at serve time (default: digest-pinned Ubuntu 24.04) | +| `PRELOOP_RUNNER_BASE_IMAGE` | Image a custom golden is baked from, used as-is plus the runner contract. `runs-on: ubuntu-latest`/`ubuntu-24.04` and an unconfigured pool use the official golden instead | | `PRELOOP_RUNNER_LABELS` | Extra `runs-on` labels the pool's runners declare (comma-separated) | | `PRELOOP_RUNNER_USER` / `PRELOOP_RUNNER_UID` | Guest runner account (default `runner`/1001, GitHub-hosted parity); `root` restores root; empty disables switching | @@ -657,65 +616,36 @@ ad-hoc `docker run`, and runs the deep-recursion probe pydantic's own - **A job misbehaves after a golden change**: the pool caches unpacked pack dirs per VM; deleting the per-VM pack cache forces a clean unpack. -- **Missing toolchain in the VM**: the toolchain is not in the curated bake -(or the workflow pins a version outside the baked toolcache). `setup-*` -actions download the exact version at job time — the intended path. If a -toolchain is needed implicitly (no setup action), add it to the curated -bake in `base_install_script`. -- **Wrong OS inside the VM**: `--base-image` was overridden; the default is -the digest-pinned Ubuntu 24.04. -- **A job VM pulls the OCI base instead of using a packed golden**: -`PRELOOP_USE_PACKED_GOLDEN` defaults to `true` in current builds. At startup, -an enabled packed path logs `Downloading pre-baked golden from release asset -(this may take several minutes)`. If the download is unavailable, Preloop -pulls the OCI base once in a machine named `-builder`, provisions it, -and packs a local artifact. That one-time builder pull is expected. - - A pull from a job machine such as `preloop-runner-0-1`, with no preceding - golden download attempt, means the running process has packed golden use - disabled. Check for an override and for a stale executable: - ```sh - printenv PRELOOP_USE_PACKED_GOLDEN - which preloop - preloop --version - ``` - - Force the current behavior while diagnosing the installed copy: - ```sh - PRELOOP_USE_PACKED_GOLDEN=true ./target/debug/preloop serve - ``` - - Rebuild first with `cargo build -p preloop-cli` if that debug binary predates - the current `versions.toml`. `PRELOOP_RUNNER_POOL_ENABLED=false` only - disables the warm pool; it does not disable packed artifacts in current - builds. -- **Docker Hub reports `TOOMANYREQUESTS` while pulling Ubuntu**: current stock -pins use `mirror.gcr.io`, so a log that says `Pulling ubuntu:24.04` or -fetches `index.docker.io` means the running CLI has an older compiled pin or -`PRELOOP_RUNNER_BASE_IMAGE` overrides it. Check both: - ```sh - which preloop - printenv PRELOOP_RUNNER_BASE_IMAGE - ``` - - Rebuild or reinstall Preloop after changing `versions.toml`. As an immediate - override, pass the current digest-pinned `ubuntu_24_04_base` value: - ```sh - PRELOOP_RUNNER_BASE_IMAGE='mirror.gcr.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90' \ - preloop serve - ``` - - The subsequent SmolVM log must name - `mirror.gcr.io/library/ubuntu:24.04@sha256:...`, not a bare - `ubuntu:24.04@sha256:...`. +- **Missing toolchain in the VM**: the image does not carry it. `setup-*` +actions download the exact version at job time — the intended path. For a tool +a workflow needs implicitly (no setup action), add it to the image you +configure (`PRELOOP_RUNNER_BASE_IMAGE` / `[golden] base_image`) and let that +golden rebuild. +- **Wrong OS inside the VM**: a configured image overrode the official golden. +Check `PRELOOP_RUNNER_BASE_IMAGE`, the `[golden] base_image` in the config +file, and `which preloop` — a stale executable keeps its old compiled default. +- **The official golden will not download**: the download is the only source. +There is no local bake and no stock-Ubuntu fallback, so the job fails and the +error names both the OCI reference it tried and the release asset. Check egress +to `ghcr.io` (or the mirror behind `PRELOOP_GOLDEN_URL`) and that +`PRELOOP_GOLDEN_OCI_REF` names a reference your registry serves. A partial +transfer is kept at `.partial` and the next attempt resumes it, so a +dropped connection costs the bytes in flight, not the artifact. +- **A job VM pulls an OCI image instead of using the prepared golden**: that is +the configured-image bake. The pool pulls the image once in a machine named +`-builder`, applies the runner contract, and packs it; the fork base +`-golden` then serves every job. A pull on every job means the pack is +not being adopted — check free disk (`PRELOOP_RUNNER_MIN_FREE_DISK_GB`, +`PRELOOP_SKIP_DISK_PREFLIGHT`) and that the payload path the engine logs +exists. ### Building the official-runner-image golden on Apple Silicon The official GitHub-hosted runner image (`ubuntu24-runner-large`, published per-arch to `ghcr.io/preloopdev/runner-images` by the runner-image-blobs fork) is tens of GiB and declares `USER=runner`. Building a golden from it -locally exercises several smolvm/preloop sharp edges that a stock Ubuntu -base never hits. All were fixed in smolvm's `src/cli/internal_boot.rs`, +locally exercises several smolvm/preloop sharp edges. All were fixed in +smolvm's `src/cli/internal_boot.rs`, `src/cli/machine.rs`, `src/pack_export.rs`, `crates/smolvm-agent/src/…`, and `crates/preloop-cli`/`preloop-vm`; the notes below describe the failure each one produced so a regression is recognizable. diff --git a/docs/vm-substrates.md b/docs/vm-substrates.md index 4f6bdd0b8..764f95c77 100644 --- a/docs/vm-substrates.md +++ b/docs/vm-substrates.md @@ -95,9 +95,10 @@ orchestrator branches on it instead of assuming SmolVM's semantics. snapshot and writes a small JSON descriptor naming it (at both `` and the `.smolmachine` sidecar the orchestrator looks for). The descriptor is portable only within that AgentENV server. The pool therefore - skips artifact build/download entirely on this backend and prepares its - golden straight from the base image, then forks per job: - `use_packed_artifact` is forced off and `use_fork` on. + skips the packed-artifact path — download or local bake — entirely on this + backend: the golden is booted from its base image, the runner contract is + applied in the guest, and `start --forkable` captures the result, which this + backend's snapshots carry into every fork. `use_fork` stays on. ## Identity, TTL, and the registry diff --git a/scripts/build-golden-x86_64.sh b/scripts/build-golden-x86_64.sh index 7ae007e43..5f7561113 100755 --- a/scripts/build-golden-x86_64.sh +++ b/scripts/build-golden-x86_64.sh @@ -6,6 +6,12 @@ set -euo pipefail cd "$(dirname "$0")/.." +# The official golden is baked from the digest-pinned official runner image. +# There is no stock-Ubuntu default: `build-golden` needs the image it bakes, +# and this script bakes — it never fetches the published packed golden. +BASE_IMAGE="${BASE_IMAGE:-$(awk -F'"' '/^official_runner_image_base_amd64 = / { print $2; exit }' official-image.toml)}" +test -n "$BASE_IMAGE" || { echo "official_runner_image_base_amd64 missing from official-image.toml" >&2; exit 1; } + echo "==> Building host CLI (release)" cargo build --release -p preloop-cli @@ -13,14 +19,16 @@ echo "==> Cross-building x86_64 runner bundle" rustup target add x86_64-unknown-linux-gnu 2>/dev/null || true cargo zigbuild --release -p preloop-runner --target x86_64-unknown-linux-gnu -echo "==> Building x86_64 golden (bakes tier 1-4: git-lfs/cmake/sshpass, pnpm/yarn/nvm, python 3.10, go 1.24, gh, yq)" +echo "==> Building x86_64 golden from $BASE_IMAGE" mkdir -p dist rm -f dist/preloop-ubuntu-24.04-x86_64 ./target/release/preloop build-golden \ --runner-bundle target/x86_64-unknown-linux-gnu/release \ + --base-image "$BASE_IMAGE" \ --output dist/preloop-ubuntu-24.04-x86_64 test -s dist/preloop-ubuntu-24.04-x86_64 echo "==> Golden built: dist/preloop-ubuntu-24.04-x86_64" -echo " Sanity-check the bake manifest inside a VM:" -echo " ./target/release/preloop serve --listen 127.0.0.1:9090 (then run a job)" +echo " The bake adds the GitHub-runner machinery only; /etc/preloop-bake.json" +echo " inside a VM records the image and what the bake resolved." +echo " Sanity-check it with ./target/release/preloop serve --listen 127.0.0.1:9090 (then run a job)" diff --git a/scripts/write-golden-provenance.py b/scripts/write-golden-provenance.py index 6822d70a6..0f7842afa 100644 --- a/scripts/write-golden-provenance.py +++ b/scripts/write-golden-provenance.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import datetime import hashlib import json import os @@ -32,8 +31,6 @@ def main() -> None: parser.add_argument("--golden", type=Path, required=True) parser.add_argument("--base-evidence", type=Path, required=True) parser.add_argument("--base-sbom", type=Path, required=True) - parser.add_argument("--apt-indices-date", default=None) - parser.add_argument("--apt-indices-max-age-days", default=None) args = parser.parse_args() evidence = json.loads(args.base_evidence.read_text(encoding="utf-8")) @@ -69,17 +66,6 @@ def main() -> None: }, "base_evidence": file_record(args.base_evidence), "base_sbom": file_record(args.base_sbom), - "apt_indices": { - key: value - for key, value in { - "baked_at": args.apt_indices_date - or datetime.date.today().isoformat(), - "max_age_days": int(args.apt_indices_max_age_days) - if args.apt_indices_max_age_days - else 7, - }.items() - if value - }, "workflow": { key: value for key, value in { diff --git a/skills/preloop/SKILL.md b/skills/preloop/SKILL.md index 8857f496e..a91831f50 100644 --- a/skills/preloop/SKILL.md +++ b/skills/preloop/SKILL.md @@ -125,13 +125,15 @@ the URL GitHub and remote runners can actually reach. Hidden alias: `engine`. | `PRELOOP_RUNNER_POOL_SIZE` | override warm runner pool size (memory-bounded, cap 8) | | `PRELOOP_RUNNER_STORAGE_GB` | persistent guest storage per runner; use `80` or more for full hosted-image OCI snapshots | | `PRELOOP_RUNNER_PACK_PROXY` / `PRELOOP_RUNNER_PACK_NO_PROXY` | proxy and bypass list for smolvm's registry export VM during golden packing | -| `PRELOOP_RUNNER_BASE_IMAGE` / `PRELOOP_RUNNER_BUNDLE` / `PRELOOP_RUNNER_DNS` / `PRELOOP_RUNNER_OVERLAY_GB` / `PRELOOP_RUNNER_NAME_PREFIX` / `PRELOOP_RUNNER_POOL_ENABLED` / `PRELOOP_USE_FORK` / `PRELOOP_USE_PACKED_GOLDEN` / `PRELOOP_WORKSPACE` | runner pool / VM tuning knobs (full semantics in `crates/preloop-cli/src/main.rs`) | - -Runner execution defaults: packed goldens enabled, warm pool disabled, one -single-use VM provisioned per queued job, and concurrency capped by host CPU. -Each VM receives 4 vCPUs and a 4096 MiB ballooned memory ceiling. Set -`PRELOOP_RUNNER_POOL_ENABLED=true` to keep warm runners (size is memory-bounded, -cap 8), or `PRELOOP_USE_PACKED_GOLDEN=false` to force cold OCI provisioning. +| `PRELOOP_RUNNER_BASE_IMAGE` / `PRELOOP_RUNNER_BUNDLE` / `PRELOOP_RUNNER_DNS` / `PRELOOP_RUNNER_OVERLAY_GB` / `PRELOOP_RUNNER_NAME_PREFIX` / `PRELOOP_RUNNER_POOL_ENABLED` / `PRELOOP_USE_FORK` | runner pool / VM tuning knobs (full semantics in `crates/preloop-cli/src/main.rs`) | + +Runner execution defaults: the official packed golden is downloaded once per +environment and each job forks from it; the warm pool is off, and concurrency +is capped by host CPU. Each VM receives 4 vCPUs and a 4096 MiB ballooned memory +ceiling. Set `PRELOOP_RUNNER_POOL_ENABLED=true` to keep warm runners (size is +memory-bounded, cap 8). A configured `PRELOOP_RUNNER_BASE_IMAGE` is baked as-is +plus the GitHub-runner machinery; `runs-on: ubuntu-latest`/`ubuntu-24.04` and a +pool with no image configured use the official golden. ## Failure classification (from `preloop/docs/09_agent_loop_retry_and_fork.md`) diff --git a/versions.toml b/versions.toml index 96c33c00e..418c9ad39 100644 --- a/versions.toml +++ b/versions.toml @@ -25,179 +25,23 @@ smolvm_min_version = "1.18.1" # floor rarely moves. smolvm_golden_version = "1.22.2" -# Freshness policy for apt package indices baked into golden images. -# Provisioning stamps each bake with its date (see prepare_artifact) and warns -# when a pack serving curated jobs is older than this. The weekly -# apt-indices-refresh workflow opens a draft reminder PR past this age; the -# golden is rebuilt manually on a host with enough disk. -# Quoted: build.rs only picks up quoted values (and Renovate's regex manager -# expects flat `key = "value"` lines). -# Golden rebuild requested 2026-10-05: published provenance is missing. -apt_indices_max_age_days = "7" - -# Image inputs for the runner golden. The OCI bases are immutable filesystem -# inputs. `github_runner_image_version` is the actions/runner-images snapshot -# used as the parity source for the tool and package versions below. +# Image inputs for the runner golden. The official golden is baked from the +# per-architecture OCI references in official-image.toml (the dump of the +# GitHub-hosted runner image) and consumed as a packed artifact; the only +# other golden source is whatever `--base-image`/`PRELOOP_RUNNER_BASE_IMAGE` +# names, which is used as-is plus the runner contract (see +# crates/preloop-orchestrator/src/lib.rs `golden_contract_script`). # crates/preloop-orchestrator/build.rs compiles this file into the CLI; rebuild # and reinstall Preloop after changing a pin. -github_runner_image_version = "20260720.247.2" - -# Keep base images digest-pinned. Tags are useful labels but are not immutable. -# Custom organization images are runtime/build inputs (`--base-image` and -# PRELOOP_RUNNER_BASE_IMAGE), not project defaults in this file. -ubuntu_24_04_base = "mirror.gcr.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0aab6c06ba2cef9ebffbc7092d90" -ubuntu_22_04_base = "mirror.gcr.io/library/ubuntu:22.04@sha256:0e0a0fc6d18feda9db1590da249ac93e8d5abfea8f4c3c0c849ce512b5ef8982" -# Provisioning version pins baked into the runner golden. -node_version = "22.23.1" -rustup_version = "1.29.0" -rust_toolchain_version = "1.97" -cargo_shear_version = "1.12.4" +# Node runtimes the guest runner mounts for JavaScript actions, pinned per +# runtime and platform. Downloaded once on the host and mounted read-only into +# every machine (never baked into a golden). node20_externals_version = "20.20.2" node24_externals_version = "24.19.0" -# GitHub-hosted parity bake list — exact versions from the official -# ubuntu-24.04 runner image toolset named above. Baked into the golden at -# build time; see docs/vm-images.md for the tier rationale. -git_version = "2.54.0" -git_lfs_version = "3.7.1" -docker_version = "28.0.4" -docker_buildx_version = "0.35.0" # Docker CLI baked into the standalone preloop-runner container image # (ci/runner.Dockerfile, published by release-runner.yml). Tracks the official -# actions/runner image's pins — NOT the golden's docker_version above, which -# follows the ubuntu-24.04 toolset. +# actions/runner image's pins. runner_image_docker_version = "29.6.1" runner_image_buildx_version = "0.35.0" -docker_compose_version = "2.38.2" -clang_versions = "16 17 18" -clang_default_version = "18" -clang_16_version = "16.0.6" -clang_17_version = "17.0.6" -clang_18_version = "18.1.3" -gcc_versions = "12 13 14" -gcc_12_version = "12.4.0" -gcc_13_version = "13.3.0" -gcc_14_version = "14.2.0" -nvm_version = "0.40.6" -yarn_version = "1.22.22" - -# Apt packages baked into the golden, pinned exactly. Versions marked -# EXACT match the official ubuntu-24.04 runner image toolset -# named by `github_runner_image_version`; the rest are the archive's current -# version where the official pin has been superseded in noble. Re-verify -# against the image readme + the noble archive when bumping. -apt_git = "1:2.43.0-1ubuntu7.3" -apt_curl = "8.5.0-2ubuntu10.11" -apt_wget = "1.21.4-1ubuntu4.4" -apt_ca_certificates = "20260601~24.04.1" -apt_gnupg2 = "2.4.4-2ubuntu17.4" -apt_sudo = "1.9.15p5-3ubuntu5.24.04.2" -apt_openssh_client = "1:9.6p1-3ubuntu13.18" -# Browser runtime libraries — the GitHub-hosted ubuntu-24.04 image ships -# these (its Chromium/playwright toolset depends on them), and workflows -# that run playwright/puppeteer tests fail without them. -apt_libnspr4 = "2:4.35-1.1build1" -apt_libnss3 = "2:3.98-1ubuntu0.2" -apt_libatk1 = "2.52.0-1build1" -apt_libatk_bridge = "2.52.0-1build1" -apt_libatspi = "2.52.0-1build1" -apt_libcairo2 = "1.18.0-3build1" -apt_libcups2t64 = "2.4.7-1.2ubuntu7.14" -apt_libdbus_1_3 = "1.14.10-4ubuntu4.1" -apt_libdrm2 = "2.4.125-1ubuntu0.1~24.04.2" -apt_libgbm1 = "25.2.8-0ubuntu0.24.04.2" -apt_libglib2 = "2.80.0-6ubuntu3.8" -apt_libpango = "1.52.1+ds-1build1" -apt_libx11_6 = "2:1.8.7-1build1" -apt_libxcb1 = "1.15-1ubuntu2" -apt_libxcomposite1 = "1:0.4.5-1build3" -apt_libxdamage1 = "1:1.1.6-1build1" -apt_libxext6 = "2:1.3.4-1build2" -apt_libxfixes3 = "1:6.0.0-2build1" -apt_libxkbcommon0 = "1.6.0-1build1" -apt_libxrandr2 = "2:1.5.2-2build1" -apt_libasound2t64 = "1.2.11-1ubuntu0.3" -# Scripting runtimes the GitHub-hosted ubuntu-24.04 image ships — real -# workflows run `gem install` (neovim's functionaltests install the neovim -# RubyGem) and `cpanm` (Perl module tests) without installing anything. -apt_ruby = "1:3.2~ubuntu1" -apt_ruby_rubygems = "3.4.20-1" -apt_perl = "5.38.2-3.2ubuntu0.3" -apt_cpanminus = "1.7047-1" -# Rest of the GitHub-hosted ubuntu-24.04 image's apt baseline (image readme -# 20260720.247.2): network/entropy/font/display utilities real workflows -# reach for (`lsb_release` in LLVM install scripts, `xvfb` for headless -# browser tests, `telnet`/`sshpass` for service probes, …). -apt_lsb_release = "12.0-2" -apt_fonts_noto_color_emoji = "2.047-0ubuntu0.24.04.1" -apt_haveged = "1.9.14-1ubuntu2" -apt_mediainfo = "24.01.1-1build2" -apt_p7zip_rar = "16.02+transitional.1" -apt_pollinate = "4.33-3.1ubuntu1.3" -apt_sshpass = "1.09-1" -apt_telnet = "0.17+2.5-3ubuntu4.2" -apt_tk = "8.6.14build1" -apt_xvfb = "2:21.1.12-1ubuntu1.6" -apt_zsync = "0.6.2-5build1" -apt_ftp = "20230507-2build3" -apt_sphinxsearch = "2.2.11-8build1" -apt_systemd_coredump = "255.4-1ubuntu8.17" -apt_libnss3_tools = "2:3.98-1ubuntu0.2" -apt_software_properties_common = "0.99.49.4" -apt_build_essential = "12.10ubuntu1" -apt_pkg_config = "1.8.1-2build1" -apt_libssl_dev = "3.0.13-0ubuntu3.12" -apt_make = "4.3-4.1build2" -apt_autoconf = "2.71-3" -apt_automake = "1:1.16.5-1.3ubuntu1" -apt_libtool = "2.4.7-7build1" -apt_m4 = "1.4.19-4build1" -apt_bison = "2:3.8.2+dfsg-1build2" -apt_flex = "2.6.4-8.2build1" -apt_texinfo = "7.1-3build2" -apt_patchelf = "0.18.0-1.1build1" -apt_swig = "4.2.0-2ubuntu1" -apt_dpkg_dev = "1.22.6ubuntu6.6" -apt_fakeroot = "1.33-1" -apt_binutils = "2.42-4ubuntu2.10" -apt_lld = "1:18.0-58ubuntu1" -apt_libicu_dev = "74.2-1ubuntu3.1" -apt_libsqlite3_dev = "3.45.1-1ubuntu2.7" -apt_libyaml_dev = "0.2.5-1build1" -apt_python3 = "3.12.3-0ubuntu2.1" -apt_python3_pip = "24.0+dfsg-1ubuntu1.3" -apt_python_is_python3 = "3.11.4-1" -apt_unzip = "6.0-28ubuntu4.1" -apt_zip = "3.0-13ubuntu0.2" -apt_xz_utils = "5.6.1+really5.4.5-1ubuntu0.3" -apt_zstd = "1.5.5+dfsg2-2build1.1" -apt_bzip2 = "1.0.8-5.1build0.1" -apt_brotli = "1.1.0-2build2" -apt_lz4 = "1.9.4-1build1.1" -apt_pigz = "2.8-1" -apt_p7zip_full = "16.02+transitional.1" -apt_tar = "1.35+dfsg-3ubuntu0.4" -apt_jq = "1.7.1-3ubuntu0.24.04.2" -apt_file = "1:5.45-3build1" -apt_tree = "2.1.1-2ubuntu3.24.04.2" -apt_shellcheck = "0.9.0-1" -apt_parallel = "20231122+ds-1" -apt_time = "1.9-0.2build1" -apt_acl = "2.3.2-1build1.1" -apt_locales = "2.39-0ubuntu8.8" -apt_tzdata = "2026c-0ubuntu0.24.04.1" -apt_rsync = "3.2.7-1ubuntu1.5" -apt_dnsutils = "1:9.18.39-0ubuntu0.24.04.5" -apt_iputils_ping = "3:20240117-1ubuntu0.1" -apt_net_tools = "2.10-0.1ubuntu4.4" -apt_iproute2 = "6.1.0-1ubuntu6.4" -apt_netcat_openbsd = "1.226-1ubuntu2" -apt_sqlite3 = "3.45.1-1ubuntu2.7" -apt_rpm = "4.18.2+dfsg-2.1build2" -apt_aria2 = "1.37.0+debian-1build3" -apt_mercurial = "6.7.2-1ubuntu2.2" -apt_libcurl4_openssl_dev = "8.5.0-2ubuntu10.11" -apt_zlib1g_dev = "1:1.3.dfsg-3.1ubuntu2.1" -apt_gettext = "0.21-14ubuntu2" -apt_libexpat1_dev = "2.6.1-2ubuntu0.4"