Skip to content

Commit 6bcad50

Browse files
committed
Option M: manual publish, fresh build each run, Make targets, attest own output
1 parent 14dc77b commit 6bcad50

4 files changed

Lines changed: 75 additions & 46 deletions

File tree

‎.github/workflows/bugbash-sandbox-image.yml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
11
# The bug-bash sandbox image (#5714): tests/bugbash/sandbox/Dockerfile, built by build.sh.
22
#
3-
# Pull requests build it with a read-only token and push nothing. Only a push to main (or a
4-
# manual run on main) publishes, from that exact commit, with no cache and no artifact from any
5-
# other run. A person then copies the digest from the job summary into image.json in a pull
3+
# Pull requests build it with a read-only token and push nothing. Publishing is manual: only a
4+
# workflow_dispatch run on main publishes, from that run's exact commit, with no cache and no
5+
# artifact from any other run. Each publish builds fresh, and the attestation names that build's
6+
# own output. A person then copies the digest from the job summary into image.json in a pull
67
# request: no workflow writes to the repo. Nothing here deletes an image.
8+
#
9+
# Until a new image is published and its digest PR merges, runners refuse a checkout whose image
10+
# inputs changed (stale inputs key).
711
name: Bug-bash sandbox image
812

913
on:
10-
push:
11-
branches: [main]
12-
paths:
13-
- tests/bugbash/sandbox/Dockerfile
14-
- tests/bugbash/sandbox/build.sh
15-
- bun.lock
16-
- .github/workflows/bugbash-sandbox-image.yml
1714
pull_request:
1815
paths:
1916
- tests/bugbash/sandbox/Dockerfile
2017
- tests/bugbash/sandbox/build.sh
2118
- bun.lock
19+
- Makefile
2220
- .github/workflows/bugbash-sandbox-image.yml
2321
workflow_dispatch: {}
2422

@@ -42,17 +40,17 @@ jobs:
4240
persist-credentials: false
4341
- name: Build
4442
run: |
45-
id=$(./tests/bugbash/sandbox/build.sh)
43+
id=$(make -s bugbash-sandbox-image)
4644
size=$(docker image inspect --format '{{.Size}}' "$id")
47-
echo "Image $id: $size bytes (inputs key $(./tests/bugbash/sandbox/build.sh --key))" | tee -a "$GITHUB_STEP_SUMMARY"
45+
echo "Image $id: $size bytes (inputs key $(make -s bugbash-sandbox-key))" | tee -a "$GITHUB_STEP_SUMMARY"
4846
4947
publish:
5048
name: Publish
51-
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
49+
if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
5250
runs-on: ubuntu-latest
5351
permissions:
5452
contents: read
55-
# Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only on main.
53+
# Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only a manual run on main.
5654
packages: write
5755
# Build provenance for audit (the runner trusts the digest in image.json).
5856
id-token: write
@@ -71,8 +69,10 @@ jobs:
7169
- name: Build and push
7270
id: push
7371
run: |
74-
./tests/bugbash/sandbox/build.sh --push > image.json
75-
echo "digest=$(jq -r .digest image.json)" >> "$GITHUB_OUTPUT"
72+
make -s bugbash-sandbox-publish > image.json
73+
# The subject of the attestation is this build's own output (image.json).
74+
echo "image=$(jq -er .image image.json)" >> "$GITHUB_OUTPUT"
75+
echo "digest=$(jq -er .digest image.json)" >> "$GITHUB_OUTPUT"
7676
{
7777
echo '### tests/bugbash/sandbox/image.json'
7878
echo
@@ -83,6 +83,6 @@ jobs:
8383
- name: Attest
8484
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
8585
with:
86-
subject-name: ghcr.io/coder/xum-bugbash-sandbox
86+
subject-name: ${{ steps.push.outputs.image }}
8787
subject-digest: ${{ steps.push.outputs.digest }}
8888
push-to-registry: true

‎Makefile‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ include fmt.mk
9191
.PHONY: all build dev start clean help
9292
.PHONY: build-renderer version build-icons build-static build-docker-runtime verify-docker-runtime-artifacts
9393
.PHONY: lint lint-fix typecheck static-check static-check-full
94-
.PHONY: test-bugbash-repros test-bugbash-known-failures
94+
.PHONY: test-bugbash-repros test-bugbash-known-failures bugbash-sandbox-key bugbash-sandbox-image bugbash-sandbox-publish
9595
.PHONY: test test-unit test-unit-ci test-integration test-watch test-coverage test-e2e test-e2e-perf perf-tape-replay smoke-test
9696
.PHONY: dist dist-mac dist-win dist-linux install-mac-arm64 ensure-mac-sharp-runtime-deps check-appimage-icons check-mac-attach-file-runtime
9797
.PHONY: vscode-ext vscode-ext-install
@@ -224,6 +224,17 @@ dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + f
224224
bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714)
225225
@bun tests/bugbash/run.ts $(BUGBASH_ARGS)
226226

227+
# The bug-bash sandbox image (tests/bugbash/sandbox/build.sh, #5714). Each prints one value on
228+
# stdout. Publishing runs only in the manual "Bug-bash sandbox image" workflow on main.
229+
bugbash-sandbox-key: ## Print the inputs key of the bug-bash sandbox image for this checkout
230+
@./tests/bugbash/sandbox/build.sh --key
231+
232+
bugbash-sandbox-image: ## Build the bug-bash sandbox image locally (linux/amd64) and print its image ID
233+
@./tests/bugbash/sandbox/build.sh
234+
235+
bugbash-sandbox-publish: ## Workflow only: build and push the bug-bash sandbox image, print its image.json record
236+
@./tests/bugbash/sandbox/build.sh --push
237+
227238
# Bug-bash repro tests (tests/bugbash/repros/*.e2e.ts): exact UI steps, no model calls, against a
228239
# seeded `xum server` (tests/bugbash/startApp.ts). e2e needs Node.js 22.22.3+ or 24.8+ (E2E_NODE)
229240
# and its own Chromium: `node_modules/.bin/e2e-web install chromium`.

‎tests/bugbash/sandbox/build.sh‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,18 @@
33
#
44
# build.sh --key print the inputs key of this checkout
55
# build.sh build for linux/amd64 into the local Docker, print the image ID
6-
# build.sh --push main publish job only: push to GHCR, print the image.json record
6+
# build.sh --push the manual publish job only: build, push to GHCR, print the image.json record
77
#
88
# The inputs key is the sha256 of three lines: the sha256 of the Dockerfile, the sha256 of this
99
# script, and the playwright-core version of @e2e-dev/web in bun.lock. The image carries it as
1010
# the label org.xum.bugbash.inputs, and the runner computes the same key for its checkout.
1111
# The script reads these files from the working tree and refuses when they differ from HEAD, so
1212
# the key always names committed source.
1313
#
14-
# The build gets an empty context, no cache and no build args from the env. --push skips the
15-
# build when the registry already has the tag inputs-<key>: one key, one published digest.
14+
# The build gets an empty context, no cache and no build args from the env. --push always builds
15+
# fresh: it never checks the registry and never reuses an image. It pushes a new tag, so it
16+
# overwrites no tag, and it reports the name and digest from its own build output. Two publishes
17+
# of one inputs key give two digests; the reviewed digest in image.json is what runners trust.
1618
# Nothing here deletes an image.
1719
set -euo pipefail
1820

@@ -64,7 +66,7 @@ case "$mode" in
6466
esac
6567

6668
context=$(mktemp -d)
67-
trap 'rm -rf "$context" "$context.json" "$context.raw"' EXIT
69+
trap 'rm -rf "$context" "$context.json"' EXIT
6870
build() {
6971
docker buildx build --platform linux/amd64 --no-cache --pull --provenance=false --sbom=false \
7072
--build-arg "PLAYWRIGHT_CORE_VERSION=$PLAYWRIGHT" --build-arg "INPUTS_SHA256=$KEY" \
@@ -77,18 +79,13 @@ if [ "$mode" = local ]; then
7779
exit 0
7880
fi
7981

80-
# --push
81-
TAG="$IMAGE:inputs-$KEY"
82-
# The digest of a manifest is the sha256 of its raw bytes. Hash the file, not a --format value:
83-
# some buildx versions ignore --format here and print a table (docker/buildx#4006).
84-
if docker buildx imagetools inspect --raw "$TAG" >"$context.raw" 2>/dev/null; then
85-
digest="sha256:$(sha "$context.raw")"
86-
echo "build.sh: $TAG exists, no build" >&2
87-
else
88-
build --push --tag "$TAG"
89-
digest=$(jq -r '."containerimage.digest"' "$context.json")
90-
fi
91-
[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest: $digest"
92-
jq -n --arg image "$IMAGE" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \
82+
# --push. One new tag per publish (inputs key, commit, UTC time), so a publish overwrites no tag.
83+
build --push --tag "$IMAGE:inputs-${KEY:0:16}-$(git rev-parse --short=12 HEAD)-$(date -u +%Y%m%d%H%M%S)"
84+
# The name and digest come from this build's own metadata, not from a registry lookup.
85+
digest=$(jq -r '."containerimage.digest"' "$context.json")
86+
pushed=$(jq -r '."image.name"' "$context.json")
87+
[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest in the build metadata: $digest"
88+
[[ $pushed == "$IMAGE:"* && $pushed != *,* ]] || die "unexpected image name in the build metadata: $pushed"
89+
jq -n --arg image "${pushed%:*}" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \
9390
--arg from "$(git rev-parse HEAD)" \
9491
'{image: $image, digest: $digest, inputsKey: $key, playwrightCore: $pw, publishedFrom: $from}'

‎tests/bugbash/sandbox/build.test.ts‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -72,23 +72,44 @@ test("the version comes only from the @e2e-dev/web copy, and must be unique", ()
7272
expect(key().err).toContain("found 0");
7373
});
7474

75-
test("--push for an existing tag reports the digest of its manifest bytes, and builds nothing", () => {
75+
test("--push always builds fresh and reports the name and digest of its own build", () => {
7676
setup();
77-
// A fake docker: `imagetools inspect --raw` prints a manifest, any other command fails.
77+
// A fake docker: `buildx build` writes build metadata to --metadata-file, and every call is
78+
// logged. A registry lookup (imagetools) would be a reuse path.
7879
const bin = path.join(repo, "bin");
7980
fs.mkdirSync(bin);
80-
const manifest = '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}\n';
81-
fs.writeFileSync(path.join(bin, "manifest.json"), manifest);
81+
const calls = path.join(bin, "calls.log");
82+
const digest = `sha256:${"ab".repeat(32)}`;
8283
fs.writeFileSync(
8384
path.join(bin, "docker"),
84-
`#!/bin/sh\n[ "$1 $2 $3 $4" = "buildx imagetools inspect --raw" ] || exit 9\ncat "${bin}/manifest.json"\n`,
85+
`#!/bin/sh
86+
echo "$*" >> "${calls}"
87+
[ "$1 $2" = "buildx build" ] || exit 9
88+
tag=""; meta=""
89+
while [ $# -gt 0 ]; do
90+
case "$1" in --tag) tag=$2 ;; --metadata-file) meta=$2 ;; esac
91+
shift
92+
done
93+
printf '{"containerimage.digest":"${digest}","image.name":"%s"}' "$tag" > "$meta"
94+
`,
8595
{ mode: 0o755 }
8696
);
87-
const r = spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], {
88-
encoding: "utf8",
89-
env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` },
90-
});
97+
const push = () =>
98+
spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], {
99+
encoding: "utf8",
100+
env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` },
101+
});
102+
const r = push();
91103
expect(r.status).toBe(0);
92-
const digest = `sha256:${new Bun.CryptoHasher("sha256").update(manifest).digest("hex")}`;
93-
expect(JSON.parse(r.stdout)).toMatchObject({ digest, playwrightCore: "1.63.0" });
104+
expect(JSON.parse(r.stdout)).toMatchObject({
105+
image: "ghcr.io/coder/xum-bugbash-sandbox",
106+
digest,
107+
inputsKey: key().out,
108+
playwrightCore: "1.63.0",
109+
});
110+
// A second publish of the same inputs builds again. Every call was a push build.
111+
expect(push().status).toBe(0);
112+
const log = fs.readFileSync(calls, "utf8").trim().split("\n");
113+
expect(log).toHaveLength(2);
114+
for (const line of log) expect(line).toMatch(/^buildx build .*--no-cache .*--push /);
94115
});

0 commit comments

Comments
 (0)