diff --git a/tests/bugbash/sandbox/image.json b/tests/bugbash/sandbox/image.json new file mode 100644 index 00000000000..c5d934b9903 --- /dev/null +++ b/tests/bugbash/sandbox/image.json @@ -0,0 +1,7 @@ +{ + "image": "ghcr.io/coder/xum-bugbash-sandbox", + "digest": "sha256:61adf1a543f2b1cc1373506556eefad52176aabd5bb1e491927eef82bde8fa73", + "inputsKey": "0d15da9c1a6a87051e21b589933ffefb000af794de69d2741661fded30e7abf7", + "playwrightCore": "1.63.0", + "publishedFrom": "752a624e15e0a332763d6fef8329d86d9faebe36" +} diff --git a/tests/bugbash/sandbox/runner.test.ts b/tests/bugbash/sandbox/runner.test.ts new file mode 100644 index 00000000000..6199f62d3da --- /dev/null +++ b/tests/bugbash/sandbox/runner.test.ts @@ -0,0 +1,204 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { readImageLock, Refusal, Session, Stopped } from "./runner"; + +// Each test runs the real build.sh in a throwaway git repo and a fake `docker` on PATH. The +// runner gives docker a stripped env, so the fake reads its answers from bin/fake.env. +const SANDBOX = "tests/bugbash/sandbox"; +const DIGEST = `sha256:${"61".repeat(32)}`; +const REF = `ghcr.io/coder/xum-bugbash-sandbox@${DIGEST}`; +const FAKE = `#!/bin/sh +bin=$(dirname "$0"); . "$bin/fake.env" +echo "$* [home=\${HOME-} cfg=\${DOCKER_CONFIG-}]" >> "$bin/calls.log" +case "$1" in + context) echo "$HOST" ;; + info) echo "$INFO" ;; + images) [ "$IMAGES_RC" = 0 ] || { echo "daemon down" >&2; exit 1; }; echo "$IMAGES" ;; + pull) if [ "$PULL" = hang ]; then trap '' TERM; exec sleep 30; fi ;; + image) echo "{\\"org.xum.bugbash.inputs\\":\\"$LABEL\\"}" ;; + ps) [ "$PS_RC" = 0 ] || exit 1; cat "$bin/containers" ;; + rm) : > "$bin/containers" ;; + *) exit 9 ;; +esac +`; + +let root = ""; +let bin = ""; +let key = ""; +const savedPath = process.env.PATH; +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-runner-")); + fs.mkdirSync(path.join(root, SANDBOX), { recursive: true }); + for (const name of ["build.sh", "Dockerfile"]) + fs.copyFileSync(path.join(import.meta.dir, name), path.join(root, SANDBOX, name)); + fs.writeFileSync( + path.join(root, "bun.lock"), + '"@e2e-dev/web/playwright-core": ["playwright-core@1.63.0", ""],\n' + ); + for (const args of [ + ["init", "-q"], + ["add", "-A"], + ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "x"], + ]) + expect(spawnSync("git", args, { cwd: root }).status).toBe(0); + key = spawnSync(path.join(root, SANDBOX, "build.sh"), ["--key"], { + encoding: "utf8", + }).stdout.trim(); + expect(key).toMatch(/^[0-9a-f]{64}$/); + lock(key); + bin = path.join(root, "bin"); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, "docker"), FAKE, { mode: 0o755 }); + fs.writeFileSync(path.join(bin, "containers"), ""); + fake(); + process.env.PATH = `${bin}:${savedPath ?? ""}`; +}); +afterEach(async () => { + await Promise.all(sessions.splice(0).map((s) => s.cleanup())); + process.env.PATH = savedPath; + fs.rmSync(root, { recursive: true, force: true }); +}); + +function lock(inputsKey: string, digest = DIGEST) { + const record = { + image: "ghcr.io/coder/xum-bugbash-sandbox", + digest, + inputsKey, + playwrightCore: "1.63.0", + publishedFrom: "7".repeat(40), + }; + fs.writeFileSync(path.join(root, SANDBOX, "image.json"), JSON.stringify(record)); +} +function fake(over: Record = {}) { + const vars = { + HOST: "unix:///var/run/docker.sock", + INFO: '{"OSType":"linux","OperatingSystem":"Ubuntu 22.04","SecurityOptions":["name=seccomp"]}', + IMAGES_RC: "0", + IMAGES: "", + PULL: "ok", + LABEL: key, + PS_RC: "0", + ...over, + }; + fs.writeFileSync( + path.join(bin, "fake.env"), + Object.entries(vars) + .map(([k, v]) => `${k}='${v}'\n`) + .join("") + ); +} +const calls = () => + fs.existsSync(path.join(bin, "calls.log")) + ? fs.readFileSync(path.join(bin, "calls.log"), "utf8") + : ""; +/** The error a promise rejects with (`expect().rejects` is not typed as awaitable here). */ +const failure = (p: Promise) => + p.then( + () => () => undefined, + (e: unknown) => () => { + throw e; + } + ); +// Every session gets cleaned up, so no private client folder stays behind. +const sessions: Session[] = []; +const session = (stop = new AbortController()) => { + const s = new Session(stop.signal, { root }); + sessions.push(s); + return s; +}; + +test("the committed image.json is a valid lock, and a malformed one refuses", () => { + expect(readImageLock().digest).toMatch(/^sha256:[0-9a-f]{64}$/); + lock(key, "sha256:short"); + expect(() => readImageLock(root)).toThrow(Refusal); +}); + +test("a stale inputs key refuses before any docker command", async () => { + lock("0".repeat(64)); + expect(await failure(session().ensureImage())).toThrow(/stale image/); + await Bun.sleep(0); + expect(calls()).toBe(""); +}); + +test("a missing image is pulled by digest, with a private client config", async () => { + const s = session(); + expect(await s.ensureImage()).toBe(REF); + const lines = calls().trim().split("\n"); + expect(lines.map((l) => l.split(" ")[0])).toEqual(["context", "info", "images", "pull", "image"]); + expect(lines[3]).toStartWith(`pull --quiet ${REF} `); + // After `context inspect`, docker gets no HOME and an empty config folder that cleanup removes. + const cfg = /cfg=(\S+)\]/.exec(lines[2])![1]; + expect(lines[2]).toContain("[home= "); + expect(fs.readdirSync(cfg)).toEqual([]); + expect(await s.cleanup()).toBe("removed"); + expect(fs.existsSync(cfg)).toBe(false); +}); + +test("a present image is not pulled", async () => { + fake({ IMAGES: "sha256:abc" }); + expect(await session().ensureImage()).toBe(REF); + expect(calls()).not.toContain("pull"); +}); + +test.each([ + [ + "a label for other inputs", + { IMAGES: "sha256:abc", LABEL: "f".repeat(64) }, + /label org.xum.bugbash.inputs/, + ], + ["an image query failure (not 'absent')", { IMAGES_RC: "1" }, /docker images: daemon down/], + ["a remote endpoint", { HOST: "tcp://10.0.0.1:2375" }, /not a local socket/], + ["no daemon", { INFO: '{"OSType":""}' }, /no daemon answered/], + [ + "Docker Desktop", + { INFO: '{"OSType":"linux","OperatingSystem":"Docker Desktop"}' }, + /Docker Desktop/, + ], + [ + "rootless Docker", + { INFO: '{"OSType":"linux","SecurityOptions":["name=rootless"]}' }, + /rootless/, + ], +])("%s refuses, and nothing is pulled", async (_name, over, message) => { + fake(over); + expect(await failure(session().ensureImage())).toThrow(message); + await Bun.sleep(50); + expect(calls()).not.toContain("pull"); +}); + +test("a stop ends a running pull; cleanup still runs and removes only the job's container", async () => { + fake({ PULL: "hang" }); + fs.writeFileSync(path.join(bin, "containers"), "c1\n"); + const stop = new AbortController(); + const s = session(stop); + const pending = s.ensureImage(); + while (!calls().includes("pull ")) await Bun.sleep(20); + const stoppedAt = Date.now(); + stop.abort("SIGTERM"); + // The fake pull ignores SIGTERM, so it ends only by SIGKILL after the grace period. + expect(await pending.catch((e: unknown) => e)).toEqual(new Stopped("SIGTERM")); + expect(Date.now() - stoppedAt).toBeGreaterThanOrEqual(4_500); + expect(await failure(s.ensureImage())).toThrow(Stopped); + + const job = { name: "xbb-1", owner: "boot:pid", checkout: "abc" }; + const first = s.cleanup(job); + expect(s.cleanup(job)).toBe(first); + expect(await first).toBe("removed"); + const ps = calls() + .split("\n") + .find((l) => l.startsWith("ps "))!; + expect(ps).toContain( + "--filter name=^/xbb-1$ --filter label=xum.bugbash.owner=boot:pid --filter label=xum.bugbash.checkout=abc" + ); + expect(calls()).toContain("rm -f c1 "); +}, 15_000); + +test("cleanup reports an unknown container state, never success", async () => { + fake({ IMAGES: "sha256:abc", PS_RC: "1" }); + const s = session(); + await s.ensureImage(); + expect(await s.cleanup({ name: "xbb-1", owner: "o", checkout: "c" })).toStartWith("unknown: "); +}); diff --git a/tests/bugbash/sandbox/runner.ts b/tests/bugbash/sandbox/runner.ts new file mode 100644 index 00000000000..d83f0d660b9 --- /dev/null +++ b/tests/bugbash/sandbox/runner.ts @@ -0,0 +1,243 @@ +/** + * The runner library of the bug-bash sandbox (#5714). It finds a local Docker daemon, gets the + * pinned image, runs commands as tracked async children, and cleans up after a job. It has no + * entry point and starts no job container: the launch command comes in a later step. + * + * The trust anchor is the digest in image.json, which a reviewed pull request sets. The runner + * pulls only `name@digest` and never builds. It refuses when the inputs key of this checkout + * (`build.sh --key`) differs from image.json: that image was built for other inputs, and a + * person must publish a new one (workflow "Bug-bash sandbox image") and update image.json. + */ +import { spawn, type ChildProcess } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +const ROOT = fs.realpathSync(path.resolve(import.meta.dir, "../../..")); +export const IMAGE = "ghcr.io/coder/xum-bugbash-sandbox"; +/** How long a child gets after SIGTERM before SIGKILL. */ +const KILL_AFTER_MS = 5_000; + +export class Refusal extends Error {} +/** The session stopped (a signal, or cleanup started) before this command could finish. */ +export class Stopped extends Error { + constructor(readonly reason: string) { + super(`stopped: ${reason}`); + } +} + +export interface ImageLock { + image: string; + digest: string; + inputsKey: string; + playwrightCore: string; + publishedFrom: string; +} + +export function readImageLock(root = ROOT): ImageLock { + const file = path.join(root, "tests/bugbash/sandbox/image.json"); + const lock = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; + const valid = + lock.image === IMAGE && + /^sha256:[0-9a-f]{64}$/.test(lock.digest ?? "") && + /^[0-9a-f]{64}$/.test(lock.inputsKey ?? "") && + /^\d+\.\d+\.\d+$/.test(lock.playwrightCore ?? "") && + /^[0-9a-f]{40}$/.test(lock.publishedFrom ?? ""); + if (!valid) throw new Refusal("tests/bugbash/sandbox/image.json is not a valid image lock"); + return lock as ImageLock; +} + +interface Result { + ok: boolean; + stdout: string; + error: string; +} +/** A job container, matched by its name and both labels, never by the name alone. */ +export interface Job { + name: string; + owner: string; + checkout: string; +} +export type CleanupState = "removed" | `unknown: ${string}`; + +export class Session { + readonly #root: string; + readonly #children = new Map>(); + #stopped: string | null = null; + #client: Record | null = null; + #clientDir: string | null = null; + #cleanup: Promise | null = null; + + /** The entry point aborts `stop` from its SIGINT and SIGTERM handlers. */ + constructor(stop: AbortSignal, options: { root?: string } = {}) { + this.#root = options.root ?? ROOT; + if (stop.aborted) this.#stop(String(stop.reason)); + else stop.addEventListener("abort", () => this.#stop(String(stop.reason)), { once: true }); + } + + /** The pinned image, pulled when missing, as `name@digest`. */ + async ensureImage(): Promise { + const lock = readImageLock(this.#root); + const key = await this.#checkoutKey(); + if (key !== lock.inputsKey) + throw new Refusal( + `stale image: image.json has inputs key ${lock.inputsKey.slice(0, 16)}, this checkout ` + + `${key.slice(0, 16)}. Publish a new image, then update tests/bugbash/sandbox/image.json.` + ); + await this.#connect(); + const ref = `${lock.image}@${lock.digest}`; + // An empty list with exit 0 means absent. Any failure refuses: it is not "absent". + const listed = await this.#must(["images", "--no-trunc", "--quiet", ref], 15_000); + if (listed.stdout === "") await this.#must(["pull", "--quiet", ref], 10 * 60_000); + const inspect = ["image", "inspect", "--format", "{{json .Config.Labels}}", ref]; + const labels = JSON.parse((await this.#must(inspect, 15_000)).stdout) as Record< + string, + string + > | null; + const label = labels?.["org.xum.bugbash.inputs"]; + if (label !== key) + throw new Refusal(`${ref}: label org.xum.bugbash.inputs is ${label}, not ${key}`); + return ref; + } + + /** + * One cleanup for success, error and stop: later calls get the same promise. It stops the + * session, waits for every child, and removes the job's container. It reports an unknown + * container state instead of success, and it never removes an image. + */ + cleanup(job?: Job): Promise { + this.#cleanup ??= this.#runCleanup(job); + return this.#cleanup; + } + + async #runCleanup(job?: Job): Promise { + this.#stop("cleanup"); + await Promise.all(this.#children.values()); + const state = + job == null || this.#client == null ? "removed" : await this.#removeContainer(job); + if (this.#clientDir != null) fs.rmSync(this.#clientDir, { recursive: true, force: true }); + return state; + } + + async #removeContainer(job: Job): Promise { + // prettier-ignore + const filters = ["--filter", `name=^/${job.name}$`, "--filter", `label=xum.bugbash.owner=${job.owner}`, + "--filter", `label=xum.bugbash.checkout=${job.checkout}`]; + // Cleanup commands still run after a stop, so they bypass #job(). + const find = () => + this.#spawn("docker", ["ps", "-aq", "--no-trunc", ...filters], this.#client!, 15_000); + const found = await find(); + if (!found.ok) return `unknown: ${found.error}`; + if (found.stdout === "") return "removed"; + await this.#spawn("docker", ["rm", "-f", ...found.stdout.split("\n")], this.#client!, 30_000); + const after = await find(); + if (!after.ok) return `unknown: ${after.error}`; + return after.stdout === "" ? "removed" : `unknown: still present: ${after.stdout}`; + } + + #stop(reason: string) { + this.#stopped ??= reason; + // Only the children of this moment: cleanup commands start later and must finish. + const victims = [...this.#children.keys()]; + for (const child of victims) child.kill("SIGTERM"); + setTimeout(() => { + for (const child of victims) if (this.#children.has(child)) child.kill("SIGKILL"); + }, KILL_AFTER_MS).unref(); + } + + async #checkoutKey(): Promise { + const script = path.join(this.#root, "tests/bugbash/sandbox/build.sh"); + const env = { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "" }; + const r = await this.#job(script, ["--key"], env, 30_000); + if (!r.ok) throw new Refusal(`build.sh --key: ${r.error}`); + return r.stdout; + } + + /** + * Resolves the endpoint of the user's docker CLI once, then uses an empty private client + * config: the CLI reads no user config (its proxies can hold passwords) after this step. + */ + async #connect() { + if (this.#client != null) return; + if (process.platform !== "linux") + throw new Refusal(`${process.platform}: the sandbox needs Linux`); + if (process.getuid?.() === 0) throw new Refusal("the sandbox does not run as root"); + const user: Record = {}; + for (const key of ["PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG"]) + if (process.env[key] != null) user[key] = process.env[key]; + const format = "{{.Endpoints.docker.Host}}"; + const context = await this.#job( + "docker", + ["context", "inspect", "--format", format], + user, + 10_000 + ); + if (!context.ok) throw new Refusal(`docker context: ${context.error}`); + // Fail closed: a remote, ssh or relative endpoint is refused, never swapped for the default. + if (!/^unix:\/\/\/./.test(context.stdout)) + throw new Refusal(`docker endpoint ${JSON.stringify(context.stdout)}: not a local socket`); + this.#clientDir = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-docker-")); + const client = { + PATH: user.PATH ?? "", + DOCKER_HOST: context.stdout, + DOCKER_CONFIG: this.#clientDir, + }; + const info = await this.#job("docker", ["info", "--format", "{{json .}}"], client, 15_000); + if (!info.ok) throw new Refusal(`docker info: ${info.error}`); + const daemon = JSON.parse(info.stdout) as { + OSType?: string; + OperatingSystem?: string; + SecurityOptions?: string[]; + }; + // `docker info` exits 0 when no daemon answers, with only the client fields. + if ((daemon.OSType ?? "") === "") throw new Refusal("docker info: no daemon answered"); + if (daemon.OSType !== "linux" || /docker desktop/i.test(daemon.OperatingSystem ?? "")) + throw new Refusal("Docker Desktop is not supported"); + if ((daemon.SecurityOptions ?? []).some((o) => o.includes("rootless"))) + throw new Refusal("rootless Docker is not supported"); + this.#client = client; + } + + async #must(args: string[], timeoutMs: number): Promise { + const r = await this.#job("docker", args, this.#client!, timeoutMs); + if (!r.ok) throw new Refusal(`docker ${args[0]}: ${r.error}`); + return r; + } + + /** A job command. After a stop it refuses, and a running one ends with Stopped. */ + async #job(cmd: string, args: string[], env: Record, timeoutMs: number) { + if (this.#stopped != null) throw new Stopped(this.#stopped); + const r = await this.#spawn(cmd, args, env, timeoutMs); + if (this.#stopped != null) throw new Stopped(this.#stopped); + return r; + } + + #spawn( + cmd: string, + args: string[], + env: Record, + timeoutMs: number + ): Promise { + const child = spawn(cmd, args, { env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs); + const result = new Promise((resolve) => { + const done = (code: number | null, why: string) => { + clearTimeout(timer); + this.#children.delete(child); + resolve({ + ok: code === 0, + stdout: stdout.trim(), + error: code === 0 ? "" : stderr.trim() || why, + }); + }; + child.once("error", (error) => done(null, error.message)); + child.once("close", (code, signal) => done(code, `exit ${code ?? signal}`)); + }); + this.#children.set(child, result); + return result; + } +}