From e520c795aeb4b6173fbb2773db6d0788504c5427 Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 18:06:08 +0000
Subject: [PATCH 1/6] bug-bash: bash-ai-proxy scenario with a loopback fake
 provider

---
 tests/bugbash/charters.bash-ai-proxy.txt |   9 +
 tests/bugbash/e2e.config.ts              |  44 ++-
 tests/bugbash/fakeProvider.ts            | 414 +++++++++++++++++++++++
 tests/bugbash/startApp.ts                |  64 +++-
 4 files changed, 519 insertions(+), 12 deletions(-)
 create mode 100644 tests/bugbash/charters.bash-ai-proxy.txt
 create mode 100644 tests/bugbash/fakeProvider.ts

diff --git a/tests/bugbash/charters.bash-ai-proxy.txt b/tests/bugbash/charters.bash-ai-proxy.txt
new file mode 100644
index 00000000000..3c15a06d2ff
--- /dev/null
+++ b/tests/bugbash/charters.bash-ai-proxy.txt
@@ -0,0 +1,9 @@
+# Charters for the bash AI proxy branch. Run with BUGBASH_AI=mock BUGBASH_SCENARIO=bash-ai-proxy and
+# BUGBASH_ARGS="--charters tests/bugbash/charters.bash-ai-proxy.txt" (see e2e.config.ts).
+# Keep explorers out of Terminal tabs: they run real shell commands on the host.
+proxy-costs|web|skeptic|Turn on Settings > Providers > 'Count AI calls from bash commands', then in the 'Bug bash playground' workspace send 'probe', '[bash:many]' and '[bash:openai]'; after each, cross-check the token and cost numbers printed by the bash command against the Stats > Cost tab (per-model rows, components, session total, Last Request) and Analytics, and report every number that does not add up
+proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch and toggle it on and off, reloading after each change; after each state send 'probe' in the 'Bug bash playground' workspace and check that the proxy variables appear only while the switch is on, that the switch state persists, and that the Cost tab never loses or double-counts earlier spend
+proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[bash:background]' and watch the Cost tab for 40 seconds without reloading; then turn the switch off while the background command still runs; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off
+proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[bash:stream]', '[bash:bad-key]', '[bash:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted
+proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[bash:many]' in each workspace; check that each workspace's Cost tab and Analytics count only its own bash calls, that the two keys differ, and that archiving and restoring a workspace keeps its numbers
+proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title, description and help, turn it on, then send 'probe' in the 'Bug bash playground' workspace and open the Cost tab; report copy that is confusing or wrong, and content that is cut off, overlapping or unreachable
diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts
index 5a0eee48217..0ecb1ba6228 100644
--- a/tests/bugbash/e2e.config.ts
+++ b/tests/bugbash/e2e.config.ts
@@ -86,8 +86,47 @@ const mockNonBugs = realAi
       "the mock echoes your text, and after a retry it may echo [CONTINUE].",
     ];
 
+// BUGBASH_SCENARIO=bash-ai-proxy swaps mock AI for a loopback fake provider (startApp.ts,
+// fakeProvider.ts) so explorers can drive the bash AI proxy end to end.
+const scenario = process.env.BUGBASH_SCENARIO ?? "";
+
+// What the explorer must know about the bash AI proxy scenario instead of the mock-AI notes.
+const bashAiProxyContext = [
+  "The app is Xum, a desktop and browser app for running parallel AI coding agents.",
+  "It starts with one project, demo-app, and one workspace, 'Bug bash playground', in the left sidebar.",
+  "The feature under test is the bash AI proxy. When Settings > Providers > 'Bash commands' >",
+  "'Count AI calls from bash commands' is on (it is off by default), every bash command the agent",
+  "runs gets ANTHROPIC_BASE_URL, OPENAI_BASE_URL and a per-workspace 'xum-proxy-...' key that point",
+  "at Xum. Xum forwards those calls with the provider key from its settings and adds their tokens and",
+  "cost to that workspace's right-sidebar Stats > Cost tab (rows per model, session total) and to",
+  "Analytics (the bar-chart button). Turning the switch off removes the variables from new commands",
+  "and refuses calls from commands that still hold a key.",
+  "AI is a local fake provider, nothing is billed. Every chat message makes the agent run ONE fixed",
+  "bash command and then reply 'Done...'. The command prints the proxy variables (or <unset>) and the",
+  "HTTP answers. A keyword in your message picks the command: none = one Anthropic call;",
+  "'[bash:stream]' = one streamed Anthropic call; '[bash:openai]' = one OpenAI chat and one OpenAI",
+  "responses call; '[bash:many]' = five Anthropic calls; '[bash:background]' = a background command",
+  "that makes six calls, one every 5 seconds; '[bash:bad-key]' = a call with a wrong key (expect 401).",
+  "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts",
+  "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI",
+  "probe reports 1234 prompt tokens of which 100 are cached (1134 uncached) and 56 output tokens, on",
+  "gpt-6.1-sol. The chat",
+  "turns themselves also cost a little on claude-sonnet-5-5 or the selected model.",
+  "Terminals are turned off for this session, so an error when opening one is expected.",
+  "Do not sign in to any provider, MCP server or external service, and do not enter real secrets.",
+  "Project secrets (Settings > Secrets) with fake values are fine to test that a secret that names a",
+  "provider variable keeps its own value and that provider gets no proxy variables.",
+  "Known and not bugs: the per-model cost column rounds amounts under $0.01 to ~$0.00; this test",
+  "browser denies clipboard writes; workspace names must be lowercase branch names.",
+  "Known and tracked, do not report again: at phone width the right sidebar (Stats > Cost) cannot be",
+  "opened (#5767); Analytics response counts leave out bash proxy rows and label them agent 'unknown'",
+  "(#5766); Analytics shows timestamps as '1.8T', repeats y-axis ticks, says '1 responses' and",
+  "overlaps at 390px (#5768); at 390px the workspace footer overflows and a long bash Script block is",
+  "cut off (#5769). By design: the 'Workspace created' row follows the first message.",
+].join(" ");
+
 // What the local app cannot do, and the explorer's own blind spots (see `e2e guide bug-bash`).
-const context = [
+const mockOrRealContext = [
   "The app is Xum, a desktop and browser app for running parallel AI coding agents.",
   "It starts with one project, demo-app, and one workspace, 'Bug bash playground', in the left sidebar.",
   ...aiContext,
@@ -123,6 +162,8 @@ const context = [
   "after about one second; Ctrl+/ cycles to the next model; Fast mode is unavailable in this setup.",
 ].join(" ");
 
+const context = scenario === "bash-ai-proxy" ? bashAiProxyContext : mockOrRealContext;
+
 // Exploration steps need large per-step budgets (`e2e guide bug-bash`, step 1).
 // Only the active provider reads its key, so both can be set at once.
 const effort = explorerEffort();
@@ -141,6 +182,7 @@ const APP_ENV_VARS = [
   "BUGBASH_AI_RESOLVED",
   "BUGBASH_AI_REASON",
   "BUGBASH_APP_MODEL",
+  "BUGBASH_SCENARIO",
   "ANTHROPIC_API_KEY",
   "ANTHROPIC_BASE_URL",
   "OPENAI_API_KEY",
diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts
new file mode 100644
index 00000000000..a6ed34623fe
--- /dev/null
+++ b/tests/bugbash/fakeProvider.ts
@@ -0,0 +1,414 @@
+/**
+ * Fake Anthropic and OpenAI provider for the `bash-ai-proxy` bug-bash scenario
+ * (BUGBASH_SCENARIO=bash-ai-proxy, see startApp.ts).
+ *
+ * It plays two roles, both on loopback, so no model call leaves the machine and nothing is billed:
+ * 1. The app's chat model. Every user message makes the agent run ONE bash command, picked from
+ *    the fixed SCRIPTS table by a keyword in the message (default: "probe"). The explorer cannot
+ *    choose the command text, only which fixed script runs.
+ * 2. The upstream behind Xum's bash AI proxy. The scripts call the proxy with curl, and the proxy
+ *    forwards here with the key from the Xum provider settings. Probe answers carry fixed token
+ *    counts, so explorers can cross-check the Costs tab against the bash output.
+ *
+ * Every request is logged as one `[fake-provider]` line (no key values) to the app log.
+ */
+import * as http from "http";
+import type { AddressInfo } from "net";
+
+/** Token counts in every probe answer, so a skeptic can add them up. */
+export const PROBE_USAGE = { input: 1234, output: 56, cacheRead: 100 } as const;
+const PROBE_MARKER = "bugbash-proxy-probe";
+
+const ANTHROPIC_JSON = `'{"model":"claude-opus-5-5","max_tokens":5,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
+const ANTHROPIC_STREAM = `'{"model":"claude-opus-5-5","max_tokens":5,"stream":true,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
+const OPENAI_CHAT = `'{"model":"gpt-6.1-sol","stream":true,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
+const OPENAI_RESPONSES = `'{"model":"gpt-6.1-sol","input":"${PROBE_MARKER}"}'`;
+
+const SHOW_ENV = [
+  'echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-<unset>}"',
+  'echo "OPENAI_BASE_URL=${OPENAI_BASE_URL:-<unset>}"',
+  'echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:+${ANTHROPIC_API_KEY:0:14}...}"',
+].join("\n");
+const NEED_PROXY =
+  'if [ -z "$ANTHROPIC_BASE_URL" ]; then echo "no proxy variables: nothing to call"; exit 0; fi';
+const anthropicCall = (data: string, key = "$ANTHROPIC_API_KEY") =>
+  `curl -sS -w '\\nHTTP %{http_code}\\n' "$ANTHROPIC_BASE_URL/v1/messages" -H "x-api-key: ${key}" -H 'content-type: application/json' -d ${data}`;
+
+interface Script {
+  keyword: string;
+  displayName: string;
+  background: boolean;
+  script: string;
+}
+
+/** Fixed scripts. The first keyword found in the user's message wins; "probe" is the default. */
+const SCRIPTS: Script[] = [
+  {
+    keyword: "[bash:stream]",
+    displayName: "Streamed Anthropic call",
+    background: false,
+    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_STREAM)].join("\n"),
+  },
+  {
+    keyword: "[bash:openai]",
+    displayName: "OpenAI calls",
+    background: false,
+    script: [
+      SHOW_ENV,
+      'if [ -z "$OPENAI_BASE_URL" ]; then echo "no OpenAI proxy variables"; exit 0; fi',
+      `curl -sS -w '\\nHTTP %{http_code}\\n' "$OPENAI_BASE_URL/chat/completions" -H "authorization: Bearer $OPENAI_API_KEY" -H 'content-type: application/json' -d ${OPENAI_CHAT}`,
+      `curl -sS -w '\\nHTTP %{http_code}\\n' "$OPENAI_BASE_URL/responses" -H "authorization: Bearer $OPENAI_API_KEY" -H 'content-type: application/json' -d ${OPENAI_RESPONSES}`,
+    ].join("\n"),
+  },
+  {
+    keyword: "[bash:many]",
+    displayName: "Five Anthropic calls",
+    background: false,
+    script: [
+      SHOW_ENV,
+      NEED_PROXY,
+      `for i in 1 2 3 4 5; do ${anthropicCall(ANTHROPIC_JSON)}; done`,
+    ].join("\n"),
+  },
+  {
+    keyword: "[bash:background]",
+    displayName: "Background calls, one every 5 s",
+    background: true,
+    script: [
+      SHOW_ENV,
+      NEED_PROXY,
+      `for i in 1 2 3 4 5 6; do ${anthropicCall(ANTHROPIC_JSON)}; sleep 5; done`,
+    ].join("\n"),
+  },
+  {
+    keyword: "[bash:bad-key]",
+    displayName: "Call with a wrong key",
+    background: false,
+    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_JSON, "xum-proxy-wrong")].join("\n"),
+  },
+  {
+    keyword: "probe",
+    displayName: "Anthropic call through the proxy",
+    background: false,
+    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_JSON)].join("\n"),
+  },
+];
+
+type Json = Record<string, unknown>;
+
+function pickScript(userText: string): Script {
+  const text = userText.toLowerCase();
+  return SCRIPTS.find((s) => s.keyword !== "probe" && text.includes(s.keyword)) ?? SCRIPTS.at(-1)!;
+}
+
+function textOf(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .map((part) =>
+      part && typeof part === "object" && (part as Json).type === "text"
+        ? String((part as Json).text)
+        : ""
+    )
+    .join(" ");
+}
+
+/**
+ * A probe is the exact one-message request a fixed script sends. Chat turns also contain the
+ * marker (the bash tool input quotes the script), so a substring match is not enough.
+ */
+function isProbe(body: Json): boolean {
+  if (body.input === PROBE_MARKER) return true;
+  const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : [];
+  return messages.length === 1 && messages[0]?.content === PROBE_MARKER;
+}
+
+function sse(res: http.ServerResponse, events: [string | null, Json | string][]): void {
+  res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
+  for (const [type, data] of events) {
+    const payload =
+      typeof data === "string" ? data : JSON.stringify(type ? { type, ...data } : data);
+    res.write(`${type ? `event: ${type}\n` : ""}data: ${payload}\n\n`);
+  }
+  res.end();
+}
+
+function json(res: http.ServerResponse, status: number, body: Json): void {
+  res.writeHead(status, { "content-type": "application/json" });
+  res.end(JSON.stringify(body));
+}
+
+let seq = 0;
+function messageStart(model: string, usage: Json): [string, Json] {
+  return [
+    "message_start",
+    {
+      message: {
+        id: `msg_fake_${++seq}`,
+        type: "message",
+        role: "assistant",
+        model,
+        content: [],
+        stop_reason: null,
+        stop_sequence: null,
+        usage,
+      },
+    },
+  ];
+}
+
+function anthropicChat(res: http.ServerResponse, body: Json): string {
+  const model = typeof body.model === "string" ? body.model : "claude-sonnet-5-5";
+  const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : [];
+  const tools = Array.isArray(body.tools) ? (body.tools as Json[]).map((t) => t.name) : [];
+  const last = messages.at(-1);
+  const answeredTool =
+    Array.isArray(last?.content) && (last.content as Json[]).some((c) => c?.type === "tool_result");
+  const usage = { input_tokens: 500, output_tokens: 1 };
+
+  if (!tools.includes("bash") || answeredTool) {
+    const text = answeredTool ? "Done. The bash command finished; see its output above." : "ok";
+    if (body.stream !== true) {
+      json(res, 200, {
+        id: `msg_fake_${++seq}`,
+        type: "message",
+        role: "assistant",
+        model,
+        content: [{ type: "text", text }],
+        stop_reason: "end_turn",
+        stop_sequence: null,
+        usage: { input_tokens: 20, output_tokens: 2 },
+      });
+      return answeredTool ? "chat-final" : "background-json";
+    }
+    sse(res, [
+      messageStart(model, usage),
+      ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
+      ["content_block_delta", { index: 0, delta: { type: "text_delta", text } }],
+      ["content_block_stop", { index: 0 }],
+      [
+        "message_delta",
+        { delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 12 } },
+      ],
+      ["message_stop", {}],
+    ]);
+    return answeredTool ? "chat-final" : "background-sse";
+  }
+
+  const userText = textOf([...messages].reverse().find((m) => m.role === "user")?.content);
+  const script = pickScript(userText);
+  const input = JSON.stringify({
+    script: script.script,
+    timeout_secs: script.background ? 120 : 30,
+    display_name: script.displayName,
+    run_in_background: script.background,
+  });
+  sse(res, [
+    messageStart(model, usage),
+    ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
+    [
+      "content_block_delta",
+      { index: 0, delta: { type: "text_delta", text: `Running: ${script.displayName}.` } },
+    ],
+    ["content_block_stop", { index: 0 }],
+    [
+      "content_block_start",
+      {
+        index: 1,
+        content_block: { type: "tool_use", id: `toolu_fake_${seq}`, name: "bash", input: {} },
+      },
+    ],
+    ["content_block_delta", { index: 1, delta: { type: "input_json_delta", partial_json: input } }],
+    ["content_block_stop", { index: 1 }],
+    [
+      "message_delta",
+      { delta: { stop_reason: "tool_use", stop_sequence: null }, usage: { output_tokens: 40 } },
+    ],
+    ["message_stop", {}],
+  ]);
+  return `chat-tool:${script.keyword}`;
+}
+
+function anthropicProbe(res: http.ServerResponse, body: Json): string {
+  const model = typeof body.model === "string" ? body.model : "claude-opus-5-5";
+  if (body.stream === true) {
+    sse(res, [
+      messageStart(model, {
+        input_tokens: PROBE_USAGE.input,
+        cache_read_input_tokens: PROBE_USAGE.cacheRead,
+        output_tokens: 1,
+      }),
+      ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
+      ["content_block_delta", { index: 0, delta: { type: "text_delta", text: "pong" } }],
+      ["content_block_stop", { index: 0 }],
+      [
+        "message_delta",
+        {
+          delta: { stop_reason: "end_turn", stop_sequence: null },
+          usage: { output_tokens: PROBE_USAGE.output },
+        },
+      ],
+      ["message_stop", {}],
+    ]);
+    return "probe-anthropic-sse";
+  }
+  json(res, 200, {
+    id: `msg_probe_${++seq}`,
+    type: "message",
+    role: "assistant",
+    model,
+    content: [{ type: "text", text: "pong" }],
+    stop_reason: "end_turn",
+    stop_sequence: null,
+    usage: {
+      input_tokens: PROBE_USAGE.input,
+      output_tokens: PROBE_USAGE.output,
+      cache_read_input_tokens: PROBE_USAGE.cacheRead,
+    },
+  });
+  return "probe-anthropic-json";
+}
+
+function openAiUsage(): Json {
+  return {
+    prompt_tokens: PROBE_USAGE.input,
+    completion_tokens: PROBE_USAGE.output,
+    prompt_tokens_details: { cached_tokens: PROBE_USAGE.cacheRead },
+  };
+}
+
+function openAiChat(res: http.ServerResponse, body: Json): string {
+  const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol";
+  const includeUsage = (body.stream_options as Json | undefined)?.include_usage === true;
+  if (body.stream !== true) {
+    json(res, 200, {
+      id: "chatcmpl_fake",
+      object: "chat.completion",
+      model,
+      choices: [
+        { index: 0, message: { role: "assistant", content: "pong" }, finish_reason: "stop" },
+      ],
+      usage: openAiUsage(),
+    });
+    return "probe-openai-chat-json";
+  }
+  const chunk = (extra: Json) =>
+    [null, { id: "chatcmpl_fake", object: "chat.completion.chunk", model, ...extra }] as [
+      null,
+      Json,
+    ];
+  sse(res, [
+    chunk({
+      choices: [{ index: 0, delta: { role: "assistant", content: "pong" }, finish_reason: null }],
+    }),
+    chunk({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }),
+    ...(includeUsage ? [chunk({ choices: [], usage: openAiUsage() })] : []),
+    [null, "[DONE]"],
+  ]);
+  return `probe-openai-chat-sse(usage=${includeUsage})`;
+}
+
+/** The app's own OpenAI calls (not probes): a short streamed or JSON "ok" with tiny usage. */
+function openAiAppResponses(res: http.ServerResponse, body: Json): string {
+  const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol";
+  const usage = {
+    input_tokens: 20,
+    input_tokens_details: { cached_tokens: 0 },
+    output_tokens: 2,
+    output_tokens_details: { reasoning_tokens: 0 },
+  };
+  const item = { type: "message", id: "msg_app", role: "assistant" };
+  const done = {
+    ...item,
+    status: "completed",
+    content: [{ type: "output_text", text: "ok", annotations: [] }],
+  };
+  const response = (status: string, extra: Json) => ({
+    id: `resp_app_${++seq}`,
+    object: "response",
+    created_at: Math.floor(Date.now() / 1000),
+    model,
+    status,
+    ...extra,
+  });
+  if (body.stream !== true) {
+    json(res, 200, response("completed", { output: [done], usage }));
+    return "app-openai-responses-json";
+  }
+  sse(res, [
+    ["response.created", { response: response("in_progress", { output: [] }) }],
+    [
+      "response.output_item.added",
+      { output_index: 0, item: { ...item, status: "in_progress", content: [] } },
+    ],
+    [
+      "response.output_text.delta",
+      { item_id: "msg_app", output_index: 0, content_index: 0, delta: "ok" },
+    ],
+    ["response.output_item.done", { output_index: 0, item: done }],
+    ["response.completed", { response: response("completed", { output: [done], usage }) }],
+  ]);
+  return "app-openai-responses-sse";
+}
+
+function openAiResponses(res: http.ServerResponse, body: Json): string {
+  const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol";
+  if (!isProbe(body)) return openAiAppResponses(res, body);
+  json(res, 200, {
+    id: "resp_fake",
+    object: "response",
+    model,
+    status: "completed",
+    output: [
+      { type: "message", role: "assistant", content: [{ type: "output_text", text: "pong" }] },
+    ],
+    usage: {
+      input_tokens: PROBE_USAGE.input,
+      input_tokens_details: { cached_tokens: PROBE_USAGE.cacheRead },
+      output_tokens: PROBE_USAGE.output,
+      output_tokens_details: { reasoning_tokens: 0 },
+    },
+  });
+  return "probe-openai-responses";
+}
+
+/** Starts the fake on a free loopback port and returns its origin (no trailing slash). */
+export async function startFakeProvider(): Promise<{ origin: string; close: () => void }> {
+  const server = http.createServer((req, res) => {
+    const chunks: Buffer[] = [];
+    req.on("data", (chunk: Buffer) => chunks.push(chunk));
+    req.on("end", () => {
+      let body: Json = {};
+      try {
+        const raw = Buffer.concat(chunks).toString("utf8");
+        body = raw ? (JSON.parse(raw) as Json) : {};
+      } catch {
+        // Not JSON: the 404 below answers it.
+      }
+      const route = (req.url ?? "").split("?")[0];
+      let kind: string;
+      if (req.method === "POST" && route.endsWith("/messages")) {
+        kind = isProbe(body) ? anthropicProbe(res, body) : anthropicChat(res, body);
+      } else if (req.method === "POST" && route.endsWith("/chat/completions")) {
+        kind = openAiChat(res, body);
+      } else if (req.method === "POST" && route.endsWith("/responses")) {
+        kind = openAiResponses(res, body);
+      } else if (req.method === "GET" && route.endsWith("/models")) {
+        kind = "models";
+        json(res, 200, { data: [{ id: "claude-opus-5-5", type: "model" }] });
+      } else {
+        kind = "404";
+        json(res, 404, { type: "error", error: { type: "not_found_error", message: "fake" } });
+      }
+      const key = String(req.headers["x-api-key"] ?? req.headers.authorization ?? "");
+      console.log(
+        `[fake-provider] ${req.method} ${route} kind=${kind} model=${String(body.model)} stream=${String(body.stream === true)} key=${key.includes("xum-proxy-") ? "PROXY-KEY-LEAKED" : key ? "xum-settings-key" : "none"}`
+      );
+    });
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  return {
+    origin: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
+    close: () => server.close(),
+  };
+}
diff --git a/tests/bugbash/startApp.ts b/tests/bugbash/startApp.ts
index d10a44959ab..340b2abb053 100644
--- a/tests/bugbash/startApp.ts
+++ b/tests/bugbash/startApp.ts
@@ -43,6 +43,7 @@ import * as net from "net";
 import * as os from "os";
 import * as path from "path";
 import { type AiMode, AiModeError, resolveAiMode } from "./aiMode";
+import { startFakeProvider } from "./fakeProvider";
 import { seedMcpChat, writeMcpConfig } from "./mcpapps/seed";
 
 const REPO_ROOT = path.resolve(import.meta.dir, "../..");
@@ -220,7 +221,8 @@ async function seed(
   projectPath: string,
   xumRoot: string,
   child: ChildProcess,
-  ai: AiMode
+  ai: AiMode,
+  fakeProviderOrigin: string | undefined
 ): Promise<string> {
   await api(base, "splashScreens/markSplashScreenViewed", { splashId: "onboarding-wizard-v1" });
   await api(base, "experiments/setOverride", { experimentId: "artifacts", enabled: true });
@@ -239,7 +241,9 @@ async function seed(
     await api(base, "config/updateModelPreferences", { defaultModel: ai.model });
   } else {
     // The composer refuses to send without a configured provider. Mock AI never calls it, and
-    // the dead loopback port keeps any stray background call from leaving the machine.
+    // the dead loopback port keeps any stray background call from leaving the machine. The
+    // bash-ai-proxy scenario points both providers at the loopback fake instead.
+    const providerBase = fakeProviderOrigin ?? "http://127.0.0.1:9";
     await api(base, "providers/setProviderConfig", {
       provider: "anthropic",
       keyPath: ["apiKey"],
@@ -248,8 +252,20 @@ async function seed(
     await api(base, "providers/setProviderConfig", {
       provider: "anthropic",
       keyPath: ["baseUrl"],
-      value: "http://127.0.0.1:9/v1",
+      value: `${providerBase}/v1`,
     });
+    if (fakeProviderOrigin !== undefined) {
+      await api(base, "providers/setProviderConfig", {
+        provider: "openai",
+        keyPath: ["apiKey"],
+        value: "sk-openai-bugbash-fake",
+      });
+      await api(base, "providers/setProviderConfig", {
+        provider: "openai",
+        keyPath: ["baseUrl"],
+        value: `${providerBase}/v1`,
+      });
+    }
   }
   await api(base, "projects/create", { projectPath });
   await api(base, "projects/setTrust", { projectPath, trusted: true });
@@ -314,6 +330,20 @@ async function main(): Promise<void> {
   // e2e writes this line to the target's command log.
   console.log(`[bugbash startApp] app AI: ${ai.mode} (${ai.reason})`);
 
+  // BUGBASH_SCENARIO=bash-ai-proxy: instead of the mock, a loopback fake provider (fakeProvider.ts)
+  // is the chat model and the upstream of Xum's bash AI proxy, so the agent runs real bash
+  // commands whose AI calls go through the proxy. The fake only ever asks for its own fixed
+  // scripts, so agent tools stay on; nothing leaves the machine. It replaces the mock, so the
+  // explorer context (e2e.config.ts) must not describe a real model.
+  const scenario = process.env.BUGBASH_SCENARIO ?? "";
+  if (scenario !== "" && scenario !== "bash-ai-proxy") {
+    fail(`BUGBASH_SCENARIO must be empty or bash-ai-proxy, got "${scenario}"`);
+  }
+  if (scenario !== "" && ai.mode !== "mock") {
+    fail("BUGBASH_SCENARIO=bash-ai-proxy replaces the mock: run it with BUGBASH_AI=mock");
+  }
+  const fakeProvider = scenario === "bash-ai-proxy" ? await startFakeProvider() : undefined;
+
   const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-"));
   const xumRoot = path.join(tempRoot, "xum");
   const home = path.join(tempRoot, "home");
@@ -326,13 +356,17 @@ async function main(): Promise<void> {
     // Real mode: a real model answers in the app on this host. It gets no tools, and the explorer
     // gets no terminal or project init hooks, so injected text in a reply that the explorer
     // follows cannot run host commands through them (see the header).
-    ...(ai.mode === "mock"
-      ? { XUM_MOCK_AI: "1" }
-      : {
-          XUM_DISABLE_AGENT_TOOLS: "1",
-          XUM_DISABLE_TERMINALS: "1",
-          XUM_DISABLE_PROJECT_AUTOMATION: "1",
-        }),
+    ...(fakeProvider
+      ? // The explorer gets no terminal. Project automation stays on: the kill switch also
+        // withholds the proxy variables from bash, and the seeded demo repo has no hooks.
+        { XUM_DISABLE_TERMINALS: "1" }
+      : ai.mode === "mock"
+        ? { XUM_MOCK_AI: "1" }
+        : {
+            XUM_DISABLE_AGENT_TOOLS: "1",
+            XUM_DISABLE_TERMINALS: "1",
+            XUM_DISABLE_PROJECT_AUTOMATION: "1",
+          }),
     // Bug-bash clicks are not product usage.
     XUM_DISABLE_TELEMETRY: "1",
     // Git inside the app must not read or write the user's real ~/.gitconfig.
@@ -361,7 +395,14 @@ async function main(): Promise<void> {
     let workspaceId: string;
     try {
       await waitForHealth(`http://127.0.0.1:${seedPort}`, current);
-      workspaceId = await seed(`http://127.0.0.1:${seedPort}`, projectPath, xumRoot, current, ai);
+      workspaceId = await seed(
+        `http://127.0.0.1:${seedPort}`,
+        projectPath,
+        xumRoot,
+        current,
+        ai,
+        fakeProvider?.origin
+      );
     } finally {
       current.kill("SIGTERM");
       await waitForExit(current);
@@ -373,6 +414,7 @@ async function main(): Promise<void> {
       code = await waitForExit(current);
     }
   } finally {
+    fakeProvider?.close();
     fs.rmSync(tempRoot, { recursive: true, force: true });
   }
   process.exit(code ?? 0);

From 3bd2aa6ce1d91c59e2d2e8a2505c918ec24ad69d Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 18:28:46 +0000
Subject: [PATCH 2/6] bug-bash: bash-ai-proxy scenario keeps agent tools off;
 the fake model calls the proxy itself

---
 tests/bugbash/charters.bash-ai-proxy.txt |  17 +-
 tests/bugbash/e2e.config.ts              |  31 ++-
 tests/bugbash/fakeProvider.ts            | 304 ++++++++++++++---------
 tests/bugbash/startApp.ts                |  18 +-
 4 files changed, 214 insertions(+), 156 deletions(-)

diff --git a/tests/bugbash/charters.bash-ai-proxy.txt b/tests/bugbash/charters.bash-ai-proxy.txt
index 3c15a06d2ff..602f54765a3 100644
--- a/tests/bugbash/charters.bash-ai-proxy.txt
+++ b/tests/bugbash/charters.bash-ai-proxy.txt
@@ -1,9 +1,10 @@
-# Charters for the bash AI proxy branch. Run with BUGBASH_AI=mock BUGBASH_SCENARIO=bash-ai-proxy and
+# Charters for the bash AI proxy. Run with BUGBASH_AI=mock BUGBASH_SCENARIO=bash-ai-proxy and
 # BUGBASH_ARGS="--charters tests/bugbash/charters.bash-ai-proxy.txt" (see e2e.config.ts).
-# Keep explorers out of Terminal tabs: they run real shell commands on the host.
-proxy-costs|web|skeptic|Turn on Settings > Providers > 'Count AI calls from bash commands', then in the 'Bug bash playground' workspace send 'probe', '[bash:many]' and '[bash:openai]'; after each, cross-check the token and cost numbers printed by the bash command against the Stats > Cost tab (per-model rows, components, session total, Last Request) and Analytics, and report every number that does not add up
-proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch and toggle it on and off, reloading after each change; after each state send 'probe' in the 'Bug bash playground' workspace and check that the proxy variables appear only while the switch is on, that the switch state persists, and that the Cost tab never loses or double-counts earlier spend
-proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[bash:background]' and watch the Cost tab for 40 seconds without reloading; then turn the switch off while the background command still runs; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off
-proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[bash:stream]', '[bash:bad-key]', '[bash:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted
-proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[bash:many]' in each workspace; check that each workspace's Cost tab and Analytics count only its own bash calls, that the two keys differ, and that archiving and restoring a workspace keeps its numbers
-proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title, description and help, turn it on, then send 'probe' in the 'Bug bash playground' workspace and open the Cost tab; report copy that is confusing or wrong, and content that is cut off, overlapping or unreachable
+# The scenario has no agent tools and no terminals: the fake model calls the proxy itself.
+proxy-costs|web|skeptic|Turn on Settings > Providers > 'Count AI calls from bash commands', then in the 'Bug bash playground' workspace send 'probe', '[proxy:many]' and '[proxy:openai]'; after each, cross-check the documented probe token counts against the Stats > Cost tab Session rows (per-model rows, components, session total) and Analytics; check that Last Request stays the chat request; report every number that does not add up
+proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch and toggle it on and off, reloading after each change; after each state send 'probe' in the 'Bug bash playground' workspace and check that calls succeed (HTTP 200) only while the switch is on and are refused (HTTP 503) while it is off, that the switch state persists, and that the Cost tab never loses or double-counts earlier spend
+proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[proxy:background]' and watch the Cost tab for about 20 seconds without reloading (it should grow with each call); then turn the switch off while the calls still run and watch for 30 more seconds; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off
+proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[proxy:stream]', '[proxy:bad-key]', '[proxy:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted
+proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[proxy:many]' in each workspace; check that each workspace's Cost tab and Analytics count only its own proxied calls, and that archiving and restoring a workspace keeps its numbers
+proxy-trust|web|state|Turn on the bash AI proxy switch and send 'probe' in the 'Bug bash playground' workspace (expect HTTP 200); then send '[proxy:background]' and, while it runs, revoke trust for demo-app in Settings > Security; calls after the revoke must fail (HTTP 403) and must not be counted in the Stats > Cost tab; trust the project again and send 'probe' (HTTP 200, counted once); report anything that leaks, sticks or double-counts
+proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title and description and report text that is confusing, wrong, cut off or overlapping; turn it on, send 'probe' in the 'Bug bash playground' workspace and check the reply and Analytics (the Cost tab is known to be unreachable on phones, do not try it)
diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts
index 0ecb1ba6228..4d9976ec7c3 100644
--- a/tests/bugbash/e2e.config.ts
+++ b/tests/bugbash/e2e.config.ts
@@ -95,27 +95,26 @@ const bashAiProxyContext = [
   "The app is Xum, a desktop and browser app for running parallel AI coding agents.",
   "It starts with one project, demo-app, and one workspace, 'Bug bash playground', in the left sidebar.",
   "The feature under test is the bash AI proxy. When Settings > Providers > 'Bash commands' >",
-  "'Count AI calls from bash commands' is on (it is off by default), every bash command the agent",
-  "runs gets ANTHROPIC_BASE_URL, OPENAI_BASE_URL and a per-workspace 'xum-proxy-...' key that point",
-  "at Xum. Xum forwards those calls with the provider key from its settings and adds their tokens and",
-  "cost to that workspace's right-sidebar Stats > Cost tab (rows per model, session total) and to",
-  "Analytics (the bar-chart button). Turning the switch off removes the variables from new commands",
-  "and refuses calls from commands that still hold a key.",
-  "AI is a local fake provider, nothing is billed. Every chat message makes the agent run ONE fixed",
-  "bash command and then reply 'Done...'. The command prints the proxy variables (or <unset>) and the",
-  "HTTP answers. A keyword in your message picks the command: none = one Anthropic call;",
-  "'[bash:stream]' = one streamed Anthropic call; '[bash:openai]' = one OpenAI chat and one OpenAI",
-  "responses call; '[bash:many]' = five Anthropic calls; '[bash:background]' = a background command",
-  "that makes six calls, one every 5 seconds; '[bash:bad-key]' = a call with a wrong key (expect 401).",
+  "'Count AI calls from bash commands' is on (it is off by default), bash commands get a",
+  "per-workspace 'xum-proxy-...' key and endpoints that point at Xum. Xum forwards those calls with",
+  "the provider key from its settings and adds their tokens and cost to that workspace's",
+  "right-sidebar Stats > Cost tab (Session rows per model and total) and to Analytics (the bar-chart",
+  "button). The Cost tab's Last Request stays the chat request: proxied calls never replace it.",
+  "Turning the switch off makes the proxy refuse calls (HTTP 503); revoking the project's trust",
+  "makes it refuse that workspace's calls (HTTP 403).",
+  "AI is a local fake provider, nothing is billed, and agent tools are off, so the agent runs no",
+  "command. Instead the fake model itself calls the proxy with the key Xum gives that workspace's",
+  "bash commands, and replies 'Done: ...' with one HTTP result per call. A keyword in your message",
+  "picks the calls: none = one Anthropic call; '[proxy:stream]' = one streamed Anthropic call;",
+  "'[proxy:openai]' = one OpenAI chat and one OpenAI responses call; '[proxy:many]' = five Anthropic",
+  "calls; '[proxy:background]' = twelve Anthropic calls, one every 5 seconds, after the reply (about",
+  "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401).",
   "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts",
   "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI",
   "probe reports 1234 prompt tokens of which 100 are cached (1134 uncached) and 56 output tokens, on",
-  "gpt-6.1-sol. The chat",
-  "turns themselves also cost a little on claude-sonnet-5-5 or the selected model.",
+  "gpt-6.1-sol. The chat turns themselves also cost a little on claude-sonnet-5-5 or the selected model.",
   "Terminals are turned off for this session, so an error when opening one is expected.",
   "Do not sign in to any provider, MCP server or external service, and do not enter real secrets.",
-  "Project secrets (Settings > Secrets) with fake values are fine to test that a secret that names a",
-  "provider variable keeps its own value and that provider gets no proxy variables.",
   "Known and not bugs: the per-model cost column rounds amounts under $0.01 to ~$0.00; this test",
   "browser denies clipboard writes; workspace names must be lowercase branch names.",
   "Known and tracked, do not report again: at phone width the right sidebar (Stats > Cost) cannot be",
diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts
index a6ed34623fe..94fadab0fd4 100644
--- a/tests/bugbash/fakeProvider.ts
+++ b/tests/bugbash/fakeProvider.ts
@@ -3,102 +3,157 @@
  * (BUGBASH_SCENARIO=bash-ai-proxy, see startApp.ts).
  *
  * It plays two roles, both on loopback, so no model call leaves the machine and nothing is billed:
- * 1. The app's chat model. Every user message makes the agent run ONE bash command, picked from
- *    the fixed SCRIPTS table by a keyword in the message (default: "probe"). The explorer cannot
- *    choose the command text, only which fixed script runs.
- * 2. The upstream behind Xum's bash AI proxy. The scripts call the proxy with curl, and the proxy
- *    forwards here with the key from the Xum provider settings. Probe answers carry fixed token
- *    counts, so explorers can cross-check the Costs tab against the bash output.
+ * 1. The app's chat model. Every agent turn makes the fake itself call Xum's bash AI proxy, with
+ *    the key Xum gives that workspace's bash commands, picked from the fixed PLANS table by a
+ *    keyword in the message (default: "probe"). The reply lists the HTTP answers.
+ *    Agent tools stay off (AGENTS.md: no agent tools or terminals in bug bashes until the app
+ *    runs in a sandbox), so the agent runs no command. The proxy cannot tell these harness-owned
+ *    calls from a bash command's; the bash env injection itself is covered by unit tests.
+ * 2. The upstream behind the proxy. The proxy forwards here with the key from the Xum provider
+ *    settings. Probe answers carry fixed token counts, so explorers can cross-check the Costs tab.
  *
  * Every request is logged as one `[fake-provider]` line (no key values) to the app log.
  */
+import * as fs from "fs";
 import * as http from "http";
 import type { AddressInfo } from "net";
+import * as path from "path";
+
+import { deriveProxyKey } from "../../src/node/services/bashAiProxy/stableIdentity";
+import { BASH_AI_PROXY_STATE_FILE } from "../../src/node/services/bashAiProxy/proxyState";
 
 /** Token counts in every probe answer, so a skeptic can add them up. */
 export const PROBE_USAGE = { input: 1234, output: 56, cacheRead: 100 } as const;
 const PROBE_MARKER = "bugbash-proxy-probe";
 
-const ANTHROPIC_JSON = `'{"model":"claude-opus-5-5","max_tokens":5,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
-const ANTHROPIC_STREAM = `'{"model":"claude-opus-5-5","max_tokens":5,"stream":true,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
-const OPENAI_CHAT = `'{"model":"gpt-6.1-sol","stream":true,"messages":[{"role":"user","content":"${PROBE_MARKER}"}]}'`;
-const OPENAI_RESPONSES = `'{"model":"gpt-6.1-sol","input":"${PROBE_MARKER}"}'`;
-
-const SHOW_ENV = [
-  'echo "ANTHROPIC_BASE_URL=${ANTHROPIC_BASE_URL:-<unset>}"',
-  'echo "OPENAI_BASE_URL=${OPENAI_BASE_URL:-<unset>}"',
-  'echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:+${ANTHROPIC_API_KEY:0:14}...}"',
-].join("\n");
-const NEED_PROXY =
-  'if [ -z "$ANTHROPIC_BASE_URL" ]; then echo "no proxy variables: nothing to call"; exit 0; fi';
-const anthropicCall = (data: string, key = "$ANTHROPIC_API_KEY") =>
-  `curl -sS -w '\\nHTTP %{http_code}\\n' "$ANTHROPIC_BASE_URL/v1/messages" -H "x-api-key: ${key}" -H 'content-type: application/json' -d ${data}`;
+type Call = "anthropic" | "anthropic-stream" | "openai-chat" | "openai-responses" | "bad-key";
 
-interface Script {
+interface Plan {
   keyword: string;
   displayName: string;
+  calls: Call[];
+  /** Calls run one every 5 s after the reply, like a background command. */
   background: boolean;
-  script: string;
 }
 
-/** Fixed scripts. The first keyword found in the user's message wins; "probe" is the default. */
-const SCRIPTS: Script[] = [
+/** Fixed plans. The first keyword found in the user's message wins; "probe" is the default. */
+const PLANS: Plan[] = [
   {
-    keyword: "[bash:stream]",
-    displayName: "Streamed Anthropic call",
+    keyword: "[proxy:stream]",
+    displayName: "one streamed Anthropic call",
+    calls: ["anthropic-stream"],
     background: false,
-    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_STREAM)].join("\n"),
   },
   {
-    keyword: "[bash:openai]",
-    displayName: "OpenAI calls",
+    keyword: "[proxy:openai]",
+    displayName: "one OpenAI chat and one OpenAI responses call",
+    calls: ["openai-chat", "openai-responses"],
     background: false,
-    script: [
-      SHOW_ENV,
-      'if [ -z "$OPENAI_BASE_URL" ]; then echo "no OpenAI proxy variables"; exit 0; fi',
-      `curl -sS -w '\\nHTTP %{http_code}\\n' "$OPENAI_BASE_URL/chat/completions" -H "authorization: Bearer $OPENAI_API_KEY" -H 'content-type: application/json' -d ${OPENAI_CHAT}`,
-      `curl -sS -w '\\nHTTP %{http_code}\\n' "$OPENAI_BASE_URL/responses" -H "authorization: Bearer $OPENAI_API_KEY" -H 'content-type: application/json' -d ${OPENAI_RESPONSES}`,
-    ].join("\n"),
   },
   {
-    keyword: "[bash:many]",
-    displayName: "Five Anthropic calls",
+    keyword: "[proxy:many]",
+    displayName: "five Anthropic calls",
+    calls: Array<Call>(5).fill("anthropic"),
     background: false,
-    script: [
-      SHOW_ENV,
-      NEED_PROXY,
-      `for i in 1 2 3 4 5; do ${anthropicCall(ANTHROPIC_JSON)}; done`,
-    ].join("\n"),
   },
   {
-    keyword: "[bash:background]",
-    displayName: "Background calls, one every 5 s",
+    keyword: "[proxy:background]",
+    displayName: "twelve background Anthropic calls, one every 5 s",
+    calls: Array<Call>(12).fill("anthropic"),
     background: true,
-    script: [
-      SHOW_ENV,
-      NEED_PROXY,
-      `for i in 1 2 3 4 5 6; do ${anthropicCall(ANTHROPIC_JSON)}; sleep 5; done`,
-    ].join("\n"),
-  },
-  {
-    keyword: "[bash:bad-key]",
-    displayName: "Call with a wrong key",
-    background: false,
-    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_JSON, "xum-proxy-wrong")].join("\n"),
   },
   {
-    keyword: "probe",
-    displayName: "Anthropic call through the proxy",
+    keyword: "[proxy:bad-key]",
+    displayName: "one Anthropic call with a wrong key",
+    calls: ["bad-key"],
     background: false,
-    script: [SHOW_ENV, NEED_PROXY, anthropicCall(ANTHROPIC_JSON)].join("\n"),
   },
+  { keyword: "probe", displayName: "one Anthropic call", calls: ["anthropic"], background: false },
 ];
 
 type Json = Record<string, unknown>;
 
-function pickScript(userText: string): Script {
+function pickPlan(userText: string): Plan {
   const text = userText.toLowerCase();
-  return SCRIPTS.find((s) => s.keyword !== "probe" && text.includes(s.keyword)) ?? SCRIPTS.at(-1)!;
+  return PLANS.find((p) => p.keyword !== "probe" && text.includes(p.keyword)) ?? PLANS.at(-1)!;
+}
+
+/** The workspace whose agent sent this turn: its system prompt names the worktree path. */
+function workspaceIdFor(xumRoot: string, system: string): string | undefined {
+  const match = /You are in a git worktree at (\S+)/.exec(system);
+  if (!match) return undefined;
+  const config = JSON.parse(fs.readFileSync(path.join(xumRoot, "config.json"), "utf8")) as {
+    projects?: [string, { workspaces?: { path?: string; id?: string }[] }][];
+  };
+  for (const [, project] of config.projects ?? []) {
+    const hit = project.workspaces?.find((w) => w.path === match[1]);
+    if (hit?.id) return hit.id;
+  }
+  return undefined;
+}
+
+const PROBE_BODIES: Record<Exclude<Call, "bad-key">, { route: string; body: Json }> = {
+  anthropic: {
+    route: "/anthropic/v1/messages",
+    body: {
+      model: "claude-opus-5-5",
+      max_tokens: 5,
+      messages: [{ role: "user", content: PROBE_MARKER }],
+    },
+  },
+  "anthropic-stream": {
+    route: "/anthropic/v1/messages",
+    body: {
+      model: "claude-opus-5-5",
+      max_tokens: 5,
+      stream: true,
+      messages: [{ role: "user", content: PROBE_MARKER }],
+    },
+  },
+  "openai-chat": {
+    route: "/openai/v1/chat/completions",
+    body: {
+      model: "gpt-6.1-sol",
+      stream: true,
+      messages: [{ role: "user", content: PROBE_MARKER }],
+    },
+  },
+  "openai-responses": {
+    route: "/openai/v1/responses",
+    body: { model: "gpt-6.1-sol", input: PROBE_MARKER },
+  },
+};
+
+/** One call through Xum's proxy, as a bash command with the injected variables would make it. */
+async function proxyCall(xumRoot: string, workspaceId: string, call: Call): Promise<string> {
+  let state: { secret?: string; port?: number };
+  try {
+    state = JSON.parse(
+      fs.readFileSync(path.join(xumRoot, BASH_AI_PROXY_STATE_FILE), "utf8")
+    ) as typeof state;
+  } catch {
+    return "no proxy: the switch was never turned on";
+  }
+  if (typeof state.port !== "number" || typeof state.secret !== "string") {
+    return "no proxy: the switch was never turned on";
+  }
+  const key = call === "bad-key" ? "xum-proxy-wrong" : deriveProxyKey(state.secret, workspaceId);
+  const probe = PROBE_BODIES[call === "bad-key" ? "anthropic" : call];
+  const headers: Record<string, string> = { "content-type": "application/json" };
+  if (probe.route.startsWith("/openai")) headers.authorization = `Bearer ${key}`;
+  else headers["x-api-key"] = key;
+  try {
+    const res = await fetch(`http://127.0.0.1:${state.port}${probe.route}`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(probe.body),
+    });
+    const text = await res.text();
+    const error = /"message":"([^"]*)"/.exec(text)?.[1];
+    return `HTTP ${res.status}${res.ok ? "" : ` ${error ?? text.slice(0, 120)}`}`;
+  } catch (error) {
+    return `connection failed: ${error instanceof Error ? error.message : String(error)}`;
+  }
 }
 
 function textOf(content: unknown): string {
@@ -157,76 +212,71 @@ function messageStart(model: string, usage: Json): [string, Json] {
   ];
 }
 
-function anthropicChat(res: http.ServerResponse, body: Json): string {
+function chatText(res: http.ServerResponse, body: Json, text: string): void {
   const model = typeof body.model === "string" ? body.model : "claude-sonnet-5-5";
-  const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : [];
-  const tools = Array.isArray(body.tools) ? (body.tools as Json[]).map((t) => t.name) : [];
-  const last = messages.at(-1);
-  const answeredTool =
-    Array.isArray(last?.content) && (last.content as Json[]).some((c) => c?.type === "tool_result");
-  const usage = { input_tokens: 500, output_tokens: 1 };
-
-  if (!tools.includes("bash") || answeredTool) {
-    const text = answeredTool ? "Done. The bash command finished; see its output above." : "ok";
-    if (body.stream !== true) {
-      json(res, 200, {
-        id: `msg_fake_${++seq}`,
-        type: "message",
-        role: "assistant",
-        model,
-        content: [{ type: "text", text }],
-        stop_reason: "end_turn",
-        stop_sequence: null,
-        usage: { input_tokens: 20, output_tokens: 2 },
-      });
-      return answeredTool ? "chat-final" : "background-json";
-    }
-    sse(res, [
-      messageStart(model, usage),
-      ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
-      ["content_block_delta", { index: 0, delta: { type: "text_delta", text } }],
-      ["content_block_stop", { index: 0 }],
-      [
-        "message_delta",
-        { delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 12 } },
-      ],
-      ["message_stop", {}],
-    ]);
-    return answeredTool ? "chat-final" : "background-sse";
+  if (body.stream !== true) {
+    json(res, 200, {
+      id: `msg_fake_${++seq}`,
+      type: "message",
+      role: "assistant",
+      model,
+      content: [{ type: "text", text }],
+      stop_reason: "end_turn",
+      stop_sequence: null,
+      usage: { input_tokens: 20, output_tokens: 2 },
+    });
+    return;
   }
-
-  const userText = textOf([...messages].reverse().find((m) => m.role === "user")?.content);
-  const script = pickScript(userText);
-  const input = JSON.stringify({
-    script: script.script,
-    timeout_secs: script.background ? 120 : 30,
-    display_name: script.displayName,
-    run_in_background: script.background,
-  });
   sse(res, [
-    messageStart(model, usage),
+    messageStart(model, { input_tokens: 500, output_tokens: 1 }),
     ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
-    [
-      "content_block_delta",
-      { index: 0, delta: { type: "text_delta", text: `Running: ${script.displayName}.` } },
-    ],
+    ["content_block_delta", { index: 0, delta: { type: "text_delta", text } }],
     ["content_block_stop", { index: 0 }],
-    [
-      "content_block_start",
-      {
-        index: 1,
-        content_block: { type: "tool_use", id: `toolu_fake_${seq}`, name: "bash", input: {} },
-      },
-    ],
-    ["content_block_delta", { index: 1, delta: { type: "input_json_delta", partial_json: input } }],
-    ["content_block_stop", { index: 1 }],
     [
       "message_delta",
-      { delta: { stop_reason: "tool_use", stop_sequence: null }, usage: { output_tokens: 40 } },
+      { delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 12 } },
     ],
     ["message_stop", {}],
   ]);
-  return `chat-tool:${script.keyword}`;
+}
+
+/**
+ * A chat request. Agent turns (their system prompt names the worktree) make the plan's proxy
+ * calls; everything else the app asks (titles, status) gets a short "ok".
+ */
+async function anthropicChat(
+  res: http.ServerResponse,
+  body: Json,
+  xumRoot: string
+): Promise<string> {
+  const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : [];
+  const workspaceId = workspaceIdFor(xumRoot, textOf(body.system));
+  if (workspaceId === undefined) {
+    chatText(res, body, "ok");
+    return "app-chat";
+  }
+  const userText = textOf([...messages].reverse().find((m) => m.role === "user")?.content);
+  const plan = pickPlan(userText);
+  if (plan.background) {
+    chatText(res, body, `Started ${plan.displayName}. Watch the Cost tab.`);
+    // Like a background command: the calls outlive the turn. Each result goes to the app log.
+    (async () => {
+      for (const [i, call] of plan.calls.entries()) {
+        if (i > 0) await new Promise((resolve) => setTimeout(resolve, 5000));
+        console.log(
+          `[fake-provider] background call ${i + 1}: ${await proxyCall(xumRoot, workspaceId, call)}`
+        );
+      }
+    })().catch((error: unknown) =>
+      console.log(`[fake-provider] background calls failed: ${String(error)}`)
+    );
+    return `chat-background:${plan.keyword}`;
+  }
+  const results: string[] = [];
+  for (const call of plan.calls)
+    results.push(`${call}: ${await proxyCall(xumRoot, workspaceId, call)}`);
+  chatText(res, body, `Done: ${plan.displayName} through the Xum proxy.\n${results.join("\n")}`);
+  return `chat-plan:${plan.keyword}`;
 }
 
 function anthropicProbe(res: http.ServerResponse, body: Json): string {
@@ -373,11 +423,13 @@ function openAiResponses(res: http.ServerResponse, body: Json): string {
 }
 
 /** Starts the fake on a free loopback port and returns its origin (no trailing slash). */
-export async function startFakeProvider(): Promise<{ origin: string; close: () => void }> {
+export async function startFakeProvider(
+  xumRoot: string
+): Promise<{ origin: string; close: () => void }> {
   const server = http.createServer((req, res) => {
     const chunks: Buffer[] = [];
     req.on("data", (chunk: Buffer) => chunks.push(chunk));
-    req.on("end", () => {
+    const handle = async (): Promise<void> => {
       let body: Json = {};
       try {
         const raw = Buffer.concat(chunks).toString("utf8");
@@ -388,7 +440,7 @@ export async function startFakeProvider(): Promise<{ origin: string; close: () =
       const route = (req.url ?? "").split("?")[0];
       let kind: string;
       if (req.method === "POST" && route.endsWith("/messages")) {
-        kind = isProbe(body) ? anthropicProbe(res, body) : anthropicChat(res, body);
+        kind = isProbe(body) ? anthropicProbe(res, body) : await anthropicChat(res, body, xumRoot);
       } else if (req.method === "POST" && route.endsWith("/chat/completions")) {
         kind = openAiChat(res, body);
       } else if (req.method === "POST" && route.endsWith("/responses")) {
@@ -404,6 +456,12 @@ export async function startFakeProvider(): Promise<{ origin: string; close: () =
       console.log(
         `[fake-provider] ${req.method} ${route} kind=${kind} model=${String(body.model)} stream=${String(body.stream === true)} key=${key.includes("xum-proxy-") ? "PROXY-KEY-LEAKED" : key ? "xum-settings-key" : "none"}`
       );
+    };
+    req.on("end", () => {
+      handle().catch((error: unknown) => {
+        console.log(`[fake-provider] request failed: ${String(error)}`);
+        if (!res.headersSent) json(res, 500, { type: "error", error: { message: "fake failed" } });
+      });
     });
   });
   await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
diff --git a/tests/bugbash/startApp.ts b/tests/bugbash/startApp.ts
index 340b2abb053..6b9908317e8 100644
--- a/tests/bugbash/startApp.ts
+++ b/tests/bugbash/startApp.ts
@@ -331,10 +331,10 @@ async function main(): Promise<void> {
   console.log(`[bugbash startApp] app AI: ${ai.mode} (${ai.reason})`);
 
   // BUGBASH_SCENARIO=bash-ai-proxy: instead of the mock, a loopback fake provider (fakeProvider.ts)
-  // is the chat model and the upstream of Xum's bash AI proxy, so the agent runs real bash
-  // commands whose AI calls go through the proxy. The fake only ever asks for its own fixed
-  // scripts, so agent tools stay on; nothing leaves the machine. It replaces the mock, so the
-  // explorer context (e2e.config.ts) must not describe a real model.
+  // is the chat model and the upstream of Xum's bash AI proxy. On each agent turn the fake calls
+  // the proxy itself with that workspace's key, so the proxy, the Cost tab and Analytics see
+  // real traffic while agent tools stay off. Nothing leaves the machine. It replaces the mock,
+  // so the explorer context (e2e.config.ts) must not describe a real model.
   const scenario = process.env.BUGBASH_SCENARIO ?? "";
   if (scenario !== "" && scenario !== "bash-ai-proxy") {
     fail(`BUGBASH_SCENARIO must be empty or bash-ai-proxy, got "${scenario}"`);
@@ -342,10 +342,9 @@ async function main(): Promise<void> {
   if (scenario !== "" && ai.mode !== "mock") {
     fail("BUGBASH_SCENARIO=bash-ai-proxy replaces the mock: run it with BUGBASH_AI=mock");
   }
-  const fakeProvider = scenario === "bash-ai-proxy" ? await startFakeProvider() : undefined;
-
   const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-"));
   const xumRoot = path.join(tempRoot, "xum");
+  const fakeProvider = scenario === "bash-ai-proxy" ? await startFakeProvider(xumRoot) : undefined;
   const home = path.join(tempRoot, "home");
   const projectPath = path.join(tempRoot, PROJECT_NAME);
   const env: NodeJS.ProcessEnv = {
@@ -357,9 +356,10 @@ async function main(): Promise<void> {
     // gets no terminal or project init hooks, so injected text in a reply that the explorer
     // follows cannot run host commands through them (see the header).
     ...(fakeProvider
-      ? // The explorer gets no terminal. Project automation stays on: the kill switch also
-        // withholds the proxy variables from bash, and the seeded demo repo has no hooks.
-        { XUM_DISABLE_TERMINALS: "1" }
+      ? // No agent tools and no terminal (AGENTS.md: not until the app runs in a sandbox).
+        // Project automation stays on: its kill switch also makes the proxy refuse every call,
+        // the fake's replies are fixed text, and the seeded demo repo has no hooks.
+        { XUM_DISABLE_AGENT_TOOLS: "1", XUM_DISABLE_TERMINALS: "1" }
       : ai.mode === "mock"
         ? { XUM_MOCK_AI: "1" }
         : {

From d3a6a857c7a45f4a197744c4f3c8201c6a114140 Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 19:27:37 +0000
Subject: [PATCH 3/6] bug-bash: one plan per message; list #5786 as tracked

---
 tests/bugbash/e2e.config.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts
index 4d9976ec7c3..b20f5103632 100644
--- a/tests/bugbash/e2e.config.ts
+++ b/tests/bugbash/e2e.config.ts
@@ -108,7 +108,8 @@ const bashAiProxyContext = [
   "picks the calls: none = one Anthropic call; '[proxy:stream]' = one streamed Anthropic call;",
   "'[proxy:openai]' = one OpenAI chat and one OpenAI responses call; '[proxy:many]' = five Anthropic",
   "calls; '[proxy:background]' = twelve Anthropic calls, one every 5 seconds, after the reply (about",
-  "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401).",
+  "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401). One plan per message:",
+  "when a message names several keywords, only the first one in this list runs.",
   "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts",
   "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI",
   "probe reports 1234 prompt tokens of which 100 are cached (1134 uncached) and 56 output tokens, on",
@@ -121,7 +122,8 @@ const bashAiProxyContext = [
   "opened (#5767); Analytics response counts leave out bash proxy rows and label them agent 'unknown'",
   "(#5766); Analytics shows timestamps as '1.8T', repeats y-axis ticks, says '1 responses' and",
   "overlaps at 390px (#5768); at 390px the workspace footer overflows and a long bash Script block is",
-  "cut off (#5769). By design: the 'Workspace created' row follows the first message.",
+  "cut off (#5769); the Archived Workspaces list shows a stale cost until reload (#5786). By design:",
+  "the 'Workspace created' row follows the first message.",
 ].join(" ");
 
 // What the local app cannot do, and the explorer's own blind spots (see `e2e guide bug-bash`).

From 3c1e2e555d703cb17accaa6550960cf76540a878 Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 19:43:04 +0000
Subject: [PATCH 4/6] bug-bash: Sonnet chat model, path match with spaces,
 project-scope Analytics in charters

---
 tests/bugbash/charters.bash-ai-proxy.txt |  2 +-
 tests/bugbash/e2e.config.ts              |  3 ++-
 tests/bugbash/fakeProvider.ts            | 12 ++++++++----
 tests/bugbash/startApp.ts                |  4 ++++
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tests/bugbash/charters.bash-ai-proxy.txt b/tests/bugbash/charters.bash-ai-proxy.txt
index 602f54765a3..00e4223c1de 100644
--- a/tests/bugbash/charters.bash-ai-proxy.txt
+++ b/tests/bugbash/charters.bash-ai-proxy.txt
@@ -5,6 +5,6 @@ proxy-costs|web|skeptic|Turn on Settings > Providers > 'Count AI calls from bash
 proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch and toggle it on and off, reloading after each change; after each state send 'probe' in the 'Bug bash playground' workspace and check that calls succeed (HTTP 200) only while the switch is on and are refused (HTTP 503) while it is off, that the switch state persists, and that the Cost tab never loses or double-counts earlier spend
 proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[proxy:background]' and watch the Cost tab for about 20 seconds without reloading (it should grow with each call); then turn the switch off while the calls still run and watch for 30 more seconds; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off
 proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[proxy:stream]', '[proxy:bad-key]', '[proxy:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted
-proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[proxy:many]' in each workspace; check that each workspace's Cost tab and Analytics count only its own proxied calls, and that archiving and restoring a workspace keeps its numbers
+proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[proxy:many]' in each workspace; check that each workspace's Cost tab counts only its own proxied calls, that Analytics (which totals the whole demo-app project) shows the sum of both workspaces, and that archiving and restoring a workspace keeps its numbers
 proxy-trust|web|state|Turn on the bash AI proxy switch and send 'probe' in the 'Bug bash playground' workspace (expect HTTP 200); then send '[proxy:background]' and, while it runs, revoke trust for demo-app in Settings > Security; calls after the revoke must fail (HTTP 403) and must not be counted in the Stats > Cost tab; trust the project again and send 'probe' (HTTP 200, counted once); report anything that leaks, sticks or double-counts
 proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title and description and report text that is confusing, wrong, cut off or overlapping; turn it on, send 'probe' in the 'Bug bash playground' workspace and check the reply and Analytics (the Cost tab is known to be unreachable on phones, do not try it)
diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts
index b20f5103632..58342889bbf 100644
--- a/tests/bugbash/e2e.config.ts
+++ b/tests/bugbash/e2e.config.ts
@@ -113,7 +113,8 @@ const bashAiProxyContext = [
   "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts",
   "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI",
   "probe reports 1234 prompt tokens of which 100 are cached (1134 uncached) and 56 output tokens, on",
-  "gpt-6.1-sol. The chat turns themselves also cost a little on claude-sonnet-5-5 or the selected model.",
+  "gpt-6.1-sol. The chat turns themselves cost a little on the workspace model (claude-sonnet-5-5 by",
+  "default), so they get their own Cost tab row. Analytics totals the whole project, not one workspace.",
   "Terminals are turned off for this session, so an error when opening one is expected.",
   "Do not sign in to any provider, MCP server or external service, and do not enter real secrets.",
   "Known and not bugs: the per-model cost column rounds amounts under $0.01 to ~$0.00; this test",
diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts
index 94fadab0fd4..2d6826127ef 100644
--- a/tests/bugbash/fakeProvider.ts
+++ b/tests/bugbash/fakeProvider.ts
@@ -78,15 +78,19 @@ function pickPlan(userText: string): Plan {
   return PLANS.find((p) => p.keyword !== "probe" && text.includes(p.keyword)) ?? PLANS.at(-1)!;
 }
 
-/** The workspace whose agent sent this turn: its system prompt names the worktree path. */
+/**
+ * The workspace whose agent sent this turn: its system prompt names the worktree path. Match the
+ * configured paths (they can hold spaces) instead of parsing the path out of the prompt.
+ */
 function workspaceIdFor(xumRoot: string, system: string): string | undefined {
-  const match = /You are in a git worktree at (\S+)/.exec(system);
-  if (!match) return undefined;
+  if (!system.includes("You are in a git worktree at ")) return undefined;
   const config = JSON.parse(fs.readFileSync(path.join(xumRoot, "config.json"), "utf8")) as {
     projects?: [string, { workspaces?: { path?: string; id?: string }[] }][];
   };
   for (const [, project] of config.projects ?? []) {
-    const hit = project.workspaces?.find((w) => w.path === match[1]);
+    const hit = project.workspaces?.find(
+      (w) => w.path !== undefined && system.includes(`You are in a git worktree at ${w.path}`)
+    );
     if (hit?.id) return hit.id;
   }
   return undefined;
diff --git a/tests/bugbash/startApp.ts b/tests/bugbash/startApp.ts
index 6b9908317e8..24d807702de 100644
--- a/tests/bugbash/startApp.ts
+++ b/tests/bugbash/startApp.ts
@@ -255,6 +255,10 @@ async function seed(
       value: `${providerBase}/v1`,
     });
     if (fakeProviderOrigin !== undefined) {
+      // Chat turns on Sonnet, probes on Opus, so the Cost tab keeps them in separate rows.
+      await api(base, "config/updateModelPreferences", {
+        defaultModel: "anthropic:claude-sonnet-5-5",
+      });
       await api(base, "providers/setProviderConfig", {
         provider: "openai",
         keyPath: ["apiKey"],

From 12062b1957c6ed47dee16e8c93bef4870655fb35 Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 20:05:10 +0000
Subject: [PATCH 5/6] bug-bash: match the whole worktree path line

---
 tests/bugbash/fakeProvider.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts
index 2d6826127ef..04e77a145f7 100644
--- a/tests/bugbash/fakeProvider.ts
+++ b/tests/bugbash/fakeProvider.ts
@@ -88,9 +88,13 @@ function workspaceIdFor(xumRoot: string, system: string): string | undefined {
     projects?: [string, { workspaces?: { path?: string; id?: string }[] }][];
   };
   for (const [, project] of config.projects ?? []) {
-    const hit = project.workspaces?.find(
-      (w) => w.path !== undefined && system.includes(`You are in a git worktree at ${w.path}`)
-    );
+    // The path ends the line: a boundary, so `.../playground` never matches `.../playground-2`.
+    const hit = project.workspaces?.find((w) => {
+      if (w.path === undefined) return false;
+      const needle = `You are in a git worktree at ${w.path}`;
+      const at = system.indexOf(needle);
+      return at !== -1 && [undefined, "\n"].includes(system[at + needle.length]);
+    });
     if (hit?.id) return hit.id;
   }
   return undefined;

From 68e5f47a2a4aefef2f45e53a6fc14a87bbf2554a Mon Sep 17 00:00:00 2001
From: Thomas Kosiewski <tk@coder.com>
Date: Tue, 6 Oct 2026 20:26:25 +0000
Subject: [PATCH 6/6] bug-bash: background results readable via [proxy:status],
 neutral result wording, trust charter waits for finished

---
 tests/bugbash/charters.bash-ai-proxy.txt |  2 +-
 tests/bugbash/e2e.config.ts              |  4 +-
 tests/bugbash/fakeProvider.ts            | 48 +++++++++++++++++++-----
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/tests/bugbash/charters.bash-ai-proxy.txt b/tests/bugbash/charters.bash-ai-proxy.txt
index 00e4223c1de..ded436b6529 100644
--- a/tests/bugbash/charters.bash-ai-proxy.txt
+++ b/tests/bugbash/charters.bash-ai-proxy.txt
@@ -6,5 +6,5 @@ proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch
 proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[proxy:background]' and watch the Cost tab for about 20 seconds without reloading (it should grow with each call); then turn the switch off while the calls still run and watch for 30 more seconds; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off
 proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[proxy:stream]', '[proxy:bad-key]', '[proxy:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted
 proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[proxy:many]' in each workspace; check that each workspace's Cost tab counts only its own proxied calls, that Analytics (which totals the whole demo-app project) shows the sum of both workspaces, and that archiving and restoring a workspace keeps its numbers
-proxy-trust|web|state|Turn on the bash AI proxy switch and send 'probe' in the 'Bug bash playground' workspace (expect HTTP 200); then send '[proxy:background]' and, while it runs, revoke trust for demo-app in Settings > Security; calls after the revoke must fail (HTTP 403) and must not be counted in the Stats > Cost tab; trust the project again and send 'probe' (HTTP 200, counted once); report anything that leaks, sticks or double-counts
+proxy-trust|web|state|Turn on the bash AI proxy switch and send 'probe' in the 'Bug bash playground' workspace (expect HTTP 200); then send '[proxy:background]' and, while it runs, revoke trust for demo-app in Settings > Security; send '[proxy:status]' until it says finished: calls after the revoke must show HTTP 403 and must not be counted in the Stats > Cost tab; only then trust the project again and send 'probe' (HTTP 200, counted once); report anything that leaks, sticks or double-counts
 proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title and description and report text that is confusing, wrong, cut off or overlapping; turn it on, send 'probe' in the 'Bug bash playground' workspace and check the reply and Analytics (the Cost tab is known to be unreachable on phones, do not try it)
diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts
index 58342889bbf..227d11d777c 100644
--- a/tests/bugbash/e2e.config.ts
+++ b/tests/bugbash/e2e.config.ts
@@ -104,11 +104,11 @@ const bashAiProxyContext = [
   "makes it refuse that workspace's calls (HTTP 403).",
   "AI is a local fake provider, nothing is billed, and agent tools are off, so the agent runs no",
   "command. Instead the fake model itself calls the proxy with the key Xum gives that workspace's",
-  "bash commands, and replies 'Done: ...' with one HTTP result per call. A keyword in your message",
+  "bash commands, and replies 'Results of ...' with one HTTP result per call (an expected refusal is a result, not a failure of the reply). A keyword in your message",
   "picks the calls: none = one Anthropic call; '[proxy:stream]' = one streamed Anthropic call;",
   "'[proxy:openai]' = one OpenAI chat and one OpenAI responses call; '[proxy:many]' = five Anthropic",
   "calls; '[proxy:background]' = twelve Anthropic calls, one every 5 seconds, after the reply (about",
-  "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401). One plan per message:",
+  "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401); '[proxy:status]' = no call, only the background results so far (it says 'finished' when all ran). Every reply also lists the background results. One plan per message:",
   "when a message names several keywords, only the first one in this list runs.",
   "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts",
   "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI",
diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts
index 04e77a145f7..ff5001efe85 100644
--- a/tests/bugbash/fakeProvider.ts
+++ b/tests/bugbash/fakeProvider.ts
@@ -68,6 +68,12 @@ const PLANS: Plan[] = [
     calls: ["bad-key"],
     background: false,
   },
+  {
+    keyword: "[proxy:status]",
+    displayName: "no new call, only the background results so far",
+    calls: [],
+    background: false,
+  },
   { keyword: "probe", displayName: "one Anthropic call", calls: ["anthropic"], background: false },
 ];
 
@@ -252,6 +258,9 @@ function chatText(res: http.ServerResponse, body: Json, text: string): void {
  * A chat request. Agent turns (their system prompt names the worktree) make the plan's proxy
  * calls; everything else the app asks (titles, status) gets a short "ok".
  */
+/** The latest background plan per workspace and its HTTP results. */
+const backgroundRuns = new Map<string, { results: string[]; done: boolean }>();
+
 async function anthropicChat(
   res: http.ServerResponse,
   body: Json,
@@ -265,25 +274,46 @@ async function anthropicChat(
   }
   const userText = textOf([...messages].reverse().find((m) => m.role === "user")?.content);
   const plan = pickPlan(userText);
+  // Background results so far, so the explorer can read them in the next reply.
+  const background = backgroundRuns.get(workspaceId);
+  const backgroundLines = background
+    ? [
+        `Background calls (${background.done ? "finished" : "still running"}):`,
+        ...background.results.map((r, i) => `  call ${i + 1}: ${r}`),
+      ]
+    : [];
   if (plan.background) {
-    chatText(res, body, `Started ${plan.displayName}. Watch the Cost tab.`);
-    // Like a background command: the calls outlive the turn. Each result goes to the app log.
+    const run = { results: [] as string[], done: false };
+    backgroundRuns.set(workspaceId, run);
+    chatText(
+      res,
+      body,
+      `Started ${plan.displayName}. Send '[proxy:status]' to see their HTTP results; it says "finished" when all calls ran.`
+    );
+    // Like a background command: the calls outlive the turn.
     (async () => {
       for (const [i, call] of plan.calls.entries()) {
         if (i > 0) await new Promise((resolve) => setTimeout(resolve, 5000));
-        console.log(
-          `[fake-provider] background call ${i + 1}: ${await proxyCall(xumRoot, workspaceId, call)}`
-        );
+        const result = await proxyCall(xumRoot, workspaceId, call);
+        run.results.push(result);
+        console.log(`[fake-provider] background call ${i + 1}: ${result}`);
       }
-    })().catch((error: unknown) =>
-      console.log(`[fake-provider] background calls failed: ${String(error)}`)
-    );
+    })()
+      .catch((error: unknown) => run.results.push(`harness error: ${String(error)}`))
+      .finally(() => {
+        run.done = true;
+      });
     return `chat-background:${plan.keyword}`;
   }
   const results: string[] = [];
   for (const call of plan.calls)
     results.push(`${call}: ${await proxyCall(xumRoot, workspaceId, call)}`);
-  chatText(res, body, `Done: ${plan.displayName} through the Xum proxy.\n${results.join("\n")}`);
+  // Neutral wording: an expected refusal (401, 403, 503) is a result, not a success.
+  const lines =
+    plan.calls.length > 0
+      ? [`Results of ${plan.displayName} through the Xum proxy:`, ...results]
+      : [];
+  chatText(res, body, [...lines, ...backgroundLines].join("\n") || "No background calls yet.");
   return `chat-plan:${plan.keyword}`;
 }