diff --git a/tests/bugbash/charters.bash-ai-proxy.txt b/tests/bugbash/charters.bash-ai-proxy.txt new file mode 100644 index 00000000000..ded436b6529 --- /dev/null +++ b/tests/bugbash/charters.bash-ai-proxy.txt @@ -0,0 +1,10 @@ +# Charters for the bash AI proxy. Run with BUGBASH_AI=mock BUGBASH_SCENARIO=bash-ai-proxy and +# BUGBASH_ARGS="--charters tests/bugbash/charters.bash-ai-proxy.txt" (see e2e.config.ts). +# The scenario has no agent tools and no terminals: the fake model calls the proxy itself. +proxy-costs|web|skeptic|Turn on Settings > Providers > 'Count AI calls from bash commands', then in the 'Bug bash playground' workspace send 'probe', '[proxy:many]' and '[proxy:openai]'; after each, cross-check the documented probe token counts against the Stats > Cost tab Session rows (per-model rows, components, session total) and Analytics; check that Last Request stays the chat request; report every number that does not add up +proxy-switch|web|state|In Settings > Providers, find the 'Bash commands' switch and toggle it on and off, reloading after each change; after each state send 'probe' in the 'Bug bash playground' workspace and check that calls succeed (HTTP 200) only while the switch is on and are refused (HTTP 503) while it is off, that the switch state persists, and that the Cost tab never loses or double-counts earlier spend +proxy-live|web|skeptic|Turn on the bash AI proxy switch, open the Stats > Cost tab of the 'Bug bash playground' workspace, then send '[proxy:background]' and watch the Cost tab for about 20 seconds without reloading (it should grow with each call); then turn the switch off while the calls still run and watch for 30 more seconds; report totals that do not grow as calls happen, grow too much, change after a reload, or keep growing after the switch is off +proxy-inputs|web|fuzzer|Turn on the bash AI proxy switch, then in the 'Bug bash playground' workspace send '[proxy:stream]', '[proxy:bad-key]', '[proxy:openai]' and messages that mix several keywords, unicode and 300 characters; report wrong or missing usage, errors shown as success, a stuck chat, or a wrong key that still gets counted +proxy-workspaces|web|state|Turn on the bash AI proxy switch, create a second workspace in demo-app, and send 'probe' and '[proxy:many]' in each workspace; check that each workspace's Cost tab counts only its own proxied calls, that Analytics (which totals the whole demo-app project) shows the sum of both workspaces, and that archiving and restoring a workspace keeps its numbers +proxy-trust|web|state|Turn on the bash AI proxy switch and send 'probe' in the 'Bug bash playground' workspace (expect HTTP 200); then send '[proxy:background]' and, while it runs, revoke trust for demo-app in Settings > Security; send '[proxy:status]' until it says finished: calls after the revoke must show HTTP 403 and must not be counted in the Stats > Cost tab; only then trust the project again and send 'probe' (HTTP 200, counted once); report anything that leaks, sticks or double-counts +proxy-phone|phone|newcomer|On this phone-sized screen, open Settings > Providers and find the 'Bash commands' section; read its title and description and report text that is confusing, wrong, cut off or overlapping; turn it on, send 'probe' in the 'Bug bash playground' workspace and check the reply and Analytics (the Cost tab is known to be unreachable on phones, do not try it) diff --git a/tests/bugbash/e2e.config.ts b/tests/bugbash/e2e.config.ts index 5a0eee48217..227d11d777c 100644 --- a/tests/bugbash/e2e.config.ts +++ b/tests/bugbash/e2e.config.ts @@ -86,8 +86,49 @@ const mockNonBugs = realAi "the mock echoes your text, and after a retry it may echo [CONTINUE].", ]; +// BUGBASH_SCENARIO=bash-ai-proxy swaps mock AI for a loopback fake provider (startApp.ts, +// fakeProvider.ts) so explorers can drive the bash AI proxy end to end. +const scenario = process.env.BUGBASH_SCENARIO ?? ""; + +// What the explorer must know about the bash AI proxy scenario instead of the mock-AI notes. +const bashAiProxyContext = [ + "The app is Xum, a desktop and browser app for running parallel AI coding agents.", + "It starts with one project, demo-app, and one workspace, 'Bug bash playground', in the left sidebar.", + "The feature under test is the bash AI proxy. When Settings > Providers > 'Bash commands' >", + "'Count AI calls from bash commands' is on (it is off by default), bash commands get a", + "per-workspace 'xum-proxy-...' key and endpoints that point at Xum. Xum forwards those calls with", + "the provider key from its settings and adds their tokens and cost to that workspace's", + "right-sidebar Stats > Cost tab (Session rows per model and total) and to Analytics (the bar-chart", + "button). The Cost tab's Last Request stays the chat request: proxied calls never replace it.", + "Turning the switch off makes the proxy refuse calls (HTTP 503); revoking the project's trust", + "makes it refuse that workspace's calls (HTTP 403).", + "AI is a local fake provider, nothing is billed, and agent tools are off, so the agent runs no", + "command. Instead the fake model itself calls the proxy with the key Xum gives that workspace's", + "bash commands, and replies 'Results of ...' with one HTTP result per call (an expected refusal is a result, not a failure of the reply). A keyword in your message", + "picks the calls: none = one Anthropic call; '[proxy:stream]' = one streamed Anthropic call;", + "'[proxy:openai]' = one OpenAI chat and one OpenAI responses call; '[proxy:many]' = five Anthropic", + "calls; '[proxy:background]' = twelve Anthropic calls, one every 5 seconds, after the reply (about", + "a minute); '[proxy:bad-key]' = a call with a wrong key (expect HTTP 401); '[proxy:status]' = no call, only the background results so far (it says 'finished' when all ran). Every reply also lists the background results. One plan per message:", + "when a message names several keywords, only the first one in this list runs.", + "Each Anthropic probe reports input_tokens 1234 plus cache_read_input_tokens 100 (Anthropic counts", + "cache reads separately, so 1334 input in total) and 56 output tokens, on claude-opus-5-5. Each OpenAI", + "probe reports 1234 prompt tokens of which 100 are cached (1134 uncached) and 56 output tokens, on", + "gpt-6.1-sol. The chat turns themselves cost a little on the workspace model (claude-sonnet-5-5 by", + "default), so they get their own Cost tab row. Analytics totals the whole project, not one workspace.", + "Terminals are turned off for this session, so an error when opening one is expected.", + "Do not sign in to any provider, MCP server or external service, and do not enter real secrets.", + "Known and not bugs: the per-model cost column rounds amounts under $0.01 to ~$0.00; this test", + "browser denies clipboard writes; workspace names must be lowercase branch names.", + "Known and tracked, do not report again: at phone width the right sidebar (Stats > Cost) cannot be", + "opened (#5767); Analytics response counts leave out bash proxy rows and label them agent 'unknown'", + "(#5766); Analytics shows timestamps as '1.8T', repeats y-axis ticks, says '1 responses' and", + "overlaps at 390px (#5768); at 390px the workspace footer overflows and a long bash Script block is", + "cut off (#5769); the Archived Workspaces list shows a stale cost until reload (#5786). By design:", + "the 'Workspace created' row follows the first message.", +].join(" "); + // What the local app cannot do, and the explorer's own blind spots (see `e2e guide bug-bash`). -const context = [ +const mockOrRealContext = [ "The app is Xum, a desktop and browser app for running parallel AI coding agents.", "It starts with one project, demo-app, and one workspace, 'Bug bash playground', in the left sidebar.", ...aiContext, @@ -123,6 +164,8 @@ const context = [ "after about one second; Ctrl+/ cycles to the next model; Fast mode is unavailable in this setup.", ].join(" "); +const context = scenario === "bash-ai-proxy" ? bashAiProxyContext : mockOrRealContext; + // Exploration steps need large per-step budgets (`e2e guide bug-bash`, step 1). // Only the active provider reads its key, so both can be set at once. const effort = explorerEffort(); @@ -141,6 +184,7 @@ const APP_ENV_VARS = [ "BUGBASH_AI_RESOLVED", "BUGBASH_AI_REASON", "BUGBASH_APP_MODEL", + "BUGBASH_SCENARIO", "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "OPENAI_API_KEY", diff --git a/tests/bugbash/fakeProvider.ts b/tests/bugbash/fakeProvider.ts new file mode 100644 index 00000000000..ff5001efe85 --- /dev/null +++ b/tests/bugbash/fakeProvider.ts @@ -0,0 +1,510 @@ +/** + * Fake Anthropic and OpenAI provider for the `bash-ai-proxy` bug-bash scenario + * (BUGBASH_SCENARIO=bash-ai-proxy, see startApp.ts). + * + * It plays two roles, both on loopback, so no model call leaves the machine and nothing is billed: + * 1. The app's chat model. Every agent turn makes the fake itself call Xum's bash AI proxy, with + * the key Xum gives that workspace's bash commands, picked from the fixed PLANS table by a + * keyword in the message (default: "probe"). The reply lists the HTTP answers. + * Agent tools stay off (AGENTS.md: no agent tools or terminals in bug bashes until the app + * runs in a sandbox), so the agent runs no command. The proxy cannot tell these harness-owned + * calls from a bash command's; the bash env injection itself is covered by unit tests. + * 2. The upstream behind the proxy. The proxy forwards here with the key from the Xum provider + * settings. Probe answers carry fixed token counts, so explorers can cross-check the Costs tab. + * + * Every request is logged as one `[fake-provider]` line (no key values) to the app log. + */ +import * as fs from "fs"; +import * as http from "http"; +import type { AddressInfo } from "net"; +import * as path from "path"; + +import { deriveProxyKey } from "../../src/node/services/bashAiProxy/stableIdentity"; +import { BASH_AI_PROXY_STATE_FILE } from "../../src/node/services/bashAiProxy/proxyState"; + +/** Token counts in every probe answer, so a skeptic can add them up. */ +export const PROBE_USAGE = { input: 1234, output: 56, cacheRead: 100 } as const; +const PROBE_MARKER = "bugbash-proxy-probe"; + +type Call = "anthropic" | "anthropic-stream" | "openai-chat" | "openai-responses" | "bad-key"; + +interface Plan { + keyword: string; + displayName: string; + calls: Call[]; + /** Calls run one every 5 s after the reply, like a background command. */ + background: boolean; +} + +/** Fixed plans. The first keyword found in the user's message wins; "probe" is the default. */ +const PLANS: Plan[] = [ + { + keyword: "[proxy:stream]", + displayName: "one streamed Anthropic call", + calls: ["anthropic-stream"], + background: false, + }, + { + keyword: "[proxy:openai]", + displayName: "one OpenAI chat and one OpenAI responses call", + calls: ["openai-chat", "openai-responses"], + background: false, + }, + { + keyword: "[proxy:many]", + displayName: "five Anthropic calls", + calls: Array(5).fill("anthropic"), + background: false, + }, + { + keyword: "[proxy:background]", + displayName: "twelve background Anthropic calls, one every 5 s", + calls: Array(12).fill("anthropic"), + background: true, + }, + { + keyword: "[proxy:bad-key]", + displayName: "one Anthropic call with a wrong key", + calls: ["bad-key"], + background: false, + }, + { + keyword: "[proxy:status]", + displayName: "no new call, only the background results so far", + calls: [], + background: false, + }, + { keyword: "probe", displayName: "one Anthropic call", calls: ["anthropic"], background: false }, +]; + +type Json = Record; + +function pickPlan(userText: string): Plan { + const text = userText.toLowerCase(); + return PLANS.find((p) => p.keyword !== "probe" && text.includes(p.keyword)) ?? PLANS.at(-1)!; +} + +/** + * The workspace whose agent sent this turn: its system prompt names the worktree path. Match the + * configured paths (they can hold spaces) instead of parsing the path out of the prompt. + */ +function workspaceIdFor(xumRoot: string, system: string): string | undefined { + if (!system.includes("You are in a git worktree at ")) return undefined; + const config = JSON.parse(fs.readFileSync(path.join(xumRoot, "config.json"), "utf8")) as { + projects?: [string, { workspaces?: { path?: string; id?: string }[] }][]; + }; + for (const [, project] of config.projects ?? []) { + // The path ends the line: a boundary, so `.../playground` never matches `.../playground-2`. + const hit = project.workspaces?.find((w) => { + if (w.path === undefined) return false; + const needle = `You are in a git worktree at ${w.path}`; + const at = system.indexOf(needle); + return at !== -1 && [undefined, "\n"].includes(system[at + needle.length]); + }); + if (hit?.id) return hit.id; + } + return undefined; +} + +const PROBE_BODIES: Record, { route: string; body: Json }> = { + anthropic: { + route: "/anthropic/v1/messages", + body: { + model: "claude-opus-5-5", + max_tokens: 5, + messages: [{ role: "user", content: PROBE_MARKER }], + }, + }, + "anthropic-stream": { + route: "/anthropic/v1/messages", + body: { + model: "claude-opus-5-5", + max_tokens: 5, + stream: true, + messages: [{ role: "user", content: PROBE_MARKER }], + }, + }, + "openai-chat": { + route: "/openai/v1/chat/completions", + body: { + model: "gpt-6.1-sol", + stream: true, + messages: [{ role: "user", content: PROBE_MARKER }], + }, + }, + "openai-responses": { + route: "/openai/v1/responses", + body: { model: "gpt-6.1-sol", input: PROBE_MARKER }, + }, +}; + +/** One call through Xum's proxy, as a bash command with the injected variables would make it. */ +async function proxyCall(xumRoot: string, workspaceId: string, call: Call): Promise { + let state: { secret?: string; port?: number }; + try { + state = JSON.parse( + fs.readFileSync(path.join(xumRoot, BASH_AI_PROXY_STATE_FILE), "utf8") + ) as typeof state; + } catch { + return "no proxy: the switch was never turned on"; + } + if (typeof state.port !== "number" || typeof state.secret !== "string") { + return "no proxy: the switch was never turned on"; + } + const key = call === "bad-key" ? "xum-proxy-wrong" : deriveProxyKey(state.secret, workspaceId); + const probe = PROBE_BODIES[call === "bad-key" ? "anthropic" : call]; + const headers: Record = { "content-type": "application/json" }; + if (probe.route.startsWith("/openai")) headers.authorization = `Bearer ${key}`; + else headers["x-api-key"] = key; + try { + const res = await fetch(`http://127.0.0.1:${state.port}${probe.route}`, { + method: "POST", + headers, + body: JSON.stringify(probe.body), + }); + const text = await res.text(); + const error = /"message":"([^"]*)"/.exec(text)?.[1]; + return `HTTP ${res.status}${res.ok ? "" : ` ${error ?? text.slice(0, 120)}`}`; + } catch (error) { + return `connection failed: ${error instanceof Error ? error.message : String(error)}`; + } +} + +function textOf(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map((part) => + part && typeof part === "object" && (part as Json).type === "text" + ? String((part as Json).text) + : "" + ) + .join(" "); +} + +/** + * A probe is the exact one-message request a fixed script sends. Chat turns also contain the + * marker (the bash tool input quotes the script), so a substring match is not enough. + */ +function isProbe(body: Json): boolean { + if (body.input === PROBE_MARKER) return true; + const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : []; + return messages.length === 1 && messages[0]?.content === PROBE_MARKER; +} + +function sse(res: http.ServerResponse, events: [string | null, Json | string][]): void { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + for (const [type, data] of events) { + const payload = + typeof data === "string" ? data : JSON.stringify(type ? { type, ...data } : data); + res.write(`${type ? `event: ${type}\n` : ""}data: ${payload}\n\n`); + } + res.end(); +} + +function json(res: http.ServerResponse, status: number, body: Json): void { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); +} + +let seq = 0; +function messageStart(model: string, usage: Json): [string, Json] { + return [ + "message_start", + { + message: { + id: `msg_fake_${++seq}`, + type: "message", + role: "assistant", + model, + content: [], + stop_reason: null, + stop_sequence: null, + usage, + }, + }, + ]; +} + +function chatText(res: http.ServerResponse, body: Json, text: string): void { + const model = typeof body.model === "string" ? body.model : "claude-sonnet-5-5"; + if (body.stream !== true) { + json(res, 200, { + id: `msg_fake_${++seq}`, + type: "message", + role: "assistant", + model, + content: [{ type: "text", text }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 20, output_tokens: 2 }, + }); + return; + } + sse(res, [ + messageStart(model, { input_tokens: 500, output_tokens: 1 }), + ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }], + ["content_block_delta", { index: 0, delta: { type: "text_delta", text } }], + ["content_block_stop", { index: 0 }], + [ + "message_delta", + { delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 12 } }, + ], + ["message_stop", {}], + ]); +} + +/** + * A chat request. Agent turns (their system prompt names the worktree) make the plan's proxy + * calls; everything else the app asks (titles, status) gets a short "ok". + */ +/** The latest background plan per workspace and its HTTP results. */ +const backgroundRuns = new Map(); + +async function anthropicChat( + res: http.ServerResponse, + body: Json, + xumRoot: string +): Promise { + const messages = Array.isArray(body.messages) ? (body.messages as Json[]) : []; + const workspaceId = workspaceIdFor(xumRoot, textOf(body.system)); + if (workspaceId === undefined) { + chatText(res, body, "ok"); + return "app-chat"; + } + const userText = textOf([...messages].reverse().find((m) => m.role === "user")?.content); + const plan = pickPlan(userText); + // Background results so far, so the explorer can read them in the next reply. + const background = backgroundRuns.get(workspaceId); + const backgroundLines = background + ? [ + `Background calls (${background.done ? "finished" : "still running"}):`, + ...background.results.map((r, i) => ` call ${i + 1}: ${r}`), + ] + : []; + if (plan.background) { + const run = { results: [] as string[], done: false }; + backgroundRuns.set(workspaceId, run); + chatText( + res, + body, + `Started ${plan.displayName}. Send '[proxy:status]' to see their HTTP results; it says "finished" when all calls ran.` + ); + // Like a background command: the calls outlive the turn. + (async () => { + for (const [i, call] of plan.calls.entries()) { + if (i > 0) await new Promise((resolve) => setTimeout(resolve, 5000)); + const result = await proxyCall(xumRoot, workspaceId, call); + run.results.push(result); + console.log(`[fake-provider] background call ${i + 1}: ${result}`); + } + })() + .catch((error: unknown) => run.results.push(`harness error: ${String(error)}`)) + .finally(() => { + run.done = true; + }); + return `chat-background:${plan.keyword}`; + } + const results: string[] = []; + for (const call of plan.calls) + results.push(`${call}: ${await proxyCall(xumRoot, workspaceId, call)}`); + // Neutral wording: an expected refusal (401, 403, 503) is a result, not a success. + const lines = + plan.calls.length > 0 + ? [`Results of ${plan.displayName} through the Xum proxy:`, ...results] + : []; + chatText(res, body, [...lines, ...backgroundLines].join("\n") || "No background calls yet."); + return `chat-plan:${plan.keyword}`; +} + +function anthropicProbe(res: http.ServerResponse, body: Json): string { + const model = typeof body.model === "string" ? body.model : "claude-opus-5-5"; + if (body.stream === true) { + sse(res, [ + messageStart(model, { + input_tokens: PROBE_USAGE.input, + cache_read_input_tokens: PROBE_USAGE.cacheRead, + output_tokens: 1, + }), + ["content_block_start", { index: 0, content_block: { type: "text", text: "" } }], + ["content_block_delta", { index: 0, delta: { type: "text_delta", text: "pong" } }], + ["content_block_stop", { index: 0 }], + [ + "message_delta", + { + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: PROBE_USAGE.output }, + }, + ], + ["message_stop", {}], + ]); + return "probe-anthropic-sse"; + } + json(res, 200, { + id: `msg_probe_${++seq}`, + type: "message", + role: "assistant", + model, + content: [{ type: "text", text: "pong" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { + input_tokens: PROBE_USAGE.input, + output_tokens: PROBE_USAGE.output, + cache_read_input_tokens: PROBE_USAGE.cacheRead, + }, + }); + return "probe-anthropic-json"; +} + +function openAiUsage(): Json { + return { + prompt_tokens: PROBE_USAGE.input, + completion_tokens: PROBE_USAGE.output, + prompt_tokens_details: { cached_tokens: PROBE_USAGE.cacheRead }, + }; +} + +function openAiChat(res: http.ServerResponse, body: Json): string { + const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol"; + const includeUsage = (body.stream_options as Json | undefined)?.include_usage === true; + if (body.stream !== true) { + json(res, 200, { + id: "chatcmpl_fake", + object: "chat.completion", + model, + choices: [ + { index: 0, message: { role: "assistant", content: "pong" }, finish_reason: "stop" }, + ], + usage: openAiUsage(), + }); + return "probe-openai-chat-json"; + } + const chunk = (extra: Json) => + [null, { id: "chatcmpl_fake", object: "chat.completion.chunk", model, ...extra }] as [ + null, + Json, + ]; + sse(res, [ + chunk({ + choices: [{ index: 0, delta: { role: "assistant", content: "pong" }, finish_reason: null }], + }), + chunk({ choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }), + ...(includeUsage ? [chunk({ choices: [], usage: openAiUsage() })] : []), + [null, "[DONE]"], + ]); + return `probe-openai-chat-sse(usage=${includeUsage})`; +} + +/** The app's own OpenAI calls (not probes): a short streamed or JSON "ok" with tiny usage. */ +function openAiAppResponses(res: http.ServerResponse, body: Json): string { + const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol"; + const usage = { + input_tokens: 20, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 2, + output_tokens_details: { reasoning_tokens: 0 }, + }; + const item = { type: "message", id: "msg_app", role: "assistant" }; + const done = { + ...item, + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }; + const response = (status: string, extra: Json) => ({ + id: `resp_app_${++seq}`, + object: "response", + created_at: Math.floor(Date.now() / 1000), + model, + status, + ...extra, + }); + if (body.stream !== true) { + json(res, 200, response("completed", { output: [done], usage })); + return "app-openai-responses-json"; + } + sse(res, [ + ["response.created", { response: response("in_progress", { output: [] }) }], + [ + "response.output_item.added", + { output_index: 0, item: { ...item, status: "in_progress", content: [] } }, + ], + [ + "response.output_text.delta", + { item_id: "msg_app", output_index: 0, content_index: 0, delta: "ok" }, + ], + ["response.output_item.done", { output_index: 0, item: done }], + ["response.completed", { response: response("completed", { output: [done], usage }) }], + ]); + return "app-openai-responses-sse"; +} + +function openAiResponses(res: http.ServerResponse, body: Json): string { + const model = typeof body.model === "string" ? body.model : "gpt-6.1-sol"; + if (!isProbe(body)) return openAiAppResponses(res, body); + json(res, 200, { + id: "resp_fake", + object: "response", + model, + status: "completed", + output: [ + { type: "message", role: "assistant", content: [{ type: "output_text", text: "pong" }] }, + ], + usage: { + input_tokens: PROBE_USAGE.input, + input_tokens_details: { cached_tokens: PROBE_USAGE.cacheRead }, + output_tokens: PROBE_USAGE.output, + output_tokens_details: { reasoning_tokens: 0 }, + }, + }); + return "probe-openai-responses"; +} + +/** Starts the fake on a free loopback port and returns its origin (no trailing slash). */ +export async function startFakeProvider( + xumRoot: string +): Promise<{ origin: string; close: () => void }> { + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + const handle = async (): Promise => { + let body: Json = {}; + try { + const raw = Buffer.concat(chunks).toString("utf8"); + body = raw ? (JSON.parse(raw) as Json) : {}; + } catch { + // Not JSON: the 404 below answers it. + } + const route = (req.url ?? "").split("?")[0]; + let kind: string; + if (req.method === "POST" && route.endsWith("/messages")) { + kind = isProbe(body) ? anthropicProbe(res, body) : await anthropicChat(res, body, xumRoot); + } else if (req.method === "POST" && route.endsWith("/chat/completions")) { + kind = openAiChat(res, body); + } else if (req.method === "POST" && route.endsWith("/responses")) { + kind = openAiResponses(res, body); + } else if (req.method === "GET" && route.endsWith("/models")) { + kind = "models"; + json(res, 200, { data: [{ id: "claude-opus-5-5", type: "model" }] }); + } else { + kind = "404"; + json(res, 404, { type: "error", error: { type: "not_found_error", message: "fake" } }); + } + const key = String(req.headers["x-api-key"] ?? req.headers.authorization ?? ""); + console.log( + `[fake-provider] ${req.method} ${route} kind=${kind} model=${String(body.model)} stream=${String(body.stream === true)} key=${key.includes("xum-proxy-") ? "PROXY-KEY-LEAKED" : key ? "xum-settings-key" : "none"}` + ); + }; + req.on("end", () => { + handle().catch((error: unknown) => { + console.log(`[fake-provider] request failed: ${String(error)}`); + if (!res.headersSent) json(res, 500, { type: "error", error: { message: "fake failed" } }); + }); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return { + origin: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + close: () => server.close(), + }; +} diff --git a/tests/bugbash/startApp.ts b/tests/bugbash/startApp.ts index d10a44959ab..24d807702de 100644 --- a/tests/bugbash/startApp.ts +++ b/tests/bugbash/startApp.ts @@ -43,6 +43,7 @@ import * as net from "net"; import * as os from "os"; import * as path from "path"; import { type AiMode, AiModeError, resolveAiMode } from "./aiMode"; +import { startFakeProvider } from "./fakeProvider"; import { seedMcpChat, writeMcpConfig } from "./mcpapps/seed"; const REPO_ROOT = path.resolve(import.meta.dir, "../.."); @@ -220,7 +221,8 @@ async function seed( projectPath: string, xumRoot: string, child: ChildProcess, - ai: AiMode + ai: AiMode, + fakeProviderOrigin: string | undefined ): Promise { await api(base, "splashScreens/markSplashScreenViewed", { splashId: "onboarding-wizard-v1" }); await api(base, "experiments/setOverride", { experimentId: "artifacts", enabled: true }); @@ -239,7 +241,9 @@ async function seed( await api(base, "config/updateModelPreferences", { defaultModel: ai.model }); } else { // The composer refuses to send without a configured provider. Mock AI never calls it, and - // the dead loopback port keeps any stray background call from leaving the machine. + // the dead loopback port keeps any stray background call from leaving the machine. The + // bash-ai-proxy scenario points both providers at the loopback fake instead. + const providerBase = fakeProviderOrigin ?? "http://127.0.0.1:9"; await api(base, "providers/setProviderConfig", { provider: "anthropic", keyPath: ["apiKey"], @@ -248,8 +252,24 @@ async function seed( await api(base, "providers/setProviderConfig", { provider: "anthropic", keyPath: ["baseUrl"], - value: "http://127.0.0.1:9/v1", + value: `${providerBase}/v1`, }); + if (fakeProviderOrigin !== undefined) { + // Chat turns on Sonnet, probes on Opus, so the Cost tab keeps them in separate rows. + await api(base, "config/updateModelPreferences", { + defaultModel: "anthropic:claude-sonnet-5-5", + }); + await api(base, "providers/setProviderConfig", { + provider: "openai", + keyPath: ["apiKey"], + value: "sk-openai-bugbash-fake", + }); + await api(base, "providers/setProviderConfig", { + provider: "openai", + keyPath: ["baseUrl"], + value: `${providerBase}/v1`, + }); + } } await api(base, "projects/create", { projectPath }); await api(base, "projects/setTrust", { projectPath, trusted: true }); @@ -314,8 +334,21 @@ async function main(): Promise { // e2e writes this line to the target's command log. console.log(`[bugbash startApp] app AI: ${ai.mode} (${ai.reason})`); + // BUGBASH_SCENARIO=bash-ai-proxy: instead of the mock, a loopback fake provider (fakeProvider.ts) + // is the chat model and the upstream of Xum's bash AI proxy. On each agent turn the fake calls + // the proxy itself with that workspace's key, so the proxy, the Cost tab and Analytics see + // real traffic while agent tools stay off. Nothing leaves the machine. It replaces the mock, + // so the explorer context (e2e.config.ts) must not describe a real model. + const scenario = process.env.BUGBASH_SCENARIO ?? ""; + if (scenario !== "" && scenario !== "bash-ai-proxy") { + fail(`BUGBASH_SCENARIO must be empty or bash-ai-proxy, got "${scenario}"`); + } + if (scenario !== "" && ai.mode !== "mock") { + fail("BUGBASH_SCENARIO=bash-ai-proxy replaces the mock: run it with BUGBASH_AI=mock"); + } const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-")); const xumRoot = path.join(tempRoot, "xum"); + const fakeProvider = scenario === "bash-ai-proxy" ? await startFakeProvider(xumRoot) : undefined; const home = path.join(tempRoot, "home"); const projectPath = path.join(tempRoot, PROJECT_NAME); const env: NodeJS.ProcessEnv = { @@ -326,13 +359,18 @@ async function main(): Promise { // Real mode: a real model answers in the app on this host. It gets no tools, and the explorer // gets no terminal or project init hooks, so injected text in a reply that the explorer // follows cannot run host commands through them (see the header). - ...(ai.mode === "mock" - ? { XUM_MOCK_AI: "1" } - : { - XUM_DISABLE_AGENT_TOOLS: "1", - XUM_DISABLE_TERMINALS: "1", - XUM_DISABLE_PROJECT_AUTOMATION: "1", - }), + ...(fakeProvider + ? // No agent tools and no terminal (AGENTS.md: not until the app runs in a sandbox). + // Project automation stays on: its kill switch also makes the proxy refuse every call, + // the fake's replies are fixed text, and the seeded demo repo has no hooks. + { XUM_DISABLE_AGENT_TOOLS: "1", XUM_DISABLE_TERMINALS: "1" } + : ai.mode === "mock" + ? { XUM_MOCK_AI: "1" } + : { + XUM_DISABLE_AGENT_TOOLS: "1", + XUM_DISABLE_TERMINALS: "1", + XUM_DISABLE_PROJECT_AUTOMATION: "1", + }), // Bug-bash clicks are not product usage. XUM_DISABLE_TELEMETRY: "1", // Git inside the app must not read or write the user's real ~/.gitconfig. @@ -361,7 +399,14 @@ async function main(): Promise { let workspaceId: string; try { await waitForHealth(`http://127.0.0.1:${seedPort}`, current); - workspaceId = await seed(`http://127.0.0.1:${seedPort}`, projectPath, xumRoot, current, ai); + workspaceId = await seed( + `http://127.0.0.1:${seedPort}`, + projectPath, + xumRoot, + current, + ai, + fakeProvider?.origin + ); } finally { current.kill("SIGTERM"); await waitForExit(current); @@ -373,6 +418,7 @@ async function main(): Promise { code = await waitForExit(current); } } finally { + fakeProvider?.close(); fs.rmSync(tempRoot, { recursive: true, force: true }); } process.exit(code ?? 0);