diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 79243ea5b5f..dcf6fbdd8ab 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -5,7 +5,6 @@ import * as path from "path"; import { EventEmitter } from "events"; import * as fs from "fs/promises"; -import { resolveXumEnvironmentValue } from "@/common/compat/legacyMux"; import assert from "@/common/utils/assert"; import { type LanguageModel, type Tool } from "ai"; @@ -93,6 +92,7 @@ import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries"; import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore"; import { getTokenizerForModel } from "@/node/utils/main/tokenizer"; import { MockAiStreamPlayer } from "./mock/mockAiStreamPlayer"; +import { isMockAiMode } from "./mock/mockAiMode"; import { ProviderModelFactory } from "./providerModelFactory"; interface ToolExecutionContext { @@ -227,7 +227,7 @@ export class AIService extends EventEmitter { void this.ensureSessionsDir(); this.mockModeEnabled = false; - if (resolveXumEnvironmentValue("MOCK_AI", process.env) === "1") { + if (isMockAiMode()) { log.info("AIService running in MUX_MOCK_AI mode"); this.enableMockMode(); } diff --git a/src/node/services/evaluationModelFactory.ts b/src/node/services/evaluationModelFactory.ts index 0fdcb564eb7..87e21c094a9 100644 --- a/src/node/services/evaluationModelFactory.ts +++ b/src/node/services/evaluationModelFactory.ts @@ -19,6 +19,7 @@ import { isSessionTapeReplayMode, SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE, } from "@/node/services/sessionTapes/sessionTapeReplaySource"; +import { isMockAiMode, MOCK_AI_MODEL_REFUSED_MESSAGE } from "@/node/services/mock/mockAiMode"; import { buildAIProviderRequestHeaders, normalizeAnthropicBaseURL, @@ -146,6 +147,10 @@ export function createEvaluationModel( if (isSessionTapeReplayMode()) { return Err({ code: "provider_disabled", message: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE }); } + // Mock AI mode (XUM_MOCK_AI, #5604): auto model routing makes no provider call either. + if (isMockAiMode()) { + return Err({ code: "provider_disabled", message: MOCK_AI_MODEL_REFUSED_MESSAGE }); + } const target = resolveEvaluationModelTarget(modelString, deps); if (!target.success) return target; const { provider, modelId, settings, organization } = target.data; diff --git a/src/node/services/mock/mockAiMode.ts b/src/node/services/mock/mockAiMode.ts new file mode 100644 index 00000000000..5094319352f --- /dev/null +++ b/src/node/services/mock/mockAiMode.ts @@ -0,0 +1,15 @@ +import { resolveXumEnvironmentValue } from "@/common/compat/legacyMux"; + +/** Why model creation refused while mock AI mode is on. */ +export const MOCK_AI_MODEL_REFUSED_MESSAGE = + "Model calls are disabled in mock AI mode (XUM_MOCK_AI=1)"; + +/** + * XUM_MOCK_AI=1 (or the legacy MUX_MOCK_AI=1): AIService plays chat turns through + * MockAiStreamPlayer, and the model factories refuse to create provider models, so no + * feature (title, status, memory, compaction summary, refine, routing) sends a real + * provider request (#5604). + */ +export function isMockAiMode(): boolean { + return resolveXumEnvironmentValue("MOCK_AI", process.env) === "1"; +} diff --git a/src/node/services/mockAiModelGate.test.ts b/src/node/services/mockAiModelGate.test.ts new file mode 100644 index 00000000000..130ae86aeed --- /dev/null +++ b/src/node/services/mockAiModelGate.test.ts @@ -0,0 +1,150 @@ +import * as http from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { Effect } from "effect"; +import { Config, ProvidersConfigStore } from "@/node/config"; +import { AIService } from "./aiService"; +import { createEvaluationModel } from "./evaluationModelFactory"; +import { HistoryService } from "./historyService"; +import { InitStateManager } from "./initStateManager"; +import { ProviderService } from "./providerService"; +import { DisposableTempDir } from "./tempDir"; +import { generateWorkspaceStatus } from "./workspaceStatusGenerator"; +import { generateWorkspaceIdentity } from "./workspaceTitleGenerator"; + +// #5604: with XUM_MOCK_AI=1 the main chat stream is mocked, but background features +// (title, status, branch summary, memory, compaction summary, refine, auto routing) +// still created real models and sent requests to the configured provider. Every +// provider here points at a loopback server that records each request it receives. + +const MODEL = "anthropic:claude-haiku-4-5"; +const MOCK_ENV_KEYS = ["XUM_MOCK_AI", "MUX_MOCK_AI"] as const; + +interface Loopback { + requests: string[]; + baseUrl: string; + close: () => Promise; +} + +async function startLoopbackProvider(): Promise { + const requests: string[] = []; + const server = http.createServer((req, res) => { + requests.push(`${req.method ?? "?"} ${req.url ?? "?"}`); + req.resume(); + // A non-retryable 400 keeps the control case fast (no SDK retry backoff). + res.writeHead(400, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + type: "error", + error: { type: "invalid_request_error", message: "loopback" }, + }) + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + requests, + baseUrl: `http://127.0.0.1:${port}/v1`, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +function createService(root: string, baseUrl: string) { + const config = new Config(root); + const providersConfigStore = new ProvidersConfigStore(config.rootDir); + providersConfigStore.saveProvidersConfig({ + anthropic: { apiKey: "test-anthropic-key", baseUrl }, + openai: { apiKey: "test-openai-key", baseUrl }, + }); + const providerService = new ProviderService(config, providersConfigStore); + // AIService reads XUM_MOCK_AI in its constructor, like the server does at startup. + const service = new AIService( + config, + new HistoryService(config), + new InitStateManager(config), + providerService, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + providersConfigStore + ); + return { service, providersConfigStore }; +} + +describe("XUM_MOCK_AI model gate", () => { + const savedEnv = new Map(); + let tempDir: DisposableTempDir; + let loopback: Loopback; + + beforeEach(async () => { + for (const key of MOCK_ENV_KEYS) { + savedEnv.set(key, process.env[key]); + delete process.env[key]; + } + tempDir = new DisposableTempDir("mock-ai-model-gate"); + loopback = await startLoopbackProvider(); + }); + + afterEach(async () => { + await loopback.close(); + tempDir[Symbol.dispose](); + for (const key of MOCK_ENV_KEYS) { + const value = savedEnv.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + + it("control: without mock mode the same callers create models and reach the provider", async () => { + const { service, providersConfigStore } = createService(tempDir.path, loopback.baseUrl); + + const result = await generateWorkspaceIdentity( + "Fix the login bug", + [{ model: MODEL }], + service + ); + expect(result.success).toBe(false); + expect(loopback.requests.length).toBeGreaterThan(0); + + // The entry points below succeed here, so their refusals in mock mode come from the gate. + expect((await service.createModel(MODEL)).success).toBe(true); + expect((await service.createModelWithPinnedMetadata(MODEL)).success).toBe(true); + expect((await service.createModelWithPinnedOptions(MODEL)).success).toBe(true); + expect((await service.createEvaluationModel(MODEL)).success).toBe(true); + const routerModel = await Effect.runPromise( + createEvaluationModel(MODEL, { providersConfigStore }) + ); + expect(routerModel.success).toBe(true); + }); + + it("with XUM_MOCK_AI=1 no background model caller reaches the provider", async () => { + process.env.XUM_MOCK_AI = "1"; + const { service, providersConfigStore } = createService(tempDir.path, loopback.baseUrl); + + // Background generators, end to end. + const title = await generateWorkspaceIdentity("Fix the login bug", [{ model: MODEL }], service); + expect(title.success).toBe(false); + const status = await generateWorkspaceStatus("user: fix the login bug", [MODEL], service); + expect(status.success).toBe(false); + if (!status.success) expect(status.error.reachedProvider).toBe(false); + + // Every model entry point those generators and the other background callers use + // (branch summary, memory consolidation and harvest, continuous compaction + // summary, refine, workflow evaluate, auto model routing). + expect((await service.createModel(MODEL)).success).toBe(false); + expect((await service.createModelWithPinnedMetadata(MODEL)).success).toBe(false); + expect((await service.createModelWithPinnedOptions(MODEL)).success).toBe(false); + expect((await service.createEvaluationModel(MODEL)).success).toBe(false); + const routerModel = await Effect.runPromise( + createEvaluationModel(MODEL, { providersConfigStore }) + ); + expect(routerModel.success).toBe(false); + + expect(loopback.requests).toEqual([]); + }); +}); diff --git a/src/node/services/providerModelFactory.ts b/src/node/services/providerModelFactory.ts index 1ebd25a2e77..88163d57891 100644 --- a/src/node/services/providerModelFactory.ts +++ b/src/node/services/providerModelFactory.ts @@ -109,6 +109,7 @@ import { isSessionTapeReplayMode, SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE, } from "@/node/services/sessionTapes/sessionTapeReplaySource"; +import { isMockAiMode, MOCK_AI_MODEL_REFUSED_MESSAGE } from "@/node/services/mock/mockAiMode"; import { resolveProviderOptionsNamespaceKey } from "@/common/utils/ai/providerOptions"; import { resolveRoute, type RouteContext } from "@/common/routing"; import { @@ -1472,6 +1473,11 @@ export class ProviderModelFactory { message: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE, }); } + // Mock AI mode (XUM_MOCK_AI, #5604): the same choke point keeps background features + // from reaching a real provider while chat turns are mocked. + if (isMockAiMode()) { + return Err({ type: "unknown", raw: MOCK_AI_MODEL_REFUSED_MESSAGE }); + } const result = yield* self.createModelCoreEffect(modelString, muxProviderOptions, opts); if (!result.success) { return result; @@ -2903,6 +2909,10 @@ export class ProviderModelFactory { log.warn("Evaluation model refused", { error: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE }); return Err({ reason: "unsupported-route" }); } + if (isMockAiMode()) { + log.debug("Evaluation model refused", { error: MOCK_AI_MODEL_REFUSED_MESSAGE }); + return Err({ reason: "unsupported-route" }); + } // ONE providers.jsonc read for the shadow check, routing, credentials and // construction (same single-snapshot rule as resolveAndCreateModelEffect). const providersConfig = self.providersConfigStore.loadProvidersConfig() ?? {};