🤖 fix: make no provider model calls under XUM_MOCK_AI

src/node/services/aiService.ts

@@ -5,7 +5,6 @@ import * as path from "path";
 import { EventEmitter } from "events";
 import * as fs from "fs/promises";
 
-import { resolveXumEnvironmentValue } from "@/common/compat/legacyMux";
 import assert from "@/common/utils/assert";
 import { type LanguageModel, type Tool } from "ai";
 
@@ -93,6 +92,7 @@ import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries";
 import { WorkflowRunStore } from "@/node/services/workflows/WorkflowRunStore";
 import { getTokenizerForModel } from "@/node/utils/main/tokenizer";
 import { MockAiStreamPlayer } from "./mock/mockAiStreamPlayer";
+import { isMockAiMode } from "./mock/mockAiMode";
 import { ProviderModelFactory } from "./providerModelFactory";
 
 interface ToolExecutionContext {
@@ -227,7 +227,7 @@ export class AIService extends EventEmitter {
     void this.ensureSessionsDir();
     this.mockModeEnabled = false;
 
-    if (resolveXumEnvironmentValue("MOCK_AI", process.env) === "1") {
+    if (isMockAiMode()) {
       log.info("AIService running in MUX_MOCK_AI mode");
       this.enableMockMode();
     }

src/node/services/evaluationModelFactory.ts

@@ -19,6 +19,7 @@ import {
   isSessionTapeReplayMode,
   SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE,
 } from "@/node/services/sessionTapes/sessionTapeReplaySource";
+import { isMockAiMode, MOCK_AI_MODEL_REFUSED_MESSAGE } from "@/node/services/mock/mockAiMode";
 import {
   buildAIProviderRequestHeaders,
   normalizeAnthropicBaseURL,
@@ -146,6 +147,10 @@ export function createEvaluationModel(
     if (isSessionTapeReplayMode()) {
       return Err({ code: "provider_disabled", message: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE });
     }
+    // Mock AI mode (XUM_MOCK_AI, #5604): auto model routing makes no provider call either.
+    if (isMockAiMode()) {
+      return Err({ code: "provider_disabled", message: MOCK_AI_MODEL_REFUSED_MESSAGE });
+    }
     const target = resolveEvaluationModelTarget(modelString, deps);
     if (!target.success) return target;
     const { provider, modelId, settings, organization } = target.data;

src/node/services/mock/mockAiMode.ts

@@ -0,0 +1,15 @@
+import { resolveXumEnvironmentValue } from "@/common/compat/legacyMux";
+
+/** Why model creation refused while mock AI mode is on. */
+export const MOCK_AI_MODEL_REFUSED_MESSAGE =
+  "Model calls are disabled in mock AI mode (XUM_MOCK_AI=1)";
+
+/**
+ * XUM_MOCK_AI=1 (or the legacy MUX_MOCK_AI=1): AIService plays chat turns through
+ * MockAiStreamPlayer, and the model factories refuse to create provider models, so no
+ * feature (title, status, memory, compaction summary, refine, routing) sends a real
+ * provider request (#5604).
+ */
+export function isMockAiMode(): boolean {
+  return resolveXumEnvironmentValue("MOCK_AI", process.env) === "1";
+}

src/node/services/mockAiModelGate.test.ts

@@ -0,0 +1,150 @@
+import * as http from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { Effect } from "effect";
+import { Config, ProvidersConfigStore } from "@/node/config";
+import { AIService } from "./aiService";
+import { createEvaluationModel } from "./evaluationModelFactory";
+import { HistoryService } from "./historyService";
+import { InitStateManager } from "./initStateManager";
+import { ProviderService } from "./providerService";
+import { DisposableTempDir } from "./tempDir";
+import { generateWorkspaceStatus } from "./workspaceStatusGenerator";
+import { generateWorkspaceIdentity } from "./workspaceTitleGenerator";
+
+// #5604: with XUM_MOCK_AI=1 the main chat stream is mocked, but background features
+// (title, status, branch summary, memory, compaction summary, refine, auto routing)
+// still created real models and sent requests to the configured provider. Every
+// provider here points at a loopback server that records each request it receives.
+
+const MODEL = "anthropic:claude-haiku-4-5";
+const MOCK_ENV_KEYS = ["XUM_MOCK_AI", "MUX_MOCK_AI"] as const;
+
+interface Loopback {
+  requests: string[];
+  baseUrl: string;
+  close: () => Promise<void>;
+}
+
+async function startLoopbackProvider(): Promise<Loopback> {
+  const requests: string[] = [];
+  const server = http.createServer((req, res) => {
+    requests.push(`${req.method ?? "?"} ${req.url ?? "?"}`);
+    req.resume();
+    // A non-retryable 400 keeps the control case fast (no SDK retry backoff).
+    res.writeHead(400, { "content-type": "application/json" });
+    res.end(
+      JSON.stringify({
+        type: "error",
+        error: { type: "invalid_request_error", message: "loopback" },
+      })
+    );
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as AddressInfo;
+  return {
+    requests,
+    baseUrl: `http://127.0.0.1:${port}/v1`,
+    close: () => new Promise<void>((resolve) => server.close(() => resolve())),
+  };
+}
+
+function createService(root: string, baseUrl: string) {
+  const config = new Config(root);
+  const providersConfigStore = new ProvidersConfigStore(config.rootDir);
+  providersConfigStore.saveProvidersConfig({
+    anthropic: { apiKey: "test-anthropic-key", baseUrl },
+    openai: { apiKey: "test-openai-key", baseUrl },
+  });
+  const providerService = new ProviderService(config, providersConfigStore);
+  // AIService reads XUM_MOCK_AI in its constructor, like the server does at startup.
+  const service = new AIService(
+    config,
+    new HistoryService(config),
+    new InitStateManager(config),
+    providerService,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    undefined,
+    providersConfigStore
+  );
+  return { service, providersConfigStore };
+}
+
+describe("XUM_MOCK_AI model gate", () => {
+  const savedEnv = new Map<string, string | undefined>();
+  let tempDir: DisposableTempDir;
+  let loopback: Loopback;
+
+  beforeEach(async () => {
+    for (const key of MOCK_ENV_KEYS) {
+      savedEnv.set(key, process.env[key]);
+      delete process.env[key];
+    }
+    tempDir = new DisposableTempDir("mock-ai-model-gate");
+    loopback = await startLoopbackProvider();
+  });
+
+  afterEach(async () => {
+    await loopback.close();
+    tempDir[Symbol.dispose]();
+    for (const key of MOCK_ENV_KEYS) {
+      const value = savedEnv.get(key);
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+  });
+
+  it("control: without mock mode the same callers create models and reach the provider", async () => {
+    const { service, providersConfigStore } = createService(tempDir.path, loopback.baseUrl);
+
+    const result = await generateWorkspaceIdentity(
+      "Fix the login bug",
+      [{ model: MODEL }],
+      service
+    );
+    expect(result.success).toBe(false);
+    expect(loopback.requests.length).toBeGreaterThan(0);
+
+    // The entry points below succeed here, so their refusals in mock mode come from the gate.
+    expect((await service.createModel(MODEL)).success).toBe(true);
+    expect((await service.createModelWithPinnedMetadata(MODEL)).success).toBe(true);
+    expect((await service.createModelWithPinnedOptions(MODEL)).success).toBe(true);
+    expect((await service.createEvaluationModel(MODEL)).success).toBe(true);
+    const routerModel = await Effect.runPromise(
+      createEvaluationModel(MODEL, { providersConfigStore })
+    );
+    expect(routerModel.success).toBe(true);
+  });
+
+  it("with XUM_MOCK_AI=1 no background model caller reaches the provider", async () => {
+    process.env.XUM_MOCK_AI = "1";
+    const { service, providersConfigStore } = createService(tempDir.path, loopback.baseUrl);
+
+    // Background generators, end to end.
+    const title = await generateWorkspaceIdentity("Fix the login bug", [{ model: MODEL }], service);
+    expect(title.success).toBe(false);
+    const status = await generateWorkspaceStatus("user: fix the login bug", [MODEL], service);
+    expect(status.success).toBe(false);
+    if (!status.success) expect(status.error.reachedProvider).toBe(false);
+
+    // Every model entry point those generators and the other background callers use
+    // (branch summary, memory consolidation and harvest, continuous compaction
+    // summary, refine, workflow evaluate, auto model routing).
+    expect((await service.createModel(MODEL)).success).toBe(false);
+    expect((await service.createModelWithPinnedMetadata(MODEL)).success).toBe(false);
+    expect((await service.createModelWithPinnedOptions(MODEL)).success).toBe(false);
+    expect((await service.createEvaluationModel(MODEL)).success).toBe(false);
+    const routerModel = await Effect.runPromise(
+      createEvaluationModel(MODEL, { providersConfigStore })
+    );
+    expect(routerModel.success).toBe(false);
+
+    expect(loopback.requests).toEqual([]);
+  });
+});

src/node/services/providerModelFactory.ts

@@ -109,6 +109,7 @@ import {
   isSessionTapeReplayMode,
   SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE,
 } from "@/node/services/sessionTapes/sessionTapeReplaySource";
+import { isMockAiMode, MOCK_AI_MODEL_REFUSED_MESSAGE } from "@/node/services/mock/mockAiMode";
 import { resolveProviderOptionsNamespaceKey } from "@/common/utils/ai/providerOptions";
 import { resolveRoute, type RouteContext } from "@/common/routing";
 import {
@@ -1472,6 +1473,11 @@ export class ProviderModelFactory {
           message: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE,
         });
       }
+      // Mock AI mode (XUM_MOCK_AI, #5604): the same choke point keeps background features
+      // from reaching a real provider while chat turns are mocked.
+      if (isMockAiMode()) {
+        return Err<SendMessageError>({ type: "unknown", raw: MOCK_AI_MODEL_REFUSED_MESSAGE });
+      }
       const result = yield* self.createModelCoreEffect(modelString, muxProviderOptions, opts);
       if (!result.success) {
         return result;
@@ -2903,6 +2909,10 @@ export class ProviderModelFactory {
         log.warn("Evaluation model refused", { error: SESSION_TAPE_REPLAY_READ_ONLY_MESSAGE });
         return Err<EvaluationResolveError>({ reason: "unsupported-route" });
       }
+      if (isMockAiMode()) {
+        log.debug("Evaluation model refused", { error: MOCK_AI_MODEL_REFUSED_MESSAGE });
+        return Err<EvaluationResolveError>({ reason: "unsupported-route" });
+      }
       // ONE providers.jsonc read for the shadow check, routing, credentials and
       // construction (same single-snapshot rule as resolveAndCreateModelEffect).
       const providersConfig = self.providersConfigStore.loadProvidersConfig() ?? {};