feat(debug): add capture trace on error admin option

packages/backend/src/db/config-repository.ts

@@ -1529,6 +1529,10 @@ export class ConfigRepository {
     return { enabled, retryableStatusCodes, retryableErrors };
   }
 
+  async getCaptureTraceOnError(): Promise<boolean> {
+    return this.getSetting<boolean>('debug.captureOnError', false);
+  }
+
   async getCooldownPolicy(): Promise<CooldownPolicy> {
     const initialMinutes = await this.getSetting<number>('cooldown.initialMinutes', 2);
     const maxMinutes = await this.getSetting<number>('cooldown.maxMinutes', 300);

packages/backend/src/index.ts

@@ -170,6 +170,11 @@ try {
   await configService.initialize();
   logger.debug('Configuration loaded from database');
 
+  // Restore the persisted "capture trace on error" toggle into DebugManager.
+  DebugManager.getInstance().setCaptureOnError(
+    await configService.getRepository().getCaptureTraceOnError()
+  );
+
   // Register pi_ai_custom_providers with the piAiModels instance so custom
   // providers can be dispatched directly by api type without remapping hacks.
   const { registerCustomProvidersWithPiAi } = await import('./inference-v2/shared/pi-ai-utils');

packages/backend/src/routes/management/config.ts

@@ -9,6 +9,7 @@ import {
   normalizeKeyConfig,
 } from '../../config';
 import { ConfigService } from '../../services/config-service';
+import { DebugManager } from '../../services/debug-manager';
 import { isValidIpRule } from '../../utils/ip-match';
 import { getCheckerDefinitions } from '../../services/quota/checker-registry';
 import { UsageStorageService } from '../../services/usage-storage';
@@ -499,6 +500,42 @@ export async function registerConfigRoutes(
     }
   });
 
+  // ─── Capture Trace on Error ──────────────────────────────────────
+  // Persisted admin toggle. When enabled, DebugManager captures traces for
+  // every request and persists them only when a request writes an inference
+  // error or triggers a cooldown, even if global/per-key debug is off.
+
+  fastify.get('/v0/management/config/capture-trace-on-error', async (_request, reply) => {
+    try {
+      const enabled = await configService.getRepository().getCaptureTraceOnError();
+      return reply.send({ enabled });
+    } catch (e: any) {
+      return reply.code(500).send({ error: 'Internal server error' });
+    }
+  });
+
+  fastify.patch('/v0/management/config/capture-trace-on-error', async (request, reply) => {
+    const body = request.body as { enabled?: unknown } | null;
+    if (!body || typeof body !== 'object' || Array.isArray(body)) {
+      return reply.code(400).send({ error: 'Object body is required' });
+    }
+    if (typeof body.enabled !== 'boolean') {
+      return reply.code(400).send({ error: 'enabled must be a boolean' });
+    }
+
+    try {
+      // Persist to the database, then mirror to the in-memory DebugManager so
+      // the change takes effect immediately without a restart.
+      await configService.setSetting('debug.captureOnError', body.enabled);
+      DebugManager.getInstance().setCaptureOnError(body.enabled);
+      logger.debug('Capture-trace-on-error updated via API');
+      return reply.send({ enabled: body.enabled });
+    } catch (e: any) {
+      logger.error('Failed to patch capture-trace-on-error config', e);
+      return reply.code(500).send({ error: 'Internal server error' });
+    }
+  });
+
   // ─── Trusted Proxies ──────────────────────────────────────────────
   fastify.get('/v0/management/config/trusted-proxies', async (_request, reply) => {
     try {

packages/backend/src/services/cooldown-manager.ts

@@ -2,6 +2,8 @@ import { logger } from '../utils/logger';
 import { getDatabase, getSchema } from '../db/client';
 import { lt, eq, sql, and, desc } from 'drizzle-orm';
 import { getConfig } from '../config';
+import { DebugManager } from './debug-manager';
+import { getCurrentRequestId } from './request-context';
 
 export interface Target {
   provider: string;
@@ -200,6 +202,11 @@ export class CooldownManager {
 
     this.cooldowns.set(key, { expiry, consecutiveFailures, lastError });
 
+    // In capture-on-error mode, persist the triggering request's debug trace.
+    // Resolved from the async-local context since this path has no requestId;
+    // no-op when the mode is off or outside a request context (e.g. probes).
+    DebugManager.getInstance().markForcePersist(getCurrentRequestId());
+
     logger.warn(
       `Provider '${provider}' model '${model}' placed on cooldown for ${duration / 1000}s ` +
         `(failure #${consecutiveFailures}) until ${new Date(expiry).toISOString()}`

packages/backend/src/services/debug-manager.ts

@@ -2,7 +2,7 @@ import { UsageStorageService } from './usage-storage';
 import { logger } from '../utils/logger';
 import { createParser, EventSourceMessage } from 'eventsource-parser';
 import { encode } from 'eventsource-encoder';
-import { getCurrentKeyName } from './request-context';
+import { getCurrentKeyName, setCurrentRequestId } from './request-context';
 
 export interface DebugLogRecord {
   requestId: string;
@@ -18,12 +18,19 @@ export interface DebugLogRecord {
   responseStatus?: number;
   provider?: string;
   createdAt?: number;
+  /**
+   * When true, this log is persisted on flush even if debug capture is not
+   * otherwise enabled for its key. Set by the "capture trace on error" mode
+   * when a request writes an inference error or triggers a cooldown.
+   */
+  forcePersist?: boolean;
 }
 
 export class DebugManager {
   private static instance: DebugManager;
   private storage: UsageStorageService | null = null;
   private enabledGlobal: boolean = false;
+  private captureOnError: boolean = false;
   private enabledKeys: Set<string> = new Set();
   private providerFilter: string[] | null = null;
   private pendingLogs: Map<string, DebugLogRecord> = new Map();
@@ -52,6 +59,19 @@ export class DebugManager {
     return this.enabledGlobal;
   }
 
+  // ─── Capture-on-error toggle ────────────────────────────────────
+  // When enabled, traces are captured in memory for every request but only
+  // persisted if the request writes an inference error or triggers a cooldown
+  // (see markForcePersist). Successful requests are discarded on flush.
+  setCaptureOnError(enabled: boolean) {
+    this.captureOnError = enabled;
+    logger.warn(`Debug capture-on-error ${enabled ? 'enabled' : 'disabled'}`);
+  }
+
+  isCaptureOnError(): boolean {
+    return this.captureOnError;
+  }
+
   // ─── Per-key toggle ─────────────────────────────────────────────
   enableForKey(keyName: string): void {
     this.enabledKeys.add(keyName);
@@ -74,7 +94,9 @@ export class DebugManager {
    * Reads the key name from the request context when one is available.
    */
   isCaptureEnabled(): boolean {
-    return this.isEnabledForKey(getCurrentKeyName() ?? null);
+    // Capture-on-error mode buffers every request so a trace exists to persist
+    // if the request later errors or triggers a cooldown.
+    return this.captureOnError || this.isEnabledForKey(getCurrentKeyName() ?? null);
   }
 
   getEnabledKeys(): string[] {
@@ -109,6 +131,9 @@ export class DebugManager {
 
   // ─── Log capture ────────────────────────────────────────────────
   startLog(requestId: string, rawRequest: any, requestHeaders?: Record<string, string | string[]>) {
+    // Seed the request id into the async-local context so the cooldown path
+    // (which lacks a requestId) can force-persist this request's trace.
+    setCurrentRequestId(requestId);
     if (!this.isCaptureEnabled()) return;
     this.pendingLogs.set(requestId, {
       requestId,
@@ -193,8 +218,9 @@ export class DebugManager {
     const log = this.pendingLogs.get(requestId);
     if (!log) return;
 
-    // Only persist to database if debug mode is enabled for this request's key
-    if (!this.isEnabledForKey(log.apiKey ?? null)) {
+    // Persist if debug mode is enabled for this request's key, or if the
+    // request was flagged for forced persistence (capture-on-error mode).
+    if (!this.isEnabledForKey(log.apiKey ?? null) && !log.forcePersist) {
       logger.debug(
         `Skipping flush for ${requestId} - debug mode not enabled for key '${log.apiKey ?? '(none)'}'`
       );
@@ -218,6 +244,18 @@ export class DebugManager {
     this.pendingLogs.delete(requestId);
   }
 
+  /**
+   * Flag a request's pending trace for persistence on flush even when debug
+   * capture is not otherwise enabled for its key. Used by capture-on-error
+   * mode when a request writes an inference error or triggers a cooldown.
+   * No-op unless capture-on-error is enabled and a pending log exists.
+   */
+  markForcePersist(requestId: string | undefined | null): void {
+    if (!this.captureOnError || !requestId) return;
+    const log = this.pendingLogs.get(requestId);
+    if (log) log.forcePersist = true;
+  }
+
   /**
    * Mark a request as ephemeral (debug data won't be persisted)
    */

packages/backend/src/services/request-context.ts

@@ -9,6 +9,7 @@ import { AsyncLocalStorage } from 'node:async_hooks';
  */
 export interface RequestContext {
   keyName?: string;
+  requestId?: string;
 }
 
 const storage = new AsyncLocalStorage<RequestContext>();
@@ -36,6 +37,26 @@ export function getCurrentKeyName(): string | undefined {
   return storage.getStore()?.keyName;
 }
 
+/**
+ * Read just the request id from the active request context.
+ */
+export function getCurrentRequestId(): string | undefined {
+  return storage.getStore()?.requestId;
+}
+
+/**
+ * Attach the request id to the active request context so downstream code
+ * (notably DebugManager, reached via the cooldown path) can resolve the
+ * request id without explicit plumbing. Replaces the store via `enterWith`
+ * rather than mutating the existing object in place, so any other holder of
+ * a reference to the prior store is unaffected. No-op when called outside a
+ * request context.
+ */
+export function setCurrentRequestId(requestId: string): void {
+  const store = storage.getStore();
+  if (store) storage.enterWith({ ...store, requestId });
+}
+
 /**
  * Run a callback with a given request context. Prefer this wrapper in tests
  * or short-lived scopes; long-lived Fastify handlers should use

packages/backend/src/services/usage-storage.ts

@@ -4,7 +4,7 @@ import { getDatabase, getSchema } from '../db/client';
 import { NewRequestUsage } from '../db/types';
 import { EventEmitter } from 'node:events';
 import { eq, and, gte, lte, like, desc, asc, sql, getTableName } from 'drizzle-orm';
-import { DebugLogRecord } from './debug-manager';
+import { DebugLogRecord, DebugManager } from './debug-manager';
 import { getCurrentKeyName } from './request-context';
 import { estimateKwhUsed } from './inference-energy';
 import { resolveModelParams, DEFAULT_GPU_PARAMS } from '@plexus/shared';
@@ -357,6 +357,10 @@ export class UsageStorageService extends EventEmitter {
         });
 
       logger.debug(`Inference error saved for request ${requestId}`);
+
+      // In capture-on-error mode, persist this request's debug trace even if
+      // debug capture isn't otherwise enabled. No-op when the mode is off.
+      DebugManager.getInstance().markForcePersist(requestId);
     } catch (e) {
       logger.error('Failed to save inference error', e);
     }

packages/frontend/src/lib/api.ts

@@ -3446,6 +3446,26 @@ export const api = {
     return res.json();
   },
 
+  // ─── Capture Trace on Error ─────────────────────────────────────────
+
+  /** Fetch whether traces are captured on error even when debug is off. */
+  getCaptureTraceOnError: async (): Promise<{ enabled: boolean }> => {
+    const res = await fetchWithAuth(`${API_BASE}/v0/management/config/capture-trace-on-error`);
+    if (!res.ok) throw new Error('Failed to fetch capture-trace-on-error setting');
+    return res.json();
+  },
+
+  /** Toggle capturing traces on error even when debug is off. */
+  setCaptureTraceOnError: async (enabled: boolean): Promise<{ enabled: boolean }> => {
+    const res = await fetchWithAuth(`${API_BASE}/v0/management/config/capture-trace-on-error`, {
+      method: 'PATCH',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ enabled }),
+    });
+    if (!res.ok) throw new Error('Failed to update capture-trace-on-error setting');
+    return res.json();
+  },
+
   // ─── Cooldown Settings ──────────────────────────────────────────────
 
   /** Fetch current cooldown policy. */

packages/frontend/src/pages/Config.tsx

@@ -147,6 +147,11 @@ export const Config = () => {
   const [statusCodesText, setStatusCodesText] = useState('');
   const [errorsText, setErrorsText] = useState('');
 
+  // Capture-trace-on-error toggle (persisted admin setting)
+  const [captureTraceOnError, setCaptureTraceOnError] = useState(false);
+  const [captureTraceLoaded, setCaptureTraceLoaded] = useState(false);
+  const [captureTraceSaving, setCaptureTraceSaving] = useState(false);
+
   // Trusted proxies (which immediate peers' forwarding headers are believed)
   const [trustedProxies, setTrustedProxies] = useState<string[]>([]);
   const [trustedProxiesLoaded, setTrustedProxiesLoaded] = useState(false);
@@ -348,6 +353,33 @@ export const Config = () => {
     }
   }, [toast]);
 
+  const loadCaptureTraceOnError = useCallback(async () => {
+    try {
+      const { enabled } = await api.getCaptureTraceOnError();
+      setCaptureTraceOnError(enabled);
+      setCaptureTraceLoaded(true);
+    } catch (e) {
+      console.error('Failed to load capture-trace-on-error setting:', e);
+      toast.error('Failed to load trace capture settings');
+    }
+  }, [toast]);
+
+  const handleToggleCaptureTraceOnError = async (checked: boolean) => {
+    const previous = captureTraceOnError;
+    setCaptureTraceOnError(checked);
+    setCaptureTraceSaving(true);
+    try {
+      const { enabled } = await api.setCaptureTraceOnError(checked);
+      setCaptureTraceOnError(enabled);
+      toast.success(`Capture trace on error ${enabled ? 'enabled' : 'disabled'}`);
+    } catch (e) {
+      setCaptureTraceOnError(previous);
+      toast.error((e as Error).message, 'Failed to update trace capture settings');
+    } finally {
+      setCaptureTraceSaving(false);
+    }
+  };
+
   const loadCooldownPolicy = useCallback(async () => {
     try {
       const policy = await api.getCooldownPolicy();
@@ -664,6 +696,7 @@ export const Config = () => {
   useEffect(() => {
     loadConfig();
     loadFailoverPolicy();
+    loadCaptureTraceOnError();
     loadCooldownPolicy();
     loadTrustedProxies();
     loadTimeoutConfig();
@@ -980,6 +1013,30 @@ export const Config = () => {
             </div>
           </Disclosure>
 
+          {/* ─── Trace Capture ──────────────────────────────────────── */}
+          <Disclosure title="Trace Capture" defaultOpen={false}>
+            <div className="flex items-center justify-between">
+              <div className="flex items-center gap-2">
+                <AlertTriangle size={16} className="text-primary" />
+                <div>
+                  <p className="font-body text-[12px] font-medium text-text">
+                    Capture Trace on Error
+                  </p>
+                  <p className="font-body text-[11px] text-text-muted">
+                    When enabled, debug traces are stored for requests that write to the inference
+                    error log or trigger a cooldown, even while global debug tracing is off.
+                  </p>
+                </div>
+              </div>
+              <Switch
+                checked={captureTraceOnError}
+                onChange={handleToggleCaptureTraceOnError}
+                disabled={!captureTraceLoaded || captureTraceSaving}
+                aria-label="Toggle capture trace on error"
+              />
+            </div>
+          </Disclosure>
+
           {/* ─── Cooldown Settings ──────────────────────────────────── */}
           <Disclosure
             title="Cooldown Settings"