From 39b06a81335e4d6733f04cd9fa039e4715538b4b Mon Sep 17 00:00:00 2001 From: Matt Cowger Date: Sat, 4 Jul 2026 05:12:19 +0000 Subject: [PATCH 1/2] feat(debug): add capture trace on error admin option Add a persisted admin toggle that forces debug trace capture and persistence for a request whenever it writes to the inference error log or triggers a provider cooldown, even when global/per-key debug tracing is disabled. Surfaced on both the Config and Debug pages. --- packages/backend/src/db/config-repository.ts | 4 ++ packages/backend/src/index.ts | 5 ++ .../backend/src/routes/management/config.ts | 37 ++++++++++++ .../backend/src/services/cooldown-manager.ts | 7 +++ .../backend/src/services/debug-manager.ts | 46 +++++++++++++-- .../backend/src/services/request-context.ts | 19 +++++++ .../backend/src/services/usage-storage.ts | 6 +- packages/frontend/src/lib/api.ts | 20 +++++++ packages/frontend/src/pages/Config.tsx | 57 +++++++++++++++++++ packages/frontend/src/pages/Debug.tsx | 43 +++++++++++++- 10 files changed, 238 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/db/config-repository.ts b/packages/backend/src/db/config-repository.ts index 749f7c8e3..daff9c929 100644 --- a/packages/backend/src/db/config-repository.ts +++ b/packages/backend/src/db/config-repository.ts @@ -1529,6 +1529,10 @@ export class ConfigRepository { return { enabled, retryableStatusCodes, retryableErrors }; } + async getCaptureTraceOnError(): Promise { + return this.getSetting('debug.captureOnError', false); + } + async getCooldownPolicy(): Promise { const initialMinutes = await this.getSetting('cooldown.initialMinutes', 2); const maxMinutes = await this.getSetting('cooldown.maxMinutes', 300); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 7c5a4b52e..311ff5c7e 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -170,6 +170,11 @@ try { await configService.initialize(); logger.debug('Configuration loaded from database'); + // Restore the persisted "capture trace on error" toggle into DebugManager. + DebugManager.getInstance().setCaptureOnError( + await configService.getRepository().getCaptureTraceOnError() + ); + // Register pi_ai_custom_providers with the piAiModels instance so custom // providers can be dispatched directly by api type without remapping hacks. const { registerCustomProvidersWithPiAi } = await import('./inference-v2/shared/pi-ai-utils'); diff --git a/packages/backend/src/routes/management/config.ts b/packages/backend/src/routes/management/config.ts index 8a667d2fd..b1ebd035d 100644 --- a/packages/backend/src/routes/management/config.ts +++ b/packages/backend/src/routes/management/config.ts @@ -9,6 +9,7 @@ import { normalizeKeyConfig, } from '../../config'; import { ConfigService } from '../../services/config-service'; +import { DebugManager } from '../../services/debug-manager'; import { isValidIpRule } from '../../utils/ip-match'; import { getCheckerDefinitions } from '../../services/quota/checker-registry'; import { UsageStorageService } from '../../services/usage-storage'; @@ -499,6 +500,42 @@ export async function registerConfigRoutes( } }); + // ─── Capture Trace on Error ────────────────────────────────────── + // Persisted admin toggle. When enabled, DebugManager captures traces for + // every request and persists them only when a request writes an inference + // error or triggers a cooldown, even if global/per-key debug is off. + + fastify.get('/v0/management/config/capture-trace-on-error', async (_request, reply) => { + try { + const enabled = await configService.getRepository().getCaptureTraceOnError(); + return reply.send({ enabled }); + } catch (e: any) { + return reply.code(500).send({ error: 'Internal server error' }); + } + }); + + fastify.patch('/v0/management/config/capture-trace-on-error', async (request, reply) => { + const body = request.body as { enabled?: unknown } | null; + if (!body || typeof body !== 'object' || Array.isArray(body)) { + return reply.code(400).send({ error: 'Object body is required' }); + } + if (typeof body.enabled !== 'boolean') { + return reply.code(400).send({ error: 'enabled must be a boolean' }); + } + + try { + // Persist to the database, then mirror to the in-memory DebugManager so + // the change takes effect immediately without a restart. + await configService.setSetting('debug.captureOnError', body.enabled); + DebugManager.getInstance().setCaptureOnError(body.enabled); + logger.debug('Capture-trace-on-error updated via API'); + return reply.send({ enabled: body.enabled }); + } catch (e: any) { + logger.error('Failed to patch capture-trace-on-error config', e); + return reply.code(500).send({ error: 'Internal server error' }); + } + }); + // ─── Trusted Proxies ────────────────────────────────────────────── fastify.get('/v0/management/config/trusted-proxies', async (_request, reply) => { try { diff --git a/packages/backend/src/services/cooldown-manager.ts b/packages/backend/src/services/cooldown-manager.ts index f67034e10..d45439bda 100644 --- a/packages/backend/src/services/cooldown-manager.ts +++ b/packages/backend/src/services/cooldown-manager.ts @@ -2,6 +2,8 @@ import { logger } from '../utils/logger'; import { getDatabase, getSchema } from '../db/client'; import { lt, eq, sql, and, desc } from 'drizzle-orm'; import { getConfig } from '../config'; +import { DebugManager } from './debug-manager'; +import { getCurrentRequestId } from './request-context'; export interface Target { provider: string; @@ -200,6 +202,11 @@ export class CooldownManager { this.cooldowns.set(key, { expiry, consecutiveFailures, lastError }); + // In capture-on-error mode, persist the triggering request's debug trace. + // Resolved from the async-local context since this path has no requestId; + // no-op when the mode is off or outside a request context (e.g. probes). + DebugManager.getInstance().markForcePersist(getCurrentRequestId()); + logger.warn( `Provider '${provider}' model '${model}' placed on cooldown for ${duration / 1000}s ` + `(failure #${consecutiveFailures}) until ${new Date(expiry).toISOString()}` diff --git a/packages/backend/src/services/debug-manager.ts b/packages/backend/src/services/debug-manager.ts index f0503cece..d65c0b4eb 100644 --- a/packages/backend/src/services/debug-manager.ts +++ b/packages/backend/src/services/debug-manager.ts @@ -2,7 +2,7 @@ import { UsageStorageService } from './usage-storage'; import { logger } from '../utils/logger'; import { createParser, EventSourceMessage } from 'eventsource-parser'; import { encode } from 'eventsource-encoder'; -import { getCurrentKeyName } from './request-context'; +import { getCurrentKeyName, setCurrentRequestId } from './request-context'; export interface DebugLogRecord { requestId: string; @@ -18,12 +18,19 @@ export interface DebugLogRecord { responseStatus?: number; provider?: string; createdAt?: number; + /** + * When true, this log is persisted on flush even if debug capture is not + * otherwise enabled for its key. Set by the "capture trace on error" mode + * when a request writes an inference error or triggers a cooldown. + */ + forcePersist?: boolean; } export class DebugManager { private static instance: DebugManager; private storage: UsageStorageService | null = null; private enabledGlobal: boolean = false; + private captureOnError: boolean = false; private enabledKeys: Set = new Set(); private providerFilter: string[] | null = null; private pendingLogs: Map = new Map(); @@ -52,6 +59,19 @@ export class DebugManager { return this.enabledGlobal; } + // ─── Capture-on-error toggle ──────────────────────────────────── + // When enabled, traces are captured in memory for every request but only + // persisted if the request writes an inference error or triggers a cooldown + // (see markForcePersist). Successful requests are discarded on flush. + setCaptureOnError(enabled: boolean) { + this.captureOnError = enabled; + logger.warn(`Debug capture-on-error ${enabled ? 'enabled' : 'disabled'}`); + } + + isCaptureOnError(): boolean { + return this.captureOnError; + } + // ─── Per-key toggle ───────────────────────────────────────────── enableForKey(keyName: string): void { this.enabledKeys.add(keyName); @@ -74,7 +94,9 @@ export class DebugManager { * Reads the key name from the request context when one is available. */ isCaptureEnabled(): boolean { - return this.isEnabledForKey(getCurrentKeyName() ?? null); + // Capture-on-error mode buffers every request so a trace exists to persist + // if the request later errors or triggers a cooldown. + return this.captureOnError || this.isEnabledForKey(getCurrentKeyName() ?? null); } getEnabledKeys(): string[] { @@ -109,6 +131,9 @@ export class DebugManager { // ─── Log capture ──────────────────────────────────────────────── startLog(requestId: string, rawRequest: any, requestHeaders?: Record) { + // Seed the request id into the async-local context so the cooldown path + // (which lacks a requestId) can force-persist this request's trace. + setCurrentRequestId(requestId); if (!this.isCaptureEnabled()) return; this.pendingLogs.set(requestId, { requestId, @@ -193,8 +218,9 @@ export class DebugManager { const log = this.pendingLogs.get(requestId); if (!log) return; - // Only persist to database if debug mode is enabled for this request's key - if (!this.isEnabledForKey(log.apiKey ?? null)) { + // Persist if debug mode is enabled for this request's key, or if the + // request was flagged for forced persistence (capture-on-error mode). + if (!this.isEnabledForKey(log.apiKey ?? null) && !log.forcePersist) { logger.debug( `Skipping flush for ${requestId} - debug mode not enabled for key '${log.apiKey ?? '(none)'}'` ); @@ -218,6 +244,18 @@ export class DebugManager { this.pendingLogs.delete(requestId); } + /** + * Flag a request's pending trace for persistence on flush even when debug + * capture is not otherwise enabled for its key. Used by capture-on-error + * mode when a request writes an inference error or triggers a cooldown. + * No-op unless capture-on-error is enabled and a pending log exists. + */ + markForcePersist(requestId: string | undefined | null): void { + if (!this.captureOnError || !requestId) return; + const log = this.pendingLogs.get(requestId); + if (log) log.forcePersist = true; + } + /** * Mark a request as ephemeral (debug data won't be persisted) */ diff --git a/packages/backend/src/services/request-context.ts b/packages/backend/src/services/request-context.ts index eb359e4fd..f0b7edb20 100644 --- a/packages/backend/src/services/request-context.ts +++ b/packages/backend/src/services/request-context.ts @@ -9,6 +9,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; */ export interface RequestContext { keyName?: string; + requestId?: string; } const storage = new AsyncLocalStorage(); @@ -36,6 +37,24 @@ export function getCurrentKeyName(): string | undefined { return storage.getStore()?.keyName; } +/** + * Read just the request id from the active request context. + */ +export function getCurrentRequestId(): string | undefined { + return storage.getStore()?.requestId; +} + +/** + * Attach the request id to the active request context. Mutates the current + * store so downstream code (notably DebugManager, reached via the cooldown + * path) can resolve the request id without explicit plumbing. No-op when + * called outside a request context. + */ +export function setCurrentRequestId(requestId: string): void { + const store = storage.getStore(); + if (store) store.requestId = requestId; +} + /** * Run a callback with a given request context. Prefer this wrapper in tests * or short-lived scopes; long-lived Fastify handlers should use diff --git a/packages/backend/src/services/usage-storage.ts b/packages/backend/src/services/usage-storage.ts index a74d58f5e..44000cf01 100644 --- a/packages/backend/src/services/usage-storage.ts +++ b/packages/backend/src/services/usage-storage.ts @@ -4,7 +4,7 @@ import { getDatabase, getSchema } from '../db/client'; import { NewRequestUsage } from '../db/types'; import { EventEmitter } from 'node:events'; import { eq, and, gte, lte, like, desc, asc, sql, getTableName } from 'drizzle-orm'; -import { DebugLogRecord } from './debug-manager'; +import { DebugLogRecord, DebugManager } from './debug-manager'; import { getCurrentKeyName } from './request-context'; import { estimateKwhUsed } from './inference-energy'; import { resolveModelParams, DEFAULT_GPU_PARAMS } from '@plexus/shared'; @@ -357,6 +357,10 @@ export class UsageStorageService extends EventEmitter { }); logger.debug(`Inference error saved for request ${requestId}`); + + // In capture-on-error mode, persist this request's debug trace even if + // debug capture isn't otherwise enabled. No-op when the mode is off. + DebugManager.getInstance().markForcePersist(requestId); } catch (e) { logger.error('Failed to save inference error', e); } diff --git a/packages/frontend/src/lib/api.ts b/packages/frontend/src/lib/api.ts index 57b0dbadc..690a2b082 100644 --- a/packages/frontend/src/lib/api.ts +++ b/packages/frontend/src/lib/api.ts @@ -3446,6 +3446,26 @@ export const api = { return res.json(); }, + // ─── Capture Trace on Error ───────────────────────────────────────── + + /** Fetch whether traces are captured on error even when debug is off. */ + getCaptureTraceOnError: async (): Promise<{ enabled: boolean }> => { + const res = await fetchWithAuth(`${API_BASE}/v0/management/config/capture-trace-on-error`); + if (!res.ok) throw new Error('Failed to fetch capture-trace-on-error setting'); + return res.json(); + }, + + /** Toggle capturing traces on error even when debug is off. */ + setCaptureTraceOnError: async (enabled: boolean): Promise<{ enabled: boolean }> => { + const res = await fetchWithAuth(`${API_BASE}/v0/management/config/capture-trace-on-error`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ enabled }), + }); + if (!res.ok) throw new Error('Failed to update capture-trace-on-error setting'); + return res.json(); + }, + // ─── Cooldown Settings ────────────────────────────────────────────── /** Fetch current cooldown policy. */ diff --git a/packages/frontend/src/pages/Config.tsx b/packages/frontend/src/pages/Config.tsx index 4441efcc8..19988770e 100644 --- a/packages/frontend/src/pages/Config.tsx +++ b/packages/frontend/src/pages/Config.tsx @@ -147,6 +147,11 @@ export const Config = () => { const [statusCodesText, setStatusCodesText] = useState(''); const [errorsText, setErrorsText] = useState(''); + // Capture-trace-on-error toggle (persisted admin setting) + const [captureTraceOnError, setCaptureTraceOnError] = useState(false); + const [captureTraceLoaded, setCaptureTraceLoaded] = useState(false); + const [captureTraceSaving, setCaptureTraceSaving] = useState(false); + // Trusted proxies (which immediate peers' forwarding headers are believed) const [trustedProxies, setTrustedProxies] = useState([]); const [trustedProxiesLoaded, setTrustedProxiesLoaded] = useState(false); @@ -348,6 +353,33 @@ export const Config = () => { } }, [toast]); + const loadCaptureTraceOnError = useCallback(async () => { + try { + const { enabled } = await api.getCaptureTraceOnError(); + setCaptureTraceOnError(enabled); + setCaptureTraceLoaded(true); + } catch (e) { + console.error('Failed to load capture-trace-on-error setting:', e); + toast.error('Failed to load trace capture settings'); + } + }, [toast]); + + const handleToggleCaptureTraceOnError = async (checked: boolean) => { + const previous = captureTraceOnError; + setCaptureTraceOnError(checked); + setCaptureTraceSaving(true); + try { + const { enabled } = await api.setCaptureTraceOnError(checked); + setCaptureTraceOnError(enabled); + toast.success(`Capture trace on error ${enabled ? 'enabled' : 'disabled'}`); + } catch (e) { + setCaptureTraceOnError(previous); + toast.error((e as Error).message, 'Failed to update trace capture settings'); + } finally { + setCaptureTraceSaving(false); + } + }; + const loadCooldownPolicy = useCallback(async () => { try { const policy = await api.getCooldownPolicy(); @@ -664,6 +696,7 @@ export const Config = () => { useEffect(() => { loadConfig(); loadFailoverPolicy(); + loadCaptureTraceOnError(); loadCooldownPolicy(); loadTrustedProxies(); loadTimeoutConfig(); @@ -980,6 +1013,30 @@ export const Config = () => { + {/* ─── Trace Capture ──────────────────────────────────────── */} + +
+
+ +
+

+ Capture Trace on Error +

+

+ When enabled, debug traces are stored for requests that write to the inference + error log or trigger a cooldown, even while global debug tracing is off. +

+
+
+ +
+ + {/* ─── Cooldown Settings ──────────────────────────────────── */} { // Provider filter state const [providers, setProviders] = useState([]); const [debugEnabled, setDebugEnabled] = useState(false); + const [captureTraceOnError, setCaptureTraceOnError] = useState(false); + const [captureTraceLoaded, setCaptureTraceLoaded] = useState(false); + const [captureTraceSaving, setCaptureTraceSaving] = useState(false); const [selectedProviders, setSelectedProviders] = useState([]); const [isFilterOpen, setIsFilterOpen] = useState(false); @@ -162,9 +166,34 @@ export const Debug: React.FC = () => { } catch (e) { console.error('Failed to fetch providers or debug status', e); } + // Capture-trace-on-error is an admin-only persisted setting. + if (isAdmin) { + try { + const { enabled } = await api.getCaptureTraceOnError(); + setCaptureTraceOnError(enabled); + setCaptureTraceLoaded(true); + } catch (e) { + console.error('Failed to fetch capture-trace-on-error setting', e); + } + } }; fetchProvidersAndStatus(); - }, []); + }, [isAdmin]); + + const handleToggleCaptureTraceOnError = async (checked: boolean) => { + const previous = captureTraceOnError; + setCaptureTraceOnError(checked); + setCaptureTraceSaving(true); + try { + const { enabled } = await api.setCaptureTraceOnError(checked); + setCaptureTraceOnError(enabled); + } catch (e) { + setCaptureTraceOnError(previous); + console.error('Failed to update capture-trace-on-error setting', e); + } finally { + setCaptureTraceSaving(false); + } + }; // Close filter dropdown when clicking outside useEffect(() => { @@ -303,6 +332,18 @@ export const Debug: React.FC = () => { } actions={ <> + {/* Capture-trace-on-error — admin-only persisted setting. */} + {isAdmin && ( + + )} {/* Provider Filter — admin-only: the global filter affects all users. */} {isAdmin && (
From e9be9b2a73b68c022f7bbd9d34026437838f7b76 Mon Sep 17 00:00:00 2001 From: Matt Cowger Date: Sat, 4 Jul 2026 05:17:57 +0000 Subject: [PATCH 2/2] fix(debug): avoid mutating shared request-context store Replace the async-local store via enterWith instead of mutating the existing object in place, so other holders of a reference to the prior store aren't affected. Addresses PR review feedback. --- packages/backend/src/services/request-context.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/services/request-context.ts b/packages/backend/src/services/request-context.ts index f0b7edb20..e8b1f18fd 100644 --- a/packages/backend/src/services/request-context.ts +++ b/packages/backend/src/services/request-context.ts @@ -45,14 +45,16 @@ export function getCurrentRequestId(): string | undefined { } /** - * Attach the request id to the active request context. Mutates the current - * store so downstream code (notably DebugManager, reached via the cooldown - * path) can resolve the request id without explicit plumbing. No-op when - * called outside a request context. + * Attach the request id to the active request context so downstream code + * (notably DebugManager, reached via the cooldown path) can resolve the + * request id without explicit plumbing. Replaces the store via `enterWith` + * rather than mutating the existing object in place, so any other holder of + * a reference to the prior store is unaffected. No-op when called outside a + * request context. */ export function setCurrentRequestId(requestId: string): void { const store = storage.getStore(); - if (store) store.requestId = requestId; + if (store) storage.enterWith({ ...store, requestId }); } /**