From f7a297b35bcd421ef67dbadb62052297a3d3382f Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 11 Aug 2026 02:18:01 -0700 Subject: [PATCH 1/2] fix(engine): bound drawElement frames so a wedged renderer falls back instead of failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single drawElement frame could kill an entire render. On one comp (caption-editorial-emphasis) drawElementImage returns normally and the renderer then stops draining its task queue: the setTimeout(…, 0) that schedules toDataURL never fires, the capture page.evaluate never settles, and 60s later the stage watchdog fails the whole render. Deterministic on 152.0.7977.30, always the same frame. Root-cause detail in PRINFRA-488. Two gaps, both closed here. 1. Nothing below the 60s stage watchdog bounded a frame. Adds a per-frame deadline (HF_DE_FRAME_TIMEOUT_MS, default 15000, 0 disables) around the WHOLE frame operation, not just the drawElementImage call — the stall surfaces at whichever page round-trip comes next, and was observed at both the capture and the seek's background-image decode. Bounding one call missed it. Deliberately NO per-frame screenshot fallback for this error: once the renderer stops scheduling it is wedged for every subsequent round-trip on that page. Measured — the screenshot fallback blew the same deadline. The frame fails fast instead, so the recovery that actually works can run. 2. That recovery was ineligible. shouldRetryViaPinnedFallback only retried a generic capture failure when the worker count was PINNED by the inversion or the router; a comp that engaged drawElement on the ordinary single-worker path had no whole-render fallback at all. A renderer stall is now retryable on any routing — the failure is a property of drawElement itself, and the retry re-renders on a fresh page via screenshot. Reported as de_fallback_reason "de_renderer_stall" so it is distinguishable from capture_error in telemetry, and counted per-session as deFrameTimeouts. Verified end to end on the repro: previously exit 1 with "stalled: no frame progress for 60000ms"; now the deadline fires, the render retries via screenshot, and RENDER_OK with a valid 1920x1080 / 240-frame / 8.0s MP4. Tests cover the predicate both ways plus the cross-package error match. Engine 1,481 and producer 587 green (audioPadTrim.integration flakes only under full-suite parallel load — passes in isolation, and on clean HEAD it is green too, so it is not from this change). Refs PRINFRA-488 --- packages/engine/src/services/frameCapture.ts | 82 ++++++++++++++++++- packages/engine/src/types.ts | 8 ++ .../src/services/renderOrchestrator.test.ts | 55 +++++++++++++ .../src/services/renderOrchestrator.ts | 30 ++++++- 4 files changed, 173 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index d7d1c59c0e..1b341f8237 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -212,6 +212,13 @@ export interface CaptureSession { deVerifyInitMs?: number; /** Count of per-frame "No cached paint record" screenshot fallbacks (telemetry). */ deNcprFallbacks?: number; + /** + * Count of drawElement frame captures that blew `HF_DE_FRAME_TIMEOUT_MS` + * because the renderer stopped scheduling after drawElementImage returned + * (PRINFRA-488). Each one aborts the drawElement attempt so the whole render + * retries via screenshot. + */ + deFrameTimeouts?: number; /** * drawElement init passed every gate but stopped before verification + * canvas injection: the session has no video-frame injector yet (probe @@ -3133,6 +3140,55 @@ function isNoCachedPaintRecordError(err: unknown): boolean { return msg.includes("No cached paint record"); } +/** + * Per-frame deadline for the drawElement capture round-trip. + * + * drawElementImage can return normally and then leave the renderer not draining + * its task queue: the `setTimeout(…, 0)` that drawAndEncode schedules to run + * `toDataURL` never fires, so the capture `page.evaluate` never settles. + * Reproduced deterministically on Chromium 152.0.7977.30, one comp, always the + * same frame (PRINFRA-488). Nothing below the render-level watchdog bounded + * this, so a single bad frame failed the ENTIRE render after a 60 s stall. + * + * This bounds the round-trip so the frame can take the same per-frame screenshot + * fallback the `No cached paint record` case already takes — one slow frame + * instead of a dead render. Tune with `HF_DE_FRAME_TIMEOUT_MS`; 0 disables. + */ +const DE_FRAME_TIMEOUT_MS = Number(process.env.HF_DE_FRAME_TIMEOUT_MS ?? "15000"); + +class DeFrameTimeoutError extends Error { + constructor(label: string, ms: number) { + super(`drawElement ${label} exceeded ${ms}ms (renderer stopped scheduling; see PRINFRA-488)`); + this.name = "DeFrameTimeoutError"; + } +} + +function isDeFrameTimeoutError(err: unknown): boolean { + return err instanceof DeFrameTimeoutError; +} + +/** + * Race `work` against a deadline. The losing promise is NOT cancellable — + * puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is + * swallowed to avoid an unhandled rejection when it eventually settles (or + * never does). + */ +async function withFrameDeadline(work: Promise, label: string, ms: number): Promise { + if (!(ms > 0)) return work; + let timer: ReturnType | undefined; + const guard = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new DeFrameTimeoutError(label, ms)), ms); + }); + try { + return await Promise.race([work, guard]); + } finally { + if (timer) clearTimeout(timer); + void work.catch(() => { + /* orphaned round-trip — see doc above */ + }); + } +} + async function captureFrameCore( session: CaptureSession, frameIndex: number, @@ -3260,6 +3316,22 @@ async function captureFrameCore( // (display toggled / detached / freshly-shown at a clip-cut boundary). This // is a per-frame condition, not a whole-comp one — fall back to screenshot // for THIS frame instead of aborting the render. See fast-capture-limitations.md. + if (isDeFrameTimeoutError(err)) { + // Deliberately NO per-frame screenshot fallback here. When the + // renderer stops scheduling, it is wedged for EVERY subsequent + // round-trip on that page — measured: the screenshot fallback blew + // the same deadline. Fail fast instead and let the producer re-render + // the whole comp on a fresh page via the screenshot path, which is + // the only recovery that actually works. + session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; + console.log( + `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + + `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + + `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + + `retries via screenshot.`, + ); + throw err; + } if (isNoCachedPaintRecordError(err)) { session.deNcprFallbacks = (session.deNcprFallbacks ?? 0) + 1; console.log( @@ -3344,7 +3416,14 @@ export async function captureFrameToBuffer( frameIndex: number, time: number, ): Promise { - const { buffer, captureTimeMs } = await captureFrameCore(session, frameIndex, time); + const { buffer, captureTimeMs } = + session.captureMode === "drawelement" + ? await withFrameDeadline( + captureFrameCore(session, frameIndex, time), + `frame ${frameIndex}`, + DE_FRAME_TIMEOUT_MS, + ) + : await captureFrameCore(session, frameIndex, time); return { buffer, captureTimeMs }; } @@ -3951,5 +4030,6 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma deVerifyInitMs: session.deVerifyInitMs ?? 0, deBoundaryFrames: session.clipBoundaryFrames?.size ?? 0, deNcprFallbacks: session.deNcprFallbacks ?? 0, + deFrameTimeouts: session.deFrameTimeouts ?? 0, }; } diff --git a/packages/engine/src/types.ts b/packages/engine/src/types.ts index 8b33b2df5a..07fa67563f 100644 --- a/packages/engine/src/types.ts +++ b/packages/engine/src/types.ts @@ -345,6 +345,14 @@ export interface CapturePerfSummary { deBoundaryFrames: number; /** Per-frame "No cached paint record" screenshot fallbacks during capture. */ deNcprFallbacks: number; + /** + * Per-frame drawElement captures that blew the `HF_DE_FRAME_TIMEOUT_MS` + * deadline and took the screenshot fallback (renderer stopped scheduling + * after drawElementImage returned — PRINFRA-488). Non-zero means the render + * completed only because the deadline caught a stall that previously failed + * the whole render. + */ + deFrameTimeouts: number; } // ── Global Augmentation ──────────────────────────────────────────────────────── diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 93c04e51d0..29d9f39c43 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -34,6 +34,7 @@ import { resolveParallelRouterRetryPlan, resetCaptureAttemptProgress, shouldRetryViaPinnedFallback, + isDeRendererStallError, countElementTags, envInt, isDeParallelRouterEnabled, @@ -2457,6 +2458,60 @@ describe("resolveParallelRouterRetryPlan (self-verify retry rollback)", () => { }); describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic capture failures, including OOM)", () => { + // PRINFRA-488: a wedged renderer must be retryable on ANY routing. Before this, + // a comp that engaged drawElement on the ordinary single-worker path had no + // whole-render fallback, so one stalled frame failed the entire render. + it("retries a drawElement renderer stall even with no pinned routing", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: true, + }), + ).toBe(true); + }); + + it("still does NOT retry a generic capture failure with no pinned routing", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: false, + }), + ).toBe(false); + }); + + it("never retries a cancellation, even for a renderer stall", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: true, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: true, + }), + ).toBe(false); + }); + + it("recognizes the engine's stall error across the package boundary", () => { + const byName = new Error("whatever"); + byName.name = "DeFrameTimeoutError"; + expect(isDeRendererStallError(byName)).toBe(true); + expect( + isDeRendererStallError( + new Error( + "drawElement frame 50 exceeded 15000ms (renderer stopped scheduling; see PRINFRA-488)", + ), + ), + ).toBe(true); + expect(isDeRendererStallError(new Error("some other capture failure"))).toBe(false); + expect(isDeRendererStallError("not an error")).toBe(false); + }); + it("always retries a drawElement self-verify failure, pinned or not", () => { expect( shouldRetryViaPinnedFallback({ diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 69100e1e1a..74c75a223e 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1808,12 +1808,34 @@ export function shouldRetryViaPinnedFallback(args: { isCancellation: boolean; deWorkerInversion: "inverted" | "reverted" | undefined; deParallelRouter: "routed" | "reverted" | undefined; + /** + * The drawElement capture wedged the renderer (PRINFRA-488). Retryable on ANY + * routing, not just a pinned one: the failure is a property of drawElement + * itself, and the retry re-renders on a fresh page via screenshot — the only + * recovery that works once the renderer stops scheduling. Without this a comp + * that engaged drawElement on the ordinary single-worker path (neither + * inverted nor routed) had NO whole-render fallback, so one wedged frame + * failed the entire render. + */ + isDeRendererStall?: boolean; }): boolean { if (args.isCancellation) return false; if (args.isVerifyError) return true; + if (args.isDeRendererStall === true) return true; return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed"; } +/** + * True for the drawElement per-frame deadline breach raised by the engine when + * the renderer stops scheduling after `drawElementImage` returns (PRINFRA-488). + * Matched on name+message rather than by class because the error crosses the + * engine/producer package boundary. + */ +export function isDeRendererStallError(err: unknown): boolean { + if (!(err instanceof Error)) return false; + return err.name === "DeFrameTimeoutError" || err.message.includes("renderer stopped scheduling"); +} + /** * When a self-verify (or pinned-fallback) retry is triggered mid-capture, the * caller may still hold a live probe session that the failed stage was passed @@ -3556,6 +3578,7 @@ async function executeRenderPipeline(input: { // spawns on retry. See shouldRetryViaPinnedFallback for exactly // which errors qualify. const isVerifyError = isDrawElementVerificationError(err); + const isDeStall = isDeRendererStallError(err); const isCancellation = err instanceof RenderCancelledError || executionSignal?.aborted === true; if ( @@ -3564,6 +3587,7 @@ async function executeRenderPipeline(input: { isCancellation, deWorkerInversion, deParallelRouter, + isDeRendererStall: isDeStall, }) ) throw err; @@ -3576,7 +3600,11 @@ async function executeRenderPipeline(input: { deFallbackFrameIndex = t.frameIndex; deFallbackThresholdDb = t.thresholdDb; } else { - deFallbackReason = isMemoryExhaustion ? "oom" : "capture_error"; + deFallbackReason = isMemoryExhaustion + ? "oom" + : isDeStall + ? "de_renderer_stall" + : "capture_error"; } log.warn( isVerifyError From 1068fd1b4f5d2d3dfbd5282b92199d611b1a4e17 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 20 Aug 2026 22:48:29 -0700 Subject: [PATCH 2/2] fix(engine,producer,cli): count drawElement frame timeouts where they happen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `session.deFrameTimeouts` was incremented inside `captureFrameCore`'s catch, but the deadline races that function from OUTSIDE it — `withFrameDeadline` is applied by `captureFrameToBuffer`, and `DeFrameTimeoutError` is constructed nowhere else. So the branch was unreachable: on a wedged renderer the counter stayed 0, the engine-side stall log never printed, and `CapturePerfSummary.deFrameTimeouts` reported 0 on every affected render. The end-to-end recovery worked throughout — `isDeRendererStallError` matches by name at the producer — only the observability was dead. Moved the increment and the log into an `onTimeout` hook that fires from the deadline itself, deleted the unreachable branch and the now-unused `isDeFrameTimeoutError`, and threaded the count out to `render_completed.de_frame_timeouts` so the rate is graphable apart from the other `capture_error` fallbacks. A stall always costs a whole-render re-run, which is why it deserves its own series. Also corrected the `fallbackReason` docstring, which still listed only `psnr | blank | oom | capture_error` after this branch added `de_renderer_stall`. `withFrameDeadline` is exported for a two-case fake-timer test: the deadline fires and calls `onTimeout` once, and a resolving inner promise passes its value through with no late timer. The first case fails if the counter moves back inside the work promise. --- packages/cli/src/commands/render.ts | 1 + packages/cli/src/telemetry/events.ts | 2 + .../frameCapture-frameDeadline.test.ts | 45 ++++++++++++++ packages/engine/src/services/frameCapture.ts | 59 +++++++++++-------- .../src/services/render/perfSummary.ts | 1 + .../src/services/renderOrchestrator.ts | 11 +++- 6 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 packages/engine/src/services/frameCapture-frameDeadline.test.ts diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 06d03da67d..a939ea9f98 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -1514,6 +1514,7 @@ function trackRenderMetrics( deBlankRecaptures: perf?.drawElement?.blankRecaptures, deBoundaryFrames: perf?.drawElement?.boundaryFrames, deNcprFallbacks: perf?.drawElement?.ncprFallbacks, + deFrameTimeouts: perf?.drawElement?.frameTimeouts, compositionDurationMs, compositionWidth: perf?.resolution.width, compositionHeight: perf?.resolution.height, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index de3e371cd5..b529dbe022 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -235,6 +235,7 @@ export function trackRenderComplete( deBlankRecaptures?: number; deBoundaryFrames?: number; deNcprFallbacks?: number; + deFrameTimeouts?: number; // "cli" when triggered by `hyperframes render` (default), "studio" when // triggered by a studio preview-server render (POST /api/projects/:id/render). source?: "cli" | "studio"; @@ -336,6 +337,7 @@ export function trackRenderComplete( de_blank_recaptures: props.deBlankRecaptures, de_boundary_frames: props.deBoundaryFrames, de_ncpr_fallbacks: props.deNcprFallbacks, + de_frame_timeouts: props.deFrameTimeouts, ...powerStateFields(), source: props.source ?? "cli", composition_duration_ms: props.compositionDurationMs, diff --git a/packages/engine/src/services/frameCapture-frameDeadline.test.ts b/packages/engine/src/services/frameCapture-frameDeadline.test.ts new file mode 100644 index 0000000000..427c09b443 --- /dev/null +++ b/packages/engine/src/services/frameCapture-frameDeadline.test.ts @@ -0,0 +1,45 @@ +/** + * Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488). + * + * The deadline races the capture round-trip from OUTSIDE `captureFrameCore`, + * because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly + * why the stall counter has to live in the `onTimeout` hook: a wedged renderer + * never returns, so no catch block inside the work promise ever runs. The first + * shipped version incremented `session.deFrameTimeouts` in that unreachable + * catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0 + * on every stalled render. + */ + +import { describe, expect, it, vi } from "vitest"; +import { withFrameDeadline } from "./frameCapture.js"; + +describe("withFrameDeadline", () => { + it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + // Never settles — the wedged-renderer shape. + const raced = withFrameDeadline(new Promise(() => {}), "frame 7", 15_000, onTimeout); + const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/); + await vi.advanceTimersByTimeAsync(15_000); + await assertion; + expect(onTimeout).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it("passes the value through and leaves onTimeout alone when work wins", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout); + await expect(raced).resolves.toBe("buffer"); + // Past the deadline: the cleared timer must not fire late. + await vi.advanceTimersByTimeAsync(30_000); + expect(onTimeout).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 1b341f8237..efc3d2092f 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -3163,21 +3163,34 @@ class DeFrameTimeoutError extends Error { } } -function isDeFrameTimeoutError(err: unknown): boolean { - return err instanceof DeFrameTimeoutError; -} - /** * Race `work` against a deadline. The losing promise is NOT cancellable — * puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is * swallowed to avoid an unhandled rejection when it eventually settles (or - * never does). + * never does). The orphaned round-trip keeps running in its Chrome worker; + * that worker is reclaimed by the outer retry rebuilding the page + * (`closeOrphanedProbeForRetry`), not by anything here. + * + * `onTimeout` fires exactly when the deadline wins, and is the ONLY place the + * stall is observable: because the deadline races `work` from outside, nothing + * inside `work` — including its own catch blocks — ever sees this error. + * + * Exported for the deadline unit test; `captureFrameToBuffer` is the only + * production caller. */ -async function withFrameDeadline(work: Promise, label: string, ms: number): Promise { +export async function withFrameDeadline( + work: Promise, + label: string, + ms: number, + onTimeout?: () => void, +): Promise { if (!(ms > 0)) return work; let timer: ReturnType | undefined; const guard = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new DeFrameTimeoutError(label, ms)), ms); + timer = setTimeout(() => { + onTimeout?.(); + reject(new DeFrameTimeoutError(label, ms)); + }, ms); }); try { return await Promise.race([work, guard]); @@ -3316,22 +3329,6 @@ async function captureFrameCore( // (display toggled / detached / freshly-shown at a clip-cut boundary). This // is a per-frame condition, not a whole-comp one — fall back to screenshot // for THIS frame instead of aborting the render. See fast-capture-limitations.md. - if (isDeFrameTimeoutError(err)) { - // Deliberately NO per-frame screenshot fallback here. When the - // renderer stops scheduling, it is wedged for EVERY subsequent - // round-trip on that page — measured: the screenshot fallback blew - // the same deadline. Fail fast instead and let the producer re-render - // the whole comp on a fresh page via the screenshot path, which is - // the only recovery that actually works. - session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; - console.log( - `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + - `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + - `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + - `retries via screenshot.`, - ); - throw err; - } if (isNoCachedPaintRecordError(err)) { session.deNcprFallbacks = (session.deNcprFallbacks ?? 0) + 1; console.log( @@ -3422,6 +3419,22 @@ export async function captureFrameToBuffer( captureFrameCore(session, frameIndex, time), `frame ${frameIndex}`, DE_FRAME_TIMEOUT_MS, + () => { + // Deliberately NO per-frame screenshot fallback. When the renderer + // stops scheduling it is wedged for EVERY subsequent round-trip on + // that page — measured: the screenshot fallback blew the same + // deadline. Fail fast and let the producer re-render the whole comp + // on a fresh page via the screenshot path, the only recovery that + // works. Counted here rather than in captureFrameCore's catch: the + // deadline rejects from outside it, so that catch never runs. + session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; + console.log( + `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + + `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + + `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + + `retries via screenshot.`, + ); + }, ) : await captureFrameCore(session, frameIndex, time); diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts index e4f4db3fc8..c100e3dac3 100644 --- a/packages/producer/src/services/render/perfSummary.ts +++ b/packages/producer/src/services/render/perfSummary.ts @@ -149,6 +149,7 @@ function aggregateDrawElement( blankRecaptures: drain?.blankRecaptures ?? 0, boundaryFrames: perfs.reduce((sum, p) => sum + (p.deBoundaryFrames ?? 0), 0), ncprFallbacks: perfs.reduce((sum, p) => sum + (p.deNcprFallbacks ?? 0), 0), + frameTimeouts: perfs.reduce((sum, p) => sum + (p.deFrameTimeouts ?? 0), 0), }; } diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 74c75a223e..3a0febf80c 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -529,9 +529,9 @@ export interface RenderPerfSummary { * `fallbackReason` being set is the "any fallback fired" signal. */ selfVerifyFallback: boolean; - /** What tripped the fallback retry: psnr | blank | oom | capture_error. */ + /** What tripped the fallback retry: psnr | blank | oom | de_renderer_stall | capture_error. */ fallbackReason?: string; - /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */ + /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for every other reason (no score exists). */ fallbackFailedDb?: number; /** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */ fallbackFrameIndex?: number; @@ -545,6 +545,13 @@ export interface RenderPerfSummary { boundaryFrames: number; /** Per-frame "No cached paint record" screenshot fallbacks. */ ncprFallbacks: number; + /** + * Frames that blew `HF_DE_FRAME_TIMEOUT_MS` — a wedged renderer + * (PRINFRA-488). Distinct from the other fallback counters: this one always + * costs a whole-render re-run via screenshot, so its rate is worth graphing + * on its own rather than inside `capture_error`. + */ + frameTimeouts: number; }; }