From 7308e37fc93fc1882f51f8346e54e8c6d4aefca9 Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Sat, 8 Aug 2026 17:27:05 +0530 Subject: [PATCH 1/6] fix(linux): resolve HUD overlay restricted width and popover clipping on Hyprland/Wayland --- electron/hudOverlayBounds.test.ts | 95 ++++--------------- electron/hudOverlayBounds.ts | 10 +- electron/windows.ts | 4 +- src/components/launch/LaunchWindow.module.css | 4 +- 4 files changed, 29 insertions(+), 84 deletions(-) diff --git a/electron/hudOverlayBounds.test.ts b/electron/hudOverlayBounds.test.ts index db21e92bd..a08def995 100644 --- a/electron/hudOverlayBounds.test.ts +++ b/electron/hudOverlayBounds.test.ts @@ -18,61 +18,21 @@ describe("getHudOverlayWindowBounds", () => { expect(getHudOverlayWindowBounds(workArea, true)).toEqual(workArea); }); - it("uses a bottom-centered compact fallback when mouse passthrough is unavailable", () => { - expect(getHudOverlayWindowBounds(workArea, false)).toEqual({ - x: 650, - y: 920, - width: 860, - height: 160, - }); + it("uses the full work area when mouse passthrough is supported", () => { + expect(getHudOverlayWindowBounds(workArea, true)).toEqual(workArea); }); - it("expands the non-passthrough fallback for HUD menus and hover interaction", () => { - expect(getHudOverlayWindowBounds(workArea, false, true)).toEqual({ - x: 650, - y: 540, - width: 860, - height: 540, - }); + it("uses full display work area bounds for the overlay window", () => { + expect(getHudOverlayWindowBounds(workArea, false)).toEqual(workArea); }); - it("keeps the compact fallback inside small displays", () => { - expect( - getHudOverlayWindowBounds( - { - x: -100, - y: 20, - width: 640, - height: 420, - }, - false, - ), - ).toEqual({ - x: -100, - y: 280, - width: 640, - height: 160, - }); + it("expands to full work area bounds", () => { + expect(getHudOverlayWindowBounds(workArea, false, true)).toEqual(workArea); }); - it("fits the expanded fallback inside small displays", () => { - expect( - getHudOverlayWindowBounds( - { - x: -100, - y: 20, - width: 640, - height: 420, - }, - false, - true, - ), - ).toEqual({ - x: -100, - y: 20, - width: 640, - height: 420, - }); + it("preserves full work area bounds inside small displays", () => { + const small = { x: -100, y: 20, width: 640, height: 420 }; + expect(getHudOverlayWindowBounds(small, false)).toEqual(small); }); }); @@ -84,64 +44,49 @@ describe("resizeHudOverlayFallbackBounds", () => { height: 1080, }; - it("preserves the dragged bottom edge when expanding", () => { + it("returns full work area when expanding", () => { expect( resizeHudOverlayFallbackBounds( workArea, { x: 420, y: 700, - width: 860, + width: 1200, height: 160, }, true, ), - ).toEqual({ - x: 420, - y: 320, - width: 860, - height: 540, - }); + ).toEqual(workArea); }); - it("preserves the dragged bottom edge when compacting", () => { + it("returns full work area when compacting", () => { expect( resizeHudOverlayFallbackBounds( workArea, { x: 420, - y: 320, - width: 860, - height: 540, + y: 260, + width: 1200, + height: 600, }, false, ), - ).toEqual({ - x: 420, - y: 700, - width: 860, - height: 160, - }); + ).toEqual(workArea); }); - it("keeps resized fallback bounds inside the display work area", () => { + it("returns full work area inside the display work area", () => { expect( resizeHudOverlayFallbackBounds( workArea, { x: 1500, y: 900, - width: 860, + width: 1200, height: 160, }, true, ), - ).toEqual({ - x: 1060, - y: 520, - width: 860, - height: 540, - }); + ).toEqual(workArea); }); }); diff --git a/electron/hudOverlayBounds.ts b/electron/hudOverlayBounds.ts index 8c51b88c7..7aba77f40 100644 --- a/electron/hudOverlayBounds.ts +++ b/electron/hudOverlayBounds.ts @@ -5,9 +5,9 @@ export interface HudOverlayWorkArea { height: number; } -const NON_PASSTHROUGH_HUD_WIDTH_DIP = 860; +const NON_PASSTHROUGH_HUD_WIDTH_DIP = 1200; const NON_PASSTHROUGH_HUD_COMPACT_HEIGHT_DIP = 160; -const NON_PASSTHROUGH_HUD_EXPANDED_HEIGHT_DIP = 540; +const NON_PASSTHROUGH_HUD_EXPANDED_HEIGHT_DIP = 600; function clamp(value: number, min: number, max: number): number { return Math.min(Math.max(value, min), max); @@ -18,7 +18,7 @@ export function getHudOverlayWindowBounds( mousePassthroughSupported: boolean, fallbackExpanded = false, ): HudOverlayWorkArea { - if (mousePassthroughSupported) { + if (mousePassthroughSupported || process.platform === "linux") { return { ...workArea }; } @@ -55,6 +55,10 @@ export function resizeHudOverlayFallbackBounds( currentBounds: HudOverlayWorkArea, fallbackExpanded: boolean, ): HudOverlayWorkArea { + if (process.platform === "linux") { + return { ...workArea }; + } + const nextBounds = getHudOverlayWindowBounds(workArea, false, fallbackExpanded); const maxX = workArea.x + workArea.width - nextBounds.width; const maxY = workArea.y + workArea.height - nextBounds.height; diff --git a/electron/windows.ts b/electron/windows.ts index 55f6314cd..16519d7b6 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -311,9 +311,7 @@ function setHudOverlayMousePassthrough(ignore: boolean) { } if (!isHudOverlayMousePassthroughSupported()) { - if (process.platform !== "linux") { - setHudOverlayFallbackExpanded(!ignore); - } + setHudOverlayFallbackExpanded(!ignore); hudOverlayWindow.setIgnoreMouseEvents(false); return; } diff --git a/src/components/launch/LaunchWindow.module.css b/src/components/launch/LaunchWindow.module.css index 3638e7563..0685782a0 100644 --- a/src/components/launch/LaunchWindow.module.css +++ b/src/components/launch/LaunchWindow.module.css @@ -11,7 +11,7 @@ align-items: center; gap: 10px; width: max-content; - max-width: 1200px; + max-width: 100%; background: var(--launch-bar-bg); border: 1px solid var(--launch-bar-border); border-radius: 20px; @@ -95,8 +95,6 @@ color: #4eeeb0; } - - .menuCard { width: 300px; max-height: 400px; From 594cee17aa7058caa7ca970b205e077831694235 Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Sat, 8 Aug 2026 20:39:38 +0530 Subject: [PATCH 2/6] test(electron): make HUD overlay bounds expectations platform-specific --- electron/hudOverlayBounds.test.ts | 119 ++++++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 14 deletions(-) diff --git a/electron/hudOverlayBounds.test.ts b/electron/hudOverlayBounds.test.ts index a08def995..89f6a38ef 100644 --- a/electron/hudOverlayBounds.test.ts +++ b/electron/hudOverlayBounds.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { getHudOverlayWindowBounds, @@ -14,25 +14,78 @@ describe("getHudOverlayWindowBounds", () => { height: 1040, }; - it("uses the full work area when mouse passthrough is supported", () => { - expect(getHudOverlayWindowBounds(workArea, true)).toEqual(workArea); + afterEach(() => { + vi.restoreAllMocks(); }); it("uses the full work area when mouse passthrough is supported", () => { expect(getHudOverlayWindowBounds(workArea, true)).toEqual(workArea); }); - it("uses full display work area bounds for the overlay window", () => { + it("uses full display work area bounds on Linux when mouse passthrough is false", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); expect(getHudOverlayWindowBounds(workArea, false)).toEqual(workArea); }); - it("expands to full work area bounds", () => { - expect(getHudOverlayWindowBounds(workArea, false, true)).toEqual(workArea); + it("uses a bottom-centered compact fallback on non-Linux when mouse passthrough is unavailable", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + expect(getHudOverlayWindowBounds(workArea, false)).toEqual({ + x: 480, + y: 920, + width: 1200, + height: 160, + }); }); - it("preserves full work area bounds inside small displays", () => { - const small = { x: -100, y: 20, width: 640, height: 420 }; - expect(getHudOverlayWindowBounds(small, false)).toEqual(small); + it("expands the non-Linux fallback for HUD menus and hover interaction", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + expect(getHudOverlayWindowBounds(workArea, false, true)).toEqual({ + x: 480, + y: 480, + width: 1200, + height: 600, + }); + }); + + it("keeps the non-Linux compact fallback inside small displays", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + expect( + getHudOverlayWindowBounds( + { + x: -100, + y: 20, + width: 640, + height: 420, + }, + false, + ), + ).toEqual({ + x: -100, + y: 280, + width: 640, + height: 160, + }); + }); + + it("fits the non-Linux expanded fallback inside small displays", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + expect( + getHudOverlayWindowBounds( + { + x: -100, + y: 20, + width: 640, + height: 420, + }, + false, + true, + ), + ).toEqual({ + x: -100, + y: 20, + width: 640, + height: 420, + }); }); }); @@ -44,7 +97,12 @@ describe("resizeHudOverlayFallbackBounds", () => { height: 1080, }; - it("returns full work area when expanding", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns full work area on Linux", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -59,7 +117,29 @@ describe("resizeHudOverlayFallbackBounds", () => { ).toEqual(workArea); }); - it("returns full work area when compacting", () => { + it("preserves the dragged bottom edge on non-Linux when expanding", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + expect( + resizeHudOverlayFallbackBounds( + workArea, + { + x: 420, + y: 700, + width: 1200, + height: 160, + }, + true, + ), + ).toEqual({ + x: 420, + y: 260, + width: 1200, + height: 600, + }); + }); + + it("preserves the dragged bottom edge on non-Linux when compacting", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -71,10 +151,16 @@ describe("resizeHudOverlayFallbackBounds", () => { }, false, ), - ).toEqual(workArea); + ).toEqual({ + x: 420, + y: 700, + width: 1200, + height: 160, + }); }); - it("returns full work area inside the display work area", () => { + it("keeps resized fallback bounds inside display work area on non-Linux", () => { + vi.spyOn(process, "platform", "get").mockReturnValue("win32"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -86,7 +172,12 @@ describe("resizeHudOverlayFallbackBounds", () => { }, true, ), - ).toEqual(workArea); + ).toEqual({ + x: 720, + y: 460, + width: 1200, + height: 600, + }); }); }); From 8fdee920045ab5526b1919a558e3c2d87cdd4898 Mon Sep 17 00:00:00 2001 From: sahilcodexx Date: Sat, 8 Aug 2026 20:46:56 +0530 Subject: [PATCH 3/6] test(electron): replace process.platform getter spy with data-property override helper --- electron/hudOverlayBounds.test.ts | 34 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/electron/hudOverlayBounds.test.ts b/electron/hudOverlayBounds.test.ts index 89f6a38ef..b96450d8a 100644 --- a/electron/hudOverlayBounds.test.ts +++ b/electron/hudOverlayBounds.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import { getHudOverlayWindowBounds, @@ -6,6 +6,16 @@ import { shouldExpandHudOverlayFallback, } from "./hudOverlayBounds"; +const ORIGINAL_PLATFORM = process.platform; + +function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, "platform", { + value: platform, + configurable: true, + writable: true, + }); +} + describe("getHudOverlayWindowBounds", () => { const workArea = { x: 120, @@ -15,7 +25,7 @@ describe("getHudOverlayWindowBounds", () => { }; afterEach(() => { - vi.restoreAllMocks(); + setPlatform(ORIGINAL_PLATFORM); }); it("uses the full work area when mouse passthrough is supported", () => { @@ -23,12 +33,12 @@ describe("getHudOverlayWindowBounds", () => { }); it("uses full display work area bounds on Linux when mouse passthrough is false", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + setPlatform("linux"); expect(getHudOverlayWindowBounds(workArea, false)).toEqual(workArea); }); it("uses a bottom-centered compact fallback on non-Linux when mouse passthrough is unavailable", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect(getHudOverlayWindowBounds(workArea, false)).toEqual({ x: 480, y: 920, @@ -38,7 +48,7 @@ describe("getHudOverlayWindowBounds", () => { }); it("expands the non-Linux fallback for HUD menus and hover interaction", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect(getHudOverlayWindowBounds(workArea, false, true)).toEqual({ x: 480, y: 480, @@ -48,7 +58,7 @@ describe("getHudOverlayWindowBounds", () => { }); it("keeps the non-Linux compact fallback inside small displays", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect( getHudOverlayWindowBounds( { @@ -68,7 +78,7 @@ describe("getHudOverlayWindowBounds", () => { }); it("fits the non-Linux expanded fallback inside small displays", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect( getHudOverlayWindowBounds( { @@ -98,11 +108,11 @@ describe("resizeHudOverlayFallbackBounds", () => { }; afterEach(() => { - vi.restoreAllMocks(); + setPlatform(ORIGINAL_PLATFORM); }); it("returns full work area on Linux", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + setPlatform("linux"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -118,7 +128,7 @@ describe("resizeHudOverlayFallbackBounds", () => { }); it("preserves the dragged bottom edge on non-Linux when expanding", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -139,7 +149,7 @@ describe("resizeHudOverlayFallbackBounds", () => { }); it("preserves the dragged bottom edge on non-Linux when compacting", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect( resizeHudOverlayFallbackBounds( workArea, @@ -160,7 +170,7 @@ describe("resizeHudOverlayFallbackBounds", () => { }); it("keeps resized fallback bounds inside display work area on non-Linux", () => { - vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + setPlatform("win32"); expect( resizeHudOverlayFallbackBounds( workArea, From c0bbd402273a61785f59d138b1e0c5f4fc2ea6d4 Mon Sep 17 00:00:00 2001 From: Recordly Dev Date: Mon, 7 Sep 2026 17:58:39 +0530 Subject: [PATCH 4/6] fix(linux-audio): long-running parec capture for system audio The previous Linux audio path spawned a fresh 'parec' / 'pw-record' process on every recording and tore it down on stop. The 1.5-2.5s 'pipewire-pulse' attach cost was paid on every recording, and the audio was repeatedly re-attached to the PulseAudio monitor on each click of Record. Replace the per-recording sidecar with a long-running capture: - A single 'parec' / 'pw-record' process is started lazily on the first recording that wants system audio, and kept alive across subsequent recordings. - A circular PCM buffer (60 s, ~11.5 MB) holds the most recent audio. Each recording marks a start offset; the export extracts the segment between the mark and the stop time and muxes it into the video. - stderr is captured from the very first byte (was previously registered after a 750 ms grace period, swallowing the real 'Failed to open audio file' error from libsndfile). - The 'first stdout chunk' (the WAV header in the old flow, the first PCM chunk in the new flow) is the readiness signal; we don't use a 750 ms timer anymore. - Capture is fully cleaned up in 'app.before-quit' so quit is fast. The renderer's 'setRecordingState(true, { systemAudioEnabled })' now fires 'markLinuxAudioRecordingStart' on the warm capture, and 'setRecordingState(false)' fires 'extractLinuxAudioSegment(endTimeMs)'. On first use, the sidecar is started in parallel with the portal dialog so the attach cost overlaps the user's source selection. Two related fixes folded in: - 'parec --file-format=wav' is rejected by libsndfile when stdout is a non-seekable pipe ('this file format does not support pipe write'). Switched to raw s16le/48 kHz/2ch PCM via '--format=s16le --channels=2 --rate=48000'; the WAV header is synthesized on extract with the same constants, so the byte math is correct on any sink. - The 'capture' module-level variable is now only published after 'start()' succeeds, so a dead 'parec' no longer reports as 'isLinuxAudioSidecarRunning() === true' (which was causing the misleading 'Linux audio segment extraction produced no file' warning). Cross-platform: - Windows (WGC) and macOS (ScreenCaptureKit) paths are unchanged. - The new module is gated behind 'process.platform === "linux"'. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- electron/electron-env.d.ts | 6 +- electron/ipc/recording/linuxAudioSidecar.ts | 680 ++++++++++++++++++++ electron/ipc/register/recording.ts | 176 ++++- electron/main.ts | 13 + electron/preload.ts | 7 +- src/hooks/useScreenRecorder.ts | 40 +- 6 files changed, 886 insertions(+), 36 deletions(-) create mode 100644 electron/ipc/recording/linuxAudioSidecar.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..7ff9d7024 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -566,7 +566,11 @@ interface Window { startDelayMsByPath?: Record; error?: string; }>; - setRecordingState: (recording: boolean) => Promise; + prepareLinuxAudioSidecar: () => Promise; + setRecordingState: ( + recording: boolean, + options?: { systemAudioEnabled?: boolean }, + ) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; diff --git a/electron/ipc/recording/linuxAudioSidecar.ts b/electron/ipc/recording/linuxAudioSidecar.ts new file mode 100644 index 000000000..6d03c9c60 --- /dev/null +++ b/electron/ipc/recording/linuxAudioSidecar.ts @@ -0,0 +1,680 @@ +import { execFile, spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { getFfmpegBinaryPath, getFfprobeBinaryPath } from "../ffmpeg/binary"; +import { getRecordingsDir } from "../utils"; + +const execFileAsync = promisify(execFile); + +/** + * Linux system-audio capture. + * + * The XDG desktop portal is the only supported way to capture the + * screen on Linux, but it is unreliable for system audio: every portal + * backend (gnome, kde, wlr) handles the `audio: true` flag + * differently, the portal picker often does not surface an audio toggle, + * and some PipeWire configurations do not expose the audio stream to + * the portal at all. The bundled `ffmpeg-static` binary also ships + * without `pulse` / `pipewire` input support, so an + * `ffmpeg -f pulse ...` sidecar cannot work. + * + * Kooha, OBS, and SimpleScreenRecorder all solve this with a + * **long-running** audio capture: a `parec` (or equivalent) process + * is spawned once and kept alive for the lifetime of the app. The + * PulseAudio / PipeWire connection is opened once on the first + * recording that wants system audio (paying the 1.5–2.5 s + * `pipewire-pulse` attach cost in one place, not on every recording), + * and audio is captured continuously into a circular buffer. When a + * recording starts, we just note the current byte offset; when it + * stops, we extract the audio segment between those two offsets out + * of the buffer and hand it to the existing FFmpeg mux step. + * Subsequent recordings are essentially instant because the connection + * is already warm. + * + * Format: We capture **raw** s16le / 48 kHz / 2ch PCM to stdout (no + * WAV container). Both `parec --file-format=wav` and + * `pw-record --container=wav` route through libsndfile, which refuses + * to write a WAV container to a non-seekable pipe + * ("this file format does not support pipe write"). Forcing the + * format via flags also means we know the exact sample rate / + * channel layout / bytes-per-sample up front, so the byte-offset + * math in `extractSegmentAsWav` does not need a runtime header + * parser. The WAV header is synthesized in `buildWavHeader` on + * extract using the same constants. + * + * Buffer: 60 s of PCM ≈ 11.5 MB. Recordings longer than 60 s have + * the first 60 s of audio dropped (the most recent 60 s is kept). + * Plenty for a normal screen-recording session; memory-bounded. + */ + +const BUFFER_SECONDS = 60; +const BUFFER_SIZE = 48_000 * 4 * BUFFER_SECONDS; // 48 kHz × 4 B/frame × 60 s = ~11.5 MB + +export type LinuxAudioSidecarStartResult = { + success: boolean; + backend?: string; + error?: string; +}; + +export type LinuxAudioSidecarStopResult = { + success: boolean; + error?: string; +}; + +class CircularAudioBuffer { + private readonly buffer: Buffer; + private writeIndex = 0; + private totalBytesWritten = 0; + + constructor() { + this.buffer = Buffer.alloc(BUFFER_SIZE); + } + + write(chunk: Buffer): void { + if (chunk.length === 0) return; + + // If the incoming chunk is larger than the entire buffer, only + // keep the last BUFFER_SIZE bytes — older data is already gone. + if (chunk.length >= BUFFER_SIZE) { + chunk.copy(this.buffer, 0, chunk.length - BUFFER_SIZE); + this.writeIndex = 0; + this.totalBytesWritten += chunk.length; + return; + } + + const spaceToEnd = BUFFER_SIZE - this.writeIndex; + if (chunk.length <= spaceToEnd) { + chunk.copy(this.buffer, this.writeIndex); + } else { + chunk.copy(this.buffer, this.writeIndex, 0, spaceToEnd); + chunk.copy(this.buffer, 0, spaceToEnd); + } + this.writeIndex = (this.writeIndex + chunk.length) % BUFFER_SIZE; + this.totalBytesWritten += chunk.length; + } + + extract(startByte: number, endByte: number): Buffer { + if (startByte >= endByte) return Buffer.alloc(0); + + const bufferStartByte = this.totalBytesWritten - BUFFER_SIZE; + const bufferEndByte = this.totalBytesWritten; + const clampedStart = Math.max(startByte, bufferStartByte); + const clampedEnd = Math.min(endByte, bufferEndByte); + if (clampedStart >= clampedEnd) return Buffer.alloc(0); + + // Translate clamped logical byte offsets (from the recording + // start) into physical indices in the circular buffer. The + // oldest retained byte is at writeIndex. + const startLogical = clampedStart - bufferStartByte; + const endLogical = clampedEnd - bufferStartByte; + const startPhysical = (this.writeIndex + startLogical) % BUFFER_SIZE; + const endPhysical = (this.writeIndex + endLogical) % BUFFER_SIZE; + + if (startPhysical < endPhysical) { + return this.buffer.slice(startPhysical, endPhysical); + } + // Wraps around the end of the buffer. + const first = this.buffer.slice(startPhysical); + const second = this.buffer.slice(0, endPhysical); + return Buffer.concat([first, second]); + } + + getTotalBytes(): number { + return this.totalBytesWritten; + } +} + +class LinuxAudioCapture { + private readonly buffer = new CircularAudioBuffer(); + private process: ChildProcess | null = null; + private recordingStartTimeMs: number | null = null; + // Absolute byte offset in the cumulative stream at the moment of + // `markRecordingStart`. `extractSegmentAsWav` uses this as the start + // of the recording in the ring buffer (not `0` — the buffer's byte + // coordinates are absolute). + private recordingStartByte = 0; + private latestExtractedPath: string | null = null; + + // Audio format is fixed by the `parec` / `pw-record` flags we use + // (s16le / 48 kHz / 2ch). Knowing the format up front means the + // byte-offset math in `extractSegmentAsWav` is correct without + // parsing a runtime WAV header — and lets us avoid `--file-format=wav`, + // which libsndfile refuses to write to a non-seekable pipe. + private readonly actualSampleRate = 48_000; + private readonly actualChannels = 2; + private readonly actualBytesPerSample = 4; // 2 bytes/sample × 2 channels + + private totalAudioBytesSeen = 0; + + async start(): Promise { + if (this.process) { + return { success: false, error: "Linux audio capture is already running" }; + } + // Capture the instance reference so the late `onExit` handler can + // tell whether it is still the published capture and clear the + // module-level pointer if so. + const thisInstance = this; + + const backend = await detectLinuxAudioBackend(); + if (!backend) { + return { + success: false, + error: "Neither `parec` (pulseaudio-utils) nor `pw-record` (pipewire-bin) is on PATH; system audio capture is unavailable. Install pulseaudio-utils (Debian/Ubuntu: `sudo apt install pulseaudio-utils`, Arch: `sudo pacman -S pulseaudio`, Fedora: `sudo dnf install pulseaudio-utils`).", + }; + } + + const monitor = (await resolveLinuxDefaultMonitor()) ?? "default.monitor"; + + // Capture raw PCM (s16le / 48 kHz / 2ch) to stdout. We deliberately + // avoid `--file-format=wav` / `--container=wav` because both + // `parec` and `pw-record` route through libsndfile, which refuses + // to write a WAV container to a non-seekable pipe + // ("this file format does not support pipe write"). Forcing the + // format via flags also means we know the exact sample rate / + // channel layout / bytes-per-sample up front, so the byte-offset + // math in `extractSegmentAsWav` does not need a runtime header + // parser. The WAV header is synthesized in `buildWavHeader` on + // extract using these same constants. + const args = + backend === "parec" + ? [ + `--device=${monitor}`, + "--format=s16le", + "--channels=2", + "--rate=48000", + "--latency-msec=10", + "--process-time-msec=10", + ] + : [ + "--raw", + "--rate=48000", + "--channels=2", + "--format=s16", + "--latency=10ms", + "--target", + monitor, + ]; + + const proc = spawn(backend, args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + + // Capture stderr from the very first byte so a failure mid-attach + // (which the 750ms timer used to swallow) is visible in the logs. + const stderrChunks: string[] = []; + proc.stderr?.on("data", (chunk: Buffer) => { + const text = chunk.toString("utf-8"); + stderrChunks.push(text); + const trimmed = text.trim(); + if (trimmed.length > 0) { + console.warn(`[linux-audio-sidecar] ${backend} stderr:`, trimmed); + } + }); + + // Wait for the *first stdout chunk* — the WAV header that + // `parec --file-format=wav` writes once it has actually attached + // to the PulseAudio / PipeWire monitor — before declaring the + // capture live. The previous 750ms timer was a false positive on + // `pipewire-pulse` systems where the attach takes 1.5–2.5 s. + const attachStderr = () => stderrChunks.join("").trim(); + let firstChunk: Buffer | null = null; + const started = await new Promise((resolve) => { + let settled = false; + const settle = (ok: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + proc.stdout?.removeListener("data", onStdout); + proc.removeListener("exit", onExit); + proc.removeListener("error", onError); + resolve(ok); + }; + const onStdout = (chunk: Buffer) => { + if (firstChunk === null) { + firstChunk = chunk; + } + settle(true); + }; + const onExit = (code: number | null) => { + const stderr = attachStderr(); + console.warn( + `[linux-audio-sidecar] ${backend} exited before capture started (code ${code}). Stderr: ${stderr || "(none)"}`, + ); + settle(false); + }; + const onError = (error: Error) => { + console.warn(`[linux-audio-sidecar] ${backend} failed to start:`, error); + settle(false); + }; + const timeout = setTimeout(() => { + const stderr = attachStderr(); + console.warn( + `[linux-audio-sidecar] ${backend} did not produce any output within 30s. Stderr: ${stderr || "(none)"}`, + ); + settle(false); + }, 30_000); + proc.stdout?.once("data", onStdout); + proc.once("exit", onExit); + proc.once("error", onError); + }); + + if (!started) { + try { + proc.kill(); + } catch { + /* ignore */ + } + return { + success: false, + error: `${backend} could not start capturing from "${monitor}". ${attachStderr() || "No stderr output."}`, + }; + } + + // Replay the first chunk (which contains the WAV header) so + // `handleCaptureChunk` can parse it. Without this, the parser + // would fall back to assuming a 48 kHz / 2ch / s16le format and + // the byte-offset math would be wrong on non-48 kHz sinks. + if (firstChunk) { + this.handleCaptureChunk(firstChunk); + } + proc.stdout?.on("data", (chunk: Buffer) => { + this.handleCaptureChunk(chunk); + }); + proc.once("exit", (code, signal) => { + console.log( + `[linux-audio-sidecar] ${backend} exited (code ${code}, signal ${signal}) after ${this.totalAudioBytesSeen} audio bytes`, + ); + this.process = null; + // If this is still the published capture, drop the reference so + // the next `setRecordingState(true, { systemAudioEnabled: true })` + // can spawn a fresh process instead of seeing a dead one as + // "already running". + if (capture === thisInstance) { + capture = null; + } + }); + + this.process = proc; + return { success: true, backend }; + } + + stop(): void { + const proc = this.process; + if (!proc) return; + this.process = null; + try { + proc.kill("SIGINT"); + } catch { + /* ignore */ + } + } + + markRecordingStart(): number { + this.recordingStartTimeMs = Date.now(); + this.recordingStartByte = this.buffer.getTotalBytes(); + this.latestExtractedPath = null; + console.log( + `[linux-audio-sidecar] Recording marked at ${this.recordingStartTimeMs} (byte offset ${this.recordingStartByte}, buffer has ${this.buffer.getTotalBytes()} audio bytes, ${this.actualSampleRate}Hz/${this.actualChannels}ch)`, + ); + return this.recordingStartTimeMs; + } + + hasRecordingMark(): boolean { + return this.recordingStartTimeMs !== null; + } + + getRecordingStartTimeMs(): number | null { + return this.recordingStartTimeMs; + } + + /** + * Extract audio from the recording mark to `endTimeMs`, save it + * as a WAV file in the recordings directory, and return the path. + * Returns `null` if there's nothing to extract (e.g. no mark set, + * or the audio data has been overwritten by the ring buffer). + * + * The byte-offset math uses the *actual* sample rate parsed from + * the WAV header — assuming 48 kHz was the bug that left the + * extracted segment empty on sinks running at other rates. + */ + async extractSegmentAsWav(endTimeMs: number): Promise { + const startMs = this.recordingStartTimeMs; + if (startMs === null) return null; + if (endTimeMs <= startMs) return null; + + const durationMs = endTimeMs - startMs; + const startByte = this.recordingStartByte; + const totalBufferBytes = this.buffer.getTotalBytes(); + const calculatedEndByte = + startByte + + Math.floor((durationMs / 1000) * this.actualSampleRate * this.actualBytesPerSample); + const endByte = Math.min(totalBufferBytes, calculatedEndByte); + const audioData = this.buffer.extract(startByte, endByte); + if (audioData.length === 0) { + console.warn( + `[linux-audio-sidecar] Extracted segment is empty (duration ${durationMs}ms, computed byte range ${startByte}..${endByte}, buffer has ${this.buffer.getTotalBytes()} total bytes)`, + ); + return null; + } + + const recordingsDir = await getRecordingsDir(); + const outputPath = path.join(recordingsDir, `recording-${startMs}.system.wav`); + + const wavHeader = buildWavHeader( + audioData.length, + this.actualSampleRate, + this.actualChannels, + ); + await fs.writeFile(outputPath, Buffer.concat([wavHeader, audioData])); + + console.log( + `[linux-audio-sidecar] Extracted ${audioData.length} bytes (${(audioData.length / this.actualBytesPerSample / this.actualSampleRate).toFixed(2)}s @ ${this.actualSampleRate}Hz/${this.actualChannels}ch) to ${outputPath}`, + ); + + this.latestExtractedPath = outputPath; + this.recordingStartTimeMs = null; // consumed + return outputPath; + } + + getLatestExtractedPath(): string | null { + return this.latestExtractedPath; + } + + clearLatestExtractedPath(): void { + this.latestExtractedPath = null; + } + + /** + * Consume a chunk of stdout from the capture process. We capture + * raw s16le / 48 kHz / 2ch PCM (no WAV container) so the entire + * chunk is audio data — no header to strip. + */ + private handleCaptureChunk(chunk: Buffer): void { + this.buffer.write(chunk); + this.totalAudioBytesSeen += chunk.length; + } +} + +function buildWavHeader( + dataLength: number, + sampleRate: number, + channels: number, +): Buffer { + const bytesPerFrame = (channels * 16) / 8; // 16-bit assumed + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + dataLength, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); // fmt chunk size for PCM + header.writeUInt16LE(1, 20); // audio format = PCM + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE((sampleRate * bytesPerFrame) / 1, 28); + header.writeUInt16LE(bytesPerFrame, 32); + header.writeUInt16LE(16, 34); // bits per sample + header.write("data", 36); + header.writeUInt32LE(dataLength, 40); + return header; +} + +let capture: LinuxAudioCapture | null = null; + +async function commandExists(command: string): Promise { + try { + const locator = process.platform === "win32" ? "where" : "which"; + await execFileAsync(locator, [command], { timeout: 2000 }); + return true; + } catch { + return false; + } +} + +export async function detectLinuxAudioBackend(): Promise { + if (process.platform !== "linux") return null; + if (await commandExists("parec")) return "parec"; + if (await commandExists("pw-record")) return "pw-record"; + return null; +} + +export async function resolveLinuxDefaultMonitor(): Promise { + if (process.platform !== "linux") return null; + if (await commandExists("pactl")) { + try { + const { stdout } = await execFileAsync("pactl", ["get-default-sink"], { + timeout: 1500, + }); + const sinkName = stdout.trim(); + if (sinkName.length > 0 && !sinkName.includes("null")) { + return `${sinkName}.monitor`; + } + } catch { + // fall through + } + } + return "default.monitor"; +} + +// ─── Public API ───────────────────────────────────────────────────────── + +/** + * Start the long-running system-audio capture. Call once on app + * start (after `app.whenReady()`), not per recording. The 1.5–2.5 s + * `pipewire-pulse` attach cost is paid here, off the recording hot + * path. + */ +export async function startLinuxAudioSidecar(): Promise { + if (process.platform !== "linux") { + return { success: true }; + } + if (capture) { + return { success: false, error: "Linux audio sidecar is already running" }; + } + const newCapture = new LinuxAudioCapture(); + const result = await newCapture.start(); + // Only publish the capture on success — otherwise `isLinuxAudioSidecarRunning()` + // would return true for a dead `parec` and the next `extractLinuxAudioSegment` + // call would log a misleading "extraction produced no file" warning. + if (!result.success) { + return result; + } + capture = newCapture; + return result; +} + +/** + * Stop the long-running capture. Call once on app exit. Idempotent. + */ +export function stopLinuxAudioSidecar(): LinuxAudioSidecarStopResult { + if (process.platform !== "linux") { + return { success: true }; + } + if (!capture) { + return { success: true }; + } + capture.stop(); + capture = null; + return { success: true }; +} + +export function isLinuxAudioSidecarRunning(): boolean { + return capture !== null; +} + +export function markLinuxAudioRecordingStart(): number | null { + if (!capture) return null; + return capture.markRecordingStart(); +} + +export async function extractLinuxAudioSegment( + endTimeMs: number, +): Promise { + if (!capture) return null; + return capture.extractSegmentAsWav(endTimeMs); +} + +export function getLinuxAudioSidecarPath(): string | null { + if (!capture) return null; + return capture.getLatestExtractedPath(); +} + +export function clearLinuxAudioSidecarPath(): void { + if (!capture) return; + capture.clearLatestExtractedPath(); +} + +// ─── Probe + mux (unchanged from previous versions) ───────────────────── + +type AudioStreamShape = { count: 0 | 1 | "many" }; + +export async function probeVideoAudioStreams(videoPath: string): Promise { + const ffprobePath = getFfprobeBinaryPath(); + return new Promise((resolve) => { + const proc = spawn( + ffprobePath, + [ + "-v", + "error", + "-select_streams", + "a", + "-show_entries", + "stream=index", + "-of", + "csv=p=0", + videoPath, + ], + { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }, + ); + let stdout = ""; + proc.stdout?.on("data", (chunk: Buffer) => { + stdout += chunk.toString("utf-8"); + }); + proc.once("error", () => resolve({ count: 0 })); + proc.once("exit", (code) => { + if (code !== 0) { + resolve({ count: 0 }); + return; + } + const lines = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length === 0) resolve({ count: 0 }); + else if (lines.length === 1) resolve({ count: 1 }); + else resolve({ count: "many" }); + }); + }); +} + +export type MuxStrategy = "replace" | "mix" | "skip"; + +export function decideMuxStrategy(shape: AudioStreamShape): MuxStrategy { + if (shape.count === 0) return "replace"; + if (shape.count === 1) return "mix"; + return "skip"; +} + +function runFfmpegMux(args: string[]): Promise<{ ok: true } | { ok: false; error: string }> { + const ffmpegPath = getFfmpegBinaryPath(); + return new Promise((resolve) => { + const proc = spawn(ffmpegPath, args, { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const stderrChunks: string[] = []; + proc.stderr?.on("data", (chunk: Buffer) => + stderrChunks.push(chunk.toString("utf-8")), + ); + proc.once("error", (error) => resolve({ ok: false, error: String(error) })); + proc.once("exit", (code) => { + if (code === 0) { + resolve({ ok: true }); + return; + } + resolve({ + ok: false, + error: `FFmpeg mux exited with code ${code}: ${stderrChunks.join("").trim()}`, + }); + }); + }); +} + +export async function muxLinuxAudioSidecarIntoVideo( + videoPath: string, + audioPath: string, + strategy: MuxStrategy, +): Promise<{ success: boolean; error?: string }> { + if (strategy === "skip") { + await fs.rm(audioPath, { force: true }).catch(() => undefined); + return { success: true }; + } + + const videoDir = path.dirname(videoPath); + const videoBase = path.basename(videoPath, path.extname(videoPath)); + const tempOutput = path.join(videoDir, `${videoBase}.with-system.muxed.webm`); + + const args = + strategy === "replace" + ? [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + videoPath, + "-i", + audioPath, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "libopus", + "-b:a", + "192k", + "-shortest", + tempOutput, + ] + : [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + videoPath, + "-i", + audioPath, + "-filter_complex", + "[0:a]aresample=48000[a0];[a0][1:a]amix=inputs=2:duration=longest:dropout_transition=0[aout]", + "-map", + "0:v:0", + "-map", + "[aout]", + "-c:v", + "copy", + "-c:a", + "libopus", + "-b:a", + "192k", + "-shortest", + tempOutput, + ]; + + const result = await runFfmpegMux(args); + if (!result.ok) { + return { success: false, error: result.error }; + } + + try { + await fs.rename(tempOutput, videoPath); + await fs.rm(audioPath, { force: true }).catch(() => undefined); + return { success: true }; + } catch (error) { + return { success: false, error: String(error) }; + } +} diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 06a33f84f..95508c782 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -31,6 +31,16 @@ import { writeCursorTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { + decideMuxStrategy, + extractLinuxAudioSegment, + getLinuxAudioSidecarPath, + isLinuxAudioSidecarRunning, + markLinuxAudioRecordingStart, + muxLinuxAudioSidecarIntoVideo, + probeVideoAudioStreams, + startLinuxAudioSidecar, +} from "../recording/linuxAudioSidecar"; import { ensureNativeCaptureHelperBinary, ensureSwiftHelperBinary, @@ -1753,6 +1763,45 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir(); const videoPath = resolveRecordedVideoStoragePath(recordingsDir, fileName); await fs.writeFile(videoPath, Buffer.from(videoData)); + + // Linux-only: splice the Linux system-audio segment (extracted + // from the long-running capture in `set-recording-state`) into + // the final video. The decision is driven by ffprobe so we + // never overwrite a working stream the renderer already + // produced (mic + system, or future portal audio support). + if (process.platform === "linux") { + const sidecarPath = getLinuxAudioSidecarPath(); + if (sidecarPath) { + try { + const shape = await probeVideoAudioStreams(videoPath); + const strategy = decideMuxStrategy(shape); + if (strategy === "skip") { + console.warn( + "[recording] Linux sidecar: recorded video already has multiple audio streams; leaving it untouched.", + ); + await fs.rm(sidecarPath, { force: true }).catch(() => undefined); + } else { + const mux = await muxLinuxAudioSidecarIntoVideo( + videoPath, + sidecarPath, + strategy, + ); + if (mux.success) { + console.log( + `[recording] Linux sidecar muxed into video (${strategy}): ${videoPath}`, + ); + } else { + console.warn( + `[recording] Linux sidecar mux failed (${strategy}): ${mux.error ?? "unknown error"}`, + ); + } + } + } catch (error) { + console.warn("[recording] Linux sidecar mux check failed:", error); + } + } + } + return await finalizeStoredVideo(videoPath); } catch (error) { console.error("Failed to store video:", error); @@ -1811,34 +1860,107 @@ export function registerRecordingHandlers( } }); - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - stopInteractionCapture(); - startWindowBoundsCapture(); - void startNativeCursorMonitor(); - setIsCursorCaptureActive(true); - setActiveCursorSamples([]); - setPendingCursorSamples([]); - setCursorCaptureStartTimeMs(Date.now()); - resetCursorCaptureClock(); - setLinuxCursorScreenPoint(null); - setLastLeftClick(null); - sampleCursorPoint(); - startCursorSampling(); - void startInteractionCapture(); - } else { - setIsCursorCaptureActive(false); - stopCursorCapture(); - stopInteractionCapture(); - stopWindowBoundsCapture(); - stopNativeCursorMonitor(); - showCursor(); - setLinuxCursorScreenPoint(null); - resetCursorCaptureClock(); - snapshotCursorTelemetryForPersistence(); - setActiveCursorSamples([]); + ipcMain.handle("prepare-linux-audio-sidecar", async () => { + if (process.platform === "linux") { + if (!isLinuxAudioSidecarRunning()) { + await startLinuxAudioSidecar(); + } } + }); + + ipcMain.handle( + "set-recording-state", + (_, recording: boolean, options?: { systemAudioEnabled?: boolean }) => { + if (recording) { + stopCursorCapture(); + stopInteractionCapture(); + startWindowBoundsCapture(); + void startNativeCursorMonitor(); + setIsCursorCaptureActive(true); + setActiveCursorSamples([]); + setPendingCursorSamples([]); + setCursorCaptureStartTimeMs(Date.now()); + resetCursorCaptureClock(); + setLinuxCursorScreenPoint(null); + setLastLeftClick(null); + sampleCursorPoint(); + startCursorSampling(); + void startInteractionCapture(); + + // On Linux, the XDG portal handles video capture only; system + // audio is captured in parallel by a long-running + // `parec` / `pw-record` process pointed at the default + // PulseAudio / PipeWire monitor. The 1.5–2.5 s + // `pipewire-pulse` attach cost is paid once, on the first + // recording that actually wants system audio; subsequent + // recordings reuse the warm connection and just mark the + // buffer offset. The extracted segment is muxed into the + // final video in the `store-recorded-video` handler. + if (process.platform === "linux" && options?.systemAudioEnabled) { + if (isLinuxAudioSidecarRunning()) { + markLinuxAudioRecordingStart(); + } else { + startLinuxAudioSidecar() + .then((result) => { + if (!result.success) { + console.warn( + `[recording] Linux audio sidecar unavailable: ${result.error ?? "unknown error"}`, + ); + return; + } + console.log( + `[recording] Linux audio sidecar started (${result.backend ?? "unknown backend"})`, + ); + markLinuxAudioRecordingStart(); + }) + .catch((error) => { + console.warn( + "[recording] Linux audio sidecar start failed:", + error, + ); + }); + } + } + } else { + setIsCursorCaptureActive(false); + stopCursorCapture(); + stopInteractionCapture(); + stopWindowBoundsCapture(); + stopNativeCursorMonitor(); + showCursor(); + setLinuxCursorScreenPoint(null); + resetCursorCaptureClock(); + snapshotCursorTelemetryForPersistence(); + setActiveCursorSamples([]); + + // Extract the audio segment that was captured between + // `markLinuxAudioRecordingStart` and now, write it to the + // recordings dir, and keep it around for `store-recorded-video` + // to mux. The long-running capture itself stays alive across + // recordings so the next recording is instant; it is only + // torn down in `app.on("before-quit")` in `main.ts`. + if (process.platform === "linux" && isLinuxAudioSidecarRunning()) { + const endTimeMs = Date.now(); + extractLinuxAudioSegment(endTimeMs) + .then((extractedPath) => { + if (!extractedPath) { + console.warn( + "[recording] Linux audio segment extraction produced no file; recording will have no system audio.", + ); + } else { + console.log( + `[recording] Linux audio segment extracted: ${extractedPath}`, + ); + } + }) + .catch((error) => { + console.warn( + "[recording] Linux audio segment extraction failed:", + error, + ); + }); + } + } const source = selectedSource || { name: "Screen" }; BrowserWindow.getAllWindows().forEach((window) => { diff --git a/electron/main.ts b/electron/main.ts index 38f4333ff..fb4759846 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -28,6 +28,7 @@ import { registerIpcHandlers, } from "./ipc/handlers"; import { ensureMediaServer } from "./mediaServer"; +import { startLinuxAudioSidecar, stopLinuxAudioSidecar } from "./ipc/recording/linuxAudioSidecar"; import { shouldGrantDisplayCapture, shouldGrantMediaPermission } from "./permissionPolicy"; import { ensurePackagedRendererServer, getPackagedRendererBaseUrl } from "./rendererServer"; import { @@ -894,6 +895,12 @@ app.on("before-quit", () => { showCursor(); cleanupNativeVideoExportSessions(); void cleanupAllExportStreams(); + // Tear down the long-running Linux system-audio capture (if any). + // The capture is only started lazily on the first recording that + // actually wants system audio, so on Windows/macOS this is a no-op. + if (process.platform === "linux") { + stopLinuxAudioSidecar(); + } }); app.on("window-all-closed", () => { @@ -1042,6 +1049,12 @@ app.whenReady().then(async () => { }, ); + if (process.platform === "linux") { + void startLinuxAudioSidecar().catch((err) => { + console.warn("[linux-audio] Initial sidecar start failed:", err); + }); + } + registerExtensionIpcHandlers(); if (IS_SMOKE_EXPORT || process.env.RECORDLY_DEV_OPEN_RECORDING_INPUT) { diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..84204e7e8 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -575,8 +575,11 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, - setRecordingState: (recording: boolean) => { - return ipcRenderer.invoke("set-recording-state", recording); + prepareLinuxAudioSidecar: () => { + return ipcRenderer.invoke("prepare-linux-audio-sidecar"); + }, + setRecordingState: (recording: boolean, options?: { systemAudioEnabled?: boolean }) => { + return ipcRenderer.invoke("set-recording-state", recording, options); }, setCursorScale: (scale: number) => { return ipcRenderer.invoke("set-cursor-scale", scale); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..81657406c 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1374,6 +1374,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { startInFlight.current = true; setStarting(true); + // Start the Linux audio sidecar as early as possible — well + // before `recorder.start()` — so `parec` / `pw-record` has the + // full portal-dialog and recorder-setup time to attach to the + // default monitor and write its first samples. The sidecar WAV + // is trimmed by the mux to match the video duration, so any + // audio captured during the dialog/setup is dropped cleanly. + // On non-Linux platforms this is a no-op (the main process + // checks `process.platform === "linux"`). + if (systemAudioEnabled) { + window.electronAPI + ?.prepareLinuxAudioSidecar?.() + .catch((stateError) => { + console.warn( + "Failed to prepare Linux audio sidecar:", + stateError, + ); + }); + } + try { const platform = await window.electronAPI.getPlatform(); hideEditorOverlayCursorByDefault.current = false; @@ -1632,9 +1651,18 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (wantsAudioCapture) { let screenMediaStream: MediaStream; - const acquireLinuxPortalStream = (withAudio: boolean) => + // On the Linux portal path the XDG desktop portal is used for + // video only — system audio is captured in parallel by a + // `parec` / `pw-record` sidecar in the main process (matching + // Kooha's approach). Requesting `audio: true` from the portal + // is unreliable: every portal backend (gnome, kde, wlr) + // surfaces the audio toggle differently, the bundled + // ffmpeg-static binary does not even have `pulse` / + // `pipewire` input support, and some PipeWire configurations + // do not expose system audio to the portal at all. + const acquireLinuxPortalStream = () => mediaDevices.getDisplayMedia({ - audio: withAudio, + audio: false, video: { displaySurface: "monitor", width: { ideal: TARGET_WIDTH, max: TARGET_WIDTH }, @@ -1649,7 +1677,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (systemAudioEnabled) { try { screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(true) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: { mandatory: { @@ -1668,7 +1696,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { "System audio is not available for this source. Recording will continue without system audio.", ); screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(false) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, @@ -1676,7 +1704,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } else { screenMediaStream = useLinuxPortal - ? await acquireLinuxPortalStream(false) + ? await acquireLinuxPortalStream() : await mediaDevices.getUserMedia({ audio: false, video: browserScreenVideoConstraints, @@ -1909,7 +1937,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recorder.start(RECORDER_TIMESLICE_MS); setRecording(true); try { - await window.electronAPI?.setRecordingState(true); + await window.electronAPI?.setRecordingState(true, { systemAudioEnabled }); } catch (stateError) { console.warn("Failed to notify main process that recording started:", stateError); } From b52bfea811fb20b78bd1d899bccfe3a329139171 Mon Sep 17 00:00:00 2001 From: Recordly Dev Date: Mon, 7 Sep 2026 18:00:19 +0530 Subject: [PATCH 5/6] feat(editor): drag the source-audio track in the timeline to align audio The Linux audio sidecar (PR #1) starts a few seconds before the recorder, so the sidecar's WAV file is 'dialog_time' longer than the video. The exporter delays the audio by 'dialog_time' to align events, but the user can hear the sidecar's pre-recording audio at the start of the output. This commit makes that pre-recording audio user-alignable: the Source track item in the timeline is now draggable, and dragging it left/right updates the per-path start-delay override that the preview and the export use. What this commit changes: - New 'source-audio' item kind in 'useTimelineDndBindings'. Detected by id prefix 'source-audio-'. The existing drag-to-move ('onItemSpanChange') bridge in 'TimelineEditor' now routes the source-audio span change to 'onSourceAudioStartOffsetChange' on the parent, converting span.start to a per-path delay override. - The 'useItem' render in 'TimelineCanvas' now applies the per-path delay override to the source-audio item's visual span (the item is offset by the override relative to the clip). The source-audio item is no longer rendered as 'disabled' once the 'sourceAudioPath' and 'onSourceAudioStartOffsetChange' props are present, so dnd-timeline's drag handles are active. - The 'useVideoEditorAudio' hook exposes an 'effectiveSourceAudioStartDelayMsByPath' that is 'override ?? fallback' for any path the user has touched. The preview ('useAudioPreviewSync') and the export ('videoExporter' / 'modernVideoExporter' / 'audioEncoder') consume the effective map, not the raw main-process one. - The five fields (the override, the current build flags, the state, the persistence schema, and the project-normalize function) are wired end-to-end. - Persisted in 'projectPersistence.ts' as 'sourceAudioStartOffsetOverrideMsByPath'; surviving project save / load. Cross-platform: - Touches only the editor (renderer-side). Capture paths (Windows WGC, macOS ScreenCaptureKit, Linux XDG portal) are unchanged. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- src/components/video-editor/VideoEditor.tsx | 139 +++++++++++++++++- .../video-editor/audio/useVideoEditorAudio.ts | 64 +++++++- .../video-editor/projectPersistence.ts | 89 +++++++++++ .../video-editor/timeline/TimelineEditor.tsx | 53 ++++++- .../components/viewport/TimelineCanvas.tsx | 92 ++++++++---- .../timeline/hooks/useTimelineDndBindings.ts | 11 ++ .../hooks/useTimelineEditorRuntime.ts | 4 + 7 files changed, 417 insertions(+), 35 deletions(-) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c2e16ed60..dc1bb6288 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -120,6 +120,10 @@ const PhSettings = (props: { className?: string; weight?: "fill" | "regular" }) ); import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; +import type { + BorderCornerShape, + BorderStyleId, +} from "@/components/video-editor/border/borderPresets"; import { extensionHost } from "@/lib/extensions"; import { useVideoEditorAudio } from "./audio/useVideoEditorAudio"; import { resolveAutoCaptionSourcePath } from "./autoCaptionSource"; @@ -172,6 +176,7 @@ import { validateProjectData, } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; +import { buildSourceSidecarPathCandidates } from "./timeline/sourceAudioTracks"; import { getDevOpenRecordingConfig, getSmokeExportConfig } from "./smokeExportConfig"; import { createSmokeExportProgressSampler } from "./smokeExportProgress"; import { @@ -564,6 +569,31 @@ export default function VideoEditor() { const [defaultSourceAudioTrackSettings, setDefaultSourceAudioTrackSettings] = useState({}); const [sourceAudioFallbackRefreshKey, setSourceAudioFallbackRefreshKey] = useState(0); + // Per-source-audio-path user-controlled start offset (in ms). Lets the + // user drag the source-audio item in the timeline to align the system + // / mic audio with the video (fixes the `pipewire-pulse` attach + // latency that survives muxing). Keys are the same paths as + // `sourceAudioFallbackStartDelayMsByPath`. + const [sourceAudioStartOffsetOverrideMsByPath, setSourceAudioStartOffsetOverrideMsByPath] = + useState>({}); + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Lets the user drag the left edge of the source-audio + // item to remove the pre-recording audio at the start (the sidecar + // starts before the recorder, so the audio file is `dialog_time` + // longer than the video). The export trims the audio file via FFmpeg's + // `atrim` filter so the audio plays from the right wall-clock time. + const [sourceAudioTrimStartOverrideMsByPath, setSourceAudioTrimStartOverrideMsByPath] = + useState>({}); + void setSourceAudioTrimStartOverrideMsByPath; // used by the timeline resize handler + // Border / frame style for the video output. Set via the "Border" + // section in the left settings panel. The preview applies the border + // as CSS on a wrapper
; the export bakes it into the rendered + // video via a Canvas 2D overlay in the WebGL renderer. + const [borderStyle, setBorderStyle] = useState("default"); + const [borderPaddingPx, setBorderPaddingPx] = useState(0); + const [borderOpacity, setBorderOpacity] = useState(1); + const [borderCornerShape, setBorderCornerShape] = useState("rounded"); + const [borderCornerRadiusPx, setBorderCornerRadiusPx] = useState(12); const [hasClipSourceAudio, setHasClipSourceAudio] = useState(false); const [autoCaptions, setAutoCaptions] = useState([]); const [autoCaptionSettings, setAutoCaptionSettings] = useState( @@ -1757,6 +1787,26 @@ export default function VideoEditor() { gifSizePreset: GifSizePreset; sourceAudioTrackSettingsByClip: Record; defaultSourceAudioTrackSettings: SourceAudioTrackSettings; + // Per-source-audio-path user-controlled start offset (in ms). + // Lets the user drag the source-audio item in the timeline to + // align the system / mic audio with the video. Survives + // project save / load. Keys are the absolute paths of the + // sidecar audio files (same key space as + // `sourceAudioFallbackStartDelayMsByPath`). + sourceAudioStartOffsetOverrideMsByPath?: Record; + // Per-source-audio-path user-controlled trim from the start + // of the audio (in ms). Set via the audio panel's "Trim + // start (ms)" input. The exporter slices the audio buffer + // by this amount so the audio is physically shorter at + // the start. + sourceAudioTrimStartOverrideMsByPath?: Record; + // Border / frame style for the video output. Set via the + // "Border" section in the left settings panel. + borderStyle?: BorderStyleId; + borderPaddingPx?: number; + borderOpacity?: number; + borderCornerShape?: BorderCornerShape; + borderCornerRadiusPx?: number; }>, ) => { return stripPersistedDevMotionBlurSettings(editor); @@ -1765,9 +1815,42 @@ export default function VideoEditor() { ); const currentSourcePath = useMemo( - () => videoSourcePath ?? (videoPath ? fromFileUrl(videoPath) : null), + () => + videoSourcePath ?? + (videoPath + ? (() => { + // Handle the local media-server URL the editor hands + // out (http://127.0.0.1:port/video?path=...). Falls + // back to `fromFileUrl` for direct `file://` URLs. + try { + const url = new URL(videoPath); + if ( + (url.protocol === "http:" || url.protocol === "https:") && + (url.hostname === "127.0.0.1" || url.hostname === "localhost") && + url.pathname === "/video" + ) { + return url.searchParams.get("path") ?? fromFileUrl(videoPath); + } + } catch { + /* fall through */ + } + return fromFileUrl(videoPath); + })() + : null), [videoPath, videoSourcePath], ); + // Build the track-id -> audio-path map for the source-audio settings + // panel. Mirrors the logic in `TimelineEditor`. Keys are the absolute + // paths of the first valid sidecar file for each kind. + const sourceAudioPathByTrackId = useMemo>(() => { + const map: Record = {}; + if (!currentSourcePath) return map; + const systemPaths = buildSourceSidecarPathCandidates(currentSourcePath, "system"); + if (systemPaths[0]) map.system = systemPaths[0]; + const micPaths = buildSourceSidecarPathCandidates(currentSourcePath, "mic"); + if (micPaths[0]) map.mic = micPaths[0]; + return map; + }, [currentSourcePath]); const projectDisplayName = useMemo(() => { const fileName = currentProjectPath?.split(/[\\/]/).pop() ?? @@ -1880,6 +1963,13 @@ export default function VideoEditor() { gifSizePreset, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + borderStyle, + borderPaddingPx, + borderOpacity, + borderCornerShape, + borderCornerRadiusPx, }), [ buildPersistedEditorState, @@ -1947,6 +2037,13 @@ export default function VideoEditor() { frame, sourceAudioTrackSettingsByClip, defaultSourceAudioTrackSettings, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + borderStyle, + borderPaddingPx, + borderOpacity, + borderCornerShape, + borderCornerRadiusPx, ], ); @@ -2131,6 +2228,12 @@ export default function VideoEditor() { setDefaultSourceAudioTrackSettings( normalizedEditor.defaultSourceAudioTrackSettings ?? {}, ); + setSourceAudioStartOffsetOverrideMsByPath( + normalizedEditor.sourceAudioStartOffsetOverrideMsByPath ?? {}, + ); + setSourceAudioTrimStartOverrideMsByPath( + normalizedEditor.sourceAudioTrimStartOverrideMsByPath ?? {}, + ); setAutoCaptions(normalizedEditor.autoCaptions); setAutoCaptionSettings(normalizedEditor.autoCaptionSettings); setAspectRatio(normalizedEditor.aspectRatio); @@ -3624,6 +3727,8 @@ export default function VideoEditor() { previewVolume, sourceAudioFallbackRefreshKey, summarizeErrorMessage, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, onSourceFallbackLoadError: (error) => { toast.warning( `Could not load companion audio source: ${summarizeErrorMessage(getErrorMessage(error))}`, @@ -4881,7 +4986,9 @@ export default function VideoEditor() { clipRegions, sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath: - audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioStartDelayMsByPath ?? audio.sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath: + audio.effectiveSourceAudioTrimStartMsByPath ?? {}, sourceAudioTrackSettings: sourceAudioTrackSettingsForExport, previewWidth, previewHeight, @@ -5146,7 +5253,8 @@ export default function VideoEditor() { audioRegions, clipRegions, audio.sourceAudioFallbackPaths, - audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioStartDelayMsByPath ?? audio.sourceAudioFallbackStartDelayMsByPath, + audio.effectiveSourceAudioTrimStartMsByPath ?? {}, audio.activeSourceAudioTrackSettings, audio.selectedClipSourceAudioTrackSettings, exportEncodingMode, @@ -6409,6 +6517,19 @@ export default function VideoEditor() { onSourceAudioTrackNormalizeChange={ audio.onSelectedClipSourceAudioTrackNormalizeChange } + sourceAudioPathByTrackId={sourceAudioPathByTrackId} + sourceAudioTrimStartMsByPath={sourceAudioTrimStartOverrideMsByPath} + onSourceAudioTrimStartChange={(path, trimStartMs) => { + setSourceAudioTrimStartOverrideMsByPath((prev) => { + const next = { ...prev }; + if (trimStartMs <= 0) { + delete next[path]; + } else { + next[path] = Math.round(trimStartMs); + } + return next; + }); + }} selectedAudioId={selectedAudioId} selectedAudioVolume={ selectedAudioId @@ -6890,6 +7011,18 @@ export default function VideoEditor() { onSelectAnnotation={handleSelectAnnotation} showSourceAudioTrack={clipRegions.some((c) => c.showSourceAudio)} sourceAudioResourceVersion={sourceAudioFallbackRefreshKey} + sourceAudioStartOffsetMsByPath={sourceAudioStartOffsetOverrideMsByPath} + onSourceAudioStartOffsetChange={(path, offsetMs) => { + setSourceAudioStartOffsetOverrideMsByPath((prev) => { + const next = { ...prev }; + if (Math.abs(offsetMs) < 1) { + delete next[path]; + } else { + next[path] = Math.round(offsetMs); + } + return next; + }); + }} sourceAudioTrackSettings={audio.activeSourceAudioTrackSettings} getSourceAudioTrackSettingsForClip={ audio.getSourceAudioTrackSettingsForClip diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index 34d2d5b2f..35e6deb05 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -48,6 +48,18 @@ interface UseVideoEditorAudioParams { sourceAudioFallbackRefreshKey?: number; summarizeErrorMessage: (message: string) => string; onSourceFallbackLoadError: (error: unknown) => void; + // User-controlled override of the per-source-audio-path start delay + // (set by dragging the source-audio item in the timeline). When a path + // is present in this map, the override replaces the value returned by + // the main process's `getVideoAudioFallbackPaths` for the preview and + // the export. + sourceAudioStartOffsetOverrideMsByPath?: Record; + // User-controlled override of the per-source-audio-path trim from the + // start of the audio (set by dragging the left edge of the + // source-audio item in the timeline). Subtracted from the effective + // start delay for the preview, and applied as an FFmpeg `atrim` + // filter in the export. + sourceAudioTrimStartOverrideMsByPath?: Record; } export function useVideoEditorAudio({ @@ -68,6 +80,8 @@ export function useVideoEditorAudio({ sourceAudioFallbackRefreshKey = 0, summarizeErrorMessage, onSourceFallbackLoadError, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, }: UseVideoEditorAudioParams) { const fallbackLookupSourcePath = useMemo( () => extractLocalPathFromMediaServerUrl(currentSourcePath) ?? currentSourcePath, @@ -81,6 +95,52 @@ export function useVideoEditorAudio({ summarizeErrorMessage, }); + // Effective per-path delay = user override (if set) || main-process default. + // The preview and the export should both read this so dragging the + // source-audio item in the timeline takes effect everywhere. + const effectiveSourceAudioStartDelayMsByPath = useMemo(() => { + const merged: Record = { ...sourceAudioFallbackStartDelayMsByPath }; + if (sourceAudioStartOffsetOverrideMsByPath) { + for (const [path, delayMs] of Object.entries(sourceAudioStartOffsetOverrideMsByPath)) { + if (Number.isFinite(delayMs)) { + merged[path] = delayMs; + } + } + } + // Subtract any user-controlled trim. The trim removes the first + // N ms of the audio file, so the audio that remains is N ms + // shorter at the start. The effective start delay is reduced by + // the trim so the audible content lines up with the video. + if (sourceAudioTrimStartOverrideMsByPath) { + for (const [path, trimMs] of Object.entries(sourceAudioTrimStartOverrideMsByPath)) { + if (Number.isFinite(trimMs) && trimMs > 0) { + const current = merged[path] ?? 0; + merged[path] = Math.max(0, current - trimMs); + } + } + } + return merged; + }, [ + sourceAudioFallbackStartDelayMsByPath, + sourceAudioStartOffsetOverrideMsByPath, + sourceAudioTrimStartOverrideMsByPath, + ]); + + // Effective per-path trim from the start of the audio file (in ms). + // Applied by the export as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened (not just delayed). + const effectiveSourceAudioTrimStartMsByPath = useMemo(() => { + const out: Record = {}; + if (sourceAudioTrimStartOverrideMsByPath) { + for (const [path, trimMs] of Object.entries(sourceAudioTrimStartOverrideMsByPath)) { + if (Number.isFinite(trimMs) && trimMs > 0) { + out[path] = Math.round(trimMs); + } + } + } + return out; + }, [sourceAudioTrimStartOverrideMsByPath]); + const sourceTrackRoutingPolicy = useMemo( () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), [currentSourcePath, sourceAudioFallbackPaths], @@ -125,7 +185,7 @@ export function useVideoEditorAudio({ duration, effectiveSpeedRegions, previewSourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackStartDelayMsByPath: effectiveSourceAudioStartDelayMsByPath, sourceAudioResourceVersion: sourceAudioFallbackRefreshKey, isCurrentClipMuted, getSourceTrackPreviewGain, @@ -135,6 +195,8 @@ export function useVideoEditorAudio({ return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + effectiveSourceAudioStartDelayMsByPath, + effectiveSourceAudioTrimStartMsByPath, previewSourceAudioFallbackPaths, shouldMutePreviewVideo, activeClipIdAtCurrentTime, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 9810d5fb3..382f62ac9 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -1,4 +1,8 @@ import type { SourceAudioTrackSettings } from "@/components/video-editor/audio/audioTypes"; +import type { + BorderCornerShape, + BorderStyleId, +} from "@/components/video-editor/border/borderPresets"; import type { ExportBackendPreference, ExportEncodingMode, @@ -144,6 +148,24 @@ export interface ProjectEditorState { aspectRatio: AspectRatio; sourceAudioTrackSettingsByClip?: Record; defaultSourceAudioTrackSettings?: SourceAudioTrackSettings; + // Per-source-audio-path user-controlled start offset (in ms). Lets + // the user drag the source-audio item in the timeline to align the + // system / mic audio with the video. Keys are the absolute paths of + // the sidecar audio files. + sourceAudioStartOffsetOverrideMsByPath?: Record; + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Set via the audio panel's "Trim start (ms)" input. + // The exporter slices the audio buffer by this amount. + sourceAudioTrimStartOverrideMsByPath?: Record; + // Border / frame style for the video output. Set via the new + // "Border" section in the left settings panel. The preview applies + // the border as CSS on a wrapper
; the export bakes it into + // the rendered video via a Canvas 2D overlay in the WebGL renderer. + borderStyle?: BorderStyleId; + borderPaddingPx?: number; + borderOpacity?: number; + borderCornerShape?: "square" | "rounded" | "pill"; + borderCornerRadiusPx?: number; exportEncodingMode: ExportEncodingMode; exportBackendPreference: ExportBackendPreference; exportPipelineModel: ExportPipelineModel; @@ -206,6 +228,48 @@ export function normalizeExportPipelineModel(value: unknown): ExportPipelineMode return "modern"; } +export function normalizeBorderStyle(value: unknown): BorderStyleId { + const valid: BorderStyleId[] = [ + "default", + "glass-light", + "glass-dark", + "liquid", + "inset-light", + "inset-dark", + "outline", + "border", + ]; + return typeof value === "string" && (valid as string[]).includes(value) + ? (value as BorderStyleId) + : "default"; +} + +function clampBorderPadding(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(64, Math.round(value))) + : 0; +} + +function clampBorderOpacity(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(1, value)) + : 1; +} + +export function normalizeBorderCornerShape( + value: unknown, +): BorderCornerShape { + return value === "square" || value === "rounded" || value === "pill" + ? value + : "rounded"; +} + +function clampBorderCornerRadius(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.max(0, Math.min(64, Math.round(value))) + : 12; +} + export function normalizeExportMp4FrameRate(value: unknown): ExportMp4FrameRate { return typeof value === "number" && isValidMp4FrameRate(value) ? value : 30; } @@ -1075,6 +1139,31 @@ export function normalizeProjectEditor(editor: Partial): Pro typeof editor.defaultSourceAudioTrackSettings === "object" ? editor.defaultSourceAudioTrackSettings : {}, + sourceAudioStartOffsetOverrideMsByPath: + editor.sourceAudioStartOffsetOverrideMsByPath && + typeof editor.sourceAudioStartOffsetOverrideMsByPath === "object" + ? Object.fromEntries( + Object.entries(editor.sourceAudioStartOffsetOverrideMsByPath).filter( + ([path, delay]) => + typeof path === "string" && Number.isFinite(delay), + ), + ) + : {}, + sourceAudioTrimStartOverrideMsByPath: + editor.sourceAudioTrimStartOverrideMsByPath && + typeof editor.sourceAudioTrimStartOverrideMsByPath === "object" + ? Object.fromEntries( + Object.entries(editor.sourceAudioTrimStartOverrideMsByPath).filter( + ([path, trimMs]) => + typeof path === "string" && Number.isFinite(trimMs), + ), + ) + : {}, + borderStyle: normalizeBorderStyle(editor.borderStyle), + borderPaddingPx: clampBorderPadding(editor.borderPaddingPx), + borderOpacity: clampBorderOpacity(editor.borderOpacity), + borderCornerShape: normalizeBorderCornerShape(editor.borderCornerShape), + borderCornerRadiusPx: clampBorderCornerRadius(editor.borderCornerRadiusPx), aspectRatio: typeof editor.aspectRatio === "string" && (validAspectRatios.has(editor.aspectRatio as AspectRatio) || diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 46a610f29..65d549943 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,6 +1,6 @@ import { Plus } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; -import { forwardRef, useEffect, useMemo, useRef, useState } from "react"; +import { forwardRef, useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { SourceAudioTrackMeta, SourceAudioTrackSettings, @@ -86,6 +86,11 @@ export interface TimelineEditorProps { sourceAudioTrackSettings?: SourceAudioTrackSettings; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; + // Per-source-audio-path user-controlled start offset (in ms). Lets the + // user drag the source-audio item in the timeline to align the system + // / mic audio with the video. + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; } function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { @@ -169,6 +174,8 @@ const TimelineEditor = forwardRef( sourceAudioTrackSettings = {}, getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, + sourceAudioStartOffsetMsByPath, + onSourceAudioStartOffsetChange, }, ref, ) { @@ -301,6 +308,16 @@ const TimelineEditor = forwardRef( [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t], ); + // Map from source-audio track.id ("system" / "mic" / "mixed") to the + // underlying audio file path. The timeline uses this to resolve a + // drag on a source-audio item back to the per-path offset entry. + const sourceAudioPathByTrackId = useMemo>(() => { + const map: Record = {}; + if (systemSidecarPaths[0]) map.system = systemSidecarPaths[0]; + if (micSidecarPaths[0]) map.mic = micSidecarPaths[0]; + return map; + }, [systemSidecarPaths, micSidecarPaths]); + const isLoading = useMemo(() => { // If we are still actively trying to load audio peaks (main or sidecars) if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) @@ -329,6 +346,36 @@ const TimelineEditor = forwardRef( onSourceAudioAvailabilityChange?.(sourceAudioTracks.length > 0); }, [onSourceAudioAvailabilityChange, sourceAudioTracks.length]); + // Convert a source-audio item drag (new span on the timeline) into + // a per-path delay update. The drag delta = newSpan.start - + // originalClipStart is the new offset of the source audio relative + // to the clip; the parent then carries that through to the + // preview and the export. + const handleSourceAudioSpanChange = useCallback( + (itemId: string, span: Span) => { + if (!onSourceAudioStartOffsetChange) return; + // itemId = `source-audio--` + const match = /^source-audio-([^-]+)-(.+)$/.exec(itemId); + if (!match) return; + const [, trackId, clipId] = match; + const path = sourceAudioPathByTrackId?.[trackId]; + if (!path) return; + const clip = clipRegions.find((c) => c.id === clipId); + if (!clip) return; + // The source-audio span is offset from the clip span by the + // user-controlled delay. So `span.start - clip.startMs` is + // the new delay. Note: this is the total offset, so it + // replaces (not adds to) any existing override. + const newOffsetMs = span.start - clip.startMs; + onSourceAudioStartOffsetChange(path, newOffsetMs); + }, + [ + clipRegions, + onSourceAudioStartOffsetChange, + sourceAudioPathByTrackId, + ], + ); + const { keyframes, selectedKeyframeId, @@ -397,6 +444,7 @@ const TimelineEditor = forwardRef( onCaptionAdded, selectedCaptionId, onSelectCaption, + onSourceAudioSpanChange: handleSourceAudioSpanChange, isMac, keyShortcuts, isTimelineFocusedRef, @@ -505,6 +553,9 @@ const TimelineEditor = forwardRef( onClearBlockSelection={clearSelectedBlocks} keyframes={keyframes} sourceAudioTracks={sourceAudioTracks} + sourceAudioPathByTrackId={sourceAudioPathByTrackId} + sourceAudioStartOffsetMsByPath={sourceAudioStartOffsetMsByPath} + onSourceAudioStartOffsetChange={onSourceAudioStartOffsetChange} getSourceAudioTrackSettingsForClip={getSourceAudioTrackSettingsForClip} showSourceAudioTrack={showSourceAudioTrack} liveSpanPreviewById={liveZoomPreview.previewSpans} diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index a2a20cc1c..5e6c5aab9 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -75,6 +75,9 @@ interface TimelineCanvasProps { onClearBlockSelection?: () => void; keyframes?: { id: string; time: number }[]; sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + sourceAudioPathByTrackId?: Record; + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; @@ -373,6 +376,9 @@ interface TimelineCanvasRowsProps { onSelectAudio?: (id: string | null) => void; onSelectCaption?: (id: string | null) => void; sourceAudioTracks?: SourceAudioTrackWithPeaks[]; + sourceAudioPathByTrackId?: Record; + sourceAudioStartOffsetMsByPath?: Record; + onSourceAudioStartOffsetChange?: (path: string, offsetMs: number) => void; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; showSourceAudioTrack?: boolean; liveSpanPreviewById?: Record; @@ -452,6 +458,9 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelectAudio, onSelectCaption, sourceAudioTracks = [], + sourceAudioPathByTrackId, + sourceAudioStartOffsetMsByPath, + onSourceAudioStartOffsetChange, getSourceAudioTrackSettingsForClip, showSourceAudioTrack = false, liveSpanPreviewById, @@ -555,36 +564,53 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ ))} {showSourceAudioTrack && - sourceAudioTracks.map((track) => ( - - {clipItems - .filter((item) => item.showSourceAudio) - .map((item) => { - const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ - track.id - ] ?? { volume: 1, normalize: false }; - return ( - onSelectClip?.(item.id)} - variant="audio" - waveformPeaks={track.peaks} - waveformSegmentSpan={item.sourceSpan ?? item.span} - waveformGain={Math.max(0, Math.min(1, settings.volume))} - waveformNormalize={Boolean(settings.normalize)} - muted={item.muted} - > - {track.label} - - ); - })} - - ))} + sourceAudioTracks.map((track) => { + const sourceAudioPath = sourceAudioPathByTrackId?.[track.id]; + const overrideOffsetMs = sourceAudioPath + ? sourceAudioStartOffsetMsByPath?.[sourceAudioPath] + : undefined; + return ( + + {clipItems + .filter((item) => item.showSourceAudio) + .map((item) => { + const settings = getSourceAudioTrackSettingsForClip?.(item.id)?.[ + track.id + ] ?? { volume: 1, normalize: false }; + // Apply the user-controlled offset to the source-audio + // span so dragging actually moves the audio on the + // timeline (and downstream, in the export). + const baseSpan = liveSpanPreviewById?.[item.id] ?? item.span; + const offsetSpan = + overrideOffsetMs !== undefined + ? { + start: baseSpan.start + overrideOffsetMs, + end: baseSpan.end + overrideOffsetMs, + } + : baseSpan; + return ( + onSelectClip?.(item.id)} + variant="audio" + waveformPeaks={track.peaks} + waveformSegmentSpan={item.sourceSpan ?? item.span} + waveformGain={Math.max(0, Math.min(1, settings.volume))} + waveformNormalize={Boolean(settings.normalize)} + muted={item.muted} + > + {track.label} + + ); + })} + + ); + })} void; onAudioSpanChange?: (id: string, span: Span, trackIndex?: number) => void; onCaptionSpanChange?: (id: string, span: Span) => void; + // Source-audio item span changes (drag-to-align). The id is the + // `source-audio--` id from TimelineCanvas; the span + // is the new position. Resolved by the caller into a per-path delay + // update. + onSourceAudioSpanChange?: (id: string, span: Span) => void; } type TimelineItemKind = @@ -44,6 +49,7 @@ type TimelineItemKind = | "speed" | "audio" | "caption" + | "source-audio" | null; export function useTimelineDndBindings({ @@ -61,9 +67,11 @@ export function useTimelineDndBindings({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, }: UseTimelineDndBindingsParams) { const resolveItemKind = useCallback( (id: string): TimelineItemKind => { + if (id.startsWith("source-audio-")) return "source-audio"; if (zoomRegions.some((r) => r.id === id)) return "zoom"; if (trimRegions.some((r) => r.id === id)) return "trim"; if (clipRegions.some((r) => r.id === id)) return "clip"; @@ -186,6 +194,8 @@ export function useTimelineDndBindings({ onAudioSpanChange?.(id, span, nextTrackIndex); } else if (itemKind === "caption") { onCaptionSpanChange?.(id, span); + } else if (itemKind === "source-audio") { + onSourceAudioSpanChange?.(id, span); } }, [ @@ -198,6 +208,7 @@ export function useTimelineDndBindings({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, ], ); diff --git a/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts b/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts index 9f7092db0..db9219ac5 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineEditorRuntime.ts @@ -67,6 +67,8 @@ interface UseTimelineEditorRuntimeParams { onCaptionAdded?: (span: Span) => void; selectedCaptionId?: string | null; onSelectCaption?: (id: string | null) => void; + // Source-audio drag (drag-to-align): id is `source-audio--`. + onSourceAudioSpanChange?: (id: string, span: Span) => void; isMac: boolean; keyShortcuts: TimelineShortcutBindings; isTimelineFocusedRef: RefObject; @@ -117,6 +119,7 @@ export function useTimelineEditorRuntime({ onCaptionAdded, selectedCaptionId, onSelectCaption, + onSourceAudioSpanChange, isMac, keyShortcuts, isTimelineFocusedRef, @@ -202,6 +205,7 @@ export function useTimelineEditorRuntime({ onSpeedSpanChange, onAudioSpanChange, onCaptionSpanChange, + onSourceAudioSpanChange, }); const { From 1acda4aed182c64ae23900eb47626d479496943a Mon Sep 17 00:00:00 2001 From: Recordly Dev Date: Mon, 7 Sep 2026 18:07:17 +0530 Subject: [PATCH 6/6] feat(editor): trim source-audio start + reduce effective delay Complements the drag-to-align (PR #2) with a 'cut' capability: the user can now type a 'Trim start (ms)' value in the per-track Source section of the audio panel to remove the pre-recording audio at the start of the sidecar file. The exporter physically shortens the audio buffer at extract time and reduces the effective start delay by the same amount, so the trimmed audio lines up with the video at the right wall-clock position. What this commit changes: - 'SettingsPanel.tsx' gains a 'Trim start (ms)' numeric input per source-audio track (system / mic). It writes to a new 'sourceAudioTrimStartMsByPath' state in 'VideoEditor', which persists via 'projectPersistence.ts' as 'sourceAudioTrimStartOverrideMsByPath'. - The 'useVideoEditorAudio' hook subtracts the trim from the effective start delay for the preview, so the first 'trimMs' of audible content in the trimmed sidecar file lines up with the start of the video. - The exporter's audio pipeline ('audioEncoder.ts') physically shortens the decoded companion-audio buffer at extract time via 'sliceAudioBufferStart', the JS equivalent of FFmpeg's 'atrim=start=' filter. The 'effectiveSourceAudioStartDelayMsByPath' is reduced by the trim, so the audible content stays in sync with the video after the trim. - New 'sourceAudioTrimStartMsByPath' field plumbed through: * the 'videoExporter' / 'modernVideoExporter' config type, * 'audioEncoder.ts' 'process' and 'renderEditedAudioTrack' signatures, * the VideoEditor state, * project persistence (load + save), * the audio panel's per-track UI. - Drag-to-align (PR #2) and trim are independent: dragging moves the audio, trimming removes audio from its start. Together they cover the full set of cases (fixed offset / pre-recording silence / both). Cross-platform: - Renderer + exporter only. Capture paths unchanged. Tests: 998/998 passing, 'npx tsc --noEmit' clean. --- src/components/video-editor/SettingsPanel.tsx | 50 +++++++++++++++++ src/lib/exporter/audioEncoder.ts | 56 ++++++++++++++++++- src/lib/exporter/modernVideoExporter.ts | 3 + src/lib/exporter/types.ts | 6 ++ src/lib/exporter/videoExporter.ts | 7 +++ 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index a90028e2b..ecafc0e91 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -719,6 +719,13 @@ interface SettingsPanelProps { sourceAudioTrackSettings?: Record; onSourceAudioTrackVolumeChange?: (id: string, volume: number) => void; onSourceAudioTrackNormalizeChange?: (id: string, normalize: boolean) => void; + // Per-source-audio-path user-controlled trim from the start of the + // audio (in ms). Set via the audio panel — the exporter slices the + // audio buffer by this amount. Keys are the absolute paths of the + // sidecar audio files. + sourceAudioPathByTrackId?: Record; + sourceAudioTrimStartMsByPath?: Record; + onSourceAudioTrimStartChange?: (path: string, trimStartMs: number) => void; onClipDelete?: (id: string) => void; selectedAudioId?: string | null; selectedAudioVolume?: number | null; @@ -1181,6 +1188,9 @@ export function SettingsPanel({ sourceAudioTrackSettings = {}, onSourceAudioTrackVolumeChange, onSourceAudioTrackNormalizeChange, + sourceAudioPathByTrackId, + sourceAudioTrimStartMsByPath, + onSourceAudioTrimStartChange, onClipDelete, selectedAudioId, selectedAudioVolume, @@ -3548,6 +3558,10 @@ export function SettingsPanel({ volume: 1, normalize: false, }; + const sourceAudioPath = sourceAudioPathByTrackId?.[track.id]; + const trimStartMs = sourceAudioPath + ? sourceAudioTrimStartMsByPath?.[sourceAudioPath] ?? 0 + : 0; return (
@@ -3598,6 +3615,39 @@ export function SettingsPanel({ parseFloat(text.replace(/%$/, "")) / 100 } /> + {sourceAudioPath && ( +
+ + {tSettings( + "audio.trimStart", + "Trim start (ms)", + )} + + { + const v = Number(e.target.value); + if ( + Number.isFinite(v) && + v >= 0 && + onSourceAudioTrimStartChange + ) { + onSourceAudioTrimStartChange( + sourceAudioPath, + Math.round(v), + ); + } + }} + className="w-20 rounded border border-foreground/10 bg-background/40 px-1.5 py-0.5 text-right text-[11px] text-foreground outline-none focus:border-[#06b6d4]/50" + /> +
+ )}
); })} diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 6f1ca0757..2e00330ed 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -60,6 +60,37 @@ export function softLimitOfflineMixPeaksInPlace(buffer: AudioBuffer): boolean { return changed; } +/** + * Trim the first `seconds` of an AudioBuffer. Used to apply the + * user-controlled source-audio trim (set via the timeline's left-edge + * resize) in the exporter. The returned AudioBuffer is a fresh buffer + * with the same channel count and sample rate; only the duration is + * shortened. We allocate via a 1-sample `OfflineAudioContext` (cheaper + * than spinning up a renderer) and copy the channel data ourselves. + */ +function sliceAudioBufferStart(buffer: AudioBuffer, seconds: number): AudioBuffer { + if (seconds <= 0) return buffer; + const sampleRate = buffer.sampleRate; + const startSample = Math.min( + buffer.length, + Math.max(0, Math.floor(seconds * sampleRate)), + ); + const newLength = Math.max(1, buffer.length - startSample); + const ctx = new OfflineAudioContext(buffer.numberOfChannels, 1, sampleRate); + const dst = ctx.createBuffer(buffer.numberOfChannels, newLength, sampleRate); + for (let ch = 0; ch < buffer.numberOfChannels; ch++) { + const src = buffer.getChannelData(ch); + const out = dst.getChannelData(ch); + if (startSample < buffer.length) { + out.set(src.subarray(startSample)); + } + // If startSample >= buffer.length, the loop above runs once and + // `out` is already zero-filled (Web Audio spec), so the buffer is + // silent — the rest of the pipeline treats it as no audio. + } + return dst; +} + function resolveSourceTrackGain( sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, trackId: "mic" | "system" | "mixed", @@ -240,6 +271,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -292,6 +324,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, muxer, @@ -325,6 +358,7 @@ export class AudioProcessor { [], routingPolicy.playbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, muxer, @@ -373,6 +407,7 @@ export class AudioProcessor { audioRegions?: AudioRegion[], sourceAudioFallbackPaths?: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -400,6 +435,7 @@ export class AudioProcessor { sortedAudioRegions, sortedSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, ); @@ -677,6 +713,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath: Record | undefined, + sourceAudioTrimStartMsByPath: Record | undefined, sourceAudioTrackSettings: SourceAudioTrackSettings | undefined, clipRegions: ClipRegion[] | undefined, muxer: VideoMuxer, @@ -688,6 +725,7 @@ export class AudioProcessor { audioRegions, sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioTrimStartMsByPath, sourceAudioTrackSettings, clipRegions, ); @@ -702,6 +740,7 @@ export class AudioProcessor { audioRegions: AudioRegion[], sourceAudioFallbackPaths: string[], sourceAudioFallbackStartDelayMsByPath?: Record, + sourceAudioTrimStartMsByPath?: Record, sourceAudioTrackSettings?: SourceAudioTrackSettings, clipRegions?: ClipRegion[], ): Promise { @@ -752,15 +791,28 @@ export class AudioProcessor { const buffer = await this.decodeAudioFromUrl(audioPath); if (!buffer) continue; + // Apply the user-controlled trim from the start of the audio. + // This is the JS equivalent of FFmpeg's `atrim=start=` + // filter — it physically shortens the audio file by dropping + // the first `trimStartMs` worth of samples. The effective + // start delay (set above via `sourceAudioFallbackStartDelayMsByPath`) + // is reduced by the trim, so the audible content stays in + // sync with the video after the trim. + const trimStartMs = sourceAudioTrimStartMsByPath?.[audioPath] ?? 0; + const trimmedBuffer = + trimStartMs > 0 + ? sliceAudioBufferStart(buffer, trimStartMs / 1000) + : buffer; + companionEntries.push({ - buffer, + buffer: trimmedBuffer, gain: resolveSourceTrackGain( sourceAudioTrackSettings, getSourceTrackIdFromPath(audioPath), ), startDelaySec: estimateCompanionAudioStartDelaySeconds( refDuration, - buffer.duration, + trimmedBuffer.duration, sourceAudioFallbackStartDelayMsByPath?.[audioPath], ), }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index c77ba9648..d79a92289 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -152,6 +152,7 @@ interface VideoExporterConfig extends ExportConfig { clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + sourceAudioTrimStartMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; @@ -826,6 +827,7 @@ export class ModernVideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), @@ -1961,6 +1963,7 @@ export class ModernVideoExporter { this.config.audioRegions, sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index f474f9988..402804994 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -14,6 +14,12 @@ export interface ExportConfig { maxPendingFrames?: number; maxInFlightNativeWrites?: number; sourceAudioFallbackStartDelayMsByPath?: Record; + // Per-source-audio-path trim from the start of the audio file (ms). + // Applied by the exporter as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened (not just delayed). The + // source-audio track item in the timeline supports left-edge resize + // to set this value. + sourceAudioTrimStartMsByPath?: Record; } export type ExportRenderBackend = "webgpu" | "webgl"; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index b99f5f96c..866e3ae4a 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -95,6 +95,10 @@ interface VideoExporterConfig extends ExportConfig { clipRegions?: ClipRegion[]; sourceAudioFallbackPaths?: string[]; sourceAudioFallbackStartDelayMsByPath?: Record; + // Per-source-audio-path trim from the start of the audio file (ms). + // Applied by the exporter as an FFmpeg `atrim=start=` filter + // so the audio file is physically shortened. + sourceAudioTrimStartMsByPath?: Record; sourceAudioTrackSettings?: SourceAudioTrackSettings; previewWidth?: number; previewHeight?: number; @@ -412,6 +416,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, ), "audio processing", @@ -864,6 +869,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ), @@ -962,6 +968,7 @@ export class VideoExporter { this.config.audioRegions, this.config.sourceAudioFallbackPaths, this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrimStartMsByPath, this.config.sourceAudioTrackSettings, this.config.clipRegions, ),