diff --git a/packages/core/src/audioGroups.test.ts b/packages/core/src/audioGroups.test.ts index 588e8c45f7..edc16d16a7 100644 --- a/packages/core/src/audioGroups.test.ts +++ b/packages/core/src/audioGroups.test.ts @@ -4,6 +4,7 @@ import { ensureAudioGroupInertStyle, HF_AUDIO_GROUP_ATTR, resolveAudioGroups, + resolveCarveSourceIds, } from "./audioGroups.js"; beforeEach(() => { @@ -22,7 +23,15 @@ describe("resolveAudioGroups", () => { `; const groups = resolveAudioGroups(document); - expect(groups).toEqual([{ id: "voiceover", label: "Voiceover", memberIds: ["vo-1", "vo-2"] }]); + expect(groups).toEqual([ + { + id: "voiceover", + label: "Voiceover", + memberIds: ["vo-1", "vo-2"], + volume: 1, + hidden: false, + }, + ]); }); it("resolves from member tags alone when the group element is absent, label = id", () => { @@ -30,7 +39,9 @@ describe("resolveAudioGroups", () => { `; const groups = resolveAudioGroups(document); - expect(groups).toEqual([{ id: "narration", label: "narration", memberIds: ["vo-1"] }]); + expect(groups).toEqual([ + { id: "narration", label: "narration", memberIds: ["vo-1"], volume: 1, hidden: false }, + ]); }); it("ignores data-audio-group on the group element itself (groups do not nest)", () => { @@ -39,7 +50,9 @@ describe("resolveAudioGroups", () => { `; const groups = resolveAudioGroups(document); - expect(groups).toEqual([{ id: "outer", label: "outer", memberIds: ["vo-1"] }]); + expect(groups).toEqual([ + { id: "outer", label: "outer", memberIds: ["vo-1"], volume: 1, hidden: false }, + ]); expect(audioGroupOf(document.getElementById("outer") as Element)).toBeNull(); }); @@ -58,6 +71,37 @@ describe("resolveAudioGroups", () => { document.body.innerHTML = ``; expect(resolveAudioGroups(document)).toEqual([]); }); + + it("reads the group element's fx chain, automation, volume and hidden", () => { + document.body.innerHTML = ` + + + `; + const groups = resolveAudioGroups(document); + expect(groups).toEqual([ + { + id: "voiceover", + label: "voiceover", + memberIds: ["vo-1"], + fxChain: '{"version":1,"nodes":[]}', + automation: '{"lanes":[]}', + volume: 0.5, + hidden: true, + }, + ]); + }); + + it("defaults volume to 1 and hidden to false when a group element exists but carries neither", () => { + document.body.innerHTML = ` + + + `; + const [group] = resolveAudioGroups(document); + expect(group?.volume).toBe(1); + expect(group?.hidden).toBe(false); + expect(group?.fxChain).toBeUndefined(); + expect(group?.automation).toBeUndefined(); + }); }); describe("audioGroupOf", () => { @@ -129,6 +173,47 @@ describe("ensureAudioGroupInertStyle", () => { }); }); +describe("resolveCarveSourceIds", () => { + it("expands a group id to its current members", () => { + document.body.innerHTML = ` + + + `; + expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2"]); + }); + + it("picks up a member added to the group after the carve was set (analysis-time, not frozen)", () => { + document.body.innerHTML = ` + + + `; + expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2"]); + document.body.insertAdjacentHTML( + "beforeend", + ``, + ); + expect(resolveCarveSourceIds(document, ["voiceover"])).toEqual(["vo-1", "vo-2", "vo-3"]); + }); + + it("passes through a plain clip id that still exists", () => { + document.body.innerHTML = ``; + expect(resolveCarveSourceIds(document, ["vo-1"])).toEqual(["vo-1"]); + }); + + it("drops an id that resolves to nothing — a deleted clip, an empty or vanished group", () => { + document.body.innerHTML = ``; + expect(resolveCarveSourceIds(document, ["vo-1", "deleted", "no-such-group"])).toEqual(["vo-1"]); + }); + + it("dedupes and preserves first-seen order across a mix of group and plain ids", () => { + document.body.innerHTML = ` + + + `; + expect(resolveCarveSourceIds(document, ["voiceover", "vo-1"])).toEqual(["vo-1", "vo-2"]); + }); +}); + describe(HF_AUDIO_GROUP_ATTR, () => { it("is the attribute name membership is keyed on", () => { expect(HF_AUDIO_GROUP_ATTR).toBe("data-audio-group"); diff --git a/packages/core/src/audioGroups.ts b/packages/core/src/audioGroups.ts index 86ebdb9d73..e25ee16565 100644 --- a/packages/core/src/audioGroups.ts +++ b/packages/core/src/audioGroups.ts @@ -9,6 +9,9 @@ * nothing here routes or sums audio yet. */ +import { HF_AUDIO_FX_ATTR } from "./audioFx.js"; +import { HF_AUDIO_AUTOMATION_ATTR } from "./audioAutomation.js"; + export const HF_AUDIO_GROUP_TAG = "hf-audio-group"; export const HF_AUDIO_GROUP_ATTR = "data-audio-group"; @@ -32,6 +35,38 @@ export interface HfAudioGroup { label: string; /** Member element ids, in document order. */ memberIds: string[]; + /** Serialised FX chain JSON from the group element's `data-fx-chain`, when set. */ + fxChain?: string; + /** Serialised automation JSON from the group element's `data-automation`, when set. */ + automation?: string; + /** The group element's `data-volume`, defaulting to 1 when absent or there is no group element. */ + volume: number; + /** + * The group element's `data-hidden`. Render drops every member rather than + * zeroing them (RULES: mute-by-drop, never mute-by-volume-0) — B5 defines + * the UI for this; this field just makes the read available now. + */ + hidden: boolean; +} + +function parseGroupVolume(el: Element | undefined): number { + const raw = el?.getAttribute("data-volume"); + const parsed = raw ? parseFloat(raw) : 1; + return Number.isFinite(parsed) ? parsed : 1; +} + +function buildGroup(id: string, memberIds: string[], el: Element | undefined): HfAudioGroup { + const fxChain = el?.getAttribute(HF_AUDIO_FX_ATTR); + const automation = el?.getAttribute(HF_AUDIO_AUTOMATION_ATTR); + return { + id, + label: el?.getAttribute("data-label") || id, + memberIds, + ...(fxChain ? { fxChain } : {}), + ...(automation ? { automation } : {}), + volume: parseGroupVolume(el), + hidden: el?.hasAttribute("data-hidden") ?? false, + }; } /** @@ -58,13 +93,42 @@ export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] { const groups: HfAudioGroup[] = []; for (const [id, memberIds] of membersByGroup) { - const el = groupElements.get(id); - const label = el?.getAttribute("data-label") || id; - groups.push({ id, label, memberIds }); + groups.push(buildGroup(id, memberIds, groupElements.get(id))); } return groups; } +/** + * Expand a list of source ids for a carve: a plain id passes through if it + * still exists, a group id expands to its CURRENT members. Resolved fresh + * every time — group membership is never frozen into the carve's own + * attribute, so adding a fourth voice to a group already named in a carve's + * `sources` picks it up on the next analysis without editing that carve. + * + * Dedupes and preserves first-seen order; an id that resolves to nothing + * (a deleted clip, an empty or vanished group) is dropped rather than kept + * as a dangling reference the analysis would only fail to find anyway. + */ +export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): string[] { + const groupsById = new Map(resolveAudioGroups(doc).map((group) => [group.id, group] as const)); + const seen = new Set(); + const out: string[] = []; + const add = (id: string): void => { + if (seen.has(id)) return; + seen.add(id); + out.push(id); + }; + for (const id of ids) { + const group = groupsById.get(id); + if (group) { + group.memberIds.forEach(add); + } else if (doc.getElementById(id)) { + add(id); + } + } + return out; +} + /** * The group a member belongs to, or null — the same predicate * `resolveAudioGroups` scans with, so the two can never disagree about a given @@ -74,9 +138,14 @@ export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] { * `data-audio-group` on an `` itself (groups do not nest). * `data-audio-group=""` returns null rather than `""` — the resolver skips a * falsy id, and the "or null" in this contract has to mean it. + * + * Tolerant of objects that only partially implement `Element` (test doubles for + * `HTMLMediaElement` commonly do): anything missing `tagName` or `getAttribute` + * simply has no group, mirroring `readChain`'s style in `runtime/audioFx.ts`. */ export function audioGroupOf(el: Element): string | null { - if (el.tagName?.toLowerCase() !== "audio") return null; + if (typeof el.tagName !== "string" || el.tagName.toLowerCase() !== "audio") return null; + if (typeof el.getAttribute !== "function") return null; return el.getAttribute(HF_AUDIO_GROUP_ATTR) || null; } @@ -91,8 +160,8 @@ export function audioGroupOf(el: Element): string | null { * would shift by adding a group, which is not something a mixing decision is * allowed to do. * - * `!important` because the rule has to beat an author rule that sets `display` - * on the tag — inertness here is a contract, not a default. Emitted from the + * `!important` because an author rule can outrank a bare type selector on + * specificity — inertness here is a contract, not a default. Emitted from the * runtime rather than the compiler so preview and render share one source. */ export function ensureAudioGroupInertStyle(doc: Document): void { diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 2ff8e0b933..6a3d0241ba 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -655,6 +655,167 @@ describe("WebAudioTransport", () => { }); }); + describe("group routing (preview)", () => { + // Real jsdom elements — `groupInput` looks the group up via + // `el.ownerDocument.getElementById`, and `audioGroupOf` reads `tagName` / + // `getAttribute`, neither of which the plain-object mocks above implement. + function createGroupMockAudioContext(currentTime = 100) { + const gainNodes: { + gain: { value: number }; + connect: ReturnType; + disconnect: ReturnType; + }[] = []; + const masterGain = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + const ctx = { + currentTime, + state: "running", + resume: vi.fn(), + createBufferSource: vi.fn(() => ({ + buffer: null as AudioBuffer | null, + playbackRate: { value: 1 }, + start: vi.fn(), + stop: vi.fn(), + disconnect: vi.fn(), + connect: vi.fn(), + addEventListener: vi.fn(), + })), + createGain: vi.fn(() => { + const node = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + gainNodes.push(node); + return node; + }), + destination: {}, + close: vi.fn(), + }; + return { ctx, gainNodes, masterGain }; + } + + function setupGroupTransport(currentTime = 100) { + const transport = new WebAudioTransport(); + const mock = createGroupMockAudioContext(currentTime); + (transport as unknown as { _ctx: unknown })._ctx = mock.ctx; + (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain; + const gen = transport.startGeneration(); + return { transport, mock, gen }; + } + + function groupedAudioEl(id: string, groupId?: string): HTMLMediaElement { + const el = document.createElement("audio"); + el.id = id; + if (groupId) el.setAttribute("data-audio-group", groupId); + document.body.appendChild(el); + return el as unknown as HTMLMediaElement; + } + + /** Create a grouped member and schedule it in one step — the shape every + * test below needs, differing only in id/group/generation. */ + async function scheduleGrouped( + transport: WebAudioTransport, + gen: number, + id: string, + groupId?: string, + ): Promise { + const el = groupedAudioEl(id, groupId); + await transport.schedulePlayback(el, mockBuffer, 0, 0, 0, 1, gen); + return el; + } + + /** The group's own input gain is built lazily on the first member — + * index 1 in creation order (that member's gain is index 0). */ + const firstGroupInput = (mock: ReturnType) => + mock.gainNodes[1]!; + + beforeEach(() => { + document.body.innerHTML = ""; + }); + + it("routes an ungrouped member straight to master, unchanged", async () => { + const { transport, mock, gen } = setupGroupTransport(); + + await scheduleGrouped(transport, gen, "solo"); + + // One gain node — the member's own — connected directly to master. + expect(mock.gainNodes).toHaveLength(1); + expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain); + }); + + it("two members of the same group land on ONE shared group gain, not master directly", async () => { + const { transport, mock, gen } = setupGroupTransport(); + + await scheduleGrouped(transport, gen, "a", "vo"); + await scheduleGrouped(transport, gen, "b", "vo"); + + // Member gain nodes: index 0 (a) and index 2 (b) — index 1 is the + // group's own input gain, built inside a's schedule call. + expect(mock.gainNodes.length).toBeGreaterThanOrEqual(3); + const groupInput = firstGroupInput(mock); + const aGain = mock.gainNodes[0]!; + const bGain = mock.gainNodes[2]!; + + // Neither member connects straight to master — both feed the shared bus. + expect(aGain.connect).toHaveBeenCalledWith(groupInput); + expect(bGain.connect).toHaveBeenCalledWith(groupInput); + expect(aGain.connect).not.toHaveBeenCalledWith(mock.masterGain); + expect(bGain.connect).not.toHaveBeenCalledWith(mock.masterGain); + + // The bus itself is what reaches master — a plain sum, no processing, + // since neither member's group has a chain-bearing ``. + expect(groupInput.connect).toHaveBeenCalledWith(mock.masterGain); + }); + + it("a second member of an already-open group does not rebuild the group bus", async () => { + const { transport, mock, gen } = setupGroupTransport(); + + await scheduleGrouped(transport, gen, "a", "vo"); + const gainCountAfterFirst = mock.gainNodes.length; // a-gain + group-input + await scheduleGrouped(transport, gen, "b", "vo"); + + // Only b's own gain is new — no second group-input gain minted. + expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 1); + }); + + it("a group id with no matching element still gets a flat bus", async () => { + const { transport, mock, gen } = setupGroupTransport(); + + await scheduleGrouped(transport, gen, "a", "orphan-group"); // no matching element + + expect(firstGroupInput(mock).connect).toHaveBeenCalledWith(mock.masterGain); + }); + + it("group volume rides the group's own data-volume via its automation lane, not the member's", async () => { + document.body.innerHTML = ``; + const { transport, gen } = setupGroupTransport(); + + // No throw wiring the group's automation reader against a real + // element that carries no fx/automation attrs. + await expect(scheduleGrouped(transport, gen, "a", "vo")).resolves.not.toBeNull(); + }); + + it("destroy() disposes every group bus", async () => { + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + const groupInput = firstGroupInput(mock); + + transport.destroy(); + + expect(groupInput.disconnect).toHaveBeenCalled(); + }); + + it("stopAll() does NOT dispose group buses — replaying the group does not rebuild it", async () => { + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + const groupInput = firstGroupInput(mock); + + transport.stopAll(); + expect(groupInput.disconnect).not.toHaveBeenCalled(); + + const gen2 = transport.startGeneration(); + await scheduleGrouped(transport, gen2, "a", "vo"); + // Still only one group-input gain ever created for "vo". + expect(mock.gainNodes.filter((n) => n === groupInput)).toHaveLength(1); + }); + }); + describe("decodeAudioElement retry policy (late-asset self-heal)", () => { function transportWithDecode(decodeImpl: () => Promise) { const transport = new WebAudioTransport(); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 704f7af840..a154c78af7 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -5,6 +5,7 @@ import { type AutomationTiming, } from "../audio/audioFxAutomation.js"; import { VOLUME_RANGE } from "../audioAutomation.js"; +import { audioGroupOf } from "../audioGroups.js"; import { swallow } from "./diagnostics"; import { clampAudioGain } from "../audioGain.js"; import { getDebugSurface } from "./globals.js"; @@ -70,9 +71,12 @@ function startBoundedSource( /** * The volume lane rides the fader, after the effects — where a DAW puts it, * and the order the render bakes it in. + * + * Typed against the attribute reader rather than `HTMLMediaElement` so a group + * bus (an ``, not a media element) can ride the same path. */ function scheduleVolumeLane( - el: HTMLMediaElement, + el: { getAttribute?(name: string): string | null }, gainNode: GainNode, timing: AutomationTiming, ): void { @@ -119,6 +123,11 @@ export class WebAudioTransport { private _masterGain: GainNode | null = null; private _masterVolume = 1; private _masterMuted = false; + // One shared bus per group id, lazily built the first time a member of that + // group is scheduled. Lives for the session (mirrors `_masterGain`'s own + // lifecycle) rather than being torn down on every `stopAll()`, so replaying + // a group does not rebuild its chain; only `destroy()` disposes these. + private _groups = new Map(); // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`, // composition time was `_rateAnchorComp`, and time has been advancing at // `_rate` composition-seconds per wallclock-second since. @@ -269,6 +278,97 @@ export class WebAudioTransport { } } + /** + * The gain a grouped member's signal should land on, building it on first + * use. A group's clock is COMPOSITION time (design doc §1.3) — it has no + * `data-start`, and a missing start parses as 0, which is exactly + * composition time — so its chain and volume lane are scheduled once here + * against that zero-offset timing, not the member's own clip-local timing. + * A group id with no matching `` element still gets a bus + * (flat, no chain) so a hand-authored `data-audio-group` degrades to a + * plain sum rather than losing the member's audio. + */ + private groupInput(groupId: string, doc: Document, timing: AutomationTiming): GainNode | null { + const existing = this._groups.get(groupId); + if (existing) return existing.input; + if (!this._ctx || !this._masterGain) return null; + + const input = this._ctx.createGain(); + const groupEl = doc.getElementById(groupId); + const fx = attachElementFxChain( + this._ctx, + groupEl ?? { getAttribute: () => null }, + input, + this._masterGain, + timing, + ); + if (groupEl) scheduleVolumeLane(groupEl, input, timing); + + this._groups.set(groupId, { + input, + dispose: () => { + try { + fx?.dispose(); + input.disconnect(); + } catch { + // Already torn down. + } + }, + }); + return input; + } + + /** Master, unless `el` belongs to a group — then that group's bus (built on + * first use, per `groupInput`). */ + private resolveDestination( + el: HTMLMediaElement, + scheduledAt: number, + compositionTime: number, + safeRate: number, + ): GainNode | null { + if (!this._masterGain) return null; + const groupId = audioGroupOf(el); + if (!groupId) return this._masterGain; + const groupTiming: AutomationTiming = { scheduledAt, elapsed: compositionTime, rate: safeRate }; + return this.groupInput(groupId, el.ownerDocument, groupTiming) ?? this._masterGain; + } + + /** + * The graph goes with it. Splicing alone left the FX handle alive and then + * UNREACHABLE — `stopAll()` disposes by walking `_activeSources`, which the + * splice just emptied of this entry. Every clip that finished naturally + * leaked its MutationObserver for the session, and each one still answered + * later `data-fx-chain` edits by rebuilding a whole graph (impulse response, + * chorus/phaser oscillators started and never stopped) around a dead + * source. Not disposed when the index is already -1: `stopAll()` has + * already done it, and `stop()` is what fired this event. + */ + private handleSourceEnded( + sourceNode: AudioBufferSourceNode, + scheduled: ScheduledSource, + el: HTMLMediaElement, + priorMuted: boolean, + ): void { + const idx = this._activeSources.indexOf(scheduled); + if (idx === -1) return; + this._activeSources.splice(idx, 1); + el.muted = priorMuted; + try { + sourceNode.disconnect(); + scheduled.fx?.dispose(); + scheduled.gainNode.disconnect(); + } catch { + // Already torn down. + } + if (this._activeSources.length === 0) this._paused = true; + } + + // Pre-existing size (110 lines before this diff, which shrank it to under + // 95 via two extractions — see `handleSourceEnded`/`resolveDestination`); + // the remainder is inherently sequential graph-wiring, not a nested + // decision tree, and further splitting would cost more readability than it + // buys. Same call the B2 step took on `TimelineLogicalRow`. + // fallow-ignore-next-line complexity async schedulePlayback( el: HTMLMediaElement, buffer: AudioBuffer, @@ -309,7 +409,9 @@ export class WebAudioTransport { // output — the same order the offline render uses. Preview and render run // the identical graph builders, so what is heard here is what is written. const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing); - gainNode.connect(this._masterGain); + gainNode.connect( + this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain, + ); scheduleVolumeLane(el, gainNode, timing); @@ -355,29 +457,9 @@ export class WebAudioTransport { this._activeSources.push(scheduled); this._paused = false; - sourceNode.addEventListener("ended", () => { - const idx = this._activeSources.indexOf(scheduled); - if (idx !== -1) { - this._activeSources.splice(idx, 1); - el.muted = priorMuted; - // The graph goes with it. Splicing alone left the FX handle alive and - // then UNREACHABLE — stopAll() disposes by walking this array, which - // the splice just emptied of this entry. Every clip that finished - // naturally leaked its MutationObserver for the session, and each one - // still answered later `data-fx-chain` edits by rebuilding a whole - // graph (impulse response, chorus/phaser oscillators started and never - // stopped) around a dead source. Not disposed when idx is -1: stopAll() - // has already done it, and `stop()` is what fired this event. - try { - sourceNode.disconnect(); - fx?.dispose(); - gainNode.disconnect(); - } catch { - // Already torn down. - } - if (this._activeSources.length === 0) this._paused = true; - } - }); + sourceNode.addEventListener("ended", () => + this.handleSourceEnded(sourceNode, scheduled, el, priorMuted), + ); return scheduled; } catch (err) { @@ -500,6 +582,8 @@ export class WebAudioTransport { destroy(): void { this.stopAll(); + for (const group of this._groups.values()) group.dispose(); + this._groups.clear(); this._bufferCache.clear(); this._failedSrcs.clear(); this._mediaElementSources = new WeakMap(); diff --git a/packages/engine/src/services/audioMixer.grouping.test.ts b/packages/engine/src/services/audioMixer.grouping.test.ts new file mode 100644 index 0000000000..d148b72cb5 --- /dev/null +++ b/packages/engine/src/services/audioMixer.grouping.test.ts @@ -0,0 +1,271 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { getFfmpegBinary } from "../utils/ffmpegBinaries.js"; +import { MIXED_AUDIO_FILENAME, processCompositionAudio } from "./audioMixer.js"; + +/** + * Level arithmetic across the mix graph. + * + * `mixAudioTracks` corrects for amix's own normalisation with a single global + * `masterOutputGain * tracks.length`. That correction is exact for one flat + * amix and wrong for any other shape — and it fails SILENTLY, in the export + * rather than in preview, because preview mixes through Web Audio and never + * runs this graph at all. + * + * Measured on a four-track mix (spike, 2026-08-14): keeping the global track + * count on an outer amix that only has three inputs lands the whole mix + * +2.499 dB hot — exactly 20·log10(4/3). Nothing errors; the file is just + * loud. These tests are the instrument that catches that class. + */ + +const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0; +const tempDirs: string[] = []; + +/** Mean level of a whole file, or of one window when `from`/`to` are given. */ +function meanVolumeDb(path: string, from?: number, to?: number): number { + const filter = + from === undefined ? "volumedetect" : `atrim=${from}:${to},asetpts=N/SR/TB,volumedetect`; + const result = spawnSync( + getFfmpegBinary(), + ["-nostdin", "-hide_banner", "-i", path, "-af", filter, "-f", "null", "-"], + { encoding: "utf-8" }, + ); + const match = result.stderr.match(/mean_volume:\s*(-?[\d.]+) dB/); + if (result.status !== 0 || !match?.[1]) { + throw new Error(`Could not measure mean volume: ${result.stderr}`); + } + return Number(match[1]); +} + +/** A sine at `freq`, scaled by `gain`, written as PCM so the source is exact. */ +function writeTone(path: string, freq: number, seconds: number, gain: number): void { + const result = spawnSync( + getFfmpegBinary(), + [ + "-nostdin", + "-v", + "error", + "-f", + "lavfi", + "-i", + `sine=frequency=${freq}:duration=${seconds}:sample_rate=48000`, + "-af", + `volume=${gain}`, + "-c:a", + "pcm_s16le", + path, + ], + { encoding: "utf-8" }, + ); + if (result.status !== 0) throw new Error(`Could not write tone: ${result.stderr}`); +} + +const track = (id: string, end: number, volume = 1) => ({ + id, + src: `${id}.wav`, + start: 0, + end, + mediaStart: 0, + layer: 0, + volume, + type: "audio" as const, +}); + +describe.skipIf(!HAS_FFMPEG)("mix level arithmetic", () => { + afterEach(() => { + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it("keeps every track at its authored level regardless of how many there are", async () => { + // The property the compensation exists to hold: adding tracks must not + // duck the ones already there. Mixing the SAME tone twice is the cleanest + // probe — two coherent copies sum to exactly +6.02 dB, so any residual + // normalisation shows up as a plain arithmetic miss rather than something + // that has to be teased out of unrelated material. + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-count-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-count-work-")); + tempDirs.push(projectDir, workDir); + + writeTone(join(projectDir, "a.wav"), 440, 2, 0.4); + writeTone(join(projectDir, "b.wav"), 440, 2, 0.4); + const oneUp = join(projectDir, `one-${MIXED_AUDIO_FILENAME}`); + const twoUp = join(projectDir, `two-${MIXED_AUDIO_FILENAME}`); + + const one = await processCompositionAudio([track("a", 2)], projectDir, workDir, oneUp, 2); + const two = await processCompositionAudio( + [track("a", 2), track("b", 2)], + projectDir, + workDir, + twoUp, + 2, + ); + expect(one.success).toBe(true); + expect(two.success).toBe(true); + + // Two coherent copies of one tone = +6.02 dB. If amix's 1/N ever survives + // the correction, this lands at 0 dB instead. + expect(meanVolumeDb(twoUp) - meanVolumeDb(oneUp)).toBeCloseTo(6.02, 0); + }); + + it("does not lift the survivors when a shorter track ends", async () => { + // amix with normalize=true rescales by the number of CURRENTLY ACTIVE + // inputs, so a track ending mid-composition would hand the remaining ones + // a level jump. `apad` to the full duration is what holds every input + // active for the whole graph and neutralises that — an invariant the + // filter string relies on without saying so. + // + // Measured without apad (spike, 2026-08-14): the tail runs +1.94 dB hot. + // Any group work that builds its own amix has to keep the padding, or + // inherit that bug one level down. + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-drop-work-")); + tempDirs.push(projectDir, workDir); + + writeTone(join(projectDir, "short.wav"), 440, 1, 0.5); + writeTone(join(projectDir, "long.wav"), 880, 3, 0.5); + const together = join(projectDir, `both-${MIXED_AUDIO_FILENAME}`); + const alone = join(projectDir, `alone-${MIXED_AUDIO_FILENAME}`); + + const both = await processCompositionAudio( + [track("short", 1), track("long", 3)], + projectDir, + workDir, + together, + 3, + ); + const solo = await processCompositionAudio([track("long", 3)], projectDir, workDir, alone, 3); + expect(both.success).toBe(true); + expect(solo.success).toBe(true); + + // After 1.5 s only `long` is sounding. It must read the same whether or not + // a second track happened to end earlier. + const tailTogether = meanVolumeDb(together, 1.5, 3); + const tailAlone = meanVolumeDb(alone, 1.5, 3); + expect(Math.abs(tailTogether - tailAlone)).toBeLessThan(0.5); + }); + + /** + * The gate for group buses (plans/audio-mixer-groups.md §1). + * + * Grouping is routing, not processing: a group whose FX chain is empty must + * be a no-op on the mix. Enable this the moment `data-audio-group` routes + * through a nested amix — it is the definition of done for §1.3, and the + * only thing standing between a wrong gain correction and a silently loud + * export. + * + * Proven reachable in the spike: nesting with each amix compensated by ITS + * OWN input count nulls against the flat mix to -inf (sample-identical), as + * does `amix=normalize=0` with no correction at all. + */ + it("mixes a grouped composition at the same level as the ungrouped one", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-level-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-level-work-")); + tempDirs.push(projectDir, workDir); + + writeTone(join(projectDir, "a.wav"), 440, 2, 0.4); + writeTone(join(projectDir, "b.wav"), 660, 2, 0.4); + const flatOut = join(projectDir, `flat-${MIXED_AUDIO_FILENAME}`); + const groupedOut = join(projectDir, `grouped-${MIXED_AUDIO_FILENAME}`); + + const flat = await processCompositionAudio( + [track("a", 2), track("b", 2)], + projectDir, + workDir, + flatOut, + 2, + ); + const grouped = await processCompositionAudio( + [ + { ...track("a", 2), groupId: "voiceover" }, + { ...track("b", 2), groupId: "voiceover" }, + ], + projectDir, + workDir, + groupedOut, + 2, + ); + expect(flat.success).toBe(true); + expect(grouped.success).toBe(true); + + // An empty group chain is pure routing — the export must read the same + // whether or not the two tones happened to share a group. + expect(Math.abs(meanVolumeDb(groupedOut) - meanVolumeDb(flatOut))).toBeLessThan(0.3); + }); + + it("a group FX chain fully cutting its members leaves an ungrouped track untouched (routing isolation)", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-fx-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-fx-work-")); + tempDirs.push(projectDir, workDir); + + writeTone(join(projectDir, "voice.wav"), 440, 2, 0.4); + writeTone(join(projectDir, "sfx.wav"), 880, 2, 0.4); + const mixedOut = join(projectDir, `mixed-${MIXED_AUDIO_FILENAME}`); + const sfxAloneOut = join(projectDir, `sfx-alone-${MIXED_AUDIO_FILENAME}`); + + const groupChain = JSON.stringify({ + version: 1, + nodes: [{ type: "gain", id: "g", params: { gain: -60 } }], + }); + + const mixed = await processCompositionAudio( + [{ ...track("voice", 2), groupId: "vo", groupFxChain: groupChain }, track("sfx", 2)], + projectDir, + workDir, + mixedOut, + 2, + ); + const sfxAlone = await processCompositionAudio( + [track("sfx", 2)], + projectDir, + workDir, + sfxAloneOut, + 2, + ); + expect(mixed.success).toBe(true); + expect(sfxAlone.success).toBe(true); + + // The voice group is cut ~60 dB — the mix should read close to sfx alone, + // and the ungrouped sfx track's own processing is unaffected by the + // group existing at all. + expect(Math.abs(meanVolumeDb(mixedOut) - meanVolumeDb(sfxAloneOut))).toBeLessThan(0.5); + }); + + it("a member's own volume envelope still applies inside a group", async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-grp-env-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-grp-env-work-")); + tempDirs.push(projectDir, workDir); + + writeTone(join(projectDir, "a.wav"), 440, 4, 0.5); + const groupedOut = join(projectDir, `grouped-${MIXED_AUDIO_FILENAME}`); + const flatOut = join(projectDir, `flat-${MIXED_AUDIO_FILENAME}`); + + const withEnvelope = { + ...track("a", 4), + volumeKeyframes: [ + { time: 0, volume: 1 }, + { time: 4, volume: 0 }, + ], + }; + + const grouped = await processCompositionAudio( + [{ ...withEnvelope, groupId: "vo" }], + projectDir, + workDir, + groupedOut, + 4, + ); + const flat = await processCompositionAudio([withEnvelope], projectDir, workDir, flatOut, 4); + expect(grouped.success).toBe(true); + expect(flat.success).toBe(true); + + // The envelope fades to silent — the tail should read the same whether + // the track is grouped or not, proving member-level processing survives + // the group path unchanged. + const groupedTail = meanVolumeDb(groupedOut, 3, 4); + const flatTail = meanVolumeDb(flatOut, 3, 4); + expect(Math.abs(groupedTail - flatTail)).toBeLessThan(0.5); + }); +}); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 6265ba0b99..0fa593deb0 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -47,6 +47,7 @@ import { parseStrictFiniteTimingNumber, readMediaStart, } from "@hyperframes/core"; +import { HF_AUDIO_GROUP_ATTR, resolveAudioGroups } from "@hyperframes/core/audio-groups"; import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js"; import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; @@ -461,6 +462,17 @@ export function parseAudioElements(html: string): AudioElement[] { return false; }; + // Resolved once per parse. A group element carrying `data-hidden` drops + // every member from the render (RULES: mute-by-drop, never + // mute-by-volume-0) — members never enter the sub-mix. + const groupsById = new Map( + resolveAudioGroups(document).map((group) => [group.id, group] as const), + ); + const memberGroupHidden = (el: AudioMediaElement): boolean => { + const groupId = el.getAttribute(HF_AUDIO_GROUP_ATTR); + return groupId ? (groupsById.get(groupId)?.hidden ?? false) : false; + }; + // ', + ); + expect(recordEdit).toHaveBeenCalledTimes(1); + expect(recordEdit.mock.calls[0]?.[0]?.label).toBe("Group 2 voice clips"); + expect( + usePlayerStore.getState().elements.find((el) => el.key === "index.html:#narration") + ?.audioGroup, + ).toBe("voiceover"); + expect( + usePlayerStore.getState().elements.find((el) => el.key === "index.html:#interview-guest") + ?.audioGroup, + ).toBe("voiceover"); + }); + + it("does nothing for fewer than two elements — grouping is a plural concept", async () => { + const recordEdit = vi.fn(); + const changedPaths = await createAudioGroupAndAssignMembers({ + projectId: "project-1", + activeCompPath: "index.html", + elements: [element({ id: "narration", domId: "narration" })], + groupId: "voiceover", + previewIframe: null, + writeProjectFile: async () => {}, + recordEdit, + domEditSaveTimestampRef: { current: 0 }, + pendingTimelineEditPathRef: { current: new Set() }, + }); + expect(changedPaths).toEqual([]); + expect(recordEdit).not.toHaveBeenCalled(); + }); + + it("reverts the optimistic live patch when the save fails", async () => { + const iframe = document.createElement("iframe"); + document.body.append(iframe); + if (iframe.contentDocument) { + iframe.contentDocument.body.innerHTML = ` + + + `; + } + // No stubbed fetch: readFileContent's request will fail, forcing the + // catch path. + const narration = element({ id: "narration", domId: "narration" }); + const guest = element({ id: "interview-guest", domId: "interview-guest" }); + + await expect( + createAudioGroupAndAssignMembers({ + projectId: "project-1", + activeCompPath: "index.html", + elements: [narration, guest], + groupId: "voiceover", + previewIframe: iframe, + writeProjectFile: async () => {}, + recordEdit: vi.fn(), + domEditSaveTimestampRef: { current: 0 }, + pendingTimelineEditPathRef: { current: new Set() }, + }), + ).rejects.toThrow(); + + expect( + iframe.contentDocument?.getElementById("narration")?.hasAttribute("data-audio-group"), + ).toBe(false); + expect( + iframe.contentDocument?.getElementById("interview-guest")?.hasAttribute("data-audio-group"), + ).toBe(false); + }); +}); diff --git a/packages/studio/src/hooks/timelineTrackVisibility.ts b/packages/studio/src/hooks/timelineTrackVisibility.ts index 96c2df2040..7c2593ab9c 100644 --- a/packages/studio/src/hooks/timelineTrackVisibility.ts +++ b/packages/studio/src/hooks/timelineTrackVisibility.ts @@ -8,6 +8,7 @@ import { } from "../player/components/timelineTrackDisplay"; import { saveProjectFilesWithHistory } from "../utils/studioFileHistory"; import { isAudioTimelineElement } from "../utils/timelineInspector"; +import { HF_AUDIO_GROUP_ATTR } from "@hyperframes/core/audio-groups"; import { readTagSnippetByTarget, type PatchOperation } from "../utils/sourcePatcher"; import { applyPatchByTarget, @@ -277,6 +278,114 @@ export async function toggleTimelineElementHidden({ }); } +function patchLiveAudioGroupState( + iframe: HTMLIFrameElement | null, + elements: readonly TimelineElement[], + groupId: string | null, + activeCompPath: string | null, +): void { + for (const element of elements) { + const target = findTimelineElementInIframe(iframe, element, activeCompPath); + if (!target) continue; + if (groupId) target.setAttribute(HF_AUDIO_GROUP_ATTR, groupId); + else target.removeAttribute(HF_AUDIO_GROUP_ATTR); + } +} + +interface CreateAudioGroupAndAssignMembersInput { + projectId: string; + activeCompPath: string | null; + elements: readonly TimelineElement[]; + groupId: string; + previewIframe: HTMLIFrameElement | null; + writeProjectFile: (path: string, content: string) => Promise; + recordEdit: (input: RecordEditInput) => Promise; + domEditSaveTimestampRef: MutableRef; + pendingTimelineEditPathRef: MutableRef>; +} + +/** + * Group two or more voice clips: write `data-audio-group=""` on + * every one of them, atomically, one undo entry — the same multi-target shape + * `setElementsHidden` uses for mute. The group needs no `` + * element of its own to exist: `resolveAudioGroups` already degrades + * gracefully to label = id when one is absent, and a naming dialog is out of + * scope here — the id itself is the default name. + */ +// fallow-ignore-next-line complexity +export async function createAudioGroupAndAssignMembers({ + projectId, + activeCompPath, + elements, + groupId, + previewIframe, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + pendingTimelineEditPathRef, +}: CreateAudioGroupAndAssignMembersInput): Promise { + if (elements.length < 2) return []; + + patchLiveAudioGroupState(previewIframe, elements, groupId, activeCompPath); + reseekPreviewRuntime(previewIframe); + + const groupOperation: PatchOperation = { + type: "attribute", + property: HF_AUDIO_GROUP_ATTR, + value: groupId, + }; + const originalByPath = new Map(); + const files: Record = {}; + + try { + for (const [targetPath, fileElements] of groupElementsByTargetPath(elements, activeCompPath)) { + let patchedContent = await readFileContent(projectId, targetPath); + originalByPath.set(targetPath, patchedContent); + + for (const element of fileElements) { + const patchTarget = buildPatchTarget(element); + if (!patchTarget) { + throw new Error(`Timeline element ${element.id} is missing a patchable target`); + } + if (readTagSnippetByTarget(patchedContent, patchTarget) === undefined) { + throw new Error(`Unable to patch timeline element ${element.id} in ${targetPath}`); + } + patchedContent = applyPatchByTarget(patchedContent, patchTarget, groupOperation); + } + + files[targetPath] = patchedContent; + pendingTimelineEditPathRef.current.add(targetPath); + } + + domEditSaveTimestampRef.current = Date.now(); + const changedPaths = await saveProjectFilesWithHistory({ + projectId, + label: `Group ${elements.length} voice clips`, + kind: "timeline", + files, + readFile: async (path) => { + const original = originalByPath.get(path); + if (original !== undefined) return original; + return readFileContent(projectId, path); + }, + writeFile: writeProjectFile, + recordEdit, + }); + domEditSaveTimestampRef.current = Date.now(); + for (const element of elements) { + usePlayerStore.getState().updateElement(element.key ?? element.id, { audioGroup: groupId }); + } + return changedPaths; + } catch (error) { + // Mirrors setElementsHidden's failure path: the optimistic live patch + // already ran, so a save failure has to be unwound or the preview shows a + // grouping that never made it to disk. + patchLiveAudioGroupState(previewIframe, elements, null, activeCompPath); + reseekPreviewRuntime(previewIframe); + throw error; + } +} + export function useTimelineTrackVisibilityEditing({ projectIdRef, activeCompPath, @@ -407,3 +516,67 @@ export function useTimelineElementVisibilityEditing({ ], ); } + +/** + * The write behind B6's auto-group: pick two or more voice clips in the carve + * picker and they land in a group instead of naming each other by id. Same + * expanded-rows resolution as element-visibility, for the same reason — a + * nested sub-composition child has no entry in the raw store list. + */ +export function useAudioGroupCarveAssignment({ + projectIdRef, + activeCompPath, + showToast, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + previewIframeRef, + pendingTimelineEditPathRef, + isRecordingRef, +}: UseTimelineElementVisibilityEditingInput): ( + clipIds: readonly string[], + groupId: string, +) => Promise { + const expandedElements = useExpandedTimelineElements(); + return useCallback( + async (clipIds: readonly string[], groupId: string) => { + if (isRecordingRef?.current) { + showToast("Cannot edit timeline while recording", "error"); + return; + } + const pid = projectIdRef.current; + if (!pid) return; + const keys = new Set(clipIds); + const elements = expandedElements.filter((item) => keys.has(item.key ?? item.id)); + try { + await createAudioGroupAndAssignMembers({ + projectId: pid, + activeCompPath, + elements, + groupId, + previewIframe: previewIframeRef.current, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + pendingTimelineEditPathRef, + }); + } catch (error) { + console.error("[Timeline] Failed to group voice clips", error); + const message = error instanceof Error ? error.message : "Failed to group voice clips"; + showToast(message); + } + }, + [ + activeCompPath, + expandedElements, + previewIframeRef, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + pendingTimelineEditPathRef, + isRecordingRef, + showToast, + projectIdRef, + ], + ); +} diff --git a/packages/studio/src/hooks/useEffectiveTimelineDuration.ts b/packages/studio/src/hooks/useEffectiveTimelineDuration.ts new file mode 100644 index 0000000000..b89fd3b827 --- /dev/null +++ b/packages/studio/src/hooks/useEffectiveTimelineDuration.ts @@ -0,0 +1,20 @@ +import { useMemo } from "react"; +import type { TimelineElement } from "../player/store/timelineElement"; + +/** + * The stored `duration` lags a moment behind an edit that pushes an element + * past it (drag, trim, paste) — this is the actual end of the timeline, the + * later of the stored duration and the furthest element's end. + */ +export function useEffectiveTimelineDuration( + timelineDuration: number, + timelineElements: readonly TimelineElement[], +): number { + return useMemo(() => { + const maxEnd = + timelineElements.length > 0 + ? Math.max(...timelineElements.map((el) => el.start + el.duration)) + : 0; + return Math.max(timelineDuration, maxEnd); + }, [timelineDuration, timelineElements]); +} diff --git a/packages/studio/src/hooks/useTimelineEditing.ts b/packages/studio/src/hooks/useTimelineEditing.ts index f59b82c2a5..453102032b 100644 --- a/packages/studio/src/hooks/useTimelineEditing.ts +++ b/packages/studio/src/hooks/useTimelineEditing.ts @@ -28,6 +28,7 @@ import { import type { PersistTimelineEditInput } from "./timelineEditingHelpers"; import type { TimelineStackingReorderIntent } from "../player/components/timelineEditing"; import { + useAudioGroupCarveAssignment, useTimelineElementVisibilityEditing, useTimelineTrackVisibilityEditing, } from "./timelineTrackVisibility"; @@ -388,6 +389,18 @@ export function useTimelineEditing({ forceReloadSdkSession, }); + const handleAutoGroupCarveSources = useAudioGroupCarveAssignment({ + projectIdRef, + activeCompPath, + showToast, + writeProjectFile, + recordEdit, + domEditSaveTimestampRef, + previewIframeRef, + pendingTimelineEditPathRef, + isRecordingRef, + }); + // fallow-ignore-next-line complexity const handleTimelineElementsDelete = useCallback( // fallow-ignore-next-line complexity @@ -558,6 +571,7 @@ export function useTimelineEditing({ handleTimelineElementResize, handleToggleTrackHidden, handleToggleElementHidden, + handleAutoGroupCarveSources, handleTimelineElementDelete, handleTimelineElementsDelete, handleTimelineElementSplit: handleRazorSplit, diff --git a/packages/studio/src/player/store/playerStore.ts b/packages/studio/src/player/store/playerStore.ts index 8cf4d59a26..63274b5de1 100644 --- a/packages/studio/src/player/store/playerStore.ts +++ b/packages/studio/src/player/store/playerStore.ts @@ -146,7 +146,14 @@ interface PlayerState extends KeyframeSlice, AutomationSelectionSlice, Thumbnail updates: Partial< Pick< TimelineElement, - "start" | "duration" | "track" | "zIndex" | "hasExplicitZIndex" | "playbackStart" | "hidden" + | "start" + | "duration" + | "track" + | "zIndex" + | "hasExplicitZIndex" + | "playbackStart" + | "hidden" + | "audioGroup" > >, ) => void; diff --git a/plans/spikes/.gitignore b/plans/spikes/.gitignore new file mode 100644 index 0000000000..3135082642 --- /dev/null +++ b/plans/spikes/.gitignore @@ -0,0 +1 @@ +amix-work/ diff --git a/plans/spikes/amix-nesting-spike.sh b/plans/spikes/amix-nesting-spike.sh new file mode 100755 index 0000000000..b6000dc5b9 --- /dev/null +++ b/plans/spikes/amix-nesting-spike.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Spike: does a nested amix (group bus) preserve the level of a flat amix? +# +# Invariant under test: a group whose FX chain is EMPTY must be a no-op on the +# mix. Grouping is routing, not processing — if grouping alone changes the +# level, every grouped export is silently wrong. +set -euo pipefail + +D="$(dirname "$0")/amix-work" +rm -rf "$D"; mkdir -p "$D" +cd "$D" + +SR=48000 +DUR=3 + +# Four tracks, distinct frequencies so nothing cancels, distinct amplitudes so a +# mis-weighted track shows up rather than averaging out. +ffmpeg -v error -f lavfi -i "sine=frequency=220:sample_rate=$SR:duration=$DUR" -af "volume=0.50" -c:a pcm_s16le t1.wav +ffmpeg -v error -f lavfi -i "sine=frequency=440:sample_rate=$SR:duration=$DUR" -af "volume=0.25" -c:a pcm_s16le t2.wav +ffmpeg -v error -f lavfi -i "sine=frequency=880:sample_rate=$SR:duration=$DUR" -af "volume=0.40" -c:a pcm_s16le t3.wav +ffmpeg -v error -f lavfi -i "sine=frequency=1760:sample_rate=$SR:duration=$DUR" -af "volume=0.15" -c:a pcm_s16le t4.wav + +rms () { # $1 = wav -> RMS dB + ffmpeg -v info -i "$1" -af astats=metadata=1:reset=0 -f null - 2>&1 \ + | awk -F'dB: ' '/Overall/{o=1} o&&/RMS level dB/{print $2; exit}' +} + +echo "=== per-track RMS (dB) ===" +for f in t1 t2 t3 t4; do printf " %-4s %s\n" "$f" "$(rms $f.wav)"; done +echo + +# --- ARM A: flat mix, today's shipped shape ------------------------------- +# amix normalizes by input count; multiply back by the same count. +ffmpeg -v error -i t1.wav -i t2.wav -i t3.wav -i t4.wav -filter_complex \ +"[0:a]apad,atrim=0:$DUR[a0];\ +[1:a]apad,atrim=0:$DUR[a1];\ +[2:a]apad,atrim=0:$DUR[a2];\ +[3:a]apad,atrim=0:$DUR[a3];\ +[a0][a1][a2][a3]amix=inputs=4:duration=longest:dropout_transition=0[mixed];\ +[mixed]volume=4[out]" -map "[out]" -c:a pcm_s16le flat.wav + +# --- ARM B: nested, compensated PER NODE --------------------------------- +# group A = t1+t2 (2 inputs -> x2). outer = groupA + t3 + t4 (3 inputs -> x3). +ffmpeg -v error -i t1.wav -i t2.wav -i t3.wav -i t4.wav -filter_complex \ +"[0:a]apad,atrim=0:$DUR[a0];\ +[1:a]apad,atrim=0:$DUR[a1];\ +[2:a]apad,atrim=0:$DUR[a2];\ +[3:a]apad,atrim=0:$DUR[a3];\ +[a0][a1]amix=inputs=2:duration=longest:dropout_transition=0[gmix];\ +[gmix]volume=2[gA];\ +[gA][a2][a3]amix=inputs=3:duration=longest:dropout_transition=0[mixed];\ +[mixed]volume=3[out]" -map "[out]" -c:a pcm_s16le nested_ok.wav + +# --- ARM C: nested, but the outer node keeps the GLOBAL track count ------- +# The plausible mistake: tracks.length is 4, the outer amix has 3 inputs. +ffmpeg -v error -i t1.wav -i t2.wav -i t3.wav -i t4.wav -filter_complex \ +"[0:a]apad,atrim=0:$DUR[a0];\ +[1:a]apad,atrim=0:$DUR[a1];\ +[2:a]apad,atrim=0:$DUR[a2];\ +[3:a]apad,atrim=0:$DUR[a3];\ +[a0][a1]amix=inputs=2:duration=longest:dropout_transition=0[gmix];\ +[gmix]volume=2[gA];\ +[gA][a2][a3]amix=inputs=3:duration=longest:dropout_transition=0[mixed];\ +[mixed]volume=4[out]" -map "[out]" -c:a pcm_s16le nested_bug.wav + +# --- ARM D: nested with normalize=0, no compensation anywhere ------------ +ffmpeg -v error -i t1.wav -i t2.wav -i t3.wav -i t4.wav -filter_complex \ +"[0:a]apad,atrim=0:$DUR[a0];\ +[1:a]apad,atrim=0:$DUR[a1];\ +[2:a]apad,atrim=0:$DUR[a2];\ +[3:a]apad,atrim=0:$DUR[a3];\ +[a0][a1]amix=inputs=2:normalize=0:duration=longest:dropout_transition=0[gA];\ +[gA][a2][a3]amix=inputs=3:normalize=0:duration=longest:dropout_transition=0[out]" \ +-map "[out]" -c:a pcm_s16le nested_norm0.wav + +# --- ARM E: FLAT with normalize=0 ---------------------------------------- +ffmpeg -v error -i t1.wav -i t2.wav -i t3.wav -i t4.wav -filter_complex \ +"[0:a]apad,atrim=0:$DUR[a0];\ +[1:a]apad,atrim=0:$DUR[a1];\ +[2:a]apad,atrim=0:$DUR[a2];\ +[3:a]apad,atrim=0:$DUR[a3];\ +[a0][a1][a2][a3]amix=inputs=4:normalize=0:duration=longest:dropout_transition=0[out]" \ +-map "[out]" -c:a pcm_s16le flat_norm0.wav + +echo "=== mix RMS (dB) ===" +for f in flat nested_ok nested_bug nested_norm0 flat_norm0; do + printf " %-14s %s\n" "$f" "$(rms $f.wav)" +done +echo + +# --- sample-exactness: null test (A inverted + B must be silence) --------- +null_test () { # $1 $2 -> peak dB of the difference + ffmpeg -v info -i "$1" -i "$2" -filter_complex \ + "[1:a]volume=-1[inv];[0:a][inv]amix=inputs=2:normalize=0,astats=metadata=1:reset=0[d]" \ + -map "[d]" -f null - 2>&1 \ + | awk -F'dB: ' '/Overall/{o=1} o&&/Peak level dB/{print $2; exit}' +} + +echo "=== null tests (peak dB of difference; -inf or < -90 = identical) ===" +printf " flat vs nested_ok %s\n" "$(null_test flat.wav nested_ok.wav)" +printf " flat vs nested_bug %s\n" "$(null_test flat.wav nested_bug.wav)" +printf " flat vs nested_norm0 %s\n" "$(null_test flat.wav nested_norm0.wav)" +printf " flat vs flat_norm0 %s\n" "$(null_test flat.wav flat_norm0.wav)" diff --git a/skills-manifest.json b/skills-manifest.json index edfaad47b2..f3c2bee0c0 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -26,7 +26,7 @@ "files": 121 }, "hyperframes-audio": { - "hash": "94aba963d262d71d", + "hash": "819ab4e70d0f1cc9", "files": 6 }, "hyperframes-cli": { diff --git a/skills/hyperframes-audio/SKILL.md b/skills/hyperframes-audio/SKILL.md index f1ee5781b2..ee13ddf367 100644 --- a/skills/hyperframes-audio/SKILL.md +++ b/skills/hyperframes-audio/SKILL.md @@ -236,6 +236,31 @@ so one analysis covers all of them: the bands come from all the speech there is, the envelopes rise wherever any of it is happening. Voices that never play while the bed does are left out; they cannot mask it. +**A carve against more than one clip id is wrong. Group the clips and carve +against the group.** This is an invariant, not a tip. Naming clips one by one has +to be exhaustively right and stays right only until the next edit — a fourth +narration clip added later plays outside the carve's awareness, and the bed +fails to duck under it silently. Naming the group instead resolves membership at +analysis time, so a clip added to the group later is covered without touching +`sources` at all: + +```html + + + + + + +``` + +A `sources` list naming two or more plain clip ids instead of a group is caught +by the `audio_carve_ungrouped_sources` lint rule — it still works, but it is the +version that silently rots when a clip is added. + **One knob.** `strength` is 0..1 and derives everything: how deep to cut, how many bands, how wide, how far to favour intelligibility over raw voice energy, how far the level may drop, how far under the voice to aim. Those six move