diff --git a/docs/concepts/data-attributes.mdx b/docs/concepts/data-attributes.mdx
index ed1caaff1..2ee08dd49 100644
--- a/docs/concepts/data-attributes.mdx
+++ b/docs/concepts/data-attributes.mdx
@@ -20,6 +20,35 @@ Hyperframes uses HTML data attributes to control timing, media playback, and [co
| `data-media-start` | `"2"` | Media playback offset / trim point in seconds. Default: `0` |
| `data-volume` | `"0.8"` | Audio/video volume, 0 to 1 |
| `data-has-audio` | `"true"` | Indicates video has an audio track |
+| `data-role` | `"voice"` | Marks narration or dialogue that should trigger ducking |
+| `data-duck` | `"-12dB"` | Lowers this track while audible `data-role="voice"` clips overlap it |
+| `data-duck-fade` | `"0.3"` | Ducking ramp duration in seconds. Default: `0.3` |
+
+## Music Ducking
+
+Use `data-duck` on background music and `data-role="voice"` on narration or dialogue. Hyperframes computes the overlap windows from clip timing and renders the result through the same volume automation path used for authored fades.
+
+```html index.html
+` or audible `` clips is not part of the compile-time duck calculation.
## Composition Attributes
diff --git a/docs/docs.json b/docs/docs.json
index c8a1caac6..5c4b2566f 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -85,6 +85,7 @@
"guides/hyperframes-vs-remotion",
"guides/gsap-animation",
"guides/keyframes",
+ "guides/music-voiceover",
"guides/rendering",
"guides/remove-background",
"guides/hdr",
diff --git a/docs/guides/music-voiceover.mdx b/docs/guides/music-voiceover.mdx
new file mode 100644
index 000000000..41b891c77
--- /dev/null
+++ b/docs/guides/music-voiceover.mdx
@@ -0,0 +1,32 @@
+---
+title: Music + Voiceover
+description: "Balance background music under narration with declarative audio ducking."
+---
+
+Use `data-duck` when a music bed should automatically drop under narration or dialogue. Mark the narration with `data-role="voice"` and put the duck amount on the music track.
+
+```html index.html
+` clips and audible `` clips. Audio started imperatively from script is outside this compile-time calculation.
diff --git a/docs/reference/html-schema.mdx b/docs/reference/html-schema.mdx
index dd6d0e70c..feee87073 100644
--- a/docs/reference/html-schema.mdx
+++ b/docs/reference/html-schema.mdx
@@ -56,6 +56,9 @@ Common sizes:
| `data-track-index` | All | Yes | Timeline track number. Controls z-ordering (higher = in front). Clips on the same track cannot overlap. |
| `data-media-start` | video, audio | No | Playback offset / trim point in source file (seconds). Default: `0`. See [Data Attributes](/concepts/data-attributes). |
| `data-volume` | audio, video | No | Volume level from `0` to `1`. Default: `1`. |
+| `data-role` | audio, video | No | Semantic audio role. Set `data-role="voice"` on narration or dialogue tracks that should trigger music ducking. |
+| `data-duck` | audio, video | No | Duck this track while any audible `data-role="voice"` clip overlaps it. Accepts dB (`"-12dB"` or `"-12"`) or a linear gain (`"0.25"`). |
+| `data-duck-fade` | audio, video | No | Ducking ramp duration in seconds. Default: `0.3`. |
| `data-composition-id` | div | On compositions | Unique composition ID. Must match the key used in `window.__timelines`. |
| `data-composition-src` | div | No | Path to external composition HTML file (for [nested compositions](#composition-clips)). |
| `data-variable-values` | div | No | JSON object of values passed to a nested composition. The framework carries the values through, but your composition script must read and apply them manually. |
@@ -129,8 +132,30 @@ Common sizes:
- `data-duration` is **optional** — defaults to the remaining duration of the source file from `data-media-start`
- Audio clips are invisible — do not add `class="clip"` (there is nothing to show/hide)
- `data-volume` controls volume — use `"0.5"` for background music at 50% volume
+ - `data-duck` lowers background music under clips marked `data-role="voice"`
- `data-media-start` trims the beginning of the audio source, just like video
- Multiple audio clips can overlap on different tracks for layered sound design
+
+ ```html
+
diff --git a/packages/core/src/compiler/htmlCompiler.ts b/packages/core/src/compiler/htmlCompiler.ts
index 264f4ec11..113670047 100644
--- a/packages/core/src/compiler/htmlCompiler.ts
+++ b/packages/core/src/compiler/htmlCompiler.ts
@@ -1,6 +1,7 @@
import { resolve } from "path";
import {
compileTimingAttrs,
+ compileAudioDucking,
injectDurations,
extractResolvedMedia,
clampDurations,
@@ -37,7 +38,7 @@ export async function compileHtml(
const { html: staticCompiled, unresolved } = compileTimingAttrs(rawHtml);
let html = staticCompiled;
- if (!probeMediaDuration) return html;
+ if (!probeMediaDuration) return compileAudioDucking(html);
// Phase 1: Resolve missing durations
const mediaUnresolved = unresolved.filter(
@@ -86,5 +87,5 @@ export async function compileHtml(
html = clampDurations(html, clampList);
}
- return html;
+ return compileAudioDucking(html);
}
diff --git a/packages/core/src/compiler/index.ts b/packages/core/src/compiler/index.ts
index d18803c92..5ad5b9c1f 100644
--- a/packages/core/src/compiler/index.ts
+++ b/packages/core/src/compiler/index.ts
@@ -1,6 +1,7 @@
// Timing compiler (browser-safe)
export {
compileTimingAttrs,
+ compileAudioDucking,
injectDurations,
extractResolvedMedia,
clampDurations,
diff --git a/packages/core/src/compiler/timingCompiler.test.ts b/packages/core/src/compiler/timingCompiler.test.ts
index b7dcf044c..2156d8b4a 100644
--- a/packages/core/src/compiler/timingCompiler.test.ts
+++ b/packages/core/src/compiler/timingCompiler.test.ts
@@ -1,10 +1,19 @@
import { describe, it, expect } from "vitest";
import {
compileTimingAttrs,
+ compileAudioDucking,
injectDurations,
extractResolvedMedia,
clampDurations,
} from "./timingCompiler.js";
+import { parseVolumeKeyframesAttribute } from "../runtime/mediaVolumeEnvelope.js";
+
+function duckKeyframesFor(html: string, id: string) {
+ const match = html.match(new RegExp(`]*id=["']${id}["'][^>]*>`, "i"));
+ if (!match) throw new Error(`Missing audio tag ${id}`);
+ const attr = match[0].match(/data-hf-duck-keyframes='([^']+)'/);
+ return parseVolumeKeyframesAttribute(attr?.[1]);
+}
describe("compileTimingAttrs", () => {
it("adds data-end when data-start and data-duration are present on a video", () => {
@@ -185,3 +194,53 @@ describe("clampDurations", () => {
expect(result).toContain('data-end="7"');
});
});
+
+describe("compileAudioDucking", () => {
+ it("emits duck keyframes around voice overlaps", () => {
+ const html = compileTimingAttrs(`
+ ();
+ for (const clip of clips) {
+ if (clip.duckGain === null || !isAudibleClip(clip)) continue;
+ const keyframes = createDuckKeyframes(clip, voiceIntervals);
+ if (keyframes.length > 0) duckKeyframesById.set(clip.id, keyframes);
+ }
+
+ return html.replace(/<(?:audio|video)\b[^>]*>/gi, (tag) => {
+ const id = getAttr(tag, "id");
+ const keyframes = id ? duckKeyframesById.get(id) : undefined;
+ if (keyframes && keyframes.length > 0) {
+ return setAttr(tag, DUCK_KEYFRAMES_ATTR, JSON.stringify(keyframes));
+ }
+ return hasAttr(tag, DUCK_KEYFRAMES_ATTR) ? removeAttr(tag, DUCK_KEYFRAMES_ATTR) : tag;
+ });
+}
+
/**
* Inject resolved durations into compiled HTML.
*
diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts
index 55ab677e0..1e703e4dc 100644
--- a/packages/core/src/index.test.ts
+++ b/packages/core/src/index.test.ts
@@ -128,6 +128,7 @@ describe("@hyperframes/core public API exports", () => {
describe("compiler exports", () => {
it("exports compiler functions", () => {
expect(typeof core.compileTimingAttrs).toBe("function");
+ expect(typeof core.compileAudioDucking).toBe("function");
expect(typeof core.injectDurations).toBe("function");
expect(typeof core.extractResolvedMedia).toBe("function");
expect(typeof core.clampDurations).toBe("function");
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 747e32abf..4187410b9 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -115,6 +115,7 @@ export type {
export {
compileTimingAttrs,
+ compileAudioDucking,
injectDurations,
extractResolvedMedia,
clampDurations,
diff --git a/packages/core/src/runtime/mediaVolumeEnvelope.ts b/packages/core/src/runtime/mediaVolumeEnvelope.ts
index 69d55ea0c..ac6680ef1 100644
--- a/packages/core/src/runtime/mediaVolumeEnvelope.ts
+++ b/packages/core/src/runtime/mediaVolumeEnvelope.ts
@@ -12,6 +12,133 @@ export interface VolumeKeyframe {
volume: number;
}
+function clampVolume(volume: number): number {
+ if (!Number.isFinite(volume)) return 1;
+ return Math.max(0, Math.min(1, volume));
+}
+
+function readNumberProperty(value: object, key: "time" | "volume"): number | null {
+ const descriptor = Object.getOwnPropertyDescriptor(value, key);
+ return typeof descriptor?.value === "number" && Number.isFinite(descriptor.value)
+ ? descriptor.value
+ : null;
+}
+
+function roundedPoint(time: number, volume: number): VolumeKeyframe {
+ return {
+ time: Number(time.toFixed(6)),
+ volume: Number(clampVolume(volume).toFixed(6)),
+ };
+}
+
+function compactVolumeKeyframes(keyframes: VolumeKeyframe[]): VolumeKeyframe[] {
+ const sorted = keyframes
+ .filter((kf) => Number.isFinite(kf.time) && Number.isFinite(kf.volume))
+ .map((kf) => roundedPoint(kf.time, kf.volume))
+ .sort((a, b) => a.time - b.time);
+
+ const deduped: VolumeKeyframe[] = [];
+ for (const point of sorted) {
+ const previous = deduped.at(-1);
+ if (previous && Math.abs(previous.time - point.time) < 0.000001) {
+ previous.volume = point.volume;
+ } else {
+ deduped.push(point);
+ }
+ }
+
+ if (deduped.length < 3) return deduped;
+
+ const compacted: VolumeKeyframe[] = [];
+ for (const point of deduped) {
+ compacted.push(point);
+ while (compacted.length >= 3) {
+ const c = compacted.at(-1)!;
+ const b = compacted.at(-2)!;
+ const a = compacted.at(-3)!;
+ const span = c.time - a.time;
+ const expected =
+ span <= 0 ? a.volume : a.volume + ((c.volume - a.volume) * (b.time - a.time)) / span;
+ if (Math.abs(expected - b.volume) > 0.000001) break;
+ compacted.splice(compacted.length - 2, 1);
+ }
+ }
+ return compacted;
+}
+
+export function parseVolumeKeyframesAttribute(raw: string | null | undefined): VolumeKeyframe[] {
+ if (!raw) return [];
+
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ return [];
+ }
+
+ if (!Array.isArray(parsed)) return [];
+
+ const keyframes: VolumeKeyframe[] = [];
+ for (const item of parsed) {
+ if (typeof item !== "object" || item === null) continue;
+ const time = readNumberProperty(item, "time");
+ const volume = readNumberProperty(item, "volume");
+ if (time === null || volume === null) continue;
+ keyframes.push({ time, volume });
+ }
+ return compactVolumeKeyframes(keyframes);
+}
+
+export function serializeVolumeKeyframesAttribute(keyframes: VolumeKeyframe[]): string {
+ return JSON.stringify(compactVolumeKeyframes(keyframes));
+}
+
+export function multiplyVolumeKeyframeEnvelopes(params: {
+ sourceKeyframes: VolumeKeyframe[];
+ multiplierKeyframes: VolumeKeyframe[];
+ trackStart: number;
+ trackEnd: number;
+ baseVolume: number;
+}): VolumeKeyframe[] {
+ const multiplier = compactVolumeKeyframes(params.multiplierKeyframes);
+ if (multiplier.length === 0) return compactVolumeKeyframes(params.sourceKeyframes);
+
+ const source = compactVolumeKeyframes(params.sourceKeyframes);
+ if (source.length === 0) return multiplier;
+
+ const baseVolume = clampVolume(params.baseVolume);
+ const sourceEnvelope = normaliseEnvelope(source, params.trackStart, baseVolume);
+ const multiplierEnvelope = normaliseEnvelope(multiplier, params.trackStart, baseVolume);
+ if (sourceEnvelope.length === 0) return multiplier;
+ if (multiplierEnvelope.length === 0) return source;
+
+ const duration = Math.max(0, params.trackEnd - params.trackStart);
+ const times = new Set([0, duration]);
+ for (const point of sourceEnvelope) {
+ if (point.time >= 0 && point.time <= duration) times.add(point.time);
+ }
+ for (const point of multiplierEnvelope) {
+ if (point.time >= 0 && point.time <= duration) times.add(point.time);
+ }
+
+ const sortedTimes = Array.from(times).sort((a, b) => a - b);
+ const sampleTimes = new Set(sortedTimes);
+ for (let i = 0; i < sortedTimes.length - 1; i += 1) {
+ const a = sortedTimes[i]!;
+ const b = sortedTimes[i + 1]!;
+ if (b > a) sampleTimes.add((a + b) / 2);
+ }
+
+ const combined: VolumeKeyframe[] = [];
+ for (const relativeTime of Array.from(sampleTimes).sort((a, b) => a - b)) {
+ const sourceVolume = interpolateVolumeGain(sourceEnvelope, relativeTime);
+ const multiplierVolume = interpolateVolumeGain(multiplierEnvelope, relativeTime);
+ const multiplierGain = baseVolume > 0 ? multiplierVolume / baseVolume : 0;
+ combined.push(roundedPoint(params.trackStart + relativeTime, sourceVolume * multiplierGain));
+ }
+ return compactVolumeKeyframes(combined);
+}
+
/**
* Normalise raw keyframes to track-relative seconds: subtract `trackStart`,
* clamp to [0,1], sort, de-duplicate, and prepend a `baseVolume` anchor at
@@ -29,7 +156,7 @@ export function normaliseEnvelope(
.filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
.map((k) => ({
time: Math.max(0, k.time - trackStart),
- volume: Math.max(0, Math.min(1, k.volume)),
+ volume: clampVolume(k.volume),
}))
.sort((a, b) => a.time - b.time);
@@ -45,7 +172,7 @@ export function normaliseEnvelope(
if (deduped.length === 0) return deduped;
if (deduped[0]!.time > 0) {
- deduped.unshift({ time: 0, volume: Math.max(0, Math.min(1, baseVolume)) });
+ deduped.unshift({ time: 0, volume: clampVolume(baseVolume) });
}
return deduped;
}
@@ -98,7 +225,7 @@ export function probeElementVolumeKeyframes(
: compositionDuration;
const staticAttr = Number.parseFloat(el.dataset.volume ?? "");
- const staticVolume = Number.isFinite(staticAttr) ? Math.max(0, Math.min(1, staticAttr)) : 1;
+ const staticVolume = Number.isFinite(staticAttr) ? clampVolume(staticAttr) : 1;
// Reset to data-volume so GSAP captures the correct FROM value.
el.volume = staticVolume;
@@ -113,7 +240,7 @@ export function probeElementVolumeKeyframes(
seekTimeline(bounded);
const raw = Number(el.volume);
if (!Number.isFinite(raw)) continue;
- const volume = Math.max(0, Math.min(1, raw));
+ const volume = clampVolume(raw);
const last = keyframes.at(-1);
if (!last || Math.abs(last.volume - volume) > 0.0001 || bounded === sampleEnd) {
keyframes.push({ time: Number(bounded.toFixed(6)), volume: Number(volume.toFixed(6)) });
@@ -140,11 +267,24 @@ export function probeAndCacheElementVolume(
compositionDuration: number,
cache: WeakMap,
): void {
- if (!timeline) return;
if (!(mediaEl instanceof HTMLAudioElement) && !(mediaEl instanceof HTMLVideoElement)) return;
- if (compositionDuration <= 0) return;
+
+ const start = Number.parseFloat(mediaEl.dataset.start ?? "0") || 0;
+ const endAttr = Number.parseFloat(mediaEl.dataset.end ?? "");
+ const durationAttr = Number.parseFloat(mediaEl.dataset.duration ?? "");
+ const end =
+ Number.isFinite(endAttr) && endAttr > start
+ ? endAttr
+ : Number.isFinite(durationAttr) && durationAttr > 0
+ ? start + durationAttr
+ : compositionDuration;
+ const staticAttr = Number.parseFloat(mediaEl.dataset.volume ?? "");
+ const staticVolume = Number.isFinite(staticAttr) ? clampVolume(staticAttr) : 1;
+ const declarativeKeyframes = parseVolumeKeyframesAttribute(mediaEl.dataset.volumeKeyframes);
+ const duckKeyframes = parseVolumeKeyframesAttribute(mediaEl.dataset.hfDuckKeyframes);
const seekFn = (t: number) => {
+ if (!timeline) return;
try {
if (typeof timeline.totalTime === "function") {
timeline.totalTime(t, true);
@@ -156,8 +296,20 @@ export function probeAndCacheElementVolume(
}
};
- const keyframes = probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60);
- if (keyframes) {
- cache.set(mediaEl, keyframes);
+ const probedKeyframes =
+ timeline && compositionDuration > 0
+ ? probeElementVolumeKeyframes(mediaEl, seekFn, compositionDuration, 60)
+ : null;
+
+ let keyframes = probedKeyframes ?? declarativeKeyframes;
+ if (duckKeyframes.length > 0) {
+ keyframes = multiplyVolumeKeyframeEnvelopes({
+ sourceKeyframes: keyframes,
+ multiplierKeyframes: duckKeyframes,
+ trackStart: start,
+ trackEnd: end,
+ baseVolume: staticVolume,
+ });
}
+ if (keyframes.length > 0) cache.set(mediaEl, keyframes);
}
diff --git a/packages/engine/src/services/audioElementParser.ts b/packages/engine/src/services/audioElementParser.ts
new file mode 100644
index 000000000..9db710d11
--- /dev/null
+++ b/packages/engine/src/services/audioElementParser.ts
@@ -0,0 +1,69 @@
+import { parseHTML } from "linkedom";
+import {
+ multiplyVolumeKeyframeEnvelopes,
+ parseVolumeKeyframesAttribute,
+} from "@hyperframes/core/media-volume-envelope";
+import { unwrapTemplate } from "../utils/htmlTemplate.js";
+import type { AudioElement } from "./audioMixer.types.js";
+
+function parseElementVolumeKeyframes(
+ el: Element,
+ start: number,
+ end: number,
+ baseVolume: number,
+): AudioElement["volumeKeyframes"] {
+ const authored = parseVolumeKeyframesAttribute(el.getAttribute("data-volume-keyframes"));
+ const duck = parseVolumeKeyframesAttribute(el.getAttribute("data-hf-duck-keyframes"));
+ if (duck.length === 0) return authored.length > 0 ? authored : undefined;
+ return multiplyVolumeKeyframeEnvelopes({
+ sourceKeyframes: authored,
+ multiplierKeyframes: duck,
+ trackStart: start,
+ trackEnd: end,
+ baseVolume,
+ });
+}
+
+function readAudioElement(el: Element, type: "audio" | "video"): AudioElement | null {
+ const id = el.getAttribute("id");
+ const src = el.getAttribute("src");
+ if (!id || !src) return null;
+
+ const startAttr = el.getAttribute("data-start");
+ const endAttr = el.getAttribute("data-end");
+ const mediaStartAttr = el.getAttribute("data-media-start");
+ const layerAttr = el.getAttribute("data-layer");
+ const volumeAttr = el.getAttribute("data-volume");
+ const start = startAttr ? parseFloat(startAttr) : 0;
+ const end = endAttr ? parseFloat(endAttr) : 0;
+ const volume = volumeAttr ? parseFloat(volumeAttr) : 1.0;
+
+ return {
+ id: type === "video" ? `${id}-audio` : id,
+ src,
+ start,
+ end,
+ mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
+ layer: layerAttr ? parseInt(layerAttr) : 0,
+ volume,
+ volumeKeyframes: parseElementVolumeKeyframes(el, start, end, volume),
+ type,
+ };
+}
+
+export function parseAudioElements(html: string): AudioElement[] {
+ const elements: AudioElement[] = [];
+ const { document } = parseHTML(unwrapTemplate(html));
+
+ for (const el of document.querySelectorAll("audio[id][src]")) {
+ const audio = readAudioElement(el, "audio");
+ if (audio) elements.push(audio);
+ }
+
+ for (const el of document.querySelectorAll('video[id][src][data-has-audio="true"]')) {
+ const audio = readAudioElement(el, "video");
+ if (audio) elements.push(audio);
+ }
+
+ return elements;
+}
diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts
index 6f6eb4756..4310d92d2 100644
--- a/packages/engine/src/services/audioMixer.test.ts
+++ b/packages/engine/src/services/audioMixer.test.ts
@@ -16,7 +16,7 @@ vi.mock("../utils/runFfmpeg.js", () => ({
runFfmpeg: runFfmpegMock,
}));
-import { processCompositionAudio } from "./audioMixer.js";
+import { parseAudioElements, processCompositionAudio } from "./audioMixer.js";
describe("processCompositionAudio", () => {
const tempDirs: string[] = [];
@@ -28,6 +28,44 @@ describe("processCompositionAudio", () => {
}
});
+ it("parses compiled duck keyframes as volume automation", () => {
+ const elements = parseAudioElements(`
+ elements
- const audioEls = document.querySelectorAll("audio[id][src]");
- for (const el of audioEls) {
- const id = el.getAttribute("id");
- const src = el.getAttribute("src");
- if (!id || !src) continue;
-
- const startAttr = el.getAttribute("data-start");
- const endAttr = el.getAttribute("data-end");
- const mediaStartAttr = el.getAttribute("data-media-start");
- const layerAttr = el.getAttribute("data-layer");
- const volumeAttr = el.getAttribute("data-volume");
-
- elements.push({
- id,
- src,
- start: startAttr ? parseFloat(startAttr) : 0,
- end: endAttr ? parseFloat(endAttr) : 0,
- mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
- layer: layerAttr ? parseInt(layerAttr) : 0,
- volume: volumeAttr ? parseFloat(volumeAttr) : 1.0,
- type: "audio",
- });
- }
-
- // Parse elements with data-has-audio="true"
- const videoEls = document.querySelectorAll('video[id][src][data-has-audio="true"]');
- for (const el of videoEls) {
- const id = el.getAttribute("id");
- const src = el.getAttribute("src");
- if (!id || !src) continue;
-
- const startAttr = el.getAttribute("data-start");
- const endAttr = el.getAttribute("data-end");
- const mediaStartAttr = el.getAttribute("data-media-start");
- const layerAttr = el.getAttribute("data-layer");
- const volumeAttr = el.getAttribute("data-volume");
-
- elements.push({
- id: `${id}-audio`,
- src,
- start: startAttr ? parseFloat(startAttr) : 0,
- end: endAttr ? parseFloat(endAttr) : 0,
- mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0,
- layer: layerAttr ? parseInt(layerAttr) : 0,
- volume: volumeAttr ? parseFloat(volumeAttr) : 1.0,
- type: "video",
- });
- }
-
- return elements;
-}
-
async function extractAudioFromVideo(
videoPath: string,
outputPath: string,
diff --git a/packages/producer/src/services/htmlCompiler.test.ts b/packages/producer/src/services/htmlCompiler.test.ts
index 6b6ff7fa6..66e72769a 100644
--- a/packages/producer/src/services/htmlCompiler.test.ts
+++ b/packages/producer/src/services/htmlCompiler.test.ts
@@ -15,6 +15,7 @@ import {
localizeRemoteFontFaces,
recompileWithResolutions,
} from "./htmlCompiler.js";
+import type { AudioVolumeKeyframe } from "@hyperframes/engine";
// ── collectExternalAssets ──────────────────────────────────────────────────
@@ -523,6 +524,56 @@ describe("detectShaderTransitionUsage", () => {
});
});
+function interpolateVolume(keyframes: AudioVolumeKeyframe[], time: number): number {
+ if (keyframes.length === 0) return 1;
+ let segment = 0;
+ while (segment < keyframes.length - 2 && time >= keyframes[segment + 1]!.time) {
+ segment += 1;
+ }
+ const a = keyframes[segment]!;
+ const b = keyframes[segment + 1] ?? a;
+ const span = b.time - a.time;
+ const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span));
+ return a.volume + (b.volume - a.volume) * progress;
+}
+
+function rmsVolume(keyframes: AudioVolumeKeyframe[], start: number, end: number): number {
+ let sumSquares = 0;
+ let count = 0;
+ for (let t = start; t <= end; t += 0.05) {
+ const volume = interpolateVolume(keyframes, t);
+ sumSquares += volume * volume;
+ count += 1;
+ }
+ return Math.sqrt(sumSquares / count);
+}
+
+describe("audio ducking compilation", () => {
+ it("passes a lower render-side RMS envelope during voice windows", async () => {
+ const projectDir = mkdtempSync(join(tmpdir(), "hf-audio-duck-"));
+ writeFileSync(
+ join(projectDir, "index.html"),
+ `
+
+`,
+ );
+
+ const compiled = await compileForRender(projectDir, join(projectDir, "index.html"), projectDir);
+ const music = compiled.audios.find((audio) => audio.id === "music");
+ if (!music?.volumeKeyframes) throw new Error("Missing music duck keyframes");
+
+ const beforeVoice = rmsVolume(music.volumeKeyframes, 0.1, 0.6);
+ const duringVoice = rmsVolume(music.volumeKeyframes, 1.1, 1.9);
+ const afterVoice = rmsVolume(music.volumeKeyframes, 3.0, 3.5);
+
+ expect(duringVoice).toBeLessThan(beforeVoice * 0.35);
+ expect(duringVoice).toBeLessThan(afterVoice * 0.35);
+ });
+});
+
describe("template-wrapped sub-composition media offsets", () => {
function writeTemplateWrappedProject(
hostAttrs: string,
diff --git a/packages/producer/src/services/htmlCompiler.ts b/packages/producer/src/services/htmlCompiler.ts
index 17dc05bcb..0233e1711 100644
--- a/packages/producer/src/services/htmlCompiler.ts
+++ b/packages/producer/src/services/htmlCompiler.ts
@@ -14,6 +14,7 @@ import { join, dirname, resolve, basename } from "path";
import { parseHTML } from "linkedom";
import {
compileTimingAttrs,
+ compileAudioDucking,
injectDurations,
extractResolvedMedia,
clampDurations,
@@ -258,6 +259,8 @@ async function compileHtmlFile(
compiledHtml = clampDurations(compiledHtml, clampList);
}
+ compiledHtml = compileAudioDucking(compiledHtml);
+
// Strip crossorigin from video elements: the render pipeline replaces them with
// injected frame images, so the browser never needs to load the source.
// Without this, videos with crossorigin="anonymous" targeting CORS-restricted
@@ -1884,7 +1887,7 @@ export async function recompileWithResolutions(
): Promise {
if (resolutions.length === 0) return compiled;
- const html = injectDurations(compiled.html, resolutions);
+ const html = compileAudioDucking(injectDurations(compiled.html, resolutions));
// Re-parse sub-compositions with the updated parent bounds
const {
diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts
index e1a0fb907..36064fde2 100644
--- a/packages/producer/src/services/render/stages/probeStage.ts
+++ b/packages/producer/src/services/render/stages/probeStage.ts
@@ -38,6 +38,7 @@ import {
initializeSession,
} from "@hyperframes/engine";
import { fpsToNumber } from "@hyperframes/core";
+import { multiplyVolumeKeyframeEnvelopes } from "@hyperframes/core/media-volume-envelope";
import type { CompiledComposition } from "../../htmlCompiler.js";
import {
discoverMediaFromBrowser,
@@ -367,9 +368,18 @@ export async function runProbeStage(input: ProbeStageInput): Promise 0
+ ? multiplyVolumeKeyframeEnvelopes({
+ sourceKeyframes: keyframes,
+ multiplierKeyframes: audio.volumeKeyframes,
+ trackStart: audio.start,
+ trackEnd: audio.end,
+ baseVolume: audio.volume ?? 1,
+ })
+ : keyframes;
log.info(`[Probe] Runtime audio volume automation: ${audio.id}`, {
- keyframeCount: keyframes.length,
+ keyframeCount: audio.volumeKeyframes.length,
});
}
}