From 3da31e399b498d0ab6b1ce32b008b8b10cdcd2dc Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Mon, 27 Jul 2026 22:51:03 -0700 Subject: [PATCH 1/3] feat(producer): lower parallel-DE router floor to 700 frames + power-state telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HF_DE_PARALLEL_MIN_FRAMES default 2000 -> 700, re-calibrated by a controlled crossover sweep (fixed content-per-frame, three synthetic profiles x {350..3000f} x {single,par2,par3} x 3 reps, resolved worker counts and capture modes verified per run): par3 beats single at EVERY size in every profile — +17-21% at 700f rising to +28-34% at 3000f. That includes a 24-sub-composition profile built specifically to reproduce the 'workers re-pay init' failure the original 2000 floor guarded against (92k tweens, ~2.5s pollSubCompositionTimelines per worker): workers initialize concurrently, so duplicated init costs CPU, not wall-clock, and the comp still parallelizes +19% at 700f. Below ~700f the win thins toward +10% while paying three hardware-GPU browsers, so a floor remains. par2 loses to par3 in every cell of every profile — the router's existing 3-worker pin is confirmed, not changed. Harness: plans/drawelement-fast-capture/de-crossover-bench.sh (docs repo). Also adds on_battery / low_power_mode to render_complete and render_error. The DE fleet is macOS laptops, and bench sweeps on an M4 Pro caught the SAME render flipping between ~9.6 and ~17.2 ms/frame power-management regimes with no existing telemetry signal to segment by — the router soak reading this change needs that dimension to interpret perf on the machines users actually render on. Sampled per event (volatile), pmset-based, darwin-only, null-safe on failure. Router stays default-off behind HF_DE_PARALLEL_ROUTER; this tunes what it will do when the soak clears it to flip. Co-Authored-By: Claude Opus 5 (1M context) --- packages/cli/src/telemetry/events.ts | 16 ++++++ packages/cli/src/telemetry/system.test.ts | 51 ++++++++++++++++++ packages/cli/src/telemetry/system.ts | 54 +++++++++++++++++++ .../src/services/renderOrchestrator.ts | 30 ++++++++--- 4 files changed, 143 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index 9e100ae3ee..bfe9056175 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -3,6 +3,20 @@ import type { SubTimelineWaitOutcome } from "@hyperframes/engine"; import { FEEDBACK_RATING_SCALE } from "../utils/feedbackRating.js"; import { flush, trackEvent } from "./client.js"; import { readConfig } from "./config.js"; +import { getPowerState } from "./system.js"; + +// Power state is volatile (a laptop docks/undocks mid-session), so it is +// sampled per render event rather than cached with SystemMeta. Attached to +// render_complete AND render_error: the DE fleet is macOS laptops whose +// power management shifts render perf ~1.8x with no other telemetry signal, +// and perf/soak analysis needs to segment by it (see getPowerState). +function powerStateFields(): { on_battery?: boolean; low_power_mode?: boolean } { + const power = getPowerState(); + return { + on_battery: power.on_battery ?? undefined, + low_power_mode: power.low_power_mode ?? undefined, + }; +} // run_id is attached only when the orchestrator set HYPERFRAMES_RUN_ID — an // absent property, never null/"" (PostHog treats those as real values). @@ -277,6 +291,7 @@ export function trackRenderComplete( de_blank_recaptures: props.deBlankRecaptures, de_boundary_frames: props.deBoundaryFrames, de_ncpr_fallbacks: props.deNcprFallbacks, + ...powerStateFields(), source: props.source ?? "cli", composition_duration_ms: props.compositionDurationMs, composition_width: props.compositionWidth, @@ -356,6 +371,7 @@ export function trackRenderError( elapsed_ms: props.elapsedMs, peak_memory_mb: props.peakMemoryMb, memory_free_mb: props.memoryFreeMb, + ...powerStateFields(), ...renderObservabilityEventProperties(props), }, props.distinctId, diff --git a/packages/cli/src/telemetry/system.test.ts b/packages/cli/src/telemetry/system.test.ts index 375ca15386..c6e90df167 100644 --- a/packages/cli/src/telemetry/system.test.ts +++ b/packages/cli/src/telemetry/system.test.ts @@ -110,3 +110,54 @@ describe("getAvailableMemoryMb", () => { expect(getAvailableMemoryMb()).toBe(6144); }); }); + +describe("power state (laptop fleet segmentation)", () => { + it("parsePmsetPowerSource reads battery vs AC", async () => { + const { parsePmsetPowerSource } = await import("./system.js"); + expect(parsePmsetPowerSource("Now drawing from 'Battery Power'\n -InternalBattery-0")).toBe( + true, + ); + expect(parsePmsetPowerSource("Now drawing from 'AC Power'\n -InternalBattery-0")).toBe(false); + expect(parsePmsetPowerSource("garbage output")).toBe(null); + }); + + it("getPowerState samples pmset on darwin", async () => { + vi.doMock("node:os", async () => ({ + ...(await vi.importActual("node:os")), + platform: () => "darwin", + })); + vi.doMock("node:child_process", async () => ({ + ...(await vi.importActual("node:child_process")), + execSync: (cmd: string) => + cmd === "pmset -g batt" + ? "Now drawing from 'Battery Power'\n" + : "SleepDisabled 0\n lowpowermode 1\n", + })); + const { getPowerState } = await import("./system.js"); + expect(getPowerState()).toEqual({ on_battery: true, low_power_mode: true }); + }); + + it("getPowerState returns nulls off-darwin (no guessing)", async () => { + vi.doMock("node:os", async () => ({ + ...(await vi.importActual("node:os")), + platform: () => "linux", + })); + const { getPowerState } = await import("./system.js"); + expect(getPowerState()).toEqual({ on_battery: null, low_power_mode: null }); + }); + + it("getPowerState survives pmset failure with nulls", async () => { + vi.doMock("node:os", async () => ({ + ...(await vi.importActual("node:os")), + platform: () => "darwin", + })); + vi.doMock("node:child_process", async () => ({ + ...(await vi.importActual("node:child_process")), + execSync: () => { + throw new Error("pmset: command failed"); + }, + })); + const { getPowerState } = await import("./system.js"); + expect(getPowerState()).toEqual({ on_battery: null, low_power_mode: null }); + }); +}); diff --git a/packages/cli/src/telemetry/system.ts b/packages/cli/src/telemetry/system.ts index d8d9bacf54..20481ec5bb 100644 --- a/packages/cli/src/telemetry/system.ts +++ b/packages/cli/src/telemetry/system.ts @@ -184,6 +184,60 @@ export function getFreeDiskMb(path: string = "."): number | null { } } +export interface PowerState { + /** true = running on battery, false = external power, null = undetectable. */ + on_battery: boolean | null; + /** macOS Low Power Mode; null off-darwin or undetectable. */ + low_power_mode: boolean | null; +} + +/** + * Parse `pmset -g batt` output for the power source line. Exported for tests. + * Example first line: `Now drawing from 'Battery Power'`. + */ +export function parsePmsetPowerSource(raw: string): boolean | null { + const m = raw.match(/Now drawing from '([^']+)'/); + if (!m) return null; + return m[1] === "Battery Power"; +} + +/** + * Sample the machine's power state. Volatile — sample at event time, never + * cache alongside SystemMeta. + * + * Why this exists: the DE fast path only engages on macOS + hardware GPU, so + * the render fleet is overwhelmingly laptops, and laptop perf is + * power-managed — bench sweeps on an M4 Pro showed the SAME render flipping + * between ~9.6 and ~17.2 ms/frame regimes with no load/thermal signal to + * explain it. Without a power-state dimension on render telemetry those + * regimes are indistinguishable noise; with it, perf distributions (and the + * DE parallel-router soak) can be segmented by the machine state real users + * actually render in. + */ +export function getPowerState(): PowerState { + if (platform() !== "darwin") { + // Linux laptops exist but the DE fleet is darwin; don't guess elsewhere. + return { on_battery: null, low_power_mode: null }; + } + let on_battery: boolean | null = null; + let low_power_mode: boolean | null = null; + try { + on_battery = parsePmsetPowerSource( + execSync("pmset -g batt", { encoding: "utf-8", timeout: 2000 }), + ); + } catch { + // pmset missing/slow — leave null rather than fail telemetry. + } + try { + const raw = execSync("pmset -g", { encoding: "utf-8", timeout: 2000 }); + const m = raw.match(/lowpowermode\s+(\d)/); + if (m) low_power_mode = m[1] === "1"; + } catch { + // Same: absence of the reading is itself acceptable. + } + return { on_battery, low_power_mode }; +} + /** * Get available memory in MB, accounting for OS-level page caching. * diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index e8d37dbf96..1bd7ae4c6b 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1305,9 +1305,14 @@ export function resolveInversionRetryPlan(args: { * but the decision itself stays gated behind its own flag pending the * telemetry soak (revert rate, de_verify_min_db distribution) on real wild * traffic — there is currently none, since nothing routes here by default. - * Takes priority over the single-worker inversion when both would fire (a - * higher minFrames than HF_DE_SINGLE_MIN_FRAMES is the intended shape: this - * only picks up the long tail the inversion's own benchmark didn't cover). + * Takes priority over the single-worker inversion when both would fire. + * Re-calibrated 2026-07-27: a controlled crossover sweep (three content + * profiles including a genuinely init-expensive 24-sub-composition comp; + * worker counts and capture modes verified per run) found par3 > single at + * every size from 350f up in every profile — workers init concurrently, so + * per-worker init duplication costs CPU, not wall-clock. minFrames therefore + * dropped below the inversion's threshold (700 vs 900): where both fire, + * parallel wins over the inversion's single-worker pick (+17–21% at 700f). */ export function shouldPreferParallelDrawElement(args: { workerCount: number; @@ -2310,18 +2315,27 @@ async function executeRenderPipeline(input: { process.env.HF_DE_PARALLEL_STREAM === "true", }); // DE parallel-router eligibility — see shouldPreferParallelDrawElement. - // Default-off (HF_DE_PARALLEL_ROUTER); HF_DE_PARALLEL_MIN_FRAMES defaults - // higher than the single-worker inversion's threshold since it targets - // the long tail the inversion's own benchmark didn't cover. + // Default-off (HF_DE_PARALLEL_ROUTER); HF_DE_PARALLEL_MIN_FRAMES default + // 700, re-calibrated 2026-07-27 from the original safe-high 2000. A + // controlled frame-count sweep (fixed content-per-frame, three synthetic + // profiles × {350..3000f} × {single,par2,par3} × 3 reps, worker counts + + // capture modes verified per run) showed par3 beating single at EVERY + // size in every profile — +17–21% at 700f rising to +28–34% at 3000f — + // including a 24-sub-composition profile built to reproduce the + // "workers re-pay init" failure (92k tweens, ~2.5s + // pollSubCompositionTimelines per worker): workers init CONCURRENTLY, so + // duplicated init costs CPU, not wall-clock. Below ~700f the win thins + // toward ~+10% while still paying 3 hardware-GPU browsers, so the floor + // stays. Harness: plans/drawelement-fast-capture/de-crossover-bench.sh. const deParallelRouterEnabled = process.env.HF_DE_PARALLEL_ROUTER === "true"; const deParallelMinFramesRaw = process.env.HF_DE_PARALLEL_MIN_FRAMES; const deParallelMinFramesNum = deParallelMinFramesRaw === undefined || deParallelMinFramesRaw.trim() === "" - ? 2000 + ? 700 : Number(deParallelMinFramesRaw); const deParallelMinFrames = Number.isFinite(deParallelMinFramesNum) ? deParallelMinFramesNum - : 2000; + : 700; // RAM floor default 24 GB: the wild black-slab report was a 16 GB // machine; every clean routed cohort in telemetry so far is >=24 GB. // HF_DE_PARALLEL_MIN_MEM_MB overrides (0 disables the guard). From b99c8038983e2c0bb2545e8eaf5fcd7c4f9abe03 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 28 Jul 2026 00:59:22 -0700 Subject: [PATCH 2/3] fix(producer): guard pmset behind shouldTrack + don't pin workers without streaming (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the floor/telemetry PR: 1. powerStateFields() is spread into the properties object at the CALL SITE, so it ran before trackEvent's own `if (!shouldTrack()) return` guard — telemetry-disabled installs paid two blocking `pmset` subprocess spawns per render for an event that was then discarded. Now short-circuits on shouldTrack() (memoized, so no cost on the tracked path). Regression test asserts pmset is not sampled when telemetry is off; fault-injection verified it fails without the guard. 2. The DE parallel router pinned workerCount to 3 and skipped calibration even when verified parallel DE STREAMING — the entire reason for the pin — could not run for that render. The common case is a composition over streamingEncodeMaxDurationSeconds (240 s default): the duration cap disables streaming before the router's force flag is consulted, so the render got a hard-coded 3 workers chosen by a benchmark for a path it was not on, instead of the calibrated count. shouldPreferParallelDrawElement now takes parallelStreamingAvailable and withholds the bet without it. Co-Authored-By: Claude Opus 5 (1M context) --- packages/cli/src/telemetry/events.test.ts | 36 +++++++++++++++++++ packages/cli/src/telemetry/events.ts | 10 +++++- .../src/services/renderOrchestrator.test.ts | 11 ++++++ .../src/services/renderOrchestrator.ts | 20 +++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 9ee0b8c040..d614f67234 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -2,9 +2,20 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; const trackEvent = vi.fn(); const flush = vi.fn(() => Promise.resolve()); +const shouldTrack = vi.fn(() => true); vi.mock("./client.js", () => ({ trackEvent: (...args: unknown[]) => trackEvent(...args), flush: () => flush(), + shouldTrack: () => shouldTrack(), +})); + +// Power state shells out to `pmset`; spy so tests can assert it is NOT +// sampled for opted-out installs (the fields are built at the call site, +// before trackEvent's own shouldTrack guard). +const getPowerState = vi.fn(() => ({ on_battery: true, low_power_mode: false })); +vi.mock("./system.js", async () => ({ + ...(await vi.importActual("./system.js")), + getPowerState: () => getPowerState(), })); // identifyUser reads the install anonymousId; pin it so the $identify alias is @@ -653,3 +664,28 @@ describe("auth login telemetry events", () => { expect(trackEvent).not.toHaveBeenCalled(); }); }); + +describe("power-state sampling respects the telemetry opt-out", () => { + beforeEach(() => { + getPowerState.mockClear(); + shouldTrack.mockReturnValue(true); + }); + + it("samples power state for a tracked render", () => { + trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false }); + expect(getPowerState).toHaveBeenCalled(); + const props = trackEvent.mock.calls.at(-1)?.[1] as Record; + expect(props.on_battery).toBe(true); + expect(props.low_power_mode).toBe(false); + }); + + it("does NOT spawn pmset when telemetry is disabled", () => { + // Regression: powerStateFields() is spread into the properties object at + // the call site, so it runs BEFORE trackEvent's `if (!shouldTrack())` + // guard — an opted-out install would otherwise pay two blocking + // subprocess spawns per render for an event that is then discarded. + shouldTrack.mockReturnValue(false); + trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false }); + expect(getPowerState).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index bfe9056175..08c2cd36fd 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -1,7 +1,7 @@ import { redactTelemetryString, type OutputResolutionIssueKind } from "@hyperframes/core"; import type { SubTimelineWaitOutcome } from "@hyperframes/engine"; import { FEEDBACK_RATING_SCALE } from "../utils/feedbackRating.js"; -import { flush, trackEvent } from "./client.js"; +import { flush, shouldTrack, trackEvent } from "./client.js"; import { readConfig } from "./config.js"; import { getPowerState } from "./system.js"; @@ -10,7 +10,15 @@ import { getPowerState } from "./system.js"; // render_complete AND render_error: the DE fleet is macOS laptops whose // power management shifts render perf ~1.8x with no other telemetry signal, // and perf/soak analysis needs to segment by it (see getPowerState). +// +// shouldTrack() is checked HERE, not just inside trackEvent: this helper is +// spread into the properties object at the CALL SITE, so it runs before +// trackEvent's own `if (!shouldTrack()) return` guard. Without this an +// opted-out install would still pay two blocking `pmset` subprocess spawns +// per render for an event that is then discarded (review finding). +// shouldTrack() memoizes, so this costs nothing on the tracked path. function powerStateFields(): { on_battery?: boolean; low_power_mode?: boolean } { + if (!shouldTrack()) return {}; const power = getPowerState(); return { on_battery: power.on_battery ?? undefined, diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index e900b68f21..ff34b7e17f 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -1842,6 +1842,7 @@ describe("shouldPreferParallelDrawElement (DE parallel router)", () => { probeDeGated: false, experimentalParallelDeOptIn: false, routerEnabled: true, + parallelStreamingAvailable: true, totalMemoryMb: 32768, minMemoryMb: 24576, }; @@ -1850,6 +1851,16 @@ describe("shouldPreferParallelDrawElement (DE parallel router)", () => { expect(shouldPreferParallelDrawElement(eligible)).toBe(true); }); + it("withholds the bet when parallel streaming can't run (e.g. over the duration cap)", () => { + // The router pins workerCount to 3 and skips calibration to serve the + // verified parallel DE STREAMING path. If streaming is off for this + // render — the >240s duration cap is the common case — firing would pay + // the whole cost of the pin for none of the benefit. + expect( + shouldPreferParallelDrawElement({ ...eligible, parallelStreamingAvailable: false }), + ).toBe(false); + }); + it("withholds the parallel bet below the RAM floor (16 GB black-slab report)", () => { expect(shouldPreferParallelDrawElement({ ...eligible, totalMemoryMb: 16384 })).toBe(false); }); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 1bd7ae4c6b..79f4e6494e 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1331,6 +1331,16 @@ export function shouldPreferParallelDrawElement(args: { experimentalParallelDeOptIn: boolean; /** HF_DE_PARALLEL_ROUTER === "true" — the router's own kill switch, default off. */ routerEnabled: boolean; + /** + * Whether verified parallel DE STREAMING can actually run for this render + * (`shouldUseStreamingEncode` at the router's worker count with + * forceParallelStream). The router's entire value is that path; without it + * firing would pin workerCount to 3 and skip calibration while delivering + * none of the benefit — e.g. a composition longer than + * `streamingEncodeMaxDurationSeconds` (240 s default), where the duration + * cap disables streaming before the router's force flag is consulted. + */ + parallelStreamingAvailable: boolean; /** Machine RAM (os.totalmem, MB). */ totalMemoryMb: number; /** RAM floor for routing; <=0 disables the guard. */ @@ -1338,6 +1348,7 @@ export function shouldPreferParallelDrawElement(args: { }): boolean { return ( args.routerEnabled && + args.parallelStreamingAvailable && args.workerCount > 1 && typeof args.requestedWorkers !== "number" && args.useDrawElement && @@ -2366,6 +2377,15 @@ async function executeRenderPipeline(input: { process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true" || process.env.HF_DE_PARALLEL_STREAM === "true", routerEnabled: deParallelRouterEnabled, + // Router pins 3 workers for the streaming path; don't pin when the + // duration cap (or any other streaming gate) would turn that path off. + parallelStreamingAvailable: shouldUseStreamingEncode( + cfg, + outputFormat, + 3, + job.duration, + true, + ), totalMemoryMb: Math.round(totalmem() / (1024 * 1024)), minMemoryMb: deParallelMinMemoryMb, }); From cddc90ae3719329f23567341814fe3bcae358ff1 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 28 Jul 2026 01:29:58 -0700 Subject: [PATCH 3/3] fix(cli): add required gpu prop to power-state test calls trackRenderComplete requires `gpu: boolean`; the two new opt-out test calls omitted it, failing Typecheck in CI. The fix already existed on the stacked branch, so only this base branch was broken. Co-Authored-By: Claude Opus 5 (1M context) --- packages/cli/src/telemetry/events.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index d614f67234..029471d717 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -672,7 +672,7 @@ describe("power-state sampling respects the telemetry opt-out", () => { }); it("samples power state for a tracked render", () => { - trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false }); + trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false, gpu: false }); expect(getPowerState).toHaveBeenCalled(); const props = trackEvent.mock.calls.at(-1)?.[1] as Record; expect(props.on_battery).toBe(true); @@ -685,7 +685,7 @@ describe("power-state sampling respects the telemetry opt-out", () => { // guard — an opted-out install would otherwise pay two blocking // subprocess spawns per render for an event that is then discarded. shouldTrack.mockReturnValue(false); - trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false }); + trackRenderComplete({ durationMs: 1, fps: 30, quality: "high", docker: false, gpu: false }); expect(getPowerState).not.toHaveBeenCalled(); }); });