Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/cli/src/commands/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1429,6 +1429,7 @@ function trackRenderMetrics(
deParallelRouter: perf?.drawElement?.parallelRouter,
dePreRouterWorkers: perf?.drawElement?.preRouterWorkers,
deGateReason: perf?.drawElement?.gateReason,
gpuRenderer: perf?.drawElement?.gpuRenderer,
deWorkerEncode: perf?.drawElement?.workerEncode,
deVerifyArmed: perf?.drawElement?.verifyArmed,
deVerifyChecked: perf?.drawElement?.verifyChecked,
Expand Down
9 changes: 9 additions & 0 deletions packages/cli/src/telemetry/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export interface RenderObservabilityTelemetryPayload {
captureDeWorkerInversion?: string;
captureDePreInversionWorkers?: number;
captureDeParallelRouter?: string;
captureDeGpuRenderer?: string;
captureDePreRouterWorkers?: number;
captureDeSelfVerifyFallback?: boolean;
captureDeFallbackReason?: string;
Expand Down Expand Up @@ -108,6 +109,7 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
de_worker_inversion: props.captureDeWorkerInversion,
de_pre_inversion_workers: props.captureDePreInversionWorkers,
de_parallel_router: props.captureDeParallelRouter,
gpu_renderer: props.captureDeGpuRenderer,
de_pre_router_workers: props.captureDePreRouterWorkers,
de_self_verify_fallback: props.captureDeSelfVerifyFallback,
de_fallback_reason: props.captureDeFallbackReason,
Expand Down Expand Up @@ -183,6 +185,8 @@ export function trackRenderComplete(
deParallelRouter?: string;
dePreRouterWorkers?: number;
deGateReason?: string;
/** Low-cardinality GPU bucket from DE session init (`<backend>/<vendor>`, e.g. `d3d11/nvidia`). */
gpuRenderer?: string;
deWorkerEncode?: boolean;
deVerifyArmed?: number;
deVerifyChecked?: number;
Expand Down Expand Up @@ -280,6 +284,7 @@ export function trackRenderComplete(
de_parallel_router: props.deParallelRouter,
de_pre_router_workers: props.dePreRouterWorkers,
de_gate_reason: props.deGateReason,
gpu_renderer: props.gpuRenderer,
de_worker_encode: props.deWorkerEncode,
de_verify_armed: props.deVerifyArmed,
de_verify_checked: props.deVerifyChecked,
Expand Down Expand Up @@ -374,6 +379,10 @@ export function trackRenderError(
elapsed_ms: props.elapsedMs,
peak_memory_mb: props.peakMemoryMb,
memory_free_mb: props.memoryFreeMb,
// gpu_renderer arrives via renderObservabilityEventProperties below:
// on the failure path perfSummary is never built, so live capture
// observability is the only source. Backend attribution matters MOST
// here — a win32 D3D11 crash is what the rollout is watching for.
...renderObservabilityEventProperties(props),
},
props.distinctId,
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/telemetry/renderObservability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ export function renderObservabilityTelemetryPayload(
captureDeWorkerInversion: capture.deWorkerInversion,
captureDePreInversionWorkers: capture.dePreInversionWorkers,
captureDeParallelRouter: capture.deParallelRouter,
captureDeGpuRenderer: capture.deGpuRenderer,
captureDePreRouterWorkers: capture.dePreRouterWorkers,
captureDeSelfVerifyFallback: capture.deSelfVerifyFallback,
captureDeFallbackReason: capture.deFallbackReason,
Expand Down
152 changes: 94 additions & 58 deletions packages/engine/src/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { join } from "node:path";
import { tmpdir } from "node:os";
import {
resolveConfig,
resolveDefaultDrawElement,
DEFAULT_CONFIG,
scaleProtocolTimeoutForComposition,
shouldClampToScreenshotForConcreteGpu,
Expand Down Expand Up @@ -232,6 +233,40 @@ describe("resolveConfig", () => {
});
});

describe("resolveDefaultDrawElement (pure host clamp)", () => {
const base = {
useDrawElement: true,
explicitOptIn: false,
browserGpuMode: "hardware" as const,
workerEncode: true,
};
it("engages on darwin and win32, not linux", () => {
expect(resolveDefaultDrawElement({ ...base, platform: "darwin" })).toBe(true);
expect(resolveDefaultDrawElement({ ...base, platform: "win32" })).toBe(true);
expect(resolveDefaultDrawElement({ ...base, platform: "linux" })).toBe(false);
});
it("software GPU clamps off even on supported platforms", () => {
expect(
resolveDefaultDrawElement({ ...base, platform: "win32", browserGpuMode: "software" }),
).toBe(false);
});
it("explicit opt-in overrides platform and GPU clamps", () => {
expect(
resolveDefaultDrawElement({
...base,
explicitOptIn: true,
platform: "linux",
browserGpuMode: "software",
}),
).toBe(true);
});
it("no worker-encode (no verify net) clamps the default off", () => {
expect(resolveDefaultDrawElement({ ...base, platform: "darwin", workerEncode: false })).toBe(
false,
);
});
});

describe("useDrawElement (PRODUCER_EXPERIMENTAL_FAST_CAPTURE)", () => {
it("default is clamped off on software-GPU hosts (page-side compositing preserved)", () => {
setEnv("PRODUCER_BROWSER_GPU_MODE", "software");
Expand All @@ -242,21 +277,20 @@ describe("resolveConfig", () => {
expect(config.enablePageSideCompositing).toBe(true);
});

it("default engages on macOS with a hardware-GPU browser", () => {
setEnv("PRODUCER_BROWSER_GPU_MODE", "hardware");
unsetEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE");
unsetEnv("HF_DE_WORKER_ENCODE");
const config = resolveConfig();
expect(config.useDrawElement).toBe(process.platform === "darwin");
});

it("default engages on macOS with auto GPU mode (the stock CLI path)", () => {
setEnv("PRODUCER_BROWSER_GPU_MODE", "auto");
unsetEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE");
unsetEnv("HF_DE_WORKER_ENCODE");
const config = resolveConfig();
expect(config.useDrawElement).toBe(process.platform === "darwin");
});
// win32 opened 2026-07-27 (was darwin-only): ~206k non-CI hardware-GPU
// Windows renders / 30d sat on the screenshot path behind the old clamp.
// "auto" is the stock CLI path; both must pass the platform clamp.
for (const gpuMode of ["hardware", "auto"] as const) {
it(`default engages on macOS/Windows with ${gpuMode} GPU mode`, () => {
setEnv("PRODUCER_BROWSER_GPU_MODE", gpuMode);
unsetEnv("PRODUCER_EXPERIMENTAL_FAST_CAPTURE");
unsetEnv("HF_DE_WORKER_ENCODE");
const config = resolveConfig();
expect(config.useDrawElement).toBe(
process.platform === "darwin" || process.platform === "win32",
);
});
}

it("default requires worker-encode (the verified drain)", () => {
setEnv("PRODUCER_BROWSER_GPU_MODE", "hardware");
Expand Down Expand Up @@ -624,40 +658,58 @@ describe("resolveConfig", () => {
Object.defineProperty(process, "platform", { value: originalPlatform, configurable: true });
});

it("auto-disables on win32 + software-GPU + workers=1 + no user opt-in", () => {
setPlatform("win32");
setEnv("PRODUCER_BROWSER_GPU_MODE", "software");
setEnv("PRODUCER_MAX_WORKERS", "1");
/**
* Shared setup/assert for the win32 software-GPU compound: fixed common
* env (low-memory on, no explicit streaming opt-in), variable platform /
* gpu / workers, then assert whether the auto-disable fired.
*/
function expectCompoundOutcome(opts: {
platform: NodeJS.Platform;
gpuMode?: string;
workers?: string;
autoDisabled: boolean;
}): void {
setPlatform(opts.platform);
if (opts.gpuMode === undefined) unsetEnv("PRODUCER_BROWSER_GPU_MODE");
else setEnv("PRODUCER_BROWSER_GPU_MODE", opts.gpuMode);
if (opts.workers === undefined) unsetEnv("PRODUCER_MAX_WORKERS");
else setEnv("PRODUCER_MAX_WORKERS", opts.workers);
setEnv("PRODUCER_LOW_MEMORY_MODE", "true");
unsetEnv("PRODUCER_ENABLE_STREAMING_ENCODE");

const config = resolveConfig();
expect(config.enableStreamingEncode).toBe(false);
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBe(true);
expect(config.enableStreamingEncode).toBe(!opts.autoDisabled);
if (opts.autoDisabled) {
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBe(true);
} else {
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBeUndefined();
}
}

it("auto-disables on win32 + software-GPU + workers=1 + no user opt-in", () => {
expectCompoundOutcome({
platform: "win32",
gpuMode: "software",
workers: "1",
autoDisabled: true,
});
});

it("leaves streaming-encode on when platform is linux", () => {
setPlatform("linux");
setEnv("PRODUCER_BROWSER_GPU_MODE", "software");
setEnv("PRODUCER_MAX_WORKERS", "1");
setEnv("PRODUCER_LOW_MEMORY_MODE", "true");
unsetEnv("PRODUCER_ENABLE_STREAMING_ENCODE");

const config = resolveConfig();
expect(config.enableStreamingEncode).toBe(true);
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBeUndefined();
expectCompoundOutcome({
platform: "linux",
gpuMode: "software",
workers: "1",
autoDisabled: false,
});
});

it("leaves streaming-encode on when workers > 1", () => {
setPlatform("win32");
setEnv("PRODUCER_BROWSER_GPU_MODE", "software");
setEnv("PRODUCER_MAX_WORKERS", "4");
setEnv("PRODUCER_LOW_MEMORY_MODE", "true");
unsetEnv("PRODUCER_ENABLE_STREAMING_ENCODE");

const config = resolveConfig();
expect(config.enableStreamingEncode).toBe(true);
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBeUndefined();
expectCompoundOutcome({
platform: "win32",
gpuMode: "software",
workers: "4",
autoDisabled: false,
});
});

it("respects explicit env opt-in (PRODUCER_ENABLE_STREAMING_ENCODE=true) on the compound", () => {
Expand Down Expand Up @@ -703,32 +755,16 @@ describe("resolveConfig", () => {
// --low-memory-mode implies screenshot capture. Compound applies even
// if browserGpuMode is not literal "software" (defense-in-depth: matches
// the OR semantics in `softwareGpuForced`).
setPlatform("win32");
unsetEnv("PRODUCER_BROWSER_GPU_MODE");
unsetEnv("PRODUCER_DISABLE_GPU");
setEnv("PRODUCER_MAX_WORKERS", "1");
setEnv("PRODUCER_LOW_MEMORY_MODE", "true");
unsetEnv("PRODUCER_ENABLE_STREAMING_ENCODE");

const config = resolveConfig();
expect(config.enableStreamingEncode).toBe(false);
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBe(true);
expectCompoundOutcome({ platform: "win32", workers: "1", autoDisabled: true });
});

it("does not trigger when concurrency is 'auto' (workers not explicitly pinned)", () => {
// Config-layer sees `concurrency === "auto"`, not a number — the
// helper's numeric workers check treats NaN as "unknown, don't trigger".
// Downstream workers may still resolve to 1 via lowMemoryMode, but the
// config-time clamp is deliberately conservative.
setPlatform("win32");
setEnv("PRODUCER_BROWSER_GPU_MODE", "software");
unsetEnv("PRODUCER_MAX_WORKERS");
setEnv("PRODUCER_LOW_MEMORY_MODE", "true");
unsetEnv("PRODUCER_ENABLE_STREAMING_ENCODE");

const config = resolveConfig();
expect(config.enableStreamingEncode).toBe(true);
expect(config.streamingEncodeAutoDisabledOnWin32Compound).toBeUndefined();
expectCompoundOutcome({ platform: "win32", gpuMode: "software", autoDisabled: false });
});
});

Expand Down
94 changes: 67 additions & 27 deletions packages/engine/src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ export interface EngineConfig {
/**
* Use drawElementImage for frame capture (requires the CanvasDrawElement
* Chrome flag, added globally in buildChromeArgs). Default ON, clamped in
* `resolveConfig` to hosts where it can actually engage (macOS + hardware-GPU
* browser); compile/init gates and the runtime self-verification net route
* `resolveConfig` to hosts where it can actually engage (macOS or Windows +
* hardware-GPU browser); compile/init gates and the runtime self-verification net route
* incompatible or damaged renders back to screenshot capture.
* Kill switch: `PRODUCER_EXPERIMENTAL_FAST_CAPTURE=false` (or the CLI
* `--experimental-fast-capture=false`).
Expand Down Expand Up @@ -708,6 +708,39 @@ function memoryAdaptiveCacheBytesMb(): number {
* Env vars provide backward compatibility during migration; explicit config
* takes precedence over everything.
*/
/**
* Platforms where default-on drawElement may engage: macOS (Metal-ANGLE,
* the original validated envelope) and Windows (D3D11-ANGLE, opened
* 2026-07-27 — see the clamp comment above resolveDefaultDrawElement's call
* site). Linux is excluded: that fleet is headless/Docker SwiftShader.
* Internal — the exported `resolveDefaultDrawElement` is the tested surface.
*/
function isDrawElementPlatform(platform: NodeJS.Platform): boolean {
return platform === "darwin" || platform === "win32";
}

/**
* Default-on drawElement host clamp. An explicit opt-in always wins (attempt
* DE, let the init-time gates route away — debugging relies on it). Otherwise
* DE stays on only where it can actually engage — a supported platform with a
* non-software-GPU browser — AND with worker-encode enabled: the runtime
* self-verification net lives in the worker-encode drain (the serial path has
* only the blank guard), so a default-on session without it would ship
* unverified drawElement frames. Pure; exported for tests.
*/
export function resolveDefaultDrawElement(args: {
useDrawElement: boolean;
explicitOptIn: boolean;
platform: NodeJS.Platform;
browserGpuMode: EngineConfig["browserGpuMode"];
workerEncode: boolean;
}): boolean {
if (!args.useDrawElement) return false;
if (args.explicitOptIn) return true;
if (!isDrawElementPlatform(args.platform) || args.browserGpuMode === "software") return false;
return args.workerEncode;
}

export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
const env = (key: string): string | undefined => process.env[key];
const envNum = (key: string, fallback: number): number => {
Expand Down Expand Up @@ -857,38 +890,45 @@ export function resolveConfig(overrides?: Partial<EngineConfig>): EngineConfig {
};

// Default-on drawElement is clamped to hosts where it can actually engage
// (macOS with a non-software-GPU browser; SwiftShader drops transparent
// sub-layers — crbug 521434899). "auto" passes the clamp: the stock CLI
// resolves GPU mode to auto, which probes to hardware on real Macs — and if
// it resolves to software after all, the SwiftShader init-time gate still
// routes the session to the screenshot baseline. Without the clamp, the
// default would needlessly disable page-side shader compositing (below) on
// Linux/Docker hosts where DE never runs. An EXPLICIT opt-in (env or caller override)
// skips the clamp and keeps the old semantics — attempt DE, let the
// init-time gates route away — which debugging relies on.
// (macOS or Windows with a non-software-GPU browser; SwiftShader drops
// transparent sub-layers — crbug 521434899). "auto" passes the clamp: the
// stock CLI resolves GPU mode to auto, which probes to hardware on real
// Macs/PCs — and if it resolves to software after all, the SwiftShader
// init-time gate still routes the session to the screenshot baseline.
// Without the clamp, the default would needlessly disable page-side shader
// compositing (below) on Linux/Docker hosts where DE never runs. An
// EXPLICIT opt-in (env or caller override) skips the clamp and keeps the
// old semantics — attempt DE, let the init-time gates route away — which
// debugging relies on.
//
// win32 opened 2026-07-27: telemetry showed ~206k non-CI hardware-GPU
// Windows renders / 30d (~78% of the win32 fleet) held on the slow
// screenshot path by the darwin-only clamp — the second-largest perf
// population after macOS. The mechanism is platform-neutral (the Chrome
// flag ships everywhere); darwin-only was a validation envelope, not an
// architectural limit. Opening it rides the same per-render safety
// contract macOS shipped with in v0.7.38: compile/init gates +
// worker-encode self-verify + screenshot fallback catch damage per
// render, and `gpu_renderer` telemetry (captured at DE session init)
// segments the D3D11/ANGLE cohort by GPU vendor so backend-specific
// damage clusters are attributable. Kill switches unchanged
// (PRODUCER_EXPERIMENTAL_FAST_CAPTURE=false; per-render --workers).
// Linux stays excluded: the fleet there is headless/Docker SwiftShader.
const explicitDrawElementOptIn =
env("PRODUCER_EXPERIMENTAL_FAST_CAPTURE") === "true" || overrides?.useDrawElement === true;
if (
merged.useDrawElement &&
!explicitDrawElementOptIn &&
!(process.platform === "darwin" && merged.browserGpuMode !== "software")
) {
merged.useDrawElement = false;
}
// The runtime self-verification net lives in the worker-encode drain — the
// serial drawElement path has only the blank guard. Default-on drawElement
// therefore requires worker-encode; disabling HF_DE_WORKER_ENCODE without an
// explicit drawElement opt-in falls back to the screenshot baseline rather
// than shipping unverified drawElement frames.
if (merged.useDrawElement && !explicitDrawElementOptIn && !merged.enableDrawElementWorkerEncode) {
merged.useDrawElement = false;
}
merged.useDrawElement = resolveDefaultDrawElement({
useDrawElement: merged.useDrawElement,
explicitOptIn: explicitDrawElementOptIn,
platform: process.platform,
browserGpuMode: merged.browserGpuMode,
workerEncode: merged.enableDrawElementWorkerEncode,
});

// Software GPU implies screenshot capture.
//
// Two existing platform gates already do most of the work: `browserManager`
// only launches BeginFrame on Linux + chrome-headless-shell + !forceScreenshot,
// and the DE clamp above turns off `useDrawElement` on non-(darwin +
// and the DE clamp above turns off `useDrawElement` on non-((darwin|win32) +
// non-software) hosts. Setting `forceScreenshot` here layers defense-in-depth
// on top:
//
Expand Down
Loading
Loading