From 59d9204b57c7df507d569fb6e9920a1374672f16 Mon Sep 17 00:00:00 2001 From: Kuitos Date: Sat, 18 Jul 2026 00:06:37 +0800 Subject: [PATCH] fix(sandbox): lazily mirror globals and expand performance benchmarks (#3148) --- benchmark/README.md | 109 ++-- benchmark/compare-results.mjs | 29 ++ benchmark/fixtures/host/garfish.html | 13 + benchmark/fixtures/host/micro-app.html | 13 + benchmark/fixtures/host/native.html | 13 + benchmark/fixtures/host/qiankun-v2.html | 13 + benchmark/fixtures/host/src/benchmark.ts | 84 ++- benchmark/fixtures/host/src/garfish.ts | 34 ++ benchmark/fixtures/host/src/micro-app.ts | 27 + benchmark/fixtures/host/src/native.ts | 141 +++++ benchmark/fixtures/host/src/qiankun-v2.ts | 24 + benchmark/fixtures/host/src/qiankun.ts | 17 +- benchmark/fixtures/host/src/wujie.ts | 23 +- benchmark/fixtures/host/vite.config.ts | 5 + benchmark/frameworks.mjs | 96 ++++ benchmark/package.json | 17 +- benchmark/runner.mjs | 377 ++++++++++---- benchmark/scenarios.mjs | 491 +++++++++++++++--- benchmark/snapshot.mjs | 4 + benchmark/src/browser.mjs | 39 +- benchmark/src/fingerprint.mjs | 43 ++ benchmark/src/harness.mjs | 103 ++++ benchmark/src/hash.mjs | 61 +++ benchmark/src/options.mjs | 40 +- benchmark/src/origins.mjs | 25 + benchmark/src/report.mjs | 108 +++- benchmark/src/result-comparison.mjs | 39 ++ benchmark/src/server.mjs | 227 +++++++- benchmark/src/site-isolation.mjs | 62 +++ benchmark/src/snapshot.mjs | 53 +- benchmark/src/stats.mjs | 44 ++ benchmark/tests/browser.integration.test.mjs | 113 +++- benchmark/tests/browser.test.mjs | 96 +++- benchmark/tests/fingerprint.test.mjs | 75 +++ benchmark/tests/frameworks.test.mjs | 39 ++ benchmark/tests/harness.test.mjs | 22 + benchmark/tests/hash.test.mjs | 38 ++ benchmark/tests/options.test.mjs | 37 +- benchmark/tests/report.test.mjs | 105 +++- benchmark/tests/result-comparison.test.mjs | 36 ++ benchmark/tests/scenarios.test.mjs | 253 ++++++++- benchmark/tests/server.test.mjs | 53 +- benchmark/tests/snapshot.test.mjs | 18 +- benchmark/tests/stats.test.mjs | 18 +- benchmark/tests/suites.test.mjs | 79 +++ benchmark/tsconfig.json | 1 + package.json | 5 + .../src/core/membrane/__tests__/index.test.ts | 149 ++++++ packages/sandbox/src/core/membrane/index.ts | 141 ++--- pnpm-lock.yaml | 132 +++++ tsconfig.json | 2 +- 51 files changed, 3379 insertions(+), 407 deletions(-) create mode 100644 benchmark/compare-results.mjs create mode 100644 benchmark/fixtures/host/garfish.html create mode 100644 benchmark/fixtures/host/micro-app.html create mode 100644 benchmark/fixtures/host/native.html create mode 100644 benchmark/fixtures/host/qiankun-v2.html create mode 100644 benchmark/fixtures/host/src/garfish.ts create mode 100644 benchmark/fixtures/host/src/micro-app.ts create mode 100644 benchmark/fixtures/host/src/native.ts create mode 100644 benchmark/fixtures/host/src/qiankun-v2.ts create mode 100644 benchmark/frameworks.mjs create mode 100644 benchmark/src/fingerprint.mjs create mode 100644 benchmark/src/harness.mjs create mode 100644 benchmark/src/hash.mjs create mode 100644 benchmark/src/origins.mjs create mode 100644 benchmark/src/result-comparison.mjs create mode 100644 benchmark/src/site-isolation.mjs create mode 100644 benchmark/tests/fingerprint.test.mjs create mode 100644 benchmark/tests/frameworks.test.mjs create mode 100644 benchmark/tests/harness.test.mjs create mode 100644 benchmark/tests/hash.test.mjs create mode 100644 benchmark/tests/result-comparison.test.mjs create mode 100644 benchmark/tests/suites.test.mjs create mode 100644 packages/sandbox/src/core/membrane/__tests__/index.test.ts diff --git a/benchmark/README.md b/benchmark/README.md index 759035bf6..828e235b4 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,33 +1,35 @@ # qiankun benchmark -This workspace compares the cold-load path of the locally built qiankun package with a pinned Wujie release. It is intentionally separate from `e2e/`: Playwright Library only drives the browser boundaries, while all timing and core-element detection run inside the page. +This workspace measures cold micro-app loading from the instant an adapter invokes its mount path until the app's core element is paintable. It compares the locally built qiankun package with qiankun 2.10.16, native iframe, Wujie 2.1.0, MicroApp 1.0.0-rc.32, and Garfish 1.19.7. Playwright Library owns browser lifecycle only; timing and paint detection run inside the page. ## Commands Run from the repository root: ```bash -# Five samples per product cell. Validates plumbing only; not performance data. +# Fast plumbing checks: one browser trial and five samples per cell. pnpm benchmark:check +pnpm benchmark:ecosystem:check +pnpm benchmark:ssr-streaming:check -# Five warmups, 100 valid attempts per product cell, and 50 samples per A/A arm. +# Formal runs: three independent browser trials, each with five warmups, +# 100 samples per product cell, and 100 samples per A/A arm. pnpm benchmark:smoke +pnpm benchmark:ecosystem +pnpm benchmark:site-isolation +pnpm benchmark:ssr-streaming ``` -To compare a performance change against an interleaved baseline, snapshot the fully bundled host before editing package code, then run the comparison after the change: +The check commands disable the A/A gate and are not performance evidence. Formal run results are written to `benchmark/results/-/`; raw warmups and measurements are retained. -```bash -# On the clean baseline revision. Fails instead of overwriting an existing snapshot. -pnpm benchmark:baseline - -# After making and building the candidate change. Runs 100 paired samples per revision. -pnpm benchmark:compare +To compare two repeated runs by absolute median, use the guarded comparison command: -# Five paired samples for plumbing only; the improvement gate is disabled. -pnpm benchmark:compare:check +```bash +pnpm --filter @qiankunjs/benchmark run compare:results -- \ + benchmark/results/ benchmark/results/ ``` -The baseline snapshot contains the complete Vite host bundle, so its qiankun, loader, sandbox, and shared dependency graph cannot mix with candidate packages. Revision samples use different host origins, fresh BrowserContexts, a balanced alternating order, and the same streamed fixture. A formal comparison passes only when every sample is valid and the paired bootstrap 95% confidence interval is entirely below 0%. +This command rejects results whose harness fingerprints differ. The fingerprint covers the environment, Chromium and Playwright versions, launch/context options, suite definition, sampling configuration, selected adapter configuration and versions, selected Vite entry graph, fixture behavior, and timing/statistics source. Cross-run medians remain diagnostic; the paired comparisons inside one run are the primary result. Install the pinned Chromium revision once if Playwright reports that it is missing: @@ -35,39 +37,80 @@ Install the pinned Chromium revision once if Playwright reports that it is missi pnpm --filter @qiankunjs/benchmark exec playwright install chromium ``` -Local snapshots are written to `benchmark/artifacts/`; run results are written to `benchmark/results/-/`. Both are gitignored. +## Suites -## Matrix +Suites are explicit and independent; adding an ecosystem framework does not alter the frozen core schedule. -The matrix is explicit rather than a cartesian product: +| Suite | Cells | Purpose | +| --- | --: | --- | +| `core` | 8 | qiankun isolation cost plus buffered and streamed comparisons with native iframe and Wujie | +| `site-isolation` | 6 | same-site anchors and cross-site entries for qiankun, native iframe, and Wujie | +| `ecosystem-html` | 6 | one canonical same-site, buffered, isolated cell for each framework/version | +| `ssr-streaming` | 6 | literal SSR progressive reveal compared with an identical delayed buffer, native iframe, qiankun v2, Wujie, and Garfish | -1. qiankun without sandbox or style isolation -2. qiankun with sandbox only -3. qiankun with sandbox and style isolation -4. Wujie with its intrinsic iframe and Shadow DOM isolation -5. qiankun with full isolation and a three-chunk HTML response -6. Wujie with the same three-chunk HTML response +The core suite contains qiankun with no isolation, sandbox only, and sandbox plus style isolation; same-site buffered native iframe and Wujie; and streamed native iframe, qiankun, and Wujie cells. -Wujie does not expose switches that disable its JavaScript or CSS isolation. Its off-cases are therefore marked N/A instead of being compared against a semantically different configuration. +The ecosystem suite intentionally adds only these canonical cells: + +- native iframe; +- qiankun 2.10.16 with its Proxy sandbox and experimental scoped CSS isolation; +- the locally built qiankun with sandbox and style isolation; +- Wujie with its intrinsic iframe and Shadow DOM isolation; +- MicroApp with its default sandbox and scoped CSS; +- Garfish with Browser VM sandbox, effect collection enabled (`cache: false`), and strict Shadow DOM isolation. + +The ecosystem suite stays intentionally small and does not duplicate its canonical cells under streamed delivery. Native HTML parsing and the locally built qiankun can render progressively; qiankun 2.10.16, Wujie, MicroApp, and Garfish accept chunked responses but currently consume the complete HTML before parsing. The core suite retains Wujie's streamed-response diagnostic because Wujie is qiankun's phase-one comparison target. Framework isolation semantics are also not identical: qiankun v2 uses selector rewriting, the local qiankun uses CSS `@scope`, and Garfish/Wujie use Shadow DOM. The suite compares observed user-visible latency under these isolated configurations, not security equivalence. + +The dedicated SSR suite is the canonical streaming comparison. Unlike the core suite's client-rendered chunk diagnostic, its response contains literal server-rendered markup: the first chunk includes the stylesheet, critical element, 12 visible rows, and native paint reporter; the second adds 44 deferred rows; and the final chunk adds another 44 rows, the stream-tail marker, serialized SSR data, the only external lifecycle entry script, and the closing tags. The default schedule is 0/50/100 ms. Keeping the entry script last lets progressive consumers reveal the SSR core while framework settlement and cleanup still wait for the complete response and lifecycle execution. An identical-byte qiankun v3 control withholds the complete response until 100 ms, isolating progressive reveal from fixture content and response-tail duration. Native iframe and qiankun v3 can expose the critical markup before EOF; the pinned qiankun v2, Wujie, and Garfish adapters currently buffer the complete HTML response first. + +This is a deterministic SSR-shaped fixture, not a React/Vue server-rendering or hydration throughput benchmark. Its entry script supplies framework lifecycles and marks the already rendered core as mounted. The streamed-versus-delayed result therefore represents earlier CSS discovery, progressive DOM insertion, and hiding response-tail latency; it must not be interpreted as pure parser, sandbox, or loader CPU throughput. + +The cross-site suite uses identical response bytes for all products. Native iframe must create an OOPIF under Chromium's `--site-per-process`; a separate preflight browser verifies that condition. Wujie 2.1.0 fetches its cross-site entry while retaining a host-origin `srcdoc` JavaScript realm, so that cell is described as a cross-site entry rather than a cross-site iframe. ## Measurement contract -- `t0`: immediately before `loadMicroApp` or `startApp` is called in the host page. -- `t1`: the core marker exists, has non-zero geometry, is visible, has the expected CSS sentinel, and survives two `requestAnimationFrame` callbacks. -- The framework must still finish mounting successfully; settlement is awaited outside the measured duration. -- Buffered and streamed entry responses contain identical bytes. Streamed HTML is emitted at 0/50/100 ms by default. -- The browser process is reused, but every attempt receives a fresh BrowserContext and page. -- No trace, video, screenshot, HAR, route interception, retry, or parallel execution is enabled. -- Raw warmup and measured samples are retained; none are silently replaced or filtered as outliers. +- `t0`: immediately before the selected adapter invokes `loadMicroApp`, `startApp`, `renderApp`, native iframe navigation, or `Garfish.loadApp`. +- `t1`: the core marker has non-zero geometry, contains `[data-benchmark-critical]`, is visible, has the expected CSS sentinel, and survives two `requestAnimationFrame` callbacks. It does not require `data-mounted=true`, so visible SSR content can finish this metric before client lifecycle settlement. +- Framework settlement and cleanup must succeed, but both remain outside the measured duration. +- After settlement, the harness separately requires `data-mounted=true`; a framework cannot pass by exposing SSR markup without completing its lifecycle. +- Every attempt gets a fresh BrowserContext and page. Every formal trial gets a newly launched Chromium process. +- Adapter module initialization happens before `t0` and may not request fixture resources; the runner fails the sample if it observes such a request. +- Native iframe reports its child-document paint timestamp through an origin-, source-, version-, and token-validated `postMessage`. +- Buffered and streamed responses contain identical bytes. Streamed HTML is emitted at 0/50/100 ms by default. +- The SSR delayed-buffer control contains the same bytes as its streamed counterpart and sends them together at the time the stream would have ended. +- HTML responses are `no-store`; deterministic static assets are immutable so qiankun's walk-ahead preload can satisfy its subsequent sandbox fetch. Fresh BrowserContexts keep every measured sample cold. +- No trace, video, screenshot, HAR, route interception, retry, outlier replacement, or parallel execution is enabled. +- Host bundle download and top-level framework initialization are outside this benchmark's scope. + +The primary metric remains `t1 - t0`. Reports also retain framework-settlement time (`settled - t0`) and the entry HTML resource's response-end time (`PerformanceResourceTiming.responseEnd - t0`) as diagnostics; response end does not represent the entire static-resource waterfall. `response end - core paint` is reported as paint lead: a positive value means this harness observed the user-visible core before the complete HTML response arrived. Native iframe navigation timing is not visible as a parent-page resource entry, so its response-end and paint-lead diagnostics are `n/a`; its child-reported primary paint duration remains valid. These per-variant medians explain the result but do not replace the primary metric or participate in paired comparisons, confidence intervals, or pass/fail gates. -The primary statistic is the paired median relative delta with a 10,000-resample bootstrap 95% confidence interval. Median, mean, p75, p95, MAD, standard deviation, and CV are also retained. Positive comparison deltas mean the candidate is slower than the reference. +Rounds are paired within each independent browser trial. The primary estimate is the median of per-trial median log ratios. Its 95% confidence interval comes from a hierarchical 10,000-resample bootstrap that resamples trials first and rounds second, so a `3 × 100` run is not treated as 300 independent observations. Median, mean, p75, p95, MAD, standard deviation, CV, and each trial's summary are also retained. Positive deltas mean the candidate is slower than the reference. ## A/A calibration -Before the product matrix, two aliases of the exact same fully isolated qiankun variant are interleaved. The formal run requires: +Each trial first interleaves two aliases of the exact same selected qiankun cell. The SSR suite uses its streamed qiankun v3 cell; the other suites use the canonical fully isolated cell. Both every trial and the aggregate must satisfy: - absolute paired median delta no greater than 3%; - the bootstrap 95% interval includes 0%; - interval width no greater than 10 percentage points. -The benchmark is manual in phase one and is not part of the regular PR CI gate. +## Revision comparison + +To validate a qiankun optimization, snapshot the fully bundled host from a clean baseline revision, then run an interleaved candidate comparison: + +```bash +# On the clean baseline revision. Existing snapshots are never overwritten. +pnpm benchmark:baseline + +# After making and building the candidate change. +pnpm benchmark:compare + +# Five paired samples for plumbing only; the improvement gate is disabled. +pnpm benchmark:compare:check +``` + +The snapshot contains the complete Vite host bundle, preventing baseline and candidate package graphs from mixing. It also records a revision-harness compatibility hash that excludes qiankun package sources but covers the adapter, fixture, timing contract, runner, statistics, browser options, and toolchain inputs. The runner rejects stale snapshots when any of those inputs change; recreate old schema snapshots before running a revision comparison. + +Revision mode remains a single browser trial with balanced baseline/candidate rounds and fresh BrowserContexts; it passes only when every sample is valid and the paired-bootstrap 95% confidence interval is entirely below 0%. Use `--scenario=sandbox` with a named snapshot to isolate the buffered sandbox-only path. + +Local snapshots are written to `benchmark/artifacts/`. Snapshots and results are gitignored, and the benchmark remains a manual gate rather than part of regular PR CI. diff --git a/benchmark/compare-results.mjs b/benchmark/compare-results.mjs new file mode 100644 index 000000000..28d1e5fa3 --- /dev/null +++ b/benchmark/compare-results.mjs @@ -0,0 +1,29 @@ +import { readFile, stat } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +import { compareBenchmarkResults } from './src/result-comparison.mjs'; + +async function readResult(argument) { + const path = resolve(argument); + const resultPath = (await stat(path)).isDirectory() ? join(path, 'result.json') : path; + return JSON.parse(await readFile(resultPath, 'utf8')); +} + +function formatPercent(value) { + return `${value >= 0 ? '+' : ''}${value.toFixed(2)}%`; +} + +const [referencePath, candidatePath, ...extra] = process.argv.slice(2); +if (!referencePath || !candidatePath || extra.length > 0) { + throw new Error('usage: node compare-results.mjs '); +} + +const rows = compareBenchmarkResults(await readResult(referencePath), await readResult(candidatePath)); +console.log('| Variant | Reference median (ms) | Candidate median (ms) | Absolute-run delta |'); +console.log('| --- | ---: | ---: | ---: |'); +for (const row of rows) { + console.log( + `| ${row.label} | ${row.referenceMedian.toFixed(2)} | ${row.candidateMedian.toFixed(2)} | ${formatPercent(row.relativeDeltaPercent)} |`, + ); +} +console.log('\nCross-run absolute medians are diagnostic only; within-run paired comparisons remain primary.'); diff --git a/benchmark/fixtures/host/garfish.html b/benchmark/fixtures/host/garfish.html new file mode 100644 index 000000000..3c71b687f --- /dev/null +++ b/benchmark/fixtures/host/garfish.html @@ -0,0 +1,13 @@ + + + + + + Garfish benchmark host + + + +
+ + + diff --git a/benchmark/fixtures/host/micro-app.html b/benchmark/fixtures/host/micro-app.html new file mode 100644 index 000000000..4bacd2b25 --- /dev/null +++ b/benchmark/fixtures/host/micro-app.html @@ -0,0 +1,13 @@ + + + + + + MicroApp benchmark host + + + +
+ + + diff --git a/benchmark/fixtures/host/native.html b/benchmark/fixtures/host/native.html new file mode 100644 index 000000000..288632c8b --- /dev/null +++ b/benchmark/fixtures/host/native.html @@ -0,0 +1,13 @@ + + + + + + Native iframe benchmark host + + + +
+ + + diff --git a/benchmark/fixtures/host/qiankun-v2.html b/benchmark/fixtures/host/qiankun-v2.html new file mode 100644 index 000000000..3f31e411b --- /dev/null +++ b/benchmark/fixtures/host/qiankun-v2.html @@ -0,0 +1,13 @@ + + + + + + qiankun v2 benchmark host + + + +
+ + + diff --git a/benchmark/fixtures/host/src/benchmark.ts b/benchmark/fixtures/host/src/benchmark.ts index bd74d111c..6a8c6199e 100644 --- a/benchmark/fixtures/host/src/benchmark.ts +++ b/benchmark/fixtures/host/src/benchmark.ts @@ -5,27 +5,39 @@ export interface BenchmarkOptions { } export interface BenchmarkMeasurement { + cleaned: boolean; duration: number; + entryResponseEndDuration: number | null; settled: boolean; + settledDuration: number; t0: number; t1: number; } -type StartMicroApp = (options: BenchmarkOptions) => Promise; +export interface BenchmarkMountHandle { + cleanup(): Promise; + settled: Promise; +} + +type MountMicroApp = (options: BenchmarkOptions, container: HTMLElement) => BenchmarkMountHandle; -function findCoreElement(): HTMLElement | null { - const direct = document.querySelector('#benchmark-core'); - if (direct) return direct; - const wujieApp = document.querySelector('wujie-app'); - return wujieApp?.shadowRoot?.querySelector('#benchmark-core') ?? null; +export interface BenchmarkPaintObserver { + assertMounted(): void; + prepare?(options: BenchmarkOptions): void; + waitForPaint(t0: number, options: BenchmarkOptions): Promise; } +export type FindCoreElement = () => HTMLElement | null; + +const findDirectCoreElement: FindCoreElement = () => document.querySelector('#benchmark-core'); + function isPaintable(element: HTMLElement): boolean { const rect = element.getBoundingClientRect(); const style = getComputedStyle(element); return ( rect.width > 0 && rect.height > 0 && + element.querySelector('[data-benchmark-critical]') !== null && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && @@ -33,7 +45,14 @@ function isPaintable(element: HTMLElement): boolean { ); } -function waitForPaint(t0: number, timeoutMs: number): Promise { +function getEntryResponseEndDuration(entry: string, t0: number): number | null { + const timing = performance.getEntriesByName(entry, 'resource').at(-1); + if (!(timing instanceof PerformanceResourceTiming)) return null; + const duration = timing.responseEnd - t0; + return Number.isFinite(duration) && duration >= 0 ? duration : null; +} + +function waitForDomPaint(t0: number, timeoutMs: number, findCoreElement: FindCoreElement): Promise { return new Promise((resolve, reject) => { const timeout = window.setTimeout(() => reject(new Error('core element paint timed out')), timeoutMs); @@ -60,22 +79,57 @@ function waitForPaint(t0: number, timeoutMs: number): Promise { }); } -export function installBenchmark(startMicroApp: StartMicroApp): void { +export function createDomPaintObserver( + findCoreElement: FindCoreElement = findDirectCoreElement, +): BenchmarkPaintObserver { + return { + assertMounted() { + const mountedCore = findCoreElement(); + if (mountedCore?.dataset.mounted !== 'true') { + throw new Error('micro app settled without mounting its core element'); + } + }, + waitForPaint(t0, options) { + return waitForDomPaint(t0, options.timeoutMs, findCoreElement); + }, + }; +} + +export function installBenchmark( + mountMicroApp: MountMicroApp, + paintObserver: BenchmarkPaintObserver = createDomPaintObserver(), +): void { window.__BENCHMARK__ = { async run(options) { const container = document.querySelector('#micro-app-container'); if (!container) throw new Error('micro app container is missing'); container.replaceChildren(); + paintObserver.prepare?.(options); const t0 = performance.now(); - const paintPromise = waitForPaint(t0, options.timeoutMs); - const settlePromise = startMicroApp(options); - const [duration] = await Promise.all([paintPromise, settlePromise]); - const mountedCore = findCoreElement(); - if (mountedCore?.dataset.mounted !== 'true') { - throw new Error('micro app settled without mounting its core element'); + const handle = mountMicroApp(options, container); + const paintPromise = paintObserver.waitForPaint(t0, options); + const settledPromise = handle.settled.then(() => performance.now() - t0); + let duration: number; + let settledDuration: number; + try { + [duration, settledDuration] = await Promise.all([paintPromise, settledPromise]); + paintObserver.assertMounted(); + } finally { + await handle.cleanup(); + if (container.childNodes.length > 0) { + throw new Error('framework adapter cleanup left nodes in the micro app container'); + } } - return { duration, settled: true, t0, t1: t0 + duration }; + return { + cleaned: true, + duration, + entryResponseEndDuration: getEntryResponseEndDuration(options.entry, t0), + settled: true, + settledDuration, + t0, + t1: t0 + duration, + }; }, }; } diff --git a/benchmark/fixtures/host/src/garfish.ts b/benchmark/fixtures/host/src/garfish.ts new file mode 100644 index 000000000..3430a5f67 --- /dev/null +++ b/benchmark/fixtures/host/src/garfish.ts @@ -0,0 +1,34 @@ +import Garfish, { type interfaces } from 'garfish'; + +import { createDomPaintObserver, installBenchmark } from './benchmark'; + +Garfish.run({ disablePreloadApp: true }); + +function findGarfishCoreElement(): HTMLElement | null { + return ( + document + .querySelector('#micro-app-container > [id^="garfish_app_for_"]') + ?.shadowRoot?.querySelector('#benchmark-core') ?? null + ); +} + +installBenchmark(({ entry, frameworkOptions }, container) => { + let app: interfaces.App | null = null; + const settled = (async () => { + app = await Garfish.loadApp('benchmark-app', { + ...(frameworkOptions as Partial>), + domGetter: () => container, + entry, + }); + if (!app) throw new Error('Garfish failed to load'); + if (!(await app.mount())) throw new Error('Garfish failed to mount'); + })(); + + return { + async cleanup() { + await settled.catch(() => {}); + if (app && !app.unmount()) throw new Error('Garfish failed to unmount'); + }, + settled, + }; +}, createDomPaintObserver(findGarfishCoreElement)); diff --git a/benchmark/fixtures/host/src/micro-app.ts b/benchmark/fixtures/host/src/micro-app.ts new file mode 100644 index 000000000..2db97dd83 --- /dev/null +++ b/benchmark/fixtures/host/src/micro-app.ts @@ -0,0 +1,27 @@ +import microApp from '@micro-zoe/micro-app'; + +import { installBenchmark } from './benchmark'; + +microApp.start(); + +installBenchmark(({ entry, frameworkOptions }, container) => { + const settled = microApp + .renderApp({ + ...frameworkOptions, + container, + name: 'benchmark-app', + url: entry, + }) + .then((mounted) => { + if (!mounted) throw new Error('MicroApp failed to mount'); + }); + + return { + async cleanup() { + await settled.catch(() => {}); + const unmounted = await microApp.unmountApp('benchmark-app', { destroy: true }); + if (!unmounted) throw new Error('MicroApp failed to unmount'); + }, + settled, + }; +}); diff --git a/benchmark/fixtures/host/src/native.ts b/benchmark/fixtures/host/src/native.ts new file mode 100644 index 000000000..9674a3827 --- /dev/null +++ b/benchmark/fixtures/host/src/native.ts @@ -0,0 +1,141 @@ +import { type BenchmarkPaintObserver, installBenchmark } from './benchmark'; + +const NATIVE_BENCHMARK_QUERY_VALUE = 'native-iframe'; +const NATIVE_CORE_PAINTED = 'native-core-painted'; +const NATIVE_APP_MOUNTED = 'native-app-mounted'; +const MESSAGE_VERSION = 1; + +interface ActiveRun { + expectedOrigin: string; + iframe?: HTMLIFrameElement; + mounted: boolean; + paintMessage?: Promise>; + token: string; +} + +let activeRun: ActiveRun | undefined; + +function getExpectedMessage( + event: MessageEvent, + run: ActiveRun, + type: string, +): Record | undefined { + if (event.origin !== run.expectedOrigin || event.source !== run.iframe?.contentWindow) return; + if (typeof event.data !== 'object' || event.data === null) return; + + const message = event.data as Record; + if (message.type !== type || message.version !== MESSAGE_VERSION || message.token !== run.token) return; + return message; +} + +function waitForIframeMessage(type: string, run: ActiveRun, timeoutMs: number): Promise> { + return new Promise((resolve, reject) => { + const timeout = window.setTimeout(() => { + cleanup(); + reject(new Error(`${type} timed out`)); + }, timeoutMs); + const onMessage = (event: MessageEvent) => { + const message = getExpectedMessage(event, run, type); + if (!message) return; + cleanup(); + resolve(message); + }; + function cleanup() { + window.clearTimeout(timeout); + window.removeEventListener('message', onMessage); + } + window.addEventListener('message', onMessage); + }); +} + +function waitForIframeLoad(iframe: HTMLIFrameElement, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timeout = window.setTimeout(() => { + cleanup(); + reject(new Error('native iframe load timed out')); + }, timeoutMs); + const onError = () => { + cleanup(); + reject(new Error('native iframe failed to load')); + }; + const onLoad = () => { + cleanup(); + resolve(); + }; + function cleanup() { + window.clearTimeout(timeout); + iframe.removeEventListener('error', onError); + iframe.removeEventListener('load', onLoad); + } + iframe.addEventListener('error', onError); + iframe.addEventListener('load', onLoad); + }); +} + +const nativePaintObserver: BenchmarkPaintObserver = { + assertMounted() { + if (!activeRun?.mounted) throw new Error('native iframe settled without mounting its core element'); + }, + prepare(options) { + const run: ActiveRun = { + expectedOrigin: new URL(options.entry).origin, + mounted: false, + token: crypto.randomUUID(), + }; + run.paintMessage = waitForIframeMessage(NATIVE_CORE_PAINTED, run, options.timeoutMs); + activeRun = run; + }, + async waitForPaint(t0, options) { + const startEpoch = performance.timeOrigin + t0; + const run = activeRun; + if (!run?.paintMessage) throw new Error('native iframe paint observer is not prepared'); + + const message = await run.paintMessage; + const paintedAt = message.paintedAt; + const receivedAt = performance.timeOrigin + performance.now(); + if ( + typeof paintedAt !== 'number' || + !Number.isFinite(paintedAt) || + paintedAt < startEpoch || + paintedAt > receivedAt + 1 || + paintedAt - startEpoch > options.timeoutMs + ) { + throw new Error('native iframe reported an invalid paint timestamp'); + } + return paintedAt - startEpoch; + }, +}; + +installBenchmark(({ entry, timeoutMs }, container) => { + if (!activeRun) throw new Error('native iframe paint observer is not ready'); + + const run = activeRun; + const iframe = document.createElement('iframe'); + iframe.style.border = '0'; + iframe.style.display = 'block'; + iframe.style.height = '480px'; + iframe.style.width = '960px'; + iframe.title = 'Native iframe benchmark app'; + run.iframe = iframe; + + const entryUrl = new URL(entry); + entryUrl.searchParams.set('benchmark', NATIVE_BENCHMARK_QUERY_VALUE); + entryUrl.searchParams.set('benchmark-parent-origin', window.location.origin); + entryUrl.searchParams.set('benchmark-token', run.token); + const loadPromise = waitForIframeLoad(iframe, timeoutMs); + const mountPromise = waitForIframeMessage(NATIVE_APP_MOUNTED, run, timeoutMs).then(() => { + run.mounted = true; + }); + + iframe.src = entryUrl.href; + container.appendChild(iframe); + const settled = Promise.all([loadPromise, mountPromise]).then(() => {}); + return { + async cleanup() { + await settled.catch(() => {}); + iframe.remove(); + activeRun = undefined; + }, + settled, + }; +}, nativePaintObserver); diff --git a/benchmark/fixtures/host/src/qiankun-v2.ts b/benchmark/fixtures/host/src/qiankun-v2.ts new file mode 100644 index 000000000..8b30fbc8e --- /dev/null +++ b/benchmark/fixtures/host/src/qiankun-v2.ts @@ -0,0 +1,24 @@ +import { type FrameworkConfiguration, loadMicroApp, start } from 'qiankun-v2'; + +import { installBenchmark } from './benchmark'; + +start({ prefetch: false }); + +installBenchmark(({ entry, frameworkOptions }, container) => { + const app = loadMicroApp( + { + container, + entry, + name: 'benchmark-app', + }, + frameworkOptions as FrameworkConfiguration, + ); + const settled = app.mountPromise.then(() => {}); + return { + async cleanup() { + await settled.catch(() => {}); + await app.unmount(); + }, + settled, + }; +}); diff --git a/benchmark/fixtures/host/src/qiankun.ts b/benchmark/fixtures/host/src/qiankun.ts index 6982cc858..9c2bfcd18 100644 --- a/benchmark/fixtures/host/src/qiankun.ts +++ b/benchmark/fixtures/host/src/qiankun.ts @@ -1,10 +1,10 @@ -import { type AppConfiguration, loadMicroApp } from 'qiankun'; +import { type AppConfiguration, loadMicroApp, start } from 'qiankun'; import { installBenchmark } from './benchmark'; -installBenchmark(async ({ entry, frameworkOptions }) => { - const container = document.querySelector('#micro-app-container'); - if (!container) throw new Error('micro app container is missing'); +start(); + +installBenchmark(({ entry, frameworkOptions }, container) => { const app = loadMicroApp( { container, @@ -13,5 +13,12 @@ installBenchmark(async ({ entry, frameworkOptions }) => { }, frameworkOptions as AppConfiguration, ); - await app.mountPromise; + const settled = app.mountPromise.then(() => {}); + return { + async cleanup() { + await settled.catch(() => {}); + await app.unmount(); + }, + settled, + }; }); diff --git a/benchmark/fixtures/host/src/wujie.ts b/benchmark/fixtures/host/src/wujie.ts index 12cd6cd67..c90ffd972 100644 --- a/benchmark/fixtures/host/src/wujie.ts +++ b/benchmark/fixtures/host/src/wujie.ts @@ -1,13 +1,24 @@ -import { startApp } from 'wujie'; +import { destroyApp, startApp } from 'wujie'; -import { installBenchmark } from './benchmark'; +import { createDomPaintObserver, installBenchmark } from './benchmark'; -installBenchmark(async ({ entry, frameworkOptions }) => { - await startApp({ +function findWujieCoreElement(): HTMLElement | null { + return document.querySelector('wujie-app')?.shadowRoot?.querySelector('#benchmark-core') ?? null; +} + +installBenchmark(({ entry, frameworkOptions }) => { + const settled = startApp({ ...frameworkOptions, el: '#micro-app-container', loading: document.createElement('span'), name: 'benchmark-app', url: entry, - }); -}); + }).then(() => {}); + return { + async cleanup() { + await settled.catch(() => {}); + await destroyApp('benchmark-app'); + }, + settled, + }; +}, createDomPaintObserver(findWujieCoreElement)); diff --git a/benchmark/fixtures/host/vite.config.ts b/benchmark/fixtures/host/vite.config.ts index db7b8caa4..efc18f6fa 100644 --- a/benchmark/fixtures/host/vite.config.ts +++ b/benchmark/fixtures/host/vite.config.ts @@ -4,11 +4,16 @@ import { defineConfig } from 'vite'; export default defineConfig({ build: { emptyOutDir: true, + manifest: true, minify: false, outDir: 'dist', rollupOptions: { input: { + garfish: fileURLToPath(new URL('garfish.html', import.meta.url)), + microApp: fileURLToPath(new URL('micro-app.html', import.meta.url)), + native: fileURLToPath(new URL('native.html', import.meta.url)), qiankun: fileURLToPath(new URL('qiankun.html', import.meta.url)), + qiankunV2: fileURLToPath(new URL('qiankun-v2.html', import.meta.url)), wujie: fileURLToPath(new URL('wujie.html', import.meta.url)), }, }, diff --git a/benchmark/frameworks.mjs b/benchmark/frameworks.mjs new file mode 100644 index 000000000..e1879fffa --- /dev/null +++ b/benchmark/frameworks.mjs @@ -0,0 +1,96 @@ +export const FRAMEWORK_ADAPTER_API_VERSION = 3; + +export const FRAMEWORK_ADAPTERS = { + garfish: { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: false, + }, + hostPage: 'garfish.html', + id: 'garfish', + initialization: { disablePreloadApp: true }, + packageName: 'garfish', + }, + 'micro-app': { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: false, + }, + hostPage: 'micro-app.html', + id: 'micro-app', + initialization: { prefetch: false, start: true }, + packageName: '@micro-zoe/micro-app', + }, + native: { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: true, + }, + hostPage: 'native.html', + id: 'native', + initialization: {}, + packageName: null, + }, + qiankun: { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: true, + }, + hostPage: 'qiankun.html', + id: 'qiankun', + initialization: { start: true }, + packageName: 'qiankun', + }, + 'qiankun-v2': { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: false, + }, + hostPage: 'qiankun-v2.html', + id: 'qiankun-v2', + initialization: { prefetch: false, start: true }, + packageName: 'qiankun-v2', + }, + wujie: { + capabilities: { + acceptsChunkedResponse: true, + crossSiteEntry: true, + htmlEntry: true, + progressiveHtmlParsing: false, + }, + hostPage: 'wujie.html', + id: 'wujie', + initialization: {}, + packageName: 'wujie', + }, +}; + +export function getFrameworkAdapter(id) { + const adapter = FRAMEWORK_ADAPTERS[id]; + if (!adapter) throw new Error(`unknown benchmark framework adapter: ${id}`); + return adapter; +} + +export function getVariantHostPage(variant) { + return getFrameworkAdapter(variant.framework).hostPage; +} + +export function assertFrameworkVariantSupported(variant) { + const adapter = getFrameworkAdapter(variant.framework); + if (variant.entrySite === 'cross-site' && !adapter.capabilities.crossSiteEntry) { + throw new Error(`${adapter.id} does not support cross-site HTML entries`); + } + if (variant.delivery === 'streamed' && !adapter.capabilities.acceptsChunkedResponse) { + throw new Error(`${adapter.id} does not accept chunked HTML responses`); + } +} diff --git a/benchmark/package.json b/benchmark/package.json index ad7e486d8..f0b874827 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -6,16 +6,25 @@ "scripts": { "build:fixture": "vite build fixtures/host", "check": "pnpm run test:unit && pnpm run typecheck && pnpm run build:fixture && node --test tests/browser.integration.test.mjs", - "compare": "pnpm run build:fixture && node runner.mjs --mode=revision --baseline-dir=artifacts/baseline --samples=100 --warmup=5 --calibration-samples=50", + "compare": "pnpm run build:fixture && node runner.mjs --mode=revision --baseline-dir=artifacts/baseline --samples=100 --warmup=5 --calibration-samples=100", + "compare:results": "node compare-results.mjs", "compare:check": "pnpm run build:fixture && node runner.mjs --mode=revision --baseline-dir=artifacts/baseline --samples=5 --warmup=2 --calibration-samples=5 --calibration-gate=false --comparison-gate=false", - "run:check": "pnpm run build:fixture && node runner.mjs --samples=5 --warmup=2 --calibration-samples=5 --calibration-gate=false", + "ecosystem": "pnpm run build:fixture && node runner.mjs --suite=ecosystem-html --samples=100 --warmup=5 --calibration-samples=100", + "ecosystem:check": "pnpm run build:fixture && node runner.mjs --suite=ecosystem-html --samples=5 --warmup=2 --calibration-samples=5 --calibration-gate=false --trials=1", + "run:check": "pnpm run build:fixture && node runner.mjs --samples=5 --warmup=2 --calibration-samples=5 --calibration-gate=false --trials=1", "snapshot:baseline": "pnpm run build:fixture && node snapshot.mjs --name=baseline", - "smoke": "pnpm run build:fixture && node runner.mjs --samples=100 --warmup=5 --calibration-samples=50", - "test:unit": "node --test tests/browser.test.mjs tests/collect.test.mjs tests/options.test.mjs tests/report.test.mjs tests/revisions.test.mjs tests/scenarios.test.mjs tests/schedule.test.mjs tests/server.test.mjs tests/snapshot.test.mjs tests/stats.test.mjs", + "site-isolation": "pnpm run build:fixture && node runner.mjs --suite=site-isolation --samples=100 --warmup=5 --calibration-samples=100", + "smoke": "pnpm run build:fixture && node runner.mjs --samples=100 --warmup=5 --calibration-samples=100", + "ssr-streaming": "pnpm run build:fixture && node runner.mjs --suite=ssr-streaming --samples=100 --warmup=5 --calibration-samples=100", + "ssr-streaming:check": "pnpm run build:fixture && node runner.mjs --suite=ssr-streaming --samples=5 --warmup=2 --calibration-samples=5 --calibration-gate=false --trials=1", + "test:unit": "node --test tests/browser.test.mjs tests/collect.test.mjs tests/fingerprint.test.mjs tests/frameworks.test.mjs tests/harness.test.mjs tests/hash.test.mjs tests/options.test.mjs tests/report.test.mjs tests/result-comparison.test.mjs tests/revisions.test.mjs tests/scenarios.test.mjs tests/schedule.test.mjs tests/server.test.mjs tests/snapshot.test.mjs tests/stats.test.mjs tests/suites.test.mjs", "typecheck": "tsc -p tsconfig.json --noEmit" }, "dependencies": { + "@micro-zoe/micro-app": "1.0.0-rc.32", + "garfish": "1.19.7", "qiankun": "workspace:*", + "qiankun-v2": "npm:qiankun@2.10.16", "wujie": "2.1.0" }, "devDependencies": { diff --git a/benchmark/runner.mjs b/benchmark/runner.mjs index bf31a4d4e..58566581f 100644 --- a/benchmark/runner.mjs +++ b/benchmark/runner.mjs @@ -7,22 +7,25 @@ import { fileURLToPath } from 'node:url'; import { chromium } from 'playwright'; +import { assertFrameworkVariantSupported, getFrameworkAdapter } from './frameworks.mjs'; import { runBrowserSample } from './src/browser.mjs'; import { collectSamples } from './src/collect.mjs'; +import { createHarnessRecord, createRevisionHarnessRecord } from './src/harness.mjs'; +import { CHROMIUM_LAUNCH_ARGS, createBenchmarkOrigins } from './src/origins.mjs'; import { parseRunnerOptions } from './src/options.mjs'; import { buildReport, renderSummaryMarkdown } from './src/report.mjs'; import { evaluateRevisionComparison, resolveVariantHostOrigin } from './src/revisions.mjs'; import { createFixtureServer } from './src/server.mjs'; -import { readBaselineSnapshot } from './src/snapshot.mjs'; +import { inspectCrossSiteIsolation } from './src/site-isolation.mjs'; +import { assertBaselineHarnessCompatible, readBaselineSnapshot } from './src/snapshot.mjs'; import { createStaticServer } from './src/static-server.mjs'; import { evaluateCalibration } from './src/stats.mjs'; import { CALIBRATION_VARIANTS, - PRODUCT_COMPARISONS, - PRODUCT_VARIANTS, REVISION_CALIBRATION_VARIANTS, REVISION_COMPARISONS, - REVISION_VARIANTS, + SUITES, + createRevisionVariants, } from './scenarios.mjs'; const execFileAsync = promisify(execFile); @@ -50,23 +53,35 @@ function materializeCalibrationVariants(productVariants, aliases) { }); } -function createRunDefinition(mode) { - if (mode === 'revision') { +function createRunDefinition(options) { + if (options.mode === 'revision') { return { calibrationAliases: REVISION_CALIBRATION_VARIANTS, comparisons: REVISION_COMPARISONS, - productTitle: 'Revision comparison', - variants: REVISION_VARIANTS, + productTitle: `Revision comparison · ${options.scenario}`, + variants: createRevisionVariants(options.scenario), }; } + const suite = SUITES[options.suite]; + if (!suite) throw new Error(`unknown benchmark suite: ${options.suite}`); + const calibrationSourceVariant = suite.calibrationSourceVariant ?? CALIBRATION_VARIANTS[0].sourceVariant; return { - calibrationAliases: CALIBRATION_VARIANTS, - comparisons: PRODUCT_COMPARISONS, - productTitle: 'Product matrix', - variants: PRODUCT_VARIANTS, + calibrationAliases: CALIBRATION_VARIANTS.map((variant) => ({ + ...variant, + sourceVariant: calibrationSourceVariant, + })), + comparisons: suite.comparisons, + productTitle: suite.title, + variants: suite.variants, }; } +function findVariant(variants, id) { + const variant = variants.find((candidate) => candidate.id === id); + if (!variant) throw new Error(`unknown benchmark variant: ${id}`); + return variant; +} + function ensureAllValid(samples, phase) { const invalid = samples.filter((sample) => !sample.valid); if (invalid.length > 0) { @@ -74,10 +89,10 @@ function ensureAllValid(samples, phase) { } } -async function collectPhase({ browser, fixtureOrigin, hostOrigins, phase, rounds, seed, timeoutMs, variants }) { +async function collectPhase({ browser, fixtureOrigins, hostOrigins, phase, rounds, seed, timeoutMs, trial, variants }) { let completed = 0; const total = rounds * variants.length; - return collectSamples({ + const samples = await collectSamples({ rounds, seed, variants, @@ -86,21 +101,51 @@ async function collectPhase({ browser, fixtureOrigin, hostOrigins, phase, rounds try { const measurement = await runBrowserSample({ browser, - fixtureOrigin, + fixtureOrigins, hostOrigin: resolveVariantHostOrigin(variant, hostOrigins), timeoutMs, variant, }); - console.log(`[benchmark] ${phase} ${completed}/${total} ${variant.id} ${measurement.duration.toFixed(2)}ms`); + console.log( + `[benchmark] trial ${trial + 1} ${phase} ${completed}/${total} ${variant.id} ${measurement.duration.toFixed(2)}ms`, + ); return { ...measurement, phase, timestamp: new Date().toISOString() }; } catch (error) { console.error( - `[benchmark] ${phase} ${completed}/${total} ${variant.id} failed: ${error instanceof Error ? error.message : String(error)}`, + `[benchmark] trial ${trial + 1} ${phase} ${completed}/${total} ${variant.id} failed: ${error instanceof Error ? error.message : String(error)}`, ); throw error; } }, }); + return samples.map((sample) => ({ ...sample, trial })); +} + +function buildCalibrationReport(samples, seed, variants) { + return buildReport({ + comparisons: [ + { + candidate: 'calibration-b', + id: 'aa-calibration', + label: 'A/A identical qiankun variants', + reference: 'calibration-a', + }, + ], + samples, + seed, + variants, + }); +} + +function combineCalibrationEvaluations(aggregate, trialEvaluations) { + const trialFailures = trialEvaluations.flatMap(({ evaluation, trial }) => + evaluation.failures.map((failure) => `trial ${trial + 1}: ${failure}`), + ); + return { + failures: [...aggregate.failures, ...trialFailures], + passed: aggregate.passed && trialFailures.length === 0, + trials: trialEvaluations, + }; } function renderRunSummary({ @@ -113,19 +158,35 @@ function renderRunSummary({ revisionEvaluation, }) { const lines = [ - metadata.options.mode === 'revision' ? '# qiankun revision benchmark' : '# qiankun vs Wujie benchmark', + metadata.options.mode === 'revision' ? '# qiankun revision benchmark' : '# micro-frontend HTML-entry benchmark', '', `- Run: ${metadata.runId}`, `${metadata.options.mode === 'revision' ? '- Candidate commit' : '- Commit'}: ${metadata.commit}${metadata.dirty ? ' (dirty)' : ''}`, `- Chromium: ${metadata.browserVersion}`, - `- Samples: ${metadata.options.samples} per product cell`, - `- Warmup: ${metadata.options.warmup} per cell`, + `- Suite: ${metadata.options.mode === 'revision' ? `revision:${metadata.options.scenario}` : metadata.options.suite}`, + `- Independent browser trials: ${metadata.options.trials}`, + `- Samples: ${metadata.options.samples} per product cell per trial`, + `- Warmup: ${metadata.options.warmup} per cell per trial`, `- Seed: ${metadata.options.seed}`, + `- Frameworks: ${Object.entries(metadata.frameworkVersions) + .map(([name, version]) => `${name}@${version}`) + .join(', ')}`, + `- Harness fingerprint: ${metadata.harness?.fingerprint ?? 'unavailable'}`, '', '> Positive comparison deltas mean the candidate is slower than the reference.', + '> Comparisons pair rounds within each browser trial, then aggregate trials with equal weight.', '', ]; + if (metadata.siteIsolation) { + lines.splice( + 4, + 0, + `- Native cross-site iframe: ${metadata.siteIsolation.nativeIframe.oopif ? 'OOPIF verified' : 'OOPIF not observed'}`, + `- Wujie app-site OOPIF: ${metadata.siteIsolation.wujie.appSiteOopif ? 'observed' : 'not observed (same-site srcdoc sandbox)'}`, + ); + } + if (metadata.baseline) { lines.splice( 4, @@ -170,7 +231,8 @@ async function main() { const { commit } = git; const runId = `${startedAt.toISOString().replace(/[:.]/gu, '-')}-${commit.slice(0, 8)}`; const resultDirectory = join(benchmarkRoot, 'results', runId); - const runDefinition = createRunDefinition(options.mode); + const runDefinition = createRunDefinition(options); + runDefinition.variants.forEach(assertFrameworkVariantSupported); const hostServers = { candidate: createStaticServer({ port: 7600, root: join(benchmarkRoot, 'fixtures/host/dist') }), }; @@ -180,85 +242,150 @@ async function main() { } const fixture = createFixtureServer({ chunkIntervalMs: options.chunkIntervalMs, port: 7601 }); const calibrationVariants = materializeCalibrationVariants(runDefinition.variants, runDefinition.calibrationAliases); + const calibrationSamples = []; + const calibrationTrialEvaluations = []; + const calibrationWarmupSamples = []; + const productSamples = []; + const productWarmupSamples = []; let baselineMetadata; - let browser; + let browserVersion = 'unavailable'; let calibrationEvaluation; let calibrationReport; - let calibrationSamples = []; - let calibrationWarmupSamples = []; + let fatalError; let productReport; - let productSamples = []; - let productWarmupSamples = []; let revisionEvaluation; - let fatalError; + let siteIsolation; try { - if (baselineDirectory) baselineMetadata = await readBaselineSnapshot(baselineDirectory); + if (baselineDirectory) { + baselineMetadata = await readBaselineSnapshot(baselineDirectory); + assertBaselineHarnessCompatible(baselineMetadata, await createRevisionHarnessRecord(benchmarkRoot)); + } await Promise.all([...Object.values(hostServers).map((host) => host.start()), fixture.start()]); - const hostOrigins = Object.fromEntries(Object.entries(hostServers).map(([role, host]) => [role, host.origin])); - browser = await chromium.launch({ headless: true }); - - calibrationWarmupSamples = await collectPhase({ - browser, + const physicalHostOrigins = Object.fromEntries( + Object.entries(hostServers).map(([role, host]) => [role, host.origin]), + ); + const { fixtureOrigins, hostOrigins } = createBenchmarkOrigins({ fixtureOrigin: fixture.origin, - hostOrigins, - phase: 'calibration-warmup', - rounds: options.warmup, - seed: options.seed - 2, - timeoutMs: options.timeoutMs, - variants: calibrationVariants, + hostOrigins: physicalHostOrigins, }); - ensureAllValid(calibrationWarmupSamples, 'calibration warmup'); + const launchOptions = { args: CHROMIUM_LAUNCH_ARGS, headless: true }; - calibrationSamples = await collectPhase({ - browser, - fixtureOrigin: fixture.origin, - hostOrigins, - phase: 'calibration', - rounds: options.calibrationSamples, - seed: options.seed - 1, - timeoutMs: options.timeoutMs, - variants: calibrationVariants, - }); - ensureAllValid(calibrationSamples, 'calibration'); - calibrationReport = buildReport({ - comparisons: [ - { - candidate: 'calibration-b', - id: 'aa-calibration', - label: 'A/A identical qiankun variants', - reference: 'calibration-a', - }, - ], - samples: calibrationSamples, - seed: options.seed, - variants: calibrationVariants, - }); - calibrationEvaluation = evaluateCalibration(calibrationReport.comparisons['aa-calibration']); + if (options.mode === 'framework' && options.suite === 'site-isolation') { + const preflightBrowser = await chromium.launch(launchOptions); + try { + browserVersion = preflightBrowser.version(); + siteIsolation = await inspectCrossSiteIsolation({ + browser: preflightBrowser, + fixtureOrigins, + hostOrigin: hostOrigins.candidate, + nativeVariant: findVariant(runDefinition.variants, 'native-iframe-cross-site'), + timeoutMs: options.timeoutMs, + wujieVariant: findVariant(runDefinition.variants, 'wujie-cross-site-entry'), + }); + } finally { + await preflightBrowser.close(); + } + } - if (!options.calibrationGate || calibrationEvaluation.passed) { - productWarmupSamples = await collectPhase({ - browser, - fixtureOrigin: fixture.origin, - hostOrigins, - phase: 'product-warmup', - rounds: options.warmup, - seed: options.seed + 1, - timeoutMs: options.timeoutMs, - variants: runDefinition.variants, - }); - ensureAllValid(productWarmupSamples, 'product warmup'); + for (let trial = 0; trial < options.trials; trial += 1) { + const trialSeed = options.seed + trial * 1009; + const browser = await chromium.launch(launchOptions); + try { + browserVersion = browser.version(); + const trialCalibrationWarmup = await collectPhase({ + browser, + fixtureOrigins, + hostOrigins, + phase: 'calibration-warmup', + rounds: options.warmup, + seed: trialSeed - 2, + timeoutMs: options.timeoutMs, + trial, + variants: calibrationVariants, + }); + calibrationWarmupSamples.push(...trialCalibrationWarmup); + ensureAllValid(trialCalibrationWarmup, `trial ${trial + 1} calibration warmup`); - productSamples = await collectPhase({ - browser, - fixtureOrigin: fixture.origin, - hostOrigins, - phase: 'product', - rounds: options.samples, - seed: options.seed, - timeoutMs: options.timeoutMs, - variants: runDefinition.variants, - }); + const trialCalibrationSamples = await collectPhase({ + browser, + fixtureOrigins, + hostOrigins, + phase: 'calibration', + rounds: options.calibrationSamples, + seed: trialSeed - 1, + timeoutMs: options.timeoutMs, + trial, + variants: calibrationVariants, + }); + calibrationSamples.push(...trialCalibrationSamples); + ensureAllValid(trialCalibrationSamples, `trial ${trial + 1} calibration`); + const trialCalibrationReport = buildCalibrationReport(trialCalibrationSamples, trialSeed, calibrationVariants); + const trialCalibrationEvaluation = evaluateCalibration(trialCalibrationReport.comparisons['aa-calibration']); + calibrationTrialEvaluations.push({ evaluation: trialCalibrationEvaluation, trial }); + if (options.calibrationGate && !trialCalibrationEvaluation.passed) { + throw new Error( + `trial ${trial + 1} A/A calibration failed: ${trialCalibrationEvaluation.failures.join('; ')}`, + ); + } + + const trialProductWarmup = await collectPhase({ + browser, + fixtureOrigins, + hostOrigins, + phase: 'product-warmup', + rounds: options.warmup, + seed: trialSeed + 1, + timeoutMs: options.timeoutMs, + trial, + variants: runDefinition.variants, + }); + productWarmupSamples.push(...trialProductWarmup); + ensureAllValid(trialProductWarmup, `trial ${trial + 1} product warmup`); + + const trialProductSamples = await collectPhase({ + browser, + fixtureOrigins, + hostOrigins, + phase: 'product', + rounds: options.samples, + seed: trialSeed, + timeoutMs: options.timeoutMs, + trial, + variants: runDefinition.variants, + }); + productSamples.push(...trialProductSamples); + ensureAllValid(trialProductSamples, `trial ${trial + 1} product`); + } finally { + await browser.close(); + } + } + } catch (error) { + fatalError = error instanceof Error ? (error.stack ?? error.message) : String(error); + } finally { + const cleanupResults = await Promise.allSettled([ + ...Object.values(hostServers).map((host) => host.close()), + fixture.close(), + ]); + const cleanupFailures = cleanupResults + .filter((result) => result.status === 'rejected') + .map((result) => (result.reason instanceof Error ? result.reason.message : String(result.reason))); + if (cleanupFailures.length > 0) { + const cleanupError = `benchmark cleanup failed: ${cleanupFailures.join('; ')}`; + fatalError = fatalError ? `${fatalError}\n${cleanupError}` : cleanupError; + } + } + + try { + if (calibrationSamples.length > 0) { + calibrationReport = buildCalibrationReport(calibrationSamples, options.seed, calibrationVariants); + calibrationEvaluation = combineCalibrationEvaluations( + evaluateCalibration(calibrationReport.comparisons['aa-calibration']), + calibrationTrialEvaluations, + ); + } + const expectedProductSamples = options.samples * options.trials * runDefinition.variants.length; + if (productSamples.length === expectedProductSamples) { productReport = buildReport({ comparisons: runDefinition.comparisons, samples: productSamples, @@ -270,38 +397,74 @@ async function main() { } } } catch (error) { - fatalError = error instanceof Error ? (error.stack ?? error.message) : String(error); - } finally { - const cleanupTasks = [...Object.values(hostServers).map((host) => host.close()), fixture.close()]; - if (browser) cleanupTasks.push(browser.close()); - const cleanupResults = await Promise.allSettled(cleanupTasks); - const cleanupFailures = cleanupResults - .filter((result) => result.status === 'rejected') - .map((result) => (result.reason instanceof Error ? result.reason.message : String(result.reason))); - if (!fatalError && cleanupFailures.length > 0) { - fatalError = `benchmark cleanup failed: ${cleanupFailures.join('; ')}`; - } + const reportError = error instanceof Error ? (error.stack ?? error.message) : String(error); + fatalError = fatalError ? `${fatalError}\nReport generation failed: ${reportError}` : reportError; } const benchmarkPackage = await readJson(join(benchmarkRoot, 'package.json')); const qiankunPackage = await readJson(join(repositoryRoot, 'packages/qiankun/package.json')); - const metadata = { + const environment = { arch: os.arch(), - baseline: baselineMetadata, - browserVersion: browser?.version() ?? 'unavailable', - commit, cpu: os.cpus()[0]?.model ?? 'unknown', cpuCount: os.cpus().length, - dirty: git.dirty, nodeVersion: process.version, - options, platform: os.platform(), platformRelease: os.release(), + }; + const frameworkVersions = {}; + for (const framework of new Set(runDefinition.variants.map((variant) => variant.framework))) { + if (framework === 'native') { + frameworkVersions[framework] = 'browser-iframe'; + continue; + } + if (framework === 'qiankun') { + frameworkVersions[framework] = qiankunPackage.version; + continue; + } + + const packageName = getFrameworkAdapter(framework).packageName; + frameworkVersions[framework] = packageName + ? (await readJson(join(benchmarkRoot, 'node_modules', packageName, 'package.json'))).version + : 'unknown'; + } + let harness; + try { + harness = await createHarnessRecord({ + baselineBundleHash: baselineMetadata?.bundleHash, + benchmarkRoot, + browserArgs: CHROMIUM_LAUNCH_ARGS, + browserVersion, + environment, + frameworkVersions, + options, + playwrightVersion: benchmarkPackage.devDependencies.playwright, + runDefinition, + }); + } catch (error) { + const harnessError = error instanceof Error ? (error.stack ?? error.message) : String(error); + fatalError = fatalError ? `${fatalError}\nHarness fingerprint failed: ${harnessError}` : harnessError; + } + + const metadata = { + arch: environment.arch, + baseline: baselineMetadata, + browserVersion, + commit, + cpu: environment.cpu, + cpuCount: environment.cpuCount, + dirty: git.dirty, + frameworkVersions, + harness, + nodeVersion: environment.nodeVersion, + options, + platform: environment.platform, + platformRelease: environment.platformRelease, qiankunVersion: qiankunPackage.version, runId, - schemaVersion: 2, + schemaVersion: 4, + siteIsolation, startedAt: startedAt.toISOString(), - wujieVersion: benchmarkPackage.dependencies.wujie, + wujieVersion: frameworkVersions.wujie, }; const hasInvalidProductSample = productSamples.some((sample) => !sample.valid); const passed = @@ -309,7 +472,7 @@ async function main() { (!options.calibrationGate || calibrationEvaluation?.passed === true) && (options.mode !== 'revision' || !options.comparisonGate || revisionEvaluation?.passed === true) && !hasInvalidProductSample && - productSamples.length === options.samples * runDefinition.variants.length; + productSamples.length === options.samples * options.trials * runDefinition.variants.length; const result = { calibration: { evaluation: calibrationEvaluation, diff --git a/benchmark/scenarios.mjs b/benchmark/scenarios.mjs index c967fe482..3ecf8bc8b 100644 --- a/benchmark/scenarios.mjs +++ b/benchmark/scenarios.mjs @@ -5,57 +5,214 @@ const WUJIE_OPTIONS = { sync: false, }; -export const PRODUCT_VARIANTS = [ - { - delivery: 'buffered', - framework: 'qiankun', - frameworkOptions: { sandbox: false, styleIsolation: false }, - id: 'qk-no-isolation', - label: 'qiankun · no isolation', - }, - { - delivery: 'buffered', - framework: 'qiankun', - frameworkOptions: { sandbox: true, styleIsolation: false }, - id: 'qk-sandbox', - label: 'qiankun · sandbox', - }, - { - delivery: 'buffered', - framework: 'qiankun', - frameworkOptions: { sandbox: true, styleIsolation: true }, - id: 'qk-full-isolation', - label: 'qiankun · sandbox + style isolation', - }, +const QK_NO_ISOLATION = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: false, styleIsolation: false }, + id: 'qk-no-isolation', + label: 'qiankun · no isolation', +}; + +const QK_SANDBOX = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: false }, + id: 'qk-sandbox', + label: 'qiankun · sandbox', +}; + +const QK_FULL_ISOLATION = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: true }, + id: 'qk-full-isolation', + label: 'qiankun · sandbox + style isolation', +}; + +const QK_V2_FULL_ISOLATION = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun-v2', + frameworkOptions: { sandbox: { experimentalStyleIsolation: true } }, + id: 'qk-v2-full-isolation', + label: 'qiankun v2.10.16 · Proxy sandbox + scoped CSS', +}; + +const NATIVE_IFRAME = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'native', + frameworkOptions: {}, + id: 'native-iframe', + label: 'Native iframe · same-site · buffered', +}; + +const WUJIE_ISOLATED = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'wujie', + frameworkOptions: WUJIE_OPTIONS, + id: 'wujie-isolated', + label: 'Wujie · same-site entry', +}; + +const MICRO_APP_DEFAULT_ISOLATION = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'micro-app', + frameworkOptions: {}, + id: 'microapp-default-isolation', + label: 'MicroApp · sandbox + scoped CSS', +}; + +const GARFISH_STRICT_ISOLATION = { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'garfish', + frameworkOptions: { cache: false, sandbox: { strictIsolation: true } }, + id: 'garfish-strict-isolation', + label: 'Garfish · VM sandbox + Shadow DOM', +}; + +const NATIVE_IFRAME_SSR_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'native', + frameworkOptions: {}, + htmlFixture: 'ssr', + id: 'native-iframe-ssr-streamed', + label: 'Native iframe · SSR streamed', +}; + +const QK_V3_SSR_DELAYED_BUFFERED = { + delivery: 'delayed-buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: true }, + htmlFixture: 'ssr', + id: 'qk-v3-ssr-delayed-buffered', + label: 'qiankun v3 · SSR delayed-buffer control', +}; + +const QK_V2_SSR_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'qiankun-v2', + frameworkOptions: { sandbox: { experimentalStyleIsolation: true } }, + htmlFixture: 'ssr', + id: 'qk-v2-ssr-streamed', + label: 'qiankun v2.10.16 · SSR streamed response', +}; + +const QK_V3_SSR_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: true }, + htmlFixture: 'ssr', + id: 'qk-v3-ssr-streamed', + label: 'qiankun v3 · SSR progressive stream', +}; + +const WUJIE_SSR_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'wujie', + frameworkOptions: WUJIE_OPTIONS, + htmlFixture: 'ssr', + id: 'wujie-ssr-streamed', + label: 'Wujie · SSR streamed response', +}; + +const GARFISH_SSR_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'garfish', + frameworkOptions: { cache: false, sandbox: { strictIsolation: true } }, + htmlFixture: 'ssr', + id: 'garfish-ssr-streamed', + label: 'Garfish · SSR streamed response', +}; + +const QK_FULL_ISOLATION_CROSS_SITE = { + delivery: 'buffered', + entrySite: 'cross-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: true }, + id: 'qk-full-isolation-cross-site', + label: 'qiankun · cross-site entry · full isolation', +}; + +const NATIVE_IFRAME_CROSS_SITE = { + delivery: 'buffered', + entrySite: 'cross-site', + framework: 'native', + frameworkOptions: {}, + id: 'native-iframe-cross-site', + label: 'Native iframe · cross-site · buffered', +}; + +const WUJIE_CROSS_SITE = { + delivery: 'buffered', + entrySite: 'cross-site', + framework: 'wujie', + frameworkOptions: WUJIE_OPTIONS, + id: 'wujie-cross-site-entry', + label: 'Wujie · cross-site entry', +}; + +const NATIVE_IFRAME_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'native', + frameworkOptions: {}, + id: 'native-iframe-streamed', + label: 'Native iframe · same-site · streamed', +}; + +const QK_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: true }, + id: 'qk-streamed', + label: 'qiankun · streamed', +}; + +const WUJIE_STREAMED = { + delivery: 'streamed', + entrySite: 'same-site', + framework: 'wujie', + frameworkOptions: WUJIE_OPTIONS, + id: 'wujie-streamed', + label: 'Wujie · streamed response', +}; + +const SAME_SITE_COMPARISONS = [ { - delivery: 'buffered', - framework: 'wujie', - frameworkOptions: WUJIE_OPTIONS, - id: 'wujie-isolated', - label: 'Wujie · iframe + Shadow DOM', + candidate: 'qk-full-isolation', + id: 'qiankun-native-isolated', + label: 'qiankun vs native iframe under same-site isolation', + reference: 'native-iframe', }, { - delivery: 'streamed', - framework: 'qiankun', - frameworkOptions: { sandbox: true, styleIsolation: true }, - id: 'qk-streamed', - label: 'qiankun · streamed', + candidate: 'wujie-isolated', + id: 'wujie-native-isolated', + label: 'Wujie vs native iframe with same-site entries', + reference: 'native-iframe', }, { - delivery: 'streamed', - framework: 'wujie', - frameworkOptions: WUJIE_OPTIONS, - id: 'wujie-streamed', - label: 'Wujie · streamed response', + candidate: 'wujie-isolated', + id: 'isolated-framework', + label: 'Wujie vs qiankun with same-site entries', + reference: 'qk-full-isolation', }, ]; -export const CALIBRATION_VARIANTS = [ - { id: 'calibration-a', label: 'A/A · A', sourceVariant: 'qk-full-isolation' }, - { id: 'calibration-b', label: 'A/A · B', sourceVariant: 'qk-full-isolation' }, -]; - -export const PRODUCT_COMPARISONS = [ +const CORE_COMPARISONS = [ { candidate: 'qk-sandbox', id: 'sandbox-cost', @@ -68,11 +225,18 @@ export const PRODUCT_COMPARISONS = [ label: 'qiankun style isolation cost', reference: 'qk-sandbox', }, + ...SAME_SITE_COMPARISONS, { - candidate: 'wujie-isolated', - id: 'isolated-framework', - label: 'Wujie vs qiankun under isolation', - reference: 'qk-full-isolation', + candidate: 'qk-streamed', + id: 'qiankun-native-streamed', + label: 'qiankun vs native iframe with streamed HTML', + reference: 'native-iframe-streamed', + }, + { + candidate: 'wujie-streamed', + id: 'wujie-native-streamed', + label: 'Wujie vs native iframe with streamed HTML', + reference: 'native-iframe-streamed', }, { candidate: 'wujie-streamed', @@ -82,27 +246,232 @@ export const PRODUCT_COMPARISONS = [ }, ]; -const REVISION_FRAMEWORK_OPTIONS = { sandbox: true, styleIsolation: true }; +const ECOSYSTEM_COMPARISONS = [ + ...SAME_SITE_COMPARISONS, + { + candidate: 'qk-v2-full-isolation', + id: 'qiankun-v2-native-isolated', + label: 'qiankun v2 vs native iframe under isolation', + reference: 'native-iframe', + }, + { + candidate: 'qk-full-isolation', + id: 'qiankun-v3-v2-isolated', + label: 'qiankun v3 vs qiankun v2 under isolation', + reference: 'qk-v2-full-isolation', + }, + { + candidate: 'microapp-default-isolation', + id: 'microapp-native-isolated', + label: 'MicroApp vs native iframe under default isolation', + reference: 'native-iframe', + }, + { + candidate: 'microapp-default-isolation', + id: 'microapp-qiankun-isolated', + label: 'MicroApp vs qiankun under default isolation', + reference: 'qk-full-isolation', + }, + { + candidate: 'garfish-strict-isolation', + id: 'garfish-native-isolated', + label: 'Garfish vs native iframe under strict isolation', + reference: 'native-iframe', + }, + { + candidate: 'garfish-strict-isolation', + id: 'garfish-qiankun-isolated', + label: 'Garfish vs qiankun under isolation', + reference: 'qk-full-isolation', + }, + { + candidate: 'garfish-strict-isolation', + id: 'garfish-qiankun-v2-isolated', + label: 'Garfish vs qiankun v2 under isolation', + reference: 'qk-v2-full-isolation', + }, +]; -export const REVISION_VARIANTS = [ +const SITE_ISOLATION_COMPARISONS = [ + ...SAME_SITE_COMPARISONS, { - delivery: 'streamed', - framework: 'qiankun', - frameworkOptions: REVISION_FRAMEWORK_OPTIONS, - hostRole: 'baseline', - id: 'revision-baseline', - label: 'baseline · qiankun streamed', + candidate: 'qk-full-isolation-cross-site', + id: 'qiankun-cross-site-cost', + label: 'qiankun cross-site entry cost', + reference: 'qk-full-isolation', }, { - delivery: 'streamed', - framework: 'qiankun', - frameworkOptions: REVISION_FRAMEWORK_OPTIONS, - hostRole: 'candidate', - id: 'revision-candidate', - label: 'candidate · qiankun streamed', + candidate: 'native-iframe-cross-site', + id: 'native-cross-site-cost', + label: 'native iframe cross-site navigation cost (OOPIF verified)', + reference: 'native-iframe', + }, + { + candidate: 'wujie-cross-site-entry', + id: 'wujie-cross-site-cost', + label: 'Wujie cross-site entry cost', + reference: 'wujie-isolated', + }, + { + candidate: 'qk-full-isolation-cross-site', + id: 'qiankun-native-cross-site', + label: 'qiankun vs native iframe with cross-site entries', + reference: 'native-iframe-cross-site', + }, + { + candidate: 'wujie-cross-site-entry', + id: 'wujie-native-cross-site', + label: 'Wujie vs native iframe with cross-site entries', + reference: 'native-iframe-cross-site', + }, + { + candidate: 'wujie-cross-site-entry', + id: 'cross-site-framework', + label: 'Wujie vs qiankun with cross-site entries', + reference: 'qk-full-isolation-cross-site', }, ]; +const SSR_STREAMING_COMPARISONS = [ + { + candidate: 'qk-v3-ssr-streamed', + id: 'qiankun-v3-ssr-streaming-gain', + label: 'qiankun v3 streamed vs delayed-buffered SSR', + reference: 'qk-v3-ssr-delayed-buffered', + }, + { + candidate: 'qk-v3-ssr-streamed', + id: 'qiankun-v3-native-ssr-streamed', + label: 'qiankun v3 vs native iframe with streamed SSR', + reference: 'native-iframe-ssr-streamed', + }, + { + candidate: 'qk-v2-ssr-streamed', + id: 'qiankun-v2-v3-ssr-streamed', + label: 'qiankun v2 vs v3 with streamed SSR', + reference: 'qk-v3-ssr-streamed', + }, + { + candidate: 'wujie-ssr-streamed', + id: 'wujie-qiankun-v3-ssr-streamed', + label: 'Wujie vs qiankun v3 with streamed SSR', + reference: 'qk-v3-ssr-streamed', + }, + { + candidate: 'garfish-ssr-streamed', + id: 'garfish-qiankun-v3-ssr-streamed', + label: 'Garfish vs qiankun v3 with streamed SSR', + reference: 'qk-v3-ssr-streamed', + }, +]; + +export const SUITES = { + core: { + comparisons: CORE_COMPARISONS, + id: 'core', + title: 'Core HTML-entry matrix', + variants: [ + QK_NO_ISOLATION, + QK_SANDBOX, + QK_FULL_ISOLATION, + NATIVE_IFRAME, + WUJIE_ISOLATED, + NATIVE_IFRAME_STREAMED, + QK_STREAMED, + WUJIE_STREAMED, + ], + }, + 'site-isolation': { + comparisons: SITE_ISOLATION_COMPARISONS, + id: 'site-isolation', + title: 'Same-site and cross-site matrix', + variants: [ + QK_FULL_ISOLATION, + NATIVE_IFRAME, + WUJIE_ISOLATED, + QK_FULL_ISOLATION_CROSS_SITE, + NATIVE_IFRAME_CROSS_SITE, + WUJIE_CROSS_SITE, + ], + }, + 'ssr-streaming': { + calibrationSourceVariant: 'qk-v3-ssr-streamed', + comparisons: SSR_STREAMING_COMPARISONS, + id: 'ssr-streaming', + title: 'SSR streaming matrix', + variants: [ + NATIVE_IFRAME_SSR_STREAMED, + QK_V3_SSR_DELAYED_BUFFERED, + QK_V2_SSR_STREAMED, + QK_V3_SSR_STREAMED, + WUJIE_SSR_STREAMED, + GARFISH_SSR_STREAMED, + ], + }, + 'ecosystem-html': { + comparisons: ECOSYSTEM_COMPARISONS, + id: 'ecosystem-html', + title: 'HTML-entry ecosystem matrix', + variants: [ + NATIVE_IFRAME, + QK_V2_FULL_ISOLATION, + QK_FULL_ISOLATION, + WUJIE_ISOLATED, + MICRO_APP_DEFAULT_ISOLATION, + GARFISH_STRICT_ISOLATION, + ], + }, +}; + +export const PRODUCT_VARIANTS = SUITES.core.variants; +export const PRODUCT_COMPARISONS = SUITES.core.comparisons; + +export const CALIBRATION_VARIANTS = [ + { id: 'calibration-a', label: 'A/A · A', sourceVariant: 'qk-full-isolation' }, + { id: 'calibration-b', label: 'A/A · B', sourceVariant: 'qk-full-isolation' }, +]; + +const REVISION_SCENARIOS = { + sandbox: { + delivery: 'buffered', + frameworkOptions: { sandbox: true, styleIsolation: false }, + label: 'qiankun sandbox', + }, + streaming: { + delivery: 'streamed', + frameworkOptions: { sandbox: true, styleIsolation: true }, + label: 'qiankun streamed', + }, +}; + +export function createRevisionVariants(scenario = 'streaming') { + const config = REVISION_SCENARIOS[scenario]; + if (!config) throw new Error(`unknown revision scenario: ${scenario}`); + + return [ + { + delivery: config.delivery, + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: config.frameworkOptions, + hostRole: 'baseline', + id: 'revision-baseline', + label: `baseline · ${config.label}`, + }, + { + delivery: config.delivery, + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: config.frameworkOptions, + hostRole: 'candidate', + id: 'revision-candidate', + label: `candidate · ${config.label}`, + }, + ]; +} + +export const REVISION_VARIANTS = createRevisionVariants(); + export const REVISION_CALIBRATION_VARIANTS = [ { id: 'calibration-a', label: 'A/A · A', sourceVariant: 'revision-candidate' }, { id: 'calibration-b', label: 'A/A · B', sourceVariant: 'revision-candidate' }, diff --git a/benchmark/snapshot.mjs b/benchmark/snapshot.mjs index 11bbee88d..9f963927a 100644 --- a/benchmark/snapshot.mjs +++ b/benchmark/snapshot.mjs @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import { promisify } from 'node:util'; import { fileURLToPath } from 'node:url'; +import { createRevisionHarnessRecord } from './src/harness.mjs'; import { assertCleanBaselineGitState, createBaselineSnapshot } from './src/snapshot.mjs'; const execFileAsync = promisify(execFile); @@ -28,12 +29,15 @@ const name = parseName(process.argv.slice(2)); const targetDirectory = join(benchmarkRoot, 'artifacts', name); const git = await getGitState(); assertCleanBaselineGitState(git); +const harness = await createRevisionHarnessRecord(benchmarkRoot); const metadata = await createBaselineSnapshot({ createdAt: new Date().toISOString(), git, + harness, sourceDirectory: join(benchmarkRoot, 'fixtures/host/dist'), targetDirectory, }); console.log(`[benchmark] snapshot: ${targetDirectory}`); console.log(`[benchmark] bundle: ${metadata.bundleHash}`); +console.log(`[benchmark] harness: ${metadata.harness.fingerprint}`); diff --git a/benchmark/src/browser.mjs b/benchmark/src/browser.mjs index 4859a840b..691b2ea2b 100644 --- a/benchmark/src/browser.mjs +++ b/benchmark/src/browser.mjs @@ -1,4 +1,7 @@ -const BROWSER_CONTEXT_OPTIONS = { +import { resolveVariantEntryOrigin } from './origins.mjs'; +import { getVariantHostPage } from '../frameworks.mjs'; + +export const BROWSER_CONTEXT_OPTIONS = { colorScheme: 'light', deviceScaleFactor: 1, locale: 'en-US', @@ -8,6 +11,13 @@ const BROWSER_CONTEXT_OPTIONS = { viewport: { height: 720, width: 1280 }, }; +export const MEASUREMENT_CONTRACT = { + diagnostics: ['framework settlement', 'HTML response end', 'paint lead before response end'], + end: 'core element is paintable and survives two animation frames', + start: 'immediately before the prepared framework adapter invokes mount', + version: 4, +}; + async function withTimeout(promise, timeoutMs) { let timeout; const timeoutPromise = new Promise((_, reject) => { @@ -21,7 +31,7 @@ async function withTimeout(promise, timeoutMs) { } } -export async function runBrowserSample({ browser, fixtureOrigin, hostOrigin, timeoutMs, variant }) { +export async function runBrowserSample({ browser, fixtureOrigins, hostOrigin, timeoutMs, variant }) { const context = await browser.newContext(BROWSER_CONTEXT_OPTIONS); try { const page = await context.newPage(); @@ -38,24 +48,41 @@ export async function runBrowserSample({ browser, fixtureOrigin, hostOrigin, tim if (status >= 400) errors.push(`response: ${response.url()} (${status})`); }); - await page.goto(`${hostOrigin}/${variant.framework}.html`, { + const fixtureOriginSet = new Set(Object.values(fixtureOrigins)); + const fixtureRequestsBeforeRun = []; + let runStarted = false; + page.on('request', (request) => { + if (runStarted) return; + const url = new URL(request.url()); + if (fixtureOriginSet.has(url.origin)) fixtureRequestsBeforeRun.push(url.href); + }); + + await page.goto(`${hostOrigin}/${getVariantHostPage(variant)}`, { timeout: timeoutMs, waitUntil: 'load', }); + if (fixtureRequestsBeforeRun.length > 0) { + throw new Error(`framework adapter fetched the fixture before t0: ${fixtureRequestsBeforeRun[0]}`); + } + runStarted = true; const measurement = await withTimeout( page.evaluate( - async ({ delivery, entryOrigin, frameworkOptions, timeout }) => { + async ({ delivery, entryOrigin, frameworkOptions, htmlFixture, timeout }) => { if (!window.__BENCHMARK__) throw new Error('benchmark host API is unavailable'); + const entryUrl = new URL('/app/index.html', entryOrigin); + entryUrl.searchParams.set('delivery', delivery); + if (htmlFixture) entryUrl.searchParams.set('fixture', htmlFixture); return window.__BENCHMARK__.run({ - entry: `${entryOrigin}/app?delivery=${delivery}`, + entry: entryUrl.href, frameworkOptions, timeoutMs: timeout, }); }, { delivery: variant.delivery, - entryOrigin: fixtureOrigin, + entryOrigin: resolveVariantEntryOrigin(variant, fixtureOrigins), frameworkOptions: variant.frameworkOptions, + htmlFixture: variant.htmlFixture ?? null, timeout: timeoutMs, }, ), diff --git a/benchmark/src/fingerprint.mjs b/benchmark/src/fingerprint.mjs new file mode 100644 index 000000000..4492dd949 --- /dev/null +++ b/benchmark/src/fingerprint.mjs @@ -0,0 +1,43 @@ +import { createHash } from 'node:crypto'; + +function canonicalize(value) { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalize(value[key])]), + ); + } + return value; +} + +function collectChangedPaths(left, right, path = '') { + if (Object.is(left, right)) return []; + if (Array.isArray(left) || Array.isArray(right)) { + return JSON.stringify(left) === JSON.stringify(right) ? [] : [path || '']; + } + if (left && right && typeof left === 'object' && typeof right === 'object') { + const keys = new Set([...Object.keys(left), ...Object.keys(right)]); + return [...keys].sort().flatMap((key) => collectChangedPaths(left[key], right[key], path ? `${path}.${key}` : key)); + } + return [path || '']; +} + +export function createHarnessFingerprint(descriptor) { + return createHash('sha256') + .update(JSON.stringify(canonicalize(descriptor))) + .digest('hex'); +} + +export function assertComparableHarness(left, right) { + const expectedLeft = createHarnessFingerprint(left.descriptor); + const expectedRight = createHarnessFingerprint(right.descriptor); + if (left.fingerprint !== expectedLeft || right.fingerprint !== expectedRight) { + throw new Error('benchmark harness fingerprint does not match its descriptor'); + } + if (left.fingerprint === right.fingerprint) return; + + const changedPaths = collectChangedPaths(left.descriptor, right.descriptor); + throw new Error(`benchmark harnesses are not comparable; changed fields: ${changedPaths.join(', ')}`); +} diff --git a/benchmark/src/harness.mjs b/benchmark/src/harness.mjs new file mode 100644 index 000000000..e2acecfbe --- /dev/null +++ b/benchmark/src/harness.mjs @@ -0,0 +1,103 @@ +import { join } from 'node:path'; + +import { FRAMEWORK_ADAPTER_API_VERSION, getFrameworkAdapter, getVariantHostPage } from '../frameworks.mjs'; +import { BROWSER_CONTEXT_OPTIONS, MEASUREMENT_CONTRACT } from './browser.mjs'; +import { createHarnessFingerprint } from './fingerprint.mjs'; +import { hashFiles, hashViteEntries } from './hash.mjs'; +import { CHROMIUM_LAUNCH_ARGS } from './origins.mjs'; + +const HARNESS_SOURCE_FILES = [ + 'runner.mjs', + 'src/browser.mjs', + 'src/collect.mjs', + 'src/fingerprint.mjs', + 'src/harness.mjs', + 'src/hash.mjs', + 'src/origins.mjs', + 'src/report.mjs', + 'src/revisions.mjs', + 'src/schedule.mjs', + 'src/server.mjs', + 'src/site-isolation.mjs', + 'src/static-server.mjs', + 'src/stats.mjs', +]; + +const REVISION_HARNESS_SOURCE_FILES = [ + ...HARNESS_SOURCE_FILES, + '../pnpm-lock.yaml', + 'fixtures/host/qiankun.html', + 'fixtures/host/src/benchmark.ts', + 'fixtures/host/src/host.css', + 'fixtures/host/src/qiankun.ts', + 'fixtures/host/vite.config.ts', + 'frameworks.mjs', + 'package.json', + 'scenarios.mjs', + 'src/options.mjs', +]; + +export async function createRevisionHarnessRecord(benchmarkRoot) { + const descriptor = { + adapterApiVersion: FRAMEWORK_ADAPTER_API_VERSION, + browser: { + args: CHROMIUM_LAUNCH_ARGS, + context: BROWSER_CONTEXT_OPTIONS, + }, + measurement: MEASUREMENT_CONTRACT, + sourceHash: await hashFiles(benchmarkRoot, REVISION_HARNESS_SOURCE_FILES), + version: 1, + }; + return { descriptor, fingerprint: createHarnessFingerprint(descriptor) }; +} + +export async function createHarnessRecord({ + baselineBundleHash, + benchmarkRoot, + browserArgs, + browserVersion, + environment, + frameworkVersions, + options, + playwrightVersion, + runDefinition, +}) { + const hostPages = [...new Set(runDefinition.variants.map(getVariantHostPage))]; + const adapterIds = [...new Set(runDefinition.variants.map((variant) => variant.framework))]; + const hostDirectory = join(benchmarkRoot, 'fixtures/host/dist'); + const descriptor = { + adapterApiVersion: FRAMEWORK_ADAPTER_API_VERSION, + adapters: Object.fromEntries(adapterIds.map((id) => [id, getFrameworkAdapter(id)])), + browser: { + args: browserArgs, + context: BROWSER_CONTEXT_OPTIONS, + driverVersion: playwrightVersion, + name: 'chromium', + version: browserVersion, + }, + environment, + fixture: { + chunkIntervalMs: options.chunkIntervalMs, + hostBundleHash: await hashViteEntries(hostDirectory, hostPages), + }, + frameworks: frameworkVersions, + harnessSourceHash: await hashFiles(benchmarkRoot, HARNESS_SOURCE_FILES), + measurement: MEASUREMENT_CONTRACT, + revision: baselineBundleHash ? { baselineBundleHash } : null, + sampling: { + calibrationSamples: options.calibrationSamples, + samples: options.samples, + seed: options.seed, + timeoutMs: options.timeoutMs, + trials: options.trials, + warmup: options.warmup, + }, + suite: { + comparisons: runDefinition.comparisons, + id: options.mode === 'revision' ? `revision:${options.scenario}` : options.suite, + variants: runDefinition.variants, + }, + }; + + return { descriptor, fingerprint: createHarnessFingerprint(descriptor) }; +} diff --git a/benchmark/src/hash.mjs b/benchmark/src/hash.mjs new file mode 100644 index 000000000..f19e43328 --- /dev/null +++ b/benchmark/src/hash.mjs @@ -0,0 +1,61 @@ +import { createHash } from 'node:crypto'; +import { readFile, readdir } from 'node:fs/promises'; +import { join, relative } from 'node:path'; + +async function listFiles(directory) { + const files = []; + const entries = await readdir(directory, { withFileTypes: true }); + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...(await listFiles(path))); + } else if (entry.isFile()) { + files.push(path); + } + } + return files; +} + +async function hashRelativeFiles(directory, relativeFiles) { + const hash = createHash('sha256'); + for (const file of [...new Set(relativeFiles)].sort()) { + hash.update(file); + hash.update('\0'); + hash.update(await readFile(join(directory, file))); + hash.update('\0'); + } + return hash.digest('hex'); +} + +export async function hashDirectory(directory) { + const files = await listFiles(directory); + return hashRelativeFiles( + directory, + files.map((file) => relative(directory, file)), + ); +} + +export async function hashFiles(directory, files) { + return hashRelativeFiles(directory, files); +} + +export async function hashViteEntries(directory, entryHtmlFiles) { + const manifest = JSON.parse(await readFile(join(directory, '.vite/manifest.json'), 'utf8')); + const files = new Set(entryHtmlFiles); + const visited = new Set(); + + function visit(key) { + if (visited.has(key)) return; + visited.add(key); + const record = manifest[key]; + if (!record) throw new Error(`Vite manifest entry is missing: ${key}`); + files.add(record.file); + record.css?.forEach((file) => files.add(file)); + record.assets?.forEach((file) => files.add(file)); + record.imports?.forEach(visit); + record.dynamicImports?.forEach(visit); + } + + entryHtmlFiles.forEach(visit); + return hashRelativeFiles(directory, [...files]); +} diff --git a/benchmark/src/options.mjs b/benchmark/src/options.mjs index 32f6cdf68..51abece50 100644 --- a/benchmark/src/options.mjs +++ b/benchmark/src/options.mjs @@ -1,13 +1,16 @@ const DEFAULT_OPTIONS = { baselineDir: null, calibrationGate: true, - calibrationSamples: 50, + calibrationSamples: 100, chunkIntervalMs: 50, comparisonGate: true, mode: 'framework', samples: 100, + scenario: null, seed: 20260711, + suite: 'core', timeoutMs: 10_000, + trials: 3, warmup: 5, }; @@ -22,11 +25,13 @@ const INTEGER_OPTIONS = new Map([ ['samples', 'samples'], ['seed', 'seed'], ['timeout', 'timeoutMs'], + ['trials', 'trials'], ['warmup', 'warmup'], ]); export function parseRunnerOptions(args) { const options = { ...DEFAULT_OPTIONS }; + let trialsExplicit = false; for (const argument of args) { const match = /^--([^=]+)=(.+)$/u.exec(argument); if (!match) throw new Error(`unknown option: ${argument}`); @@ -49,6 +54,27 @@ export function parseRunnerOptions(args) { continue; } + if (name === 'scenario') { + if (rawValue !== 'streaming' && rawValue !== 'sandbox') { + throw new Error('scenario must be streaming or sandbox'); + } + options.scenario = rawValue; + continue; + } + + if (name === 'suite') { + if ( + rawValue !== 'core' && + rawValue !== 'site-isolation' && + rawValue !== 'ssr-streaming' && + rawValue !== 'ecosystem-html' + ) { + throw new Error('suite must be core, site-isolation, ssr-streaming, or ecosystem-html'); + } + options.suite = rawValue; + continue; + } + if (name === 'baseline-dir') { options.baselineDir = rawValue; continue; @@ -59,6 +85,7 @@ export function parseRunnerOptions(args) { const value = Number(rawValue); if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); options[property] = value; + if (name === 'trials') trialsExplicit = true; } if (options.mode === 'revision' && !options.baselineDir) { @@ -67,6 +94,17 @@ export function parseRunnerOptions(args) { if (options.mode === 'framework' && options.baselineDir) { throw new Error('baseline-dir requires revision mode'); } + if (options.mode === 'framework' && options.scenario) { + throw new Error('scenario requires revision mode'); + } + if (options.mode === 'revision' && !options.scenario) { + options.scenario = 'streaming'; + } + if (options.mode === 'revision') { + if (options.suite !== 'core') throw new Error('revision mode requires the core suite'); + if (trialsExplicit && options.trials !== 1) throw new Error('revision mode requires trials=1'); + options.trials = 1; + } return options; } diff --git a/benchmark/src/origins.mjs b/benchmark/src/origins.mjs new file mode 100644 index 000000000..2fb4fe0c1 --- /dev/null +++ b/benchmark/src/origins.mjs @@ -0,0 +1,25 @@ +export const CROSS_SITE_APP_HOSTNAME = 'benchmark-app.localhost'; + +export const CHROMIUM_LAUNCH_ARGS = ['--site-per-process']; + +function replaceHostname(origin, hostname) { + const url = new URL(origin); + url.hostname = hostname; + return url.origin; +} + +export function createBenchmarkOrigins({ fixtureOrigin, hostOrigins }) { + return { + fixtureOrigins: { + 'cross-site': replaceHostname(fixtureOrigin, CROSS_SITE_APP_HOSTNAME), + 'same-site': fixtureOrigin, + }, + hostOrigins, + }; +} + +export function resolveVariantEntryOrigin(variant, fixtureOrigins) { + const origin = fixtureOrigins[variant.entrySite]; + if (!origin) throw new Error(`fixture origin is unavailable for entry site: ${variant.entrySite}`); + return origin; +} diff --git a/benchmark/src/report.mjs b/benchmark/src/report.mjs index 4c7f93393..5289cd356 100644 --- a/benchmark/src/report.mjs +++ b/benchmark/src/report.mjs @@ -1,44 +1,93 @@ -import { comparePairedSamples, summarize } from './stats.mjs'; +import { comparePairedTrials, summarize } from './stats.mjs'; function formatSignedPercent(value) { return `${value >= 0 ? '+' : ''}${value.toFixed(2)}%`; } +function indexByRound(samples, comparisonId, trial, role) { + const byRound = new Map(); + for (const sample of samples) { + if (byRound.has(sample.round)) { + throw new Error(`${comparisonId} trial ${trial} has duplicate ${role} round ${sample.round}`); + } + byRound.set(sample.round, sample.duration); + } + return byRound; +} + +function summarizeMetric(samples, select) { + const values = samples.map(select).filter((value) => Number.isFinite(value)); + return values.length > 0 ? summarize(values) : null; +} + +function buildPhaseDiagnostics(samples) { + return { + entryResponseEndDuration: summarizeMetric(samples, (sample) => sample.entryResponseEndDuration), + paintLeadBeforeResponseEnd: summarizeMetric(samples, (sample) => + Number.isFinite(sample.entryResponseEndDuration) ? sample.entryResponseEndDuration - sample.duration : null, + ), + settledDuration: summarizeMetric(samples, (sample) => sample.settledDuration), + }; +} + export function buildReport({ comparisons, samples, seed, variants }) { const variantReports = {}; for (const variant of variants) { const matching = samples.filter((sample) => sample.variant === variant.id); const valid = matching.filter((sample) => sample.valid); + const trialIds = [...new Set(valid.map((sample) => sample.trial ?? 0))].sort((left, right) => left - right); variantReports[variant.id] = { + diagnostics: buildPhaseDiagnostics(valid), invalidCount: matching.length - valid.length, label: variant.label, summary: valid.length === 0 ? null : summarize(valid.map((sample) => sample.duration)), + trialSummaries: trialIds.map((trial) => ({ + summary: summarize(valid.filter((sample) => (sample.trial ?? 0) === trial).map((sample) => sample.duration)), + trial, + })), }; } const comparisonReports = {}; comparisons.forEach((comparison, comparisonIndex) => { - const referenceByRound = new Map( - samples - .filter((sample) => sample.valid && sample.variant === comparison.reference) - .map((sample) => [sample.round, sample.duration]), + const validComparisonSamples = samples.filter( + (sample) => sample.valid && (sample.variant === comparison.reference || sample.variant === comparison.candidate), ); - const candidateByRound = new Map( - samples - .filter((sample) => sample.valid && sample.variant === comparison.candidate) - .map((sample) => [sample.round, sample.duration]), - ); - const pairedRounds = [...referenceByRound.keys()].filter((round) => candidateByRound.has(round)); - const measured = comparePairedSamples( - pairedRounds.map((round) => referenceByRound.get(round)), - pairedRounds.map((round) => candidateByRound.get(round)), - { seed: seed + comparisonIndex }, + const trialIds = [...new Set(validComparisonSamples.map((sample) => sample.trial ?? 0))].sort( + (left, right) => left - right, ); + const pairedTrials = trialIds.flatMap((trial) => { + const trialSamples = validComparisonSamples.filter((sample) => (sample.trial ?? 0) === trial); + const referenceByRound = indexByRound( + trialSamples.filter((sample) => sample.variant === comparison.reference), + comparison.id, + trial, + 'reference', + ); + const candidateByRound = indexByRound( + trialSamples.filter((sample) => sample.variant === comparison.candidate), + comparison.id, + trial, + 'candidate', + ); + const pairedRounds = [...referenceByRound.keys()].filter((round) => candidateByRound.has(round)); + if (referenceByRound.size !== candidateByRound.size || pairedRounds.length !== referenceByRound.size) { + throw new Error(`${comparison.id} trial ${trial} has unpaired rounds`); + } + if (pairedRounds.length === 0) return []; + pairedRounds.sort((left, right) => left - right); + return [ + { + candidate: pairedRounds.map((round) => candidateByRound.get(round)), + reference: pairedRounds.map((round) => referenceByRound.get(round)), + }, + ]; + }); + const measured = comparePairedTrials(pairedTrials, { seed: seed + comparisonIndex }); comparisonReports[comparison.id] = { ...measured, candidate: comparison.candidate, label: comparison.label, - pairedCount: pairedRounds.length, reference: comparison.reference, }; }); @@ -68,17 +117,38 @@ export function renderSummaryMarkdown(report, { headingLevel = 1, title }) { ); }); + const hasPhaseDiagnostics = Object.values(report.variants).some((variant) => + Object.values(variant.diagnostics).some(Boolean), + ); + if (hasPhaseDiagnostics) { + lines.push( + '', + `${sectionHeading} Phase diagnostics`, + '', + '> Paint lead is HTML response end minus core paint; positive values prove the core painted before the full response completed.', + '', + '| Variant | Mount settled median (ms) | HTML response end median (ms) | Paint lead median (ms) |', + '| --- | ---: | ---: | ---: |', + ); + Object.values(report.variants).forEach((variant) => { + const diagnostics = variant.diagnostics; + lines.push( + `| ${variant.label} | ${diagnostics.settledDuration?.median.toFixed(2) ?? 'n/a'} | ${diagnostics.entryResponseEndDuration?.median.toFixed(2) ?? 'n/a'} | ${diagnostics.paintLeadBeforeResponseEnd?.median.toFixed(2) ?? 'n/a'} |`, + ); + }); + } + lines.push( '', `${sectionHeading} Comparisons`, '', - '| Comparison | Relative delta | 95% CI | Paired samples |', - '| --- | ---: | ---: | ---: |', + '| Comparison | Relative delta | 95% CI | Trials | Paired samples |', + '| --- | ---: | ---: | ---: | ---: |', ); Object.values(report.comparisons).forEach((comparison) => { const [lower, upper] = comparison.confidenceInterval95; lines.push( - `| ${comparison.label} | ${formatSignedPercent(comparison.relativeDeltaPercent)} | ${formatSignedPercent(lower)} to ${formatSignedPercent(upper)} | ${comparison.pairedCount} |`, + `| ${comparison.label} | ${formatSignedPercent(comparison.relativeDeltaPercent)} | ${formatSignedPercent(lower)} to ${formatSignedPercent(upper)} | ${comparison.trialCount} | ${comparison.pairedCount} |`, ); }); diff --git a/benchmark/src/result-comparison.mjs b/benchmark/src/result-comparison.mjs new file mode 100644 index 000000000..c5155c87d --- /dev/null +++ b/benchmark/src/result-comparison.mjs @@ -0,0 +1,39 @@ +import { assertComparableHarness } from './fingerprint.mjs'; + +function getHarness(result) { + const harness = result?.metadata?.harness; + if (!harness) throw new Error('benchmark result is missing its harness fingerprint'); + return harness; +} + +function getVariantMedians(result) { + const variants = result?.product?.report?.variants; + if (!variants) throw new Error('benchmark result is missing its product report'); + return variants; +} + +export function compareBenchmarkResults(reference, candidate) { + assertComparableHarness(getHarness(reference), getHarness(candidate)); + const referenceVariants = getVariantMedians(reference); + const candidateVariants = getVariantMedians(candidate); + const referenceIds = Object.keys(referenceVariants).sort(); + const candidateIds = Object.keys(candidateVariants).sort(); + if (JSON.stringify(referenceIds) !== JSON.stringify(candidateIds)) { + throw new Error('benchmark results contain different variant sets'); + } + + return referenceIds.map((id) => { + const referenceMedian = referenceVariants[id].summary?.median; + const candidateMedian = candidateVariants[id].summary?.median; + if (!Number.isFinite(referenceMedian) || !Number.isFinite(candidateMedian) || referenceMedian <= 0) { + throw new Error(`benchmark result has an invalid median for ${id}`); + } + return { + candidateMedian, + id, + label: candidateVariants[id].label, + relativeDeltaPercent: (candidateMedian / referenceMedian - 1) * 100, + referenceMedian, + }; + }); +} diff --git a/benchmark/src/server.mjs b/benchmark/src/server.mjs index 0d1c8ddac..c249eaec9 100644 --- a/benchmark/src/server.mjs +++ b/benchmark/src/server.mjs @@ -33,6 +33,9 @@ const ENTRY = ` return Promise.resolve(); } + global.mount = mountedRoot; + global.unmount = unmount; + global['benchmark-app'] = { bootstrap: function () { return Promise.resolve(); }, mount: function () { mountedRoot(); return Promise.resolve(); }, @@ -40,6 +43,25 @@ const ENTRY = ` }; global.__WUJIE_MOUNT = mountedRoot; global.__WUJIE_UNMOUNT = unmount; + + if (typeof __GARFISH_EXPORTS__ !== 'undefined') { + __GARFISH_EXPORTS__.provider = function () { + return { + destroy: unmount, + render: mountedRoot, + }; + }; + } + + if (global.location.search.indexOf('benchmark=native-iframe') !== -1) { + var benchmarkParams = new URLSearchParams(global.location.search); + var parentOrigin = benchmarkParams.get('benchmark-parent-origin'); + var token = benchmarkParams.get('benchmark-token'); + if (parentOrigin && token) { + mountedRoot(); + global.parent.postMessage({ type: 'native-app-mounted', version: 1, token: token }, parentOrigin); + } + } })(window); `.trim(); @@ -60,7 +82,52 @@ const HTML_CHUNKS = [ for (var index = 0; index < 100; index++) { rows += '
' + index + 'deterministic benchmark row
'; } - root.innerHTML = '

micro app core

' + rows + '
'; + root.innerHTML = '

micro app core

' + rows + '
'; + + if (window.location.search.indexOf('benchmark=native-iframe') !== -1) { + var benchmarkParams = new URLSearchParams(window.location.search); + var parentOrigin = benchmarkParams.get('benchmark-parent-origin'); + var token = benchmarkParams.get('benchmark-token'); + if (parentOrigin && token) { + var isPaintable = function isPaintable(element) { + var rect = element.getBoundingClientRect(); + var style = getComputedStyle(element); + return rect.width > 0 && + rect.height > 0 && + element.querySelector('[data-benchmark-critical]') !== null && + style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + style.getPropertyValue('--benchmark-style-ready').trim() === '1'; + }; + + var waitForPaint = function waitForPaint() { + var core = document.querySelector('#benchmark-core'); + if (!core || !isPaintable(core)) { + requestAnimationFrame(waitForPaint); + return; + } + + requestAnimationFrame(function () { + requestAnimationFrame(function () { + var paintedCore = document.querySelector('#benchmark-core'); + if (!paintedCore || !isPaintable(paintedCore)) { + requestAnimationFrame(waitForPaint); + return; + } + window.parent.postMessage({ + paintedAt: performance.timeOrigin + performance.now(), + token: token, + type: 'native-core-painted', + version: 1 + }, parentOrigin); + }); + }); + }; + + requestAnimationFrame(waitForPaint); + } + } })(); `, ` @@ -70,13 +137,106 @@ const HTML_CHUNKS = [ const FULL_HTML = HTML_CHUNKS.join(''); -function setSharedHeaders(response) { +function renderSsrRows(start, count, label) { + return Array.from( + { length: count }, + (_, offset) => `
${start + offset}${label} ${start + offset}
`, + ).join(''); +} + +const SSR_NATIVE_PAINT_REPORTER = ``; + +const SSR_HTML_CHUNKS = [ + ` + + + + benchmark SSR dashboard + + + +
+
+
+

Account overview

+

Server-rendered critical account content

+
+ ${renderSsrRows(0, 12, 'critical transaction')} +
+ ${SSR_NATIVE_PAINT_REPORTER}`, + `
+

Recommended actions

+ ${renderSsrRows(12, 44, 'deferred recommendation')} +
`, + `
+

Recent activity

+ ${renderSsrRows(56, 44, 'deferred activity')} +
+
+
+ + + +`, +]; + +const SSR_FULL_HTML = SSR_HTML_CHUNKS.join(''); +const ASSET_CACHE_CONTROL = 'public, max-age=31536000, immutable'; +const HTML_CACHE_CONTROL = 'no-store'; + +function setSharedHeaders(response, cacheControl = HTML_CACHE_CONTROL) { response.setHeader('Access-Control-Allow-Origin', '*'); - response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Cache-Control', cacheControl); + response.setHeader('Timing-Allow-Origin', '*'); } -function sendText(response, statusCode, contentType, body) { - setSharedHeaders(response); +function sendText(response, statusCode, contentType, body, cacheControl = HTML_CACHE_CONTROL) { + setSharedHeaders(response, cacheControl); response.writeHead(statusCode, { 'Content-Length': Buffer.byteLength(body), 'Content-Type': contentType, @@ -103,9 +263,15 @@ function close(server) { export function createFixtureServer({ chunkIntervalMs = 50, host = '127.0.0.1', port = 7601 } = {}) { let origin; + const requestCounts = new Map(); const server = createServer((request, response) => { - setSharedHeaders(response); + const url = new URL(request.url ?? '/', origin ?? `http://${host}:${port}`); + if (request.method !== 'OPTIONS') { + requestCounts.set(url.pathname, (requestCounts.get(url.pathname) ?? 0) + 1); + } + if (request.method === 'OPTIONS') { + setSharedHeaders(response); response.writeHead(204, { 'Access-Control-Allow-Headers': '*', 'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS', @@ -114,23 +280,36 @@ export function createFixtureServer({ chunkIntervalMs = 50, host = '127.0.0.1', return; } - const url = new URL(request.url ?? '/', origin ?? `http://${host}:${port}`); - if (url.pathname === '/style.css') { - sendText(response, 200, 'text/css; charset=utf-8', FIXTURE_STYLE); + if (url.pathname === '/app/style.css') { + sendText(response, 200, 'text/css; charset=utf-8', FIXTURE_STYLE, ASSET_CACHE_CONTROL); return; } - if (url.pathname === '/entry.js') { - sendText(response, 200, 'text/javascript; charset=utf-8', ENTRY); + if (url.pathname === '/app/entry.js') { + sendText(response, 200, 'text/javascript; charset=utf-8', ENTRY, ASSET_CACHE_CONTROL); return; } - if (url.pathname !== '/app') { + if (url.pathname !== '/app/index.html') { sendText(response, 404, 'text/plain; charset=utf-8', 'Not Found'); return; } + const fixtureName = url.searchParams.get('fixture') ?? 'client-rendered'; + const chunks = fixtureName === 'ssr' ? SSR_HTML_CHUNKS : fixtureName === 'client-rendered' ? HTML_CHUNKS : null; + if (!chunks) { + sendText(response, 400, 'text/plain; charset=utf-8', 'Unknown HTML fixture'); + return; + } + + const fullHtml = fixtureName === 'ssr' ? SSR_FULL_HTML : FULL_HTML; const delivery = url.searchParams.get('delivery') ?? 'buffered'; if (delivery === 'buffered') { - sendText(response, 200, 'text/html; charset=utf-8', FULL_HTML); + sendText(response, 200, 'text/html; charset=utf-8', fullHtml); + return; + } + if (delivery === 'delayed-buffered') { + const delay = chunkIntervalMs * (chunks.length - 1); + const timer = setTimeout(() => sendText(response, 200, 'text/html; charset=utf-8', fullHtml), delay); + response.once('close', () => clearTimeout(timer)); return; } if (delivery !== 'streamed') { @@ -138,13 +317,19 @@ export function createFixtureServer({ chunkIntervalMs = 50, host = '127.0.0.1', return; } + setSharedHeaders(response); response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); - response.write(HTML_CHUNKS[0]); - const secondTimer = setTimeout(() => response.write(HTML_CHUNKS[1]), chunkIntervalMs); - const finalTimer = setTimeout(() => response.end(HTML_CHUNKS[2]), chunkIntervalMs * 2); + response.write(chunks[0]); + const timers = chunks + .slice(1) + .map((chunk, index) => + setTimeout( + () => (index === chunks.length - 2 ? response.end(chunk) : response.write(chunk)), + chunkIntervalMs * (index + 1), + ), + ); response.once('close', () => { - clearTimeout(secondTimer); - clearTimeout(finalTimer); + timers.forEach((timer) => clearTimeout(timer)); }); }); @@ -156,6 +341,12 @@ export function createFixtureServer({ chunkIntervalMs = 50, host = '127.0.0.1', if (!origin) throw new Error('fixture server has not started'); return origin; }, + getRequestCount(pathname) { + return requestCounts.get(pathname) ?? 0; + }, + resetRequestCounts() { + requestCounts.clear(); + }, async start() { await listen(server, port, host); const address = server.address(); diff --git a/benchmark/src/site-isolation.mjs b/benchmark/src/site-isolation.mjs new file mode 100644 index 000000000..fbde7a876 --- /dev/null +++ b/benchmark/src/site-isolation.mjs @@ -0,0 +1,62 @@ +import { runBrowserSample } from './browser.mjs'; + +function isIframeTargetAtOrigin(targetInfo, origin) { + if (targetInfo.type !== 'iframe') return false; + try { + return new URL(targetInfo.url).origin === origin; + } catch { + return false; + } +} + +async function observeTargetsDuring(session, sample) { + const targets = new Map(); + const recordTarget = ({ targetInfo }) => targets.set(targetInfo.targetId, targetInfo); + session.on('Target.targetCreated', recordTarget); + session.on('Target.targetInfoChanged', recordTarget); + try { + await sample(); + return [...targets.values()]; + } finally { + session.off('Target.targetCreated', recordTarget); + session.off('Target.targetInfoChanged', recordTarget); + } +} + +export async function inspectCrossSiteIsolation({ + browser, + fixtureOrigins, + hostOrigin, + nativeVariant, + timeoutMs, + wujieVariant, +}) { + const session = await browser.newBrowserCDPSession(); + const runVariant = (variant) => runBrowserSample({ browser, fixtureOrigins, hostOrigin, timeoutMs, variant }); + try { + await session.send('Target.setDiscoverTargets', { discover: true }); + const nativeTargets = await observeTargetsDuring(session, () => runVariant(nativeVariant)); + const crossSiteOrigin = fixtureOrigins['cross-site']; + const nativeOopifTarget = nativeTargets.find((target) => isIframeTargetAtOrigin(target, crossSiteOrigin)); + if (!nativeOopifTarget) { + throw new Error(`cross-site native iframe did not create an OOPIF target at ${crossSiteOrigin}`); + } + + const wujieTargets = await observeTargetsDuring(session, () => runVariant(wujieVariant)); + const wujieAppSiteTarget = wujieTargets.find((target) => isIframeTargetAtOrigin(target, crossSiteOrigin)); + return { + crossSiteEntryOrigin: crossSiteOrigin, + nativeIframe: { + oopif: true, + targetUrl: nativeOopifTarget.url, + }, + wujie: { + appSiteOopif: Boolean(wujieAppSiteTarget), + targetUrl: wujieAppSiteTarget?.url ?? null, + }, + }; + } finally { + await session.send('Target.setDiscoverTargets', { discover: false }).catch(() => {}); + await session.detach().catch(() => {}); + } +} diff --git a/benchmark/src/snapshot.mjs b/benchmark/src/snapshot.mjs index aeeb609cb..ea157d5ea 100644 --- a/benchmark/src/snapshot.mjs +++ b/benchmark/src/snapshot.mjs @@ -1,31 +1,8 @@ -import { createHash } from 'node:crypto'; -import { access, cp, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, relative } from 'node:path'; +import { access, cp, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; -async function listFiles(directory) { - const files = []; - const entries = await readdir(directory, { withFileTypes: true }); - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - const path = join(directory, entry.name); - if (entry.isDirectory()) { - files.push(...(await listFiles(path))); - } else if (entry.isFile()) { - files.push(path); - } - } - return files; -} - -async function hashDirectory(directory) { - const hash = createHash('sha256'); - for (const file of await listFiles(directory)) { - hash.update(relative(directory, file)); - hash.update('\0'); - hash.update(await readFile(file)); - hash.update('\0'); - } - return hash.digest('hex'); -} +import { assertComparableHarness } from './fingerprint.mjs'; +import { hashDirectory } from './hash.mjs'; async function assertMissing(path) { try { @@ -37,7 +14,8 @@ async function assertMissing(path) { throw new Error(`baseline snapshot already exists: ${path}`); } -export async function createBaselineSnapshot({ createdAt, git, sourceDirectory, targetDirectory }) { +export async function createBaselineSnapshot({ createdAt, git, harness, sourceDirectory, targetDirectory }) { + assertComparableHarness(harness, harness); await assertMissing(targetDirectory); const stagingDirectory = `${targetDirectory}.tmp-${process.pid}`; await assertMissing(stagingDirectory); @@ -47,7 +25,7 @@ export async function createBaselineSnapshot({ createdAt, git, sourceDirectory, await mkdir(stagingDirectory); await cp(sourceDirectory, join(stagingDirectory, 'host'), { recursive: true }); const bundleHash = await hashDirectory(join(stagingDirectory, 'host')); - const metadata = { bundleHash, createdAt, git, schemaVersion: 1 }; + const metadata = { bundleHash, createdAt, git, harness, schemaVersion: 2 }; await writeFile(join(stagingDirectory, 'metadata.json'), `${JSON.stringify(metadata, null, 2)}\n`); await rename(stagingDirectory, targetDirectory); return metadata; @@ -62,23 +40,36 @@ export function assertCleanBaselineGitState(git) { } function validateMetadata(metadata) { + if (metadata?.schemaVersion !== 2) { + throw new Error('baseline snapshot metadata is incompatible; recreate the snapshot'); + } if ( !metadata || typeof metadata !== 'object' || - metadata.schemaVersion !== 1 || typeof metadata.bundleHash !== 'string' || !/^[a-f\d]{64}$/u.test(metadata.bundleHash) || typeof metadata.createdAt !== 'string' || !metadata.git || typeof metadata.git !== 'object' || typeof metadata.git.commit !== 'string' || - typeof metadata.git.dirty !== 'boolean' + typeof metadata.git.dirty !== 'boolean' || + !metadata.harness || + typeof metadata.harness !== 'object' ) { throw new Error('baseline snapshot metadata is invalid'); } + try { + assertComparableHarness(metadata.harness, metadata.harness); + } catch { + throw new Error('baseline snapshot metadata is invalid'); + } return metadata; } +export function assertBaselineHarnessCompatible(metadata, currentHarness) { + assertComparableHarness(metadata.harness, currentHarness); +} + export async function readBaselineSnapshot(directory) { const metadata = validateMetadata(JSON.parse(await readFile(join(directory, 'metadata.json'), 'utf8'))); const actualHash = await hashDirectory(join(directory, 'host')); diff --git a/benchmark/src/stats.mjs b/benchmark/src/stats.mjs index 13fe8af90..2ecb294fc 100644 --- a/benchmark/src/stats.mjs +++ b/benchmark/src/stats.mjs @@ -80,6 +80,50 @@ export function comparePairedSamples(reference, candidate, { iterations = 10_000 }; } +export function comparePairedTrials(trials, { iterations = 10_000, seed = 20260711 } = {}) { + if (trials.length === 0) throw new Error('at least one trial is required'); + if (!Number.isInteger(iterations) || iterations <= 0) { + throw new Error('bootstrap iterations must be a positive integer'); + } + + const logRatiosByTrial = trials.map(({ candidate, reference }) => { + if (reference.length !== candidate.length) throw new Error('paired sample arrays must have the same length'); + validateSamples(reference, { positive: true }); + validateSamples(candidate, { positive: true }); + return reference.map((sample, index) => Math.log(candidate[index] / sample)); + }); + const trialMedianLogRatios = logRatiosByTrial.map((logRatios) => + medianSorted([...logRatios].sort((left, right) => left - right)), + ); + const estimate = medianSorted([...trialMedianLogRatios].sort((left, right) => left - right)); + const random = createSeededRandom(seed); + const bootstrap = []; + + for (let iteration = 0; iteration < iterations; iteration += 1) { + const resampledTrialMedians = []; + for (let trialIndex = 0; trialIndex < logRatiosByTrial.length; trialIndex += 1) { + const selectedTrial = logRatiosByTrial[Math.floor(random() * logRatiosByTrial.length)]; + const resampled = []; + for (let index = 0; index < selectedTrial.length; index += 1) { + resampled.push(selectedTrial[Math.floor(random() * selectedTrial.length)]); + } + resampled.sort((left, right) => left - right); + resampledTrialMedians.push(medianSorted(resampled)); + } + resampledTrialMedians.sort((left, right) => left - right); + bootstrap.push(Math.expm1(medianSorted(resampledTrialMedians)) * 100); + } + bootstrap.sort((left, right) => left - right); + + return { + confidenceInterval95: [quantileSorted(bootstrap, 0.025), quantileSorted(bootstrap, 0.975)], + pairedCount: logRatiosByTrial.reduce((total, logRatios) => total + logRatios.length, 0), + relativeDeltaPercent: Math.expm1(estimate) * 100, + trialCount: trials.length, + trialRelativeDeltas: trialMedianLogRatios.map((value) => Math.expm1(value) * 100), + }; +} + export function evaluateCalibration( comparison, { maxAbsoluteDeltaPercent = 3, maxIntervalWidthPercentPoints = 10 } = {}, diff --git a/benchmark/tests/browser.integration.test.mjs b/benchmark/tests/browser.integration.test.mjs index 449d8ed24..5f4830d7b 100644 --- a/benchmark/tests/browser.integration.test.mjs +++ b/benchmark/tests/browser.integration.test.mjs @@ -5,9 +5,11 @@ import test from 'node:test'; import { chromium } from 'playwright'; import { runBrowserSample } from '../src/browser.mjs'; +import { CHROMIUM_LAUNCH_ARGS, createBenchmarkOrigins } from '../src/origins.mjs'; import { createFixtureServer } from '../src/server.mjs'; +import { inspectCrossSiteIsolation } from '../src/site-isolation.mjs'; import { createStaticServer } from '../src/static-server.mjs'; -import { PRODUCT_VARIANTS } from '../scenarios.mjs'; +import { SUITES } from '../scenarios.mjs'; const hostRoot = fileURLToPath(new URL('../fixtures/host/dist', import.meta.url)); @@ -15,20 +17,113 @@ test('every benchmark variant reaches a painted core element and a settled app', const fixture = createFixtureServer({ chunkIntervalMs: 10, port: 0 }); const host = createStaticServer({ port: 0, root: hostRoot }); await Promise.all([fixture.start(), host.start()]); - const browser = await chromium.launch({ headless: true }); + const browser = await chromium.launch({ args: CHROMIUM_LAUNCH_ARGS, headless: true }); try { - for (const variant of PRODUCT_VARIANTS) { - const measurement = await runBrowserSample({ + const { fixtureOrigins } = createBenchmarkOrigins({ + fixtureOrigin: fixture.origin, + hostOrigins: { candidate: host.origin }, + }); + const siteIsolationVariants = SUITES['site-isolation'].variants; + const nativeCrossSite = siteIsolationVariants.find(({ id }) => id === 'native-iframe-cross-site'); + const wujieCrossSite = siteIsolationVariants.find(({ id }) => id === 'wujie-cross-site-entry'); + assert.ok(nativeCrossSite); + assert.ok(wujieCrossSite); + + const siteIsolation = await inspectCrossSiteIsolation({ + browser, + fixtureOrigins, + hostOrigin: host.origin, + nativeVariant: nativeCrossSite, + timeoutMs: 10_000, + wujieVariant: wujieCrossSite, + }); + assert.equal(siteIsolation.crossSiteEntryOrigin, fixtureOrigins['cross-site']); + assert.equal(siteIsolation.nativeIframe.oopif, true); + assert.match(siteIsolation.nativeIframe.targetUrl, /^http:\/\/benchmark-app\.localhost:/u); + assert.equal(siteIsolation.wujie.appSiteOopif, false); + assert.equal(siteIsolation.wujie.targetUrl, null); + + const uniqueVariants = [ + ...new Map( + Object.values(SUITES) + .flatMap((suite) => suite.variants) + .map((variant) => [variant.id, variant]), + ).values(), + ]; + for (const variant of uniqueVariants) { + fixture.resetRequestCounts(); + let measurement; + try { + measurement = await runBrowserSample({ + browser, + fixtureOrigins, + hostOrigin: host.origin, + timeoutMs: 10_000, + variant, + }); + } catch (error) { + throw new Error( + `${variant.id}: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? { cause: error } : undefined, + ); + } + assert.ok(measurement.duration > 0, variant.id); + assert.ok(measurement.t1 >= measurement.t0, variant.id); + assert.equal(measurement.cleaned, true, variant.id); + assert.equal(measurement.settled, true, variant.id); + if (variant.id === 'qk-full-isolation') { + assert.equal(fixture.getRequestCount('/app/entry.js'), 1, 'qiankun preload must satisfy the sandbox fetch'); + assert.equal(fixture.getRequestCount('/app/style.css'), 1, 'qiankun preload must satisfy the style fetch'); + } + } + } finally { + await browser.close(); + await Promise.all([fixture.close(), host.close()]); + } +}); + +test('SSR phase diagnostics prove progressive paint before the full response', async () => { + const chunkIntervalMs = 100; + const fixture = createFixtureServer({ chunkIntervalMs, port: 0 }); + const host = createStaticServer({ port: 0, root: hostRoot }); + await Promise.all([fixture.start(), host.start()]); + const browser = await chromium.launch({ args: CHROMIUM_LAUNCH_ARGS, headless: true }); + + try { + const { fixtureOrigins } = createBenchmarkOrigins({ + fixtureOrigin: fixture.origin, + hostOrigins: { candidate: host.origin }, + }); + const variants = new Map(SUITES['ssr-streaming'].variants.map((variant) => [variant.id, variant])); + const run = (id) => + runBrowserSample({ browser, - fixtureOrigin: fixture.origin, + fixtureOrigins, hostOrigin: host.origin, timeoutMs: 10_000, - variant, + variant: variants.get(id), }); - assert.ok(measurement.duration > 0, variant.id); - assert.ok(measurement.t1 >= measurement.t0, variant.id); - assert.equal(measurement.settled, true, variant.id); + + const progressive = await run('qk-v3-ssr-streamed'); + assert.ok(progressive.entryResponseEndDuration >= chunkIntervalMs * 2); + assert.ok(progressive.duration < progressive.entryResponseEndDuration); + assert.ok(progressive.settledDuration >= progressive.entryResponseEndDuration); + + const native = await run('native-iframe-ssr-streamed'); + assert.ok(native.duration < chunkIntervalMs * 2); + assert.ok(native.settledDuration >= chunkIntervalMs * 2); + assert.equal(native.entryResponseEndDuration, null); + + for (const id of [ + 'qk-v3-ssr-delayed-buffered', + 'qk-v2-ssr-streamed', + 'wujie-ssr-streamed', + 'garfish-ssr-streamed', + ]) { + const buffered = await run(id); + assert.ok(buffered.entryResponseEndDuration >= chunkIntervalMs * 2, id); + assert.ok(buffered.duration >= buffered.entryResponseEndDuration, id); } } finally { await browser.close(); diff --git a/benchmark/tests/browser.test.mjs b/benchmark/tests/browser.test.mjs index 5e2a42d9d..16dae1948 100644 --- a/benchmark/tests/browser.test.mjs +++ b/benchmark/tests/browser.test.mjs @@ -2,13 +2,73 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { runBrowserSample } from '../src/browser.mjs'; +import { createBenchmarkOrigins, resolveVariantEntryOrigin } from '../src/origins.mjs'; const VARIANT = { delivery: 'buffered', + entrySite: 'same-site', framework: 'qiankun', frameworkOptions: {}, + htmlFixture: 'ssr', }; +const FIXTURE_ORIGINS = { + 'cross-site': 'http://benchmark-app.localhost:7601', + 'same-site': 'http://127.0.0.1:7601', +}; + +test('createBenchmarkOrigins preserves the fixture port while changing the cross-site hostname', () => { + const hostOrigins = { candidate: 'http://127.0.0.1:7600' }; + const origins = createBenchmarkOrigins({ fixtureOrigin: FIXTURE_ORIGINS['same-site'], hostOrigins }); + + assert.deepEqual(origins, { fixtureOrigins: FIXTURE_ORIGINS, hostOrigins }); + assert.notEqual( + new URL(origins.fixtureOrigins['cross-site']).hostname, + new URL(origins.fixtureOrigins['same-site']).hostname, + ); +}); + +test('resolveVariantEntryOrigin requires an origin for the declared entry site', () => { + assert.equal(resolveVariantEntryOrigin(VARIANT, FIXTURE_ORIGINS), FIXTURE_ORIGINS['same-site']); + assert.equal( + resolveVariantEntryOrigin({ ...VARIANT, entrySite: 'cross-site' }, FIXTURE_ORIGINS), + FIXTURE_ORIGINS['cross-site'], + ); + assert.throws( + () => resolveVariantEntryOrigin({ ...VARIANT, entrySite: 'missing' }, FIXTURE_ORIGINS), + /fixture origin is unavailable for entry site: missing/u, + ); +}); + +test('runBrowserSample sends the variant entry-site origin to the host page', async () => { + let evaluationOptions; + const page = { + evaluate: async (_callback, options) => { + evaluationOptions = options; + return { duration: 10, settled: true, t0: 1, t1: 11 }; + }, + goto: async () => {}, + on: () => {}, + }; + const browser = { + newContext: async () => ({ + close: async () => {}, + newPage: async () => page, + }), + }; + + await runBrowserSample({ + browser, + fixtureOrigins: FIXTURE_ORIGINS, + hostOrigin: 'http://127.0.0.1:7600', + timeoutMs: 10, + variant: { ...VARIANT, entrySite: 'cross-site', framework: 'native' }, + }); + + assert.equal(evaluationOptions.entryOrigin, FIXTURE_ORIGINS['cross-site']); + assert.equal(evaluationOptions.htmlFixture, 'ssr'); +}); + test('runBrowserSample times out a hung benchmark and closes its context', { timeout: 1_000 }, async () => { let contextClosed = false; const page = { @@ -29,7 +89,7 @@ test('runBrowserSample times out a hung benchmark and closes its context', { tim await assert.rejects( runBrowserSample({ browser, - fixtureOrigin: 'http://127.0.0.1:7601', + fixtureOrigins: FIXTURE_ORIGINS, hostOrigin: 'http://127.0.0.1:7600', timeoutMs: 10, variant: VARIANT, @@ -55,7 +115,7 @@ test('runBrowserSample closes its context when page creation fails', async () => await assert.rejects( runBrowserSample({ browser, - fixtureOrigin: 'http://127.0.0.1:7601', + fixtureOrigins: FIXTURE_ORIGINS, hostOrigin: 'http://127.0.0.1:7600', timeoutMs: 10, variant: VARIANT, @@ -88,7 +148,7 @@ test('runBrowserSample rejects HTTP error responses', async () => { await assert.rejects( runBrowserSample({ browser, - fixtureOrigin: 'http://127.0.0.1:7601', + fixtureOrigins: FIXTURE_ORIGINS, hostOrigin: 'http://127.0.0.1:7600', timeoutMs: 10, variant: VARIANT, @@ -96,3 +156,33 @@ test('runBrowserSample rejects HTTP error responses', async () => { /response: http:\/\/127\.0\.0\.1:7601\/missing\.css \(404\)/u, ); }); + +test('runBrowserSample rejects adapters that fetch fixture resources before t0', async () => { + const handlers = new Map(); + const page = { + evaluate: async () => ({ cleaned: true, duration: 10, settled: true, t0: 1, t1: 11 }), + goto: async () => { + handlers.get('request')?.({ + url: () => `${FIXTURE_ORIGINS['same-site']}/app/index.html?delivery=buffered`, + }); + }, + on: (event, handler) => handlers.set(event, handler), + }; + const browser = { + newContext: async () => ({ + close: async () => {}, + newPage: async () => page, + }), + }; + + await assert.rejects( + runBrowserSample({ + browser, + fixtureOrigins: FIXTURE_ORIGINS, + hostOrigin: 'http://127.0.0.1:7600', + timeoutMs: 10, + variant: VARIANT, + }), + /framework adapter fetched the fixture before t0/u, + ); +}); diff --git a/benchmark/tests/fingerprint.test.mjs b/benchmark/tests/fingerprint.test.mjs new file mode 100644 index 000000000..74a766bfe --- /dev/null +++ b/benchmark/tests/fingerprint.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { assertComparableHarness, createHarnessFingerprint } from '../src/fingerprint.mjs'; + +const descriptor = { + browser: { + launchArgs: ['--site-per-process'], + name: 'chromium', + version: '149.0.7756.0', + }, + fixture: { + bundleHash: 'fixture-sha256', + chunkIntervalMs: 50, + }, + frameworks: { + qiankun: '3.0.0-alpha.0', + wujie: '2.1.0', + }, + harnessSourceHash: 'harness-sha256', + measurement: { + end: 'core-element-visible-after-two-animation-frames', + start: 'immediately-before-framework-load', + }, + variants: [ + { delivery: 'buffered', framework: 'native', id: 'native-iframe' }, + { delivery: 'buffered', framework: 'qiankun', id: 'qiankun-full' }, + ], +}; + +function harnessRecord(value) { + return { descriptor: value, fingerprint: createHarnessFingerprint(value) }; +} + +test('createHarnessFingerprint is canonical and sensitive to execution-path inputs', () => { + const reordered = { + variants: descriptor.variants.map(({ delivery, framework, id }) => ({ id, framework, delivery })), + measurement: { start: descriptor.measurement.start, end: descriptor.measurement.end }, + harnessSourceHash: descriptor.harnessSourceHash, + frameworks: { wujie: descriptor.frameworks.wujie, qiankun: descriptor.frameworks.qiankun }, + fixture: { chunkIntervalMs: descriptor.fixture.chunkIntervalMs, bundleHash: descriptor.fixture.bundleHash }, + browser: { + version: descriptor.browser.version, + name: descriptor.browser.name, + launchArgs: descriptor.browser.launchArgs, + }, + }; + const fingerprint = createHarnessFingerprint(descriptor); + + assert.match(fingerprint, /^[a-f\d]{64}$/u); + assert.equal(createHarnessFingerprint(reordered), fingerprint); + + const changedLaunchArgs = structuredClone(descriptor); + changedLaunchArgs.browser.launchArgs.push('--disable-background-timer-throttling'); + assert.notEqual(createHarnessFingerprint(changedLaunchArgs), fingerprint); + + const changedFramework = structuredClone(descriptor); + changedFramework.frameworks.wujie = '2.2.0'; + assert.notEqual(createHarnessFingerprint(changedFramework), fingerprint); + + const changedVariants = structuredClone(descriptor); + changedVariants.variants.reverse(); + assert.notEqual(createHarnessFingerprint(changedVariants), fingerprint); +}); + +test('assertComparableHarness rejects cross-run comparisons and identifies the changed field', () => { + const left = harnessRecord(descriptor); + const sameHarness = harnessRecord(structuredClone(descriptor)); + assert.doesNotThrow(() => assertComparableHarness(left, sameHarness)); + + const changed = structuredClone(descriptor); + changed.browser.launchArgs.push('--disable-renderer-backgrounding'); + assert.throws(() => assertComparableHarness(left, harnessRecord(changed)), /browser\.launchArgs/u); + assert.throws(() => assertComparableHarness(left, { descriptor, fingerprint: '0'.repeat(64) }), /fingerprint/u); +}); diff --git a/benchmark/tests/frameworks.test.mjs b/benchmark/tests/frameworks.test.mjs new file mode 100644 index 000000000..3c41e61b3 --- /dev/null +++ b/benchmark/tests/frameworks.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + FRAMEWORK_ADAPTERS, + assertFrameworkVariantSupported, + getFrameworkAdapter, + getVariantHostPage, +} from '../frameworks.mjs'; +import { SUITES } from '../scenarios.mjs'; + +test('every suite variant resolves through an explicit framework adapter', () => { + for (const variant of Object.values(SUITES).flatMap((suite) => suite.variants)) { + const adapter = getFrameworkAdapter(variant.framework); + assert.equal(getVariantHostPage(variant), adapter.hostPage); + assert.equal(adapter.capabilities.htmlEntry, true); + assert.doesNotThrow(() => assertFrameworkVariantSupported(variant)); + } +}); + +test('framework adapters own host filenames and reject unknown ids', () => { + assert.deepEqual(Object.keys(FRAMEWORK_ADAPTERS).sort(), [ + 'garfish', + 'micro-app', + 'native', + 'qiankun', + 'qiankun-v2', + 'wujie', + ]); + assert.equal(getVariantHostPage({ framework: 'qiankun' }), 'qiankun.html'); + assert.equal(getVariantHostPage({ framework: 'qiankun-v2' }), 'qiankun-v2.html'); + assert.equal(FRAMEWORK_ADAPTERS.native.capabilities.progressiveHtmlParsing, true); + assert.equal(FRAMEWORK_ADAPTERS.qiankun.capabilities.progressiveHtmlParsing, true); + for (const id of ['garfish', 'micro-app', 'qiankun-v2', 'wujie']) { + assert.equal(FRAMEWORK_ADAPTERS[id].capabilities.acceptsChunkedResponse, true); + assert.equal(FRAMEWORK_ADAPTERS[id].capabilities.progressiveHtmlParsing, false); + } + assert.throws(() => getFrameworkAdapter('unknown'), /unknown benchmark framework adapter: unknown/u); +}); diff --git a/benchmark/tests/harness.test.mjs b/benchmark/tests/harness.test.mjs new file mode 100644 index 000000000..a38de9dcd --- /dev/null +++ b/benchmark/tests/harness.test.mjs @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +import { createRevisionHarnessRecord } from '../src/harness.mjs'; + +const benchmarkRoot = fileURLToPath(new URL('..', import.meta.url)); + +test('revision harness compatibility record is stable and covers source inputs', async () => { + const first = await createRevisionHarnessRecord(benchmarkRoot); + const second = await createRevisionHarnessRecord(benchmarkRoot); + + assert.deepEqual(second, first); + assert.match(first.fingerprint, /^[a-f\d]{64}$/u); + assert.match(first.descriptor.sourceHash, /^[a-f\d]{64}$/u); + assert.equal(first.descriptor.measurement.version, 4); + assert.deepEqual(first.descriptor.measurement.diagnostics, [ + 'framework settlement', + 'HTML response end', + 'paint lead before response end', + ]); +}); diff --git a/benchmark/tests/hash.test.mjs b/benchmark/tests/hash.test.mjs new file mode 100644 index 000000000..fac887934 --- /dev/null +++ b/benchmark/tests/hash.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { hashViteEntries } from '../src/hash.mjs'; + +test('hashViteEntries follows only the selected entry graph', async () => { + const root = await mkdtemp(join(tmpdir(), 'qiankun-benchmark-hash-')); + try { + await Promise.all([mkdir(join(root, '.vite')), mkdir(join(root, 'assets'))]); + await Promise.all([ + writeFile(join(root, 'core.html'), '
core
'), + writeFile(join(root, 'ecosystem.html'), '
ecosystem
'), + writeFile(join(root, 'assets/core.js'), 'import "./shared.js";'), + writeFile(join(root, 'assets/ecosystem.js'), 'console.log("ecosystem");'), + writeFile(join(root, 'assets/shared.js'), 'console.log("shared");'), + writeFile( + join(root, '.vite/manifest.json'), + JSON.stringify({ + '_shared.js': { file: 'assets/shared.js' }, + 'core.html': { file: 'assets/core.js', imports: ['_shared.js'], isEntry: true }, + 'ecosystem.html': { file: 'assets/ecosystem.js', isEntry: true }, + }), + ), + ]); + + const initial = await hashViteEntries(root, ['core.html']); + await writeFile(join(root, 'assets/ecosystem.js'), 'console.log("changed ecosystem");'); + assert.equal(await hashViteEntries(root, ['core.html']), initial); + + await writeFile(join(root, 'assets/shared.js'), 'console.log("changed shared");'); + assert.notEqual(await hashViteEntries(root, ['core.html']), initial); + } finally { + await rm(root, { force: true, recursive: true }); + } +}); diff --git a/benchmark/tests/options.test.mjs b/benchmark/tests/options.test.mjs index a180d7181..3e5057804 100644 --- a/benchmark/tests/options.test.mjs +++ b/benchmark/tests/options.test.mjs @@ -7,13 +7,16 @@ test('parseRunnerOptions provides the formal smoke defaults', () => { assert.deepEqual(parseRunnerOptions([]), { baselineDir: null, calibrationGate: true, - calibrationSamples: 50, + calibrationSamples: 100, chunkIntervalMs: 50, comparisonGate: true, mode: 'framework', samples: 100, + scenario: null, seed: 20260711, + suite: 'core', timeoutMs: 10_000, + trials: 3, warmup: 5, }); }); @@ -26,8 +29,10 @@ test('parseRunnerOptions accepts explicit check-run overrides', () => { '--calibration-samples=5', '--calibration-gate=false', '--seed=9', + '--suite=core', '--chunk-interval=10', '--timeout=2000', + '--trials=4', ]), { baselineDir: null, @@ -37,31 +42,46 @@ test('parseRunnerOptions accepts explicit check-run overrides', () => { comparisonGate: true, mode: 'framework', samples: 5, + scenario: null, seed: 9, + suite: 'core', timeoutMs: 2_000, + trials: 4, warmup: 2, }, ); }); +test('parseRunnerOptions accepts the dedicated SSR streaming suite', () => { + assert.equal(parseRunnerOptions(['--suite=ssr-streaming']).suite, 'ssr-streaming'); +}); + +test('parseRunnerOptions defaults revision comparisons to the streaming scenario', () => { + assert.equal(parseRunnerOptions(['--mode=revision', '--baseline-dir=artifacts/baseline']).scenario, 'streaming'); +}); + test('parseRunnerOptions accepts revision comparison options', () => { assert.deepEqual( parseRunnerOptions([ '--mode=revision', '--baseline-dir=artifacts/baseline', '--comparison-gate=false', + '--scenario=sandbox', '--samples=5', ]), { baselineDir: 'artifacts/baseline', calibrationGate: true, - calibrationSamples: 50, + calibrationSamples: 100, chunkIntervalMs: 50, comparisonGate: false, mode: 'revision', samples: 5, + scenario: 'sandbox', seed: 20260711, + suite: 'core', timeoutMs: 10_000, + trials: 1, warmup: 5, }, ); @@ -74,4 +94,17 @@ test('parseRunnerOptions rejects unknown, non-integer, and non-positive values', assert.throws(() => parseRunnerOptions(['--calibration-gate=maybe']), /calibration-gate must be true or false/); assert.throws(() => parseRunnerOptions(['--mode=revision']), /baseline-dir is required/u); assert.throws(() => parseRunnerOptions(['--mode=unknown']), /mode must be framework or revision/u); + assert.throws( + () => parseRunnerOptions(['--mode=revision', '--baseline-dir=artifacts/baseline', '--scenario=unknown']), + /scenario must be streaming or sandbox/u, + ); + assert.throws(() => parseRunnerOptions(['--scenario=sandbox']), /scenario requires revision mode/u); + assert.throws( + () => parseRunnerOptions(['--mode=revision', '--baseline-dir=artifacts/baseline', '--trials=2']), + /revision mode requires trials=1/u, + ); + assert.throws( + () => parseRunnerOptions(['--mode=revision', '--baseline-dir=artifacts/baseline', '--suite=ecosystem-html']), + /revision mode requires the core suite/u, + ); }); diff --git a/benchmark/tests/report.test.mjs b/benchmark/tests/report.test.mjs index 20f9fbe71..a2dea6f9d 100644 --- a/benchmark/tests/report.test.mjs +++ b/benchmark/tests/report.test.mjs @@ -9,11 +9,51 @@ const variants = [ ]; const samples = [ - { duration: 10, position: 0, round: 0, sequence: 0, valid: true, variant: 'a' }, - { duration: 20, position: 1, round: 0, sequence: 1, valid: true, variant: 'b' }, - { duration: 30, position: 1, round: 1, sequence: 2, valid: true, variant: 'a' }, - { duration: 60, position: 0, round: 1, sequence: 3, valid: true, variant: 'b' }, - { error: 'timeout', position: 0, round: 2, sequence: 4, valid: false, variant: 'a' }, + { + duration: 10, + entryResponseEndDuration: 40, + position: 0, + round: 0, + sequence: 0, + settledDuration: 50, + trial: 0, + valid: true, + variant: 'a', + }, + { + duration: 20, + entryResponseEndDuration: 40, + position: 1, + round: 0, + sequence: 1, + settledDuration: 55, + trial: 0, + valid: true, + variant: 'b', + }, + { + duration: 30, + entryResponseEndDuration: 50, + position: 1, + round: 1, + sequence: 2, + settledDuration: 60, + trial: 0, + valid: true, + variant: 'a', + }, + { + duration: 60, + entryResponseEndDuration: 70, + position: 0, + round: 1, + sequence: 3, + settledDuration: 80, + trial: 0, + valid: true, + variant: 'b', + }, + { error: 'timeout', position: 0, round: 2, sequence: 4, trial: 0, valid: false, variant: 'a' }, ]; test('buildReport summarizes valid samples while retaining invalid counts', () => { @@ -25,9 +65,62 @@ test('buildReport summarizes valid samples while retaining invalid counts', () = }); assert.equal(report.variants.a.summary.count, 2); + assert.equal(report.variants.a.diagnostics.settledDuration.median, 55); + assert.equal(report.variants.a.diagnostics.entryResponseEndDuration.median, 45); + assert.equal(report.variants.a.diagnostics.paintLeadBeforeResponseEnd.median, 25); assert.equal(report.variants.a.invalidCount, 1); assert.equal(report.variants.b.summary.count, 2); assert.ok(Math.abs(report.comparisons['b-vs-a'].relativeDeltaPercent - 100) < 1e-10); + assert.equal(report.comparisons['b-vs-a'].trialCount, 1); +}); + +test('buildReport pairs repeated round numbers within their trial instead of overwriting them', () => { + const report = buildReport({ + comparisons: [{ candidate: 'b', id: 'b-vs-a', label: 'B vs A', reference: 'a' }], + samples: [ + { duration: 10, round: 0, trial: 0, valid: true, variant: 'a' }, + { duration: 20, round: 0, trial: 0, valid: true, variant: 'b' }, + { duration: 10, round: 0, trial: 1, valid: true, variant: 'a' }, + { duration: 10, round: 0, trial: 1, valid: true, variant: 'b' }, + ], + seed: 5, + variants, + }); + + assert.equal(report.comparisons['b-vs-a'].pairedCount, 2); + assert.equal(report.comparisons['b-vs-a'].trialCount, 2); + assert.deepEqual(report.comparisons['b-vs-a'].trialRelativeDeltas, [100, 0]); +}); + +test('buildReport rejects incomplete and duplicate pairs instead of dropping them silently', () => { + const comparison = [{ candidate: 'b', id: 'b-vs-a', label: 'B vs A', reference: 'a' }]; + assert.throws( + () => + buildReport({ + comparisons: comparison, + samples: [ + { duration: 10, round: 0, trial: 0, valid: true, variant: 'a' }, + { duration: 20, round: 1, trial: 0, valid: true, variant: 'b' }, + ], + seed: 5, + variants, + }), + /b-vs-a trial 0 has unpaired rounds/u, + ); + assert.throws( + () => + buildReport({ + comparisons: comparison, + samples: [ + { duration: 10, round: 0, trial: 0, valid: true, variant: 'a' }, + { duration: 11, round: 0, trial: 0, valid: true, variant: 'a' }, + { duration: 20, round: 0, trial: 0, valid: true, variant: 'b' }, + ], + seed: 5, + variants, + }), + /b-vs-a trial 0 has duplicate reference round 0/u, + ); }); test('renderSummaryMarkdown exposes counts, medians, and comparison confidence intervals', () => { @@ -41,6 +134,8 @@ test('renderSummaryMarkdown exposes counts, medians, and comparison confidence i assert.match(markdown, /# Smoke result/); assert.match(markdown, /Variant A \| 2 \| 1 \| 20\.00/); + assert.match(markdown, /## Phase diagnostics/); + assert.match(markdown, /Variant A \| 55\.00 \| 45\.00 \| 25\.00/); assert.match(markdown, /B vs A \| \+100\.00%/); }); diff --git a/benchmark/tests/result-comparison.test.mjs b/benchmark/tests/result-comparison.test.mjs new file mode 100644 index 000000000..69cbc6cb7 --- /dev/null +++ b/benchmark/tests/result-comparison.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { createHarnessFingerprint } from '../src/fingerprint.mjs'; +import { compareBenchmarkResults } from '../src/result-comparison.mjs'; + +function createResult(descriptor, median) { + return { + metadata: { + harness: { descriptor, fingerprint: createHarnessFingerprint(descriptor) }, + }, + product: { + report: { + variants: { + native: { label: 'Native iframe', summary: { median } }, + }, + }, + }, + }; +} + +test('compareBenchmarkResults permits only matching harnesses', () => { + const descriptor = { browser: { args: ['--site-per-process'] }, suite: 'core' }; + const rows = compareBenchmarkResults(createResult(descriptor, 40), createResult(descriptor, 44)); + assert.equal(rows[0].candidateMedian, 44); + assert.equal(rows[0].id, 'native'); + assert.equal(rows[0].label, 'Native iframe'); + assert.equal(rows[0].referenceMedian, 40); + assert.ok(Math.abs(rows[0].relativeDeltaPercent - 10) < 1e-10); + + assert.throws( + () => + compareBenchmarkResults(createResult(descriptor, 40), createResult({ browser: { args: [] }, suite: 'core' }, 44)), + /benchmark harnesses are not comparable; changed fields: browser\.args/u, + ); +}); diff --git a/benchmark/tests/scenarios.test.mjs b/benchmark/tests/scenarios.test.mjs index 47605e8ed..e2c03338f 100644 --- a/benchmark/tests/scenarios.test.mjs +++ b/benchmark/tests/scenarios.test.mjs @@ -5,10 +5,35 @@ import * as scenarios from '../scenarios.mjs'; const { CALIBRATION_VARIANTS, PRODUCT_COMPARISONS, PRODUCT_VARIANTS } = scenarios; -test('the product matrix contains six explicit variants instead of a cartesian product', () => { +test('the core product matrix contains eight explicit variants instead of a cartesian product', () => { assert.deepEqual( PRODUCT_VARIANTS.map((variant) => variant.id), - ['qk-no-isolation', 'qk-sandbox', 'qk-full-isolation', 'wujie-isolated', 'qk-streamed', 'wujie-streamed'], + [ + 'qk-no-isolation', + 'qk-sandbox', + 'qk-full-isolation', + 'native-iframe', + 'wujie-isolated', + 'native-iframe-streamed', + 'qk-streamed', + 'wujie-streamed', + ], + ); +}); + +test('every product variant declares its entry site explicitly', () => { + assert.deepEqual( + PRODUCT_VARIANTS.map(({ entrySite, id }) => ({ entrySite, id })), + [ + { entrySite: 'same-site', id: 'qk-no-isolation' }, + { entrySite: 'same-site', id: 'qk-sandbox' }, + { entrySite: 'same-site', id: 'qk-full-isolation' }, + { entrySite: 'same-site', id: 'native-iframe' }, + { entrySite: 'same-site', id: 'wujie-isolated' }, + { entrySite: 'same-site', id: 'native-iframe-streamed' }, + { entrySite: 'same-site', id: 'qk-streamed' }, + { entrySite: 'same-site', id: 'wujie-streamed' }, + ], ); }); @@ -33,10 +58,189 @@ test('wujie variants use its fastest cold-load isolated configuration', () => { } }); -test('the matrix defines the four comparisons used to diagnose optimization directions', () => { +test('core native iframe variants cover same-site buffered and streamed delivery', () => { + assert.deepEqual( + PRODUCT_VARIANTS.filter(({ framework }) => framework === 'native').map( + ({ delivery, entrySite, frameworkOptions, id }) => ({ + delivery, + entrySite, + frameworkOptions, + id, + }), + ), + [ + { delivery: 'buffered', entrySite: 'same-site', frameworkOptions: {}, id: 'native-iframe' }, + { delivery: 'streamed', entrySite: 'same-site', frameworkOptions: {}, id: 'native-iframe-streamed' }, + ], + ); +}); + +test('ecosystem suite adds canonical isolated cells for qiankun v2, MicroApp, and Garfish', () => { + const ecosystem = scenarios.SUITES['ecosystem-html']; + const qiankunV2 = ecosystem.variants.find(({ id }) => id === 'qk-v2-full-isolation'); + const microApp = ecosystem.variants.find(({ id }) => id === 'microapp-default-isolation'); + const garfish = ecosystem.variants.find(({ id }) => id === 'garfish-strict-isolation'); + + assert.deepEqual(qiankunV2, { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun-v2', + frameworkOptions: { sandbox: { experimentalStyleIsolation: true } }, + id: 'qk-v2-full-isolation', + label: 'qiankun v2.10.16 · Proxy sandbox + scoped CSS', + }); + assert.deepEqual(microApp, { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'micro-app', + frameworkOptions: {}, + id: 'microapp-default-isolation', + label: 'MicroApp · sandbox + scoped CSS', + }); + assert.deepEqual(garfish, { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'garfish', + frameworkOptions: { cache: false, sandbox: { strictIsolation: true } }, + id: 'garfish-strict-isolation', + label: 'Garfish · VM sandbox + Shadow DOM', + }); + assert.deepEqual( + ecosystem.comparisons.map(({ id }) => id), + [ + 'qiankun-native-isolated', + 'wujie-native-isolated', + 'isolated-framework', + 'qiankun-v2-native-isolated', + 'qiankun-v3-v2-isolated', + 'microapp-native-isolated', + 'microapp-qiankun-isolated', + 'garfish-native-isolated', + 'garfish-qiankun-isolated', + 'garfish-qiankun-v2-isolated', + ], + ); +}); + +test('SSR streaming suite isolates progressive reveal from full-response buffering', () => { + const suite = scenarios.SUITES['ssr-streaming']; + + assert.deepEqual( + suite.variants.map(({ delivery, framework, htmlFixture, id }) => ({ delivery, framework, htmlFixture, id })), + [ + { + delivery: 'streamed', + framework: 'native', + htmlFixture: 'ssr', + id: 'native-iframe-ssr-streamed', + }, + { + delivery: 'delayed-buffered', + framework: 'qiankun', + htmlFixture: 'ssr', + id: 'qk-v3-ssr-delayed-buffered', + }, + { + delivery: 'streamed', + framework: 'qiankun-v2', + htmlFixture: 'ssr', + id: 'qk-v2-ssr-streamed', + }, + { + delivery: 'streamed', + framework: 'qiankun', + htmlFixture: 'ssr', + id: 'qk-v3-ssr-streamed', + }, + { + delivery: 'streamed', + framework: 'wujie', + htmlFixture: 'ssr', + id: 'wujie-ssr-streamed', + }, + { + delivery: 'streamed', + framework: 'garfish', + htmlFixture: 'ssr', + id: 'garfish-ssr-streamed', + }, + ], + ); + assert.deepEqual( + suite.comparisons.map(({ candidate, id, reference }) => ({ candidate, id, reference })), + [ + { + candidate: 'qk-v3-ssr-streamed', + id: 'qiankun-v3-ssr-streaming-gain', + reference: 'qk-v3-ssr-delayed-buffered', + }, + { + candidate: 'qk-v3-ssr-streamed', + id: 'qiankun-v3-native-ssr-streamed', + reference: 'native-iframe-ssr-streamed', + }, + { + candidate: 'qk-v2-ssr-streamed', + id: 'qiankun-v2-v3-ssr-streamed', + reference: 'qk-v3-ssr-streamed', + }, + { + candidate: 'wujie-ssr-streamed', + id: 'wujie-qiankun-v3-ssr-streamed', + reference: 'qk-v3-ssr-streamed', + }, + { + candidate: 'garfish-ssr-streamed', + id: 'garfish-qiankun-v3-ssr-streamed', + reference: 'qk-v3-ssr-streamed', + }, + ], + ); +}); + +test('the matrix defines framework and native-baseline comparisons', () => { assert.deepEqual( PRODUCT_COMPARISONS.map((comparison) => comparison.id), - ['sandbox-cost', 'style-isolation-cost', 'isolated-framework', 'streaming-framework'], + [ + 'sandbox-cost', + 'style-isolation-cost', + 'qiankun-native-isolated', + 'wujie-native-isolated', + 'isolated-framework', + 'qiankun-native-streamed', + 'wujie-native-streamed', + 'streaming-framework', + ], + ); + + assert.deepEqual( + PRODUCT_COMPARISONS.filter(({ id }) => id.includes('native')).map(({ candidate, id, reference }) => ({ + candidate, + id, + reference, + })), + [ + { + candidate: 'qk-full-isolation', + id: 'qiankun-native-isolated', + reference: 'native-iframe', + }, + { + candidate: 'wujie-isolated', + id: 'wujie-native-isolated', + reference: 'native-iframe', + }, + { + candidate: 'qk-streamed', + id: 'qiankun-native-streamed', + reference: 'native-iframe-streamed', + }, + { + candidate: 'wujie-streamed', + id: 'wujie-native-streamed', + reference: 'native-iframe-streamed', + }, + ], ); }); @@ -49,29 +253,33 @@ test('A/A calibration aliases the exact same qiankun variant', () => { }); test('revision comparison variants differ only by revision host role', () => { - const { REVISION_CALIBRATION_VARIANTS, REVISION_COMPARISONS, REVISION_VARIANTS } = scenarios; + const { REVISION_CALIBRATION_VARIANTS, REVISION_COMPARISONS, createRevisionVariants } = scenarios; + const revisionVariants = createRevisionVariants('streaming'); assert.deepEqual( - REVISION_VARIANTS.map(({ hostRole, id }) => ({ hostRole, id })), + revisionVariants.map(({ hostRole, id }) => ({ hostRole, id })), [ { hostRole: 'baseline', id: 'revision-baseline' }, { hostRole: 'candidate', id: 'revision-candidate' }, ], ); assert.deepEqual( - REVISION_VARIANTS.map(({ delivery, framework, frameworkOptions }) => ({ + revisionVariants.map(({ delivery, entrySite, framework, frameworkOptions }) => ({ delivery, + entrySite, framework, frameworkOptions, })), [ { delivery: 'streamed', + entrySite: 'same-site', framework: 'qiankun', frameworkOptions: { sandbox: true, styleIsolation: true }, }, { delivery: 'streamed', + entrySite: 'same-site', framework: 'qiankun', frameworkOptions: { sandbox: true, styleIsolation: true }, }, @@ -90,3 +298,34 @@ test('revision comparison variants differ only by revision host role', () => { ['revision-candidate', 'revision-candidate'], ); }); + +test('sandbox revision scenario isolates sandbox work from streaming and style isolation', () => { + const revisionVariants = scenarios.createRevisionVariants('sandbox'); + + assert.deepEqual( + revisionVariants.map(({ delivery, entrySite, framework, frameworkOptions }) => ({ + delivery, + entrySite, + framework, + frameworkOptions, + })), + [ + { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: false }, + }, + { + delivery: 'buffered', + entrySite: 'same-site', + framework: 'qiankun', + frameworkOptions: { sandbox: true, styleIsolation: false }, + }, + ], + ); +}); + +test('revision scenarios reject unknown names', () => { + assert.throws(() => scenarios.createRevisionVariants('unknown'), /unknown revision scenario: unknown/u); +}); diff --git a/benchmark/tests/server.test.mjs b/benchmark/tests/server.test.mjs index 17dc0caaf..4ca656b4f 100644 --- a/benchmark/tests/server.test.mjs +++ b/benchmark/tests/server.test.mjs @@ -32,8 +32,8 @@ test('streamed and buffered HTML produce identical bytes with observable chunk g await fixture.start(); try { - const buffered = await request(`${fixture.origin}/app?delivery=buffered`); - const streamed = await request(`${fixture.origin}/app?delivery=streamed`); + const buffered = await request(`${fixture.origin}/app/index.html?delivery=buffered`); + const streamed = await request(`${fixture.origin}/app/index.html?delivery=streamed`); assert.equal(buffered.statusCode, 200); assert.equal(streamed.statusCode, 200); @@ -47,20 +47,65 @@ test('streamed and buffered HTML produce identical bytes with observable chunk g } }); +test('SSR streaming reveals literal critical content before an identical delayed buffer', async () => { + const fixture = createFixtureServer({ chunkIntervalMs: 20, port: 0 }); + await fixture.start(); + + try { + const streamed = await request(`${fixture.origin}/app/index.html?fixture=ssr&delivery=streamed`); + const delayed = await request(`${fixture.origin}/app/index.html?fixture=ssr&delivery=delayed-buffered`); + + assert.equal(streamed.statusCode, 200); + assert.equal(delayed.statusCode, 200); + assert.equal(streamed.body, delayed.body); + assert.ok(streamed.arrivals.length >= 3); + assert.ok(streamed.arrivals.at(-1) - streamed.arrivals[0] >= 30); + assert.ok(delayed.arrivals[0] >= 30); + assert.match(streamed.body, /data-benchmark-critical/u); + assert.doesNotMatch(streamed.body, /root\.innerHTML/u); + assert.ok(streamed.body.indexOf('data-benchmark-critical') < streamed.body.indexOf('benchmark-stream-tail')); + assert.ok( + streamed.body.indexOf('benchmark-stream-tail') < streamed.body.indexOf('', ); assert.match(created.bundleHash, /^[a-f\d]{64}$/u); + assert.equal(created.schemaVersion, 2); assert.deepEqual(await readBaselineSnapshot(targetDirectory), created); + assert.doesNotThrow(() => assertBaselineHarnessCompatible(created, createHarness())); + assert.throws( + () => assertBaselineHarnessCompatible(created, createHarness('source-b')), + /benchmark harnesses are not comparable; changed fields: sourceHash/u, + ); await assert.rejects( createBaselineSnapshot({ createdAt: '2026-07-11T00:00:00.000Z', git: { commit: 'abc123', dirty: false }, + harness: createHarness(), sourceDirectory, targetDirectory, }), diff --git a/benchmark/tests/stats.test.mjs b/benchmark/tests/stats.test.mjs index 6af49c50b..f69ded8a1 100644 --- a/benchmark/tests/stats.test.mjs +++ b/benchmark/tests/stats.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { comparePairedSamples, evaluateCalibration, summarize } from '../src/stats.mjs'; +import { comparePairedSamples, comparePairedTrials, evaluateCalibration, summarize } from '../src/stats.mjs'; test('summarize reports robust and tail statistics without removing samples', () => { const summary = summarize([1, 2, 3, 4, 5]); @@ -37,6 +37,22 @@ test('comparePairedSamples reports a deterministic 100 percent regression', () = assert.ok(Math.abs(comparison.confidenceInterval95[1] - 100) < 1e-10); }); +test('comparePairedTrials aggregates independent browser trials with equal trial weight', () => { + const neutralReference = Array.from({ length: 101 }, () => 10); + const neutralCandidate = Array.from({ length: 101 }, () => 10); + const trials = [ + { candidate: [20], reference: [10] }, + { candidate: [20], reference: [10] }, + { candidate: neutralCandidate, reference: neutralReference }, + ]; + const comparison = comparePairedTrials(trials, { iterations: 1_000, seed: 7 }); + + assert.equal(comparison.trialCount, 3); + assert.deepEqual(comparison.trialRelativeDeltas, [100, 100, 0]); + assert.ok(Math.abs(comparison.relativeDeltaPercent - 100) < 1e-10); + assert.deepEqual(comparePairedTrials(trials, { iterations: 1_000, seed: 7 }), comparison); +}); + test('evaluateCalibration enforces delta, zero coverage, and interval width', () => { assert.deepEqual(evaluateCalibration({ relativeDeltaPercent: 1, confidenceInterval95: [-2, 3] }), { passed: true, diff --git a/benchmark/tests/suites.test.mjs b/benchmark/tests/suites.test.mjs new file mode 100644 index 000000000..c7b5465dd --- /dev/null +++ b/benchmark/tests/suites.test.mjs @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { SUITES } from '../scenarios.mjs'; +import { createHarnessFingerprint } from '../src/fingerprint.mjs'; +import { createBalancedSchedule } from '../src/schedule.mjs'; + +test('suite memberships stay explicit while the core fingerprint remains frozen', () => { + assert.equal( + createHarnessFingerprint(SUITES.core), + 'cfdd7caaf5aa7f3a8579699c2b740ea24624b10b5e0b2d23c99aab7a7fc71665', + ); + assert.deepEqual( + SUITES.core.variants.map(({ id }) => id), + [ + 'qk-no-isolation', + 'qk-sandbox', + 'qk-full-isolation', + 'native-iframe', + 'wujie-isolated', + 'native-iframe-streamed', + 'qk-streamed', + 'wujie-streamed', + ], + ); + assert.deepEqual( + SUITES['site-isolation'].variants.map(({ id }) => id), + [ + 'qk-full-isolation', + 'native-iframe', + 'wujie-isolated', + 'qk-full-isolation-cross-site', + 'native-iframe-cross-site', + 'wujie-cross-site-entry', + ], + ); + assert.deepEqual( + SUITES['ecosystem-html'].variants.map(({ id }) => id), + [ + 'native-iframe', + 'qk-v2-full-isolation', + 'qk-full-isolation', + 'wujie-isolated', + 'microapp-default-isolation', + 'garfish-strict-isolation', + ], + ); + assert.deepEqual( + SUITES['ssr-streaming'].variants.map(({ id }) => id), + [ + 'native-iframe-ssr-streamed', + 'qk-v3-ssr-delayed-buffered', + 'qk-v2-ssr-streamed', + 'qk-v3-ssr-streamed', + 'wujie-ssr-streamed', + 'garfish-ssr-streamed', + ], + ); + assert.equal(SUITES['ssr-streaming'].calibrationSourceVariant, 'qk-v3-ssr-streamed'); +}); + +test('adding an ecosystem variant cannot change the core balanced schedule', () => { + const coreIds = SUITES.core.variants.map(({ id }) => id); + const before = createBalancedSchedule(coreIds, 24, 20260711); + const expandedEcosystem = [...SUITES['ecosystem-html'].variants, { id: 'future-framework' }]; + + assert.equal(expandedEcosystem.length, SUITES['ecosystem-html'].variants.length + 1); + assert.deepEqual(createBalancedSchedule(coreIds, 24, 20260711), before); +}); + +test('each suite owns comparisons whose variants exist in that suite', () => { + for (const suite of Object.values(SUITES)) { + const variantIds = new Set(suite.variants.map(({ id }) => id)); + for (const comparison of suite.comparisons) { + assert.ok(variantIds.has(comparison.reference), `${suite.id}:${comparison.id}:reference`); + assert.ok(variantIds.has(comparison.candidate), `${suite.id}:${comparison.id}:candidate`); + } + } +}); diff --git a/benchmark/tsconfig.json b/benchmark/tsconfig.json index 8e4ad2afb..9407fef6f 100644 --- a/benchmark/tsconfig.json +++ b/benchmark/tsconfig.json @@ -7,5 +7,6 @@ "skipLibCheck": true, "types": ["node"] }, + "exclude": [], "include": ["fixtures/host/src/**/*.ts", "fixtures/host/vite.config.ts"] } diff --git a/package.json b/package.json index 23325db8d..f25f6ccaa 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,12 @@ "benchmark:check": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run run:check", "benchmark:compare": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run compare", "benchmark:compare:check": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run compare:check", + "benchmark:ecosystem": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run ecosystem", + "benchmark:ecosystem:check": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run ecosystem:check", + "benchmark:site-isolation": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run site-isolation", "benchmark:smoke": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run smoke", + "benchmark:ssr-streaming": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run ssr-streaming", + "benchmark:ssr-streaming:check": "pnpm run build:packages && pnpm --filter @qiankunjs/benchmark run ssr-streaming:check", "build": "pnpm -r run build", "build:packages": "pnpm --filter './packages/**' run build", "ci:version": "changeset version && node scripts/sanitize-pre-state.mjs", diff --git a/packages/sandbox/src/core/membrane/__tests__/index.test.ts b/packages/sandbox/src/core/membrane/__tests__/index.test.ts new file mode 100644 index 000000000..4ffe562ed --- /dev/null +++ b/packages/sandbox/src/core/membrane/__tests__/index.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; +import { Membrane } from '..'; + +function createCountedGlobal() { + const rawGlobal: Record = {}; + for (let index = 0; index < 128; index++) { + rawGlobal[`configurable${index}`] = index; + } + Object.defineProperties(rawGlobal, { + locked: { + configurable: false, + enumerable: true, + value: 'host', + writable: false, + }, + lockedGetter: { + configurable: false, + enumerable: true, + get() { + return this.marker; + }, + }, + lockedWritable: { + configurable: false, + enumerable: true, + value: 'host', + writable: true, + }, + marker: { + configurable: true, + enumerable: true, + value: 'host-marker', + writable: true, + }, + }); + + let descriptorReads = 0; + const globalContext = new Proxy(rawGlobal, { + getOwnPropertyDescriptor(target, property) { + descriptorReads++; + return Reflect.getOwnPropertyDescriptor(target, property); + }, + }); + + return { + getDescriptorReads: () => descriptorReads, + globalContext: globalContext as unknown as WindowProxy, + rawGlobal, + }; +} + +describe('non-configurable global properties', () => { + it('materializes host descriptors only when proxy invariants require them', () => { + const { getDescriptorReads, globalContext } = createCountedGlobal(); + const membrane = new Membrane(globalContext, {}); + + expect(getDescriptorReads()).toBe(0); + expect(Object.prototype.hasOwnProperty.call(membrane.target, 'locked')).toBe(false); + expect(Reflect.ownKeys(membrane.realmGlobal)).toContain('locked'); + expect(getDescriptorReads()).toBe(0); + + expect(Object.getOwnPropertyDescriptor(membrane.realmGlobal, 'locked')).toEqual({ + configurable: false, + enumerable: true, + value: 'host', + writable: false, + }); + expect(Object.prototype.hasOwnProperty.call(membrane.target, 'locked')).toBe(true); + }); + + it('keeps non-configurable data and accessor behavior after lazy materialization', () => { + const { globalContext, rawGlobal } = createCountedGlobal(); + const membrane = new Membrane(globalContext, {}); + const proxy = membrane.realmGlobal as unknown as Record; + + expect(proxy.lockedGetter).toBe('host-marker'); + expect(Object.prototype.hasOwnProperty.call(membrane.target, 'lockedGetter')).toBe(false); + expect(Object.getOwnPropertyDescriptor(proxy, 'lockedGetter')?.configurable).toBe(false); + expect(proxy.lockedGetter).toBe('host-marker'); + + expect(Reflect.set(proxy, 'lockedWritable', 'sandbox')).toBe(true); + expect(proxy.lockedWritable).toBe('sandbox'); + expect(rawGlobal.lockedWritable).toBe('host'); + + expect(() => Object.defineProperty(proxy, 'locked', { value: 'sandbox' })).toThrow(TypeError); + expect(proxy.locked).toBe('host'); + expect(() => Reflect.deleteProperty(proxy, 'locked')).toThrow(TypeError); + }); + + it('supports descriptor enumeration without violating proxy invariants', () => { + const { globalContext } = createCountedGlobal(); + const membrane = new Membrane(globalContext, {}); + + expect(Object.getOwnPropertyNames(membrane.realmGlobal)).toContain('locked'); + expect(Object.getOwnPropertyDescriptors(membrane.realmGlobal).locked).toEqual({ + configurable: false, + enumerable: true, + value: 'host', + writable: false, + }); + }); + + it('never materializes a shielded internal key on set, keeping the host accessor unreachable', () => { + const rawGlobal: Record = {}; + const hostAccessor = () => 'host-realm-accessor'; + Object.defineProperty(rawGlobal, '__qk_r_test', { + configurable: false, + enumerable: false, + value: hostAccessor, + writable: false, + }); + const membrane = new Membrane(rawGlobal as unknown as WindowProxy, {}); + const proxy = membrane.realmGlobal as unknown as Record; + + // shielded before any write + expect(proxy.__qk_r_test).toBeUndefined(); + expect('__qk_r_test' in proxy).toBe(false); + + // a write behaves like defining a brand-new app-own global rather than + // copying the non-configurable host descriptor onto the target + proxy.__qk_r_test = 'sandbox-own'; + + expect(proxy.__qk_r_test).toBe('sandbox-own'); + expect(Object.getOwnPropertyDescriptor(membrane.target, '__qk_r_test')?.configurable).toBe(true); + expect(rawGlobal.__qk_r_test).toBe(hostAccessor); + }); + + it('keeps endowments ahead of the host global without scanning host descriptors', () => { + const { getDescriptorReads, globalContext } = createCountedGlobal(); + const membrane = new Membrane( + globalContext, + {}, + { + endowments: { + locked: { + configurable: false, + enumerable: true, + value: 'endowment', + writable: false, + }, + }, + }, + ); + + expect(getDescriptorReads()).toBe(0); + expect((membrane.realmGlobal as unknown as Record).locked).toBe('endowment'); + expect(getDescriptorReads()).toBe(0); + }); +}); diff --git a/packages/sandbox/src/core/membrane/index.ts b/packages/sandbox/src/core/membrane/index.ts index 9bcbdb80c..b13643a5f 100644 --- a/packages/sandbox/src/core/membrane/index.ts +++ b/packages/sandbox/src/core/membrane/index.ts @@ -5,7 +5,6 @@ import { esmInternalPrefix, freeze, getOwnPropertyDescriptor, - getOwnPropertyNames, hasOwnProperty, keys, } from '@qiankunjs/shared'; @@ -138,8 +137,8 @@ export class Membrane { const { endowments = {} } = opts || {}; const whitelistVars = [...(opts?.whitelist || []), ...globalVariableWhiteList]; const descriptorTargetMap = new Map(); - - const { target, propertiesWithGetter } = createMembraneTarget(endowments, incubatorContext); + const propertiesWithGetter = new Map(); + const target = createMembraneTarget(endowments); this.target = target; @@ -152,14 +151,30 @@ export class Membrane { incubatorContext[p as never] = value; } else { // We must keep its description while the property existed in incubatorContext before - if (!hasOwnProperty(membraneTarget, p) && hasOwnProperty(incubatorContext, p)) { - const descriptor = getOwnPropertyDescriptor(incubatorContext, p); - const { writable, configurable, enumerable } = descriptor!; - // only writable property can be overwritten - // here we ignored accessor descriptor of incubatorContext as it makes no sense to trigger its logic(which might make sandbox escaping instead) - // we force to set value by data descriptor - if (writable || hasOwnProperty(descriptor, 'set')) { - defineProperty(membraneTarget, p, { configurable, enumerable, writable: true, value }); + if (!hasOwnProperty(membraneTarget, p)) { + /* + * never consult the host descriptor for shielded internal keys (e.g. the ESM realm + * accessor): materializing it would turn the key into an own prop and permanently + * unshield the real accessor for this app. As the app cannot observe the host key + * (get/has/getOwnPropertyDescriptor all shield it), the write falls through to the + * brand-new-own-global branch below instead. + */ + const descriptor = isShieldedInternalGlobal(membraneTarget, p) + ? undefined + : getOwnPropertyDescriptor(incubatorContext, p); + if (descriptor && !descriptor.configurable) { + materializeNonConfigurableProperty(membraneTarget, propertiesWithGetter, p, descriptor); + membraneTarget[p] = value; + } else if (descriptor) { + const { writable, configurable, enumerable } = descriptor; + // only writable property can be overwritten + // here we ignored accessor descriptor of incubatorContext as it makes no sense to trigger its logic(which might make sandbox escaping instead) + // we force to set value by data descriptor + if (writable || hasOwnProperty(descriptor, 'set')) { + defineProperty(membraneTarget, p, { configurable, enumerable, writable: true, value }); + } + } else { + membraneTarget[p] = value; } } else { membraneTarget[p] = value; @@ -239,9 +254,9 @@ export class Membrane { p: string | number | symbol, ): PropertyDescriptor | undefined { /* - as the descriptor of top/self/window/mockTop in raw window are configurable but not in proxy membraneTarget, we need to get it from membraneTarget to avoid TypeError - see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy/handler/getOwnPropertyDescriptor - > A property cannot be reported as non-configurable, if it does not exist as an own property of the membraneTarget object or if it exists as a configurable own property of the membraneTarget object. + * Proxy invariants require a non-configurable property to exist on the target before its + * descriptor is returned. Mirror only the requested property instead of scanning every + * property on the host window while constructing each sandbox. */ if (hasOwnProperty(membraneTarget, p)) { const descriptor = getOwnPropertyDescriptor(membraneTarget, p); @@ -251,13 +266,15 @@ export class Membrane { if (isShieldedInternalGlobal(membraneTarget, p)) return undefined; - if (hasOwnProperty(incubatorContext, p)) { - const descriptor = getOwnPropertyDescriptor(incubatorContext, p); - descriptorTargetMap.set(p, 'globalContext'); - // A property cannot be reported as non-configurable, if it does not exist as an own property of the membraneTarget object - if (descriptor && !descriptor.configurable) { - descriptor.configurable = true; + const descriptor = getOwnPropertyDescriptor(incubatorContext, p); + if (descriptor) { + if (!descriptor.configurable) { + materializeNonConfigurableProperty(membraneTarget, propertiesWithGetter, p, descriptor); + descriptorTargetMap.set(p, 'target'); + return getOwnPropertyDescriptor(membraneTarget, p); } + + descriptorTargetMap.set(p, 'globalContext'); return descriptor; } @@ -274,7 +291,14 @@ export class Membrane { }, defineProperty: (membraneTarget, p, attributes) => { - const from = descriptorTargetMap.get(p); + let from = descriptorTargetMap.get(p); + if (!from && !hasOwnProperty(membraneTarget, p) && !isShieldedInternalGlobal(membraneTarget, p)) { + const descriptor = getOwnPropertyDescriptor(incubatorContext, p); + if (descriptor && !descriptor.configurable) { + materializeNonConfigurableProperty(membraneTarget, propertiesWithGetter, p, descriptor); + from = 'target'; + } + } /* Descriptor must be defined to native window while it comes from native window via Object.getOwnPropertyDescriptor(window, p), otherwise it would cause a TypeError with illegal invocation. @@ -292,6 +316,13 @@ export class Membrane { }, deleteProperty: (membraneTarget, p) => { + if (!hasOwnProperty(membraneTarget, p) && !isShieldedInternalGlobal(membraneTarget, p)) { + const descriptor = getOwnPropertyDescriptor(incubatorContext, p); + if (descriptor && !descriptor.configurable) { + materializeNonConfigurableProperty(membraneTarget, propertiesWithGetter, p, descriptor); + } + } + if (hasOwnProperty(membraneTarget, p)) { delete membraneTarget[p]; this.modifications.delete(p); @@ -334,61 +365,35 @@ export class Membrane { } } -function createMembraneTarget( - endowments: Endowments = {}, - incubatorContext: WindowProxy, -): { - target: MembraneTarget; - propertiesWithGetter: Map; -} { - // map always has the best performance in `has` check scenario - // see https://jsperf.com/array-indexof-vs-set-has/23 - const propertiesWithGetter = new Map(); - const target: MembraneTarget = keys(endowments).reduce((acc, key) => { +function createMembraneTarget(endowments: Endowments = {}): MembraneTarget { + return keys(endowments).reduce((target, key) => { const value = endowments[key]; if (isPropertyDescriptor(value)) { - defineProperty(acc, key, value); + defineProperty(target, key, value); } else { - acc[key] = value; + target[key] = value; } - return acc; + return target; }, {} as MembraneTarget); +} - /* - copy the non-configurable property of incubatorContext to membrane target to avoid TypeError - see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy/handler/getOwnPropertyDescriptor - > A property cannot be reported as non-configurable, if it does not exist as an own property of the target object or if it exists as a configurable own property of the target object. - */ - getOwnPropertyNames(incubatorContext) - .filter((p) => { - const descriptor = getOwnPropertyDescriptor(incubatorContext, p); - // never copy qiankun-internal keys (e.g. the ESM realm accessor) into the target: once copied - // they become own props and isShieldedInternalGlobal would stop shielding them for this app, - // re-exposing the realm accessor to the 2nd+ instance through the proxy (defence in depth) - return !hasOwnProperty(endowments, p) && !descriptor?.configurable && !p.startsWith(esmInternalPrefix); - }) - .forEach((p) => { - const descriptor = getOwnPropertyDescriptor(incubatorContext, p); - if (descriptor) { - const hasGetter = hasOwnProperty(descriptor, 'get'); - if (hasGetter) { - propertiesWithGetter.set(p, true); - } - - defineProperty( - target, - p, - // freeze the descriptor to avoid being modified by zone.js - // see https://github.com/angular/zone.js/blob/a5fe09b0fac27ac5df1fa746042f96f05ccb6a00/lib/browser/define-property.ts#L71 - freeze(descriptor), - ); - } - }); +function materializeNonConfigurableProperty( + target: MembraneTarget, + propertiesWithGetter: Map, + property: PropertyKey, + descriptor: PropertyDescriptor, +): void { + if (hasOwnProperty(descriptor, 'get')) { + propertiesWithGetter.set(property, true); + } - return { + defineProperty( target, - propertiesWithGetter, - }; + property, + // freeze the descriptor to avoid being modified by zone.js + // see https://github.com/angular/zone.js/blob/a5fe09b0fac27ac5df1fa746042f96f05ccb6a00/lib/browser/define-property.ts#L71 + freeze(descriptor), + ); } /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c256701b9..4c927216e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -80,9 +80,18 @@ importers: benchmark: dependencies: + '@micro-zoe/micro-app': + specifier: 1.0.0-rc.32 + version: 1.0.0-rc.32 + garfish: + specifier: 1.19.7 + version: 1.19.7 qiankun: specifier: workspace:* version: link:../packages/qiankun + qiankun-v2: + specifier: npm:qiankun@2.10.16 + version: qiankun@2.10.16 wujie: specifier: 2.1.0 version: 2.1.0 @@ -523,6 +532,9 @@ packages: resolution: {integrity: sha512-z1oyLq5/UVkohVXNDEY70mJbT/sv/t6HYtCvCwNrOri6pxBJDomP9R83KOlwcat+xqBQEdJHjbrPh36f1avmZA==} engines: {node: '>= 14.0.0'} + '@alioth-org/es-module-lexer@1.1.0': + resolution: {integrity: sha512-2GrWjl8Whptm2Za2Zx+z8dm8Dz62hLWy1KPYEyplil4m28c6OR900iHNrYAYXS9QFCO1tqMABrl1daj6Mt66QQ==} + '@antfu/install-pkg@1.1.0': resolution: {integrity: sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==} @@ -1340,6 +1352,27 @@ packages: '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + '@garfish/browser-snapshot@1.19.7': + resolution: {integrity: sha512-OclfcZzML8Jioo7FvrWV2sPMUEWXIDTlquc2rHf0ck1chzsyux7GKb0QWR8xUYkJd9fU/XMimjqTtSgLUXqPzw==} + + '@garfish/browser-vm@1.19.7': + resolution: {integrity: sha512-EMWfGGJYWca4AdfKe0pRJuFDNkk94HTMuALKrlXiKSXdrC06GquAYmuxwvEaRzPKabWrgder/CkEtsGRx7Yc2Q==} + + '@garfish/core@1.19.7': + resolution: {integrity: sha512-zprhusHwMScpbqeO0cV12Bqz6q8FNowl6f84gQ8WSl8Eu7NNcyRWPwzevAbo7aQL+rxSJ/y41d2w5QEOZ9EEKw==} + + '@garfish/hooks@1.19.7': + resolution: {integrity: sha512-27Yr+/IMQuvXuuP44iOe5P8MjhPhaxK64of7SI3Nt6PTNGTUkK+2/5lkj2fa8xrjbJpxHgfloNYsU5B6FS/6YA==} + + '@garfish/loader@1.19.7': + resolution: {integrity: sha512-mOM/x3meYMcXfp1WBtgBuIqDwcQOPpDa8hP2bDj/2lbgZzsy5TinUX7dcPfaCs17hhvA+YRKKK3L4QgZOZWA8g==} + + '@garfish/router@1.19.7': + resolution: {integrity: sha512-YYQ3M2CCBa0bv4cqMcgyhwq7122/Xkq3kB9BaWGGuSQFKiIkfxoQQ4ye2Mpuu4YfDIJK5iQnKqFOllAsUGsOIg==} + + '@garfish/utils@1.19.7': + resolution: {integrity: sha512-q8dFBokCwt1j2WXS/wraMz440edTx48Ai+rLCQJuRDZLTrqHYYFQWIFCLZ1tfXi9lLWzlQv/CowghItM/Etqew==} + '@hapi/bourne@3.0.0': resolution: {integrity: sha512-Waj1cwPXJDucOib4a3bAISsKJVb15MKi9IvmTI/7ssVEm6sywXGjVJDhl6/umt1pK1ZS7PacXU3A1PmFKHEZ2w==} @@ -1541,6 +1574,9 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} + '@micro-zoe/micro-app@1.0.0-rc.32': + resolution: {integrity: sha512-1M34tdcZB9yEvOozMJ/1tOnFVnX3QJtWe4kDevpO/GjNJ5oFjq6gE9jiInJSvvcYZcZTKrIWcM9xUzbFbIFG0A==} + '@microsoft/api-extractor-model@7.28.17': resolution: {integrity: sha512-b2AfLP33oEVtWLeNavSBRdyDa8sKlXjN4pdhBnC4HLontOtjILhL1ERAmZObF4PWSyChnnC2vjb47C9WKCFRGg==} @@ -4666,6 +4702,9 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventemitter2@6.4.9: + resolution: {integrity: sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==} + eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} @@ -4877,6 +4916,9 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + garfish@1.19.7: + resolution: {integrity: sha512-w2i6pr1j2sQ9w0EK7PYGnqgWUu77dOWpGC66AWnS9Xs10hPHcQ9WlcOrulVdAOkkkINKRr2i6NtpuCwk41VSSQ==} + generator-function@2.0.1: resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} engines: {node: '>= 0.4'} @@ -5208,6 +5250,9 @@ packages: import-html-entry@1.12.0: resolution: {integrity: sha512-wloMEMwupKJ8DWvKsEzJTXhHVieEH8ylu9ebeQg7T9JUsPTo0Zwa1EkuSKgKJvOiA2MxAFkeYYvd/E2pKiFtWQ==} + import-html-entry@1.17.0: + resolution: {integrity: sha512-2SDsRlGlE8bqdnGqsOyiDPEWlzJR0jNW4LWopnZl5QE1Yd0nJ7fykWo2GaKUF7Jq7pR0g3dElhuJHyamTt1gPQ==} + import-lazy@4.0.0: resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} engines: {node: '>=8'} @@ -6968,6 +7013,9 @@ packages: resolution: {integrity: sha512-KTqnxsgGiQ6ZAzZCVlJH5eOjSnvlyEgx1m8bkRJfOhmGRqfo5KLvmAlACQkrjEtOQ4B7wF9TdSLIs9O90MX9xA==} engines: {node: '>=16.0.0'} + qiankun@2.10.16: + resolution: {integrity: sha512-Q3tSVUrPnzx8ckEOKIoPnhb5LE28FPKyan/r6jEuGJGqTbIy+3rp6E2/KfU82ZI4yZpef9LFTrnxdj49jAEsmw==} + qs@6.14.1: resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} engines: {node: '>=0.6'} @@ -7438,6 +7486,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + single-spa@5.9.5: + resolution: {integrity: sha512-9SQdmsyz4HSP+3gs6PJzhkaMEg+6zTlu9oxIghnwUX3eq+ajq4ft5egl0iyR55LAmO/UwvU8NgIWs/ZyQMa6dw==} + single-spa@6.0.0: resolution: {integrity: sha512-wFHWD0dNHb1oJF1yGWIkvIpovUUGyES2ljkqy80wMIC73AdCTcKCyJBP1DqoU8pUZrfu2ccLaFIKL2PVrAOpOg==} @@ -8558,6 +8609,8 @@ snapshots: dependencies: '@algolia/client-common': 5.47.0 + '@alioth-org/es-module-lexer@1.1.0': {} + '@antfu/install-pkg@1.1.0': dependencies: package-manager-detector: 1.6.0 @@ -9382,6 +9435,58 @@ snapshots: '@gar/promisify@1.1.3': {} + '@garfish/browser-snapshot@1.19.7': + dependencies: + '@garfish/core': 1.19.7 + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + + '@garfish/browser-vm@1.19.7': + dependencies: + '@garfish/core': 1.19.7 + '@garfish/hooks': 1.19.7 + '@garfish/loader': 1.19.7 + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + + '@garfish/core@1.19.7': + dependencies: + '@alioth-org/es-module-lexer': 1.1.0 + '@garfish/hooks': 1.19.7 + '@garfish/loader': 1.19.7 + '@garfish/utils': 1.19.7 + eventemitter2: 6.4.9 + transitivePeerDependencies: + - supports-color + + '@garfish/hooks@1.19.7': + dependencies: + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + + '@garfish/loader@1.19.7': + dependencies: + '@garfish/hooks': 1.19.7 + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + + '@garfish/router@1.19.7': + dependencies: + '@garfish/core': 1.19.7 + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + + '@garfish/utils@1.19.7': + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + '@hapi/bourne@3.0.0': {} '@humanwhocodes/config-array@0.13.0': @@ -9607,6 +9712,8 @@ snapshots: dependencies: langium: 3.3.1 + '@micro-zoe/micro-app@1.0.0-rc.32': {} + '@microsoft/api-extractor-model@7.28.17(@types/node@18.19.130)': dependencies: '@microsoft/tsdoc': 0.14.2 @@ -13214,6 +13321,8 @@ snapshots: etag@1.8.1: {} + eventemitter2@6.4.9: {} + eventemitter3@4.0.7: {} events@3.3.0: {} @@ -13532,6 +13641,16 @@ snapshots: functions-have-names@1.2.3: {} + garfish@1.19.7: + dependencies: + '@garfish/browser-snapshot': 1.19.7 + '@garfish/browser-vm': 1.19.7 + '@garfish/core': 1.19.7 + '@garfish/router': 1.19.7 + '@garfish/utils': 1.19.7 + transitivePeerDependencies: + - supports-color + generator-function@2.0.1: {} gensync@1.0.0-beta.2: {} @@ -13934,6 +14053,10 @@ snapshots: dependencies: '@babel/runtime': 7.26.10 + import-html-entry@1.17.0: + dependencies: + '@babel/runtime': 7.28.6 + import-lazy@4.0.0: {} import-local@3.2.0: @@ -15900,6 +16023,13 @@ snapshots: pvutils@1.1.5: {} + qiankun@2.10.16: + dependencies: + '@babel/runtime': 7.28.6 + import-html-entry: 1.17.0 + lodash: 4.17.23 + single-spa: 5.9.5 + qs@6.14.1: dependencies: side-channel: 1.1.0 @@ -16479,6 +16609,8 @@ snapshots: signal-exit@4.1.0: {} + single-spa@5.9.5: {} + single-spa@6.0.0: {} sisteransi@1.0.5: {} diff --git a/tsconfig.json b/tsconfig.json index f9208ae60..4d5aa05b3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,5 +29,5 @@ "@qiankunjs/*": ["packages/*/src"] } }, - "exclude": ["node_modules", "examples", "packages/**/dist"] + "exclude": ["node_modules", "examples", "benchmark", "packages/**/dist"] }