From e1da385c5d93d1789f04c017ee68a27c3d0e38e3 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Thu, 16 Jul 2026 14:13:03 -0400 Subject: [PATCH 1/4] Add declarative CI benchmark matrix and `vx-bench matrix` CLI ### Motivation - Centralize CI benchmark coverage in Python so workflow YAMLs do not embed large, duplicated JSON matrices. - Provide a reproducible way to emit GitHub Actions `include` arrays for named profiles to keep coverage and workflow shape consistent. ### Description - Add `bench_orchestrator/matrix.py` which implements `TargetSet`, `BenchmarkDef`, `Profile`, storage handling, and `resolve_matrix` to render profile entries into GitHub Actions matrix entries. - Add `bench_orchestrator/benchmarks.py` to declare `BENCHMARKS` and `PROFILES` as the canonical source of benchmark definitions and CI profiles. - Add a new CLI command `matrix` in `bench_orchestrator/cli.py` to list profiles or emit JSON with `resolve_matrix`, and wire in the new modules. - Update `README.md` to document the `matrix` command and the declarative benchmark matrix approach. ### Testing - Added unit tests in `bench-orchestrator/tests/test_matrix.py` covering policy narrowing, resolver output shape, profile roles, preserved display/scale values, and CLI behavior, and these tests were run with `pytest` and passed. - Exercised the CLI behavior using the `typer` test runner to assert `vx-bench matrix develop` emits JSON and unknown profiles return a failing exit code, and these checks succeeded. Signed-off-by: Codex Signed-off-by: Connor Tsui --- bench-orchestrator/README.md | 35 ++++ .../bench_orchestrator/benchmarks.py | 133 +++++++++++++ bench-orchestrator/bench_orchestrator/cli.py | 28 +++ .../bench_orchestrator/matrix.py | 178 ++++++++++++++++++ bench-orchestrator/tests/test_matrix.py | 127 +++++++++++++ 5 files changed, 501 insertions(+) create mode 100644 bench-orchestrator/bench_orchestrator/benchmarks.py create mode 100644 bench-orchestrator/bench_orchestrator/matrix.py create mode 100644 bench-orchestrator/tests/test_matrix.py diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 0b267008a85..f6ced77e646 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -70,6 +70,23 @@ vx-bench prepare-data [options] - `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'` - `--opt`: Benchmark-specific options such as `scale-factor=10.0` + +### `matrix` - Resolve CI Benchmark Matrices + +Emit the GitHub Actions `include:` array for a named profile. The benchmark workflows can use this +to keep benchmark coverage in Python instead of copying large JSON matrices between YAML files. + +```bash +vx-bench matrix # list available profiles +vx-bench matrix develop # emit compact JSON +vx-bench matrix nightly --pretty +``` + +**Options:** + +- `--list`: List available profiles and exit +- `--pretty`: Pretty-print the JSON output + ### `compare` - Compare Results Compare benchmark results within a run or across multiple runs. Results are displayed in a pivot table format. @@ -137,6 +154,24 @@ vx-bench clean --older-than "30 days" [options] - `--keep-labeled`: Don't delete labeled runs (default: true) - `--dry-run, -n`: Show what would be deleted +## Declarative Benchmark Matrix + +CI benchmark coverage is declared in `bench_orchestrator/benchmarks.py` and rendered by +`bench_orchestrator/matrix.py`. This keeps the source of truth out of workflow YAML while still +emitting the JSON shape GitHub Actions expects. + +The model separates three review concerns: + +- **Benchmark definitions** declare the suite, storage location, scale factor, and supported + engine/format targets. +- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, and + `nightly`). +- **Matrix rendering** converts a profile into stable GitHub Actions `include` entries with fields + such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage metadata. + +When adding coverage, update the declarations first and add focused tests for the resolved profile +entries rather than duplicating large inline JSON matrices in workflow files. + ## Example Workflows ### 1. Basic Performance Comparison diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py new file mode 100644 index 00000000000..1c6e490183f --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""SQL benchmark declarations and CI profiles. + +Edit this file when changing benchmark coverage. Matrix rendering lives in +``bench_orchestrator.matrix`` so workflow shape and benchmark coverage do not drift together. +""" + +from .config import Benchmark, Engine, Format +from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck + + +def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: + suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}" + if storage is Storage.NVME: + target_set = df( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.LANCE, + ) | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB) + local_dir = None + remote_storage = None + else: + target_set = STANDARD + local_dir = f"vortex-bench/data/tpch/{scale_factor:.1f}" + remote_storage = f"s3://vortex-ci-benchmark-datasets/${{{{github.ref_name}}}}/${{{{github.run_id}}}}/tpch/{scale_factor:.1f}/" + + name = f"TPC-H on {storage.label}" if scale_factor == 100 else f"TPC-H SF={scale_factor:g} on {storage.label}" + return BenchmarkDef( + id=f"tpch-{storage.value}{suffix}", + benchmark=Benchmark.TPCH, + name=name, + targets=target_set, + storage=storage, + scale_factor=scale_factor, + iterations=iterations, + nightly=scale_factor == 100, + local_dir=local_dir, + remote_storage=remote_storage, + ) + + +def _clickbench(benchmark: Benchmark, name: str) -> BenchmarkDef: + return BenchmarkDef( + id=f"{benchmark.value}-nvme", + benchmark=benchmark, + name=name, + targets=df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.LANCE) + | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB), + ) + + +def _fineweb(storage: Storage) -> BenchmarkDef: + if storage is Storage.NVME: + return BenchmarkDef( + id="fineweb", + benchmark=Benchmark.FINEWEB, + name="FineWeb NVMe", + targets=STANDARD, + scale_factor=100, + ) + return BenchmarkDef( + id="fineweb-s3", + benchmark=Benchmark.FINEWEB, + name="FineWeb S3", + targets=STANDARD, + storage=Storage.S3, + scale_factor=100, + local_dir="vortex-bench/data/fineweb", + remote_storage="s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + ) + + +BENCHMARKS: list[BenchmarkDef] = [ + _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"), + _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"), + _tpch(1.0, Storage.NVME), + _tpch(1.0, Storage.S3), + _tpch(10.0, Storage.NVME), + _tpch(10.0, Storage.S3), + _tpch(100, Storage.NVME, iterations=None), + _tpch(100.0, Storage.S3, iterations=None), + BenchmarkDef( + id="tpcds-nvme", + benchmark=Benchmark.TPCDS, + name="TPC-DS SF=1 on NVME", + targets=STANDARD | duck(Format.DUCKDB), + scale_factor=1.0, + ), + BenchmarkDef( + id="statpopgen", + benchmark=Benchmark.STATPOPGEN, + name="Statistical and Population Genetics", + targets=STANDARD.only(Engine.DUCKDB), + scale_factor=100, + local_dir="vortex-bench/data/statpopgen", + ), + _fineweb(Storage.NVME), + _fineweb(Storage.S3), + BenchmarkDef( + id="polarsignals", + benchmark=Benchmark.POLARSIGNALS, + name="PolarSignals Profiling", + targets=df(Format.VORTEX), + scale_factor=1, + ), + BenchmarkDef( + id="appian-nvme", + benchmark=Benchmark.APPIAN, + name="Appian on NVME", + targets=STANDARD | duck(Format.DUCKDB), + iterations=10, + ), +] + +PROFILES: dict[str, Profile] = { + "develop": Profile( + targets=all_targets, + description="Every regular SQL benchmark at full target coverage.", + ), + "pr": Profile( + targets=defaults, + description="Every regular SQL benchmark at default targets.", + ), + "nightly": Profile( + nightly=True, + targets=defaults, + description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", + ), +} diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index b9d7f9bb2ef..cbe5ac3ac0e 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -3,6 +3,7 @@ """CLI for benchmark orchestration.""" +import json import subprocess from contextlib import contextmanager from datetime import datetime, timedelta @@ -15,6 +16,7 @@ from rich.console import Console from rich.table import Table +from .benchmarks import BENCHMARKS, PROFILES from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -29,6 +31,7 @@ parse_targets_json, resolve_axis_targets, ) +from .matrix import resolve_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -214,6 +217,31 @@ def prepare_data( raise typer.Exit(1) from exc +@app.command("matrix") +def matrix( + profile: Annotated[ + str | None, + typer.Argument(help="Profile to resolve; omit to list available profiles"), + ] = None, + list_profiles: Annotated[bool, typer.Option("--list", help="List available profiles and exit")] = False, + pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False, +) -> None: + """Emit the GitHub Actions benchmark matrix for a profile.""" + if profile is None or list_profiles: + for name, prof in PROFILES.items(): + console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + return + + prof = PROFILES.get(profile) + if prof is None: + known = ", ".join(PROFILES) + console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") + raise typer.Exit(1) + + entries = resolve_matrix(prof, BENCHMARKS) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) + + @app.command() def run( benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")], diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py new file mode 100644 index 00000000000..b97df9714ac --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Resolve declarative benchmark definitions into GitHub Actions matrices.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from enum import Enum + +from .config import Benchmark, BenchmarkTarget, Engine, Format + + +class Storage(Enum): + """Where a benchmark's data lives when it runs.""" + + NVME = "nvme" + S3 = "s3" + + @property + def label(self) -> str: + """Human-facing name used in benchmark display names.""" + return "NVME" if self is Storage.NVME else "S3" + + +def _dedupe(targets: Iterable[BenchmarkTarget]) -> tuple[BenchmarkTarget, ...]: + """Normalize and de-duplicate targets, preserving first-seen order.""" + return tuple(dict.fromkeys(target.normalized() for target in targets)) + + +@dataclass(frozen=True) +class TargetSet: + """An ordered set of engine/format targets with small set algebra.""" + + targets: tuple[BenchmarkTarget, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "targets", _dedupe(self.targets)) + + def __or__(self, other: TargetSet) -> TargetSet: + """Return the ordered union of two target sets.""" + return TargetSet(self.targets + other.targets) + + def only(self, *engines: Engine) -> TargetSet: + """Restrict the target set to the given engines.""" + keep = set(engines) + return TargetSet(tuple(target for target in self.targets if target.engine in keep)) + + def formats(self) -> list[Format]: + """Return referenced formats in first-seen order.""" + return list(dict.fromkeys(target.format for target in self.targets)) + + def __iter__(self): + return iter(self.targets) + + def __len__(self) -> int: + return len(self.targets) + + +def targets(engine: Engine, *formats: Format) -> TargetSet: + """Build targets for one engine across several formats.""" + return TargetSet(tuple(BenchmarkTarget(engine=engine, format=fmt) for fmt in formats)) + + +def df(*formats: Format) -> TargetSet: + """Build DataFusion targets across several formats.""" + return targets(Engine.DATAFUSION, *formats) + + +def duck(*formats: Format) -> TargetSet: + """Build DuckDB targets across several formats.""" + return targets(Engine.DUCKDB, *formats) + + +STANDARD = df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT) | duck( + Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT +) +DEFAULTS = df(Format.PARQUET, Format.VORTEX) | duck(Format.PARQUET, Format.VORTEX) +_NOT_GENERATED = frozenset({Format.ARROW, Format.LANCE}) +_FORMAT_ORDER = ( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.VORTEX_NATIVE, + Format.DUCKDB, + Format.LANCE, +) + + +@dataclass(frozen=True) +class BenchmarkDef: + """A benchmark and the canonical target superset it can run.""" + + id: str + benchmark: Benchmark + name: str + targets: TargetSet + storage: Storage = Storage.NVME + scale_factor: float | int | None = None + iterations: int | None = None + nightly: bool = False + local_dir: str | None = None + remote_storage: str | None = None + + @property + def subcommand(self) -> str: + """Return the ``vx-bench`` subcommand for this benchmark.""" + return self.benchmark.value + + +TargetPolicy = Callable[[BenchmarkDef], TargetSet] + + +def all_targets(benchmark: BenchmarkDef) -> TargetSet: + """Run every target declared by a benchmark.""" + return benchmark.targets + + +def defaults(benchmark: BenchmarkDef) -> TargetSet: + """Run the cheap default lane intersected with a benchmark's declared targets.""" + return TargetSet(tuple(target for target in benchmark.targets if target in DEFAULTS.targets)) + + +@dataclass(frozen=True) +class Profile: + """A named CI benchmark configuration.""" + + nightly: bool = False + targets: TargetPolicy = all_targets + description: str = "" + + +def _valid_for_storage(target_set: TargetSet, storage: Storage) -> TargetSet: + """Drop targets that are invalid for the storage backend.""" + if storage is Storage.S3: + return TargetSet(tuple(target for target in target_set if target.format is not Format.LANCE)) + return target_set + + +def _data_formats(target_set: TargetSet) -> list[Format]: + """Return data formats that the data-generation step must produce.""" + present = set(target_set.formats()) + return [fmt for fmt in _FORMAT_ORDER if fmt in present and fmt not in _NOT_GENERATED] + + +def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet) -> dict[str, object]: + """Build one GitHub Actions ``include`` entry.""" + entry: dict[str, object] = { + "id": benchmark.id, + "subcommand": benchmark.subcommand, + "name": benchmark.name, + "targets": [target.to_dict() for target in run_targets], + "data_formats": [fmt.value for fmt in _data_formats(run_targets)], + } + if benchmark.scale_factor is not None: + entry["scale_factor"] = str(benchmark.scale_factor) + if benchmark.iterations is not None: + entry["iterations"] = str(benchmark.iterations) + if benchmark.local_dir is not None: + entry["local_dir"] = benchmark.local_dir + if benchmark.remote_storage is not None: + entry["remote_storage"] = benchmark.remote_storage + return entry + + +def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: + """Resolve a profile into GitHub Actions matrix entries.""" + entries: list[dict[str, object]] = [] + for benchmark in benchmarks: + if benchmark.nightly != profile.nightly: + continue + run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) + if len(run_targets) == 0: + continue + entries.append(_matrix_entry(benchmark, run_targets)) + return entries diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py new file mode 100644 index 00000000000..77eb91f9012 --- /dev/null +++ b/bench-orchestrator/tests/test_matrix.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Contract tests for the declarative CI benchmark matrix.""" + +import json +from typing import cast + +from bench_orchestrator import cli as cli_module +from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES +from bench_orchestrator.config import Benchmark, Engine, Format +from bench_orchestrator.matrix import ( + DEFAULTS, + BenchmarkDef, + Profile, + Storage, + all_targets, + defaults, + df, + duck, + resolve_matrix, +) +from typer.testing import CliRunner + +runner = CliRunner() + + +def _targets(entry: dict[str, object]) -> list[dict[str, str]]: + return cast("list[dict[str, str]]", entry["targets"]) + + +def test_default_policy_only_narrows_declared_targets() -> None: + benchmark = BenchmarkDef( + id="duckdb-only", + benchmark=Benchmark.TPCH, + name="DuckDB only", + targets=duck(Format.VORTEX, Format.DUCKDB), + ) + + assert set(defaults(benchmark)) == set(duck(Format.VORTEX)) + + +def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: + benchmark = BenchmarkDef( + id="remote", + benchmark=Benchmark.TPCH, + name="Remote", + targets=df(Format.ARROW, Format.PARQUET, Format.LANCE, Format.VORTEX) | duck(Format.DUCKDB), + storage=Storage.S3, + scale_factor=1, + iterations=10, + local_dir="data/tpch", + remote_storage="s3://bucket/tpch/1.0/", + ) + + [entry] = resolve_matrix(Profile(targets=all_targets), [benchmark]) + + assert entry == { + "id": "remote", + "subcommand": "tpch", + "name": "Remote", + "targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "duckdb"}, + ], + "data_formats": ["parquet", "vortex", "duckdb"], + "scale_factor": "1", + "iterations": "10", + "local_dir": "data/tpch", + "remote_storage": "s3://bucket/tpch/1.0/", + } + + +def test_ci_profiles_have_distinct_and_consistent_roles() -> None: + assert set(PROFILES) == {"develop", "pr", "nightly"} + regular_ids = {benchmark.id for benchmark in BENCHMARKS if not benchmark.nightly} + nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.nightly} + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + pr = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert set(develop) == regular_ids + assert set(pr) == regular_ids + assert set(nightly) == nightly_ids + + default_targets = set(DEFAULTS) + for entry in (*pr.values(), *nightly.values()): + targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} + assert targets + assert targets <= {(target.engine, target.format) for target in default_targets} + + tpch = develop["tpch-nvme"] + assert [(target["engine"], target["format"]) for target in _targets(tpch)] == [ + ("datafusion", "arrow"), + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("datafusion", "vortex-compact"), + ("datafusion", "lance"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + ("duckdb", "vortex-compact"), + ("duckdb", "duckdb"), + ] + + +def test_existing_display_and_scale_values_are_preserved() -> None: + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert develop["tpch-nvme"]["scale_factor"] == "1.0" + assert develop["statpopgen"]["scale_factor"] == "100" + assert develop["polarsignals"]["scale_factor"] == "1" + assert nightly["tpch-nvme"]["name"] == "TPC-H on NVME" + assert nightly["tpch-nvme"]["scale_factor"] == "100" + assert nightly["tpch-s3"]["name"] == "TPC-H on S3" + assert nightly["tpch-s3"]["scale_factor"] == "100.0" + + +def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: + result = runner.invoke(cli_module.app, ["matrix", "develop"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) + + result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) + assert result.exit_code == 1 From 995a5b6020e4176fea79f3a91a33a34a8dae55c7 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 20 Jul 2026 11:35:09 -0400 Subject: [PATCH 2/4] Wire declarative benchmark profiles into CI Signed-off-by: Connor Tsui --- .github/workflows/bench-pr.yml | 1 + .github/workflows/bench.yml | 24 +- .github/workflows/nightly-bench.yml | 44 +- .github/workflows/sql-benchmarks.yml | 569 ++---------------- .github/workflows/sql-pr.yml | 2 +- .github/workflows/sql-vortex-pr.yml | 23 +- bench-orchestrator/README.md | 7 +- .../bench_orchestrator/benchmarks.py | 54 +- .../bench_orchestrator/matrix.py | 59 +- bench-orchestrator/tests/test_matrix.py | 68 ++- 10 files changed, 211 insertions(+), 640 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index b5b011150a6..9d2ada7dfdf 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -139,3 +139,4 @@ jobs: secrets: inherit with: mode: "pr" + benchmark_profile: "pr-full" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b1e29277f00..11409c9349e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -171,31 +171,11 @@ jobs: secrets: inherit with: mode: "develop" + benchmark_profile: "develop" sql-vortex: uses: ./.github/workflows/sql-benchmarks.yml secrets: inherit with: mode: "develop" - benchmark_matrix: | - [ - { - "id": "vortex-queries", - "subcommand": "vortex", - "name": "Vortex queries", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "iterations": "100" - } - ] + benchmark_profile: "vortex" diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index b89395516be..ff3f52337a9 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -23,49 +23,7 @@ jobs: with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} - benchmark_matrix: | - [ - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H on S3", - "local_dir": "vortex-bench/data/tpch/100.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100.0" - } - ] + benchmark_profile: "nightly" strategy: # A single run not should kill the others fail-fast: false diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 59dd9571b36..9391a475841 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -6,509 +6,49 @@ on: mode: required: true type: string + benchmark_profile: + required: true + type: string + description: "Declarative benchmark profile to resolve" machine_type: required: false type: string default: i7i.metal-24xl - benchmark_matrix: - required: false - type: string - description: "JSON string containing the full matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpch-s3-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on S3", - "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - }, - { - "id": "appian-nvme", - "subcommand": "appian", - "name": "Appian on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "iterations": "10" - } - ] - base_benchmark_matrix: - required: false - type: string - description: "JSON string containing the base matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - } - ] - benchmark_profile: - required: false - type: string - description: "Benchmark profile to run: full or base" - default: "full" jobs: + resolve-matrix: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + benchmark_matrix: ${{ steps.resolve.outputs.benchmark_matrix }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + - name: Resolve benchmark matrix + id: resolve + shell: bash + env: + BENCHMARK_PROFILE: ${{ inputs.benchmark_profile }} + run: | + set -Eeuo pipefail + + matrix="$(uv run --project bench-orchestrator vx-bench matrix "$BENCHMARK_PROFILE")" + { + echo 'benchmark_matrix<<__BENCHMARK_MATRIX__' + echo "$matrix" + echo '__BENCHMARK_MATRIX__' + } >> "$GITHUB_OUTPUT" + bench: + needs: resolve-matrix timeout-minutes: 120 env: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" @@ -518,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.benchmark_profile == 'base' && inputs.base_benchmark_matrix || inputs.benchmark_matrix) }} + include: ${{ fromJSON(needs.resolve-matrix.outputs.benchmark_matrix) }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -555,21 +95,6 @@ jobs: - uses: ./.github/actions/system-info - - name: Resolve benchmark targets - id: targets - shell: bash - run: | - if [ "${{ inputs.mode }}" = "pr" ]; then - targets='${{ toJSON(matrix.pr_targets) }}' - else - targets='${{ toJSON(matrix.develop_targets) }}' - fi - { - echo 'targets_json<<__TARGETS_JSON__' - echo "$targets" - echo '__TARGETS_JSON__' - } >> "$GITHUB_OUTPUT" - - name: Build binaries shell: bash env: @@ -600,13 +125,14 @@ jobs: role-duration-seconds: 7200 - name: Upload data - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" + REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/" run: | - aws s3 rm --recursive ${{ matrix.remote_storage }} - aws s3 cp --recursive ${{matrix.local_dir}} ${{ matrix.remote_storage }} + aws s3 rm --recursive "$REMOTE_STORAGE" + aws s3 cp --recursive "${{ matrix.local_dir }}" "$REMOTE_STORAGE" - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false @@ -619,7 +145,7 @@ jobs: extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - name: Run ${{ matrix.name }} benchmark - if: matrix.remote_storage == null || github.event.pull_request.head.repo.fork == true + if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true shell: bash env: OTEL_SERVICE_NAME: "vortex-bench" @@ -629,7 +155,7 @@ jobs: OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ @@ -638,7 +164,7 @@ jobs: ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Run ${{ matrix.name }} benchmark (remote) - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" @@ -647,19 +173,20 @@ jobs: OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}" OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" + REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ --runner "ec2_${{ inputs.machine_type }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ - --opt remote-data-dir=${{ matrix.remote_storage }} \ + --opt remote-data-dir="$REMOTE_STORAGE" \ ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Capture file sizes - if: matrix.remote_storage == null + if: matrix.remote_key == null shell: bash run: | uv run --no-project scripts/capture-file-sizes.py \ diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/sql-pr.yml index 45b0ed1d675..8ffe7f163ae 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/sql-pr.yml @@ -39,4 +39,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: ${{ inputs.benchmark_profile || 'base' }} + benchmark_profile: ${{ inputs.benchmark_profile == 'full' && 'pr-full' || 'pr' }} diff --git a/.github/workflows/sql-vortex-pr.yml b/.github/workflows/sql-vortex-pr.yml index 2b99eb0e4d9..d0ab4d06c8e 100644 --- a/.github/workflows/sql-vortex-pr.yml +++ b/.github/workflows/sql-vortex-pr.yml @@ -23,25 +23,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_matrix: | - [ - { - "id": "vortex-queries", - "subcommand": "vortex", - "name": "Vortex queries", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "iterations": "100" - } - ] + benchmark_profile: "vortex" diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index f6ced77e646..16f2ce3bf39 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -79,6 +79,7 @@ to keep benchmark coverage in Python instead of copying large JSON matrices betw ```bash vx-bench matrix # list available profiles vx-bench matrix develop # emit compact JSON +vx-bench matrix pr-full # emit full pull-request coverage vx-bench matrix nightly --pretty ``` @@ -164,10 +165,10 @@ The model separates three review concerns: - **Benchmark definitions** declare the suite, storage location, scale factor, and supported engine/format targets. -- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, and - `nightly`). +- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, + `pr-full`, `nightly`, and `vortex`). - **Matrix rendering** converts a profile into stable GitHub Actions `include` entries with fields - such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage metadata. + such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage keys. When adding coverage, update the declarations first and add focused tests for the resolved profile entries rather than duplicating large inline JSON matrices in workflow files. diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py index 1c6e490183f..015517cbc40 100644 --- a/bench-orchestrator/bench_orchestrator/benchmarks.py +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -8,7 +8,21 @@ """ from .config import Benchmark, Engine, Format -from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck +from .matrix import ( + DEFAULTS, + STANDARD, + BenchmarkDef, + BenchmarkGroup, + Profile, + Storage, + all_targets, + defaults, + df, + duck, + pr_base_benchmarks, + pr_defaults, + pr_full, +) def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: @@ -22,11 +36,11 @@ def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None Format.LANCE, ) | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB) local_dir = None - remote_storage = None + remote_key = None else: target_set = STANDARD local_dir = f"vortex-bench/data/tpch/{scale_factor:.1f}" - remote_storage = f"s3://vortex-ci-benchmark-datasets/${{{{github.ref_name}}}}/${{{{github.run_id}}}}/tpch/{scale_factor:.1f}/" + remote_key = f"tpch/{scale_factor:.1f}" name = f"TPC-H on {storage.label}" if scale_factor == 100 else f"TPC-H SF={scale_factor:g} on {storage.label}" return BenchmarkDef( @@ -37,9 +51,10 @@ def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None storage=storage, scale_factor=scale_factor, iterations=iterations, - nightly=scale_factor == 100, + group=BenchmarkGroup.NIGHTLY if scale_factor == 100 else BenchmarkGroup.REGULAR, + pr_base=storage is Storage.NVME or scale_factor != 10, local_dir=local_dir, - remote_storage=remote_storage, + remote_key=remote_key, ) @@ -50,6 +65,7 @@ def _clickbench(benchmark: Benchmark, name: str) -> BenchmarkDef: name=name, targets=df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.LANCE) | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB), + pr_targets=DEFAULTS | duck(Format.DUCKDB), ) @@ -70,7 +86,7 @@ def _fineweb(storage: Storage) -> BenchmarkDef: storage=Storage.S3, scale_factor=100, local_dir="vortex-bench/data/fineweb", - remote_storage="s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + remote_key="fineweb", ) @@ -112,8 +128,18 @@ def _fineweb(storage: Storage) -> BenchmarkDef: benchmark=Benchmark.APPIAN, name="Appian on NVME", targets=STANDARD | duck(Format.DUCKDB), + pr_targets=DEFAULTS | duck(Format.DUCKDB), + pr_base=False, iterations=10, ), + BenchmarkDef( + id="vortex-queries", + benchmark=Benchmark.VORTEX_QUERIES, + name="Vortex queries", + targets=DEFAULTS, + group=BenchmarkGroup.VORTEX, + iterations=100, + ), ] PROFILES: dict[str, Profile] = { @@ -122,12 +148,22 @@ def _fineweb(storage: Storage) -> BenchmarkDef: description="Every regular SQL benchmark at full target coverage.", ), "pr": Profile( - targets=defaults, - description="Every regular SQL benchmark at default targets.", + benchmarks=pr_base_benchmarks, + targets=pr_defaults, + description="The cheaper pull-request SQL benchmark lane.", + ), + "pr-full": Profile( + targets=pr_full, + data_formats=all_targets, + description="Every regular SQL benchmark at full PR target coverage.", ), "nightly": Profile( - nightly=True, + group=BenchmarkGroup.NIGHTLY, targets=defaults, description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", ), + "vortex": Profile( + group=BenchmarkGroup.VORTEX, + description="The Vortex query suite run on pushes and pull requests.", + ), } diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py index b97df9714ac..f28e74c8957 100644 --- a/bench-orchestrator/bench_orchestrator/matrix.py +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -24,6 +24,14 @@ def label(self) -> str: return "NVME" if self is Storage.NVME else "S3" +class BenchmarkGroup(Enum): + """A separately scheduled group of benchmark definitions.""" + + REGULAR = "regular" + NIGHTLY = "nightly" + VORTEX = "vortex" + + def _dedupe(targets: Iterable[BenchmarkTarget]) -> tuple[BenchmarkTarget, ...]: """Normalize and de-duplicate targets, preserving first-seen order.""" return tuple(dict.fromkeys(target.normalized() for target in targets)) @@ -100,9 +108,11 @@ class BenchmarkDef: storage: Storage = Storage.NVME scale_factor: float | int | None = None iterations: int | None = None - nightly: bool = False + group: BenchmarkGroup = BenchmarkGroup.REGULAR + pr_targets: TargetSet | None = None + pr_base: bool = True local_dir: str | None = None - remote_storage: str | None = None + remote_key: str | None = None @property def subcommand(self) -> str: @@ -111,6 +121,7 @@ def subcommand(self) -> str: TargetPolicy = Callable[[BenchmarkDef], TargetSet] +BenchmarkPolicy = Callable[[BenchmarkDef], bool] def all_targets(benchmark: BenchmarkDef) -> TargetSet: @@ -123,12 +134,39 @@ def defaults(benchmark: BenchmarkDef) -> TargetSet: return TargetSet(tuple(target for target in benchmark.targets if target in DEFAULTS.targets)) +def pr_full(benchmark: BenchmarkDef) -> TargetSet: + """Return the full target set supported by pull-request runners.""" + if benchmark.pr_targets is not None: + return benchmark.pr_targets + return TargetSet(tuple(target for target in benchmark.targets if target.format is not Format.LANCE)) + + +def pr_defaults(benchmark: BenchmarkDef) -> TargetSet: + """Return the cheap PR lane while preserving Arrow coverage for regular TPC-H.""" + allowed = set(DEFAULTS) + if benchmark.benchmark is Benchmark.TPCH: + allowed.update(df(Format.ARROW)) + return TargetSet(tuple(target for target in pr_full(benchmark) if target in allowed)) + + +def all_benchmarks(_: BenchmarkDef) -> bool: + """Include every benchmark in a profile's group.""" + return True + + +def pr_base_benchmarks(benchmark: BenchmarkDef) -> bool: + """Include benchmarks assigned to the cheaper PR lane.""" + return benchmark.pr_base + + @dataclass(frozen=True) class Profile: """A named CI benchmark configuration.""" - nightly: bool = False + group: BenchmarkGroup = BenchmarkGroup.REGULAR + benchmarks: BenchmarkPolicy = all_benchmarks targets: TargetPolicy = all_targets + data_formats: TargetPolicy | None = None description: str = "" @@ -145,14 +183,14 @@ def _data_formats(target_set: TargetSet) -> list[Format]: return [fmt for fmt in _FORMAT_ORDER if fmt in present and fmt not in _NOT_GENERATED] -def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet) -> dict[str, object]: +def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet, data_format_targets: TargetSet) -> dict[str, object]: """Build one GitHub Actions ``include`` entry.""" entry: dict[str, object] = { "id": benchmark.id, "subcommand": benchmark.subcommand, "name": benchmark.name, "targets": [target.to_dict() for target in run_targets], - "data_formats": [fmt.value for fmt in _data_formats(run_targets)], + "data_formats": [fmt.value for fmt in _data_formats(data_format_targets)], } if benchmark.scale_factor is not None: entry["scale_factor"] = str(benchmark.scale_factor) @@ -160,8 +198,8 @@ def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet) -> dict[str, entry["iterations"] = str(benchmark.iterations) if benchmark.local_dir is not None: entry["local_dir"] = benchmark.local_dir - if benchmark.remote_storage is not None: - entry["remote_storage"] = benchmark.remote_storage + if benchmark.remote_key is not None: + entry["remote_key"] = benchmark.remote_key return entry @@ -169,10 +207,13 @@ def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list """Resolve a profile into GitHub Actions matrix entries.""" entries: list[dict[str, object]] = [] for benchmark in benchmarks: - if benchmark.nightly != profile.nightly: + if benchmark.group is not profile.group or not profile.benchmarks(benchmark): continue run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) if len(run_targets) == 0: continue - entries.append(_matrix_entry(benchmark, run_targets)) + data_format_targets = run_targets + if profile.data_formats is not None: + data_format_targets = _valid_for_storage(profile.data_formats(benchmark), benchmark.storage) + entries.append(_matrix_entry(benchmark, run_targets, data_format_targets)) return entries diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 77eb91f9012..06e9bd5eb63 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -12,12 +12,14 @@ from bench_orchestrator.matrix import ( DEFAULTS, BenchmarkDef, + BenchmarkGroup, Profile, Storage, all_targets, defaults, df, duck, + pr_full, resolve_matrix, ) from typer.testing import CliRunner @@ -50,7 +52,7 @@ def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: scale_factor=1, iterations=10, local_dir="data/tpch", - remote_storage="s3://bucket/tpch/1.0/", + remote_key="tpch/1.0", ) [entry] = resolve_matrix(Profile(targets=all_targets), [benchmark]) @@ -69,27 +71,39 @@ def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: "scale_factor": "1", "iterations": "10", "local_dir": "data/tpch", - "remote_storage": "s3://bucket/tpch/1.0/", + "remote_key": "tpch/1.0", } def test_ci_profiles_have_distinct_and_consistent_roles() -> None: - assert set(PROFILES) == {"develop", "pr", "nightly"} - regular_ids = {benchmark.id for benchmark in BENCHMARKS if not benchmark.nightly} - nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.nightly} + assert set(PROFILES) == {"develop", "pr", "pr-full", "nightly", "vortex"} + regular_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.REGULAR} + nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.NIGHTLY} develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} pr = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr"], BENCHMARKS)} + pr_full_matrix = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr-full"], BENCHMARKS)} nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + vortex = resolve_matrix(PROFILES["vortex"], BENCHMARKS) assert set(develop) == regular_ids - assert set(pr) == regular_ids + assert set(pr_full_matrix) == regular_ids + assert set(pr) == regular_ids - {"appian-nvme", "tpch-s3-10"} assert set(nightly) == nightly_ids - - default_targets = set(DEFAULTS) - for entry in (*pr.values(), *nightly.values()): + assert [entry["id"] for entry in vortex] == ["vortex-queries"] + assert develop["tpch-s3"]["remote_key"] == "tpch/1.0" + assert nightly["tpch-s3"]["remote_key"] == "tpch/100.0" + assert "${{" not in json.dumps([*develop.values(), *pr.values(), *pr_full_matrix.values(), *nightly.values()]) + + default_targets = {(target.engine, target.format) for target in DEFAULTS} + pr_targets = default_targets | {(Engine.DATAFUSION, Format.ARROW)} + for entry in pr.values(): + targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} + assert targets + assert targets <= pr_targets + for entry in nightly.values(): targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} assert targets - assert targets <= {(target.engine, target.format) for target in default_targets} + assert targets <= default_targets tpch = develop["tpch-nvme"] assert [(target["engine"], target["format"]) for target in _targets(tpch)] == [ @@ -104,6 +118,28 @@ def test_ci_profiles_have_distinct_and_consistent_roles() -> None: ("duckdb", "duckdb"), ] + assert [(target["engine"], target["format"]) for target in _targets(pr_full_matrix["clickbench-nvme"])] == [ + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + ("duckdb", "duckdb"), + ] + assert pr_full_matrix["clickbench-nvme"]["data_formats"] == [ + "parquet", + "vortex", + "vortex-compact", + "duckdb", + ] + assert [(target["engine"], target["format"]) for target in _targets(pr["tpch-nvme"])] == [ + ("datafusion", "arrow"), + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + ] + assert all(target["format"] != "lance" for entry in pr_full_matrix.values() for target in _targets(entry)) + def test_existing_display_and_scale_values_are_preserved() -> None: develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} @@ -125,3 +161,15 @@ def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) assert result.exit_code == 1 + + +def test_pr_full_policy_honors_benchmark_override() -> None: + benchmark = BenchmarkDef( + id="override", + benchmark=Benchmark.APPIAN, + name="Override", + targets=df(Format.PARQUET, Format.VORTEX, Format.LANCE), + pr_targets=df(Format.VORTEX), + ) + + assert pr_full(benchmark) == df(Format.VORTEX) From 57221e0e50841d0df63c3616649a9b3c82e74d57 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 20 Jul 2026 11:55:07 -0400 Subject: [PATCH 3/4] Harden declarative benchmark profiles Signed-off-by: Connor Tsui --- .../bench_orchestrator/benchmarks.py | 25 ++++++---- .../bench_orchestrator/matrix.py | 7 ++- bench-orchestrator/tests/test_matrix.py | 49 +++++++++++++++++-- 3 files changed, 67 insertions(+), 14 deletions(-) diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py index 015517cbc40..abe06df81dd 100644 --- a/bench-orchestrator/bench_orchestrator/benchmarks.py +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -25,7 +25,14 @@ ) -def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: +def _tpch( + scale_factor: float | int, + storage: Storage, + *, + group: BenchmarkGroup, + pr_base: bool, + iterations: int | None = 10, +) -> BenchmarkDef: suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}" if storage is Storage.NVME: target_set = df( @@ -51,8 +58,8 @@ def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None storage=storage, scale_factor=scale_factor, iterations=iterations, - group=BenchmarkGroup.NIGHTLY if scale_factor == 100 else BenchmarkGroup.REGULAR, - pr_base=storage is Storage.NVME or scale_factor != 10, + group=group, + pr_base=pr_base, local_dir=local_dir, remote_key=remote_key, ) @@ -93,12 +100,12 @@ def _fineweb(storage: Storage) -> BenchmarkDef: BENCHMARKS: list[BenchmarkDef] = [ _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"), _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"), - _tpch(1.0, Storage.NVME), - _tpch(1.0, Storage.S3), - _tpch(10.0, Storage.NVME), - _tpch(10.0, Storage.S3), - _tpch(100, Storage.NVME, iterations=None), - _tpch(100.0, Storage.S3, iterations=None), + _tpch(1.0, Storage.NVME, group=BenchmarkGroup.REGULAR, pr_base=True), + _tpch(1.0, Storage.S3, group=BenchmarkGroup.REGULAR, pr_base=True), + _tpch(10.0, Storage.NVME, group=BenchmarkGroup.REGULAR, pr_base=True), + _tpch(10.0, Storage.S3, group=BenchmarkGroup.REGULAR, pr_base=False), + _tpch(100, Storage.NVME, group=BenchmarkGroup.NIGHTLY, pr_base=True, iterations=None), + _tpch(100.0, Storage.S3, group=BenchmarkGroup.NIGHTLY, pr_base=True, iterations=None), BenchmarkDef( id="tpcds-nvme", benchmark=Benchmark.TPCDS, diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py index f28e74c8957..04584f2092b 100644 --- a/bench-orchestrator/bench_orchestrator/matrix.py +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -206,12 +206,17 @@ def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet, data_format_t def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: """Resolve a profile into GitHub Actions matrix entries.""" entries: list[dict[str, object]] = [] + seen_ids: set[str] = set() for benchmark in benchmarks: if benchmark.group is not profile.group or not profile.benchmarks(benchmark): continue + if benchmark.id in seen_ids: + raise ValueError(f"Duplicate benchmark ID {benchmark.id!r} in resolved profile") + seen_ids.add(benchmark.id) + run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) if len(run_targets) == 0: - continue + raise ValueError(f"Benchmark {benchmark.id!r} resolved to no runnable targets") data_format_targets = run_targets if profile.data_formats is not None: data_format_targets = _valid_for_storage(profile.data_formats(benchmark), benchmark.storage) diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 06e9bd5eb63..76757631826 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -6,6 +6,7 @@ import json from typing import cast +import pytest from bench_orchestrator import cli as cli_module from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES from bench_orchestrator.config import Benchmark, Engine, Format @@ -15,6 +16,7 @@ BenchmarkGroup, Profile, Storage, + TargetSet, all_targets, defaults, df, @@ -75,15 +77,54 @@ def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: } +def test_resolver_rejects_benchmarks_without_runnable_targets() -> None: + benchmark = BenchmarkDef( + id="empty", + benchmark=Benchmark.TPCH, + name="Empty", + targets=df(Format.PARQUET), + ) + + with pytest.raises(ValueError, match="Benchmark 'empty' resolved to no runnable targets"): + _ = resolve_matrix(Profile(targets=lambda _benchmark: TargetSet()), [benchmark]) + + +def test_resolver_rejects_duplicate_ids_within_a_profile() -> None: + benchmarks = [ + BenchmarkDef( + id="duplicate", + benchmark=Benchmark.TPCH, + name="First", + targets=df(Format.PARQUET), + ), + BenchmarkDef( + id="duplicate", + benchmark=Benchmark.TPCDS, + name="Second", + targets=df(Format.VORTEX), + ), + ] + + with pytest.raises(ValueError, match="Duplicate benchmark ID 'duplicate' in resolved profile"): + _ = resolve_matrix(Profile(), benchmarks) + + def test_ci_profiles_have_distinct_and_consistent_roles() -> None: assert set(PROFILES) == {"develop", "pr", "pr-full", "nightly", "vortex"} regular_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.REGULAR} nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.NIGHTLY} - develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} - pr = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr"], BENCHMARKS)} - pr_full_matrix = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr-full"], BENCHMARKS)} - nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + develop_entries = resolve_matrix(PROFILES["develop"], BENCHMARKS) + pr_entries = resolve_matrix(PROFILES["pr"], BENCHMARKS) + pr_full_entries = resolve_matrix(PROFILES["pr-full"], BENCHMARKS) + nightly_entries = resolve_matrix(PROFILES["nightly"], BENCHMARKS) vortex = resolve_matrix(PROFILES["vortex"], BENCHMARKS) + for entries in (develop_entries, pr_entries, pr_full_entries, nightly_entries, vortex): + assert len(entries) == len({entry["id"] for entry in entries}) + + develop = {entry["id"]: entry for entry in develop_entries} + pr = {entry["id"]: entry for entry in pr_entries} + pr_full_matrix = {entry["id"]: entry for entry in pr_full_entries} + nightly = {entry["id"]: entry for entry in nightly_entries} assert set(develop) == regular_ids assert set(pr_full_matrix) == regular_ids From a07192243f79982f26d439e44de1bc2622087f84 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 20 Jul 2026 12:13:42 -0400 Subject: [PATCH 4/4] Simplify CI benchmark matrix presets Signed-off-by: Connor Tsui --- .github/workflows/bench-dispatch.yml | 2 +- .github/workflows/bench-pr.yml | 2 +- .github/workflows/bench.yml | 4 +- .github/workflows/nightly-bench.yml | 2 +- .github/workflows/sql-benchmarks.yml | 30 +-- .github/workflows/sql-pr.yml | 18 +- .github/workflows/sql-vortex-pr.yml | 2 +- bench-orchestrator/README.md | 29 +- .../bench_orchestrator/benchmarks.py | 52 +--- bench-orchestrator/bench_orchestrator/cli.py | 27 +- .../bench_orchestrator/matrix.py | 80 +++--- bench-orchestrator/tests/test_matrix.py | 249 +++++------------- 12 files changed, 167 insertions(+), 330 deletions(-) diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/bench-dispatch.yml index d1116d964b5..6313e37c64a 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/bench-dispatch.yml @@ -64,4 +64,4 @@ jobs: uses: ./.github/workflows/sql-pr.yml secrets: inherit with: - benchmark_profile: "full" + matrix_preset: "pr-full" diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 9d2ada7dfdf..9c8408cf901 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -139,4 +139,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: "pr-full" + matrix_preset: "pr-full" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 11409c9349e..0045c1ad214 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -171,11 +171,11 @@ jobs: secrets: inherit with: mode: "develop" - benchmark_profile: "develop" + matrix_preset: "develop" sql-vortex: uses: ./.github/workflows/sql-benchmarks.yml secrets: inherit with: mode: "develop" - benchmark_profile: "vortex" + matrix_preset: "vortex" diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index ff3f52337a9..4564ef681ca 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -23,7 +23,7 @@ jobs: with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} - benchmark_profile: "nightly" + matrix_preset: "nightly" strategy: # A single run not should kill the others fail-fast: false diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 9391a475841..552bb7d0e01 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -6,10 +6,10 @@ on: mode: required: true type: string - benchmark_profile: + matrix_preset: required: true type: string - description: "Declarative benchmark profile to resolve" + description: "Named benchmark matrix to run" machine_type: required: false type: string @@ -19,15 +19,15 @@ jobs: resolve-matrix: runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read outputs: benchmark_matrix: ${{ steps.resolve.outputs.benchmark_matrix }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: inputs.mode == 'pr' with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: inputs.mode != 'pr' + ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: @@ -36,16 +36,12 @@ jobs: id: resolve shell: bash env: - BENCHMARK_PROFILE: ${{ inputs.benchmark_profile }} + MATRIX_PRESET: ${{ inputs.matrix_preset }} run: | set -Eeuo pipefail - matrix="$(uv run --project bench-orchestrator vx-bench matrix "$BENCHMARK_PROFILE")" - { - echo 'benchmark_matrix<<__BENCHMARK_MATRIX__' - echo "$matrix" - echo '__BENCHMARK_MATRIX__' - } >> "$GITHUB_OUTPUT" + matrix="$(uv run --project bench-orchestrator vx-bench matrix "$MATRIX_PRESET")" + echo "benchmark_matrix=$matrix" >> "$GITHUB_OUTPUT" bench: needs: resolve-matrix @@ -70,12 +66,8 @@ jobs: with: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: inputs.mode == 'pr' with: - ref: ${{ github.event.pull_request.head.sha }} - - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: inputs.mode != 'pr' + ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - uses: ./.github/actions/setup-rust @@ -225,7 +217,7 @@ jobs: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 - Benchmark `${{ matrix.name }}` (${{ inputs.benchmark_profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + Benchmark `${{ matrix.name }}` (${{ inputs.matrix_preset }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. comment-tag: bench-pr-comment-${{ matrix.id }} - name: Upload Benchmark Results diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/sql-pr.yml index 8ffe7f163ae..b14016030c0 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/sql-pr.yml @@ -5,7 +5,7 @@ name: PR SQL Benchmarks concurrency: group: >- - ${{ github.workflow }}-${{ inputs.benchmark_profile || 'base' }}-${{ + ${{ github.workflow }}-${{ inputs.matrix_preset || 'pr' }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -13,20 +13,20 @@ concurrency: on: workflow_call: inputs: - benchmark_profile: + matrix_preset: required: false type: string - default: "base" + default: "pr" workflow_dispatch: inputs: - benchmark_profile: - description: "SQL benchmark profile to run" + matrix_preset: + description: "SQL benchmark matrix to run" required: false type: choice - default: "base" + default: "pr" options: - - "base" - - "full" + - "pr" + - "pr-full" permissions: contents: read @@ -39,4 +39,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: ${{ inputs.benchmark_profile == 'full' && 'pr-full' || 'pr' }} + matrix_preset: ${{ inputs.matrix_preset }} diff --git a/.github/workflows/sql-vortex-pr.yml b/.github/workflows/sql-vortex-pr.yml index d0ab4d06c8e..f368bd01935 100644 --- a/.github/workflows/sql-vortex-pr.yml +++ b/.github/workflows/sql-vortex-pr.yml @@ -23,4 +23,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: "vortex" + matrix_preset: "vortex" diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 16f2ce3bf39..0dd938f861e 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -71,13 +71,13 @@ vx-bench prepare-data [options] - `--opt`: Benchmark-specific options such as `scale-factor=10.0` -### `matrix` - Resolve CI Benchmark Matrices +### `matrix` - Render a CI Benchmark Matrix -Emit the GitHub Actions `include:` array for a named profile. The benchmark workflows can use this +Emit the GitHub Actions `include:` array for a named preset. The benchmark workflows use this to keep benchmark coverage in Python instead of copying large JSON matrices between YAML files. ```bash -vx-bench matrix # list available profiles +vx-bench matrix # list available presets vx-bench matrix develop # emit compact JSON vx-bench matrix pr-full # emit full pull-request coverage vx-bench matrix nightly --pretty @@ -85,7 +85,7 @@ vx-bench matrix nightly --pretty **Options:** -- `--list`: List available profiles and exit +- `--list`: List available presets and exit - `--pretty`: Pretty-print the JSON output ### `compare` - Compare Results @@ -155,23 +155,14 @@ vx-bench clean --older-than "30 days" [options] - `--keep-labeled`: Don't delete labeled runs (default: true) - `--dry-run, -n`: Show what would be deleted -## Declarative Benchmark Matrix +## CI Benchmark Matrices -CI benchmark coverage is declared in `bench_orchestrator/benchmarks.py` and rendered by -`bench_orchestrator/matrix.py`. This keeps the source of truth out of workflow YAML while still -emitting the JSON shape GitHub Actions expects. +CI benchmark entries live in `bench_orchestrator/benchmarks.py`. The `vx-bench matrix` command +renders one of the `develop`, `pr`, `pr-full`, `nightly`, or `vortex` presets as the JSON expected +by GitHub Actions. -The model separates three review concerns: - -- **Benchmark definitions** declare the suite, storage location, scale factor, and supported - engine/format targets. -- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, - `pr-full`, `nightly`, and `vortex`). -- **Matrix rendering** converts a profile into stable GitHub Actions `include` entries with fields - such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage keys. - -When adding coverage, update the declarations first and add focused tests for the resolved profile -entries rather than duplicating large inline JSON matrices in workflow files. +When adding coverage, update the benchmark declarations and the expected preset membership in +`tests/test_matrix.py`. ## Example Workflows diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py index abe06df81dd..d75a0a4c307 100644 --- a/bench-orchestrator/bench_orchestrator/benchmarks.py +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""SQL benchmark declarations and CI profiles. +"""SQL benchmark declarations used by CI matrices. Edit this file when changing benchmark coverage. Matrix rendering lives in ``bench_orchestrator.matrix`` so workflow shape and benchmark coverage do not drift together. @@ -13,15 +13,9 @@ STANDARD, BenchmarkDef, BenchmarkGroup, - Profile, Storage, - all_targets, - defaults, df, duck, - pr_base_benchmarks, - pr_defaults, - pr_full, ) @@ -30,7 +24,7 @@ def _tpch( storage: Storage, *, group: BenchmarkGroup, - pr_base: bool, + run_in_pr: bool = True, iterations: int | None = 10, ) -> BenchmarkDef: suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}" @@ -59,7 +53,7 @@ def _tpch( scale_factor=scale_factor, iterations=iterations, group=group, - pr_base=pr_base, + run_in_pr=run_in_pr, local_dir=local_dir, remote_key=remote_key, ) @@ -100,12 +94,12 @@ def _fineweb(storage: Storage) -> BenchmarkDef: BENCHMARKS: list[BenchmarkDef] = [ _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"), _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"), - _tpch(1.0, Storage.NVME, group=BenchmarkGroup.REGULAR, pr_base=True), - _tpch(1.0, Storage.S3, group=BenchmarkGroup.REGULAR, pr_base=True), - _tpch(10.0, Storage.NVME, group=BenchmarkGroup.REGULAR, pr_base=True), - _tpch(10.0, Storage.S3, group=BenchmarkGroup.REGULAR, pr_base=False), - _tpch(100, Storage.NVME, group=BenchmarkGroup.NIGHTLY, pr_base=True, iterations=None), - _tpch(100.0, Storage.S3, group=BenchmarkGroup.NIGHTLY, pr_base=True, iterations=None), + _tpch(1.0, Storage.NVME, group=BenchmarkGroup.REGULAR), + _tpch(1.0, Storage.S3, group=BenchmarkGroup.REGULAR), + _tpch(10.0, Storage.NVME, group=BenchmarkGroup.REGULAR), + _tpch(10.0, Storage.S3, group=BenchmarkGroup.REGULAR, run_in_pr=False), + _tpch(100, Storage.NVME, group=BenchmarkGroup.NIGHTLY, iterations=None), + _tpch(100.0, Storage.S3, group=BenchmarkGroup.NIGHTLY, iterations=None), BenchmarkDef( id="tpcds-nvme", benchmark=Benchmark.TPCDS, @@ -136,7 +130,7 @@ def _fineweb(storage: Storage) -> BenchmarkDef: name="Appian on NVME", targets=STANDARD | duck(Format.DUCKDB), pr_targets=DEFAULTS | duck(Format.DUCKDB), - pr_base=False, + run_in_pr=False, iterations=10, ), BenchmarkDef( @@ -148,29 +142,3 @@ def _fineweb(storage: Storage) -> BenchmarkDef: iterations=100, ), ] - -PROFILES: dict[str, Profile] = { - "develop": Profile( - targets=all_targets, - description="Every regular SQL benchmark at full target coverage.", - ), - "pr": Profile( - benchmarks=pr_base_benchmarks, - targets=pr_defaults, - description="The cheaper pull-request SQL benchmark lane.", - ), - "pr-full": Profile( - targets=pr_full, - data_formats=all_targets, - description="Every regular SQL benchmark at full PR target coverage.", - ), - "nightly": Profile( - group=BenchmarkGroup.NIGHTLY, - targets=defaults, - description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", - ), - "vortex": Profile( - group=BenchmarkGroup.VORTEX, - description="The Vortex query suite run on pushes and pull requests.", - ), -} diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index cbe5ac3ac0e..783c48c5adb 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -16,7 +16,7 @@ from rich.console import Console from rich.table import Table -from .benchmarks import BENCHMARKS, PROFILES +from .benchmarks import BENCHMARKS from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -31,7 +31,7 @@ parse_targets_json, resolve_axis_targets, ) -from .matrix import resolve_matrix +from .matrix import MATRIX_PRESETS, resolve_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -219,26 +219,25 @@ def prepare_data( @app.command("matrix") def matrix( - profile: Annotated[ + preset: Annotated[ str | None, - typer.Argument(help="Profile to resolve; omit to list available profiles"), + typer.Argument(help="Matrix preset to render; omit to list available presets"), ] = None, - list_profiles: Annotated[bool, typer.Option("--list", help="List available profiles and exit")] = False, + list_presets: Annotated[bool, typer.Option("--list", help="List available presets and exit")] = False, pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False, ) -> None: - """Emit the GitHub Actions benchmark matrix for a profile.""" - if profile is None or list_profiles: - for name, prof in PROFILES.items(): - console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + """Emit a GitHub Actions benchmark matrix.""" + if preset is None or list_presets: + for name, description in MATRIX_PRESETS.items(): + console.print(f"[bold cyan]{name}[/bold cyan]: {description}") return - prof = PROFILES.get(profile) - if prof is None: - known = ", ".join(PROFILES) - console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") + if preset not in MATRIX_PRESETS: + known = ", ".join(MATRIX_PRESETS) + console.print(f"[red]Unknown matrix preset '{preset}'. Available: {known}[/red]") raise typer.Exit(1) - entries = resolve_matrix(prof, BENCHMARKS) + entries = resolve_matrix(preset, BENCHMARKS) typer.echo(json.dumps(entries, indent=2 if pretty else None)) diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py index 04584f2092b..5d59a7017bd 100644 --- a/bench-orchestrator/bench_orchestrator/matrix.py +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""Resolve declarative benchmark definitions into GitHub Actions matrices.""" +"""Render benchmark definitions as GitHub Actions matrices.""" from __future__ import annotations -from collections.abc import Callable, Iterable +from collections.abc import Iterable from dataclasses import dataclass from enum import Enum @@ -110,7 +110,7 @@ class BenchmarkDef: iterations: int | None = None group: BenchmarkGroup = BenchmarkGroup.REGULAR pr_targets: TargetSet | None = None - pr_base: bool = True + run_in_pr: bool = True local_dir: str | None = None remote_key: str | None = None @@ -120,54 +120,51 @@ def subcommand(self) -> str: return self.benchmark.value -TargetPolicy = Callable[[BenchmarkDef], TargetSet] -BenchmarkPolicy = Callable[[BenchmarkDef], bool] - - -def all_targets(benchmark: BenchmarkDef) -> TargetSet: - """Run every target declared by a benchmark.""" - return benchmark.targets - - -def defaults(benchmark: BenchmarkDef) -> TargetSet: - """Run the cheap default lane intersected with a benchmark's declared targets.""" +def _default_targets(benchmark: BenchmarkDef) -> TargetSet: + """Return the cheap default targets supported by a benchmark.""" return TargetSet(tuple(target for target in benchmark.targets if target in DEFAULTS.targets)) -def pr_full(benchmark: BenchmarkDef) -> TargetSet: +def _pr_full_targets(benchmark: BenchmarkDef) -> TargetSet: """Return the full target set supported by pull-request runners.""" if benchmark.pr_targets is not None: return benchmark.pr_targets return TargetSet(tuple(target for target in benchmark.targets if target.format is not Format.LANCE)) -def pr_defaults(benchmark: BenchmarkDef) -> TargetSet: +def _pr_targets(benchmark: BenchmarkDef) -> TargetSet: """Return the cheap PR lane while preserving Arrow coverage for regular TPC-H.""" allowed = set(DEFAULTS) if benchmark.benchmark is Benchmark.TPCH: allowed.update(df(Format.ARROW)) - return TargetSet(tuple(target for target in pr_full(benchmark) if target in allowed)) + return TargetSet(tuple(target for target in _pr_full_targets(benchmark) if target in allowed)) -def all_benchmarks(_: BenchmarkDef) -> bool: - """Include every benchmark in a profile's group.""" - return True +MATRIX_PRESETS = { + "develop": "Every regular SQL benchmark at full target coverage.", + "pr": "The quicker pull-request SQL benchmark matrix.", + "pr-full": "Every regular SQL benchmark at full PR target coverage.", + "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", + "vortex": "The Vortex query suite run on pushes and pull requests.", +} -def pr_base_benchmarks(benchmark: BenchmarkDef) -> bool: - """Include benchmarks assigned to the cheaper PR lane.""" - return benchmark.pr_base +def _include_benchmark(preset: str, benchmark: BenchmarkDef) -> bool: + if preset == "nightly": + return benchmark.group is BenchmarkGroup.NIGHTLY + if preset == "vortex": + return benchmark.group is BenchmarkGroup.VORTEX + return benchmark.group is BenchmarkGroup.REGULAR and (preset != "pr" or benchmark.run_in_pr) -@dataclass(frozen=True) -class Profile: - """A named CI benchmark configuration.""" - - group: BenchmarkGroup = BenchmarkGroup.REGULAR - benchmarks: BenchmarkPolicy = all_benchmarks - targets: TargetPolicy = all_targets - data_formats: TargetPolicy | None = None - description: str = "" +def _targets_for(preset: str, benchmark: BenchmarkDef) -> TargetSet: + if preset in {"develop", "vortex"}: + return benchmark.targets + if preset == "pr-full": + return _pr_full_targets(benchmark) + if preset == "pr": + return _pr_targets(benchmark) + return _default_targets(benchmark) def _valid_for_storage(target_set: TargetSet, storage: Storage) -> TargetSet: @@ -203,22 +200,25 @@ def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet, data_format_t return entry -def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: - """Resolve a profile into GitHub Actions matrix entries.""" +def resolve_matrix(preset: str, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: + """Render a named preset as GitHub Actions matrix entries.""" + if preset not in MATRIX_PRESETS: + known = ", ".join(MATRIX_PRESETS) + raise ValueError(f"Unknown matrix preset {preset!r}. Available: {known}") + entries: list[dict[str, object]] = [] seen_ids: set[str] = set() for benchmark in benchmarks: - if benchmark.group is not profile.group or not profile.benchmarks(benchmark): + if not _include_benchmark(preset, benchmark): continue if benchmark.id in seen_ids: - raise ValueError(f"Duplicate benchmark ID {benchmark.id!r} in resolved profile") + raise ValueError(f"Duplicate benchmark ID {benchmark.id!r} in matrix preset {preset!r}") seen_ids.add(benchmark.id) - run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) + run_targets = _valid_for_storage(_targets_for(preset, benchmark), benchmark.storage) if len(run_targets) == 0: raise ValueError(f"Benchmark {benchmark.id!r} resolved to no runnable targets") - data_format_targets = run_targets - if profile.data_formats is not None: - data_format_targets = _valid_for_storage(profile.data_formats(benchmark), benchmark.storage) + data_format_targets = benchmark.targets if preset == "pr-full" else run_targets + data_format_targets = _valid_for_storage(data_format_targets, benchmark.storage) entries.append(_matrix_entry(benchmark, run_targets, data_format_targets)) return entries diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 76757631826..4d8490fce01 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -1,216 +1,103 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""Contract tests for the declarative CI benchmark matrix.""" +"""Tests for CI benchmark matrices.""" import json +from dataclasses import replace from typing import cast import pytest from bench_orchestrator import cli as cli_module -from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES -from bench_orchestrator.config import Benchmark, Engine, Format -from bench_orchestrator.matrix import ( - DEFAULTS, - BenchmarkDef, - BenchmarkGroup, - Profile, - Storage, - TargetSet, - all_targets, - defaults, - df, - duck, - pr_full, - resolve_matrix, -) +from bench_orchestrator.benchmarks import BENCHMARKS +from bench_orchestrator.matrix import MATRIX_PRESETS, TargetSet, resolve_matrix from typer.testing import CliRunner runner = CliRunner() +REGULAR_IDS = ( + "clickbench-nvme", + "clickbench-sorted-nvme", + "tpch-nvme", + "tpch-s3", + "tpch-nvme-10", + "tpch-s3-10", + "tpcds-nvme", + "statpopgen", + "fineweb", + "fineweb-s3", + "polarsignals", + "appian-nvme", +) +EXPECTED_IDS = { + "develop": REGULAR_IDS, + "pr": tuple(benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id not in {"tpch-s3-10", "appian-nvme"}), + "pr-full": REGULAR_IDS, + "nightly": ("tpch-nvme", "tpch-s3"), + "vortex": ("vortex-queries",), +} -def _targets(entry: dict[str, object]) -> list[dict[str, str]]: - return cast("list[dict[str, str]]", entry["targets"]) - - -def test_default_policy_only_narrows_declared_targets() -> None: - benchmark = BenchmarkDef( - id="duckdb-only", - benchmark=Benchmark.TPCH, - name="DuckDB only", - targets=duck(Format.VORTEX, Format.DUCKDB), - ) - - assert set(defaults(benchmark)) == set(duck(Format.VORTEX)) - - -def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: - benchmark = BenchmarkDef( - id="remote", - benchmark=Benchmark.TPCH, - name="Remote", - targets=df(Format.ARROW, Format.PARQUET, Format.LANCE, Format.VORTEX) | duck(Format.DUCKDB), - storage=Storage.S3, - scale_factor=1, - iterations=10, - local_dir="data/tpch", - remote_key="tpch/1.0", - ) - - [entry] = resolve_matrix(Profile(targets=all_targets), [benchmark]) - - assert entry == { - "id": "remote", - "subcommand": "tpch", - "name": "Remote", - "targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"}, - ], - "data_formats": ["parquet", "vortex", "duckdb"], - "scale_factor": "1", - "iterations": "10", - "local_dir": "data/tpch", - "remote_key": "tpch/1.0", - } +def _entries(preset: str) -> list[dict[str, object]]: + return resolve_matrix(preset, BENCHMARKS) -def test_resolver_rejects_benchmarks_without_runnable_targets() -> None: - benchmark = BenchmarkDef( - id="empty", - benchmark=Benchmark.TPCH, - name="Empty", - targets=df(Format.PARQUET), - ) - with pytest.raises(ValueError, match="Benchmark 'empty' resolved to no runnable targets"): - _ = resolve_matrix(Profile(targets=lambda _benchmark: TargetSet()), [benchmark]) - - -def test_resolver_rejects_duplicate_ids_within_a_profile() -> None: - benchmarks = [ - BenchmarkDef( - id="duplicate", - benchmark=Benchmark.TPCH, - name="First", - targets=df(Format.PARQUET), - ), - BenchmarkDef( - id="duplicate", - benchmark=Benchmark.TPCDS, - name="Second", - targets=df(Format.VORTEX), - ), - ] - - with pytest.raises(ValueError, match="Duplicate benchmark ID 'duplicate' in resolved profile"): - _ = resolve_matrix(Profile(), benchmarks) - - -def test_ci_profiles_have_distinct_and_consistent_roles() -> None: - assert set(PROFILES) == {"develop", "pr", "pr-full", "nightly", "vortex"} - regular_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.REGULAR} - nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.group is BenchmarkGroup.NIGHTLY} - develop_entries = resolve_matrix(PROFILES["develop"], BENCHMARKS) - pr_entries = resolve_matrix(PROFILES["pr"], BENCHMARKS) - pr_full_entries = resolve_matrix(PROFILES["pr-full"], BENCHMARKS) - nightly_entries = resolve_matrix(PROFILES["nightly"], BENCHMARKS) - vortex = resolve_matrix(PROFILES["vortex"], BENCHMARKS) - for entries in (develop_entries, pr_entries, pr_full_entries, nightly_entries, vortex): - assert len(entries) == len({entry["id"] for entry in entries}) - - develop = {entry["id"]: entry for entry in develop_entries} - pr = {entry["id"]: entry for entry in pr_entries} - pr_full_matrix = {entry["id"]: entry for entry in pr_full_entries} - nightly = {entry["id"]: entry for entry in nightly_entries} - - assert set(develop) == regular_ids - assert set(pr_full_matrix) == regular_ids - assert set(pr) == regular_ids - {"appian-nvme", "tpch-s3-10"} - assert set(nightly) == nightly_ids - assert [entry["id"] for entry in vortex] == ["vortex-queries"] - assert develop["tpch-s3"]["remote_key"] == "tpch/1.0" - assert nightly["tpch-s3"]["remote_key"] == "tpch/100.0" - assert "${{" not in json.dumps([*develop.values(), *pr.values(), *pr_full_matrix.values(), *nightly.values()]) - - default_targets = {(target.engine, target.format) for target in DEFAULTS} - pr_targets = default_targets | {(Engine.DATAFUSION, Format.ARROW)} - for entry in pr.values(): - targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} - assert targets - assert targets <= pr_targets - for entry in nightly.values(): - targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} - assert targets - assert targets <= default_targets - - tpch = develop["tpch-nvme"] - assert [(target["engine"], target["format"]) for target in _targets(tpch)] == [ - ("datafusion", "arrow"), - ("datafusion", "parquet"), - ("datafusion", "vortex"), - ("datafusion", "vortex-compact"), - ("datafusion", "lance"), - ("duckdb", "parquet"), - ("duckdb", "vortex"), - ("duckdb", "vortex-compact"), - ("duckdb", "duckdb"), - ] +def _targets(entry: dict[str, object]) -> set[tuple[str, str]]: + targets = cast("list[dict[str, str]]", entry["targets"]) + return {(target["engine"], target["format"]) for target in targets} - assert [(target["engine"], target["format"]) for target in _targets(pr_full_matrix["clickbench-nvme"])] == [ - ("datafusion", "parquet"), - ("datafusion", "vortex"), - ("duckdb", "parquet"), - ("duckdb", "vortex"), - ("duckdb", "duckdb"), - ] - assert pr_full_matrix["clickbench-nvme"]["data_formats"] == [ - "parquet", - "vortex", - "vortex-compact", - "duckdb", - ] - assert [(target["engine"], target["format"]) for target in _targets(pr["tpch-nvme"])] == [ + +@pytest.mark.parametrize(("preset", "expected_ids"), EXPECTED_IDS.items()) +def test_matrix_presets(preset: str, expected_ids: tuple[str, ...]) -> None: + entries = _entries(preset) + ids = [entry["id"] for entry in entries] + + assert tuple(ids) == expected_ids + assert len(ids) == len(set(ids)) + assert all(entry["targets"] for entry in entries) + assert "${{" not in json.dumps(entries) + + +def test_pr_target_selection() -> None: + develop = {entry["id"]: entry for entry in _entries("develop")} + pr = {entry["id"]: entry for entry in _entries("pr")} + pr_full = {entry["id"]: entry for entry in _entries("pr-full")} + + assert _targets(pr["tpch-nvme"]) == { ("datafusion", "arrow"), ("datafusion", "parquet"), ("datafusion", "vortex"), ("duckdb", "parquet"), ("duckdb", "vortex"), - ] - assert all(target["format"] != "lance" for entry in pr_full_matrix.values() for target in _targets(entry)) + } + assert ("datafusion", "lance") in _targets(develop["tpch-nvme"]) + assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values()) + assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"]) + + +def test_resolver_rejects_empty_targets() -> None: + benchmark = replace(BENCHMARKS[0], id="empty", targets=TargetSet()) + + with pytest.raises(ValueError, match="Benchmark 'empty' resolved to no runnable targets"): + _ = resolve_matrix("develop", [benchmark]) -def test_existing_display_and_scale_values_are_preserved() -> None: - develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} - nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} +def test_resolver_rejects_duplicate_ids() -> None: + benchmark = BENCHMARKS[0] - assert develop["tpch-nvme"]["scale_factor"] == "1.0" - assert develop["statpopgen"]["scale_factor"] == "100" - assert develop["polarsignals"]["scale_factor"] == "1" - assert nightly["tpch-nvme"]["name"] == "TPC-H on NVME" - assert nightly["tpch-nvme"]["scale_factor"] == "100" - assert nightly["tpch-s3"]["name"] == "TPC-H on S3" - assert nightly["tpch-s3"]["scale_factor"] == "100.0" + with pytest.raises(ValueError, match="Duplicate benchmark ID 'clickbench-nvme'"): + _ = resolve_matrix("develop", [benchmark, replace(benchmark, name="Duplicate")]) -def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: +def test_matrix_command() -> None: result = runner.invoke(cli_module.app, ["matrix", "develop"]) assert result.exit_code == 0 - assert json.loads(result.stdout) + assert json.loads(result.stdout) == _entries("develop") result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) assert result.exit_code == 1 + assert "Unknown matrix preset" in result.stdout -def test_pr_full_policy_honors_benchmark_override() -> None: - benchmark = BenchmarkDef( - id="override", - benchmark=Benchmark.APPIAN, - name="Override", - targets=df(Format.PARQUET, Format.VORTEX, Format.LANCE), - pr_targets=df(Format.VORTEX), - ) - - assert pr_full(benchmark) == df(Format.VORTEX) +def test_preset_names_are_stable() -> None: + assert set(MATRIX_PRESETS) == set(EXPECTED_IDS)