diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 0b267008a85..f6ced77e646 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -70,6 +70,23 @@ vx-bench prepare-data [options] - `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'` - `--opt`: Benchmark-specific options such as `scale-factor=10.0` + +### `matrix` - Resolve CI Benchmark Matrices + +Emit the GitHub Actions `include:` array for a named profile. The benchmark workflows can use this +to keep benchmark coverage in Python instead of copying large JSON matrices between YAML files. + +```bash +vx-bench matrix # list available profiles +vx-bench matrix develop # emit compact JSON +vx-bench matrix nightly --pretty +``` + +**Options:** + +- `--list`: List available profiles and exit +- `--pretty`: Pretty-print the JSON output + ### `compare` - Compare Results Compare benchmark results within a run or across multiple runs. Results are displayed in a pivot table format. @@ -137,6 +154,24 @@ vx-bench clean --older-than "30 days" [options] - `--keep-labeled`: Don't delete labeled runs (default: true) - `--dry-run, -n`: Show what would be deleted +## Declarative Benchmark Matrix + +CI benchmark coverage is declared in `bench_orchestrator/benchmarks.py` and rendered by +`bench_orchestrator/matrix.py`. This keeps the source of truth out of workflow YAML while still +emitting the JSON shape GitHub Actions expects. + +The model separates three review concerns: + +- **Benchmark definitions** declare the suite, storage location, scale factor, and supported + engine/format targets. +- **Profiles** choose how much declared coverage each workflow should run (`develop`, `pr`, and + `nightly`). +- **Matrix rendering** converts a profile into stable GitHub Actions `include` entries with fields + such as `targets`, `data_formats`, `scale_factor`, `iterations`, and remote-storage metadata. + +When adding coverage, update the declarations first and add focused tests for the resolved profile +entries rather than duplicating large inline JSON matrices in workflow files. + ## Example Workflows ### 1. Basic Performance Comparison diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py new file mode 100644 index 00000000000..1c6e490183f --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""SQL benchmark declarations and CI profiles. + +Edit this file when changing benchmark coverage. Matrix rendering lives in +``bench_orchestrator.matrix`` so workflow shape and benchmark coverage do not drift together. +""" + +from .config import Benchmark, Engine, Format +from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck + + +def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: + suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}" + if storage is Storage.NVME: + target_set = df( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.LANCE, + ) | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB) + local_dir = None + remote_storage = None + else: + target_set = STANDARD + local_dir = f"vortex-bench/data/tpch/{scale_factor:.1f}" + remote_storage = f"s3://vortex-ci-benchmark-datasets/${{{{github.ref_name}}}}/${{{{github.run_id}}}}/tpch/{scale_factor:.1f}/" + + name = f"TPC-H on {storage.label}" if scale_factor == 100 else f"TPC-H SF={scale_factor:g} on {storage.label}" + return BenchmarkDef( + id=f"tpch-{storage.value}{suffix}", + benchmark=Benchmark.TPCH, + name=name, + targets=target_set, + storage=storage, + scale_factor=scale_factor, + iterations=iterations, + nightly=scale_factor == 100, + local_dir=local_dir, + remote_storage=remote_storage, + ) + + +def _clickbench(benchmark: Benchmark, name: str) -> BenchmarkDef: + return BenchmarkDef( + id=f"{benchmark.value}-nvme", + benchmark=benchmark, + name=name, + targets=df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.LANCE) + | duck(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB), + ) + + +def _fineweb(storage: Storage) -> BenchmarkDef: + if storage is Storage.NVME: + return BenchmarkDef( + id="fineweb", + benchmark=Benchmark.FINEWEB, + name="FineWeb NVMe", + targets=STANDARD, + scale_factor=100, + ) + return BenchmarkDef( + id="fineweb-s3", + benchmark=Benchmark.FINEWEB, + name="FineWeb S3", + targets=STANDARD, + storage=Storage.S3, + scale_factor=100, + local_dir="vortex-bench/data/fineweb", + remote_storage="s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + ) + + +BENCHMARKS: list[BenchmarkDef] = [ + _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"), + _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"), + _tpch(1.0, Storage.NVME), + _tpch(1.0, Storage.S3), + _tpch(10.0, Storage.NVME), + _tpch(10.0, Storage.S3), + _tpch(100, Storage.NVME, iterations=None), + _tpch(100.0, Storage.S3, iterations=None), + BenchmarkDef( + id="tpcds-nvme", + benchmark=Benchmark.TPCDS, + name="TPC-DS SF=1 on NVME", + targets=STANDARD | duck(Format.DUCKDB), + scale_factor=1.0, + ), + BenchmarkDef( + id="statpopgen", + benchmark=Benchmark.STATPOPGEN, + name="Statistical and Population Genetics", + targets=STANDARD.only(Engine.DUCKDB), + scale_factor=100, + local_dir="vortex-bench/data/statpopgen", + ), + _fineweb(Storage.NVME), + _fineweb(Storage.S3), + BenchmarkDef( + id="polarsignals", + benchmark=Benchmark.POLARSIGNALS, + name="PolarSignals Profiling", + targets=df(Format.VORTEX), + scale_factor=1, + ), + BenchmarkDef( + id="appian-nvme", + benchmark=Benchmark.APPIAN, + name="Appian on NVME", + targets=STANDARD | duck(Format.DUCKDB), + iterations=10, + ), +] + +PROFILES: dict[str, Profile] = { + "develop": Profile( + targets=all_targets, + description="Every regular SQL benchmark at full target coverage.", + ), + "pr": Profile( + targets=defaults, + description="Every regular SQL benchmark at default targets.", + ), + "nightly": Profile( + nightly=True, + targets=defaults, + description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", + ), +} diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index b9d7f9bb2ef..cbe5ac3ac0e 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -3,6 +3,7 @@ """CLI for benchmark orchestration.""" +import json import subprocess from contextlib import contextmanager from datetime import datetime, timedelta @@ -15,6 +16,7 @@ from rich.console import Console from rich.table import Table +from .benchmarks import BENCHMARKS, PROFILES from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -29,6 +31,7 @@ parse_targets_json, resolve_axis_targets, ) +from .matrix import resolve_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -214,6 +217,31 @@ def prepare_data( raise typer.Exit(1) from exc +@app.command("matrix") +def matrix( + profile: Annotated[ + str | None, + typer.Argument(help="Profile to resolve; omit to list available profiles"), + ] = None, + list_profiles: Annotated[bool, typer.Option("--list", help="List available profiles and exit")] = False, + pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False, +) -> None: + """Emit the GitHub Actions benchmark matrix for a profile.""" + if profile is None or list_profiles: + for name, prof in PROFILES.items(): + console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + return + + prof = PROFILES.get(profile) + if prof is None: + known = ", ".join(PROFILES) + console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") + raise typer.Exit(1) + + entries = resolve_matrix(prof, BENCHMARKS) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) + + @app.command() def run( benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")], diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py new file mode 100644 index 00000000000..b97df9714ac --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Resolve declarative benchmark definitions into GitHub Actions matrices.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from enum import Enum + +from .config import Benchmark, BenchmarkTarget, Engine, Format + + +class Storage(Enum): + """Where a benchmark's data lives when it runs.""" + + NVME = "nvme" + S3 = "s3" + + @property + def label(self) -> str: + """Human-facing name used in benchmark display names.""" + return "NVME" if self is Storage.NVME else "S3" + + +def _dedupe(targets: Iterable[BenchmarkTarget]) -> tuple[BenchmarkTarget, ...]: + """Normalize and de-duplicate targets, preserving first-seen order.""" + return tuple(dict.fromkeys(target.normalized() for target in targets)) + + +@dataclass(frozen=True) +class TargetSet: + """An ordered set of engine/format targets with small set algebra.""" + + targets: tuple[BenchmarkTarget, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "targets", _dedupe(self.targets)) + + def __or__(self, other: TargetSet) -> TargetSet: + """Return the ordered union of two target sets.""" + return TargetSet(self.targets + other.targets) + + def only(self, *engines: Engine) -> TargetSet: + """Restrict the target set to the given engines.""" + keep = set(engines) + return TargetSet(tuple(target for target in self.targets if target.engine in keep)) + + def formats(self) -> list[Format]: + """Return referenced formats in first-seen order.""" + return list(dict.fromkeys(target.format for target in self.targets)) + + def __iter__(self): + return iter(self.targets) + + def __len__(self) -> int: + return len(self.targets) + + +def targets(engine: Engine, *formats: Format) -> TargetSet: + """Build targets for one engine across several formats.""" + return TargetSet(tuple(BenchmarkTarget(engine=engine, format=fmt) for fmt in formats)) + + +def df(*formats: Format) -> TargetSet: + """Build DataFusion targets across several formats.""" + return targets(Engine.DATAFUSION, *formats) + + +def duck(*formats: Format) -> TargetSet: + """Build DuckDB targets across several formats.""" + return targets(Engine.DUCKDB, *formats) + + +STANDARD = df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT) | duck( + Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT +) +DEFAULTS = df(Format.PARQUET, Format.VORTEX) | duck(Format.PARQUET, Format.VORTEX) +_NOT_GENERATED = frozenset({Format.ARROW, Format.LANCE}) +_FORMAT_ORDER = ( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.VORTEX_NATIVE, + Format.DUCKDB, + Format.LANCE, +) + + +@dataclass(frozen=True) +class BenchmarkDef: + """A benchmark and the canonical target superset it can run.""" + + id: str + benchmark: Benchmark + name: str + targets: TargetSet + storage: Storage = Storage.NVME + scale_factor: float | int | None = None + iterations: int | None = None + nightly: bool = False + local_dir: str | None = None + remote_storage: str | None = None + + @property + def subcommand(self) -> str: + """Return the ``vx-bench`` subcommand for this benchmark.""" + return self.benchmark.value + + +TargetPolicy = Callable[[BenchmarkDef], TargetSet] + + +def all_targets(benchmark: BenchmarkDef) -> TargetSet: + """Run every target declared by a benchmark.""" + return benchmark.targets + + +def defaults(benchmark: BenchmarkDef) -> TargetSet: + """Run the cheap default lane intersected with a benchmark's declared targets.""" + return TargetSet(tuple(target for target in benchmark.targets if target in DEFAULTS.targets)) + + +@dataclass(frozen=True) +class Profile: + """A named CI benchmark configuration.""" + + nightly: bool = False + targets: TargetPolicy = all_targets + description: str = "" + + +def _valid_for_storage(target_set: TargetSet, storage: Storage) -> TargetSet: + """Drop targets that are invalid for the storage backend.""" + if storage is Storage.S3: + return TargetSet(tuple(target for target in target_set if target.format is not Format.LANCE)) + return target_set + + +def _data_formats(target_set: TargetSet) -> list[Format]: + """Return data formats that the data-generation step must produce.""" + present = set(target_set.formats()) + return [fmt for fmt in _FORMAT_ORDER if fmt in present and fmt not in _NOT_GENERATED] + + +def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet) -> dict[str, object]: + """Build one GitHub Actions ``include`` entry.""" + entry: dict[str, object] = { + "id": benchmark.id, + "subcommand": benchmark.subcommand, + "name": benchmark.name, + "targets": [target.to_dict() for target in run_targets], + "data_formats": [fmt.value for fmt in _data_formats(run_targets)], + } + if benchmark.scale_factor is not None: + entry["scale_factor"] = str(benchmark.scale_factor) + if benchmark.iterations is not None: + entry["iterations"] = str(benchmark.iterations) + if benchmark.local_dir is not None: + entry["local_dir"] = benchmark.local_dir + if benchmark.remote_storage is not None: + entry["remote_storage"] = benchmark.remote_storage + return entry + + +def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: + """Resolve a profile into GitHub Actions matrix entries.""" + entries: list[dict[str, object]] = [] + for benchmark in benchmarks: + if benchmark.nightly != profile.nightly: + continue + run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) + if len(run_targets) == 0: + continue + entries.append(_matrix_entry(benchmark, run_targets)) + return entries diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py new file mode 100644 index 00000000000..77eb91f9012 --- /dev/null +++ b/bench-orchestrator/tests/test_matrix.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Contract tests for the declarative CI benchmark matrix.""" + +import json +from typing import cast + +from bench_orchestrator import cli as cli_module +from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES +from bench_orchestrator.config import Benchmark, Engine, Format +from bench_orchestrator.matrix import ( + DEFAULTS, + BenchmarkDef, + Profile, + Storage, + all_targets, + defaults, + df, + duck, + resolve_matrix, +) +from typer.testing import CliRunner + +runner = CliRunner() + + +def _targets(entry: dict[str, object]) -> list[dict[str, str]]: + return cast("list[dict[str, str]]", entry["targets"]) + + +def test_default_policy_only_narrows_declared_targets() -> None: + benchmark = BenchmarkDef( + id="duckdb-only", + benchmark=Benchmark.TPCH, + name="DuckDB only", + targets=duck(Format.VORTEX, Format.DUCKDB), + ) + + assert set(defaults(benchmark)) == set(duck(Format.VORTEX)) + + +def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: + benchmark = BenchmarkDef( + id="remote", + benchmark=Benchmark.TPCH, + name="Remote", + targets=df(Format.ARROW, Format.PARQUET, Format.LANCE, Format.VORTEX) | duck(Format.DUCKDB), + storage=Storage.S3, + scale_factor=1, + iterations=10, + local_dir="data/tpch", + remote_storage="s3://bucket/tpch/1.0/", + ) + + [entry] = resolve_matrix(Profile(targets=all_targets), [benchmark]) + + assert entry == { + "id": "remote", + "subcommand": "tpch", + "name": "Remote", + "targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "duckdb"}, + ], + "data_formats": ["parquet", "vortex", "duckdb"], + "scale_factor": "1", + "iterations": "10", + "local_dir": "data/tpch", + "remote_storage": "s3://bucket/tpch/1.0/", + } + + +def test_ci_profiles_have_distinct_and_consistent_roles() -> None: + assert set(PROFILES) == {"develop", "pr", "nightly"} + regular_ids = {benchmark.id for benchmark in BENCHMARKS if not benchmark.nightly} + nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.nightly} + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + pr = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert set(develop) == regular_ids + assert set(pr) == regular_ids + assert set(nightly) == nightly_ids + + default_targets = set(DEFAULTS) + for entry in (*pr.values(), *nightly.values()): + targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} + assert targets + assert targets <= {(target.engine, target.format) for target in default_targets} + + tpch = develop["tpch-nvme"] + assert [(target["engine"], target["format"]) for target in _targets(tpch)] == [ + ("datafusion", "arrow"), + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("datafusion", "vortex-compact"), + ("datafusion", "lance"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + ("duckdb", "vortex-compact"), + ("duckdb", "duckdb"), + ] + + +def test_existing_display_and_scale_values_are_preserved() -> None: + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert develop["tpch-nvme"]["scale_factor"] == "1.0" + assert develop["statpopgen"]["scale_factor"] == "100" + assert develop["polarsignals"]["scale_factor"] == "1" + assert nightly["tpch-nvme"]["name"] == "TPC-H on NVME" + assert nightly["tpch-nvme"]["scale_factor"] == "100" + assert nightly["tpch-s3"]["name"] == "TPC-H on S3" + assert nightly["tpch-s3"]["scale_factor"] == "100.0" + + +def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: + result = runner.invoke(cli_module.app, ["matrix", "develop"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) + + result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) + assert result.exit_code == 1