diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 12137726122..b5b011150a6 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -113,10 +113,8 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md - echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ - >> comment.md + > comment.md cat comment.md >> $GITHUB_STEP_SUMMARY - name: Comment PR diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 243db83aff8..59dd9571b36 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -678,10 +678,8 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo "# Benchmarks: ${{ matrix.name }}" > comment.md - echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ - >> comment.md + > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md new file mode 100644 index 00000000000..df1027e2bd0 --- /dev/null +++ b/benchmarks/compress-bench/README.md @@ -0,0 +1,16 @@ +# Compression benchmark + +Measures compression and decompression throughput, plus resulting file sizes, for Vortex +versus Parquet (and optionally Lance) across a range of datasets: NYC taxi data, several +[Public BI](https://github.com/cwida/public_bi_benchmark) tables (Arade, Bimbo, +CMSprovider, Euro2016, Food, HashTags), TPC-H `l_comment` variants, and synthetic nested +data. This is the workload behind the `Compression` PR comment. + +See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--formats`, +`--datasets`, `--ops compress,decompress`). + +## Running locally + +```bash +cargo run -p compress-bench --profile release_debug +``` diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 8d9fa73f915..1cbf3f47c55 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -113,6 +113,9 @@ fn get_compressor(format: Format) -> Box { /// The benchmark ID used for output path. const BENCHMARK_ID: &str = "compress"; +/// Repo-relative path of the suite explainer linked from CI benchmark PR comments. +const DOC_PATH: &str = "benchmarks/compress-bench/README.md"; + async fn run_compress( iterations: usize, datasets_filter: Option, @@ -212,8 +215,8 @@ async fn run_compress( ) } DisplayFormat::GhJson => { - print_measurements_json(&mut writer, measurements.timings)?; - print_measurements_json(&mut writer, measurements.ratios) + print_measurements_json(&mut writer, measurements.timings, DOC_PATH)?; + print_measurements_json(&mut writer, measurements.ratios, DOC_PATH) } } } diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md new file mode 100644 index 00000000000..14949c85fcb --- /dev/null +++ b/benchmarks/random-access-bench/README.md @@ -0,0 +1,22 @@ +# Random Access benchmark + +Measures point-lookup latency: fetching individual rows by index from a file, rather than +scanning it. This is the workload behind the `Random Access` PR comment. + +Two access patterns are generated with a fixed seed (see [`src/main.rs`](./src/main.rs)): + +- **correlated**: several clusters of consecutive indices scattered across the dataset, + simulating lookups with spatial locality; +- **uniform**: indices drawn from a Poisson process spread uniformly across the dataset, + simulating lookups with no locality. + +Each pattern runs over four datasets (`taxi`, `feature-vectors`, `nested-lists`, +`nested-structs`) in Parquet, Lance, and Vortex, both with a cached open file handle and +reopening the file per lookup. CI drives the full matrix via +[`scripts/random-access-split.py`](../../scripts/random-access-split.py). + +## Running locally + +```bash +cargo run -p random-access-bench --profile release_debug --features lance +``` diff --git a/benchmarks/random-access-bench/src/lib.rs b/benchmarks/random-access-bench/src/lib.rs index 7d4b7d2c389..27be40d0342 100644 --- a/benchmarks/random-access-bench/src/lib.rs +++ b/benchmarks/random-access-bench/src/lib.rs @@ -282,6 +282,9 @@ async fn open_accessor( /// The benchmark ID used for output path. const BENCHMARK_ID: &str = "random-access"; +/// Repo-relative path of the suite explainer linked from CI benchmark PR comments. +const DOC_PATH: &str = "benchmarks/random-access-bench/README.md"; + /// Fixed indices used by the original taxi benchmark (preserved for historical continuity). const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000]; @@ -409,7 +412,7 @@ pub async fn run(config: RunConfig) -> Result<()> { } DisplayFormat::GhJson => { let timings: Vec = runs.into_iter().map(|r| r.timing).collect(); - print_measurements_json(&mut writer, timings)?; + print_measurements_json(&mut writer, timings, DOC_PATH)?; } } diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 7d560c144a3..d7cd34793db 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -12,6 +12,7 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors import math +import os import re import sys from dataclasses import dataclass @@ -763,6 +764,27 @@ def format_within_engine_summary(analyses: dict[str, dict[str, Any]]) -> str | N return " · ".join(summaries) +def format_title(benchmark_name: str, pr: pd.DataFrame) -> str: + """Render the comment title, linking the suite explainer doc emitted by the benchmark binary. + + The doc path is a repo-relative markdown path carried on the PR result rows (the `doc` + field, populated from `Benchmark::doc_path` in Rust), so the benchmark code is the single + source of truth for where each suite is documented. The link pins the PR's own commit so + it resolves before the PR merges and stays valid afterwards. + """ + + title = f"# Benchmarks: {benchmark_name}" if benchmark_name else "# Benchmarks" + if "doc" in pr.columns: + docs = pr["doc"].dropna().unique() + if len(docs) > 0: + server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repository = os.environ.get("GITHUB_REPOSITORY", "vortex-data/vortex") + commits = pr["commit_id"].dropna().unique() if "commit_id" in pr.columns else [] + ref = commits[0] if len(commits) > 0 else "develop" + title += f" [\N{OPEN BOOK}]({server_url}/{repository}/blob/{ref}/{docs[0]})" + return title + + def format_report_help() -> str: """Render explanatory markdown for the benchmark report headline fields.""" @@ -823,6 +845,7 @@ def main() -> None: benchmark_name = sys.argv[3] if len(sys.argv) > 3 else "" pr = pd.read_json(sys.argv[2], lines=True) + title = format_title(benchmark_name, pr) base = read_latest_baseline_rows(sys.argv[1], pr) base_commit_id = set(base["commit_id"].unique()) @@ -896,6 +919,8 @@ def main() -> None: shifts += f" · Median polish {format_ratio_change(float(np.exp(polish.overall)))}" summary_fields.append(f"**Shifts**: {shifts}") + print(title) + print("") print("
".join(summary_fields)) print("") print(format_report_help()) diff --git a/vortex-bench/REUSE.toml b/vortex-bench/REUSE.toml index 5fb9bd33466..a39e10c12b5 100644 --- a/vortex-bench/REUSE.toml +++ b/vortex-bench/REUSE.toml @@ -10,6 +10,12 @@ path = "**/*.csv" SPDX-FileCopyrightText = "Copyright the Vortex contributors" SPDX-License-Identifier = "Apache-2.0" +# Benchmark suite explainer docs linked from CI PR comments. +[[annotations]] +path = "sql/**/*.md" +SPDX-FileCopyrightText = "Copyright the Vortex contributors" +SPDX-License-Identifier = "CC-BY-4.0" + # `insta` snapshot files do not allow leading comment lines. [[annotations]] path = "src/snapshots/**.snap" diff --git a/vortex-bench/sql/appian/README.md b/vortex-bench/sql/appian/README.md new file mode 100644 index 00000000000..9304bcee2a0 --- /dev/null +++ b/vortex-bench/sql/appian/README.md @@ -0,0 +1,22 @@ +# Appian benchmark + +Mirrors the queries from DuckDB's in-tree +[`appian_benchmarks`](https://github.com/duckdb/duckdb/tree/main/benchmark/appian_benchmarks) +suite — a real-world business-application workload (customers, orders, addresses) with +selective filters and joins over camelCase-named columns. + +The eight queries live in this directory (`q1.sql` ... `q8.sql`). Upstream ships the data +as a single `.duckdb` blob (~593 MB); the harness downloads it once and projects each +table into Parquet with lowercased column names so that DataFusion and DuckDB resolve the +verbatim queries identically — see [`src/appian`](../../src/appian) for the details. + +## CI variant + +CI runs this suite from local NVMe as the `Appian on NVME` PR comment, comparing +DataFusion and DuckDB over Parquet and Vortex files (plus a native DuckDB baseline). + +## Running locally + +```bash +vx-bench run appian --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/clickbench.md b/vortex-bench/sql/clickbench.md new file mode 100644 index 00000000000..eb3db810649 --- /dev/null +++ b/vortex-bench/sql/clickbench.md @@ -0,0 +1,30 @@ +# ClickBench benchmark + +[ClickBench](https://github.com/ClickHouse/ClickBench) is ClickHouse's web-analytics +benchmark: 43 queries over a single wide `hits` table (~100M rows of real-ish traffic +data). It is heavy on aggregations, `GROUP BY`s over high-cardinality columns, and +selective string filters, and is the main "wide table scan" workload in CI. + +The queries live in [`clickbench_queries.sql`](./clickbench_queries.sql) (one query per +line, numbered from Q0 in file order). The harness lives in +[`src/clickbench`](../src/clickbench). + +## CI variant + +CI runs this suite from local NVMe as the `Clickbench on NVME` PR comment, comparing +DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files (plus a native +DuckDB baseline). + +## Sorted variant + +`Clickbench Sorted on NVME` runs the same table sorted by event time, split into 100 +shards whose filenames are shuffled so engines cannot rely on file order, exercising sort +pushdown and zone-map-style pruning on a subset of the queries. See +[`ClickBenchSortedBenchmark`](../src/clickbench/benchmark.rs). + +## Running locally + +```bash +vx-bench run clickbench --engine datafusion,duckdb --format parquet,vortex +vx-bench run clickbench-sorted --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/fineweb.md b/vortex-bench/sql/fineweb.md new file mode 100644 index 00000000000..13421cc5456 --- /dev/null +++ b/vortex-bench/sql/fineweb.md @@ -0,0 +1,21 @@ +# FineWeb benchmark + +A string-heavy benchmark over a sample shard of the HuggingFace +[FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) web-crawl dataset +(URLs, dates, and full page text). The dataset exercises dictionary and FSST string +compression heavily, and the hand-crafted queries in [`fineweb.sql`](./fineweb.sql) +(numbered from Q0 in file order) focus on string predicates: equality filters, `LIKE` +prefix and containment patterns, and aggregations over string columns. + +The harness lives in [`src/fineweb`](../src/fineweb). + +## CI variants + +CI runs this suite from local NVMe (`FineWeb NVMe`) and from S3 (`FineWeb S3`), comparing +DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files. + +## Running locally + +```bash +vx-bench run fineweb --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/gharchive.md b/vortex-bench/sql/gharchive.md new file mode 100644 index 00000000000..16cebd2d1a9 --- /dev/null +++ b/vortex-bench/sql/gharchive.md @@ -0,0 +1,14 @@ +# GitHub Archive benchmark + +A deeply nested real-world dataset from the [GitHub Archive](https://www.gharchive.org/): +GitHub event records whose `payload`, `repo`, `actor`, and `org` fields are nested structs. +The queries in [`gharchive.sql`](./gharchive.sql) (numbered from Q0 in file order) filter +and aggregate on those nested fields, making this the main struct-field-pushdown workload. + +The harness lives in [`src/realnest/gharchive.rs`](../src/realnest/gharchive.rs). + +## Running locally + +```bash +cargo run --bin datafusion-bench --profile release_debug -- --benchmark gharchive +``` diff --git a/vortex-bench/sql/polarsignals.md b/vortex-bench/sql/polarsignals.md new file mode 100644 index 00000000000..499466593e7 --- /dev/null +++ b/vortex-bench/sql/polarsignals.md @@ -0,0 +1,23 @@ +# PolarSignals Profiling benchmark + +A benchmark over continuous-profiling stacktrace data from +[Polar Signals](https://www.polarsignals.com/). The schema features a sparse struct (ten +nullable label fields covering five fill-rate tiers), deeply nested locations +(`List>>>`), and several low-cardinality string columns — +making it the main deeply-nested / sparse-data workload in CI. + +The queries in [`polarsignals.sql`](./polarsignals.sql) select stacktrace samples over +increasingly wide time ranges with equality filters on the low-cardinality metadata +columns. The harness lives in [`src/polarsignals`](../src/polarsignals). + +## CI variant + +CI runs this suite from local NVMe as the `PolarSignals Profiling` PR comment. It runs +DataFusion over Vortex only (there is no Parquet control for this suite, so its geomean is +reported without a drift-corrected verdict). + +## Running locally + +```bash +vx-bench run polarsignals --engine datafusion --format vortex +``` diff --git a/vortex-bench/sql/public-bi.md b/vortex-bench/sql/public-bi.md new file mode 100644 index 00000000000..60859677741 --- /dev/null +++ b/vortex-bench/sql/public-bi.md @@ -0,0 +1,15 @@ +# Public BI benchmark + +Real-world BI datasets from the +[Public BI benchmark](https://github.com/cwida/public_bi_benchmark) — anonymized Tableau +workbooks with messy, wide, string-heavy tables. A subset of the datasets (Arade, Bimbo, +CMSprovider, Euro2016, Food, HashTags) also feeds the +[Compression benchmark](../../benchmarks/compress-bench/README.md). + +The datasets, schemas, and queries are defined in [`src/public_bi.rs`](../src/public_bi.rs). + +## Running locally + +```bash +vx-bench run public-bi +``` diff --git a/vortex-bench/sql/spatialbench.md b/vortex-bench/sql/spatialbench.md new file mode 100644 index 00000000000..f959e006887 --- /dev/null +++ b/vortex-bench/sql/spatialbench.md @@ -0,0 +1,15 @@ +# SpatialBench benchmark + +The [Apache Sedona SpatialBench](https://sedona.apache.org/spatialbench/) geospatial +analytics benchmark: twelve queries (Q1 ... Q12 in [`spatialbench.sql`](./spatialbench.sql), +DuckDB dialect) over a trips/zones schema, exercising spatial predicates and functions such +as `ST_DWithin`, `ST_Intersects`, and `ST_Distance`. The query logic matches upstream +`sedona-spatialbench`; only formatting differs. + +The harness lives in [`src/spatialbench`](../src/spatialbench). + +## Running locally + +```bash +vx-bench run spatialbench +``` diff --git a/vortex-bench/sql/statpopgen.md b/vortex-bench/sql/statpopgen.md new file mode 100644 index 00000000000..c0b90a54f54 --- /dev/null +++ b/vortex-bench/sql/statpopgen.md @@ -0,0 +1,25 @@ +# Statistical and Population Genetics benchmark + +A genomics analytics benchmark using the gnomAD v3.1.2 release of the jointly called One +Thousand Genomes + Human Genome Diversity Project (1kG+HGDP) dataset — a prefix of +chromosome 21. The data source is . + +The distinguishing feature is the `GT` genotype column: a large variable-length list of +small nullable integers per row, which stresses list encodings and list-aware compute. +The queries in [`statpopgen.sql`](./statpopgen.sql) cover common genomics patterns: +allele-frequency calculations, Hardy-Weinberg equilibrium statistics, variant filtering by +frequency and by locus interval, and random access to a specific variant. + +The harness lives in [`src/statpopgen`](../src/statpopgen). + +## CI variant + +CI runs this suite from local NVMe as the `Statistical and Population Genetics` PR +comment. Only DuckDB is exercised (over Parquet, Vortex, and vortex-compact files), +because the queries rely on DuckDB list lambdas. + +## Running locally + +```bash +vx-bench run statpopgen --engine duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/tpcds/README.md b/vortex-bench/sql/tpcds/README.md new file mode 100644 index 00000000000..3e4258ce251 --- /dev/null +++ b/vortex-bench/sql/tpcds/README.md @@ -0,0 +1,20 @@ +# TPC-DS benchmark + +The industry-standard [TPC-DS](https://www.tpc.org/tpcds/) decision-support benchmark: 99 +analytical queries (`01.sql` ... `99.sql`) over a retail snowflake schema. Compared to +TPC-H it has many more queries, a wider schema, and heavier use of joins, subqueries, and +window functions, making it a broad regression net for scan and pushdown behavior. + +The benchmark harness lives in [`src/tpcds`](../../src/tpcds). + +## CI variant + +CI runs this suite at scale factor 1 from local NVMe as the `TPC-DS SF=1 on NVME` PR +comment, comparing DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files +(plus a native DuckDB baseline). + +## Running locally + +```bash +vx-bench run tpcds --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0 +``` diff --git a/vortex-bench/sql/tpch/README.md b/vortex-bench/sql/tpch/README.md new file mode 100644 index 00000000000..aa5a16fbab2 --- /dev/null +++ b/vortex-bench/sql/tpch/README.md @@ -0,0 +1,23 @@ +# TPC-H benchmark + +The industry-standard [TPC-H](https://www.tpc.org/tpch/) decision-support benchmark: 22 +analytical queries over an 8-table warehouse schema (`lineitem`, `orders`, `customer`, ...). +It stresses scan throughput, filter pushdown, joins, and aggregations over mostly numeric +and date columns. + +The queries in this directory (`q1.sql` ... `q22.sql`) are the standard TPC-H queries. +Data is generated deterministically by [`tpchgen`](../../src/tpch/tpchgen.rs); the benchmark +harness lives in [`src/tpch`](../../src/tpch). + +## CI variants + +CI runs this suite at scale factor 1 and 10, reading either from local NVMe or from S3 +(the data is uploaded to a bucket first), as the `TPC-H SF={1,10} on {NVME,S3}` PR +comments. Each variant compares DataFusion and DuckDB over Parquet, Vortex, and +vortex-compact files (plus in-memory Arrow and native DuckDB baselines on NVMe). + +## Running locally + +```bash +vx-bench run tpch --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0 +``` diff --git a/vortex-bench/sql/vortex/README.md b/vortex-bench/sql/vortex/README.md new file mode 100644 index 00000000000..793f7fb0f67 --- /dev/null +++ b/vortex-bench/sql/vortex/README.md @@ -0,0 +1,26 @@ +# Vortex queries benchmark + +A small suite of microbenchmark queries targeting Vortex-specific scan paths, run on +**every PR commit** (unlike the label-gated suites) with a high iteration count, so it is +the most sensitive — and most frequently seen — benchmark comment. + +[`init.sql`](./init.sql) generates a 25M-row two-column table; the numbered queries each +pin down one scan behavior: + +- [`0_sum-with-filter.sql`](./0_sum-with-filter.sql): a filtered aggregation that forces a + linear scan today; once statistics are propagated to arrays it should use zone maps + instead of decoding every row. +- [`1_sum.sql`](./1_sum.sql): an unfiltered aggregation that should eventually be answered + from footer statistics without decoding data. + +## CI variant + +CI runs this as the `Vortex queries` PR comment (see +[`.github/workflows/sql-vortex-pr.yml`](../../../.github/workflows/sql-vortex-pr.yml)), +comparing DataFusion and DuckDB over Parquet and Vortex files with 100 iterations. + +## Running locally + +```bash +vx-bench run vortex --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/src/appian/mod.rs b/vortex-bench/src/appian/mod.rs index cdc82ac200a..20dabb5676a 100644 --- a/vortex-bench/src/appian/mod.rs +++ b/vortex-bench/src/appian/mod.rs @@ -121,6 +121,10 @@ impl AppianBenchmark { #[async_trait::async_trait] impl Benchmark for AppianBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/appian/README.md" + } + fn queries(&self) -> anyhow::Result> { Ok(appian_queries().collect()) } diff --git a/vortex-bench/src/benchmark.rs b/vortex-bench/src/benchmark.rs index d59708886a6..b7015d242a7 100644 --- a/vortex-bench/src/benchmark.rs +++ b/vortex-bench/src/benchmark.rs @@ -64,6 +64,10 @@ pub trait Benchmark: Send + Sync { fn dataset(&self) -> BenchmarkDataset; + /// Repo-relative path of the markdown explainer for this benchmark suite, linked from the + /// title of CI benchmark PR comments. Required: every suite must ship a doc. + fn doc_path(&self) -> &'static str; + /// Get the name of the benchmark dataset fn dataset_name(&self) -> &str; diff --git a/vortex-bench/src/clickbench/benchmark.rs b/vortex-bench/src/clickbench/benchmark.rs index 346496dbda0..fcba1b9f9c1 100644 --- a/vortex-bench/src/clickbench/benchmark.rs +++ b/vortex-bench/src/clickbench/benchmark.rs @@ -76,6 +76,10 @@ fn read_clickbench_queries(queries_file: Option<&str>) -> Result &'static str { + "vortex-bench/sql/clickbench.md" + } + fn queries(&self) -> Result> { read_clickbench_queries(self.queries_file.as_deref()) } @@ -120,6 +124,10 @@ impl Benchmark for ClickBenchBenchmark { #[async_trait::async_trait] impl Benchmark for ClickBenchSortedBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/clickbench.md#sorted-variant" + } + fn queries(&self) -> Result> { Ok(read_clickbench_queries(self.queries_file.as_deref())? .into_iter() diff --git a/vortex-bench/src/display.rs b/vortex-bench/src/display.rs index a8e903df25a..343c1a06612 100644 --- a/vortex-bench/src/display.rs +++ b/vortex-bench/src/display.rs @@ -105,9 +105,14 @@ pub fn render_table( pub fn print_measurements_json( writer: &mut dyn Write, all_measurements: Vec, + doc: &str, ) -> anyhow::Result<()> { for measurement in all_measurements { - writeln!(writer, "{}", measurement.to_json())?; + let mut json = measurement.to_json(); + if let Some(obj) = json.as_object_mut() { + obj.insert("doc".to_string(), doc.into()); + } + writeln!(writer, "{json}")?; } Ok(()) diff --git a/vortex-bench/src/fineweb/mod.rs b/vortex-bench/src/fineweb/mod.rs index b8c86925905..aefa796af3a 100644 --- a/vortex-bench/src/fineweb/mod.rs +++ b/vortex-bench/src/fineweb/mod.rs @@ -53,6 +53,10 @@ impl FinewebBenchmark { #[async_trait::async_trait] impl Benchmark for FinewebBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/fineweb.md" + } + /// Some basic string-focused queries, numbered from Q0 in `sql/fineweb.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/polarsignals/benchmark.rs b/vortex-bench/src/polarsignals/benchmark.rs index fce692ceb68..d1097e83bd1 100644 --- a/vortex-bench/src/polarsignals/benchmark.rs +++ b/vortex-bench/src/polarsignals/benchmark.rs @@ -59,6 +59,10 @@ impl PolarSignalsBenchmark { #[async_trait::async_trait] impl Benchmark for PolarSignalsBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/polarsignals.md" + } + fn queries(&self) -> Result> { let queries_file = workspace_root() .join("vortex-bench") diff --git a/vortex-bench/src/public_bi.rs b/vortex-bench/src/public_bi.rs index f5c22bdea8f..3e651df5bb7 100644 --- a/vortex-bench/src/public_bi.rs +++ b/vortex-bench/src/public_bi.rs @@ -530,6 +530,10 @@ impl PublicBiBenchmark { #[async_trait] impl Benchmark for PublicBiBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/public-bi.md" + } + fn queries(&self) -> anyhow::Result> { self.pbi_benchmark().queries() } diff --git a/vortex-bench/src/realnest/gharchive.rs b/vortex-bench/src/realnest/gharchive.rs index a84395f4a10..6596598d5ba 100644 --- a/vortex-bench/src/realnest/gharchive.rs +++ b/vortex-bench/src/realnest/gharchive.rs @@ -66,6 +66,10 @@ impl GithubArchiveBenchmark { #[async_trait::async_trait] impl Benchmark for GithubArchiveBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/gharchive.md" + } + /// GitHub Archive queries, numbered from Q0 in `sql/gharchive.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index 1983c63ba05..9e5c4db3d08 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -75,6 +75,7 @@ pub struct SqlBenchmarkRunner { formats: Vec, memory_tracker: Option, hide_progress_bar: bool, + doc: &'static str, query_measurements: Vec, memory_measurements: Vec, } @@ -106,6 +107,7 @@ impl SqlBenchmarkRunner { formats, memory_tracker, hide_progress_bar, + doc: benchmark.doc_path(), query_measurements: Vec::new(), memory_measurements: Vec::new(), }) @@ -223,6 +225,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, f, ) } @@ -232,6 +235,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, std::io::stdout().lock(), ), } @@ -249,6 +253,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, output, ) } @@ -457,6 +462,7 @@ pub fn export_results( display_format: &DisplayFormat, engine: Engine, formats: &[Format], + doc: &str, mut output: W, ) -> anyhow::Result<()> { let targets = formats @@ -467,13 +473,13 @@ pub fn export_results( if !memory.is_empty() { match display_format { DisplayFormat::Table => render_table(&mut output, memory, &targets)?, - DisplayFormat::GhJson => print_measurements_json(&mut output, memory)?, + DisplayFormat::GhJson => print_measurements_json(&mut output, memory, doc)?, }; } match display_format { DisplayFormat::Table => render_table(&mut output, queries, &targets)?, - DisplayFormat::GhJson => print_measurements_json(&mut output, queries)?, + DisplayFormat::GhJson => print_measurements_json(&mut output, queries, doc)?, }; Ok(()) diff --git a/vortex-bench/src/spatialbench/benchmark.rs b/vortex-bench/src/spatialbench/benchmark.rs index 48fa84f8ec1..5df7671bd1f 100644 --- a/vortex-bench/src/spatialbench/benchmark.rs +++ b/vortex-bench/src/spatialbench/benchmark.rs @@ -58,6 +58,10 @@ impl SpatialBenchBenchmark { #[async_trait::async_trait] impl Benchmark for SpatialBenchBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/spatialbench.md" + } + /// All SpatialBench queries, numbered started at Q1 in `spatialbench.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs index 0861f7bfc88..2c677fb79a3 100644 --- a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs +++ b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs @@ -103,6 +103,10 @@ impl StatPopGenBenchmark { #[async_trait::async_trait] impl Benchmark for StatPopGenBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/statpopgen.md" + } + fn queries(&self) -> Result> { let queries_file = workspace_root() .join("vortex-bench") diff --git a/vortex-bench/src/tpcds/tpcds_benchmark.rs b/vortex-bench/src/tpcds/tpcds_benchmark.rs index 7f8f2d5281e..5bacae73516 100644 --- a/vortex-bench/src/tpcds/tpcds_benchmark.rs +++ b/vortex-bench/src/tpcds/tpcds_benchmark.rs @@ -56,6 +56,10 @@ impl TpcDsBenchmark { #[async_trait::async_trait] impl Benchmark for TpcDsBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/tpcds/README.md" + } + fn queries(&self) -> Result> { Ok(tpcds_queries().collect()) } diff --git a/vortex-bench/src/tpch/benchmark.rs b/vortex-bench/src/tpch/benchmark.rs index 6851e5f5f06..4dfdaf1207d 100644 --- a/vortex-bench/src/tpch/benchmark.rs +++ b/vortex-bench/src/tpch/benchmark.rs @@ -75,6 +75,10 @@ impl TpcHBenchmark { #[async_trait::async_trait] impl Benchmark for TpcHBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/tpch/README.md" + } + fn queries(&self) -> anyhow::Result> { Ok(tpch_queries().collect()) } diff --git a/vortex-bench/src/vortex_queries.rs b/vortex-bench/src/vortex_queries.rs index f611238bb6b..63e9a658068 100644 --- a/vortex-bench/src/vortex_queries.rs +++ b/vortex-bench/src/vortex_queries.rs @@ -89,6 +89,10 @@ impl VortexBenchmark { #[async_trait::async_trait] impl Benchmark for VortexBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/vortex/README.md" + } + fn queries(&self) -> Result> { self.query_files()? .iter()