Add documentation links to benchmark suite PR comments

.github/workflows/bench-pr.yml

@@ -113,10 +113,8 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
-          echo '' >> comment.md
           uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
-            >> comment.md
+            > comment.md
           cat comment.md >> $GITHUB_STEP_SUMMARY
 
       - name: Comment PR

.github/workflows/sql-benchmarks.yml

@@ -678,10 +678,8 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          echo "# Benchmarks: ${{ matrix.name }}" > comment.md
-          echo '' >> comment.md
           uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \
-            >> comment.md
+            > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR

benchmarks/compress-bench/README.md

@@ -0,0 +1,16 @@
+# Compression benchmark
+
+Measures compression and decompression throughput, plus resulting file sizes, for Vortex
+versus Parquet (and optionally Lance) across a range of datasets: NYC taxi data, several
+[Public BI](https://github.com/cwida/public_bi_benchmark) tables (Arade, Bimbo,
+CMSprovider, Euro2016, Food, HashTags), TPC-H `l_comment` variants, and synthetic nested
+data. This is the workload behind the `Compression` PR comment.
+
+See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--formats`,
+`--datasets`, `--ops compress,decompress`).
+
+## Running locally
+
+```bash
+cargo run -p compress-bench --profile release_debug
+```

benchmarks/compress-bench/src/main.rs

@@ -113,6 +113,9 @@ fn get_compressor(format: Format) -> Box<dyn Compressor> {
 /// The benchmark ID used for output path.
 const BENCHMARK_ID: &str = "compress";
 
+/// Repo-relative path of the suite explainer linked from CI benchmark PR comments.
+const DOC_PATH: &str = "benchmarks/compress-bench/README.md";
+
 async fn run_compress(
     iterations: usize,
     datasets_filter: Option<Regex>,
@@ -212,8 +215,8 @@ async fn run_compress(
             )
         }
         DisplayFormat::GhJson => {
-            print_measurements_json(&mut writer, measurements.timings)?;
-            print_measurements_json(&mut writer, measurements.ratios)
+            print_measurements_json(&mut writer, measurements.timings, DOC_PATH)?;
+            print_measurements_json(&mut writer, measurements.ratios, DOC_PATH)
         }
     }
 }

benchmarks/random-access-bench/README.md

@@ -0,0 +1,22 @@
+# Random Access benchmark
+
+Measures point-lookup latency: fetching individual rows by index from a file, rather than
+scanning it. This is the workload behind the `Random Access` PR comment.
+
+Two access patterns are generated with a fixed seed (see [`src/main.rs`](./src/main.rs)):
+
+- **correlated**: several clusters of consecutive indices scattered across the dataset,
+  simulating lookups with spatial locality;
+- **uniform**: indices drawn from a Poisson process spread uniformly across the dataset,
+  simulating lookups with no locality.
+
+Each pattern runs over four datasets (`taxi`, `feature-vectors`, `nested-lists`,
+`nested-structs`) in Parquet, Lance, and Vortex, both with a cached open file handle and
+reopening the file per lookup. CI drives the full matrix via
+[`scripts/random-access-split.py`](../../scripts/random-access-split.py).
+
+## Running locally
+
+```bash
+cargo run -p random-access-bench --profile release_debug --features lance
+```

benchmarks/random-access-bench/src/lib.rs

@@ -282,6 +282,9 @@ async fn open_accessor(
 /// The benchmark ID used for output path.
 const BENCHMARK_ID: &str = "random-access";
 
+/// Repo-relative path of the suite explainer linked from CI benchmark PR comments.
+const DOC_PATH: &str = "benchmarks/random-access-bench/README.md";
+
 /// Fixed indices used by the original taxi benchmark (preserved for historical continuity).
 const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000];
 
@@ -409,7 +412,7 @@ pub async fn run(config: RunConfig) -> Result<()> {
         }
         DisplayFormat::GhJson => {
             let timings: Vec<TimingMeasurement> = runs.into_iter().map(|r| r.timing).collect();
-            print_measurements_json(&mut writer, timings)?;
+            print_measurements_json(&mut writer, timings, DOC_PATH)?;
         }
     }
 

scripts/compare-benchmark-jsons.py

@@ -12,6 +12,7 @@
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 import math
+import os
 import re
 import sys
 from dataclasses import dataclass
@@ -763,6 +764,27 @@ def format_within_engine_summary(analyses: dict[str, dict[str, Any]]) -> str | N
     return " · ".join(summaries)
 
 
+def format_title(benchmark_name: str, pr: pd.DataFrame) -> str:
+    """Render the comment title, linking the suite explainer doc emitted by the benchmark binary.
+
+    The doc path is a repo-relative markdown path carried on the PR result rows (the `doc`
+    field, populated from `Benchmark::doc_path` in Rust), so the benchmark code is the single
+    source of truth for where each suite is documented. The link pins the PR's own commit so
+    it resolves before the PR merges and stays valid afterwards.
+    """
+
+    title = f"# Benchmarks: {benchmark_name}" if benchmark_name else "# Benchmarks"
+    if "doc" in pr.columns:
+        docs = pr["doc"].dropna().unique()
+        if len(docs) > 0:
+            server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
+            repository = os.environ.get("GITHUB_REPOSITORY", "vortex-data/vortex")
+            commits = pr["commit_id"].dropna().unique() if "commit_id" in pr.columns else []
+            ref = commits[0] if len(commits) > 0 else "develop"
+            title += f" [\N{OPEN BOOK}]({server_url}/{repository}/blob/{ref}/{docs[0]})"
+    return title
+
+
 def format_report_help() -> str:
     """Render explanatory markdown for the benchmark report headline fields."""
 
@@ -823,6 +845,7 @@ def main() -> None:
     benchmark_name = sys.argv[3] if len(sys.argv) > 3 else ""
 
     pr = pd.read_json(sys.argv[2], lines=True)
+    title = format_title(benchmark_name, pr)
     base = read_latest_baseline_rows(sys.argv[1], pr)
 
     base_commit_id = set(base["commit_id"].unique())
@@ -896,6 +919,8 @@ def main() -> None:
                 shifts += f" · Median polish {format_ratio_change(float(np.exp(polish.overall)))}"
         summary_fields.append(f"**Shifts**: {shifts}")
 
+    print(title)
+    print("")
     print("<br>".join(summary_fields))
     print("")
     print(format_report_help())

vortex-bench/REUSE.toml

@@ -10,6 +10,12 @@ path = "**/*.csv"
 SPDX-FileCopyrightText = "Copyright the Vortex contributors"
 SPDX-License-Identifier = "Apache-2.0"
 
+# Benchmark suite explainer docs linked from CI PR comments.
+[[annotations]]
+path = "sql/**/*.md"
+SPDX-FileCopyrightText = "Copyright the Vortex contributors"
+SPDX-License-Identifier = "CC-BY-4.0"
+
 # `insta` snapshot files do not allow leading comment lines.
 [[annotations]]
 path = "src/snapshots/**.snap"

vortex-bench/sql/appian/README.md

@@ -0,0 +1,22 @@
+# Appian benchmark
+
+Mirrors the queries from DuckDB's in-tree
+[`appian_benchmarks`](https://github.com/duckdb/duckdb/tree/main/benchmark/appian_benchmarks)
+suite — a real-world business-application workload (customers, orders, addresses) with
+selective filters and joins over camelCase-named columns.
+
+The eight queries live in this directory (`q1.sql` ... `q8.sql`). Upstream ships the data
+as a single `.duckdb` blob (~593 MB); the harness downloads it once and projects each
+table into Parquet with lowercased column names so that DataFusion and DuckDB resolve the
+verbatim queries identically — see [`src/appian`](../../src/appian) for the details.
+
+## CI variant
+
+CI runs this suite from local NVMe as the `Appian on NVME` PR comment, comparing
+DataFusion and DuckDB over Parquet and Vortex files (plus a native DuckDB baseline).
+
+## Running locally
+
+```bash
+vx-bench run appian --engine datafusion,duckdb --format parquet,vortex
+```

vortex-bench/sql/clickbench.md

@@ -0,0 +1,30 @@
+# ClickBench benchmark
+
+[ClickBench](https://github.com/ClickHouse/ClickBench) is ClickHouse's web-analytics
+benchmark: 43 queries over a single wide `hits` table (~100M rows of real-ish traffic
+data). It is heavy on aggregations, `GROUP BY`s over high-cardinality columns, and
+selective string filters, and is the main "wide table scan" workload in CI.
+
+The queries live in [`clickbench_queries.sql`](./clickbench_queries.sql) (one query per
+line, numbered from Q0 in file order). The harness lives in
+[`src/clickbench`](../src/clickbench).
+
+## CI variant
+
+CI runs this suite from local NVMe as the `Clickbench on NVME` PR comment, comparing
+DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files (plus a native
+DuckDB baseline).
+
+## Sorted variant
+
+`Clickbench Sorted on NVME` runs the same table sorted by event time, split into 100
+shards whose filenames are shuffled so engines cannot rely on file order, exercising sort
+pushdown and zone-map-style pruning on a subset of the queries. See
+[`ClickBenchSortedBenchmark`](../src/clickbench/benchmark.rs).
+
+## Running locally
+
+```bash
+vx-bench run clickbench --engine datafusion,duckdb --format parquet,vortex
+vx-bench run clickbench-sorted --engine datafusion,duckdb --format parquet,vortex
+```

vortex-bench/sql/fineweb.md

@@ -0,0 +1,21 @@
+# FineWeb benchmark
+
+A string-heavy benchmark over a sample shard of the HuggingFace
+[FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) web-crawl dataset
+(URLs, dates, and full page text). The dataset exercises dictionary and FSST string
+compression heavily, and the hand-crafted queries in [`fineweb.sql`](./fineweb.sql)
+(numbered from Q0 in file order) focus on string predicates: equality filters, `LIKE`
+prefix and containment patterns, and aggregations over string columns.
+
+The harness lives in [`src/fineweb`](../src/fineweb).
+
+## CI variants
+
+CI runs this suite from local NVMe (`FineWeb NVMe`) and from S3 (`FineWeb S3`), comparing
+DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files.
+
+## Running locally
+
+```bash
+vx-bench run fineweb --engine datafusion,duckdb --format parquet,vortex
+```

vortex-bench/sql/gharchive.md

@@ -0,0 +1,14 @@
+# GitHub Archive benchmark
+
+A deeply nested real-world dataset from the [GitHub Archive](https://www.gharchive.org/):
+GitHub event records whose `payload`, `repo`, `actor`, and `org` fields are nested structs.
+The queries in [`gharchive.sql`](./gharchive.sql) (numbered from Q0 in file order) filter
+and aggregate on those nested fields, making this the main struct-field-pushdown workload.
+
+The harness lives in [`src/realnest/gharchive.rs`](../src/realnest/gharchive.rs).
+
+## Running locally
+
+```bash
+cargo run --bin datafusion-bench --profile release_debug -- --benchmark gharchive
+```

vortex-bench/sql/polarsignals.md

@@ -0,0 +1,23 @@
+# PolarSignals Profiling benchmark
+
+A benchmark over continuous-profiling stacktrace data from
+[Polar Signals](https://www.polarsignals.com/). The schema features a sparse struct (ten
+nullable label fields covering five fill-rate tiers), deeply nested locations
+(`List<Struct<..., List<Struct<...>>>>`), and several low-cardinality string columns —
+making it the main deeply-nested / sparse-data workload in CI.
+
+The queries in [`polarsignals.sql`](./polarsignals.sql) select stacktrace samples over
+increasingly wide time ranges with equality filters on the low-cardinality metadata
+columns. The harness lives in [`src/polarsignals`](../src/polarsignals).
+
+## CI variant
+
+CI runs this suite from local NVMe as the `PolarSignals Profiling` PR comment. It runs
+DataFusion over Vortex only (there is no Parquet control for this suite, so its geomean is
+reported without a drift-corrected verdict).
+
+## Running locally
+
+```bash
+vx-bench run polarsignals --engine datafusion --format vortex
+```

vortex-bench/sql/public-bi.md

@@ -0,0 +1,15 @@
+# Public BI benchmark
+
+Real-world BI datasets from the
+[Public BI benchmark](https://github.com/cwida/public_bi_benchmark) — anonymized Tableau
+workbooks with messy, wide, string-heavy tables. A subset of the datasets (Arade, Bimbo,
+CMSprovider, Euro2016, Food, HashTags) also feeds the
+[Compression benchmark](../../benchmarks/compress-bench/README.md).
+
+The datasets, schemas, and queries are defined in [`src/public_bi.rs`](../src/public_bi.rs).
+
+## Running locally
+
+```bash
+vx-bench run public-bi
+```

vortex-bench/sql/spatialbench.md

@@ -0,0 +1,15 @@
+# SpatialBench benchmark
+
+The [Apache Sedona SpatialBench](https://sedona.apache.org/spatialbench/) geospatial
+analytics benchmark: twelve queries (Q1 ... Q12 in [`spatialbench.sql`](./spatialbench.sql),
+DuckDB dialect) over a trips/zones schema, exercising spatial predicates and functions such
+as `ST_DWithin`, `ST_Intersects`, and `ST_Distance`. The query logic matches upstream
+`sedona-spatialbench`; only formatting differs.
+
+The harness lives in [`src/spatialbench`](../src/spatialbench).
+
+## Running locally
+
+```bash
+vx-bench run spatialbench
+```

vortex-bench/sql/statpopgen.md

@@ -0,0 +1,25 @@
+# Statistical and Population Genetics benchmark
+
+A genomics analytics benchmark using the gnomAD v3.1.2 release of the jointly called One
+Thousand Genomes + Human Genome Diversity Project (1kG+HGDP) dataset — a prefix of
+chromosome 21. The data source is <https://gnomad.broadinstitute.org/>.
+
+The distinguishing feature is the `GT` genotype column: a large variable-length list of
+small nullable integers per row, which stresses list encodings and list-aware compute.
+The queries in [`statpopgen.sql`](./statpopgen.sql) cover common genomics patterns:
+allele-frequency calculations, Hardy-Weinberg equilibrium statistics, variant filtering by
+frequency and by locus interval, and random access to a specific variant.
+
+The harness lives in [`src/statpopgen`](../src/statpopgen).
+
+## CI variant
+
+CI runs this suite from local NVMe as the `Statistical and Population Genetics` PR
+comment. Only DuckDB is exercised (over Parquet, Vortex, and vortex-compact files),
+because the queries rely on DuckDB list lambdas.
+
+## Running locally
+
+```bash
+vx-bench run statpopgen --engine duckdb --format parquet,vortex
+```

vortex-bench/sql/tpcds/README.md

@@ -0,0 +1,20 @@
+# TPC-DS benchmark
+
+The industry-standard [TPC-DS](https://www.tpc.org/tpcds/) decision-support benchmark: 99
+analytical queries (`01.sql` ... `99.sql`) over a retail snowflake schema. Compared to
+TPC-H it has many more queries, a wider schema, and heavier use of joins, subqueries, and
+window functions, making it a broad regression net for scan and pushdown behavior.
+
+The benchmark harness lives in [`src/tpcds`](../../src/tpcds).
+
+## CI variant
+
+CI runs this suite at scale factor 1 from local NVMe as the `TPC-DS SF=1 on NVME` PR
+comment, comparing DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files
+(plus a native DuckDB baseline).
+
+## Running locally
+
+```bash
+vx-bench run tpcds --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0
+```

vortex-bench/sql/tpch/README.md

@@ -0,0 +1,23 @@
+# TPC-H benchmark
+
+The industry-standard [TPC-H](https://www.tpc.org/tpch/) decision-support benchmark: 22
+analytical queries over an 8-table warehouse schema (`lineitem`, `orders`, `customer`, ...).
+It stresses scan throughput, filter pushdown, joins, and aggregations over mostly numeric
+and date columns.
+
+The queries in this directory (`q1.sql` ... `q22.sql`) are the standard TPC-H queries.
+Data is generated deterministically by [`tpchgen`](../../src/tpch/tpchgen.rs); the benchmark
+harness lives in [`src/tpch`](../../src/tpch).
+
+## CI variants
+
+CI runs this suite at scale factor 1 and 10, reading either from local NVMe or from S3
+(the data is uploaded to a bucket first), as the `TPC-H SF={1,10} on {NVME,S3}` PR
+comments. Each variant compares DataFusion and DuckDB over Parquet, Vortex, and
+vortex-compact files (plus in-memory Arrow and native DuckDB baselines on NVMe).
+
+## Running locally
+
+```bash
+vx-bench run tpch --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0
+```