diff --git a/benchmarks/README.md b/benchmarks/README.md index 489f2d485a9ff..ec9cf01e7fa8b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -923,6 +923,22 @@ cargo run --release --bin dfbench -- h2o --join-paths ./benchmarks/data/h2o/J1_1 # Micro-Benchmarks +## ASOF Join + +This benchmark exercises ASOF joins across relative input sizes, available +input ordering, equality-group cardinality and skew, match direction, and +payload width. One keyed workload reads pre-sorted Parquet inputs with declared +ordering so it measures the join without including an input sort. + +### Example Run + +```bash +# No external data is required. The pre-sorted Parquet inputs are generated by +# the SQL benchmark setup outside the measured query. + +./bench.sh run asof_join +``` + ## Nested Loop Join This benchmark focuses on the performance of queries with nested loop joins, minimizing other overheads such as scanning data sources or evaluating predicates. diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh index df33d2aec7e54..97c15e7cf09d1 100755 --- a/benchmarks/bench.sh +++ b/benchmarks/bench.sh @@ -160,6 +160,7 @@ imdb: Join Order Benchmark (JOB) using the IMDB dataset conver # Micro-Benchmarks (specific operators and features) cancellation: How long cancelling a query takes +asof_join: ASOF join workloads varying size, ordering, grouping, match direction, and payload width nlj: Benchmark for simple nested loop joins, testing various join scenarios hj: Benchmark for simple hash joins, testing various join scenarios smj: Benchmark for simple sort merge joins, testing various join scenarios @@ -269,6 +270,10 @@ main() { # Data is generated inline by the suite's load SQL (COPY). echo "parquet_row_filter_skip: no external data to generate" ;; + asof_join) + # The ordered Parquet case is generated inline by the suite's load SQL. + echo "asof_join: no external data to generate" + ;; tpcds) data_tpcds ;; @@ -479,6 +484,7 @@ main() { run_h2o_join "BIG" "PARQUET" "join" run_imdb run_external_aggr + run_asof_join run_nlj run_hj run_tpcds @@ -512,6 +518,9 @@ main() { parquet_row_filter_skip) run_parquet_row_filter_skip ;; + asof_join) + run_asof_join + ;; tpcds) run_tpcds ;; @@ -1632,6 +1641,14 @@ run_topk_sorted_tpch() { $CARGO_COMMAND --bin dfbench -- sort-tpch --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" --sorted --limit 100 ${QUERY_ARG} ${LATENCY_ARG} } +# Runs the ASOF join benchmark +run_asof_join() { + RESULTS_FILE="${RESULTS_DIR}/asof_join.json" + echo "RESULTS_FILE: ${RESULTS_FILE}" + echo "Running ASOF join benchmark..." + debug_run $CARGO_COMMAND --bin benchmark_runner -- asof_join --iterations 5 -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG} +} + # Runs the nlj benchmark run_nlj() { RESULTS_FILE="${RESULTS_DIR}/nlj.json" diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.md index b8bf8adedede2..c5a4136e8e5ea 100644 --- a/benchmarks/sql_benchmarks/README.md +++ b/benchmarks/sql_benchmarks/README.md @@ -29,6 +29,7 @@ in the community: | Benchmark Suite | Description | |-----------------------|--------------------------------------------------------------------| +| `asof_join` | ASOF join benchmarks across size, ordering, grouping, match direction, and payload width | | `clickbench` | ClickBench benchmark | | `clickbench extended` | 12 additional, more complex queries against the Clickbench dataset | | `clickbench_sorted` | ClickBench benchmark using a pre-sorted hits file. | diff --git a/benchmarks/sql_benchmarks/asof_join/asof_join.suite b/benchmarks/sql_benchmarks/asof_join/asof_join.suite new file mode 100644 index 0000000000000..773afb83feca4 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/asof_join.suite @@ -0,0 +1,18 @@ +# Workloads vary these axes: +# - relative left and right input sizes +# - required ordering already available or included in the measured query +# - no equality key, many balanced groups, or heavily skewed groups +# - latest right row at or before, or earliest right row at or after, each left row +# - narrow integer or wide UTF-8 payloads + +description = "ASOF join SQL benchmarks" + +query_pattern = "q{QUERY_ID_PADDED}.benchmark" + +[[examples]] +command = "cargo run --release --bin benchmark_runner -- asof_join" +description = "Run all ASOF join queries." + +[[examples]] +command = "cargo run --release --bin benchmark_runner -- asof_join --query 7" +description = "Run the pre-sorted keyed ASOF join query." diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmark new file mode 100644 index 0000000000000..fdbea8bd456e8 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmark @@ -0,0 +1,17 @@ +name Q01 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left: 1M rows; right: 10K rows; no equality key; ascending timestamps. +-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts). +WITH left_input AS ( + SELECT value AS ts, value AS payload FROM range(1000000) +), +right_input AS ( + SELECT value AS ts, value AS payload FROM range(10000) +) +SELECT l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts); diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmark new file mode 100644 index 0000000000000..0c857192c4648 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmark @@ -0,0 +1,25 @@ +name Q02 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left and right: 1M rows each; 10K balanced equality groups; integer payloads. +-- Inputs are not ordered by (key, ts), so required ordering is included. +-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts). +WITH left_input AS ( + SELECT value % 10000 AS key, + value / 10000 + 1 AS ts, + value AS payload + FROM range(1000000) +), +right_input AS ( + SELECT value % 10000 AS key, + value / 10000 AS ts, + value AS payload + FROM range(1000000) +) +SELECT l.key, l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts) +ON l.key = r.key; diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmark new file mode 100644 index 0000000000000..af051fc106f82 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmark @@ -0,0 +1,26 @@ +name Q03 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left and right: 250K rows each; 10K balanced equality groups. +-- Inputs are not ordered by (key, ts), so required ordering is included. +-- Matches the latest right row at or before each left timestamp and materializes +-- 256-byte UTF-8 payloads from both inputs. +WITH left_input AS ( + SELECT value % 10000 AS key, + value / 10000 + 1 AS ts, + repeat('x', 256) AS payload + FROM range(250000) +), +right_input AS ( + SELECT value % 10000 AS key, + value / 10000 AS ts, + repeat('y', 256) AS payload + FROM range(250000) +) +SELECT l.key, l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts) +ON l.key = r.key; diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmark new file mode 100644 index 0000000000000..1c58063bbf378 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmark @@ -0,0 +1,18 @@ +name Q04 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left and right: 500K rows each; no equality key; ascending source timestamps. +-- Matches the earliest right row at or after each left timestamp (l.ts <= r.ts), +-- so the measured query includes the required descending ordering. +WITH left_input AS ( + SELECT value AS ts, value AS payload FROM range(500000) +), +right_input AS ( + SELECT value AS ts, value AS payload FROM range(500000) +) +SELECT l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts <= r.ts); diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmark new file mode 100644 index 0000000000000..b1d10c66df28d --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmark @@ -0,0 +1,18 @@ +name Q05 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left: 100K rows; right: 1M rows; no equality key; ascending timestamps. +-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts). +-- Exercises a right input that is ten times larger than the left input. +WITH left_input AS ( + SELECT value AS ts, value AS payload FROM range(100000) +), +right_input AS ( + SELECT value AS ts, value AS payload FROM range(1000000) +) +SELECT l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts); diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmark new file mode 100644 index 0000000000000..c6cb12a45eba9 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmark @@ -0,0 +1,25 @@ +name Q06 +group asof_join + +expect_plan AsOfJoinExec + +run +-- Left and right: 1M rows each; one equality group contains 95% of rows. +-- Inputs are not ordered by (key, ts), so required ordering is included. +-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts). +WITH left_input AS ( + SELECT CASE WHEN value % 100 < 95 THEN 0 ELSE value % 16 + 1 END AS key, + value / 100 + 1 AS ts, + value AS payload + FROM range(1000000) +), +right_input AS ( + SELECT CASE WHEN value % 100 < 95 THEN 0 ELSE value % 16 + 1 END AS key, + value / 100 AS ts, + value AS payload + FROM range(1000000) +) +SELECT l.key, l.ts, l.payload, r.payload AS right_payload +FROM left_input l +ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts) +ON l.key = r.key; diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmark new file mode 100644 index 0000000000000..7d878f468b1f0 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmark @@ -0,0 +1,25 @@ +name Q07 +group asof_join + +load sql_benchmarks/asof_join/init/load_q07.sql + +assert II +SELECT + (SELECT count(*) FROM asof_left_sorted) AS left_rows, + (SELECT count(*) FROM asof_right_sorted) AS right_rows +---- +100000|100000 + +expect_plan AsOfJoinExec + +run +-- Left and right: 100K rows each; 10K balanced equality groups; integer payloads. +-- Both Parquet inputs declare (group_key, ts) ordering, so the measured query does not +-- include the input sorts required by the other keyed workloads. +-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts). +SELECT l.group_key, l.ts, l.payload, r.payload AS right_payload +FROM asof_left_sorted l +ASOF JOIN asof_right_sorted r MATCH_CONDITION (l.ts >= r.ts) +ON l.group_key = r.group_key; + +cleanup sql_benchmarks/asof_join/init/cleanup_q07.sql diff --git a/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sql b/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sql new file mode 100644 index 0000000000000..85887a15ab19c --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS asof_left_sorted; +DROP TABLE IF EXISTS asof_right_sorted; diff --git a/benchmarks/sql_benchmarks/asof_join/init/load_q07.sql b/benchmarks/sql_benchmarks/asof_join/init/load_q07.sql new file mode 100644 index 0000000000000..5905081d0f6f3 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/init/load_q07.sql @@ -0,0 +1,37 @@ +COPY ( + SELECT value / 10 AS group_key, + value % 10 + 1 AS ts, + value AS payload + FROM range(100000) + ORDER BY group_key, ts +) +TO 'sql_benchmarks/asof_join/scratch/q07_left.parquet' +STORED AS PARQUET; + +COPY ( + SELECT value / 10 AS group_key, + value % 10 AS ts, + value AS payload + FROM range(100000) + ORDER BY group_key, ts +) +TO 'sql_benchmarks/asof_join/scratch/q07_right.parquet' +STORED AS PARQUET; + +CREATE EXTERNAL TABLE asof_left_sorted ( + group_key BIGINT, + ts BIGINT, + payload BIGINT +) +STORED AS PARQUET +LOCATION 'sql_benchmarks/asof_join/scratch/q07_left.parquet' +WITH ORDER (group_key ASC NULLS FIRST, ts ASC NULLS FIRST); + +CREATE EXTERNAL TABLE asof_right_sorted ( + group_key BIGINT, + ts BIGINT, + payload BIGINT +) +STORED AS PARQUET +LOCATION 'sql_benchmarks/asof_join/scratch/q07_right.parquet' +WITH ORDER (group_key ASC NULLS FIRST, ts ASC NULLS FIRST); diff --git a/benchmarks/sql_benchmarks/asof_join/scratch/.gitignore b/benchmarks/sql_benchmarks/asof_join/scratch/.gitignore new file mode 100644 index 0000000000000..4bed5da93fb28 --- /dev/null +++ b/benchmarks/sql_benchmarks/asof_join/scratch/.gitignore @@ -0,0 +1 @@ +*.parquet