Add Benchmark

dev/benchmark/.gitignore

@@ -0,0 +1,8 @@
+*.arrow
+*.avro
+*.json
+*.parquet
+*.tsv
+*.csv
+*.gz
+.DS_Store

dev/benchmark/Makefile

@@ -0,0 +1,43 @@
+.PHONY: corpus # Create the data corpus.
+corpus: mk-datasets.sh amazon.sql export-hacknernews.sql hackernews.sql logs.sql taxi_trips.sql
+	@bash $<
+
+.PHONY: sync # Upload the datasets to S3
+sync: corpus
+	@aws s3 sync logs s3://chdb-lakedata-public/logs
+	@aws s3 sync taxi_trips s3://chdb-lakedata-public/taxi_trips
+# 	@aws s3 sync amazon s3://chdb-lakedata-public/amazon
+# 	@aws s3 sync hackernews s3://chdb-lakedata-public/hackernews
+
+PSQL_OPTS := --quiet --no-psqlrc --no-align --tuples-only --set ON_ERROR_STOP=1
+results.txt: pg_lake/results.tsv pg_duckdb/results.tsv aws_s3/results.tsv chdb/results.tsv
+	@echo 'dataset	format	compression	run1	run2	run3	average' > $@
+	@cat $^ >> $@
+
+chdb/results.tsv: chdb/*.sql
+	@psql $(PSQL_OPTS) -f chdb/logs.sql > $@
+	@psql $(PSQL_OPTS) -f chdb/taxi.sql >> $@
+# 	@psql $(PSQL_OPTS) -f chdb/amazon.sql >> $@
+# 	@psql $(PSQL_OPTS) -f chdb/hackernews.sql >> $@
+
+aws_s3/results.tsv: aws_s3/*.sql
+	@psql $(PSQL_OPTS) -f aws_s3/logs.sql > $@
+	@psql $(PSQL_OPTS) -f aws_s3/taxi.sql >> $@
+# 	@psql $(PSQL_OPTS) -f aws_s3/amazon.sql >> $@
+# 	@psql $(PSQL_OPTS) -f aws_s3/hackernews.sql >> $@
+
+pg_duckdb/results.tsv: pg_duckdb/*.sql
+	@psql $(PSQL_OPTS) -f pg_duckdb/logs.sql > $@
+	@psql $(PSQL_OPTS) -f pg_duckdb/taxi.sql >> $@
+# 	@psql $(PSQL_OPTS) -f pg_duckdb/amazon.sql >> $@
+# 	@psql $(PSQL_OPTS) -f pg_duckdb/hackernews.sql >> $@
+
+pg_lake/results.tsv: pg_lake/*.sql
+	@psql $(PSQL_OPTS) -f pg_lake/logs.sql > $@
+	@psql $(PSQL_OPTS) -f pg_lake/taxi.sql >> $@
+# 	@psql $(PSQL_OPTS) -f pg_lake/amazon.sql >> $@
+# 	@psql $(PSQL_OPTS) -f pg_lake/hackernews.sql >> $@
+
+.PHONY: sync # Summarize results by extension with averages for each data set and format.
+summary: summary.sql results.txt
+	@psql $(PSQL_OPTS) -f $<

dev/benchmark/README.md

@@ -0,0 +1,102 @@
+Postgres Lake Copy Benchmark
+============================
+
+Create data sets in various formats and test importing that data from S3 with
+various extensions.
+
+The benchmark scripts for each extension execute in a PL/pgSQL function to
+minimize overhead timing, except for pg_duckdb, which cannot run inside a
+function. Each establishes an overhead cost, executes each query three times
+and subtracts the overhead from each run, then summarizes the output.
+
+## Charts
+
+Test configurations:
+
+*   chdb 0.1.1, PG 18, r8id.xlarge, 4 vCPUs and 32 GB RAM
+*   pg_lake 3.5, PG 18, r8id.xlarge, 4 vCPUs and 32 GB RAM
+*   pg_duckdb 1.2.0 (ee38d3b), PG 18, r8id.xlarge, 4 vCPUs and 32 GB RAM
+*   aws_s3 1.2.0, PG 18, db.r8g.xlarge, 4 vCPUs and 32 GB RAM
+
+### NYC Taxi Data
+
+![NYC Taxi Data Benchmark](taxi-bench.png)
+
+### "Logs" Data
+
+!["Logs" Data Benchmark](logs-bench.png)
+
+### chdb Data Formats
+
+![chdb Data Formats](chdb-bench.png)
+
+## Data Sets
+
+```sh
+make corpus
+```
+
+Generates the benchmark data sets in the `amazon`, `hacknernews`, `logs`, and
+`taxi_trips` directories.
+
+## Upload
+
+```sh
+export AWS_ACCESS_KEY_ID="xxxxxxxxxxxxxxxxx"
+export AWS_SECRET_ACCESS_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+export AWS_SESSION_TOKEN="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+make sync-s3
+```
+
+Syncs the data sets to the `chdb-lakedata-public` S3 bucket. Requires the
+`aws` CLI.
+
+## Benchmark
+
+```sh
+export PGHOST=chdb_host PGUSER=chdb_user PGPASSWORD=chdb_password
+make chdb/results.tsv
+
+export PGHOST=aws_s3_host PGUSER=aws_s3_user PGPASSWORD=aws_s3_password
+make aws_s3/results.tsv
+
+export PGHOST=pg_duckdb_host PGUSER=pg_duckdb_user PGPASSWORD=pg_duckdb_password
+make pg_duckdb/results.tsv
+
+export PGHOST=pg_lake_host PGUSER=pg_lake_user PGPASSWORD=pg_lake_password
+make pg_lake/results.tsv
+
+make results.txt
+```
+
+Run each of the tests with any Postgres-specific environment configuration for
+each, then collect them all into `results.txt`.
+
+### Summarize
+
+```sh
+make summary
+```
+
+Summarizes the results by extension, with average runtimes for each data set
+and format. Paste into a spreadsheet to generate charts and graphs.
+
+## TOC
+
+*   `Makefile`: Execute tasks
+*   `amazon.sql`: Generate Amazon Reviews dataset in `amazon` directory
+*   `aws_s3/`: Scripts to test data import with [aws_s3]
+*   `chdb/`: Scripts to test data import with [chdb_hook]
+*   `export-hacknernews.sql`:  Export Hacknernews dataset from ClickHouse
+*   `hackernews.sql`: Generate Hacknernews dataset in `hacknernews` directory
+*   `logs.sql`: Generate faux logs output in `logs` directory
+*   `mk-datasets.sh`: Generates all datasets
+*   `pg_duckdb/`: Scripts to test data import with [pg_duckdb]
+*   `pg_lake/`:  Scripts to test data import with [pg_lake]
+*   `results.sql`: Reformat results into table for graph generation
+*   `taxi_trips.sql`: Generate NYC Taxi dataset in `taxi_trips` directory
+
+  [chdb_hook]: https://pgxn.org/dist/chdb/doc/chdb_hook.html
+  [aws_s3]: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_PostgreSQL.S3Import.html
+  [pg_duckdb]: https://github.com/duckdb/pg_duckdb
+  [pg_lake]: https://github.com/Snowflake-Labs/pg_lake

dev/benchmark/amazon.sql

@@ -0,0 +1,76 @@
+SET client_min_messages TO WARNING;
+SET datestyle TO ISO;
+SET timezone TO UTC;
+\getenv workdir WORKDIR
+\set file_base file:// :workdir
+
+LOAD 'chdb_hook';
+
+-- https://clickhouse.com/docs/get-started/sample-datasets/amazon-reviews
+
+CREATE TABLE IF NOT EXISTS amazon_reviews (
+    review_date       DATE     NOT NULL,
+    marketplace       TEXT     NOT NULL,
+    customer_id       BIGINT   NOT NULL,
+    review_id         TEXT     NOT NULL,
+    product_id        TEXT     NOT NULL,
+    product_parent    BIGINT   NOT NULL,
+    product_title     TEXT     NOT NULL,
+    product_category  TEXT     NOT NULL,
+    star_rating       SMALLINT NOT NULL,
+    helpful_votes     BIGINT   NOT NULL,
+    total_votes       BIGINT   NOT NULL,
+    vine              BOOL     NOT NULL,
+    verified_purchase BOOL     NOT NULL,
+    review_headline   TEXT     NOT NULL,
+    review_body       TEXT     NOT NULL
+);
+
+-- Insert the records if they're not present.
+SELECT NOT EXISTS (SELECT 1 FROM amazon_reviews LIMIT 1) AS is_empty \gset
+\if :is_empty
+    \echo Loading amazon_reviews_2015.snappy.parquet
+    \set orig_file :file_base /amazon_reviews_2015.snappy.parquet
+    COPY amazon_reviews FROM :'orig_file';
+\endif
+
+\set structure 'review_date Date32, marketplace String, customer_id UInt64, review_id String, product_id String, product_parent UInt64, product_title String, product_category String, star_rating UInt8, helpful_votes UInt32, total_votes UInt32, vine Bool, verified_purchase Bool, review_headline String, review_body String'
+
+\if `[ ! -f amazon_reviews_2015.tsv ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.tsv
+    \set tsv_file :file_base /amazon_reviews_2015.tsv
+    COPY amazon_reviews TO :'tsv_file' ( format 'TabSeparatedWithNames', structure :'structure');
+    \! gzip -fk amazon_reviews_2015.tsv
+\endif
+
+\if `[ ! -f amazon_reviews_2015.csv ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.csv
+    \set csv_file :file_base /amazon_reviews_2015.csv
+    COPY amazon_reviews TO :'csv_file' ( format 'CSVWithNames', structure :'structure');
+    \! gzip -fk amazon_reviews_2015.csv
+\endif
+
+\if `[ ! -f amazon_reviews_2015.json ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.json
+    \set json_file :file_base /amazon_reviews_2015.json
+    COPY amazon_reviews TO :'json_file' ( format 'JSONEachRow', structure :'structure');
+    \! gzip -fk amazon_reviews_2015.json
+\endif
+
+\if `[ ! -f amazon_reviews_2015.parquet ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.parquet
+    \set parquet_file :file_base /amazon_reviews_2015.parquet
+    COPY amazon_reviews TO :'parquet_file' ( format 'parquet', structure :'structure');
+\endif
+
+\if `[ ! -f amazon_reviews_2015.arrow ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.arrow
+    \set arrow_file :file_base /amazon_reviews_2015.arrow
+    COPY amazon_reviews TO :'arrow_file' ( format 'arrow', structure :'structure');
+\endif
+
+\if `[ ! -f amazon_reviews_2015.avro ] && echo 1 || echo 0`
+    \echo Exporting to amazon_reviews_2015.avro
+    \set avro_file :file_base /amazon_reviews_2015.avro
+    COPY amazon_reviews TO :'avro_file' ( format 'avro', structure :'structure');
+\endif

dev/benchmark/aws_s3/logs.sql

@@ -0,0 +1,25 @@
+BEGIN;
+SET client_min_messages TO WARNING;
+
+CREATE TYPE http_method AS ENUM(
+  'GET', 'HEAD', 'POST', 'PUT', 'DELETE',
+  'CONNECT', 'OPTIONS', 'TRACE', 'PATCH', 'QUERY'
+);
+
+-- Table name `target` required by util.pgsql/load_target().
+CREATE TABLE  target (
+    req_id    BIGINT,
+    start_at  TIMESTAMPTZ,
+    duration  INTEGER,
+    resource  TEXT,
+    method    http_method,
+    node_id   BIGINT,
+    response  INTEGER
+);
+
+\ir util.pgsql
+
+COPY (SELECT * FROM load_target('logs', 'Text', 'none', 'logs/logs-2026-08-26.tsv', 'logs/logs-2026-08-27.tsv', 'logs/logs-2026-08-28.tsv')) TO STDOUT;
+COPY (SELECT * FROM load_target('logs', 'CSV', 'none', 'logs/logs-2026-08-26.csv', 'logs/logs-2026-08-27.csv', 'logs/logs-2026-08-28.csv')) TO STDOUT;
+
+ROLLBACK;

dev/benchmark/aws_s3/taxi.sql

@@ -0,0 +1,62 @@
+-- https://clickhouse.com/docs/get-started/quickstarts/tutorial
+
+BEGIN;
+SET client_min_messages TO WARNING;
+
+CREATE EXTENSION IF NOT EXISTS aws_s3 CASCADE;
+
+-- Table name `target` required by util.pgsql/load_target().
+CREATE TABLE IF NOT EXISTS target (
+    trip_id                 bigint,
+    vendor_id               bigint,
+    pickup_date             date,
+    pickup_datetime         timestamp with time zone,
+    dropoff_date            date,
+    dropoff_datetime        timestamp with time zone,
+    store_and_fwd_flag      smallint,
+    rate_code_id            smallint,
+    pickup_longitude        double precision,
+    pickup_latitude         double precision,
+    dropoff_longitude       double precision,
+    dropoff_latitude        double precision,
+    passenger_count         smallint,
+    trip_distance           double precision,
+    fare_amount             numeric(10,2),
+    extra                   numeric(10,2),
+    mta_tax                 numeric(10,2),
+    tip_amount              numeric(10,2),
+    tolls_amount            numeric(10,2),
+    ehail_fee               numeric(10,2),
+    improvement_surcharge   numeric(10,2),
+    total_amount            numeric(10,2),
+    payment_type            text,
+    trip_type               smallint,
+    pickup                  character varying(25),
+    dropoff                 character varying(25),
+    cab_type                text,
+    pickup_nyct2010_gid     smallint,
+    pickup_ctlabel          double precision,
+    pickup_borocode         smallint,
+    pickup_ct2010           text,
+    pickup_boroct2010       text,
+    pickup_cdeligibil       text,
+    pickup_ntacode          character varying(4),
+    pickup_ntaname          text,
+    pickup_puma             integer,
+    dropoff_nyct2010_gid    smallint,
+    dropoff_ctlabel         double precision,
+    dropoff_borocode        smallint,
+    dropoff_ct2010          text,
+    dropoff_boroct2010      text,
+    dropoff_cdeligibil      text,
+    dropoff_ntacode         character varying(4),
+    dropoff_ntaname         text,
+    dropoff_puma            integer
+);
+
+\ir util.pgsql
+
+COPY (SELECT * FROM load_target('taxi', 'Text', 'none', 'taxi_trips/taxi_trips.tsv')) TO STDOUT;
+COPY (SELECT * FROM load_target('taxi', 'CSV', 'none', 'taxi_trips/taxi_trips.csv')) TO STDOUT;
+
+ROLLBACK;