From da75e675e51e57b6724a87524208bec026ee4362 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 2 Mar 2026 17:30:44 +0800 Subject: [PATCH 01/43] x (#301) Co-authored-by: luofucong --- datafusion-pg-catalog/src/sql/rules.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index 25bdbe3..bc80aba 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -462,6 +462,7 @@ impl RewriteRegclassCastToSubqueryVisitor { data_type, expr: inner_expr, format: _, + .. } = expr { if *kind == CastKind::DoubleColon { @@ -480,6 +481,7 @@ impl RewriteRegclassCastToSubqueryVisitor { data_type, expr: _, format: _, + .. } = expr { if *kind == CastKind::DoubleColon { @@ -496,6 +498,7 @@ impl RewriteRegclassCastToSubqueryVisitor { data_type, expr: inner_expr, format: _, + .. } = expr { if *kind == CastKind::DoubleColon { @@ -506,6 +509,7 @@ impl RewriteRegclassCastToSubqueryVisitor { data_type: inner_data_type, expr: inner_inner_expr, format: _, + .. } = inner_expr.as_ref() { if *inner_kind == CastKind::DoubleColon { From e95783996e5e6eddaefefd22e7c5210dd25e5998 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 2 Mar 2026 17:32:49 +0800 Subject: [PATCH 02/43] chore: Release --- Cargo.lock | 2 +- arrow-pg/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a911e7..1904916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,7 +379,7 @@ dependencies = [ [[package]] name = "arrow-pg" -version = "0.12.1" +version = "0.12.2" dependencies = [ "arrow", "arrow-schema", diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 88eee11..47ab581 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "arrow-pg" description = "Arrow data mapping and encoding/decoding for Postgres" -version = "0.12.1" +version = "0.12.2" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 7f013f8..881e1e1 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -13,7 +13,7 @@ readme = "../README.md" rust-version.workspace = true [dependencies] -arrow-pg = { path = "../arrow-pg", version = "0.12.1", default-features = false, features = ["datafusion"] } +arrow-pg = { path = "../arrow-pg", version = "0.12.2", default-features = false, features = ["datafusion"] } bytes.workspace = true async-trait = "0.1" chrono.workspace = true From e27f450b5e31d6356c560c5f7cfff3b9a7011804 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 2 Mar 2026 17:35:35 +0800 Subject: [PATCH 03/43] chore: Release --- Cargo.lock | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1904916..38ebf92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1598,7 +1598,7 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.15.0" +version = "0.15.1" dependencies = [ "async-trait", "criterion", diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 2419927..1feb987 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-pg-catalog" description = "pg_catalog compatibility for datafusion" -version = "0.15.0" +version = "0.15.1" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 881e1e1..3286792 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -18,7 +18,7 @@ bytes.workspace = true async-trait = "0.1" chrono.workspace = true datafusion.workspace = true -datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.15.0" } +datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.15.1" } geodatafusion = { version = "0.3", optional = true } futures.workspace = true getset = "0.1" From 8b3acd5ee6e0a13aa5b2ae40998fe4108fa0791b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Mar 2026 10:27:01 +0800 Subject: [PATCH 04/43] chore(deps): bump tokio from 1.49.0 to 1.50.0 (#302) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.49.0 to 1.50.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.49.0...tokio-1.50.0) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.50.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 13 ++++++------- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38ebf92..fdabf2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1896,7 +1896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3694,7 +3694,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3991,7 +3991,6 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4109,7 +4108,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4218,9 +4217,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -4607,7 +4606,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 47ab581..45b28bc 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -37,4 +37,4 @@ geo-traits = { version = "0.3", optional = true } [dev-dependencies] async-trait = "0.1" -tokio = { version = "1.49", features = ["full"]} +tokio = { version = "1.50", features = ["full"]} diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 1feb987..434b018 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -23,7 +23,7 @@ datafusion = { workspace = true, features = ["sql"] } futures.workspace = true log = "0.4" postgres-types.workspace = true -tokio = { version = "1.49", features = ["sync"] } +tokio = { version = "1.50", features = ["sync"] } [dev-dependencies] env_logger = "0.11" diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 3286792..d3b0b38 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -26,7 +26,7 @@ log = "0.4" pgwire = { workspace = true, features = ["server-api-ring"] } postgres-types.workspace = true rust_decimal.workspace = true -tokio = { version = "1.49", features = ["sync", "net"] } +tokio = { version = "1.50", features = ["sync", "net"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } rustls-pemfile = "2.0" rustls-pki-types = "1.14" From 47bdf5e4b7b929a85bc48b2a107e0bd7edf8262c Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Mon, 9 Mar 2026 10:07:17 +0000 Subject: [PATCH 05/43] Avoid creating query string when not needed (#303) Since the query String is only used later, move its creation lower down, so we avoid it entirely if query hooks process the query. --- datafusion-postgres/src/handlers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index 37671cf..0672d5d 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -138,8 +138,6 @@ impl SimpleQueryHandler for DfSessionService { let mut results = vec![]; 'stmt: for statement in statements { - let query = statement.to_string(); - // Call query hooks with the parsed statement for hook in &self.query_hooks { if let Some(result) = hook @@ -152,6 +150,8 @@ impl SimpleQueryHandler for DfSessionService { } let df_result = { + let query = statement.to_string(); + let timeout = client::get_statement_timeout(client); if let Some(timeout_duration) = timeout { tokio::time::timeout(timeout_duration, self.session_context.sql(&query)) From f7b01b8ed0484e21835d90386e1bb251a3679839 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Mar 2026 10:58:31 +0800 Subject: [PATCH 06/43] chore(deps): bump datafusion from 52.2.0 to 52.3.0 (#306) Bumps [datafusion](https://github.com/apache/datafusion) from 52.2.0 to 52.3.0. - [Changelog](https://github.com/apache/datafusion/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion/compare/52.2.0...52.3.0) --- updated-dependencies: - dependency-name: datafusion dependency-version: 52.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 137 +++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdabf2b..8b6dba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,9 +1078,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f" +checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c" dependencies = [ "arrow", "arrow-schema", @@ -1134,9 +1134,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399" +checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74" dependencies = [ "arrow", "async-trait", @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10" +checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c" dependencies = [ "arrow", "async-trait", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6" +checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -1207,9 +1207,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7" +checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85" dependencies = [ "futures", "log", @@ -1218,9 +1218,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129" +checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5" dependencies = [ "arrow", "async-compression", @@ -1253,9 +1253,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc" +checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f" dependencies = [ "arrow", "arrow-ipc", @@ -1277,9 +1277,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69ce35d9df5c672747f79df4b8f4967b39a3514c3af30b9a7b5426f83d4be814" +checksum = "18de2e914c2c9ed4b31a4920940b181b0957bc164eec4fc04c294533219bf0a7" dependencies = [ "apache-avro", "arrow", @@ -1297,9 +1297,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba" +checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51" dependencies = [ "arrow", "async-trait", @@ -1320,9 +1320,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e" +checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c" dependencies = [ "arrow", "async-trait", @@ -1342,9 +1342,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957" +checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32" dependencies = [ "arrow", "async-trait", @@ -1372,15 +1372,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e" +checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462" [[package]] name = "datafusion-execution" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c" +checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660" dependencies = [ "arrow", "async-trait", @@ -1399,9 +1399,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9" +checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465" dependencies = [ "arrow", "async-trait", @@ -1422,9 +1422,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0" +checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f" dependencies = [ "arrow", "datafusion-common", @@ -1435,9 +1435,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6" +checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245" dependencies = [ "arrow", "arrow-buffer", @@ -1466,9 +1466,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413" +checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787" dependencies = [ "ahash 0.8.12", "arrow", @@ -1487,9 +1487,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf" +checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae" dependencies = [ "ahash 0.8.12", "arrow", @@ -1500,9 +1500,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887" +checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320" dependencies = [ "arrow", "arrow-ord", @@ -1523,9 +1523,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae" +checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752" dependencies = [ "arrow", "async-trait", @@ -1539,9 +1539,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77" +checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8" dependencies = [ "arrow", "datafusion-common", @@ -1557,9 +1557,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729" +checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1567,9 +1567,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81" +checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2" dependencies = [ "datafusion-doc", "quote", @@ -1578,9 +1578,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb" +checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047" dependencies = [ "arrow", "chrono", @@ -1612,9 +1612,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e" +checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6" dependencies = [ "ahash 0.8.12", "arrow", @@ -1636,9 +1636,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63" +checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b" dependencies = [ "arrow", "datafusion-common", @@ -1651,9 +1651,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157" +checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce" dependencies = [ "ahash 0.8.12", "arrow", @@ -1668,9 +1668,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac" +checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944" dependencies = [ "arrow", "datafusion-common", @@ -1687,9 +1687,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261" +checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61" dependencies = [ "ahash 0.8.12", "arrow", @@ -1754,9 +1754,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd" +checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589" dependencies = [ "arrow", "datafusion-common", @@ -1771,9 +1771,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f" +checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6" dependencies = [ "async-trait", "datafusion-common", @@ -1785,9 +1785,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb" +checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87" dependencies = [ "arrow", "bigdecimal", @@ -1896,7 +1896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3694,7 +3694,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3991,6 +3991,7 @@ dependencies = [ "cfg-if", "libc", "psm", + "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4108,7 +4109,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4606,7 +4607,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From a4758ddd003e98dbd08e0c4e7a848ecc70c9bfe3 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Fri, 13 Mar 2026 08:35:05 +0000 Subject: [PATCH 07/43] Improve pg_catalog type mapping (#305) * Improve pg_catalog type mapping Instead of PgAttributeTable deciding how to map from arrow types to postgres types, delegate that decision to arrow-pg. This avoids duplicationm, brings consistency, and makes us more robust against future changes and additions to the mapping in arrow-pg. * Revert back to signed ints for iod Revert back to signed ints for iod since postgresql does not have support for unsigned ints. --- Cargo.lock | 1 + datafusion-pg-catalog/Cargo.toml | 1 + .../src/pg_catalog/pg_attribute.rs | 45 ++++++++----------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b6dba4..e05cf35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1600,6 +1600,7 @@ dependencies = [ name = "datafusion-pg-catalog" version = "0.15.1" dependencies = [ + "arrow-pg", "async-trait", "criterion", "datafusion", diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 434b018..e67a3a2 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -20,6 +20,7 @@ include = [ [dependencies] async-trait = "0.1" datafusion = { workspace = true, features = ["sql"] } +arrow-pg = { path = "../arrow-pg", version = "0.12.2", default-features = false, features = ["datafusion"] } futures.workspace = true log = "0.4" postgres-types.workspace = true diff --git a/datafusion-pg-catalog/src/pg_catalog/pg_attribute.rs b/datafusion-pg-catalog/src/pg_catalog/pg_attribute.rs index 9ffa802..57e0c7f 100644 --- a/datafusion-pg-catalog/src/pg_catalog/pg_attribute.rs +++ b/datafusion-pg-catalog/src/pg_catalog/pg_attribute.rs @@ -10,7 +10,7 @@ use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::streaming::PartitionStream; -use postgres_types::Oid; +use postgres_types::{Oid, Type}; use tokio::sync::RwLock; use crate::pg_catalog::catalog_info::CatalogInfo; @@ -141,7 +141,7 @@ impl PgAttributeTable { attrelids.push(table_oid as i32); attnames.push(field.name().clone()); - atttypids.push(pg_type_oid); + atttypids.push(pg_type_oid as i32); attstattargets.push(-1); // Default statistics target attlens.push(type_len); attnums.push(attnum); @@ -211,31 +211,22 @@ impl PgAttributeTable { } /// Map DataFusion data types to PostgreSQL type information - fn datafusion_to_pg_type(data_type: &DataType) -> (i32, i16, bool, &'static str, &'static str) { - match data_type { - DataType::Boolean => (16, 1, true, "c", "p"), // bool - DataType::Int8 => (18, 1, true, "c", "p"), // char - DataType::Int16 => (21, 2, true, "s", "p"), // int2 - DataType::Int32 => (23, 4, true, "i", "p"), // int4 - DataType::Int64 => (20, 8, true, "d", "p"), // int8 - DataType::UInt8 => (18, 2, true, "s", "p"), // char - DataType::UInt16 => (21, 4, true, "i", "p"), // int2 - DataType::UInt32 => (23, 8, true, "d", "p"), // int4 - DataType::UInt64 => (20, -1, false, "i", "m"), // int8 - DataType::Float32 => (700, 4, true, "i", "p"), // float4 - DataType::Float64 => (701, 8, true, "d", "p"), // float8 - DataType::Utf8 => (25, -1, false, "i", "x"), // text - DataType::LargeUtf8 => (25, -1, false, "i", "x"), // text - DataType::Binary => (17, -1, false, "i", "x"), // bytea - DataType::LargeBinary => (17, -1, false, "i", "x"), // bytea - DataType::Date32 => (1082, 4, true, "i", "p"), // date - DataType::Date64 => (1082, 4, true, "i", "p"), // date - DataType::Time32(_) => (1083, 8, true, "d", "p"), // time - DataType::Time64(_) => (1083, 8, true, "d", "p"), // time - DataType::Timestamp(_, _) => (1114, 8, true, "d", "p"), // timestamp - DataType::Decimal128(_, _) => (1700, -1, false, "i", "m"), // numeric - DataType::Decimal256(_, _) => (1700, -1, false, "i", "m"), // numeric - _ => (25, -1, false, "i", "x"), // Default to text for unknown types + fn datafusion_to_pg_type(data_type: &DataType) -> (u32, i16, bool, &'static str, &'static str) { + match arrow_pg::datatypes::into_pg_type(data_type) { + Ok(t @ Type::BOOL) => (t.oid(), 1, true, "c", "p"), + Ok(t @ Type::CHAR) => (t.oid(), 1, true, "c", "p"), + Ok(t @ Type::INT2) => (t.oid(), 2, true, "s", "p"), + Ok(t @ Type::INT4) => (t.oid(), 4, true, "i", "p"), + Ok(t @ Type::INT8) => (t.oid(), 8, true, "d", "p"), + Ok(t @ Type::FLOAT4) => (t.oid(), 4, true, "i", "p"), + Ok(t @ Type::FLOAT8) => (t.oid(), 8, true, "d", "p"), + Ok(t @ Type::TEXT) => (t.oid(), -1, false, "i", "x"), + Ok(t @ Type::BYTEA) => (t.oid(), -1, false, "i", "x"), + Ok(t @ Type::DATE) => (t.oid(), 4, true, "i", "p"), + Ok(t @ Type::TIME) => (t.oid(), 8, true, "d", "p"), + Ok(t @ Type::TIMESTAMP) => (t.oid(), 8, true, "d", "p"), + Ok(t @ Type::NUMERIC) => (t.oid(), -1, false, "i", "m"), + _ => (Type::TEXT.oid(), -1, false, "i", "x"), // Default to text for unknown types } } } From 179e01f702d41a79642098fa84cd708508d062ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:27:30 +0800 Subject: [PATCH 08/43] chore(deps): bump pgwire from 0.38.0 to 0.38.1 (#307) Bumps [pgwire](https://github.com/sunng87/pgwire) from 0.38.0 to 0.38.1. - [Release notes](https://github.com/sunng87/pgwire/releases) - [Changelog](https://github.com/sunng87/pgwire/blob/master/CHANGELOG.md) - [Commits](https://github.com/sunng87/pgwire/compare/v0.38.0...v0.38.1) --- updated-dependencies: - dependency-name: pgwire dependency-version: 0.38.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e05cf35..d111964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1897,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3131,9 +3131,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.38.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d5e5a60d3f6e40c91f6a2a7f8d09665e636272bd5611977253559b6651aabb" +checksum = "f2a798d130b8975a566c2cf6d8955746e1f09a9ee2c3ff2e6020a2c6528c5bd1" dependencies = [ "async-trait", "base64", @@ -3695,7 +3695,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3992,7 +3992,6 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4110,7 +4109,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4608,7 +4607,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From 23615fc0c9b4d989c89f8b68fec4e35efefc39d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:24:54 +0800 Subject: [PATCH 09/43] chore(deps): bump datafusion from 52.3.0 to 52.4.0 (#308) Bumps [datafusion](https://github.com/apache/datafusion) from 52.3.0 to 52.4.0. - [Changelog](https://github.com/apache/datafusion/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion/compare/52.3.0...52.4.0) --- updated-dependencies: - dependency-name: datafusion dependency-version: 52.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 137 +++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d111964..8b9d569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,9 +1078,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c" +checksum = "43c18ba387f9c05ac1f3be32a73f8f3cc6c1cfc43e5d4b7a8e5b0d3a5eb48dc7" dependencies = [ "arrow", "arrow-schema", @@ -1134,9 +1134,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74" +checksum = "3c75a4ce672b27fb8423810efb92a3600027717a1664d06a2c307eeeabcec694" dependencies = [ "arrow", "async-trait", @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c" +checksum = "2c8b9a3795ffb46bf4957a34c67d89a67558b311ae455c8d4295ff2115eeea50" dependencies = [ "arrow", "async-trait", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd" +checksum = "205dc1e20441973f470e6b7ef87626a3b9187970e5106058fef1b713047f770c" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -1207,9 +1207,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85" +checksum = "8cf5880c02ff6f5f11fb5bc19211789fb32fd3c53d79b7d6cb2b12e401312ba0" dependencies = [ "futures", "log", @@ -1218,9 +1218,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5" +checksum = "bc614d6e709450e29b7b032a42c1bdb705f166a6b2edef7bed7c7897eb905499" dependencies = [ "arrow", "async-compression", @@ -1253,9 +1253,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f" +checksum = "6e497d5fc48dac7ce86f6b4fb09a3a494385774af301ff20ec91aebfae9b05b4" dependencies = [ "arrow", "arrow-ipc", @@ -1277,9 +1277,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18de2e914c2c9ed4b31a4920940b181b0957bc164eec4fc04c294533219bf0a7" +checksum = "474d9b26f185b57f549a0f7ce9183428dd0042014a2e0d093f5430fdc9dae289" dependencies = [ "apache-avro", "arrow", @@ -1297,9 +1297,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51" +checksum = "0dfc250cad940d0327ca2e9109dc98830892d17a3d6b2ca11d68570e872cf379" dependencies = [ "arrow", "async-trait", @@ -1320,9 +1320,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c" +checksum = "c91e9677ed62833b0e8129dec0d1a8f3c9bb7590bd6dd714a43e4c3b663e4aa0" dependencies = [ "arrow", "async-trait", @@ -1342,9 +1342,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32" +checksum = "23798383465e0c569bd442d1453b50691261f8ad6511d840c48457b3bf51ae21" dependencies = [ "arrow", "async-trait", @@ -1372,15 +1372,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462" +checksum = "3e13e5fe3447baa0584b61ee8644086e007e1ef6e58f4be48bc8a72417854729" [[package]] name = "datafusion-execution" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660" +checksum = "48a6cc03e34899a54546b229235f7b192634c8e832f78a267f0989b18216c56d" dependencies = [ "arrow", "async-trait", @@ -1399,9 +1399,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465" +checksum = "ee3315d87eca7a7df58e52a1fb43b4c4171b545fd30ffc3102945c162a9f6ddb" dependencies = [ "arrow", "async-trait", @@ -1422,9 +1422,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f" +checksum = "98c6d83feae0753799f933a2c47dfd15980c6947960cb95ed60f5c1f885548b3" dependencies = [ "arrow", "datafusion-common", @@ -1435,9 +1435,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245" +checksum = "49b82962015cc3db4d7662459c9f7fcda0591b5edacb8af1cf3bc3031f274800" dependencies = [ "arrow", "arrow-buffer", @@ -1466,9 +1466,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787" +checksum = "4e42c227d9e55a6c8041785d4a8a117e4de531033d480aae10984247ac62e27e" dependencies = [ "ahash 0.8.12", "arrow", @@ -1487,9 +1487,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae" +checksum = "cead3cfed825b0b688700f4338d281cd7857e4907775a5b9554c083edd5f3f95" dependencies = [ "ahash 0.8.12", "arrow", @@ -1500,9 +1500,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320" +checksum = "62ea99612970aebab8cf864d02eb3d296bbab7f4881e1023d282b57fe431b201" dependencies = [ "arrow", "arrow-ord", @@ -1523,9 +1523,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752" +checksum = "d83dbf3ab8b9af6f209b068825a7adbd3b88bf276f2a1ec14ba09567b97f5674" dependencies = [ "arrow", "async-trait", @@ -1539,9 +1539,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8" +checksum = "732edabe07496e2fc5a1e57a284d7a36edcea445a2821119770a0dea624b472c" dependencies = [ "arrow", "datafusion-common", @@ -1557,9 +1557,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b" +checksum = "e0c6e30e09700799bd52adce8c377ab03dda96e73a623e4803a31ad94fe7ce14" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1567,9 +1567,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2" +checksum = "402f2a8ed70fb99a18f71580a1fe338604222a3d32ddeac6e72c5b34feea2d4d" dependencies = [ "datafusion-doc", "quote", @@ -1578,9 +1578,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047" +checksum = "99f32edb8ba12f08138f86c09b80fae3d4a320551262fa06b91d8a8cb3065a5b" dependencies = [ "arrow", "chrono", @@ -1613,9 +1613,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6" +checksum = "987c5e29e96186589301b42e25aa7d11bbe319a73eb02ef8d755edc55b5b89fc" dependencies = [ "ahash 0.8.12", "arrow", @@ -1637,9 +1637,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b" +checksum = "1de89d0afa08b6686697bd8a6bac4ba2cd44c7003356e1bce6114d5a93f94b5c" dependencies = [ "arrow", "datafusion-common", @@ -1652,9 +1652,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce" +checksum = "602d1970c0fe87f1c3a36665d131fbfe1c4379d35f8fc5ec43a362229ad2954d" dependencies = [ "ahash 0.8.12", "arrow", @@ -1669,9 +1669,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944" +checksum = "b24d704b6385ebe27c756a12e5ba15684576d3b47aeca79cc9fb09480236dc32" dependencies = [ "arrow", "datafusion-common", @@ -1688,9 +1688,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61" +checksum = "c21d94141ea5043e98793f170798e9c1887095813b8291c5260599341e383a38" dependencies = [ "ahash 0.8.12", "arrow", @@ -1755,9 +1755,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589" +checksum = "1a68cce43d18c0dfac95cacd74e70565f7e2fb12b9ed41e2d312f0fa837626b1" dependencies = [ "arrow", "datafusion-common", @@ -1772,9 +1772,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6" +checksum = "6b4e1c40a0b1896aed4a4504145c2eb7fa9b9da13c2d04b40a4767a09f076199" dependencies = [ "async-trait", "datafusion-common", @@ -1786,9 +1786,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.3.0" +version = "52.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87" +checksum = "2f1891e5b106d1d73c7fe403bd8a265d19c3977edc17f60808daf26c2fe65ffb" dependencies = [ "arrow", "bigdecimal", @@ -1897,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3695,7 +3695,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3992,6 +3992,7 @@ dependencies = [ "cfg-if", "libc", "psm", + "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4109,7 +4110,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4607,7 +4608,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] From fbca72d4ea017dd6a737672ee98642561099160f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:58:10 +0800 Subject: [PATCH 10/43] chore(deps): bump env_logger from 0.11.9 to 0.11.10 (#310) Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.11.9 to 0.11.10. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.11.9...v0.11.10) --- updated-dependencies: - dependency-name: env_logger dependency-version: 0.11.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b9d569..8a20304 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,9 +89,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -110,9 +110,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -1873,9 +1873,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -1897,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2633,9 +2633,9 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "log", @@ -2646,9 +2646,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -3695,7 +3695,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3992,7 +3992,6 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4110,7 +4109,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4608,7 +4607,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From 83d9fdfd159ffa060bf3c6644dbe8267d160054a Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Tue, 24 Mar 2026 17:52:41 +0800 Subject: [PATCH 11/43] fix: use int2 for arrow 8-bit int (#311) --- arrow-pg/src/datatypes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arrow-pg/src/datatypes.rs b/arrow-pg/src/datatypes.rs index 4dce871..8898313 100644 --- a/arrow-pg/src/datatypes.rs +++ b/arrow-pg/src/datatypes.rs @@ -24,7 +24,7 @@ pub fn into_pg_type(arrow_type: &DataType) -> PgWireResult { let datatype = match arrow_type { DataType::Null => Type::UNKNOWN, DataType::Boolean => Type::BOOL, - DataType::Int8 => Type::CHAR, + DataType::Int8 => Type::INT2, DataType::Int16 | DataType::UInt8 => Type::INT2, DataType::Int32 | DataType::UInt16 => Type::INT4, DataType::Int64 | DataType::UInt32 => Type::INT8, @@ -53,7 +53,7 @@ pub fn into_pg_type(arrow_type: &DataType) -> PgWireResult { | DataType::ListView(field) | DataType::LargeListView(field) => match field.data_type() { DataType::Boolean => Type::BOOL_ARRAY, - DataType::Int8 => Type::CHAR_ARRAY, + DataType::Int8 => Type::INT2_ARRAY, DataType::Int16 | DataType::UInt8 => Type::INT2_ARRAY, DataType::Int32 | DataType::UInt16 => Type::INT4_ARRAY, DataType::Int64 | DataType::UInt32 => Type::INT8_ARRAY, From 58ca2973d429abf7d9809ef2c7322440068b643e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 10:25:01 +0800 Subject: [PATCH 12/43] chore(deps): bump pgwire from 0.38.1 to 0.38.2 (#314) Bumps [pgwire](https://github.com/sunng87/pgwire) from 0.38.1 to 0.38.2. - [Release notes](https://github.com/sunng87/pgwire/releases) - [Changelog](https://github.com/sunng87/pgwire/blob/master/CHANGELOG.md) - [Commits](https://github.com/sunng87/pgwire/compare/v0.38.1...v0.38.2) --- updated-dependencies: - dependency-name: pgwire dependency-version: 0.38.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a20304..af4ba21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1897,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3131,9 +3131,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.38.1" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a798d130b8975a566c2cf6d8955746e1f09a9ee2c3ff2e6020a2c6528c5bd1" +checksum = "3a1bdf05fc8231cc5024572fe056e3ce34eb6b9b755ba7aba110e1c64119cec3" dependencies = [ "async-trait", "base64", @@ -3695,7 +3695,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4109,7 +4109,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4607,7 +4607,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 9006d820c0efbf2399ed587d5812fbc2a14e6cf7 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sun, 29 Mar 2026 21:10:10 +0800 Subject: [PATCH 13/43] chore: update readme --- README.md | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8ea09d6..d24e32e 100644 --- a/README.md +++ b/README.md @@ -74,21 +74,11 @@ let server_options = ServerOptions::new() serve(session_context, &server_options).await ``` -### Security Features - -The server automatically includes: - -- User authentication (default postgres superuser) -- Role-based access control with predefined roles: - - readonly: SELECT permissions - - readwrite: SELECT, INSERT, UPDATE, DELETE permissions - - dbadmin: Full administrative permissions -- SSL/TLS encryption when certificates are provided -- Query-level permission checking - ### The CLI `datafusion-postgres-cli` -Command-line tool to serve JSON/CSV/Arrow/Parquet/Avro files as PostgreSQL-compatible tables. +Command-line tool to serve JSON/CSV/Arrow/Parquet/Avro files as +PostgreSQL-compatible tables. This is like a `SimpleHTTPServer` for hosting data +files, but with Postgres protocol and datafusion query engine. ``` datafusion-postgres-cli 0.6.1 @@ -114,7 +104,7 @@ OPTIONS: --tls-key Path to TLS private key file for SSL/TLS encryption ``` -#### 🔒 Security Options +#### Security Options ```bash # Run with SSL/TLS encryption @@ -127,7 +117,7 @@ datafusion-postgres-cli \ datafusion-postgres-cli --csv data:sample.csv ``` -## 📋 Example Usage +## Example Usage ### Basic Example @@ -145,8 +135,6 @@ Listening on 127.0.0.1:5432 (unencrypted) ### Connect with psql -> **🔐 Authentication**: The default setup allows connections without authentication for development. For secure deployments, use `DfAuthSource` with standard pgwire authentication handlers (cleartext, MD5, or SCRAM). See `auth.rs` for implementation examples. - ```bash psql -h 127.0.0.1 -p 5432 -U postgres ``` @@ -179,7 +167,7 @@ postgres=> COMMIT; COMMIT ``` -### 🔐 Production Setup with SSL/TLS +### SSL/TLS ```bash # Generate SSL certificates @@ -199,6 +187,13 @@ TLS enabled using cert: server.crt and key: server.key Listening on 127.0.0.1:5432 with TLS encryption ``` +## PostGIS/Geodatafusion + +With [geodatafusion](https://github.com/datafusion-contrib/geodatafusion), we +can also simulate PostGIS interface (UDF and datatypes) with +datafusion-postgres. To enable this feature, turn on the feature flag `postgis` +for `datafusion-postgres`. + ## Community ### Developer Mailing List From 00b4f4878273ceb7c4480d302714c99c60c4df02 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:23:53 +0800 Subject: [PATCH 14/43] chore(deps): bump rust_decimal from 1.40.0 to 1.41.0 (#316) Bumps [rust_decimal](https://github.com/paupino/rust-decimal) from 1.40.0 to 1.41.0. - [Release notes](https://github.com/paupino/rust-decimal/releases) - [Changelog](https://github.com/paupino/rust-decimal/blob/master/CHANGELOG.md) - [Commits](https://github.com/paupino/rust-decimal/compare/1.40.0...1.41.0) --- updated-dependencies: - dependency-name: rust_decimal dependency-version: 1.41.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 14 ++++++++------ Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af4ba21..1d178d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1897,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3661,9 +3661,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" dependencies = [ "arrayvec", "borsh", @@ -3674,6 +3674,7 @@ dependencies = [ "rkyv", "serde", "serde_json", + "wasm-bindgen", ] [[package]] @@ -3695,7 +3696,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4109,7 +4110,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4482,6 +4483,7 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -4607,7 +4609,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 896d61a..ae2875b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ datafusion = { version = "52" } futures = "0.3" pgwire = { version = "0.38", default-features = false } postgres-types = "0.2" -rust_decimal = { version = "1.40", features = ["db-postgres"] } +rust_decimal = { version = "1.41", features = ["db-postgres"] } tokio = { version = "1", default-features = false } [profile.release] From a301e5e1bda6bde4fa48eb8a87a1d263b7f1d2b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 10:17:49 +0800 Subject: [PATCH 15/43] chore(deps): bump postgres-types from 0.2.12 to 0.2.13 (#319) Bumps [postgres-types](https://github.com/rust-postgres/rust-postgres) from 0.2.12 to 0.2.13. - [Release notes](https://github.com/rust-postgres/rust-postgres/releases) - [Commits](https://github.com/rust-postgres/rust-postgres/compare/postgres-types-v0.2.12...postgres-types-v0.2.13) --- updated-dependencies: - dependency-name: postgres-types dependency-version: 0.2.13 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d178d5..f84505c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3267,9 +3267,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" +checksum = "8dc729a129e682e8d24170cd30ae1aa01b336b096cbb56df6d534ffec133d186" dependencies = [ "array-init", "bytes", From 764c14776127259dea82363d1b2f632c828e44c3 Mon Sep 17 00:00:00 2001 From: Martin Garton Date: Wed, 1 Apr 2026 03:11:37 +0100 Subject: [PATCH 16/43] Upgrade to datafusion 53 (#321) * Upgrade to datafusion 53 Upgrade to datafusion 53 and upgrade some other dependencies to match the versions used by datafusion 53 (arrow libraries, etc) Also make a few minor changes to fix new deprecation warnings. * Fix tests --- Cargo.lock | 377 +++++++++++++------------ Cargo.toml | 6 +- arrow-pg/Cargo.toml | 4 +- datafusion-pg-catalog/src/sql/rules.rs | 36 +-- datafusion-postgres-cli/src/main.rs | 4 +- datafusion-postgres/Cargo.toml | 2 +- 6 files changed, 222 insertions(+), 207 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f84505c..dad9b5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,9 +210,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ "arrow-arith", "arrow-array", @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ "arrow-array", "arrow-buffer", @@ -298,9 +298,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ "arrow-array", "arrow-cast", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ "arrow-buffer", "arrow-schema", @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,9 +342,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,9 +366,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ "arrow-array", "arrow-buffer", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "serde_core", "serde_json", @@ -425,9 +425,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ "arrow-array", "arrow-buffer", @@ -456,9 +456,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.37" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" dependencies = [ "compression-codecs", "compression-core", @@ -474,7 +474,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -605,7 +605,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -628,7 +628,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -846,9 +846,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.36" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "bzip2", "compression-core", @@ -1048,7 +1048,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1059,7 +1059,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1078,9 +1078,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c18ba387f9c05ac1f3be32a73f8f3cc6c1cfc43e5d4b7a8e5b0d3a5eb48dc7" +checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" dependencies = [ "arrow", "arrow-schema", @@ -1134,9 +1134,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c75a4ce672b27fb8423810efb92a3600027717a1664d06a2c307eeeabcec694" +checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" dependencies = [ "arrow", "async-trait", @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8b9a3795ffb46bf4957a34c67d89a67558b311ae455c8d4295ff2115eeea50" +checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" dependencies = [ "arrow", "async-trait", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205dc1e20441973f470e6b7ef87626a3b9187970e5106058fef1b713047f770c" +checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -1194,6 +1194,7 @@ dependencies = [ "half", "hashbrown 0.16.1", "indexmap", + "itertools 0.14.0", "libc", "log", "object_store", @@ -1207,9 +1208,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf5880c02ff6f5f11fb5bc19211789fb32fd3c53d79b7d6cb2b12e401312ba0" +checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" dependencies = [ "futures", "log", @@ -1218,9 +1219,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc614d6e709450e29b7b032a42c1bdb705f166a6b2edef7bed7c7897eb905499" +checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" dependencies = [ "arrow", "async-compression", @@ -1253,9 +1254,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e497d5fc48dac7ce86f6b4fb09a3a494385774af301ff20ec91aebfae9b05b4" +checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" dependencies = [ "arrow", "arrow-ipc", @@ -1277,9 +1278,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474d9b26f185b57f549a0f7ce9183428dd0042014a2e0d093f5430fdc9dae289" +checksum = "49dda81c79b6ba57b1853a9158abc66eb85a3aa1cede0c517dabec6d8a4ed3aa" dependencies = [ "apache-avro", "arrow", @@ -1297,9 +1298,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dfc250cad940d0327ca2e9109dc98830892d17a3d6b2ca11d68570e872cf379" +checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" dependencies = [ "arrow", "async-trait", @@ -1320,9 +1321,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91e9677ed62833b0e8129dec0d1a8f3c9bb7590bd6dd714a43e4c3b663e4aa0" +checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" dependencies = [ "arrow", "async-trait", @@ -1337,14 +1338,16 @@ dependencies = [ "datafusion-session", "futures", "object_store", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23798383465e0c569bd442d1453b50691261f8ad6511d840c48457b3bf51ae21" +checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" dependencies = [ "arrow", "async-trait", @@ -1372,22 +1375,24 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e13e5fe3447baa0584b61ee8644086e007e1ef6e58f4be48bc8a72417854729" +checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" [[package]] name = "datafusion-execution" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48a6cc03e34899a54546b229235f7b192634c8e832f78a267f0989b18216c56d" +checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" dependencies = [ "arrow", + "arrow-buffer", "async-trait", "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1399,9 +1404,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3315d87eca7a7df58e52a1fb43b4c4171b545fd30ffc3102945c162a9f6ddb" +checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" dependencies = [ "arrow", "async-trait", @@ -1422,9 +1427,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c6d83feae0753799f933a2c47dfd15980c6947960cb95ed60f5c1f885548b3" +checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" dependencies = [ "arrow", "datafusion-common", @@ -1435,9 +1440,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b82962015cc3db4d7662459c9f7fcda0591b5edacb8af1cf3bc3031f274800" +checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" dependencies = [ "arrow", "arrow-buffer", @@ -1456,6 +1461,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.2", "regex", @@ -1466,9 +1472,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e42c227d9e55a6c8041785d4a8a117e4de531033d480aae10984247ac62e27e" +checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" dependencies = [ "ahash 0.8.12", "arrow", @@ -1482,14 +1488,15 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cead3cfed825b0b688700f4338d281cd7857e4907775a5b9554c083edd5f3f95" +checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" dependencies = [ "ahash 0.8.12", "arrow", @@ -1500,9 +1507,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ea99612970aebab8cf864d02eb3d296bbab7f4881e1023d282b57fe431b201" +checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" dependencies = [ "arrow", "arrow-ord", @@ -1516,16 +1523,18 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", + "itoa", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83dbf3ab8b9af6f209b068825a7adbd3b88bf276f2a1ec14ba09567b97f5674" +checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" dependencies = [ "arrow", "async-trait", @@ -1539,9 +1548,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732edabe07496e2fc5a1e57a284d7a36edcea445a2821119770a0dea624b472c" +checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" dependencies = [ "arrow", "datafusion-common", @@ -1557,9 +1566,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c6e30e09700799bd52adce8c377ab03dda96e73a623e4803a31ad94fe7ce14" +checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1567,20 +1576,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f2a8ed70fb99a18f71580a1fe338604222a3d32ddeac6e72c5b34feea2d4d" +checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "datafusion-optimizer" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99f32edb8ba12f08138f86c09b80fae3d4a320551262fa06b91d8a8cb3065a5b" +checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" dependencies = [ "arrow", "chrono", @@ -1613,9 +1622,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "987c5e29e96186589301b42e25aa7d11bbe319a73eb02ef8d755edc55b5b89fc" +checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" dependencies = [ "ahash 0.8.12", "arrow", @@ -1637,9 +1646,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de89d0afa08b6686697bd8a6bac4ba2cd44c7003356e1bce6114d5a93f94b5c" +checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" dependencies = [ "arrow", "datafusion-common", @@ -1652,9 +1661,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602d1970c0fe87f1c3a36665d131fbfe1c4379d35f8fc5ec43a362229ad2954d" +checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" dependencies = [ "ahash 0.8.12", "arrow", @@ -1669,9 +1678,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b24d704b6385ebe27c756a12e5ba15684576d3b47aeca79cc9fb09480236dc32" +checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" dependencies = [ "arrow", "datafusion-common", @@ -1688,9 +1697,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21d94141ea5043e98793f170798e9c1887095813b8291c5260599341e383a38" +checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" dependencies = [ "ahash 0.8.12", "arrow", @@ -1712,6 +1721,7 @@ dependencies = [ "indexmap", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -1755,9 +1765,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a68cce43d18c0dfac95cacd74e70565f7e2fb12b9ed41e2d312f0fa837626b1" +checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" dependencies = [ "arrow", "datafusion-common", @@ -1772,9 +1782,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4e1c40a0b1896aed4a4504145c2eb7fa9b9da13c2d04b40a4767a09f076199" +checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" dependencies = [ "async-trait", "datafusion-common", @@ -1786,15 +1796,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.4.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f1891e5b106d1d73c7fe403bd8a265d19c3977edc17f60808daf26c2fe65ffb" +checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-functions-nested", "indexmap", "log", "recursive", @@ -1820,7 +1831,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1842,7 +1853,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1936,13 +1947,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -2040,7 +2051,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2134,9 +2145,9 @@ dependencies = [ [[package]] name = "geoarrow" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd02b95a09c124fdf0c5e4480e4cf0a508d31a88837d89a4107e24efb1a6f3f8" +checksum = "ec42ac7fb4fdcd6982dab92d24faf436f18c36e47c3f813a33619a2728718a30" dependencies = [ "geoarrow-array", "geoarrow-schema", @@ -2144,9 +2155,9 @@ dependencies = [ [[package]] name = "geoarrow-array" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1cc4106ac0a0a512c398961ce95d8150475c84a84e17c4511c3643fa120a17" +checksum = "dafe7b7de3fab1a8b7099fd6a6434ca955fa65065f9c19f0f8a133693f3c2b0e" dependencies = [ "arrow-array", "arrow-buffer", @@ -2160,9 +2171,9 @@ dependencies = [ [[package]] name = "geoarrow-expr-geo" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84300361ce57fb875bcaa6e32b95b0aff5c6b1af692b936bdd58ff343f4394" +checksum = "8e4a62ac19c86827c6ec81ea584594b3ee96db5a8119b9774d3466c6b373c434" dependencies = [ "arrow-array", "arrow-buffer", @@ -2174,9 +2185,9 @@ dependencies = [ [[package]] name = "geoarrow-schema" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97be4e9f523f92bd6a0e0458323f4b783d073d011664decd8dbf05651704f34" +checksum = "4d4a7edb2a1d87024a93805332a9c8184a0354836271d42c0d18cf628a5e3cd0" dependencies = [ "arrow-schema", "geo-traits", @@ -2187,9 +2198,9 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cb8faa9b3bf4ae9f49b1f023b82d20626826f6448a7055498376146c10c4ead" +checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" dependencies = [ "arrow-arith", "arrow-array", @@ -2270,7 +2281,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2652,7 +2663,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2695,7 +2706,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2775,15 +2786,15 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.179" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "liblzma" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" dependencies = [ "liblzma-sys", ] @@ -2805,15 +2816,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" -[[package]] -name = "libz-rs-sys" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" -dependencies = [ - "zlib-rs", -] - [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -2843,9 +2845,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] @@ -2868,9 +2870,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miniz_oxide" @@ -2951,7 +2953,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2965,14 +2967,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "bytes", "chrono", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http", "humantime", "itertools 0.14.0", @@ -3049,14 +3053,13 @@ dependencies = [ [[package]] name = "parquet" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" +checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" dependencies = [ "ahash 0.8.12", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -3306,7 +3309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3361,7 +3364,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3411,9 +3414,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quote" -version = "1.0.43" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3543,7 +3546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3586,9 +3589,9 @@ checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "rend" @@ -3825,7 +3828,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -3957,9 +3960,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "recursive", @@ -3968,13 +3971,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4052,7 +4055,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4074,9 +4077,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.114" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -4091,7 +4094,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4148,7 +4151,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4159,7 +4162,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4242,7 +4245,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4255,6 +4258,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -4317,7 +4332,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4424,11 +4439,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.19.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.1", "js-sys", "serde_core", "wasm-bindgen", @@ -4520,7 +4535,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4639,7 +4654,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4650,7 +4665,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4887,7 +4902,7 @@ dependencies = [ "heck 0.5.0", "indexmap", "prettyplease", - "syn 2.0.114", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4903,7 +4918,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5023,7 +5038,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure", ] @@ -5044,7 +5059,7 @@ checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -5064,7 +5079,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure", ] @@ -5085,7 +5100,7 @@ checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -5118,14 +5133,14 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index ae2875b..b29bcf4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ repository = "https://github.com/datafusion-contrib/datafusion-postgres/" documentation = "https://docs.rs/crate/datafusion-postgres/" [workspace.dependencies] -arrow = "57" -arrow-schema = "57" +arrow = "58" +arrow-schema = "58" bytes = "1.11.1" chrono = { version = "0.4", features = ["std"] } -datafusion = { version = "52" } +datafusion = { version = "53" } futures = "0.3" pgwire = { version = "0.38", default-features = false } postgres-types = "0.2" diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 45b28bc..5cacde9 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -25,8 +25,8 @@ bytes.workspace = true chrono.workspace = true datafusion = { workspace = true, optional = true } futures.workspace = true -geoarrow = { version = "0.7", optional = true } -geoarrow-schema = { version = "0.7", optional = true } +geoarrow = { version = "0.8", optional = true } +geoarrow-schema = { version = "0.8", optional = true } pg_interval = { version = "0.5.1", package = "pg_interval_2" } pgwire = { workspace = true, default-features = false, features = ["server-api", "pg-ext-types"] } postgres-types.workspace = true diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index bc80aba..5848731 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -1160,7 +1160,7 @@ mod tests { assert_rewrite!( &rules, "SELECT n.oid, n.* FROM pg_catalog.pg_namespace n", - "SELECT n.oid AS __alias_oid, n.* FROM pg_catalog.pg_namespace AS n" + "SELECT n.oid AS __alias_oid, n.* FROM pg_catalog.pg_namespace n" ); assert_rewrite!( @@ -1181,7 +1181,7 @@ mod tests { let statement = rewrite(statement, &rules); assert_eq!( statement.to_string(), - "SELECT n.oid AS __alias_oid, n.*, d.description FROM pg_catalog.pg_namespace AS n LEFT OUTER JOIN pg_catalog.pg_description AS d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY nspsname" + "SELECT n.oid AS __alias_oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY nspsname" ); } @@ -1193,7 +1193,7 @@ mod tests { assert_rewrite!( &rules, "SELECT n.* FROM pg_catalog.pg_namespace n WHERE nspname = 'pg_catalog' ORDER BY nspname", - "SELECT n.* FROM pg_catalog.pg_namespace AS n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" + "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" ); assert_rewrite!( @@ -1205,12 +1205,12 @@ mod tests { assert_rewrite!( &rules, "SELECT n.oid,n.*,d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid=n.oid AND d.objsubid=0 AND d.classoid='pg_namespace' ORDER BY nspsname", - "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace AS n LEFT OUTER JOIN pg_catalog.pg_description AS d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY n.nspsname" + "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY n.nspsname" ); assert_rewrite!(&rules, "SELECT i.*,i.indkey as keys,c.relname,c.relnamespace,c.relam,c.reltablespace,tc.relname as tabrelname,dsc.description FROM pg_catalog.pg_index i INNER JOIN pg_catalog.pg_class c ON c.oid=i.indexrelid INNER JOIN pg_catalog.pg_class tc ON tc.oid=i.indrelid LEFT OUTER JOIN pg_catalog.pg_description dsc ON i.indexrelid=dsc.objoid WHERE i.indrelid=1 ORDER BY tabrelname, c.relname", - "SELECT i.*, i.indkey AS keys, c.relname, c.relnamespace, c.relam, c.reltablespace, tc.relname AS tabrelname, dsc.description FROM pg_catalog.pg_index AS i INNER JOIN pg_catalog.pg_class AS c ON c.oid = i.indexrelid INNER JOIN pg_catalog.pg_class AS tc ON tc.oid = i.indrelid LEFT OUTER JOIN pg_catalog.pg_description AS dsc ON i.indexrelid = dsc.objoid WHERE i.indrelid = 1 ORDER BY tabrelname, c.relname" + "SELECT i.*, i.indkey AS keys, c.relname, c.relnamespace, c.relam, c.reltablespace, tc.relname AS tabrelname, dsc.description FROM pg_catalog.pg_index i INNER JOIN pg_catalog.pg_class c ON c.oid = i.indexrelid INNER JOIN pg_catalog.pg_class tc ON tc.oid = i.indrelid LEFT OUTER JOIN pg_catalog.pg_description dsc ON i.indexrelid = dsc.objoid WHERE i.indrelid = 1 ORDER BY tabrelname, c.relname" ); } @@ -1224,25 +1224,25 @@ mod tests { assert_rewrite!( &rules, "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog'::regclass ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace AS n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" + "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" ); assert_rewrite!( &rules, "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.oid = 1 AND n.nspname = 'pg_catalog'::regclass ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace AS n WHERE n.oid = 1 AND n.nspname = 'pg_catalog' ORDER BY n.nspname" + "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.oid = 1 AND n.nspname = 'pg_catalog' ORDER BY n.nspname" ); assert_rewrite!( &rules, - "SELECT n.oid,n.*,d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid=n.oid AND d.objsubid=0 AND d.classoid='pg_namespace'::regclass ORDER BY nspname", - "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace AS n LEFT OUTER JOIN pg_catalog.pg_description AS d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY nspname" + "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace'::regclass ORDER BY nspname", + "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY nspname" ); assert_rewrite!( &rules, "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname", - "SELECT n.* FROM pg_catalog.pg_namespace AS n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" + "SELECT n.* FROM pg_catalog.pg_namespace n WHERE n.nspname = 'pg_catalog' ORDER BY n.nspname" ); assert_rewrite!( @@ -1252,7 +1252,7 @@ mod tests { LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) WHERE c.oid = '16386'", - "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::TEXT END, c.relpersistence, c.relreplident, am.amname FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_class AS tc ON (c.reltoastrelid = tc.oid) LEFT JOIN pg_catalog.pg_am AS am ON (c.relam = am.oid) WHERE c.oid = '16386'" + "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, false AS relhasoids, c.relispartition, '', c.reltablespace, CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::TEXT END, c.relpersistence, c.relreplident, am.amname FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid) LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid) WHERE c.oid = '16386'" ); } @@ -1264,25 +1264,25 @@ mod tests { assert_rewrite!( &rules, "SELECT $1::regclass::oid", - "SELECT (SELECT c.oid FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) AS p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" + "SELECT (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" ); assert_rewrite!( &rules, "SELECT $1::pg_catalog.regclass::oid", - "SELECT (SELECT c.oid FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) AS p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" + "SELECT (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" ); assert_rewrite!( &rules, "SELECT $1::pg_catalog.regclass::pg_catalog.oid", - "SELECT (SELECT c.oid FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) AS p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" + "SELECT (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" ); assert_rewrite!( &rules, "SELECT * FROM pg_catalog.pg_class WHERE oid = 't'::pg_catalog.regclass::pg_catalog.oid", - "SELECT * FROM pg_catalog.pg_class WHERE oid = (SELECT c.oid FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident('t'::TEXT) AS parts) AS p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" + "SELECT * FROM pg_catalog.pg_class WHERE oid = (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace CROSS JOIN (SELECT parse_ident('t'::TEXT) AS parts) p WHERE n.nspname = COALESCE(CASE WHEN array_length(p.parts, 1) > 1 THEN p.parts[1] END, current_schema()) AND c.relname = p.parts[-1])" ); } @@ -1394,7 +1394,7 @@ mod tests { fn test_collate_fix() { let rules: Vec> = vec![Arc::new(FixCollate)]; - assert_rewrite!(&rules, "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname OPERATOR(pg_catalog.~) '^(tablename)$' COLLATE pg_catalog.default AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3;", "SELECT c.oid, c.relname FROM pg_catalog.pg_class AS c WHERE c.relname ~ '^(tablename)$' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3"); + assert_rewrite!(&rules, "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname OPERATOR(pg_catalog.~) '^(tablename)$' COLLATE pg_catalog.default AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3;", "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname ~ '^(tablename)$' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3"); } #[test] @@ -1404,7 +1404,7 @@ mod tests { assert_rewrite!(&rules, "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) FROM pg_catalog.pg_attrdef d WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef), a.attnotnull, (SELECT c.collname FROM pg_catalog.pg_collation c, pg_catalog.pg_type t WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation LIMIT 1) AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum;", - "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NULL, a.attnotnull, NULL AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute AS a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum"); + "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NULL, a.attnotnull, NULL AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum"); } #[test] @@ -1425,7 +1425,7 @@ mod tests { assert_rewrite!(&rules, "SELECT a.attname, (SELECT COUNT(*) FROM pg_catalog.pg_attribute WHERE attrelid = a.oid) AS count FROM pg_catalog.pg_attribute a", - "SELECT a.attname, NULL AS count FROM pg_catalog.pg_attribute AS a" + "SELECT a.attname, NULL AS count FROM pg_catalog.pg_attribute a" ); } diff --git a/datafusion-postgres-cli/src/main.rs b/datafusion-postgres-cli/src/main.rs index 7b0f296..080201a 100644 --- a/datafusion-postgres-cli/src/main.rs +++ b/datafusion-postgres-cli/src/main.rs @@ -3,7 +3,7 @@ use std::fs; use std::sync::Arc; use datafusion::execution::options::{ - ArrowReadOptions, AvroReadOptions, CsvReadOptions, NdJsonReadOptions, ParquetReadOptions, + ArrowReadOptions, AvroReadOptions, CsvReadOptions, JsonReadOptions, ParquetReadOptions, }; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_postgres::auth::AuthManager; @@ -139,7 +139,7 @@ async fn setup_session_context( // Register JSON tables for (table_name, table_path) in opts.json_tables.iter().map(|s| parse_table_def(s.as_ref())) { session_context - .register_json(table_name, table_path, NdJsonReadOptions::default()) + .register_json(table_name, table_path, JsonReadOptions::default()) .await .map_err(|e| format!("Failed to register JSON table '{table_name}': {e}"))?; info!("Loaded {table_path} as table {table_name}"); diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index d3b0b38..754a487 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -19,7 +19,7 @@ async-trait = "0.1" chrono.workspace = true datafusion.workspace = true datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.15.1" } -geodatafusion = { version = "0.3", optional = true } +geodatafusion = { version = "0.4", optional = true } futures.workspace = true getset = "0.1" log = "0.4" From a958f1f7038aa9adbc11c92fb9d0541b0c19dce8 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Wed, 1 Apr 2026 10:17:23 +0800 Subject: [PATCH 17/43] chore: Release --- Cargo.lock | 8 ++++---- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 4 ++-- datafusion-postgres-cli/Cargo.toml | 4 ++-- datafusion-postgres/Cargo.toml | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dad9b5a..3686eb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,7 +379,7 @@ dependencies = [ [[package]] name = "arrow-pg" -version = "0.12.2" +version = "0.13.0" dependencies = [ "arrow", "arrow-schema", @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.15.1" +version = "0.16.0" dependencies = [ "arrow-pg", "async-trait", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "datafusion-postgres" -version = "0.15.0" +version = "0.16.0" dependencies = [ "arrow-pg", "async-trait", @@ -1753,7 +1753,7 @@ dependencies = [ [[package]] name = "datafusion-postgres-cli" -version = "0.15.0" +version = "0.16.0" dependencies = [ "datafusion", "datafusion-postgres", diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 5cacde9..8adf044 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "arrow-pg" description = "Arrow data mapping and encoding/decoding for Postgres" -version = "0.12.2" +version = "0.13.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index e67a3a2..4f3400f 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-pg-catalog" description = "pg_catalog compatibility for datafusion" -version = "0.15.1" +version = "0.16.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -20,7 +20,7 @@ include = [ [dependencies] async-trait = "0.1" datafusion = { workspace = true, features = ["sql"] } -arrow-pg = { path = "../arrow-pg", version = "0.12.2", default-features = false, features = ["datafusion"] } +arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, features = ["datafusion"] } futures.workspace = true log = "0.4" postgres-types.workspace = true diff --git a/datafusion-postgres-cli/Cargo.toml b/datafusion-postgres-cli/Cargo.toml index 59692a8..25fbc0d 100644 --- a/datafusion-postgres-cli/Cargo.toml +++ b/datafusion-postgres-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-postgres-cli" description = "Command-line tool for DataFusion's Postgres Protocol frontend" -version = "0.15.0" +version = "0.16.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -15,7 +15,7 @@ rust-version.workspace = true [dependencies] datafusion = { workspace = true, features = ["avro"] } tokio = { workspace = true, features = ["full"] } -datafusion-postgres = { path = "../datafusion-postgres", version = "0.15.0" } +datafusion-postgres = { path = "../datafusion-postgres", version = "0.16.0" } structopt = { version = "0.3", default-features = false } log = "0.4" env_logger = "0.11" diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 754a487..d543ed4 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-postgres" description = "Exporting datafusion query engine with postgres wire protocol" -version = "0.15.0" +version = "0.16.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -13,12 +13,12 @@ readme = "../README.md" rust-version.workspace = true [dependencies] -arrow-pg = { path = "../arrow-pg", version = "0.12.2", default-features = false, features = ["datafusion"] } +arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, features = ["datafusion"] } bytes.workspace = true async-trait = "0.1" chrono.workspace = true datafusion.workspace = true -datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.15.1" } +datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.16.0" } geodatafusion = { version = "0.4", optional = true } futures.workspace = true getset = "0.1" From 9442098e85fc1f2bdb166801ad9e91868a011729 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 10:33:21 +0800 Subject: [PATCH 18/43] chore(deps): bump tokio from 1.50.0 to 1.51.0 (#322) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.50.0 to 1.51.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.50.0...tokio-1.51.0) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.51.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 112 ++++++------------------------- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 4 files changed, 22 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3686eb9..9999fee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2886,9 +2886,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -3928,12 +3928,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4222,9 +4222,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" dependencies = [ "bytes", "libc", @@ -4239,9 +4239,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -4698,7 +4698,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -4707,16 +4707,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -4734,31 +4725,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4767,96 +4741,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.14" diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 8adf044..1c29f16 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -37,4 +37,4 @@ geo-traits = { version = "0.3", optional = true } [dev-dependencies] async-trait = "0.1" -tokio = { version = "1.50", features = ["full"]} +tokio = { version = "1.51", features = ["full"]} diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 4f3400f..ed8fdc0 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -24,7 +24,7 @@ arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, futures.workspace = true log = "0.4" postgres-types.workspace = true -tokio = { version = "1.50", features = ["sync"] } +tokio = { version = "1.51", features = ["sync"] } [dev-dependencies] env_logger = "0.11" diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index d543ed4..52088a0 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -26,7 +26,7 @@ log = "0.4" pgwire = { workspace = true, features = ["server-api-ring"] } postgres-types.workspace = true rust_decimal.workspace = true -tokio = { version = "1.50", features = ["sync", "net"] } +tokio = { version = "1.51", features = ["sync", "net"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } rustls-pemfile = "2.0" rustls-pki-types = "1.14" From c2459b7ff9940dd0e3cfa7144bafe42ee1b8df64 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Wed, 8 Apr 2026 15:06:56 +0800 Subject: [PATCH 19/43] fix: use text format as default in describe statement (#323) --- datafusion-postgres/src/handlers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index 0672d5d..d237039 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -408,7 +408,7 @@ impl QueryParser for Parser { let schema = plan.schema(); let fields = arrow_schema_to_pg_fields( schema.as_arrow(), - column_format.unwrap_or(&Format::UnifiedBinary), + column_format.unwrap_or(&Format::UnifiedText), None, )?; From 60b3dac9492ba62fd427fef8683833ec832beeae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:49:29 +0800 Subject: [PATCH 20/43] chore(deps): bump tokio from 1.51.0 to 1.51.1 (#324) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.51.0 to 1.51.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.51.0...tokio-1.51.1) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.51.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9999fee..bee2017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4222,9 +4222,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.0" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "bytes", "libc", From 4ba7615245583309608eef815d06054dc588ba66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 10:49:44 +0800 Subject: [PATCH 21/43] chore(deps): bump pgwire from 0.38.2 to 0.38.3 (#325) Bumps [pgwire](https://github.com/sunng87/pgwire) from 0.38.2 to 0.38.3. - [Release notes](https://github.com/sunng87/pgwire/releases) - [Changelog](https://github.com/sunng87/pgwire/blob/v0.38.3/CHANGELOG.md) - [Commits](https://github.com/sunng87/pgwire/compare/v0.38.2...v0.38.3) --- updated-dependencies: - dependency-name: pgwire dependency-version: 0.38.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bee2017..1c63378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1908,7 +1908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3134,9 +3134,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.38.2" +version = "0.38.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1bdf05fc8231cc5024572fe056e3ce34eb6b9b755ba7aba110e1c64119cec3" +checksum = "24bd4e6b1bfddc5c6420dee6602ec80946700b4c31ddcb64ee190ad6d979c210" dependencies = [ "async-trait", "base64", @@ -3699,7 +3699,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4113,7 +4113,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4624,7 +4624,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From d3de9f42b978aa8230966c3269da81e4f04c8ae7 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 13 Apr 2026 20:27:40 +0800 Subject: [PATCH 22/43] feat: improve query parameter parsing for datafusion (#326) * feat: improve query parameter parsing for datafusion * feat: add range check and tests * chore: fmt * Simplify float & int casting (#327) Instead of reusing `checked_int_cast` also for the float code paths and requiring a `source` parameter, split the int & float paths completely, making the callsites simpler, and reducing the overall code length. --------- Co-authored-by: Martin Garton --- arrow-pg/src/datatypes/df.rs | 1370 ++++++++++++++++++++++++++++++---- 1 file changed, 1239 insertions(+), 131 deletions(-) diff --git a/arrow-pg/src/datatypes/df.rs b/arrow-pg/src/datatypes/df.rs index c782095..bca9580 100644 --- a/arrow-pg/src/datatypes/df.rs +++ b/arrow-pg/src/datatypes/df.rs @@ -13,7 +13,7 @@ use pg_interval::Interval; use pgwire::api::portal::{Format, Portal}; use pgwire::api::results::QueryResponse; use pgwire::api::Type; -use pgwire::error::{PgWireError, PgWireResult}; +use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use pgwire::messages::data::DataRow; use pgwire::types::format::FormatOptions; use rust_decimal::prelude::ToPrimitive; @@ -51,6 +51,227 @@ pub async fn encode_dataframe( Ok(QueryResponse::new(fields, pg_row_stream)) } +fn invalid_parameter_error(msg: impl Into) -> PgWireError { + PgWireError::UserError(Box::new(ErrorInfo::new( + "ERROR".to_owned(), + "22023".to_owned(), + msg.into(), + ))) +} + +fn out_of_range_error(source: &str, value: i64, target: &DataType) -> PgWireError { + invalid_parameter_error(format!( + "{} value {} is out of range for {:?}", + source, value, target + )) +} + +fn checked_int_cast(value: i64, target: &DataType) -> PgWireResult +where + T: TryFrom, +{ + T::try_from(value).map_err(|_| out_of_range_error("INT", value, target)) +} + +fn coerce_int_value(value: Option, target: &DataType) -> PgWireResult { + match target { + DataType::Int8 => Ok(ScalarValue::Int8( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::Int16 => Ok(ScalarValue::Int16( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::Int32 => Ok(ScalarValue::Int32( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::Int64 => Ok(ScalarValue::Int64(value)), + DataType::UInt8 => Ok(ScalarValue::UInt8( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::UInt16 => Ok(ScalarValue::UInt16( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::UInt32 => Ok(ScalarValue::UInt32( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::UInt64 => Ok(ScalarValue::UInt64( + value.map(|n| checked_int_cast(n, target)).transpose()?, + )), + DataType::Float32 => Ok(ScalarValue::Float32(value.map(|n| n as f32))), + DataType::Float64 => Ok(ScalarValue::Float64(value.map(|n| n as f64))), + DataType::Timestamp(TimeUnit::Second, _) => Ok(ScalarValue::TimestampSecond(value, None)), + DataType::Timestamp(TimeUnit::Millisecond, _) => Ok(ScalarValue::TimestampMillisecond( + value.map(|n| n * 1000), + None, + )), + DataType::Timestamp(TimeUnit::Microsecond, _) => Ok(ScalarValue::TimestampMicrosecond( + value.map(|n| n * 1_000_000), + None, + )), + DataType::Timestamp(TimeUnit::Nanosecond, _) => Ok(ScalarValue::TimestampNanosecond( + value.map(|n| n * 1_000_000_000), + None, + )), + _ => Err(invalid_parameter_error(format!( + "Cannot coerce integer value to {:?}", + target + ))), + } +} + +fn checked_float_cast(value: f64, target: &DataType) -> PgWireResult +where + T: TryFrom, +{ + if !value.is_finite() { + return Err(invalid_parameter_error(format!( + "FLOAT value {} is out of range for {:?}", + value, target + ))); + } + let n = value as i64; + if (n as f64 - value).abs() > f64::EPSILON { + return Err(out_of_range_error("FLOAT", n, target)); + } + T::try_from(n).map_err(|_| out_of_range_error("FLOAT", n, target)) +} + +fn coerce_float_value(value: Option, target: &DataType) -> PgWireResult { + match target { + DataType::Int8 => Ok(ScalarValue::Int8( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::Int16 => Ok(ScalarValue::Int16( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::Int32 => Ok(ScalarValue::Int32( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::Int64 => Ok(ScalarValue::Int64( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::UInt8 => Ok(ScalarValue::UInt8( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::UInt16 => Ok(ScalarValue::UInt16( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::UInt32 => Ok(ScalarValue::UInt32( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::UInt64 => Ok(ScalarValue::UInt64( + value.map(|n| checked_float_cast(n, target)).transpose()?, + )), + DataType::Float32 => Ok(ScalarValue::Float32(value.map(|n| n as f32))), + DataType::Float64 => Ok(ScalarValue::Float64(value)), + _ => Err(invalid_parameter_error(format!( + "Cannot coerce float value to {:?}", + target + ))), + } +} + +fn coerce_timestamp_value( + value: Option, + target: &DataType, +) -> PgWireResult { + match target { + DataType::Timestamp(TimeUnit::Second, _) => Ok(ScalarValue::TimestampSecond( + value.map(|t| t.and_utc().timestamp()), + None, + )), + DataType::Timestamp(TimeUnit::Millisecond, _) => Ok(ScalarValue::TimestampMillisecond( + value.map(|t| t.and_utc().timestamp_millis()), + None, + )), + DataType::Timestamp(TimeUnit::Microsecond, _) => Ok(ScalarValue::TimestampMicrosecond( + value.map(|t| t.and_utc().timestamp_micros()), + None, + )), + DataType::Timestamp(TimeUnit::Nanosecond, _) => Ok(ScalarValue::TimestampNanosecond( + value.and_then(|t| t.and_utc().timestamp_nanos_opt()), + None, + )), + _ => Err(invalid_parameter_error(format!( + "Cannot coerce TIMESTAMP to {:?}", + target + ))), + } +} + +fn coerce_timestamptz_value( + value: Option>, + target: &DataType, +) -> PgWireResult { + match target { + DataType::Timestamp(unit, tz) => { + let tz = tz.clone(); + match unit { + TimeUnit::Second => Ok(ScalarValue::TimestampSecond( + value.map(|t| t.timestamp()), + tz, + )), + TimeUnit::Millisecond => Ok(ScalarValue::TimestampMillisecond( + value.map(|t| t.timestamp_millis()), + tz, + )), + TimeUnit::Microsecond => Ok(ScalarValue::TimestampMicrosecond( + value.map(|t| t.timestamp_micros()), + tz, + )), + TimeUnit::Nanosecond => Ok(ScalarValue::TimestampNanosecond( + value.and_then(|t| t.timestamp_nanos_opt()), + tz, + )), + } + } + _ => Err(invalid_parameter_error(format!( + "Cannot coerce TIMESTAMPTZ to {:?}", + target + ))), + } +} + +fn coerce_interval_value(value: Option, target: &DataType) -> PgWireResult { + match target { + DataType::Interval(IntervalUnit::YearMonth) => Ok(match value { + Some(i) => { + if i.days != 0 || i.microseconds != 0 { + return Err(invalid_parameter_error( + "Cannot coerce INTERVAL with days or time components to YearMonth interval", + )); + } + ScalarValue::IntervalYearMonth(Some(i.months)) + } + None => ScalarValue::IntervalYearMonth(None), + }), + DataType::Interval(IntervalUnit::DayTime) => Ok(match value { + Some(i) => { + if i.months != 0 { + return Err(invalid_parameter_error( + "Cannot coerce INTERVAL with months component to DayTime interval", + )); + } + if i.microseconds % 1000 != 0 { + return Err(invalid_parameter_error( + "Cannot coerce INTERVAL with sub-millisecond precision to DayTime interval", + )); + } + ScalarValue::new_interval_dt(i.days, (i.microseconds / 1000) as i32) + } + None => ScalarValue::IntervalDayTime(None), + }), + DataType::Interval(IntervalUnit::MonthDayNano) | DataType::Duration(_) => Ok(match value { + Some(i) => ScalarValue::new_interval_mdn(i.months, i.days, i.microseconds * 1_000i64), + None => ScalarValue::IntervalMonthDayNano(None), + }), + _ => Err(invalid_parameter_error(format!( + "Cannot coerce INTERVAL to {:?}", + target + ))), + } +} + /// Deserialize client provided parameter data. /// /// First we try to use the type information from `pg_type_hint`, which is @@ -90,9 +311,10 @@ where .and_then(|f| f.clone()), inferenced_type, )?; + // enumerate all supported parameter types and deserialize the + // type to ScalarValue, with data coercion when server-inferred + // types are available match pg_type { - // enumerate all supported parameter types and deserialize the - // type to ScalarValue Type::BOOL => { let value = portal.parameter::(i, &pg_type)?; deserialized_params.push(ScalarValue::Boolean(value)); @@ -103,19 +325,49 @@ where } Type::INT2 => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Int16(value)); + match inferenced_type { + Some(target) => { + deserialized_params + .push(coerce_int_value(value.map(|n| n as i64), target)?); + } + None => { + deserialized_params.push(ScalarValue::Int16(value)); + } + } } Type::INT4 => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Int32(value)); + match inferenced_type { + Some(target) => { + deserialized_params + .push(coerce_int_value(value.map(|n| n as i64), target)?); + } + None => { + deserialized_params.push(ScalarValue::Int32(value)); + } + } } Type::INT8 => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Int64(value)); + match inferenced_type { + Some(target) => { + deserialized_params.push(coerce_int_value(value, target)?); + } + None => { + deserialized_params.push(ScalarValue::Int64(value)); + } + } } Type::TEXT | Type::VARCHAR => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Utf8(value)); + match inferenced_type { + Some(DataType::LargeUtf8) => { + deserialized_params.push(ScalarValue::LargeUtf8(value)); + } + _ => { + deserialized_params.push(ScalarValue::Utf8(value)); + } + } } Type::BYTEA => { let value = portal.parameter::>(i, &pg_type)?; @@ -124,39 +376,113 @@ where Type::FLOAT4 => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Float32(value)); + match inferenced_type { + Some(target) => { + deserialized_params + .push(coerce_float_value(value.map(|n| n as f64), target)?); + } + None => { + deserialized_params.push(ScalarValue::Float32(value)); + } + } } Type::FLOAT8 => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::Float64(value)); + match inferenced_type { + Some(target) => { + deserialized_params.push(coerce_float_value(value, target)?); + } + None => { + deserialized_params.push(ScalarValue::Float64(value)); + } + } } Type::NUMERIC => { - let value = match portal.parameter::(i, &pg_type)? { - None => ScalarValue::Decimal128(None, 0, 0), - Some(value) => { - let precision = match value.mantissa() { - 0 => 1, - m => (m.abs() as f64).log10().floor() as u8 + 1, + let value = portal.parameter::(i, &pg_type)?; + match inferenced_type { + Some(DataType::Decimal128(p, s)) => { + deserialized_params.push(ScalarValue::Decimal128( + value.map(|mut n| { + n.rescale(*s as u32); + n.mantissa() + }), + *p, + *s, + )); + } + Some(DataType::UInt64) => { + deserialized_params + .push(ScalarValue::UInt64(value.and_then(|n| n.to_u64()))); + } + Some(DataType::UInt32) => { + deserialized_params + .push(ScalarValue::UInt32(value.and_then(|n| n.to_u32()))); + } + Some(DataType::Int64) => { + deserialized_params + .push(ScalarValue::Int64(value.and_then(|n| n.to_i64()))); + } + Some(DataType::Int32) => { + deserialized_params + .push(ScalarValue::Int32(value.and_then(|n| n.to_i32()))); + } + Some(DataType::Float64) => { + deserialized_params + .push(ScalarValue::Float64(value.and_then(|n| n.to_f64()))); + } + Some(DataType::Float32) => { + deserialized_params + .push(ScalarValue::Float32(value.and_then(|n| n.to_f32()))); + } + Some(target) => { + return Err(invalid_parameter_error(format!( + "Cannot coerce NUMERIC to {:?}", + target + ))); + } + None => { + let scalar = match value { + None => ScalarValue::Decimal128(None, 0, 0), + Some(v) => { + let precision = match v.mantissa() { + 0 => 1, + m => (m.abs() as f64).log10().floor() as u8 + 1, + }; + let scale = v.scale() as i8; + ScalarValue::Decimal128(Some(v.mantissa()), precision, scale) + } }; - let scale = value.scale() as i8; - ScalarValue::Decimal128(value.to_i128(), precision, scale) + deserialized_params.push(scalar); } - }; - deserialized_params.push(value); + } } Type::TIMESTAMP => { let value = portal.parameter::(i, &pg_type)?; - deserialized_params.push(ScalarValue::TimestampMicrosecond( - value.map(|t| t.and_utc().timestamp_micros()), - None, - )); + match inferenced_type { + Some(target) => { + deserialized_params.push(coerce_timestamp_value(value, target)?); + } + None => { + deserialized_params.push(ScalarValue::TimestampMicrosecond( + value.map(|t| t.and_utc().timestamp_micros()), + None, + )); + } + } } Type::TIMESTAMPTZ => { let value = portal.parameter::>(i, &pg_type)?; - deserialized_params.push(ScalarValue::TimestampMicrosecond( - value.map(|t| t.timestamp_micros()), - value.map(|t| t.offset().to_string().into()), - )); + match inferenced_type { + Some(target) => { + deserialized_params.push(coerce_timestamptz_value(value, target)?); + } + None => { + deserialized_params.push(ScalarValue::TimestampMicrosecond( + value.map(|t| t.timestamp_micros()), + None, + )); + } + } } Type::DATE => { let value = portal.parameter::(i, &pg_type)?; @@ -183,15 +509,7 @@ where Some(DataType::Time32(TimeUnit::Second)) => { ScalarValue::Time32Second(ns.map(|ns| (ns / 1_000_000_000) as _)) } - _ => { - return Err(PgWireError::ApiError( - format!( - "Unable to deserialise time parameter type {:?} to type {:?}", - value, inferenced_type - ) - .into(), - )) - } + _ => ScalarValue::Time64Nanosecond(ns), }; deserialized_params.push(scalar_value); @@ -208,106 +526,309 @@ where } Type::INTERVAL => { let value = portal.parameter::(i, &pg_type)?; - let scalar_value = if let Some(i) = value { - ScalarValue::new_interval_mdn(i.months, i.days, i.microseconds * 1_000i64) - } else { - ScalarValue::IntervalMonthDayNano(None) - }; + match inferenced_type { + Some(target) => { + deserialized_params.push(coerce_interval_value(value, target)?); + } + None => { + let scalar_value = if let Some(i) = value { + ScalarValue::new_interval_mdn( + i.months, + i.days, + i.microseconds * 1_000i64, + ) + } else { + ScalarValue::IntervalMonthDayNano(None) + }; - deserialized_params.push(scalar_value); + deserialized_params.push(scalar_value); + } + } } // Array types support Type::BOOL_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Boolean).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Boolean, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Boolean).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Boolean), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Boolean, + true, + 1, + )); + } + } } Type::INT2_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Int16).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Int16, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Int16).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Int16), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Int16, + true, + 1, + )); + } + } } Type::INT4_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Int32).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Int32, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Int32).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Int32), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Int32, + true, + 1, + )); + } + } } Type::INT8_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Int64).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Int64, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Int64).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Int64), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Int64, + true, + 1, + )); + } + } } Type::FLOAT4_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Float32).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Float32, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Float32).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Float32), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Float32, + true, + 1, + )); + } + } } Type::FLOAT8_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Float64).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Float64, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Float64).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Float64), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Float64, + true, + 1, + )); + } + } } Type::TEXT_ARRAY | Type::VARCHAR_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter().map(ScalarValue::Utf8).collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Utf8, - ))); + match value { + Some(values) => { + let scalar_values: Vec = + values.into_iter().map(ScalarValue::Utf8).collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, &DataType::Utf8), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Utf8, + true, + 1, + )); + } + } } Type::INTERVAL_ARRAY => { let value = portal.parameter::>>(i, &pg_type)?; - let scalar_values: Vec = value.map_or(Vec::new(), |v| { - v.into_iter() - .map(|i| { - if let Some(i) = i { - ScalarValue::new_interval_mdn( - i.months, - i.days, - i.microseconds * 1_000i64, - ) - } else { - ScalarValue::IntervalMonthDayNano(None) + match value { + Some(values) => { + let scalar_values: Vec = values + .into_iter() + .map(|i| { + if let Some(i) = i { + ScalarValue::new_interval_mdn( + i.months, + i.days, + i.microseconds * 1_000i64, + ) + } else { + ScalarValue::IntervalMonthDayNano(None) + } + }) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::Interval(IntervalUnit::MonthDayNano), + ), + )); + } + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Interval(IntervalUnit::MonthDayNano), + true, + 1, + )); + } + } + } + Type::NUMERIC_ARRAY => { + let value = portal.parameter::>>(i, &pg_type)?; + match value { + Some(values) => match inferenced_type { + Some(DataType::List(field)) => match field.data_type() { + DataType::Decimal128(p, s) => { + let scalar_values: Vec = values + .into_iter() + .map(|n| { + ScalarValue::Decimal128( + n.map(|mut n| { + n.rescale(*s as u32); + n.mantissa() + }), + *p, + *s, + ) + }) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::Decimal128(*p, *s), + ), + )); } - }) - .collect() - }); - deserialized_params.push(ScalarValue::List(ScalarValue::new_list_nullable( - &scalar_values, - &DataType::Interval(IntervalUnit::MonthDayNano), - ))); + DataType::UInt64 => { + let scalar_values: Vec = values + .into_iter() + .map(|n| ScalarValue::UInt64(n.and_then(|n| n.to_u64()))) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::UInt64, + ), + )); + } + DataType::Float64 => { + let scalar_values: Vec = values + .into_iter() + .map(|n| ScalarValue::Float64(n.and_then(|n| n.to_f64()))) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::Float64, + ), + )); + } + DataType::Float32 => { + let scalar_values: Vec = values + .into_iter() + .map(|n| ScalarValue::Float32(n.and_then(|n| n.to_f32()))) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::Float32, + ), + )); + } + other => { + let scalar_values: Vec = values + .into_iter() + .map(|n| { + ScalarValue::Decimal128( + n.map(|mut n| { + n.rescale(0); + n.mantissa() + }), + 38, + 0, + ) + }) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable(&scalar_values, other), + )); + } + }, + _ => { + let scalar_values: Vec = values + .into_iter() + .map(|n| match n { + None => ScalarValue::Decimal128(None, 0, 0), + Some(v) => { + let precision = match v.mantissa() { + 0 => 1, + m => (m.abs() as f64).log10().floor() as u8 + 1, + }; + let scale = v.scale() as i8; + ScalarValue::Decimal128( + Some(v.mantissa()), + precision, + scale, + ) + } + }) + .collect(); + deserialized_params.push(ScalarValue::List( + ScalarValue::new_list_nullable( + &scalar_values, + &DataType::Decimal128(38, 0), + ), + )); + } + }, + None => { + deserialized_params.push(ScalarValue::new_null_list( + DataType::Decimal128(38, 0), + true, + 1, + )); + } + } } // Advanced types Type::MONEY => { @@ -363,58 +884,645 @@ mod tests { use std::sync::Arc; use arrow::datatypes::DataType; - use bytes::Bytes; - use datafusion::{common::ParamValues, scalar::ScalarValue}; + use bytes::{Bytes, BytesMut}; + use chrono::{FixedOffset, NaiveDate, NaiveTime, TimeZone}; + use datafusion::{ + arrow::datatypes::{IntervalUnit, TimeUnit}, + common::ParamValues, + scalar::ScalarValue, + }; + use pg_interval::Interval; use pgwire::{ api::{portal::Portal, stmt::StoredStatement}, messages::{data::FORMAT_CODE_BINARY, extendedquery::Bind}, }; - use postgres_types::Type; + use postgres_types::{ToSql, Type}; + use rust_decimal::Decimal; use crate::datatypes::df::deserialize_parameters; - #[test] - fn test_deserialise_time_params() { - let postgres_types = vec![Some(Type::TIME)]; - - let us: i64 = 1_000_000; // 1 second + fn encode_param(value: &T, pg_type: &Type) -> Bytes { + let mut buf = BytesMut::new(); + value.to_sql(pg_type, &mut buf).unwrap(); + buf.freeze() + } + fn make_portal(value: &T, pg_type: Type) -> Portal<&'static str> { + let data = encode_param(value, &pg_type); let bind = Bind::new( None, None, vec![FORMAT_CODE_BINARY], - vec![Some(Bytes::from(i64::to_be_bytes(us).to_vec()))], + vec![Some(data)], vec![], ); + let stmt = StoredStatement::new("id".into(), "s", vec![Some(pg_type)]); + Portal::try_new(&bind, Arc::new(stmt)).unwrap() + } + + fn make_null_portal(pg_type: Type) -> Portal<&'static str> { + let bind = Bind::new(None, None, vec![FORMAT_CODE_BINARY], vec![None], vec![]); + let stmt = StoredStatement::new("id".into(), "s", vec![Some(pg_type)]); + Portal::try_new(&bind, Arc::new(stmt)).unwrap() + } + + fn get_scalar(result: ParamValues) -> ScalarValue { + let ParamValues::List(list) = result else { + panic!("expected list"); + }; + assert_eq!(list.len(), 1); + list.into_iter().next().unwrap().value().clone() + } + + fn get_result(portal: &Portal<&'static str>, inferred: Option<&DataType>) -> ScalarValue { + let result = deserialize_parameters(portal, &[inferred]).unwrap(); + get_scalar(result) + } + + // -- Basic types -- + + #[test] + fn test_bool() { + let portal = make_portal(&true, Type::BOOL); + assert_eq!(get_result(&portal, None), ScalarValue::Boolean(Some(true))); + + let portal = make_null_portal(Type::BOOL); + assert_eq!(get_result(&portal, None), ScalarValue::Boolean(None)); + } + + #[test] + fn test_char() { + let portal = make_portal(&42i8, Type::CHAR); + assert_eq!(get_result(&portal, None), ScalarValue::Int8(Some(42))); + } + + #[test] + fn test_text_varchar() { + for pg_type in [Type::TEXT, Type::VARCHAR] { + let portal = make_portal(&"hello".to_string(), pg_type.clone()); + assert_eq!( + get_result(&portal, None), + ScalarValue::Utf8(Some("hello".to_string())) + ); + } + } + + #[test] + fn test_text_large_utf8() { + let portal = make_portal(&"hello".to_string(), Type::TEXT); + assert_eq!( + get_result(&portal, Some(&DataType::LargeUtf8)), + ScalarValue::LargeUtf8(Some("hello".to_string())) + ); + } + + #[test] + fn test_bytea() { + let portal = make_portal(&vec![1u8, 2, 3], Type::BYTEA); + assert_eq!( + get_result(&portal, None), + ScalarValue::Binary(Some(vec![1, 2, 3])) + ); + } + + // -- INT2 coercion -- + + #[test] + fn test_int2_direct() { + let portal = make_portal(&42i16, Type::INT2); + assert_eq!(get_result(&portal, None), ScalarValue::Int16(Some(42))); + } + + #[test] + fn test_int2_coerce_to_int32() { + let portal = make_portal(&100i16, Type::INT2); + assert_eq!( + get_result(&portal, Some(&DataType::Int32)), + ScalarValue::Int32(Some(100)) + ); + } + + #[test] + fn test_int2_coerce_to_int64() { + let portal = make_portal(&100i16, Type::INT2); + assert_eq!( + get_result(&portal, Some(&DataType::Int64)), + ScalarValue::Int64(Some(100)) + ); + } + + #[test] + fn test_int2_coerce_to_float64() { + let portal = make_portal(&100i16, Type::INT2); + assert_eq!( + get_result(&portal, Some(&DataType::Float64)), + ScalarValue::Float64(Some(100.0)) + ); + } + + #[test] + fn test_int2_coerce_to_uint32() { + let portal = make_portal(&100i16, Type::INT2); + assert_eq!( + get_result(&portal, Some(&DataType::UInt32)), + ScalarValue::UInt32(Some(100)) + ); + } + + #[test] + fn test_int2_null_coercion() { + let portal = make_null_portal(Type::INT2); + assert_eq!( + get_result(&portal, Some(&DataType::Int64)), + ScalarValue::Int64(None) + ); + } + + // -- INT4 coercion -- + + #[test] + fn test_int4_coerce_to_int8() { + let portal = make_portal(&42i32, Type::INT4); + assert_eq!( + get_result(&portal, Some(&DataType::Int8)), + ScalarValue::Int8(Some(42)) + ); + } + + #[test] + fn test_int4_coerce_to_int64() { + let portal = make_portal(&100i32, Type::INT4); + assert_eq!( + get_result(&portal, Some(&DataType::Int64)), + ScalarValue::Int64(Some(100)) + ); + } + + #[test] + fn test_int4_coerce_to_uint64() { + let portal = make_portal(&100i32, Type::INT4); + assert_eq!( + get_result(&portal, Some(&DataType::UInt64)), + ScalarValue::UInt64(Some(100)) + ); + } + + #[test] + fn test_int4_coerce_out_of_range() { + let portal = make_portal(&300i32, Type::INT4); + assert!(deserialize_parameters(&portal, &[Some(&DataType::Int8)]).is_err()); + } + + #[test] + fn test_int4_coerce_negative_to_unsigned() { + let portal = make_portal(&(-1i32), Type::INT4); + assert!(deserialize_parameters(&portal, &[Some(&DataType::UInt64)]).is_err()); + } + + // -- INT8 coercion -- + + #[test] + fn test_int8_direct() { + let portal = make_portal(&42i64, Type::INT8); + assert_eq!(get_result(&portal, None), ScalarValue::Int64(Some(42))); + } + + #[test] + fn test_int8_coerce_to_int16() { + let portal = make_portal(&100i64, Type::INT8); + assert_eq!( + get_result(&portal, Some(&DataType::Int16)), + ScalarValue::Int16(Some(100)) + ); + } + + #[test] + fn test_int8_coerce_to_uint32() { + let portal = make_portal(&100i64, Type::INT8); + assert_eq!( + get_result(&portal, Some(&DataType::UInt32)), + ScalarValue::UInt32(Some(100)) + ); + } + + #[test] + fn test_int8_coerce_out_of_range() { + let portal = make_portal(&100000i64, Type::INT8); + assert!(deserialize_parameters(&portal, &[Some(&DataType::Int8)]).is_err()); + } + + // -- FLOAT4 coercion -- + + #[test] + fn test_float4_direct() { + let portal = make_portal(&3.14f32, Type::FLOAT4); + assert_eq!( + get_result(&portal, None), + ScalarValue::Float32(Some(3.14f32)) + ); + } + + #[test] + fn test_float4_coerce_to_float64() { + let portal = make_portal(&3.14f32, Type::FLOAT4); + assert_eq!( + get_result(&portal, Some(&DataType::Float64)), + ScalarValue::Float64(Some(3.14f32 as f64)) + ); + } + + #[test] + fn test_float4_coerce_to_int32() { + let portal = make_portal(&42.0f32, Type::FLOAT4); + assert_eq!( + get_result(&portal, Some(&DataType::Int32)), + ScalarValue::Int32(Some(42)) + ); + } + + // -- FLOAT8 coercion -- + + #[test] + fn test_float8_direct() { + let portal = make_portal(&3.14f64, Type::FLOAT8); + assert_eq!(get_result(&portal, None), ScalarValue::Float64(Some(3.14))); + } + + #[test] + fn test_float8_coerce_to_float32() { + let portal = make_portal(&3.14f64, Type::FLOAT8); + assert_eq!( + get_result(&portal, Some(&DataType::Float32)), + ScalarValue::Float32(Some(3.14f64 as f32)) + ); + } - let stmt = StoredStatement::new("statement_id".into(), "statement", postgres_types); - let portal = Portal::try_new(&bind, Arc::new(stmt)).unwrap(); + #[test] + fn test_float8_coerce_to_int64() { + let portal = make_portal(&42.0f64, Type::FLOAT8); + assert_eq!( + get_result(&portal, Some(&DataType::Int64)), + ScalarValue::Int64(Some(42)) + ); + } + + #[test] + fn test_float8_coerce_fractional_to_int_error() { + let portal = make_portal(&3.14f64, Type::FLOAT8); + assert!(deserialize_parameters(&portal, &[Some(&DataType::Int64)]).is_err()); + } + + // -- NUMERIC coercion -- + + #[test] + fn test_numeric_direct() { + let portal = make_portal(&Decimal::new(123, 1), Type::NUMERIC); + let result = get_result(&portal, None); + match result { + ScalarValue::Decimal128(Some(v), p, s) => { + assert_eq!(v, 123); + assert_eq!(s, 1); + assert!(p > 0); + } + other => panic!("expected Decimal128, got {other:?}"), + } + } + + #[test] + fn test_numeric_coerce_to_decimal128_with_rescale() { + // Decimal::new(123, 1) = 12.3, rescale to scale 3 => 12.300 => mantissa 12300 + let portal = make_portal(&Decimal::new(123, 1), Type::NUMERIC); + assert_eq!( + get_result(&portal, Some(&DataType::Decimal128(10, 3))), + ScalarValue::Decimal128(Some(12300), 10, 3) + ); + } + + #[test] + fn test_numeric_coerce_to_uint64() { + let portal = make_portal(&Decimal::from(42u64), Type::NUMERIC); + assert_eq!( + get_result(&portal, Some(&DataType::UInt64)), + ScalarValue::UInt64(Some(42)) + ); + } + + #[test] + fn test_numeric_coerce_to_float64() { + let portal = make_portal(&Decimal::new(314, 2), Type::NUMERIC); + assert_eq!( + get_result(&portal, Some(&DataType::Float64)), + ScalarValue::Float64(Some(3.14)) + ); + } + + #[test] + fn test_numeric_coerce_to_int32() { + let portal = make_portal(&Decimal::from(42i32), Type::NUMERIC); + assert_eq!( + get_result(&portal, Some(&DataType::Int32)), + ScalarValue::Int32(Some(42)) + ); + } + + #[test] + fn test_numeric_null() { + let portal = make_null_portal(Type::NUMERIC); + assert_eq!( + get_result(&portal, Some(&DataType::Decimal128(10, 2))), + ScalarValue::Decimal128(None, 10, 2) + ); + } + + // -- TIMESTAMP coercion -- + + #[test] + fn test_timestamp_direct() { + let ts = chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(); + let portal = make_portal(&ts, Type::TIMESTAMP); + assert_eq!( + get_result(&portal, None), + ScalarValue::TimestampMicrosecond(Some(ts.and_utc().timestamp_micros()), None) + ); + } + + #[test] + fn test_timestamp_coerce_to_seconds() { + let ts = chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(); + let portal = make_portal(&ts, Type::TIMESTAMP); + assert_eq!( + get_result(&portal, Some(&DataType::Timestamp(TimeUnit::Second, None))), + ScalarValue::TimestampSecond(Some(1700000000), None) + ); + } + + #[test] + fn test_timestamp_coerce_to_milliseconds() { + let ts = chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(); + let portal = make_portal(&ts, Type::TIMESTAMP); + assert_eq!( + get_result( + &portal, + Some(&DataType::Timestamp(TimeUnit::Millisecond, None)) + ), + ScalarValue::TimestampMillisecond(Some(1700000000000), None) + ); + } + + #[test] + fn test_timestamp_coerce_to_nanoseconds() { + let ts = chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(); + let portal = make_portal(&ts, Type::TIMESTAMP); + assert_eq!( + get_result( + &portal, + Some(&DataType::Timestamp(TimeUnit::Nanosecond, None)) + ), + ScalarValue::TimestampNanosecond( + Some(ts.and_utc().timestamp_nanos_opt().unwrap()), + None + ) + ); + } + + // -- TIMESTAMPTZ coercion -- + + #[test] + fn test_timestamptz_direct() { + let ts = FixedOffset::east_opt(3600).unwrap().from_utc_datetime( + &chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(), + ); + let portal = make_portal(&ts, Type::TIMESTAMPTZ); + assert_eq!( + get_result(&portal, None), + ScalarValue::TimestampMicrosecond(Some(ts.timestamp_micros()), None) + ); + } + + #[test] + fn test_timestamptz_coerce_to_seconds() { + let ts = FixedOffset::east_opt(3600).unwrap().from_utc_datetime( + &chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(), + ); + let portal = make_portal(&ts, Type::TIMESTAMPTZ); + assert_eq!( + get_result(&portal, Some(&DataType::Timestamp(TimeUnit::Second, None))), + ScalarValue::TimestampSecond(Some(1700000000), None) + ); + } + + #[test] + fn test_timestamptz_coerce_to_nanoseconds() { + let ts = FixedOffset::east_opt(3600).unwrap().from_utc_datetime( + &chrono::DateTime::from_timestamp(1700000000, 0) + .unwrap() + .naive_utc(), + ); + let portal = make_portal(&ts, Type::TIMESTAMPTZ); + assert_eq!( + get_result( + &portal, + Some(&DataType::Timestamp(TimeUnit::Nanosecond, None)) + ), + ScalarValue::TimestampNanosecond(Some(ts.timestamp_nanos_opt().unwrap()), None) + ); + } + + // -- DATE -- + + #[test] + fn test_date() { + let date = NaiveDate::from_ymd_opt(2024, 1, 15).unwrap(); + let portal = make_portal(&date, Type::DATE); + assert_eq!( + get_result(&portal, None), + ScalarValue::Date32(Some( + datafusion::arrow::datatypes::Date32Type::from_naive_date(date) + )) + ); + } + + // -- TIME with all unit coercions -- + + #[test] + fn test_deserialise_time_params() { + let portal = make_portal( + &NaiveTime::from_num_seconds_from_midnight_opt(1, 0).unwrap(), + Type::TIME, + ); for (arrow_type, expected) in [ ( - DataType::Time32(arrow::datatypes::TimeUnit::Second), + DataType::Time32(TimeUnit::Second), ScalarValue::Time32Second(Some(1)), ), ( - DataType::Time32(arrow::datatypes::TimeUnit::Millisecond), + DataType::Time32(TimeUnit::Millisecond), ScalarValue::Time32Millisecond(Some(1000)), ), ( - DataType::Time64(arrow::datatypes::TimeUnit::Microsecond), + DataType::Time64(TimeUnit::Microsecond), ScalarValue::Time64Microsecond(Some(1000000)), ), ( - DataType::Time64(arrow::datatypes::TimeUnit::Nanosecond), + DataType::Time64(TimeUnit::Nanosecond), ScalarValue::Time64Nanosecond(Some(1000000000)), ), ] { - let result = deserialize_parameters(&portal, &[Some(&arrow_type)]).unwrap(); - let ParamValues::List(list) = result else { - panic!("expected list"); - }; - - assert_eq!(list.len(), 1); - assert_eq!(list[0].value(), &expected) + assert_eq!(get_result(&portal, Some(&arrow_type)), expected); } } + + #[test] + fn test_time_default_nanosecond() { + let time = NaiveTime::from_num_seconds_from_midnight_opt(1, 0).unwrap(); + let portal = make_portal(&time, Type::TIME); + // No inferred type: should default to Time64Nanosecond + assert_eq!( + get_result(&portal, None), + ScalarValue::Time64Nanosecond(Some(1_000_000_000)) + ); + } + + // -- INTERVAL coercion -- + + #[test] + fn test_interval_direct() { + let interval = Interval::new(1, 2, 3_000_000); + let portal = make_portal(&interval, Type::INTERVAL); + assert_eq!( + get_result(&portal, None), + ScalarValue::new_interval_mdn(1, 2, 3_000_000_000i64) + ); + } + + #[test] + fn test_interval_coerce_to_month_day_nano() { + let interval = Interval::new(1, 2, 3_000_000); + let portal = make_portal(&interval, Type::INTERVAL); + assert_eq!( + get_result( + &portal, + Some(&DataType::Interval(IntervalUnit::MonthDayNano)) + ), + ScalarValue::new_interval_mdn(1, 2, 3_000_000_000i64) + ); + } + + #[test] + fn test_interval_coerce_to_year_month() { + let interval = Interval::new(3, 0, 0); + let portal = make_portal(&interval, Type::INTERVAL); + assert_eq!( + get_result(&portal, Some(&DataType::Interval(IntervalUnit::YearMonth))), + ScalarValue::IntervalYearMonth(Some(3)) + ); + } + + #[test] + fn test_interval_coerce_to_year_month_with_days_error() { + let interval = Interval::new(3, 1, 0); + let portal = make_portal(&interval, Type::INTERVAL); + assert!(deserialize_parameters( + &portal, + &[Some(&DataType::Interval(IntervalUnit::YearMonth))] + ) + .is_err()); + } + + #[test] + fn test_interval_coerce_to_day_time() { + let interval = Interval::new(0, 5, 3_000_000); + let portal = make_portal(&interval, Type::INTERVAL); + assert_eq!( + get_result(&portal, Some(&DataType::Interval(IntervalUnit::DayTime))), + ScalarValue::new_interval_dt(5, 3000) + ); + } + + #[test] + fn test_interval_coerce_to_day_time_with_months_error() { + let interval = Interval::new(1, 5, 3_000_000); + let portal = make_portal(&interval, Type::INTERVAL); + assert!(deserialize_parameters( + &portal, + &[Some(&DataType::Interval(IntervalUnit::DayTime))] + ) + .is_err()); + } + + #[test] + fn test_interval_coerce_to_day_time_sub_millis_error() { + let interval = Interval::new(0, 5, 500); + let portal = make_portal(&interval, Type::INTERVAL); + assert!(deserialize_parameters( + &portal, + &[Some(&DataType::Interval(IntervalUnit::DayTime))] + ) + .is_err()); + } + + #[test] + fn test_interval_null() { + let portal = make_null_portal(Type::INTERVAL); + assert_eq!( + get_result(&portal, None), + ScalarValue::IntervalMonthDayNano(None) + ); + } + + // -- UUID, JSON -- + // These types don't have FromSql support in pgwire, so they + // fall through to the `_` wildcard branch which calls + // `portal.parameter::()`. Testing them requires a postgres + // round-trip, so we skip unit tests for these. + + // -- Advanced types (MONEY, INET, MACADDR) -- + // Same as above: pgwire's FromSql doesn't accept MONEY/INET/MACADDR. + + // -- Null parameters -- + + #[test] + fn test_null_int4() { + let portal = make_null_portal(Type::INT4); + assert_eq!( + get_result(&portal, Some(&DataType::Int32)), + ScalarValue::Int32(None) + ); + } + + #[test] + fn test_null_timestamp() { + let portal = make_null_portal(Type::TIMESTAMP); + assert_eq!( + get_result( + &portal, + Some(&DataType::Timestamp(TimeUnit::Millisecond, None)) + ), + ScalarValue::TimestampMillisecond(None, None) + ); + } + + // -- Fallback: unknown type as string -- + + #[test] + fn test_unknown_type_string() { + let portal = make_portal(&"hello".to_string(), Type::NAME); + assert_eq!( + get_result(&portal, None), + ScalarValue::Utf8(Some("hello".to_string())) + ); + } } From 764be81bcf48b278ef1d2bb0c84bba3a65307ee5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:29:23 +0800 Subject: [PATCH 23/43] chore(deps): bump tokio from 1.51.1 to 1.52.0 (#328) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.51.1 to 1.52.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.51.1...tokio-1.52.0) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.52.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c63378..00b14f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1908,7 +1908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3699,7 +3699,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4113,7 +4113,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4222,9 +4222,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.1" +version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" dependencies = [ "bytes", "libc", @@ -4624,7 +4624,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 1c29f16..5c30366 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -37,4 +37,4 @@ geo-traits = { version = "0.3", optional = true } [dev-dependencies] async-trait = "0.1" -tokio = { version = "1.51", features = ["full"]} +tokio = { version = "1.52", features = ["full"]} diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index ed8fdc0..54bf979 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -24,7 +24,7 @@ arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, futures.workspace = true log = "0.4" postgres-types.workspace = true -tokio = { version = "1.51", features = ["sync"] } +tokio = { version = "1.52", features = ["sync"] } [dev-dependencies] env_logger = "0.11" diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 52088a0..1f13a8c 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -26,7 +26,7 @@ log = "0.4" pgwire = { workspace = true, features = ["server-api-ring"] } postgres-types.workspace = true rust_decimal.workspace = true -tokio = { version = "1.51", features = ["sync", "net"] } +tokio = { version = "1.52", features = ["sync", "net"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } rustls-pemfile = "2.0" rustls-pki-types = "1.14" From fe83376165fa8c6e892a0df0ed422d88f522c193 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Wed, 15 Apr 2026 16:05:24 +0800 Subject: [PATCH 24/43] fix: only return schema for query (#329) --- datafusion-postgres/src/handlers.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index d237039..a98b337 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -405,14 +405,18 @@ impl QueryParser for Parser { column_format: Option<&Format>, ) -> PgWireResult> { if let (_, Some((_, plan))) = stmt { - let schema = plan.schema(); - let fields = arrow_schema_to_pg_fields( - schema.as_arrow(), - column_format.unwrap_or(&Format::UnifiedText), - None, - )?; - - Ok(fields) + if !matches!(plan, LogicalPlan::Ddl(_) | LogicalPlan::Dml(_)) { + let schema = plan.schema(); + let fields = arrow_schema_to_pg_fields( + schema.as_arrow(), + column_format.unwrap_or(&Format::UnifiedText), + None, + )?; + + Ok(fields) + } else { + Ok(vec![]) + } } else { Ok(vec![]) } From f27a050ffa8efe80636b3d90896a6f754b50415e Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Tue, 21 Apr 2026 12:54:21 +0800 Subject: [PATCH 25/43] fix: lint for rust 1.95 (#334) --- datafusion-pg-catalog/src/sql/rules.rs | 53 +++++++++++--------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index 5848731..96a83bf 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -269,16 +269,15 @@ impl ResolveUnqualifiedIdentifer { projection_aliases: &HashSet, ) { match expr { - Expr::Identifier(ident) => { - // If the identifier is not a table alias itself and not already aliased in projection, rewrite it. + // If the identifier is not a table alias itself and not already aliased in projection, rewrite it. + Expr::Identifier(ident) if !table_aliases.contains(&ident.value) - && !projection_aliases.contains(&ident.value) - { - *expr = Expr::CompoundIdentifier(vec![ - Ident::new(wildcard_alias.to_string()), - ident.clone(), - ]); - } + && !projection_aliases.contains(&ident.value) => + { + *expr = Expr::CompoundIdentifier(vec![ + Ident::new(wildcard_alias.to_string()), + ident.clone(), + ]); } Expr::BinaryOp { left, right, .. } => { Self::rewrite_expr(left, wildcard_alias, table_aliases, projection_aliases); @@ -347,27 +346,23 @@ impl VisitorMut for RemoveUnsupportedTypesVisitor<'_> { data_type, value, uses_odbc_syntax: _, - }) => { - if self - .unsupported_types - .contains(data_type.to_string().to_lowercase().as_str()) - { - *expr = - Expr::Value(Value::SingleQuotedString(value.to_string()).with_empty_span()); - } + }) if self + .unsupported_types + .contains(data_type.to_string().to_lowercase().as_str()) => + { + *expr = Expr::Value(Value::SingleQuotedString(value.to_string()).with_empty_span()); } Expr::Cast { data_type, expr: value, .. - } => { - if self - .unsupported_types - .contains(data_type.to_string().to_lowercase().as_str()) - { - *expr = *value.clone(); - } + } if self + .unsupported_types + .contains(data_type.to_string().to_lowercase().as_str()) => + { + *expr = *value.clone(); } + // Add more match arms for other expression types (e.g., `Function`, `InList`) as needed. _ => {} } @@ -1007,12 +1002,10 @@ impl Visitor for CorrelationCheckVisitor<'_> { }) => { *self.0 = true; } - Expr::CompoundIdentifier(idents) => { - if !idents.is_empty() { - let table_name = &idents[0].value; - if !self.1.contains(table_name) { - *self.0 = true; - } + Expr::CompoundIdentifier(idents) if !idents.is_empty() => { + let table_name = &idents[0].value; + if !self.1.contains(table_name) { + *self.0 = true; } } _ => {} From 66a3242082deb6af9e605ef1ea2651dffcaf948e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:22:13 +0800 Subject: [PATCH 26/43] chore(deps): bump datafusion from 53.0.0 to 53.1.0 (#332) Bumps [datafusion](https://github.com/apache/datafusion) from 53.0.0 to 53.1.0. - [Changelog](https://github.com/apache/datafusion/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion/compare/53.0.0...53.1.0) --- updated-dependencies: - dependency-name: datafusion dependency-version: 53.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 137 +++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00b14f7..0f40172 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,9 +1078,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ "arrow", "arrow-schema", @@ -1134,9 +1134,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ "arrow", "async-trait", @@ -1159,9 +1159,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -1208,9 +1208,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" dependencies = [ "futures", "log", @@ -1219,9 +1219,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ "arrow", "async-compression", @@ -1254,9 +1254,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ "arrow", "arrow-ipc", @@ -1278,9 +1278,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49dda81c79b6ba57b1853a9158abc66eb85a3aa1cede0c517dabec6d8a4ed3aa" +checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" dependencies = [ "apache-avro", "arrow", @@ -1298,9 +1298,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ "arrow", "async-trait", @@ -1321,9 +1321,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ "arrow", "async-trait", @@ -1345,9 +1345,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ "arrow", "async-trait", @@ -1375,15 +1375,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ "arrow", "arrow-buffer", @@ -1404,9 +1404,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ "arrow", "async-trait", @@ -1427,9 +1427,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", "datafusion-common", @@ -1440,9 +1440,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ "arrow", "arrow-buffer", @@ -1472,9 +1472,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", @@ -1494,9 +1494,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", @@ -1507,9 +1507,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", @@ -1532,9 +1532,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", @@ -1548,9 +1548,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", "datafusion-common", @@ -1566,9 +1566,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1576,9 +1576,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ "datafusion-doc", "quote", @@ -1587,9 +1587,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", @@ -1622,9 +1622,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", @@ -1646,9 +1646,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", "datafusion-common", @@ -1661,9 +1661,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash 0.8.12", "arrow", @@ -1678,9 +1678,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", "datafusion-common", @@ -1697,9 +1697,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash 0.8.12", "arrow", @@ -1765,9 +1765,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", "datafusion-common", @@ -1782,9 +1782,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", "datafusion-common", @@ -1796,9 +1796,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ "arrow", "bigdecimal", @@ -1908,7 +1908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3699,7 +3699,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3996,6 +3996,7 @@ dependencies = [ "cfg-if", "libc", "psm", + "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4113,7 +4114,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4624,7 +4625,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 466df241bfa59c6613719071ae8f9ca8a98396bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:22:22 +0800 Subject: [PATCH 27/43] chore(deps): bump tokio from 1.52.0 to 1.52.1 (#331) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.52.0 to 1.52.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.0...tokio-1.52.1) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.52.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f40172..2307d3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4223,9 +4223,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", From 1b9c12cce39ad4677280ddbacfa4e4f9f63f6567 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Wed, 22 Apr 2026 17:52:09 +0800 Subject: [PATCH 28/43] feat: update pgwire to 0.39 (#330) --- Cargo.lock | 4 +-- Cargo.toml | 2 +- datafusion-postgres/src/handlers.rs | 39 ++++++++++++++++++++++++----- datafusion-postgres/src/testing.rs | 17 ++++++++++--- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2307d3c..85236f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3134,9 +3134,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.38.3" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24bd4e6b1bfddc5c6420dee6602ec80946700b4c31ddcb64ee190ad6d979c210" +checksum = "b3cee243b682091188b90f22b07585ad6b43e699b52bc29422bd9ca6ce2c2deb" dependencies = [ "async-trait", "base64", diff --git a/Cargo.toml b/Cargo.toml index b29bcf4..21208e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ bytes = "1.11.1" chrono = { version = "0.4", features = ["std"] } datafusion = { version = "53" } futures = "0.3" -pgwire = { version = "0.38", default-features = false } +pgwire = { version = "0.39", default-features = false } postgres-types = "0.2" rust_decimal = { version = "1.41", features = ["db-postgres"] } tokio = { version = "1", default-features = false } diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index a98b337..ac5a945 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -11,11 +11,12 @@ use datafusion::sql::sqlparser; use log::info; use pgwire::api::auth::noop::NoopStartupHandler; use pgwire::api::auth::StartupHandler; +use pgwire::api::cancel::{CancelHandler, DefaultCancelHandler}; use pgwire::api::portal::{Format, Portal}; use pgwire::api::query::{ExtendedQueryHandler, SimpleQueryHandler}; use pgwire::api::results::{FieldInfo, Response, Tag}; use pgwire::api::stmt::QueryParser; -use pgwire::api::{ClientInfo, ErrorHandler, PgWireServerHandlers, Type}; +use pgwire::api::{ClientInfo, ConnectionManager, ErrorHandler, PgWireServerHandlers, Type}; use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::PgWireBackendMessage; use pgwire::types::format::FormatOptions; @@ -29,19 +30,34 @@ use arrow_pg::datatypes::{arrow_schema_to_pg_fields, into_pg_type}; use datafusion_pg_catalog::sql::PostgresCompatibilityParser; /// Simple startup handler that does no authentication -pub struct SimpleStartupHandler; +pub struct SimpleStartupHandler { + connection_manager: Arc, +} #[async_trait::async_trait] -impl NoopStartupHandler for SimpleStartupHandler {} +impl NoopStartupHandler for SimpleStartupHandler { + fn connection_manager(&self) -> Option> { + Some(self.connection_manager.clone()) + } +} pub struct HandlerFactory { pub session_service: Arc, + cancel_handler: Arc, + startup_handler: Arc, } impl HandlerFactory { pub fn new(session_context: Arc) -> Self { let session_service = Arc::new(DfSessionService::new(session_context)); - HandlerFactory { session_service } + let connection_manager = Arc::new(ConnectionManager::new()); + HandlerFactory { + session_service, + cancel_handler: Arc::new(DefaultCancelHandler::new(connection_manager.clone())), + startup_handler: Arc::new(SimpleStartupHandler { + connection_manager: connection_manager.clone(), + }), + } } pub fn new_with_hooks( @@ -52,7 +68,14 @@ impl HandlerFactory { session_context, query_hooks, )); - HandlerFactory { session_service } + let connection_manager = Arc::new(ConnectionManager::new()); + HandlerFactory { + session_service, + cancel_handler: Arc::new(DefaultCancelHandler::new(connection_manager.clone())), + startup_handler: Arc::new(SimpleStartupHandler { + connection_manager: connection_manager.clone(), + }), + } } } @@ -66,12 +89,16 @@ impl PgWireServerHandlers for HandlerFactory { } fn startup_handler(&self) -> Arc { - Arc::new(SimpleStartupHandler) + self.startup_handler.clone() } fn error_handler(&self) -> Arc { Arc::new(LoggingErrorHandler) } + + fn cancel_handler(&self) -> Arc { + self.cancel_handler.clone() + } } struct LoggingErrorHandler; diff --git a/datafusion-postgres/src/testing.rs b/datafusion-postgres/src/testing.rs index 4f9a7b2..26529b0 100644 --- a/datafusion-postgres/src/testing.rs +++ b/datafusion-postgres/src/testing.rs @@ -4,7 +4,10 @@ use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_pg_catalog::pg_catalog::setup_pg_catalog; use futures::Sink; use pgwire::{ - api::{ClientInfo, ClientPortalStore, PgWireConnectionState, METADATA_USER}, + api::{ + store::MemPortalStore, ClientInfo, ClientPortalStore, PgWireConnectionState, + SessionExtensions, METADATA_USER, + }, messages::{ response::TransactionStatus, startup::SecretKey, PgWireBackendMessage, ProtocolVersion, }, @@ -29,8 +32,9 @@ pub fn setup_handlers() -> DfSessionService { #[derive(Debug, Default)] pub struct MockClient { metadata: HashMap, - portal_store: HashMap, + portal_store: MemPortalStore, pub sent_messages: Vec, + session_extensions: SessionExtensions, } impl MockClient { @@ -40,8 +44,9 @@ impl MockClient { MockClient { metadata, - portal_store: HashMap::default(), + portal_store: MemPortalStore::new(), sent_messages: Vec::new(), + session_extensions: SessionExtensions::new(), } } @@ -98,10 +103,14 @@ impl ClientInfo for MockClient { fn sni_server_name(&self) -> Option<&str> { None } + + fn session_extensions(&self) -> &pgwire::api::SessionExtensions { + &self.session_extensions + } } impl ClientPortalStore for MockClient { - type PortalStore = HashMap; + type PortalStore = MemPortalStore; fn portal_store(&self) -> &Self::PortalStore { &self.portal_store } From 789dc234a2fbfaf4376a983ce5566f3991d7cd5b Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Thu, 23 Apr 2026 10:07:33 +0800 Subject: [PATCH 29/43] refactor: update for rust 2024 edition (#335) --- Cargo.toml | 2 +- arrow-pg/src/datatypes.rs | 2 +- arrow-pg/src/datatypes/df.rs | 36 +-- arrow-pg/src/list_encoder.rs | 20 +- arrow-pg/src/struct_encoder.rs | 4 +- .../benches/parser_benchmark.rs | 2 +- datafusion-pg-catalog/src/pg_catalog.rs | 8 +- .../src/pg_catalog/format_type.rs | 6 +- .../src/pg_catalog/pg_attribute.rs | 2 +- .../src/pg_catalog/pg_class.rs | 4 +- .../src/pg_catalog/pg_database.rs | 2 +- .../src/pg_catalog/pg_namespace.rs | 2 +- .../src/pg_catalog/quote_ident_udf.rs | 9 +- datafusion-pg-catalog/src/sql/rules.rs | 263 +++++++++--------- datafusion-postgres-cli/src/main.rs | 2 +- datafusion-postgres/src/auth.rs | 20 +- datafusion-postgres/src/handlers.rs | 48 ++-- datafusion-postgres/src/hooks/mod.rs | 2 +- datafusion-postgres/src/hooks/permissions.rs | 4 +- datafusion-postgres/src/hooks/set_show.rs | 15 +- datafusion-postgres/src/hooks/transactions.rs | 4 +- datafusion-postgres/src/lib.rs | 6 +- datafusion-postgres/src/planner.rs | 21 +- datafusion-postgres/src/testing.rs | 8 +- datafusion-postgres/tests/pgadbc.rs | 2 - 25 files changed, 243 insertions(+), 251 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 21208e7..11447a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["datafusion-postgres", "datafusion-postgres-cli", "arrow-pg", "datafusion-pg-catalog"] [workspace.package] -edition = "2021" +edition = "2024" license = "Apache-2.0" rust-version = "1.89" authors = ["Ning Sun "] diff --git a/arrow-pg/src/datatypes.rs b/arrow-pg/src/datatypes.rs index 8898313..48dfaf5 100644 --- a/arrow-pg/src/datatypes.rs +++ b/arrow-pg/src/datatypes.rs @@ -7,9 +7,9 @@ use arrow_schema::extension::ExtensionType; #[cfg(feature = "datafusion")] use datafusion::arrow::{datatypes::*, record_batch::RecordBatch}; +use pgwire::api::Type; use pgwire::api::portal::Format; use pgwire::api::results::FieldInfo; -use pgwire::api::Type; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use pgwire::messages::data::DataRow; use pgwire::types::format::FormatOptions; diff --git a/arrow-pg/src/datatypes/df.rs b/arrow-pg/src/datatypes/df.rs index bca9580..ae7a34b 100644 --- a/arrow-pg/src/datatypes/df.rs +++ b/arrow-pg/src/datatypes/df.rs @@ -8,16 +8,16 @@ use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::ParamValues; use datafusion::prelude::*; use datafusion::scalar::ScalarValue; -use futures::{stream, StreamExt}; +use futures::{StreamExt, stream}; use pg_interval::Interval; +use pgwire::api::Type; use pgwire::api::portal::{Format, Portal}; use pgwire::api::results::QueryResponse; -use pgwire::api::Type; use pgwire::error::{ErrorInfo, PgWireError, PgWireResult}; use pgwire::messages::data::DataRow; use pgwire::types::format::FormatOptions; -use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; +use rust_decimal::prelude::ToPrimitive; use super::{arrow_schema_to_pg_fields, encode_recordbatch, into_pg_type}; @@ -1435,11 +1435,13 @@ mod tests { fn test_interval_coerce_to_year_month_with_days_error() { let interval = Interval::new(3, 1, 0); let portal = make_portal(&interval, Type::INTERVAL); - assert!(deserialize_parameters( - &portal, - &[Some(&DataType::Interval(IntervalUnit::YearMonth))] - ) - .is_err()); + assert!( + deserialize_parameters( + &portal, + &[Some(&DataType::Interval(IntervalUnit::YearMonth))] + ) + .is_err() + ); } #[test] @@ -1456,22 +1458,20 @@ mod tests { fn test_interval_coerce_to_day_time_with_months_error() { let interval = Interval::new(1, 5, 3_000_000); let portal = make_portal(&interval, Type::INTERVAL); - assert!(deserialize_parameters( - &portal, - &[Some(&DataType::Interval(IntervalUnit::DayTime))] - ) - .is_err()); + assert!( + deserialize_parameters(&portal, &[Some(&DataType::Interval(IntervalUnit::DayTime))]) + .is_err() + ); } #[test] fn test_interval_coerce_to_day_time_sub_millis_error() { let interval = Interval::new(0, 5, 500); let portal = make_portal(&interval, Type::INTERVAL); - assert!(deserialize_parameters( - &portal, - &[Some(&DataType::Interval(IntervalUnit::DayTime))] - ) - .is_err()); + assert!( + deserialize_parameters(&portal, &[Some(&DataType::Interval(IntervalUnit::DayTime))]) + .is_err() + ); } #[test] diff --git a/arrow-pg/src/list_encoder.rs b/arrow-pg/src/list_encoder.rs index f605fb7..1a0788b 100644 --- a/arrow-pg/src/list_encoder.rs +++ b/arrow-pg/src/list_encoder.rs @@ -3,40 +3,40 @@ use std::{str::FromStr, sync::Arc}; #[cfg(not(feature = "datafusion"))] use arrow::{ array::{ - timezone::Tz, Array, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, + Array, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray, DurationSecondArray, IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray, PrimitiveArray, StringArray, StringViewArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, - TimestampSecondArray, + TimestampSecondArray, timezone::Tz, }, datatypes::{ - DataType, Date32Type, Date64Type, Float32Type, Float64Type, Int16Type, Int32Type, - Int64Type, Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, + DataType, Date32Type, Date64Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, + Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, - TimeUnit, UInt16Type, UInt32Type, UInt64Type, UInt8Type, + TimeUnit, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }, temporal_conversions::{as_date, as_time}, }; #[cfg(feature = "datafusion")] use datafusion::arrow::{ array::{ - timezone::Tz, Array, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, + Array, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, DurationMicrosecondArray, DurationMillisecondArray, DurationNanosecondArray, DurationSecondArray, IntervalDayTimeArray, IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray, PrimitiveArray, StringArray, StringViewArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, - TimestampSecondArray, + TimestampSecondArray, timezone::Tz, }, datatypes::{ - DataType, Date32Type, Date64Type, Float32Type, Float64Type, Int16Type, Int32Type, - Int64Type, Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, + DataType, Date32Type, Date64Type, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, + Int64Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, - TimeUnit, UInt16Type, UInt32Type, UInt64Type, UInt8Type, + TimeUnit, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }, temporal_conversions::{as_date, as_time}, }; diff --git a/arrow-pg/src/struct_encoder.rs b/arrow-pg/src/struct_encoder.rs index 8f8f956..4fbfe3e 100644 --- a/arrow-pg/src/struct_encoder.rs +++ b/arrow-pg/src/struct_encoder.rs @@ -12,11 +12,11 @@ use bytes::{BufMut, BytesMut}; use pgwire::api::results::{FieldFormat, FieldInfo}; use pgwire::error::PgWireResult; use pgwire::types::format::FormatOptions; -use pgwire::types::{ToSqlText, QUOTE_CHECK, QUOTE_ESCAPE}; +use pgwire::types::{QUOTE_CHECK, QUOTE_ESCAPE, ToSqlText}; use postgres_types::{Field, IsNull, ToSql, Type}; use crate::datatypes::field_into_pg_type; -use crate::encoder::{encode_value, Encoder}; +use crate::encoder::{Encoder, encode_value}; #[derive(Debug)] struct BytesWrapper(BytesMut, bool); diff --git a/datafusion-pg-catalog/benches/parser_benchmark.rs b/datafusion-pg-catalog/benches/parser_benchmark.rs index 099c46c..951d04a 100644 --- a/datafusion-pg-catalog/benches/parser_benchmark.rs +++ b/datafusion-pg-catalog/benches/parser_benchmark.rs @@ -1,6 +1,6 @@ use std::hint::black_box; -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{Criterion, criterion_group, criterion_main}; use datafusion_pg_catalog::sql::PostgresCompatibilityParser; fn bench_parser_parse(c: &mut Criterion) { diff --git a/datafusion-pg-catalog/src/pg_catalog.rs b/datafusion-pg-catalog/src/pg_catalog.rs index 5703d98..6749f30 100644 --- a/datafusion-pg-catalog/src/pg_catalog.rs +++ b/datafusion-pg-catalog/src/pg_catalog.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; -use std::sync::atomic::AtomicU32; use std::sync::Arc; +use std::sync::atomic::AtomicU32; use async_trait::async_trait; use datafusion::arrow::array::{ - as_boolean_array, ArrayRef, AsArray, BooleanBuilder, Int32Builder, RecordBatch, StringArray, - StringBuilder, + ArrayRef, AsArray, BooleanBuilder, Int32Builder, RecordBatch, StringArray, StringBuilder, + as_boolean_array, }; use datafusion::arrow::datatypes::{DataType, Field, Int32Type, SchemaRef}; use datafusion::arrow::ipc::reader::FileReader; @@ -18,7 +18,7 @@ use datafusion::logical_expr::{ ColumnarValue, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; use datafusion::physical_plan::streaming::PartitionStream; -use datafusion::prelude::{create_udf, Expr, SessionContext}; +use datafusion::prelude::{Expr, SessionContext, create_udf}; use postgres_types::Oid; use tokio::sync::RwLock; diff --git a/datafusion-pg-catalog/src/pg_catalog/format_type.rs b/datafusion-pg-catalog/src/pg_catalog/format_type.rs index 6588c4a..24da102 100644 --- a/datafusion-pg-catalog/src/pg_catalog/format_type.rs +++ b/datafusion-pg-catalog/src/pg_catalog/format_type.rs @@ -6,8 +6,8 @@ use datafusion::{ datatypes::{DataType, Int32Type}, }, common::{ - cast::{as_int32_array, as_int64_array}, DataFusionError, + cast::{as_int32_array, as_int64_array}, }, logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, @@ -27,7 +27,7 @@ pub(crate) fn format_type_impl(args: &[ColumnarValue]) -> Result Result bool { // Check if identifier starts with a letter or underscore and contains only letters, digits, underscores let mut chars = ident.chars(); - if let Some(first_char) = chars.next() { - if !first_char.is_alphabetic() && first_char != '_' { - return true; - } + if let Some(first_char) = chars.next() + && !first_char.is_alphabetic() + && first_char != '_' + { + return true; } // Check remaining characters diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index 96a83bf..dd2f8fe 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -136,10 +136,10 @@ impl AliasDuplicatedProjectionRewrite { impl SqlStatementRewriteRule for AliasDuplicatedProjectionRewrite { fn rewrite(&self, mut statement: Statement) -> Statement { - if let Statement::Query(query) = &mut statement { - if let SetExpr::Select(select) = query.body.as_mut() { - Self::rewrite_select_with_alias(select); - } + if let Statement::Query(query) = &mut statement + && let SetExpr::Select(select) = query.body.as_mut() + { + Self::rewrite_select_with_alias(select); } statement @@ -459,12 +459,11 @@ impl RewriteRegclassCastToSubqueryVisitor { format: _, .. } = expr + && *kind == CastKind::DoubleColon { - if *kind == CastKind::DoubleColon { - let dt_lower = data_type.to_string().to_lowercase(); - if dt_lower == "oid" || dt_lower == "pg_catalog.oid" { - return self.is_regclass_cast(inner_expr); - } + let dt_lower = data_type.to_string().to_lowercase(); + if dt_lower == "oid" || dt_lower == "pg_catalog.oid" { + return self.is_regclass_cast(inner_expr); } } false @@ -478,11 +477,10 @@ impl RewriteRegclassCastToSubqueryVisitor { format: _, .. } = expr + && *kind == CastKind::DoubleColon { - if *kind == CastKind::DoubleColon { - let dt_lower = data_type.to_string().to_lowercase(); - return dt_lower == "regclass" || dt_lower == "pg_catalog.regclass"; - } + let dt_lower = data_type.to_string().to_lowercase(); + return dt_lower == "regclass" || dt_lower == "pg_catalog.regclass"; } false } @@ -495,27 +493,22 @@ impl RewriteRegclassCastToSubqueryVisitor { format: _, .. } = expr + && *kind == CastKind::DoubleColon { - if *kind == CastKind::DoubleColon { - let dt_lower = data_type.to_string().to_lowercase(); - if dt_lower == "oid" || dt_lower == "pg_catalog.oid" { - if let Expr::Cast { - kind: inner_kind, - data_type: inner_data_type, - expr: inner_inner_expr, - format: _, - .. - } = inner_expr.as_ref() - { - if *inner_kind == CastKind::DoubleColon { - let inner_dt_lower = inner_data_type.to_string().to_lowercase(); - if inner_dt_lower == "regclass" - || inner_dt_lower == "pg_catalog.regclass" - { - return Some((**inner_inner_expr).clone()); - } - } - } + let dt_lower = data_type.to_string().to_lowercase(); + if (dt_lower == "oid" || dt_lower == "pg_catalog.oid") + && let Expr::Cast { + kind: inner_kind, + data_type: inner_data_type, + expr: inner_inner_expr, + format: _, + .. + } = inner_expr.as_ref() + && *inner_kind == CastKind::DoubleColon + { + let inner_dt_lower = inner_data_type.to_string().to_lowercase(); + if inner_dt_lower == "regclass" || inner_dt_lower == "pg_catalog.regclass" { + return Some((**inner_inner_expr).clone()); } } } @@ -527,10 +520,10 @@ impl VisitorMut for RewriteRegclassCastToSubqueryVisitor { type Break = (); fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { - if self.is_regclass_to_oid_cast(expr) { - if let Some(inner_expr) = self.extract_inner_expr(expr) { - *expr = self.create_subquery(&inner_expr); - } + if self.is_regclass_to_oid_cast(expr) + && let Some(inner_expr) = self.extract_inner_expr(expr) + { + *expr = self.create_subquery(&inner_expr); } ControlFlow::Continue(()) } @@ -669,15 +662,15 @@ impl VisitorMut for PrependUnqualifiedPgTableNameVisitor { ) -> ControlFlow { if let TableFactor::Table { name, args, .. } = table_factor { // not a table function - if args.is_none() && name.0.len() == 1 { - if let ObjectNamePart::Identifier(ident) = &name.0[0] { - if ident.value.starts_with("pg_") { - *name = ObjectName(vec![ - ObjectNamePart::Identifier(Ident::new("pg_catalog")), - name.0[0].clone(), - ]); - } - } + if args.is_none() + && name.0.len() == 1 + && let ObjectNamePart::Identifier(ident) = &name.0[0] + && ident.value.starts_with("pg_") + { + *name = ObjectName(vec![ + ObjectNamePart::Identifier(Ident::new("pg_catalog")), + name.0[0].clone(), + ]); } } @@ -718,47 +711,38 @@ impl VisitorMut for FixArrayLiteralVisitor { data_type, .. } = expr + && kind == &CastKind::DoubleColon + && let DataType::Array(arr) = data_type { - if kind == &CastKind::DoubleColon { - if let DataType::Array(arr) = data_type { - // cast some to - if let Expr::Value(ValueWithSpan { - value: Value::SingleQuotedString(array_literal), - .. - }) = expr.as_ref() - { - let items = - array_literal.trim_matches(|c| c == '{' || c == '}' || c == ' '); - let items = items.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()); - - let is_text = match arr { - ArrayElemTypeDef::AngleBracket(dt) => Self::is_string_type(dt.as_ref()), - ArrayElemTypeDef::SquareBracket(dt, _) => { - Self::is_string_type(dt.as_ref()) - } - ArrayElemTypeDef::Parenthesis(dt) => Self::is_string_type(dt.as_ref()), - _ => false, - }; - - let elems = items - .map(|s| { - if is_text { - Expr::Value( - Value::SingleQuotedString(s.to_string()).with_empty_span(), - ) - } else { - Expr::Value( - Value::Number(s.to_string(), false).with_empty_span(), - ) - } - }) - .collect(); - **expr = Expr::Array(Array { - elem: elems, - named: true, - }); - } - } + // cast some to + if let Expr::Value(ValueWithSpan { + value: Value::SingleQuotedString(array_literal), + .. + }) = expr.as_ref() + { + let items = array_literal.trim_matches(|c| c == '{' || c == '}' || c == ' '); + let items = items.split(',').map(|s| s.trim()).filter(|s| !s.is_empty()); + + let is_text = match arr { + ArrayElemTypeDef::AngleBracket(dt) => Self::is_string_type(dt.as_ref()), + ArrayElemTypeDef::SquareBracket(dt, _) => Self::is_string_type(dt.as_ref()), + ArrayElemTypeDef::Parenthesis(dt) => Self::is_string_type(dt.as_ref()), + _ => false, + }; + + let elems = items + .map(|s| { + if is_text { + Expr::Value(Value::SingleQuotedString(s.to_string()).with_empty_span()) + } else { + Expr::Value(Value::Number(s.to_string(), false).with_empty_span()) + } + }) + .collect(); + **expr = Expr::Array(Array { + elem: elems, + named: true, + }); } } @@ -794,14 +778,14 @@ impl VisitorMut for RemoveQualifierVisitor { table_factor: &mut TableFactor, ) -> ControlFlow { // remove table function qualifier - if let TableFactor::Table { name, args, .. } = table_factor { - if args.is_some() { - // multiple idents in name, which means it's a qualified table name - if name.0.len() > 1 { - if let Some(last_ident) = name.0.pop() { - *name = ObjectName(vec![last_ident]); - } - } + if let TableFactor::Table { name, args, .. } = table_factor + && args.is_some() + { + // multiple idents in name, which means it's a qualified table name + if name.0.len() > 1 + && let Some(last_ident) = name.0.pop() + { + *name = ObjectName(vec![last_ident]); } } ControlFlow::Continue(()) @@ -828,10 +812,10 @@ impl VisitorMut for RemoveQualifierVisitor { Expr::Function(function) => { // remove qualifier from pg_catalog.function let name = &mut function.name; - if name.0.len() > 1 { - if let Some(last_ident) = name.0.pop() { - *name = ObjectName(vec![last_ident]); - } + if name.0.len() > 1 + && let Some(last_ident) = name.0.pop() + { + *name = ObjectName(vec![last_ident]); } } @@ -860,19 +844,20 @@ impl VisitorMut for CurrentUserVariableToSessionUserFunctionCallVisitor { type Break = (); fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { - if let Expr::Identifier(ident) = expr { - if ident.quote_style.is_none() && ident.value.to_lowercase() == "current_user" { - *expr = Expr::Function(Function { - name: ObjectName::from(vec![Ident::new("session_user")]), - args: FunctionArguments::None, - uses_odbc_syntax: false, - parameters: FunctionArguments::None, - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }); - } + if let Expr::Identifier(ident) = expr + && ident.quote_style.is_none() + && ident.value.to_lowercase() == "current_user" + { + *expr = Expr::Function(Function { + name: ObjectName::from(vec![Ident::new("session_user")]), + args: FunctionArguments::None, + uses_odbc_syntax: false, + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }); } if let Expr::Function(func) = expr { @@ -916,10 +901,10 @@ impl VisitorMut for FixCollateVisitor { *expr = inner.as_ref().clone(); } Expr::BinaryOp { op, .. } => { - if let BinaryOperator::PGCustomBinaryOperator(ops) = op { - if *ops == ["pg_catalog", "~"] { - *op = BinaryOperator::PGRegexMatch; - } + if let BinaryOperator::PGCustomBinaryOperator(ops) = op + && *ops == ["pg_catalog", "~"] + { + *op = BinaryOperator::PGRegexMatch; } } _ => {} @@ -1081,20 +1066,16 @@ impl VisitorMut for FixVersionColumnNameVisitor { fn pre_visit_query(&mut self, query: &mut Query) -> ControlFlow { if let SetExpr::Select(select) = query.body.as_mut() { for projection in &mut select.projection { - if let SelectItem::UnnamedExpr(Expr::Function(f)) = projection { - if f.name.0.len() == 1 { - if let ObjectNamePart::Identifier(part) = &f.name.0[0] { - if part.value == "version" { - if let FunctionArguments::List(args) = &f.args { - if args.args.is_empty() { - *projection = SelectItem::ExprWithAlias { - expr: Expr::Function(f.clone()), - alias: Ident::new("version"), - } - } - } - } - } + if let SelectItem::UnnamedExpr(Expr::Function(f)) = projection + && f.name.0.len() == 1 + && let ObjectNamePart::Identifier(part) = &f.name.0[0] + && part.value == "version" + && let FunctionArguments::List(args) = &f.args + && args.args.is_empty() + { + *projection = SelectItem::ExprWithAlias { + expr: Expr::Function(f.clone()), + alias: Ident::new("version"), } } } @@ -1201,7 +1182,8 @@ mod tests { "SELECT n.oid, n.*, d.description FROM pg_catalog.pg_namespace n LEFT OUTER JOIN pg_catalog.pg_description d ON d.objoid = n.oid AND d.objsubid = 0 AND d.classoid = 'pg_namespace' ORDER BY n.nspsname" ); - assert_rewrite!(&rules, + assert_rewrite!( + &rules, "SELECT i.*,i.indkey as keys,c.relname,c.relnamespace,c.relam,c.reltablespace,tc.relname as tabrelname,dsc.description FROM pg_catalog.pg_index i INNER JOIN pg_catalog.pg_class c ON c.oid=i.indexrelid INNER JOIN pg_catalog.pg_class tc ON tc.oid=i.indrelid LEFT OUTER JOIN pg_catalog.pg_description dsc ON i.indexrelid=dsc.objoid WHERE i.indrelid=1 ORDER BY tabrelname, c.relname", "SELECT i.*, i.indkey AS keys, c.relname, c.relnamespace, c.relam, c.reltablespace, tc.relname AS tabrelname, dsc.description FROM pg_catalog.pg_index i INNER JOIN pg_catalog.pg_class c ON c.oid = i.indexrelid INNER JOIN pg_catalog.pg_class tc ON tc.oid = i.indrelid LEFT OUTER JOIN pg_catalog.pg_description dsc ON i.indexrelid = dsc.objoid WHERE i.indrelid = 1 ORDER BY tabrelname, c.relname" ); @@ -1387,7 +1369,11 @@ mod tests { fn test_collate_fix() { let rules: Vec> = vec![Arc::new(FixCollate)]; - assert_rewrite!(&rules, "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname OPERATOR(pg_catalog.~) '^(tablename)$' COLLATE pg_catalog.default AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3;", "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname ~ '^(tablename)$' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3"); + assert_rewrite!( + &rules, + "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname OPERATOR(pg_catalog.~) '^(tablename)$' COLLATE pg_catalog.default AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3;", + "SELECT c.oid, c.relname FROM pg_catalog.pg_class c WHERE c.relname ~ '^(tablename)$' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 2, 3" + ); } #[test] @@ -1395,9 +1381,11 @@ mod tests { let rules: Vec> = vec![Arc::new(RemoveSubqueryFromProjection)]; - assert_rewrite!(&rules, + assert_rewrite!( + &rules, "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), (SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) FROM pg_catalog.pg_attrdef d WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef), a.attnotnull, (SELECT c.collname FROM pg_catalog.pg_collation c, pg_catalog.pg_type t WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation LIMIT 1) AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum;", - "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NULL, a.attnotnull, NULL AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum"); + "SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NULL, a.attnotnull, NULL AS attcollation, a.attidentity, a.attgenerated FROM pg_catalog.pg_attribute a WHERE a.attrelid = '16384' AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum" + ); } #[test] @@ -1405,7 +1393,8 @@ mod tests { let rules: Vec> = vec![Arc::new(RemoveSubqueryFromProjection)]; - assert_rewrite!(&rules, + assert_rewrite!( + &rules, "SELECT id, (SELECT COUNT(*) FROM pg_catalog.pg_attribute) AS attr_count FROM pg_catalog.pg_class", "SELECT id, (SELECT COUNT(*) FROM pg_catalog.pg_attribute LIMIT 1) AS attr_count FROM pg_catalog.pg_class" ); @@ -1416,7 +1405,8 @@ mod tests { let rules: Vec> = vec![Arc::new(RemoveSubqueryFromProjection)]; - assert_rewrite!(&rules, + assert_rewrite!( + &rules, "SELECT a.attname, (SELECT COUNT(*) FROM pg_catalog.pg_attribute WHERE attrelid = a.oid) AS count FROM pg_catalog.pg_attribute a", "SELECT a.attname, NULL AS count FROM pg_catalog.pg_attribute a" ); @@ -1427,7 +1417,8 @@ mod tests { let rules: Vec> = vec![Arc::new(RemoveSubqueryFromProjection)]; - assert_rewrite!(&rules, + assert_rewrite!( + &rules, "SELECT id, (SELECT attname FROM pg_catalog.pg_attribute LIMIT 1) AS first_attr FROM pg_catalog.pg_class", "SELECT id, (SELECT attname FROM pg_catalog.pg_attribute LIMIT 1) AS first_attr FROM pg_catalog.pg_class" ); diff --git a/datafusion-postgres-cli/src/main.rs b/datafusion-postgres-cli/src/main.rs index 080201a..9e20690 100644 --- a/datafusion-postgres-cli/src/main.rs +++ b/datafusion-postgres-cli/src/main.rs @@ -8,7 +8,7 @@ use datafusion::execution::options::{ use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_postgres::auth::AuthManager; use datafusion_postgres::datafusion_pg_catalog::setup_pg_catalog; -use datafusion_postgres::{serve, ServerOptions}; +use datafusion_postgres::{ServerOptions, serve}; use env_logger::Env; use log::info; use structopt::StructOpt; diff --git a/datafusion-postgres/src/auth.rs b/datafusion-postgres/src/auth.rs index 034bf3f..84cb982 100644 --- a/datafusion-postgres/src/auth.rs +++ b/datafusion-postgres/src/auth.rs @@ -567,11 +567,11 @@ impl AuthSource for SimpleAuthSource { let username = login.user().unwrap_or("anonymous"); // Check if user exists and can login - if let Some(user) = self.auth_manager.get_user(username).await { - if user.can_login { - // Return empty password for now (no authentication required) - return Ok(Password::new(None, vec![])); - } + if let Some(user) = self.auth_manager.get_user(username).await + && user.can_login + { + // Return empty password for now (no authentication required) + return Ok(Password::new(None, vec![])); } // For postgres user, always allow @@ -619,10 +619,12 @@ mod tests { // Test postgres user authentication assert!(auth_manager.authenticate("postgres", "").await.unwrap()); - assert!(!auth_manager - .authenticate("nonexistent", "password") - .await - .unwrap()); + assert!( + !auth_manager + .authenticate("nonexistent", "password") + .await + .unwrap() + ); } #[tokio::test] diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index ac5a945..f774a3b 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -9,8 +9,8 @@ use datafusion::prelude::*; use datafusion::sql::parser::Statement; use datafusion::sql::sqlparser; use log::info; -use pgwire::api::auth::noop::NoopStartupHandler; use pgwire::api::auth::StartupHandler; +use pgwire::api::auth::noop::NoopStartupHandler; use pgwire::api::cancel::{CancelHandler, DefaultCancelHandler}; use pgwire::api::portal::{Format, Portal}; use pgwire::api::query::{ExtendedQueryHandler, SimpleQueryHandler}; @@ -21,9 +21,9 @@ use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::PgWireBackendMessage; use pgwire::types::format::FormatOptions; +use crate::hooks::QueryHook; use crate::hooks::set_show::SetShowHook; use crate::hooks::transactions::TransactionStatementHook; -use crate::hooks::QueryHook; use crate::{client, planner}; use arrow_pg::datatypes::df; use arrow_pg::datatypes::{arrow_schema_to_pg_fields, into_pg_type}; @@ -242,28 +242,28 @@ impl ExtendedQueryHandler for DfSessionService { let query = &portal.statement.statement.0; log::debug!("Received execute extended query: {query}"); // Check query hooks first - if !self.query_hooks.is_empty() { - if let (_, Some((statement, plan))) = &portal.statement.statement { - // TODO: in the case where query hooks all return None, we do the param handling again later. - let param_types = planner::get_inferred_parameter_types(plan) - .map_err(|e| PgWireError::ApiError(Box::new(e)))?; - - let param_values: ParamValues = - df::deserialize_parameters(portal, &ordered_param_types(¶m_types))?; - - for hook in &self.query_hooks { - if let Some(result) = hook - .handle_extended_query( - statement, - plan, - ¶m_values, - &self.session_context, - client, - ) - .await - { - return result; - } + if !self.query_hooks.is_empty() + && let (_, Some((statement, plan))) = &portal.statement.statement + { + // TODO: in the case where query hooks all return None, we do the param handling again later. + let param_types = planner::get_inferred_parameter_types(plan) + .map_err(|e| PgWireError::ApiError(Box::new(e)))?; + + let param_values: ParamValues = + df::deserialize_parameters(portal, &ordered_param_types(¶m_types))?; + + for hook in &self.query_hooks { + if let Some(result) = hook + .handle_extended_query( + statement, + plan, + ¶m_values, + &self.session_context, + client, + ) + .await + { + return result; } } } diff --git a/datafusion-postgres/src/hooks/mod.rs b/datafusion-postgres/src/hooks/mod.rs index c1c6f58..48d5c96 100644 --- a/datafusion-postgres/src/hooks/mod.rs +++ b/datafusion-postgres/src/hooks/mod.rs @@ -9,8 +9,8 @@ use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::SessionContext; use datafusion::sql::sqlparser::ast::Statement; use futures::Sink; -use pgwire::api::results::Response; use pgwire::api::ClientInfo; +use pgwire::api::results::Response; use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::PgWireBackendMessage; diff --git a/datafusion-postgres/src/hooks/permissions.rs b/datafusion-postgres/src/hooks/permissions.rs index ac663e1..42f7594 100644 --- a/datafusion-postgres/src/hooks/permissions.rs +++ b/datafusion-postgres/src/hooks/permissions.rs @@ -5,13 +5,13 @@ use datafusion::common::ParamValues; use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::SessionContext; use datafusion::sql::sqlparser::ast::Statement; -use pgwire::api::results::Response; use pgwire::api::ClientInfo; +use pgwire::api::results::Response; use pgwire::error::{PgWireError, PgWireResult}; +use crate::QueryHook; use crate::auth::AuthManager; use crate::hooks::HookClient; -use crate::QueryHook; use datafusion_pg_catalog::pg_catalog::context::{Permission, ResourceType}; diff --git a/datafusion-postgres/src/hooks/set_show.rs b/datafusion-postgres/src/hooks/set_show.rs index 3d15179..91fd36d 100644 --- a/datafusion-postgres/src/hooks/set_show.rs +++ b/datafusion-postgres/src/hooks/set_show.rs @@ -8,18 +8,18 @@ use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::SessionContext; use datafusion::sql::sqlparser::ast::{Expr, Set, Statement}; use log::{info, warn}; +use pgwire::api::ClientInfo; use pgwire::api::auth::DefaultServerParameterProvider; use pgwire::api::results::{DataRowEncoder, FieldFormat, FieldInfo, QueryResponse, Response, Tag}; -use pgwire::api::ClientInfo; use pgwire::error::{PgWireError, PgWireResult}; -use pgwire::messages::startup::ParameterStatus; use pgwire::messages::PgWireBackendMessage; +use pgwire::messages::startup::ParameterStatus; use pgwire::types::format::FormatOptions; use postgres_types::Type; +use crate::QueryHook; use crate::client; use crate::hooks::HookClient; -use crate::QueryHook; #[derive(Debug)] pub struct SetShowHook; @@ -186,15 +186,14 @@ async fn try_respond_set_statements( if let Expr::Value(value) = value { let val_str = value.into_string().unwrap_or_else(|| "".to_string()); client.metadata_mut().insert(var.clone(), val_str); - if let Some((name, value)) = parameter_status_for_var(&var, &*client) { - if let Err(e) = client + if let Some((name, value)) = parameter_status_for_var(&var, &*client) + && let Err(e) = client .send_message(PgWireBackendMessage::ParameterStatus( ParameterStatus::new(name, value), )) .await - { - return Some(Err(e)); - } + { + return Some(Err(e)); } return Some(Ok(Response::Execution(Tag::new("SET")))); } diff --git a/datafusion-postgres/src/hooks/transactions.rs b/datafusion-postgres/src/hooks/transactions.rs index 13ef260..6ae98e8 100644 --- a/datafusion-postgres/src/hooks/transactions.rs +++ b/datafusion-postgres/src/hooks/transactions.rs @@ -5,13 +5,13 @@ use datafusion::common::ParamValues; use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::SessionContext; use datafusion::sql::sqlparser::ast::Statement; -use pgwire::api::results::{Response, Tag}; use pgwire::api::ClientInfo; +use pgwire::api::results::{Response, Tag}; use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::response::TransactionStatus; -use crate::hooks::HookClient; use crate::QueryHook; +use crate::hooks::HookClient; /// Hook for processing transaction related statements /// diff --git a/datafusion-postgres/src/lib.rs b/datafusion-postgres/src/lib.rs index e455ca6..936b21a 100644 --- a/datafusion-postgres/src/lib.rs +++ b/datafusion-postgres/src/lib.rs @@ -19,8 +19,8 @@ use rustls_pemfile::{certs, pkcs8_private_keys}; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use tokio::net::TcpListener; use tokio::sync::Semaphore; -use tokio_rustls::rustls::{self, ServerConfig}; use tokio_rustls::TlsAcceptor; +use tokio_rustls::rustls::{self, ServerConfig}; use handlers::HandlerFactory; pub use handlers::{DfSessionService, Parser}; @@ -170,7 +170,9 @@ pub async fn serve_with_handlers( match semaphore.try_acquire() { Ok(permit) => Some(permit), Err(_) => { - warn!("Connection rejected from {addr}: max connections ({max_conn_count}) reached"); + warn!( + "Connection rejected from {addr}: max connections ({max_conn_count}) reached" + ); return; } } diff --git a/datafusion-postgres/src/planner.rs b/datafusion-postgres/src/planner.rs index db33ef7..290015e 100644 --- a/datafusion-postgres/src/planner.rs +++ b/datafusion-postgres/src/planner.rs @@ -13,19 +13,18 @@ fn extract_placeholder_cast_types(plan: &LogicalPlan) -> Result DfSessionService { let session_config = SessionConfig::new().with_information_schema(true); diff --git a/datafusion-postgres/tests/pgadbc.rs b/datafusion-postgres/tests/pgadbc.rs index cd7a515..670fa56 100644 --- a/datafusion-postgres/tests/pgadbc.rs +++ b/datafusion-postgres/tests/pgadbc.rs @@ -4,8 +4,6 @@ use datafusion_postgres::testing::*; const PGADBC_QUERIES: &[&str] = &[ "SELECT attname, atttypid FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_attribute AS attr ON cls.oid = attr.attrelid INNER JOIN pg_catalog.pg_type AS typ ON attr.atttypid = typ.oid WHERE attr.attnum >= 0 AND cls.oid = 'clubs'::regclass::oid ORDER BY attr.attnum", - - ]; #[tokio::test] From 77af31f299e8a4ae9c7d1677420f340dbc31ab62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:24:22 +0800 Subject: [PATCH 30/43] chore(deps): bump rustls-pki-types from 1.14.0 to 1.14.1 (#336) Bumps [rustls-pki-types](https://github.com/rustls/pki-types) from 1.14.0 to 1.14.1. - [Release notes](https://github.com/rustls/pki-types/releases) - [Commits](https://github.com/rustls/pki-types/compare/v/1.14.0...v/1.14.1) --- updated-dependencies: - dependency-name: rustls-pki-types dependency-version: 1.14.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85236f0..7fba1c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1908,7 +1908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3699,7 +3699,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3728,9 +3728,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -3996,7 +3996,6 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -4114,7 +4113,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4625,7 +4624,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From b14ca2fcafd9ad3fdf982ce36fd49ddb23cc0728 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 20:34:42 +0800 Subject: [PATCH 31/43] chore(deps): bump arrow-schema from 58.1.0 to 58.2.0 (#337) Bumps [arrow-schema](https://github.com/apache/arrow-rs) from 58.1.0 to 58.2.0. - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/compare/58.1.0...58.2.0) --- updated-dependencies: - dependency-name: arrow-schema dependency-version: 58.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7fba1c2..9e5b89d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "58.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" dependencies = [ "serde_core", "serde_json", From f86dae39202bff59280f1249e48a428cefbcca11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 10:18:53 +0800 Subject: [PATCH 32/43] chore(deps): bump tokio from 1.52.1 to 1.52.2 (#340) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.52.1 to 1.52.2. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.1...tokio-1.52.2) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.52.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e5b89d..c6c0c0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4222,9 +4222,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" dependencies = [ "bytes", "libc", From fc02f8e7f593cd34594964227826ad4daa518bbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 10:19:04 +0800 Subject: [PATCH 33/43] chore(deps): bump rust_decimal from 1.41.0 to 1.42.0 (#341) Bumps [rust_decimal](https://github.com/paupino/rust-decimal) from 1.41.0 to 1.42.0. - [Release notes](https://github.com/paupino/rust-decimal/releases) - [Changelog](https://github.com/paupino/rust-decimal/blob/master/CHANGELOG.md) - [Commits](https://github.com/paupino/rust-decimal/compare/1.41.0...1.42.0) --- updated-dependencies: - dependency-name: rust_decimal dependency-version: 1.42.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6c0c0a..2bee5e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3664,9 +3664,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.41.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" dependencies = [ "arrayvec", "borsh", diff --git a/Cargo.toml b/Cargo.toml index 11447a9..b855ed6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ datafusion = { version = "53" } futures = "0.3" pgwire = { version = "0.39", default-features = false } postgres-types = "0.2" -rust_decimal = { version = "1.41", features = ["db-postgres"] } +rust_decimal = { version = "1.42", features = ["db-postgres"] } tokio = { version = "1", default-features = false } [profile.release] From 6aaf9b355d9d532856969adc38e93d86d1eccf1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 10:22:38 +0800 Subject: [PATCH 34/43] chore(deps): bump tokio from 1.52.2 to 1.52.3 (#343) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.52.2 to 1.52.3. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.52.2...tokio-1.52.3) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.52.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bee5e4..5257834 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4222,9 +4222,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", From 646dde3048c70dfc7160b42fd5e3ae29e4d48e5d Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 11 May 2026 10:23:59 +0800 Subject: [PATCH 35/43] feat: cursor and fetch support (#339) * feat: cursor and fetch support * feat: update cursor to support extended query * feat: use pgwire 0.40 fetch api * test: update tests for empty fetch * test: add an integration test for foreign data wrapper * ci: install podman * test: refactor integration test to use flake environment * ci: remove podman info * chore: add back podman info --- .github/workflows/ci.yml | 11 +- Cargo.lock | 22 +- Cargo.toml | 2 +- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/src/sql/parser.rs | 10 + datafusion-postgres/src/handlers.rs | 272 +++++++++++++++++++++++- datafusion-postgres/src/hooks/cursor.rs | 223 +++++++++++++++++++ datafusion-postgres/src/hooks/mod.rs | 17 +- datafusion-postgres/src/testing.rs | 8 +- flake.nix | 2 + tests-integration/test.sh | 106 +++++++-- tests-integration/test_fdw.py | 154 ++++++++++++++ 12 files changed, 778 insertions(+), 51 deletions(-) create mode 100644 datafusion-postgres/src/hooks/cursor.rs create mode 100644 tests-integration/test_fdw.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4334596..6d76796 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,15 +74,12 @@ jobs: needs: [test] steps: - uses: actions/checkout@v6 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - run: | - pip install psycopg + - uses: cachix/install-nix-action@v31 + - name: Initialize podman + run: nix develop --command podman info >/dev/null 2>&1 || true - run: | cd tests-integration - ./test.sh + nix develop --command bash test.sh msrv: name: MSRV diff --git a/Cargo.lock b/Cargo.lock index 5257834..76b2867 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -392,7 +392,7 @@ dependencies = [ "geo-traits", "geoarrow", "geoarrow-schema", - "pg_interval_2", + "pg_interval", "pgwire", "postgis", "postgres-types", @@ -1908,7 +1908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3122,10 +3122,10 @@ dependencies = [ ] [[package]] -name = "pg_interval_2" -version = "0.5.1" +name = "pg_interval" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469827e70c8c74562f88b9434cf8a8fe35665281d2442304e99efcadf8f76a8f" +checksum = "c386dd54fce258fc04e668126ae68589a0d92e03a90ea67881d1300f70fd6170" dependencies = [ "bytes", "chrono", @@ -3134,9 +3134,9 @@ dependencies = [ [[package]] name = "pgwire" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3cee243b682091188b90f22b07585ad6b43e699b52bc29422bd9ca6ce2c2deb" +checksum = "c764131468c2049ee0d9324e1f465d45f9b16d54797cb75d401a60a39429dfa9" dependencies = [ "async-trait", "base64", @@ -3147,7 +3147,7 @@ dependencies = [ "hex", "lazy-regex", "md5", - "pg_interval_2", + "pg_interval", "postgis", "postgres-types", "rand 0.10.0", @@ -3699,7 +3699,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4113,7 +4113,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4624,7 +4624,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b855ed6..4b095d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ bytes = "1.11.1" chrono = { version = "0.4", features = ["std"] } datafusion = { version = "53" } futures = "0.3" -pgwire = { version = "0.39", default-features = false } +pgwire = { version = "0.40", default-features = false } postgres-types = "0.2" rust_decimal = { version = "1.42", features = ["db-postgres"] } tokio = { version = "1", default-features = false } diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 5c30366..1daf5ca 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -27,7 +27,7 @@ datafusion = { workspace = true, optional = true } futures.workspace = true geoarrow = { version = "0.8", optional = true } geoarrow-schema = { version = "0.8", optional = true } -pg_interval = { version = "0.5.1", package = "pg_interval_2" } +pg_interval = { version = "0.5.0" } pgwire = { workspace = true, default-features = false, features = ["server-api", "pg-ext-types"] } postgres-types.workspace = true rust_decimal.workspace = true diff --git a/datafusion-pg-catalog/src/sql/parser.rs b/datafusion-pg-catalog/src/sql/parser.rs index 719df90..a918b5b 100644 --- a/datafusion-pg-catalog/src/sql/parser.rs +++ b/datafusion-pg-catalog/src/sql/parser.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use datafusion::sql::sqlparser::ast::Statement; use datafusion::sql::sqlparser::dialect::PostgreSqlDialect; +use datafusion::sql::sqlparser::keywords::Keyword; use datafusion::sql::sqlparser::parser::Parser; use datafusion::sql::sqlparser::parser::ParserError; use datafusion::sql::sqlparser::tokenizer::Token; @@ -247,10 +248,19 @@ impl PostgresCompatibilityParser { // Get token values (without spans) and filter out only whitespace // Keep semicolons as they separate statements + // Also rewrite ABORT to ROLLBACK for postgres compatibility + // remove this when https://github.com/apache/datafusion-sqlparser-rs/pull/2332 is ready let filtered_tokens: Vec = tokens .iter() .map(|t| t.token.clone()) .filter(|t| !matches!(t, Token::Whitespace(_))) + .map(|t| { + if matches!(&t, Token::Word(w) if w.keyword == Keyword::ABORT) { + Token::make_keyword("ROLLBACK") + } else { + t + } + }) .collect(); // Handle empty input diff --git a/datafusion-postgres/src/handlers.rs b/datafusion-postgres/src/handlers.rs index f774a3b..c7d5320 100644 --- a/datafusion-postgres/src/handlers.rs +++ b/datafusion-postgres/src/handlers.rs @@ -16,12 +16,16 @@ use pgwire::api::portal::{Format, Portal}; use pgwire::api::query::{ExtendedQueryHandler, SimpleQueryHandler}; use pgwire::api::results::{FieldInfo, Response, Tag}; use pgwire::api::stmt::QueryParser; -use pgwire::api::{ClientInfo, ConnectionManager, ErrorHandler, PgWireServerHandlers, Type}; +use pgwire::api::store::PortalStore; +use pgwire::api::{ + ClientInfo, ClientPortalStore, ConnectionManager, ErrorHandler, PgWireServerHandlers, Type, +}; use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::PgWireBackendMessage; use pgwire::types::format::FormatOptions; use crate::hooks::QueryHook; +use crate::hooks::cursor::CursorStatementHook; use crate::hooks::set_show::SetShowHook; use crate::hooks::transactions::TransactionStatementHook; use crate::{client, planner}; @@ -121,8 +125,11 @@ pub struct DfSessionService { impl DfSessionService { pub fn new(session_context: Arc) -> DfSessionService { - let hooks: Vec> = - vec![Arc::new(SetShowHook), Arc::new(TransactionStatementHook)]; + let hooks: Vec> = vec![ + Arc::new(CursorStatementHook), + Arc::new(SetShowHook), + Arc::new(TransactionStatementHook), + ]; Self::new_with_hooks(session_context, hooks) } @@ -147,11 +154,18 @@ impl DfSessionService { impl SimpleQueryHandler for DfSessionService { async fn do_query(&self, client: &mut C, query: &str) -> PgWireResult> where - C: ClientInfo + futures::Sink + Unpin + Send + Sync, + C: ClientInfo + + ClientPortalStore + + futures::Sink + + Unpin + + Send + + Sync, + C::PortalStore: PortalStore, C::Error: std::fmt::Debug, PgWireError: From<>::Error>, { log::debug!("Received query: {query}"); + let statements = self .parser .sql_parser @@ -235,7 +249,13 @@ impl ExtendedQueryHandler for DfSessionService { _max_rows: usize, ) -> PgWireResult where - C: ClientInfo + futures::Sink + Unpin + Send + Sync, + C: ClientInfo + + ClientPortalStore + + futures::Sink + + Unpin + + Send + + Sync, + C::PortalStore: PortalStore, C::Error: std::fmt::Debug, PgWireError: From<>::Error>, { @@ -636,4 +656,246 @@ mod tests { assert!(!has_ps, "statement_timeout should not send ParameterStatus"); } + + fn assert_execution_tag(response: &Response, expected: &str) { + match response { + Response::Execution(tag) => { + let cc = pgwire::messages::response::CommandComplete::from(tag.clone()); + assert_eq!(cc.tag, expected, "Unexpected execution tag"); + } + other => panic!("Expected Execution response, got: {other:?}"), + } + } + + async fn assert_query_response_empty(response: &mut Response) { + use futures::StreamExt; + + let Response::Query(qr) = response else { + panic!("Expected Query response, got: {response:?}"); + }; + + let mut count = 0; + while qr.data_rows().next().await.is_some() { + count += 1; + } + assert_eq!(count, 0, "Expected no rows from exhausted cursor"); + } + + #[tokio::test] + async fn test_declare_fetch_close_cursor() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + let responses = ::do_query( + &service, + &mut client, + "DECLARE test_cursor CURSOR FOR SELECT 1 AS col", + ) + .await + .unwrap(); + + assert_eq!(responses.len(), 1); + assert_execution_tag(&responses[0], "DECLARE CURSOR"); + + let responses = ::do_query( + &service, + &mut client, + "FETCH NEXT FROM test_cursor", + ) + .await + .unwrap(); + + assert_eq!(responses.len(), 1); + assert!( + matches!(&responses[0], Response::Query(_)), + "Expected Query response for FETCH" + ); + + let mut responses = ::do_query( + &service, + &mut client, + "FETCH NEXT FROM test_cursor", + ) + .await + .unwrap(); + + assert_eq!(responses.len(), 1); + assert_query_response_empty(&mut responses[0]).await; + + let responses = ::do_query( + &service, + &mut client, + "CLOSE test_cursor", + ) + .await + .unwrap(); + + assert_eq!(responses.len(), 1); + assert_execution_tag(&responses[0], "CLOSE CURSOR"); + } + + #[tokio::test] + async fn test_fetch_nonexistent_cursor() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + let result = ::do_query( + &service, + &mut client, + "FETCH NEXT FROM nonexistent", + ) + .await; + + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_close_all_portals() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + ::do_query( + &service, + &mut client, + "DECLARE c1 CURSOR FOR SELECT 1", + ) + .await + .unwrap(); + + ::do_query( + &service, + &mut client, + "DECLARE c2 CURSOR FOR SELECT 2", + ) + .await + .unwrap(); + + let responses = + ::do_query(&service, &mut client, "CLOSE ALL") + .await + .unwrap(); + + assert!(matches!(&responses[0], Response::Execution(_)),); + + let result = ::do_query( + &service, + &mut client, + "FETCH NEXT FROM c1", + ) + .await; + assert!(result.is_err(), "c1 should be closed"); + } + + #[tokio::test] + async fn test_fetch_forward_n() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + ::do_query( + &service, + &mut client, + "CREATE TABLE nums AS SELECT 1 AS n UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5", + ) + .await + .unwrap(); + + ::do_query( + &service, + &mut client, + "DECLARE mycur CURSOR FOR SELECT n FROM nums ORDER BY n", + ) + .await + .unwrap(); + + let responses = ::do_query( + &service, + &mut client, + "FETCH FORWARD 3 FROM mycur", + ) + .await + .unwrap(); + + assert!( + matches!(&responses[0], Response::Query(_)), + "Expected Query response for FORWARD 3" + ); + + let responses = ::do_query( + &service, + &mut client, + "FETCH FORWARD ALL FROM mycur", + ) + .await + .unwrap(); + + let resp_desc = match &responses[0] { + Response::Query(_) => "Query".to_string(), + Response::Execution(tag) => { + let cc = pgwire::messages::response::CommandComplete::from(tag.clone()); + format!("Execution({})", cc.tag) + } + other => format!("{:?}", other), + }; + assert!( + matches!(&responses[0], Response::Query(_)), + "Expected Query response for remaining rows, got: {resp_desc}" + ); + + let mut responses = ::do_query( + &service, + &mut client, + "FETCH NEXT FROM mycur", + ) + .await + .unwrap(); + + assert_query_response_empty(&mut responses[0]).await; + } + + #[tokio::test] + async fn test_scroll_cursor_error() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + ::do_query( + &service, + &mut client, + "DECLARE mycur CURSOR FOR SELECT 1", + ) + .await + .unwrap(); + + let result = ::do_query( + &service, + &mut client, + "FETCH PRIOR FROM mycur", + ) + .await; + + assert!(result.is_err(), "PRIOR should fail on forward-only cursor"); + } + + #[tokio::test] + async fn test_move_cursor() { + let service = crate::testing::setup_handlers(); + let mut client = MockClient::new(); + + ::do_query( + &service, + &mut client, + "DECLARE mycur CURSOR FOR SELECT generate_series(1, 5) AS n", + ) + .await + .unwrap(); + + let responses = ::do_query( + &service, + &mut client, + "FETCH FORWARD 3 FROM mycur", + ) + .await + .unwrap(); + + assert!(matches!(&responses[0], Response::Query(_))); + } } diff --git a/datafusion-postgres/src/hooks/cursor.rs b/datafusion-postgres/src/hooks/cursor.rs new file mode 100644 index 0000000..2538ea5 --- /dev/null +++ b/datafusion-postgres/src/hooks/cursor.rs @@ -0,0 +1,223 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use datafusion::common::ParamValues; +use datafusion::logical_expr::LogicalPlan; +use datafusion::prelude::SessionContext; +use datafusion::sql::sqlparser; +use datafusion::sql::sqlparser::ast::{CloseCursor, DeclareType, FetchDirection}; +use pgwire::api::ClientInfo; +use pgwire::api::portal::{Format, Portal}; +use pgwire::api::results::{Response, Tag}; +use pgwire::api::stmt::StoredStatement; +use pgwire::api::store::{MemPortalStore, PortalStore}; +use pgwire::error::{PgWireError, PgWireResult}; + +use super::{HookClient, QueryHook}; +use crate::arrow_pg::datatypes::df; + +pub(crate) type DfStatement = (String, Option<(sqlparser::ast::Statement, LogicalPlan)>); + +/// Hook for processing cursor-related statements (DECLARE/FETCH/CLOSE) +#[derive(Debug)] +pub struct CursorStatementHook; + +#[async_trait] +impl QueryHook for CursorStatementHook { + async fn handle_simple_query( + &self, + statement: &sqlparser::ast::Statement, + session_context: &SessionContext, + client: &mut dyn HookClient, + ) -> Option> { + let store = client.portal_store(); + + match statement { + sqlparser::ast::Statement::Declare { stmts } => { + Some(handle_declare(store, stmts, session_context).await) + } + sqlparser::ast::Statement::Fetch { + name, direction, .. + } => Some(handle_fetch(store, name, direction).await), + sqlparser::ast::Statement::Close { cursor } => Some(handle_close(store, cursor)), + _ => None, + } + } + + async fn handle_extended_parse_query( + &self, + statement: &sqlparser::ast::Statement, + _session_context: &SessionContext, + _client: &(dyn ClientInfo + Send + Sync), + ) -> Option> { + match statement { + sqlparser::ast::Statement::Declare { .. } + | sqlparser::ast::Statement::Fetch { .. } + | sqlparser::ast::Statement::Close { .. } => Some(Ok(LogicalPlan::EmptyRelation( + datafusion::logical_expr::EmptyRelation { + produce_one_row: false, + schema: Arc::new(datafusion::common::DFSchema::empty()), + }, + ))), + _ => None, + } + } + + async fn handle_extended_query( + &self, + statement: &sqlparser::ast::Statement, + _logical_plan: &LogicalPlan, + _params: &ParamValues, + session_context: &SessionContext, + client: &mut dyn HookClient, + ) -> Option> { + let store = client.portal_store(); + + match statement { + sqlparser::ast::Statement::Declare { stmts } => { + Some(handle_declare(store, stmts, session_context).await) + } + sqlparser::ast::Statement::Fetch { + name, direction, .. + } => Some(handle_fetch(store, name, direction).await), + sqlparser::ast::Statement::Close { cursor } => Some(handle_close(store, cursor)), + _ => None, + } + } +} + +async fn handle_declare( + store: &MemPortalStore, + stmts: &[datafusion::sql::sqlparser::ast::Declare], + session_context: &SessionContext, +) -> PgWireResult { + for declare in stmts { + if declare.declare_type != Some(DeclareType::Cursor) { + return Err(PgWireError::UserError(Box::new( + pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "42601".to_string(), + format!("unsupported DECLARE type: {:?}", declare.declare_type), + ), + ))); + } + + let cursor_name = match declare.names.first() { + Some(name) => name.value.clone(), + None => { + return Err(PgWireError::UserError(Box::new( + pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "42601".to_string(), + "cursor name is required".to_string(), + ), + ))); + } + }; + + let for_query = match &declare.for_query { + Some(q) => q.to_string(), + None => { + return Err(PgWireError::UserError(Box::new( + pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "42601".to_string(), + "DECLARE CURSOR requires a FOR query".to_string(), + ), + ))); + } + }; + + let df = session_context + .sql(&for_query) + .await + .map_err(|e| PgWireError::ApiError(Box::new(e)))?; + + let query_response = df::encode_dataframe(df, &Format::UnifiedText, None).await?; + + let stored_stmt = Arc::new(StoredStatement::new( + cursor_name.clone(), + (for_query, None), + vec![], + )); + + let portal = Portal::new_cursor(cursor_name.clone(), stored_stmt); + + portal.start(query_response).await; + + store.put_portal(Arc::new(portal)); + } + + Ok(Response::Execution(Tag::new("DECLARE CURSOR"))) +} + +async fn handle_fetch( + store: &MemPortalStore, + name: &datafusion::sql::sqlparser::ast::Ident, + direction: &FetchDirection, +) -> PgWireResult { + let cursor_name = &name.value; + + let max_rows = match direction { + FetchDirection::Next | FetchDirection::Forward { limit: None } => Some(1), + FetchDirection::Forward { limit: Some(v) } | FetchDirection::Count { limit: v } => { + parse_value_as_usize(v) + } + FetchDirection::ForwardAll | FetchDirection::All => None, + FetchDirection::Prior | FetchDirection::Backward { .. } | FetchDirection::BackwardAll => { + return Err(PgWireError::UserError(Box::new( + pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "42000".to_string(), + "cursor can only scan forward".to_string(), + ), + ))); + } + FetchDirection::First + | FetchDirection::Last + | FetchDirection::Absolute { .. } + | FetchDirection::Relative { .. } => { + return Err(PgWireError::UserError(Box::new( + pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "42000".to_string(), + "cursor can only scan forward".to_string(), + ), + ))); + } + }; + + let portal = store.get_portal(cursor_name).ok_or_else(|| { + PgWireError::UserError(Box::new(pgwire::error::ErrorInfo::new( + "ERROR".to_string(), + "34000".to_string(), + format!("cursor \"{cursor_name}\" does not exist"), + ))) + })?; + + let fetch_result = portal.fetch(max_rows.unwrap_or(0)).await?; + + Ok(Response::Query(fetch_result.response)) +} + +fn handle_close( + store: &MemPortalStore, + cursor: &CloseCursor, +) -> PgWireResult { + match cursor { + CloseCursor::All => { + store.clear_portals(); + } + CloseCursor::Specific { name } => { + store.rm_portal(&name.value); + } + } + Ok(Response::Execution(Tag::new("CLOSE CURSOR"))) +} + +fn parse_value_as_usize(value: &datafusion::sql::sqlparser::ast::Value) -> Option { + match value { + datafusion::sql::sqlparser::ast::Value::Number(s, _) => s.parse().ok(), + _ => None, + } +} diff --git a/datafusion-postgres/src/hooks/mod.rs b/datafusion-postgres/src/hooks/mod.rs index 48d5c96..c43b5b4 100644 --- a/datafusion-postgres/src/hooks/mod.rs +++ b/datafusion-postgres/src/hooks/mod.rs @@ -1,3 +1,4 @@ +pub mod cursor; pub mod permissions; pub mod set_show; pub mod transactions; @@ -10,21 +11,35 @@ use datafusion::prelude::SessionContext; use datafusion::sql::sqlparser::ast::Statement; use futures::Sink; use pgwire::api::ClientInfo; +use pgwire::api::ClientPortalStore; use pgwire::api::results::Response; +use pgwire::api::store::{MemPortalStore, PortalStore}; use pgwire::error::{PgWireError, PgWireResult}; use pgwire::messages::PgWireBackendMessage; +use crate::hooks::cursor::DfStatement; + #[async_trait] pub trait HookClient: ClientInfo + Send + Sync { + fn portal_store(&self) -> &MemPortalStore; + async fn send_message(&mut self, item: PgWireBackendMessage) -> PgWireResult<()>; } #[async_trait] impl HookClient for S where - S: ClientInfo + Sink + Send + Sync + Unpin, + S: ClientInfo + ClientPortalStore + Sink + Send + Sync + Unpin, PgWireError: From<>::Error>, + S::PortalStore: PortalStore, { + fn portal_store(&self) -> &MemPortalStore { + self.portal_store() + .as_any() + .downcast_ref::>() + .expect("portal store is not MemPortalStore") + } + async fn send_message(&mut self, item: PgWireBackendMessage) -> PgWireResult<()> { use futures::SinkExt; self.send(item).await.map_err(PgWireError::from) diff --git a/datafusion-postgres/src/testing.rs b/datafusion-postgres/src/testing.rs index a1a322e..8fb25e9 100644 --- a/datafusion-postgres/src/testing.rs +++ b/datafusion-postgres/src/testing.rs @@ -1,6 +1,8 @@ use std::{collections::HashMap, sync::Arc}; +use datafusion::logical_expr::LogicalPlan; use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion::sql::sqlparser; use datafusion_pg_catalog::pg_catalog::setup_pg_catalog; use futures::Sink; use pgwire::{ @@ -29,10 +31,12 @@ pub fn setup_handlers() -> DfSessionService { DfSessionService::new(Arc::new(session_context)) } +type DfStatement = (String, Option<(sqlparser::ast::Statement, LogicalPlan)>); + #[derive(Debug, Default)] pub struct MockClient { metadata: HashMap, - portal_store: MemPortalStore, + portal_store: MemPortalStore, pub sent_messages: Vec, session_extensions: SessionExtensions, } @@ -110,7 +114,7 @@ impl ClientInfo for MockClient { } impl ClientPortalStore for MockClient { - type PortalStore = MemPortalStore; + type PortalStore = MemPortalStore; fn portal_store(&self) -> &Self::PortalStore { &self.portal_store } diff --git a/flake.nix b/flake.nix index 4591903..fe3516b 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,8 @@ cargo-nextest cargo-release curl + lsof + podman pythonEnv postgresql_18.out ]; diff --git a/tests-integration/test.sh b/tests-integration/test.sh index 044ca7e..b5b276a 100755 --- a/tests-integration/test.sh +++ b/tests-integration/test.sh @@ -5,11 +5,14 @@ set -e # Function to cleanup processes cleanup() { echo "🧹 Cleaning up processes..." - for pid in $CSV_PID $TRANSACTION_PID $PARQUET_PID $RBAC_PID $SSL_PID $POSTGIS_PID; do + for pid in $CSV_PID $TRANSACTION_PID $PARQUET_PID $RBAC_PID $SSL_PID $POSTGIS_PID $FDW_PID; do if [ ! -z "$pid" ]; then kill -9 $pid 2>/dev/null || true fi done + if [ ! -z "$FDW_PG_CONTAINER" ]; then + podman rm -f $FDW_PG_CONTAINER 2>/dev/null || true + fi } # Trap to cleanup on exit @@ -41,19 +44,6 @@ cd .. cargo build --features datafusion-postgres/postgis cd tests-integration -# Set up test environment - -# Create virtual environment if it doesn't exist -if [ ! -d "test_env" ]; then - echo "Creating Python virtual environment..." - python3 -m venv test_env -fi - -# Activate virtual environment and install dependencies -echo "Setting up Python dependencies..." -source test_env/bin/activate -pip install -q psycopg - # Test 1: CSV data loading and PostgreSQL compatibility echo "" echo "📊 Test 1: Enhanced CSV Data Loading & PostgreSQL Compatibility" @@ -69,7 +59,7 @@ if ! ps -p $CSV_PID > /dev/null 2>&1; then exit 1 fi -if python3 test_csv.py; then +if python test_csv.py; then echo "✅ Enhanced CSV test passed" else echo "❌ Enhanced CSV test failed" @@ -80,16 +70,85 @@ fi kill -9 $CSV_PID 2>/dev/null || true sleep 3 -# Test 2: Transaction support +# Test 2: Foreign Data Wrapper (postgres_fdw) +echo "" +echo "🌍 Test 2: Foreign Data Wrapper (postgres_fdw)" +echo "-----------------------------------------------" + +# Start a PostgreSQL container for the FDW test +echo "Starting PostgreSQL container..." +FDW_PG_CONTAINER=$(podman run -d \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_DB=fdw_test \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -p 5435:5432 \ + docker.io/library/postgres:17) + +if [ -z "$FDW_PG_CONTAINER" ]; then + echo "⚠️ Could not start PostgreSQL container, skipping FDW test" +else + echo "Waiting for PostgreSQL container to be ready..." + timeout=60 + count=0 + until pg_isready -h 127.0.0.1 -p 5435 -q 2>/dev/null; do + if [ $count -ge $timeout ]; then + echo "❌ PostgreSQL container did not become ready within ${timeout}s" + echo "--- podman logs ---" + podman logs $FDW_PG_CONTAINER 2>&1 || true + echo "--- end logs ---" + podman rm -f $FDW_PG_CONTAINER 2>/dev/null || true + FDW_PG_CONTAINER="" + exit 1 + fi + sleep 1 + count=$((count + 1)) + done + echo " PostgreSQL container is ready (waited ${count}s)" + + # Start datafusion-postgres with CSV data for FDW target + wait_for_port 5433 + ../target/debug/datafusion-postgres-cli --host 0.0.0.0 -p 5433 --csv delhi:delhiclimate.csv & + FDW_PID=$! + sleep 5 + + if ! ps -p $FDW_PID > /dev/null 2>&1; then + echo "❌ DataFusion server for FDW test failed to start" + podman rm -f $FDW_PG_CONTAINER 2>/dev/null || true + exit 1 + fi + + # Run FDW test + export PGHOST=127.0.0.1 + export PGPORT=5435 + export PGUSER=postgres + export PGDATABASE=fdw_test + export DF_PORT=5433 + + if python test_fdw.py; then + echo "✅ FDW test passed" + else + echo "❌ FDW test failed" + kill -9 $FDW_PID 2>/dev/null || true + podman rm -f $FDW_PG_CONTAINER 2>/dev/null || true + exit 1 + fi + + kill -9 $FDW_PID 2>/dev/null || true + podman rm -f $FDW_PG_CONTAINER 2>/dev/null || true + FDW_PG_CONTAINER="" + sleep 3 +fi + +# Test 3: Transaction support echo "" -echo "🔐 Test 2: Transaction Support" +echo "🔐 Test 3: Transaction Support" echo "------------------------------" wait_for_port 5433 ../target/debug/datafusion-postgres-cli -p 5433 --csv delhi:delhiclimate.csv & TRANSACTION_PID=$! sleep 5 -if python3 test_transactions.py; then +if python test_transactions.py; then echo "✅ Transaction test passed" else echo "❌ Transaction test failed" @@ -100,16 +159,16 @@ fi kill -9 $TRANSACTION_PID 2>/dev/null || true sleep 3 -# Test 3: Parquet data loading and advanced data types +# Test 4: Parquet data loading and advanced data types echo "" -echo "📦 Test 3: Enhanced Parquet Data Loading & Advanced Data Types" +echo "📦 Test 4: Enhanced Parquet Data Loading & Advanced Data Types" echo "--------------------------------------------------------------" wait_for_port 5434 ../target/debug/datafusion-postgres-cli -p 5434 --parquet all_types:all_types.parquet & PARQUET_PID=$! sleep 5 -if python3 test_parquet.py; then +if python test_parquet.py; then echo "✅ Enhanced Parquet test passed" else echo "❌ Enhanced Parquet test failed" @@ -135,7 +194,7 @@ if ! ps -p $SSL_PID > /dev/null 2>&1; then exit 1 fi -if python3 test_ssl.py; then +if python test_ssl.py; then echo "✅ SSL/TLS test passed" else echo "❌ SSL/TLS test failed" @@ -161,7 +220,7 @@ if ! ps -p $POSTGIS_PID > /dev/null 2>&1; then exit 1 fi -if python3 test_postgis.py; then +if python test_postgis.py; then echo "✅ PostGIS test passed" else echo "❌ PostGIS test failed" @@ -177,6 +236,7 @@ echo "==========================================" echo "" echo "📈 Test Summary:" echo " ✅ Enhanced CSV data loading with PostgreSQL compatibility" +echo " ✅ Foreign Data Wrapper (postgres_fdw) support" echo " ✅ Complete transaction support (BEGIN/COMMIT/ROLLBACK)" echo " ✅ Enhanced Parquet data loading with advanced data types" echo " ✅ Array types and complex data type support" diff --git a/tests-integration/test_fdw.py b/tests-integration/test_fdw.py new file mode 100644 index 0000000..3ddd8ef --- /dev/null +++ b/tests-integration/test_fdw.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +Test postgres_fdw foreign data wrapper support. + +Requires a running PostgreSQL instance accessible via PGHOST/PGPORT/PGUSER/PGDATABASE env vars, +or defaults to localhost:5432 with user postgres and database fdw_test. + +The datafusion-postgres server should be running on port specified by DF_PORT (default 5433). +""" + +import os +import sys +import psycopg + + +DF_PORT = os.environ.get("DF_PORT", "5433") +PG_HOST = os.environ.get("PGHOST", "127.0.0.1") +PG_PORT = os.environ.get("PGPORT", "5432") +PG_USER = os.environ.get("PGUSER", "postgres") +PG_DB = os.environ.get("PGDATABASE", "fdw_test") + + +def main(): + print("🌍 Testing Foreign Data Wrapper (postgres_fdw)") + print("=" * 50) + + try: + conn = psycopg.connect( + f"host={PG_HOST} port={PG_PORT} user={PG_USER} dbname={PG_DB}", + autocommit=True, + ) + pg_version = conn.info.server_version + print(f" Connected to PostgreSQL {pg_version // 10000}.{pg_version % 10000 // 100}") + + setup_fdw(conn) + test_basic_query(conn) + test_aggregate_query(conn) + test_multiple_rows(conn) + test_order_by(conn) + test_cursor_lifecycle(conn) + cleanup_fdw(conn) + + conn.close() + print("\n✅ All FDW tests passed!") + return 0 + + except Exception as e: + print(f"\n❌ FDW tests failed: {e}") + import traceback + traceback.print_exc() + return 1 + + +def setup_fdw(conn): + """Set up the foreign data wrapper connecting to datafusion-postgres.""" + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS postgres_fdw") + print(" ✓ postgres_fdw extension installed") + + cur.execute("DROP SERVER IF EXISTS df_server CASCADE") + cur.execute(f""" + CREATE SERVER df_server + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host 'host.containers.internal', port '{DF_PORT}', dbname 'postgres') + """) + print(" ✓ Foreign server df_server created") + + cur.execute(f""" + CREATE USER MAPPING FOR current_user + SERVER df_server + OPTIONS (user 'postgres', password '') + """) + print(" ✓ User mapping created") + + cur.execute(""" + IMPORT FOREIGN SCHEMA public + LIMIT TO (delhi) + FROM SERVER df_server + INTO public + """) + print(" ✓ Foreign table delhi imported") + + +def test_basic_query(conn): + """Test basic SELECT through FDW.""" + with conn.cursor() as cur: + cur.execute("SELECT count(*) FROM delhi") + result = cur.fetchone()[0] + assert result > 0, f"Expected rows in delhi, got {result}" + print(f" ✓ Basic query: {result} rows in delhi") + + +def test_aggregate_query(conn): + """Test aggregate functions through FDW.""" + with conn.cursor() as cur: + cur.execute("SELECT avg(meantemp), max(humidity) FROM delhi") + row = cur.fetchone() + assert row[0] is not None, "Expected non-null avg(meantemp)" + assert row[1] is not None, "Expected non-null max(humidity)" + print(f" ✓ Aggregate query: avg(meantemp)={row[0]:.2f}, max(humidity)={row[1]}") + + +def test_multiple_rows(conn): + """Test fetching multiple rows through FDW.""" + with conn.cursor() as cur: + cur.execute("SELECT date, meantemp FROM delhi ORDER BY date LIMIT 5") + rows = cur.fetchall() + assert len(rows) == 5, f"Expected 5 rows, got {len(rows)}" + print(f" ✓ Multiple rows: fetched {len(rows)} rows") + + +def test_order_by(conn): + """Test ORDER BY through FDW.""" + with conn.cursor() as cur: + cur.execute("SELECT date, meantemp FROM delhi ORDER BY meantemp DESC LIMIT 3") + rows = cur.fetchall() + temps = [row[1] for row in rows] + assert temps == sorted(temps, reverse=True), "Expected descending order" + print(f" ✓ ORDER BY: top 3 temps = {temps}") + + +def test_cursor_lifecycle(conn): + """Test DECLARE/FETCH/CLOSE cursor through FDW.""" + with conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("DECLARE fdw_cur CURSOR FOR SELECT date, meantemp FROM delhi ORDER BY date") + print(" ✓ DECLARE CURSOR") + + cur.execute("FETCH FORWARD 3 FROM fdw_cur") + rows = cur.fetchall() + assert len(rows) == 3, f"Expected 3 rows from FETCH, got {len(rows)}" + print(f" ✓ FETCH FORWARD 3: got {len(rows)} rows") + + cur.execute("FETCH NEXT FROM fdw_cur") + row = cur.fetchone() + assert row is not None, "Expected a row from FETCH NEXT" + print(" ✓ FETCH NEXT: got 1 row") + + cur.execute("CLOSE fdw_cur") + cur.execute("COMMIT") + print(" ✓ CLOSE + COMMIT") + + +def cleanup_fdw(conn): + """Clean up FDW objects.""" + with conn.cursor() as cur: + cur.execute("DROP FOREIGN TABLE IF EXISTS delhi CASCADE") + cur.execute("DROP USER MAPPING IF EXISTS FOR current_user SERVER df_server") + cur.execute("DROP SERVER IF EXISTS df_server CASCADE") + print(" ✓ FDW objects cleaned up") + + +if __name__ == "__main__": + sys.exit(main()) From a5a1780293525816e6992b7da4db61e3c8e098e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 16:16:13 +0800 Subject: [PATCH 36/43] chore(deps): bump arrow-schema from 58.2.0 to 58.3.0 (#344) Bumps [arrow-schema](https://github.com/apache/arrow-rs) from 58.2.0 to 58.3.0. - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/arrow-rs/commits) --- updated-dependencies: - dependency-name: arrow-schema dependency-version: 58.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76b2867..fd62cc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ "serde_core", "serde_json", From e384928b1462dc929830fb9b24d5a766dc36153d Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sun, 17 May 2026 10:50:14 -0700 Subject: [PATCH 37/43] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'fenix': 'github:nix-community/fenix/f1cf6dcfc51a08e154ec753f11146dc1d95be1e5?narHash=sha256-EIP2/UXzbkb1ZWoa0pTwNv2ZivlfbRT%2B7j0fj36IhWg%3D' (2026-01-25) → 'github:nix-community/fenix/3c09e6c592c6ebfb9cde94bba990af23b2c06a87?narHash=sha256-AIbVmDAJFVRl%2BPUkgoOHVvzulVLQW0pcltKNrBVWxww%3D' (2026-05-17) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/cc76a8a0c6039abb17e005aa5f96c9d13d4d9665?narHash=sha256-LoyIkk68hFEwE49o8rzYK50bqX8jWhjcxpYLz2lBCbI%3D' (2026-01-24) → 'github:rust-lang/rust-analyzer/ace47025ddfc7fec399e11594318e64443c0e1c2?narHash=sha256-bNCTsqjqpSVXDKsCKHTyNEaw6LpaiRUoUVKlVDSzjHk%3D' (2026-05-16) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/48698d12cc10555a4f3e3222d9c669b884a49dfe?narHash=sha256-yxgb4AmkVHY5OOBrC79Vv6EVd4QZEotqv%2B6jcvA212M%3D' (2026-01-25) → 'github:NixOS/nixpkgs/d233902339c02a9c334e7e593de68855ad26c4cb?narHash=sha256-30sZNZoA1cqF5JNO9fVX%2BwgiQYjB7HJqqJ4ztCDeBZE%3D' (2026-05-15) --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 125dfd9..eac01da 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1769323446, - "narHash": "sha256-EIP2/UXzbkb1ZWoa0pTwNv2ZivlfbRT+7j0fj36IhWg=", + "lastModified": 1779007028, + "narHash": "sha256-AIbVmDAJFVRl+PUkgoOHVvzulVLQW0pcltKNrBVWxww=", "owner": "nix-community", "repo": "fenix", - "rev": "f1cf6dcfc51a08e154ec753f11146dc1d95be1e5", + "rev": "3c09e6c592c6ebfb9cde94bba990af23b2c06a87", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1769330179, - "narHash": "sha256-yxgb4AmkVHY5OOBrC79Vv6EVd4QZEotqv+6jcvA212M=", + "lastModified": 1778869304, + "narHash": "sha256-30sZNZoA1cqF5JNO9fVX+wgiQYjB7HJqqJ4ztCDeBZE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "48698d12cc10555a4f3e3222d9c669b884a49dfe", + "rev": "d233902339c02a9c334e7e593de68855ad26c4cb", "type": "github" }, "original": { @@ -65,11 +65,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1769275938, - "narHash": "sha256-LoyIkk68hFEwE49o8rzYK50bqX8jWhjcxpYLz2lBCbI=", + "lastModified": 1778958756, + "narHash": "sha256-bNCTsqjqpSVXDKsCKHTyNEaw6LpaiRUoUVKlVDSzjHk=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "cc76a8a0c6039abb17e005aa5f96c9d13d4d9665", + "rev": "ace47025ddfc7fec399e11594318e64443c0e1c2", "type": "github" }, "original": { From 84c43e961d591447eecc317de66a21045fd26784 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 25 May 2026 11:34:18 -0700 Subject: [PATCH 38/43] chore: Release --- Cargo.lock | 8 ++++---- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 4 ++-- datafusion-postgres-cli/Cargo.toml | 4 ++-- datafusion-postgres/Cargo.toml | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd62cc3..c90743d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,7 +379,7 @@ dependencies = [ [[package]] name = "arrow-pg" -version = "0.13.0" +version = "0.14.0" dependencies = [ "arrow", "arrow-schema", @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.16.0" +version = "0.17.0" dependencies = [ "arrow-pg", "async-trait", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "datafusion-postgres" -version = "0.16.0" +version = "0.17.0" dependencies = [ "arrow-pg", "async-trait", @@ -1753,7 +1753,7 @@ dependencies = [ [[package]] name = "datafusion-postgres-cli" -version = "0.16.0" +version = "0.17.0" dependencies = [ "datafusion", "datafusion-postgres", diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 1daf5ca..289ca5e 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "arrow-pg" description = "Arrow data mapping and encoding/decoding for Postgres" -version = "0.13.0" +version = "0.14.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 54bf979..00b2ce9 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-pg-catalog" description = "pg_catalog compatibility for datafusion" -version = "0.16.0" +version = "0.17.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -20,7 +20,7 @@ include = [ [dependencies] async-trait = "0.1" datafusion = { workspace = true, features = ["sql"] } -arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, features = ["datafusion"] } +arrow-pg = { path = "../arrow-pg", version = "0.14.0", default-features = false, features = ["datafusion"] } futures.workspace = true log = "0.4" postgres-types.workspace = true diff --git a/datafusion-postgres-cli/Cargo.toml b/datafusion-postgres-cli/Cargo.toml index 25fbc0d..f0805dc 100644 --- a/datafusion-postgres-cli/Cargo.toml +++ b/datafusion-postgres-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-postgres-cli" description = "Command-line tool for DataFusion's Postgres Protocol frontend" -version = "0.16.0" +version = "0.17.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -15,7 +15,7 @@ rust-version.workspace = true [dependencies] datafusion = { workspace = true, features = ["avro"] } tokio = { workspace = true, features = ["full"] } -datafusion-postgres = { path = "../datafusion-postgres", version = "0.16.0" } +datafusion-postgres = { path = "../datafusion-postgres", version = "0.17.0" } structopt = { version = "0.3", default-features = false } log = "0.4" env_logger = "0.11" diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 1f13a8c..e1df902 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-postgres" description = "Exporting datafusion query engine with postgres wire protocol" -version = "0.16.0" +version = "0.17.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -13,12 +13,12 @@ readme = "../README.md" rust-version.workspace = true [dependencies] -arrow-pg = { path = "../arrow-pg", version = "0.13.0", default-features = false, features = ["datafusion"] } +arrow-pg = { path = "../arrow-pg", version = "0.14.0", default-features = false, features = ["datafusion"] } bytes.workspace = true async-trait = "0.1" chrono.workspace = true datafusion.workspace = true -datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.16.0" } +datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.17.0" } geodatafusion = { version = "0.4", optional = true } futures.workspace = true getset = "0.1" From f6f5f211ff05575e61cc0dff3c56ba9b26342b21 Mon Sep 17 00:00:00 2001 From: discord9 Date: Thu, 25 Jun 2026 20:52:54 +0800 Subject: [PATCH 39/43] fix(pg-catalog): restrict unqualified pg table rewrite (#364) --- datafusion-pg-catalog/src/sql/rules.rs | 33 ++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index dd2f8fe..d57e7d2 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -2,6 +2,8 @@ use std::collections::HashSet; use std::fmt::Debug; use std::ops::ControlFlow; +use crate::pg_catalog::PG_CATALOG_TABLES; + use datafusion::sql::sqlparser::ast::Array; use datafusion::sql::sqlparser::ast::ArrayElemTypeDef; use datafusion::sql::sqlparser::ast::BinaryOperator; @@ -647,10 +649,19 @@ impl SqlStatementRewriteRule for RewriteArrayAnyAllOperation { /// Prepend qualifier to table_name /// /// Postgres has pg_catalog in search_path by default so it allow access to -/// `pg_namespace` without `pg_catalog.` qualifier +/// `pg_namespace` without `pg_catalog.` qualifier. +/// +/// Only names of known pg_catalog relations are qualified. A simple `pg_` +/// prefix check would also rewrite user tables such as `pg_compat_test`. #[derive(Debug)] pub struct PrependUnqualifiedPgTableName; +fn is_pg_catalog_table(name: &str) -> bool { + PG_CATALOG_TABLES + .iter() + .any(|table| table.eq_ignore_ascii_case(name)) +} + struct PrependUnqualifiedPgTableNameVisitor; impl VisitorMut for PrependUnqualifiedPgTableNameVisitor { @@ -665,7 +676,7 @@ impl VisitorMut for PrependUnqualifiedPgTableNameVisitor { if args.is_none() && name.0.len() == 1 && let ObjectNamePart::Identifier(ident) = &name.0[0] - && ident.value.starts_with("pg_") + && is_pg_catalog_table(&ident.value) { *name = ObjectName(vec![ ObjectNamePart::Identifier(Ident::new("pg_catalog")), @@ -1308,11 +1319,29 @@ mod tests { "SELECT * FROM pg_catalog.pg_namespace" ); + assert_rewrite!( + &rules, + "SELECT * FROM pg_type", + "SELECT * FROM pg_catalog.pg_type" + ); + assert_rewrite!( &rules, "SELECT typtype, typname, pg_type.oid FROM pg_catalog.pg_type LEFT JOIN pg_namespace as ns ON ns.oid = oid", "SELECT typtype, typname, pg_type.oid FROM pg_catalog.pg_type LEFT JOIN pg_catalog.pg_namespace AS ns ON ns.oid = oid" ); + + assert_rewrite!( + &rules, + "SELECT * FROM pg_compat_test", + "SELECT * FROM pg_compat_test" + ); + + assert_rewrite!( + &rules, + "SELECT * FROM pg_custom_table", + "SELECT * FROM pg_custom_table" + ); } #[test] From b0269e473a4d9058f2bf427651d5b00d44363da3 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Thu, 25 Jun 2026 21:54:02 +0800 Subject: [PATCH 40/43] chore: Release --- Cargo.lock | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c90743d..3313555 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.17.0" +version = "0.17.2" dependencies = [ "arrow-pg", "async-trait", diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 00b2ce9..954ed13 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-pg-catalog" description = "pg_catalog compatibility for datafusion" -version = "0.17.0" +version = "0.17.2" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index e1df902..082a401 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -18,7 +18,7 @@ bytes.workspace = true async-trait = "0.1" chrono.workspace = true datafusion.workspace = true -datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.17.0" } +datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.17.2" } geodatafusion = { version = "0.4", optional = true } futures.workspace = true getset = "0.1" From 911bf4702ebe03c09a33854761b3c8aa8c8b2c9f Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sat, 4 Jul 2026 21:58:52 +0800 Subject: [PATCH 41/43] feat: add support for select current_catalog, select current_schema --- datafusion-pg-catalog/src/sql/rules.rs | 141 +++++++++++++++++++++---- 1 file changed, 118 insertions(+), 23 deletions(-) diff --git a/datafusion-pg-catalog/src/sql/rules.rs b/datafusion-pg-catalog/src/sql/rules.rs index d57e7d2..436d878 100644 --- a/datafusion-pg-catalog/src/sql/rules.rs +++ b/datafusion-pg-catalog/src/sql/rules.rs @@ -845,42 +845,119 @@ impl SqlStatementRewriteRule for RemoveQualifier { } } -/// Replace `current_user` with `session_user()` +/// Rewrite postgres built-in special registers that may be referenced +/// without parentheses into the function-call form DataFusion understands. +/// +/// Postgres allows several built-in special registers to be used without +/// parentheses -- e.g. `current_user`, `current_catalog`, `current_schema`. +/// DataFusion only exposes these as function calls, so this rule rewrites the +/// parenthesis-less forms. sqlparser represents them in two different ways, +/// so the rule handles both: +/// +/// * `current_user` and `current_catalog` are sqlparser-special-cased +/// keywords that already parse as a zero-argument `Function` (with +/// `FunctionArguments::None`). They only need renaming, because DataFusion +/// exposes them under different names: `current_user` -> `session_user`, +/// `current_catalog` -> `current_database`. +/// * `current_schema` parses as a bare identifier and is wrapped into a +/// zero-argument function call: `current_schema` -> `current_schema()`. +/// +/// Because the rewritten AST is re-serialised and re-parsed by DataFusion +/// (see `handlers.rs`), the emitted form must round-trip through the postgres +/// dialect. sqlparser only accepts the parenthesis-less spelling for the +/// special-cased keywords [`PARENTHESIS_LESS_KEYWORDS`]; every other target +/// (e.g. `current_database`) must be emitted with explicit `()`. +/// +/// Note: `version` is deliberately NOT handled here -- in postgres it is a +/// regular function, not a special register, so `SELECT version` (no parens) +/// is invalid and never reaches this rule. #[derive(Debug)] pub struct CurrentUserVariableToSessionUserFunctionCall; +/// `(postgres function name, target function name)` for the parenthesis-less +/// `Function` nodes produced by sqlparser that need renaming. +const BUILTIN_FUNCTION_RENAME: &[(&str, &str)] = &[ + ("current_user", "session_user"), + ("current_catalog", "current_database"), +]; + +/// sqlparser special-cases these keywords into a parenthesis-less zero-argument +/// `Function` (see sqlparser's `Parser::parse_word`). They -- and only they -- +/// may be emitted without parentheses; `name()` does not re-parse for them, so +/// any other target must be serialised as an explicit `name()` call. +const PARENTHESIS_LESS_KEYWORDS: &[&str] = + &["current_catalog", "current_user", "session_user", "user"]; + +/// `(identifier, target function name)` for bare identifiers that must be +/// wrapped into a zero-argument function call. +const BARE_IDENTIFIER_TO_FUNCTION: &[(&str, &str)] = &[("current_schema", "current_schema")]; + +fn empty_function_call(name: &str) -> Expr { + Expr::Function(Function { + name: ObjectName::from(vec![Ident::new(name)]), + args: FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }), + uses_odbc_syntax: false, + parameters: FunctionArguments::None, + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} + struct CurrentUserVariableToSessionUserFunctionCallVisitor; +impl CurrentUserVariableToSessionUserFunctionCallVisitor { + fn last_ident_lower(name: &ObjectName) -> Option { + name.0 + .last() + .and_then(|part| part.as_ident()) + .map(|ident| ident.value.to_lowercase()) + } +} + impl VisitorMut for CurrentUserVariableToSessionUserFunctionCallVisitor { type Break = (); fn pre_visit_expr(&mut self, expr: &mut Expr) -> ControlFlow { + // Bare identifier -> zero-argument function call. if let Expr::Identifier(ident) = expr && ident.quote_style.is_none() - && ident.value.to_lowercase() == "current_user" { - *expr = Expr::Function(Function { - name: ObjectName::from(vec![Ident::new("session_user")]), - args: FunctionArguments::None, - uses_odbc_syntax: false, - parameters: FunctionArguments::None, - filter: None, - null_treatment: None, - over: None, - within_group: vec![], - }); + let lower = ident.value.to_lowercase(); + if let Some(&(_, target)) = BARE_IDENTIFIER_TO_FUNCTION + .iter() + .find(|(name, _)| *name == lower) + { + *expr = empty_function_call(target); + } } - if let Expr::Function(func) = expr { - let fname = func - .name - .0 + // Parenthesis-less `Function` (or explicit call) -> rename where needed. + // A parenthesis-less `FunctionArguments::None` is only kept as-is when + // the *target* is itself a parenthesis-less special keyword; otherwise + // it is normalised to an empty argument list so the rewritten SQL + // re-parses as a real `name()` call (the parenthesis-less spelling does + // not round-trip for non-special names such as `current_database`). + if let Expr::Function(func) = expr + && let Some(fname) = Self::last_ident_lower(&func.name) + && let Some(&(_, target)) = BUILTIN_FUNCTION_RENAME .iter() - .map(|ident| ident.to_string()) - .collect::>() - .join("."); - if fname.to_lowercase() == "current_user" { - func.name = ObjectName::from(vec![Ident::new("session_user")]) + .find(|(name, _)| *name == fname) + { + func.name = ObjectName::from(vec![Ident::new(target)]); + if matches!(func.args, FunctionArguments::None) + && !PARENTHESIS_LESS_KEYWORDS.contains(&target) + { + func.args = FunctionArguments::List(FunctionArgumentList { + args: vec![], + duplicate_treatment: None, + clauses: vec![], + }); } } @@ -1383,15 +1460,33 @@ mod tests { let rules: Vec> = vec![Arc::new(CurrentUserVariableToSessionUserFunctionCall)]; + // current_user -> session_user. `session_user` is a sqlparser-special + // keyword, so it must stay parenthesis-less to round-trip through the + // postgres dialect (re-parsed by DataFusion downstream). assert_rewrite!(&rules, "SELECT current_user", "SELECT session_user"); - assert_rewrite!(&rules, "SELECT CURRENT_USER", "SELECT session_user"); - assert_rewrite!( &rules, "SELECT is_null(current_user)", "SELECT is_null(session_user)" ); + + // current_catalog -> current_database. `current_database` is NOT a + // special keyword, so it needs explicit `()` to re-parse as a call. + assert_rewrite!( + &rules, + "SELECT current_catalog", + "SELECT current_database()" + ); + + // version is deliberately not rewritten: in postgres it is a regular + // function, not a special register, so `SELECT version` (no parens) is + // invalid and `SELECT version()` is already the canonical form. + assert_rewrite!(&rules, "SELECT version", "SELECT version"); + assert_rewrite!(&rules, "SELECT version()", "SELECT version()"); + + // current_schema -> current_schema() + assert_rewrite!(&rules, "SELECT current_schema", "SELECT current_schema()"); } #[test] From b1f524292699bff9c380e242cc80ab0220c92748 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Sat, 4 Jul 2026 22:36:37 +0800 Subject: [PATCH 42/43] chore: Release --- Cargo.lock | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-postgres/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3313555..7c48f71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1607,7 +1607,7 @@ dependencies = [ [[package]] name = "datafusion-pg-catalog" -version = "0.17.2" +version = "0.17.3" dependencies = [ "arrow-pg", "async-trait", diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index 954ed13..fb18a35 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datafusion-pg-catalog" description = "pg_catalog compatibility for datafusion" -version = "0.17.2" +version = "0.17.3" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/datafusion-postgres/Cargo.toml b/datafusion-postgres/Cargo.toml index 082a401..1518338 100644 --- a/datafusion-postgres/Cargo.toml +++ b/datafusion-postgres/Cargo.toml @@ -18,7 +18,7 @@ bytes.workspace = true async-trait = "0.1" chrono.workspace = true datafusion.workspace = true -datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.17.2" } +datafusion-pg-catalog = { path = "../datafusion-pg-catalog", version = "0.17.3" } geodatafusion = { version = "0.4", optional = true } futures.workspace = true getset = "0.1" From f1bfdcbc26304e1ff2a509a2e97f3efc6aabe488 Mon Sep 17 00:00:00 2001 From: discord9 Date: Thu, 16 Jul 2026 10:43:37 +0800 Subject: [PATCH 43/43] feat: support DataFusion 54 Signed-off-by: discord9 --- Cargo.lock | 1269 +++++++++++++---- arrow-pg/Cargo.toml | 2 +- datafusion-pg-catalog/Cargo.toml | 2 +- datafusion-pg-catalog/src/pg_catalog.rs | 69 +- .../src/pg_catalog/format_type.rs | 4 - .../src/pg_catalog/has_privilege_udf.rs | 4 - .../src/pg_catalog/pg_get_expr_udf.rs | 4 - .../src/pg_catalog/quote_ident_udf.rs | 4 - 8 files changed, 1080 insertions(+), 278 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c48f71..bee9876 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,7 +153,7 @@ dependencies = [ "bon", "bzip2", "crc32fast", - "digest", + "digest 0.10.7", "liblzma", "log", "miniz_oxide", @@ -210,9 +210,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ "arrow-arith", "arrow-array", @@ -231,9 +231,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -256,7 +256,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ "arrow-array", "arrow-buffer", @@ -298,9 +298,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ "arrow-array", "arrow-cast", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,15 +342,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap", @@ -366,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -386,7 +387,7 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion", + "datafusion 54.0.0", "futures", "geo-postgis", "geo-traits", @@ -402,9 +403,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ "arrow-array", "arrow-buffer", @@ -425,9 +426,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -439,9 +440,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ "arrow-array", "arrow-buffer", @@ -558,7 +559,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -583,6 +584,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bon" version = "3.8.2" @@ -871,6 +881,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1006,6 +1022,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1088,32 +1113,32 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-arrow", + "datafusion-catalog 53.1.0", + "datafusion-catalog-listing 53.1.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-datasource-arrow 53.1.0", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-datasource-csv 53.1.0", + "datafusion-datasource-json 53.1.0", + "datafusion-datasource-parquet 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-nested 53.1.0", + "datafusion-functions-table 53.1.0", + "datafusion-functions-window 53.1.0", + "datafusion-optimizer 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-optimizer 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "datafusion-sql 53.1.0", "flate2", "futures", "itertools 0.14.0", @@ -1124,7 +1149,60 @@ dependencies = [ "parquet", "rand 0.9.2", "regex", - "sqlparser", + "sqlparser 0.61.0", + "tempfile", + "tokio", + "url", + "uuid", + "zstd", +] + +[[package]] +name = "datafusion" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog 54.0.0", + "datafusion-catalog-listing 54.0.0", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-datasource-arrow 54.0.0", + "datafusion-datasource-csv 54.0.0", + "datafusion-datasource-json 54.0.0", + "datafusion-datasource-parquet 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-nested 54.0.0", + "datafusion-functions-table 54.0.0", + "datafusion-functions-window 54.0.0", + "datafusion-optimizer 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-optimizer 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", + "datafusion-sql 54.0.0", + "flate2", + "futures", + "indexmap", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser 0.62.0", "tempfile", "tokio", "url", @@ -1141,14 +1219,39 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1165,15 +1268,38 @@ checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1201,11 +1327,37 @@ dependencies = [ "parquet", "paste", "recursive", - "sqlparser", + "sqlparser 0.61.0", "tokio", "web-time", ] +[[package]] +name = "datafusion-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "libc", + "log", + "object_store", + "parquet", + "recursive", + "sqlparser 0.62.0", + "tokio", + "uuid", + "web-time", +] + [[package]] name = "datafusion-common-runtime" version = "53.1.0" @@ -1217,6 +1369,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-common-runtime" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-datasource" version = "53.1.0" @@ -1229,15 +1392,50 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "liblzma", + "log", + "object_store", + "rand 0.9.2", + "tokio", + "tokio-util", + "url", + "zstd", +] + +[[package]] +name = "datafusion-datasource" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "flate2", "futures", "glob", @@ -1245,6 +1443,7 @@ dependencies = [ "liblzma", "log", "object_store", + "parking_lot", "rand 0.9.2", "tokio", "tokio-util", @@ -1262,14 +1461,38 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "object_store", @@ -1286,11 +1509,11 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "num-traits", "object_store", @@ -1305,14 +1528,37 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", "futures", "object_store", "regex", @@ -1328,14 +1574,14 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "object_store", "serde_json", @@ -1343,6 +1589,29 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "datafusion-datasource-json" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-session 54.0.0", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" @@ -1352,18 +1621,49 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", + "datafusion-session 53.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-adapter 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", + "datafusion-session 54.0.0", "futures", "itertools 0.14.0", "log", @@ -1379,6 +1679,12 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +[[package]] +name = "datafusion-doc" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" + [[package]] name = "datafusion-execution" version = "53.1.0" @@ -1390,9 +1696,31 @@ dependencies = [ "async-trait", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", "futures", "log", "object_store", @@ -1411,18 +1739,41 @@ dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "indexmap", "itertools 0.14.0", "paste", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-expr" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "indexmap", + "itertools 0.14.0", + "recursive", + "serde_json", + "sqlparser 0.62.0", ] [[package]] @@ -1432,12 +1783,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.1.0", "indexmap", "itertools 0.14.0", "paste", ] +[[package]] +name = "datafusion-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "indexmap", + "itertools 0.14.0", +] + [[package]] name = "datafusion-functions" version = "53.1.0" @@ -1451,25 +1814,57 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-macros 53.1.0", "hex", "itertools 0.14.0", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-traits", "rand 0.9.2", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] +[[package]] +name = "datafusion-functions" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "hex", + "itertools 0.14.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.2", + "regex", + "sha2 0.11.0", + "uuid", +] + [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" @@ -1478,20 +1873,41 @@ checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "log", "num-traits", "paste", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" @@ -1500,9 +1916,21 @@ checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] @@ -1513,16 +1941,16 @@ checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr-common 53.1.0", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -1530,6 +1958,31 @@ dependencies = [ "paste", ] +[[package]] +name = "datafusion-functions-nested" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + [[package]] name = "datafusion-functions-table" version = "53.1.0" @@ -1538,14 +1991,30 @@ checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "parking_lot", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 54.0.0", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-plan 54.0.0", + "parking_lot", +] + [[package]] name = "datafusion-functions-window" version = "53.1.0" @@ -1553,25 +2022,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "log", "paste", ] +[[package]] +name = "datafusion-functions-window" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-doc 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-macros 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "log", +] + [[package]] name = "datafusion-functions-window-common" version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +dependencies = [ + "datafusion-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", ] [[package]] @@ -1580,7 +2076,18 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.1.0", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-macros" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +dependencies = [ + "datafusion-doc 54.0.0", "quote", "syn 2.0.117", ] @@ -1593,10 +2100,30 @@ checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "indexmap", + "itertools 0.14.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", "indexmap", "itertools 0.14.0", "log", @@ -1612,7 +2139,7 @@ dependencies = [ "arrow-pg", "async-trait", "criterion", - "datafusion", + "datafusion 54.0.0", "env_logger", "futures", "log", @@ -1628,11 +2155,11 @@ checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "hashbrown 0.16.1", "indexmap", @@ -1644,6 +2171,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-expr" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" @@ -1651,11 +2200,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", "itertools 0.14.0", ] @@ -1668,14 +2232,31 @@ dependencies = [ "ahash 0.8.12", "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "parking_lot", ] +[[package]] +name = "datafusion-physical-expr-common" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr-common 54.0.0", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" @@ -1683,14 +2264,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", + "itertools 0.14.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "datafusion-pruning 54.0.0", "itertools 0.14.0", "recursive", ] @@ -1706,15 +2306,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "half", "hashbrown 0.16.1", @@ -1727,6 +2327,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-physical-plan" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common 54.0.0", + "datafusion-common-runtime 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions 54.0.0", + "datafusion-functions-aggregate-common 54.0.0", + "datafusion-functions-window-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-postgres" version = "0.17.0" @@ -1735,7 +2368,7 @@ dependencies = [ "async-trait", "bytes", "chrono", - "datafusion", + "datafusion 53.1.0", "datafusion-pg-catalog", "env_logger", "futures", @@ -1755,7 +2388,7 @@ dependencies = [ name = "datafusion-postgres-cli" version = "0.17.0" dependencies = [ - "datafusion", + "datafusion 53.1.0", "datafusion-postgres", "env_logger", "log", @@ -1770,16 +2403,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", "itertools 0.14.0", "log", ] +[[package]] +name = "datafusion-pruning" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +dependencies = [ + "arrow", + "datafusion-common 54.0.0", + "datafusion-datasource 54.0.0", + "datafusion-expr-common 54.0.0", + "datafusion-physical-expr 54.0.0", + "datafusion-physical-expr-common 54.0.0", + "datafusion-physical-plan 54.0.0", + "log", +] + [[package]] name = "datafusion-session" version = "53.1.0" @@ -1787,10 +2436,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +dependencies = [ + "async-trait", + "datafusion-common 54.0.0", + "datafusion-execution 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-physical-plan 54.0.0", "parking_lot", ] @@ -1803,14 +2466,33 @@ dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-nested 53.1.0", "indexmap", "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.61.0", +] + +[[package]] +name = "datafusion-sql" +version = "54.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common 54.0.0", + "datafusion-expr 54.0.0", + "datafusion-functions-nested 54.0.0", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser 0.62.0", ] [[package]] @@ -1819,7 +2501,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -1840,11 +2522,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1908,7 +2601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2205,7 +2898,7 @@ dependencies = [ "arrow-arith", "arrow-array", "arrow-schema", - "datafusion", + "datafusion 53.1.0", "geo", "geo-traits", "geoarrow-array", @@ -2348,6 +3041,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "heapless" version = "0.8.0" @@ -2385,7 +3089,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -2404,6 +3108,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "i_float" version = "1.15.0" @@ -2587,12 +3300,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2786,9 +3499,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "liblzma" @@ -2859,7 +3572,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -3053,9 +3776,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -3071,7 +3794,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", "num-bigint", "num-integer", @@ -3184,6 +3907,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3261,10 +4004,10 @@ dependencies = [ "bytes", "fallible-iterator", "hmac", - "md-5", + "md-5 0.10.6", "memchr", "rand 0.9.2", - "sha2", + "sha2 0.10.9", "stringprep", ] @@ -3699,7 +4442,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3852,7 +4595,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3969,6 +4723,17 @@ dependencies = [ "sqlparser_derive", ] +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + [[package]] name = "sqlparser_derive" version = "0.5.0" @@ -4113,7 +4878,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4352,9 +5117,9 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-bidi" @@ -4624,7 +5389,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/arrow-pg/Cargo.toml b/arrow-pg/Cargo.toml index 289ca5e..5a4660e 100644 --- a/arrow-pg/Cargo.toml +++ b/arrow-pg/Cargo.toml @@ -23,7 +23,7 @@ arrow = { workspace = true, optional = true } arrow-schema = { workspace = true} bytes.workspace = true chrono.workspace = true -datafusion = { workspace = true, optional = true } +datafusion = { version = "=54.0.0", optional = true } futures.workspace = true geoarrow = { version = "0.8", optional = true } geoarrow-schema = { version = "0.8", optional = true } diff --git a/datafusion-pg-catalog/Cargo.toml b/datafusion-pg-catalog/Cargo.toml index fb18a35..1425966 100644 --- a/datafusion-pg-catalog/Cargo.toml +++ b/datafusion-pg-catalog/Cargo.toml @@ -19,7 +19,7 @@ include = [ [dependencies] async-trait = "0.1" -datafusion = { workspace = true, features = ["sql"] } +datafusion = { version = "=54.0.0", features = ["sql"] } arrow-pg = { path = "../arrow-pg", version = "0.14.0", default-features = false, features = ["datafusion"] } futures.workspace = true log = "0.4" diff --git a/datafusion-pg-catalog/src/pg_catalog.rs b/datafusion-pg-catalog/src/pg_catalog.rs index 6749f30..8c91d65 100644 --- a/datafusion-pg-catalog/src/pg_catalog.rs +++ b/datafusion-pg-catalog/src/pg_catalog.rs @@ -206,10 +206,6 @@ pub struct PgCatalogSchemaProvider { #[async_trait] impl SchemaProvider for PgCatalogSchemaProvider { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn table_names(&self) -> Vec { PG_CATALOG_TABLES.iter().map(ToString::to_string).collect() } @@ -1369,10 +1365,6 @@ pub fn create_pg_get_constraintdef() -> ScalarUDF { } impl ScalarUDFImpl for GetConstraintDefUDF { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "pg_get_constraintdef" } @@ -1490,7 +1482,11 @@ where #[cfg(test)] mod test { + use datafusion::arrow::array::Array; + use datafusion::arrow::datatypes::Int64Type; + use super::*; + use crate::pg_catalog::context::EmptyContextProvider; #[test] fn test_load_arrow_data() { @@ -1975,4 +1971,61 @@ mod test { ) .expect("Failed to load ipc data"); } + + #[tokio::test] + async fn test_pg_type_schema_and_query() { + let session_context = SessionContext::new(); + setup_pg_catalog(&session_context, "datafusion", EmptyContextProvider).unwrap(); + + let batches = session_context + .sql("SELECT oid, typname FROM pg_catalog.pg_type WHERE oid = 23") + .await + .unwrap() + .collect() + .await + .unwrap(); + + let batch = &batches[0]; + assert_eq!(batch.schema().field(0).name(), "oid"); + assert_eq!(batch.schema().field(0).data_type(), &DataType::Int32); + assert_eq!(batch.schema().field(1).name(), "typname"); + assert_eq!(batch.column(0).as_primitive::().value(0), 23); + assert_eq!(batch.column(1).as_string::().value(0), "int4"); + } + + #[tokio::test] + async fn test_pg_type_coercion_and_introspection_query() { + let session_context = SessionContext::new(); + setup_pg_catalog(&session_context, "datafusion", EmptyContextProvider).unwrap(); + + let batches = session_context + .sql( + "SELECT oid, typname, CAST(oid AS BIGINT) AS oid_as_bigint, NULL = typname AS unknown_comparison \ + FROM pg_catalog.pg_type WHERE oid = 23 AND NULL IS NULL", + ) + .await + .unwrap() + .collect() + .await + .unwrap(); + + let batch = &batches[0]; + assert_eq!(batch.schema().field(0).data_type(), &DataType::Int32); + assert_eq!(batch.schema().field(1).data_type(), &DataType::Utf8); + assert_eq!(batch.schema().field(2).data_type(), &DataType::Int64); + assert_eq!(batch.schema().field(3).data_type(), &DataType::Boolean); + assert_eq!(batch.column(0).as_primitive::().value(0), 23); + assert_eq!(batch.column(1).as_string::().value(0), "int4"); + assert_eq!(batch.column(2).as_primitive::().value(0), 23); + assert!(batch.column(3).as_boolean().is_null(0)); + + let batches = session_context + .sql("SELECT typname FROM pg_catalog.pg_type WHERE oid = '23'") + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!(batches[0].column(0).as_string::().value(0), "int4"); + } } diff --git a/datafusion-pg-catalog/src/pg_catalog/format_type.rs b/datafusion-pg-catalog/src/pg_catalog/format_type.rs index 24da102..78319d2 100644 --- a/datafusion-pg-catalog/src/pg_catalog/format_type.rs +++ b/datafusion-pg-catalog/src/pg_catalog/format_type.rs @@ -267,10 +267,6 @@ impl ScalarUDFImpl for FormatTypeUDF { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { format_type_impl(&args.args) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_format_type_udf() -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs b/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs index 0e17c62..b505f47 100644 --- a/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/has_privilege_udf.rs @@ -60,10 +60,6 @@ impl ScalarUDFImpl for PgHasPrivilegeUDF { Ok(ColumnarValue::Array(array)) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_has_privilege_udf(name: &str) -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs b/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs index 7750536..510d38f 100644 --- a/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/pg_get_expr_udf.rs @@ -65,10 +65,6 @@ impl ScalarUDFImpl for PgGetExprUDF { Ok(ColumnarValue::Array(array)) } - - fn as_any(&self) -> &dyn std::any::Any { - self - } } pub fn create_pg_get_expr_udf() -> ScalarUDF { diff --git a/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs b/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs index f189e1b..1eeefde 100644 --- a/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs +++ b/datafusion-pg-catalog/src/pg_catalog/quote_ident_udf.rs @@ -81,10 +81,6 @@ impl Default for ParseIdentUDF { } impl ScalarUDFImpl for ParseIdentUDF { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn signature(&self) -> &Signature { &self.signature }