From 8cbe49111d82bce0b80def668b87870775ffc984 Mon Sep 17 00:00:00 2001 From: Cora Sutton Date: Sun, 13 Sep 2026 11:04:04 +0000 Subject: [PATCH 1/2] DuckDB: Support CREATE TABLE PARTITIONED BY expressions --- src/ast/ddl.rs | 10 ++ src/ast/helpers/stmt_create_table.rs | 10 ++ src/ast/spans.rs | 6 +- src/dialect/duckdb.rs | 4 + src/dialect/mod.rs | 5 + src/parser/mod.rs | 23 +++- tests/sqlparser_common.rs | 22 ++++ tests/sqlparser_duckdb.rs | 171 +++++++++++++++++++++++++++ tests/sqlparser_mssql.rs | 2 + tests/sqlparser_postgres.rs | 1 + 10 files changed, 251 insertions(+), 3 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 66f2cad3eb..dded84beeb 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2991,6 +2991,8 @@ pub struct CreateTable { /// Hive: Table clustering column list. /// pub clustered_by: Option, + /// DuckDB partition expressions, distinct from Hive partition columns. + pub partitioned_by: Option>, /// Postgres `INHERITs` clause, which contains the list of tables from which /// the new table inherits. /// @@ -3215,6 +3217,14 @@ impl fmt::Display for CreateTable { _ => (), } + if let Some(partitioned_by) = &self.partitioned_by { + write!( + f, + " PARTITIONED BY ({})", + display_comma_separated(partitioned_by) + )?; + } + if let Some(clustered_by) = &self.clustered_by { write!(f, " {clustered_by}")?; } diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index 84a93dd20c..74e513424e 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -127,6 +127,8 @@ pub struct CreateTableBuilder { pub cluster_by: Option>>, /// Optional `CLUSTERED BY` clause. pub clustered_by: Option, + /// Optional expression-based `PARTITIONED BY` clause. + pub partitioned_by: Option>, /// Optional parent tables (`INHERITS`). pub inherits: Option>, /// Optional partitioned table (`PARTITION OF`) @@ -231,6 +233,7 @@ impl CreateTableBuilder { partition_by: None, cluster_by: None, clustered_by: None, + partitioned_by: None, inherits: None, partition_of: None, for_values: None, @@ -417,6 +420,11 @@ impl CreateTableBuilder { self.clustered_by = clustered_by; self } + /// Set expression-based partitioning. + pub fn partitioned_by(mut self, partitioned_by: Option>) -> Self { + self.partitioned_by = partitioned_by; + self + } /// Set parent tables via `INHERITS`. pub fn inherits(mut self, inherits: Option>) -> Self { self.inherits = inherits; @@ -632,6 +640,7 @@ impl CreateTableBuilder { partition_by: self.partition_by, cluster_by: self.cluster_by, clustered_by: self.clustered_by, + partitioned_by: self.partitioned_by, inherits: self.inherits, partition_of: self.partition_of, for_values: self.for_values, @@ -718,6 +727,7 @@ impl From for CreateTableBuilder { partition_by: table.partition_by, cluster_by: table.cluster_by, clustered_by: table.clustered_by, + partitioned_by: table.partitioned_by, inherits: table.inherits, partition_of: table.partition_of, for_values: table.for_values, diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 7acbd7d0b4..eb1f8e7040 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -580,7 +580,8 @@ impl Spanned for CreateTable { partition_by: _, // todo, BigQuery specific cluster_by: _, // todo, BigQuery specific clustered_by: _, // todo, Hive specific - inherits: _, // todo, PostgreSQL specific + partitioned_by, + inherits: _, // todo, PostgreSQL specific partition_of, for_values, strict: _, // bool @@ -624,7 +625,8 @@ impl Spanned for CreateTable { .chain(query.iter().map(|i| i.span())) .chain(clone.iter().map(|i| i.span())) .chain(partition_of.iter().map(|i| i.span())) - .chain(for_values.iter().map(|i| i.span())), + .chain(for_values.iter().map(|i| i.span())) + .chain(partitioned_by.iter().flatten().map(|i| i.span())), ) } } diff --git a/src/dialect/duckdb.rs b/src/dialect/duckdb.rs index 2e3673bc4c..34b19d50bf 100644 --- a/src/dialect/duckdb.rs +++ b/src/dialect/duckdb.rs @@ -24,6 +24,10 @@ pub struct DuckDbDialect; // In most cases the redshift dialect is identical to [`PostgresSqlDialect`]. impl Dialect for DuckDbDialect { + fn supports_create_table_partitioned_by_expressions(&self) -> bool { + true + } + fn supports_trailing_commas(&self) -> bool { true } diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 7c4744c5a7..16a42dda23 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -769,6 +769,11 @@ pub trait Dialect: Debug + Any { false } + /// Uses expressions rather than column declarations in `PARTITIONED BY`. + fn supports_create_table_partitioned_by_expressions(&self) -> bool { + false + } + /// Returns true if the dialect supports MySQL-specific SELECT modifiers /// like `HIGH_PRIORITY`, `STRAIGHT_JOIN`, `SQL_SMALL_RESULT`, etc. /// diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 15f135fffa..21d893ab49 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8775,6 +8775,7 @@ impl<'a> Parser<'a> { }; // parse optional column list (schema) + let has_columns = self.peek_token_ref().token == Token::LParen; let (columns, constraints) = self.parse_columns()?; let comment_after_column_def = if dialect_of!(self is HiveDialect) && self.parse_keyword(Keyword::COMMENT) { @@ -8804,7 +8805,22 @@ impl<'a> Parser<'a> { // SQLite supports `WITHOUT ROWID` at the end of `CREATE TABLE` let without_rowid = self.parse_keywords(&[Keyword::WITHOUT, Keyword::ROWID]); - let hive_distribution = self.parse_hive_distribution()?; + let (partitioned_by, hive_distribution) = if self + .dialect + .supports_create_table_partitioned_by_expressions() + { + let expressions = if self.parse_keywords(&[Keyword::PARTITIONED, Keyword::BY]) { + self.expect_token(&Token::LParen)?; + let expressions = self.parse_comma_separated(Parser::parse_expr)?; + self.expect_token(&Token::RParen)?; + Some(expressions) + } else { + None + }; + (expressions, HiveDistributionStyle::NONE) + } else { + (None, self.parse_hive_distribution()?) + }; let clustered_by = self.parse_optional_clustered_by()?; let hive_formats = self.parse_hive_formats()?; @@ -8898,6 +8914,10 @@ impl<'a> Parser<'a> { None }; + if query.is_none() && !has_columns && partitioned_by.is_some() { + return self.expected_ref("AS query or a table schema", self.peek_token_ref()); + } + // `WITH DATA` clause only applies if there is a query body. let with_data = if query.is_some() { self.maybe_parse_with_data()? @@ -8928,6 +8948,7 @@ impl<'a> Parser<'a> { .on_commit(on_commit) .on_cluster(on_cluster) .clustered_by(clustered_by) + .partitioned_by(partitioned_by) .partition_by(partition_by) .cluster_by(create_table_config.cluster_by) .inherits(create_table_config.inherits) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 2de6062b28..32a310809d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -20060,3 +20060,25 @@ fn parse_insert_by_name() { _ => unreachable!(), } } + +#[test] +fn create_table_partitioned_by_dialect_isolation() { + let sql = "CREATE TABLE t (id INTEGER) PARTITIONED BY (id + 1)"; + all_dialects_where(|dialect| dialect.supports_create_table_partitioned_by_expressions()) + .verified_stmt(sql); + all_dialects_where(|dialect| !dialect.supports_create_table_partitioned_by_expressions()) + .one_of_identical_results(|dialect| assert!(Parser::parse_sql(dialect, sql).is_err())); + let hive = TestedDialects::new(vec![Box::new(HiveDialect {}), Box::new(GenericDialect {})]); + let Statement::CreateTable(table) = hive.verified_stmt( + "CREATE TABLE t (id INT) PARTITIONED BY (category STRING) CLUSTERED BY (id) SORTED BY (id DESC) INTO 4 BUCKETS", + ) else { + unreachable!() + }; + let HiveDistributionStyle::PARTITIONED { columns } = table.hive_distribution else { + unreachable!() + }; + assert_eq!(columns[0].name, Ident::new("category")); + assert_eq!(columns[0].data_type, DataType::String(None)); + assert!(table.partitioned_by.is_none()); + assert!(table.clustered_by.unwrap().sorted_by.is_some()); +} diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index a338ef7a82..a8bce74120 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -764,6 +764,7 @@ fn test_duckdb_union_datatype() { partition_by: Default::default(), cluster_by: Default::default(), clustered_by: Default::default(), + partitioned_by: None, inherits: Default::default(), partition_of: Default::default(), for_values: Default::default(), @@ -910,3 +911,173 @@ fn test_duckdb_lambda_function() { let sql_transform = "SELECT list_transform([1, 2, 3], lambda x : x * 2)"; duckdb().verified_stmt(sql_transform); } + +#[test] +fn create_table_partitioned_by_expressions() { + for sql in [ + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 1)", + "CREATE TABLE events (id INTEGER) PARTITIONED BY (abs(id), id % 2)", + "CREATE TABLE events PARTITIONED BY (id + 1) AS SELECT 1 AS id", + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 1) WITH (flag = true)", + ] { + let statement = duckdb().verified_stmt(sql); + assert_eq!(statement, duckdb().verified_stmt(&statement.to_string())); + } + duckdb().one_statement_parses_to( + "CREATE TABLE events (id INTEGER) PARTITIONED /* before BY */ BY (id + 1,);", + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 1)", + ); +} + +#[test] +fn create_table_partitioned_by_errors() { + for (sql, expected) in [ + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY ()", + "Expected: an expression, found: )", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY id", + "Expected: (, found: id", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id", + "Expected: ), found: EOF", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id INTEGER)", + "Expected: ), found: INTEGER", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id) PARTITIONED BY (id)", + "Expected: end of statement, found: PARTITIONED", + ), + ( + "CREATE TABLE events PARTITIONED BY (id)", + "Expected: AS query or a table schema, found: EOF", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id ASC)", + "Expected: ), found: ASC", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id DESC NULLS LAST)", + "Expected: ), found: DESC", + ), + ( + "CREATE TABLE events (id INTEGER) PARTITIONED BY (id AS alias)", + "Expected: ), found: AS", + ), + ( + "CREATE TABLE events (id INTEGER) WITH (flag = true) PARTITIONED BY (id)", + "Expected: end of statement, found: PARTITIONED", + ), + ] { + assert_eq!( + duckdb().parse_sql_statements(sql).unwrap_err(), + ParserError::ParserError(expected.to_owned()), + "{sql}" + ); + } +} + +#[test] +fn create_table_partitioned_by_ast_and_builder() { + let sql = "CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 1)"; + let Statement::CreateTable(table) = duckdb().verified_stmt(sql) else { + unreachable!() + }; + let expressions = vec![Expr::BinaryOp { + left: Box::new(Expr::Identifier(Ident::new("id"))), + op: BinaryOperator::Plus, + right: Box::new(Expr::Value( + Value::Number("1".parse().unwrap(), false).into(), + )), + }]; + assert_eq!(table.partitioned_by, Some(expressions.clone())); + assert_eq!(table.hive_distribution, HiveDistributionStyle::NONE); + assert!(table.partition_by.is_none()); + let rebuilt = helpers::stmt_create_table::CreateTableBuilder::from(table.clone()).build(); + assert_eq!(rebuilt, table); + let built = helpers::stmt_create_table::CreateTableBuilder::new(table.name.clone()) + .columns(table.columns.clone()) + .partitioned_by(Some(expressions.clone())) + .build(); + assert_eq!(built.partitioned_by, Some(expressions)); + assert_eq!(built.to_string(), sql); + let Statement::CreateTable(unpartitioned) = + duckdb().verified_stmt("CREATE TABLE events (id INTEGER)") + else { + unreachable!() + }; + assert!(unpartitioned.partitioned_by.is_none()); +} + +#[test] +fn create_table_partitioned_by_spans() { + use sqlparser::parser::Parser; + use sqlparser::tokenizer::Location; + let sql = "CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 2)"; + let Statement::CreateTable(table) = + Parser::parse_sql(&DuckDbDialect {}, sql).unwrap().remove(0) + else { + unreachable!() + }; + let expression_start = sql.find("id + 2").unwrap() as u64 + 1; + let expression_end = sql.len() as u64; + assert_eq!( + table.partitioned_by.as_ref().unwrap()[0].span(), + Span::new( + Location::new(1, expression_start), + Location::new(1, expression_end) + ) + ); + assert_eq!( + table.span(), + Span::new(Location::new(1, 14), Location::new(1, expression_end)) + ); +} + +#[test] +#[cfg(feature = "json_example")] +fn create_table_partitioned_by_serialization() { + let statement = + duckdb().verified_stmt("CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 1)"); + let json = serde_json::to_string(&statement).unwrap(); + assert_eq!(statement, serde_json::from_str::(&json).unwrap()); + let mut table = serde_json::to_value( + helpers::stmt_create_table::CreateTableBuilder::new(Ident::new("events").into()).build(), + ) + .unwrap(); + table.as_object_mut().unwrap().remove("partitioned_by"); + assert!(serde_json::from_value::(table) + .unwrap() + .partitioned_by + .is_none()); +} + +#[test] +#[cfg(feature = "visitor")] +fn create_table_partitioned_by_visitors() { + use core::ops::ControlFlow; + let mut statement = + duckdb().verified_stmt("CREATE TABLE events (id INTEGER) PARTITIONED BY (id + 2)"); + let mut expressions = vec![]; + let _ = visit_expressions(&statement, |expression| { + expressions.push(expression.to_string()); + ControlFlow::<()>::Continue(()) + }); + assert_eq!(expressions, ["id + 2", "id", "2"]); + let _ = visit_expressions_mut(&mut statement, |expression| { + if let Expr::Identifier(ident) = expression { + if ident.value == "id" { + ident.value = "value".to_owned(); + } + } + ControlFlow::<()>::Continue(()) + }); + assert_eq!( + statement.to_string(), + "CREATE TABLE events (id INTEGER) PARTITIONED BY (value + 2)" + ); +} diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 4510f953e5..b3540e6552 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1982,6 +1982,7 @@ fn parse_create_table_with_valid_options() { partition_by: None, cluster_by: None, clustered_by: None, + partitioned_by: None, inherits: None, partition_of: None, for_values: None, @@ -2163,6 +2164,7 @@ fn parse_create_table_with_identity_column() { partition_by: None, cluster_by: None, clustered_by: None, + partitioned_by: None, inherits: None, partition_of: None, for_values: None, diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index d71e49b27a..112a68a3f2 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -7047,6 +7047,7 @@ fn parse_trigger_related_functions() { partition_by: None, cluster_by: None, clustered_by: None, + partitioned_by: None, inherits: None, partition_of: None, for_values: None, From fdccfdfbd7e8fca6260c8cac8cfb32373661fe85 Mon Sep 17 00:00:00 2001 From: Cora Sutton Date: Tue, 15 Sep 2026 05:26:59 +0000 Subject: [PATCH 2/2] DuckDB: Add PARTITIONED BY documentation links --- src/ast/ddl.rs | 3 +++ src/dialect/mod.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index dded84beeb..ed18451e4b 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2992,6 +2992,9 @@ pub struct CreateTable { /// pub clustered_by: Option, /// DuckDB partition expressions, distinct from Hive partition columns. + /// + /// [DuckDB syntax example (Iceberg catalog)](https://duckdb.org/docs/current/core_extensions/iceberg/writing#partitioning) + /// [DuckDB syntax introduction](https://github.com/duckdb/duckdb/pull/20431) pub partitioned_by: Option>, /// Postgres `INHERITs` clause, which contains the list of tables from which /// the new table inherits. diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 16a42dda23..05118b1892 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -770,6 +770,9 @@ pub trait Dialect: Debug + Any { } /// Uses expressions rather than column declarations in `PARTITIONED BY`. + /// + /// [DuckDB syntax example (Iceberg catalog)](https://duckdb.org/docs/current/core_extensions/iceberg/writing#partitioning) + /// [DuckDB syntax introduction](https://github.com/duckdb/duckdb/pull/20431) fn supports_create_table_partitioned_by_expressions(&self) -> bool { false }