From 6f1ae6d7498996657703c6d59f47268cb4d75bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=98=E5=85=B8?= Date: Sun, 13 Sep 2026 17:29:43 +0800 Subject: [PATCH 1/4] feat(parser): support ClickHouse COLUMNS(...) APPLY/EXCEPT/REPLACE transformers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a dedicated select-item branch for a COLUMNS('regexp') matcher followed by one or more transformers, fixing #2631 (APPLY with a lambda such as x -> round(x, 2) failed to parse; without a lambda, APPLY(...) was silently mis-parsed as the select item alias). - new ColumnsExpression / ColumnsTransformer AST nodes with an ordered transformer list, since ClickHouse parses transformers in a loop (they may repeat and combine in any order) - semantic lookahead isColumnsExpressionAhead() keeps a plain COLUMNS('regexp') on the regular Function path - expression visitor, adapter, deparser, validator and TablesNamesFinder traverse the new node Signed-off-by: 付典 --- .../expression/ColumnsExpression.java | 73 +++++++ .../expression/ColumnsTransformer.java | 95 +++++++++ .../expression/ExpressionVisitor.java | 6 + .../expression/ExpressionVisitorAdapter.java | 20 ++ .../sf/jsqlparser/util/TablesNamesFinder.java | 20 ++ .../util/deparser/ExpressionDeParser.java | 6 + .../validator/ExpressionValidator.java | 22 ++ .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 190 ++++++++++++++++++ .../statement/select/ClickHouseTest.java | 140 +++++++++++++ 9 files changed, 572 insertions(+) create mode 100644 src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java create mode 100644 src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java diff --git a/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java b/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java new file mode 100644 index 000000000..3ee369aa9 --- /dev/null +++ b/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java @@ -0,0 +1,73 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.expression; + +import java.util.ArrayList; +import java.util.List; +import net.sf.jsqlparser.parser.ASTNodeAccessImpl; + +/** + * A ClickHouse {@code COLUMNS('regexp')} matcher with one or more transformers, for example + * {@code COLUMNS('^metric_') APPLY(x -> round(x, 2))}. + * + * Without a transformer, {@code COLUMNS('regexp')} keeps parsing as a regular + * {@link net.sf.jsqlparser.expression.Function}. + */ +public class ColumnsExpression extends ASTNodeAccessImpl implements Expression { + + private Expression columns; + private List transformers; + + public ColumnsExpression(Expression columns) { + this.columns = columns; + this.transformers = new ArrayList<>(); + } + + public ColumnsExpression(Expression columns, List transformers) { + this.columns = columns; + this.transformers = transformers; + } + + public Expression getColumns() { + return columns; + } + + public ColumnsExpression setColumns(Expression columns) { + this.columns = columns; + return this; + } + + public List getTransformers() { + return transformers; + } + + public ColumnsExpression setTransformers(List transformers) { + this.transformers = transformers; + return this; + } + + public StringBuilder appendTo(StringBuilder builder) { + builder.append(columns); + for (ColumnsTransformer transformer : transformers) { + builder.append(" ").append(transformer); + } + return builder; + } + + @Override + public String toString() { + return appendTo(new StringBuilder()).toString(); + } + + @Override + public T accept(ExpressionVisitor expressionVisitor, S context) { + return expressionVisitor.visit(this, context); + } +} diff --git a/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java new file mode 100644 index 000000000..bfbbe1381 --- /dev/null +++ b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java @@ -0,0 +1,95 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.expression; + +import java.util.List; +import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList; +import net.sf.jsqlparser.parser.ASTNodeAccessImpl; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectItem; + +/** + * A ClickHouse transformer following a {@code COLUMNS(...)} matcher, for example the + * {@code APPLY(x -> round(x, 2))} in {@code SELECT COLUMNS('^m') APPLY(x -> round(x, 2))}. + * + * ClickHouse parses its transformers in a loop, so they may repeat and combine in any order. + */ +public class ColumnsTransformer extends ASTNodeAccessImpl { + + public enum ColumnsTransformerType { + APPLY, EXCEPT, REPLACE + } + + private ColumnsTransformerType type; + private Expression applyExpression; + private ParenthesedExpressionList exceptColumns; + private List> replaceItems; + + public ColumnsTransformer(ColumnsTransformerType type) { + this.type = type; + } + + public ColumnsTransformerType getType() { + return type; + } + + public ColumnsTransformer setType(ColumnsTransformerType type) { + this.type = type; + return this; + } + + public Expression getApplyExpression() { + return applyExpression; + } + + public ColumnsTransformer setApplyExpression(Expression applyExpression) { + this.applyExpression = applyExpression; + return this; + } + + public ParenthesedExpressionList getExceptColumns() { + return exceptColumns; + } + + public ColumnsTransformer setExceptColumns(ParenthesedExpressionList exceptColumns) { + this.exceptColumns = exceptColumns; + return this; + } + + public List> getReplaceItems() { + return replaceItems; + } + + public ColumnsTransformer setReplaceItems(List> replaceItems) { + this.replaceItems = replaceItems; + return this; + } + + public StringBuilder appendTo(StringBuilder builder) { + switch (type) { + case APPLY: + builder.append("APPLY(").append(applyExpression).append(")"); + break; + case EXCEPT: + builder.append("EXCEPT ").append(exceptColumns); + break; + case REPLACE: + builder.append("REPLACE(").append(Select.getStringList(replaceItems)).append(")"); + break; + } + return builder; + } + + @Override + public String toString() { + return appendTo(new StringBuilder()).toString(); + } +} diff --git a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java index 589f5b195..ea397970c 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java +++ b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java @@ -796,6 +796,12 @@ default void visit(LambdaExpression lambdaExpression) { this.visit(lambdaExpression, null); } + T visit(ColumnsExpression columnsExpression, S context); + + default void visit(ColumnsExpression columnsExpression) { + this.visit(columnsExpression, null); + } + T visit(HighExpression highExpression, S context); default void visit(HighExpression highExpression) { diff --git a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java index 435557344..8c89da29c 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java +++ b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java @@ -875,6 +875,26 @@ public T visit(LambdaExpression lambdaExpression, S context) { return lambdaExpression.getExpression().accept(this, context); } + @Override + public T visit(ColumnsExpression columnsExpression, S context) { + if (columnsExpression.getColumns() != null) { + columnsExpression.getColumns().accept(this, context); + } + for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { + if (transformer.getApplyExpression() != null) { + transformer.getApplyExpression().accept(this, context); + } + if (transformer.getReplaceItems() != null) { + for (SelectItem selectItem : transformer.getReplaceItems()) { + if (selectItem.getExpression() != null) { + selectItem.getExpression().accept(this, context); + } + } + } + } + return null; + } + @Override public T visit(HighExpression highExpression, S context) { return highExpression.getExpression().accept(this, context); diff --git a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java index 5931c2fa6..f5d434e73 100644 --- a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java +++ b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java @@ -2227,6 +2227,26 @@ public Void visit(LambdaExpression lambdaExpression, S context) { return null; } + @Override + public Void visit(ColumnsExpression columnsExpression, S context) { + if (columnsExpression.getColumns() != null) { + columnsExpression.getColumns().accept(this, context); + } + for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { + if (transformer.getApplyExpression() != null) { + transformer.getApplyExpression().accept(this, context); + } + if (transformer.getReplaceItems() != null) { + for (SelectItem selectItem : transformer.getReplaceItems()) { + if (selectItem.getExpression() != null) { + selectItem.getExpression().accept(this, context); + } + } + } + } + return null; + } + @Override public Void visit(HighExpression highExpression, S context) { highExpression.getExpression().accept(this, context); diff --git a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java index 304998b1a..0e8293ecb 100644 --- a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java +++ b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java @@ -25,6 +25,7 @@ import net.sf.jsqlparser.expression.CaseExpression; import net.sf.jsqlparser.expression.CastExpression; import net.sf.jsqlparser.expression.CollateExpression; +import net.sf.jsqlparser.expression.ColumnsExpression; import net.sf.jsqlparser.expression.ConnectByPriorOperator; import net.sf.jsqlparser.expression.ConnectByRootOperator; import net.sf.jsqlparser.expression.DateTimeLiteralExpression; @@ -1926,6 +1927,11 @@ public StringBuilder visit(LambdaExpression lambdaExpression, S context) { return builder; } + @Override + public StringBuilder visit(ColumnsExpression columnsExpression, S context) { + return columnsExpression.appendTo(builder); + } + @Override public StringBuilder visit(HighExpression highExpression, S context) { return builder.append(highExpression.toString()); diff --git a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java index f6e493177..a89885a95 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java @@ -20,6 +20,8 @@ import net.sf.jsqlparser.expression.CaseExpression; import net.sf.jsqlparser.expression.CastExpression; import net.sf.jsqlparser.expression.CollateExpression; +import net.sf.jsqlparser.expression.ColumnsExpression; +import net.sf.jsqlparser.expression.ColumnsTransformer; import net.sf.jsqlparser.expression.ConnectByPriorOperator; import net.sf.jsqlparser.expression.ConnectByRootOperator; import net.sf.jsqlparser.expression.DateTimeLiteralExpression; @@ -1218,6 +1220,26 @@ public Void visit(LambdaExpression lambdaExpression, S context) { return null; } + @Override + public Void visit(ColumnsExpression columnsExpression, S context) { + if (columnsExpression.getColumns() != null) { + columnsExpression.getColumns().accept(this, context); + } + for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { + if (transformer.getApplyExpression() != null) { + transformer.getApplyExpression().accept(this, context); + } + if (transformer.getReplaceItems() != null) { + for (SelectItem selectItem : transformer.getReplaceItems()) { + if (selectItem.getExpression() != null) { + selectItem.getExpression().accept(this, context); + } + } + } + } + return null; + } + @Override public Void visit(HighExpression highExpression, S context) { highExpression.getExpression().accept(this, context); diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 30b324fec..84c37fc04 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -1274,6 +1274,125 @@ public class CCJSqlParser extends AbstractJSqlParser { return false; } + /** + * Upper bound for the token scans of {@link #isColumnsExpressionAhead()}; content + * beyond the budget cannot come from the alias-columns grammar, and the inputs it + * belongs to are rejected by the baseline parser anyway, so declining to commit + * keeps the scan linear on pathological content. + */ + private static final int CONTENT_SCAN_BUDGET = 4096; + + /** + * True when a COLUMNS(...) function is followed by at least one ClickHouse transformer + * (APPLY, EXCEPT, REPLACE). Scans the balanced parentheses of the function call and + * checks the token after the closing bracket. A plain COLUMNS('regexp') without a + * transformer stays on the regular expression path and keeps parsing as a Function. + * + * For APPLY and REPLACE the follower's own parameter content is also checked: when + * it is shaped like the alias-columns grammar T(name[type], ...) that the regular + * alias path reads (a depth-1 comma for APPLY, or two adjacent identifier-kind + * tokens at depth 1 for both), this predicate returns false so the input keeps its + * exact baseline parse instead of committing to a transformer body that cannot + * parse it. The scan is bounded and exception-safe: on pathological content it + * returns false, which hands the input to the baseline paths. + */ + protected boolean isColumnsExpressionAhead() { + Token columnsToken = getToken(1); + if (columnsToken.kind != K_COLUMNS || getToken(2).kind != OPENING_BRACKET) { + return false; + } + try { + int depth = 1; + int index = 3; + while (true) { + Token token = getToken(index++); + if (token.kind == EOF) { + return false; + } + if (token.kind == OPENING_BRACKET) { + depth++; + } else if (token.kind == CLOSING_BRACKET) { + if (--depth == 0) { + int followerKind = getToken(index).kind; + if (followerKind != K_APPLY && followerKind != K_REPLACE) { + return followerKind == K_EXCEPT; + } + return !isAliasColumnsContent(index + 2, followerKind == K_APPLY); + } + } + if (index > CONTENT_SCAN_BUDGET) { + return false; + } + } + } catch (TokenMgrException e) { + return false; + } + } + + /** + * Walks the balanced parentheses of a follower's parameter list and reports whether + * its depth-1 content is shaped like the alias-columns grammar T(name[type], ...) + * rather than like a transformer body: a depth-1 comma (which a single expression or + * lambda body can never have, checked for APPLY only), or two adjacent + * identifier-kind tokens at depth 1 (the name/type adjacency of a typed alias + * column, e.g. T(a INT); expression grammar never places two such tokens side by + * side, which keeps predicate keywords like AS, IS or AND out of the rule). When + * this returns true the input belongs to the regular alias path, whose parse is + * identical to the baseline parser's. + */ + protected boolean isAliasColumnsContent(int startIndex, boolean commaDisqualifies) { + int depth = 1; + int index = startIndex; + int previousKind = -1; + while (true) { + Token token = getToken(index++); + if (token.kind == EOF) { + return false; + } + if (token.kind == OPENING_BRACKET) { + depth++; + } else if (token.kind == CLOSING_BRACKET) { + if (--depth == 0) { + return false; + } + } else if (depth == 1) { + if (token.kind == K_COMMA && commaDisqualifies) { + return true; + } + if (isIdentifierKind(previousKind) && isIdentifierKind(token.kind)) { + return true; + } + } + if (index - startIndex > CONTENT_SCAN_BUDGET) { + return false; + } + previousKind = token.kind; + } + } + + /** + * The token kinds that RelObjectName and ColDataType are built from, plus the + * reserved keywords usable as identifiers in their context-guarded positions. + * DT_ZONE (a compound "TIMESTAMP WITH TIME ZONE" style type lexed as one token) + * belongs here because ColDataType accepts it as a type name. + */ + private static boolean isIdentifierKind(int kind) { + switch (kind) { + case S_IDENTIFIER: case S_QUOTED_IDENTIFIER: case DATA_TYPE: + case K_DATETIMELITERAL: case K_DATE_LITERAL: case DT_ZONE: + case K_ALL: case K_ANY: case K_CASEWHEN: case K_CONNECT: case K_CREATE: + case K_DEFAULT: case K_GLOBAL: case K_GROUP: case K_GROUPING: case K_IF: + case K_IIF: case K_IGNORE: case K_IN: case K_INTERVAL: case K_LEFT: + case K_LIMIT: case K_NEXTVAL: case K_OFFSET: case K_ON: case K_OPTIMIZE: + case K_ORDER: case K_PROCEDURE: case K_PUBLIC: case K_QUALIFY: case K_RIGHT: + case K_FILE: case K_SET: case K_SOME: case K_START: case K_TABLES: case K_TOP: + case K_VALUE: case K_VALUES: + return true; + default: + return kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD; + } + } + private static void requireTypeDdlSyntax(boolean valid, String message) throws ParseException { if (!valid) { throw new ParseException(message); } } @@ -7010,11 +7129,15 @@ SelectItem SelectItem() #SelectItem: { Expression expression; Alias alias = null; + Expression columnsItemExpression; } { // @fixme: Oracle's SEQUENCE.nextval is parsed as COLUMN with a name part nextval // @todo: parse a proper SEQUENCE instead of a COLUMN ( + LOOKAHEAD( { isColumnsExpressionAhead() } ) columnsItemExpression = ColumnsExpression() + { expression = columnsItemExpression; } + | LOOKAHEAD( 3 ) expression = ConnectByPriorOperator() | ( expression=Condition() { expression = prattExpressionRest(expression, 2); } ) @@ -7027,6 +7150,73 @@ SelectItem SelectItem() #SelectItem: } } +/** + * Parses a ClickHouse COLUMNS('regexp') matcher followed by zero or more transformers, + * for example COLUMNS('^m') APPLY(x -> round(x, 2)). Each transformer branch carries a + * syntactic lookahead, and when no transformer materializes the plain Function is + * returned, so the committed select-item branch degrades to the exact regular parse + * instead of failing: EXCEPT stays a set operation and bare APPLY/REPLACE stay aliases, + * exactly as without this production. + */ +Expression ColumnsExpression() #ColumnsExpression: +{ + Expression columns; + ColumnsTransformer transformer; + List transformers = new ArrayList(); +} +{ + columns = Function() + ( + LOOKAHEAD( | | ParenthesedColumnList() ) + transformer = ColumnsTransformer() + { transformers.add(transformer); } + )* + { + if (transformers.isEmpty()) { + linkAST(columns, jjtThis); + return columns; + } + ColumnsExpression columnsExpression = new ColumnsExpression(columns, transformers); + linkAST(columnsExpression, jjtThis); + return columnsExpression; + } +} + +/** + * Parses a single ClickHouse transformer: APPLY(expression), EXCEPT (columns) or + * REPLACE (selectItems). ClickHouse parses these in a loop, so they may repeat + * and combine in any order. + */ +ColumnsTransformer ColumnsTransformer(): +{ + Expression applyExpression; + ParenthesedExpressionList exceptColumns; + List> replaceItems; + ColumnsTransformer transformer; +} +{ + ( + "(" + ( + LOOKAHEAD( RelObjectName() "->" ) applyExpression = LambdaExpression() + | + applyExpression = Expression() + ) + ")" + { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.APPLY) + .setApplyExpression(applyExpression); } + + | exceptColumns = ParenthesedColumnList() + { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.EXCEPT) + .setExceptColumns(exceptColumns); } + + | "(" replaceItems = SelectItemsList() ")" + { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.REPLACE) + .setReplaceItems(replaceItems); } + ) + { return transformer; } +} + /** * Parses the AllColumns-Pattern '*'. * diff --git a/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java b/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java index cfdca4e77..73ba93171 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java @@ -12,7 +12,10 @@ import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.ColumnsExpression; +import net.sf.jsqlparser.expression.ColumnsTransformer; import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LambdaExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -257,4 +260,141 @@ public void testCastToNestedParametricTypeIssue2441() throws JSQLParserException sql = "SELECT CAST(x AS LowCardinality(Decimal(10, 2))) FROM cast_demo"; assertSqlCanBeParsedAndDeparsed(sql, true); } + + @Test + public void testColumnsApplyLambdaIssue2631() throws JSQLParserException { + // ClickHouse applies a function or lambda expression to all columns matching + // the regular expression: https://github.com/JSQLParser/JSqlParser/issues/2631 + String sql = "SELECT COLUMNS('^metric_') APPLY(x -> round(x, 2)) FROM metrics"; + Select select = (Select) CCJSqlParserUtil.parse(sql); + PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); + ColumnsExpression columnsExpression = Assertions.assertInstanceOf(ColumnsExpression.class, + plainSelect.getSelectItems().get(0).getExpression()); + Assertions.assertEquals(1, columnsExpression.getTransformers().size()); + ColumnsTransformer transformer = columnsExpression.getTransformers().get(0); + Assertions.assertEquals(ColumnsTransformer.ColumnsTransformerType.APPLY, + transformer.getType()); + Assertions.assertInstanceOf(LambdaExpression.class, transformer.getApplyExpression()); + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsApplyFunctionName() throws JSQLParserException { + // chained APPLY modifiers, taken from the ClickHouse SELECT documentation + String sql = "SELECT COLUMNS('[jk]') APPLY(toString) APPLY(length) APPLY(max)" + + " FROM columns_transformers"; + Select select = (Select) CCJSqlParserUtil.parse(sql); + PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); + ColumnsExpression columnsExpression = Assertions.assertInstanceOf(ColumnsExpression.class, + plainSelect.getSelectItems().get(0).getExpression()); + Assertions.assertEquals(3, columnsExpression.getTransformers().size()); + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsExcept() throws JSQLParserException { + String sql = "SELECT COLUMNS('^metric_') EXCEPT (metric_disk) FROM metrics"; + Select select = (Select) CCJSqlParserUtil.parse(sql); + PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); + ColumnsExpression columnsExpression = Assertions.assertInstanceOf(ColumnsExpression.class, + plainSelect.getSelectItems().get(0).getExpression()); + Assertions.assertEquals(ColumnsTransformer.ColumnsTransformerType.EXCEPT, + columnsExpression.getTransformers().get(0).getType()); + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsReplace() throws JSQLParserException { + String sql = + "SELECT COLUMNS('^metric_') REPLACE(metric_cpu * 100 AS metric_cpu) FROM metrics"; + Select select = (Select) CCJSqlParserUtil.parse(sql); + PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); + ColumnsExpression columnsExpression = Assertions.assertInstanceOf(ColumnsExpression.class, + plainSelect.getSelectItems().get(0).getExpression()); + Assertions.assertEquals(ColumnsTransformer.ColumnsTransformerType.REPLACE, + columnsExpression.getTransformers().get(0).getType()); + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsCombinedTransformers() throws JSQLParserException { + // ClickHouse parses its transformers in a loop, so they combine in any order + String sql = + "SELECT COLUMNS('m') APPLY(x -> round(x, 2)) EXCEPT (metric_disk) FROM metrics"; + assertSqlCanBeParsedAndDeparsed(sql, true); + + sql = "SELECT COLUMNS('m') EXCEPT (metric_disk) APPLY(x -> round(x, 2)) FROM metrics"; + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsApplyMultiParamLambda() throws JSQLParserException { + String sql = "SELECT COLUMNS('m') APPLY((k, v) -> v > 5) FROM metrics"; + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testBareColumnsExpressionStaysFunction() throws JSQLParserException { + // without a transformer, COLUMNS(...) keeps parsing as a regular function + String sql = "SELECT COLUMNS('^metric_') FROM metrics"; + Select select = (Select) CCJSqlParserUtil.parse(sql); + PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); + Assertions.assertInstanceOf(Function.class, + plainSelect.getSelectItems().get(0).getExpression()); + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @Test + public void testColumnsItemFollowedBySetOperation() throws JSQLParserException { + // a follower keyword without its parenthesis never starts a transformer, so + // EXCEPT stays a set operation (a bare APPLY or REPLACE stays an alias) + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') EXCEPT SELECT b FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') EXCEPT ALL SELECT b FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT a, COLUMNS('m') EXCEPT SELECT b FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') EXCEPT (SELECT b FROM t)", true); + // parenthesized set operation operands that do not start a column list + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') EXCEPT ((SELECT b FROM t))", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') EXCEPT (VALUES (1, 2))", true); + } + + @Test + public void testBareTransformerKeywordRemainsAlias() throws JSQLParserException { + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') REPLACE FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY", true); + } + + @Test + public void testAliasColumnsBodyRemainsAlias() throws JSQLParserException { + // T(name[type], ...) is the alias-columns reading of the regular alias path, + // not a transformer body (APPLY takes a function name or a lambda in ClickHouse) + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(a, b) FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(a, b)", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(a INT) FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(`a`, `b`) FROM t", true); + assertSqlCanBeParsedAndDeparsed( + "SELECT COLUMNS('m') REPLACE(a INT, b VARCHAR(10)) FROM t", true); + } + + @Test + public void testReplaceWithBareIdentifierAsAlias() throws JSQLParserException { + // the plain ClickHouse rename form: REPLACE(column AS alias) + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') REPLACE(b AS c) FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') REPLACE(b AS c, d AS e) FROM t", true); + // keyword-predicate lambda bodies stay on the transformer reading + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(x IS NULL) FROM t", true); + assertSqlCanBeParsedAndDeparsed("SELECT COLUMNS('m') APPLY(x AND y) FROM t", true); + } + + @Test + public void testTypedAliasColumnWithCompoundTimeZoneType() throws JSQLParserException { + // compound timezone types are lexed as a single DT_ZONE token but are still + // alias-columns types, so the input belongs to the alias path + assertSqlCanBeParsedAndDeparsed( + "SELECT COLUMNS('m') APPLY(a TIMESTAMP WITH TIME ZONE) FROM t", true); + assertSqlCanBeParsedAndDeparsed( + "SELECT COLUMNS('m') APPLY(a TIMESTAMP(3) WITH TIME ZONE) FROM t", true); + assertSqlCanBeParsedAndDeparsed( + "SELECT COLUMNS('m') APPLY(a TIMESTAMP WITHOUT TIME ZONE) FROM t", true); + } } From 2fba7fa4ca455ead0a5d887082315f12a7e9eaba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=98=E5=85=B8?= Date: Mon, 14 Sep 2026 00:47:36 +0800 Subject: [PATCH 2/4] fix(parser): add default case to the ColumnsTransformer type switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Satisfies the PMD exhaustive-switch rule (Codacy '1 new issue') and fails loudly if a future ColumnsTransformerType constant is added without updating appendTo; unreachable for the current APPLY/EXCEPT/REPLACE constants, so behavior is unchanged. Same pattern as JsonFunction's Unhandled JsonOnResponseBehavior guard. Signed-off-by: 付典 --- .../java/net/sf/jsqlparser/expression/ColumnsTransformer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java index bfbbe1381..a241c7a1a 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java +++ b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java @@ -84,6 +84,8 @@ public StringBuilder appendTo(StringBuilder builder) { case REPLACE: builder.append("REPLACE(").append(Select.getStringList(replaceItems)).append(")"); break; + default: + throw new IllegalStateException("Unhandled ColumnsTransformerType: " + type); } return builder; } From 4b91117a3e28bb534e987ce858237c47ed0297ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=98=E5=85=B8?= Date: Mon, 14 Sep 2026 17:54:26 +0800 Subject: [PATCH 3/4] fix: preserve COLUMNS traversal and alias boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse alias grammar at each transformer boundary, preserve JJTree parents and lookahead visibility, and route all child expressions through visitor hooks. Add consumer and grammar regression coverage. Signed-off-by: 付典 --- README.md | 1 + .../expression/ColumnsExpression.java | 11 + .../expression/ColumnsTransformer.java | 24 ++ .../expression/ExpressionVisitor.java | 9 +- .../expression/ExpressionVisitorAdapter.java | 17 +- .../sf/jsqlparser/util/TablesNamesFinder.java | 16 +- .../util/deparser/ExpressionDeParser.java | 35 ++- .../validator/ExpressionValidator.java | 17 +- .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 231 ++++++------------ .../expression/ColumnsExpressionTest.java | 200 +++++++++++++++ .../statement/select/ClickHouseTest.java | 91 +++++++ 11 files changed, 449 insertions(+), 203 deletions(-) create mode 100644 src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java diff --git a/README.md b/README.md index bbb230d98..6df4adb09 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ and missing syntax gets added on demand — [open an issue](https://github.com/J |---|---| | **Queries** | `SELECT` · `WITH …` · Piped SQL | | **ksqlDB windows** | JOIN `WITHIN`, window `GRACE PERIOD`, and `EMIT CHANGES`/`FINAL` | +| **ClickHouse column selection** | `COLUMNS('regexp')` select items with chained `APPLY`, `EXCEPT`, and `REPLACE` transformers | | **DML** | `INSERT` · `UPDATE` · `UPSERT` · `MERGE` · `DELETE` · `TRUNCATE TABLE` | | **DDL** | `CREATE …` · `ALTER …` · `DROP …` | | **PostgreSQL RLS** | `CREATE POLICY` · `ALTER TABLE … ENABLE`/`DISABLE`/`FORCE`/`NO FORCE ROW LEVEL SECURITY` | diff --git a/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java b/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java index 3ee369aa9..aa19d555f 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java +++ b/src/main/java/net/sf/jsqlparser/expression/ColumnsExpression.java @@ -53,6 +53,17 @@ public ColumnsExpression setTransformers(List transformers) return this; } + public List getAllExpressions() { + List expressions = new ArrayList<>(); + if (columns != null) { + expressions.add(columns); + } + for (ColumnsTransformer transformer : transformers) { + transformer.collectExpressions(expressions); + } + return expressions; + } + public StringBuilder appendTo(StringBuilder builder) { builder.append(columns); for (ColumnsTransformer transformer : transformers) { diff --git a/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java index a241c7a1a..fcf6517ce 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java +++ b/src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java @@ -73,6 +73,30 @@ public ColumnsTransformer setReplaceItems(List> replaceItems) { return this; } + void collectExpressions(List expressions) { + switch (type) { + case APPLY: + if (applyExpression != null) { + expressions.add(applyExpression); + } + break; + case EXCEPT: + if (exceptColumns != null) { + expressions.addAll(exceptColumns); + } + break; + case REPLACE: + if (replaceItems != null) { + for (SelectItem item : replaceItems) { + expressions.add(item.getExpression()); + } + } + break; + default: + throw new IllegalStateException("Unhandled ColumnsTransformerType: " + type); + } + } + public StringBuilder appendTo(StringBuilder builder) { switch (type) { case APPLY: diff --git a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java index ea397970c..ecc332aea 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java +++ b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitor.java @@ -796,7 +796,14 @@ default void visit(LambdaExpression lambdaExpression) { this.visit(lambdaExpression, null); } - T visit(ColumnsExpression columnsExpression, S context); + default T visit(ColumnsExpression columnsExpression, S context) { + for (Expression expression : columnsExpression.getAllExpressions()) { + if (expression != null) { + expression.accept(this, context); + } + } + return null; + } default void visit(ColumnsExpression columnsExpression) { this.visit(columnsExpression, null); diff --git a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java index 8c89da29c..fed7b6f0c 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java +++ b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java @@ -877,22 +877,7 @@ public T visit(LambdaExpression lambdaExpression, S context) { @Override public T visit(ColumnsExpression columnsExpression, S context) { - if (columnsExpression.getColumns() != null) { - columnsExpression.getColumns().accept(this, context); - } - for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { - if (transformer.getApplyExpression() != null) { - transformer.getApplyExpression().accept(this, context); - } - if (transformer.getReplaceItems() != null) { - for (SelectItem selectItem : transformer.getReplaceItems()) { - if (selectItem.getExpression() != null) { - selectItem.getExpression().accept(this, context); - } - } - } - } - return null; + return visitExpressions(columnsExpression, context, columnsExpression.getAllExpressions()); } @Override diff --git a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java index f5d434e73..9a261cf8e 100644 --- a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java +++ b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java @@ -2229,19 +2229,9 @@ public Void visit(LambdaExpression lambdaExpression, S context) { @Override public Void visit(ColumnsExpression columnsExpression, S context) { - if (columnsExpression.getColumns() != null) { - columnsExpression.getColumns().accept(this, context); - } - for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { - if (transformer.getApplyExpression() != null) { - transformer.getApplyExpression().accept(this, context); - } - if (transformer.getReplaceItems() != null) { - for (SelectItem selectItem : transformer.getReplaceItems()) { - if (selectItem.getExpression() != null) { - selectItem.getExpression().accept(this, context); - } - } + for (Expression expression : columnsExpression.getAllExpressions()) { + if (expression != null) { + expression.accept(this, context); } } return null; diff --git a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java index 0e8293ecb..b791201df 100644 --- a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java +++ b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java @@ -26,6 +26,7 @@ import net.sf.jsqlparser.expression.CastExpression; import net.sf.jsqlparser.expression.CollateExpression; import net.sf.jsqlparser.expression.ColumnsExpression; +import net.sf.jsqlparser.expression.ColumnsTransformer; import net.sf.jsqlparser.expression.ConnectByPriorOperator; import net.sf.jsqlparser.expression.ConnectByRootOperator; import net.sf.jsqlparser.expression.DateTimeLiteralExpression; @@ -1929,7 +1930,39 @@ public StringBuilder visit(LambdaExpression lambdaExpression, S context) { @Override public StringBuilder visit(ColumnsExpression columnsExpression, S context) { - return columnsExpression.appendTo(builder); + columnsExpression.getColumns().accept(this, context); + for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { + switch (transformer.getType()) { + case APPLY: + builder.append(" APPLY("); + transformer.getApplyExpression().accept(this, context); + builder.append(")"); + break; + case EXCEPT: + builder.append(" EXCEPT "); + transformer.getExceptColumns().accept(this, context); + break; + case REPLACE: + builder.append(" REPLACE("); + boolean first = true; + for (SelectItem item : transformer.getReplaceItems()) { + if (!first) { + builder.append(", "); + } + first = false; + item.getExpression().accept(this, context); + if (item.getAlias() != null) { + builder.append(item.getAlias()); + } + } + builder.append(")"); + break; + default: + throw new IllegalStateException( + "Unhandled ColumnsTransformerType: " + transformer.getType()); + } + } + return builder; } @Override diff --git a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java index a89885a95..7b08c009e 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java @@ -21,7 +21,6 @@ import net.sf.jsqlparser.expression.CastExpression; import net.sf.jsqlparser.expression.CollateExpression; import net.sf.jsqlparser.expression.ColumnsExpression; -import net.sf.jsqlparser.expression.ColumnsTransformer; import net.sf.jsqlparser.expression.ConnectByPriorOperator; import net.sf.jsqlparser.expression.ConnectByRootOperator; import net.sf.jsqlparser.expression.DateTimeLiteralExpression; @@ -1222,19 +1221,9 @@ public Void visit(LambdaExpression lambdaExpression, S context) { @Override public Void visit(ColumnsExpression columnsExpression, S context) { - if (columnsExpression.getColumns() != null) { - columnsExpression.getColumns().accept(this, context); - } - for (ColumnsTransformer transformer : columnsExpression.getTransformers()) { - if (transformer.getApplyExpression() != null) { - transformer.getApplyExpression().accept(this, context); - } - if (transformer.getReplaceItems() != null) { - for (SelectItem selectItem : transformer.getReplaceItems()) { - if (selectItem.getExpression() != null) { - selectItem.getExpression().accept(this, context); - } - } + for (Expression expression : columnsExpression.getAllExpressions()) { + if (expression != null) { + expression.accept(this, context); } } return null; diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 84c37fc04..0e1cf2730 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -1274,125 +1274,6 @@ public class CCJSqlParser extends AbstractJSqlParser { return false; } - /** - * Upper bound for the token scans of {@link #isColumnsExpressionAhead()}; content - * beyond the budget cannot come from the alias-columns grammar, and the inputs it - * belongs to are rejected by the baseline parser anyway, so declining to commit - * keeps the scan linear on pathological content. - */ - private static final int CONTENT_SCAN_BUDGET = 4096; - - /** - * True when a COLUMNS(...) function is followed by at least one ClickHouse transformer - * (APPLY, EXCEPT, REPLACE). Scans the balanced parentheses of the function call and - * checks the token after the closing bracket. A plain COLUMNS('regexp') without a - * transformer stays on the regular expression path and keeps parsing as a Function. - * - * For APPLY and REPLACE the follower's own parameter content is also checked: when - * it is shaped like the alias-columns grammar T(name[type], ...) that the regular - * alias path reads (a depth-1 comma for APPLY, or two adjacent identifier-kind - * tokens at depth 1 for both), this predicate returns false so the input keeps its - * exact baseline parse instead of committing to a transformer body that cannot - * parse it. The scan is bounded and exception-safe: on pathological content it - * returns false, which hands the input to the baseline paths. - */ - protected boolean isColumnsExpressionAhead() { - Token columnsToken = getToken(1); - if (columnsToken.kind != K_COLUMNS || getToken(2).kind != OPENING_BRACKET) { - return false; - } - try { - int depth = 1; - int index = 3; - while (true) { - Token token = getToken(index++); - if (token.kind == EOF) { - return false; - } - if (token.kind == OPENING_BRACKET) { - depth++; - } else if (token.kind == CLOSING_BRACKET) { - if (--depth == 0) { - int followerKind = getToken(index).kind; - if (followerKind != K_APPLY && followerKind != K_REPLACE) { - return followerKind == K_EXCEPT; - } - return !isAliasColumnsContent(index + 2, followerKind == K_APPLY); - } - } - if (index > CONTENT_SCAN_BUDGET) { - return false; - } - } - } catch (TokenMgrException e) { - return false; - } - } - - /** - * Walks the balanced parentheses of a follower's parameter list and reports whether - * its depth-1 content is shaped like the alias-columns grammar T(name[type], ...) - * rather than like a transformer body: a depth-1 comma (which a single expression or - * lambda body can never have, checked for APPLY only), or two adjacent - * identifier-kind tokens at depth 1 (the name/type adjacency of a typed alias - * column, e.g. T(a INT); expression grammar never places two such tokens side by - * side, which keeps predicate keywords like AS, IS or AND out of the rule). When - * this returns true the input belongs to the regular alias path, whose parse is - * identical to the baseline parser's. - */ - protected boolean isAliasColumnsContent(int startIndex, boolean commaDisqualifies) { - int depth = 1; - int index = startIndex; - int previousKind = -1; - while (true) { - Token token = getToken(index++); - if (token.kind == EOF) { - return false; - } - if (token.kind == OPENING_BRACKET) { - depth++; - } else if (token.kind == CLOSING_BRACKET) { - if (--depth == 0) { - return false; - } - } else if (depth == 1) { - if (token.kind == K_COMMA && commaDisqualifies) { - return true; - } - if (isIdentifierKind(previousKind) && isIdentifierKind(token.kind)) { - return true; - } - } - if (index - startIndex > CONTENT_SCAN_BUDGET) { - return false; - } - previousKind = token.kind; - } - } - - /** - * The token kinds that RelObjectName and ColDataType are built from, plus the - * reserved keywords usable as identifiers in their context-guarded positions. - * DT_ZONE (a compound "TIMESTAMP WITH TIME ZONE" style type lexed as one token) - * belongs here because ColDataType accepts it as a type name. - */ - private static boolean isIdentifierKind(int kind) { - switch (kind) { - case S_IDENTIFIER: case S_QUOTED_IDENTIFIER: case DATA_TYPE: - case K_DATETIMELITERAL: case K_DATE_LITERAL: case DT_ZONE: - case K_ALL: case K_ANY: case K_CASEWHEN: case K_CONNECT: case K_CREATE: - case K_DEFAULT: case K_GLOBAL: case K_GROUP: case K_GROUPING: case K_IF: - case K_IIF: case K_IGNORE: case K_IN: case K_INTERVAL: case K_LEFT: - case K_LIMIT: case K_NEXTVAL: case K_OFFSET: case K_ON: case K_OPTIMIZE: - case K_ORDER: case K_PROCEDURE: case K_PUBLIC: case K_QUALIFY: case K_RIGHT: - case K_FILE: case K_SET: case K_SOME: case K_START: case K_TABLES: case K_TOP: - case K_VALUE: case K_VALUES: - return true; - default: - return kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD; - } - } - private static void requireTypeDdlSyntax(boolean valid, String message) throws ParseException { if (!valid) { throw new ParseException(message); } } @@ -7129,19 +7010,9 @@ SelectItem SelectItem() #SelectItem: { Expression expression; Alias alias = null; - Expression columnsItemExpression; } { - // @fixme: Oracle's SEQUENCE.nextval is parsed as COLUMN with a name part nextval - // @todo: parse a proper SEQUENCE instead of a COLUMN - ( - LOOKAHEAD( { isColumnsExpressionAhead() } ) columnsItemExpression = ColumnsExpression() - { expression = columnsItemExpression; } - | - LOOKAHEAD( 3 ) expression = ConnectByPriorOperator() - | - ( expression=Condition() { expression = prattExpressionRest(expression, 2); } ) - ) + expression = SelectItemExpression() [ LOOKAHEAD({ isAliasAhead() }) alias=Alias() ] { SelectItem selectItem = new SelectItem(expression, alias); @@ -7151,32 +7022,68 @@ SelectItem SelectItem() #SelectItem: } /** - * Parses a ClickHouse COLUMNS('regexp') matcher followed by zero or more transformers, - * for example COLUMNS('^m') APPLY(x -> round(x, 2)). Each transformer branch carries a - * syntactic lookahead, and when no transformer materializes the plain Function is - * returned, so the committed select-item branch degrades to the exact regular parse - * instead of failing: EXCEPT stays a set operation and bare APPLY/REPLACE stay aliases, - * exactly as without this production. + * Recognizes a transformer without consuming it. APPLY(functionName) intentionally takes + * precedence over its ambiguous alias-column reading. Other aliases use the existing Alias + * grammar, including typed columns, at every transformer boundary. Only the transformer + * prefix is probed: expression bodies are parsed normally by the Pratt parser. */ -Expression ColumnsExpression() #ColumnsExpression: +boolean ColumnsTransformerAhead() #void: +{} { - Expression columns; + LOOKAHEAD( "(" RelObjectName() ")" ) { return true; } + | LOOKAHEAD( ( | ) AliasColumns() ) { return false; } + | LOOKAHEAD( "(" | "(" | ParenthesedColumnList() ) + { return true; } + | { return false; } +} + +/** + * Keeps transformer consumption visible to enclosing syntactic lookaheads. The normal + * expression is parsed first; only a direct COLUMNS function can acquire transformers. + * The conditional node encloses both the matcher and its transformers, but adds no AST + * parent for ordinary expressions or alias-column lists. + */ +Expression SelectItemExpression() #ColumnsExpression(transformers != null): +{ + Expression expression; ColumnsTransformer transformer; - List transformers = new ArrayList(); + List transformers = null; } { - columns = Function() + // @fixme: Oracle's SEQUENCE.nextval is parsed as COLUMN with a name part nextval + // @todo: parse a proper SEQUENCE instead of a COLUMN + ( + LOOKAHEAD(3) expression = ConnectByPriorOperator() + | + ( expression = Condition() { expression = prattExpressionRest(expression, 2); } ) + ) ( - LOOKAHEAD( | | ParenthesedColumnList() ) + LOOKAHEAD( "(" | "(" | ParenthesedColumnList() ) + { + if (!(expression instanceof Function)) { + break; + } + Function function = (Function) expression; + if (!"COLUMNS".equalsIgnoreCase(function.getName()) || function.isEscaped() + || function.getAttribute() != null || function.getChainedParameters() != null + || function.getKeep() != null || function.isIgnoreNullsOutside() + || !ColumnsTransformerAhead()) { + break; + } + } transformer = ColumnsTransformer() - { transformers.add(transformer); } + { + if (transformers == null) { + transformers = new ArrayList(); + } + transformers.add(transformer); + } )* { - if (transformers.isEmpty()) { - linkAST(columns, jjtThis); - return columns; + if (transformers == null) { + return expression; } - ColumnsExpression columnsExpression = new ColumnsExpression(columns, transformers); + ColumnsExpression columnsExpression = new ColumnsExpression(expression, transformers); linkAST(columnsExpression, jjtThis); return columnsExpression; } @@ -7264,8 +7171,7 @@ Alias Alias(): Token token = null; boolean useAs = false; Alias alias; - String colname; - ColDataType colDataType = null; + List aliasColumns; } { ( @@ -7277,12 +7183,7 @@ Alias Alias(): [ LOOKAHEAD(2) name=RelObjectName() ] { alias = new Alias(name, true ); } - "(" { List list = new ArrayList(); } - colname = RelObjectName() [ colDataType = ColDataType() ] { list.add(new Alias.AliasColumn(colname, colDataType)); } - ( - "," { colDataType=null; } colname = RelObjectName() [ colDataType = ColDataType()] { list.add(new Alias.AliasColumn(colname, colDataType)); } - )* - ")" { alias.setAliasColumns(list); } + aliasColumns = AliasColumns() { alias.setAliasColumns(aliasColumns); } ) | @@ -7296,16 +7197,30 @@ Alias Alias(): ( name=RelObjectName() | token= { name=token.image; } ) { alias = new Alias(name,useAs); } - [ LOOKAHEAD(2) "(" { List list = new ArrayList(); } - colname = RelObjectName() [ colDataType = ColDataType() ] { list.add(new Alias.AliasColumn(colname, colDataType)); } - ("," { colDataType=null; } colname = RelObjectName() [ colDataType = ColDataType()] { list.add(new Alias.AliasColumn(colname, colDataType)); } )* - ")" { alias.setAliasColumns(list); } ] + [ LOOKAHEAD(2) aliasColumns = AliasColumns() { alias.setAliasColumns(aliasColumns); } ] ) ) { return alias; } } +List AliasColumns(): +{ + List columns = new ArrayList(); + String name; + ColDataType type = null; +} +{ + "(" name = RelObjectName() [ type = ColDataType() ] + { columns.add(new Alias.AliasColumn(name, type)); } + ( + "," { type = null; } name = RelObjectName() [ type = ColDataType() ] + { columns.add(new Alias.AliasColumn(name, type)); } + )* + ")" + { return columns; } +} + void SQLServerHint(SQLServerHints hints) : { String str; } diff --git a/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java b/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java new file mode 100644 index 000000000..727bae54a --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java @@ -0,0 +1,200 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.util.TablesNamesFinder; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; +import net.sf.jsqlparser.util.deparser.StatementDeParser; +import net.sf.jsqlparser.util.validation.ValidationContext; +import net.sf.jsqlparser.util.validation.validator.ExpressionValidator; +import org.junit.jupiter.api.Test; + +class ColumnsExpressionTest { + + private static final String SQL = "SELECT COLUMNS('m') APPLY(x -> x + 1)" + + " EXCEPT (skipped) REPLACE(value + 2 AS value) FROM metrics"; + + @Test + void testDeparserVisitsTransformerChildren() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(SQL); + StringBuilder buffer = new StringBuilder(); + List columns = new ArrayList<>(); + ExpressionDeParser expressions = new ExpressionDeParser() { + @Override + public StringBuilder visit(StringValue value, S context) { + return getBuilder().append("'renamed'"); + } + + @Override + public StringBuilder visit(LongValue value, S context) { + return getBuilder().append('?'); + } + + @Override + public StringBuilder visit(Column column, S context) { + columns.add(column.getColumnName()); + return super.visit(column, context); + } + }; + select.accept(new StatementDeParser(expressions, new SelectDeParser(), buffer), null); + + assertEquals("SELECT COLUMNS('renamed') APPLY(x -> x + ?)" + + " EXCEPT (skipped) REPLACE(value + ? AS value) FROM metrics", buffer.toString()); + assertEquals(List.of("x", "skipped", "value"), columns); + assertEquals(SQL, select.toString()); + assertSqlCanBeParsedAndDeparsed(SQL, true); + } + + @Test + void testAdapterVisitsEveryTransformerInOrder() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(SQL); + ColumnsExpression expression = assertInstanceOf(ColumnsExpression.class, + select.getSelectItems().get(0).getExpression()); + List visited = new ArrayList<>(); + Object marker = new Object(); + expression.accept(new ExpressionVisitorAdapter() { + @Override + public Void visit(StringValue value, S context) { + assertSame(marker, context); + visited.add(value.getValue()); + return null; + } + + @Override + public Void visit(Column column, S context) { + assertSame(marker, context); + visited.add(column.getColumnName()); + return null; + } + }, marker); + assertEquals(List.of("m", "x", "skipped", "value"), visited); + } + + @Test + void testAdapterPreservesCollectedResult() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse( + "SELECT COLUMNS('m') EXCEPT (skipped) FROM metrics"); + Expression expression = select.getSelectItems().get(0).getExpression(); + ExpressionVisitorAdapter> collector = new ExpressionVisitorAdapter<>() { + @Override + protected List applyExpression(Expression value, S context) { + return List.of(value.toString()); + } + + @Override + protected List visitExpressions(Expression value, S context, + Collection children) { + List result = new ArrayList<>(); + for (Expression child : children) { + if (child != null) { + result.addAll(child.accept(this, context)); + } + } + return result; + } + }; + assertEquals(List.of("'m'", "skipped"), expression.accept(collector, null)); + } + + @Test + void testVisitorDefaultDispatchesChildren() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(SQL); + ColumnsExpression expression = assertInstanceOf(ColumnsExpression.class, + select.getSelectItems().get(0).getExpression()); + ExpressionVisitor visitor = mock(ExpressionVisitor.class, CALLS_REAL_METHODS); + Object context = new Object(); + expression.accept(visitor, context); + + verify(visitor).visit((Function) expression.getColumns(), context); + verify(visitor).visit((LambdaExpression) expression.getTransformers().get(0) + .getApplyExpression(), context); + verify(visitor).visit(expression.getTransformers().get(1).getExceptColumns().get(0), + context); + verify(visitor).visit( + (net.sf.jsqlparser.expression.operators.arithmetic.Addition) expression + .getTransformers().get(2).getReplaceItems().get(0).getExpression(), + context); + } + + @Test + void testValidatorVisitsExceptColumns() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(SQL); + ColumnsExpression expression = assertInstanceOf(ColumnsExpression.class, + select.getSelectItems().get(0).getExpression()); + List visited = new ArrayList<>(); + ExpressionValidator validator = new ExpressionValidator() { + @Override + public Void visit(Column column, S context) { + visited.add(column.getColumnName()); + return null; + } + }; + validator.setContext(new ValidationContext().setCapabilities(Collections.emptyList())); + expression.accept(validator, null); + assertEquals(List.of("x", "skipped", "value"), visited); + } + + @Test + void testFinderVisitsExceptColumns() throws JSQLParserException { + List visited = new ArrayList<>(); + TablesNamesFinder finder = new TablesNamesFinder<>() { + @Override + public Void visit(Column column, S context) { + visited.add(column.getColumnName()); + return super.visit(column, context); + } + }; + assertEquals(Set.of("metrics"), finder.getTables(CCJSqlParserUtil.parse(SQL))); + assertEquals(List.of("x", "skipped", "value"), visited); + } + + @Test + void testAstTokenRangeIncludesMatcher() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse( + "SELECT COLUMNS('m') APPLY(sum) AS result FROM metrics"); + ColumnsExpression expression = assertInstanceOf(ColumnsExpression.class, + select.getSelectItem(0).getExpression()); + assertEquals("COLUMNS", expression.getASTNode().jjtGetFirstToken().image); + assertEquals(")", expression.getASTNode().jjtGetLastToken().image); + assertEquals("AS", expression.getASTNode().jjtGetLastToken().next.image); + assertEquals("COLUMNS", expression.getColumns().getASTNode().jjtGetFirstToken().image); + assertEquals("APPLY", expression.getColumns().getASTNode().jjtGetLastToken().next.image); + assertSame(expression, ((Function) expression.getColumns()).getParent()); + assertSame(expression, ((Function) expression.getColumns()) + .getParent(ColumnsExpression.class)); + } + + @Test + void testFinderVisitsTransformerSubqueries() throws JSQLParserException { + String sql = "SELECT COLUMNS('m') APPLY(x -> (SELECT max(v) FROM applied))" + + " REPLACE((SELECT v FROM replacement) AS c) FROM metrics"; + assertEquals(Set.of("metrics", "applied", "replacement"), + TablesNamesFinder.findTables(sql)); + assertSqlCanBeParsedAndDeparsed(sql, true); + } +} diff --git a/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java b/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java index 73ba93171..696bf88a0 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java @@ -12,6 +12,7 @@ import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.Alias; import net.sf.jsqlparser.expression.ColumnsExpression; import net.sf.jsqlparser.expression.ColumnsTransformer; import net.sf.jsqlparser.expression.Function; @@ -19,6 +20,8 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class ClickHouseTest { @@ -397,4 +400,92 @@ public void testTypedAliasColumnWithCompoundTimeZoneType() throws JSQLParserExce assertSqlCanBeParsedAndDeparsed( "SELECT COLUMNS('m') APPLY(a TIMESTAMP WITHOUT TIME ZONE) FROM t", true); } + + @ParameterizedTest + @ValueSource(strings = {"[x, x + 1]", "x IN (1, 2)", + "CASE x WHEN 1 THEN 0 ELSE 1 END"}) + public void testColumnsApplyExpressionBodies(String body) throws JSQLParserException { + for (String prefix : new String[] {"", " APPLY(sum)"}) { + String sql = "SELECT COLUMNS('m')" + prefix + " APPLY(x -> " + body + ") FROM t"; + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sql, true); + ColumnsExpression expression = Assertions.assertInstanceOf(ColumnsExpression.class, + select.getSelectItem(0).getExpression()); + ColumnsTransformer transformer = expression.getTransformers() + .get(expression.getTransformers().size() - 1); + LambdaExpression lambda = Assertions.assertInstanceOf(LambdaExpression.class, + transformer.getApplyExpression()); + Assertions.assertEquals(body, lambda.getExpression().toString()); + } + } + + @Test + public void testColumnsReplaceCaseExpression() throws JSQLParserException { + String sql = "SELECT COLUMNS('m') REPLACE(CASE m WHEN 1 THEN 0 ELSE 1 END AS m) FROM t"; + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sql, true); + ColumnsExpression expression = Assertions.assertInstanceOf(ColumnsExpression.class, + select.getSelectItem(0).getExpression()); + SelectItem replacement = expression.getTransformers().get(0).getReplaceItems().get(0); + Assertions.assertEquals("CASE m WHEN 1 THEN 0 ELSE 1 END", + replacement.getExpression().toString()); + Assertions.assertEquals("m", replacement.getAlias().getName()); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT * FROM (SELECT COLUMNS('m') APPLY(sum) FROM t) q", + "WITH q AS (SELECT COLUMNS('m') APPLY(x -> x + 1) FROM t) SELECT * FROM q", + "SELECT (SELECT COLUMNS('m') APPLY(sum) FROM t) AS result", + "SELECT COLUMNS('m') APPLY(sum) EXCEPT (SELECT b FROM t)", + "SELECT 1 UNION SELECT COLUMNS('m') APPLY(sum) FROM t", + "INSERT INTO dst SELECT COLUMNS('m') APPLY(sum) FROM t", + "SELECT JSON_VALUE(doc, '$' DEFAULT (SELECT COLUMNS('m') APPLY(sum) APPLY(max) FROM t) ON EMPTY) FROM src"}) + public void testColumnsInSelectContexts(String sql) throws JSQLParserException { + assertSqlCanBeParsedAndDeparsed(sql, true); + } + + @ParameterizedTest + @ValueSource(strings = {"COLUMNS('m').x", "{fn COLUMNS('m')}", "COLUMNS('m')(x)"}) + public void testDecoratedColumnsFunctionKeepsAlias(String function) throws JSQLParserException { + String sql = "SELECT " + function + " APPLY(sum) FROM t"; + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sql, true); + SelectItem item = select.getSelectItem(0); + Function expression = Assertions.assertInstanceOf(Function.class, item.getExpression()); + Assertions.assertSame(item, expression.getParent()); + Assertions.assertEquals("APPLY", item.getAlias().getName()); + Assertions.assertEquals("sum", item.getAlias().getAliasColumns().get(0).name); + } + + @ParameterizedTest + @ValueSource(strings = {"APPLY(a, b)", "APPLY(a INT)", "REPLACE(a INT)", + "REPLACE(a)", "REPLACE(a, b)"}) + public void testColumnsAliasAstAtEachBoundary(String suffix) throws JSQLParserException { + for (String prefix : new String[] {"", " APPLY(sum)"}) { + String sql = "SELECT COLUMNS('m')" + prefix + " " + suffix + " FROM t"; + PlainSelect select = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sql, true); + SelectItem item = select.getSelectItem(0); + if (prefix.isEmpty()) { + Assertions.assertInstanceOf(Function.class, item.getExpression()); + } else { + ColumnsExpression expression = Assertions.assertInstanceOf(ColumnsExpression.class, + item.getExpression()); + Assertions.assertEquals(1, expression.getTransformers().size()); + Assertions.assertEquals("sum", expression.getTransformers().get(0) + .getApplyExpression().toString()); + } + Alias alias = item.getAlias(); + Assertions.assertNotNull(alias); + Assertions.assertEquals(suffix.substring(0, suffix.indexOf('(')), alias.getName()); + Assertions.assertFalse(alias.isUseAs()); + Assertions.assertEquals("a", alias.getAliasColumns().get(0).name); + if (suffix.contains("INT")) { + Assertions.assertEquals("INT", + alias.getAliasColumns().get(0).colDataType.toString()); + } else { + Assertions.assertNull(alias.getAliasColumns().get(0).colDataType); + if (suffix.contains(",")) { + Assertions.assertEquals("b", alias.getAliasColumns().get(1).name); + } + } + } + } } From 37d48b3138292cdc349904ea506079a9f7a4cc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=98=E5=85=B8?= Date: Wed, 16 Sep 2026 21:50:47 +0800 Subject: [PATCH 4/4] style: use simple name for the Addition cast in ColumnsExpressionTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fully qualified name trips the UnnecessaryFullyQualifiedName code style check. Import the type instead, matching the sibling ExpressionPrecedenceTest. Signed-off-by: 付典 --- .../net/sf/jsqlparser/expression/ColumnsExpressionTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java b/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java index 727bae54a..bdfa302ef 100644 --- a/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java +++ b/src/test/java/net/sf/jsqlparser/expression/ColumnsExpressionTest.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Set; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.operators.arithmetic.Addition; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.PlainSelect; @@ -136,7 +137,7 @@ void testVisitorDefaultDispatchesChildren() throws JSQLParserException { verify(visitor).visit(expression.getTransformers().get(1).getExceptColumns().get(0), context); verify(visitor).visit( - (net.sf.jsqlparser.expression.operators.arithmetic.Addition) expression + (Addition) expression .getTransformers().get(2).getReplaceItems().get(0).getExpression(), context); }