Fix parsing of ClickHouse transformers (APPLY/EXCEPT/EXCLUDE/REPLACE) after '*'

src/main/java/net/sf/jsqlparser/expression/ColumnsTransformer.java

@@ -17,15 +17,15 @@
 import net.sf.jsqlparser.statement.select.SelectItem;
 
 /**
- * A ClickHouse transformer following a {@code COLUMNS(...)} matcher, for example the
+ * A ClickHouse transformer following a {@code COLUMNS(...)} matcher or a {@code *}, for example the
  * {@code APPLY(x -> round(x, 2))} in {@code SELECT COLUMNS('^m') APPLY(x -> round(x, 2))}.
  *
  * ClickHouse parses its transformers in a loop, so they may repeat and combine in any order.
  */
 public class ColumnsTransformer extends ASTNodeAccessImpl {
 
     public enum ColumnsTransformerType {
-        APPLY, EXCEPT, REPLACE
+        APPLY, EXCEPT, EXCLUDE, REPLACE
     }
 
     private ColumnsTransformerType type;
@@ -81,6 +81,7 @@ void collectExpressions(List<Expression> expressions) {
                 }
                 break;
             case EXCEPT:
+            case EXCLUDE:
                 if (exceptColumns != null) {
                     expressions.addAll(exceptColumns);
                 }
@@ -105,6 +106,9 @@ public StringBuilder appendTo(StringBuilder builder) {
             case EXCEPT:
                 builder.append("EXCEPT ").append(exceptColumns);
                 break;
+            case EXCLUDE:
+                builder.append("EXCLUDE ").append(exceptColumns);
+                break;
             case REPLACE:
                 builder.append("REPLACE(").append(Select.getStringList(replaceItems)).append(")");
                 break;

src/main/java/net/sf/jsqlparser/statement/select/AllColumns.java

@@ -9,92 +9,45 @@
  */
 package net.sf.jsqlparser.statement.select;
 
+import java.util.ArrayList;
+import java.util.List;
+import net.sf.jsqlparser.expression.ColumnsTransformer;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.ExpressionVisitor;
-import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
 import net.sf.jsqlparser.parser.ASTNodeAccessImpl;
-import net.sf.jsqlparser.schema.Column;
-
-import java.util.ArrayList;
-import java.util.List;
 
 public class AllColumns extends ASTNodeAccessImpl implements Expression {
-    protected ExpressionList<Column> exceptColumns;
-    protected List<SelectItem<?>> replaceExpressions;
-    private String exceptKeyword;
-
-    public AllColumns(ExpressionList<Column> exceptColumns,
-            List<SelectItem<?>> replaceExpressions) {
-        this.exceptColumns = exceptColumns;
-        this.replaceExpressions = replaceExpressions;
-        this.exceptKeyword = exceptColumns != null ? "Except" : null;
-    }
-
-    public AllColumns(ExpressionList<Column> exceptColumns,
-            List<SelectItem<?>> replaceExpressions, String exceptKeyword) {
-        this.exceptColumns = exceptColumns;
-        this.replaceExpressions = replaceExpressions;
-        this.exceptKeyword = exceptKeyword;
-    }
+    private List<ColumnsTransformer> transformers;
 
     public AllColumns() {
-        this(null, null);
-    }
-
-    public ExpressionList<Column> getExceptColumns() {
-        return exceptColumns;
+        this(null);
     }
 
-    public AllColumns setExceptColumns(ExpressionList<Column> exceptColumns) {
-        this.exceptColumns = exceptColumns;
-        return this;
+    public AllColumns(List<ColumnsTransformer> transformers) {
+        this.transformers = transformers;
     }
 
-    public ExpressionList<Column> addExceptColumn(Column column) {
-        if (exceptColumns == null) {
-            exceptColumns = new ExpressionList<>();
+    public List<ColumnsTransformer> getTransformers() {
+        if (transformers == null) {
+            transformers = new ArrayList<>();
         }
-        exceptColumns.add(column);
-        return exceptColumns;
+        return transformers;
     }
 
-    public List<SelectItem<?>> getReplaceExpressions() {
-        return replaceExpressions;
-    }
-
-    public AllColumns setReplaceExpressions(List<SelectItem<?>> replaceExpressions) {
-        this.replaceExpressions = replaceExpressions;
+    public AllColumns setTransformers(List<ColumnsTransformer> transformers) {
+        this.transformers = transformers;
         return this;
     }
 
-    public List<SelectItem<?>> addReplaceExpression(SelectItem<?> selectItem) {
-        if (replaceExpressions == null) {
-            replaceExpressions = new ArrayList<>();
-        }
-        replaceExpressions.add(selectItem);
-        return replaceExpressions;
-    }
-
-    public String getExceptKeyword() {
-        return exceptKeyword;
-    }
-
-    public AllColumns setExceptKeyword(String exceptKeyword) {
-        this.exceptKeyword = exceptKeyword;
+    public AllColumns addTransformer(ColumnsTransformer transformer) {
+        getTransformers().add(transformer);
         return this;
     }
 
     public StringBuilder appendTo(StringBuilder builder) {
         builder.append("*");
-        if (exceptColumns != null && !exceptColumns.isEmpty()) {
-            builder.append(" ").append(exceptKeyword).append("( ");
-            exceptColumns.appendTo(builder);
-            builder.append(" )");
-        }
-        if (replaceExpressions != null && !replaceExpressions.isEmpty()) {
-            builder.append(" REPLACE( ");
-            builder.append(Select.getStringList(replaceExpressions));
-            builder.append(" )");
+        for (ColumnsTransformer transformer : getTransformers()) {
+            builder.append(" ").append(transformer);
         }
         return builder;
     }

src/main/java/net/sf/jsqlparser/statement/select/AllTableColumns.java

@@ -10,9 +10,8 @@
 package net.sf.jsqlparser.statement.select;
 
 import java.util.List;
+import net.sf.jsqlparser.expression.ColumnsTransformer;
 import net.sf.jsqlparser.expression.ExpressionVisitor;
-import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
-import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.ReturningReferenceType;
 
@@ -22,24 +21,17 @@ public class AllTableColumns extends AllColumns {
     private ReturningReferenceType returningReferenceType = null;
     private String returningQualifier = null;
 
-    public AllTableColumns(Table table, ExpressionList<Column> exceptColumns,
-            List<SelectItem<?>> replaceExpressions, String exceptKeyword) {
-        super(exceptColumns, replaceExpressions, exceptKeyword);
+    public AllTableColumns(Table table, List<ColumnsTransformer> transformers) {
+        super(transformers);
         this.table = table;
     }
 
-    public AllTableColumns(Table table, ExpressionList<Column> exceptColumns,
-            List<SelectItem<?>> replaceExpressions) {
-        this(table, exceptColumns, replaceExpressions, "EXCEPT");
-    }
-
     public AllTableColumns(Table table) {
-        this(table, null, null);
+        this(table, (List<ColumnsTransformer>) null);
     }
 
     public AllTableColumns(Table table, AllColumns allColumns) {
-        this(table, allColumns.exceptColumns, allColumns.replaceExpressions,
-                allColumns.getExceptKeyword());
+        this(table, allColumns.getTransformers());
     }
 
     public Table getTable() {

src/main/java/net/sf/jsqlparser/statement/select/FunctionAllColumns.java

@@ -17,7 +17,7 @@ public class FunctionAllColumns extends AllColumns {
     private Function function;
 
     public FunctionAllColumns(Function function) {
-        super(null, null, null);
+        super(null);
         this.function = function;
     }
 

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -7846,7 +7846,27 @@ boolean ColumnsTransformerAhead() #void:
 {
       LOOKAHEAD( <K_APPLY> "(" RelObjectName() ")" ) { return true; }
     | LOOKAHEAD( ( <K_APPLY> | <K_REPLACE> ) AliasColumns() ) { return false; }
-    | LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "(" | <K_EXCEPT> ParenthesedColumnList() )
+    | LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
+        | ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
+      { return true; }
+    | { return false; }
+}
+
+/**
+ * Recognizes a transformer after the AllColumns-Pattern '*'.
+ *
+ * This differs from ColumnsTransformerAhead() in one point: after '*' an alias is not a
+ * meaningful reading of REPLACE, so REPLACE (col) stays a transformer here whereas
+ * APPLY (col1, col2) remains an alias. Reusing the COLUMNS(...) rules verbatim would turn
+ * '* REPLACE (x)' into an alias.
+ */
+boolean AllColumnsTransformerAhead() #void:
+{}
+{
+      LOOKAHEAD( <K_APPLY> "(" RelObjectName() ")" ) { return true; }
+    | LOOKAHEAD( <K_APPLY> AliasColumns() ) { return false; }
+    | LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
+        | ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
       { return true; }
     | { return false; }
 }
@@ -7914,6 +7934,7 @@ ColumnsTransformer ColumnsTransformer():
     ParenthesedExpressionList<Column> exceptColumns;
     List<SelectItem<?>> replaceItems;
     ColumnsTransformer transformer;
+    Token tk;
 }
 {
     (
@@ -7927,8 +7948,11 @@ ColumnsTransformer ColumnsTransformer():
           { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.APPLY)
                   .setApplyExpression(applyExpression); }
 
-        | <K_EXCEPT> exceptColumns = ParenthesedColumnList()
-          { transformer = new ColumnsTransformer(ColumnsTransformer.ColumnsTransformerType.EXCEPT)
+        | ( tk=<K_EXCEPT> | tk=<K_EXCLUDE> ) exceptColumns = ParenthesedColumnList()
+          { transformer = new ColumnsTransformer(
+                  tk.kind == K_EXCLUDE
+                          ? ColumnsTransformer.ColumnsTransformerType.EXCLUDE
+                          : ColumnsTransformer.ColumnsTransformerType.EXCEPT)
                   .setExceptColumns(exceptColumns); }
 
         | <K_REPLACE> "(" replaceItems = SelectItemsList() ")"
@@ -7945,20 +7969,32 @@ ColumnsTransformer ColumnsTransformer():
  */
 AllColumns AllColumns(boolean allowAdditions):
 {
-    ParenthesedExpressionList<Column> exceptColumns = null;
-    List<SelectItem<?>> replaceExpressions = null;
-    String exceptKeyword=null;
-    Token tk;
+    ColumnsTransformer transformer;
+    List<ColumnsTransformer> transformers = null;
 }
 {
     "*"
-    // BigData allows EXCEPT, DuckDB allows EXCLUDE
-    [ LOOKAHEAD(2, { allowAdditions }) ( tk=<K_EXCEPT> | tk=<K_EXCLUDE> ) exceptColumns = ParenthesedColumnList() { exceptKeyword=tk.image; } ]
-    // BigData allows REPLACE
-    [ LOOKAHEAD(2, { allowAdditions }) <K_REPLACE> "(" replaceExpressions = SelectItemsList() ")" ]
+    // BigData allows EXCEPT, DuckDB allows EXCLUDE, ClickHouse allows APPLY;
+    // they may repeat and combine in any order
+    (
+        LOOKAHEAD( <K_APPLY> "(" | <K_REPLACE> "("
+            | ( <K_EXCEPT> | <K_EXCLUDE> ) ParenthesedColumnList() )
+        {
+            if (!allowAdditions || !AllColumnsTransformerAhead()) {
+                break;
+            }
+        }
+        transformer = ColumnsTransformer()
+        {
+            if (transformers == null) {
+                transformers = new ArrayList<ColumnsTransformer>();
+            }
+            transformers.add(transformer);
+        }
+    )*
 
     {
-        return new AllColumns(exceptColumns, replaceExpressions, exceptKeyword);
+        return new AllColumns(transformers);
     }
 }
 

src/test/java/net/sf/jsqlparser/statement/select/ClickHouseTest.java

@@ -14,13 +14,12 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
-
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Alias;
 import net.sf.jsqlparser.expression.ColumnsExpression;
 import net.sf.jsqlparser.expression.ColumnsTransformer;
-import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
+import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.LambdaExpression;
 import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
@@ -337,6 +336,28 @@ public void testColumnsCombinedTransformers() throws JSQLParserException {
         assertSqlCanBeParsedAndDeparsed(sql, true);
     }
 
+    @Test
+    public void testAllColumnsApplyIssue2636() throws JSQLParserException {
+        // ClickHouse accepts the same transformers after '*':
+        // https://github.com/JSQLParser/JSqlParser/issues/2636
+        String sql = "SELECT * APPLY(sum) FROM t";
+        Select select = (Select) CCJSqlParserUtil.parse(sql);
+        PlainSelect plainSelect = (PlainSelect) select.getSelectBody();
+        SelectItem<?> selectItem = plainSelect.getSelectItems().get(0);
+
+        Assertions.assertNull(selectItem.getAlias(),
+                "APPLY must not fall back to an alias");
+        Assertions.assertInstanceOf(AllColumns.class, selectItem.getExpression());
+        assertSqlCanBeParsedAndDeparsed(sql, true);
+    }
+
+    @Test
+    public void testAllColumnsChainedTransformersIssue2636() throws JSQLParserException {
+        // ClickHouse parses the transformers in a loop, so they may repeat after '*'
+        String sql = "SELECT * APPLY(sum) EXCEPT (a) FROM t";
+        assertSqlCanBeParsedAndDeparsed(sql, true);
+    }
+
     @Test
     public void testColumnsApplyMultiParamLambda() throws JSQLParserException {
         String sql = "SELECT COLUMNS('m') APPLY((k, v) -> v > 5) FROM metrics";