fix(spark): #501 fix syntax error for subtraction between column and function call

src/grammar/spark/SparkSqlParser.g4

@@ -1272,17 +1272,8 @@ qualifiedName
     : identifier (DOT identifier)*
     ;
 
-// this rule is used for explicitly capturing wrong identifiers such as test-table, which should actually be `test-table`
-// replace identifier with errorCapturingIdentifier where the immediate follow symbol is not an expression, otherwise
-// valid expressions such as "a-b" can be recognized as an identifier
 errorCapturingIdentifier
-    : identifier errorCapturingIdentifierExtra
-    ;
-
-// extra left-factoring grammar
-errorCapturingIdentifierExtra
-    : (MINUS identifier)+
-    |
+    : identifier
     ;
 
 identifier

src/lib/spark/SparkSqlParserListener.ts

@@ -281,7 +281,6 @@ import { FunctionNameContext } from "./SparkSqlParser.js";
 import { FunctionNameCreateContext } from "./SparkSqlParser.js";
 import { QualifiedNameContext } from "./SparkSqlParser.js";
 import { ErrorCapturingIdentifierContext } from "./SparkSqlParser.js";
-import { ErrorCapturingIdentifierExtraContext } from "./SparkSqlParser.js";
 import { IdentifierContext } from "./SparkSqlParser.js";
 import { StrictIdentifierContext } from "./SparkSqlParser.js";
 import { QuotedIdentifierContext } from "./SparkSqlParser.js";
@@ -3230,16 +3229,6 @@ export class SparkSqlParserListener implements ParseTreeListener {
      * @param ctx the parse tree
      */
     exitErrorCapturingIdentifier?: (ctx: ErrorCapturingIdentifierContext) => void;
-    /**
-     * Enter a parse tree produced by `SparkSqlParser.errorCapturingIdentifierExtra`.
-     * @param ctx the parse tree
-     */
-    enterErrorCapturingIdentifierExtra?: (ctx: ErrorCapturingIdentifierExtraContext) => void;
-    /**
-     * Exit a parse tree produced by `SparkSqlParser.errorCapturingIdentifierExtra`.
-     * @param ctx the parse tree
-     */
-    exitErrorCapturingIdentifierExtra?: (ctx: ErrorCapturingIdentifierExtraContext) => void;
     /**
      * Enter a parse tree produced by `SparkSqlParser.identifier`.
      * @param ctx the parse tree

src/lib/spark/SparkSqlParserVisitor.ts

@@ -281,7 +281,6 @@ import { FunctionNameContext } from "./SparkSqlParser.js";
 import { FunctionNameCreateContext } from "./SparkSqlParser.js";
 import { QualifiedNameContext } from "./SparkSqlParser.js";
 import { ErrorCapturingIdentifierContext } from "./SparkSqlParser.js";
-import { ErrorCapturingIdentifierExtraContext } from "./SparkSqlParser.js";
 import { IdentifierContext } from "./SparkSqlParser.js";
 import { StrictIdentifierContext } from "./SparkSqlParser.js";
 import { QuotedIdentifierContext } from "./SparkSqlParser.js";
@@ -2040,12 +2039,6 @@ export class SparkSqlParserVisitor<Result> extends AbstractParseTreeVisitor<Resu
      * @return the visitor result
      */
     visitErrorCapturingIdentifier?: (ctx: ErrorCapturingIdentifierContext) => Result;
-    /**
-     * Visit a parse tree produced by `SparkSqlParser.errorCapturingIdentifierExtra`.
-     * @param ctx the parse tree
-     * @return the visitor result
-     */
-    visitErrorCapturingIdentifierExtra?: (ctx: ErrorCapturingIdentifierExtraContext) => Result;
     /**
      * Visit a parse tree produced by `SparkSqlParser.identifier`.
      * @param ctx the parse tree

test/parser/spark/contextCollect/entityCollector.test.ts

@@ -660,4 +660,40 @@ describe('SparkSQL entity collector tests', () => {
         expect(queryResults[0].columns?.[0].text).toBe('name');
         expect(queryResults[1].text).toBe('id');
     });
+
+    // https://github.com/DTStack/dt-sql-parser/issues/501
+    test('should collect subtraction expressions as expression columns', () => {
+        const spark = new SparkSQL();
+        const allEntities = spark.getAllEntities(
+            'SELECT a - b, t.a - t.b AS diff, a - f(x), (price - if(`symbol` is null, open_price, 0)) * quantity AS total FROM t;'
+        );
+        const queryResult = allEntities.find(
+            (e) => e.entityContextType === EntityContextType.QUERY_RESULT
+        ) as CommonEntityContext;
+
+        expect(queryResult).toBeDefined();
+        expect(queryResult.columns?.length).toBe(4);
+
+        expect(queryResult.columns[0].text).toBe('a - b');
+        expect(queryResult.columns[0].declareType).toBe(ColumnDeclareType.EXPRESSION);
+        expect(queryResult.columns[0][AttrName.alias]).toBeFalsy();
+
+        expect(queryResult.columns[1].text).toBe('t.a - t.b');
+        expect(queryResult.columns[1].declareType).toBe(ColumnDeclareType.EXPRESSION);
+        expect(queryResult.columns[1][AttrName.alias]).toEqual(
+            expect.objectContaining({ text: 'diff' })
+        );
+
+        expect(queryResult.columns[2].text).toBe('a - f(x)');
+        expect(queryResult.columns[2].declareType).toBe(ColumnDeclareType.EXPRESSION);
+        expect(queryResult.columns[2][AttrName.alias]).toBeFalsy();
+
+        expect(queryResult.columns[3].text).toBe(
+            '(price - if(`symbol` is null, open_price, 0)) * quantity'
+        );
+        expect(queryResult.columns[3].declareType).toBe(ColumnDeclareType.EXPRESSION);
+        expect(queryResult.columns[3][AttrName.alias]).toEqual(
+            expect.objectContaining({ text: 'total' })
+        );
+    });
 });

test/parser/spark/syntax/fixtures/reset.sql

@@ -1,9 +1,11 @@
 -- RESET;
-
 -- RESET configuration_key;
-
 -- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
 
 -- If you start your application with --conf spark.foo=bar and set spark.foo=foobar in runtime, the example below will restore it to 'bar'. If spark.foo is not specified during starting, the example below will remove this config from the SQLConf. It will ignore nonexistent keys.
 RESET spark.abc;
+
+RESET write.target-file-size-bytes;
+
+RESET tez.grouping.max-size;

test/parser/spark/syntax/fixtures/set.sql

@@ -13,3 +13,12 @@ SET;
 
 -- List the value of specified property key.
 SET spark.sql.variable.substitute;
+
+SET tez.grouping.max-size = 12800;
+SET tez.grouping.max-size;
+
+SET spark.sql.kafka.sasl.mechanism = SCRAM-SHA-512;
+
+SET fs.s3a.server-side-encryption-algorithm = AES-256;
+
+SET fs.s3a.server-side-encryption.key=arn:aws:kms;

test/parser/spark/syntax/selectStatement.test.ts

@@ -162,4 +162,39 @@ describe('Spark Select Syntax Tests', () => {
             expect(spark.validate(itemSql).length).toBe(0);
         });
     });
+
+    // https://github.com/DTStack/dt-sql-parser/issues/501
+    describe('subtraction expression with column and function call (#501)', () => {
+        const validSqlList = [
+            // Original issue SQL and comparison variations
+            'SELECT (price - if(`symbol` is null, open_price, 0)) * quantity FROM t;',
+            'SELECT price - if(`symbol` is null, open_price, 0) FROM t;',
+            'SELECT (price - if(`symbol` is null, open_price, 0) * quantity) FROM t;',
+            'SELECT (price + if(`symbol` is null, open_price, 0)) * quantity FROM t;',
+            'SELECT (1 - if(`symbol` is null, open_price, 0)) * quantity FROM t;',
+            // Qualified column names, backquoted hyphenated identifiers, and nested parentheses/functions
+            'SELECT (t.price - coalesce(t.discount, 0)) * t.quantity FROM orders t;',
+            'SELECT (`col-a` - if(`col-b` is null, `col-c`, 0)) * `col-d` FROM `my-db`.`my-table`;',
+            'SELECT ((a - coalesce(b, 0)) - if(c > 0, d, 1)) / total FROM t;',
+            'SELECT round((price - if(`symbol` is null, open_price, 0)) * quantity, 2) FROM t;',
+            // Subtraction with function call without FROM clause or followed by other clauses
+            'SELECT price - if(`symbol` is null, open_price, 0);',
+            'SELECT price - if(`symbol` is null, open_price, 0) WHERE price > 0;',
+            'SELECT price - if(`symbol` is null, open_price, 0) GROUP BY price HAVING price > 0;',
+            'SELECT price - if(`symbol` is null, open_price, 0) ORDER BY price LIMIT 10;',
+            // Zero-argument functions and compact subtraction without whitespace
+            'SELECT a - f(), (a - f()) * b, a-f(x), (a-b) FROM t;',
+            // Parenthesized subtraction in WHERE, JOIN ON, GROUP BY, HAVING, ORDER BY, and CASE WHEN
+            'SELECT * FROM t WHERE (price - if(`symbol` is null, open_price, 0)) > 0;',
+            'SELECT * FROM t1 JOIN t2 ON (t1.a - nvl(t2.b, 0)) = 0;',
+            'SELECT (a - coalesce(b, 0)), count(*) FROM t GROUP BY (a - coalesce(b, 0)) HAVING (max(a) - coalesce(min(b), 0)) > 0 ORDER BY (max(a) - coalesce(min(b), 0)) DESC;',
+            'SELECT CASE WHEN (price - if(discount is null, 0, discount)) > 0 THEN (price - coalesce(tax, 0)) ELSE 0 END FROM t;',
+        ];
+
+        validSqlList.forEach((sql) => {
+            it(sql, () => {
+                expect(spark.validate(sql).length).toBe(0);
+            });
+        });
+    });
 });

test/parser/spark/validateInvalidSql.test.ts

@@ -13,4 +13,10 @@ describe('Spark SQL validate invalid sql', () => {
     test('validate unComplete sql', () => {
         expect(spark.validate(unCompleteSQL).length).not.toBe(0);
     });
+
+    test('validate unquoted hyphenated identifiers', () => {
+        expect(spark.validate('CREATE DATABASE test-db;').length).not.toBe(0);
+        expect(spark.validate('CREATE TABLE test-tbl (id-col INT);').length).not.toBe(0);
+        expect(spark.validate('SELECT * FROM test-tbl;').length).not.toBe(0);
+    });
 });