Skip to content

Commit 5fd7241

Browse files
fudianchnfudianchn
committed
fix(parser): remove WITH lookahead token limits
Signed-off-by: 付典 <fudianchn@gmail.com>
1 parent d083de5 commit 5fd7241

2 files changed

Lines changed: 128 additions & 55 deletions

File tree

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 27 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ import net.sf.jsqlparser.statement.export.*;
8989
import net.sf.jsqlparser.statement.lock.*;
9090
import java.util.*;
9191
import java.util.AbstractMap.SimpleEntry;
92+
import java.util.concurrent.CancellationException;
9293
import net.sf.jsqlparser.statement.select.SetOperationList.SetOperationType;
9394

9495
import java.util.logging.Level;
@@ -738,57 +739,31 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
738739
}
739740
}
740741

741-
/**
742-
* Skips "." name continuations starting at token position {@code i}; returns the
743-
* position of the first token after the qualified name.
744-
*/
745-
private int skipQualifiedObjectName(int i) {
746-
int guard = 0;
747-
while (guard++ < 64 && ".".equals(getToken(i).image)) {
748-
i += 2;
742+
private Token nextWithItemToken(Token current) {
743+
if (current.next == null) {
744+
current.next = token_source.getNextToken();
749745
}
750-
return i;
746+
return current.next;
751747
}
752748

753-
/**
754-
* Skips the balanced round-bracket group opening at token position {@code i};
755-
* returns the position after the matching closing bracket, or -1 when the
756-
* group is unbalanced or the scan would run past the token stream.
757-
*/
758-
private int skipBalancedBracketGroup(int i) {
749+
private Token skipBalancedBracketGroup(Token current) {
759750
int depth = 0;
760-
int guard = 0;
761-
while (guard++ < 4096) {
762-
int kind = getToken(i).kind;
763-
if (kind == 0) {
764-
return -1;
765-
}
766-
if (kind == OPENING_BRACKET) {
751+
while (!interrupted && current.kind != EOF) {
752+
if (current.kind == OPENING_BRACKET) {
767753
depth++;
768-
} else if (kind == CLOSING_BRACKET) {
769-
depth--;
770-
if (depth == 0) {
771-
return i + 1;
754+
} else if (current.kind == CLOSING_BRACKET) {
755+
if (--depth == 0) {
756+
return nextWithItemToken(current);
772757
}
773758
}
774-
i++;
759+
current = nextWithItemToken(current);
775760
}
776-
return -1;
761+
if (interrupted) {
762+
throw new CancellationException("Parsing interrupted");
763+
}
764+
return null;
777765
}
778766

779-
/**
780-
* True when the upcoming WITH item is ClickHouse's expression alias form
781-
* {@code WITH <expression> AS <identifier>} (including lambda aliases such as
782-
* {@code x -> x * 2} or {@code (x -> x * 2)}), and false for the classic CTE
783-
* shape {@code name [(cols)] AS [NOT] MATERIALIZED (statement)} or for the
784-
* other WithItem alternatives (FUNCTION, RECURSIVE).
785-
*
786-
* <p>Everything that cannot start a CTE name (literals, brackets, operators,
787-
* function calls) can only be an expression. A name-shaped head followed by
788-
* anything but an AS decision point also continues as an expression
789-
* ({@code total * 2}, {@code case when .. end}); the CTE shape is exactly
790-
* the name-shaped head with AS before a parenthesized statement.</p>
791-
*/
792767
private boolean isWithExpressionAliasAhead() {
793768
try {
794769
Token t1 = getToken(1);
@@ -798,27 +773,24 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
798773
if (!isIdentifierishNameToken(t1)) {
799774
return true;
800775
}
801-
int i = skipQualifiedObjectName(2);
802-
if (getToken(i).kind == OPENING_BRACKET) {
803-
// a column list (CTE) or call arguments (expression); the decision
804-
// point sits behind the balanced group either way
805-
i = skipBalancedBracketGroup(i);
806-
if (i < 0) {
776+
Token next = getToken(2);
777+
if (next.kind == OPENING_BRACKET) {
778+
next = skipBalancedBracketGroup(next);
779+
if (next == null) {
807780
return false;
808781
}
809782
}
810-
if (getToken(i).kind != K_AS) {
783+
if (next.kind != K_AS) {
811784
return true;
812785
}
813-
i++;
814-
if (getToken(i).kind == K_NOT) {
815-
i++;
786+
next = nextWithItemToken(next);
787+
if (next.kind == K_NOT) {
788+
next = nextWithItemToken(next);
816789
}
817-
if (getToken(i).kind == K_MATERIALIZED) {
818-
i++;
790+
if (next.kind == K_MATERIALIZED) {
791+
next = nextWithItemToken(next);
819792
}
820-
// the CTE shape requires a parenthesized statement here
821-
return getToken(i).kind != OPENING_BRACKET;
793+
return next.kind != OPENING_BRACKET;
822794
} catch (TokenMgrException e) {
823795
return false;
824796
}

‎src/test/java/net/sf/jsqlparser/statement/select/WithItemTest.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,117 @@
1010
package net.sf.jsqlparser.statement.select;
1111

1212
import static org.junit.jupiter.api.Assertions.assertEquals;
13+
import static org.junit.jupiter.api.Assertions.assertFalse;
14+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
1315
import static org.junit.jupiter.api.Assertions.assertNotNull;
16+
import static org.junit.jupiter.api.Assertions.assertNull;
17+
import static org.junit.jupiter.api.Assertions.assertThrows;
1418

19+
import java.util.concurrent.CancellationException;
20+
import java.util.concurrent.atomic.AtomicBoolean;
21+
import java.util.stream.Collectors;
22+
import java.util.stream.IntStream;
1523
import net.sf.jsqlparser.JSQLParserException;
24+
import net.sf.jsqlparser.expression.Function;
25+
import net.sf.jsqlparser.parser.CCJSqlParser;
26+
import net.sf.jsqlparser.parser.CCJSqlParserConstants;
27+
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
28+
import net.sf.jsqlparser.parser.ParseException;
29+
import net.sf.jsqlparser.parser.Token;
1630
import net.sf.jsqlparser.test.TestUtils;
1731
import org.junit.jupiter.api.Test;
32+
import org.junit.jupiter.api.Timeout;
1833
import org.junit.jupiter.params.ParameterizedTest;
34+
import org.junit.jupiter.params.provider.CsvSource;
1935
import org.junit.jupiter.params.provider.ValueSource;
36+
import org.mockito.Mockito;
2037

2138
class WithItemTest {
2239

40+
@ParameterizedTest
41+
@CsvSource({"2047, 1", "2047, -1", "2048, 1", "4096, 1"})
42+
void testLargeExpressionAlias(int arguments, String firstArgument) throws JSQLParserException {
43+
String expression = "coalesce(" + firstArgument + ", 1".repeat(arguments - 1) + ")";
44+
Select select = (Select) TestUtils.assertSqlCanBeParsedAndDeparsed(
45+
"WITH " + expression + " AS v, t AS (SELECT v) SELECT v FROM t", true);
46+
WithItem<?> item = select.getWithItemsList().get(0);
47+
Function function = assertInstanceOf(Function.class, item.getExpression());
48+
assertEquals("v", item.getAliasName());
49+
assertNull(item.getParenthesedStatement());
50+
assertEquals("coalesce", function.getName());
51+
assertEquals(arguments, function.getParameters().size());
52+
assertEquals(firstArgument, function.getParameters().get(0).toString());
53+
assertEquals("1", function.getParameters().get(arguments - 1).toString());
54+
assertNotNull(select.getWithItemsList().get(1).getParenthesedStatement());
55+
}
56+
57+
@ParameterizedTest
58+
@ValueSource(ints = {2047, 2048})
59+
void testLargeCteColumnList(int columns) throws JSQLParserException {
60+
String names = IntStream.range(0, columns).mapToObj(i -> "c" + i)
61+
.collect(Collectors.joining(", "));
62+
Select select = (Select) TestUtils.assertSqlCanBeParsedAndDeparsed(
63+
"WITH t(" + names + ") AS (SELECT 1" + ", 1".repeat(columns - 1)
64+
+ ") SELECT * FROM t",
65+
true);
66+
WithItem<?> item = select.getWithItemsList().get(0);
67+
assertEquals("t", item.getAliasName());
68+
assertNull(item.getExpression());
69+
assertNotNull(item.getParenthesedStatement());
70+
assertEquals(columns, item.getWithItemList().size());
71+
assertEquals("c" + (columns - 1), item.getWithItemList().get(columns - 1).toString());
72+
}
73+
74+
@Test
75+
void testLargeNestedExpressionAlias() throws JSQLParserException {
76+
String expression = "concat(concat('(', ')')" + ", 'x'".repeat(2047) + ")";
77+
Select select = (Select) TestUtils.assertSqlCanBeParsedAndDeparsed(
78+
"WITH " + expression + " AS v SELECT v", true);
79+
Function function = assertInstanceOf(Function.class,
80+
select.getWithItemsList().get(0).getExpression());
81+
assertEquals(2048, function.getParameters().size());
82+
assertEquals("concat('(', ')')", function.getParameters().get(0).toString());
83+
}
84+
85+
@ParameterizedTest
86+
@ValueSource(ints = {1, 64, 65})
87+
void testQualifiedExpressionAlias(int qualifiers) throws JSQLParserException {
88+
String name = "s.".repeat(qualifiers) + "coalesce";
89+
Select select = (Select) TestUtils.assertSqlCanBeParsedAndDeparsed(
90+
"WITH " + name + "(1, 2) AS v SELECT v", true);
91+
Function function = assertInstanceOf(Function.class,
92+
select.getWithItemsList().get(0).getExpression());
93+
assertEquals(name, function.getName());
94+
assertEquals(2, function.getParameters().size());
95+
}
96+
97+
@ParameterizedTest
98+
@ValueSource(ints = {2, 2048})
99+
@Timeout(5)
100+
void testUnclosedExpressionAlias(int arguments) {
101+
String sql = "WITH coalesce(1" + ", 1".repeat(arguments - 1);
102+
assertThrows(ParseException.class, () -> CCJSqlParserUtil.newParser(sql).Statement());
103+
}
104+
105+
@Test
106+
void testInterruptedWithItemLookaheadStopsReading() {
107+
CCJSqlParser parser = CCJSqlParserUtil.newParser(
108+
"WITH coalesce(1" + ", 1".repeat(512) + ") AS v SELECT v");
109+
AtomicBoolean reachedAlias = new AtomicBoolean();
110+
parser.token_source = Mockito.spy(parser.token_source);
111+
Mockito.doAnswer(invocation -> {
112+
Token token = (Token) invocation.callRealMethod();
113+
if (token.kind == CCJSqlParserConstants.S_LONG) {
114+
parser.interrupted = true;
115+
} else if (token.kind == CCJSqlParserConstants.K_AS) {
116+
reachedAlias.set(true);
117+
}
118+
return token;
119+
}).when(parser.token_source).getNextToken();
120+
assertThrows(CancellationException.class, parser::Statement);
121+
assertFalse(reachedAlias.get());
122+
}
123+
23124
@Test
24125
void testNotMaterializedIssue2251() throws JSQLParserException {
25126
String sqlStr = "WITH devices AS NOT MATERIALIZED (\n"

0 commit comments

Comments
 (0)