Skip to content

Commit 3e7012d

Browse files
a49aa49a
committed
remove add file statment
1 parent 67c2378 commit 3e7012d

3 files changed

Lines changed: 111 additions & 1 deletion

File tree

‎core/src/main/java/com/dtstack/flink/sql/parser/SqlParser.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,11 @@
2828
import com.google.common.collect.Lists;
2929
import com.google.common.base.Strings;
3030

31+
import java.util.ArrayList;
3132
import java.util.List;
3233
import java.util.Set;
34+
import java.util.regex.Matcher;
35+
import java.util.regex.Pattern;
3336

3437
/**
3538
* Reason:
@@ -51,6 +54,8 @@ public static void setLocalSqlPluginRoot(String localSqlPluginRoot){
5154
LOCAL_SQL_PLUGIN_ROOT = localSqlPluginRoot;
5255
}
5356

57+
private static final Pattern ADD_FIlE_PATTERN = Pattern.compile("(?i).*add\\s+file\\s+.+");
58+
5459
/**
5560
* flink support sql syntax
5661
* CREATE TABLE sls_stream() with ();
@@ -70,6 +75,7 @@ public static SqlTree parseSql(String sql, String pluginLoadMode) throws Excepti
7075
.replace("\t", " ").trim();
7176

7277
List<String> sqlArr = DtStringUtil.splitIgnoreQuota(sql, SQL_DELIMITER);
78+
sqlArr = removeAddFileStmt(sqlArr);
7379
SqlTree sqlTree = new SqlTree();
7480
AbstractTableInfoParser tableInfoParser = new AbstractTableInfoParser();
7581
for(String childSql : sqlArr){
@@ -150,4 +156,18 @@ public static SqlTree parseSql(String sql, String pluginLoadMode) throws Excepti
150156

151157
return sqlTree;
152158
}
159+
160+
/**
161+
* remove add file with statment etc. add file /etc/krb5.conf;
162+
*/
163+
private static List<String> removeAddFileStmt(List<String> stmts) {
164+
List<String> cleanedStmts = new ArrayList<>();
165+
for (String stmt : stmts) {
166+
Matcher matcher = ADD_FIlE_PATTERN.matcher(stmt);
167+
if(!matcher.matches()) {
168+
cleanedStmts.add(stmt);
169+
}
170+
}
171+
return cleanedStmts;
172+
}
153173
}

‎core/src/test/java/com/dtstack/flink/sql/parser/SqlParserTest.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.parser;
20+
21+
import org.junit.Assert;
22+
import org.junit.Test;
23+
import org.powermock.reflect.Whitebox;
24+
25+
import java.util.ArrayList;
26+
import java.util.List;
27+
28+
public class SqlParserTest {
29+
30+
@Test
31+
public void testRemoveAddFileStmt() throws Exception {
32+
List<String> rawStmts = new ArrayList<>();
33+
String sql1 = " add file asdasdasd ";
34+
String sql2 = " aDd fIle With asdasdasd ";
35+
String sql3 = " INSERT INTO dwd_foo SELECT id, name FROM ods_foo";
36+
String sql4 = " ADD FILE asb ";
37+
rawStmts.add(sql1);
38+
rawStmts.add(sql2);
39+
rawStmts.add(sql3);
40+
rawStmts.add(sql4);
41+
42+
List<String> stmts = Whitebox.invokeMethod(SqlParser.class, "removeAddFileStmt", rawStmts);
43+
Assert.assertEquals(stmts.get(0), sql3);
44+
}
45+
46+
}

‎pom.xml‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,57 @@
4040

4141
</modules>
4242

43-
4443
<properties>
4544
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
4645
<flink.version>1.10.0</flink.version>
4746
<hadoop.version>2.7.3</hadoop.version>
47+
<junit.version>4.12</junit.version>
48+
<mockito.version>2.21.0</mockito.version>
49+
<powermock.version>2.0.4</powermock.version>
50+
<jacoco.version>0.7.8</jacoco.version>
4851
</properties>
4952

53+
<dependencies>
54+
<!-- test dependencies -->
55+
<dependency>
56+
<groupId>junit</groupId>
57+
<artifactId>junit</artifactId>
58+
<version>${junit.version}</version>
59+
<scope>test</scope>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.mockito</groupId>
63+
<artifactId>mockito-core</artifactId>
64+
<version>${mockito.version}</version>
65+
<scope>test</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.powermock</groupId>
69+
<artifactId>powermock-module-junit4</artifactId>
70+
<version>${powermock.version}</version>
71+
<scope>test</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.powermock</groupId>
75+
<artifactId>powermock-api-mockito2</artifactId>
76+
<version>${powermock.version}</version>
77+
<scope>test</scope>
78+
<exclusions>
79+
<exclusion>
80+
<groupId>org.mockito</groupId>
81+
<artifactId>mockito-core</artifactId>
82+
</exclusion>
83+
</exclusions>
84+
</dependency>
85+
<dependency>
86+
<groupId>org.jacoco</groupId>
87+
<artifactId>org.jacoco.agent</artifactId>
88+
<classifier>runtime</classifier>
89+
<scope>test</scope>
90+
<version>${jacoco.version}</version>
91+
</dependency>
92+
</dependencies>
93+
5094
<build>
5195
<plugins>
5296
<plugin>

0 commit comments

Comments
 (0)