Skip to content

Commit 3361be9

Browse files
todd5167todd5167
committed
fix conflicts
2 parents 7ffe70b + 3b264e0 commit 3361be9

94 files changed

Lines changed: 6188 additions & 141 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,32 @@
88
> > * 扩展了输入和输出的性能指标到promethus
99
1010
## 新特性:
11-
1.kafka源表支持not null语法,支持字符串类型的时间转换。
12-
2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。
13-
3.异步维表支持非等值连接，比如：<>,<,>。
11+
* 1.kafka源表支持not null语法,支持字符串类型的时间转换。
12+
* 2.rdb维表与DB建立连接时，周期进行连接，防止连接断开。rdbsink写入时，对连接进行检查。
13+
* 3.异步维表支持非等值连接，比如：<>,<,>。
14+
* 4.增加kafka数组解析
15+
* 5.增加kafka1.0以上版本的支持
16+
* 6.增加postgresql、kudu、clickhouse维表、结果表的支持
17+
* 7.支持插件的依赖方式,参考pluginLoadMode参数
18+
* 8.支持cep处理
19+
* 9.支持udaf
20+
* 10.支持谓词下移
1421

1522
## BUG修复：
16-
1.修复不能解析sql中orderby,union语法。
17-
2.修复yarnPer模式提交失败的异常。
23+
* 1.修复不能解析sql中orderby,union语法。
24+
* 2.修复yarnPer模式提交失败的异常。
25+
* 3.一些bug的修复
1826

1927
# 已支持
20-
* 源表：kafka 0.9，1.x版本
21-
* 维表：mysql，SQlServer,oracle,hbase，mongo，redis,cassandra,serversocket
22-
* 结果表：mysql，SQlServer,oracle,hbase，elasticsearch5.x，mongo，redis,cassandra,console
28+
* 源表：kafka 0.9、0.10、0.11、1.x版本
29+
* 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, serversocket, kudu, postgresql, clickhouse
30+
* 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, console, kudu, postgresql, clickhouse
2331

2432
# 后续开发计划
25-
* 增加SQL支持CEP
2633
* 维表快照
27-
* sql优化（谓词下移等）
2834
* kafka avro格式
2935
* topN
3036

31-
3237
## 1 快速起步
3338
### 1.1 运行模式
3439

@@ -40,7 +45,7 @@
4045
### 1.2 执行环境
4146

4247
* Java: JDK8及以上
43-
* Flink集群: 1.4,1.5（单机模式不需要安装Flink集群）
48+
* Flink集群: 1.4,1.5,1.8（单机模式不需要安装Flink集群）
4449
* 操作系统：理论上不限
4550

4651
### 1.3 打包
@@ -150,6 +155,11 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack
150155
* 必选：否
151156
* 默认值：false
152157

158+
* **pluginLoadMode**
159+
* 描述：per_job 模式下的插件包加载方式。classpath:从每台机器加载插件包，shipfile:将需要插件从提交的节点上传到hdfs,不需要每台安装插件
160+
* 必选：否
161+
* 默认值：classpath
162+
153163
* **yarnSessionConf**
154164
* 描述：yarn session 模式下指定的运行的一些参数，[可参考](https://ci.apache.org/projects/flink/flink-docs-release-1.8/ops/cli.html),目前只支持指定yid
155165
* 必选：否
@@ -166,13 +176,19 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack
166176
* [mongo 结果表插件](docs/mongoSink.md)
167177
* [redis 结果表插件](docs/redisSink.md)
168178
* [cassandra 结果表插件](docs/cassandraSink.md)
179+
* [kudu 结果表插件](docs/kuduSink.md)
180+
* [postgresql 结果表插件](docs/postgresqlSink.md)
181+
* [clickhouse 结果表插件](docs/clickhouseSink.md)
169182

170183
### 2.3 维表插件
171184
* [hbase 维表插件](docs/hbaseSide.md)
172185
* [mysql 维表插件](docs/mysqlSide.md)
173186
* [mongo 维表插件](docs/mongoSide.md)
174187
* [redis 维表插件](docs/redisSide.md)
175188
* [cassandra 维表插件](docs/cassandraSide.md)
189+
* [kudu 维表插件](docs/kuduSide.md)
190+
* [postgresql 维表插件](docs/postgresqlSide.md)
191+
* [clickhouse 维表插件](docs/clickhouseSide.md)
176192

177193
## 3 性能指标(新增)
178194

@@ -203,7 +219,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack
203219

204220
```
205221
206-
CREATE (scala|table) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun
222+
CREATE (scala|table|aggregate) FUNCTION CHARACTER_LENGTH WITH com.dtstack.Kun;
207223
208224
209225
CREATE TABLE MyTable(

‎clickhouse/clickhouse-side/clickhouse-all-side/pom.xml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>sql.side.clickhouse</artifactId>
7+
<groupId>com.dtstack.flink</groupId>
8+
<version>1.0-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>sql.side.all.clickhouse</artifactId>
13+
<name>clickhouse-all-side</name>
14+
15+
<packaging>jar</packaging>
16+
17+
<properties>
18+
<sql.side.clickhouse.core.version>1.0-SNAPSHOT</sql.side.clickhouse.core.version>
19+
</properties>
20+
21+
<dependencies>
22+
<dependency>
23+
<groupId>com.dtstack.flink</groupId>
24+
<artifactId>sql.side.clickhouse.core</artifactId>
25+
<version>${sql.side.clickhouse.core.version}</version>
26+
</dependency>
27+
</dependencies>
28+
29+
<build>
30+
<plugins>
31+
<plugin>
32+
<groupId>org.apache.maven.plugins</groupId>
33+
<artifactId>maven-shade-plugin</artifactId>
34+
<version>1.4</version>
35+
<executions>
36+
<execution>
37+
<phase>package</phase>
38+
<goals>
39+
<goal>shade</goal>
40+
</goals>
41+
<configuration>
42+
<artifactSet>
43+
<excludes>
44+
45+
</excludes>
46+
</artifactSet>
47+
<filters>
48+
<filter>
49+
<artifact>*:*</artifact>
50+
<excludes>
51+
<exclude>META-INF/*.SF</exclude>
52+
<exclude>META-INF/*.DSA</exclude>
53+
<exclude>META-INF/*.RSA</exclude>
54+
</excludes>
55+
</filter>
56+
</filters>
57+
</configuration>
58+
</execution>
59+
</executions>
60+
</plugin>
61+
62+
<plugin>
63+
<artifactId>maven-antrun-plugin</artifactId>
64+
<version>1.2</version>
65+
<executions>
66+
<execution>
67+
<id>copy-resources</id>
68+
<!-- here the phase you need -->
69+
<phase>package</phase>
70+
<goals>
71+
<goal>run</goal>
72+
</goals>
73+
<configuration>
74+
<tasks>
75+
<copy todir="${basedir}/../../../plugins/clickhouseallside">
76+
<fileset dir="target/">
77+
<include name="${project.artifactId}-${project.version}.jar"/>
78+
</fileset>
79+
</copy>
80+
81+
<move file="${basedir}/../../../plugins/clickhouseallside/${project.artifactId}-${project.version}.jar"
82+
tofile="${basedir}/../../../plugins/clickhouseallside/${project.name}-${git.branch}.jar"/>
83+
</tasks>
84+
</configuration>
85+
</execution>
86+
</executions>
87+
</plugin>
88+
</plugins>
89+
</build>
90+
91+
92+
</project>

‎clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllReqRow.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.side.clickhouse;
20+
21+
import com.dtstack.flink.sql.side.FieldInfo;
22+
import com.dtstack.flink.sql.side.JoinInfo;
23+
import com.dtstack.flink.sql.side.SideTableInfo;
24+
import com.dtstack.flink.sql.side.rdb.all.RdbAllReqRow;
25+
import com.dtstack.flink.sql.util.DtStringUtil;
26+
import com.dtstack.flink.sql.util.JDBCUtils;
27+
import org.apache.flink.api.java.typeutils.RowTypeInfo;
28+
import org.apache.flink.shaded.guava18.com.google.common.collect.Maps;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
import java.sql.Connection;
33+
import java.sql.DriverManager;
34+
import java.util.List;
35+
import java.util.Map;
36+
37+
public class ClickhouseAllReqRow extends RdbAllReqRow {
38+
39+
private static final Logger LOG = LoggerFactory.getLogger(ClickhouseAllReqRow.class);
40+
41+
private static final String CLICKHOUSE_DRIVER = "ru.yandex.clickhouse.ClickHouseDriver";
42+
43+
public ClickhouseAllReqRow(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
44+
super(new ClickhouseAllSideInfo(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo));
45+
}
46+
47+
@Override
48+
public Connection getConn(String dbURL, String userName, String passWord) {
49+
try {
50+
Connection connection ;
51+
JDBCUtils.forName(CLICKHOUSE_DRIVER, getClass().getClassLoader());
52+
// ClickHouseProperties contains all properties
53+
if (userName == null) {
54+
connection = DriverManager.getConnection(dbURL);
55+
} else {
56+
connection = DriverManager.getConnection(dbURL, userName, passWord);
57+
}
58+
return connection;
59+
} catch (Exception e) {
60+
LOG.error("", e);
61+
throw new RuntimeException("", e);
62+
}
63+
}
64+
65+
}

‎clickhouse/clickhouse-side/clickhouse-all-side/src/main/java/com/dtstack/flink/sql/side/clickhouse/ClickhouseAllSideInfo.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.side.clickhouse;
20+
21+
import com.dtstack.flink.sql.side.FieldInfo;
22+
import com.dtstack.flink.sql.side.JoinInfo;
23+
import com.dtstack.flink.sql.side.SideTableInfo;
24+
import com.dtstack.flink.sql.side.rdb.all.RdbAllSideInfo;
25+
import org.apache.flink.api.java.typeutils.RowTypeInfo;
26+
27+
import java.util.List;
28+
29+
30+
public class ClickhouseAllSideInfo extends RdbAllSideInfo {
31+
public ClickhouseAllSideInfo(RowTypeInfo rowTypeInfo, JoinInfo joinInfo, List<FieldInfo> outFieldInfoList, SideTableInfo sideTableInfo) {
32+
super(rowTypeInfo, joinInfo, outFieldInfoList, sideTableInfo);
33+
}
34+
}

‎clickhouse/clickhouse-side/clickhouse-async-side/pom.xml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>sql.side.clickhouse</artifactId>
7+
<groupId>com.dtstack.flink</groupId>
8+
<version>1.0-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>sql.side.async.clickhouse</artifactId>
13+
<name>clickhouse-async-side</name>
14+
15+
<packaging>jar</packaging>
16+
17+
<properties>
18+
<sql.side.clickhouse.core.version>1.0-SNAPSHOT</sql.side.clickhouse.core.version>
19+
</properties>
20+
21+
<dependencies>
22+
<dependency>
23+
<groupId>com.dtstack.flink</groupId>
24+
<artifactId>sql.side.clickhouse.core</artifactId>
25+
<version>${sql.side.clickhouse.core.version}</version>
26+
</dependency>
27+
</dependencies>
28+
29+
<build>
30+
<plugins>
31+
<plugin>
32+
<groupId>org.apache.maven.plugins</groupId>
33+
<artifactId>maven-shade-plugin</artifactId>
34+
<version>1.4</version>
35+
<executions>
36+
<execution>
37+
<phase>package</phase>
38+
<goals>
39+
<goal>shade</goal>
40+
</goals>
41+
<configuration>
42+
<artifactSet>
43+
<excludes>
44+
45+
</excludes>
46+
</artifactSet>
47+
<filters>
48+
<filter>
49+
<artifact>*:*</artifact>
50+
<excludes>
51+
<exclude>META-INF/*.SF</exclude>
52+
<exclude>META-INF/*.DSA</exclude>
53+
<exclude>META-INF/*.RSA</exclude>
54+
</excludes>
55+
</filter>
56+
</filters>
57+
</configuration>
58+
</execution>
59+
</executions>
60+
</plugin>
61+
62+
<plugin>
63+
<artifactId>maven-antrun-plugin</artifactId>
64+
<version>1.2</version>
65+
<executions>
66+
<execution>
67+
<id>copy-resources</id>
68+
<!-- here the phase you need -->
69+
<phase>package</phase>
70+
<goals>
71+
<goal>run</goal>
72+
</goals>
73+
<configuration>
74+
<tasks>
75+
<copy todir="${basedir}/../../../plugins/clickhouseasyncside">
76+
<fileset dir="target/">
77+
<include name="${project.artifactId}-${project.version}.jar"/>
78+
</fileset>
79+
</copy>
80+
81+
<move file="${basedir}/../../../plugins/clickhouseasyncside/${project.artifactId}-${project.version}.jar"
82+
tofile="${basedir}/../../../plugins/clickhouseasyncside/${project.name}-${git.branch}.jar"/>
83+
</tasks>
84+
</configuration>
85+
</execution>
86+
</executions>
87+
</plugin>
88+
</plugins>
89+
</build>
90+
91+
</project>

0 commit comments

Comments
 (0)