Merge branch 'v1.8.0_dev_merge1.5.0' into 'v1.8.0_dev'

V1.8.0 dev merge1.5.0

1.8 合并1.5 ,
调整按名称插入时的api

See merge request !53

Author: zoudaokoulife

.gitignore

@@ -10,6 +10,7 @@ target/
 plugins/
 lib/
 .vertx/
+.DS_Store
 bin/nohup.out
-
+.DS_Store
 bin/sideSql.txt

cassandra/cassandra-side/cassandra-async-side/src/main/java/com/dtstack/flink/sql/side/cassandra/CassandraAsyncReqRow.java

@@ -256,7 +256,7 @@ public void onFailure(Throwable t) {
                         t.getMessage());
                 System.out.println("Failed to retrieve the data: " + t.getMessage());
                 cluster.closeAsync();
-                resultFuture.complete(null);
+                resultFuture.completeExceptionally(t);
             }
         });
     }

core/src/main/java/com/dtstack/flink/sql/Main.java

@@ -23,7 +23,12 @@
 import com.dtstack.flink.sql.classloader.DtClassLoader;
 import com.dtstack.flink.sql.enums.ECacheType;
 import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment;
-import com.dtstack.flink.sql.parser.*;
+import com.dtstack.flink.sql.exec.FlinkSQLExec;
+import com.dtstack.flink.sql.parser.CreateFuncParser;
+import com.dtstack.flink.sql.parser.CreateTmpTableParser;
+import com.dtstack.flink.sql.parser.InsertSqlParser;
+import com.dtstack.flink.sql.parser.SqlParser;
+import com.dtstack.flink.sql.parser.SqlTree;
 import com.dtstack.flink.sql.side.SideSqlExec;
 import com.dtstack.flink.sql.side.SideTableInfo;
 import com.dtstack.flink.sql.table.SourceTableInfo;
@@ -49,19 +54,21 @@
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
-import org.apache.flink.calcite.shaded.com.google.common.base.Preconditions;
-import org.apache.flink.calcite.shaded.com.google.common.base.Strings;
-import org.apache.flink.calcite.shaded.com.google.common.collect.Lists;
-import org.apache.flink.calcite.shaded.com.google.common.collect.Maps;
-import org.apache.flink.calcite.shaded.com.google.common.collect.Sets;
 import org.apache.flink.client.program.ContextEnvironment;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.shaded.guava18.com.google.common.base.Preconditions;
+import org.apache.flink.shaded.guava18.com.google.common.base.Strings;
+import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
+import org.apache.flink.shaded.guava18.com.google.common.collect.Maps;
+import org.apache.flink.shaded.guava18.com.google.common.collect.Sets;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamContextEnvironment;
+
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.java.StreamTableEnvironment;
+
 import org.apache.flink.table.sinks.TableSink;
 import org.apache.flink.types.Row;
 import org.slf4j.Logger;
@@ -205,7 +212,10 @@ public static void main(String[] args) throws Exception {
                         //sql-dimensional table contains the dimension table of execution
                         sideSqlExec.exec(result.getExecSql(), sideTableMap, tableEnv, registerTableCache);
                     }else{
-                        tableEnv.sqlUpdate(result.getExecSql());
+                        FlinkSQLExec.sqlUpdate(tableEnv, result.getExecSql());
+                        if(LOG.isInfoEnabled()){
+                            LOG.info("exec sql: " + result.getExecSql());
+                        }
                     }
                 }
             }
@@ -288,6 +298,9 @@ private static void registerTable(SqlTree sqlTree, StreamExecutionEnvironment en
 
                 Table regTable = tableEnv.fromDataStream(adaptStream, fields);
                 tableEnv.registerTable(tableInfo.getName(), regTable);
+                if(LOG.isInfoEnabled()){
+                    LOG.info("registe table {} success.", tableInfo.getName());
+                }
                 registerTableCache.put(tableInfo.getName(), regTable);
                 classPathSet.add(PluginUtil.getRemoteJarFilePath(tableInfo.getType(), SourceTableInfo.SOURCE_SUFFIX, remoteSqlPluginPath, localSqlPluginPath));
             } else if (tableInfo instanceof TargetTableInfo) {
@@ -320,7 +333,6 @@ private static StreamExecutionEnvironment getStreamExeEnv(Properties confPropert
         StreamExecutionEnvironment env = !ClusterMode.local.name().equals(deployMode) ?
                 StreamExecutionEnvironment.getExecutionEnvironment() :
                 new MyLocalStreamEnvironment();
-
         env.getConfig().disableClosureCleaner();
         env.setParallelism(FlinkUtil.getEnvParallelism(confProperties));
         Configuration globalJobParameters = new Configuration();

core/src/main/java/com/dtstack/flink/sql/exec/FlinkSQLExec.java

@@ -0,0 +1,66 @@
+package com.dtstack.flink.sql.exec;
+
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlInsert;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.api.java.StreamTableEnvironment;
+import org.apache.flink.table.calcite.FlinkPlannerImpl;
+import org.apache.flink.table.plan.logical.LogicalRelNode;
+import org.apache.flink.table.plan.schema.TableSinkTable;
+import org.apache.flink.table.plan.schema.TableSourceSinkTable;
+import scala.Option;
+
+import java.lang.reflect.Method;
+
+/**
+ * @description:  mapping by name when insert into sink table
+ * @author: maqi
+ * @create: 2019/08/15 11:09
+ */
+public class FlinkSQLExec {
+
+    public static void sqlUpdate(StreamTableEnvironment tableEnv, String stmt) throws Exception {
+
+        FlinkPlannerImpl planner = new FlinkPlannerImpl(tableEnv.getFrameworkConfig(), tableEnv.getPlanner(), tableEnv.getTypeFactory());
+        SqlNode insert = planner.parse(stmt);
+
+        if (!(insert instanceof SqlInsert)) {
+            throw new TableException(
+                    "Unsupported SQL query! sqlUpdate() only accepts SQL statements of type INSERT.");
+        }
+        SqlNode query = ((SqlInsert) insert).getSource();
+
+        SqlNode validatedQuery = planner.validate(query);
+
+        Table queryResult = new Table(tableEnv, new LogicalRelNode(planner.rel(validatedQuery).rel));
+        String targetTableName = ((SqlIdentifier) ((SqlInsert) insert).getTargetTable()).names.get(0);
+
+        Method method = TableEnvironment.class.getDeclaredMethod("getTable", String.class);
+        method.setAccessible(true);
+        Option sinkTab = (Option)method.invoke(tableEnv, targetTableName);
+
+        if (sinkTab.isEmpty()) {
+            throw  new ValidationException("Sink table " + targetTableName + "not found in flink");
+        }
+
+        TableSourceSinkTable targetTable = (TableSourceSinkTable) sinkTab.get();
+        TableSinkTable tableSinkTable = (TableSinkTable)targetTable.tableSinkTable().get();
+        String[] fieldNames = tableSinkTable.tableSink().getFieldNames();
+
+        Table newTable = null;
+        try {
+            newTable = queryResult.select(String.join(",", fieldNames));
+        } catch (Exception e) {
+            throw new ValidationException(
+                    "Field name of query result and registered TableSink "+targetTableName +" do not match.\n" +
+                    "Query result schema: " + String.join(",", queryResult.getSchema().getColumnNames()) + "\n" +
+                    "TableSink schema: " + String.join(",", fieldNames));
+        }
+
+        tableEnv.insertInto(newTable, targetTableName, tableEnv.queryConfig());
+    }
+}

core/src/main/java/com/dtstack/flink/sql/side/AsyncReqRow.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
- 
+
 
 package com.dtstack.flink.sql.side;
 
@@ -28,9 +28,12 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.async.ResultFuture;
 import org.apache.flink.streaming.api.functions.async.RichAsyncFunction;
+import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry;
 import org.apache.flink.types.Row;
 
+import java.util.Collection;
 import java.util.Collections;
+import java.util.concurrent.TimeoutException;
 
 /**
  * All interfaces inherit naming rules: type + "AsyncReqRow" such as == "MysqlAsyncReqRow
@@ -50,6 +53,18 @@ public AsyncReqRow(SideInfo sideInfo){
         this.sideInfo = sideInfo;
     }
 
+    @Override
+    public void timeout(Row input, ResultFuture<Row> resultFuture) throws Exception {
+        StreamRecordQueueEntry<Row> future = (StreamRecordQueueEntry<Row>)resultFuture;
+        try {
+            if (null == future.get()) {
+                new TimeoutException("Async function call has timed out.");
+            }
+        } catch (Exception e) {
+            throw new Exception(e);
+        }
+    }
+
     private void initCache(){
         SideTableInfo sideTableInfo = sideInfo.getSideTableInfo();
         if(sideTableInfo.getCacheType() == null || ECacheType.NONE.name().equalsIgnoreCase(sideTableInfo.getCacheType())){

core/src/main/java/com/dtstack/flink/sql/side/SideInfo.java

@@ -89,7 +89,7 @@ public void parseSelectFields(JoinInfo joinInfo){
         for( int i=0; i<outFieldInfoList.size(); i++){
             FieldInfo fieldInfo = outFieldInfoList.get(i);
             if(fieldInfo.getTable().equalsIgnoreCase(sideTableName)){
-                fields.add(fieldInfo.getFieldName());
+                fields.add(sideTableInfo.getPhysicalFields().getOrDefault(fieldInfo.getFieldName(), fieldInfo.getFieldName()));
                 sideFieldIndex.put(i, sideIndex);
                 sideFieldNameIndex.put(i, fieldInfo.getFieldName());
                 sideIndex++;