Merge branch 'feat_1.8_autoJsonArray_mergedTest' into '1.8_test_3.10.x'

Feat 1.8 auto json array merged test

See merge request dt-insight-engine/flinkStreamSQL!76

Author: zoudaokoulife

.gitignore

@@ -8,6 +8,7 @@ target/
 *.eclipse.*
 *.iml
 plugins/
+sqlplugins/
 lib/
 .vertx/
 .DS_Store

core/pom.xml

@@ -122,6 +122,7 @@
             <artifactId>junit</artifactId>
             <version>4.12</version>
         </dependency>
+
     </dependencies>
 
     <build>

core/src/main/java/com/dtstack/flink/sql/format/dtnest/DtNestRowDeserializationSchema.java

@@ -24,6 +24,7 @@
 import org.apache.flink.api.common.serialization.AbstractDeserializationSchema;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
@@ -34,7 +35,7 @@
 import org.apache.flink.types.Row;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.lang.reflect.Array;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
@@ -77,30 +78,7 @@ public Row deserialize(byte[] message) throws IOException {
         String decoderStr = new String(message, charsetName);
         JsonNode root = objectMapper.readTree(decoderStr);
         this.parseTree(root, null);
-        Row row = new Row(fieldNames.length);
-
-        try {
-            for (int i = 0; i < fieldNames.length; i++) {
-                JsonNode node = getIgnoreCase(fieldNames[i]);
-                AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i);
-
-                if (node == null) {
-                    if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) {
-                        throw new IllegalStateException("Failed to find field with name '"
-                                + fieldNames[i] + "'.");
-                    } else {
-                        row.setField(i, null);
-                    }
-                } else {
-                    // Read the value as specified type
-                    Object value = convert(node, fieldTypes[i]);
-                    row.setField(i, value);
-                }
-            }
-            return row;
-        } finally {
-            nodeAndJsonNodeMapping.clear();
-        }
+        return convertTopRow();
     }
 
     private void parseTree(JsonNode jsonNode, String prefix) {
@@ -174,6 +152,10 @@ private Object convert(JsonNode node, TypeInformation<?> info) {
         } else if (info.getTypeClass().equals(Types.SQL_TIMESTAMP.getTypeClass())) {
             // local zone
             return Timestamp.valueOf(node.asText());
+        }  else if (info instanceof RowTypeInfo) {
+            return convertRow(node, (RowTypeInfo) info);
+        } else if (info instanceof ObjectArrayTypeInfo) {
+            return convertObjectArray(node, ((ObjectArrayTypeInfo) info).getComponentInfo());
         } else {
             // for types that were specified without JSON schema
             // e.g. POJOs
@@ -185,5 +167,55 @@ private Object convert(JsonNode node, TypeInformation<?> info) {
         }
     }
 
+    private Row convertTopRow() {
+        Row row = new Row(fieldNames.length);
+        try {
+            for (int i = 0; i < fieldNames.length; i++) {
+                JsonNode node = getIgnoreCase(fieldNames[i]);
+                AbstractTableInfo.FieldExtraInfo fieldExtraInfo = fieldExtraInfos.get(i);
 
+                if (node == null) {
+                    if (fieldExtraInfo != null && fieldExtraInfo.getNotNull()) {
+                        throw new IllegalStateException("Failed to find field with name '"
+                            + fieldNames[i] + "'.");
+                    } else {
+                        row.setField(i, null);
+                    }
+                } else {
+                    // Read the value as specified type
+                    Object value = convert(node, fieldTypes[i]);
+                    row.setField(i, value);
+                }
+            }
+            return row;
+        } finally {
+            nodeAndJsonNodeMapping.clear();
+        }
+    }
+
+    private Row convertRow(JsonNode node, RowTypeInfo info) {
+        final String[] names = info.getFieldNames();
+        final TypeInformation<?>[] types = info.getFieldTypes();
+
+        final Row row = new Row(names.length);
+        for (int i = 0; i < names.length; i++) {
+            final String name = names[i];
+            final JsonNode subNode = node.get(name);
+            if (subNode == null) {
+                row.setField(i, null);
+            } else {
+                row.setField(i, convert(subNode, types[i]));
+            }
+        }
+
+        return row;
+    }
+
+    private Object convertObjectArray(JsonNode node, TypeInformation<?> elementType) {
+        final Object[] array = (Object[]) Array.newInstance(elementType.getTypeClass(), node.size());
+        for (int i = 0; i < node.size(); i++) {
+            array[i] = convert(node.get(i), elementType);
+        }
+        return array;
+    }
 }

core/src/main/java/com/dtstack/flink/sql/parser/CreateTmpTableParser.java

@@ -22,11 +22,7 @@
 
 import com.dtstack.flink.sql.util.DtStringUtil;
 import org.apache.calcite.config.Lex;
-import org.apache.calcite.sql.SqlBasicCall;
-import org.apache.calcite.sql.SqlJoin;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.*;
 import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.sql.parser.SqlParser;
 import com.google.common.collect.Lists;
@@ -164,6 +160,10 @@ private static void parseNode(SqlNode sqlNode, CreateTmpTableParser.SqlParserRes
                     parseNode(unionRight, sqlParseResult);
                 }
                 break;
+            case MATCH_RECOGNIZE:
+                SqlMatchRecognize node = (SqlMatchRecognize) sqlNode;
+                sqlParseResult.addSourceTable(node.getTableRef().toString());
+                break;
             default:
                 //do nothing
                 break;

core/src/main/java/com/dtstack/flink/sql/table/AbstractTableParser.java

@@ -16,8 +16,6 @@
  * limitations under the License.
  */
 
-
-
 package com.dtstack.flink.sql.table;
 
 import com.dtstack.flink.sql.util.ClassUtil;
@@ -46,7 +44,7 @@ public abstract class AbstractTableParser {
     private static final String CHAR_TYPE_NO_LENGTH = "CHAR";
 
     private static Pattern primaryKeyPattern = Pattern.compile("(?i)PRIMARY\\s+KEY\\s*\\((.*)\\)");
-    private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(\\w+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$");
+    private static Pattern nestJsonFieldKeyPattern = Pattern.compile("(?i)((@*\\S+\\.)*\\S+)\\s+(.+)\\s+AS\\s+(\\w+)(\\s+NOT\\s+NULL)?$");
     private static Pattern physicalFieldFunPattern = Pattern.compile("\\w+\\((\\w+)\\)$");
     private static Pattern charTypePattern = Pattern.compile("(?i)CHAR\\((\\d*)\\)$");
 
@@ -84,32 +82,32 @@ public boolean dealKeyPattern(String fieldRow, AbstractTableInfo tableInfo){
         return false;
     }
 
-    public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){
+    public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo) {
 
         List<String> fieldRows = DtStringUtil.splitIgnoreQuota(fieldsInfo, ',');
-        for(String fieldRow : fieldRows){
+
+        for (String fieldRow : fieldRows) {
             fieldRow = fieldRow.trim();
 
-            if(StringUtils.isBlank(fieldRow)){
+            if (StringUtils.isBlank(fieldRow)) {
                 throw new RuntimeException(String.format("table [%s],exists field empty.", tableInfo.getName()));
             }
 
-            String[] filedInfoArr = fieldRow.split("\\s+");
-            if(filedInfoArr.length < 2 ){
-                throw new RuntimeException(String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow));
-            }
+            String[] fieldInfoArr = fieldRow.split("\\s+");
+
+            String errorMsg = String.format("table [%s] field [%s] format error.", tableInfo.getName(), fieldRow);
+            Preconditions.checkState(fieldInfoArr.length >= 2, errorMsg);
 
             boolean isMatcherKey = dealKeyPattern(fieldRow, tableInfo);
-            if(isMatcherKey){
+            if (isMatcherKey) {
                 continue;
             }
 
             //Compatible situation may arise in space in the fieldName
-            String[] filedNameArr = new String[filedInfoArr.length - 1];
-            System.arraycopy(filedInfoArr, 0, filedNameArr, 0, filedInfoArr.length - 1);
+            String[] filedNameArr = new String[fieldInfoArr.length - 1];
+            System.arraycopy(fieldInfoArr, 0, filedNameArr, 0, fieldInfoArr.length - 1);
             String fieldName = String.join(" ", filedNameArr);
-            String fieldType = filedInfoArr[filedInfoArr.length - 1 ].trim();
-
+            String fieldType = fieldInfoArr[fieldInfoArr.length - 1 ].trim();
 
             Class fieldClass = null;
             AbstractTableInfo.FieldExtraInfo fieldExtraInfo = null;
@@ -123,7 +121,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){
                 fieldClass = dbTypeConvertToJavaType(fieldType);
             }
 
-            tableInfo.addPhysicalMappings(filedInfoArr[0],filedInfoArr[0]);
+            tableInfo.addPhysicalMappings(fieldInfoArr[0], fieldInfoArr[0]);
             tableInfo.addField(fieldName);
             tableInfo.addFieldClass(fieldClass);
             tableInfo.addFieldType(fieldType);
@@ -133,7 +131,7 @@ public void parseFieldsInfo(String fieldsInfo, AbstractTableInfo tableInfo){
         tableInfo.finish();
     }
 
-    public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo){
+    public void dealPrimaryKey(Matcher matcher, AbstractTableInfo tableInfo) {
         String primaryFields = matcher.group(1).trim();
         String[] splitArry = primaryFields.split(",");
         List<String> primaryKes = Lists.newArrayList(splitArry);
@@ -172,4 +170,5 @@ protected void addParserHandler(String parserName, Pattern pattern, ITableFieldD
         patternMap.put(parserName, pattern);
         handlerMap.put(parserName, handler);
     }
+
 }

core/src/main/java/com/dtstack/flink/sql/util/ClassUtil.java

@@ -20,6 +20,7 @@
 
 package com.dtstack.flink.sql.util;
 
+import java.lang.reflect.Array;
 import java.math.BigDecimal;
 import java.sql.Date;
 import java.sql.Time;
@@ -34,7 +35,14 @@
 public class ClassUtil {
 
     public static Class<?> stringConvertClass(String str) {
-        switch (str.toLowerCase()) {
+
+        // 这部分主要是告诉Class转TypeInfomation的方法，字段是Array类型
+        String lowerStr = str.toLowerCase();
+        if (lowerStr.startsWith("array")) {
+            return Array.newInstance(Integer.class, 0).getClass();
+        }
+
+        switch (lowerStr) {
             case "boolean":
             case "bit":
                 return Boolean.class;
@@ -61,6 +69,7 @@ public static Class<?> stringConvertClass(String str) {
             case "varchar":
             case "char":
             case "text":
+            case "string":
                 return String.class;
 
             case "real":