[fix-35378] optimize sonar scan result, update gitlab ci script

a49a · a49a · commit 88d1230cebec · 2021-02-23T17:49:21.000+08:00
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,10 +1,22 @@
-build:
+stages:
+  - validate
+  - test
+
+test-job:
   stage: test
   script:
     - mvn clean org.jacoco:jacoco-maven-plugin:0.7.8:prepare-agent package -Dmaven.test.failure.ignore=true -q
+  only:
+    - v1.10.0_dev
+  tags:
+    - dt-insight-engine
+
+validate-job:
+  stage: validate
+  script:
     - mvn sonar:sonar -Dsonar.projectKey="dt-insight-engine/flinkStreamSQL"  -Dsonar.login=11974c5e9a29625efa09fdc3c3fdc031efb1aab1 -Dsonar.host.url=http://172.16.100.198:9000 -Dsonar.jdbc.url=jdbc:postgresql://172.16.100.198:5432/sonar -Dsonar.java.binaries=target/sonar
     - sh ci/sonar_notify.sh
   only:
-    - 1.10_release_4.0.x
+    - v1.10.0_dev
   tags:
-    - dt-insight-engine
+    - dt-insight-engine
diff --git a/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java b/core/src/main/java/com/dtstack/flink/sql/side/JoinNodeDealer.java
@@ -300,6 +300,7 @@ private JoinInfo dealNestJoin(SqlJoin joinNode,
 
         SqlNode newLeftNode = joinNode.getLeft();
 
+        // 双流JOIN再维表JOIN情况
         if(newLeftNode.getKind() != AS && parentRightIsSide){
 
             String leftTbAlias = buildAs.getOperands()[1].toString();
@@ -975,12 +976,10 @@ public Map<String, String> buildTmpTableFieldRefOriField(Set<String> fieldSet, S
             } else {
                 fields = StringUtils.splitByWholeSeparator(field, " as ");
             }
-            if (fields != null) {
-                fields = Stream
-                        .of(fields)
-                        .map(StringUtils::trimToNull)
-                        .toArray(String[]::new);
-            }
+            fields = Stream
+                    .of(fields)
+                    .map(StringUtils::trimToNull)
+                    .toArray(String[]::new);
             String oldKey = field;
             String[] oldFieldInfo = StringUtils.splitByWholeSeparator(fields[0], ".");
             String oldFieldName = oldFieldInfo.length == 2 ? oldFieldInfo[1] : oldFieldInfo[0];
diff --git a/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java b/impala/impala-sink/src/main/java/com/dtstack/flink/sql/sink/impala/ImpalaOutputFormat.java
@@ -178,12 +178,11 @@ private void init() throws SQLException {
             if (!storeType.equalsIgnoreCase(KUDU_TYPE)) {
                 throw new IllegalArgumentException("update mode not support for non-kudu table!");
             }
-
             updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys));
-            return;
+        } else {
+            valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields);
         }
 
-        valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields);
     }
 
     private void initScheduledTask(Long batchWaitInterval) {
@@ -331,7 +330,7 @@ private void putRowIntoMap(Map<String, ArrayList<String>> rowDataMap, Tuple2<Str
     }
 
     private List<String> rebuildFieldNameListAndTypeList(List<String> fieldNames, List<String> staticPartitionFields, List<String> fieldTypes, String partitionFields) {
-        if (partitionFields.isEmpty()) {
+        if (partitionFields == null || partitionFields.isEmpty()) {
             return fieldNames;
         }
 
diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/LauncherMain.java
@@ -31,6 +31,8 @@
 import com.dtstack.flink.sql.option.Options;
 import com.dtstack.flink.sql.util.PluginUtil;
 import org.apache.commons.io.Charsets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.FileInputStream;
@@ -51,7 +53,7 @@
 
 public class LauncherMain {
 
-
+    private static final Logger LOG = LoggerFactory.getLogger(LauncherMain.class);
 
     public static JobParamsInfo parseArgs(String[] args) throws Exception {
         if (args.length == 1 && args[0].endsWith(".json")) {
@@ -97,30 +99,24 @@ public static JobParamsInfo parseArgs(String[] args) throws Exception {
     }
 
     private static String[] parseJson(String[] args) {
-        BufferedReader reader = null;
         StringBuilder lastStr = new StringBuilder();
         try {
-            FileInputStream fileInputStream = new FileInputStream(args[0]);
-            InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, StandardCharsets.UTF_8);
-            reader = new BufferedReader(inputStreamReader);
-            String tempString;
-            while ((tempString = reader.readLine()) != null) {
-                lastStr.append(tempString);
-            }
-            reader.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        } finally {
-            if (reader != null) {
-                try {
-                    reader.close();
-                } catch (IOException e) {
-                    e.printStackTrace();
+            try (FileInputStream fileInputStream = new FileInputStream(args[0])) {
+                try (InputStreamReader inputStreamReader =
+                             new InputStreamReader(fileInputStream, StandardCharsets.UTF_8)) {
+                    try (BufferedReader reader = new BufferedReader(inputStreamReader)) {
+                        String tempString;
+                        while ((tempString = reader.readLine()) != null) {
+                            lastStr.append(tempString);
+                        }
+                    }
                 }
             }
+        } catch (IOException e) {
+            LOG.error("", e);
         }
-        Map<String, Object> map = JSON.parseObject(lastStr.toString(), new TypeReference<Map<String, Object>>() {
-        });
+
+        Map<String, Object> map = JSON.parseObject(lastStr.toString(), new TypeReference<Map<String, Object>>() {});
         List<String> list = new LinkedList<>();
 
         for (Map.Entry<String, Object> entry : map.entrySet()) {
@@ -130,7 +126,6 @@ private static String[] parseJson(String[] args) {
         return list.toArray(new String[0]);
     }
 
-
     public static void main(String[] args) throws Exception {
         JobParamsInfo jobParamsInfo = parseArgs(args);
         ClusterMode execMode = ClusterMode.valueOf(jobParamsInfo.getMode());
diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/StandaloneExecutor.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/StandaloneExecutor.java
@@ -19,6 +19,7 @@
 package com.dtstack.flink.sql.launcher.executor;
 
 import com.dtstack.flink.sql.enums.EPluginLoadMode;
+import com.dtstack.flink.sql.launcher.LauncherMain;
 import com.dtstack.flink.sql.launcher.entity.JobParamsInfo;
 import com.dtstack.flink.sql.launcher.factory.StandaloneClientFactory;
 import com.dtstack.flink.sql.launcher.utils.JobGraphBuildUtil;
@@ -32,13 +33,18 @@
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Date: 2020/3/6
  * Company: www.dtstack.com
  * @author maqi
  */
 public class StandaloneExecutor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(StandaloneExecutor.class);
+
     JobParamsInfo jobParamsInfo;
 
     public StandaloneExecutor(JobParamsInfo jobParamsInfo) {
@@ -60,15 +66,21 @@ public void exec() throws Exception {
         JobGraphBuildUtil.fillJobGraphClassPath(jobGraph);
 
         ClusterDescriptor clusterDescriptor = StandaloneClientFactory.INSTANCE.createClusterDescriptor("", flinkConfiguration);
-        ClusterClientProvider clusterClientProvider = clusterDescriptor.retrieve(StandaloneClusterId.getInstance());
-        ClusterClient clusterClient = clusterClientProvider.getClusterClient();
 
+        try {
+            ClusterClientProvider clusterClientProvider = clusterDescriptor.retrieve(StandaloneClusterId.getInstance());
+            ClusterClient clusterClient = clusterClientProvider.getClusterClient();
 
-        JobExecutionResult jobExecutionResult = ClientUtils.submitJob(clusterClient, jobGraph);
-        String jobId = jobExecutionResult.getJobID().toString();
-        System.out.println("jobID:" + jobId);
+            JobExecutionResult jobExecutionResult = ClientUtils.submitJob(clusterClient, jobGraph);
+            String jobId = jobExecutionResult.getJobID().toString();
+            LOG.info("jobID:" + jobId);
+        } finally {
+            try {
+                clusterDescriptor.close();
+            } catch (Exception e) {
+                LOG.info("Could not properly close the yarn cluster descriptor.", e);
+            }
+        }
 
     }
-
-
 }
diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/YarnJobClusterExecutor.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/YarnJobClusterExecutor.java
@@ -76,26 +76,33 @@ public void exec() throws Exception {
         YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) YarnClusterClientFactory.INSTANCE
                 .createClusterDescriptor(jobParamsInfo.getYarnConfDir(), flinkConfiguration);
 
-        List<File> shipFiles = getShipFiles(jobParamsInfo.getFlinkJarPath(), jobParamsInfo.getPluginLoadMode(), jobGraph, clusterDescriptor);
-
-        if (jobParamsInfo.getAddShipFile() != null) {
-            List<String> addShipFilesPath = parsePathFromStr(jobParamsInfo.getAddShipFile());
-            for (String path : addShipFilesPath) {
-                shipFiles.add(getAddShipFile(path));
+        try {
+            List<File> shipFiles = getShipFiles(jobParamsInfo.getFlinkJarPath(), jobParamsInfo.getPluginLoadMode(), jobGraph, clusterDescriptor);
+
+            if (jobParamsInfo.getAddShipFile() != null) {
+                List<String> addShipFilesPath = parsePathFromStr(jobParamsInfo.getAddShipFile());
+                for (String path : addShipFilesPath) {
+                    shipFiles.add(getAddShipFile(path));
+                }
             }
-        }
-
-        dumpSameKeytab(flinkConfiguration, shipFiles);
 
-        clusterDescriptor.addShipFiles(shipFiles);
+            dumpSameKeytab(flinkConfiguration, shipFiles);
+            clusterDescriptor.addShipFiles(shipFiles);
 
-        ClusterSpecification clusterSpecification = YarnClusterClientFactory.INSTANCE.getClusterSpecification(flinkConfiguration);
-        ClusterClientProvider<ApplicationId> applicationIdClusterClientProvider = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
+            ClusterSpecification clusterSpecification = YarnClusterClientFactory.INSTANCE.getClusterSpecification(flinkConfiguration);
+            ClusterClientProvider<ApplicationId> applicationIdClusterClientProvider = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
 
-        String applicationId = applicationIdClusterClientProvider.getClusterClient().getClusterId().toString();
-        String flinkJobId = jobGraph.getJobID().toString();
+            String applicationId = applicationIdClusterClientProvider.getClusterClient().getClusterId().toString();
+            String flinkJobId = jobGraph.getJobID().toString();
 
-        LOG.info(String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId));
+            LOG.info(String.format("deploy per_job with appId: %s, jobId: %s", applicationId, flinkJobId));
+        } finally {
+            try {
+                clusterDescriptor.close();
+            } catch (Exception e) {
+                LOG.info("Could not properly close the yarn cluster descriptor.", e);
+            }
+        }
     }
 
     private void dumpSameKeytab(Configuration flinkConfiguration, List<File> shipFiles) {
diff --git a/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/YarnSessionClusterExecutor.java b/launcher/src/main/java/com/dtstack/flink/sql/launcher/executor/YarnSessionClusterExecutor.java
@@ -33,6 +33,8 @@
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -41,6 +43,9 @@
  * @author maqi
  */
 public class YarnSessionClusterExecutor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(YarnJobClusterExecutor.class);
+
     JobParamsInfo jobParamsInfo;
 
     public YarnSessionClusterExecutor(JobParamsInfo jobParamsInfo) {
@@ -52,28 +57,37 @@ public void exec() throws Exception {
         Configuration flinkConfiguration = JobGraphBuildUtil.getFlinkConfiguration(jobParamsInfo.getFlinkConfDir(), jobParamsInfo.getConfProperties());
         ClusterDescriptor clusterDescriptor = YarnClusterClientFactory.INSTANCE.createClusterDescriptor(jobParamsInfo.getYarnConfDir(), flinkConfiguration);
 
-        Object yid = jobParamsInfo.getYarnSessionConfProperties().get("yid");
-        if (null == yid) {
-            throw new RuntimeException("yarnSessionMode yid is required");
-        }
+        try {
+            Object yid = jobParamsInfo.getYarnSessionConfProperties().get("yid");
+            if (null == yid) {
+                throw new RuntimeException("yarnSessionMode yid is required");
+            }
 
-        ApplicationId applicationId = ConverterUtils.toApplicationId(yid.toString());
-        ClusterClientProvider<ApplicationId> retrieve = clusterDescriptor.retrieve(applicationId);
-        ClusterClient<ApplicationId> clusterClient = retrieve.getClusterClient();
+            ApplicationId applicationId = ConverterUtils.toApplicationId(yid.toString());
+            ClusterClientProvider<ApplicationId> retrieve = clusterDescriptor.retrieve(applicationId);
+            ClusterClient<ApplicationId> clusterClient = retrieve.getClusterClient();
 
-        if (StringUtils.equalsIgnoreCase(jobParamsInfo.getPluginLoadMode(), EPluginLoadMode.SHIPFILE.name())) {
-            jobGraph.getUserArtifacts().clear();
-        } else {
-            JobGraphBuildUtil.fillJobGraphClassPath(jobGraph);
-        }
+            if (StringUtils.equalsIgnoreCase(jobParamsInfo.getPluginLoadMode(), EPluginLoadMode.SHIPFILE.name())) {
+                jobGraph.getUserArtifacts().clear();
+            } else {
+                JobGraphBuildUtil.fillJobGraphClassPath(jobGraph);
+            }
+
+            if (!StringUtils.isEmpty(jobParamsInfo.getUdfJar())) {
+                JobGraphBuildUtil.fillUserJarForJobGraph(jobParamsInfo.getUdfJar(), jobGraph);
+            }
 
-        if (!StringUtils.isEmpty(jobParamsInfo.getUdfJar())) {
-            JobGraphBuildUtil.fillUserJarForJobGraph(jobParamsInfo.getUdfJar(), jobGraph);
+            JobExecutionResult jobExecutionResult = ClientUtils.submitJob(clusterClient, jobGraph);
+            String jobId = jobExecutionResult.getJobID().toString();
+            LOG.info("jobID:" + jobId);
+        } finally {
+            try {
+                clusterDescriptor.close();
+            } catch (Exception e) {
+                LOG.info("Could not properly close the yarn cluster descriptor.", e);
+            }
         }
 
-        JobExecutionResult jobExecutionResult = ClientUtils.submitJob(clusterClient, jobGraph);
-        String jobId = jobExecutionResult.getJobID().toString();
-        System.out.println("jobID:" + jobId);
 
     }
 
diff --git a/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java b/rdb/rdb-side/src/main/java/com/dtstack/flink/sql/side/rdb/async/RdbAsyncReqRow.java
@@ -407,10 +407,8 @@ private void handleQuery(SQLConnection connection, Map<String, Object> inputPara
                 // and close the connection
                 connection.close(done -> {
                     if (done.failed()) {
-                        throw new SuppressRestartsException(
-                                new Throwable(
-                                        ExceptionTrace.traceOriginalCause(done.cause())
-                                )
+                        LOG.error("sql connection close failed! " +
+                                ExceptionTrace.traceOriginalCause(done.cause())
                         );
                     }
                 });

Original file line number	Diff line number	Diff line change
`@@ -178,12 +178,11 @@ private void init() throws SQLException {`
`178`	`178`	`if (!storeType.equalsIgnoreCase(KUDU_TYPE)) {`
`179`	`179`	`throw new IllegalArgumentException("update mode not support for non-kudu table!");`
`180`	`180`	`}`
`181`		`-`
`182`	`181`	`updateStatement = connection.prepareStatement(buildUpdateSql(schema, tableName, fieldNames, primaryKeys));`
`183`		`- return;`
	`182`	`+ } else {`
	`183`	`+ valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields);`
`184`	`184`	`}`
`185`	`185`
`186`		`- valueFieldNames = rebuildFieldNameListAndTypeList(fieldNames, staticPartitionFields, fieldTypes, partitionFields);`
`187`	`186`	`}`
`188`	`187`
`189`	`188`	`private void initScheduledTask(Long batchWaitInterval) {`
`@@ -331,7 +330,7 @@ private void putRowIntoMap(Map<String, ArrayList<String>> rowDataMap, Tuple2<Str`
`331`	`330`	`}`
`332`	`331`
`333`	`332`	`private List<String> rebuildFieldNameListAndTypeList(List<String> fieldNames, List<String> staticPartitionFields, List<String> fieldTypes, String partitionFields) {`
`334`		`- if (partitionFields.isEmpty()) {`
	`333`	`+ if (partitionFields == null \|\| partitionFields.isEmpty()) {`
`335`	`334`	`return fieldNames;`
`336`	`335`	`}`
`337`	`336`