remote plugin path not required

Author: todd5167

core/src/main/java/com/dtstack/flink/sql/GetPlan.java

@@ -34,10 +34,12 @@ public class GetPlan {
 
     public static String getExecutionPlan(String[] args) {
         try {
+            long start = System.currentTimeMillis();
             ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args);
             StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo);
             String executionPlan = env.getExecutionPlan();
-            return ApiResult.createSuccessResultJsonStr(executionPlan);
+            long end = System.currentTimeMillis();
+            return ApiResult.createSuccessResultJsonStr(executionPlan, end - start);
         } catch (Exception e) {
             return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e));
         }

core/src/main/java/com/dtstack/flink/sql/exec/ApiResult.java

@@ -15,23 +15,25 @@
 public class ApiResult {
 
     private static final Logger LOG = LoggerFactory.getLogger(ApiResult.class);
-    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
     public static final Integer FAIL = 0;
     public static final Integer SUCCESS = 1;
 
     private int code;
+    private long space;
     private Object data;
     private String errorMsg;
 
     public ApiResult() {
     }
 
-    public static String createSuccessResultJsonStr(String message) {
+    public static String createSuccessResultJsonStr(String message,long space) {
         ApiResult apiResult = createSuccessResult(SUCCESS, message);
+        apiResult.setSpace(space);
         String result;
         try {
-            result = objectMapper.writeValueAsString(apiResult);
+            result = OBJECT_MAPPER.writeValueAsString(apiResult);
         } catch (Exception e) {
             LOG.error("", e);
             result = "code:" + SUCCESS + ",message:" + message;
@@ -50,7 +52,7 @@ public static String createErrorResultJsonStr(String message) {
         ApiResult apiResult = createErrorResult(message, FAIL);
         String result;
         try {
-            result = objectMapper.writeValueAsString(apiResult);
+            result = OBJECT_MAPPER.writeValueAsString(apiResult);
         } catch (Exception e) {
             LOG.error("", e);
             result = "code:" + FAIL + ",message:" + message;
@@ -88,4 +90,12 @@ public String getErrorMsg() {
     public void setErrorMsg(String errorMsg) {
         this.errorMsg = errorMsg;
     }
+
+    public long getSpace() {
+        return space;
+    }
+
+    public void setSpace(long space) {
+        this.space = space;
+    }
 }

core/src/main/java/com/dtstack/flink/sql/exec/ExecuteProcessHelper.java

@@ -22,6 +22,7 @@
 import com.dtstack.flink.sql.config.CalciteConfig;
 import com.dtstack.flink.sql.enums.ClusterMode;
 import com.dtstack.flink.sql.enums.ECacheType;
+import com.dtstack.flink.sql.enums.EPluginLoadMode;
 import com.dtstack.flink.sql.environment.MyLocalStreamEnvironment;
 import com.dtstack.flink.sql.environment.StreamEnvConfigManager;
 import com.dtstack.flink.sql.function.FunctionManager;
@@ -43,13 +44,15 @@
 import com.dtstack.flink.sql.util.PluginUtil;
 import com.dtstack.flink.sql.watermarker.WaterMarkerAssigner;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.calcite.sql.SqlInsert;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.commons.io.Charsets;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
@@ -106,6 +109,8 @@ public static ParamsInfo parseParams(String[] args) throws Exception {
         String pluginLoadMode = options.getPluginLoadMode();
         String deployMode = options.getMode();
 
+        Preconditions.checkArgument(checkRemoteSqlPluginPath(remoteSqlPluginPath, deployMode, pluginLoadMode),
+                "Non-local mode or shipfile deployment mode, remoteSqlPluginPath is required");
         String confProp = URLDecoder.decode(options.getConfProp(), Charsets.UTF_8.toString());
         Properties confProperties = PluginUtil.jsonStrToObject(confProp, Properties.class);
 
@@ -124,6 +129,22 @@ public static ParamsInfo parseParams(String[] args) throws Exception {
 
     }
 
+    /**
+     *   非local模式或者shipfile部署模式，remoteSqlPluginPath必填
+     * @param remoteSqlPluginPath
+     * @param deployMode
+     * @param pluginLoadMode
+     * @return
+     */
+    public static boolean checkRemoteSqlPluginPath(String remoteSqlPluginPath, String deployMode, String pluginLoadMode) {
+        if (StringUtils.isEmpty(remoteSqlPluginPath)) {
+            return StringUtils.equalsIgnoreCase(pluginLoadMode, EPluginLoadMode.SHIPFILE.name())
+                    || StringUtils.equalsIgnoreCase(deployMode, ClusterMode.local.name());
+        }
+        return true;
+    }
+
+
     public static StreamExecutionEnvironment getStreamExecution(ParamsInfo paramsInfo) throws Exception {
         StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExeEnv(paramsInfo.getConfProp(), paramsInfo.getDeployMode());
         StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);

core/src/main/java/com/dtstack/flink/sql/option/Options.java

@@ -48,7 +48,7 @@ public class Options {
     @OptionRequired(required = true,description = "Sql local plugin root")
     private  String localSqlPluginPath;
 
-    @OptionRequired(required = true,description = "Sql remote plugin root")
+    @OptionRequired(required = false,description = "Sql remote plugin root")
     private  String remoteSqlPluginPath ;
 
     @OptionRequired(description = "sql ext jar,eg udf jar")

core/src/test/java/com/dtstack/flink/sql/TestGetPlan.java

@@ -46,13 +46,10 @@ public class TestGetPlan {
     @Test
     public void testGetExecutionPlan() throws Exception {
         List<URL> urls = new ArrayList<URL>();
-        urls.addAll(getJarUrl("/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins/"));
         urls.addAll(getJarUrl("/Users/maqi/tmp/flink/flink-1.8.1/lib"));
 
         ClassLoader parentClassLoader = Thread.currentThread().getContextClassLoader();
         DtClassLoader childClassLoader = new DtClassLoader(urls.toArray(new URL[urls.size()]), parentClassLoader);
-
-        Thread.currentThread().setContextClassLoader(childClassLoader);
         Class<?> aClass = childClassLoader.loadClass("com.dtstack.flink.sql.GetPlan");
 
         String[] params = {
@@ -62,8 +59,6 @@ public void testGetExecutionPlan() throws Exception {
                 "%0d%0aCREATE+TABLE+MyTable(%0d%0a++++id+INT%2c%0d%0a++++channel+VARCHAR%2c%0d%0a++++pv+varchar%2c%0d%0a++++xctime+varchar%2c%0d%0a++++name+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest02%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+MyTable2(%0d%0a++++id2+INT%2c%0d%0a++++channel2+VARCHAR%2c%0d%0a++++pv2+varchar%2c%0d%0a++++xctime2+varchar%2c%0d%0a++++name2+varchar%0d%0a+)WITH(%0d%0a++++type+%3d%27kafka11%27%2c%0d%0a++++bootstrapServers+%3d%27172.16.8.107%3a9092%27%2c%0d%0a++++zookeeperQuorum+%3d%27172.16.8.107%3a2181%2fkafka%27%2c%0d%0a++++offsetReset+%3d%27latest%27%2c%0d%0a++++topic+%3d%27mqTest03%27%2c%0d%0a++++timezone%3d%27Asia%2fShanghai%27%2c%0d%0a++++topicIsPattern+%3d%27false%27%2c%0d%0a++++parallelism+%3d%271%27%0d%0a+)%3b%0d%0a%0d%0a%0d%0aCREATE+TABLE+sideTableA(%0d%0a++++id1+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++time_info+varchar%2c%0d%0a++++name1+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)+%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimA%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%0d%0a%0d%0a+)%3b%0d%0a%0d%0a+CREATE+TABLE+sideTableB(%0d%0a++++id+INT%2c%0d%0a++++channel1+varchar%2c%0d%0a++++address+varchar%2c%0d%0a++++PRIMARY+KEY(channel1)%2c%0d%0a++++PERIOD+FOR+SYSTEM_TIME%0d%0a+)WITH(%0d%0a++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimB%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a++++cache+%3d+%27LRU%27%2c%0d%0a++++asyncTimeout+%3d+%271000000%27%0d%0a+)%3b%0d%0a%0d%0aCREATE+TABLE+MyResult(%0d%0a++++xctime+VARCHAR%2c%0d%0a++++name+VARCHAR%2c%0d%0a++++time_info+VARCHAR%2c%0d%0a++++address+VARCHAR%0d%0a+)WITH(%0d%0a+++++type+%3d%27mysql%27%2c%0d%0a++++url+%3d%27jdbc%3amysql%3a%2f%2f172.16.8.109%3a3306%2ftest%27%2c%0d%0a++++userName+%3d%27dtstack%27%2c%0d%0a++++password+%3d%27abc123%27%2c%0d%0a++++tableName+%3d%27dimC%27%2c%0d%0a++++parallelism+%3d%271%27%2c%0d%0a+)%3b%0d%0a%0d%0ainsert+%0d%0ainto%0d%0a++MyResult%0d%0a++++select%0d%0a++++++++t1.xctime+as+xctime%2c%0d%0a++++++++t1.name+as+name%2c%0d%0a++++++++t3.channel1+as+time_info%2c%0d%0a++++++++t3.address+as+address+++++%0d%0a++++from%0d%0a++++++++MyTable+as+t1%0d%0a++++left+join%0d%0a++++++++--MyTable2+m2+%0d%0a++++++++sideTableA+m2+++++++++++++++++++++++++++%0d%0a++++++++on++t1.channel+%3d+m2.channel1%0d%0a++++left+join+sideTableB+t3%0d%0a++++++++on+t1.channel+%3d+t3.channel1%0d%0a%0d%0a%0d%0a",
                 "-localSqlPluginPath",
                 "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins",
-                "-remoteSqlPluginPath",
-                "/Users/maqi/code/dtstack/dt-center-flinkStreamSQL/plugins",
                 "-name","test"};
 
         Method getExecutionPlan = aClass.getMethod("getExecutionPlan", String[].class);