Merge remote-tracking branch 'origin/master'

src/main/java/com/winhc/dataworks/flow/touch/Main.java

@@ -10,6 +10,7 @@ import com.helospark.lightdi.annotation.Service;
 import com.winhc.dataworks.flow.touch.bean.*;
 import com.winhc.dataworks.flow.touch.configuration.SchemaInit;
 import com.winhc.dataworks.flow.touch.service.TouchService;
+import com.winhc.dataworks.flow.touch.utils.DateUtils;
 import com.winhc.dataworks.flow.touch.utils.DingUtils;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -39,7 +40,7 @@ public class Main {
 
     static {
         options.addOption("s", "singleJob", false, "选填，是否单独触发一个业务流程");
-        options.addOption("b", "bizDate", true, "必填，业务时间");
+        options.addOption("b", "bizDate", true, "选填，业务时间[2020-07-07],默认为昨天");
         options.addOption("f", "fileName", true, "必填，yaml文件");
 
         options.addOption("flow", "flow", true, "单任务必填，业务流程名");
@@ -48,7 +49,7 @@ public class Main {
 
     private static void verify(CommandLine commandLine) {
         try {
-            if (!commandLine.hasOption("b") || !commandLine.hasOption("f")) {
+            if (!commandLine.hasOption("f")) {
                 throw new RuntimeException();
             }
             if (commandLine.hasOption("s")) {
@@ -71,7 +72,7 @@ public class Main {
         try {
             CommandLine commandLine = parser.parse(options, args);
             verify(commandLine);
-            String bizDate = commandLine.getOptionValue("b");
+            String bizDate = commandLine.getOptionValue("b", DateUtils.getYesterday());
             String fileName = commandLine.getOptionValue("f");
             List<DataWorksFlowJob> jobs = SchemaInit.getJobs(fileName);
 

src/main/java/com/winhc/dataworks/flow/touch/bean/DingMsg.java

@@ -22,6 +22,12 @@ import java.time.format.DateTimeFormatter;
 public class DingMsg {
     private static final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 
+    public static String msg2Md(String msg) {
+        return "#### " + msg +
+                "\n\n" + "> 时间：" + LocalDateTime.now().format(dtf) +
+                "\n\n" + "> 系统信息：" + os();
+    }
+
     public DingMsg(String msgLevel, String project, String flow, String nodeName, String status) {
         this.msgLevel = msgLevel;
         this.project = project;

src/main/java/com/winhc/dataworks/flow/touch/utils/DateUtils.java

@@ -0,0 +1,23 @@
+package com.winhc.dataworks.flow.touch.utils;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Locale;
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/7/20 13:48
+ * @Description:
+ */
+public class DateUtils {
+    private static final DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd")
+            .withLocale(Locale.CHINA)
+            .withZone(ZoneId.systemDefault());
+
+    public static String getYesterday() {
+        Instant instant = Instant.now().minus(1, ChronoUnit.DAYS);
+        return df.format(instant);
+    }
+}

src/main/java/com/winhc/dataworks/flow/touch/utils/DingUtils.java

@@ -36,7 +36,7 @@ public class DingUtils {
     }
 
     public boolean send(String msg) {
-        return sendByBody(getMsgBody(msg));
+        return sendByBody(getMdBody(msg));
     }
 
     private boolean sendByBody(String body) {
@@ -73,6 +73,10 @@ public class DingUtils {
         return response == null ? "" : response.isSuccessful() ? Objects.requireNonNull(response.body()).string() : "";
     }
 
+    private static String getMdBody(String msg) {
+        return String.format("{\"msgtype\":\"markdown\",\"markdown\":{\"title\":\"%s\",\"text\":\"%s\"}}", "任务通知", DingMsg.msg2Md(msg));
+    }
+
     private static String getMdBody(DingMsg msg) {
         return String.format("{\"msgtype\":\"markdown\",\"markdown\":{\"title\":\"%s\",\"text\":\"%s\"}}", msg.getMsgLevel(), msg.toMd());
     }

src/main/resources/bid&land&punish&etc.yaml

@@ -0,0 +1,236 @@
+job:
+#  #------<公司基本信息
+#  - project: winhc_test
+#    flow: inc_company_spark
+#    task:
+#      - taskName: company_inc
+#        param:
+#          - _nodeId: 700003381602
+#            project: winhc_eci_dev
+#            bizdate: 20200618
+#  #------>
+##------<环保处罚
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_env_punishment
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_env_punishment
+#            DIM_COLUMS: id,name,department,publish_time,punish_number,punish_basis,law_break,reason,content,execution,source_url,source_flag,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,name,source_url,punish_number
+#            DS: 20200605
+##------>
+##------<行政处罚-信用中国
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_punishment_info_creditchina
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_punishment_info_creditchina
+#            DIM_COLUMS: id,company_name,punish_number,punish_name,punish_status,legal_name,type,type_2,result,reason,evidence,area,department,decision_date,term,status,appro_time,source,source_flag,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,company_name,source,punish_number
+#            DS: 20200605
+##------>
+##------<行政处罚
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_punishment_info
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_punishment_info
+#            DIM_COLUMS: id,base,punish_number,name,reg_number,legal_person_name,legal_name_id,legal_name_type,type,content,department_name,decision_date,publish_date,description,desc_file_path,desc_oss_path,punish_desc_mod_time,punish_desc_remark,source,tag,remark,remark_clean,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,name,source,desc_file_path,punish_number
+#            DS: 20200605
+##------>
+##------<地块公示:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_land_publicity
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_publicity
+#            DIM_COLUMS: id,title,district,land_num,location,use_for,area,project_name,application_name,transfer_period,final_price,remark,public_announcement_period,feedback_method,contact_organize,organize_location,postal_code,contact_number,contact_person,electronic_mail,publication_organize,publication_date,publication_start_date,publication_end_date,source_url,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,title,project_name,source_url
+#            DS: 20200605
+##------>
+##------<招投标:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cids_sql
+#    task:
+#      - taskName: company_bid
+#        param:
+#          - _nodeId: 700003375027
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_bid
+#            DIM_COLUMS: id,title,link,intro,abs,publish_time,purchaser,proxy,province,base,type,items,create_time,update_time,deleted
+#            DIM_LIST_COLUMS: id,title,publish_time,province,base,type,items,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,title,publish_time
+#            DS: 20200605
+##------>
+##------<购地信息:ODPS SQL
+#  - project: winhc_test
+#    flow: inc_cid_sql
+#    task:
+#      - taskName: company_land_announcement
+#        param:
+#          - _nodeId: 700003375028
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_announcement
+#            DIM_COLUMS: id,e_number,application_name,district,project_name,project_loc,area,source,source_view,use_type,supply_method,use_period,category,land_level,tran_price,volume_rate,volume_rate_ceiling,commit_time,agreement_start_time,actual_start_time,scheduled_completion,actual_completion_time,authority,contract_date,instalment_payment,source_url,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,source,e_number,application_name
+#            DS: 20200605
+##------>
+#------<土地抵押
+#  - project: winhc_test
+#    flow: inc_company_land_mortgage_sql
+#    task:
+#      - taskName: company_land_mortgage
+#        param:
+#          - _nodeId: 700003375909
+#            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_mortgage
+#            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
+#            DS: 20200605
+#          - _nodeId: 700003375910
+#            PROJECT: winhc_test_dev
+#            DIM_TABLE: company_land_mortgage
+#            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
+#            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
+#            DS: 20200605
+##------>
+#------<招投标:Spark
+  - project: winhc_test
+    flow: incr_calc_intellectual
+    task:
+      - taskName: company_bid
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_bid
+            dupliCols: new_cid,title,link,publish_time
+            flag: cids
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_bid_list
+            cidField: new_cid
+            dupliCols: new_cid,title,link,publish_time
+#------>
+#------<环保处罚
+      - taskName: company_env_punishment
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_env_punishment
+            dupliCols: new_cid,name,source_url,punish_number
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_land_publicity
+            cidField: new_cid
+            dupliCols: new_cid,title,project_name,source_url
+#------>
+#------<购地信息:Spark
+      - taskName: company_land_announcement
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_land_announcement
+            dupliCols: new_cid,source,e_number,application_name
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_land_announcement
+            cidField: new_cid
+            dupliCols: new_cid,source,e_number,application_name
+#------>
+#------<地块公示:Spark
+      - taskName: company_land_publicity
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_land_publicity
+            dupliCols: new_cid,title,project_name,source_url
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_land_publicity
+            cidField: new_cid
+            dupliCols: new_cid,title,project_name,source_url
+#------>
+#------<行政处罚-信用中国:SPARK
+      - taskName: company_punishment_info_creditchina
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_punishment_info_creditchina
+            dupliCols: new_cid,company_name,source,punish_number
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_punishment_info_creditchina
+            cidField: new_cid
+            dupliCols: new_cid,company_name,source,punish_number
+#------>
+#------<行政处罚:SPARK
+      - taskName: company_punishment_info
+        param:
+          - _nodeId: 700003375026
+            project: winhc_eci_dev
+            tableName: company_punishment_info
+            dupliCols: new_cid,name,source,desc_file_path,punish_number
+            flag: cid
+          - _nodeId: 700003380225
+            project: winhc_eci_dev
+            tableName: company_punishment_info
+            cidField: new_cid
+            dupliCols: new_cid,name,source,desc_file_path,punish_number
+#------>
+#------<土地抵押:ODPS SQL
+  - project: winhc_test
+    flow: inc_company_land_mortgage_sql
+    task:
+      - taskName: company_land_mortgage
+        param:
+          - _nodeId: 700003375909
+            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_mortgage
+          - _nodeId: 700003375910
+            PROJECT: winhc_eci_dev
+            DIM_TABLE: company_land_mortgage
+            DIM_COLUMS: type,id,land_mark,land_num,land_aministrative_area,land_loc,land_area,other_item_num,use_right_num,mortgagor,mortgagee,nature,use_for,use_type,area,evaluate_amount,mortgage_amount,source_url,start_date,end_date,create_time,update_time,deleted
+            DUPLI_COLS: new_cid,type,source_url,land_mark,land_num
+            MD5_COLS: type,source_url,land_mark,land_num
+          - _nodeId: 700003422526
+            PROJECT: winhc_eci_dev
+            DIM_TABLE: company_land_mortgage
+#------>
+#------<土地转让:ODPS SQL
+  - project: winhc_test
+    flow: inc_company_land_transfer_sql
+    task:
+      - taskName: company_land_transfer
+        param:
+          - _nodeId: 700003377079
+            PROJECT: winhc_eci_dev
+#            DIM_TABLE: company_land_transfer
+            DS: 20200717
+          - _nodeId: 700003377080
+            PROJECT: winhc_eci_dev
+            DIM_TABLE: company_land_transfer
+            DIM_COLUMS: type,id,mark,num,location,aministrative_area,user_pre,user_now,area,use_for,use_type,years_of_use,situation,level,merchandise_type,merchandise_price,merchandise_time,url,create_time,update_time,deleted
+            DUPLI_COLS: new_cid,type,url,num,mark,location
+            MD5_COLS: type,url,num,mark,location
+            DS: 20200717
+          - _nodeId: 700003421692
+            PROJECT: winhc_eci_dev
+            DIM_TABLE: company_land_transfer
+#------>