Merge remote-tracking branch 'origin/master'

# Conflicts:
#	src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/CompanyDynamic.scala

pom.xml

@@ -307,6 +307,12 @@
             <artifactId>fastjson</artifactId>
             <version>1.2.72</version>
         </dependency>
+        <!--图处理框架-->
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-graphx_${scala.binary.version}</artifactId>
+            <version>2.4.6</version>
+        </dependency>
 
     </dependencies>
 

src/main/scala/com/winhc/bigdata/spark/implicits/BaseHelper.scala

@@ -16,7 +16,7 @@ object BaseHelper {
     def isRegCapital(): Boolean = pattern matches str
 
 
-    def getOrNull(): String = if (str == null) "null" else "\"" + str + "\""
+    def getOrNull(): String = if (str == null) "\"\"" else "\"" + str + "\""
 
 
     def toRegCapital(): Double = str match {

src/main/scala/com/winhc/bigdata/spark/implicits/DataFrame2HBaseHelper.scala

@@ -26,7 +26,7 @@ object DataFrame2HBaseHelper {
         for (f <- fields) {
           val v = row.getAs[String](f.toLowerCase)
           if (v != null) {
-            put.addColumn(f_bytes, Bytes.toBytes(f), Bytes.toBytes(v))
+            put.addColumn(f_bytes, Bytes.toBytes(f.toUpperCase()), Bytes.toBytes(v))
           }
         }
         (new ImmutableBytesWritable, put)

src/main/scala/com/winhc/bigdata/spark/jobs/CalcIncrTotal.scala

@@ -42,6 +42,18 @@ object CalcIncrTotal {
   //  winhc_eci_dev company_double_random_check_result_info main_id,check_item,new_cid cid
 
   //  winhc_eci_dev company_judicial_sale_combine main_id,new_cid cids
+  //  winhc_eci_dev company_zxr_final_case new_cid,case_no cid
+
+  //  winhc_eci_dev company_license new_cid,license_number cid
+  //  winhc_eci_dev company_license_creditchina new_cid,licence_number cid
+  //  winhc_eci_dev company_license_entpub new_cid,license_number cid
+
+  //  winhc_eci_dev company_liquidating_info new_cid,brief_cancel_result,announcement_apply_date cid
+
+  //  winhc_eci_dev wenshu_detail_combine new_cid,case_no,cname cid
+
+
+
 
   def main(args: Array[String]): Unit = {
 

src/main/scala/com/winhc/bigdata/spark/jobs/CompanyAnnualReport.scala

@@ -0,0 +1,425 @@
+package com.winhc.bigdata.spark.jobs
+
+import com.winhc.bigdata.spark.config.EsConfig
+import com.winhc.bigdata.spark.udf.BaseFunc
+import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
+import com.winhc.bigdata.spark.utils.{CompanyIncSummary, DataTypeUtils, LoggingUtils, SparkUtils}
+import org.apache.commons.lang3.StringUtils
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+
+import scala.annotation.meta.getter
+import scala.collection.mutable
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/13 14:00
+ * @Description:
+ */
+object CompanyAnnualReport {
+
+  case class CompanyAnnualReportUtils(s: SparkSession,
+                                      project: String //表所在工程名
+                                     ) extends LoggingUtils with Logging with BaseFunc {
+    @(transient@getter) val spark: SparkSession = s
+
+
+    def getDDL(targetTablePre: String = "ads_", tableName: String, otherCols: Seq[(String, String)]): String = {
+      val colsSet = otherCols.map(_._1).toSet
+
+
+      val cols = otherCols.map(f => {
+        s"${f._1} ${f._2} \n"
+      }) ++
+        spark.table(s"$project.ods_$tableName").schema.fields
+          .filter(f => {
+            !f.name.equals("ds")
+          }).filter(f => {
+          !colsSet.contains(f.name)
+        }).map(f => {
+          val name = f.name
+          val dataType = f.dataType
+          s"$name ${DataTypeUtils.getDataType(dataType)} COMMENT '${f.getComment().getOrElse("")}'\n"
+        })
+
+      s"""
+         |CREATE TABLE IF NOT EXISTS $project.$targetTablePre$tableName
+         |(
+         |    ${cols.mkString(",")}
+         |)
+         |COMMENT 'TABLE COMMENT'
+         |PARTITIONED BY (ds STRING COMMENT '分区')
+         |""".stripMargin
+    }
+
+
+    def getMainTmpTab(tempView: String): Unit = {
+      val ds = getLastPartitionsOrElse(s"$project.ads_company_annual_report", "0")
+      sql(
+        s"""
+           |SELECT  rowkey
+           |        ,main_id
+           |FROM    (
+           |            SELECT  *
+           |                    ,ROW_NUMBER() OVER(PARTITION BY rowkey ORDER BY ds DESC ) AS num
+           |            FROM    (
+           |                        SELECT  rowkey
+           |                                ,id AS main_id
+           |                                ,ds
+           |                        FROM    $project.ads_company_annual_report
+           |                        WHERE   ds = $ds
+           |                        UNION ALL
+           |                        SELECT  rowkey
+           |                                ,id AS main_id
+           |                                ,ds
+           |                        FROM    $project.inc_ads_company_annual_report
+           |                        WHERE   ds > $ds
+           |                    ) AS t1
+           |        ) AS t2
+           |WHERE   t2.num = 1
+           |""".stripMargin)
+        .createOrReplaceTempView(tempView)
+    }
+  }
+
+
+  case class CompanyAnnualReportHandle(s: SparkSession,
+                                       project: String //表所在工程名
+                                      ) extends LoggingUtils with Logging with BaseFunc {
+    @(transient@getter) val spark: SparkSession = s
+
+
+    def main_table_all(): Unit = {
+      //创建表
+      val utils = CompanyAnnualReportUtils(spark, project)
+      val ddl = utils.getDDL(tableName = "company_annual_report", otherCols = Seq(("rowkey", "string"), ("new_cid", "string"), ("cid", "string")))
+      if (!spark.catalog.tableExists(s"$project.ads_company_annual_report")) {
+        println(ddl)
+        println("\n请手动建表")
+        return
+      }
+
+      val ds = getLastPartitionsOrElse(s"$project.ods_company_annual_report", "0")
+
+      val columns = getColumns(s"$project.ads_company_annual_report").diff(Seq("ds", "rowkey", "new_cid", "cid"))
+
+      //写入到主表 ads
+      sql(
+        s"""
+           |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.ads_company_annual_report PARTITION(ds=${ds})
+           |SELECT  rowkey,new_cid,cid,${columns.mkString(",")}
+           |FROM    (
+           |        SELECT
+           |                *
+           |                ,ROW_NUMBER() OVER (PARTITION BY new_cid,report_year ORDER BY id DESC ) num
+           |                ,CONCAT_WS('_',new_cid,report_year) AS rowkey
+           |        FROM    (
+           |                SELECT
+           |                        a.*
+           |                        ,coalesce(b.new_cid,a.cid) AS new_cid
+           |                FROM    (
+           |                        select *
+           |                        from  $project.ods_company_annual_report
+           |                        WHERE   ds = $ds
+           |                        AND  cid IS NOT NULL
+           |                ) a
+           |                LEFT JOIN (
+           |                        select *
+           |                        from  $project.base_company_mapping
+           |                        where ds = $ds
+           |                 ) b
+           |                ON      a.cid = b.cid
+           |            ) c
+           |        ) d
+           |WHERE   num =1  AND report_year is not null AND report_year <> ''
+           |""".stripMargin)
+    }
+
+    def main_table_inc(): Unit = {
+      val utils = CompanyAnnualReportUtils(spark, project)
+      val ddl = utils.getDDL(targetTablePre = "inc_ads_", tableName = "company_annual_report", otherCols = Seq(("rowkey", "string"), ("new_cid", "string"), ("cid", "string")))
+      if (!spark.catalog.tableExists(s"$project.inc_ads_company_annual_report")) {
+        println(ddl)
+        println("\n请手动建表")
+        return
+      }
+
+      val inc_ods_end_ds = getLastPartitionsOrElse(s"$project.inc_ods_company_annual_report", "0")
+      val ads_end_ds = getLastPartitionsOrElse(s"$project.ads_company_annual_report", "0")
+      var inc_ads_end_ds = getLastPartitionsOrElse(s"$project.inc_ads_company_annual_report", "0")
+
+      if (inc_ods_end_ds.equals(inc_ads_end_ds)) {
+        println("inc_ods_end_ds = inc_ads_end_ds ! ")
+        inc_ads_end_ds = getSecondLastPartitionOrElse(s"$project.inc_ads_company_annual_report", "0")
+      }
+      val columns = getColumns(s"$project.ads_company_annual_report").diff(Seq("ds", "rowkey", "new_cid", "cid"))
+
+      sql(
+        s"""
+           |SELECT  cid,current_cid as new_cid
+           |FROM    $project.inc_ods_company
+           |WHERE   ds > ${inc_ads_end_ds}
+           |AND     cid IS NOT NULL
+           |AND     current_cid IS NOT NULL
+           |GROUP BY cid,current_cid
+           |""".stripMargin).createOrReplaceTempView("mapping")
+
+      sql(
+        s"""
+           |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.inc_ads_company_annual_report PARTITION(ds=${inc_ods_end_ds})
+           |SELECT
+           |        rowkey
+           |        ,new_cid
+           |        ,cid
+           |        ,${columns.mkString(",")}
+           |FROM    (
+           |        SELECT
+           |                *
+           |                ,ROW_NUMBER() OVER(PARTITION BY new_cid,report_year ORDER BY id DESC ) num
+           |                ,CONCAT_WS('_',new_cid,report_year) AS rowkey
+           |        FROM    (
+           |                SELECT
+           |                        coalesce(b.new_cid,a.cid) AS new_cid
+           |                        ,a.cid
+           |                        ,${columns.map(s => s"a.$s").mkString(",")}
+           |                FROM    (
+           |                        SELECT  *
+           |                        FROM    $project.inc_ods_company_annual_report
+           |                        WHERE   ds > $inc_ads_end_ds and cid is not null
+           |                        ) a
+           |                LEFT JOIN (
+           |                        select *
+           |                        from $project.base_company_mapping
+           |                        where ds = ${getLastPartitionsOrElse(s"$project.base_company_mapping", "0")}
+           |                ) b
+           |                ON      a.cid = b.cid
+           |                UNION ALL
+           |                SELECT
+           |                        a.new_cid
+           |                        ,b.cid
+           |                        ,${columns.mkString(",")}
+           |                FROM    mapping a
+           |                JOIN (
+           |                        SELECT
+           |                                new_cid AS cid
+           |                                ,${columns.mkString(",")}
+           |                        FROM    $project.ads_company_annual_report
+           |                        WHERE   ds = ${ads_end_ds}
+           |                        UNION ALL
+           |                        SELECT
+           |                                new_cid AS cid
+           |                                ,${columns.mkString(",")}
+           |                        FROM    $project.inc_ads_company_annual_report
+           |                        WHERE   ds > ${inc_ads_end_ds}
+           |                     ) b
+           |                ON      a.cid = b.cid
+           |                ) c
+           |        ) d
+           |WHERE   num =1 AND report_year IS NOT NULL AND report_year <> ''
+           |""".stripMargin)
+
+      //todo 只写入hbase
+
+      val writCols = getColumns("winhc_eci_dev.inc_ads_company_annual_report").diff(Seq(
+        "ds"
+        , "id"
+        , "new_cid"
+        , "cid"
+      ))
+
+      import com.winhc.bigdata.spark.implicits.DataFrame2HBaseHelper._
+      sql(
+        s"""
+           |SELECT  new_cid as cid,${writCols.mkString(",")}
+           |FROM    winhc_eci_dev.inc_ads_company_annual_report
+           |WHERE   ds = '$inc_ods_end_ds'
+           |""".stripMargin)
+        .save2HBase("COMPANY_ANNUAL_REPORT"
+          , "rowkey"
+          , "cid" +: writCols)
+
+      CompanyIncSummary(spark, project, "company_annual_report", "new_cid", Seq("new_cid","report_year")).calc
+
+    }
+
+
+    def sublist_all(tableName: String, primaryFields: Seq[String]): Unit = {
+      cleanup()
+      val utils = CompanyAnnualReportUtils(spark, project)
+
+      //创建表
+      val ddl = utils.getDDL(tableName = tableName, otherCols = Seq(("rowkey", "String"), ("new_cid", "string")))
+      if (!spark.catalog.tableExists(s"$project.ads_$tableName")) {
+        println(ddl)
+        println("\n请手动建表")
+        return
+      }
+
+      utils.getMainTmpTab("main_table_tmp")
+
+      val ods_ds = getLastPartitionsOrElse(s"$project.ods_$tableName", "0")
+
+      sql(
+        s"""
+           |INSERT OVERWRITE TABLE $project.ads_$tableName PARTITION(ds='$ods_ds')
+           |SELECT  ${getColumns(s"$project.ads_$tableName").diff(Seq("ds")).mkString(",")}
+           |FROM    (
+           |            SELECT  t3.*
+           |                    ,split(rowkey,'_')[0] as new_cid
+           |                    ,ROW_NUMBER() OVER(PARTITION BY rowkey ORDER BY id DESC) AS num
+           |            FROM    (
+           |                        SELECT  ${
+          if (primaryFields.exists(s => StringUtils.isNotBlank(s))) {
+            s"""
+               |concat_ws(
+               |  '_'
+               |  ,t2.rowkey
+               | ,md5(
+               |    cleanup(concat_ws('',${primaryFields.mkString(",")}))
+               |      )
+               | )
+               |""".stripMargin
+          } else "t2.rowkey"
+        }
+           |
+           |                                 AS rowkey
+           |                                ,t1.*
+           |                        FROM    (
+           |                                    SELECT  *
+           |                                    FROM    winhc_eci_dev.ods_$tableName
+           |                                    WHERE   ds = $ods_ds
+           |                                ) AS t1
+           |                        JOIN main_table_tmp AS t2
+           |                        ON      t1.main_id = t2.main_id
+           |                    ) AS t3
+           |        ) AS t4
+           |WHERE   t4.num = 1
+           |""".stripMargin)
+    }
+
+
+    def sublist_inc(tableName: String, primaryFields: Seq[String]): Unit = {
+      cleanup()
+      val utils = CompanyAnnualReportUtils(spark, project)
+
+      val ddl = utils.getDDL(targetTablePre = "inc_ads_", tableName = tableName, otherCols = Seq(("rowkey", "String"), ("new_cid", "string")))
+      if (!spark.catalog.tableExists(s"$project.inc_ads_$tableName")) {
+        println(ddl)
+        println("\n请手动建表")
+        return
+      }
+
+      utils.getMainTmpTab("main_table_tmp")
+
+
+      var inc_ads_end_ds = getLastPartitionsOrElse(s"$project.inc_ads_$tableName", "0")
+      val inc_ods_end_ds = getLastPartitionsOrElse(s"$project.inc_ods_$tableName", "0")
+
+      if (inc_ads_end_ds.equals(inc_ods_end_ds)) {
+        println("inc_ads_end_ds = inc_ods_end_ds !")
+        inc_ads_end_ds = getSecondLastPartitionOrElse(s"$project.inc_ads_$tableName", "0")
+      }
+
+      sql(
+        s"""
+           |INSERT OVERWRITE TABLE $project.inc_ads_$tableName PARTITION(ds='$inc_ods_end_ds')
+           |SELECT  ${getColumns(s"$project.inc_ads_$tableName").diff(Seq("ds")).mkString(",")}
+           |FROM    (
+           |            SELECT  t3.*
+           |                    ,split(rowkey,'_')[0] as new_cid
+           |                    ,ROW_NUMBER() OVER(PARTITION BY rowkey ORDER BY ds DESC) AS num
+           |            FROM    (
+           |                        SELECT  ${
+          if (primaryFields.exists(s => StringUtils.isNotBlank(s))) {
+            s"""
+               |concat_ws(
+               |  '_'
+               |  ,t2.rowkey
+               | ,md5(
+               |    cleanup(concat_ws('',${primaryFields.mkString(",")}))
+               |      )
+               | )
+               |""".stripMargin
+          } else "t2.rowkey"
+        } AS rowkey
+           |                                ,t1.*
+           |                        FROM    (
+           |                                    SELECT  *
+           |                                    FROM    winhc_eci_dev.inc_ods_$tableName
+           |                                    WHERE   ds > $inc_ads_end_ds
+           |                                ) AS t1
+           |                        JOIN main_table_tmp AS t2
+           |                        ON      t1.main_id = t2.main_id
+           |                    ) AS t3
+           |        ) AS t4
+           |WHERE   t4.num = 1
+           |""".stripMargin)
+
+      val writeCols = getColumns(s"winhc_eci_dev.inc_ads_$tableName").diff(Seq(
+        "ds"
+        , "id"
+        , "new_cid"
+        , "cid"
+      ))
+
+      //todo 只写入hbase
+      import com.winhc.bigdata.spark.implicits.DataFrame2HBaseHelper._
+      sql(
+        s"""
+           |SELECT  new_cid as cid,${writeCols.mkString(",")}
+           |FROM    winhc_eci_dev.inc_ads_$tableName
+           |WHERE   ds = '$inc_ods_end_ds'
+           |""".stripMargin)
+        .save2HBase(tableName.toUpperCase
+          , "rowkey"
+          , "cid" +: writeCols)
+    }
+  }
+
+
+  def main(args: Array[String]): Unit = {
+    val Array(project, ds) = args
+
+    println(
+      s"""
+         |project: $project
+         |ds: $ds
+         |""".stripMargin)
+
+    val config = EsConfig.getEsConfigMap ++ mutable.Map(
+      "spark.hadoop.odps.project.name" -> project,
+      "spark.debug.maxToStringFields" -> "200",
+      "spark.hadoop.odps.spark.local.partition.amt" -> "1000"
+    )
+    val spark = SparkUtils.InitEnv("company_annual_report", config)
+
+    val sublist_map = Map(
+      "company_annual_report_out_investment" -> "out_investment_name" //年报对外投资
+      , "company_annual_report_change" -> "change_item,change_time" //年报-变更
+      , "company_annual_report_equity_change" -> "investor_name,change_time" //年报-股权变更
+      , "company_annual_report_holder" -> "investor_name" //年报-股东
+      //      , "company_annual_report_out_guarantee" -> "id" //年报-对外担保
+      , "company_annual_report_webinfo" -> "website" //年报-网站
+      , "company_annual_report_social_security" -> "" //年报-社保 (不采取去重，和main_id一对一)
+    )
+
+    val all_flag = false
+
+    if (all_flag) {
+      //存量
+     /* CompanyAnnualReportHandle(spark, project).main_table_all()
+      for (elem <- sublist_map) {
+        println("xjk:" + elem._1)
+        CompanyAnnualReportHandle(spark, project).sublist_all(elem._1, elem._2.split(","))
+      }*/
+    } else {
+      //增量
+      CompanyAnnualReportHandle(spark, project).main_table_inc()
+      for (e <- sublist_map) {
+        println("xjk:" + e._1)
+        CompanyAnnualReportHandle(spark, project).sublist_inc(e._1, e._2.split(","))
+      }
+    }
+    spark.stop()
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/CompanyForCid.scala

@@ -20,11 +20,21 @@ object CompanyForCid {
       "ods_company_mortgage_info" -> Seq("reg_date","reg_num","amount", "new_cid") //产品信息
     )
 //  winhc_eci_dev ods_company_own_tax tax_balance,tax_category,tax_num,new_cid
+  // winhc_eci_dev ods_company_change new_cid,change_item,change_time
+  // winhc_eci_dev ods_company_illegal_info new_cid,put_reason,put_date,put_department
 //  winhc_eci_dev company_check_info check_org,check_date,new_cid
 //  winhc_eci_dev company_tax_contravention taxpayer_number,case_info,new_cid
 //  winhc_eci_dev company_double_random_check_info check_task_num,new_cid
 //  winhc_eci_dev company_double_random_check_result_info main_id,check_item,new_cid
+//  winhc_eci_dev company_zxr_final_case case_no,new_cid
 
+//  winhc_eci_dev company_license license_number,new_cid
+//  winhc_eci_dev company_license_creditchina licence_number,new_cid
+//  winhc_eci_dev company_license_entpub license_number,new_cid
+//  winhc_eci_dev ods_company_finance new_cid,round,money
+//  winhc_eci_dev ods_company_dishonest_info new_cid,case_no
+
+//  winhc_eci_dev wenshu_detail_combine new_cid,case_no,cname,name_type
   def main(args: Array[String]): Unit = {
     val Array(space, sourceTable, cols) = args
 

src/main/scala/com/winhc/bigdata/spark/jobs/CompanyIncrForCid.scala

@@ -16,6 +16,14 @@ object CompanyIncrForCid {
   //winhc_eci_dev company_app_info icon_oss_path,brief,name,new_cid
   //winhc_eci_dev ads_company_tm app_date,tm_name,reg_no,new_cid
   //winhc_eci_dev company_wechat title,public_num,new_cid
+
+  //  winhc_eci_dev company_license license_number,new_cid
+  //  winhc_eci_dev company_license_creditchina licence_number,new_cid
+  //  winhc_eci_dev company_license_entpub license_number,new_cid
+  // winhc_eci_dev company_change new_cid,change_item,change_time
+  // winhc_eci_dev company_illegal_info new_cid,put_reason,put_date,put_department
+  //  winhc_eci_dev company_finance new_cid,round,money
+  // winhc_eci_dev company_dishonest_info new_cid,case_no
   def main(args: Array[String]): Unit = {
     val Array(project, tableName, dupliCols) = args
     println(

src/main/scala/com/winhc/bigdata/spark/jobs/CompanyWenshuDetailCombine.scala

@@ -0,0 +1,161 @@
+package com.winhc.bigdata.spark.jobs
+
+import com.winhc.bigdata.spark.udf.CompanyMapping
+import com.winhc.bigdata.spark.utils.{BaseUtil, LoggingUtils, SparkUtils}
+import org.apache.commons.lang3.StringUtils
+import org.apache.spark.sql.SparkSession
+
+import scala.collection.mutable
+
+
+/**
+ * @Description: 文书增量添加cid
+ * @author π
+ * @date 2020/8/17
+ */
+object CompanyWenshuDetailCombine {
+  def main(args: Array[String]): Unit = {
+    val project ="winhc_eci_dev"
+    val tableName ="inc_ods_wenshu_detail_combine"
+    println(
+      s"""
+         |project: $project
+         |tableName: $tableName
+         |""".stripMargin)
+
+    val config = mutable.Map(
+      "spark.hadoop.odps.project.name" -> "winhc_eci_dev",
+      "spark.hadoop.odps.spark.local.partition.amt" -> "100"
+    )
+    val spark: SparkSession = SparkUtils.InitEnv(this.getClass.getSimpleName, config)
+    CompanyWenshuDetailCombine(spark,project,tableName).calc
+    spark.stop()
+  }
+
+}
+case class CompanyWenshuDetailCombine(s: SparkSession,
+                                      project: String, //表所在工程名
+                                      tableName: String //表名（不加前后辍）
+                                     ) extends LoggingUtils with CompanyMapping{
+  override protected val spark: SparkSession = s
+
+  def calc ={
+    prepareFunctions(spark)
+    var v1 = BaseUtil.getPartion("winhc_eci_dev.inc_ods_wenshu_detail_combine", spark)//添加cid后文书-最新分区
+    val v2 = BaseUtil.getPartion("winhc_eci.ods_wenshu_detail", spark)//缺cid文书-最新分区
+
+    val mapDs = BaseUtil.getPartion("winhc_eci_dev.base_company_mapping", spark)//cid映射最新分区
+
+    if(StringUtils.isBlank(v1)){
+      v1="20200604"
+    }
+
+    println(
+      s"""
+        |v1:$v1
+        |v2:$v2
+        |mapDs:$mapDs
+        |""".stripMargin)
+
+    sql(
+      s"""
+        |INSERT OVERWRITE TABLE winhc_eci_dev.inc_ods_wenshu_detail_combine PARTITION(ds='$v2')
+        |SELECT  name_type
+        |        ,e.cname
+        |        ,f.new_cid AS cid
+        |        ,case_id
+        |        ,uuid
+        |        ,docid
+        |        ,case_no
+        |        ,doc_type
+        |        ,case_type
+        |        ,case_reason_level2
+        |        ,case_reason_level3
+        |        ,case_reason_level4
+        |        ,case_reason
+        |        ,case_reason_levelnum
+        |        ,case_stage
+        |        ,case_amt
+        |        ,party_info
+        |        ,court_name
+        |        ,court_province
+        |        ,court_city
+        |        ,court_level
+        |        ,yg_info
+        |        ,yg_type
+        |        ,yg_name
+        |        ,yg_wtdlr
+        |        ,yg_faren
+        |        ,yg_lawyer
+        |        ,bg_info
+        |        ,bg_type
+        |        ,bg_name
+        |        ,bg_wtdlr
+        |        ,bg_faren
+        |        ,bg_lawyer
+        |        ,third_party
+        |        ,danbao
+        |        ,fact
+        |        ,court_view
+        |        ,judge
+        |        ,clerk
+        |        ,judge_date_cn
+        |        ,judge_date
+        |        ,judge_year
+        |        ,judge_result
+        |        ,is_success
+        |        ,url
+        |        ,head
+        |        ,title
+        |        ,legal_basis
+        |        ,keywords
+        |        ,plaintiffs
+        |        ,defendants
+        |        ,crawl_date
+        |        ,update_date
+        |        ,sample_type
+        |        ,judge_main
+        |FROM    (
+        |            SELECT  *
+        |            FROM    (
+        |                        SELECT  *
+        |                                ,ROW_NUMBER() OVER (PARTITION BY case_no,cname,name_type ORDER BY update_date DESC) num
+        |                        FROM    (
+        |                                    SELECT  *
+        |                                    FROM    (
+        |                                                SELECT  "y" AS name_type
+        |                                                        ,*
+        |                                                FROM    winhc_eci.ods_wenshu_detail
+        |                                                LATERAL VIEW explode(split(yg_name, '\n')) tmpTable AS cname
+        |                                                WHERE   ds > '$v1'
+        |                                                AND     yg_type = '企业'
+        |                                            ) c
+        |                                    UNION ALL
+        |                                    SELECT  *
+        |                                    FROM    (
+        |                                                SELECT  "b" AS name_type
+        |                                                        ,*
+        |                                                FROM    winhc_eci.ods_wenshu_detail
+        |                                                LATERAL VIEW explode(split(bg_name, '\n')) tmpTable AS cname
+        |                                                WHERE   ds > '$v1'
+        |                                                AND     bg_type = '企业'
+        |                                            ) d
+        |                                ) e
+        |                    ) x
+        |            WHERE   num = 1
+        |        ) e
+        |JOIN    (
+        |            SELECT  *
+        |            FROM    (
+        |                        SELECT  *
+        |                                ,ROW_NUMBER() OVER(PARTITION BY CLEANUP(cname) ORDER BY update_time DESC) num
+        |                        FROM    winhc_eci_dev.base_company_mapping
+        |                        WHERE   ds = '$mapDs'
+        |                    ) k
+        |            WHERE   num = 1
+        |            AND     length(CLEANUP(cname)) > 4
+        |        ) f
+        |ON      CLEANUP(e.cname) = CLEANUP(f.cname)
+        |""".stripMargin)
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/ChangeExtract.scala

@@ -2,7 +2,7 @@ package com.winhc.bigdata.spark.jobs.chance
 
 import com.winhc.bigdata.spark.config.EsConfig
 import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
-import com.winhc.bigdata.spark.utils.{BaseUtil, LoggingUtils, ReflectUtils, SparkUtils}
+import com.winhc.bigdata.spark.utils._
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.types.{MapType, StringType, StructField, StructType}
@@ -21,7 +21,7 @@ object ChangeExtract {
   //判断两个map在指定key上是否相等，如不等反回不相等字段
   def getDoubleDataMap(iterable: Iterable[Map[String, String]]): (Map[String, String], Map[String, String]) = {
     val map = iterable.map(m => (m("change_flag"), m)).toMap
-    (map("0"), map("1"))
+    (map.getOrElse("0", null), map.getOrElse("1", null))
   }
 
   def getHandleClazz(tableName: String, equCols: Seq[String]): {def handle(rowkey: String, oldMap: Map[String, String], newMap: Map[String, String]): (String, String, String, Map[String, String], String, String, String, String, Map[String, String])} = {
@@ -47,6 +47,10 @@ object ChangeExtract {
 
     val target_eci_change_extract = "ads_change_extract"
 
+    val updateTimeMapping = Map(
+      "wenshu_detail_combine" -> "update_date" //文书排序时间
+    )
+
     def calc(isCopy: Boolean = true): Unit = {
       val cols = primaryFields.filter(!_.equals(primaryKey)).seq
 
@@ -101,7 +105,7 @@ object ChangeExtract {
                    |                     ,c
                    |             FROM    (
                    |                         SELECT  a.*
-                   |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY update_time DESC) c
+                   |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC) c
                    |                         FROM    (
                    |                                     SELECT  ${intersectCols.mkString(",")}
                    |                                     FROM    $project.ads_$tableName
@@ -137,7 +141,7 @@ object ChangeExtract {
                    |                     ,coalesce(mm.new_cid,tmp.$cid) AS $cid
                    |             FROM    (
                    |                         SELECT  a.*
-                   |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY update_time DESC) c
+                   |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC) c
                    |                         FROM    (
                    |                                     SELECT  ${intersectCols.mkString(",")}
                    |                                     FROM    $project.ads_$tableName
@@ -176,7 +180,7 @@ object ChangeExtract {
                |             SELECT  tmp.*
                |             FROM    (
                |                         SELECT  a.*
-               |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY update_time DESC) c
+               |                                 ,row_number() OVER (PARTITION BY a.${primaryKey} ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC) c
                |                         FROM    (
                |                                     SELECT  ${intersectCols.mkString(",")}
                |                                     FROM    $project.ads_$tableName
@@ -204,17 +208,29 @@ object ChangeExtract {
             val rowkey = x._1
             val map_list = x._2
             //          try {
-            if (map_list.size == 1) {
+            //            if (map_list.size == 1) {
+            //              val res = handle.handle(rowkey, null, map_list.head)
+            //              Row(res._1, res._2, tableName, res._3, res._4, res._5, res._6, res._7, res._8, update_time, res._9)
+            //            } else {
+            //              if (map_list.size > 2) {
+            //                logInfo("list.size > 2! rowkey:" + rowkey)
+            //              }
+            val m = getDoubleDataMap(map_list)
+
+            val new_map = m._1
+            val old_map = m._2
+            if (new_map == null && old_map == null) {
+              null
+            } else if (old_map == null) {
               val res = handle.handle(rowkey, null, map_list.head)
-              Row(res._1, res._2, tableName, res._3, res._4, res._5, res._6, res._7, res._8, update_time, res._9)
-            } else {
-              if (map_list.size > 2) {
-                logInfo("list.size > 2! rowkey:" + rowkey)
+              if (res == null) {
+                null
+              } else {
+                Row(res._1, res._2, tableName, res._3, res._4, res._5, res._6, res._7, res._8, update_time, res._9)
               }
-              val m = getDoubleDataMap(map_list)
-
-              val new_map = m._1
-              val old_map = m._2
+            } else if (new_map == null) {
+              null
+            } else {
               val res = handle.handle(rowkey, old_map, new_map)
               if (res == null) {
                 null
@@ -222,6 +238,7 @@ object ChangeExtract {
                 Row(res._1, res._2, tableName, res._3, res._4, res._5, res._6, res._7, res._8, update_time, res._9)
               }
             }
+            //            }
             /* } catch {
                case e: Exception => {
                  logError(s"xjk rowkey:$rowkey msg:${e.getMessage} equCols:$cols")
@@ -247,14 +264,14 @@ object ChangeExtract {
       ))
 
       spark.createDataFrame(rdd, schema)
-        .createOrReplaceTempView("tmp_change_extract_view") //
+        .createOrReplaceTempView(s"tmp_change_extract_view$tableName") //
 
       sql(
         s"""
            |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE ${project}.$target_eci_change_extract PARTITION(ds='$ds',tn='$tableName')
            |SELECT *
            |FROM
-           |    tmp_change_extract_view
+           |    tmp_change_extract_view$tableName
            |""".stripMargin)
     }
   }
@@ -292,24 +309,33 @@ object ChangeExtract {
     , Args(tableName = "company_land_announcement", primaryFields = "e_number,project_name")
     , Args(tableName = "company_bid_list", primaryFields = "title")
     , Args(tableName = "company_land_transfer", primaryFields = "num,location")
-    , Args(tableName = "company_employment", primaryFields = "title,cid,url_path")
-    , Args(tableName = "company_brief_cancel_announcement_result", primaryFields = "cid,main_id")
+    , Args(tableName = "company_employment", primaryFields = "title,url_path")
     , Args(tableName = "company_env_punishment", primaryFields = "punish_number")
     , Args(tableName = "company_icp", primaryFields = "domain")
     , Args(tableName = "company_punishment_info", primaryFields = "punish_number")
     , Args(tableName = "company_punishment_info_creditchina", primaryFields = "punish_number")
-    , Args(tableName = "bankruptcy_open_case", primaryFields = "case_no", isCopy=false) //破产重整
-    , Args(tableName = "company_public_announcement2_list", primaryFields = "applicant_cid,owner_cid,drawer_cid,gather_name_cid,bill_num")//公示催告
-    , Args(tableName = "company_mortgage_info", primaryFields = "reg_num")//动产抵押
-    , Args(tableName = "company_stock_announcement", primaryFields = "title")//企业公告
-    , Args(tableName = "company_check_info", primaryFields = "check_result")//抽查检查
-    , Args(tableName = "company_court_announcement_list", primaryFields = "content")//法院公告
-    , Args(tableName = "company_court_open_announcement_list", primaryFields = "case_reason")//开庭公告
-    , Args(tableName = "company_court_register_list", primaryFields = "area")//立案信息
-    , Args(tableName = "company_double_random_check_info", primaryFields = "check_plan_name")//双随机抽查
-    , Args(tableName = "company_judicial_sale_combine_list", primaryFields = "title")//司法拍卖
-    , Args(tableName = "company_tax_contravention", primaryFields = "case_type")//税收违法
-    , Args(tableName = "company_send_announcement_list", primaryFields = "title")//送达公告
+    , Args(tableName = "bankruptcy_open_case", primaryFields = "case_no", isCopy = false) //破产重整
+    , Args(tableName = "company_public_announcement2_list", primaryFields = "applicant_cid,owner_cid,drawer_cid,gather_name_cid,bill_num") //公示催告
+    , Args(tableName = "company_mortgage_info", primaryFields = "reg_num") //动产抵押
+    , Args(tableName = "company_stock_announcement", primaryFields = "title") //企业公告
+    , Args(tableName = "company_check_info", primaryFields = "check_result") //抽查检查
+    , Args(tableName = "company_court_announcement_list", primaryFields = "content") //法院公告
+    , Args(tableName = "company_court_open_announcement_list", primaryFields = "case_reason") //开庭公告
+    , Args(tableName = "company_court_register_list", primaryFields = "area") //立案信息
+    , Args(tableName = "company_double_random_check_info", primaryFields = "check_plan_name") //双随机抽查
+    , Args(tableName = "company_judicial_sale_combine_list", primaryFields = "title") //司法拍卖
+    , Args(tableName = "company_tax_contravention", primaryFields = "case_type") //税收违法
+    , Args(tableName = "company_send_announcement_list", primaryFields = "title") //送达公告
+    , Args(tableName = "company_annual_report_out_guarantee", primaryFields = "id") //年报-对外担保
+    , Args(tableName = "company_zxr_restrict", primaryFields = "status") //限制消费令，发现最新状态
+    , Args(tableName = "company_brief_cancel_announcement", primaryFields = "credit_code") //简易注销
+    , Args(tableName = "company_liquidating_info", primaryFields = "id") //清算信息
+
+    , Args(tableName = "company_zxr_final_case", primaryFields = "identity_num") //终本案件
+    , Args(tableName = "company_license_creditchina", primaryFields = "licence_content") //行政许可-信用中国
+    , Args(tableName = "company_license_entpub", primaryFields = "license_name") //行政许可-企业公示
+    , Args(tableName = "company_license", primaryFields = "license_name") //行政许可
+    , Args(tableName = "wenshu_detail_combine", primaryFields = "cname") //文书
 
     , Args(tableName = "company_certificate", primaryFields = "type")
     , Args(tableName = "company_abnormal_info", primaryFields = "remove_reason")
@@ -318,65 +344,58 @@ object ChangeExtract {
 
     , Args(tableName = "company_equity_info", primaryKey = "id", primaryFields = "reg_number", isCopy = false)
     , Args(tableName = "company_staff", primaryFields = "staff_type")
-    //公司名称,法人ID：人标识或公司标识,公司类型,注册地址,营业期限终止日期,经营范围,登记机关,企业状态                 ,注册资本,实收资本金额（单位：分）,注销日期,注销原因
-    , Args(tableName = "company", primaryKey = "cid", primaryFields = "name,legal_entity_id,company_org_type,reg_location,to_time,business_scope,reg_institute,reg_status,reg_capital,actual_capital_amount,cancel_date,cancel_reason")
-    , Args(tableName = "company_illegal_info",  primaryFields = "remove_reason")
+    //公司名称,法人ID：人标识或公司标识,公司类型,注册地址,营业期限终止日期,经营范围,登记机关,企业状态                 ,注册资本,注销日期,注销原因
+    , Args(tableName = "company", primaryKey = "cid", primaryFields = "name,legal_entity_id,company_org_type,reg_location,to_time,business_scope,reg_institute,reg_status,reg_capital,cancel_date,cancel_reason")
+    , Args(tableName = "company_illegal_info", primaryFields = "remove_reason")
+    , Args(tableName = "company_finance", primaryFields = "round")
+    , Args(tableName = "company_dishonest_info", primaryFields = "case_no")
+    , Args(tableName = "company_holder", primaryFields = "amount")
+    , Args(tableName = "company_annual_report_out_investment", primaryFields = "main_id")
   )
 
+
   private case class Args(project: String = "winhc_eci_dev"
                           , tableName: String
                           , primaryKey: String = "rowkey"
                           , primaryFields: String
                           , isCopy: Boolean = true)
 
+
   def main(args: Array[String]): Unit = {
-    if (args.length == 2) {
-      val Array(tableName, inc_ds) = args
-
-      val e = startArgs.filter(_.tableName.equals(tableName)).head
-      val config = EsConfig.getEsConfigMap ++ mutable.Map(
-        "spark.hadoop.odps.project.name" -> e.project,
-        "spark.hadoop.odps.spark.local.partition.amt" -> "10"
-      )
-      val spark = SparkUtils.InitEnv("ChangeExtract", config)
-
-      ChangeExtractHandle(spark, e.project, tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
-      spark.stop()
-    } else {
-      val ds = args(0)
-      val project = "winhc_eci_dev"
-      val config = EsConfig.getEsConfigMap ++ mutable.Map(
-        "spark.hadoop.odps.project.name" -> project,
-        "spark.hadoop.odps.spark.local.partition.amt" -> "10"
-      )
-      val spark = SparkUtils.InitEnv("ChangeExtract", config)
-
-      startArgs.foreach(e => {
-        ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, ds, e.primaryFields.split(",")).calc(e.isCopy)
-      })
-
-
-      /* val rows =
-         """winhc_eci_dev company_tm rowkey 20200717 status_new
-           |winhc_eci_dev company_patent_list rowkey 20200717 lprs
-           |winhc_eci_dev company_copyright_works_list rowkey 20200717 type
-           |winhc_eci_dev company_copyright_reg_list rowkey 20200717 version
-           |winhc_eci_dev company_land_publicity rowkey 20200717 title,location,use_for
-           |winhc_eci_dev company_land_announcement rowkey 20200717 e_number,project_name
-           |winhc_eci_dev company_bid_list rowkey 20200717 title
-           |winhc_eci_dev company_land_transfer rowkey 20200717 num,location
-           |winhc_eci_dev company_employment rowkey 20200717 source
-           |winhc_eci_dev company_env_punishment rowkey 20200717 punish_number
-           |winhc_eci_dev company_icp rowkey 20200717 domain
-           |""".stripMargin.replace("20200717", ds)
-       for (r <- rows.split("\r\n")) {
-         if (StringUtils.isNotEmpty(r)) {
-           val as = r.split(" ")
-           val Array(tmp, tableName, rowkey, inc_ds, pf, isCopy) = if (as.length == 6) as else as :+ "true"
-           ChangeExtractHandle(spark, project, tableName, rowkey, inc_ds, pf.split(",")).calc(isCopy.toBoolean)
-         }
-       }*/
-      spark.stop()
+    val Array(tableName, inc_ds) = args
+
+    val config = EsConfig.getEsConfigMap ++ mutable.Map(
+      "spark.hadoop.odps.project.name" -> "winhc_eci_dev",
+      "spark.hadoop.odps.spark.local.partition.amt" -> "10"
+    )
+    val spark = SparkUtils.InitEnv("ChangeExtract", config)
+
+
+    var start = startArgs
+    if (!tableName.equals("all")) {
+      val set = tableName.split(",").toSet
+      start = start.filter(a => set.contains(a.tableName))
     }
+
+    val a = start.map(e => (e.tableName, () => {
+      ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
+      true
+    }))
+
+    AsyncExtract.startAndWait(spark, a)
+
+    /* if (tableName.equals("all")) {
+       startArgs.foreach(e => {
+         ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
+       })
+     } else {
+       val set = tableName.split(",").toSet
+       startArgs.filter(a => set.contains(a.tableName)).foreach(e => {
+         ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
+       })
+     }*/
+
+    spark.stop()
   }
+
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/creditor_info_add_other.scala

@@ -315,7 +315,7 @@ object creditor_info_add_other {
     //    val ds = "20200721"
     val spark = SparkUtils.InitEnv("add_cols_other", map)
 
-    val table = "xjk_tmp_ads_cre_info_test_v2"
+    val table = "xjk_tmp_ads_cre_info_test_v3"
 
     val v = add_cols_other(spark, project)
     v.prefix(ds, table)

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/bankruptcy_open_case.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,7 +20,7 @@ case class bankruptcy_open_case(equCols: Seq[String]) extends CompanyChangeHandl
     str
   }
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("public_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("public_date"),newMap("update_time"))
 
   override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("case_no"), s"${newMap("case_no")}破产重整发生变更")
 

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company.scala

@@ -13,7 +13,7 @@ import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
 case class company(equCols: Seq[String]) extends CompanyChangeHandle with Serializable {
   override def getCid(rowkey: String, newMap: Map[String, String]): String = rowkey
 
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = "1"
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = null
 
   override def getBizTime(newMap: Map[String, String]): String = newMap("update_time")
 

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_abnormal_info.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: XuJiakai
@@ -16,5 +16,5 @@ case class company_abnormal_info(equCols: Seq[String]) extends CompanyChangeHand
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "经营异常", Array("put_department", "remove_department", "put_reason", "put_date", "remove_date", "remove_reason"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("put_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("put_date"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement_result.scala

@@ -5,20 +5,20 @@ import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
 import com.winhc.bigdata.spark.utils.ChangeExtractUtils
 
 /**
- * @Author: XuJiakai
- * @Date: 2020/7/9 16:44
+ * @Author: yyn
+ * @Date: 2020/8/13
  * @Description:
  */
 
 
-//
+//年报-对外担保
 
-case class company_brief_cancel_announcement_result(equCols: Seq[String]) extends CompanyChangeHandle {
-  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}简易注销信息发生变更")
+case class company_annual_report_out_guarantee(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "对外担保发生变更"
 
-  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}简易注销信息")
+  override def getInsertTitle(newMap: Map[String, String]): String = "新增对外担保"
 
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("cname", "brief_cancel_result"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "对外担保", Array("credito_type", "credito_amount", "credito_term", "guarantee_term", "guarantee_way"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = newMap("update_time")
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement_result.scala

@@ -1,4 +1,3 @@
-
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
@@ -6,19 +5,15 @@ import com.winhc.bigdata.spark.utils.ChangeExtractUtils
 
 /**
  * @Author: XuJiakai
- * @Date: 2020/7/9 16:44
- * @Description:
+ * @Date: 2020/8/21 13:41
+ * @Description: 年报对外投资
  */
+case class company_annual_report_out_investment(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "对外投资发生变更"
 
+  override def getInsertTitle(newMap: Map[String, String]): String = s"新增一家对外投资：${newMap.getOrElse("out_investment_name", "")}"
 
-//
-
-case class company_brief_cancel_announcement_result(equCols: Seq[String]) extends CompanyChangeHandle {
-  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}简易注销信息发生变更")
-
-  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}简易注销信息")
-
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("cname", "brief_cancel_result"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "对外投资", Array("out_investment_cid", "out_investment_name", "reg_number", "credit_code"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = newMap("update_time")
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_bid_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_bid_list(equCols: Seq[String]) extends CompanyChangeHandle {
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "招投标", Array("publish_time", "title", "purchaser", "province", "abs"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("publish_time")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("publish_time"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+
+/**
+ * @Author: Yan Yongnian
+ * @Date: 2020/8/19
+ * @Description:
+ */
+
+
+//简易注销
+
+case class company_brief_cancel_announcement(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("company_name"), s"${newMap("company_name")}简易注销发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("company_name"), s"新增${newMap("company_name")}简易注销")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("company_name", "reg_number", "credit_code", "announcement_term", "reg_authority"))
+
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("announcement_end_date"),newMap("update_time"))
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_check_info.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_check_info(equCols: Seq[String]) extends CompanyChangeHandle
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "抽查检查", Array("check_org", "check_date"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("check_date"))){
-      newMap("update_time")
-    }else{
-      newMap("check_date")
-    }
+    DateUtils.getBizDate(newMap("check_date"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_court_announcement_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_court_announcement_list(equCols: Seq[String]) extends Company
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "法院公告", Array("bltn_no", "publish_date", "case_no"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("publish_date"))){
-      newMap("update_time")
-    }else{
-      newMap("publish_date")
-    }
+    DateUtils.getBizDate(newMap("publish_date"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_court_open_announcement_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_court_open_announcement_list(equCols: Seq[String]) extends Co
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "开庭公告", Array("case_no", "start_date"))
 
   override def getBizTime(newMap: Map[String, String]): String ={
-    if(StringUtils.isBlank(newMap("start_date"))){
-      newMap("update_time")
-    }else{
-      newMap("start_date")
-    }
+    DateUtils.getBizDate(newMap("start_date"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_court_register_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_court_register_list(equCols: Seq[String]) extends CompanyChan
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "立案信息", Array("case_no", "filing_date"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("filing_date"))){
-      newMap("update_time")
-    }else{
-      newMap("filing_date")
-    }
+    DateUtils.getBizDate(newMap("filing_date"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_dishonest_info.scala

@@ -0,0 +1,19 @@
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/12 18:50
+ * @Description: 企业失信被执
+ */
+case class company_dishonest_info(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "企业失信被执发生变化"
+
+  override def getInsertTitle(newMap: Map[String, String]): String = s"新增1条企业失信信息:${newMap.getOrElse("court","")}"
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "企业失信被执", Array("name", "case_no", "court", "gist_dd", "gist_unit"))
+
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("pub_date"),newMap("reg_time"),newMap("appro_time"),newMap("update_time"))
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_double_random_check_info.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_double_random_check_info(equCols: Seq[String]) extends Compan
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "双随机抽查", Array("check_task_num", "check_date"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("check_date"))){
-      newMap("update_time")
-    }else{
-      newMap("check_date")
-    }
+    DateUtils.getBizDate(newMap("check_date"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_env_punishment.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: XuJiakai
@@ -20,5 +20,5 @@ case class company_env_punishment(equCols: Seq[String]) extends CompanyChangeHan
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "环保处罚", Array("name", "department", "punish_number", "punish_basis", "law_break", "reason", "content", "deleted"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("publish_time")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("publish_time"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement_result.scala

@@ -1,4 +1,3 @@
-
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
@@ -6,19 +5,15 @@ import com.winhc.bigdata.spark.utils.ChangeExtractUtils
 
 /**
  * @Author: XuJiakai
- * @Date: 2020/7/9 16:44
- * @Description:
+ * @Date: 2020/8/12 17:42
+ * @Description: 融资历史
  */
+case class company_finance(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "融资历史发生变化"
 
+  override def getInsertTitle(newMap: Map[String, String]): String = s"获得了${newMap.getOrElse("round", "")}融资，由${newMap.getOrElse("inverstors", "")}投资"
 
-//
-
-case class company_brief_cancel_announcement_result(equCols: Seq[String]) extends CompanyChangeHandle {
-  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}简易注销信息发生变更")
-
-  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}简易注销信息")
-
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("cname", "brief_cancel_result"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "融资历史", Array("company_name", "finance_time", "money", "round", "inverstors"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = newMap("report_date")
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement_result.scala

@@ -1,4 +1,3 @@
-
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
@@ -6,19 +5,15 @@ import com.winhc.bigdata.spark.utils.ChangeExtractUtils
 
 /**
  * @Author: XuJiakai
- * @Date: 2020/7/9 16:44
- * @Description:
+ * @Date: 2020/8/19 14:10
+ * @Description: 股东
  */
+case class company_holder(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "股东发生更新"
 
+  override def getInsertTitle(newMap: Map[String, String]): String = "新增股东"
 
-//
-
-case class company_brief_cancel_announcement_result(equCols: Seq[String]) extends CompanyChangeHandle {
-  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}简易注销信息发生变更")
-
-  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}简易注销信息")
-
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("cname", "brief_cancel_result"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "股东信息", Array("holder_id", "holder_type", "amount", "capital", "capital_actual"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = newMap("update_time")
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_illegal_info.scala

@@ -13,7 +13,7 @@ case class company_illegal_info(equCols: Seq[String]) extends CompanyChangeHandl
 
   override def getInsertTitle(newMap: Map[String, String]): String = "新增一条严重违法"
 
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String =  ChangeExtractUtils.getTags(newMap, "严重违法", Array("put_reason", "put_date", "put_department", "remove_reason", "remove_date", "remove_department", "type"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "严重违法", Array("put_reason", "put_date", "put_department", "remove_reason", "remove_date", "remove_department", "type"))
 
-  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getMaxDate(newMap.getOrElse("put_date", ""), newMap.getOrElse("remove_date", ""))
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("put_date"), newMap("remove_date"), newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_judicial_sale_combine_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -11,7 +11,7 @@ import org.apache.commons.lang3.StringUtils
  * @Description:司法拍卖
  */
 
-case class company_judicial_sale_combine_list(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+case class company_judicial_sale_combine_list(equCols: Seq[String]) extends CompanyChangeHandle with Serializable {
   override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}司法拍卖发生变更")
 
   override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}司法拍卖")
@@ -19,10 +19,6 @@ case class company_judicial_sale_combine_list(equCols: Seq[String]) extends Comp
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "司法拍卖", Array("title", "pub_time"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("pub_time"))){
-      newMap("update_time")
-    }else{
-      newMap("pub_time")
-    }
+    DateUtils.getBizDate(newMap("pub_time"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_land_announcement.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_land_announcement(equCols: Seq[String]) extends CompanyChange
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "购地信息", Array("project_name", "project_loc", "area", "tran_price", "e_number"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("actual_start_time")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("actual_start_time"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_land_mortgage.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_land_mortgage(equCols: Seq[String]) extends CompanyChangeHand
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "土地抵押", Array("land_mark", "land_num", "land_aministrative_area", "land_loc", "land_area", "use_right_num", "use_for", "area", "evaluate_amount", "mortgage_amount", "start_date", "end_date"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("start_date"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_land_publicity.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,7 +20,7 @@ case class company_land_publicity(equCols: Seq[String]) extends CompanyChangeHan
     str
   }
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("publication_start_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("publication_start_date"),newMap("update_time"))
 
   override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}地块公示发生变更")
 

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_land_transfer.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_land_transfer(equCols: Seq[String]) extends CompanyChangeHand
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "土地转让", Array("merchandise_time", "user_pre", "user_now", "location", "area", "merchandise_price", "aministrative_area"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("merchandise_time")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("merchandise_time"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_license.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11
+ * @Description:行政许可
+ */
+
+case class company_license(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("license_name"), s"行政许可发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("license_name"), s"新增行政许可")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "行政许可", Array("license_name","start_date"))
+
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("start_date"), newMap("update_time"))
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_license_creditchina.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11
+ * @Description:行政许可-信用中国
+ */
+
+case class company_license_creditchina(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("licence_number"), s"行政许可-信用中国发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("licence_number"), s"新增行政许可-信用中国")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "行政许可-信用中国", Array("licence_content","decision_date"))
+
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("decision_date"), newMap("update_time"))
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_license_entpub.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11
+ * @Description:行政许可-企业公示
+ */
+
+case class company_license_entpub(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("license_name"), s"行政许可-企业公示发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("license_name"), s"新增行政许可-企业公示")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "行政许可-企业公示", Array("license_name","start_date"))
+
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("start_date"), newMap("update_time"))
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_brief_cancel_announcement_result.scala

@@ -5,20 +5,20 @@ import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
 import com.winhc.bigdata.spark.utils.ChangeExtractUtils
 
 /**
- * @Author: XuJiakai
- * @Date: 2020/7/9 16:44
+ * @Author: Yan Yongnian
+ * @Date: 2020/8/19
  * @Description:
  */
 
 
-//
+//清算信息
 
-case class company_brief_cancel_announcement_result(equCols: Seq[String]) extends CompanyChangeHandle {
-  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"${newMap("title")}简易注销信息发生变更")
+case class company_liquidating_info(equCols: Seq[String]) extends CompanyChangeHandle {
+  override def getUpdateTitle(newMap: Map[String, String]): String = "清算信息发生变更"
 
-  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("title"), s"新增${newMap("title")}简易注销信息")
+  override def getInsertTitle(newMap: Map[String, String]): String = s"新增清算信息"
 
-  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "简易注销", Array("cname", "brief_cancel_result"))
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "清算信息", Array("manager", "member"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = newMap("update_time")
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_public_announcement2_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_public_announcement2_list(equCols: Seq[String]) extends Compa
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "公示催告", Array("applicant_cid", "owner_cid", "drawer_cid", "gather_name_cid", "bill_type", "bill_num", "bill_amt"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("publish_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("publish_date"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_punishment_info.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,7 +20,7 @@ case class company_punishment_info(equCols: Seq[String]) extends CompanyChangeHa
     str
   }
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("decision_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("decision_date"),newMap("update_time"))
 
   override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("punish_number"), s"${newMap("punish_number")}行政处罚发生变更")
 

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_punishment_info_creditchina.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,7 +20,7 @@ case class company_punishment_info_creditchina(equCols: Seq[String]) extends Com
     str
   }
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("decision_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("decision_date"),newMap("update_time"))
 
   override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("punish_number"), s"${newMap("punish_number")}行政处罚-信用中国发生变更")
 

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_send_announcement_list.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: Yan Yongnian
@@ -20,5 +20,5 @@ case class company_send_announcement_list(equCols: Seq[String]) extends CompanyC
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "送达公告", Array("court", "case_no", "area", "case_reason", "plaintiff", "defendant"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("start_date")
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("start_date"),newMap("update_time"))
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_tax_contravention.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 import org.apache.commons.lang3.StringUtils
 
 /**
@@ -19,10 +19,6 @@ case class company_tax_contravention(equCols: Seq[String]) extends CompanyChange
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "税收违法", Array("taxpayer_number", "publish_time"))
 
   override def getBizTime(newMap: Map[String, String]): String = {
-    if(StringUtils.isBlank(newMap("publish_time"))){
-      newMap("update_time")
-    }else{
-      newMap("publish_time")
-    }
+    DateUtils.getBizDate(newMap("publish_time"), newMap("update_time"))
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_tm.scala

@@ -2,7 +2,7 @@
 package com.winhc.bigdata.spark.jobs.chance.table
 
 import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
-import com.winhc.bigdata.spark.utils.ChangeExtractUtils
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
 
 /**
  * @Author: XuJiakai
@@ -20,5 +20,7 @@ case class company_tm(equCols: Seq[String]) extends CompanyChangeHandle {
 
   override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.get_ip_tags("商标", newMap("tm_name"), newMap("app_date"), newMap("reg_no"))
 
-  override def getBizTime(newMap: Map[String, String]): String = newMap("app_date")
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("app_date"), newMap("update_time"))
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_zxr_final_case.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11
+ * @Description:终本案件
+ */
+
+case class company_zxr_final_case(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("name"), s"终本案件发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("name"), s"新增终本案件")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "终本案件", Array("identity_num","case_create_time"))
+
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("case_create_time"), newMap("update_time"))
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/company_zxr_restrict.scala

@@ -0,0 +1,25 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+
+/**
+ * @Author: Yan Yongnian
+ * @Date: 2020/8/14
+ * @Description:限制消费令
+ */
+//限制消费令
+case class company_zxr_restrict(equCols: Seq[String]) extends CompanyChangeHandle with Serializable {
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = {
+    val str = ChangeExtractUtils.getTags(newMap, "被限制高消费", Array("name", "identity_num", "court_name", "case_create_time", "case_no"))
+    str
+  }
+
+  override def getBizTime(newMap: Map[String, String]): String = DateUtils.getNotNullStr(newMap("case_create_time"),newMap("update_time"))
+
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("case_no"), s"${newMap("case_no")}限制消费令发生变更")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("case_no"), s"新增${newMap("case_no")}限制消费令")
+}

src/main/scala/com/winhc/bigdata/spark/jobs/chance/table/wenshu_detail_combine.scala

@@ -0,0 +1,24 @@
+
+package com.winhc.bigdata.spark.jobs.chance.table
+
+import com.winhc.bigdata.spark.jobs.chance.CompanyChangeHandle
+import com.winhc.bigdata.spark.utils.{ChangeExtractUtils, DateUtils}
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/18
+ * @Description:裁判文书
+ */
+
+case class wenshu_detail_combine(equCols: Seq[String]) extends CompanyChangeHandle with Serializable  {
+  override def getUpdateTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("cname"), s"裁判文书")
+
+  override def getInsertTitle(newMap: Map[String, String]): String = getValueOrNull(newMap("cname"), s"裁判文书")
+
+  override def getLabel(oldMap: Map[String, String], newMap: Map[String, String]): String = ChangeExtractUtils.getTags(newMap, "裁判文书", Array("case_no", "cname"))
+
+  override def getBizTime(newMap: Map[String, String]): String = {
+    DateUtils.getBizDate(newMap("judge_date"), newMap("update_date"))
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/CompanyDynamic.scala

@@ -5,7 +5,7 @@ import java.util.Date
 import com.winhc.bigdata.spark.config.EsConfig
 import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
 import com.winhc.bigdata.spark.utils.ReflectUtils.getClazz
-import com.winhc.bigdata.spark.utils.{LoggingUtils, SparkUtils}
+import com.winhc.bigdata.spark.utils.{AsyncExtract, LoggingUtils, SparkUtils}
 import org.apache.commons.lang3.time.DateFormatUtils
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.types.StringType
@@ -143,7 +143,7 @@ object CompanyDynamic {
           None
         }
         else {
-          result.map(res => Row(CompanyDynamicHandleUtils.getDynamicId(cid, res._4, res._7, res._6), res._1, res._2, res._3, res._4, res._5, res._6, res._7, res._8, res._9, res._10, DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss")))
+          result.map(res => Row(CompanyDynamicHandleUtils.getDynamicId(res._1, res._4, res._7, res._6), res._1, res._2, res._3, res._4, res._5, res._6, res._7, res._8, res._9, res._10, DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss")))
         }
       })
 
@@ -162,7 +162,7 @@ object CompanyDynamic {
         , "create_time" -> StringType
       ))
       spark.createDataFrame(rdd, schema)
-        .createOrReplaceTempView("company_dynamic_tmp")
+        .createOrReplaceTempView(s"company_dynamic_tmp$tableName")
 
       val cols = getColumns(s"$project.$targetTab").filter(!_.equals("ds")).filter(!_.equals("tn"))
 
@@ -171,31 +171,53 @@ object CompanyDynamic {
            |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE ${getEnvProjectName(env, project)}.$targetTab PARTITION(ds='$ds',tn='$tableName')
            |SELECT ${cols.mkString(",")}
            |FROM
-           |    company_dynamic_tmp
+           |    company_dynamic_tmp$tableName
            |""".stripMargin)
     }
   }
 
   private val startArgs = Seq(
-    Args(tableName = "company_abnormal_info", bName = 0)
+    Args(tableName = "company_abnormal_info", bName = 1)
     , Args(tableName = "company_equity_info")
-    , Args(tableName = "company_staff", bName = 0)
+    , Args(tableName = "company_staff", bName = 1)
     , Args(tableName = "company", bName = 0)
     , Args(tableName = "bankruptcy_open_case", bName = 1)
-    , Args(tableName = "company_illegal_info", bName = 0)
+    , Args(tableName = "company_illegal_info", bName = 1)
     , Args(tableName = "company_land_publicity", bName = 1)
-    , Args(tableName = "company_employment", bName = 1)
+//    , Args(tableName = "company_employment", bName = 1)
     , Args(tableName = "company_land_announcement", bName = 1)
-    , Args(tableName = "company_bid_list", bName = 1)
+    , Args(tableName = "company_bid_list", bName = 2)
     , Args(tableName = "company_land_transfer", bName = 1)
     , Args(tableName = "company_env_punishment", bName = 1)
     , Args(tableName = "company_punishment_info", bName = 1)
     , Args(tableName = "company_punishment_info_creditchina", bName = 1)
     , Args(tableName = "bankruptcy_open_case", bName = 1)
-    , Args(tableName = "company_public_announcement2_list", bName = 1)
+    , Args(tableName = "company_public_announcement2_list", bName = 2)
     , Args(tableName = "company_mortgage_info", bName = 1)
     , Args(tableName = "company_stock_announcement", bName = 1)
+    , Args(tableName = "company_finance", bName = 1)
+    , Args(tableName = "company_dishonest_info", bName = 1)
     , Args(tableName = "company_send_announcement_list", bName = 1)
+    , Args(tableName = "company_annual_report_out_guarantee", bName = 1)
+    , Args(tableName = "company_zxr_restrict", bName = 1)
+    , Args(tableName = "company_brief_cancel_announcement", bName = 1)
+    , Args(tableName = "company_liquidating_info", bName = 1)
+
+    , Args(tableName = "company_zxr_final_case", bName = 1) //终本案件
+    , Args(tableName = "company_license_creditchina", bName = 1) //行政许可-信用中国
+    , Args(tableName = "company_license_entpub", bName = 1) //行政许可-企业公示
+    , Args(tableName = "company_license", bName = 1) //行政许可
+    , Args(tableName = "company_check_info", bName = 1) //抽查检查
+    , Args(tableName = "company_court_announcement_list", bName = 2) //法院公告
+    , Args(tableName = "company_court_open_announcement_list", bName = 2) //开庭公告
+    , Args(tableName = "company_court_register_list", bName = 2) //立案信息
+    , Args(tableName = "company_double_random_check_info", bName = 1) //双随机抽查
+    , Args(tableName = "company_judicial_sale_combine_list", bName = 1) //司法拍卖
+    , Args(tableName = "company_tax_contravention", bName = 1) //税收违法
+    , Args(tableName = "wenshu_detail_combine", bName = 1) //裁判文书
+    , Args(tableName = "company_holder", bName = 1) //股东
+    , Args(tableName = "company_annual_report_out_investment", bName = 1) //裁判文书
+    , Args(tableName = "company_own_tax", bName = 1) //欠税公告
   )
 
   private case class Args(project: String = "winhc_eci_dev"
@@ -209,7 +231,7 @@ object CompanyDynamic {
       println(
         s"""
            |Please enter the legal parameters !
-           |<project> <ds> <tableNames>
+           |<project> <tableNames> <ds>
            |""".stripMargin)
       sys.exit(-99)
     }
@@ -225,13 +247,32 @@ object CompanyDynamic {
 
     val config = EsConfig.getEsConfigMap ++ mutable.Map(
       "spark.hadoop.odps.project.name" -> project,
-      "spark.hadoop.odps.spark.local.partition.amt" -> "10"
+      "spark.hadoop.odps.spark.local.partition.amt" -> "1000"
     )
     val spark = SparkUtils.InitEnv("CompanyDynamic", config)
     val cd = CompanyDynamicUtil(spark, project, ds)
     cd.init()
 
-    val ts = tableNames.split(",").toSet
+    var start = startArgs
+    if (!tableNames.equals("all")) {
+      val set = tableNames.split(",").toSet
+      start = start.filter(a => set.contains(a.tableName))
+    }
+
+    val a = start.map(e => (e.tableName, () => {
+      cd.calc(e.tableName, e.bName)
+      true
+    }))
+
+    AsyncExtract.startAndWait(spark, a)
+
+
+    /*if (tableNames.equals("all")) {
+      startArgs.foreach(e => {
+        cd.calc(e.tableName, e.bName)
+      })
+    } else {
+      val ts = tableNames.split(",").toSet
 
     startArgs.filter(e => {
       ts.contains(e.tableName)
@@ -243,7 +284,7 @@ object CompanyDynamic {
         cd.calc(e.tableName, e.bName)
       }
 
-    })
+    })*/
     spark.stop()
   }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/CompanyDynamicHandle.scala

@@ -7,6 +7,7 @@ package com.winhc.bigdata.spark.jobs.dynamic
  */
 trait CompanyDynamicHandle {
 
+  //废弃
   private val table_2_sub_info_type_map = Map(
     "CompanyDynamicHandleTest" -> "MyTest"
     , "company" -> "eci_detail" //工商信息
@@ -16,52 +17,61 @@ trait CompanyDynamicHandle {
     , "company_land_mortgage" -> "land_mortgage" //土地抵押
     , "company_bid_list" -> "tender_es" //中标信息ES
     , "" -> "enterprise_shixin" //失信
-    , "" -> "enterprise_zhixing" //被执
+    , "company_dishonest_info" -> "enterprise_zhixing" //被执
     , "" -> "shareholder_shixin" //股东失信
     , "" -> "shareholder_zhixing" //股东被执
     , "" -> "tender_qichacha" //中标信息企查查
     , "company_abnormal_info" -> "eci_exception" //经营异常
     , "" -> "eci_zscq" //知识产权
-    , "" -> "eci_wenshu" //裁判文书
-    , "" -> "court_announcement" //法院公告
-    , "" -> "" //对外投资
+    , "wenshu_detail_combine" -> "eci_wenshu" //裁判文书
+    , "company_court_announcement_list" -> "court_announcement" //法院公告
+    , "company_annual_report_out_investment" -> "company_annual_report_out_investment" //对外投资
     , "company_punishment_info" -> "punishment_info" //行政处罚
     , "company_punishment_info_creditchina" -> "punishment_info_creditchina" //行政处罚-信用中国
     , "company_mortgage_info" -> "eci_chattel" //动产抵押
     , "company_env_punishment" -> "env_punishment" //环保处罚
     , "" -> "judicial_assistance" //股权冻结
     , "company_public_announcement2_list" -> "company_public_announcement2_list" //公示催告
-    , "" -> "serious_violation" //严重违法
-    , "" -> "simple_cancellation" //简易注销
+    , "company_illegal_info" -> "serious_violation" //严重违法
+    , "company_brief_cancel_announcement" -> "company_brief_cancel_announcement" //简易注销
     , "company_equity_info" -> "stock_pledge" //股权出质
     , "company_tax_contravention" -> "tax_illegal" //税收违法
     , "company_own_tax" -> "tax_owenotice" //欠税公告
     , "company_judicial_sale_combine_list" -> "judicial" //司法拍卖
     , "company_employment" -> "recruit" //招聘信息
-    , "" -> "liquidation_information" //清算信息
+    , "company_liquidating_info" -> "company_liquidating_info" //清算信息
     , "" -> "investor_equity_change" //大股东变更
     , "" -> "actual_controller_change" //实际控制人变更
     , "company_court_open_announcement_list" -> "court_notice" //开庭公告
     , "bankruptcy_open_case" -> "bankruptcy_open_case" //破产重整
     , "company_stock_announcement" -> "company_stock_announcement" //企业公告
     , "company_send_announcement_list" -> "company_send_announcement_list" //送达公告
+    , "company_annual_report_out_guarantee" -> "company_annual_report_out_guarantee" //年报-对外担保
+    , "company_zxr_restrict" -> "company_zxr_restrict" //年报-对外担保
 
     , "company_staff" -> "company_staff" //主要成员
+    , "company_finance" -> "company_finance" //融资动态
     , "company_check_info" -> "spot_check" //抽查检查
     , "company_double_random_check_info" -> "company_double_random_check_info" //双随机抽查
-    , "company_court_register" -> "company_court_register" //立案信息
+    , "company_court_register_list" -> "company_court_register" //立案信息
+    , "company_zxr_final_case" -> "company_zxr_final_case" //终本案件
+    , "company_license_creditchina" -> "company_license_creditchina" //行政许可-信用中国
+    , "company_license_entpub" -> "company_license_entpub" //行政许可-企业公示
+    , "company_license" -> "company_license" //行政许可
+    , "company_holder" -> "company_holder" //行政许可
   )
 
+  //废弃
   private val table_2_info_type = Map(
     "CompanyDynamicHandleTest" -> "0"
     , "company" -> "1" //工商信息
-    , "" -> "2" // 企业失信被执
+    , "company_dishonest_info" -> "2" // 企业失信被执
     , "" -> "3" // 企业股东失信被执
     , "company_abnormal_info" -> "4" // 经营异常
     , "" -> "5" // 知识产权
-    , "" -> "6" // 裁判文书
+    , "wenshu_detail_combine" -> "6" // 裁判文书
     , "company_court_announcement_list" -> "7" // 法院公告
-    , "" -> "8" // 对外投资
+    , "company_annual_report_out_investment" -> "8" // 对外投资
     , "company_mortgage_info" -> "9" // 动产抵押
     , "company_judicial_sale_combine_list" -> "10" // 司法拍卖
     , "company_land_publicity" -> "11-1" // 土地信息-地块公示
@@ -75,41 +85,45 @@ trait CompanyDynamicHandle {
     , "company_public_announcement2_list" -> "15" // 公示催告
     , "company_env_punishment" -> "16" // 环保处罚
     , "company_equity_info" -> "17" // 股权出质
-    , "" -> "18" // 严重违法
-    , "" -> "19" // 简易注销
+    , "company_illegal_info" -> "18" // 严重违法
+    , "company_brief_cancel_announcement" -> "19" // 简易注销
     , "company_own_tax" -> "20" // 欠税公告
     , "company_tax_contravention" -> "21" // 税收违法
     , "" -> "22" // 股权冻结
-    , "" -> "23" // 清算信息
+    , "company_liquidating_info" -> "23" // 清算信息
     , "" -> "24" // 大股东变更
     , "" -> "25" // 实际控制人变更
     , "company_court_open_announcement_list" -> "26" // 开庭公告
     , "" -> "27" // 新闻信息
-    , "" -> "28" // 股东信息
+    , "company_holder" -> "28" // 股东信息
     , "" -> "29" // 最终受益人
     , "company_staff" -> "30" // 主要成员
-    , "" -> "31" // 融资动态
+    , "company_finance" -> "31" // 融资动态
     , "company_stock_announcement" -> "32" // 企业公告
     , "company_check_info" -> "33" // 抽查检查
-    , "" -> "34" // 行政许可
+    , "company_license" -> "34-1" // 行政许可
+    , "company_license_creditchina" -> "34-2" // 行政许可-信用中国
+    , "company_license_entpub" -> "34-3" // 行政许可-企业公示
     , "company_double_random_check_info" -> "35" // 双随机抽查
-    , "" -> "36" // 限制高消费
+    , "company_zxr_restrict" -> "36" // 限制高消费
     , "" -> "37" // 被执行人
     , "company_send_announcement_list" -> "38" // 送达报告
     , "bankruptcy_open_case" -> "39" // 破产重整
-    , "company_court_register" -> "40" // 立案信息
+    , "company_court_register_list" -> "40" // 立案信息
+    , "company_annual_report_out_guarantee" -> "41" // 年报-对外担保
+    , "company_zxr_final_case" -> "41" // 终本案件
   )
 
 
   //风险等级映射
   private val info_risk_level_map = Map(
-    "" -> "4" //企业失信被执情况
+    "company_dishonest_info" -> "4" //企业失信被执情况
     , "" -> "4" //股东失信被执情况
     , "" -> "4" //股权冻结
     , "" -> "4" //司法拍卖
-    , "" -> "4" //清算信息
-    , "" -> "4" //简易注销
-    , "" -> "4" //严重违法
+    , "company_liquidating_info" -> "4" //清算信息
+    , "company_brief_cancel_announcement" -> "4" //简易注销
+    , "company_illegal_info" -> "4" //严重违法
     , "" -> "3" //裁判文书（被告）
     , "" -> "3" //裁判文书（被执行人）
     , "" -> "3" //法院公告（被告/被执行人）
@@ -135,10 +149,13 @@ trait CompanyDynamicHandle {
     , "" -> "2" //对外投资企业注销/吊销/经营异常
     , "" -> "2" //分支机构注销/吊销/经营异常
     , "" -> "2" //新闻舆论（中立、消极）
+    , "company_staff" -> "2" //主要成员
+    , "company_holder" -> "2" //股东信息
+    , "company_finance" -> "2" //融资
     , "" -> "1" //增资
     , "" -> "1" //裁判文书（原告）
     , "" -> "1" //裁判文书（申请执行人）
-    , "" -> "1" //对外投资
+    , "company_annual_report_out_investment" -> "1" //对外投资
     , "" -> "1" //分支机构
     , "" -> "1" //购地信息
     , "" -> "1" //地块公示
@@ -190,7 +207,7 @@ trait CompanyDynamicHandle {
       , get_biz_id(rowkey)
       , get_sub_info_type()
       , get_info_risk_level(old_map, new_map)
-      , if (suggestion == null) "被监控企业流动资金紧张，可能存在经营困难的情况。建议立即与被监控企业书面对账，适当催促其履行债务并持续监控。" else suggestion
+      , if (suggestion == null) null else suggestion
     ))
   }
 
@@ -216,7 +233,7 @@ trait CompanyDynamicHandle {
    *
    * @return
    */
-  protected def get_info_type(): String = table_2_info_type(getClass.getSimpleName)
+  protected def get_info_type(): String = getClass.getSimpleName //table_2_info_type(getClass.getSimpleName)
 
   /**
    * 变更内容
@@ -225,7 +242,7 @@ trait CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String
+  protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = null
 
   /**
    * 变更时间
@@ -233,7 +250,13 @@ trait CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  protected def get_change_time(bizDate: String, new_map: Map[String, String]): String = bizDate
+  protected def get_change_time(bizDate: String, new_map: Map[String, String]): String = {
+    var res = bizDate
+    if (bizDate.length == 10) {
+      res = res.concat(" 00:00:00")
+    }
+    res
+  }
 
   /**
    * 业务id
@@ -248,7 +271,7 @@ trait CompanyDynamicHandle {
    *
    * @return
    */
-  protected def get_sub_info_type(): String = table_2_sub_info_type_map(getClass.getSimpleName)
+  protected def get_sub_info_type(): String = null //table_2_sub_info_type_map(getClass.getSimpleName)
 
   /**
    * 风险等级

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/bankruptcy_open_case.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/5
- * @Description TODO
+ * @Description
  */
 //破产公告
 case class bankruptcy_open_case() extends CompanyDynamicHandle {
@@ -26,10 +26,10 @@ case class bankruptcy_open_case() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""案号:$new_map("case_no")\n
-       |被申请人：$new_map("respondent")\n
-       |申请人：$new_map("applicant")\n
-       |公开日期：$new_map("public_date")\n""".stripMargin
+    s"""案号:${new_map("case_no")}\n
+       |被申请人：${new_map("respondent")}\n
+       |申请人：${new_map("applicant")}\n
+       |公开日期：${new_map("public_date")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company.scala

@@ -19,7 +19,7 @@ case class company() extends CompanyDynamicHandle {
     , "reg_institute" -> "登记机关"
     , "reg_status" -> "企业状态"
     , "reg_capital" -> "注册资本"
-    , "actual_capital_amount" -> "实收资本"
+//    , "actual_capital_amount" -> "实收资本"
   )
 
 
@@ -82,12 +82,12 @@ case class company() extends CompanyDynamicHandle {
         , new_map("name")
         , super.get_info_type()
         , s"${change_type_map(e)}发生变更"
-        , s"""{"变更后内容": ${new_map(e).getOrNull()},"变更事项": ${change_type_map(e).getOrNull()},"变更日期": "$bizDate","变更前内容": ${old_map(e).getOrNull()}}"""
+        , s"""{"content_after": ${new_map(e).getOrNull()},"change_item": ${change_type_map(e).getOrNull()},"change_time": "$bizDate","content_before": ${old_map(e).getOrNull()}}"""
         , bizDate
         , rowkey
         , super.get_sub_info_type()
         , v
-        , "建议"
+        , null
       )
     })
   }
@@ -101,12 +101,4 @@ case class company() extends CompanyDynamicHandle {
    */
   override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = null
 
-  /**
-   * 变更内容
-   *
-   * @param old_map
-   * @param new_map
-   * @return
-   */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = null
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_abnormal_info.scala

@@ -1,7 +1,6 @@
 package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
-import com.winhc.bigdata.spark.implicits.MapHelper._
 
 //经营异常
 case class company_abnormal_info() extends CompanyDynamicHandle {
@@ -14,13 +13,13 @@ case class company_abnormal_info() extends CompanyDynamicHandle {
    */
   override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = new_map.getOrElse("put_reason", null)
 
-  /**
+  /*
    * 变更内容
    *
    * @param old_map
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = new_map.toJson(Seq("put_department->做出决定机关", "remove_department->移出决定机关", "put_reason->列入经营异常目录原因", "put_date->列入日期", "remove_date->移出日期", "remove_reason->移出经营异常目录原因"))
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = new_map.toJson(Seq("put_department->做出决定机关", "remove_department->移出决定机关", "put_reason->列入经营异常目录原因", "put_date->列入日期", "remove_date->移出日期", "remove_reason->移出经营异常目录原因"))
 
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_annual_report_out_guarantee.scala

@@ -0,0 +1,46 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author yyn
+ * @Date 2020/8/13
+ * @Description
+ */
+//年报-对外担保
+case class company_annual_report_out_guarantee()extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = "对外担保"
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = new_map("credito_term") + new_map("guarantee_way")
+
+  /**
+   * 变更时间
+   *
+   * @param new_map
+   * @return
+   */
+//  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
+
+  /**
+   * 风险等级
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_annual_report_out_investment.scala

@@ -0,0 +1,30 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/21 13:44
+ * @Description:
+ */
+case class company_annual_report_out_investment() extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = s"新增一家对外投资：${new_map.getOrElse("out_investment_name", "未知")}"
+
+  /*
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+ /* override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(
+    Seq("out_investment_cid", "out_investment_name", "reg_number", "credit_code"
+    )
+  )*/
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_bid_list.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/7/28
- * @Description TODO
+ * @Description
  */
 case class company_bid_list() extends CompanyDynamicHandle {
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_brief_cancel_announcement.scala

@@ -0,0 +1,48 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author yyn
+ * @Date 2020/8/19
+ * @Description
+ */
+//简易注销
+case class company_brief_cancel_announcement() extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = new_map("announcement_term")
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
+    "进行了简易注销"
+  }
+
+  /**
+   * 变更时间
+   *
+   * @param new_map
+   * @return
+   */
+//  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
+
+  /**
+   * 风险等级
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "高风险信息"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_check_info.scala

@@ -39,7 +39,7 @@ case class company_check_info() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
 
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "2"
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_court_announcement_list.scala

@@ -2,6 +2,7 @@ package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.implicits.MapHelper._
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+import org.apache.commons.lang3.StringUtils
 
 /**
  * @Author: π
@@ -41,21 +42,30 @@ case class company_court_announcement_list() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
-    "announcement_type_code->公告类型"
-    , "plaintiff->原告"
-    , "bltn_no->公告号"
-    , "court_name->法院名"
-    , "deal_grade->处理等级名称"
-    , "litigant->当事人"
-    , "judge->法官"
-    , "province->省份"
-    , "judge_phone->法官电话"
-    , "case_no->案件号"
-    , "content->案件内容"
-    , "publish_page->刊登版面"
-    , "publish_date->刊登日期"
-  ))
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+//    "announcement_type_code->公告类型"
+//    , "plaintiff->原告"
+//    , "bltn_no->公告号"
+//    , "court_name->法院名"
+//    , "deal_grade->处理等级名称"
+//    , "litigant->当事人"
+//    , "judge->法官"
+//    , "province->省份"
+//    , "judge_phone->法官电话"
+//    , "case_no->案件号"
+//    , "content->案件内容"
+//    , "publish_page->刊登版面"
+//    , "publish_date->刊登日期"
+//  ))
 
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    val oldcname = new_map("cname")
+    val litigant = new_map("litigant")
+    if (StringUtils.isNotBlank(oldcname)) {
+      if (oldcname.contains(litigant)) {
+        return "3"
+      }
+    }
+    "2"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_court_open_announcement_list.scala

@@ -2,6 +2,7 @@ package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.implicits.MapHelper._
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+import org.apache.commons.lang3.StringUtils
 
 /**
  * @Author: π
@@ -41,18 +42,27 @@ case class company_court_open_announcement_list() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
-    "case_reason->案由"
-    ,"case_no->案号"
-    ,"start_date->开庭时间"
-    ,"area->地区"
-    ,"plan_date->排期日期"
-    ,"judge->审判长/主审人"
-    ,"litigant->当事人"
-    ,"court->法院"
-    ,"court_room->法庭"
-    ,"content->公告内容"
-  ))
-
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+//    "case_reason->案由"
+//    , "case_no->案号"
+//    , "start_date->开庭时间"
+//    , "area->地区"
+//    , "plan_date->排期日期"
+//    , "judge->审判长/主审人"
+//    , "litigant->当事人"
+//    , "court->法院"
+//    , "court_room->法庭"
+//    , "content->公告内容"
+//  ))
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    val oldcname = new_map("cname")
+    val defendant = new_map("defendant")//被告
+    if (StringUtils.isNotBlank(oldcname)) {
+      if (oldcname.contains(defendant)) {
+        return "3"
+      }
+    }
+    "2"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_court_register_list.scala

@@ -2,6 +2,7 @@ package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.implicits.MapHelper._
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+import org.apache.commons.lang3.StringUtils
 
 /**
  * @Author: π
@@ -41,21 +42,30 @@ case class company_court_register_list() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
-    "case_reason->案由"
-    ,"case_no->案号"
-    ,"filing_date->立案日期"
-    ,"start_time->开庭时间"
-    ,"department->承办部门"
-    ,"court->法院"
-    ,"judge->承办法官"
-    ,"assistant->法官助理"
-    ,"case_type->案件类型"
-    ,"case_status->案件状态"
-    ,"plaintiff->上诉人"
-    ,"defendant->被上诉人"
-    ,"third->第三人"
-  ))
-
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+//    "case_reason->案由"
+//    ,"case_no->案号"
+//    ,"filing_date->立案日期"
+//    ,"start_time->开庭时间"
+//    ,"department->承办部门"
+//    ,"court->法院"
+//    ,"judge->承办法官"
+//    ,"assistant->法官助理"
+//    ,"case_type->案件类型"
+//    ,"case_status->案件状态"
+//    ,"plaintiff->上诉人"
+//    ,"defendant->被上诉人"
+//    ,"third->第三人"
+//  ))
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    val oldcname = new_map("cname")
+    val defendant = new_map("defendant")
+    if (StringUtils.isNotBlank(oldcname)) {
+      if (oldcname.contains(defendant)) {
+        return "3"
+      }
+    }
+    "2"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_dishonest_info.scala

@@ -0,0 +1,52 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/12 18:50
+ * @Description: 企业失信被执
+ */
+case class company_dishonest_info() extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = s"新增1条企业失信信息:${new_map.getOrElse("court", "")}"
+
+  /*
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  /*override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+    "name->失信人名称"
+    , "age->年龄"
+    , "sexy->性别"
+    , "case_no->案号"
+    , "focus_num->关注数"
+    , "card_num"
+    , "legal_name->法人、负责人姓名"
+    , "legal_name_id->法人标识"
+    , "legal_name_type"
+    , "staff->法定负责人/主要负责人信息"
+    , "area->省份地区"
+    , "court->法院"
+    , "gist_dd->执行依据文号"
+    , "reg_time->立案时间"
+    , "gist_unit->做出执行的依据单位"
+    , "duty->生效法律文书确定的义务"
+    , "performance->履行情况"
+    , "unperform_part->未履行"
+    , "performed_part->已履行"
+    , "action_content->失信被执行人行为具体情形"
+    , "pub_date->发布时间"
+    , "lawsuit_url"
+    , "appro_time->与官网核准的时间"
+  ))*/
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_double_random_check_info.scala

@@ -40,7 +40,9 @@ case class company_double_random_check_info() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
 
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    "2"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_env_punishment.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/7/27
- * @Description TODO
+ * @Description
  */
 //环保处罚
 case class company_env_punishment()extends CompanyDynamicHandle {

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_equity_info.scala

@@ -1,7 +1,6 @@
 package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
-import com.winhc.bigdata.spark.implicits.MapHelper._
 
 /**
  * @Author: XuJiakai
@@ -32,27 +31,28 @@ case class company_equity_info() extends CompanyDynamicHandle {
     if ("1".equals(new_map("deleted"))) {
       return Seq.empty
     }
-    var list: Seq[(String, String, String)] = Seq((new_map("cid"), cname, "2")) //标的企业
+
+    var map = Map(new_map("cid") -> (cname, "2")) //标的企业
     if ("2".equals(new_map.getOrElse("pledgor_type", "0"))) {
-      list = list :+ (new_map("pledgor_id"), new_map("pledgor"), "3") //出质人
+      map = map ++ Map(new_map("pledgor_id") -> (new_map("pledgor"), "3")) //出质人
     }
     if ("2".equals(new_map.getOrElse("pledgee_type", "0"))) {
-      list = list :+ (new_map("pledgee_id"), new_map("pledgee"), "1") //质权人
+      map = map ++ Map(new_map("pledgee_id") -> (new_map("pledgee"), "1")) //质权人
     }
-
-    list.map(t => {
-      (t._1
-        , t._2
-        , get_info_type()
-        , get_rta_desc(old_map, new_map)
-        , get_change_content(old_map, new_map, cname)
-        , get_change_time(bizDate, new_map)
-        , get_biz_id(rowkey)
-        , get_sub_info_type()
-        , t._3
-        , "被监控企业流动资金紧张，可能存在经营困难的情况。建议立即与被监控企业书面对账，适当催促其履行债务并持续监控。"
-      )
-    }).seq
+    map.map(e => (e._1, e._2._1, e._2._2)).toSeq
+      .map(t => {
+        (t._1
+          , t._2
+          , get_info_type()
+          , get_rta_desc(old_map, new_map)
+          , get_change_content(old_map, new_map, cname)
+          , get_change_time(bizDate, new_map)
+          , get_biz_id(rowkey)
+          , get_sub_info_type()
+          , t._3
+          , null
+        )
+      }).seq
   }
 
   /**
@@ -64,25 +64,17 @@ case class company_equity_info() extends CompanyDynamicHandle {
    */
   override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = "新增1条股权出质信息"
 
-  /**
+  /*
    * 变更内容
    *
    * @param old_map
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
+  /*override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
+    import com.winhc.bigdata.spark.implicits.MapHelper._
     val json = new_map.toJson(Seq("equity_amount->出质股权数额", "state->出质状态", "reg_date->股权出质设立登记日期", "reg_number->质权登记编号"))
     json.substring(0, json.length - 1) +
       s""","出质股权标的企业": {"企业名称": "$cname","企业KeyNo": "${new_map("cid")}"},"出质人信息": {"证件号": "${new_map("certif_number_l")}","出质人": "${new_map("pledgor")}"},"质权人信息": {"质权人": "${new_map("pledgee")}","证件号": "${new_map("certif_number_r")}"}}"""
-  }
-
-  /**
-   * 风险等级
-   *
-   * @param old_map
-   * @param new_map
-   * @return
-   */
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = null
+  }*/
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_finance.scala

@@ -0,0 +1,34 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/12 17:43
+ * @Description: 融资历史
+ */
+case class company_finance() extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = s"获得了${new_map.getOrElse("round", "")}融资，由${new_map.getOrElse("inverstors", "")}投资"
+
+  /*
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  /*override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+    "company_name->公司名称"
+    , "finance_time->融资时间"
+    , "money->融资金额"
+    , "round->轮次"
+    , "inverstors->投资人"
+  ))*/
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_holder.scala

@@ -0,0 +1,62 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/19 14:15
+ * @Description: 股东
+ */
+case class company_holder() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert", "update")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    if (old_map == null) {
+      "insert"
+    } else {
+      "update"
+    }
+  }
+
+  /*
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = {
+    var out = Map(
+      "after" -> new_map("percent")
+      , "holder_id" -> new_map("holder_id")
+      , "holder_type" -> new_map("holder_type")
+    )
+    if (old_map == null) {
+      out = out ++ Map(
+        "type" -> "insert"
+      )
+    } else {
+      out = out ++ Map(
+        "type" -> "update"
+        , "before" -> old_map("percent")
+      )
+    }
+    import com.winhc.bigdata.spark.implicits.MapHelper._
+    val keys = out.keys.toSeq
+    out.toJson(keys)
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_illegal_info.scala

@@ -1,7 +1,6 @@
 package com.winhc.bigdata.spark.jobs.dynamic.tables
 
 import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
-import com.winhc.bigdata.spark.implicits.MapHelper._
 
 /**
  * @Author: XuJiakai
@@ -27,22 +26,22 @@ case class company_illegal_info() extends CompanyDynamicHandle {
    */
   override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
     if (old_map == null)
-      return s"新增严重违法记录：${new_map.getOrElse("put_reason", "")}"
+      return s"新增严重违法记录：${new_map.getOrElse("put_reason", "未知")}"
 
     if (old_map.get("remove_date") == null && new_map("remove_date") != null)
-      return s"移除严重违法记录：${new_map.getOrElse("remove_reason", "")}"
+      return s"移除严重违法记录：${new_map.getOrElse("remove_reason", "未知")}"
 
     null
   }
 
-  /**
+  /*
    * 变更内容
    *
    * @param old_map
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+  /*override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
     "put_reason->列入原因"
     , "put_date->列入时间"
     , "remove_date->移除时间"
@@ -50,5 +49,5 @@ case class company_illegal_info() extends CompanyDynamicHandle {
     , "remove_reason->移除原因"
     , "remove_department->移除决定机关"
     , "type->类型"
-  ))
+  ))*/
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_judicial_sale_combine_list.scala

@@ -41,9 +41,11 @@ case class company_judicial_sale_combine_list() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
-    "source_id->address"
-  ))
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+//    "source_id->address"
+//  ))
 
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "提示信息"
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    "4"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_land_announcement.scala

@@ -5,10 +5,10 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //购地信息
-case class company_land_announcement()extends CompanyDynamicHandle {
+case class company_land_announcement() extends CompanyDynamicHandle {
   /**
    * 信息描述
    *
@@ -33,7 +33,7 @@ case class company_land_announcement()extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-//  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
+  //  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
 
   /**
    * 风险等级

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_land_mortgage.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //土地抵押
 case class company_land_mortgage() extends CompanyDynamicHandle {
@@ -26,10 +26,10 @@ case class company_land_mortgage() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""抵押土地评估金额:$new_map("evaluate_amount")万元\n
-       |抵押金额：$new_map("mortgage_amount")万元\n
-       |抵押开始时间：$new_map("start_date")\n
-       |抵押结束时间：$new_map("end_date")\n""".stripMargin
+    s"""抵押土地评估金额:${new_map("evaluate_amount")}万元\n
+       |抵押金额：${new_map("mortgage_amount")}万元\n
+       |抵押开始时间：${new_map("start_date")}\n
+       |抵押结束时间：${new_map("end_date")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_land_publicity.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //地块公示
 case class company_land_publicity() extends CompanyDynamicHandle {

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_land_transfer.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //土地转让
 case class company_land_transfer() extends CompanyDynamicHandle {
@@ -26,8 +26,8 @@ case class company_land_transfer() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""转让价格:$new_map("merchandise_price")万元\n
-       |成交时间：$new_map("merchandise_time")\n""".stripMargin
+    s"""转让价格:${new_map("merchandise_price")}万元\n
+       |成交时间：${new_map("merchandise_time")}\n""".stripMargin
   }
 
   /**
@@ -54,7 +54,7 @@ case class company_land_transfer() extends CompanyDynamicHandle {
         "提示信息"
       }
       else{//原有土地使用权人
-        "提示信息"
+        "警示信息"
       }
     }
     else{

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_license.scala

@@ -0,0 +1,48 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11 17:39
+ * @Description: 行政许可
+ */
+case class company_license() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+
+    s"""
+       |许可证编号：${new_map("license_number")}\n
+       |有效期自：${new_map("start_date")}\n
+       |有效期至：${new_map("end_date")}\n
+       |许可机关：${new_map("department")}\n
+       |许可内容：${new_map("scope")}
+       |""".stripMargin
+  }
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "2"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_license_creditchina.scala

@@ -0,0 +1,49 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.implicits.MapHelper._
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11 17:39
+ * @Description: 行政许可-信用中国
+ */
+case class company_license_creditchina() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+
+    s"""
+       |许可证编号：${new_map("licence_number")}\n
+       |有效期自：${new_map("decision_date")}\n
+       |有效期至：${new_map("end_date")}\n
+       |许可机关：${new_map("department")}\n
+       |许可内容：${new_map("resume")}
+       |""".stripMargin
+  }
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "2"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_license_entpub.scala

@@ -0,0 +1,48 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11 17:39
+ * @Description: 行政许可-企业公示
+ */
+case class company_license_entpub() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+
+    s"""
+       |许可证编号：${new_map("license_number")}\n
+       |有效期自：${new_map("start_date")}\n
+       |有效期至：${new_map("end_date")}\n
+       |许可机关：${new_map("department")}\n
+       |许可内容：${new_map("scope")}
+       |""".stripMargin
+  }
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "2"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_liquidating_info.scala

@@ -0,0 +1,48 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author yyn
+ * @Date 2020/8/19
+ * @Description
+ */
+//清算信息
+case class company_liquidating_info() extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = "清算信息"
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
+    s"进行了清算，清算组负责人为${new_map("manager")},清算组成员为${new_map("member")}"
+  }
+
+  /**
+   * 变更时间
+   *
+   * @param new_map
+   * @return
+   */
+  //  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
+
+  /**
+   * 风险等级
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "高风险信息"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_mortgage_info.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //动产抵押
 case class company_mortgage_info() extends CompanyDynamicHandle {
@@ -26,8 +26,8 @@ case class company_mortgage_info() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""被担保债权数额:$new_map("amount")\n
-       |债务人履行债务期限：$new_map("term")\n""".stripMargin
+    s"""被担保债权数额:${new_map("amount")}\n
+       |债务人履行债务期限：${new_map("term")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_own_tax.scala

@@ -19,11 +19,9 @@ case class company_own_tax() extends CompanyDynamicHandle {
   /**
    * 变更内容
    *
-   * @param old_map
-   * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = new_map.toJson(Seq("tax_balance->欠税余额","tax_category->欠税税种","put_reason->列入经营异常目录原因","new_tax_balance->当前新发生的欠税余额","publish_date->发布日期","department->发布单位"))
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = new_map.toJson(Seq("tax_balance->欠税余额","tax_category->欠税税种","put_reason->列入经营异常目录原因","new_tax_balance->当前新发生的欠税余额","publish_date->发布日期","department->发布单位"))
 
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_public_announcement2_list.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/6
- * @Description TODO
+ * @Description
  */
 //公示催告
 case class company_public_announcement2_list()extends CompanyDynamicHandle {
@@ -26,10 +26,10 @@ case class company_public_announcement2_list()extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""票号:$new_map("bill_num")\n
-       |申请人：$new_map("cname")\n
-       |票面金额：$new_map("start_date")\n
-       |公告日期：$new_map("end_date")\n""".stripMargin
+    s"""票号:${new_map("bill_num")}\n
+       |申请人：${new_map("cname")}\n
+       |票面金额：${new_map("start_date")}\n
+       |公告日期：${new_map("end_date")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_punishment_info.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //行政处罚
 case class company_punishment_info() extends CompanyDynamicHandle {
@@ -26,9 +26,9 @@ case class company_punishment_info() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""决定日期:$new_map("decision_date")\n
-       |违法行为类型：$new_map("type")\n
-       |行政处罚内容：$new_map("content")\n""".stripMargin
+    s"""决定日期:${new_map("decision_date")}\n
+       |违法行为类型：${new_map("type")}\n
+       |行政处罚内容：${new_map("content")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_punishment_info_creditchina.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/4
- * @Description TODO
+ * @Description
  */
 //行政处罚-信用中国
 case class company_punishment_info_creditchina() extends CompanyDynamicHandle {
@@ -26,9 +26,9 @@ case class company_punishment_info_creditchina() extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""决定日期:$new_map("decision_date")\n
-       |违法行为类型：$new_map("type")\n
-       |行政处罚结果：$new_map("result")\n""".stripMargin
+    s"""决定日期:${new_map("decision_date")}\n
+       |违法行为类型：${new_map("type")}\n
+       |行政处罚结果：${new_map("result")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_send_announcement_list.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/12
- * @Description TODO
+ * @Description
  */
 //送达公告
 case class company_send_announcement_list()extends CompanyDynamicHandle {
@@ -26,8 +26,8 @@ case class company_send_announcement_list()extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""发布日期:$new_map("start_date")\n
-       |法院名称：$new_map("court")\n""".stripMargin
+    s"""发布日期:${new_map("start_date")}\n
+       |法院名称：${new_map("court")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_staff.scala

@@ -25,14 +25,7 @@ case class company_staff() extends CompanyDynamicHandle {
    * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq("hid","staff_type->职位"))
+  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq("hid", "staff_type"))
+
 
-  /**
-   * 风险等级
-   *
-   * @param old_map
-   * @param new_map
-   * @return
-   */
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "1"
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_stock_announcement.scala

@@ -5,7 +5,7 @@ import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
 /**
  * @Author yyn
  * @Date 2020/8/11
- * @Description TODO
+ * @Description
  */
 //企业公告
 case class company_stock_announcement()extends CompanyDynamicHandle {
@@ -26,8 +26,8 @@ case class company_stock_announcement()extends CompanyDynamicHandle {
    * @return
    */
   override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
-    s"""公告名称:$new_map("title")\n
-       |公告日期：$new_map("time")\n""".stripMargin
+    s"""公告名称:${new_map("title")}\n
+       |公告日期：${new_map("time")}\n""".stripMargin
   }
 
   /**

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_tax_contravention.scala

@@ -36,31 +36,29 @@ case class company_tax_contravention() extends CompanyDynamicHandle {
   /**
    * 变更内容
    *
-   * @param old_map
-   * @param new_map
    * @return
    */
-  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = {
-      new_map.toJson(Seq(
-        "taxpayer_name->纳税人名称"
-        , "taxpayer_number->纳税人识别号"
-        , "taxpayer_code->组织机构代码"
-        , "address->注册地址"
-        , "publish_time->发布日期"
-        , "case_type->案件性质"
-        , "department->所属税务机关"
-        , "case_info->主要违法事实"
-        , "legal_person_info->法定代表人或负责人"
-//        , s"${splitInfo(new_map("legal_person_info"))._2}->性别"
-//        , s"${splitInfo(new_map("legal_person_info"))._3}->证件名称"
-//        , s"${splitInfo(new_map("legal_person_info"))._4}->证件号码"
-        , "responsible_person_info->负有责任的财务负责人"
-//        , s"${splitInfo(new_map("legal_person_info"))._2}->性别"
-//        , s"${splitInfo(new_map("legal_person_info"))._3}->证件名称"
-//        , s"${splitInfo(new_map("legal_person_info"))._4}->证件号码"
-        , "responsible_department_info->负有直接责任的中介机构"
-      ))
-  }
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = {
+//      new_map.toJson(Seq(
+//        "taxpayer_name->纳税人名称"
+//        , "taxpayer_number->纳税人识别号"
+//        , "taxpayer_code->组织机构代码"
+//        , "address->注册地址"
+//        , "publish_time->发布日期"
+//        , "case_type->案件性质"
+//        , "department->所属税务机关"
+//        , "case_info->主要违法事实"
+//        , "legal_person_info->法定代表人或负责人"
+////        , s"${splitInfo(new_map("legal_person_info"))._2}->性别"
+////        , s"${splitInfo(new_map("legal_person_info"))._3}->证件名称"
+////        , s"${splitInfo(new_map("legal_person_info"))._4}->证件号码"
+//        , "responsible_person_info->负有责任的财务负责人"
+////        , s"${splitInfo(new_map("legal_person_info"))._2}->性别"
+////        , s"${splitInfo(new_map("legal_person_info"))._3}->证件名称"
+////        , s"${splitInfo(new_map("legal_person_info"))._4}->证件号码"
+//        , "responsible_department_info->负有直接责任的中介机构"
+//      ))
+//  }
 
   def splitInfo(s: String) = {
     if (StringUtils.isNotBlank(s) && s.replaceAll(":", ",").split(",").size == 4) {
@@ -71,5 +69,7 @@ case class company_tax_contravention() extends CompanyDynamicHandle {
     }
   }
 
-  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "警示信息"
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    "3"
+  }
 }

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_zxr_final_case.scala

@@ -0,0 +1,60 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.implicits.MapHelper._
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author: π
+ * @Date: 2020/8/11 17:39
+ * @Description: 终本案件
+ */
+case class company_zxr_final_case() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+
+    s"""
+       |案号：${new_map("case_no")}\n
+       |执行法院：${new_map("court_name")}\n
+       |立案日期：${new_map("case_create_time")}\n
+       |终本日期：${new_map("case_final_time")}
+       |""".stripMargin
+  }
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = new_map.toJson(Seq(
+//    "name->被执行人姓名"
+//    ,"identity_num->身份证号码/组织机构代码"
+//    ,"sex->性别"
+//    ,"case_no->案号"
+//    ,"court_name->执行法院"
+//    ,"case_create_time->立案时间"
+//    ,"case_final_time->终本日期"
+//    ,"exec_amount->执行标的（元）"
+//    ,"no_exec_amount->未履行金额"
+//  ))
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    "4"
+  }
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/company_zxr_restrict.scala

@@ -0,0 +1,51 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+
+/**
+ * @Author yyn
+ * @Date 2020/8/14
+ * @Description
+ */
+//限制消费令
+case class company_zxr_restrict()extends CompanyDynamicHandle {
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = new_map("case_no")
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String = null): String = {
+    s"""案号:${new_map("case_no")}\n
+       |限消令对象:${new_map("name")}\n
+       |立案日期:${new_map("case_create_time")}\n
+       |发布日期:${new_map("update_time")}\n""".stripMargin
+  }
+
+  /**
+   * 变更时间
+   *
+   * @param new_map
+   * @return
+   */
+//  override def get_change_time(new_map: Map[String, String]): String = new_map("biz_date")
+
+  /**
+   * 风险等级
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = "高风险"
+}

src/main/scala/com/winhc/bigdata/spark/jobs/dynamic/tables/wenshu_detail_combine.scala

@@ -0,0 +1,68 @@
+package com.winhc.bigdata.spark.jobs.dynamic.tables
+
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamicHandle
+import org.apache.commons.lang3.StringUtils
+
+/**
+ * @Author: π
+ * @Date: 2020/8/18
+ * @Description: 裁判文书
+ */
+case class wenshu_detail_combine() extends CompanyDynamicHandle {
+
+
+  /**
+   * 来源表的变更类型，insert or update
+   *
+   * @return
+   */
+  override def org_type(): Seq[String] = Seq("insert")
+
+  /**
+   * 信息描述
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+  override protected def get_rta_desc(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    val t1 = new_map("name_type")
+    var t2 = ""
+    if (StringUtils.isNotBlank(new_map("name_type"))) {
+      if ("y".equals(t1)) {
+        t2 = "原告"
+      } else {
+        t2 = "被告"
+      }
+    }
+    s"""
+       |案由：${new_map("case_reason_level3")}\n
+       |案号：${new_map("case_no")}\n
+       |诉讼身份：${t2}\n
+       |发布日期：${new_map("judge_date")}
+       |""".stripMargin
+  }
+
+  /**
+   * 变更内容
+   *
+   * @param old_map
+   * @param new_map
+   * @return
+   */
+//  override protected def get_change_content(old_map: Map[String, String], new_map: Map[String, String], cname: String): String = ""
+
+  override protected def get_info_risk_level(old_map: Map[String, String], new_map: Map[String, String]): String = {
+    val t1 = new_map("name_type")
+    var t2 = "2"
+    if (StringUtils.isNotBlank(new_map("name_type"))) {
+      if ("y".equals(t1)) {
+        t2 = "2"
+      } else {
+        t2 = "3"
+      }
+    }
+    t2
+  }
+
+}

src/main/scala/com/winhc/bigdata/spark/jobs/inc_company_equity_info.scala

@@ -30,12 +30,12 @@ object inc_company_equity_info {
       cleanup()
       val cols = getColumns(s"winhc_eci_dev.ads_company_equity_info").diff(Seq("ds", "rowkey", "id"))
 
-      val startPart = getLastPartitionsOrElse(s"winhc_eci_dev.inc_ads_company_equity_info", "0")
+      var startPart = getLastPartitionsOrElse(s"winhc_eci_dev.inc_ads_company_equity_info", "0")
       val endPart = getLastPartitionsOrElse(s"winhc_eci_dev.inc_ods_company_equity_info", BaseUtil.getYesterday())
 
       if (startPart.equals(endPart)) {
         println("start partition = end partition!")
-        sys.exit(-1)
+        startPart = getSecondLastPartitionOrElse(s"winhc_eci_dev.inc_ads_company_equity_info", "0")
       }
 
       sql(
@@ -172,6 +172,8 @@ object inc_company_equity_info {
         , "CONCAT_WS('_',cid,main_id)").syn()
       import com.winhc.bigdata.spark.implicits.DataFrame2HBaseHelper._
 
+      //todo 没有写出摘要
+
       val outFields = getColumns("winhc_eci_dev.inc_ads_company_equity_info").map(_.toUpperCase)
       sql(
         s"""

src/main/scala/com/winhc/bigdata/spark/jobs/increment/inc_phx_cid_ads.scala

@@ -6,13 +6,12 @@ import com.winhc.bigdata.spark.utils.{BaseUtil, LoggingUtils, SparkUtils}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.functions.col
 
-import scala.annotation.meta.getter
 import scala.collection.mutable
 
 /**
  * @Author yyn
  * @Date 2020/7/15
- * @Description TODO
+ * @Description
  */
 object inc_phx_cid_ads extends LoggingUtils {
   var config = mutable.Map(

src/main/scala/com/winhc/bigdata/spark/test/Justicase.scala

@@ -0,0 +1,68 @@
+package com.winhc.bigdata.spark.test
+
+import org.apache.spark.graphx._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.{SparkConf, SparkContext}
+import org.json4s.jackson.JsonMethods.parse
+import org.json4s.jackson.Serialization
+import org.json4s.{JValue, ShortTypeHints}
+
+object Justicase {
+  implicit val formats = Serialization.formats(ShortTypeHints(List()))
+  def getJsonValue(parseJson: JValue, key: String): String = {
+    (parseJson \ key).extractOrElse[String]("0")//后面以Edge构造图,故孤立的顶点都连上编号为0的顶点以形成边
+  }
+  def main(args: Array[String]): Unit = {
+    val sparkConf = new SparkConf()
+    sparkConf.setAppName("JusticaseTest")
+    sparkConf.setMaster("local[*]")
+    val sparkContext = new SparkContext(sparkConf)
+
+    // 文书或公告
+    val case_relations: RDD[String] = sparkContext.parallelize(
+      Seq("""{"case_no":"3","from":"2"}"""
+        ,"""{"case_no":"2","from":"1"}"""
+        ,"""{"case_no":"4","from":"3"}"""
+
+        ,"""{"case_no":"8","from":"7"}"""
+
+        ,"""{"case_no":"14","from":"13"}"""
+        ,"""{"case_no":"13","from":"12"}"""
+
+        ,"""{"case_no":"19"}"""
+
+        ,"""{"case_no":"28","from":"27"}"""
+        ,"""{"case_no":"27","from":"26"}"""
+        ,"""{"case_no":"26","from":"25"}"""
+        ,"""{"case_no":"29","from":"28"}"""
+        ,"""{"case_no":"30","from":"29"}"""
+        ,"""{"case_no":"31","from":"30"}"""
+        ,"""{"case_no":"55","from":"30"}"""
+      )
+    )
+
+    // 构造GraphX中Edge对象
+    val edges = case_relations.map(relation => {
+      val jv = parse(relation)
+
+      val from = getJsonValue(jv, "from").toLong
+      val to = getJsonValue(jv, "case_no").toLong
+      Edge(from, to, relation)
+    })
+
+    // 根据边构造图
+    val graph: Graph[String, String] = Graph.fromEdges(edges, defaultValue = "")
+
+    // 获取连通分量
+    val connetedGraph: Graph[VertexId, String] = graph.connectedComponents()
+
+    // 将同一连通分量中各个边聚合
+    val tripleGroup: RDD[(VertexId, Set[VertexId])] = connetedGraph.triplets.map(t => (t.srcAttr, Set(t.dstId)))
+      .reduceByKey(_ ++ _)
+
+    //逐一输出所有极大连通子图结果：起点Vertex,与其连通的Vertex集合。Vertex可以是案号形成的Long
+    tripleGroup.foreach(i=>{
+      println(i._1 + ": " + i._2.mkString(","))
+    })
+  }
+}

src/main/scala/com/winhc/bigdata/spark/test/TestChangeExtract.scala

@@ -0,0 +1,132 @@
+package com.winhc.bigdata.spark.test
+
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import com.winhc.bigdata.spark.config.EsConfig
+import com.winhc.bigdata.spark.jobs.chance.ChangeExtract.ChangeExtractHandle
+import com.winhc.bigdata.spark.utils.{SparkUtils, Watching}
+import org.apache.spark.sql.SparkSession
+
+import scala.collection.mutable
+
+/**
+ * @Description:
+ * @author π
+ * @date 2020/8/1914:38
+ */
+object TestChangeExtract {
+
+  private case class Args(project: String = "winhc_eci_dev"
+                          , tableName: String
+                          , primaryKey: String = "rowkey"
+                          , primaryFields: String
+                          , isCopy: Boolean = true)
+
+  private val startArgs = Seq(
+    Args(tableName = "company_tm", primaryFields = "status_new")
+    , Args(tableName = "company_patent_list", primaryFields = "lprs")
+    , Args(tableName = "company_copyright_works_list", primaryFields = "type")
+    , Args(tableName = "company_copyright_reg_list", primaryFields = "version")
+    , Args(tableName = "company_land_publicity", primaryFields = "title,location,use_for")
+    , Args(tableName = "company_land_announcement", primaryFields = "e_number,project_name")
+    , Args(tableName = "company_bid_list", primaryFields = "title")
+    , Args(tableName = "company_land_transfer", primaryFields = "num,location")
+    , Args(tableName = "company_employment", primaryFields = "title,cid,url_path")
+    , Args(tableName = "company_brief_cancel_announcement_result", primaryFields = "cid,main_id")
+    , Args(tableName = "company_env_punishment", primaryFields = "punish_number")
+    , Args(tableName = "company_icp", primaryFields = "domain")
+    , Args(tableName = "company_punishment_info", primaryFields = "punish_number")
+    , Args(tableName = "company_punishment_info_creditchina", primaryFields = "punish_number")
+    , Args(tableName = "bankruptcy_open_case", primaryFields = "case_no", isCopy = false) //破产重整
+    , Args(tableName = "company_public_announcement2_list", primaryFields = "applicant_cid,owner_cid,drawer_cid,gather_name_cid,bill_num") //公示催告
+    , Args(tableName = "company_mortgage_info", primaryFields = "reg_num") //动产抵押
+    , Args(tableName = "company_stock_announcement", primaryFields = "title") //企业公告
+    , Args(tableName = "company_check_info", primaryFields = "check_result") //抽查检查
+    , Args(tableName = "company_court_announcement_list", primaryFields = "content") //法院公告
+    , Args(tableName = "company_court_open_announcement_list", primaryFields = "case_reason") //开庭公告
+    , Args(tableName = "company_court_register_list", primaryFields = "area") //立案信息
+    , Args(tableName = "company_double_random_check_info", primaryFields = "check_plan_name") //双随机抽查
+    , Args(tableName = "company_judicial_sale_combine_list", primaryFields = "title") //司法拍卖
+    , Args(tableName = "company_tax_contravention", primaryFields = "case_type") //税收违法
+    , Args(tableName = "company_send_announcement_list", primaryFields = "title") //送达公告
+    , Args(tableName = "company_annual_report_out_guarantee", primaryFields = "id") //年报-对外担保
+    , Args(tableName = "company_zxr_restrict", primaryFields = "status") //限制消费令，发现最新状态
+
+    , Args(tableName = "company_zxr_final_case", primaryFields = "identity_num") //终本案件
+    , Args(tableName = "company_license_creditchina", primaryFields = "licence_content") //行政许可-信用中国
+    , Args(tableName = "company_license_entpub", primaryFields = "license_name") //行政许可-企业公示
+    , Args(tableName = "company_license", primaryFields = "license_name") //行政许可
+    , Args(tableName = "wenshu_detail_combine", primaryFields = "cname") //文书
+
+    , Args(tableName = "company_certificate", primaryFields = "type")
+    , Args(tableName = "company_abnormal_info", primaryFields = "remove_reason")
+
+    , Args(tableName = "company_own_tax", primaryFields = "tax_balance,tax_category,tax_num")
+
+    , Args(tableName = "company_equity_info", primaryKey = "id", primaryFields = "reg_number", isCopy = false)
+    , Args(tableName = "company_staff", primaryFields = "staff_type")
+    //公司名称,法人ID：人标识或公司标识,公司类型,注册地址,营业期限终止日期,经营范围,登记机关,企业状态                 ,注册资本,实收资本金额（单位：分）,注销日期,注销原因
+    , Args(tableName = "company", primaryKey = "cid", primaryFields = "name,legal_entity_id,company_org_type,reg_location,to_time,business_scope,reg_institute,reg_status,reg_capital,actual_capital_amount,cancel_date,cancel_reason")
+    , Args(tableName = "company_illegal_info", primaryFields = "remove_reason")
+    , Args(tableName = "company_finance", primaryFields = "round")
+    , Args(tableName = "company_dishonest_info", primaryFields = "case_no")
+  )
+
+  def main(args: Array[String]): Unit = {
+
+    val ds = args(0)
+    val project = "winhc_eci_dev"
+    val config = EsConfig.getEsConfigMap ++ mutable.Map(
+      "spark.hadoop.odps.project.name" -> project,
+      "spark.hadoop.odps.spark.local.partition.amt" -> "1000"
+    )
+    val spark = SparkUtils.InitEnv("ChangeExtract", config)
+    TestChangeExtract(spark, ds).calc()
+    spark.stop()
+  }
+
+  val seq = List(
+    "company_own_tax"
+    , "company_check_info"
+    , "company_court_announcement_list"
+    , "company_court_open_announcement_list"
+    , "company_court_register_list"
+    , "company_double_random_check_info"
+    , "company_judicial_sale_combine_list"
+    , "company_tax_contravention"
+    , "company_zxr_final_case"
+    , "company_license_creditchina"
+    , "company_license_entpub"
+    , "company_license"
+    , "wenshu_detail_combine"
+  )
+
+  case class TestChangeExtract(s: SparkSession,
+                               ds: String //表名（不加前后辍）
+                              ) extends Watching {
+
+    override protected val spark: SparkSession = s
+
+
+    def calc2(): Unit = {
+      startArgs.filter(s => seq.contains(s.tableName)).foreach(e => {
+        println("______________________________" + e.tableName + "___________________________________")
+        ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, ds, e.primaryFields.split(",")).calc(e.isCopy)
+      })
+    }
+
+    def calc(): Unit = {
+      val latch = new CountDownLatch(seq.length)
+      startArgs.filter(s => seq.contains(s.tableName)).foreach(e => {
+        asyncWatch(e.tableName, () => {
+          println("______________________________" + e.tableName + "___________________________________")
+          ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, ds, e.primaryFields.split(",")).calc(e.isCopy)
+          latch.countDown()
+        })
+      })
+      latch.await(60, TimeUnit.MINUTES)
+    }
+
+  }
+
+}

src/main/scala/com/winhc/bigdata/spark/test/TestCompanyDynamic.scala

@@ -0,0 +1,126 @@
+package com.winhc.bigdata.spark.test
+
+import java.util.concurrent.CountDownLatch
+
+import com.winhc.bigdata.spark.config.EsConfig
+import com.winhc.bigdata.spark.jobs.dynamic.CompanyDynamic.{CompanyDynamicUtil}
+import com.winhc.bigdata.spark.utils.{SparkUtils, Watching}
+import org.apache.spark.sql.SparkSession
+
+import scala.collection.mutable
+
+/**
+ * @Description: ${todo}
+ * @author π
+ * @date 2020/8/1918:08
+ */
+object TestCompanyDynamic {
+
+  private case class Args(project: String = "winhc_eci_dev"
+                          , tableName: String
+                          , bName: Int = 1)
+
+  private val startArgs = Seq(
+    Args(tableName = "company_abnormal_info", bName = 0)
+    , Args(tableName = "company_equity_info")
+    , Args(tableName = "company_staff", bName = 0)
+    , Args(tableName = "company", bName = 0)
+    , Args(tableName = "bankruptcy_open_case", bName = 1)
+    , Args(tableName = "company_illegal_info", bName = 0)
+    , Args(tableName = "company_land_publicity", bName = 1)
+    , Args(tableName = "company_employment", bName = 1)
+    , Args(tableName = "company_land_announcement", bName = 1)
+    , Args(tableName = "company_bid_list", bName = 1)
+    , Args(tableName = "company_land_transfer", bName = 1)
+    , Args(tableName = "company_env_punishment", bName = 1)
+    , Args(tableName = "company_punishment_info", bName = 1)
+    , Args(tableName = "company_punishment_info_creditchina", bName = 1)
+    , Args(tableName = "bankruptcy_open_case", bName = 1)
+    , Args(tableName = "company_public_announcement2_list", bName = 1)
+    , Args(tableName = "company_mortgage_info", bName = 1)
+    , Args(tableName = "company_stock_announcement", bName = 1)
+    , Args(tableName = "company_finance", bName = 1)
+    , Args(tableName = "company_dishonest_info", bName = 1)
+    , Args(tableName = "company_send_announcement_list", bName = 1)
+    , Args(tableName = "company_annual_report_out_guarantee", bName = 1)
+    , Args(tableName = "company_zxr_restrict", bName = 1)
+
+    , Args(tableName = "company_own_tax", bName = 1) //终本案件
+    , Args(tableName = "company_zxr_final_case", bName = 1) //终本案件
+    , Args(tableName = "company_license_creditchina", bName = 1) //行政许可-信用中国
+    , Args(tableName = "company_license_entpub", bName = 1) //行政许可-企业公示
+    , Args(tableName = "company_license", bName = 1) //行政许可
+    , Args(tableName = "company_check_info", bName = 1) //抽查检查
+    , Args(tableName = "company_court_announcement_list", bName = 2) //法院公告
+    , Args(tableName = "company_court_open_announcement_list", bName = 2) //开庭公告
+    , Args(tableName = "company_court_register_list", bName = 2) //立案信息
+    , Args(tableName = "company_double_random_check_info", bName = 1) //双随机抽查
+    , Args(tableName = "company_judicial_sale_combine_list", bName = 1) //司法拍卖
+    , Args(tableName = "company_tax_contravention", bName = 1) //税收违法
+    , Args(tableName = "wenshu_detail_combine", bName = 1) //裁判文书
+  )
+  val seq = List(
+      "company_own_tax"
+    , "company_check_info"
+    , "company_court_announcement_list"
+    , "company_court_open_announcement_list"
+    , "company_court_register_list"
+    , "company_double_random_check_info"
+    , "company_judicial_sale_combine_list"
+    , "company_tax_contravention"
+    , "company_zxr_final_case"
+    , "company_license_creditchina"
+    , "company_license_entpub"
+    , "company_license"
+    , "wenshu_detail_combine"
+  )
+
+  def main(args: Array[String]): Unit = {
+
+    val ds = args(0)
+    println(s"$ds")
+    val project = "winhc_eci_dev"
+
+    val config = EsConfig.getEsConfigMap ++ mutable.Map(
+      "spark.hadoop.odps.project.name" -> project,
+      "spark.hadoop.odps.spark.local.partition.amt" -> "1000"
+    )
+    val spark = SparkUtils.InitEnv("CompanyDynamic", config)
+    val s = spark.sparkContext.statusTracker
+      s.getExecutorInfos.map(x=>{
+      x.numRunningTasks()
+    })
+    TestCompanyDynamic(spark, project, ds).calc()
+    spark.stop()
+  }
+
+
+  case class TestCompanyDynamic(s: SparkSession,
+                                project: String, //表名（不加前后辍）
+                                ds: String //表名（不加前后辍）
+                               ) extends Watching {
+
+    override protected val spark: SparkSession = s
+
+    def calc2(): Unit = {
+      startArgs.filter(s => seq.contains(s.tableName)).foreach(e => {
+        println("______________________________" + e.tableName + "___________________________________")
+        CompanyDynamicUtil(spark, project, ds).calc(e.tableName, e.bName)
+      })
+    }
+
+    def calc(): Unit = {
+      val latch = new CountDownLatch(seq.length)
+      startArgs.filter(s => seq.contains(s.tableName)).foreach(e => {
+        asyncWatch(e.tableName, () => {
+          println("______________________________" + e.tableName + "___________________________________")
+          CompanyDynamicUtil(spark, project, ds).calc(e.tableName, e.bName)
+          latch.countDown()
+        })
+      })
+      latch.await()
+    }
+
+  }
+
+}

src/main/scala/com/winhc/bigdata/spark/utils/AsyncExtract.scala

@@ -0,0 +1,121 @@
+package com.winhc.bigdata.spark.utils
+
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import org.apache.commons.lang3.time.StopWatch
+import org.apache.spark.JobExecutionStatus
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+
+/**
+ * @Author: XuJiakai
+ * @Date: 2020/8/21 11:36
+ * @Description:
+ */
+object AsyncExtract {
+
+  def startAndWait(spark: SparkSession, seq: Seq[(String, () => Boolean)]): Unit = {
+    start(seq)
+    wait(spark)
+  }
+
+  def start(seq: Seq[(String, () => Boolean)]): Unit = {
+    AsyncExtract(seq).start()
+  }
+
+  def wait(spark: SparkSession): Unit = {
+    val tracker = spark.sparkContext.statusTracker
+
+    val ints = tracker.getJobIdsForGroup(null)
+    println(ints.mkString(","))
+    val failed = ints
+      .map(i => tracker.getJobInfo(i))
+      .map(o => o.orNull)
+      .filter(_ != null)
+      .map(i => i.status()).exists(i => JobExecutionStatus.FAILED.equals(i))
+    if (failed) {
+      println("There are failed jobs !!!")
+      sys.exit(-999)
+    }
+  }
+}
+
+case class AsyncExtract(seq: Seq[(String, () => Boolean)]) extends Logging {
+
+  def start(): Unit = {
+    val latch = new CountDownLatch(seq.length)
+    seq.foreach(e => {
+      asyncWatch(e._1, () => {
+        try {
+          println("______________________________" + e._1 + "___________________________________")
+          val res = e._2()
+          println(s"---${e._1}---------$res-------------")
+        } catch {
+          case ex: Exception => {
+            ex.printStackTrace()
+          }
+        } finally {
+          latch.countDown()
+        }
+      })
+    })
+    latch.await()
+  }
+
+  private def asyncWatch[T](name: String, f: () => Unit): Unit = {
+    val t = new Thread(new Runnable {
+      override def run(): Unit = {
+        watch(name, f)
+      }
+    })
+    t.setName(name)
+    t.start()
+  }
+
+  private def watch[T](name: String, fun: () => T): T = {
+    val stopWatch = new StopWatch
+    stopWatch.start()
+    println(
+      s"""
+         |
+         |----------------------- submit [action=$name] begin... -----------------------
+         |""".stripMargin)
+    logInfo(
+      s"""
+         |
+         |----------------------- submit [action=$name] begin... -----------------------
+       """.stripMargin
+    )
+
+    val r = fun()
+    stopWatch.stop()
+    val totalSeconds = stopWatch.getTime(TimeUnit.SECONDS)
+
+    println(
+      s"""
+         |-------------- submit [action=$name] end, cost ${toTimeStr(totalSeconds)} -------------
+         |
+         |""".stripMargin)
+    logInfo(
+      s"""
+         |-------------- submit [action=$name] end, cost ${toTimeStr(totalSeconds)} -------------
+         |
+       """.stripMargin
+    )
+    r
+  }
+
+  private def toTimeStr(totalSeconds: Long): String = {
+    val hours = TimeUnit.SECONDS.toHours(totalSeconds)
+    val minutes = TimeUnit.SECONDS.toMinutes(totalSeconds - hours * 3600)
+    val seconds = totalSeconds - hours * 3600 - minutes * 60
+    val timeStr = if (hours > 0) {
+      s"${hours}h${minutes}m${seconds}s"
+    } else if (minutes > 0) {
+      s"${minutes}m${seconds}s"
+    } else {
+      s"${seconds}s"
+    }
+    timeStr
+  }
+}

src/main/scala/com/winhc/bigdata/spark/utils/CompanyForCidUtils.scala

@@ -12,7 +12,7 @@ import scala.annotation.meta.getter
  * cid转换
  */
 
-case class CompanyForCidUtils(s: SparkSession, space: String, sourceTable: String, cols: Seq[String]) extends LoggingUtils  with CompanyMapping{
+case class CompanyForCidUtils(s: SparkSession, space: String, sourceTable: String, cols: Seq[String]) extends LoggingUtils with CompanyMapping {
   @(transient@getter) val spark: SparkSession = s
 
   val rowKeyMapping =
@@ -35,15 +35,16 @@ case class CompanyForCidUtils(s: SparkSession, space: String, sourceTable: Strin
     //rowkey前缀匹配
     val rowKeyPre = rowKeyMapping.getOrElse(sourceTable,"new_cid")
 
-   val ddl =  spark.table(odsTable).schema.filter(s=>{!"ds".equals(s.name)}).map(s=>{
-
+    val ddl = spark.table(odsTable).schema.filter(s => {
+      !"ds".equals(s.name)
+    }).map(s => {
       val name = s.name
       val dataType = s.dataType
       s"$name ${DataTypeUtils.getDataType(dataType)} COMMENT '${s.getComment().getOrElse("")}'\n"
     }).mkString(",")
 
 
-    sql(
+    println(
       s"""
          |CREATE TABLE IF NOT EXISTS ${adsTable}
          |(
@@ -54,6 +55,10 @@ case class CompanyForCidUtils(s: SparkSession, space: String, sourceTable: Strin
          |COMMENT 'TABLE COMMENT'
          |PARTITIONED BY (ds STRING COMMENT '分区')
          |""".stripMargin)
+    if (!spark.catalog.tableExists(adsTable)) {
+      return
+    }
+
 
     //替换字段
     sql(
@@ -78,12 +83,12 @@ case class CompanyForCidUtils(s: SparkSession, space: String, sourceTable: Strin
          |        ) d
          |WHERE   num =1  AND cols is not null AND trim(cols) <> ''
          |""".stripMargin)
-//      .createOrReplaceTempView(s"t2")
+    //      .createOrReplaceTempView(s"t2")
 
-//    sql(s"select rowkey,new_cid,${columns.mkString(",")} from t2").show(10)
+    //    sql(s"select rowkey,new_cid,${columns.mkString(",")} from t2").show(10)
 
     //写表
-//    sql(s"insert into table ${adsTable} partition (ds=${ds}) select rowkey,new_cid,${columns.mkString(",")} from t2")
+    //    sql(s"insert into table ${adsTable} partition (ds=${ds}) select rowkey,new_cid,${columns.mkString(",")} from t2")
 
     println(s"${this.getClass.getSimpleName} calc end! " + new Date().toString)
   }

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncSummary.scala

@@ -24,6 +24,9 @@ case class CompanyIncSummary(s: SparkSession,
   @(transient@getter) val spark: SparkSession = s
   private val f_bytes: Array[Byte] = Bytes.toBytes("F")
   private val name_bytes: Array[Byte] = Bytes.toBytes(tableName.toUpperCase)
+  val updateTimeMapping = Map(
+    "wenshu_detail_combine" -> "update_date" //文书排序时间
+  )
 
   def calc(): Unit = {
     val ads_table = s"${project}.ads_$tableName" //存量ads表
@@ -57,7 +60,7 @@ case class CompanyIncSummary(s: SparkSession,
          |SELECT  ${new_cols.map(getCastCols(_, "")).mkString(",")}
          |FROM    (
          |            SELECT  tmp.*
-         |                    ,ROW_NUMBER() OVER(PARTITION BY ${dupliCols.mkString(",")} ORDER BY update_time DESC ) c
+         |                    ,ROW_NUMBER() OVER(PARTITION BY ${dupliCols.mkString(",")} ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC ) c
          |            FROM    (
          |                        SELECT  ${new_cols.map(getCastCols(_, "org_tab.")).mkString(",")}
          |                        FROM    (
@@ -87,19 +90,19 @@ case class CompanyIncSummary(s: SparkSession,
          |SELECT  ${cidField} as cid
          |        ,COUNT(1) as num
          |FROM    inc_tmp_view
+         |where $cidField is not null
          |GROUP BY $cidField
+         |having count(1) >0
          |""".stripMargin)
       .select(Seq("cid", "num").map(column => col(column).cast("string")): _*)
       .rdd
-      .filter(r => {
-        r.get(1) != null && !"0".equals(r.getString(1))
-      }).map(row => {
-      val id = row.getString(0)
-      val num = row.getString(1)
-      val put = new Put(Bytes.toBytes(id))
-      put.addColumn(f_bytes, name_bytes, Bytes.toBytes(num))
-      (new ImmutableBytesWritable, put)
-    }).filter(_ != null).saveAsHadoopDataset(jobConf)
+      .map(row => {
+        val id = row.getString(0)
+        val num = row.getString(1)
+        val put = new Put(Bytes.toBytes(id))
+        put.addColumn(f_bytes, name_bytes, Bytes.toBytes(num))
+        (new ImmutableBytesWritable, put)
+      }).filter(_ != null).saveAsHadoopDataset(jobConf)
   }
 
   private def getCastCols(name: String, pre: String): String = {

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidUtils.scala

@@ -16,12 +16,16 @@ case class CompanyIncrForCidUtils(s: SparkSession,
                                   project: String, //表所在工程名
                                   tableName: String, //表名（不加前后辍）
                                   dupliCols: Seq[String] // 去重列
-                                 ) extends LoggingUtils with CompanyMapping{
+                                 ) extends LoggingUtils with CompanyMapping {
   @(transient@getter) val spark: SparkSession = s
  //主键字段
   val rowKeyMapping =
     Map("company_double_random_check_result_info" -> "new_cid,main_id" //双随机抽查-结果公示
     )
+//排序时间
+  val updateTimeMapping =
+    Map("wenshu_detail_combine" -> "update_date" //文书排序时间
+    )
 
   def calc(): Unit = {
     println(s"${this.getClass.getSimpleName} calc start! " + new Date().toString)
@@ -33,6 +37,39 @@ case class CompanyIncrForCidUtils(s: SparkSession,
     val inc_ods_company_tb = s"${project}.inc_ods_$tableName" //增量ods表
     val inc_ads_company_tb = s"${project}.inc_ads_$tableName" //增量ads表
 
+
+
+    //table字段
+    val columns: Seq[String] = spark.table(ads_company_tb).schema.map(_.name).filter(s => {
+      !s.equals("ds") && !s.equals("cid") && !s.equals("new_cid") && !s.equals("rowkey")
+    })
+
+    val colsSet = columns.toSet
+
+    val ddl = spark.table(inc_ods_company_tb).schema.filter(s => colsSet.contains(s.name)).map(s => {
+      val name = s.name
+      val dataType = s.dataType
+      s"$name ${DataTypeUtils.getDataType(dataType)} COMMENT '${s.getComment().getOrElse("")}'\n"
+    }).mkString(",")
+
+    println(
+      s"""
+         |CREATE TABLE IF NOT EXISTS ${inc_ads_company_tb}
+         |(
+         |    rowkey  STRING COMMENT 'FIELD'
+         |    ,flag STRING COMMENT 'FIELD'
+         |    ,new_cid STRING COMMENT 'FIELD'
+         |    ,cid STRING COMMENT 'FIELD'
+         |    ,$ddl
+         |)
+         |COMMENT 'TABLE COMMENT'
+         |PARTITIONED BY (ds STRING COMMENT '分区')
+         |""".stripMargin)
+
+    if (!spark.catalog.tableExists(inc_ads_company_tb)) {
+      return
+    }
+
     //存量表ads最新分区
     val remainDs = BaseUtil.getPartion(ads_company_tb, spark)
 
@@ -58,7 +95,13 @@ case class CompanyIncrForCidUtils(s: SparkSession,
     //增量ods和增量ads最后一个分区相等,跳出
     if (lastDsIncOds.equals(lastDsIncAds)) {
       println("inc_ods equals inc_ads ds ,please delete last ds !!!")
-      runDs = lastDsIncOds
+      //runDs = lastDsIncOds
+      val l1 = sql(s"show partitions $inc_ads_company_tb").collect.toList.map(_.getString(0).split("=")(1)).sorted
+      if (l1.size > 1) {
+        runDs = BaseUtil.atDaysAfter(1, l1(l1.size - 2))
+      }else{
+        runDs = firstDsIncOds
+      }
       //sys.exit(-1)
     }
 
@@ -72,14 +115,12 @@ case class CompanyIncrForCidUtils(s: SparkSession,
          |firstDsIncOds:$firstDsIncOds
          |""".stripMargin)
 
-    //table字段
-    val columns: Seq[String] = spark.table(ads_company_tb).schema.map(_.name).filter(s => {
-      !s.equals("ds") && !s.equals("cid") && !s.equals("new_cid") && !s.equals("rowkey")
-    })
-
     //rowkey前缀匹配
     val rowKeyPre = rowKeyMapping.getOrElse(tableName,"new_cid")
 
+    //排序时间
+    val update_time = updateTimeMapping.getOrElse(tableName,"update_time")
+
     sql(
       s"""
          |SELECT  cid,current_cid as new_cid
@@ -105,7 +146,7 @@ case class CompanyIncrForCidUtils(s: SparkSession,
          |                    ,new_cid
          |                    ,cid
          |                    ,${columns.mkString(",")}
-         |                    ,ROW_NUMBER() OVER (PARTITION BY cleanup(CONCAT_WS('',${dupliCols.mkString(",")})) ORDER BY update_time DESC ) num
+         |                    ,ROW_NUMBER() OVER (PARTITION BY cleanup(CONCAT_WS('',${dupliCols.mkString(",")})) ORDER BY $update_time DESC ) num
          |            FROM    (
          |                        SELECT  "0" AS flag
          |                                ,a.new_cid

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidWithoutMD5Utils.scala

@@ -17,8 +17,9 @@ import scala.collection.mutable
 case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
                                             project: String, //表所在工程名
                                             tableName: String, //表名（不加前后辍）
-                                            dupliCols: Seq[String] // 去重列
-                                 ) extends LoggingUtils with CompanyMapping{
+                                            dupliCols: Seq[String], // 去重列
+                                            updateCol: String = "update_time" //ROW_NUMBER窗口函数的ORDER BY字段，默认（可以不传参数）为update_time
+                                           ) extends LoggingUtils with CompanyMapping {
   @(transient@getter) val spark: SparkSession = s
 
   def calc(): Unit = {
@@ -100,7 +101,7 @@ case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
          |                    ,new_cid
          |                    ,cid
          |                    ,${columns.mkString(",")}
-         |                    ,ROW_NUMBER() OVER (PARTITION BY cleanup(CONCAT_WS('',${dupliCols.mkString(",")})) ORDER BY update_time DESC ) num
+         |                    ,ROW_NUMBER() OVER (PARTITION BY cleanup(CONCAT_WS('',${dupliCols.mkString(",")})) ORDER BY NVL($updateCol,update_time) DESC ) num
          |            FROM    (
          |                        SELECT  "0" AS flag
          |                                ,a.new_cid
@@ -150,11 +151,12 @@ case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
     println(s"${this.getClass.getSimpleName} calc end! " + new Date().toString)
   }
 }
+
 object CompanyIncrForCidWithoutMD5Utils {
 
   def main(args: Array[String]): Unit = {
 
-    val Array(project, tableName, dupliCols, flag) = args
+    val Array(project, tableName, dupliCols, flag, _) = args
     println(
       s"""
          |project: $project
@@ -162,17 +164,22 @@ object CompanyIncrForCidWithoutMD5Utils {
          |dupliCols: $dupliCols
          |flag: $flag
          |""".stripMargin)
-    if (args.length != 4) {
-      println("请输入 project:项目, tableName:表名, dupliCols:去重字段, flag:标识 !!!")
+    if (args.length < 4) {
+      println("请输入 project:项目, tableName:表名, dupliCols:去重字段, flag:标识, [updateCol:排序列]!!!")
       sys.exit(-1)
     }
+    //ROW_NUMBER窗口函数的ORDER BY字段，默认（可以不传参数）为update_time
+    var updateCol: String = "update_time"
+    if (args.length == 5 && !args(4).endsWith("}")) {
+      updateCol = args(4)
+    }
     val config = mutable.Map(
       "spark.hadoop.odps.project.name" -> "winhc_eci_dev",
       "spark.hadoop.odps.spark.local.partition.amt" -> "1"
     )
     val spark: SparkSession = SparkUtils.InitEnv(this.getClass.getSimpleName, config)
     flag match {
-      case "cid" => CompanyIncrForCidWithoutMD5Utils(spark, project, tableName, (dupliCols.split(",").toSeq)).calc()
+      case "cid" => CompanyIncrForCidWithoutMD5Utils(spark, project, tableName, (dupliCols.split(",").toSeq), updateCol).calc()
     }
     spark.stop()
   }

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidsUtils.scala

@@ -93,7 +93,13 @@ case class CompanyIncrForCidsUtils(s: SparkSession,
     //增量ods和增量ads最后一个分区相等,跳出
     if (lastDsIncOds.equals(lastDsIncAds)) {
       println("inc_ods equals inc_ads ds ,please delete last ds !!!")
-      runDs = lastDsIncOds
+      //runDs = lastDsIncOds
+      val l1 = sql(s"show partitions $inc_ads_company_tb").collect.toList.map(_.getString(0).split("=")(1)).sorted
+      if (l1.size > 1) {
+        runDs = BaseUtil.atDaysAfter(1, l1(l1.size - 2))
+      }else{
+        runDs = firstDsIncOds
+      }
       //sys.exit(-1)
     }
 

src/main/scala/com/winhc/bigdata/spark/utils/DateUtils.scala

@@ -2,6 +2,8 @@ package com.winhc.bigdata.spark.utils
 
 import java.text.SimpleDateFormat
 
+import org.apache.commons.lang3.StringUtils
+
 /**
  * @Author: XuJiakai
  * @Date: 2020/8/10 11:19
@@ -10,23 +12,64 @@ import java.text.SimpleDateFormat
 object DateUtils {
 
   def toUnixTimestamp(date: String, pattern: String = "yyyy-MM-dd HH:mm:ss"): Long = {
-    val fm = new SimpleDateFormat(pattern)
+    var p = "yyyy-MM-dd HH:mm:ss"
+    if (date.length == 10) {
+      p = "yyyy-MM-dd"
+    }
+    val fm = new SimpleDateFormat(p)
     fm.parse(date).getTime / 1000
   }
 
+  /**
+   * 获取第一个不为空的字符串
+   *
+   * @param date
+   * @return
+   */
+  def getNotNullStr(date: String*): String = date.filter(_ != null).head
+
+  /**
+   * 获取最小的一个日期，如果有异常情况则反回第一个日期
+   *
+   * @param date
+   * @return
+   */
+  def getMinDate(date: String*): String = {
+    val notNullDate = date.filter(_ != null)
+    if (notNullDate.map(_.length).distinct.length != 1) {
+      return notNullDate.head
+    }
+    var minDate: String = null
+    for (i <- 0 until notNullDate.length - 1) {
+      minDate = getMaxDateBy2(notNullDate(i), notNullDate(i + 1), true)
+    }
+    minDate
+  }
 
   /**
    * 获取最大的一个日期，如果有异常情况则反回第一个日期
    *
-   * @param date1
-   * @param date2
+   * @param date
    */
-  def getMaxDate(date1: String, date2: String): String = {
+  def getMaxDate(date: String*): String = {
+    val notNullDate = date.filter(_ != null)
+    if (notNullDate.map(_.length).distinct.length != 1) {
+      return notNullDate.head
+    }
+    var maxDate: String = null
+    for (i <- 0 until notNullDate.length - 1) {
+      maxDate = getMaxDateBy2(notNullDate(i), notNullDate(i + 1))
+    }
+    maxDate
+  }
+
+  private def getMaxDateBy2(date1: String, date2: String, reverse: Boolean = false): String = {
     try {
       if (date1.length != date2.length) {
         return date1
       }
-      date1.compareTo(date2) match {
+      val re = if (reverse) -1 else 1
+      date1.compareTo(date2) * re match {
         case -1 => date2
         case 1 => date1
         case _ => date1
@@ -36,8 +79,44 @@ object DateUtils {
     }
   }
 
+  def isLegalDate(date: String): Boolean = {
+    try {
+      var p = "yyyy-MM-dd HH:mm:ss"
+      if (date.length == 10) {
+        p = "yyyy-MM-dd"
+      }
+      val fm = new SimpleDateFormat(p)
+      val date1 = fm.parse(date)
+      date.equals(fm.format(date1))
+    } catch {
+      case e: Exception => false
+    }
+  }
+
+  /**
+   * 判断bizdate是否合法,否则用update_time替代
+   *
+   * @param date1
+   * @param date2
+   * @return
+   */
+  def getBizDate(date1: String, date2: String): String = {
+    if (StringUtils.isBlank(date1)) {
+      return date2
+    }
+    val s = date1.replaceAll("年", "-").replaceAll("月", "-")
+      .replaceAll("日", "").replaceAll("/", "-")
+
+    if (s.length >= 10 && isLegalDate(s.substring(0, 10))) {
+      s.substring(0, 10)
+    } else {
+      date2
+    }
+  }
+
   def main(args: Array[String]): Unit = {
-    println(getMaxDate("2003-10-12 10:00:00", "2003-11-12 00:00:01"))
+    println(isLegalDate("2003-10-12 10:00:00"))
+    //    println(getNotNullStr(null, "2003-10-12 10:00:00", null, "2003-11-12 00:00:02"))
   }
 
 }

src/main/scala/com/winhc/bigdata/spark/utils/LoggingUtils.scala

@@ -121,6 +121,15 @@ trait LoggingUtils extends Logging {
     sql(sql_s).collect.toList.map(_.getString(0).split("=")(1)).seq
   }
 
+  def getSecondLastPartitionOrElse(t: String, default: String): String = {
+    val ps = getPartitions(t)
+    if (ps.length >= 2) {
+      ps(ps.length - 2)
+    } else {
+      default
+    }
+  }
+
   def getLastPartitionsOrElse(t: String, default: String): String = {
     val ps = getPartitions(t)
     if (ps.nonEmpty) {