剔除法人id监控

src/main/scala/com/winhc/bigdata/spark/ng/relation/inc_company_relation_v2.scala

@@ -38,6 +38,9 @@ case class inc_company_relation_v2(s: SparkSession,
   val inc_ads_relation_staff = "winhc_ng.inc_ads_relation_staff"
   val inc_ads_relation_legal_entity = "winhc_ng.inc_ads_relation_legal_entity"
 
+  //val change_field = " change_fields LIKE '%update_time%' OR "
+  val change_field = ""
+
   def register_fun(): Unit = {
     prepareFunctions(spark)
 
@@ -94,6 +97,7 @@ case class inc_company_relation_v2(s: SparkSession,
          |""".stripMargin)
 
     //增量法人表(新增,移除法人)
+    // 法人id监控 剔除 -- OR change_fields LIKE '%legal_entity_id%'
     sql(
       s"""
          |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $inc_ads_company_legal_entity PARTITION(ds= '$ds')
@@ -108,7 +112,7 @@ case class inc_company_relation_v2(s: SparkSession,
          |        ,new_data['deleted'] AS deleted
          |FROM    $ads_change_extract
          |WHERE   ds = '$ds' AND     tn = 'company'
-         |AND     (change_fields LIKE '%legal_entity_name%' or change_fields is null or change_fields LIKE '%deleted%' OR change_fields LIKE '%legal_entity_id%')
+         |AND     ($change_field change_fields LIKE '%legal_entity_name%' or change_fields is null or change_fields LIKE '%deleted%')
          |AND     length(trim(new_data['legal_entity_name'])) > 0
          |UNION ALL
          |SELECT  CONCAT_WS('_',company_id,hash(old_data['legal_entity_name'])) AS rowkey
@@ -315,7 +319,7 @@ case class inc_company_relation_v2(s: SparkSession,
          |         SELECT rowkey
          |         FROM $ads_change_extract
          |         WHERE   ds = '$ds' AND tn = 'company_holder'
-         |         AND    (change_fields LIKE '%percent%' OR change_fields is null OR change_fields LIKE '%deleted%')
+         |         AND    ($change_field change_fields LIKE '%percent%' OR change_fields is null OR change_fields LIKE '%deleted%')
          |         UNION ALL
          |         SELECT rowkey
          |         FROM incr_human_rowkey
@@ -336,7 +340,7 @@ case class inc_company_relation_v2(s: SparkSession,
          |    SELECT rowkey
          |    FROM $ads_change_extract
          |    WHERE   ds = '$ds' AND tn = 'company_holder'
-         |    AND    (change_fields LIKE '%percent%' OR change_fields is null OR change_fields LIKE '%deleted%' OR change_fields LIKE '%holder_id%')
+         |    AND    ($change_field change_fields LIKE '%percent%' OR change_fields is null OR change_fields LIKE '%deleted%' OR change_fields LIKE '%holder_id%')
          |   ) b
          |ON    a.rowkey = b.rowkey
          |)
@@ -371,7 +375,7 @@ case class inc_company_relation_v2(s: SparkSession,
          |       SELECT rowkey
          |         FROM $ads_change_extract
          |         WHERE   ds = '$ds' AND tn = 'company_staff'
-         |       AND    (change_fields LIKE '%staff_type%' OR change_fields is null OR change_fields LIKE '%deleted%')
+         |       AND    ($change_field change_fields LIKE '%staff_type%' OR change_fields is null OR change_fields LIKE '%deleted%')
          |       UNION ALL
          |       SELECT rowkey
          |         FROM incr_human_rowkey