表名修改,name去重

src/main/scala/com/winhc/bigdata/spark/jobs/judicial/JudicialCaseRelationPreNew.scala

@@ -147,7 +147,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
          |        ,case_amt
          |FROM    (
          |            SELECT  *
-         |            FROM $project.xjk_ads_judicial_case_relation1_tmp
+         |            FROM $project.ads_judicial_case_relation_graph
          |        ) a
          |RIGHT JOIN (
          |            SELECT *,md5(cleanup(case_no)) as new_judicase_id
@@ -305,9 +305,9 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
       t1_ds = BaseUtil.getPartion(t1, spark)
     }
     //司法案件id交换表
-    val t3 = "tmp_xf_judicial_case_relation_replace"
+    val t3 = "ads_judicial_case_relation_replace"
     val second_ds = getSecondLastPartitionOrElse(t3, "0")
-    val t4 = "tmp_xf_judicial_case_incr_mapping"
+    val t4 = "ads_judicial_case_incr_mapping"
     println(s"calc ds: $t2_ds, par ds : $t1_ds, second_ds : $second_ds")
 
     //替换司法案件id
@@ -404,7 +404,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
     //司法案件主表
     sql(
       s"""
-         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci.tmp_xf_judicial_case_relation_r1
+         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci.ads_judicial_case_relation_r1
          |SELECT  judicase_id
          |        ,max(first_title) title
          |        ,max(case_type) case_type
@@ -441,7 +441,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
     //明细表
     sql(
       s"""
-         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci.tmp_xf_judicial_case_relation_r2
+         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci.ads_judicial_case_relation_r2
          |SELECT  md5(concat_ws('',judicase_id,CLEANUP(case_no),case_stage)) id
          |        ,judicase_id
          |        ,max(first_title) title

src/main/scala/com/winhc/bigdata/spark/utils/BaseUtil.scala

@@ -131,7 +131,7 @@ object BaseUtil {
           .replaceAll("：", ",")
           .replaceAll("\\s+", ",").split(",")
 
-      val list = arr.filter(_.length > 1).toList
+      val list = arr.toSet.filter(_.length > 1).toList
       if (list.nonEmpty) {
         val sb = new StringBuilder
         for (a <- list) {
@@ -226,7 +226,7 @@ object BaseUtil {
   }
 
   def title(ygname: String, bgname: String, reason: String): String = {
-     Seq(ygname,bgname,reason).filter(s=>StringUtils.isNotBlank(s)).mkString(",")
+     Seq(replaceChar(ygname),replaceChar(bgname),reason).filter(s=>StringUtils.isNotBlank(s)).mkString(",")
   }
 
   def trimBlack(s: String): String = {