Merge remote-tracking branch 'origin/master'

src/main/scala/com/winhc/bigdata/spark/ng/credit_punishment/CreditPunishmentCaseAgg.scala

@@ -113,6 +113,8 @@ case class CreditPunishmentCaseAgg(s: SparkSession
            |                    )
            |        )
            |WHERE   num = 1
+           |AND     court_name is not null
+           |AND     trim(court_name) <> ''
            |""".stripMargin)
         .createTempView(tmp_view)
     } else {
@@ -144,6 +146,8 @@ case class CreditPunishmentCaseAgg(s: SparkSession
            |                    )
            |        )
            |WHERE   num = 1
+           |AND     court_name is not null
+           |AND     trim(court_name) <> ''
            |""".stripMargin)
         .createTempView(tmp_view)
     }
@@ -264,14 +268,12 @@ case class CreditPunishmentCaseAgg(s: SparkSession
        |        ,cast(mm['deleted'] as bigint) AS deleted
        |        ,mm['all_rowkey'] AS all_rowkey
        |FROM    (
-       |            SELECT  name
-       |                    ,court_name
-       |                    ,case_no
+       |            SELECT  collect_set(name)[0] as name
+       |                    ,collect_set(court_name)[0] as court_name
+       |                    ,collect_set(case_no)[0] as case_no
        |                    ,credit_punishment_case_agg(rowkey,tn,keyno,card_num,case_create_time,deleted,detail_data) AS mm
        |            FROM    $org_tab
-       |            GROUP BY name
-       |                     ,court_name
-       |                     ,case_no
+       |            GROUP BY cleanup(concat_ws('',name,court_name,case_no))
        |        )
        |WHERE   name IS NOT NULL and trim(name) <> ''
        |""".stripMargin

src/main/scala/com/winhc/bigdata/spark/ng/credit_punishment/CreditPunishmentDataExtraction.scala

@@ -249,14 +249,16 @@ case class CreditPunishmentDataExtraction(s: SparkSession
          |SELECT xjk_rowkey as rowkey
          |       ,tn as tn
          |       ,xjk_keyno as keyno
-         |       ,xjk_name as name
+         |       ,trim(xjk_name) as name
          |       ,xjk_card_num as card_num
-         |       ,xjk_court_name as court_name
+         |       ,trim(xjk_court_name) as court_name
          |       ,case_no_replace(xjk_case_no) as case_no
          |       ,xjk_case_create_time as case_create_time
          |       ,xjk_deleted as deleted
          |       ,detail_data
          |FROM   credit_punishment_tmp_view
+         |WHERE  xjk_court_name is not null
+         |AND    trim(xjk_court_name) <> ''
          |""".stripMargin)
   }