债权表优化

src/main/scala/com/winhc/bigdata/spark/jobs/judicial/JudicialCaseRelationDebtorRelation.scala

@@ -101,6 +101,50 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
       !s.equals("ds")
     })
 
+    //标记历史和非历史债权
+    sql(
+      s"""
+         |SELECT r2 as rowkey,label,status,name
+         |from (
+         |SELECT *,ROW_NUMBER() OVER(PARTITION BY rowkey,LABEL ORDER BY update_time DESC ) num
+         |from (
+         |SELECT rowkey ,'3' LABEL,status,update_time,name,rowkey r2   from $project.inc_ads_company_dishonest_info where ds > '0'
+         |UNION ALL
+         |SELECT rowkey ,'3' LABEL,status,update_time,name,rowkey r2   from $project.ads_company_dishonest_info where ds > '0'
+         |UNION ALL
+         |SELECT rowkey ,'7' LABEL,status,update_time,cname name,split(rowkey,'_')[1] r2   from $project.inc_ads_company_zxr_list where ds > '0'
+         |UNION ALL
+         |SELECT rowkey ,'7' LABEL,status,update_time,cname name,split(rowkey,'_')[1] r2 from $project.ads_company_zxr_list where ds > '0'
+         |UNION ALL
+         |SELECT rowkey ,'5' LABEL,status,update_time,name,rowkey r2   from $project.inc_ads_company_zxr_restrict where ds > '0'
+         |UNION ALL
+         |SELECT rowkey ,'5' LABEL,status,update_time,name,rowkey r2  from $project.ads_company_zxr_restrict where ds > '0'
+         | )
+         |)
+         |where num = 1 and cast(status as string) = '0'
+         |""".stripMargin).createOrReplaceTempView("t1")
+//
+    sql(
+      s"""
+         |insert ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.tmp_xf_case_status_mapping
+         |SELECT   a.judicase_id,MIN(b.status) status,concat_ws(',',collect_set(b.name)) bg_name
+         |FROM    (
+         |            SELECT  detail_id
+         |                    ,LABle
+         |                    ,judicase_id
+         |                    ,flag
+         |            FROM    $project.ads_judicial_case_relation_replace_cids
+         |            WHERE   ds = '$t1_ds'
+         |        ) a
+         |JOIN    (
+         |            SELECT  *
+         |            FROM    t1
+         |        ) b
+         |ON      a.detail_id = b.rowkey
+         |AND     a.flag = b.LABEL
+         |GROUP by a.judicase_id
+         |""".stripMargin)
+
     sql(
       s"""
          |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.ads_judicial_case_relation_r1
@@ -115,11 +159,11 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
          |    lable       ,
          |    detail      ,
          |    yg_name,
-         |    bg_name,
+         |    y.bg_name as bg_name,
          |    case_amt    ,
          |    date        ,
          |    court_level ,
-         |    0 as deleted     ,
+         |    y.status as deleted     ,
          |    cids
          |FROM
          |(
@@ -138,27 +182,37 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
          |        ,concat_ws(',',collect_set(cids)) cids
          |        ,concat_ws(',',collect_set(yg_name)) yg_name
          |        ,concat_ws(',',collect_set(bg_name)) bg_name
+         |        -- ,min(status) status
          |FROM    (
-         |        SELECT  *
+         |        SELECT  a.*
          |        FROM    (
          |                   SELECT  judicase_id,flag,title,case_type,case_reason,case_no,court_name,case_stage,lable,yg_name,bg_name,date,case_amt,cids,detail_id
          |                   ,court_level(court_name) court_level
          |                   FROM    $project.$t6
          |                   WHERE   ds = '$t1_ds'
-         |                )
+         |                )a
+         |                -- JOIN
+         |                -- (
+         |                -- select judicase_id,status from $project.tmp_xf_case_status_mapping
+         |                -- )b on a.judicase_id = b.judicase_id
          |        )
          |GROUP BY judicase_id
          |)x
+         |join
+         |(
+         |  select judicase_id,status,bg_name from $project.tmp_xf_case_status_mapping
+         |)y on x.judicase_id = y.judicase_id
          |""".stripMargin).show(20, false)
 
     sql(
       s"""
          |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.tmp_xf_yg_bg_name
-         |SELECT md5(concat_ws('',yg_name,bg_name)) id,yg_name,bg_name
+         |SELECT md5(concat_ws('',yg_name,bg_name)) id,yg_name,bg_name,min(deleted) deleted
          |FROM (
          |SELECT
          |yg_name_x as yg_name
          |,bg_name_x as bg_name
+         |,deleted
          |FROM winhc_eci_dev.ads_judicial_case_relation_r1
          |LATERAL VIEW explode(split(yg_name,',')) a AS yg_name_x
          |LATERAL VIEW explode(split(bg_name,',')) b AS bg_name_x
@@ -272,15 +326,18 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
          |d.reg_capital bg_reg_capital,
          |d.phones bg_phones,
          |d.emails bg_emails,
-         |0 as deleted
+         |c.deleted as deleted
          |FROM    (
          |            SELECT  a.id
          |                    ,a.yg_name
+         |                    ,a.deleted
          |                    ,b.*
          |            FROM    (
          |                        SELECT  id
          |                                ,yg_name
+         |                                ,deleted
          |                        FROM    $project.tmp_xf_yg_bg_name
+         |                        WHERE   deleted  = '0'
          |                    ) a
          |            JOIN    (
          |                        SELECT  *
@@ -299,6 +356,7 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
          |                        SELECT  id
          |                                ,bg_name
          |                        FROM    $project.tmp_xf_yg_bg_name
+         |                        WHERE   deleted  = '0'
          |                    ) a
          |            JOIN    (
          |                        SELECT  *
@@ -311,21 +369,21 @@ case class JudicialCaseRelationDebtorRelation(s: SparkSession, project: String,
          |        )d
          |ON      c.id = d.id
          |)e
-         |JOIN
-         |(
-         |  SELECT  cid
-         |  FROM    (
-         |              SELECT  cid,rowkey,
-         |                      max(deleted) f
-         |              FROM    $project.$deadbeat_company
-         |              WHERE   ds > '0'
-         |              AND     tn <> 'company_zxr_final_case'
-         |              GROUP by cid,rowkey
-         |          )
-         |  WHERE   f = '0'
-         |  GROUP by cid
-         |)f
-         |on e.bg_cid = f.cid
+         |-- JOIN
+         |-- (
+         |--   SELECT  cid
+         |--   FROM    (
+         |--               SELECT  cid,rowkey,
+         |--                       max(deleted) f
+         |--               FROM    $project.$deadbeat_company
+         |--               WHERE   ds > '0'
+         |--               AND     tn <> 'company_zxr_final_case'
+         |--               GROUP by cid,rowkey
+         |--           )
+         |--   WHERE   f = '0'
+         |--   GROUP by cid
+         |-- )f
+         |-- on e.bg_cid = f.cid
          |where e.num = 1
          |)
          |where cnt > 1