优化了计算逻辑

src/main/scala/com/winhc/bigdata/spark/jobs/judicial/JudicialCaseRelationPreNew.scala

@@ -310,7 +310,8 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
     var t1_ds = ds
     if (StringUtils.isBlank(ds)) {
       t2_ds = BaseUtil.getPartion(t2, "wenshu", spark)
-      t1_ds = BaseUtil.getPartion(t1, spark)
+      //t1_ds = BaseUtil.getPartion(t1, spark)
+      t1_ds = t2_ds
     }
 
     val t3 = "ads_judicial_case_relation_replace" //司法案件id交换表
@@ -507,7 +508,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
       s"""
          |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci_dev.ads_judicial_case_relation_r1
          |SELECT
-         |    judicase_id ,
+         |    x.judicase_id ,
          |    title       ,
          |    case_type   ,
          |    case_reason ,
@@ -521,42 +522,40 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
          |    case_amt    ,
          |    date        ,
          |    court_level ,
-         |    deleted     ,
+         |    y.deleted     ,
          |    cids
          |FROM
          |(
          |SELECT  judicase_id
-         |        ,max(first_title) title
+         |        ,max(title) title
          |        ,concat_ws(',',collect_set(case_type)) case_type
          |        ,case_reason(case_reason,date,flag) case_reason
          |        ,concat_ws(',',collect_set(case_no)) case_no
          |        ,concat_ws(',',collect_set(court_name)) court_name
          |        ,last_stage(concat_ws(' ',collect_set(case_stage))) case_stage
          |        ,trim_black(concat_ws(',',max(case_type),collect_set(lable))) lable
-         |        -- ,concat('[',concat_ws(',',collect_set(detail)),']') detail
          |        ,null as detail
          |        ,max(case_amt) AS case_amt
          |        ,max(date) AS date
          |        ,trim_black(concat_ws(',',collect_set(court_level))) court_level
-         |        ,max(deleted) deleted
          |        ,concat_ws(',',collect_set(cids)) cids
          |        ,name_aggs(yg_name,bg_name,flag,date) name_aggs
          |FROM    (
          |        SELECT  a.*
-         |                ,first_value(title) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_title
-         |                ,b.deleted
          |        FROM    (
-         |                   SELECT  *,court_level(court_name) court_level
+         |                   SELECT  judicase_id,flag,title,case_type,case_reason,case_no,court_name,case_stage,lable,yg_name,bg_name,date,case_amt,cids
+         |                   ,court_level(court_name) court_level
          |                   FROM    $project.$t6
          |                   WHERE   ds >= '$second_ds'
-         |                ) a JOIN
-         |                (
-         |                   select *
-         |                   from $project.$t4
-         |                ) b on a.judicase_id = b.judicase_id
+         |                ) a
          |        )
          |GROUP BY judicase_id
-         |)
+         |)x
+         |JOIN
+         |(
+         |   select *
+         |   from $project.$t4
+         |) y on x.judicase_id = y.judicase_id
          |""".stripMargin).show(20, false)
 
     //明细表
@@ -565,7 +564,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
          |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE winhc_eci_dev.ads_judicial_case_relation_r2
          |SELECT
          |    id    ,
-         |    judicase_id ,
+         |    x.judicase_id ,
          |    title       ,
          |    case_type   ,
          |    case_reason ,
@@ -577,7 +576,7 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
          |    name_aggs['yg_name'] yg_name,
          |    name_aggs['bg_name'] bg_name,
          |    last_date   ,
-         |    deleted
+         |    y.deleted
          |FROM
          |(
          |SELECT  md5(concat_ws('',judicase_id,CLEANUP(case_no))) id
@@ -591,26 +590,25 @@ case class JudicialCaseRelationPreNew(s: SparkSession, project: String, ds: Stri
          |        ,trim_black(concat_ws(',',max(case_type),collect_set(lable))) lable
          |        ,concat('[',concat_ws(',',collect_set(detail)),']') detail
          |        ,max(last_date) last_date
-         |        ,max(deleted) deleted
          |        ,name_aggs(yg_name,bg_name,flag,date) name_aggs
          |FROM    (
          |        SELECT  a.*
          |                ,first_value(title) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_title
          |                ,first_value(date) OVER (PARTITION BY a.judicase_id ORDER BY date DESC ) AS last_date
-         |                ,b.deleted
          |        FROM    (
          |                   SELECT  *
          |                   FROM    $project.$t6
          |                   WHERE   ds >= '$second_ds' AND length(lable) > 0
-         |                )a JOIN
-         |                (
-         |                   select *
-         |                   from $project.$t4
-         |                )b on a.judicase_id = b.judicase_id
+         |                )a
          |)
          |GROUP BY judicase_id
          |         ,case_no
-         |)
+         |) x
+         |JOIN
+         |(
+         |   select *
+         |   from $project.$t4
+         |) y on x.judicase_id = y.judicase_id
          |""".stripMargin).show(10, false)
 
   }