立案信息到司法案件

src/main/scala/com/winhc/bigdata/spark/jobs/JudicialCaseRelationPreCourtRegister.scala

@@ -109,176 +109,6 @@ case class JudicialCaseRelationPre2(s: SparkSession, project: String
          |""".stripMargin).show(10, false)
   }
 
-  def calc(): Unit = {
-    prepareFunctions(spark)
-    map_2_json()
-    registerCourtRank()
-    //预处理数据
-    //precalc()
-    val cols = Seq("flag", "date", "detail_id")
-
-    val t1 = s"$project.inc_ads_company_court_register"
-    val t2 = s"$project.ads_judicial_case_relation_pre"
-    val t1_ds = BaseUtil.getPartion(t1, spark)
-    val t2_ds = BaseUtil.getPartion(t2, "wenshu", spark)
-
-    //替换司法案件id
-    sql(
-      s"""
-         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.tmp_xf_judicial_case_relation_replace_2 partition (ds = '$t1_ds')
-         |SELECT  COALESCE(b.judicase_id,a.new_judicase_id) judicase_id
-         |        ,a.flag
-         |        ,a.title
-         |        ,a.case_type
-         |        ,a.case_reason
-         |        ,a.case_no
-         |        ,a.court_name
-         |        ,a.case_stage
-         |        ,case_label(a.flag) lable
-         |        ,map_2_json(${getStrToMap(cols)}) as detail
-         |        ,a.yg_name
-         |        ,a.bg_name
-         |        ,a.date
-         |        ,a.detail_id
-         |        ,a.case_amt
-         |FROM    (
-         |  select
-         |  *,md5(CLEANUP(case_no)) as new_judicase_id
-         |  from $project.ads_judicial_case_relation_pre
-         |  where ds= '$t2_ds' and tn <> 'wenshu'
-         |) a
-         |LEFT JOIN (
-         |  select case_no,max(judicase_id) judicase_id
-         |  from $project.ads_judicial_case_relation_pre
-         |  where ds = '$t2_ds' and tn ='wenshu' and  length(trim(case_no)) > 0
-         |  group by case_no
-         |) b
-         |ON  CLEANUP(a.case_no) = CLEANUP(b.case_no)
-         |union all
-         |SELECT   judicase_id
-         |        ,flag
-         |        ,title
-         |        ,case_type
-         |        ,case_reason
-         |        ,case_no
-         |        ,court_name
-         |        ,case_stage
-         |        ,case_label(flag) lable
-         |        ,map_2_json(${getStrToMap(cols)}) as detail
-         |        ,yg_name
-         |        ,bg_name
-         |        ,date
-         |        ,detail_id
-         |        ,case_amt
-         |from $project.ads_judicial_case_relation_pre
-         |where ds = '$t2_ds' and tn ='wenshu' and length(trim(case_no)) > 0
-         |""".stripMargin).show(10, false)
-
-    //找出增量数据
-    sql(
-      s"""
-         |INSERT OVERWRITE TABLE $project.tmp_xf_judicial_case_incr_mapping
-         |SELECT  coalesce(a.judicase_id,b.judicase_id)judicase_id
-         |        ,CASE WHEN a.judicase_id IS NULL THEN 1 ELSE 0 END
-         |FROM    (
-         |            SELECT  judicase_id
-         |                    ,md5(concat_ws('',judicase_id, sort(concat_ws('\001',collect_set(case_no))))) r1
-         |            FROM    $project.tmp_xf_judicial_case_relation_replace_2
-         |            WHERE   ds = '$t1_ds'
-         |            GROUP BY judicase_id
-         |        ) a
-         |FULL JOIN (
-         |              SELECT  judicase_id
-         |                      ,md5(concat_ws('',judicase_id, sort(concat_ws('\001',collect_set(case_no))))) r2
-         |              FROM    $project.tmp_xf_judicial_case_relation_replace_2
-         |              WHERE   ds < '$t1_ds'
-         |              GROUP BY judicase_id
-         |          ) b
-         |ON  r1 = r2
-         |WHERE   r1 IS NULL OR r2 IS NULL
-         |""".stripMargin)
 
-    sql(
-      s"""
-         |SELECT  court_name,court_level(court_name) court_level
-         |FROM    $project.tmp_xf_judicial_case_relation_replace_2
-         |WHERE   ds = '$t1_ds'
-         |""".stripMargin).show(200, false)
-
-    //司法案件主表
-    sql(
-      s"""
-         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.tmp_xf_judicial_case_relation_r1
-         |SELECT  judicase_id
-         |        ,max(first_title) title
-         |        ,max(case_type) case_type
-         |        ,max(case_reason) case_reason
-         |        ,concat_ws(',',collect_set(case_no)) case_no
-         |        ,concat_ws(',',collect_set(court_name)) court_name
-         |        ,last_stage(concat_ws(' ',collect_set(case_stage))) case_stage
-         |        ,concat_ws(',',max(case_type),collect_set(lable)) lable
-         |        ,concat('[',concat_ws(',',collect_set(detail)),']') detail
-         |        ,max(first_yg_name) AS yg_name
-         |        ,max(first_bg_name) AS bg_name
-         |        ,max(case_amt) AS case_amt
-         |        ,max(date) AS date
-         |        ,trim_black(concat_ws(',',collect_set(court_level))) court_level
-         |        ,max(deleted) deleted
-         |FROM    (
-         |        SELECT  a.* ,first_value(yg_name) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_yg_name
-         |                ,first_value(bg_name) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_bg_name
-         |                ,first_value(title) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_title
-         |                ,b.deleted
-         |        FROM    (
-         |                   SELECT  *,court_level(court_name) court_level
-         |                   FROM    $project.tmp_xf_judicial_case_relation_replace_2
-         |                   WHERE   ds = '$t1_ds'
-         |                ) a JOIN
-         |                (
-         |                   select *
-         |                   from $project.tmp_xf_judicial_case_incr_mapping
-         |                ) b on a.judicase_id = b.judicase_id
-         |        )
-         |GROUP BY judicase_id
-         |""".stripMargin).show(20, false)
-
-    //明细表
-    sql(
-      s"""
-         |INSERT ${if (isWindows) "INTO" else "OVERWRITE"} TABLE $project.tmp_xf_judicial_case_relation_r2
-         |SELECT  md5(concat_ws('',judicase_id,CLEANUP(case_no),case_stage)) id
-         |        ,judicase_id
-         |        ,max(first_title) title
-         |        ,max(case_type) case_type
-         |        ,max(case_reason) case_reason
-         |        ,case_no
-         |        ,max(court_name) court_name
-         |        ,case_stage
-         |        ,concat_ws(',',max(case_type),collect_set(lable)) lable
-         |        ,concat('[',concat_ws(',',collect_set(detail)),']') detail
-         |        ,max(first_yg_name) yg_name
-         |        ,max(first_bg_name) bg_name
-         |        ,max(deleted) deleted
-         |FROM    (
-         |        SELECT  a.* ,first_value(yg_name) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_yg_name
-         |                ,first_value(bg_name) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_bg_name
-         |                ,first_value(title) OVER (PARTITION BY a.judicase_id ORDER BY date ASC ) AS first_title
-         |                ,b.deleted
-         |        FROM    (
-         |                   SELECT  *
-         |                   FROM    $project.tmp_xf_judicial_case_relation_replace_2
-         |                   WHERE   ds = '$t1_ds'
-         |                )a JOIN
-         |                (
-         |                   select *
-         |                   from $project.tmp_xf_judicial_case_incr_mapping
-         |                )b on a.judicase_id = b.judicase_id
-         |)
-         |GROUP BY judicase_id
-         |         ,case_no
-         |         ,case_stage
-         |""".stripMargin).show(10, false)
-
-  }
 
 }