Forráskód Böngészése

fix: base_company_mapping修复bug

许家凯 4 éve
szülő
commit
e9c8021b8a

+ 1 - 2
src/main/scala/com/winhc/bigdata/spark/jobs/CompanyBaseMapping.scala

@@ -104,7 +104,7 @@ case class CompanyBaseMapping(s: SparkSession,
          |        ,t1.cname AS cname
          |        ,COALESCE(t2.new_cid,t1.new_cid) AS new_cid
          |        ,COALESCE(t2.update_time,t1.update_time) AS update_time
-         |        ,${other_cols.map(f=>s"t2.$f").mkString(",")}
+         |        ,${other_cols.map(f=>s"t1.$f").mkString(",")}
          |FROM    (
          |            SELECT  ${getColumns(target_tab).diff(Seq("ds")).mkString(",")}
          |            FROM    (
@@ -131,7 +131,6 @@ case class CompanyBaseMapping(s: SparkSession,
          |              SELECT  cid
          |                      ,current_cid AS new_cid
          |                      ,now() AS update_time
-         |                      ,${other_cols.mkString(",")}
          |              FROM    winhc_eci_dev.inc_ods_company
          |              WHERE   ds = '$ds'
          |              AND     cid IS NOT NULL

+ 2 - 5
src/main/scala/com/winhc/bigdata/spark/jobs/CompanyNameMappingPro.scala

@@ -1,26 +1,23 @@
 package com.winhc.bigdata.spark.jobs
 
-import com.aliyun.odps.utils.StringUtils
 import com.winhc.bigdata.spark.config.HBaseConfig
 import com.winhc.bigdata.spark.utils.{CompanyNameMappingUtil, SparkUtils}
-import org.apache.hadoop.hbase.client.{Get, Put, Table}
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.TableName
 import org.apache.hadoop.hbase.spark.HBaseContext
 import org.apache.hadoop.hbase.spark.HBaseRDDFunctions._
 import org.apache.hadoop.hbase.util.Bytes
-import org.apache.hadoop.hbase.{CellUtil, TableName}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.types.{StringType, StructField, StructType}
 
 import scala.collection.mutable
-import scala.collection.mutable.Set
 
 /**
  * @Author: XuJiakai
  * @Date: 2020/5/26 10:41
  * @Description: 公司名称映射关系计算
  */
+@deprecated
 object CompanyNameMappingPro extends Logging {
   val f_bytes: Array[Byte] = Bytes.toBytes("f")
   val cname_bytes: Array[Byte] = Bytes.toBytes("name")

+ 1 - 0
src/main/scala/com/winhc/bigdata/spark/jobs/CompanyNameMappingPro_stage_01.scala

@@ -15,6 +15,7 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
  * @Date: 2020/5/26 10:41
  * @Description: 公司名称映射关系计算  只获取最新的公司id
  */
+@deprecated
 object CompanyNameMappingPro_stage_01 extends Logging {
   val f_bytes: Array[Byte] = Bytes.toBytes("f")
   val cname_bytes: Array[Byte] = Bytes.toBytes("name")