|
@@ -3,12 +3,19 @@ package com.winhc.bigdata.spark.ng.dynamic
|
|
import com.winhc.bigdata.spark.ng.dynamic.utils.CompanyDynamicUtils
|
|
import com.winhc.bigdata.spark.ng.dynamic.utils.CompanyDynamicUtils
|
|
import org.apache.spark.internal.Logging
|
|
import org.apache.spark.internal.Logging
|
|
|
|
|
|
|
|
+import scala.annotation.meta.{getter, setter}
|
|
|
|
+
|
|
/**
|
|
/**
|
|
* @author: XuJiakai
|
|
* @author: XuJiakai
|
|
* @date: 2021/6/23 10:35
|
|
* @date: 2021/6/23 10:35
|
|
*/
|
|
*/
|
|
trait NgCompanyDynamicHandle extends Serializable with Logging {
|
|
trait NgCompanyDynamicHandle extends Serializable with Logging {
|
|
|
|
|
|
|
|
+ @getter
|
|
|
|
+ @setter
|
|
|
|
+ protected val is_inc: Boolean //false 为存量
|
|
|
|
+
|
|
|
|
+
|
|
def filter: (String, String, Seq[String], Map[String, String], Map[String, String]) => Boolean = CompanyDynamicUtils.default_filter
|
|
def filter: (String, String, Seq[String], Map[String, String], Map[String, String]) => Boolean = CompanyDynamicUtils.default_filter
|
|
|
|
|
|
def flat_map: (ChangeExtract) => Seq[CompanyDynamicRecord]
|
|
def flat_map: (ChangeExtract) => Seq[CompanyDynamicRecord]
|
|
@@ -41,10 +48,11 @@ trait NgCompanyDynamicHandle extends Serializable with Logging {
|
|
val biz_date = change_extract.biz_date
|
|
val biz_date = change_extract.biz_date
|
|
val rowkey = change_extract.rowkey
|
|
val rowkey = change_extract.rowkey
|
|
val update_time = change_extract.update_time
|
|
val update_time = change_extract.update_time
|
|
|
|
+ val date = if(is_inc) update_time else biz_date
|
|
|
|
|
|
|
|
|
|
CompanyDynamicRecord(
|
|
CompanyDynamicRecord(
|
|
- id = CompanyDynamicUtils.generateId(rowkey, biz_date, tn, update_field)
|
|
|
|
|
|
+ id = CompanyDynamicUtils.generateId(rowkey, date, tn, update_field)
|
|
, association_entity_info = association_entity_info
|
|
, association_entity_info = association_entity_info
|
|
, rowkey = rowkey
|
|
, rowkey = rowkey
|
|
, tn = tn
|
|
, tn = tn
|
|
@@ -53,9 +61,11 @@ trait NgCompanyDynamicHandle extends Serializable with Logging {
|
|
, dynamic_info = dynamic_info
|
|
, dynamic_info = dynamic_info
|
|
, agg_detail_text = null
|
|
, agg_detail_text = null
|
|
, agg_detail_rowkey = null
|
|
, agg_detail_rowkey = null
|
|
- , change_time = biz_date
|
|
|
|
|
|
+ , biz_time = biz_date
|
|
|
|
+ , dynamic_time = date
|
|
, update_time = update_time
|
|
, update_time = update_time
|
|
, create_time = create_time
|
|
, create_time = create_time
|
|
)
|
|
)
|
|
}
|
|
}
|
|
|
|
+
|
|
}
|
|
}
|