|
@@ -19,7 +19,7 @@ case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
|
|
tableName: String, //表名(不加前后辍)
|
|
tableName: String, //表名(不加前后辍)
|
|
dupliCols: Seq[String], // 去重列
|
|
dupliCols: Seq[String], // 去重列
|
|
updateCol: String = "update_time" //ROW_NUMBER窗口函数的ORDER BY字段,默认(可以不传参数)为update_time
|
|
updateCol: String = "update_time" //ROW_NUMBER窗口函数的ORDER BY字段,默认(可以不传参数)为update_time
|
|
- ) extends LoggingUtils with CompanyMapping{
|
|
|
|
|
|
+ ) extends LoggingUtils with CompanyMapping {
|
|
@(transient@getter) val spark: SparkSession = s
|
|
@(transient@getter) val spark: SparkSession = s
|
|
|
|
|
|
def calc(): Unit = {
|
|
def calc(): Unit = {
|
|
@@ -151,6 +151,7 @@ case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
|
|
println(s"${this.getClass.getSimpleName} calc end! " + new Date().toString)
|
|
println(s"${this.getClass.getSimpleName} calc end! " + new Date().toString)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+
|
|
object CompanyIncrForCidWithoutMD5Utils {
|
|
object CompanyIncrForCidWithoutMD5Utils {
|
|
|
|
|
|
def main(args: Array[String]): Unit = {
|
|
def main(args: Array[String]): Unit = {
|
|
@@ -169,7 +170,7 @@ object CompanyIncrForCidWithoutMD5Utils {
|
|
}
|
|
}
|
|
//ROW_NUMBER窗口函数的ORDER BY字段,默认(可以不传参数)为update_time
|
|
//ROW_NUMBER窗口函数的ORDER BY字段,默认(可以不传参数)为update_time
|
|
var updateCol: String = "update_time"
|
|
var updateCol: String = "update_time"
|
|
- if(args.length == 5){
|
|
|
|
|
|
+ if (args.length == 5 && !args(4).endsWith("}")) {
|
|
updateCol = args(4)
|
|
updateCol = args(4)
|
|
}
|
|
}
|
|
val config = mutable.Map(
|
|
val config = mutable.Map(
|