|
@@ -57,13 +57,12 @@ case class CompanyIncSummary(s: SparkSession,
|
|
|
println("cols not equals!")
|
|
|
println(new_cols.sorted)
|
|
|
}
|
|
|
-
|
|
|
sql(
|
|
|
s"""
|
|
|
|SELECT ${new_cols.map(getCastCols(_, "")).mkString(",")}
|
|
|
|FROM (
|
|
|
| SELECT tmp.*
|
|
|
- | ,ROW_NUMBER() OVER(PARTITION BY ${dupliCols.mkString(",")} ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC ) c
|
|
|
+ | ,ROW_NUMBER() OVER(PARTITION BY CLEANUP(concat_ws('',${dupliCols.mkString(",")}))) ORDER BY ${updateTimeMapping.getOrElse(tableName, "update_time")} DESC ) c
|
|
|
| FROM (
|
|
|
| SELECT ${new_cols.map(getCastCols(_, "org_tab.")).mkString(",")}
|
|
|
| FROM (
|