Bladeren bron

fix: 重跑数据分区设置

许家凯 3 jaren geleden
bovenliggende
commit
4fcda8c11e

+ 1 - 1
src/main/scala/com/winhc/bigdata/spark/ng/jobs/CompanySummaryNg.scala

@@ -357,7 +357,7 @@ object CompanySummaryNg {
       "spark.hadoop.odps.spark.local.partition.amt" -> "10000"
     )
 
-    val spark = SparkUtils.InitEnv(getClass.getSimpleName, config)
+    val spark = SparkUtils.InitEnv(getClass.getSimpleName + ":" + args(0), config)
 
     if (args.length == 2 && !args(1).endsWith("}")) {
       run(spark, args(0), args(1))

+ 2 - 2
src/main/scala/com/winhc/bigdata/spark/ng/jobs/inc_company_ng.scala

@@ -55,9 +55,9 @@ case class inc_company_ng(s: SparkSession,
 
   def inc(): Unit = {
     val inc_ods_ds = getLastPartitionsOrElse(inc_ods_tab, getLastPartitionsOrElse(ods_tab, null))
-    var ads_ds = getLastPartitionsOrElse(ads_tab, null)
+    var ads_ds = getLastPartitionsOrElse(inc_ads_tab, null)
     if (inc_ods_ds.equals(ads_ds)) {
-      ads_ds = getSecondLastPartitionOrElse(ads_tab, null)
+      ads_ds = getSecondLastPartitionOrElse(inc_ads_tab, null)
       if (ads_ds == null) {
         all()
         return