bugfix,兼容重跑逻辑

src/main/resources/env.yaml

@@ -1,5 +1,5 @@
 profile:
-  activate: dev
+  activate: prod
 
 ---
 env:

src/main/scala/com/winhc/bigdata/spark/model/CompanyEmploymentScore.scala

@@ -86,7 +86,7 @@ case class CompanyEmploymentScore(s: SparkSession, sourceTable: String, tableVie
     val cnt1 = r.getAs[Long]("cnt1")
     val cnt2 = r.getAs[Long]("cnt2")
     flag match {
-      case "302" => employmentScore(id, cid, cnt1,cnt2, kind, prpject)
+      case "208" => employmentScore(id, cid, cnt1,cnt2, kind, prpject)
     }
   }
 
@@ -119,7 +119,7 @@ object CompanyEmploymentScore {
       "spark.hadoop.odps.spark.local.partition.amt" -> "10"
     )
     val spark: SparkSession = SparkUtils.InitEnv(this.getClass.getSimpleName, config)
-    CompanyEmploymentScore(spark, "company_employment","", "302", "start_date", "经营情况", "招聘", "0", "winhc_eci_dev").calc()
+    CompanyEmploymentScore(spark, "company_employment","", "208", "start_date", "经营情况", "招聘", "0", "winhc_eci_dev").calc()
     spark.stop()
   }
 }

src/main/scala/com/winhc/bigdata/spark/model/CompanyIntellectualsScore.scala

@@ -79,7 +79,7 @@ object CompanyIntellectualsScore {
       return
     }
     //招聘
-    if (flag.equals("302")) {
+    if (flag.equals("208")) {
       new CompanyEmploymentScore(spark, sourceTable, tableView, flag, time, kind, project, "1", namespace).calc()
       return
     }

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidUtils.scala

@@ -51,6 +51,13 @@ case class CompanyIncrForCidUtils(s: SparkSession,
 
     val cols_md5 = dupliCols.filter(!_.equals("new_cid"))
 
+    //增量ods和增量ads最后一个分区相等,跳出
+    if (lastDsIncOds.equals(lastDsIncAds)) {
+      println("inc_ods equals inc_ads ds ,please delete last ds !!!")
+      runDs = lastDsIncOds
+      //sys.exit(-1)
+    }
+
     println(
       s"""
          |cols_md5:$cols_md5
@@ -61,12 +68,6 @@ case class CompanyIncrForCidUtils(s: SparkSession,
          |firstDsIncOds:$firstDsIncOds
          |""".stripMargin)
 
-    //增量ods和增量ads最后一个分区相等,跳出
-    if (lastDsIncOds.equals(lastDsIncAds)) {
-      println("inc_ods equals inc_ads ds ,please delete last ds !!!")
-      sys.exit(-1)
-    }
-
     //table字段
     val columns: Seq[String] = spark.table(ads_company_tb).schema.map(_.name).filter(s => {
       !s.equals("ds") && !s.equals("cid") && !s.equals("new_cid") && !s.equals("rowkey")
@@ -113,7 +114,7 @@ case class CompanyIncrForCidUtils(s: SparkSession,
          |                                    SELECT  new_cid AS cid
          |                                            ,${columns.mkString(",")}
          |                                    FROM    ${inc_ads_company_tb}
-         |                                    WHERE   ds >= ${runDs}
+         |                                    WHERE   ds > ${remainDs}
          |                                ) b
          |                        ON      a.cid = b.cid
          |                        UNION ALL

src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidsUtils.scala

@@ -59,6 +59,13 @@ case class CompanyIncrForCidsUtils(s: SparkSession,
 
     val cols_md5 = dupliCols.filter(!_.equals("new_cid"))
 
+    //增量ods和增量ads最后一个分区相等,跳出
+    if (lastDsIncOds.equals(lastDsIncAds)) {
+      println("inc_ods equals inc_ads ds ,please delete last ds !!!")
+      runDs = lastDsIncOds
+      //sys.exit(-1)
+    }
+
     println(
       s"""
          |cols_md5:$cols_md5
@@ -69,12 +76,6 @@ case class CompanyIncrForCidsUtils(s: SparkSession,
          |firstDsIncOds:$firstDsIncOds
          |""".stripMargin)
 
-    //增量ods和增量ads最后一个分区相等,跳出
-    if (lastDsIncOds.equals(lastDsIncAds)) {
-      println("inc_ods equals inc_ads ds ,please delete last ds !!!")
-      sys.exit(-1)
-    }
-
     //table字段
     val columns: Seq[String] = spark.table(ads_company_tb).schema.map(_.name).filter(s => {
       !s.equals("ds") && !s.equals("cid") && !s.equals("new_cid") && !s.equals("rowkey") && !s.equals("cids") && !s.equals("new_cids")
@@ -143,7 +144,7 @@ case class CompanyIncrForCidsUtils(s: SparkSession,
          |                                                SELECT  new_cid AS cid
          |                                                        ,${sublistTableFieldName.mkString(",")}
          |                                                FROM    ${inc_ads_company_tb_list}
-         |                                                WHERE   ds >= ${runDs}
+         |                                                WHERE   ds > ${remainDs}
          |                                                UNION ALL
          |                                                SELECT  new_cid AS cid
          |                                                        ,${sublistTableFieldName.mkString(",")}