Explorar o código

执行人和终本,人的增量,以及司法案件预处理

lyb %!s(int64=4) %!d(string=hai) anos
pai
achega
5749b8a1cd

+ 2 - 2
src/main/scala/com/winhc/bigdata/spark/etl/PersonIncrSync.scala

@@ -106,7 +106,7 @@ case class PersonIncrSync(s: SparkSession,
     var runDs = ""
     //table字段
     val columns: Seq[String] = spark.table(ads_company_tb).schema.map(_.name).filter(s => {
-      !s.equals("ds") && !s.equals(s"new_$cidCol") && !s.equals("rowkey")
+      !s.equals("ds") && !s.equals(s"new_$cidCol")  && !s.equals("rowkey") && !s.equals("cids")
     })
     //第一次run
     if (StringUtils.isBlank(lastDsIncAds)) {
@@ -174,7 +174,7 @@ case class PersonIncrSync(s: SparkSession,
 
     MaxComputer2Phoenix(
       spark,
-      colsTotal.filter(!_.equals("cid")).filter(!_.equals("new_cid")),
+      colsTotal.filter(!_.equals("cid")).filter(!_.equals("new_cid")).filter(!_.equals("cids")).filter(!_.equals("new_cids")),
       inc_ads_company_tb,
       tableName+"_PERSON",
       lastDsIncOds,