许家凯 пре 4 година
родитељ
комит
fb5470b0dc

+ 1 - 1
src/main/scala/com/winhc/bigdata/spark/implicits/DataFrame2HBaseHelper.scala

@@ -21,7 +21,7 @@ object DataFrame2HBaseHelper {
 
       val stringDf = df.select(df.columns.map(column => col(column).cast("string")): _*)
       stringDf.rdd.map(row => {
-        val id = row.getAs[String](rowkeyFieldName)
+        val id = row.getAs[String](rowkeyFieldName.toLowerCase())
         val put = new Put(Bytes.toBytes(id))
         for (f <- fields) {
           val v = row.getAs[String](f.toLowerCase)

+ 2 - 0
src/main/scala/com/winhc/bigdata/spark/utils/CompanyIncrForCidWithoutMD5Utils.scala

@@ -148,6 +148,8 @@ case class CompanyIncrForCidWithoutMD5Utils(s: SparkSession,
       s"CONCAT_WS('_',new_cid,${cols_md5.mkString(",")})"
     ).syn()
 
+    CompanyIncSummary(spark, project, tableName, "new_cid", dupliCols).calc
+
     println(s"${this.getClass.getSimpleName} calc end! " + new Date().toString)
   }
 }