许家凯 5 лет назад
Родитель
Сommit
157bf5a727

+ 6 - 18
src/main/scala/com/winhc/bigdata/spark/jobs/CompanySummaryBySingle.scala

@@ -1,9 +1,9 @@
 package com.winhc.bigdata.spark.jobs
 
 import com.mongodb.spark.MongoSpark
-import com.winhc.bigdata.spark.utils.CompanySummaryUtils._
 import com.winhc.bigdata.spark.utils.SparkUtils
-import org.bson.Document
+
+import scala.collection.mutable
 
 /**
  * @Author: XuJiakai
@@ -16,7 +16,7 @@ object CompanySummaryBySingle {
        sys.exit(-1)
      }
      val tableName = args(0)*/
-    val map = Map(
+    val map = mutable.Map[String, String](
       "spark.mongodb.output.uri" -> "mongodb://itslaw:itslaw_168@dds-uf6ff5dfd9aef3641.mongodb.rds.aliyuncs.com:3717,dds-uf6ff5dfd9aef3642.mongodb.rds.aliyuncs.com:3717/itslaw.company_summary?replicaSet=mgset-6501997"
       , "spark.hadoop.odps.cupid.vpc.domain.list" -> "{\"regionId\":\"cn-shanghai\",\"vpcs\":[{\"vpcId\":\"vpc-11hby9xee\",\"zones\":[{\"urls\":[{\"domain\":\"dds-uf6ff5dfd9aef3641.mongodb.rds.aliyuncs.com\",\"port\":3717},{\"domain\":\"dds-uf6ff5dfd9aef3642.mongodb.rds.aliyuncs.com\",\"port\":3717}]}]}]}"
     )
@@ -24,7 +24,7 @@ object CompanySummaryBySingle {
     val spark = SparkUtils.InitEnv("CompanySummaryCalculator", map)
     import spark._
 
-  /*  var df = sql("select * from ads_company_summary limit 100")
+    var df = sql("select * from ads_company_summary limit 100")
     val lookup = Map(
       "company_id" -> "_id"
       , "ads_judrisk_court_annc_list_num" -> "judrisk_court_annc_list_num"
@@ -35,21 +35,9 @@ object CompanySummaryBySingle {
       if (df.columns.contains(elem._1)) {
         df = df.withColumnRenamed(elem._1, elem._2)
       }
-    }*/
-
-
-    val document1 = new Document()
-    document1.append("name", "sunshangxiang").append("age", 18).append("sex", "female")
-    val document2 = new Document()
-    document2.append("name", "diaochan").append("age", 24).append("sex", "female")
-    val document3 = new Document()
-    document3.append("name", "huangyueying").append("age", 23).append("sex", "female")
-
-    val seq = Seq(document1, document2, document3)
-    val df = spark.sparkContext.parallelize(seq)
-
+    }
 
-    MongoSpark.save(df)
+    MongoSpark.save(df.write)
     spark.stop()
   }
 }