|
@@ -1,9 +1,9 @@
|
|
|
package com.winhc.bigdata.spark.jobs
|
|
|
|
|
|
import com.mongodb.spark.MongoSpark
|
|
|
-import com.winhc.bigdata.spark.utils.CompanySummaryUtils._
|
|
|
import com.winhc.bigdata.spark.utils.SparkUtils
|
|
|
-import org.bson.Document
|
|
|
+
|
|
|
+import scala.collection.mutable
|
|
|
|
|
|
/**
|
|
|
* @Author: XuJiakai
|
|
@@ -16,7 +16,7 @@ object CompanySummaryBySingle {
|
|
|
sys.exit(-1)
|
|
|
}
|
|
|
val tableName = args(0)*/
|
|
|
- val map = Map(
|
|
|
+ val map = mutable.Map[String, String](
|
|
|
"spark.mongodb.output.uri" -> "mongodb://itslaw:itslaw_168@dds-uf6ff5dfd9aef3641.mongodb.rds.aliyuncs.com:3717,dds-uf6ff5dfd9aef3642.mongodb.rds.aliyuncs.com:3717/itslaw.company_summary?replicaSet=mgset-6501997"
|
|
|
, "spark.hadoop.odps.cupid.vpc.domain.list" -> "{\"regionId\":\"cn-shanghai\",\"vpcs\":[{\"vpcId\":\"vpc-11hby9xee\",\"zones\":[{\"urls\":[{\"domain\":\"dds-uf6ff5dfd9aef3641.mongodb.rds.aliyuncs.com\",\"port\":3717},{\"domain\":\"dds-uf6ff5dfd9aef3642.mongodb.rds.aliyuncs.com\",\"port\":3717}]}]}]}"
|
|
|
)
|
|
@@ -24,7 +24,7 @@ object CompanySummaryBySingle {
|
|
|
val spark = SparkUtils.InitEnv("CompanySummaryCalculator", map)
|
|
|
import spark._
|
|
|
|
|
|
- /* var df = sql("select * from ads_company_summary limit 100")
|
|
|
+ var df = sql("select * from ads_company_summary limit 100")
|
|
|
val lookup = Map(
|
|
|
"company_id" -> "_id"
|
|
|
, "ads_judrisk_court_annc_list_num" -> "judrisk_court_annc_list_num"
|
|
@@ -35,21 +35,9 @@ object CompanySummaryBySingle {
|
|
|
if (df.columns.contains(elem._1)) {
|
|
|
df = df.withColumnRenamed(elem._1, elem._2)
|
|
|
}
|
|
|
- }*/
|
|
|
-
|
|
|
-
|
|
|
- val document1 = new Document()
|
|
|
- document1.append("name", "sunshangxiang").append("age", 18).append("sex", "female")
|
|
|
- val document2 = new Document()
|
|
|
- document2.append("name", "diaochan").append("age", 24).append("sex", "female")
|
|
|
- val document3 = new Document()
|
|
|
- document3.append("name", "huangyueying").append("age", 23).append("sex", "female")
|
|
|
-
|
|
|
- val seq = Seq(document1, document2, document3)
|
|
|
- val df = spark.sparkContext.parallelize(seq)
|
|
|
-
|
|
|
+ }
|
|
|
|
|
|
- MongoSpark.save(df)
|
|
|
+ MongoSpark.save(df.write)
|
|
|
spark.stop()
|
|
|
}
|
|
|
}
|