|
@@ -1,5 +1,6 @@
|
|
package com.winhc.bigdata.flink.utils
|
|
package com.winhc.bigdata.flink.utils
|
|
|
|
|
|
|
|
+import com.alibaba.fastjson.JSONObject
|
|
import com.fasterxml.jackson.databind.JsonNode
|
|
import com.fasterxml.jackson.databind.JsonNode
|
|
import com.fasterxml.jackson.databind.node.ObjectNode
|
|
import com.fasterxml.jackson.databind.node.ObjectNode
|
|
import com.github.fge.jsonschema.main.JsonSchemaFactory
|
|
import com.github.fge.jsonschema.main.JsonSchemaFactory
|
|
@@ -9,8 +10,8 @@ import com.winhc.bigdata.flink.utils.BaseUtils.md5
|
|
import org.json4s.JsonAST.{JArray, JObject, JString}
|
|
import org.json4s.JsonAST.{JArray, JObject, JString}
|
|
import org.json4s.JsonDSL.jobject2assoc
|
|
import org.json4s.JsonDSL.jobject2assoc
|
|
import org.json4s.jackson.JsonMethods._
|
|
import org.json4s.jackson.JsonMethods._
|
|
-import java.io.{File, InputStream}
|
|
|
|
|
|
|
|
|
|
+import java.io.{File, InputStream}
|
|
import scala.collection.JavaConverters._
|
|
import scala.collection.JavaConverters._
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -21,7 +22,7 @@ object RowDataUtils {
|
|
val schemas: Map[String, JsonNode] = createAllSchema()
|
|
val schemas: Map[String, JsonNode] = createAllSchema()
|
|
|
|
|
|
def createAllSchema(): Map[String, JsonNode] = {
|
|
def createAllSchema(): Map[String, JsonNode] = {
|
|
-// val schema_dir: String = this.getClass.getClassLoader.getResource("/schema").getPath
|
|
|
|
|
|
+ // val schema_dir: String = this.getClass.getClassLoader.getResource("/schema").getPath
|
|
|
|
|
|
Seq(
|
|
Seq(
|
|
("company",
|
|
("company",
|
|
@@ -340,6 +341,21 @@ object RowDataUtils {
|
|
.toMap
|
|
.toMap
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ def generateRowKey(dim: String, json: JSONObject): String = {
|
|
|
|
+
|
|
|
|
+ ArgsCompanyJob.job_args.get(dim).map(acj => {
|
|
|
|
+ val md5_fields = acj.md5_fields
|
|
|
|
+ val companyId = json.getString("company_id")
|
|
|
|
+ if (md5_fields != null) {
|
|
|
|
+ val fields = md5(md5_fields.map(field => json.getString(field)).filter(s => s != null && s.nonEmpty).mkString)
|
|
|
|
+ s"${companyId}_$fields".stripMargin
|
|
|
|
+ } else if (acj.rowkey_udf != null) {
|
|
|
|
+ acj.rowkey_udf(companyId, null, json)
|
|
|
|
+ } else {
|
|
|
|
+ null
|
|
|
|
+ }
|
|
|
|
+ }).orNull
|
|
|
|
+ }
|
|
|
|
|
|
/**
|
|
/**
|
|
* TODO 未实现
|
|
* TODO 未实现
|
|
@@ -356,7 +372,7 @@ object RowDataUtils {
|
|
val fields = md5(md5_fields.map(field => json.get(field).asText()).filter(s => s != null && s.nonEmpty).mkString)
|
|
val fields = md5(md5_fields.map(field => json.get(field).asText()).filter(s => s != null && s.nonEmpty).mkString)
|
|
s"${companyId}_$fields".stripMargin
|
|
s"${companyId}_$fields".stripMargin
|
|
} else if (acj.rowkey_udf != null) {
|
|
} else if (acj.rowkey_udf != null) {
|
|
- acj.rowkey_udf(companyId, json)
|
|
|
|
|
|
+ acj.rowkey_udf(companyId, json, null)
|
|
} else {
|
|
} else {
|
|
null
|
|
null
|
|
}
|
|
}
|
|
@@ -380,6 +396,23 @@ object RowDataUtils {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ def generateMetaInfoJava(dim: String, companyId: String, companyName: String): com.winhc.bigdata.flink.java.entity.MetaInfo = {
|
|
|
|
+
|
|
|
|
+ val m = ArgsCompanyJob.job_args.get(dim).orNull
|
|
|
|
+ if (m == null) {
|
|
|
|
+ val company = new com.winhc.bigdata.flink.java.entity.MetaInfo.CompanyInfo(companyId, companyName)
|
|
|
|
+ new com.winhc.bigdata.flink.java.entity.MetaInfo(company, null, null, null)
|
|
|
|
+
|
|
|
|
+ } else {
|
|
|
|
+ import com.winhc.bigdata.flink.java.entity.MetaInfo._
|
|
|
|
+ val company = new CompanyInfo(companyId, companyName)
|
|
|
|
+ val es = new ElasticSearchInfo(m.esInfo._index, m.esInfo._type)
|
|
|
|
+ val hbase = new HbaseInfo(m.hbaseInfo.table, m.hbaseInfo.cf)
|
|
|
|
+ val holo = new HologresInfo(m.holoInfo.table, m.holoInfo.shema)
|
|
|
|
+ new com.winhc.bigdata.flink.java.entity.MetaInfo(company, es, hbase, holo)
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
|
|
/**
|
|
/**
|
|
* 实体类校验
|
|
* 实体类校验
|
|
@@ -452,7 +485,7 @@ object RowDataUtils {
|
|
}
|
|
}
|
|
|
|
|
|
def main(args: Array[String]): Unit = {
|
|
def main(args: Array[String]): Unit = {
|
|
-println("")
|
|
|
|
|
|
+ println("")
|
|
}
|
|
}
|
|
|
|
|
|
}
|
|
}
|