|
@@ -1,9 +1,6 @@
|
|
package com.winhc.bigdata.spark.jobs.chance
|
|
package com.winhc.bigdata.spark.jobs.chance
|
|
|
|
|
|
-import java.util.concurrent.{CountDownLatch, TimeUnit}
|
|
|
|
-
|
|
|
|
import com.winhc.bigdata.spark.config.EsConfig
|
|
import com.winhc.bigdata.spark.config.EsConfig
|
|
-import com.winhc.bigdata.spark.test.TestChangeExtract.seq
|
|
|
|
import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
|
|
import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
|
|
import com.winhc.bigdata.spark.utils._
|
|
import com.winhc.bigdata.spark.utils._
|
|
import org.apache.spark.internal.Logging
|
|
import org.apache.spark.internal.Logging
|
|
@@ -357,35 +354,13 @@ object ChangeExtract {
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
- case class ChangeExtract(s: SparkSession,
|
|
|
|
- ds: String //表名(不加前后辍)
|
|
|
|
- ) extends Watching {
|
|
|
|
-
|
|
|
|
- override protected val spark: SparkSession = s
|
|
|
|
-
|
|
|
|
- def calc(): Unit = {
|
|
|
|
- val latch = new CountDownLatch(seq.length)
|
|
|
|
- startArgs
|
|
|
|
- // .filter(s => seq.contains(s.tableName))
|
|
|
|
- .foreach(e => {
|
|
|
|
- asyncWatch(e.tableName, () => {
|
|
|
|
- println("______________________________" + e.tableName + "___________________________________")
|
|
|
|
- ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
- latch.countDown()
|
|
|
|
- })
|
|
|
|
- })
|
|
|
|
- latch.await(60, TimeUnit.MINUTES)
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
-
|
|
|
|
private case class Args(project: String = "winhc_eci_dev"
|
|
private case class Args(project: String = "winhc_eci_dev"
|
|
, tableName: String
|
|
, tableName: String
|
|
, primaryKey: String = "rowkey"
|
|
, primaryKey: String = "rowkey"
|
|
, primaryFields: String
|
|
, primaryFields: String
|
|
, isCopy: Boolean = true)
|
|
, isCopy: Boolean = true)
|
|
|
|
|
|
|
|
+
|
|
def main(args: Array[String]): Unit = {
|
|
def main(args: Array[String]): Unit = {
|
|
val Array(tableName, inc_ds) = args
|
|
val Array(tableName, inc_ds) = args
|
|
|
|
|
|
@@ -395,16 +370,32 @@ object ChangeExtract {
|
|
)
|
|
)
|
|
val spark = SparkUtils.InitEnv("ChangeExtract", config)
|
|
val spark = SparkUtils.InitEnv("ChangeExtract", config)
|
|
|
|
|
|
- if (tableName.equals("all")) {
|
|
|
|
- startArgs.foreach(e => {
|
|
|
|
- ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
- })
|
|
|
|
- } else {
|
|
|
|
|
|
+
|
|
|
|
+ var start = startArgs
|
|
|
|
+ if (!tableName.equals("all")) {
|
|
val set = tableName.split(",").toSet
|
|
val set = tableName.split(",").toSet
|
|
- startArgs.filter(a => set.contains(a.tableName)).foreach(e => {
|
|
|
|
- ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
- })
|
|
|
|
|
|
+ start = start.filter(a => set.contains(a.tableName))
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ val a = start.map(e => (e.tableName, () => {
|
|
|
|
+ ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
+ true
|
|
|
|
+ }))
|
|
|
|
+
|
|
|
|
+ AsyncExtract.startAndWait(spark, a)
|
|
|
|
+
|
|
|
|
+ /* if (tableName.equals("all")) {
|
|
|
|
+ startArgs.foreach(e => {
|
|
|
|
+ ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
+ })
|
|
|
|
+ } else {
|
|
|
|
+ val set = tableName.split(",").toSet
|
|
|
|
+ startArgs.filter(a => set.contains(a.tableName)).foreach(e => {
|
|
|
|
+ ChangeExtractHandle(spark, e.project, e.tableName, e.primaryKey, inc_ds, e.primaryFields.split(",")).calc(e.isCopy)
|
|
|
|
+ })
|
|
|
|
+ }*/
|
|
|
|
+
|
|
spark.stop()
|
|
spark.stop()
|
|
}
|
|
}
|
|
|
|
+
|
|
}
|
|
}
|