|
@@ -5,7 +5,7 @@ import java.util.Date
|
|
import com.winhc.bigdata.spark.config.EsConfig
|
|
import com.winhc.bigdata.spark.config.EsConfig
|
|
import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
|
|
import com.winhc.bigdata.spark.utils.BaseUtil.isWindows
|
|
import com.winhc.bigdata.spark.utils.ReflectUtils.getClazz
|
|
import com.winhc.bigdata.spark.utils.ReflectUtils.getClazz
|
|
-import com.winhc.bigdata.spark.utils.{LoggingUtils, SparkUtils}
|
|
|
|
|
|
+import com.winhc.bigdata.spark.utils.{AsyncExtract, LoggingUtils, SparkUtils}
|
|
import org.apache.commons.lang3.time.DateFormatUtils
|
|
import org.apache.commons.lang3.time.DateFormatUtils
|
|
import org.apache.spark.internal.Logging
|
|
import org.apache.spark.internal.Logging
|
|
import org.apache.spark.sql.types.StringType
|
|
import org.apache.spark.sql.types.StringType
|
|
@@ -253,7 +253,21 @@ object CompanyDynamic {
|
|
val cd = CompanyDynamicUtil(spark, project, ds)
|
|
val cd = CompanyDynamicUtil(spark, project, ds)
|
|
cd.init()
|
|
cd.init()
|
|
|
|
|
|
- if (tableNames.equals("all")) {
|
|
|
|
|
|
+ var start = startArgs
|
|
|
|
+ if (!tableNames.equals("all")) {
|
|
|
|
+ val set = tableNames.split(",").toSet
|
|
|
|
+ start = start.filter(a => set.contains(a.tableName))
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ val a = start.map(e => (e.tableName, () => {
|
|
|
|
+ cd.calc(e.tableName, e.bName)
|
|
|
|
+ true
|
|
|
|
+ }))
|
|
|
|
+
|
|
|
|
+ AsyncExtract.startAndWait(spark, a)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+ /*if (tableNames.equals("all")) {
|
|
startArgs.foreach(e => {
|
|
startArgs.foreach(e => {
|
|
cd.calc(e.tableName, e.bName)
|
|
cd.calc(e.tableName, e.bName)
|
|
})
|
|
})
|
|
@@ -265,7 +279,7 @@ object CompanyDynamic {
|
|
}).foreach(e => {
|
|
}).foreach(e => {
|
|
cd.calc(e.tableName, e.bName)
|
|
cd.calc(e.tableName, e.bName)
|
|
})
|
|
})
|
|
- }
|
|
|
|
|
|
+ }*/
|
|
|
|
|
|
spark.stop()
|
|
spark.stop()
|
|
}
|
|
}
|