|
@@ -6,22 +6,26 @@ import com.alibaba.fastjson.JSONObject;
|
|
|
import com.winhc.bigdata.task.service.DingTalkService;
|
|
|
import com.winhc.bigdata.task.util.ElasticsearchQueryUtil;
|
|
|
import com.winhc.bigdata.task.util.ThrowableUtils;
|
|
|
+import lombok.Data;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
import lombok.experimental.ExtensionMethod;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import lombok.var;
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
import org.elasticsearch.common.TriFunction;
|
|
|
import org.elasticsearch.common.collect.Tuple;
|
|
|
+import org.frameworkset.elasticsearch.bulk.BulkProcessor;
|
|
|
import org.frameworkset.elasticsearch.client.ClientInterface;
|
|
|
+import org.frameworkset.elasticsearch.entity.ESDatas;
|
|
|
+import org.frameworkset.elasticsearch.entity.MetaMap;
|
|
|
+import org.frameworkset.elasticsearch.entity.SearchHit;
|
|
|
import org.frameworkset.util.CollectionUtils;
|
|
|
+import org.frameworkset.util.ObjectUtils;
|
|
|
import org.springframework.kafka.annotation.KafkaListener;
|
|
|
import org.springframework.kafka.core.KafkaTemplate;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
-import java.util.HashMap;
|
|
|
-import java.util.List;
|
|
|
-import java.util.Map;
|
|
|
-import java.util.Set;
|
|
|
+import java.util.*;
|
|
|
import java.util.concurrent.CompletableFuture;
|
|
|
import java.util.concurrent.ConcurrentHashMap;
|
|
|
import java.util.concurrent.ExecutionException;
|
|
@@ -44,6 +48,7 @@ public class DynamicPersonIdUpdateJob {
|
|
|
private final ClientInterface restClient;
|
|
|
private final KafkaTemplate<String, String> kafkaTemplate;
|
|
|
private final DingTalkService dingTalkService;
|
|
|
+ private final BulkProcessor processor;
|
|
|
|
|
|
public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
|
|
|
Set<Object> seen = ConcurrentHashMap.newKeySet();
|
|
@@ -115,11 +120,14 @@ public class DynamicPersonIdUpdateJob {
|
|
|
batchFromCreate(collect);
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+ /*=======================flag 0 8 9================*/
|
|
|
private void batchFromCreate(Map<String, List<JSONObject>> params) {
|
|
|
CompletableFuture<Void> do0 = doAll(params.get("0"), ElasticsearchQueryUtil::createPersonId);
|
|
|
CompletableFuture<Void> do9 = doAll(params.get("9"), ElasticsearchQueryUtil::deleteDynamic);
|
|
|
+ CompletableFuture<Void> do8 = do8(params.get("8"));
|
|
|
try {
|
|
|
- CompletableFuture.allOf(do0, do9).get();
|
|
|
+ CompletableFuture.allOf(do0, do9, do8).get();
|
|
|
} catch (InterruptedException | ExecutionException e) {
|
|
|
log.error("waiting processor error", e);
|
|
|
dingTalkService.error("{}\n{}", e.getMessage(), ThrowableUtils.getStackTraceByPn(e, "com.winhc.bigdata.task"));
|
|
@@ -127,6 +135,7 @@ public class DynamicPersonIdUpdateJob {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+
|
|
|
private CompletableFuture<Void> doAll(List<JSONObject> params, TriFunction<String, String, List<Tuple<String, String>>, String> function) {
|
|
|
if (CollectionUtils.isEmpty(params)) {
|
|
|
return CompletableFuture.completedFuture(null);
|
|
@@ -168,12 +177,86 @@ public class DynamicPersonIdUpdateJob {
|
|
|
}, TASK_FJ_POOL);
|
|
|
}
|
|
|
|
|
|
+ private CompletableFuture<Void> do8(List<JSONObject> params) {
|
|
|
+ if (CollectionUtils.isEmpty(params)) {
|
|
|
+ return CompletableFuture.completedFuture(null);
|
|
|
+ }
|
|
|
+ List<CompletableFuture<Void>> cr = params.stream().map(j -> new ScanParams(j.getString("tn"), j.getString("company_id"), j.getString("relation_id"))).map(p -> CompletableFuture.runAsync(() -> scan(p))).collect(Collectors.toList());
|
|
|
+ List<CompletableFuture<Void>> rc = params.stream().map(j -> new ScanParams(j.getString("tn"), j.getString("relation_id"), j.getString("company_id"))).map(p -> CompletableFuture.runAsync(() -> scan(p))).collect(Collectors.toList());
|
|
|
+ cr.addAll(rc);
|
|
|
+ return CompletableFuture.allOf(cr.toArray(new CompletableFuture[0]));
|
|
|
+ }
|
|
|
+
|
|
|
+ @RequiredArgsConstructor
|
|
|
+ @Data
|
|
|
+ public static class ScanParams {
|
|
|
+ private final String tn;
|
|
|
+ private final String keyno;
|
|
|
+ private final String validId;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void scan(ScanParams params) {
|
|
|
+ restClient.scroll("winhc-company-dynamic/_search", "scrollQuery", "1m", params.toMap(), MetaMap.class, (response, handlerInfo) -> {
|
|
|
+ Optional
|
|
|
+ .ofNullable(response)
|
|
|
+ .map(ESDatas::getDatas)
|
|
|
+ .ifPresent(
|
|
|
+ l -> l.forEach(m -> remove8(m.getId(), m.getIndex(), params.getKeyno(), params.getValidId(), m))
|
|
|
+// l -> l.forEach(hit-> remove8(hit.getId(), params.getKeyno(), params.getValidId(),hit.asMap()))
|
|
|
+ );
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ private void remove8(String id, String index, String keyno, String validId, Map<String, Object> data) {
|
|
|
+ JSONObject jData = new JSONObject(data);
|
|
|
+ String dynamic_info = jData.getString("dynamic_info");
|
|
|
+ if (StringUtils.isEmpty(dynamic_info)) return;
|
|
|
+ try {
|
|
|
+ JSONObject jsonObject = JSON.parseObject(dynamic_info);
|
|
|
+ JSONArray content = jsonObject.getJSONArray("content");
|
|
|
+ if (ObjectUtils.isEmpty(content)) {
|
|
|
+ log.warn("content为空");
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ List<JSONObject> collect = content.toJavaList(JSONObject.class).stream().filter(j -> {
|
|
|
+ JSONObject entity = j.getJSONObject("entity");
|
|
|
+ if (!keyno.equals(entity.getString("keyno"))) return true;
|
|
|
+ return !j.anyMatchByNestedObject(validId, "keyno", "before", "after");
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+ if (collect.isEmpty()) {
|
|
|
+ Map<String, Object> map = new HashMap<>();
|
|
|
+ map.put("deleted", "9");
|
|
|
+ map.put("id", id);
|
|
|
+ processor.updateData(index, "dynamic", map);
|
|
|
+ } else {
|
|
|
+ jsonObject.put("content", collect);
|
|
|
+ jsonObject.put("id", id);
|
|
|
+ processor.insertData(index, "dynamic", jsonObject);
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ log.error("parse json error", e);
|
|
|
+ dingTalkService.error("{}\n{}", e.getMessage(), ThrowableUtils.getStackTraceByPn(e, "com.winhc.bigdata.task"));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
@SuppressWarnings("unused")
|
|
|
public static class StringStringMapExt {
|
|
|
+ public static boolean anyMatchByNestedObject(JSONObject jo, String val, String key, String... fields) {
|
|
|
+ return Stream.of(fields).map(jo::getJSONObject).filter(Objects::nonNull).map(j -> j.getString(key)).anyMatch(val::equals);
|
|
|
+ }
|
|
|
+
|
|
|
public static Stream<JSONObject> toJSONStream(List<String> strings) {
|
|
|
return strings.stream().map(JSONObject::parseObject);
|
|
|
}
|
|
|
|
|
|
+ public static Map<String, Object> toMap(ScanParams sp) {
|
|
|
+ Map<String, Object> stringObjectHashMap = new HashMap<>();
|
|
|
+ stringObjectHashMap.put("tn", sp.getTn());
|
|
|
+ stringObjectHashMap.put("keyno", sp.getKeyno());
|
|
|
+ return stringObjectHashMap;
|
|
|
+ }
|
|
|
+
|
|
|
public static JSONArray orDefault(JSONArray j, JSONArray dj) {
|
|
|
return CollectionUtils.isEmpty(j) ? dj : j;
|
|
|
}
|