|
@@ -7,6 +7,7 @@ import com.mongodb.client.model.Filters;
|
|
import com.winhc.common.constant.Base;
|
|
import com.winhc.common.constant.Base;
|
|
import com.winhc.config.ConfigConstant;
|
|
import com.winhc.config.ConfigConstant;
|
|
import com.winhc.kafka.KafkaProduce;
|
|
import com.winhc.kafka.KafkaProduce;
|
|
|
|
+import com.winhc.service.EsQueryService;
|
|
import lombok.AllArgsConstructor;
|
|
import lombok.AllArgsConstructor;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.bson.Document;
|
|
import org.bson.Document;
|
|
@@ -38,13 +39,15 @@ public class AsynMergePersonTask {
|
|
|
|
|
|
private final MongoTemplate mongoTemplate;
|
|
private final MongoTemplate mongoTemplate;
|
|
private final KafkaProduce kafkaProduce;
|
|
private final KafkaProduce kafkaProduce;
|
|
|
|
+ private final EsQueryService esQueryService;
|
|
@Autowired
|
|
@Autowired
|
|
ConfigConstant configConstant;
|
|
ConfigConstant configConstant;
|
|
|
|
|
|
@Scheduled(cron = "*/15 * * * * ?")
|
|
@Scheduled(cron = "*/15 * * * * ?")
|
|
//@Scheduled(cron = "0 /2 * * * ? ")
|
|
//@Scheduled(cron = "0 /2 * * * ? ")
|
|
public void start() throws InterruptedException {
|
|
public void start() throws InterruptedException {
|
|
- if(isWindows()) return;
|
|
|
|
|
|
+ if (isWindows()) return;
|
|
|
|
+ log.info("start AsynMergePersonTask !!! ");
|
|
MongoCollection<Document> collection = mongoTemplate.getCollection(Base.PID_WAIT_MERGE_V9);
|
|
MongoCollection<Document> collection = mongoTemplate.getCollection(Base.PID_WAIT_MERGE_V9);
|
|
while (true) {
|
|
while (true) {
|
|
//1.查询mongo 2分钟之前数据
|
|
//1.查询mongo 2分钟之前数据
|
|
@@ -52,8 +55,12 @@ public class AsynMergePersonTask {
|
|
FindIterable<Document> documents = collection.find(query).batchSize(200).noCursorTimeout(true);
|
|
FindIterable<Document> documents = collection.find(query).batchSize(200).noCursorTimeout(true);
|
|
List<String> ids = new ArrayList<>();
|
|
List<String> ids = new ArrayList<>();
|
|
for (Document d : documents) {
|
|
for (Document d : documents) {
|
|
- ids.add(d.get("_id").toString());
|
|
|
|
- kafkaProduce.produce(configConstant.topic_node_relation_union, JSON.toJSONString(d.get("data")));
|
|
|
|
|
|
+ String human_pid = d.get("_id").toString();
|
|
|
|
+ ids.add(human_pid);
|
|
|
|
+ Integer count = esQueryService.queryByDsl(human_pid);
|
|
|
|
+ if (count < 100) {
|
|
|
|
+ kafkaProduce.produce(configConstant.topic_node_relation_union, JSON.toJSONString(d.get("data")));
|
|
|
|
+ }
|
|
}
|
|
}
|
|
//2.成功后删除
|
|
//2.成功后删除
|
|
if (!ids.isEmpty()) {
|
|
if (!ids.isEmpty()) {
|
|
@@ -63,5 +70,6 @@ public class AsynMergePersonTask {
|
|
break;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+ log.info("stop AsynMergePersonTask !!! ");
|
|
}
|
|
}
|
|
}
|
|
}
|