|
|
@@ -0,0 +1,224 @@
|
|
|
+package com.winhc.test;
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.google.common.base.Charsets;
|
|
|
+import com.google.common.io.Resources;
|
|
|
+import com.mongodb.bulk.BulkWriteResult;
|
|
|
+import com.mongodb.client.model.UpdateOneModel;
|
|
|
+import com.mongodb.client.model.UpdateOptions;
|
|
|
+import com.mongodb.client.model.WriteModel;
|
|
|
+import com.winhc.kafka.KafkaProduce;
|
|
|
+import com.winhc.service.impl.PersonMergeImpl;
|
|
|
+import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.bson.Document;
|
|
|
+import org.frameworkset.elasticsearch.ElasticSearchHelper;
|
|
|
+import org.frameworkset.elasticsearch.boot.BBossESStarter;
|
|
|
+import org.frameworkset.elasticsearch.client.ClientInterface;
|
|
|
+import org.frameworkset.elasticsearch.entity.ESDatas;
|
|
|
+import org.frameworkset.elasticsearch.entity.RestResponse;
|
|
|
+import org.frameworkset.elasticsearch.entity.SearchHit;
|
|
|
+import org.junit.Test;
|
|
|
+import org.junit.runner.RunWith;
|
|
|
+import org.neo4j.driver.Driver;
|
|
|
+import org.neo4j.driver.Record;
|
|
|
+import org.neo4j.driver.Session;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
+import org.springframework.boot.test.context.SpringBootTest;
|
|
|
+import org.springframework.data.mongodb.core.MongoTemplate;
|
|
|
+import org.springframework.test.context.junit4.SpringRunner;
|
|
|
+
|
|
|
+import javax.annotation.PostConstruct;
|
|
|
+import java.io.IOException;
|
|
|
+import java.net.URL;
|
|
|
+import java.util.*;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author π
|
|
|
+ * @Description:人员移除测试
|
|
|
+ * @date 2023/2/23 16:37
|
|
|
+ */
|
|
|
+@RunWith(SpringRunner.class)
|
|
|
+@SpringBootTest
|
|
|
+@Slf4j
|
|
|
+public class TestPersonUpdate {
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier("bbossESStarterEs5")
|
|
|
+ public BBossESStarter bbossESStarterEs5;
|
|
|
+
|
|
|
+ public ClientInterface restClient5;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier("bbossESStarterEs6")
|
|
|
+ public BBossESStarter bbossESStarterEs6;
|
|
|
+
|
|
|
+ public ClientInterface restClient6;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ public MongoTemplate mongoTemplate;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier("DriverV1")
|
|
|
+ Driver driver;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ KafkaProduce kafkaProduce;
|
|
|
+
|
|
|
+ public ClientInterface clientUtil;
|
|
|
+
|
|
|
+ public List<String> indexs = Arrays.asList("winhc_index_rt_company", "winhc_index_rt_company_holder", "winhc_index_rt_company_staff");
|
|
|
+
|
|
|
+ @PostConstruct
|
|
|
+ public void init() {
|
|
|
+ restClient5 = bbossESStarterEs5.getRestClient("es5");
|
|
|
+ restClient6 = bbossESStarterEs6.getRestClient("es6");
|
|
|
+ clientUtil = ElasticSearchHelper.getConfigRestClientUtil("es6", "esmapper/scroll.xml");
|
|
|
+ }
|
|
|
+
|
|
|
+ private String loadResourceAsString(String resource) throws IOException {
|
|
|
+ URL url = Resources.getResource(resource);
|
|
|
+ return Resources.toString(url, Charsets.UTF_8);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void startUpdate() throws IOException {
|
|
|
+
|
|
|
+ //String out_mongo = "xf_test1";
|
|
|
+ String out_mongo = "company_back_update_v9";
|
|
|
+ //人员id映射表
|
|
|
+ //String mapping_index = "winhc_company_human_pid_mapping";
|
|
|
+ String mapping_index = "winhc_company_human_pid_mapping_v9";
|
|
|
+ //老板映射表
|
|
|
+ //String relation_index = "winhc-company-human-relation";
|
|
|
+ String relation_index = "winhc-company-human-relation-v9";
|
|
|
+
|
|
|
+ Arrays.stream(loadResourceAsString("data/update_pid.txt")
|
|
|
+ .split("\n"))
|
|
|
+ .filter(StringUtils::isNotBlank)
|
|
|
+ .distinct()
|
|
|
+ .forEach(pid -> {
|
|
|
+ //执行删除
|
|
|
+ run(pid, out_mongo, mapping_index, relation_index);
|
|
|
+ });
|
|
|
+
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ private void run(String pid, String out_mongo, String mapping_index, String relation_index) {
|
|
|
+ //更新映射表摘要 -ES
|
|
|
+ updateMappingEsIndex(pid, mapping_index);
|
|
|
+ //更新老板摘要 -ES
|
|
|
+ updateRelationEsIndex(pid, relation_index);
|
|
|
+ //移除图库人员 -Neo4j
|
|
|
+ deletedGraphNode(pid);
|
|
|
+
|
|
|
+ //待更新 数据回推 -HBase -Mongo
|
|
|
+ callBackUpdate(pid, out_mongo);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void callBackUpdate(String pid, String out_mongo) {
|
|
|
+ indexs.forEach(index1 -> callRowkey(pid, index1, out_mongo));
|
|
|
+ }
|
|
|
+
|
|
|
+ public void callRowkey(String pid, String queryIndex, String out_mongo) {
|
|
|
+
|
|
|
+ //回推mongo
|
|
|
+ Map params = new HashMap();
|
|
|
+ params.put("from", 0);
|
|
|
+ params.put("size", 10000);
|
|
|
+ params.put("pid", pid);
|
|
|
+ ESDatas<Map> redata = clientUtil.searchList(queryIndex + "/_search",
|
|
|
+ "searchPagineDatas", params, Map.class);
|
|
|
+
|
|
|
+ RestResponse restResponse = (RestResponse) redata.getRestResponse();
|
|
|
+ List<SearchHit> searchHits = restResponse.getSearchHits().getHits();
|
|
|
+ List<Document> collect = searchHits.stream().map(d -> {
|
|
|
+ JSONObject doc = new JSONObject();
|
|
|
+ String rowkey = d.getId();
|
|
|
+ String company_id = rowkey.split("_")[0];
|
|
|
+ doc.put("rowkey", rowkey);
|
|
|
+ doc.put("company_id", company_id);
|
|
|
+ String index = d.getIndex();
|
|
|
+ String tn;
|
|
|
+ if (index.contains("staff")) {
|
|
|
+ tn = "company_staff";
|
|
|
+ } else if (index.contains("holder")) {
|
|
|
+ tn = "company_holder";
|
|
|
+ } else {
|
|
|
+ tn = "company";
|
|
|
+ }
|
|
|
+ doc.put("tn", tn);
|
|
|
+ JSONObject j = new JSONObject();
|
|
|
+ j.put(company_id, Collections.singletonList(doc));
|
|
|
+ j.put("_id", rowkey + tn);
|
|
|
+ return new Document(j);
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+
|
|
|
+ if (!collect.isEmpty()) {
|
|
|
+ UpdateOptions uo = new UpdateOptions().upsert(true).bypassDocumentValidation(true);
|
|
|
+ List<WriteModel<Document>> list = new ArrayList<>();
|
|
|
+ collect.forEach(m -> {
|
|
|
+ list.add(new UpdateOneModel<>(new Document("_id", m.get("_id")), new Document("$set", m), uo));
|
|
|
+ });
|
|
|
+ BulkWriteResult writeResult = mongoTemplate.getCollection(out_mongo).bulkWrite(list);
|
|
|
+ System.out.println(writeResult.getMatchedCount());
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ public void updateMappingEsIndex(String pid, String mapping_index) {
|
|
|
+ //移除映射表
|
|
|
+ String dsl1 = "{\n" +
|
|
|
+ " \"query\": {\n" +
|
|
|
+ " \"term\": {\n" +
|
|
|
+ " \"human_pid\": {\n" +
|
|
|
+ " \"value\": \"" + pid + "\"\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ "}";
|
|
|
+ System.out.println(dsl1);
|
|
|
+ String res1 = restClient5.updateByQuery(mapping_index + "/_delete_by_query?conflicts=proceed&refresh=true&wait_for_completion=false", dsl1);
|
|
|
+ System.out.println(res1);
|
|
|
+ }
|
|
|
+
|
|
|
+ public void updateRelationEsIndex(String pid, String relation_index) {
|
|
|
+
|
|
|
+ //移除老板聚合表
|
|
|
+ String dsl2 = "{\n" +
|
|
|
+ " \"query\": {\n" +
|
|
|
+ " \"terms\": {\n" +
|
|
|
+ " \"_id\": [\n" +
|
|
|
+ " \"" + pid + "\"\n" +
|
|
|
+ " ]\n" +
|
|
|
+ " }\n" +
|
|
|
+ " },\n" +
|
|
|
+ " \"script\": {\n" +
|
|
|
+ " \"inline\": \" ctx._source['deleted'] = '9' \"\n" +
|
|
|
+ " }\n" +
|
|
|
+ "}";
|
|
|
+ System.out.println(dsl2);
|
|
|
+ String res2 = restClient5.updateByQuery(relation_index + "/_update_by_query?conflicts=proceed&refresh=true&wait_for_completion=false", dsl2);
|
|
|
+ System.out.println(res2);
|
|
|
+
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ public void deletedGraphNode(String pid) {
|
|
|
+ //移除图库节点
|
|
|
+ long start = System.currentTimeMillis();
|
|
|
+ Session session = driver.session();
|
|
|
+ final String cql =
|
|
|
+ "MATCH (n:`个人`{person_id:\"" + pid + "\"}) \n" +
|
|
|
+ "DETACH DELETE n";
|
|
|
+ log.info("consumer cql:{}", cql);
|
|
|
+
|
|
|
+ List<Map<String, Object>> re = session.writeTransaction(tx -> tx.run(cql).list()).stream().map(Record::asMap).collect(Collectors.toList());
|
|
|
+ System.out.println(JSONObject.toJSONString(re));
|
|
|
+ session.close();
|
|
|
+ log.info("class:{} | cost:{}", PersonMergeImpl.class.getSimpleName(), (System.currentTimeMillis() - start));
|
|
|
+ }
|
|
|
+}
|
xufei