|
@@ -0,0 +1,260 @@
|
|
|
+package com.winhc.task;
|
|
|
+
|
|
|
+import com.alibaba.excel.EasyExcel;
|
|
|
+import com.alibaba.fastjson.JSONArray;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.winhc.task.bean.CaseOut;
|
|
|
+import com.winhc.task.bean.Summary;
|
|
|
+import com.winhc.task.job.CompanyBackUpdate;
|
|
|
+import com.winhc.task.listener.NoModelDataListener;
|
|
|
+import com.winhc.task.util.BaseUtils;
|
|
|
+import com.winhc.task.util.EasyExcelUtil;
|
|
|
+import com.winhc.task.util.QueryBuilderUtils;
|
|
|
+import lombok.val;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.apache.hadoop.hbase.TableName;
|
|
|
+import org.apache.hadoop.hbase.client.Connection;
|
|
|
+import org.apache.hadoop.hbase.client.Get;
|
|
|
+import org.apache.hadoop.hbase.client.ResultScanner;
|
|
|
+import org.apache.hadoop.hbase.client.Scan;
|
|
|
+import org.apache.hadoop.hbase.filter.PageFilter;
|
|
|
+import org.elasticsearch.action.search.SearchRequest;
|
|
|
+import org.elasticsearch.action.search.SearchResponse;
|
|
|
+import org.elasticsearch.client.RestHighLevelClient;
|
|
|
+import org.elasticsearch.index.query.WrapperQueryBuilder;
|
|
|
+import org.elasticsearch.search.builder.SearchSourceBuilder;
|
|
|
+import org.junit.jupiter.api.Test;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
+import org.springframework.boot.test.context.SpringBootTest;
|
|
|
+
|
|
|
+import java.io.IOException;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.util.*;
|
|
|
+import java.util.concurrent.CompletableFuture;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+import java.util.stream.Stream;
|
|
|
+import java.util.stream.StreamSupport;
|
|
|
+
|
|
|
+import static com.winhc.task.common.Constant.hbase_scan_v8;
|
|
|
+import static com.winhc.task.common.Constant.staff_head;
|
|
|
+
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author π
|
|
|
+ * @Description:
|
|
|
+ * @date 2023/7/19 15:16
|
|
|
+ */
|
|
|
+@SpringBootTest
|
|
|
+public class JudicialCaseQueryJob {
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ @Qualifier(value = "v6")
|
|
|
+ RestHighLevelClient getClient;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ Connection connection;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ CompanyBackUpdate companyBackUpdate;
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void start1() throws Exception {
|
|
|
+ String fileName = "C:\\Users\\batmr\\Documents\\WeChat Files\\XU1759380082\\FileStorage\\File\\2023-07\\2023-3-31破产信息.xls";
|
|
|
+ // 这里 只要,然后读取第一个sheet
|
|
|
+ List<Map<Integer, String>> dataList = new ArrayList<>();
|
|
|
+ EasyExcel.read(fileName, new NoModelDataListener(dataList)).sheet().doRead();
|
|
|
+ System.out.println(dataList.size());
|
|
|
+ val futures = new ArrayList<CompletableFuture<JSONObject>>();
|
|
|
+
|
|
|
+ dataList.stream()
|
|
|
+ .map(m -> m.get(4))
|
|
|
+ .forEach(d -> {
|
|
|
+ futures.add(fetchEs(d));
|
|
|
+ });
|
|
|
+
|
|
|
+ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get();
|
|
|
+ List<JSONObject> reList = futures.stream().map(x -> {
|
|
|
+ try {
|
|
|
+ return x.get();
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw new RuntimeException("error 1111 :" + e.getMessage(), e);
|
|
|
+ }
|
|
|
+ }).filter(Objects::nonNull).collect(Collectors.toList());
|
|
|
+ System.out.println(JSONObject.toJSONString(reList.size()));
|
|
|
+
|
|
|
+
|
|
|
+ val futures2 = new ArrayList<CompletableFuture<List<JSONObject>>>();
|
|
|
+ reList.forEach((keys) -> {
|
|
|
+ futures2.add(fetchHbaseById(keys));
|
|
|
+ });
|
|
|
+
|
|
|
+
|
|
|
+ CompletableFuture.allOf(futures2.toArray(new CompletableFuture[0])).get();
|
|
|
+
|
|
|
+ List<List<JSONObject>> reList2 = futures2.stream().map(x -> {
|
|
|
+ try {
|
|
|
+ return x.get();
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw new RuntimeException("error 1111 :" + e.getMessage(), e);
|
|
|
+ }
|
|
|
+ }).filter(Objects::nonNull).collect(Collectors.toList());
|
|
|
+
|
|
|
+
|
|
|
+ List<List<String>> head_list = Arrays.stream("企业@原告@原告电话@原告法人@原告地址".split("@"))
|
|
|
+ .map(Collections::singletonList)
|
|
|
+ .collect(Collectors.toList());
|
|
|
+
|
|
|
+ EasyExcelUtil case_xlsx = new EasyExcelUtil().init("D:\\tmp\\case\\" + "case_list.xlsx", "sheet1", head_list, new HashSet<>());
|
|
|
+ List<CaseOut> final_list = new ArrayList<>();
|
|
|
+
|
|
|
+ reList2.forEach(d1 -> {
|
|
|
+ d1.forEach(d2 -> {
|
|
|
+ String bg_name = d2.getString("bg_name");
|
|
|
+ String yg_name = d2.getString("name");
|
|
|
+ String reg_location = d2.getString("reg_location");
|
|
|
+ String legals = Optional.ofNullable(d2.getJSONArray("legal_entities")).orElse(new JSONArray())
|
|
|
+ .toJavaList(JSONObject.class)
|
|
|
+ .stream()
|
|
|
+ .map(x -> {
|
|
|
+ JSONObject tj = new JSONObject();
|
|
|
+ tj.fluentPut("legal", x.getString("name"))
|
|
|
+ .fluentPut("id", x.getString("id"))
|
|
|
+ .fluentPut("legal_deleted", x.getInteger("deleted"));
|
|
|
+ return tj;
|
|
|
+ }).filter(x -> x.getString("legal_deleted").equals("0") && StringUtils.isNotBlank(x.getString("legal")))
|
|
|
+ .map(r1 -> r1.getString("legal"))
|
|
|
+ .collect(Collectors.joining("\n"));
|
|
|
+
|
|
|
+
|
|
|
+ String phones = Optional.ofNullable(d2.getJSONArray("tel_info")).orElse(new JSONArray())
|
|
|
+ .toJavaList(JSONObject.class).stream()
|
|
|
+ .filter(x -> StringUtils.isNotBlank(x.getString("tel"))).map(x -> x.getString("tel"))
|
|
|
+ .distinct().collect(Collectors.joining("\n"));
|
|
|
+
|
|
|
+
|
|
|
+ CaseOut cc = CaseOut.builder().bg_name(bg_name)
|
|
|
+ .yg_name(yg_name).phones(phones)
|
|
|
+ .legal(legals).reg_location(reg_location).build();
|
|
|
+ final_list.add(cc);
|
|
|
+ });
|
|
|
+ });
|
|
|
+
|
|
|
+ case_xlsx.doExportExcel(final_list);
|
|
|
+ case_xlsx.finish();
|
|
|
+ }
|
|
|
+
|
|
|
+ @Test
|
|
|
+ public void start2() throws Exception {
|
|
|
+ companyBackUpdate.start();
|
|
|
+ }
|
|
|
+
|
|
|
+ private CompletableFuture<JSONObject> fetchEs(String companyName) {
|
|
|
+ return CompletableFuture.supplyAsync(() -> {
|
|
|
+ try {
|
|
|
+ List<String> ids = queryCaseByCompanyName(companyName);
|
|
|
+ return new JSONObject()
|
|
|
+ .fluentPut("company_name", companyName)
|
|
|
+ .fluentPut("ids", ids);
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ return new JSONObject();
|
|
|
+ }
|
|
|
+ }, BaseUtils.COMMON_POOL);
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ private CompletableFuture<List<JSONObject>> fetchHbaseById(JSONObject keys) {
|
|
|
+ String tableName = "NG_RT_COMPANY";
|
|
|
+ return CompletableFuture.supplyAsync(() -> {
|
|
|
+ List<JSONObject> re = new ArrayList<>();
|
|
|
+ List<Get> gs = keys.getJSONArray("ids").toJavaList(String.class)
|
|
|
+ .stream().
|
|
|
+ distinct().map(vk -> vk.getBytes(StandardCharsets.UTF_8))
|
|
|
+ .map(Get::new).collect(Collectors.toList());
|
|
|
+
|
|
|
+ try (val table = connection.getTable(TableName.valueOf(tableName))) {
|
|
|
+ val rs = table.get(gs);
|
|
|
+ if (rs != null) {
|
|
|
+ re = Stream.of(rs)
|
|
|
+ .filter(result -> result != null && !result.isEmpty())
|
|
|
+ .map(BaseUtils::toJSONObjectLowerCase)
|
|
|
+ .map(data -> data.fluentPut("bg_name", keys.getString("company_name")))
|
|
|
+ .collect(Collectors.toList());
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ throw new RuntimeException("fetchHbase error" + e.getMessage(), e);
|
|
|
+ }
|
|
|
+ return re;
|
|
|
+ }, BaseUtils.COMMON_POOL);
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<String> queryCaseByCompanyName(String companyName) throws IOException {
|
|
|
+ String dsl = "{\n" +
|
|
|
+ " \"query\": {\n" +
|
|
|
+ " \"bool\": {\n" +
|
|
|
+ " \"must\": [\n" +
|
|
|
+ " {\n" +
|
|
|
+ " \"term\": {\n" +
|
|
|
+ " \"bg_name.name.keyword\": {\n" +
|
|
|
+ " \"value\": \"&company_name\"\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " },\n" +
|
|
|
+ " {\n" +
|
|
|
+ " \"term\": {\n" +
|
|
|
+ " \"deleted\": {\n" +
|
|
|
+ " \"value\": \"0\"\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " },\n" +
|
|
|
+ " {\n" +
|
|
|
+ " \"exists\": {\n" +
|
|
|
+ " \"field\": \"yg_name.litigant_id\"\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " ],\n" +
|
|
|
+ " \"must_not\": [\n" +
|
|
|
+ " {\n" +
|
|
|
+ " \"term\": {\n" +
|
|
|
+ " \"yg_name.litigant_id\": {\n" +
|
|
|
+ " \"value\": \"\"\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " }\n" +
|
|
|
+ " ]\n" +
|
|
|
+ " }\n" +
|
|
|
+ " },\n" +
|
|
|
+ " \"_source\": {\n" +
|
|
|
+ " \"includes\": [\n" +
|
|
|
+ " \"bg_name\",\n" +
|
|
|
+ " \"yg_name\"\n" +
|
|
|
+ " ],\n" +
|
|
|
+ " \"excludes\": []\n" +
|
|
|
+ " }\n" +
|
|
|
+ "}";
|
|
|
+
|
|
|
+ String ds1 = dsl.replaceAll("&company_name", companyName);
|
|
|
+ SearchRequest searchRequest = new SearchRequest("winhc_judicial_case_v9").types("_doc");
|
|
|
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
|
|
+ searchSourceBuilder.size(10000);
|
|
|
+ if (StringUtils.isNotEmpty(ds1)) {
|
|
|
+ WrapperQueryBuilder query = QueryBuilderUtils.getQuery(JSONObject.parseObject(ds1));
|
|
|
+ searchSourceBuilder.query(query);
|
|
|
+ }
|
|
|
+ searchRequest.source(searchSourceBuilder);
|
|
|
+ SearchResponse searchResponse = getClient.search(searchRequest);
|
|
|
+ List<String> ids = Arrays.stream(searchResponse.getHits().getHits())
|
|
|
+ .flatMap(x -> {
|
|
|
+ Map<String, Object> sourceAsMap = x.getSourceAsMap();
|
|
|
+ String str = JSONObject.toJSONString(sourceAsMap);
|
|
|
+ JSONObject c = JSONObject.parseObject(str);
|
|
|
+ return c.getJSONArray("yg_name").toJavaList(JSONObject.class)
|
|
|
+ .stream()
|
|
|
+ .map(m -> m.getString("litigant_id"))
|
|
|
+ .filter(d -> StringUtils.isNotBlank(d) && d.length() == 32).distinct()
|
|
|
+ ;
|
|
|
+ }).collect(Collectors.toList());
|
|
|
+ return ids;
|
|
|
+ }
|
|
|
+}
|