#!/usr/bin/env python # -*- coding: utf-8 -*- """ @author: Deepcold @file: fromEs.py @time: 2019/8/3 14:49 """ import datetime import json from bin.common.esQuery import es_query from bin.common.handleDate import set_date from bin.db.parseDate.parseData import parse_data from bin.db.parseDate.parseData_YG import parse_data_yg class GetDataFromEs(object): def __init__(self, db_config, push_date): self.db_config = db_config self.save_items = [] self.push_date = push_date self.table_name = self.db_config.table_name self.query_condition = self.db_config.query_condition def get_data(self, start_date): # 遍历数据表信息 print("正在读取表" + self.table_name) table_info = self.db_config[self.table_name] date = start_date[self.db_config.event_type + self.db_config.event_sub_type + self.table_name] end_date = datetime.datetime.strftime(self.push_date, "%Y-%m-%d") fields = [x for x in table_info.ALL_FIELDS.keys()] es_type = table_info.TYPE query_condition = json.dumps(self.query_condition) query_condition = query_condition.replace("start", date).replace("end", end_date) # 查询审判结果里包含终结执行程序的案件 query = { "_source": fields, "query": json.loads(query_condition)} print(query) query_data = es_query(query, self.table_name, es_type) if query_data: # 查询有结果,将日期更新至最新 set_date(self.db_config.event_type + self.db_config.event_sub_type + self.table_name, datetime.datetime.strftime(self.push_date, "%Y-%m-%d")) # 遍历查询结果 for each in query_data: _id = each["_id"] # 案件id each = each["_source"] # es内容主体 each["_id"] = _id party = self.db_config.party if party == "PLAINTIFF": parse_data_yg(self.db_config, each, self.push_date, self.save_items) else: parse_data(self.db_config, each, self.push_date, self.save_items) return self.save_items