DailyEvent/bin/db/getData/fromEs.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: Deepcold
@file: fromEs.py
@time: 2019/8/3 14:49
"""
import datetime
import json
from bin.common.esQuery import es_query
from bin.common.handleDate import set_date
from bin.db.parseDate.parseData import parse_data
from bin.db.parseDate.parseData_YG import parse_data_yg


class GetDataFromEs(object):
    def __init__(self, db_config, push_date):
        self.db_config = db_config
        self.save_items = []
        self.push_date = push_date
        self.table_name = self.db_config.table_name
        self.query_condition = self.db_config.query_condition

    def get_data(self, start_date):
        # 遍历数据表信息
        print("正在读取表" + self.table_name)
        table_info = self.db_config[self.table_name]
        date = start_date[self.db_config.event_type + self.db_config.event_sub_type + self.table_name]
        end_date = datetime.datetime.strftime(self.push_date, "%Y-%m-%d")
        fields = [x for x in table_info.ALL_FIELDS.keys()]
        es_type = table_info.TYPE
        query_condition = json.dumps(self.query_condition)
        query_condition = query_condition.replace("start", date).replace("end", end_date)

        # 查询审判结果里包含终结执行程序的案件
        query = {
            "_source": fields,
            "query": json.loads(query_condition)}
        print(query)
        query_data = es_query(query, self.table_name, es_type)
        if query_data:
            # 查询有结果,将日期更新至最新
            set_date(self.db_config.event_type + self.db_config.event_sub_type + self.table_name, datetime.datetime.strftime(self.push_date, "%Y-%m-%d"))
        # 遍历查询结果
        for each in query_data:
            _id = each["_id"]  # 案件id
            each = each["_source"]  # es内容主体
            each["_id"] = _id
            party = self.db_config.party
            if party == "PLAINTIFF":
                parse_data_yg(self.db_config, each, self.push_date, self.save_items)
            else:
                parse_data(self.db_config, each, self.push_date, self.save_items)
        return self.save_items