DailyEvent/bin/db/getData/fromMongo.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: Deepcold
@file: fromMysql.py
@time: 2019/8/3 14:49
"""
import datetime
from pymongo import MongoClient
from bin.common.handleDate import set_date
from bin.db.parseDate.parseData import parse_data


class GetDataFromMongo(object):
    def __init__(self, db_config, push_date):
        self.db_config = db_config
        self.db_info = self.db_config.db_info
        # 连接mongodb
        self.client = MongoClient(self.db_info.HOST, replicaSet=self.db_info.REPLICASET)
        self.client[self.db_info.DBNAME].authenticate(self.db_info.USER, self.db_info.PASSWD)

        self.save_items = []
        self.push_date = push_date
        self.table_name = self.db_config.table_name

    def get_data(self, start_date):
        # 遍历数据表信息
        print("正在读取表" + self.table_name)
        table_info = self.db_config[self.table_name]
        query_words = table_info.QUERY_WORDS  # 查询关键字
        date = start_date[self.db_config.event_type + self.db_config.event_sub_type + self.table_name]
        end_date = datetime.datetime.strftime(self.push_date, "%Y-%m-%d")  # 要查询的结束日期
        db = self.client[self.db_info.DBNAME][self.table_name]
        limit_key = {}
        for temp in table_info.ALL_FIELDS.keys():
            limit_key[temp] = 1
        query = {query_words["start_date"]: {"$gt": date, "$lt": end_date}}
        query_data = db.find(query, limit_key, no_cursor_timeout=True)

        if query_data:
            #     # 查询有结果,将日期更新至最新
            set_date(self.db_config.event_type + self.db_config.event_sub_type + self.table_name,
                     datetime.datetime.strftime(self.push_date, "%Y-%m-%d"))
        # 解析数据
        for each in query_data:
            _id = str(each["_id"])
            each["_id"] = str(each["_id"])
            parse_data(self.db_config, each, self.push_date, self.save_items)

        return self.save_items