DailyEvent/bin/db/getData/fromMysql.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@author: Deepcold
@file: fromMysql.py
@time: 2019/8/3 14:49
"""""
import datetime
import pymysql
from bin.common.handleDate import set_date
from bin.db.parseDate.parseData import parse_data


class GetDataFromMysql(object):
    def __init__(self, db_config, push_date):
        # 连接mysql数据库
        self.db_config = db_config
        db_info = self.db_config.db_info
        connect = pymysql.connect(
            host=db_info.HOST, user=db_info.USER, passwd=db_info.PASSWD,
            db=db_info.DBNAME, charset='utf8', use_unicode=True)
        self.cursor = connect.cursor()
        self.save_items = []
        self.push_date = push_date
        self.table_name = self.db_config.table_name

    def get_data(self, start_date):
        # 遍历数据表信息
        print("正在读取表" + self.table_name)
        table_info = self.db_config[self.table_name]
        query_words = table_info.QUERY_WORDS  # 查询关键字
        date = start_date[self.db_config.event_type + self.db_config.event_sub_type + self.table_name]
        date = "'" + date + "'"  # 要查询的开始日期

        end_date = "'" + datetime.datetime.strftime(self.push_date, "%Y-%m-%d") + "'"  # 要查询的结束日期
        fields = ",".join(table_info.ALL_FIELDS.keys())
        try:
            self.cursor.execute("SELECT %s FROM %s where %s >= %s and %s < %s" % (
                fields, self.table_name, query_words["start_date"], date, query_words["start_date"], end_date))
            query_data = self.cursor.fetchall()
            print("读取完毕")
        except Exception as e:
            query_data = []
            print(e)
        if query_data:
            # 查询有结果,将日期更新至最新
            set_date(self.db_config.event_type + self.db_config.event_sub_type + self.table_name, datetime.datetime.strftime(self.push_date, "%Y-%m-%d"))
        for each in query_data:
            each = list(each)
            each = dict(zip(table_info.ALL_FIELDS.keys(), each))
            each["_id"] = each["id"]
            parse_data(self.db_config, each, self.push_date, self.save_items)

        return self.save_items