#!/usr/bin/env python # -*- coding: utf-8 -*- """ @author: Deepcold @file: fromMysql.py @time: 2019/8/3 14:49 """"" import datetime import pymysql from bin.common.handleDate import set_date from bin.db.parseDate.parseData import parse_data class GetDataFromMysql(object): def __init__(self, db_config, push_date): # 连接mysql数据库 self.db_config = db_config db_info = self.db_config.db_info connect = pymysql.connect( host=db_info.HOST, user=db_info.USER, passwd=db_info.PASSWD, db=db_info.DBNAME, charset='utf8', use_unicode=True) self.cursor = connect.cursor() self.save_items = [] self.push_date = push_date self.table_name = self.db_config.table_name def get_data(self, start_date): # 遍历数据表信息 print("正在读取表" + self.table_name) table_info = self.db_config[self.table_name] query_words = table_info.QUERY_WORDS # 查询关键字 date = start_date[self.db_config.event_type + self.db_config.event_sub_type + self.table_name] date = "'" + date + "'" # 要查询的开始日期 end_date = "'" + datetime.datetime.strftime(self.push_date, "%Y-%m-%d") + "'" # 要查询的结束日期 fields = ",".join(table_info.ALL_FIELDS.keys()) try: self.cursor.execute("SELECT %s FROM %s where %s >= %s and %s < %s" % ( fields, self.table_name, query_words["start_date"], date, query_words["start_date"], end_date)) query_data = self.cursor.fetchall() print("读取完毕") except Exception as e: query_data = [] print(e) if query_data: # 查询有结果,将日期更新至最新 set_date(self.db_config.event_type + self.db_config.event_sub_type + self.table_name, datetime.datetime.strftime(self.push_date, "%Y-%m-%d")) for each in query_data: each = list(each) each = dict(zip(table_info.ALL_FIELDS.keys(), each)) each["_id"] = each["id"] parse_data(self.db_config, each, self.push_date, self.save_items) return self.save_items