feat: 添加清理mongo

spider/check_queue.py

@@ -6,6 +6,7 @@
 from utils.rabbit_mq_info import get_all_queue_info
 from utils.datetime_utils import get_now
 import requests as r
+from spider.delete_mongo_history import remove_mongo_history
 
 queue_info = get_all_queue_info()
 cpa_qcc_spider_num = queue_info['cpa_qcc_spider']
@@ -18,7 +19,7 @@ now_date = get_now()
 msg = f'企查查队列剩余数据：{cpa_qcc_spider_num}\n赢火虫队列剩余数据：{cpa_winhc_spider_num}'
 
 res = r.post(base_url + '/ding/send-msg', data=msg.encode('utf-8'))
-print(res.text)
+print(f"发送队列信息到钉钉：{res.text}")
 
 if cpa_qcc_spider_num > 0 or cpa_winhc_spider_num > 0:
     body = {
@@ -27,7 +28,13 @@ if cpa_qcc_spider_num > 0 or cpa_winhc_spider_num > 0:
 
     }
     res = r.post(base_url + '/sms/send?sign=ansdvhhanvisdnhacamjklvrvrsjka', json=body)
-    print(res.text)
+    print(f"发送队列信息到短信：{res.text}")
+    pass
+
+del_info = remove_mongo_history()
+if del_info:
+    res = r.post(base_url + '/ding/send-msg', data=del_info.encode('utf-8'))
+    print(f"发送删除信息到钉钉：{res.text}")
     pass
 
 if __name__ == '__main__':

spider/delete_mongo_history.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# @Time : 2023/1/5 10:00
+# @Author : XuJiakai
+# @File : delete_mongo_history
+# @Software: PyCharm
+
+import datetime
+from project_const import MONGODB_NAME
+from sdk.WinhcAllClient import get_all_client
+
+all_client = get_all_client()
+del_col = all_client.get_mongo_collection(MONGODB_NAME + '_del')
+
+
+def _get_remove_ds(start_ds):
+    start_date = datetime.datetime.strptime(start_ds, "%Y%m%d")
+    last_week = datetime.datetime.today() + datetime.timedelta(days=-7)
+    result = []
+    while start_date < last_week:
+        date_str = start_date.strftime("%Y%m%d")
+        result.append(date_str)
+        start_date += datetime.timedelta(days=1)
+        pass
+    return result
+    pass
+
+
+def remove_mongo_history():
+    res = del_col.find_one()
+    del_info = None
+    if res:
+        start_ds = res['_id'].split('_')[0]
+        result_date = _get_remove_ds(start_ds)
+        del_list = []
+        for i in result_date:
+            print(f"delete prefix: {i} ...")
+            res = del_col.delete_many({"_id": {"$regex": "^" + i}})
+            str = f"deleted prefix: {i} , deleted_count: {res.deleted_count}"
+            del_list.append(str)
+            print(str)
+            pass
+        del_info = '\n'.join(del_list)
+    return del_info
+    pass
+
+
+if __name__ == '__main__':
+    remove_mongo_history()
+    pass