|
|
@@ -10,11 +10,9 @@ Created on 2018-12-08 16:50
|
|
|
import logging
|
|
|
import os
|
|
|
import sys
|
|
|
-import time
|
|
|
from logging.handlers import BaseRotatingHandler
|
|
|
|
|
|
import loguru
|
|
|
-import pymongo
|
|
|
from better_exceptions import format_exception
|
|
|
|
|
|
import feapder.setting as setting
|
|
|
@@ -41,47 +39,38 @@ class RotatingFileHandler(BaseRotatingHandler):
|
|
|
self.max_bytes = max_bytes
|
|
|
self.backup_count = backup_count
|
|
|
self.placeholder = str(len(str(backup_count)))
|
|
|
- self._to_db = None
|
|
|
- self.filename = filename
|
|
|
-
|
|
|
-
|
|
|
- @property
|
|
|
- def to_db(self):
|
|
|
- if not self._to_db:
|
|
|
- self._to_db = pymongo.MongoClient(setting.MONGO_IP, setting.MONGO_PORT)
|
|
|
-
|
|
|
- return self._to_db.pyspider
|
|
|
|
|
|
+ def doRollover(self):
|
|
|
+ if self.stream:
|
|
|
+ self.stream.close()
|
|
|
+ self.stream = None
|
|
|
+ if self.backup_count > 0:
|
|
|
+ fmt = f"%s.%0{self.placeholder}d"
|
|
|
+ for i in range(self.backup_count - 1, 0, -1):
|
|
|
+ sfn = self.rotation_filename(fmt % (self.baseFilename, i))
|
|
|
+ dfn = self.rotation_filename(fmt % (self.baseFilename, i + 1))
|
|
|
+ if os.path.exists(sfn):
|
|
|
+ if os.path.exists(dfn):
|
|
|
+ os.remove(dfn)
|
|
|
+ os.rename(sfn, dfn)
|
|
|
+ dfn = self.rotation_filename(fmt % (self.baseFilename, 1))
|
|
|
+ if os.path.exists(dfn):
|
|
|
+ os.remove(dfn)
|
|
|
+ self.rotate(self.baseFilename, dfn)
|
|
|
+ if not self.delay:
|
|
|
+ self.stream = self._open()
|
|
|
|
|
|
def shouldRollover(self, record):
|
|
|
- parmars = {
|
|
|
- "spider_name":record.name,
|
|
|
- "msg":record.msg,
|
|
|
- "Message":str(record.getMessage)
|
|
|
- }
|
|
|
- if record.levelname == "ERROR":
|
|
|
- crawl_type = 'list'
|
|
|
- if 'detail' in record.name:
|
|
|
- crawl_type = 'detail'
|
|
|
- url = ''
|
|
|
- item={
|
|
|
- "recordname":record.name,
|
|
|
- "spidercode":"spidercode",
|
|
|
- "author":self.filename,
|
|
|
- "account":"",
|
|
|
- "crawl_time":time.time(),
|
|
|
- "crawl_type": crawl_type,
|
|
|
- "status_code":"status_code",
|
|
|
- "url":url,
|
|
|
- "reason":record.msg,
|
|
|
- 'parmars': parmars,
|
|
|
- }
|
|
|
-
|
|
|
- # print('<<<<<<<<<<<<<<<<<<<<<<<插入error_info')
|
|
|
- # print(item)
|
|
|
- # print(self.to_db.error_info)
|
|
|
- # self.to_db.error_info.insert_one(item)
|
|
|
+ if self.stream is None: # delay was set...
|
|
|
+ self.stream = self._open()
|
|
|
|
|
|
+ if self.max_bytes > 0: # are we rolling over?
|
|
|
+ # print('record >>>> ', record)
|
|
|
+ msg = "%s\n" % self.format(record)
|
|
|
+ self.stream.seek(0, 2) # due to non-posix-compliant Windows feature
|
|
|
+ if self.stream.tell() + len(msg) >= self.max_bytes:
|
|
|
+ return 1
|
|
|
+ return 0
|
|
|
|
|
|
|
|
|
def get_logger(
|
|
|
@@ -144,8 +133,8 @@ def get_logger(
|
|
|
|
|
|
# 定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大10M
|
|
|
if is_write_to_file:
|
|
|
- # if path and not os.path.exists(os.path.dirname(path)):
|
|
|
- # os.makedirs(os.path.dirname(path))
|
|
|
+ if path and not os.path.exists(os.path.dirname(path)):
|
|
|
+ os.makedirs(os.path.dirname(path))
|
|
|
|
|
|
rf_handler = RotatingFileHandler(
|
|
|
path,
|
