crawlab_feader/FworkSpider/feapder/utils/log.py

# -*- coding: utf-8 -*-
"""
Created on 2018-12-08 16:50
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
import logging
import os
import sys
import time
from logging.handlers import BaseRotatingHandler

import loguru
import pymongo
from better_exceptions import format_exception

import feapder.setting as setting

LOG_FORMAT = "%(threadName)s|%(asctime)s|%(filename)s|%(funcName)s|line:%(lineno)d|%(levelname)s| %(message)s"
PRINT_EXCEPTION_DETAILS = True


class InterceptHandler(logging.Handler):
    def emit(self, record):
        # Retrieve context where the logging call occurred, this happens to be in the 6th frame upward
        logger_opt = loguru.logger.opt(depth=6, exception=record.exc_info)
        logger_opt.log(record.levelname, record.getMessage())


# 重写 RotatingFileHandler 自定义log的文件名
# 原来 xxx.log xxx.log.1 xxx.log.2 xxx.log.3 文件由近及远
# 现在 xxx.log xxx1.log xxx2.log  如果backup_count 是2位数时  则 01  02  03 三位数 001 002 .. 文件由近及远
class RotatingFileHandler(BaseRotatingHandler):
    def __init__(
        self, filename, mode="a", max_bytes=0, backup_count=0, encoding=None, delay=0
    ):
        BaseRotatingHandler.__init__(self, filename, mode, encoding, delay)
        self.max_bytes = max_bytes
        self.backup_count = backup_count
        self.placeholder = str(len(str(backup_count)))
        self._to_db = None
        self.filename = filename


    @property
    def to_db(self):
        if not self._to_db:
            self._to_db = pymongo.MongoClient(setting.MONGO_IP, setting.MONGO_PORT)

        return self._to_db.pyspider


    def shouldRollover(self, record):
        parmars = {
            "spider_name":record.name,
            "msg":record.msg,
            "Message":str(record.getMessage)
        }
        if record.levelname == "ERROR":
            crawl_type = 'list'
            if 'detail' in record.name:
                crawl_type = 'detail'
            url = ''
            item={
                "recordname":record.name,
                "spidercode":"spidercode",
                "author":self.filename,
                "account":"",
                "crawl_time":time.time(),
                "crawl_type": crawl_type,
                "status_code":"status_code",
                "url":url,
                "reason":record.msg,
                'parmars': parmars,
            }

            # print('<<<<<<<<<<<<<<<<<<<<<<<插入error_info')
            # print(item)
            # print(self.to_db.error_info)
            # self.to_db.error_info.insert_one(item)



def get_logger(
    name=None,
    path=None,
    log_level=None,
    is_write_to_console=None,
    is_write_to_file=None,
    color=None,
    mode=None,
    max_bytes=None,
    backup_count=None,
    encoding=None,
):
    """
    @summary: 获取log
    ---------
    @param name: log名
    @param path: log文件存储路径 如 D://xxx.log
    @param log_level: log等级 CRITICAL/ERROR/WARNING/INFO/DEBUG
    @param is_write_to_console: 是否输出到控制台
    @param is_write_to_file: 是否写入到文件 默认否
    @param color：是否有颜色
    @param mode：写文件模式
    @param max_bytes： 每个日志文件的最大字节数
    @param backup_count：日志文件保留数量
    @param encoding：日志文件编码
    ---------
    @result:
    """
    # 加载setting里最新的值
    name = name or setting.LOG_NAME
    path = path or setting.LOG_PATH
    log_level = log_level or setting.LOG_LEVEL
    is_write_to_console = (
        is_write_to_console
        if is_write_to_console is not None
        else setting.LOG_IS_WRITE_TO_CONSOLE
    )
    is_write_to_file = (
        is_write_to_file
        if is_write_to_file is not None
        else setting.LOG_IS_WRITE_TO_FILE
    )
    color = color if color is not None else setting.LOG_COLOR
    mode = mode or setting.LOG_MODE
    max_bytes = max_bytes or setting.LOG_MAX_BYTES
    backup_count = backup_count or setting.LOG_BACKUP_COUNT
    encoding = encoding or setting.LOG_ENCODING

    # logger 配置
    name = name.split(os.sep)[-1].split(".")[0]  # 取文件名

    logger = logging.getLogger(name)
    logger.setLevel(log_level)

    formatter = logging.Formatter(LOG_FORMAT)
    if PRINT_EXCEPTION_DETAILS:
        formatter.formatException = lambda exc_info: format_exception(*exc_info)

    # 定义一个RotatingFileHandler，最多备份5个日志文件，每个日志文件最大10M
    if is_write_to_file:
        # if path and not os.path.exists(os.path.dirname(path)):
        #     os.makedirs(os.path.dirname(path))

        rf_handler = RotatingFileHandler(
            path,
            mode=mode,
            max_bytes=max_bytes,
            backup_count=backup_count,
            encoding=encoding,
        )
        rf_handler.setFormatter(formatter)
        logger.addHandler(rf_handler)
    if color and is_write_to_console:
        loguru_handler = InterceptHandler()
        loguru_handler.setFormatter(formatter)
        # logging.basicConfig(handlers=[loguru_handler], level=0)
        logger.addHandler(loguru_handler)
    elif is_write_to_console:
        stream_handler = logging.StreamHandler()
        stream_handler.stream = sys.stdout
        stream_handler.setFormatter(formatter)
        logger.addHandler(stream_handler)

    _handler_list = []
    _handler_name_list = []
    # 检查是否存在重复handler
    for _handler in logger.handlers:
        if str(_handler) not in _handler_name_list:
            _handler_name_list.append(str(_handler))
            _handler_list.append(_handler)
    logger.handlers = _handler_list
    return logger


# logging.disable(logging.DEBUG) # 关闭所有log

# 不让打印log的配置
STOP_LOGS = [
    # ES
    "urllib3.response",
    "urllib3.connection",
    "elasticsearch.trace",
    "requests.packages.urllib3.util",
    "requests.packages.urllib3.util.retry",
    "urllib3.util",
    "requests.packages.urllib3.response",
    "requests.packages.urllib3.contrib.pyopenssl",
    "requests.packages",
    "urllib3.util.retry",
    "requests.packages.urllib3.contrib",
    "requests.packages.urllib3.connectionpool",
    "requests.packages.urllib3.poolmanager",
    "urllib3.connectionpool",
    "requests.packages.urllib3.connection",
    "elasticsearch",
    "log_request_fail",
    # requests
    "requests",
    "selenium.webdriver.remote.remote_connection",
    "selenium.webdriver.remote",
    "selenium.webdriver",
    "selenium",
    # markdown
    "MARKDOWN",
    "build_extension",
    # newspaper
    "calculate_area",
    "largest_image_url",
    "newspaper.images",
    "newspaper",
    "Importing",
    "PIL",
]

# 关闭日志打印
for STOP_LOG in STOP_LOGS:
    log_level = eval("logging." + setting.OTHERS_LOG_LEVAL)
    logging.getLogger(STOP_LOG).setLevel(log_level)

# print(logging.Logger.manager.loggerDict) # 取使用debug模块的name

# 日志级别大小关系为：CRITICAL > ERROR > WARNING > INFO > DEBUG


class Log:
    log = None

    def __getattr__(self, name):
        # 调用log时再初始化，为了加载最新的setting
        if self.__class__.log is None:
            self.__class__.log = get_logger()
        return getattr(self.__class__.log, name)

    @property
    def debug(self):
        return self.__class__.log.debug

    @property
    def info(self):
        return self.__class__.log.info

    @property
    def warning(self):
        return self.__class__.log.warning

    @property
    def exception(self):
        return self.__class__.log.exception

    @property
    def error(self):
        return self.__class__.log.error

    @property
    def critical(self):
        return self.__class__.log.critical


log = Log()