crawlab_feader/FworkSpider/mongo_pipeline.py

# -*- coding: utf-8 -*-
"""
Created on 2021-04-18 14:12:21
---------
@summary: 导出数据
---------
@author: 马国鹏
@email:  305021384@qq.com
"""
from typing import Dict, List, Tuple
import time
# from feapder.db.mongodb import MongoDB
from feapder.db.redisdb import RedisDB
from feapder.dedup import Dedup
from feapder.pipelines import BasePipeline
from feapder.utils.log import log
from untils.tools import *
# from crawlab import save_item



class MongoPipeline(BasePipeline):
    def __init__(self):
        self._to_db = None

    @property
    def to_db(self):
        if not self._to_db:
            self._to_db = RedisDB()

        return self._to_db

    def save_items(self, table, items: List[Dict]) -> bool:
        """
        保存数据
        Args:
            table: 表名
            items: 数据，[{},{},...]

        Returns: 是否保存成功 True / False
                 若False，不会将本批数据入到去重库，以便再次入库
        """
        try:
            add_count = self.to_db.lpush(table="savemongo:"+table, values=items)
            # add_count = self.to_db.lpop(table="savemongo:"+table, values=items)
            datas_size = len(items)
            log.info(
                "共导出 %s 条数据到 %s,  新增 %s条, 重复 %s 条"
                % (datas_size, table, len(items), datas_size - len(items))
            )

            return True
        except Exception as e:
            log.exception(e)
            return False