crawlab_feader/FworkSpider/items/base_item.py

# -*- coding: utf-8 -*-

import feapder.utils.tools as tools
from feapder import Item


class SwordFishProjectItem(Item):

    def __init__(self):
        super(SwordFishProjectItem, self).__init__(
            save=True  # 是否入库,默认入库=True,不入库=False
        )

    @property
    def save(self) -> bool:
        return self.__dict__["save"]

    @save.setter
    def save(self, enable: bool):
        self.__dict__["save"] = enable

    @property
    def fingerprint(self):
        args = []
        # 1、入库字段去重
        for key, value in self.to_dict.items():
            if value:
                if (self.unique_key and key in self.unique_key) or not self.unique_key:
                    args.append(str(value))

        # 2、入库字段去重_招投标字段
        bidding = self.to_dict.get("item", {})
        for key, value in bidding.items():
            if value:
                if (self.unique_key and key in self.unique_key) or not self.unique_key:
                    args.append(str(value))

        # 3、非入库字段去重
        if self.unique_key:
            for key in filter(lambda x: x not in bidding and x not in self.to_dict, self.unique_key):
                if key not in args:
                    args.append(str(key))

        if args:
            args = sorted(args)
            return tools.get_md5(*args) + "&&" + tools.get_sha256(*args)
        else:
            return None