topic_spider/find_source/crawler/validate.py

from crawler.bloom_filter.RedisBloomFilter import RedisFilter
from settings import REQUIREMENT_PHRASE


class Validator:

    def __init__(self, redis_key='Validator_'):
        self._validator_name = redis_key
        self._rbf = RedisFilter(redis_key=self._validator_name)
        self._rbf.start(1000000000, 0.00001)

    @staticmethod
    def _requirement_phrase(val: str):
        """关键词"""
        for word in REQUIREMENT_PHRASE:
            if val.find(word) != -1:
                return True
        return False

    def add_data(self, val: str):
        return self._rbf.add(val)

    def data(self, val: str):
        return self._rbf.is_exists(val)

    def phrase(self, val: str):
        return self._requirement_phrase(val)

    @property
    def length(self):
        return len(self._rbf)