topic_spider/find_source/crawler/validate.py

from crawler.bloom_filter.RedisBloomFilter import RedisFilter
from settings import FILTER_WORDS


class Validator:

    def __init__(self, redis_key='Validator_'):
        self._validator_name = redis_key
        self._rbf = RedisFilter(redis_key=self._validator_name)
        self._rbf.start(1000000000, 0.00001)

    @staticmethod
    def _filter_words(val: str):
        """过滤词"""
        for word in FILTER_WORDS:
            if val.find(word) != -1:
                return True
        return False

    def add_data(self, val: str):
        return self._rbf.add(val)

    def data(self, val: str):
        return self._rbf.is_exists(val)

    def words(self, val: str):
        return self._filter_words(val)

    @property
    def length(self):
        return len(self._rbf)