topic_spider/find_source/crawler/constants.py

from common.databases import mongo_table, redis_client
from crawler.bloom_filter import RedisBloomFilter

__all__ = [
    'MGO_VISIT',
    'RBF',
    'REDIS',
    'DEFAULT_QUERY',
]
'''搜索关键词组与域名的集合表'''
MGO_VISIT = mongo_table('shujuziyuan', 'visit')
'''判重库'''
MGO_FILTER = mongo_table(
    host='192.168.3.167',
    port='27082',
    db='shujuziyuan',
    name='repetition_url'
)
'''redis'''
REDIS = redis_client()
'''redis布隆过滤器'''
RBF = RedisBloomFilter()
RBF.start(1000000000, 0.00001)
'''详情采集标识'''
DEFAULT_QUERY = {}