topic_spider/find_source/settings.py

from common.databases import mongo_table, redis_client


'''Mongo'''
MGO_DATABASE = 'shujuziyuan'
'''去重表'''
MGO_REMOVAL_DUPLICATE = mongo_table(db=MGO_DATABASE, name='removal_duplicate')
'''数据挖掘结果表'''
MGO_DOMAIN = mongo_table(db=MGO_DATABASE, name='new_domains')
'''查询结果表'''
MGO_QUERY = mongo_table(db=MGO_DATABASE, name='data_query')
''''数据采集记录表'''
MGO_RECORDS = mongo_table(db=MGO_DATABASE, name='excavate_records')
'''组织|单位表'''
MGO_ORGS = mongo_table(db=MGO_DATABASE, name='retrieve_orgs')
'''搜索词'''
MGO_KEYWORDS = mongo_table(db=MGO_DATABASE, name='retrieve_keywords')
'''种子列表'''
MGO_URLS = mongo_table(db=MGO_DATABASE, name='retrieve_urls')
'''竞品列表'''
MGO_COMPETING_GOODS = mongo_table(db=MGO_DATABASE, name='retrieve_competing_goods')
'''网站栏目表'''
MGO_WEBSITE = mongo_table(db=MGO_DATABASE, name='website')
'''luaconfig'''
MGO_LUA_SPIDERS = mongo_table(db='editor', name='luaconfig')

'''redis'''
REDIS = redis_client()
'''redis键名前缀'''
REDIS_QUERY_KEYWORD = 'query_keyword'
REDIS_QUERY_ORGS = 'query_org'
REDIS_EXCAVATE = 'data_excavate'

'''过滤词'''
FILTER_WORDS = [
    '竞谈', '发包', '比价', '开标', '邀标', '采购', '招标', '中标', '废标', '成交', '单一', '询价',
    '项目结果', '邀请', '磋商', '流标', '谈判', '竞争', '遴选', '比选', '招募', '评标', '资格预审',
    '议价', '中选', '答疑', '合同', '竞价', '变更', '更正', '预告', '集采', '抽取', '抽签',
    '中止公告', '终止公告', '竞卖', '竞买', '论证', '拟建', '审批', '环评'
]

'''搜索引擎屏蔽的网址'''
ENGINE_FEATURE_RETRIEVES = [
    'microsoft.com',
    'cn.bing.com',
    'beian.miit.gov.cn',
    'beian.gov.cn/portal/registerSystemInfo',
    'baike.baidu.com'
]