|
|
@@ -50,6 +50,11 @@ WEBDRIVER = dict(
|
|
|
service_log_path=os.devnull # 日志路径，默认置空
|
|
|
)
|
|
|
|
|
|
+ITEM_FILTER_ENABLE = True # item 去重
|
|
|
+ITEM_FILTER_SETTING = dict(
|
|
|
+ filter_type=5 # redis去重
|
|
|
+)
|
|
|
+
|
|
|
# 设置代理
|
|
|
PROXY_EXTRACT_API = "http://proxy.spdata.jianyu360.com/proxy/getallip" # 代理提取API ，返回的代理分割符为\r\n
|
|
|
PROXY_ENABLE = True
|
