|
|
@@ -70,12 +70,17 @@ WEBDRIVER = dict(
|
|
|
service_log_path=os.devnull # 日志路径
|
|
|
)
|
|
|
|
|
|
+# splash渲染
|
|
|
+SPLASH_API = "http://splash.spdata.jianyu360.com/render.json"
|
|
|
+
|
|
|
# request网络请求超时时间
|
|
|
REQUEST_TIMEOUT = 60
|
|
|
|
|
|
# 设置代理，代理提取API ，返回的代理分割符为\r\n
|
|
|
PROXY_EXTRACT_API = "http://proxy.spdata.jianyu360.com/proxy/getallip"
|
|
|
PROXY_ENABLE = True
|
|
|
+JY_PROXY_URL = "http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch"
|
|
|
+JY_PROXY_AUTHOR = "Basic amlhbnl1MDAxOjEyM3F3ZSFB"
|
|
|
|
|
|
# item去重
|
|
|
ITEM_FILTER_ENABLE = True
|
|
|
@@ -112,13 +117,6 @@ LOG_BACKUP_COUNT = 20 # 日志文件保留数量
|
|
|
LOG_ENCODING = "utf8" # 日志文件编码
|
|
|
OTHERS_LOG_LEVAL = "ERROR" # 第三方库的log等级 一般用不到
|
|
|
|
|
|
-# 自建代理池
|
|
|
-SWORDFISH_PROXY_URL = "http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch"
|
|
|
-SWORDFISH_PROXY_AUTHOR = "Basic amlhbnl1MDAxOjEyM3F3ZSFB"
|
|
|
-
|
|
|
-# splash 渲染服务
|
|
|
-SWORDFISH_RENDER_URL = "http://splash.spdata.jianyu360.com/render.json"
|
|
|
-
|
|
|
# 远程bucket配置
|
|
|
ALI_BUCKET_CONFIG = {
|
|
|
"key_id": "LTAI4G5x9aoZx8dDamQ7vfZi",
|
