update

find_source/common/databases.py

@@ -153,10 +153,7 @@ def redis_client(cfg=None, host=None, port=None, db=None, password=None):
     port = (port or _cfg['port'])
     password = (password or _cfg['pwd'])
     db = (db or _cfg['db'])
-    pool = redis.ConnectionPool(
-        host=host,
-        port=port,
-        password=password,
-        db=db
-    )
-    return redis.Redis(connection_pool=pool, decode_responses=True)
+    return redis.StrictRedis(host=host,
+                             port=port,
+                             password=password,
+                             db=db, decode_responses=True)

find_source/config/conf.yaml

@@ -1,13 +1,13 @@
 mongo:
-  host: 192.168.20.248
+  host: 192.168.3.182
   port: !!int 27017
 
 
 redis:
-  host: 192.168.3.109
-  port: !!int 16379
-  pwd: 'top@123'
-  db: !!int 3
+  host: 192.168.3.182
+  port: !!int 6379
+  pwd: 'jianyu@python'
+  db: !!int 5
 
 
 es:

find_source/crawler/download.py

@@ -130,7 +130,7 @@ class Downloader:
 class RenderDownloader(Downloader):
 
     def get(self, url, **kw):
-        splash_url = 'http://8.131.72.226:8998/render.json'
+        splash_url = 'http://splash.spdata.jianyu360.com/render.json'
         args = {
             'url': url,
             'html': 1,

find_source/crawler/utils.py

@@ -77,7 +77,7 @@ def extract_page_title(source):
 
 
 def is_url(url):
-    """判断url格式畸形与否"""
+    """判断url格式"""
     _regex = re.compile(
         r'^(?:http|ftp)s?://'  # http:// or https://
         r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...

find_source/settings.py

@@ -22,7 +22,7 @@ MGO_URLS = mongo_table(db=MGO_DATABASE, name='retrieve_urls')
 '''竞品列表'''
 MGO_COMPETING_GOODS = mongo_table(db=MGO_DATABASE, name='retrieve_competing_goods')
 '''luaconfig'''
-MGO_LUA_SPIDERS = mongo_table(host='192.168.20.248', port=27080, db='editor', name='luaconfig')
+MGO_LUA_SPIDERS = mongo_table(host='192.168.3.182', port=27017, db='editor', name='luaconfig')
 
 '''redis'''
 REDIS = redis_client()