配置工具包

find_source/config/conf.yaml

@@ -0,0 +1,31 @@
+# mongo
+mongo:
+  host: 172.17.4.87
+  port: !!int 27080
+#  host: 127.0.0.1
+#  port: !!int 27017
+
+
+# redis
+redis:
+  host: 127.0.0.1
+  port: !!int 6379
+  pwd: ""
+  db: !!int 10
+
+
+# es
+es:
+  host: 172.17.145.170
+#  host: 192.168.3.206
+#  host: 127.0.0.1
+  port: !!int 9800
+  db: biddingall # es库别名
+
+
+# 代理
+proxy:
+  socks5:
+    url: http://cc.spdata.jianyu360.com/crawl/proxy/socks5/fetch
+    auth:
+      Authorization: Basic amlhbnl1MDAxOjEyM3F3ZSFB

find_source/config/constants.yaml

@@ -0,0 +1,2 @@
+headers:
+  User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36

find_source/config/load.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+import yaml
+
+__all__ = [
+    'mongo_conf',
+    'redis_conf',
+    'es_conf',
+    'jy_proxy',
+    'node_module_path',
+    'headers',
+    'analyze_url'
+]
+
+_base_path = Path(__file__).parent
+_yaml_conf = (_base_path / 'conf.yaml').resolve()
+_yaml_constants = (_base_path / 'constants.yaml').resolve()
+_node_modules = (_base_path.parent / 'node_modules').resolve()
+
+with open(_yaml_conf, encoding="utf-8") as f:
+    _conf = yaml.safe_load(f)
+    mongo_conf = _conf['mongo']
+    redis_conf = _conf['redis']
+    es_conf: dict = _conf['es']
+    jy_proxy: dict = _conf['proxy']
+    node_module_path = _node_modules
+    analyze_url = f'http://{es_conf["host"]}:{es_conf["port"]}/{es_conf["db"]}/_analyze'
+
+
+with open(_yaml_constants, encoding="utf-8") as fp:
+    _constants = yaml.safe_load(fp)
+    headers: dict = _constants['headers']