浏览代码

luaconfig表迁移

maxiaoshan 2 年之前
父节点
当前提交
afd94024bf
共有 9 个文件被更改,包括 365 次插入83 次删除
  1. 8 1
      src/config.json
  2. 283 0
      src/logs/spider.log
  3. 22 23
      src/main.go
  4. 12 14
      src/spider/front.go
  5. 22 24
      src/spider/handler.go
  6. 1 1
      src/spider/history.go
  7. 2 3
      src/spider/msgservice.go
  8. 3 3
      src/spider/spider.go
  9. 12 14
      src/spider/store.go

+ 8 - 1
src/config.json

@@ -1,9 +1,16 @@
 {
     "webport": "7100",
     "mongodb_spider": "192.168.3.207:27092",
-    "spider_dbsize": 10,
+    "spider_dbsize": 50,
     "mongodb_editor": "192.168.3.207:27092",
     "editor_dbsize": 2,
+    "bideditor": {
+        "addr": "192.168.3.207:27001",
+        "db": "editor",
+        "size": 5,
+        "username": "root",
+        "password": "root"
+    },
     "editoraddr": "http://127.0.0.1:6011/spider/infos",
     "msgname": "爬虫采集平台7100",
     "msgserveraddr": "spdata.jianyu360.com:803",

+ 283 - 0
src/logs/spider.log

@@ -9175,3 +9175,286 @@ stack traceback:
 2022/08/23 18:42:42 spider.go:779: info  Running Code: a_zghxzbtbdzjypt_cggg Stop: false
 2022/08/23 18:42:42 spider.go:845: info  Thread Info:	Code: a_zghxzbtbdzjypt_cggg 	count: 1 	thread num: 0
 2022/08/23 18:43:18 handler.go:1318: info  更新心跳个数: 1
+2022/08/30 15:59:24 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 15:59:24 main.go:139: debug  7100
+2022/08/30 15:59:24 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 15:59:24 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 16:00:20 handler.go:1284: error  send to editor:  Post "http://127.0.0.1:6011/spider/infos": dial tcp 127.0.0.1:6011: connectex: No connection could be made because the target machine actively refused it.
+2022/08/30 16:00:22 spider.go:180: debug  a_zgzfcgw_cggg_zbgg 中国政府采购网 频率: 30 , 150
+2022/08/30 16:00:47 handler.go:136: info  高性能模式:LUA加载完成
+2022/08/30 16:00:47 handler.go:142: info  总共加载脚本数: 1
+2022/08/30 16:01:01 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 16:01:26 main.go:139: debug  7100
+2022/08/30 16:01:26 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 16:01:26 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 16:01:26 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 16:01:30 spider.go:180: debug  a_zgzfcgw_cggg_zbgg 中国政府采购网 频率: 30 , 150
+2022/08/30 16:01:31 handler.go:136: info  高性能模式:LUA加载完成
+2022/08/30 16:01:31 handler.go:142: info  总共加载脚本数: 1
+2022/08/30 16:01:43 handler.go:1284: error  send to editor:  Post "http://127.0.0.1:6011/spider/infos": dial tcp 127.0.0.1:6011: connectex: No connection could be made because the target machine actively refused it.
+2022/08/30 16:02:26 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 16:02:39 main.go:139: debug  7100
+2022/08/30 16:02:39 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 16:02:39 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 16:02:39 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/08/30 16:03:43 spider.go:1090: info  Detail Download All Thread: 0
+2022/08/30 16:03:43 handler.go:1284: error  send to editor:  Post "http://127.0.0.1:6011/spider/infos": dial tcp 127.0.0.1:6011: connectex: No connection could be made because the target machine actively refused it.
+2022/08/30 16:03:44 spider.go:180: debug  a_zgzfcgw_cggg_zbgg 中国政府采购网 频率: 30 , 150
+2022/08/30 16:03:46 handler.go:136: info  高性能模式:LUA加载完成
+2022/08/30 16:03:46 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:24:18 main.go:139: debug  7100
+2022/09/05 14:24:18 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:24:18 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:24:18 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:24:18 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:24:18 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:24:18 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:26:58 main.go:139: debug  7100
+2022/09/05 14:26:58 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:26:58 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:26:58 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:26:58 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:26:58 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:26:58 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:27:58 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:28:53 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:28:53 main.go:139: debug  7100
+2022/09/05 14:28:53 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:28:53 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:28:53 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:28:53 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:28:53 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:29:56 main.go:139: debug  7100
+2022/09/05 14:29:56 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:29:56 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:29:56 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:29:56 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:29:56 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:29:56 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:31:43 main.go:139: debug  7100
+2022/09/05 14:31:43 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:31:43 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:43 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:43 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:31:43 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:31:43 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:31:51 main.go:139: debug  7100
+2022/09/05 14:31:51 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:31:51 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:51 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:51 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:31:51 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:31:51 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:31:59 main.go:139: debug  7100
+2022/09/05 14:31:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:31:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:31:59 spider.go:180: debug  a_gjggzyjypt_zfcg_zbgs_new 国家公共资源交易平台 频率: 30 , 150
+2022/09/05 14:31:59 handler.go:136: info  高性能模式:LUA加载完成
+2022/09/05 14:31:59 handler.go:142: info  总共加载脚本数: 1
+2022/09/05 14:32:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:33:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:34:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:35:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:36:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:37:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:38:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:39:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:40:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:41:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:42:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:43:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:44:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:45:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:46:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 14:46:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:47:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:48:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:49:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:50:34 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 14:50:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:51:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:52:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:53:28 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 14:53:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:54:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:55:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:56:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:57:43 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 14:57:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:58:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 14:59:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:00:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:01:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 15:01:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:02:05 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:02:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:03:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:04:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:05:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:06:08 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:06:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:07:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:08:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:09:12 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:09:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:10:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:11:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:12:45 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:12:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:13:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:14:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:15:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:16:02 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:16:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 15:16:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:17:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:18:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:19:35 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:19:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:20:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:21:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:22:12 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:22:34 handler.go:415: info  下架脚本 ah_fysggzyjyzx_fscg
+2022/09/05 15:22:34 handler.go:464: info  ah_fysggzyjyzx_fscg 脚本下架成功
+2022/09/05 15:22:34 handler.go:729: info  上下架: true <nil>
+2022/09/05 15:22:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:23:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:24:54 handler.go:415: info  下架脚本 ah_ahhfggzyjyzx_qtjy
+2022/09/05 15:24:54 handler.go:464: info  ah_ahhfggzyjyzx_qtjy 脚本下架成功
+2022/09/05 15:24:54 handler.go:729: info  上下架: true <nil>
+2022/09/05 15:24:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:25:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:26:28 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:26:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:27:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:28:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:29:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:30:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:31:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 15:31:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:32:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:33:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:34:40 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:34:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:35:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:36:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:37:28 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:37:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:38:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:39:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:40:29 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:40:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:41:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:42:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:43:51 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:43:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:44:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:45:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:46:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 15:46:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:47:58 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:47:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:48:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:49:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:50:44 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:50:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:51:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:52:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:53:27 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:53:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:54:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:55:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:56:12 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:56:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:57:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:58:46 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 15:58:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 15:59:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:00:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:01:16 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:01:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 16:01:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:02:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:03:47 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:03:59 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:05:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:06:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:06:34 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:07:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:08:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:09:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:09:14 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:10:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:11:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:12:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:13:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:13:54 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:14:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:15:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:16:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:16:24 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:16:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 16:17:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:18:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:19:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:19:15 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:20:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:21:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:21:51 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:22:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:23:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:24:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:24:22 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:25:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:26:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:27:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:27:46 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:28:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:29:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:30:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:31:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:31:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 16:32:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:33:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:34:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:35:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:36:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:37:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:38:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:39:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:40:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:41:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:41:12 handler.go:415: info  下架脚本 hl_hebsngqzfcgw_zhbcjgg
+2022/09/05 16:41:12 handler.go:464: info  hl_hebsngqzfcgw_zhbcjgg 脚本下架成功
+2022/09/05 16:41:12 handler.go:729: info  上下架: true <nil>
+2022/09/05 16:41:13 handler.go:415: info  下架脚本 hl_hebsngqzfcgw_cggg
+2022/09/05 16:41:13 handler.go:464: info  hl_hebsngqzfcgw_cggg 脚本下架成功
+2022/09/05 16:41:13 handler.go:729: info  上下架: true <nil>
+2022/09/05 16:42:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:43:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:44:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:45:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:45:17 download.go:129: error  a_gjggzyjypt_zfcg_zbgs_new方法DownloadAdv,url:http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp,err:timeout 150
+2022/09/05 16:46:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:46:59 handler.go:405: info  节点 7100 脚本文件爬虫数 0
+2022/09/05 16:47:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:48:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:49:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:50:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:51:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:51:38 handler.go:415: info  下架脚本 yn_hhzggzyjydzfwxt_zfcg_bgtz
+2022/09/05 16:51:38 handler.go:464: info  yn_hhzggzyjydzfwxt_zfcg_bgtz 脚本下架成功
+2022/09/05 16:51:38 handler.go:729: info  上下架: true <nil>
+2022/09/05 16:51:49 handler.go:415: info  下架脚本 yn_hhzggzyjydzfwxt_zfcg_cggg
+2022/09/05 16:51:49 handler.go:464: info  yn_hhzggzyjydzfwxt_zfcg_cggg 脚本下架成功
+2022/09/05 16:51:49 handler.go:729: info  上下架: true <nil>
+2022/09/05 16:52:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:53:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:54:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:55:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:56:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:57:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:58:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 16:59:00 spider.go:1090: info  Detail Download All Thread: 0
+2022/09/05 17:00:00 spider.go:1090: info  Detail Download All Thread: 0

+ 22 - 23
src/main.go

@@ -11,7 +11,6 @@ import (
 	mgo "mongodb"
 	qu "qfw/util"
 	es "qfw/util/elastic"
-	mgu "qfw/util/mongodbutil"
 	"regexp"
 	"runtime"
 	. "spiderutil"
@@ -30,21 +29,7 @@ func init() {
 	//初始化lua重载函数
 	lua.Disablelib(Config.Luadisablelib)
 	//初始化数据库
-	initConfig(Config.Mongodb_spider, "spider", "spider", Config.Spider_dbsize)
-	initConfig(Config.Mongodb_editor, "editor", "editor", Config.Editor_dbsize)
-	mgu.InitMongodbPool()
-	spider.MgoS = &mgo.MongodbSim{
-		MongodbAddr: Config.Mongodb_spider,
-		Size:        50,
-		DbName:      "spider",
-	}
-	spider.MgoS.InitPool()
-	spider.MgoE = &mgo.MongodbSim{
-		MongodbAddr: Config.Mongodb_editor,
-		Size:        Config.Editor_dbsize,
-		DbName:      "editor",
-	}
-	spider.MgoE.InitPool()
+	InitMgo()
 	spider.InitOther() //加载其他信息
 	//验证码识别client
 	codegrpc.InitCodeGrpcClient()
@@ -98,13 +83,27 @@ func init() {
 }
 
 //初始化数据源
-func initConfig(addr, alias, db string, dbsize int) {
-	conf := *new(mgu.PoolConfig)
-	conf.Addr = addr
-	conf.Alias = alias
-	conf.DB = db
-	conf.Size = dbsize
-	mgu.Config = append(mgu.Config, conf)
+func InitMgo() {
+	spider.MgoS = &mgo.MongodbSim{
+		MongodbAddr: Config.Mongodb_spider,
+		Size:        Config.Spider_dbsize,
+		DbName:      "spider",
+	}
+	spider.MgoS.InitPool()
+	spider.MgoE = &mgo.MongodbSim{
+		MongodbAddr: Config.Mongodb_editor,
+		Size:        Config.Editor_dbsize,
+		DbName:      "editor",
+	}
+	spider.MgoE.InitPool()
+	spider.MgoEB = &mgo.MongodbSim{
+		MongodbAddr: Config.BidEditor.Addr,
+		Size:        Config.BidEditor.Size,
+		DbName:      Config.BidEditor.Db,
+		UserName:    Config.BidEditor.Username,
+		Password:    Config.BidEditor.Password,
+	}
+	spider.MgoEB.InitPool()
 }
 
 //

+ 12 - 14
src/spider/front.go

@@ -7,8 +7,6 @@ import (
 	mu "mfw/util"
 	"os"
 	"qfw/util"
-	mgu "qfw/util/mongodbutil"
-
 	"sort"
 	lu "spiderutil"
 	"strings"
@@ -49,7 +47,7 @@ const (
 //调用保存信息接口
 func (i *Front) Sendinfo() error {
 	id := i.GetString("id")
-	info := mgu.FindById("data_bak", "spider", "spider", id, nil)
+	info, _ := MgoS.FindById("data_bak", id, nil)
 	delete(*info, "_id")
 	tmp, _ := json.Marshal([]interface{}{"title", []interface{}{&info}})
 	bs, err := Msclient.Call("", mu.UUID(8), 4002, mu.SENDTO_TYPE_ALL_RECIVER, tmp, 30)
@@ -83,9 +81,9 @@ func (i *Front) Login() error {
 	name := i.GetString("s_name")
 	pass := i.GetString("s_pwd")
 	if name != "" && pass != "" {
-		tmp := *mgu.FindOne("spider_admin", "spider", "spider", `{"loginname":"`+name+`","password":"`+util.GetMd5String(pass)+`"}`)
-		if len(tmp) > 0 {
-			i.SetSession("loginuser", tmp)
+		tmp, _ := MgoS.FindOne("spider_admin", map[string]interface{}{"loginname": name, "password": util.GetMd5String(pass)})
+		if len(*tmp) > 0 {
+			i.SetSession("loginuser", *tmp)
 		}
 		return i.Redirect("/front")
 	} else {
@@ -518,9 +516,9 @@ func (i *Front) SearchJob() error {
 //错误查看
 func (i *Front) ViewErrIndex(code string) error {
 	i.T["code"] = code
-	tmp := *mgu.FindOne("spider_ldtime", "spider", "spider", `{"code":"`+code+`"}`)
-	if len(tmp) > 1 {
-		i.T["lastpubtime"] = tmp["lastpubtimestr"]
+	tmp, _ := MgoS.FindOne("spider_ldtime", map[string]interface{}{"code": code})
+	if len(*tmp) > 1 {
+		i.T["lastpubtime"] = (*tmp)["lastpubtimestr"]
 	}
 	return i.Render("viewErr.html", &i.T)
 }
@@ -544,14 +542,14 @@ func (i *Front) ViewErrDetail() error {
 		res["lowlimit"] = v.LowerLimit
 		res["uplimit"] = v.UpperLimit
 	}
-	list := *mgu.Find("spider_errlog", "spider", "spider", `{"code":"`+code+`"}`, `{"comeintime":-1}`, `{"comeintime":1,"url":1,"content":1}`, false, -1, -1)
-	for _, v := range list {
+	list, _ := MgoS.Find("spider_errlog", map[string]interface{}{"code": code}, map[string]interface{}{"comeintime": -1}, map[string]interface{}{"comeintime": 1, "url": 1, "content": 1}, false, -1, -1)
+	for _, v := range *list {
 		v["comeintime"] = time.Unix(util.Int64All(v["comeintime"]), 0).Format(util.Date_Full_Layout)
 		if v["url"] == nil {
 			v["url"] = "监控错误日志"
 		}
 	}
-	res["list"] = list
+	res["list"] = *list
 	i.ServeJson(&res)
 	return nil
 }
@@ -565,7 +563,7 @@ func (i *Front) ClearErr() error {
 	if value, ok := Allspiders.Load(code); ok {
 		v := value.(*Spider)
 		atomic.StoreInt32(&v.ErrorNum, 0)
-		mgu.Del("spider_errlog", "spider", "spider", `{"code":"`+code+`"}`)
+		MgoS.Del("spider_errlog", map[string]interface{}{"code": code})
 	}
 	res := make(map[string]interface{})
 	res["qcode"] = qcode
@@ -598,7 +596,7 @@ func (i *Front) UpdateDownLimit() error {
 		time, _ := time.ParseInLocation(util.Date_Full_Layout, lastpubtime, time.Local)
 		updata = map[string]interface{}{"$set": map[string]interface{}{"uplimit": uplimit, "lowlimit": lowlimit, "lastpubtime": time.Unix(), "lastpubtimestr": lastpubtime}}
 	}
-	b := mgu.Update("spider_ldtime", "spider", "spider", `{"code":"`+code+`"}`, updata, true, false)
+	b := MgoS.Update("spider_ldtime", map[string]interface{}{"code": code}, updata, true, false)
 	if b {
 		if value, ok := Allspiders.Load(code); ok {
 			v := value.(*Spider)

+ 22 - 24
src/spider/handler.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"path/filepath"
 	qu "qfw/util"
-	mgu "qfw/util/mongodbutil"
 	"regexp"
 	util "spiderutil"
 	"strings"
@@ -20,7 +19,6 @@ import (
 
 	"github.com/donnie4w/go-logger/logger"
 	"github.com/yuin/gopher-lua"
-	. "gopkg.in/mgo.v2/bson"
 )
 
 var SpiderHeart sync.Map = sync.Map{} //爬虫心跳
@@ -64,7 +62,7 @@ func InitSpider() {
 
 //高性能模式
 func NoQueueScript() {
-	list := *mgu.Find("spider_ldtime", "spider", "spider", ``, nil, `{"code":1,"uplimit":1,"lowlimit":1}`, false, -1, -1)
+	list, _ := MgoS.Find("spider_ldtime", nil, nil, map[string]interface{}{"code": 1, "uplimit": 1, "lowlimit": 1}, false, -1, -1)
 	LoopListPath.Range(func(key, temp interface{}) bool {
 		if info, ok := temp.(map[string]string); ok {
 			code := info["code"]
@@ -82,7 +80,7 @@ func NoQueueScript() {
 				sp.MUserName = info["modifyuser"]
 				sp.MUserEmail = info["modifyemail"]
 				Allspiders.Store(sp.Code, sp)
-				for _, tmp := range list {
+				for _, tmp := range *list {
 					if qu.ObjToString(tmp["code"]) == sp.Code {
 						sp.UpperLimit = qu.IntAll(tmp["uplimit"])
 						//sp2.UpperLimit = qu.IntAll(tmp["uplimit"])
@@ -110,7 +108,7 @@ func NoQueueScript() {
 				if sp != nil {
 					username = sp.MUserName
 				}
-				mgu.Update("spider_loadfail", "spider", "spider",
+				MgoS.Update("spider_loadfail",
 					map[string]interface{}{
 						"code": code,
 						"modifytime": map[string]interface{}{
@@ -189,7 +187,7 @@ func QueueUpScriptList() {
 						if sp != nil {
 							username = sp.MUserName
 						}
-						mgu.Update("spider_loadfail", "spider", "spider",
+						MgoS.Update("spider_loadfail",
 							map[string]interface{}{
 								"code": code,
 								"modifytime": map[string]interface{}{
@@ -212,7 +210,7 @@ func QueueUpScriptList() {
 					if sp != nil && sp.IsHistoricalMend { //下载历史的爬虫执行一次后删除
 						DelLen++
 						LoopListPath.Delete(key)
-						b := mgu.Update("luaconfig", "editor", "editor", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"state": 6}}, false, false)
+						MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"state": 6}}, false, false)
 						logger.Debug("Delete History Code:", code, b)
 					}
 				}
@@ -298,16 +296,16 @@ func QueueUpScriptDetail() {
 //获取所有爬虫脚本--数据库
 func getSpiderScriptDB(code string) map[string]map[string]string {
 	scriptSpider := map[string]map[string]string{}
-	query := ``
+	query := map[string]interface{}{}
 	if code == "all" { //初始化所有脚本
-		query = `{"state":5,"event":` + fmt.Sprint(util.Config.Uploadevent) + `}`
+		query = map[string]interface{}{"state": 5, "event": util.Config.Uploadevent}
 	} else { //消息在线上传
-		query = `{"code":"` + code + `","event":` + fmt.Sprint(util.Config.Uploadevent) + `}`
+		query = map[string]interface{}{"code": code, "event": util.Config.Uploadevent}
 		//query = `{"$or":[{"iupload":1},{"iupload":3}],"event":` + fmt.Sprint(util.Config.Uploadevent) + `,"modifytime":{"$gt":1502937042}}`
 	}
-	listdb := mgu.Find("luaconfig", "editor", "editor", query, `{"_id":-1}`, nil, false, -1, -1)
+	listdb, _ := MgoEB.Find("luaconfig", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
 	//临时历史附件
-	//listdb := mgu.Find("luaconfig_test", "editor", "editor", query, `{"_id":-1}`, nil, false, -1, -1)
+	//listdb, _ := MgoEB.Find("luaconfig_test", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
 
 	for _, v := range *listdb {
 		old := qu.IntAll(v["old_lua"])
@@ -653,7 +651,7 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 					} else {
 						err = errors.New("新增失败")
 						nowT := time.Now().Unix()
-						mgu.Update("spider_loadfail", "spider", "spider",
+						MgoS.Update("spider_loadfail",
 							map[string]interface{}{
 								"code": k,
 								"modifytime": map[string]interface{}{
@@ -801,7 +799,7 @@ func ReloadSpiderFile() {
 				} else {
 					if i == 0 {
 						nowT := time.Now().Unix()
-						mgu.Update("spider_loadfail", "spider", "spider",
+						MgoS.Update("spider_loadfail",
 							map[string]interface{}{
 								"code": k,
 								"modifytime": map[string]interface{}{
@@ -1068,11 +1066,11 @@ func NewSpiderForThread(code, luafile string) (*Spider, string) {
 //下载量入库
 func SaveDownCount(code string, addtotal bool, todayDowncount, todayRequestNum, yesterdayDowncount, yestoDayRequestNum int32) {
 	date := time.Unix(time.Now().Unix(), 0).Format(qu.Date_Short_Layout)
-	updata := M{}
+	updata := map[string]interface{}{}
 	if addtotal {
-		updata = M{
-			"$inc": M{"totaldown": todayDowncount, "totalreq": todayRequestNum},
-			"$set": M{
+		updata = map[string]interface{}{
+			"$inc": map[string]interface{}{"totaldown": todayDowncount, "totalreq": todayRequestNum},
+			"$set": map[string]interface{}{
 				"yesdowncount":   yesterdayDowncount,
 				"yesdownreq":     yestoDayRequestNum,
 				"todaydowncount": todayDowncount,
@@ -1084,8 +1082,8 @@ func SaveDownCount(code string, addtotal bool, todayDowncount, todayRequestNum,
 			},
 		}
 	} else {
-		updata = M{
-			"$set": M{
+		updata = map[string]interface{}{
+			"$set": map[string]interface{}{
 				"yesdowncount":   yesterdayDowncount,
 				"yesdownreq":     yestoDayRequestNum,
 				"todaydowncount": todayDowncount,
@@ -1097,14 +1095,14 @@ func SaveDownCount(code string, addtotal bool, todayDowncount, todayRequestNum,
 			},
 		}
 	}
-	mgu.Update("spider_downlog", "spider", "spider", M{"code": code, "date": date}, updata, true, false)
+	MgoS.Update("spider_downlog", map[string]interface{}{"code": code, "date": date}, updata, true, false)
 }
 
 //获取下载的上下限(没用)
 func GetLimitDownload(code string) (uplimit, lowlimit int) {
 	defer mu.Catch()
-	ret := mgu.FindOne("spider_ldtime", "spider", "spider", `{"code":"`+code+`"}`)
-	if *ret != nil {
+	ret, _ := MgoS.FindOne("spider_ldtime", map[string]interface{}{"code": code})
+	if ret != nil && len(*ret) > 0 {
 		uplimit = qu.IntAll((*ret)["uplimit"])
 		lowlimit = qu.IntAll((*ret)["lowlimit"])
 		return uplimit, lowlimit
@@ -1344,7 +1342,7 @@ func SpiderCodeSendToEditor(code string) {
 		}
 	}
 	logger.Info("Code:", code, "	Send Move Event:", ok)
-	mgu.Save("luamovelog", "editor", "editor", map[string]interface{}{
+	MgoE.Save("luamovelog", map[string]interface{}{
 		"code":       code,
 		"comeintime": time.Now().Unix(),
 		"type":       "sendfail",

+ 1 - 1
src/spider/history.go

@@ -157,7 +157,7 @@ func GetHistoryDownloadSpider() {
 			count := qu.IntAll(tmp["count"])
 			logger.Info("code:", code, "	当前待采集历史信息量:", count)
 			//查询爬虫信息
-			lua, _ := MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+			lua, _ := MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 			if len(*lua) > 0 {
 				state := qu.IntAll((*lua)["state"])
 				if state >= 7 || state == 4 { //已作废、无发布、需登录、无法处理、已删除、已上线(python)这些状态爬虫的数据不再下载

+ 2 - 3
src/spider/msgservice.go

@@ -8,7 +8,6 @@ import (
 	"math/rand"
 	mu "mfw/util"
 	qu "qfw/util"
-	mgu "qfw/util/mongodbutil"
 	"strings"
 
 	util "spiderutil"
@@ -241,7 +240,7 @@ func SaveObjBlak(event int, checkAtrr string, c string, data []map[string]interf
 			}
 			logger.Error("未成功传送信息-批量", event, len(data), data[0]["spidercode"])
 		}
-		mgu.SaveBulk("data_bak", "spider", "spider", data...)
+		MgoS.SaveBulk("data_bak", data...)
 	}
 }
 
@@ -286,7 +285,7 @@ func SaveObj(event int, checkAtrr string, data map[string]interface{}, saveredis
 				data["biddingid"] = arr[0]
 				data["biddingcoll"] = arr[1]
 			}
-			mgu.Save("data_bak", "spider", "spider", data)
+			MgoS.Save("data_bak", data)
 			if !isExists {
 				util.RedisClusterSet(hashHref, "", -1)
 			}

+ 3 - 3
src/spider/spider.go

@@ -14,7 +14,6 @@ import (
 	mu "mfw/util"
 	mgo "mongodb"
 	qu "qfw/util"
-	mgu "qfw/util/mongodbutil"
 	"strconv"
 	"sync"
 
@@ -92,6 +91,7 @@ var EsIndex string
 var EsType string
 var MgoS *mgo.MongodbSim
 var MgoE *mgo.MongodbSim
+var MgoEB *mgo.MongodbSim
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)   //更新要重下数据的状态
 var UpdataHeartCache = make(chan []map[string]interface{}, 1000) //更新爬虫心跳信息
 var SaveMgoCache = make(chan map[string]interface{}, 1000)       //保存爬虫采集非本站点数据
@@ -207,7 +207,7 @@ func (s *Spider) ExecJob(reload bool) {
 			//if s.IsMustDownload { //历史数据下载,只跑一轮
 			if s.IsHistoricalMend && util.Config.IsHistoryEvent { //历史节点7000,高性能模式,历史补漏只下载一轮
 				UpdateSpiderByCodeState(s.Code, "6") //爬虫在该节点下架
-				b := mgu.Update("luaconfig", "editor", "editor", map[string]interface{}{"code": s.Code}, map[string]interface{}{"$set": map[string]interface{}{"state": 6}}, false, false)
+				b := MgoEB.Update("luaconfig", map[string]interface{}{"code": s.Code}, map[string]interface{}{"$set": map[string]interface{}{"state": 6}}, false, false)
 				logger.Info("Delete History Code:", s.Code, b)
 			} else {
 				if !s.Stop { //未下架定时执行
@@ -407,7 +407,7 @@ func (s *Spider) DownListPageItem() (errs interface{}) {
 		util.TimeSleepFunc(100*time.Millisecond, TimeSleepChan)
 	}
 	logger.Info(s.Code, "本轮列表页采集详情:", downloadAllNum, repeatAllNum, start)
-	if !util.Config.IsHistoryEvent { //非历史节点统计下载率
+	if !util.Config.IsHistoryEvent && !s.Stop { //非历史节点统计下载率
 		nowTime := time.Now()
 		sDate := qu.FormatDate(&nowTime, qu.Date_Short_Layout)
 		set := map[string]interface{}{

+ 12 - 14
src/spider/store.go

@@ -3,13 +3,11 @@ package spider
 import (
 	"encoding/json"
 	"fmt"
-	mu "mfw/util"
-	"qfw/util"
-	mgu "qfw/util/mongodbutil"
-
 	"github.com/cron"
 	"github.com/donnie4w/go-logger/logger"
 	"github.com/yuin/gopher-lua"
+	mu "mfw/util"
+	"qfw/util"
 
 	//"qfw/util/redis"
 	lu "spiderutil"
@@ -46,7 +44,7 @@ func StoreBlak(mode, event int, c, coverAttr string, data []map[string]interface
 		}
 	}
 	if mode == 1 {
-		b := mgu.SaveBulk(c, "spider", "spider", data...)
+		b := MgoS.SaveBulk(c, data...)
 		if b {
 			logger.Debug(c, mode, "保存成功")
 		} else {
@@ -66,11 +64,11 @@ func Store(mode, event int, c, coverAttr string, data map[string]interface{}, fl
 	if mode == 1 {
 		if strings.HasPrefix(c, "ecps") { //公示入库标识值
 			data["signvalue"] = int(1)
-			if mgu.Update(c, "spider", "spider", `{"`+coverAttr+`":"`+fmt.Sprint(data["title"])+`"}`, map[string]interface{}{"$set": data}, true, false) {
+			if MgoS.Update(c, map[string]interface{}{"coverAttr": fmt.Sprint(data["title"])}, map[string]interface{}{"$set": data}, true, false) {
 				logger.Debug(c, data["title"], mode, "保存成功")
 			}
 		} else {
-			id := mgu.Save(c, "spider", "spider", data)
+			id := MgoS.Save(c, data)
 			if id != "" {
 				logger.Debug(c, mode, "保存成功")
 			} else {
@@ -102,14 +100,14 @@ func saveVerificationLog(code, name, url, content string) {
 	data["content"] = content
 	data["comeintime"] = time.Now().Unix()
 	data["event"] = lu.Config.Uploadevent
-	mgu.Save("spider_errlog", "spider", "spider", data)
+	MgoS.Save("spider_errlog", data)
 }
 
 //查找信息是否存在
 func findHasExit(c, q string) bool {
 	defer mu.Catch()
-	ret := mgu.FindOne(c, "spider", "spider", q)
-	if *ret != nil {
+	ret, _ := MgoS.FindOne(c, q)
+	if ret != nil && len(*ret) > 0 {
 		return true
 	} else {
 		return false
@@ -122,7 +120,7 @@ var spider_ldtime = map[string]map[string]interface{}{}
 func GetLastPubtime(code string) int64 {
 	defer mu.Catch()
 	if len(spider_ldtime) < 1 {
-		list := mgu.Find("spider_ldtime", "spider", "spider", ``, nil, ``, false, -1, -1)
+		list, _ := MgoS.Find("spider_ldtime", nil, nil, nil, false, -1, -1)
 		for _, v := range *list {
 			spider_ldtime[fmt.Sprint(v["code"])] = v
 		}
@@ -139,7 +137,7 @@ func GetLastPubtime(code string) int64 {
 func GetLastExectime(code string) int64 {
 	defer mu.Catch()
 	if len(spider_ldtime) < 1 {
-		list := mgu.Find("spider_ldtime", "spider", "spider", ``, nil, ``, false, -1, -1)
+		list, _ := MgoS.Find("spider_ldtime", nil, nil, nil, false, -1, -1)
 		for _, v := range *list {
 			spider_ldtime[fmt.Sprint(v["code"])] = v
 		}
@@ -158,7 +156,7 @@ var spider_downlog = map[string]map[string]interface{}{}
 func GetDownloadLast(code, date string) map[string]interface{} {
 	defer mu.Catch()
 	if len(spider_downlog) < 1 {
-		list := mgu.Find("spider_downlog", "spider", "spider", `{"date":"`+date+`"}`, nil, ``, false, -1, -1)
+		list, _ := MgoS.Find("spider_downlog", map[string]interface{}{"date": date}, nil, nil, false, -1, -1)
 		for _, v := range *list {
 			spider_downlog[fmt.Sprint(v["code"])] = v
 		}
@@ -190,7 +188,7 @@ func GcCount() {
 		})
 		//删除15天前的数据
 		//mgu.Del("data_bak", "spider", "spider", `{"comeintime":{"$lt":`+fmt.Sprint(t.Add(-200*24*time.Hour).Unix())+`}}`)
-		mgu.Del("spider_errlog", "spider", "spider", `{"comeintime":{"$lt":`+fmt.Sprint(t.Add(-30*24*time.Hour).Unix())+`}}`)
+		MgoS.Del("spider_errlog", map[string]interface{}{"comeintime": map[string]interface{}{"$lt": t.Add(-30 * 24 * time.Hour).Unix()}})
 		lu.TimeSleepFunc(1*time.Hour, TimeSleepChan)
 	}
 	lu.TimeAfterFunc(30*time.Minute, GcCount, TimeChan)