Prechádzať zdrojové kódy

新增配置三级页下载线程数

maxiaoshan 4 rokov pred
rodič
commit
978bea098a
4 zmenil súbory, kde vykonal 5 pridanie a 4 odobranie
  1. 1 0
      src/config.json
  2. 2 2
      src/main.go
  3. 1 1
      src/mainT.go
  4. 1 1
      src/spider/handler.go

+ 1 - 0
src/config.json

@@ -11,6 +11,7 @@
 	"isdelay":false,
     "working": 0,
     "chansize": 4,
+    "detailchansize": 20,
     "uploadevent": 7100,
     "logLevel": 1,
     "daynum": 6,

+ 2 - 2
src/main.go

@@ -1,11 +1,11 @@
 package main
 
 import (
+	_ "filter"
 	"fmt"
 	"io/ioutil"
-	_ "lua/filter"
-	"lua/spider"
 	"os"
+	"spider"
 
 	mgo "mongodb"
 	qu "qfw/util"

+ 1 - 1
src/mainT.go

@@ -4,8 +4,8 @@ package main
 import (
 	"io/ioutil"
 	"log"
-	"lua/spider"
 	"os"
+	"spider"
 	. "spiderutil"
 	"strings"
 )

+ 1 - 1
src/spider/handler.go

@@ -203,7 +203,7 @@ func QueueUpScriptList() {
 //排队模式下载三级页数据
 func QueueUpScriptDetail() {
 	logger.Info("节能模式三级页")
-	chanSize := 2 * util.Config.Chansize
+	chanSize := util.Config.DetailChansize
 	CC2 = make(chan *lua.LState, chanSize)
 	for i := 0; i < chanSize; i++ { //目前初始化Allspiders,Allspiders2两个爬虫池,线程乘2
 		CC2 <- lua.NewState(lua.Options{