更新

src/spider/handler.go

@@ -39,7 +39,7 @@ var InitCount int
 var InitAllLuaOver = make(chan bool, 1) //所有脚本是否加载完毕
 
 func InitSpider() {
-	scriptMap := getSpiderScriptDB("all", "online") //加载爬虫，初始化模板
+	scriptMap := getSpiderScriptDB("all") //加载爬虫，初始化模板
 	scriptMapFile := getSpiderScriptFile(false)
 	for code, v := range scriptMap {
 		LoopListPath.Store(code, v)
@@ -294,7 +294,7 @@ func QueueUpScriptDetail() {
 }
 
 // 获取所有爬虫脚本--数据库
-func getSpiderScriptDB(code, stype string) map[string]map[string]string {
+func getSpiderScriptDB(code string) map[string]map[string]string {
 	scriptSpider := map[string]map[string]string{}
 	query := map[string]interface{}{}
 	if Supplement { //数据采集
@@ -312,11 +312,7 @@ func getSpiderScriptDB(code, stype string) map[string]map[string]string {
 		query = map[string]interface{}{"code": code, "event": util.Config.Uploadevent}
 		//query = `{"$or":[{"iupload":1},{"iupload":3}],"event":` + fmt.Sprint(util.Config.Uploadevent) + `,"modifytime":{"$gt":1502937042}}`
 	}
-	coll := "luaconfig"
-	if stype == "online" {
-		coll = "luaconfig_online"
-	}
-	listdb, _ := MgoEB.Find(coll, query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
+	listdb, _ := MgoEB.Find("luaconfig", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
 	//临时历史附件
 	//listdb, _ := MgoEB.Find("luaconfig_test", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
 
@@ -461,7 +457,7 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 		up = true
 		err = nil
 	} else if state == "-1" { //爬虫重采更新线上爬虫
-		scriptMap := getSpiderScriptDB(code, "init")
+		scriptMap := getSpiderScriptDB(code)
 		logger.Info("更新线上脚本,库中是否已存在该脚本:", code, len(scriptMap) > 0, scriptMap[code] != nil)
 		if util.Config.Working == 1 { //排队模式
 			for _, v := range scriptMap {
@@ -530,7 +526,7 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 			}
 		}
 	} else { //脚本上架
-		scriptMap := getSpiderScriptDB(code, "init")
+		scriptMap := getSpiderScriptDB(code)
 		logger.Info("上架新增脚本,库中是否已存在该脚本:", code, len(scriptMap) > 0, scriptMap[code] != nil)
 		if util.Config.Modal == 1 && !util.Config.IsHistoryEvent { //分开采集
 			go UpdateHighListDataByCode(code)

src/spider/script.go

@@ -14,7 +14,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"github.com/shopspring/decimal"
-	gojs "gorunjs/client"
 	"io"
 	"io/ioutil"
 	mu "mfw/util"
@@ -1158,14 +1157,14 @@ func (s *Script) LoadScript(site, channel, user *string, code, script_file strin
 		S.Push(lua.LString(respCookie))
 		return 3
 	}))
-	s.L.SetGlobal("goRunJs", s.L.NewFunction(func(S *lua.LState) int {
-		param := S.ToString(-2) //list or detail
-		step := S.ToString(-1)  //参数
-		result := gojs.GoRunJsGetResult(s.SCode, param, step)
-		qu.Debug("Go Run Js Result:", param, step, result)
-		S.Push(lua.LString(result))
-		return 1
-	}))
+	//s.L.SetGlobal("goRunJs", s.L.NewFunction(func(S *lua.LState) int {
+	//	param := S.ToString(-2) //list or detail
+	//	step := S.ToString(-1)  //参数
+	//	result := gojs.GoRunJsGetResult(s.SCode, param, step)
+	//	qu.Debug("Go Run Js Result:", param, step, result)
+	//	S.Push(lua.LString(result))
+	//	return 1
+	//}))
 	s.L.SetGlobal("newDownloadFile", s.L.NewFunction(func(S *lua.LState) int {
 		cookie := S.ToString(-1)
 		head := S.ToTable(-2)

src/spider/util.go

@@ -18,22 +18,24 @@ func InitOther() {
 	defer qu.Catch()
 	DelaySiteMap = map[string]*DelaySite{}
 	DelaySiteLock = &sync.Mutex{}
-	for {
-		list, _ := MgoS.Find("spider_compete", nil, nil, nil, false, -1, -1)
-		for _, l := range *list {
-			site := qu.ObjToString(l["site"])
-			delayTime := qu.IntAll(l["delaytime"])
-			compete, _ := l["compete"].(bool)
-			DelaySiteLock.Lock()
-			DelaySiteMap[site] = &DelaySite{
-				DelayTime: delayTime,
-				Compete:   compete,
+	go func() {
+		for {
+			list, _ := MgoS.Find("spider_compete", nil, nil, nil, false, -1, -1)
+			for _, l := range *list {
+				site := qu.ObjToString(l["site"])
+				delayTime := qu.IntAll(l["delaytime"])
+				compete, _ := l["compete"].(bool)
+				DelaySiteLock.Lock()
+				DelaySiteMap[site] = &DelaySite{
+					DelayTime: delayTime,
+					Compete:   compete,
+				}
+				DelaySiteLock.Unlock()
 			}
-			DelaySiteLock.Unlock()
+			logger.Info("重载延时采集配置...")
+			time.Sleep(12 * time.Hour)
 		}
-		logger.Info("重载延时采集配置...")
-		time.Sleep(12 * time.Hour)
-	}
+	}()
 }
 
 // 获取第day天凌晨的时间戳