新增spidercompete新老爬虫判断

src/spider/handler.go

@@ -95,8 +95,8 @@ func NoQueueScript() {
 			code := info["code"]
 			script := info["script"]
 			sp := NewSpider(code, script)
-			sp.Index = qu.IntAll(key)
 			if sp != nil && sp.Code != "nil" { //脚本加载成功
+				sp.Index = qu.IntAll(key)
 				if info["createuser"] != "" {
 					sp.UserName = info["createuser"]
 				}
@@ -196,7 +196,7 @@ func NewSpider(code, luafile string) *Spider {
 	//qu.Debug("-------", spider.GetBoolVar("spiderIsHistoricalMend"), spider.GetBoolVar("spiderIsMustDownload"))
 	spider.IsHistoricalMend = spider.GetBoolVar("spiderIsHistoricalMend")
 	spider.IsMustDownload = spider.GetBoolVar("spiderIsMustDownload")
-
+	spider.IsCompete = spider.GetBoolVar("spiderIsCompete")
 	return spider
 }
 
@@ -313,7 +313,7 @@ func GetScriptByTmp(luaconfig map[string]interface{}) string {
 		} else {
 			script_content = luaconfig["str_content"].(string)
 		}
-		script += fmt.Sprintf(util.Tmp_Other, luaconfig["spidertype"], luaconfig["spiderhistorymaxpage"], luaconfig["spidermovevent"])
+		script += fmt.Sprintf(util.Tmp_Other, luaconfig["spidertype"], luaconfig["spiderhistorymaxpage"], luaconfig["spidermovevent"], luaconfig["spidercompete"])
 		script += ` 
 			` + script_time + `
 			` + script_list + `

src/spider/spider.go

@@ -58,6 +58,7 @@ type Spider struct {
 	//历史补漏
 	IsHistoricalMend bool //是否是历史补漏爬虫
 	IsMustDownload   bool //是否强制下载
+	IsCompete        bool //区分新老爬虫
 }
 
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000) //更新要重下数据的状态
@@ -211,6 +212,7 @@ func DownloadHighDetail(code string) {
 					}
 					data["spidercode"] = sp.Code
 					data["dataging"] = 0
+					data["iscompete"] = sp.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 					Store(sp.StoreMode, sp.StoreToMsgEvent, sp.Collection, sp.CoverAttr, data, true)
 					set := map[string]interface{}{"$set": map[string]interface{}{"state": 1}} //下载成功state置为1
 					MgoS.Update("spider_highlistdata", query, set, false, false)