新增iscompete是否是新爬虫的判断

src/spider/handler.go

@@ -40,7 +40,7 @@ var InitCount int
 var InitAllLuaOver = make(chan bool, 1) //所有脚本是否加载完毕
 
 func InitSpider() {
-	scriptMap := getSpiderScriptDB("all")
+	scriptMap := getSpiderScriptDB("all") //加载爬虫，初始化模板
 	scriptMapFile := getSpiderScriptFile(false)
 	k := 0
 	for _, v := range scriptMap {
@@ -104,6 +104,10 @@ func NoQueueScript() {
 			} else {
 				logger.Info(code, "脚本加载失败，请检查！")
 				nowT := time.Now().Unix()
+				username := "异常"
+				if sp != nil {
+					username = sp.MUserName
+				}
 				mgu.Update("spider_loadfail", "spider", "spider",
 					map[string]interface{}{
 						"code": code,
@@ -118,7 +122,7 @@ func NoQueueScript() {
 							"type":       "初始化",
 							"script":     script,
 							"updatetime": nowT,
-							"modifyuser": sp.MUserName,
+							"modifyuser": username,
 							"event":      util.Config.Uploadevent,
 							"err":        errstr,
 						},
@@ -168,7 +172,6 @@ func QueueUpScriptList() {
 				if !old_is_running { //判断当前爬虫是否正在执行
 					script := info["script"]
 					sp, errstr := NewSpider_New(code, script, false)
-					isHistoricalMend := sp.IsHistoricalMend
 					//logger.Info("初始化脚本是否成功：", sp != nil, e.Value)
 					if errstr == "" && sp != nil && sp.Code != "nil" { //初始化脚本成功
 						sp.Index = qu.IntAll(key)
@@ -184,6 +187,10 @@ func QueueUpScriptList() {
 						sp.StartJob()
 					} else {
 						nowT := time.Now().Unix()
+						username := "异常"
+						if sp != nil {
+							username = sp.MUserName
+						}
 						mgu.Update("spider_loadfail", "spider", "spider",
 							map[string]interface{}{
 								"code": code,
@@ -198,13 +205,13 @@ func QueueUpScriptList() {
 									"type":       "初始化",
 									"script":     script,
 									"updatetime": nowT,
-									"modifyuser": sp.MUserName,
+									"modifyuser": username,
 									"event":      util.Config.Uploadevent,
 									"err":        errstr,
 								},
 							}, true, false)
 					}
-					if isHistoricalMend { //下载历史的爬虫执行一次后删除
+					if sp != nil && sp.IsHistoricalMend { //下载历史的爬虫执行一次后删除
 						DelLen++
 						LoopListPath.Delete(key)
 						b := mgu.Update("luaconfig", "editor", "editor", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"state": 6}}, false, false)
@@ -935,6 +942,8 @@ func NewSpider_New(code, luafile string, newstate bool) (*Spider, string) {
 	//新增历史补漏
 	spider.IsHistoricalMend = spider.GetBoolVar("spiderIsHistoricalMend")
 	spider.IsMustDownload = spider.GetBoolVar("spiderIsMustDownload")
+	//新老爬虫
+	spider.IsCompete = spider.GetBoolVar("spiderIsCompete")
 	return spider, ""
 }
 
@@ -995,7 +1004,8 @@ func NewSpider(code, luafile string) (*Spider, string) {
 	//qu.Debug("-------", spider.GetBoolVar("spiderIsHistoricalMend"), spider.GetBoolVar("spiderIsMustDownload"))
 	spider.IsHistoricalMend = spider.GetBoolVar("spiderIsHistoricalMend")
 	spider.IsMustDownload = spider.GetBoolVar("spiderIsMustDownload")
-
+	//新老爬虫
+	spider.IsCompete = spider.GetBoolVar("spiderIsCompete")
 	return spider, ""
 }
 
@@ -1110,7 +1120,7 @@ func GetScriptByTmp(luaconfig map[string]interface{}) string {
 		} else {
 			script_content = luaconfig["str_content"].(string)
 		}
-		script += fmt.Sprintf(util.Tmp_Other, luaconfig["spidertype"], luaconfig["spiderhistorymaxpage"], luaconfig["spidermovevent"])
+		script += fmt.Sprintf(util.Tmp_Other, luaconfig["spidertype"], luaconfig["spiderhistorymaxpage"], luaconfig["spidermovevent"], luaconfig["spidercompete"])
 		script += ` 
 			` + script_time + `
 			` + script_list + `

src/spider/spider.go

@@ -68,6 +68,7 @@ type Spider struct {
 	//历史补漏
 	IsHistoricalMend bool //是否是历史补漏爬虫
 	IsMustDownload   bool //是否强制下载
+	IsCompete        bool //区分新老爬虫
 }
 
 var UpdataMgoCache = make(chan []map[string]interface{}, 1000)   //更新要重下数据的状态
@@ -418,6 +419,7 @@ func (s *Spider) HistoricalMendDownloadDetailItem(p interface{}) {
 	atomic.AddInt32(&s.TotalDowncount, 1)
 	data["spidercode"] = s.Code
 	//qu.Debug("--------------开始保存---------------")
+	data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 	Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, flag)
 	//qu.Debug("--------------保存结束---------------")
 }
@@ -498,7 +500,7 @@ func (s *Spider) DownloadDetailItem(p interface{}, num *int) {
 	// 	},
 	// })
 	// UpdataMgoCache <- update
-
+	data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 	Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
 	//qu.Debug("-----增量保存结束-----")
 }
@@ -712,6 +714,7 @@ func (s *Spider) DownloadHighDetail() {
 					}
 					data["spidercode"] = s.Code
 					data["dataging"] = 0
+					data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 					Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
 					set := map[string]interface{}{"$set": map[string]interface{}{"state": 1}} //下载成功state置为1
 					Mgo.Update("spider_highlistdata", query, set, false, false)
@@ -832,6 +835,7 @@ func (s *Spider) DownloadListDetail() {
 			}
 			data["spidercode"] = s.Code
 			data["dataging"] = 0
+			data["iscompete"] = s.IsCompete //2021-11-01以后新增的爬虫不在展示原文链接(保存服务判断)
 			Store(s.StoreMode, s.StoreToMsgEvent, s.Collection, s.CoverAttr, data, true)
 			set := map[string]interface{}{"$set": map[string]interface{}{"state": 1}} //下载成功state置为1
 			Mgo.Update("spider_highlistdata", query, set, false, false)