修改加载失败爬虫保存

src/mainT.go

@@ -24,7 +24,7 @@ func main1() {
 		name := arrs[len(arrs)-1]
 		if strings.HasPrefix(name, "spider_") && strings.HasSuffix(name, ".lua") {
 			bs, _ := ioutil.ReadAll(f)
-			sp := spider.NewSpider("test", string(bs))
+			sp, _ := spider.NewSpider("test", string(bs))
 			go sp.StartJob()
 			log.Println("成功加载脚本", testdir)
 		} else {

src/spider/handler.go

@@ -71,11 +71,11 @@ func NoQueueScript() {
 		if info, ok := temp.(map[string]string); ok {
 			code := info["code"]
 			script := info["script"]
-			sp := NewSpider(code, script)
-			sp2 := NewSpider(code, script)
-			sp.Index = qu.IntAll(key)
-			sp2.Index = qu.IntAll(key)
-			if sp != nil && sp.Code != "nil" { //脚本加载成功
+			sp, errstr := NewSpider(code, script)
+			if errstr == "" && sp != nil && sp.Code != "nil" { //脚本加载成功
+				sp2, _ := NewSpider(code, script)
+				sp.Index = qu.IntAll(key)
+				sp2.Index = qu.IntAll(key)
 				if info["createuser"] != "" {
 					sp.UserName = info["createuser"]
 					sp2.UserName = info["createuser"]
@@ -103,13 +103,18 @@ func NoQueueScript() {
 				//util.TimeSleepFunc(10*time.Millisecond, TimeSleepChan)
 			} else {
 				logger.Info(code, "脚本加载失败，请检查！")
-				mgu.Save("spider_loadfail", "spider", "spider", map[string]interface{}{
-					"code":   code,
-					"type":   "初始化",
-					"script": script,
-					"intime": time.Now().Format(qu.Date_Full_Layout),
-					"event":  util.Config.Uploadevent,
-				})
+				mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{
+					"$set": map[string]interface{}{
+						"code":       code,
+						"type":       "初始化",
+						"script":     script,
+						"updatetime": time.Now().Unix(),
+						"modifyuser": sp.MUserName,
+						"event":      util.Config.Uploadevent,
+						"err":        errstr,
+						"ok":         false,
+					},
+				}, true, false)
 			}
 			time.Sleep(1 * time.Second)
 		}
@@ -154,11 +159,11 @@ func QueueUpScriptList() {
 				logger.Info("Code：", code, "Is Downloading List:", old_is_running)
 				if !old_is_running { //判断当前爬虫是否正在执行
 					script := info["script"]
-					sp := NewSpider_New(code, script, false)
+					sp, errstr := NewSpider_New(code, script, false)
 					isHistoricalMend := sp.IsHistoricalMend
-					sp.Index = qu.IntAll(key)
 					//logger.Info("初始化脚本是否成功：", sp != nil, e.Value)
-					if sp != nil && sp.Code != "nil" { //初始化脚本成功
+					if errstr == "" && sp != nil && sp.Code != "nil" { //初始化脚本成功
+						sp.Index = qu.IntAll(key)
 						if info["createuser"] != "" {
 							sp.UserName = info["createuser"]
 						}
@@ -170,13 +175,18 @@ func QueueUpScriptList() {
 						Allspiders.Store(code, sp)
 						sp.StartJob()
 					} else {
-						mgu.Save("spider_loadfail", "spider", "spider", map[string]interface{}{
-							"code":   code,
-							"type":   "初始化",
-							"script": script,
-							"intime": time.Now().Format(qu.Date_Full_Layout),
-							"event":  util.Config.Uploadevent,
-						})
+						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{
+							"$set": map[string]interface{}{
+								"code":       code,
+								"type":       "初始化",
+								"script":     script,
+								"updatetime": time.Now().Unix(),
+								"modifyuser": sp.MUserName,
+								"event":      util.Config.Uploadevent,
+								"err":        errstr,
+								"ok":         false,
+							},
+						}, true, false)
 					}
 					if isHistoricalMend { //下载历史的爬虫执行一次后删除
 						DelLen++
@@ -239,9 +249,9 @@ func QueueUpScriptDetail() {
 				logger.Info("Code：", code, "Is Downloading Detail:", old_is_running)
 				if !old_is_running { //判断当前爬虫是否正在执行
 					script := info["script"]
-					sp := NewSpider_New(code, script, true)
-					sp.Index = qu.IntAll(key)
-					if sp != nil && sp.Code != "nil" { //初始化脚本成功
+					sp, errstr := NewSpider_New(code, script, true)
+					if errstr == "" && sp != nil && sp.Code != "nil" { //初始化脚本成功
+						sp.Index = qu.IntAll(key)
 						if info["createuser"] != "" {
 							sp.UserName = info["createuser"]
 						}
@@ -603,11 +613,8 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 					err = nil
 					logger.Info("上架重载脚本", sp.Code)
 				} else { //新增脚本
-					sp := NewSpider(k, v["script"])
-					sp.Stop = false
-					sp.StartJob()
-					//go sp.DownloadHighDetail() //根据列表页数据下载三级页
-					if sp != nil && sp.Code != "nil" {
+					sp, errstr := NewSpider(k, v["script"])
+					if errstr == "" && sp != nil && sp.Code != "nil" {
 						if v["createuser"] != "" {
 							sp.UserName = v["createuser"]
 						}
@@ -616,20 +623,26 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 						}
 						sp.MUserName = v["modifyuser"]
 						sp.MUserEmail = v["modifyemail"]
-
 						Allspiders.Store(k, sp)
+						sp.Stop = false
+						sp.StartJob()
 						up = true
 						err = nil
 						logger.Info("上架新增脚本", sp.Code)
 					} else {
 						err = errors.New("新增失败")
-						mgu.Save("spider_loadfail", "spider", "spider", map[string]interface{}{
-							"code":   k,
-							"type":   "新增脚本失败",
-							"script": v["script"],
-							"intime": time.Now().Format(qu.Date_Full_Layout),
-							"event":  util.Config.Uploadevent,
-						})
+						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": k}, map[string]interface{}{
+							"$set": map[string]interface{}{
+								"code":       k,
+								"type":       "新增初始化脚本",
+								"script":     v["script"],
+								"updatetime": time.Now().Unix(),
+								"modifyuser": sp.MUserName,
+								"event":      util.Config.Uploadevent,
+								"err":        errstr,
+								"ok":         false,
+							},
+						}, true, false)
 					}
 				}
 				//2、Allspiders2对应7100、7400上架采集三级页数据
@@ -653,11 +666,8 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 						// err = nil
 						logger.Info("Allspiders2上架重载脚本", sp2.Code)
 					} else { //新增脚本
-						sp2 := NewSpider(k, v["script"])
-						sp2.Stop = false
-						//sp.StartJob()
-						go sp2.DownloadHighDetail() //根据列表页数据下载三级页
-						if sp2 != nil && sp2.Code != "nil" {
+						sp2, errstr := NewSpider(k, v["script"])
+						if errstr == "" && sp2 != nil && sp2.Code != "nil" {
 							if v["createuser"] != "" {
 								sp2.UserName = v["createuser"]
 							}
@@ -666,7 +676,8 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 							}
 							sp2.MUserName = v["modifyuser"]
 							sp2.MUserEmail = v["modifyemail"]
-
+							sp2.Stop = false
+							go sp2.DownloadHighDetail() //根据列表页数据下载三级页
 							Allspiders2.Store(k, sp2)
 							// up = true
 							// err = nil
@@ -713,6 +724,7 @@ func ReloadSpiderFile() {
 				as.Store(k, sp)
 			} else { //新增脚本
 				var sp *Spider
+				var errstr string
 				if util.Config.Working == 1 { //排队模式
 					if i == 0 {
 						length := 0
@@ -721,14 +733,14 @@ func ReloadSpiderFile() {
 							return true
 						})
 						LoopListPath.Store(length, v) //排队模式Allspiders,Allspiders2共用一个LoopListPath,新增一次即可
-						sp = NewSpider_New(k, v["script"], false)
+						sp, errstr = NewSpider_New(k, v["script"], false)
 					} else {
-						sp = NewSpider_New(k, v["script"], true)
+						sp, errstr = NewSpider_New(k, v["script"], true)
 					}
 				} else {
-					sp = NewSpider(k, v["script"])
+					sp, errstr = NewSpider(k, v["script"])
 				}
-				if sp != nil && sp.Code != "nil" {
+				if errstr == "" && sp != nil && sp.Code != "nil" {
 					if v["createuser"] != "" {
 						sp.UserName = v["createuser"]
 					}
@@ -757,13 +769,18 @@ func ReloadSpiderFile() {
 					logger.Info("定时重载脚本--新增", sp.Code)
 				} else {
 					if i == 0 {
-						mgu.Save("spider_loadfail", "spider", "spider", map[string]interface{}{
-							"code":   k,
-							"type":   "定时重载--新增失败",
-							"script": v["script"],
-							"intime": time.Now().Format(qu.Date_Full_Layout),
-							"event":  util.Config.Uploadevent,
-						})
+						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": k}, map[string]interface{}{
+							"$set": map[string]interface{}{
+								"code":       k,
+								"type":       "定时重载--新增失败",
+								"script":     v["script"],
+								"updatetime": time.Now().Unix(),
+								"modifyuser": sp.MUserName,
+								"event":      util.Config.Uploadevent,
+								"err":        errstr,
+								"ok":         false,
+							},
+						}, true, false)
 					}
 				}
 			}
@@ -832,14 +849,13 @@ func ReloadSpiderFile() {
 }
 
 //生成爬虫,排队模式
-func NewSpider_New(code, luafile string, newstate bool) *Spider {
+func NewSpider_New(code, luafile string, newstate bool) (*Spider, string) {
 	defer mu.Catch()
 	spider := &Spider{}
 	err := spider.LoadScript(code, luafile, newstate)
 	if err != "" {
-		return nil
+		return nil, err
 	}
-
 	spider.Code = spider.GetVar("spiderCode")
 	spider.Script.SCode = spider.Code
 	spider.Name = spider.GetVar("spiderName")
@@ -887,17 +903,16 @@ func NewSpider_New(code, luafile string, newstate bool) *Spider {
 	//新增历史补漏
 	spider.IsHistoricalMend = spider.GetBoolVar("spiderIsHistoricalMend")
 	spider.IsMustDownload = spider.GetBoolVar("spiderIsMustDownload")
-
-	return spider
+	return spider, ""
 }
 
 //生成爬虫
-func NewSpider(code, luafile string) *Spider {
+func NewSpider(code, luafile string) (*Spider, string) {
 	defer mu.Catch()
 	spider := &Spider{}
 	err := spider.LoadScript(code, luafile, true)
 	if err != "" {
-		return nil
+		return nil, err
 	}
 	spider.Code = spider.GetVar("spiderCode")
 	spider.SCode = spider.Code
@@ -949,7 +964,7 @@ func NewSpider(code, luafile string) *Spider {
 	spider.IsHistoricalMend = spider.GetBoolVar("spiderIsHistoricalMend")
 	spider.IsMustDownload = spider.GetBoolVar("spiderIsMustDownload")
 
-	return spider
+	return spider, ""
 }
 
 //下载量入库

src/spider/script.go

@@ -95,6 +95,7 @@ func (s *Script) LoadScript(code, script_file string, newstate bool) string {
 
 	if err := s.L.DoString(script_file); err != nil {
 		logger.Debug(code + ",加载lua脚本错误：" + err.Error())
+		return "加载lua脚本错误：" + err.Error()
 		//panic(code + ",加载lua脚本错误：" + err.Error())
 	}
 	s.Encoding = s.GetVar("spiderPageEncoding")