爬虫优化

spider2/src/client/ecps/ln/parse.go

@@ -114,6 +114,10 @@ func addJob(tmplist []interface{}) {
 				cm["query"] = tmp["Inv"]
 			}
 			if cm["query"] != nil && cm["query"] != "" && len(cm["query"].(string)) > 5 {
+				//防止死循环
+				if cm["query"] == infoMap["CerNo"] || cm["query"] == infoMap["EntName"] {
+					continue
+				}
 				cms := *new(com.CmdMsg)
 				cms.Nodetype = com.NODE_TYPE_ECPS
 				cms.Param = cm

spider2/src/client/ecps/ln/spider.go

@@ -85,26 +85,18 @@ func downList() {
 			}
 			ent := *mongodb.FindOne("entintime", `{"$and":[{"intime":{"$gt":`+fmt.Sprint(time.Now().Unix()-int64(rate)*24*60*60)+`}},{"$or":[{"RegNo":"`+fmt.Sprint(tmp["query"])+`"},{"EntName":"`+fmt.Sprint(tmp["query"])+`"}]}]}`)
 			if ent != nil {
-<<<<<<< HEAD
 				log.Println(ent["EntName"], ent["RegNo"], ",根据配置频率跳过")
 				putLogs(":"+fmt.Sprint(ent["EntName"])+","+fmt.Sprint(ent["RegNo"])+",根据配置频率跳过", 1)
-=======
-				log.Println(fmt.Sprint(ent["EntName"]) + ",根据配置频率跳过")
-				putLogs(":"+fmt.Sprint(ent["EntName"])+",根据配置频率跳过", 1)
->>>>>>> release
+
 				Lists.Remove(e)
 				continue
 			}
 			b, info := downloadContent(tmp)
 			Lists.Remove(e)
 			if b {
-<<<<<<< HEAD
 				log.Println("成功下载：", info["EntName"], info["RegNo"])
 				putLogs("成功下载："+fmt.Sprint(info["EntName"])+","+fmt.Sprint(info["RegNo"]), 1)
-=======
-				log.Println("成功下载：", info["EntName"])
-				putLogs("成功下载："+fmt.Sprint(info["EntName"]), 1)
->>>>>>> release
+
 			} else {
 				log.Println("下载失败：", tmp)
 				putLogs("下载失败："+fmt.Sprint(tmp), 1)

spider2/src/server/rpc.go

@@ -23,7 +23,6 @@ const (
 	RUNNING         = iota
 	PASS
 	STOP
-	OFFLINE //掉线
 )
 
 //一个节点
@@ -33,6 +32,7 @@ type Node struct {
 	Descript       string
 	Dtype          string     //节点分类
 	Status         int        //状态
+	IsOnline       bool       //是否在线
 	Logs           *list.List //日志
 	Command        *list.List //命令
 	LastAccesstime int64      //最后访问时间
@@ -153,13 +153,20 @@ func initRpc(port string) {
 func Gc() {
 	timestamp := time.Now().Unix()
 	lock.Lock()
-	for k, v := range Mirror {
+	for _, v := range Mirror {
 		log.Println(v.Name, "Status:", v.Status)
 		remain := timestamp - v.LastAccesstime
-		if remain > 10 { //5分钟掉线删除
-			delete(Mirror, k)
-		} else if remain > ACTIVECHECKTIME {
-			v.Status = OFFLINE
+		/*
+			if remain > 30 { //30秒钟掉线删除
+				delete(Mirror, k)
+			} else if remain > ACTIVECHECKTIME {
+				v.Status = OFFLINE
+			}
+		*/
+		if remain > ACTIVECHECKTIME {
+			v.IsOnline = false
+		} else {
+			v.IsOnline = true
 		}
 	}
 	lock.Unlock()