爬虫节点转移基础数据统计新增字段

src/timetask/luamove.go

@@ -14,6 +14,9 @@ type Spider struct {
 	Code         string `json:"code"`
 	Site         string `json:"site"`
 	Channel      string `json:"channel"`
+	Href         string `json:"href"`
+	MaxPage      int    `json:"maxpage"`
+	CycleTime    int    `json:"cycletime"`
 	FromEvent    int    `json:"fromevent"`    //现节点
 	ToEvent      int    `json:"toevent"`      //目标节点
 	DataNum      int    `json:"datanum"`      //采集量
@@ -65,10 +68,11 @@ func GetLuaInfo() {
 		},
 	}
 	fields := map[string]interface{}{
-		"event":   1,
-		"code":    1,
-		"site":    1,
-		"channel": 1,
+		"event":        1,
+		"code":         1,
+		"site":         1,
+		"channel":      1,
+		"param_common": 1,
 	}
 	count := util.MgoEB.Count("luaconfig", query)
 	logger.Debug("共加载线上爬虫个数:", count)
@@ -86,6 +90,13 @@ func GetLuaInfo() {
 			site := qu.ObjToString(tmp["site"])
 			channel := qu.ObjToString(tmp["channel"])
 			event := qu.IntAll(tmp["event"])
+			href := ""
+			maxPage, cycletime := 0, 0
+			if param_common := tmp["param_common"].([]interface{}); len(param_common) >= 12 {
+				href = qu.ObjToString(param_common[11])
+				maxPage = qu.IntAll(param_common[5])
+				cycletime = qu.IntAll(param_common[6])
+			}
 			lock.Lock()
 			if event != 7410 && event != 7700 {
 				if util.CodeEventType[event] == 1 {
@@ -100,6 +111,9 @@ func GetLuaInfo() {
 				Code:       code,
 				Site:       site,
 				Channel:    channel,
+				Href:       href,
+				MaxPage:    maxPage,
+				CycleTime:  cycletime,
 				FromEvent:  event,
 				Comeintime: time.Now().Unix(),
 			}