流量统计优化

src/spider/handler.go

@@ -542,7 +542,7 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 					}
 					sp2.MUserName = v["modifyuser"]
 					sp2.MUserEmail = v["modifyemail"]
-					sp2.LoadScript(sp2.Name, sp2.Channel, sp2.MUserName, k, sp2.ScriptFile, true) //更新上架，重载脚本
+					sp2.LoadScript(&sp2.Name, &sp2.Channel, &sp2.MUserName, k, sp2.ScriptFile, true) //更新上架，重载脚本
 					Allspiders2.Store(k, sp2)
 					// up = true
 					// err = nil
@@ -694,9 +694,9 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 						}
 						sp2.MUserName = v["modifyuser"]
 						sp2.MUserEmail = v["modifyemail"]
-						sp2.LoadScript(sp2.Name, sp2.Channel, sp2.MUserName, k, sp2.ScriptFile, true) //更新上架，重载脚本
-						Allspiders2.Store(k, sp2)                                                     //重载后放入集合
-						UpdateHighListDataByCode(k)                                                   //爬虫更新上架后，重置数据state=0
+						sp2.LoadScript(&sp2.Name, &sp2.Channel, &sp2.MUserName, k, sp2.ScriptFile, true) //更新上架，重载脚本
+						Allspiders2.Store(k, sp2)                                                        //重载后放入集合
+						UpdateHighListDataByCode(k)                                                      //爬虫更新上架后，重置数据state=0
 						// up = true
 						// err = nil
 						logger.Info("Allspiders2上架重载脚本", sp2.Code)
@@ -896,7 +896,7 @@ func ReloadSpiderFile() {
 func NewSpider_New(code, luafile string, newstate bool) (*Spider, string) {
 	defer mu.Catch()
 	spider := &Spider{}
-	err := spider.LoadScript(spider.Name, spider.Channel, spider.MUserName, code, luafile, newstate)
+	err := spider.LoadScript(&spider.Name, &spider.Channel, &spider.MUserName, code, luafile, newstate)
 	if err != "" {
 		return nil, err
 	}
@@ -957,7 +957,7 @@ func NewSpider_New(code, luafile string, newstate bool) (*Spider, string) {
 func NewSpider(code, luafile string) (*Spider, string) {
 	defer mu.Catch()
 	spider := &Spider{}
-	err := spider.LoadScript(spider.Name, spider.Channel, spider.MUserName, code, luafile, true)
+	err := spider.LoadScript(&spider.Name, &spider.Channel, &spider.MUserName, code, luafile, true)
 	if err != "" {
 		return nil, err
 	}

src/spider/script.go

@@ -70,7 +70,7 @@ func init() {
 var TimeSleepChan = make(chan bool, 1)
 
 //加载文件
-func (s *Script) LoadScript(site, channel, user, code, script_file string, newstate bool) string {
+func (s *Script) LoadScript(site, channel, user *string, code, script_file string, newstate bool) string {
 	defer mu.Catch()
 	s.SCode = code
 	s.ScriptFile = script_file
@@ -126,18 +126,18 @@ func (s *Script) LoadScript(site, channel, user, code, script_file string, newst
 			if sf, ok := SpiderFlowMap.Load(key); ok && sf != nil {
 				if sfMap, ok := sf.(*SpiderFlow); ok {
 					sfMap.Flow += retLen
-					sfMap.Site = site
-					sfMap.Channel = channel
-					sfMap.ModifyUser = user
+					//sfMap.Site = *site
+					//sfMap.Channel = *channel
+					//sfMap.ModifyUser = *user
 					SpiderFlowMap.Store(key, sfMap)
 				}
 			} else {
 				SpiderFlowMap.Store(key, &SpiderFlow{
 					//Code:       code,
-					Site:       site,
-					Channel:    channel,
+					Site:       *site,
+					Channel:    *channel,
 					Flow:       retLen,
-					ModifyUser: user,
+					ModifyUser: *user,
 				})
 			}
 		}
@@ -187,18 +187,18 @@ func (s *Script) LoadScript(site, channel, user, code, script_file string, newst
 			if sf, ok := SpiderFlowMap.Load(key); ok && sf != nil {
 				if sfMap, ok := sf.(*SpiderFlow); ok {
 					sfMap.Flow += retLen
-					sfMap.Site = site
-					sfMap.Channel = channel
-					sfMap.ModifyUser = user
+					//sfMap.Site = *site
+					//sfMap.Channel = *channel
+					//sfMap.ModifyUser = *user
 					SpiderFlowMap.Store(key, sfMap)
 				}
 			} else {
 				SpiderFlowMap.Store(key, &SpiderFlow{
 					//Code:       code,
-					Site:       site,
-					Channel:    channel,
+					Site:       *site,
+					Channel:    *channel,
 					Flow:       retLen,
-					ModifyUser: user,
+					ModifyUser: *user,
 				})
 			}
 		}
@@ -247,18 +247,18 @@ func (s *Script) LoadScript(site, channel, user, code, script_file string, newst
 			if sf, ok := SpiderFlowMap.Load(key); ok && sf != nil {
 				if sfMap, ok := sf.(*SpiderFlow); ok {
 					sfMap.Flow += retLen
-					sfMap.Site = site
-					sfMap.Channel = channel
-					sfMap.ModifyUser = user
+					//sfMap.Site = *site
+					//sfMap.Channel = *channel
+					//sfMap.ModifyUser = *user
 					SpiderFlowMap.Store(key, sfMap)
 				}
 			} else {
 				SpiderFlowMap.Store(key, &SpiderFlow{
 					//Code:       code,
-					Site:       site,
-					Channel:    channel,
+					Site:       *site,
+					Channel:    *channel,
 					Flow:       retLen,
-					ModifyUser: user,
+					ModifyUser: *user,
 				})
 			}
 		}
@@ -359,7 +359,7 @@ func (s *Script) LoadScript(site, channel, user, code, script_file string, newst
 		table := S.ToTable(-1)
 		list := util.TableToMap(table)
 		if len(list) > 0 {
-			UpdateHeart(site, channel, code, user, "findlist") //记录列表页实际采集数据量心跳
+			UpdateHeart(*site, *channel, code, *user, "findlist") //记录列表页实际采集数据量心跳
 		}
 		return 1
 	}))

src/spider/spider.go

@@ -175,7 +175,7 @@ func (s *Spider) ExecJob(reload bool) {
 		}
 	}()
 	if reload && util.Config.Working == 0 { //高效模式，轮询调度时重载脚本
-		s.LoadScript(s.Name, s.Channel, s.MUserName, s.Code, s.ScriptFile, true)
+		s.LoadScript(&s.Name, &s.Channel, &s.MUserName, s.Code, s.ScriptFile, true)
 	}
 	logger.Debug(s.Code, s.Name, "频率：", s.SpiderRunRate, ",", s.Timeout)
 	s.LastDowncount = 0
@@ -955,7 +955,7 @@ func (s *Spider) DownloadHighDetail() {
 					Mgo.Update("spider_highlistdata_test", query, set, false, false)
 				}
 				//重载spider
-				s.LoadScript(s.Name, s.Channel, s.MUserName, s.Code, s.ScriptFile, true)
+				s.LoadScript(&s.Name, &s.Channel, &s.MUserName, s.Code, s.ScriptFile, true)
 			} else { //没有数据
 				time.Sleep(2 * time.Minute)
 			}

src/spider/store.go

@@ -431,6 +431,7 @@ func UpdateSpiderFlow() {
 					"channel":    sfMap.Channel,
 					"modifyuser": sfMap.ModifyUser,
 					"comeintime": time.Now().Unix(),
+					"event":      lu.Config.Uploadevent,
 				})
 				SpiderFlowMap.Delete(key) //统计完成后删除非当天数据
 			}