فهرست منبع

爬虫补采模块调整参数

mxs 1 سال پیش
والد
کامیت
c8a90dc505
2فایلهای تغییر یافته به همراه7 افزوده شده و 5 حذف شده
  1. 0 1
      src/spider/spider.go
  2. 7 4
      src/spider/supplement.go

+ 0 - 1
src/spider/spider.go

@@ -977,7 +977,6 @@ func (s *Spider) SupplementDownListPageItem() (errs interface{}) {
 		lv := s.L.Get(-1)
 		s.L.Pop(1)
 		if tbl, ok := lv.(*lua.LTable); ok {
-			qu.Debug(tbl.Len())
 			if tabLen := tbl.Len(); tabLen > 0 { //列表页有数据,根据列表页信息下载三级页
 				repeatListNum := 0 // 当前列表页连接重复个数
 				isBreak := false

+ 7 - 4
src/spider/supplement.go

@@ -25,20 +25,23 @@ func InitSupplement() {
 	flag.StringVar(&Supplement_Cycle, "c", "day", "day:每天定点执行;week:每周定点执行")
 	flag.IntVar(&Supplement_Day, "d", 1, "补采几天的数据")
 	flag.IntVar(&Supplement_MaxErrorTimes, "e", 5, "连续几页异常采集中断")
+	flag.Parse()
 	if Supplement {
 		Supplement_Publishtime = GetTime(-Supplement_Day)
 		if Supplement_Cycle == "day" {
-			Supplement_StartCron = "0 16 9 ? * *"
-			//InitSpider()
+			Supplement_StartCron = "0 0 22 ? * *"
 			Supplement_EndCron = "0 0 9 ? * *"
+			//InitSpider()
 		} else if Supplement_Cycle == "week" {
 			Supplement_StartCron = "0 0 0 ? * SAT"
 			Supplement_EndCron = "0 0 0 ? * MON"
 		}
 		c := cron.New()
 		c.Start()
-		c.AddFunc(Supplement_StartCron, SupplementStart)
-		c.AddFunc(Supplement_EndCron, SupplementEnd)
+		if Supplement_StartCron != "" && Supplement_EndCron != "" {
+			c.AddFunc(Supplement_StartCron, SupplementStart)
+			c.AddFunc(Supplement_EndCron, SupplementEnd)
+		}
 	}
 }