Эх сурвалжийг харах

7000节点新爬虫转增量节点模式调整

maxiaoshan 2 жил өмнө
parent
commit
fae7a5e132

+ 29 - 24
src/spider/handler.go

@@ -1320,32 +1320,37 @@ func SaveHeartInfo() {
 //信息提交编辑器
 func SpiderCodeSendToEditor(code string) {
 	defer qu.Catch()
-	ok := false
-	for i := 1; i <= 3; i++ {
-		logger.Info("Code:", code, "	times:", i, "	Send Move Event")
-		list := []interface{}{}
-		list = append(list, code)
-		bs, _ := json.Marshal(list)
-		value := url.Values{
-			"data": []string{util.Se.EncodeString(string(bs))},
-			"type": []string{"code"},
-		}
-		res, err := http.PostForm(util.Config.Editoraddr, value)
-		if err != nil {
-			logger.Error("Send To Editor For Move Event Failed,Code:", code)
-		} else {
-			if res != nil {
-				res.Body.Close()
-			}
-			ok = true
-			break
-		}
-	}
-	logger.Info("Code:", code, "	Send Move Event:", ok)
 	MgoEB.Save("luamovelog", map[string]interface{}{
 		"code":       code,
 		"comeintime": time.Now().Unix(),
-		"type":       "sendfail",
-		"ok":         ok,
+		"ok":         false,
 	})
+	//ok := false
+	//for i := 1; i <= 3; i++ {
+	//	logger.Info("Code:", code, "	times:", i, "	Send Move Event")
+	//	list := []interface{}{}
+	//	list = append(list, code)
+	//	bs, _ := json.Marshal(list)
+	//	value := url.Values{
+	//		"data": []string{util.Se.EncodeString(string(bs))},
+	//		"type": []string{"code"},
+	//	}
+	//	res, err := http.PostForm(util.Config.Editoraddr, value)
+	//	if err != nil {
+	//		logger.Error("Send To Editor For Move Event Failed,Code:", code)
+	//	} else {
+	//		if res != nil {
+	//			res.Body.Close()
+	//		}
+	//		ok = true
+	//		break
+	//	}
+	//}
+	//logger.Info("Code:", code, "	Send Move Event:", ok)
+	//MgoEB.Save("luamovelog", map[string]interface{}{
+	//	"code":       code,
+	//	"comeintime": time.Now().Unix(),
+	//	"type":       "sendfail",
+	//	"ok":         ok,
+	//})
 }

+ 1 - 1
src/spider/spider.go

@@ -193,7 +193,7 @@ func (s *Spider) ExecJob(reload bool) {
 	}
 	if util.Config.IsHistoryEvent && s.GetVar("spiderType") == "history" { //判断爬虫是增量还是历史爬虫(目前只会在7000节点上有历史爬虫)
 		UpdateSpiderByCodeState(s.Code, "6") //爬虫在该节点下架
-		SpiderCodeSendToEditor(s.Code)       //发送编辑器
+		SpiderCodeSendToEditor(s.Code)       //历史转增量爬虫发送编辑器,切换节点上下架
 		return
 	} else {
 		if util.Config.Working == 0 { //高性能模式