机检修改

src/front/spider.go

@@ -1607,6 +1607,11 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 					upset["l_uploadtime"] = time.Now().Unix()
 				} else if state == Sp_state_2 { //打回原因
 					upset["reason"] = reason
+					//清理机检结果
+					unset["check"] = ""
+					unset["checkok"] = ""
+					unset["checktime"] = ""
+					LuaCheckSaveLog(one) //保存机检日志
 				} else if state == Sp_state_5 { //上架
 					upset["l_checktime"] = 0 //核对时间重置
 					//清理机检结果

src/luacheck/luacheck.go

@@ -44,12 +44,6 @@ type Data struct {
 // TimeTaskGetLua 定时取爬虫放入通道
 func TimeTaskGetLua() {
 	query := map[string]interface{}{
-		"event": map[string]interface{}{
-			"$ne": 7410,
-		},
-		"incrementevent": map[string]interface{}{
-			"$ne": 7410,
-		},
 		"platform": "golua平台",
 		"state":    1,
 		"check": map[string]interface{}{
@@ -273,19 +267,19 @@ func listResultCheck(pageNum int, lists map[int][]map[string]interface{}, err, w
 		warn["列表页翻页异常"] = "列表页可以翻页，最大页却为1"
 	}
 	var fileNum, downloadOkNum, fileFailedNum int //附件下载量，详情页下载量
-	n := 0
 	wg := &sync.WaitGroup{}
 	lock := &sync.Mutex{}
 	ch := make(chan bool, 10)
-	hrefs := make([][]string, 2) //记录前两页数据链接
+	hrefs := make([][]string, len(lists)) //记录前两页数据链接
+	var i, num int
 	for _, list := range lists {
 		pageHrefs := []string{}
 		for _, l := range list {
 			pageHrefs = append(pageHrefs, qu.ObjToString(l["href"]))
-			if n > 50 {
+			if num > 50 {
 				break
 			}
-			n++
+			num++
 			ch <- true
 			wg.Add(1)
 			go func(tmp map[string]interface{}) {
@@ -330,24 +324,32 @@ func listResultCheck(pageNum int, lists map[int][]map[string]interface{}, err, w
 				lock.Unlock()
 			}(l)
 		}
-		hrefs = append(hrefs, pageHrefs)
+		hrefs[i] = pageHrefs
+		i++
 	}
 	wg.Wait()
-	hrefByte1, _ := json.Marshal(hrefs[0])
-	hrefByte2, _ := json.Marshal(hrefs[1])
-	if sp.GetHashKey(hrefByte1) == sp.GetHashKey(hrefByte2) {
-		err["前两页数据一致"] = "前两页数据一致"
+
+	if len(hrefs) >= 2 {
+		hrefByte1, _ := json.Marshal(hrefs[0])
+		hrefByte2, _ := json.Marshal(hrefs[1])
+		if sp.GetHashKey(hrefByte1) == sp.GetHashKey(hrefByte2) {
+			err["前两页数据一致"] = "前两页数据一致"
+		}
 	}
 	if fileNum == 0 {
 		warn["未下载任何附件"] = "未下载任何附件"
 	}
 	if fileFailedNum > 0 {
+		href := warn["附件未成功下载"]
+		delete(warn, "附件未成功下载")
 		rate := float64(fileFailedNum) / float64(fileNum)
-		warn["部分附件未成功下载"] = "部分附件未成功下载，一共+" + fmt.Sprint(len(lists)) + "个，失败占比：" + fmt.Sprintf("%.1f", rate) + "（" + warn["附件未成功下载"] + "）"
+		warn["部分附件未成功下载"] = "部分附件未成功下载，一共" + fmt.Sprint(fileNum) + "个，失败占比：" + fmt.Sprintf("%.1f%%", rate*100) + "（" + href + "）"
 	}
-	if n := len(lists) - downloadOkNum; n > 0 {
-		rate := float64(n) / float64(len(lists))
-		warn["部分详情页未下载成功"] = "部分详情页未成功下载，一共+" + fmt.Sprint(len(lists)) + "条，失败占比：" + fmt.Sprintf("%.1f", rate) + "（" + warn["详情页下载失败"] + "）"
+	if n := num - downloadOkNum; n > 0 {
+		href := warn["详情页下载失败"]
+		delete(warn, "详情页下载失败")
+		rate := float64(n) / float64(num)
+		warn["部分详情页未下载成功"] = "部分详情页未成功下载，一共" + fmt.Sprint(num) + "条，失败占比：" + fmt.Sprintf("%.1f%%", rate*100) + "（" + href + "）"
 	}
 }
 
@@ -360,7 +362,7 @@ func downloadDetail(fileNum, fileFailedNumTmp, downloadOkNum *int, err, warn map
 		param[k] = qu.ObjToString(v)
 	}
 	var downloadDetailOk bool
-	for i := 1; i <= 3; i++ { //重试三次
+	for i := 1; i <= 2; i++ { //重试三次
 		_, rep_err := s.DownloadDetailPage(param, data)
 		if rep_err == nil && len(data) > 0 {
 			downloadDetailOk = true
@@ -375,20 +377,21 @@ func downloadDetail(fileNum, fileFailedNumTmp, downloadOkNum *int, err, warn map
 						tmpMap := tmp.(map[string]interface{})
 						if qu.ObjToString(tmpMap["filename"]) == "附件中含有乱码" {
 							getFile = true
+							*fileNum++
 							warn["附件含乱码"] = "部分附件中含有乱码（" + qu.ObjToString(param["href"]) + "）"
 						} else if qu.ObjToString(tmpMap["org_url"]) != "" && qu.ObjToString(tmpMap["fid"]) == "" { //附件未下载成功
 							warn["附件未成功下载"] = qu.ObjToString(param["href"])
 							getFile = true
+							*fileNum++
 							*fileFailedNumTmp++
 						} else if qu.ObjToString(tmpMap["fid"]) != "" {
 							getFile = true
+							*fileNum++
 						}
 					}
 				}
 			}
-			if getFile {
-				*fileNum++ //记录附件下载个数
-			} else {
+			if !getFile {
 				contenthtml := qu.ObjToString(data["contenthtml"])
 				//if !checkFile(contenthtml) {
 				//	err["contenthtml中含有可下载内容"] = "contenthtml中含有可下载内容"