调整spider_loadfail入库机制

src/spider/handler.go

@@ -103,18 +103,26 @@ func NoQueueScript() {
 				//util.TimeSleepFunc(10*time.Millisecond, TimeSleepChan)
 			} else {
 				logger.Info(code, "脚本加载失败，请检查！")
-				mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{
-					"$set": map[string]interface{}{
-						"code":       code,
-						"type":       "初始化",
-						"script":     script,
-						"updatetime": time.Now().Unix(),
-						"modifyuser": sp.MUserName,
-						"event":      util.Config.Uploadevent,
-						"err":        errstr,
-						"ok":         false,
+				nowT := time.Now().Unix()
+				mgu.Update("spider_loadfail", "spider", "spider",
+					map[string]interface{}{
+						"code": code,
+						"modifytime": map[string]interface{}{
+							"$gte": nowT - 12*3600,
+							"$lte": nowT + 12*3600,
+						},
 					},
-				}, true, false)
+					map[string]interface{}{
+						"$set": map[string]interface{}{
+							"code":       code,
+							"type":       "初始化",
+							"script":     script,
+							"updatetime": nowT,
+							"modifyuser": sp.MUserName,
+							"event":      util.Config.Uploadevent,
+							"err":        errstr,
+						},
+					}, true, false)
 			}
 			time.Sleep(1 * time.Second)
 		}
@@ -175,18 +183,26 @@ func QueueUpScriptList() {
 						Allspiders.Store(code, sp)
 						sp.StartJob()
 					} else {
-						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{
-							"$set": map[string]interface{}{
-								"code":       code,
-								"type":       "初始化",
-								"script":     script,
-								"updatetime": time.Now().Unix(),
-								"modifyuser": sp.MUserName,
-								"event":      util.Config.Uploadevent,
-								"err":        errstr,
-								"ok":         false,
+						nowT := time.Now().Unix()
+						mgu.Update("spider_loadfail", "spider", "spider",
+							map[string]interface{}{
+								"code": code,
+								"modifytime": map[string]interface{}{
+									"$gte": nowT - 12*3600,
+									"$lte": nowT + 12*3600,
+								},
 							},
-						}, true, false)
+							map[string]interface{}{
+								"$set": map[string]interface{}{
+									"code":       code,
+									"type":       "初始化",
+									"script":     script,
+									"updatetime": nowT,
+									"modifyuser": sp.MUserName,
+									"event":      util.Config.Uploadevent,
+									"err":        errstr,
+								},
+							}, true, false)
 					}
 					if isHistoricalMend { //下载历史的爬虫执行一次后删除
 						DelLen++
@@ -631,18 +647,26 @@ func UpdateSpiderByCodeState(code, state string) (bool, error) {
 						logger.Info("上架新增脚本", sp.Code)
 					} else {
 						err = errors.New("新增失败")
-						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": k}, map[string]interface{}{
-							"$set": map[string]interface{}{
-								"code":       k,
-								"type":       "新增初始化脚本",
-								"script":     v["script"],
-								"updatetime": time.Now().Unix(),
-								"modifyuser": sp.MUserName,
-								"event":      util.Config.Uploadevent,
-								"err":        errstr,
-								"ok":         false,
+						nowT := time.Now().Unix()
+						mgu.Update("spider_loadfail", "spider", "spider",
+							map[string]interface{}{
+								"code": k,
+								"modifytime": map[string]interface{}{
+									"$gte": nowT - 12*3600,
+									"$lte": nowT + 12*3600,
+								},
 							},
-						}, true, false)
+							map[string]interface{}{
+								"$set": map[string]interface{}{
+									"code":       k,
+									"type":       "新增初始化脚本",
+									"script":     v["script"],
+									"updatetime": nowT,
+									"modifyuser": sp.MUserName,
+									"event":      util.Config.Uploadevent,
+									"err":        errstr,
+								},
+							}, true, false)
 					}
 				}
 				//2、Allspiders2对应7100、7400上架采集三级页数据
@@ -769,18 +793,26 @@ func ReloadSpiderFile() {
 					logger.Info("定时重载脚本--新增", sp.Code)
 				} else {
 					if i == 0 {
-						mgu.Update("spider_loadfail", "spider", "spider", map[string]interface{}{"code": k}, map[string]interface{}{
-							"$set": map[string]interface{}{
-								"code":       k,
-								"type":       "定时重载--新增失败",
-								"script":     v["script"],
-								"updatetime": time.Now().Unix(),
-								"modifyuser": sp.MUserName,
-								"event":      util.Config.Uploadevent,
-								"err":        errstr,
-								"ok":         false,
+						nowT := time.Now().Unix()
+						mgu.Update("spider_loadfail", "spider", "spider",
+							map[string]interface{}{
+								"code": k,
+								"modifytime": map[string]interface{}{
+									"$gte": nowT - 12*3600,
+									"$lte": nowT + 12*3600,
+								},
 							},
-						}, true, false)
+							map[string]interface{}{
+								"$set": map[string]interface{}{
+									"code":       k,
+									"type":       "定时重载--新增失败",
+									"script":     v["script"],
+									"updatetime": nowT,
+									"modifyuser": sp.MUserName,
+									"event":      util.Config.Uploadevent,
+									"err":        errstr,
+								},
+							}, true, false)
 					}
 				}
 			}