Browse Source

爬虫导入取消href重复校验

maxiaoshan 2 years ago
parent
commit
f4bf9a80c2
2 changed files with 2 additions and 2 deletions
  1. 1 1
      src/front/front.go
  2. 1 1
      src/taskManager/taskManager.go

+ 1 - 1
src/front/front.go

@@ -563,7 +563,7 @@ func (f *Front) ImportLua() {
 						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
 						if len(*one) > 0 {
 							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",该栏目已存在,请确认"
-							continue
+							//continue
 						}
 						o["channeladdr"] = href
 						o["author"] = cells[5].Value

+ 1 - 1
src/taskManager/taskManager.go

@@ -478,7 +478,7 @@ func (t *TaskM) SaveNewTask() {
 	newTask["i_times"] = 0
 	newTask["i_pendstate"] = 0
 	newTask["l_checktime"] = time.Now().Unix()
-	newTask["i_frequencyerrtimes"] = time.Now().Unix() //采集频率异常次数
+	newTask["i_frequencyerrtimes"] = 0 //采集频率异常次数
 	newTask["s_platform"] = "golua平台"
 	newTask["i_num"] = 0
 	queryL := map[string]interface{}{