xg

src/taskManager/taskManager.go

@@ -60,12 +60,7 @@ func (t *TaskM) Taskfile() {
 	if t.Method() == "POST" {
 		mf, _, err := t.GetFile("xlsx")
 		errorinfo := map[string]interface{}{}
-		o := map[string]interface{}{
-			"s_date":   time.Now().Format("2006-01-02"),
-			"s_source": "人工",
-			"s_type":   "0",
-		}
-		var ug string = ""
+		o := map[string]interface{}{}
 		if err == nil {
 			binary, _ := ioutil.ReadAll(mf)
 			xls, _ := xlsx.OpenBinary(binary)
@@ -74,189 +69,85 @@ func (t *TaskM) Taskfile() {
 			for k, v := range rows {
 				if k != 0 {
 					cells := v.Cells
-					if len(cells) == 0 {
-						continue
-					}
-					if cells[0].Value == "" { //没有code
-						continue
-					} else { //有code
-						o["s_code"] = cells[0].Value
-						queryT := bson.M{
-							"s_code": cells[0].Value,
+					if len(cells) == 5 {
+						code := cells[0].Value
+						if code == "" {
+							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行未指定爬虫"
+							continue
+						}
+						query := bson.M{
+							"s_code": code,
 							"i_state": bson.M{
-								"$in": []int{1, 2, 5},
+								"$in": []int{0, 1, 2, 3, 5},
 							},
 						}
-						task := *mgdb.FindOne("task", queryT)
-						//if len(task) != 0 { //任务已存在 追加描述 并修改紧急度和最迟完成时间
-						//errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行数据已导入"
-						//continue
-						//} else { //任务不存在
-						if len(cells) == 4 && cells[3].Value != "" { //必有code和最终完成时间
-							if cells[1].Value == "" || cells[2].Value == "" { //没有描述或者紧急度
-								errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行数据不完整"
-								continue
-							} else {
-								if !strings.Contains(cells[3].Value, "-") { //日期格式不对
-									errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行日期格式错误"
-									continue
-								} else {
-									if len(task) != 0 { //有该任务 追加描述 比较紧急度和最迟完成时间
-										//追加描述
-										text := time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + cells[1].Value + "\n"
-										s_descript := task["s_descript"].(string) + text
-										//比较、更新最迟完成时间
-										var l_complete int64
-										var s_urgency string
-										s_urgency = isUrgency(cells)
-										if s_urgency == "0" {
-											errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-											continue
-										}
-										timeDate, _ := time.ParseInLocation("2006-01-02", cells[3].Value, time.Local)
-										complete := timeDate.Unix() + 64800
-										taskComplete := qu.Int64All(task["l_complete"])
-										if complete <= taskComplete { //取excel中的最迟完成时间和紧急度
-											l_complete = complete
-										} else { //取task中的最迟完成时间和紧急度
-											l_complete = taskComplete
-											s_urgency = task["s_urgency"].(string)
-										}
-
-										if task["i_state"].(int) == 2 {
-											errorinfo[cells[0].Value] = "任务已存在"
-										}
-										//更新task
-										UpdateOldTask(task["_id"].(bson.ObjectId).Hex(), s_descript, s_urgency, l_complete)
-
-									} else {
-										timeDate, err := time.ParseInLocation("2006-01-02", cells[3].Value, time.Local)
-										if err == nil {
-											o["l_complete"] = timeDate.Unix() + 64800
-										}
-										o["s_descript"] = cells[1].Value + "\n"
-										ug = isUrgency(cells)
-										if ug == "0" {
-											errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-											continue
-										}
-										o["s_urgency"] = ug
-									}
-								}
-							}
-						} else if len(cells) == 4 && cells[3].Value == "" { //避免添加最终时间后又手动删除
-							if len(task) != 0 {
-								//追加描述
-								text := time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + cells[1].Value + "\n"
-								s_descript := task["s_descript"].(string) + text
-								//比较、更新最迟完成时间
-								var l_complete int64
-								var s_urgency string
-								s_urgency = isUrgency(cells)
-								if s_urgency == "0" {
-									errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-									continue
-								}
-								complete := ft.CompleteTime(s_urgency)
-								taskComplete := qu.Int64All(task["l_complete"])
-								if complete <= taskComplete { //取excel中的最迟完成时间和紧急度
-									l_complete = complete
-								} else { //取task中的最迟完成时间和紧急度
-									l_complete = taskComplete
-									s_urgency = task["s_urgency"].(string)
-								}
-								if task["i_state"].(int) == 2 {
-									errorinfo[cells[0].Value] = "任务已存在"
-								}
-
-								//更新task
-								UpdateOldTask(task["_id"].(bson.ObjectId).Hex(), s_descript, s_urgency, l_complete)
-							} else {
-								if cells[1].Value != "" {
-									ug = isUrgency(cells)
-									if ug == "0" {
-										errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-										continue
-									}
-									o["l_complete"] = ft.CompleteTime(ug)
-									o["s_descript"] = cells[1].Value + "\n"
-									o["s_urgency"] = ug
-								} else {
-									errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行数据不完整"
-								}
-							}
-
-						} else if len(cells) == 3 { //必没有最终完成时间 自动生成
-							if len(task) != 0 {
-								//追加描述
-								text := time.Now().Format(qu.Date_Short_Layout) + "追加描述：------------------------------\n" + cells[1].Value + "\n"
-								s_descript := task["s_descript"].(string) + text
-								//比较、更新最迟完成时间
-								var l_complete int64
-								var s_urgency string
-								s_urgency = isUrgency(cells)
-								if s_urgency == "0" {
-									errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-									continue
-								}
-								complete := ft.CompleteTime(s_urgency)
-								taskComplete := qu.Int64All(task["l_complete"])
-								if complete <= taskComplete { //取excel中的最迟完成时间和紧急度
-									l_complete = complete
-								} else { //取task中的最迟完成时间和紧急度
-									l_complete = taskComplete
-									s_urgency = task["s_urgency"].(string)
-								}
-								if task["i_state"].(int) == 2 {
-									errorinfo[cells[0].Value] = "任务已存在"
-								}
-
-								//更新task
-								UpdateOldTask(task["_id"].(bson.ObjectId).Hex(), s_descript, s_urgency, l_complete)
-							} else {
-								if cells[1].Value != "" {
-									ug = isUrgency(cells)
-									if ug == "0" {
-										errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行紧急度填写错误"
-										continue
-									}
-									o["l_complete"] = ft.CompleteTime(ug)
-									o["s_descript"] = cells[1].Value + "\n"
-									o["s_urgency"] = ug
-								} else {
-									errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行数据不完整"
-								}
-							}
+						task := *mgdb.FindOne("task", query)
+						if len(task) > 0 { //任务已存在
+							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行爬虫任务已存在"
+							continue
+						}
+						urgency := cells[2].Value //紧急程度
+						if urgency == "特别紧急" {
+							urgency = "4"
+						} else if urgency == "非常紧急" {
+							urgency = "3"
+						} else if urgency == "紧急" {
+							urgency = "2"
+						} else if urgency == "普通" {
+							urgency = "1"
+						}
+						stype := cells[4].Value //任务类型
+						if stype == "采集频率异常" {
+							stype = "8"
+						} else if stype == "列表页异常" {
+							stype = "7"
+						} else if stype == "404异常" {
+							stype = "6"
+						} else if stype == "下载异常" {
+							stype = "5"
+						} else if stype == "运行异常" {
+							stype = "4"
+						} else if stype == "时间异常" {
+							stype = "3"
+						} else if stype == "数据异常" {
+							stype = "2"
+						} else if stype == "数量异常" {
+							stype = "1"
 						} else {
-							errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行数据不完整"
+							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行任务类型填写错误"
 							continue
 						}
-						if len(task) == 0 {
-							queryL := bson.M{"code": cells[0].Value}
-							lua := *mgdb.FindOne("luaconfig", queryL)
-							if lua != nil {
-								param := lua["param_common"]
-								o["s_site"] = param.([]interface{})[1] //取数组中的某个值
-								o["s_channel"] = param.([]interface{})[2]
-								o["i_state"] = 1
-								o["s_modify"] = lua["createuser"]
-								o["s_modifyid"] = lua["createuserid"]
-								o["l_comeintime"] = time.Now().Unix()
-								o["i_event"] = lua["event"]
-								o["i_times"] = 0
-							} else {
-								errorinfo[cells[0].Value] = "第" + strconv.Itoa(k+1) + "行爬虫代码填写错误"
-								continue
-							}
+						completeTime := ft.CompleteTime(urgency)
+						if completeTimeStr := cells[3].Value; completeTimeStr != "" { //完成时间
+							timeDate, _ := time.ParseInLocation(qu.Date_Full_Layout, completeTimeStr, time.Local)
+							completeTime = timeDate.Unix()
+						}
+						lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+						if len(lua) > 0 {
+							param := lua["param_common"]
+							o["s_site"] = param.([]interface{})[1] //取数组中的某个值
+							o["s_channel"] = param.([]interface{})[2]
+							o["i_state"] = 1
+							o["s_modify"] = lua["createuser"]
+							o["s_modifyid"] = lua["createuserid"]
+							o["l_comeintime"] = time.Now().Unix()
+							o["i_event"] = lua["event"]
+							o["i_times"] = 0
+							o["s_date"] = time.Now().Format("2006-01-02")
+							o["s_source"] = "人工"
+							o["s_code"] = code
+							o["s_descript"] = cells[1].Value //描述
+							o["s_urgency"] = urgency
+							o["s_stype"] = stype
+							o["l_complete"] = completeTime
 							mgdb.Save("task", o)
 							//清空map
-							o = map[string]interface{}{
-								"s_date":   time.Now().Format("2006-01-02"),
-								"s_source": "人工",
-								"s_type":   "0",
-							}
+							o = map[string]interface{}{}
+						} else {
+							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行爬虫代码填写错误"
 						}
-						//}
+					} else {
+						break
 					}
 				}
 			}
@@ -514,13 +405,14 @@ func (t *TaskM) CheckCode() {
 		query := bson.M{
 			"s_code": code,
 			"i_state": bson.M{
-				"$in": []int{1, 2, 5},
+				"$in": []int{0, 1, 2, 3, 5},
 			},
 		}
 		task := *mgdb.FindOne("task", query)
 		if task != nil {
 			task["l_complete"] = time.Unix(task["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
 			status = "hasCode"
+
 		} else {
 			luaQuery := bson.M{
 				"code": code,
@@ -973,21 +865,6 @@ func (t *TaskM) Audit() {
 	}
 }
 
-//判断紧急度
-func isUrgency(cells []*xlsx.Cell) string {
-	ug := "0"
-	if cells[2].Value == "普通" { //不紧急
-		ug = "1" //5天
-	} else if cells[2].Value == "紧急" {
-		ug = "2" //2天
-	} else if cells[2].Value == "非常紧急" {
-		ug = "3" //6小时
-	} else if cells[2].Value == "特别紧急" { //非常紧急
-		ug = "4" //2小时
-	}
-	return ug
-}
-
 //更新task
 func UpdateOldTask(id, descript, urgency string, complete int64) {
 	queryT := bson.M{