周报；爬虫修改维护人

src/config.json

@@ -134,6 +134,7 @@
 	"movelistdatacron": "0 0 0 ? * *",
 	"spiderweeklyreportcron": "0 50 6 ? * SUN",
 	"luamovecron": "0 0 8 1 * *",
+	"updateluausercron": "0 30 23 ? * *",
 	"closenum": 2,
 	"daynum": 6,
 	"mail": {

src/luatask/task.go

@@ -210,7 +210,9 @@ func InitInfo() {
 	SameDayHref = map[string]string{}
 	DataBakAllHref = map[string]string{}
 	UserTaskNum = map[string]map[string]int{}
-	StartTime, EndTime = util.GetWorkDayTimeUnix()
+	//StartTime, EndTime = util.GetWorkDayTimeUnix()
+	StartTime = util.GetTime(-1)
+	EndTime = util.GetTime(0)
 	Publishtime = qu.FormatDateByInt64(&StartTime, qu.Date_Short_Layout)
 	//StartTime = util.GetTime(-1)
 	//EndTime = util.GetTime(0)
@@ -1061,29 +1063,45 @@ func GetSpiderWarnErrData() {
 				wg.Done()
 			}()
 			infoText := qu.ObjToString(tmp["info"])
-			code := qu.ObjToString(tmp["code"])
-			href := qu.ObjToString(tmp["href"])
-			if qu.IntAll("level") == 2 {
-				stype := ""
-				/*if infoText == "Publishtime Is Too Late" {  //发布时间超前
-					query["info"] = infoText
-					stype = "publishtime"
-				} else if infoText == "Publishtime Is Less Than Zero" { //发布时间小于0
-					query["info"] = infoText
-					stype = "publishtime"
-				} else if infoText == "Publishtime Is Too Early" { //发布时间过小
-					query["info"] = infoText
-					stype = "publishtime"
-				} else*/
+			level := qu.IntAll("level")
+			stype := "" //记录异常类型
+			if level == 2 {
 				if infoText == "Field Value Not Contains Chinese" { //title、detail不含中文（数据分析处理每日异常数据）
 					stype = "text"
 				} else if infoText == "Publishtime Is Less Than Zero" { //发布时间小于0
 					stype = "publishtime"
 				} else if infoText == "Field Value Contains Random Code" { //title、detail含乱码
 					stype = "text"
-				} else {
-					return
 				}
+			} else if level == 1 {
+				if infoText == "Attachment Upload Failed" { //附件异常信息
+					flag := false
+					if data, ok := tmp["data"].(map[string]interface{}); ok {
+						if projectinfo, ok := data["projectinfo"].(map[string]interface{}); ok {
+							if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok {
+								for _, dataInfo := range attachments {
+									if d, ok := dataInfo.(map[string]interface{}); ok {
+										org_url := qu.ObjToString(d["org_url"])
+										if strings.HasSuffix(org_url, "javascript:void(0)") {
+											flag = true
+											break
+										}
+									}
+								}
+							}
+						}
+					}
+					if flag { //列入下载异常任务类型
+						infoText = "File Err" //infoText改为File Err，附件异常
+						stype = "annex"
+					}
+				} else if infoText == "Publishtime Is Zero" { //发布时间为0
+					stype = "publishtime"
+				}
+			}
+			if stype != "" {
+				code := qu.ObjToString(tmp["code"])
+				href := qu.ObjToString(tmp["href"])
 				oneErrInfo := &ErrRemark{
 					Href:   href,
 					Remark: infoText,
@@ -1094,6 +1112,7 @@ func GetSpiderWarnErrData() {
 						errMap.Num += 1                //叠加某种异常stype的数量
 						if !errMap.ErrInfo[infoText] { //记录相同stype不同infoText的异常信息
 							errMap.Err = append(errMap.Err, oneErrInfo)
+							errMap.ErrInfo[infoText] = true
 						}
 					} else {
 						spider.Error[stype] = &ErrorInfo{
@@ -1106,45 +1125,91 @@ func GetSpiderWarnErrData() {
 					}
 				}
 				lock.Unlock()
-			} else if infoText == "Attachment Upload Failed" { //检测附件链接含javascript:void(0)
-				flag := false
-				if data, ok := tmp["data"].(map[string]interface{}); ok {
-					if projectinfo, ok := data["projectinfo"].(map[string]interface{}); ok {
-						if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok {
-							for _, dataInfo := range attachments {
-								if d, ok := dataInfo.(map[string]interface{}); ok {
-									org_url := qu.ObjToString(d["org_url"])
-									if strings.HasSuffix(org_url, "javascript:void(0)") {
-										flag = true
-										break
-									}
-								}
-							}
-						}
+			}
+			/*
+				if level == 2 {
+					stype := ""
+					//if infoText == "Publishtime Is Too Late" {  //发布时间超前
+					//	query["info"] = infoText
+					//	stype = "publishtime"
+					//} else if infoText == "Publishtime Is Less Than Zero" { //发布时间小于0
+					//	query["info"] = infoText
+					//	stype = "publishtime"
+					//} else if infoText == "Publishtime Is Too Early" { //发布时间过小
+					//	query["info"] = infoText
+					//	stype = "publishtime"
+					//} else
+					if infoText == "Field Value Not Contains Chinese" { //title、detail不含中文（数据分析处理每日异常数据）
+						stype = "text"
+					} else if infoText == "Publishtime Is Less Than Zero" { //发布时间小于0
+						stype = "publishtime"
+					} else if infoText == "Field Value Contains Random Code" { //title、detail含乱码
+						stype = "text"
+					} else {
+						return
 					}
-				}
-				if flag { //列入下载异常任务类型
 					oneErrInfo := &ErrRemark{
 						Href:   href,
-						Remark: "File Err",
+						Remark: infoText,
 					}
 					lock.Lock()
 					if spider := CodeInfoMap[code]; spider != nil {
-						if errMap := spider.Error["annex"]; errMap != nil {
-							errMap.Num += 1 //叠加某种异常stype的数量
-							errMap.Err = append(errMap.Err, oneErrInfo)
+						if errMap := spider.Error[stype]; errMap != nil {
+							errMap.Num += 1                //叠加某种异常stype的数量
+							if !errMap.ErrInfo[infoText] { //记录相同stype不同infoText的异常信息
+								errMap.Err = append(errMap.Err, oneErrInfo)
+								errMap.ErrInfo[infoText] = true
+							}
 						} else {
-							spider.Error["annex"] = &ErrorInfo{
+							spider.Error[stype] = &ErrorInfo{
 								Num: 1,
 								Err: []*ErrRemark{
 									oneErrInfo,
 								},
+								ErrInfo: map[string]bool{infoText: true},
 							}
 						}
 					}
 					lock.Unlock()
-				}
-			}
+				} else if infoText == "Attachment Upload Failed" { //检测附件链接含javascript:void(0)
+					flag := false
+					if data, ok := tmp["data"].(map[string]interface{}); ok {
+						if projectinfo, ok := data["projectinfo"].(map[string]interface{}); ok {
+							if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok {
+								for _, dataInfo := range attachments {
+									if d, ok := dataInfo.(map[string]interface{}); ok {
+										org_url := qu.ObjToString(d["org_url"])
+										if strings.HasSuffix(org_url, "javascript:void(0)") {
+											flag = true
+											break
+										}
+									}
+								}
+							}
+						}
+					}
+					if flag { //列入下载异常任务类型
+						oneErrInfo := &ErrRemark{
+							Href:   href,
+							Remark: "File Err",
+						}
+						lock.Lock()
+						if spider := CodeInfoMap[code]; spider != nil {
+							if errMap := spider.Error["annex"]; errMap != nil {
+								errMap.Num += 1 //叠加某种异常stype的数量
+								errMap.Err = append(errMap.Err, oneErrInfo)
+							} else {
+								spider.Error["annex"] = &ErrorInfo{
+									Num: 1,
+									Err: []*ErrRemark{
+										oneErrInfo,
+									},
+								}
+							}
+						}
+						lock.Unlock()
+					}
+				}*/
 		}(tmp)
 		if n%10 == 0 {
 			logger.Debug(n)

src/main.go

@@ -48,6 +48,7 @@ func main() {
 	//
 	c.AddFunc(util.QyworkRemindModifyuserCron, timetask.SendInfoToWxWork_Tomodifyuser) //企业微信日常警告,爬虫开发人员告警信息
 	c.AddFunc(util.QyworkRemindAuditorCron, timetask.SendInfoToWxWork_ToAuditor)       //企业微信日常警告,审核人员告警信息
+	c.AddFunc(util.UpdateLuaUserCron, timetask.UpdateLuaUser)                          //更新外包爬虫所属人
 
 	c.AddFunc(util.MoveListDataCron, timetask.MoveListData) //列表页数据迁移
 	//爬虫任务

src/timetask/report.go

@@ -51,7 +51,7 @@ func SpiderWeeklyReport() {
 	//新建任务同比增减
 	AddTaskIncDecRatio := float64(0)
 	if ThisWeekAddTaskNum != 0 {
-		AddTaskIncDecRatio = float64(ThisWeekAddTaskNum-LastWeekAddTaskAllNum) / float64(ThisWeekAddTaskNum)
+		AddTaskIncDecRatio = float64(ThisWeekAddTaskNum-LastWeekAddTaskAllNum) / float64(LastWeekAddTaskAllNum)
 	}
 	AddTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddTaskIncDecRatio), 64)
 	qu.Debug("新建任务同比增减：", AddTaskIncDecRatio)
@@ -72,16 +72,15 @@ func SpiderWeeklyReport() {
 	//核实任务同比增减
 	CheckTaskIncDecRatio := float64(0)
 	if ThisWeekCheckTaskNum != 0 {
-		CheckTaskIncDecRatio = float64(ThisWeekCheckTaskNum-LastWeekCheckTaskNum) / float64(ThisWeekCheckTaskNum)
+		CheckTaskIncDecRatio = float64(ThisWeekCheckTaskNum-LastWeekCheckTaskNum) / float64(LastWeekCheckTaskNum)
 	}
 	CheckTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CheckTaskIncDecRatio), 64)
 	qu.Debug("核实任务同比增减：", CheckTaskIncDecRatio)
 	//待核实任务总数
 	query = map[string]interface{}{
-		//"i_state": map[string]interface{}{
-		//	"$lte": 1, //任务状态：待确认、待处理
-		//},
-		"i_state": 1, //待处理
+		"i_state": map[string]interface{}{
+			"$lte": 1, //任务状态：待确认、待处理
+		},
 	}
 	NeedToCheckTaskAllNum := util.MgoEB.Count("task", query)
 	qu.Debug("待核实任务总数：", NeedToCheckTaskAllNum)
@@ -102,7 +101,7 @@ func SpiderWeeklyReport() {
 	//lua新建爬虫同比增减
 	AddLuaIncDecRatio := float64(0)
 	if ThisWeekAddLuaNum != 0 {
-		AddLuaIncDecRatio = float64(ThisWeekAddLuaNum-LastWeekAddLuaNum) / float64(ThisWeekAddLuaNum)
+		AddLuaIncDecRatio = float64(ThisWeekAddLuaNum-LastWeekAddLuaNum) / float64(LastWeekAddLuaNum)
 	}
 	AddLuaIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddLuaIncDecRatio), 64)
 	qu.Debug("lua新建爬虫同比增减：", AddLuaIncDecRatio)
@@ -120,7 +119,7 @@ func SpiderWeeklyReport() {
 	//lua完成新建爬虫同比增减
 	FinishLuaIncDecRatio := float64(0)
 	if ThisWeekFinishLuaNum != 0 {
-		FinishLuaIncDecRatio = float64(ThisWeekFinishLuaNum-LastWeekFinishLuaNum) / float64(ThisWeekFinishLuaNum)
+		FinishLuaIncDecRatio = float64(ThisWeekFinishLuaNum-LastWeekFinishLuaNum) / float64(LastWeekFinishLuaNum)
 	}
 	FinishLuaIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishLuaIncDecRatio), 64)
 	qu.Debug("lua完成新建爬虫同比增减：", FinishLuaIncDecRatio)
@@ -154,7 +153,7 @@ func SpiderWeeklyReport() {
 	//新建待维护任务同比增减
 	AddEffectTaskIncDecRatio := float64(0)
 	if ThisWeekAddEffectTaskNum != 0 {
-		AddEffectTaskIncDecRatio = float64(ThisWeekAddEffectTaskNum-LastWeekAddEffectTaskNum) / float64(ThisWeekAddEffectTaskNum)
+		AddEffectTaskIncDecRatio = float64(ThisWeekAddEffectTaskNum-LastWeekAddEffectTaskNum) / float64(LastWeekAddEffectTaskNum)
 	}
 	AddEffectTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddEffectTaskIncDecRatio), 64)
 	qu.Debug("新建待维护任务同比增减：", AddEffectTaskIncDecRatio)
@@ -173,7 +172,7 @@ func SpiderWeeklyReport() {
 	//完成待维护任务同比增减
 	FinishEffectTaskIncDecRatio := float64(0)
 	if ThisWeekFinishEffectTaskNum != 0 {
-		FinishEffectTaskIncDecRatio = float64(ThisWeekFinishEffectTaskNum-LastWeekFinishEffectTaskNum) / float64(ThisWeekFinishEffectTaskNum)
+		FinishEffectTaskIncDecRatio = float64(ThisWeekFinishEffectTaskNum-LastWeekFinishEffectTaskNum) / float64(LastWeekFinishEffectTaskNum)
 	}
 	FinishEffectTaskIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishEffectTaskIncDecRatio), 64)
 	qu.Debug("完成待维护任务同比增减：", FinishEffectTaskIncDecRatio)
@@ -202,7 +201,7 @@ func SpiderWeeklyReport() {
 	//新建爬虫数量同比增减
 	AddPythonIncDecRatio := float64(0)
 	if ThisWeekAddPythonNum != 0 {
-		AddPythonIncDecRatio = float64(ThisWeekAddPythonNum-LastWeekAddPythonNum) / float64(ThisWeekAddPythonNum)
+		AddPythonIncDecRatio = float64(ThisWeekAddPythonNum-LastWeekAddPythonNum) / float64(LastWeekAddPythonNum)
 	}
 	AddPythonIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AddPythonIncDecRatio), 64)
 	qu.Debug("python新建爬虫同比增减：", AddPythonIncDecRatio)
@@ -222,7 +221,7 @@ func SpiderWeeklyReport() {
 	//已完成爬虫数量同比增减
 	FinishPythonIncDecRatio := float64(0)
 	if ThisWeekFinishPythonNum != 0 {
-		FinishPythonIncDecRatio = float64(ThisWeekFinishPythonNum-LastWeekFinishPythonNum) / float64(ThisWeekFinishPythonNum)
+		FinishPythonIncDecRatio = float64(ThisWeekFinishPythonNum-LastWeekFinishPythonNum) / float64(LastWeekFinishPythonNum)
 	}
 	FinishPythonIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", FinishPythonIncDecRatio), 64)
 	qu.Debug("python已完成爬虫数量同比增减：", FinishPythonIncDecRatio)
@@ -253,14 +252,14 @@ func SpiderWeeklyReport() {
 	//爬虫总量增长率
 	AllCodeIncDecRatio := float64(0)
 	if ThisWeekCodeAllNum != 0 {
-		AllCodeIncDecRatio = float64(ThisWeekCodeAllNum-LastWeekCodeAllNum) / float64(ThisWeekCodeAllNum)
+		AllCodeIncDecRatio = float64(ThisWeekCodeAllNum-LastWeekCodeAllNum) / float64(LastWeekCodeAllNum)
 	}
 	AllCodeIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllCodeIncDecRatio), 64)
 	qu.Debug("爬虫总量增长率：", AllCodeIncDecRatio)
 	//站点总量增长率
 	AllSiteIncDecRatio := float64(0)
 	if ThisWeekSiteAllNum != 0 {
-		AllSiteIncDecRatio = float64(ThisWeekSiteAllNum-LastWeekSiteAllNum) / float64(ThisWeekSiteAllNum)
+		AllSiteIncDecRatio = float64(ThisWeekSiteAllNum-LastWeekSiteAllNum) / float64(LastWeekSiteAllNum)
 	}
 	AllSiteIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllSiteIncDecRatio), 64)
 	qu.Debug("爬虫总量增长率：", AllSiteIncDecRatio)
@@ -278,7 +277,7 @@ func SpiderWeeklyReport() {
 	//mgo采集量增长率
 	AllMgoNumIncDecRatio := float64(0)
 	if ThisWeekMgoNum != 0 {
-		AllMgoNumIncDecRatio = float64(ThisWeekMgoNum-LastWeekMgoNum) / float64(ThisWeekMgoNum)
+		AllMgoNumIncDecRatio = float64(ThisWeekMgoNum-LastWeekMgoNum) / float64(LastWeekMgoNum)
 	}
 	AllMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllMgoNumIncDecRatio), 64)
 	qu.Debug("mgo采集量增长率：", AllMgoNumIncDecRatio)
@@ -290,7 +289,7 @@ func SpiderWeeklyReport() {
 	//es索引量增长率
 	AllEsNumIncDecRatio := float64(0)
 	if ThisWeekEsNum != 0 {
-		AllEsNumIncDecRatio = (float64(ThisWeekEsNum) - float64(LastWeekEsNum)) / float64(ThisWeekEsNum)
+		AllEsNumIncDecRatio = (float64(ThisWeekEsNum) - float64(LastWeekEsNum)) / float64(LastWeekEsNum)
 	}
 	AllEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", AllEsNumIncDecRatio), 64)
 	qu.Debug("es索引量增长率：", AllEsNumIncDecRatio)
@@ -301,7 +300,7 @@ func SpiderWeeklyReport() {
 	//竞品mgo采集量增长率：
 	CompeteMgoNumIncDecRatio := float64(0)
 	if ThisWeekEsNum != 0 {
-		CompeteMgoNumIncDecRatio = float64(ThisWeekCompeteMgoNum-LastWeekCompeteMgoNum) / float64(ThisWeekCompeteMgoNum)
+		CompeteMgoNumIncDecRatio = float64(ThisWeekCompeteMgoNum-LastWeekCompeteMgoNum) / float64(LastWeekCompeteMgoNum)
 	}
 	CompeteMgoNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteMgoNumIncDecRatio), 64)
 	qu.Debug("竞品mgo采集量增长率：", CompeteMgoNumIncDecRatio)
@@ -318,7 +317,7 @@ func SpiderWeeklyReport() {
 	//竞品es索引量增长率
 	CompeteEsNumIncDecRatio := float64(0)
 	if ThisWeekEsNum != 0 {
-		CompeteEsNumIncDecRatio = float64(ThisWeekCompeteEsNum-LastWeekCompeteEsNum) / float64(ThisWeekCompeteEsNum)
+		CompeteEsNumIncDecRatio = float64(ThisWeekCompeteEsNum-LastWeekCompeteEsNum) / float64(LastWeekCompeteEsNum)
 	}
 	CompeteEsNumIncDecRatio, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", CompeteEsNumIncDecRatio), 64)
 	qu.Debug("竞品es索引量增长率：", CompeteEsNumIncDecRatio)

src/timetask/updateluauser.go

@@ -0,0 +1,106 @@
+package timetask
+
+import (
+	"github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
+	"sync"
+	"util"
+)
+
+var LuaUserNum int                     //lua开发人员个数
+var LuaUserIndexMap = map[int]string{} //lua开发人员集合
+
+//每天将外包已上架爬虫分配给内部人员
+func UpdateLuaUser() {
+	defer qu.Catch()
+	sess := util.MgoEB.GetMgoConn()
+	defer util.MgoEB.DestoryMongoConn(sess)
+	query := map[string]interface{}{
+		"state":    5,
+		"platform": "golua平台",
+	}
+	fieles := map[string]interface{}{
+		"code":       1,
+		"modifyuser": 1,
+	}
+	it := sess.DB(util.MgoEB.DbName).C("luaconfig").Find(&query).Select(&fieles).Iter()
+	n := 1
+	lock := &sync.Mutex{}
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 2)
+	luaArr := [][]map[string]interface{}{}
+	taskArr := [][]map[string]interface{}{}
+	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
+		wg.Add(1)
+		ch <- true
+		go func(tmp map[string]interface{}, num int) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			modifyuser := qu.ObjToString(tmp["modifyuser"])
+			if LuaModifyUserInfoMap[modifyuser] == nil { //外包人员
+				spidercode := qu.ObjToString(tmp["code"])
+				index := num % LuaUserNum //取余，判断爬虫分给谁
+				luaUser := LuaUserIndexMap[index]
+				luaUserInfo := LuaModifyUserInfoMap[luaUser]
+				if luaUserInfo == nil {
+					logger.Info(luaUser, "初始化信息失败")
+					return
+				}
+				//更新爬虫
+				updateLua := []map[string]interface{}{}
+				updateLua = append(updateLua, map[string]interface{}{"_id": tmp["_id"]})
+				updateLua = append(updateLua, map[string]interface{}{
+					"$set": map[string]interface{}{
+						"modifyuser":      luaUser,
+						"modifyuserid":    luaUserInfo.UserID,
+						"createuser":      luaUser,
+						"createuserid":    luaUserInfo.UserID,
+						"createuseremail": luaUserInfo.Email,
+						"next":            luaUserInfo.Email,
+					},
+				})
+				//更新任务
+				updateTask := []map[string]interface{}{}
+				updateTask = append(updateTask, map[string]interface{}{
+					"s_code": spidercode,
+					"i_state": map[string]interface{}{
+						"$lte": 2, //更新待确认、待处理、处理中任务的所属人
+					},
+				})
+				updateTask = append(updateTask, map[string]interface{}{
+					"$set": map[string]interface{}{
+						"s_modify":   luaUser,
+						"s_modifyid": luaUserInfo.UserID,
+					},
+				})
+				lock.Lock()
+				//lua
+				luaArr = append(luaArr, updateLua)
+				if len(luaArr) > 100 {
+					util.MgoEB.UpdateBulk("luaconfig", luaArr...)
+					luaArr = [][]map[string]interface{}{}
+				}
+				//task
+				taskArr = append(taskArr, updateTask)
+				if len(taskArr) > 100 {
+					util.MgoEB.UpdateBulk("task", taskArr...)
+					taskArr = [][]map[string]interface{}{}
+				}
+				lock.Unlock()
+			}
+		}(tmp, n)
+		tmp = map[string]interface{}{}
+	}
+	wg.Wait()
+	if len(luaArr) > 0 {
+		util.MgoEB.UpdateBulk("luaconfig", luaArr...)
+		luaArr = [][]map[string]interface{}{}
+	}
+	if len(taskArr) > 0 {
+		util.MgoEB.UpdateBulk("task", taskArr...)
+		taskArr = [][]map[string]interface{}{}
+	}
+	logger.Debug("重新分配外包爬虫完成...")
+}

src/timetask/wxworkwarn.go

@@ -36,6 +36,8 @@ type PythonUserTextInfo struct {
 
 type UserInfo struct {
 	Username string
+	UserID   string
+	Email    string
 	Mobile   string
 	Auth     string
 }
@@ -84,6 +86,8 @@ func GetLuaUserInfo() {
 			if auth == "1" { //开发人员
 				LuaModifyUserInfoMap[eu] = &UserInfo{
 					Username: info["username"],
+					UserID:   info["userid"],
+					Email:    info["email"],
 					Mobile:   info["mobile"],
 					Auth:     auth,
 				}
@@ -105,9 +109,17 @@ func GetLuaUserInfo() {
 			}
 		}
 	}
+	//
+	index := 0
+	for name, _ := range LuaModifyUserInfoMap {
+		LuaUserIndexMap[index] = name
+		index++
+	}
+	LuaUserNum = index
 	qu.Debug(LuaModifyUserInfoMap)
 	qu.Debug(LuaAuditorInfoMap)
 	qu.Debug(PythonModifyUserInfoMap)
+	qu.Debug(LuaUserIndexMap, LuaUserNum)
 }
 
 // 统计爬虫开发人员未完成爬虫和任务

src/user.json

@@ -1,18 +1,24 @@
 {
 	"ssc":{
 		"username": "施顺才",
+		"email": "shishuncai@topnet.net.cn",
+		"userid": "5cbd57caa9c3320223906552",
 		"mobile": "13523457747",
 		"role": "lua",
 		"auth": "1"
 	},
 	"lyf":{
 		"username": "刘一帆",
+		"email": "liuyifan@topnet.net.cn",
+		"userid": "5cbd57e9a9c3320223906553",
 		"mobile": "15896901897",
 		"role": "lua",
 		"auth": "1"
 	},
 	"jiaoyubo":{
 		"username": "焦宇波",
+		"email": "jiaoyubo@topnet.net.cn",
+		"userid": "5bd67dac18920107b024c6dc",
 		"mobile": "15516197109",
 		"role": "lua",
 		"auth": "1"

src/util/config.go

@@ -30,6 +30,7 @@ var (
 	MoveListDataCron           string        //迁移spider_highlistdata、spider_listdata数据
 	SpiderWeeklyReportCron     string        //周报统计
 	LuamoveCron                string        //每月1日统计要转移节点的爬虫
+	UpdateLuaUserCron          string        //每天更新外包爬虫到内部人员
 	CloseNum                   int           //关闭几天的任务
 	DayNum                     int           //更新数据天数
 	CodeEventModel             map[int]int   //节点对应的采集模式0：老模式；1：新模式
@@ -99,6 +100,7 @@ func InitOther() {
 	MoveListDataCron = qu.ObjToString(Config["movelistdatacron"])
 	SpiderWeeklyReportCron = qu.ObjToString(Config["spiderweeklyreportcron"])
 	LuamoveCron = qu.ObjToString(Config["luamovecron"])
+	UpdateLuaUserCron = qu.ObjToString(Config["updateluausercron"])
 
 	CloseNum = qu.IntAll(Config["closenum"])
 	DayNum = qu.IntAll(Config["daynum"])