package front import ( "fmt" qu "qfw/util" "strings" "sync" "time" . "util" ) var ( //爬虫认领 ClaimMaxNum = 10 //已认领爬虫上限 ClaimLock = &sync.Mutex{} ClaimPriorityLimit = 350 //区分外包和内部人员认领爬虫的优先级界限 ClaimPriorityOut = 200 ClaimPriorityIn = 700 ClaimQueryFields = map[string]interface{}{ "site": 1, "code": 1, "channel": 1, "priority": 1, "spiderimportant": 1, "modifyuser": 1, "claimtime": 1, "claimtype": 1, "recovertime": 1, //"grade": 1, } ) //type ClaimLog struct { // Site string `bson:"site"` // Code string `bson:"code"` // Channel string `bson:"channel"` // ModifyUser string `bson:"modifyuser"` // Priority int `bson:"priority"` // Comeintime int64 `bson:"comeintime"` // Important bool `bson:"important"` // ClaimTime int64 `bson:"claimtime"` // RecoverTime int64 `bson:"recovertime"` // ReturnTime int64 `bson:"returntime"` // Stype string `bson:"stype"` //认领、回收、归还 // ReturnReason string `bson:"returnreason"` //归还原因 // ClaimRecoverType int `bson:"claimrecovertype"` //认领或回收类型 1:主动;0:被动(爬虫分配) //} const CLAIMTYPEUNCLAIMED, CLAIMTYPECLAIMED, CLAIMTYPEHISTORY = 0, 1, 2 //未认领、已认领、历史爬虫 func (f *Front) ClaimCode() { ClaimLock.Lock() defer ClaimLock.Unlock() modifyuser := f.GetSession("loginuser") identity := qu.IntAll(f.GetSession("identity")) //1、已认领爬虫个数 q := map[string]interface{}{ "modifyuser": modifyuser, "claimtype": CLAIMTYPECLAIMED, "state": map[string]interface{}{ "$in": []int{0, 2}, //待完成、未通过 }, "platform": map[string]interface{}{ "$in": []string{"golua平台", "chrome"}, }, } num := MgoEB.Count("luaconfig", q) if num >= ClaimMaxNum { f.ServeJson(map[string]interface{}{"msg": "已超爬虫认领上限!", "ok": false}) return } //2、认领爬虫逻辑 q = map[string]interface{}{ "claimtype": CLAIMTYPEUNCLAIMED, //待认领 "platform": map[string]interface{}{ "$in": []string{"golua平台", "chrome"}, }, "state": Sp_state_0, } s := 1 if identity == 1 { //内部人员 s = -1 q["priority"] = map[string]interface{}{ "$gte": ClaimPriorityLimit, } } else { //外包指定优先级范围且简单的爬虫 q["priority"] = map[string]interface{}{ "$gt": 0, "$lt": ClaimPriorityLimit, } q["grade"] = 0 } sort := map[string]interface{}{ "priority": s, } qu.Debug(f.GetSession("loginuser"), "认领爬虫:", q) lua, _ := MgoEB.Find("luaconfig", q, sort, ClaimQueryFields, false, -1, -1) if len(*lua) > 0 { claimNum := 0 //本次认领爬虫个数 claimSite := map[string]bool{} //本次认领的站点 for _, l := range *lua { site := qu.ObjToString(l["site"]) if claimSite[site] { continue } claimSite[site] = true q["site"] = site //查询指定站点爬虫 //delete(q, "priority") //这里不限制优先级 tmpLua, _ := MgoEB.Find("luaconfig", q, nil, ClaimQueryFields, false, -1, -1) claimNum += len(*tmpLua) //更新数据、新增日志 UpdateCodeAndSaveLog(*tmpLua, f) if num+claimNum >= ClaimMaxNum { break } } f.ServeJson(map[string]interface{}{"msg": "成功认领爬虫" + fmt.Sprint(claimNum) + "个", "ok": true}) return } f.ServeJson(map[string]interface{}{"msg": "暂无可认领爬虫!", "ok": false}) } func (f *Front) ReturnCode() { identity := qu.IntAll(f.GetSession("identity")) if identity != 0 { f.ServeJson(map[string]interface{}{"msg": "无权限归还爬虫!", "ok": false}) return } codes := f.GetString("codes") returnreason := f.GetString("returnreason") qu.Debug(codes) cs := strings.Split(codes, ",") q := map[string]interface{}{ "code": map[string]interface{}{ "$in": cs, }, } luas, _ := MgoEB.Find("luaconfig", q, nil, ClaimQueryFields, false, -1, -1) if len(*luas) > 0 { update := [][]map[string]interface{}{} save := []map[string]interface{}{} for _, l := range *luas { now := time.Now().Unix() up := []map[string]interface{}{ {"code": l["code"]}, {"$set": map[string]interface{}{ "claimtype": CLAIMTYPEUNCLAIMED, "claimtime": int64(0), "recovertime": int64(0), "priority": ClaimPriorityIn, "state": 0, "grade": 1, //外包归还爬虫后,爬虫难易度更新为困难 }}, } update = append(update, up) save = append(save, map[string]interface{}{ "site": l["site"], "code": l["code"], "channel": l["channel"], "modifyuser": l["modifyuser"], "priority": l["priority"], "stype": "归还", "comeintime": now, "claimtime": l["claimtime"], "recovertime": l["recovertime"], "returntime": now, "important": l["spiderimportant"], "returnreason": returnreason, "claimrecovertype": 1, "source": "golua平台爬虫归还", }) } //更新爬虫信息 MgoEB.UpdateBulk("luaconfig", update...) //保存认领日志 MgoEB.SaveBulk("lua_logs_claim", save...) f.ServeJson(map[string]interface{}{"msg": "爬虫归还成功!", "ok": true}) return } f.ServeJson(map[string]interface{}{"msg": "爬虫归还失败!", "ok": false}) } func UpdateCodeAndSaveLog(lua []map[string]interface{}, f *Front) { update := [][]map[string]interface{}{} save := []map[string]interface{}{} for _, l := range lua { loginuser := f.GetSession("loginuser") userid := f.GetSession("userid") priority := qu.IntAll(l["priority"]) spiderimportant, _ := l["spiderimportant"].(bool) recovertime := CreateRecovertime(spiderimportant, priority) now := time.Now().Unix() up := []map[string]interface{}{ {"code": l["code"]}, {"$set": map[string]interface{}{ "claimtype": CLAIMTYPECLAIMED, "createuseremail": f.GetSession("email"), "createuser": loginuser, "createuserid": userid, "modifyuser": loginuser, "modifyuserid": userid, "claimtime": now, "state": Sp_state_0, //通用平台退回爬虫状态为:无法标注12 "recovertime": recovertime, }}, } update = append(update, up) save = append(save, map[string]interface{}{ "site": l["site"], "code": l["code"], "channel": l["channel"], "modifyuser": loginuser, "priority": priority, "stype": "认领", "comeintime": now, "claimtime": now, "recovertime": recovertime, "returntime": int64(0), "important": spiderimportant, "returnreason": "", "claimrecovertype": 1, "source": "golua平台爬虫认领", }) } //更新爬虫信息 MgoEB.UpdateBulk("luaconfig", update...) //保存认领日志 MgoEB.SaveBulk("lua_logs_claim", save...) } // CreateRecovertime 生成归还时间 func CreateRecovertime(important bool, priority int) int64 { if important || priority >= ClaimPriorityLimit { return time.Now().Add(24 * time.Hour).Unix() } else { return time.Now().Add(24 * 5 * time.Hour).Unix() } } // UpdateLuaClaimtype 更新爬虫认领状态 func UpdateLuaClaimtype(code string, claimtype int) { MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{ "claimtype": claimtype, "claimtime": int64(0), "recovertime": int64(0), }}, false, false) }