spider_editor/src/front/claim.go

package front

import (
	"fmt"
	qu "qfw/util"
	"strings"
	"sync"
	"time"
	. "util"
)

var (
	//爬虫认领
	ClaimMaxNum        = 10 //已认领爬虫上限
	ClaimLock          = &sync.Mutex{}
	ClaimPriorityLimit = 350 //区分外包和内部人员认领爬虫的优先级界限
	ClaimPriorityOut   = 200
	ClaimPriorityIn    = 700
	ClaimQueryFields   = map[string]interface{}{
		"site":            1,
		"code":            1,
		"channel":         1,
		"priority":        1,
		"spiderimportant": 1,
		"modifyuser":      1,
		"claimtime":       1,
		"claimtype":       1,
		"recovertime":     1,
		//"grade":           1,
	}
)

//type ClaimLog struct {
//	Site             string `bson:"site"`
//	Code             string `bson:"code"`
//	Channel          string `bson:"channel"`
//	ModifyUser       string `bson:"modifyuser"`
//	Priority         int    `bson:"priority"`
//	Comeintime       int64  `bson:"comeintime"`
//	Important        bool   `bson:"important"`
//	ClaimTime        int64  `bson:"claimtime"`
//	RecoverTime      int64  `bson:"recovertime"`
//	ReturnTime       int64  `bson:"returntime"`
//	Stype            string `bson:"stype"`            //认领、回收、归还
//	ReturnReason     string `bson:"returnreason"`     //归还原因
//	ClaimRecoverType int    `bson:"claimrecovertype"` //认领或回收类型 1：主动；0：被动（爬虫分配）
//}

const CLAIMTYPEUNCLAIMED, CLAIMTYPECLAIMED, CLAIMTYPEHISTORY = 0, 1, 2 //未认领、已认领、历史爬虫

func (f *Front) ClaimCode() {
	ClaimLock.Lock()
	defer ClaimLock.Unlock()
	modifyuser := f.GetSession("loginuser")
	identity := qu.IntAll(f.GetSession("identity"))
	//1、已认领爬虫个数
	q := map[string]interface{}{
		"modifyuser": modifyuser,
		"claimtype":  CLAIMTYPECLAIMED,
		"state": map[string]interface{}{
			"$in": []int{0, 2}, //待完成、未通过
		},
		"platform": map[string]interface{}{
			"$in": []string{"golua平台", "chrome"},
		},
	}
	num := MgoEB.Count("luaconfig", q)
	if num >= ClaimMaxNum {
		f.ServeJson(map[string]interface{}{"msg": "已超爬虫认领上限！", "ok": false})
		return
	}
	//2、认领爬虫逻辑
	q = map[string]interface{}{
		"claimtype": CLAIMTYPEUNCLAIMED, //待认领
		"platform": map[string]interface{}{
			"$in": []string{"golua平台", "chrome"},
		},
		"state": Sp_state_0,
	}

	s := 1
	if identity == 1 { //内部人员
		s = -1
		q["priority"] = map[string]interface{}{
			"$gte": ClaimPriorityLimit,
		}
	} else { //外包指定优先级范围且简单的爬虫
		q["priority"] = map[string]interface{}{
			"$gt": 0,
			"$lt": ClaimPriorityLimit,
		}
		q["grade"] = 0
	}
	sort := map[string]interface{}{
		"priority": s,
	}
	qu.Debug(f.GetSession("loginuser"), "认领爬虫：", q)
	lua, _ := MgoEB.Find("luaconfig", q, sort, ClaimQueryFields, false, -1, -1)
	if len(*lua) > 0 {
		claimNum := 0                  //本次认领爬虫个数
		claimSite := map[string]bool{} //本次认领的站点
		for _, l := range *lua {
			site := qu.ObjToString(l["site"])
			if claimSite[site] {
				continue
			}
			claimSite[site] = true
			q["site"] = site //查询指定站点爬虫
			//delete(q, "priority") //这里不限制优先级
			tmpLua, _ := MgoEB.Find("luaconfig", q, nil, ClaimQueryFields, false, -1, -1)
			claimNum += len(*tmpLua)
			//更新数据、新增日志
			UpdateCodeAndSaveLog(*tmpLua, f)
			if num+claimNum >= ClaimMaxNum {
				break
			}
		}
		f.ServeJson(map[string]interface{}{"msg": "成功认领爬虫" + fmt.Sprint(claimNum) + "个", "ok": true})
		return
	}
	f.ServeJson(map[string]interface{}{"msg": "暂无可认领爬虫！", "ok": false})
}

func (f *Front) ReturnCode() {
	identity := qu.IntAll(f.GetSession("identity"))
	if identity != 0 {
		f.ServeJson(map[string]interface{}{"msg": "无权限归还爬虫！", "ok": false})
		return
	}
	codes := f.GetString("codes")
	returnreason := f.GetString("returnreason")
	qu.Debug(codes)
	cs := strings.Split(codes, ",")
	q := map[string]interface{}{
		"code": map[string]interface{}{
			"$in": cs,
		},
	}
	luas, _ := MgoEB.Find("luaconfig", q, nil, ClaimQueryFields, false, -1, -1)
	if len(*luas) > 0 {
		update := [][]map[string]interface{}{}
		save := []map[string]interface{}{}
		for _, l := range *luas {
			now := time.Now().Unix()
			up := []map[string]interface{}{
				{"code": l["code"]},
				{"$set": map[string]interface{}{
					"claimtype":   CLAIMTYPEUNCLAIMED,
					"claimtime":   int64(0),
					"recovertime": int64(0),
					"priority":    ClaimPriorityIn,
					"state":       0,
					"grade":       1, //外包归还爬虫后，爬虫难易度更新为困难
				}},
			}
			update = append(update, up)
			save = append(save, map[string]interface{}{
				"site":             l["site"],
				"code":             l["code"],
				"channel":          l["channel"],
				"modifyuser":       l["modifyuser"],
				"priority":         l["priority"],
				"stype":            "归还",
				"comeintime":       now,
				"claimtime":        l["claimtime"],
				"recovertime":      l["recovertime"],
				"returntime":       now,
				"important":        l["spiderimportant"],
				"returnreason":     returnreason,
				"claimrecovertype": 1,
				"source":           "golua平台爬虫归还",
			})
		}
		//更新爬虫信息
		MgoEB.UpdateBulk("luaconfig", update...)
		//保存认领日志
		MgoEB.SaveBulk("lua_logs_claim", save...)
		f.ServeJson(map[string]interface{}{"msg": "爬虫归还成功！", "ok": true})
		return
	}
	f.ServeJson(map[string]interface{}{"msg": "爬虫归还失败！", "ok": false})
}

func UpdateCodeAndSaveLog(lua []map[string]interface{}, f *Front) {
	update := [][]map[string]interface{}{}
	save := []map[string]interface{}{}
	for _, l := range lua {
		loginuser := f.GetSession("loginuser")
		userid := f.GetSession("userid")
		priority := qu.IntAll(l["priority"])
		spiderimportant, _ := l["spiderimportant"].(bool)
		recovertime := CreateRecovertime(spiderimportant, priority)
		now := time.Now().Unix()
		up := []map[string]interface{}{
			{"code": l["code"]},
			{"$set": map[string]interface{}{
				"claimtype":       CLAIMTYPECLAIMED,
				"createuseremail": f.GetSession("email"),
				"createuser":      loginuser,
				"createuserid":    userid,
				"modifyuser":      loginuser,
				"modifyuserid":    userid,
				"claimtime":       now,
				"state":           Sp_state_0, //通用平台退回爬虫状态为：无法标注12
				"recovertime":     recovertime,
			}},
		}
		update = append(update, up)
		save = append(save, map[string]interface{}{
			"site":             l["site"],
			"code":             l["code"],
			"channel":          l["channel"],
			"modifyuser":       loginuser,
			"priority":         priority,
			"stype":            "认领",
			"comeintime":       now,
			"claimtime":        now,
			"recovertime":      recovertime,
			"returntime":       int64(0),
			"important":        spiderimportant,
			"returnreason":     "",
			"claimrecovertype": 1,
			"source":           "golua平台爬虫认领",
		})
	}
	//更新爬虫信息
	MgoEB.UpdateBulk("luaconfig", update...)
	//保存认领日志
	MgoEB.SaveBulk("lua_logs_claim", save...)
}

// CreateRecovertime 生成归还时间
func CreateRecovertime(important bool, priority int) int64 {
	if important || priority >= ClaimPriorityLimit {
		return time.Now().Add(24 * time.Hour).Unix()
	} else {
		return time.Now().Add(24 * 5 * time.Hour).Unix()
	}
}

// UpdateLuaClaimtype 更新爬虫认领状态
func UpdateLuaClaimtype(code string, claimtype int) {
	MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
		"claimtype":   claimtype,
		"claimtime":   int64(0),
		"recovertime": int64(0),
	}}, false, false)
}