|
@@ -48,22 +48,23 @@ type Front struct {
|
|
|
tagCode xweb.Mapper `xweb:"/center/spider/tagcode"` //标记爬虫
|
|
|
spiderRemark xweb.Mapper `xweb:"/center/spider/spiderremark"` //标记爬虫
|
|
|
|
|
|
- spiderModel xweb.Mapper `xweb:"/center/model"` //获取补充模型
|
|
|
- runStep xweb.Mapper `xweb:"/center/run"` //方法测试
|
|
|
- spiderPass xweb.Mapper `xweb:"/center/spider/pass"` //整体测试
|
|
|
- runPinYin xweb.Mapper `xweb:"/center/runpy"` //获取拼音
|
|
|
- saveStep xweb.Mapper `xweb:"/center/save"` //爬虫保存
|
|
|
- saveJs xweb.Mapper `xweb:"/center/save/js"` //保存js
|
|
|
- loadModel xweb.Mapper `xweb:"/center/gmodel/(.*)"` //加载模型
|
|
|
- importdata xweb.Mapper `xweb:"/center/importdata"` //导入爬虫列表页面
|
|
|
- importLua xweb.Mapper `xweb:"/center/importlua"` //导入爬虫
|
|
|
- oldedit xweb.Mapper `xweb:"/center/oldedit"` //老文件编辑
|
|
|
- findName xweb.Mapper `xweb:"/center/findname"` //即时查询名称
|
|
|
- checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
|
|
|
- heart xweb.Mapper `xweb:"/center/data/heart"` //心跳监控
|
|
|
- spiderCopy xweb.Mapper `xweb:"/center/spider/copy"` //补采复制爬虫
|
|
|
- spiderSplitCopy xweb.Mapper `xweb:"/center/spider/splitcopy"` //拆分复制爬虫
|
|
|
- spiderCloseErr xweb.Mapper `xweb:"/center/spider/closeerr"` //删除错误爬虫
|
|
|
+ spiderModel xweb.Mapper `xweb:"/center/model"` //获取补充模型
|
|
|
+ runStep xweb.Mapper `xweb:"/center/run"` //方法测试
|
|
|
+ spiderPass xweb.Mapper `xweb:"/center/spider/pass"` //整体测试
|
|
|
+ runPinYin xweb.Mapper `xweb:"/center/runpy"` //获取拼音
|
|
|
+ saveStep xweb.Mapper `xweb:"/center/save"` //爬虫保存
|
|
|
+ saveJs xweb.Mapper `xweb:"/center/save/js"` //保存js
|
|
|
+ loadModel xweb.Mapper `xweb:"/center/gmodel/(.*)"` //加载模型
|
|
|
+ importdata xweb.Mapper `xweb:"/center/importdata"` //导入爬虫列表页面
|
|
|
+ importLua xweb.Mapper `xweb:"/center/importlua"` //导入爬虫
|
|
|
+ oldedit xweb.Mapper `xweb:"/center/oldedit"` //老文件编辑
|
|
|
+ findName xweb.Mapper `xweb:"/center/findname"` //即时查询名称
|
|
|
+ checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
|
|
|
+ heart xweb.Mapper `xweb:"/center/data/heart"` //心跳监控
|
|
|
+ spiderCopy xweb.Mapper `xweb:"/center/spider/copy"` //补采复制爬虫
|
|
|
+ spiderSplitCopy xweb.Mapper `xweb:"/center/spider/splitcopy"` //拆分复制爬虫
|
|
|
+ spiderCloseErr xweb.Mapper `xweb:"/center/spider/closeerr"` //删除错误爬虫
|
|
|
+ spiderUpdatePlatform xweb.Mapper `xweb:"/center/spider/updateplatform"` //修改爬虫信息
|
|
|
|
|
|
Base Base
|
|
|
OtherBase OtherBase
|
|
@@ -719,14 +720,14 @@ func (f *Front) Assign() {
|
|
|
codesarr := strings.Split(codes, ",") //被分配的爬虫
|
|
|
user, _ := u.MgoEB.FindById("user", userid, nil)
|
|
|
if user != nil && len(*user) > 0 {
|
|
|
- platform := (*user)["s_platform"]
|
|
|
+ //platform := (*user)["s_platform"]
|
|
|
name := qu.ObjToString((*user)["s_name"])
|
|
|
query := map[string]interface{}{
|
|
|
"code": map[string]interface{}{
|
|
|
"$in": codesarr,
|
|
|
},
|
|
|
}
|
|
|
- luas, _ := u.MgoEB.Find("luaconfig", query, nil, ClaimQueryFields, false, -1, -1)
|
|
|
+ luas, _ := u.MgoEB.Find("luaconfig", query, nil, nil, false, -1, -1)
|
|
|
update := [][]map[string]interface{}{}
|
|
|
save := []map[string]interface{}{}
|
|
|
for _, l := range *luas {
|
|
@@ -735,75 +736,58 @@ func (f *Front) Assign() {
|
|
|
claimtype := qu.IntAll(l["claimtype"])
|
|
|
priority := qu.IntAll(l["priority"])
|
|
|
spiderimportant, _ := l["spiderimportant"].(bool)
|
|
|
- if claimtype <= CLAIMTYPECLAIMED { //非历史爬虫分配
|
|
|
- //recovertime := CreateRecovertime(spiderimportant, priority)
|
|
|
- recovertime := time.Now().Add(24 * 5 * time.Hour).Unix()
|
|
|
- set = map[string]interface{}{
|
|
|
- "claimtype": CLAIMTYPECLAIMED,
|
|
|
- "claimtime": now,
|
|
|
- "recovertime": recovertime,
|
|
|
- "createuseremail": (*user)["s_email"],
|
|
|
- "createuser": name,
|
|
|
- "createuserid": userid,
|
|
|
- "modifyuser": name,
|
|
|
- "modifyuserid": userid,
|
|
|
- "platform": platform,
|
|
|
- }
|
|
|
- if claimtype == CLAIMTYPECLAIMED { //已认领的爬虫增加回收日志
|
|
|
- //回收日志
|
|
|
- recovelog := map[string]interface{}{
|
|
|
- "site": l["site"],
|
|
|
- "code": l["code"],
|
|
|
- "channel": l["channel"],
|
|
|
- "modifyuser": l["modifyuser"],
|
|
|
- "priority": priority,
|
|
|
- "stype": "回收",
|
|
|
- "comeintime": now,
|
|
|
- "claimtime": l["claimtime"],
|
|
|
- "recovertime": l["recovertime"],
|
|
|
- "returntime": int64(0),
|
|
|
- "important": spiderimportant,
|
|
|
- "returnreason": "",
|
|
|
- "claimrecovertype": 0,
|
|
|
- "source": "分配爬虫回收",
|
|
|
- }
|
|
|
- save = append(save, recovelog)
|
|
|
- }
|
|
|
- //被重新分配的爬虫视为重新认领(注意:lua爬虫分配到除chrome外的平台,变为历史爬虫)
|
|
|
- if platform == "golua平台" { //爬虫分配到非lua平台,不视为认领
|
|
|
- //认领日志
|
|
|
- claimlog := map[string]interface{}{
|
|
|
- "site": l["site"],
|
|
|
- "code": l["code"],
|
|
|
- "channel": l["channel"],
|
|
|
- "modifyuser": name,
|
|
|
- "priority": priority,
|
|
|
- "stype": "认领",
|
|
|
- "comeintime": now,
|
|
|
- "claimtime": now,
|
|
|
- "recovertime": recovertime,
|
|
|
- "returntime": int64(0),
|
|
|
- "important": spiderimportant,
|
|
|
- "returnreason": "",
|
|
|
- "claimrecovertype": 0,
|
|
|
- "source": "分配爬虫认领",
|
|
|
- }
|
|
|
- save = append(save, claimlog)
|
|
|
- } else if platform != "golua平台" && platform != "chrome" { //分配到其他平台改为历史爬虫
|
|
|
- set["claimtype"] = CLAIMTYPEHISTORY
|
|
|
- set["claimtime"] = 0
|
|
|
- set["recovertime"] = 0
|
|
|
- }
|
|
|
- } else { //历史爬虫
|
|
|
- set = map[string]interface{}{
|
|
|
- "createuserid": userid,
|
|
|
- "createuser": name,
|
|
|
- "createuseremail": (*user)["s_email"],
|
|
|
- "modifyuser": name,
|
|
|
- "modifyuserid": userid,
|
|
|
- "platform": (*user)["s_platform"],
|
|
|
+ //recovertime := CreateRecovertime(spiderimportant, priority)
|
|
|
+ recovertime := time.Now().Add(24 * 5 * time.Hour).Unix()
|
|
|
+ set = map[string]interface{}{
|
|
|
+ //"claimtype": CLAIMTYPECLAIMED,
|
|
|
+ //"claimtime": now,
|
|
|
+ //"recovertime": recovertime,
|
|
|
+ "createuseremail": (*user)["s_email"],
|
|
|
+ "createuser": name,
|
|
|
+ "createuserid": userid,
|
|
|
+ "modifyuser": name,
|
|
|
+ "modifyuserid": userid,
|
|
|
+ //"platform": platform,
|
|
|
+ }
|
|
|
+ if claimtype == CLAIMTYPECLAIMED { //已认领的爬虫增加回收日志
|
|
|
+ //回收日志
|
|
|
+ recovelog := map[string]interface{}{
|
|
|
+ "site": l["site"],
|
|
|
+ "code": l["code"],
|
|
|
+ "channel": l["channel"],
|
|
|
+ "modifyuser": l["modifyuser"],
|
|
|
+ "priority": priority,
|
|
|
+ "stype": "回收",
|
|
|
+ "comeintime": now,
|
|
|
+ "claimtime": l["claimtime"],
|
|
|
+ "recovertime": l["recovertime"],
|
|
|
+ "returntime": int64(0),
|
|
|
+ "important": spiderimportant,
|
|
|
+ "returnreason": "",
|
|
|
+ "claimrecovertype": 0,
|
|
|
+ "source": "分配爬虫回收",
|
|
|
}
|
|
|
+ save = append(save, recovelog)
|
|
|
+ }
|
|
|
+ //被重新分配的爬虫视为重新认领(注意:lua爬虫分配到除chrome外的平台,变为历史爬虫)
|
|
|
+ //认领日志
|
|
|
+ claimlog := map[string]interface{}{
|
|
|
+ "site": l["site"],
|
|
|
+ "code": l["code"],
|
|
|
+ "channel": l["channel"],
|
|
|
+ "modifyuser": name,
|
|
|
+ "priority": priority,
|
|
|
+ "stype": "认领",
|
|
|
+ "comeintime": now,
|
|
|
+ "claimtime": now,
|
|
|
+ "recovertime": recovertime,
|
|
|
+ "returntime": int64(0),
|
|
|
+ "important": spiderimportant,
|
|
|
+ "returnreason": "",
|
|
|
+ "claimrecovertype": 0,
|
|
|
+ "source": "分配爬虫认领",
|
|
|
}
|
|
|
+ save = append(save, claimlog)
|
|
|
//更新爬虫
|
|
|
up := []map[string]interface{}{
|
|
|
{"code": l["code"]},
|
|
@@ -829,6 +813,85 @@ func (f *Front) Assign() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+// SpiderUpdatePlatform 修改平台
|
|
|
+func (f *Front) SpiderUpdatePlatform() {
|
|
|
+ defer qu.Catch()
|
|
|
+ codes := f.GetString("codes")
|
|
|
+ platform := f.GetString("platform")
|
|
|
+ query := map[string]interface{}{
|
|
|
+ "code": map[string]interface{}{
|
|
|
+ "$in": strings.Split(codes, ","),
|
|
|
+ },
|
|
|
+ }
|
|
|
+ luas, _ := u.MgoEB.Find("luaconfig", query, nil, nil, false, -1, -1)
|
|
|
+ arr := [][]map[string]interface{}{}
|
|
|
+ save := []map[string]interface{}{}
|
|
|
+ for _, l := range *luas {
|
|
|
+ code := qu.ObjToString(l["code"])
|
|
|
+ update := []map[string]interface{}{
|
|
|
+ {"code": code},
|
|
|
+ }
|
|
|
+ set := map[string]interface{}{}
|
|
|
+ set["platform"] = platform
|
|
|
+ if platform == "python" && platform == "jschrome" {
|
|
|
+ var err error
|
|
|
+ var b bool
|
|
|
+ b = u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
|
|
|
+ pf := qu.ObjToString(l["platform"])
|
|
|
+ if pf == "golua平台" || pf == "chrome" { //爬虫所属golua平台
|
|
|
+ b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll(l["event"])) //下架
|
|
|
+ if b && err == nil {
|
|
|
+ //历史节点下架为了避免线上运行爬虫待完成时改为7000采集历史,但是又转到其他平台,导致原线上运行节点爬虫并未下线,心跳异常
|
|
|
+ if incrementevent := l["incrementevent"]; incrementevent != nil {
|
|
|
+ b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll(incrementevent))
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if b && err == nil {
|
|
|
+ //关闭任务
|
|
|
+ u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
|
|
|
+ } else {
|
|
|
+ qu.Debug("修改平台爬虫下架失败", code)
|
|
|
+ f.Write("n")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //更新状态
|
|
|
+ set["state"] = 0
|
|
|
+ //爬虫认领状态更新
|
|
|
+ set["claimtype"] = CLAIMTYPECLAIMED
|
|
|
+ set["claimtime"] = time.Now().Unix()
|
|
|
+ set["recovertime"] = int64(0)
|
|
|
+ //换平台爬虫回收,保存日志
|
|
|
+ recovelog := map[string]interface{}{
|
|
|
+ "site": l["site"],
|
|
|
+ "code": l["code"],
|
|
|
+ "channel": l["channel"],
|
|
|
+ "modifyuser": l["modifyuser"],
|
|
|
+ "priority": l["priority"],
|
|
|
+ "stype": "回收",
|
|
|
+ "comeintime": time.Now().Unix(),
|
|
|
+ "claimtime": l["claimtime"],
|
|
|
+ "recovertime": l["recovertime"],
|
|
|
+ "returntime": int64(0),
|
|
|
+ "important": l["spiderimportant"],
|
|
|
+ "returnreason": "转平台",
|
|
|
+ "claimrecovertype": 0,
|
|
|
+ "source": "爬虫转平台回收",
|
|
|
+ }
|
|
|
+ save = append(save, recovelog)
|
|
|
+ update = append(update, map[string]interface{}{"$set": set})
|
|
|
+ arr = append(arr, update)
|
|
|
+ }
|
|
|
+ u.MgoEB.UpdateBulk("luaconfig", arr...)
|
|
|
+ if len(save) > 0 {
|
|
|
+ u.MgoEB.SaveBulk("lua_logs_claim", save...)
|
|
|
+ }
|
|
|
+ arr = [][]map[string]interface{}{}
|
|
|
+ save = []map[string]interface{}{}
|
|
|
+ f.Write("y")
|
|
|
+}
|
|
|
+
|
|
|
// 审核日志导出
|
|
|
func (f *Front) AuditExport() {
|
|
|
auth := qu.IntAll(f.GetSession("auth"))
|
|
@@ -1015,12 +1078,13 @@ func (f *Front) UpdateESP() {
|
|
|
} else if w == "platform" {
|
|
|
//常规字段更新
|
|
|
set["platform"] = val
|
|
|
- set["comeintime"] = time.Now().Unix()
|
|
|
- if val != "golua平台" && val != "chrome" {
|
|
|
+ //set["comeintime"] = time.Now().Unix()
|
|
|
+ if val == "python" && val == "jschrome" {
|
|
|
b := u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
|
|
|
qu.Debug("Del Heart:", b)
|
|
|
var err error
|
|
|
- if qu.ObjToString((*one)["platform"]) == "golua平台" {
|
|
|
+ pf := qu.ObjToString((*one)["platform"])
|
|
|
+ if pf == "golua平台" || pf == "chrome" { //爬虫所属golua平台
|
|
|
b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*one)["event"])) //下架
|
|
|
if b && err == nil {
|
|
|
//历史节点下架为了避免线上运行爬虫待完成时改为7000采集历史,但是又转到其他平台,导致原线上运行节点爬虫并未下线,心跳异常
|
|
@@ -1032,28 +1096,16 @@ func (f *Front) UpdateESP() {
|
|
|
if b && err == nil {
|
|
|
//关闭任务
|
|
|
u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
|
|
|
- if b && err == nil {
|
|
|
- set["state"] = 0 //更新状态
|
|
|
- set["luauser"] = map[string]interface{}{
|
|
|
- "createuser": (*one)["createuser"],
|
|
|
- "createuserid": (*one)["createuserid"],
|
|
|
- "createuseremail": (*one)["createuseremail"],
|
|
|
- "modifyuser": (*one)["modifyuser"],
|
|
|
- "modifyuserid": (*one)["modifyuserid"],
|
|
|
- }
|
|
|
- } else {
|
|
|
- qu.Debug("历史节点下架失败")
|
|
|
- f.Write("n")
|
|
|
- return
|
|
|
- }
|
|
|
} else {
|
|
|
- qu.Debug("增量节点下架失败")
|
|
|
- f.Write("n")
|
|
|
- return
|
|
|
+ qu.Debug("修改平台爬虫下架失败", code)
|
|
|
}
|
|
|
}
|
|
|
+ //更新状态
|
|
|
+ set["state"] = 0
|
|
|
//爬虫认领状态更新
|
|
|
- UpdateLuaClaimtype(code)
|
|
|
+ set["claimtype"] = CLAIMTYPECLAIMED
|
|
|
+ set["claimtime"] = time.Now().Unix()
|
|
|
+ set["recovertime"] = int64(0)
|
|
|
//换平台爬虫回收,保存日志
|
|
|
recovelog := map[string]interface{}{
|
|
|
"site": (*one)["site"],
|