|
|
@@ -5,20 +5,16 @@ import (
|
|
|
"errors"
|
|
|
"fmt"
|
|
|
"log"
|
|
|
+ "mongodb"
|
|
|
"sort"
|
|
|
"spider"
|
|
|
|
|
|
- // "math/rand"
|
|
|
mu "mfw/util"
|
|
|
qu "qfw/util"
|
|
|
- mgdb "qfw/util/mongodb"
|
|
|
- mgu "qfw/util/mongodbutil"
|
|
|
util "spiderutil"
|
|
|
"strings"
|
|
|
"time"
|
|
|
u "util"
|
|
|
-
|
|
|
- "gopkg.in/mgo.v2/bson"
|
|
|
)
|
|
|
|
|
|
type Base struct {
|
|
|
@@ -98,59 +94,58 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
|
|
|
if f.Method() == "GET" {
|
|
|
code := util.Se.Decode4Hex(code)
|
|
|
f.T["actiontext"] = "编辑"
|
|
|
- lua := *mgdb.FindOne("luaconfig", bson.M{"code": code})
|
|
|
+ lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- if lua["createuserid"].(string) == f.GetSession("userid").(string) || auth >= 1 {
|
|
|
- if len(lua) > 0 {
|
|
|
- luacopy := map[string]interface{}{}
|
|
|
+ if qu.ObjToString((*lua)["createuserid"]) == f.GetSession("userid").(string) || auth >= 1 {
|
|
|
+ if len(*lua) > 0 {
|
|
|
if copy != "" {
|
|
|
- luacopy = *mgdb.FindOne("luaconfig", bson.M{"code": copy})
|
|
|
- if len(luacopy) > 0 {
|
|
|
- lua["model"] = luacopy["model"]
|
|
|
- common_copy := luacopy["param_common"].([]interface{})
|
|
|
- common := lua["param_common"].([]interface{})
|
|
|
+ luacopy, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": copy})
|
|
|
+ if len(*luacopy) > 0 {
|
|
|
+ (*lua)["model"] = (*luacopy)["model"]
|
|
|
+ common_copy := (*luacopy)["param_common"].([]interface{})
|
|
|
+ common := (*lua)["param_common"].([]interface{})
|
|
|
common_copy[0] = common[0]
|
|
|
common_copy[1] = common[1]
|
|
|
common_copy[2] = common[2]
|
|
|
common_copy[11] = common[11]
|
|
|
- lua["param_common"] = luacopy["param_common"]
|
|
|
- lua["param_time"] = luacopy["param_time"]
|
|
|
- lua["param_list"] = luacopy["param_list"]
|
|
|
- lua["param_content"] = luacopy["param_content"]
|
|
|
- lua["str_list"] = luacopy["str_list"]
|
|
|
- lua["str_time"] = luacopy["str_time"]
|
|
|
- lua["str_content"] = luacopy["str_content"]
|
|
|
- lua["Thref"] = luacopy["Thref"]
|
|
|
- lua["Tpublishtime"] = luacopy["Tpublishtime"]
|
|
|
- lua["Ttitle"] = luacopy["Ttitle"]
|
|
|
- lua["Tdate"] = luacopy["Tdate"]
|
|
|
- lua["type_content"] = luacopy["type_content"]
|
|
|
- lua["type_list"] = luacopy["type_list"]
|
|
|
- lua["type_time"] = luacopy["type_time"]
|
|
|
+ (*lua)["param_common"] = (*luacopy)["param_common"]
|
|
|
+ (*lua)["param_time"] = (*luacopy)["param_time"]
|
|
|
+ (*lua)["param_list"] = (*luacopy)["param_list"]
|
|
|
+ (*lua)["param_content"] = (*luacopy)["param_content"]
|
|
|
+ (*lua)["str_list"] = (*luacopy)["str_list"]
|
|
|
+ (*lua)["str_time"] = (*luacopy)["str_time"]
|
|
|
+ (*lua)["str_content"] = (*luacopy)["str_content"]
|
|
|
+ (*lua)["Thref"] = (*luacopy)["Thref"]
|
|
|
+ (*lua)["Tpublishtime"] = (*luacopy)["Tpublishtime"]
|
|
|
+ (*lua)["Ttitle"] = (*luacopy)["Ttitle"]
|
|
|
+ (*lua)["Tdate"] = (*luacopy)["Tdate"]
|
|
|
+ (*lua)["type_content"] = (*luacopy)["type_content"]
|
|
|
+ (*lua)["type_list"] = (*luacopy)["type_list"]
|
|
|
+ (*lua)["type_time"] = (*luacopy)["type_time"]
|
|
|
}
|
|
|
}
|
|
|
- if lua["listcheck"] != nil {
|
|
|
- listcheck := lua["listcheck"].(string)
|
|
|
+ if (*lua)["listcheck"] != nil {
|
|
|
+ listcheck := (*lua)["listcheck"].(string)
|
|
|
listcheck = strings.Replace(listcheck, "\\n", "\n", -1)
|
|
|
listcheck = strings.Replace(listcheck, "\\", "", -1)
|
|
|
- lua["listcheck"] = listcheck
|
|
|
+ (*lua)["listcheck"] = listcheck
|
|
|
}
|
|
|
- if lua["contentcheck"] != nil {
|
|
|
- contentcheck := lua["contentcheck"].(string)
|
|
|
+ if (*lua)["contentcheck"] != nil {
|
|
|
+ contentcheck := (*lua)["contentcheck"].(string)
|
|
|
contentcheck = strings.Replace(contentcheck, "\\n", "\n", -1)
|
|
|
contentcheck = strings.Replace(contentcheck, "\\", "", -1)
|
|
|
- lua["contentcheck"] = contentcheck
|
|
|
+ (*lua)["contentcheck"] = contentcheck
|
|
|
}
|
|
|
- js, _ := json.MarshalIndent(lua["model"], "", " ")
|
|
|
- lua["js"] = string(js)
|
|
|
+ js, _ := json.MarshalIndent((*lua)["model"], "", " ")
|
|
|
+ (*lua)["js"] = string(js)
|
|
|
f.T["lua"] = lua
|
|
|
f.T["taskId"] = taskId
|
|
|
f.T["restate"] = restate
|
|
|
- f.T["isflow"] = lua["isflow"]
|
|
|
- f.T["spidertype"] = lua["spidertype"]
|
|
|
- f.T["spidermovevent"] = lua["spidermovevent"]
|
|
|
- f.T["spiderhistorymaxpage"] = lua["spiderhistorymaxpage"]
|
|
|
- if lua["oldlua"] != nil {
|
|
|
+ f.T["isflow"] = (*lua)["isflow"]
|
|
|
+ f.T["spidertype"] = (*lua)["spidertype"]
|
|
|
+ f.T["spidermovevent"] = (*lua)["spidermovevent"]
|
|
|
+ f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
|
|
|
+ if (*lua)["oldlua"] != nil {
|
|
|
return f.Render("oldedit.html", &f.T)
|
|
|
}
|
|
|
return f.Render("spideredit.html", &f.T)
|
|
|
@@ -169,28 +164,28 @@ func (f *Front) ViewSpider(id string) error {
|
|
|
if f.Method() == "GET" {
|
|
|
code := util.Se.Decode4Hex(id)
|
|
|
f.T["actiontext"] = "编辑"
|
|
|
- lua := *mgdb.FindOne("luaconfig", bson.M{"code": code})
|
|
|
- if len(lua) > 0 {
|
|
|
- if lua["listcheck"] != nil {
|
|
|
- listcheck := lua["listcheck"].(string)
|
|
|
+ lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
+ if len(*lua) > 0 {
|
|
|
+ if (*lua)["listcheck"] != nil {
|
|
|
+ listcheck := (*lua)["listcheck"].(string)
|
|
|
listcheck = strings.Replace(listcheck, "\\n", "\n", -1)
|
|
|
listcheck = strings.Replace(listcheck, "\\", "", -1)
|
|
|
- lua["listcheck"] = listcheck
|
|
|
+ (*lua)["listcheck"] = listcheck
|
|
|
}
|
|
|
- if lua["contentcheck"] != nil {
|
|
|
- contentcheck := lua["contentcheck"].(string)
|
|
|
+ if (*lua)["contentcheck"] != nil {
|
|
|
+ contentcheck := (*lua)["contentcheck"].(string)
|
|
|
contentcheck = strings.Replace(contentcheck, "\\n", "\n", -1)
|
|
|
contentcheck = strings.Replace(contentcheck, "\\", "", -1)
|
|
|
- lua["contentcheck"] = contentcheck
|
|
|
+ (*lua)["contentcheck"] = contentcheck
|
|
|
}
|
|
|
- js, _ := json.MarshalIndent(lua["model"], "", " ")
|
|
|
- lua["js"] = string(js)
|
|
|
+ js, _ := json.MarshalIndent((*lua)["model"], "", " ")
|
|
|
+ (*lua)["js"] = string(js)
|
|
|
f.T["lua"] = lua
|
|
|
- f.T["isflow"] = lua["isflow"]
|
|
|
- f.T["spidertype"] = lua["spidertype"]
|
|
|
- f.T["spidermovevent"] = lua["spidermovevent"]
|
|
|
- f.T["spiderhistorymaxpage"] = lua["spiderhistorymaxpage"]
|
|
|
- if lua["oldlua"] != nil {
|
|
|
+ f.T["isflow"] = (*lua)["isflow"]
|
|
|
+ f.T["spidertype"] = (*lua)["spidertype"]
|
|
|
+ f.T["spidermovevent"] = (*lua)["spidermovevent"]
|
|
|
+ f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
|
|
|
+ if (*lua)["oldlua"] != nil {
|
|
|
return f.Render("oldedit.html", &f.T)
|
|
|
}
|
|
|
return f.Render("spiderview.html", &f.T)
|
|
|
@@ -208,9 +203,9 @@ func (f *Front) ViewSpider(id string) error {
|
|
|
|
|
|
func (f *Front) LoadModel(id string) error {
|
|
|
if f.Method() == "GET" {
|
|
|
- lua := *mgdb.Find("luaconfig", bson.M{"code": id}, nil, bson.M{"model": 1}, true, -1, -1)
|
|
|
- if len(lua) > 0 {
|
|
|
- f.ServeJson(lua[0])
|
|
|
+ lua, _ := u.MgoE.Find("luaconfig", map[string]interface{}{"code": id}, nil, map[string]interface{}{"model": 1}, true, -1, -1)
|
|
|
+ if len(*lua) > 0 {
|
|
|
+ f.ServeJson((*lua)[0])
|
|
|
}
|
|
|
}
|
|
|
return f.Redirect("/center")
|
|
|
@@ -222,17 +217,17 @@ func (f *Front) SaveStep() {
|
|
|
rep := map[string]interface{}{}
|
|
|
if f.GetString("oldlua") != "" {
|
|
|
id := f.GetString("code")
|
|
|
- one := *mgdb.FindOne("luaconfig", bson.M{"code": id})
|
|
|
- id = one["code"].(string)
|
|
|
+ one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
|
|
|
+ id = (*one)["code"].(string)
|
|
|
script := f.GetStringComm("script")
|
|
|
if strings.Index(script, id) == -1 {
|
|
|
rep["msg"] = "code/名称都不能更改"
|
|
|
f.ServeJson(rep)
|
|
|
return
|
|
|
} else {
|
|
|
- upset := bson.M{"luacontent": script}
|
|
|
+ upset := map[string]interface{}{"luacontent": script}
|
|
|
upset["modifytime"] = time.Now().Unix()
|
|
|
- b := mgdb.Update("luaconfig", bson.M{"code": id}, bson.M{"$set": upset}, true, false)
|
|
|
+ b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
|
|
|
if b {
|
|
|
rep["msg"] = "保存成功"
|
|
|
rep["code"] = util.Se.Encode2Hex(id)
|
|
|
@@ -243,20 +238,20 @@ func (f *Front) SaveStep() {
|
|
|
} else {
|
|
|
if f.Base.SpiderName != "" && f.Base.SpiderCode != "" {
|
|
|
code := f.Base.SpiderCode
|
|
|
- one := *mgdb.FindOne("luaconfig", bson.M{"code": f.Base.SpiderCode})
|
|
|
- state := qu.IntAllDef(one["state"], 0)
|
|
|
- restate := qu.IntAll(one["restate"])
|
|
|
+ one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": f.Base.SpiderCode})
|
|
|
+ state := qu.IntAllDef((*one)["state"], 0)
|
|
|
+ restate := qu.IntAll((*one)["restate"])
|
|
|
comeintime := time.Now().Unix()
|
|
|
- if len(one) > 0 {
|
|
|
- comeintime = qu.Int64All(one["comeintime"])
|
|
|
- ouserid := one["createuserid"].(string)
|
|
|
+ if len((*one)) > 0 {
|
|
|
+ comeintime = qu.Int64All((*one)["comeintime"])
|
|
|
+ ouserid := (*one)["createuserid"].(string)
|
|
|
if ouserid != userid && auth == role_dev {
|
|
|
f.Write("权限不够，不能修改他人脚本")
|
|
|
return
|
|
|
} else {
|
|
|
- code = one["code"].(string)
|
|
|
- f.Base.SpiderCode = one["code"].(string)
|
|
|
- f.Base.SpiderName = (one["param_common"].([]interface{}))[1].(string)
|
|
|
+ code = (*one)["code"].(string)
|
|
|
+ f.Base.SpiderCode = (*one)["code"].(string)
|
|
|
+ f.Base.SpiderName = ((*one)["param_common"].([]interface{}))[1].(string)
|
|
|
}
|
|
|
} else {
|
|
|
if auth != role_admin {
|
|
|
@@ -333,12 +328,12 @@ func (f *Front) SaveStep() {
|
|
|
model["model"] = s_model
|
|
|
param["code"] = f.Base.SpiderCode
|
|
|
param["model"] = model
|
|
|
- if len(one) > 0 {
|
|
|
- param["createuser"] = one["createuser"]
|
|
|
- param["createuserid"] = one["createuserid"]
|
|
|
- param["code"] = one["code"]
|
|
|
+ if len((*one)) > 0 {
|
|
|
+ param["createuser"] = (*one)["createuser"]
|
|
|
+ param["createuserid"] = (*one)["createuserid"]
|
|
|
+ param["code"] = (*one)["code"]
|
|
|
//开发员关联任务修改爬虫状态
|
|
|
- state = qu.IntAll(one["state"])
|
|
|
+ state = qu.IntAll((*one)["state"])
|
|
|
if auth == role_dev && state >= Sp_state_3 && restate != 1 { //开发员修改,已经审核通过(不包含已上架),状态重置为待完成(restate!=1判断，重采修改保存爬虫时不修改爬虫状态)
|
|
|
param["state"] = 0
|
|
|
} else {
|
|
|
@@ -352,7 +347,7 @@ func (f *Front) SaveStep() {
|
|
|
param["next"] = f.GetSession("email")
|
|
|
param["state"] = 0
|
|
|
}
|
|
|
- if qu.ObjToString(one["modifyuser"]) == "" {
|
|
|
+ if qu.ObjToString((*one)["modifyuser"]) == "" {
|
|
|
param["modifyuser"] = param["createuser"]
|
|
|
param["modifyuserid"] = param["createuserid"]
|
|
|
}
|
|
|
@@ -367,7 +362,7 @@ func (f *Front) SaveStep() {
|
|
|
param["spiderhistorymaxpage"] = f.OtherBase.SpiderHistoryMaxPage
|
|
|
if f.OtherBase.SpiderType == "history" { //爬虫类型是history的放到7000节点,并记录历史节点
|
|
|
param["event"] = 7000
|
|
|
- if event := qu.IntAll(one["event"]); event != 7000 {
|
|
|
+ if event := qu.IntAll((*one)["event"]); event != 7000 {
|
|
|
param["historyevent"] = event
|
|
|
}
|
|
|
}
|
|
|
@@ -383,7 +378,7 @@ func (f *Front) SaveStep() {
|
|
|
if !matchLua {
|
|
|
issave := spider.SaveSpider(code, param) //保存脚本
|
|
|
if issave {
|
|
|
- for k, v := range one {
|
|
|
+ for k, v := range *one {
|
|
|
if k != "_id" && param[k] == nil {
|
|
|
param[k] = v
|
|
|
}
|
|
|
@@ -411,36 +406,36 @@ func LuaTextCheck(list, detail string) bool {
|
|
|
func (f *Front) Assort() {
|
|
|
state, _ := f.GetInteger("state")
|
|
|
code := f.GetString("code")
|
|
|
- sql := bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ sql := map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"state": state,
|
|
|
"modifytime": time.Now().Unix(),
|
|
|
},
|
|
|
}
|
|
|
- queryT := bson.M{
|
|
|
+ queryT := map[string]interface{}{
|
|
|
"code": code,
|
|
|
}
|
|
|
//下架爬虫
|
|
|
- lua := *mgdb.FindOne("luaconfig", queryT)
|
|
|
- upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll(lua["event"]))
|
|
|
+ lua, _ := u.MgoE.FindOne("luaconfig", queryT)
|
|
|
+ upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
|
|
|
qu.Debug("下架爬虫：", code, upresult, err)
|
|
|
if upresult && err == nil {
|
|
|
//更新爬虫
|
|
|
- mgdb.Update("luaconfig", queryT, sql, false, false)
|
|
|
+ u.MgoE.Update("luaconfig", queryT, sql, false, false)
|
|
|
//关闭任务
|
|
|
- query := bson.M{
|
|
|
+ query := map[string]interface{}{
|
|
|
"s_code": code,
|
|
|
}
|
|
|
- sql = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ sql = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 6,
|
|
|
},
|
|
|
}
|
|
|
- mgdb.Update("task", query, sql, false, true)
|
|
|
+ u.MgoE.Update("task", query, sql, false, true)
|
|
|
//删除心跳
|
|
|
- mgu.Update("spider_heart", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
|
|
|
+ DelSpiderHeart(code)
|
|
|
}
|
|
|
- f.ServeJson(bson.M{"upresult": upresult})
|
|
|
+ f.ServeJson(map[string]interface{}{"upresult": upresult})
|
|
|
}
|
|
|
|
|
|
//方法测试
|
|
|
@@ -570,6 +565,7 @@ func (f *Front) GetJson() {
|
|
|
data, _ := f.GetSession("dataInfo").(map[string]interface{})
|
|
|
descript := f.GetSession("task_descript")
|
|
|
remark := f.GetSession("task_remark")
|
|
|
+ rateremark := f.GetSession("task_rateremark")
|
|
|
reason := f.GetSession("reason")
|
|
|
username := f.GetSession("username").(string)
|
|
|
msg := f.GetSession(username + "_msg")
|
|
|
@@ -584,6 +580,7 @@ func (f *Front) GetJson() {
|
|
|
f.T["data"] = data
|
|
|
f.T["descript"] = descript
|
|
|
f.T["remark"] = remark
|
|
|
+ f.T["rateremark"] = rateremark
|
|
|
f.T["reason"] = reason
|
|
|
f.T["msg"] = msg
|
|
|
|
|
|
@@ -591,6 +588,7 @@ func (f *Front) GetJson() {
|
|
|
f.DelSession("dataInfo")
|
|
|
f.DelSession("task_descript")
|
|
|
f.DelSession("task_remark")
|
|
|
+ f.DelSession("task_rateremark")
|
|
|
f.DelSession("reason")
|
|
|
|
|
|
f.Render("jsonInfo.html", &f.T)
|
|
|
@@ -598,6 +596,7 @@ func (f *Front) GetJson() {
|
|
|
|
|
|
//整体测试
|
|
|
func (f *Front) SpiderPass() {
|
|
|
+ qu.Debug("111111111111")
|
|
|
defer mu.Catch()
|
|
|
list := []map[string]interface{}{}
|
|
|
data := map[string]interface{}{}
|
|
|
@@ -605,22 +604,25 @@ func (f *Front) SpiderPass() {
|
|
|
code := f.GetString("code")
|
|
|
downloadnode := f.GetString("node")
|
|
|
//根据code查询待确认任务
|
|
|
- query := bson.M{
|
|
|
+ query := map[string]interface{}{
|
|
|
"s_code": code,
|
|
|
"i_state": 3,
|
|
|
}
|
|
|
- task := *mgdb.FindOne("task", query)
|
|
|
+ task, _ := u.MgoE.FindOne("task", query)
|
|
|
descript := "null"
|
|
|
remark := "null"
|
|
|
remarktmp := []string{}
|
|
|
- if len(task) > 0 {
|
|
|
- descript = task["s_descript"].(string)
|
|
|
- if mrecord, ok := task["a_mrecord"].([]interface{}); ok {
|
|
|
+ rateremarktmp := []string{}
|
|
|
+ if len(*task) > 0 {
|
|
|
+ descript = (*task)["s_descript"].(string)
|
|
|
+ if mrecord, ok := (*task)["a_mrecord"].([]interface{}); ok {
|
|
|
for _, m := range mrecord {
|
|
|
remarkInfo := m.(map[string]interface{})
|
|
|
- r := remarkInfo["s_mrecord_remark"].(string)
|
|
|
- if r != "" {
|
|
|
- remarktmp = append(remarktmp, r+";")
|
|
|
+ if remark := qu.ObjToString(remarkInfo["s_mrecord_remark"]); remark != "" {
|
|
|
+ remarktmp = append(remarktmp, remark+";")
|
|
|
+ }
|
|
|
+ if rateremark := qu.ObjToString(remarkInfo["s_mrecord_rateremark"]); rateremark != "" {
|
|
|
+ rateremarktmp = append(rateremarktmp, rateremark+";")
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -631,14 +633,15 @@ func (f *Front) SpiderPass() {
|
|
|
}
|
|
|
|
|
|
f.SetSession("task_remark", remark)
|
|
|
+ f.SetSession("task_rateremark", rateremarktmp)
|
|
|
f.SetSession("task_descript", descript)
|
|
|
//基本信息、方法一、方法二、方法三、总请求次数、go方法一、go方法二、go方法三、列表页条数
|
|
|
steps := []interface{}{false, false, false, false, 0, 0, 0, 0, 0}
|
|
|
- one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
|
|
|
- reason, _ := one["reason"].(string)
|
|
|
+ one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
+ reason, _ := (*one)["reason"].(string)
|
|
|
f.SetSession("reason", reason)
|
|
|
- if len(one) > 0 && one["oldlua"] == nil {
|
|
|
- common := one["param_common"].([]interface{})
|
|
|
+ if len(*one) > 0 && (*one)["oldlua"] == nil {
|
|
|
+ common := (*one)["param_common"].([]interface{})
|
|
|
if len(common) < 13 {
|
|
|
f.ServeJson(steps)
|
|
|
return
|
|
|
@@ -649,13 +652,13 @@ func (f *Front) SpiderPass() {
|
|
|
steps[0] = true
|
|
|
}
|
|
|
script, liststr, contentstr := "", "", ""
|
|
|
- if one["oldlua"] == nil {
|
|
|
+ if (*one)["oldlua"] == nil {
|
|
|
script, liststr, contentstr = spider.GetScript(code)
|
|
|
} else {
|
|
|
- script = one["luacontent"].(string)
|
|
|
+ script = (*one)["luacontent"].(string)
|
|
|
}
|
|
|
if liststr != "" && contentstr != "" {
|
|
|
- msg1 = u.SpiderPassCheckLua(liststr, contentstr, one)
|
|
|
+ msg1 = u.SpiderPassCheckLua(liststr, contentstr, (*one))
|
|
|
}
|
|
|
s := spider.CreateSpider(downloadnode, script)
|
|
|
s.SpiderMaxPage = 1
|
|
|
@@ -714,7 +717,7 @@ func (f *Front) SpiderPass() {
|
|
|
//校验
|
|
|
msg := u.SpiderPassCheckListAndDetail(list, data)
|
|
|
if msg1 != "" {
|
|
|
- msg = msg1 + "," + msg
|
|
|
+ msg = msg1 + "；" + msg
|
|
|
}
|
|
|
username := f.GetSession("username").(string)
|
|
|
f.SetSession(username+"_msg", msg)
|
|
|
@@ -724,18 +727,19 @@ func (f *Front) SpiderPass() {
|
|
|
func (f *Front) DownSpider(id string) {
|
|
|
//auth := qu.IntAll(f.GetSession("auth"))
|
|
|
//if auth > role_dev {
|
|
|
- one := *mgdb.FindOne("luaconfig", bson.M{"code": id})
|
|
|
+ one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
|
|
|
+ createuserid := qu.ObjToString((*one)["createuserid"])
|
|
|
script := ""
|
|
|
filename := id + ".lua"
|
|
|
- if len(one) > 0 {
|
|
|
- if one["oldlua"] != nil {
|
|
|
- if one["luacontent"] != nil {
|
|
|
- script = one["luacontent"].(string)
|
|
|
+ if len(*one) > 0 {
|
|
|
+ if (*one)["oldlua"] != nil {
|
|
|
+ if (*one)["luacontent"] != nil {
|
|
|
+ script = (*one)["luacontent"].(string)
|
|
|
}
|
|
|
} else {
|
|
|
- user := *mgdb.FindOne("user", bson.M{"_id": bson.ObjectIdHex(one["createuserid"].(string))})
|
|
|
- name := one["createuser"]
|
|
|
- email := user["s_email"]
|
|
|
+ user, _ := u.MgoE.FindById("user", createuserid, nil)
|
|
|
+ name := (*one)["createuser"]
|
|
|
+ email := (*user)["s_email"]
|
|
|
upload := time.Now().Format("2006-01-02 15:04:05")
|
|
|
script, _, _ = spider.GetScript(id, name, email, upload)
|
|
|
}
|
|
|
@@ -801,8 +805,8 @@ func (f *Front) UpState() error {
|
|
|
SaveRemark(taskid, "", username)
|
|
|
}
|
|
|
} else if b && state == Sp_state_6 { //下架
|
|
|
- //下架成功删除download数据
|
|
|
- flag := delDownloadData(code)
|
|
|
+ //下架成功删除心跳数据
|
|
|
+ flag := DelSpiderHeart(code)
|
|
|
log.Println(code, "---下架删除download数据：", flag)
|
|
|
} else if b && state == Sp_state_7 { //反馈
|
|
|
taskid = checkTask(codeArr, 7)
|
|
|
@@ -824,18 +828,23 @@ func (f *Front) UpState() error {
|
|
|
return nil
|
|
|
}
|
|
|
|
|
|
-//下架删除download数据
|
|
|
-func delDownloadData(code string) bool {
|
|
|
- return mgu.Del("download", "spider", "spider", `{"code":"`+code+`"}`)
|
|
|
+//下架删除心跳
|
|
|
+func DelSpiderHeart(code string) bool {
|
|
|
+ return u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
|
|
|
}
|
|
|
|
|
|
+//下架删除download数据
|
|
|
+//func delDownloadData(code string) bool {
|
|
|
+// return mgu.Del("download", "spider", "spider", `{"code":"`+code+`"}`)
|
|
|
+//}
|
|
|
+
|
|
|
//批量作废删除download数据
|
|
|
-func disableDelDownloadData(code []string) {
|
|
|
- for _, v := range code {
|
|
|
- flag := delDownloadData(v)
|
|
|
- log.Println(code, "---批量删除download数据：", flag)
|
|
|
- }
|
|
|
-}
|
|
|
+//func disableDelDownloadData(code []string) {
|
|
|
+// for _, v := range code {
|
|
|
+// flag := delDownloadData(v)
|
|
|
+// log.Println(code, "---批量删除download数据：", flag)
|
|
|
+// }
|
|
|
+//}
|
|
|
|
|
|
//爬虫核对
|
|
|
func (f *Front) Checktime() {
|
|
|
@@ -844,7 +853,9 @@ func (f *Front) Checktime() {
|
|
|
if auth != role_admin {
|
|
|
f.ServeJson(false)
|
|
|
} else {
|
|
|
- b := mgdb.Update("luaconfig", `{"code":"`+code+`"}`, `{"$set":{"l_checktime":`+fmt.Sprint(time.Now().Unix())+`}}`, true, false)
|
|
|
+ b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
|
|
|
+ "l_checktime": time.Now().Unix(),
|
|
|
+ }}, true, false)
|
|
|
f.ServeJson(b)
|
|
|
}
|
|
|
}
|
|
|
@@ -900,17 +911,16 @@ func (f *Front) BatchShelves() {
|
|
|
} else { //批量下架
|
|
|
for _, code := range codes {
|
|
|
b, err = UpStateAndUpSpider(code, "", "", "", Sp_state_6)
|
|
|
- if err != nil {
|
|
|
+ if !b || err != nil {
|
|
|
errCode = append(errCode, code)
|
|
|
}
|
|
|
//下架删除download数据
|
|
|
- if b {
|
|
|
- flag := delDownloadData(code)
|
|
|
- log.Println(code, "---删除download数据：", flag)
|
|
|
- }
|
|
|
+ //if b {
|
|
|
+ // flag := delDownloadData(code)
|
|
|
+ // log.Println(code, "---删除download数据：", flag)
|
|
|
+ //}
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
} else {
|
|
|
errCode = append(errCode, "没有权限")
|
|
|
}
|
|
|
@@ -921,17 +931,17 @@ func (f *Front) BatchShelves() {
|
|
|
func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, error) {
|
|
|
upresult := false
|
|
|
var err error
|
|
|
- one := map[string]interface{}{}
|
|
|
+ one := &map[string]interface{}{}
|
|
|
if code != "" {
|
|
|
- one = *mgdb.FindOne("luaconfig", bson.M{"code": code})
|
|
|
+ one, _ = u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
} else {
|
|
|
- one = *mgdb.FindOne("luaconfig", bson.M{"_id": bson.ObjectIdHex(id)})
|
|
|
- code = one["code"].(string)
|
|
|
+ one, _ = u.MgoE.FindById("luaconfig", id, nil)
|
|
|
+ code = qu.ObjToString((*one)["code"])
|
|
|
}
|
|
|
- if len(one) > 0 {
|
|
|
+ if len(*one) > 0 {
|
|
|
var event int
|
|
|
- if one["event"] != nil {
|
|
|
- event = qu.IntAll(one["event"])
|
|
|
+ if (*one)["event"] != nil {
|
|
|
+ event = qu.IntAll((*one)["event"])
|
|
|
} else {
|
|
|
for k, _ := range util.Config.Uploadevents { //?
|
|
|
event = qu.IntAll(k)
|
|
|
@@ -968,18 +978,19 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
|
|
|
}
|
|
|
|
|
|
if upresult && err == nil {
|
|
|
- upset := bson.M{"state": state} //修改状态
|
|
|
- if one["oldlua"] != nil { //老脚本上传
|
|
|
- upresult = mgdb.Update("luaconfig", bson.M{"code": code}, bson.M{"$set": upset}, true, false)
|
|
|
+ upset := map[string]interface{}{"state": state} //修改状态
|
|
|
+ if (*one)["oldlua"] != nil { //老脚本上传
|
|
|
+ upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
|
|
|
} else {
|
|
|
if state == Sp_state_1 { //提交审核
|
|
|
upset["l_complete"] = time.Now().Unix()
|
|
|
upset["report"] = ""
|
|
|
- } else if state == Sp_state_3 { //发布
|
|
|
- if one["event"] == nil {
|
|
|
+ } else if state == Sp_state_3 { //审核通过
|
|
|
+ if (*one)["event"] == nil {
|
|
|
upset["event"] = event
|
|
|
upset["modifytime"] = time.Now().Unix()
|
|
|
}
|
|
|
+ upset["frequencyerrtimes"] = 0 //爬虫审核通过，重置采集频率异常次数
|
|
|
upset["l_uploadtime"] = time.Now().Unix()
|
|
|
} else if state == Sp_state_2 { //打回原因
|
|
|
upset["reason"] = reason
|
|
|
@@ -992,37 +1003,23 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
|
|
|
upset["disablereason"] = reason
|
|
|
upset["modifytime"] = time.Now().Unix()
|
|
|
}
|
|
|
- qu.Debug("set:", upset)
|
|
|
upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
|
|
|
- //upresult = mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
|
|
|
- // if state == Sp_state_1 { //提交审核，验证是否提交成功
|
|
|
- // for i := 1; i <= 5; i++ { //解决提交不上，重试5次
|
|
|
- // lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
- // tmpState := qu.IntAll(lua["state"])
|
|
|
- // if state == tmpState {
|
|
|
- // break
|
|
|
- // } else {
|
|
|
- // upresult = mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
|
|
|
- // upresult = false
|
|
|
- // }
|
|
|
- // }
|
|
|
- // }
|
|
|
qu.Debug("提交日志：", code, upset, upresult)
|
|
|
if upresult && (state == Sp_state_2 || state == Sp_state_3) { //打回、审核记录日志
|
|
|
types := "打回"
|
|
|
if state == Sp_state_3 {
|
|
|
types = "审核"
|
|
|
}
|
|
|
- obj := bson.M{
|
|
|
+ obj := map[string]interface{}{
|
|
|
"code": code,
|
|
|
"auditor": username,
|
|
|
"types": types,
|
|
|
"comeintime": time.Now().Unix(),
|
|
|
"reason": reason,
|
|
|
- "spideruser": one["createuser"],
|
|
|
- "modifytime": one["modifytime"],
|
|
|
+ "spideruser": (*one)["createuser"],
|
|
|
+ "modifytime": (*one)["modifytime"],
|
|
|
}
|
|
|
- mgdb.Save("lua_logs_auditor", obj)
|
|
|
+ u.MgoE.Save("lua_logs_auditor", obj)
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -1037,12 +1034,9 @@ func SaveRemark(taskid []string, reason, username string) {
|
|
|
reason = "审核通过"
|
|
|
}
|
|
|
for _, id := range taskid {
|
|
|
- query := bson.M{
|
|
|
- "_id": bson.ObjectIdHex(string(id)),
|
|
|
- }
|
|
|
- task := *mgdb.FindOne("task", query)
|
|
|
- if task != nil {
|
|
|
- checkData := task["a_check"]
|
|
|
+ task, _ := u.MgoE.FindById("task", id, nil)
|
|
|
+ if task != nil && len(*task) > 0 {
|
|
|
+ checkData := (*task)["a_check"]
|
|
|
var checkArr []map[string]interface{}
|
|
|
newData := make(map[string]interface{})
|
|
|
|
|
|
@@ -1058,51 +1052,48 @@ func SaveRemark(taskid []string, reason, username string) {
|
|
|
}
|
|
|
}
|
|
|
checkArr = append(checkArr, newData)
|
|
|
- task["a_check"] = checkArr
|
|
|
-
|
|
|
- mgdb.Update("task", query, map[string]interface{}{
|
|
|
- "$set": task,
|
|
|
- }, false, false)
|
|
|
+ (*task)["a_check"] = checkArr
|
|
|
+ u.MgoE.UpdateById("task", id, map[string]interface{}{"$set": &task})
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
//修改任务状态
|
|
|
func UpTaskState(code []string, num int, reason string, startTime int64) {
|
|
|
- query := bson.M{}
|
|
|
- update := bson.M{}
|
|
|
+ query := map[string]interface{}{}
|
|
|
+ update := map[string]interface{}{}
|
|
|
for _, v := range code {
|
|
|
- if num == 1 || num == 2 || num == 3 || num == 7 {
|
|
|
- query = bson.M{
|
|
|
- "_id": bson.ObjectIdHex(v),
|
|
|
+ if num == 1 || num == 2 || num == 3 || num == 7 { //id
|
|
|
+ query = map[string]interface{}{
|
|
|
+ "_id": mongodb.StringTOBsonId(v),
|
|
|
}
|
|
|
} else {
|
|
|
- query = bson.M{
|
|
|
+ query = map[string]interface{}{ //code
|
|
|
"s_code": v,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if num == 1 { //提交审核
|
|
|
- update = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ update = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 3,
|
|
|
},
|
|
|
}
|
|
|
} else if num == 2 { //打回 -->未通过
|
|
|
- update = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ update = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 5,
|
|
|
},
|
|
|
}
|
|
|
} else if num == 3 { //发布(审核通过) -->审核通过
|
|
|
- update = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ update = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 4,
|
|
|
},
|
|
|
}
|
|
|
} else if num == 4 { //批量作废 -->关闭
|
|
|
- update = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ update = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 6,
|
|
|
"l_complete": time.Now().Unix(),
|
|
|
},
|
|
|
@@ -1115,15 +1106,15 @@ func UpTaskState(code []string, num int, reason string, startTime int64) {
|
|
|
}
|
|
|
mrecord := []interface{}{}
|
|
|
mrecord = append(mrecord, newData)
|
|
|
- update = bson.M{
|
|
|
- "$set": bson.M{
|
|
|
+ update = map[string]interface{}{
|
|
|
+ "$set": map[string]interface{}{
|
|
|
"i_state": 3,
|
|
|
"l_complete": time.Now().Unix(),
|
|
|
"a_mrecord": mrecord,
|
|
|
},
|
|
|
}
|
|
|
}
|
|
|
- flag := mgdb.Update("task", query, update, false, true)
|
|
|
+ flag := u.MgoE.Update("task", query, update, false, true)
|
|
|
log.Println("codeOrId:", query, " 修改任务状态：", flag)
|
|
|
}
|
|
|
}
|
|
|
@@ -1147,10 +1138,10 @@ func (f *Front) ChangeEvent() {
|
|
|
f.ServeJson("没有对应节点")
|
|
|
return
|
|
|
}
|
|
|
- info := *mgdb.FindOne("luaconfig", `{"code":"`+code+`"}`)
|
|
|
- if len(info) > 0 {
|
|
|
- oldevent := qu.IntAll(info["event"])
|
|
|
- if qu.IntAll(info["state"]) == Sp_state_5 {
|
|
|
+ info, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
|
|
|
+ if len(*info) > 0 {
|
|
|
+ oldevent := qu.IntAll((*info)["event"])
|
|
|
+ if qu.IntAll((*info)["state"]) == Sp_state_5 {
|
|
|
//源节点下架
|
|
|
_, err := spider.UpdateSpiderByCodeState(code, fmt.Sprint(Sp_state_6), oldevent)
|
|
|
set := map[string]interface{}{
|
|
|
@@ -1159,7 +1150,7 @@ func (f *Front) ChangeEvent() {
|
|
|
"state": Sp_state_6,
|
|
|
},
|
|
|
}
|
|
|
- mgdb.Update("luaconfig", `{"code":"`+code+`"}`, set, true, false)
|
|
|
+ u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
|
|
|
if err != nil && strings.Contains(err.Error(), "timeout") {
|
|
|
f.ServeJson("连接节点" + fmt.Sprint(oldevent) + "超时")
|
|
|
} else {
|
|
|
@@ -1171,7 +1162,7 @@ func (f *Front) ChangeEvent() {
|
|
|
"event": qu.IntAll(event),
|
|
|
},
|
|
|
}
|
|
|
- mgdb.Update("luaconfig", `{"code":"`+code+`"}`, set, true, false)
|
|
|
+ u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
|
|
|
}
|
|
|
} else {
|
|
|
f.ServeJson("没有对应记录")
|
|
|
@@ -1215,32 +1206,37 @@ func (f *Front) LuaList() {
|
|
|
searchStr := f.GetString("search[value]")
|
|
|
//search := strings.Replace(searchStr, " ", "", -1)
|
|
|
search := strings.TrimSpace(searchStr)
|
|
|
- query := bson.M{}
|
|
|
+ platform := f.GetString("platform")
|
|
|
+ query := map[string]interface{}{}
|
|
|
|
|
|
- q1 := bson.M{}
|
|
|
+ q1 := map[string]interface{}{}
|
|
|
q1["$or"] = []interface{}{
|
|
|
- bson.M{"code": bson.M{"$regex": search}},
|
|
|
- bson.M{"createuser": bson.M{"$regex": search}},
|
|
|
- bson.M{"param_common.1": bson.M{"$regex": search}},
|
|
|
+ map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
|
|
|
+ map[string]interface{}{"createuser": map[string]interface{}{"$regex": search}},
|
|
|
+ map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": search}},
|
|
|
}
|
|
|
- q2 := bson.M{}
|
|
|
+ q2 := map[string]interface{}{}
|
|
|
if state > -1 {
|
|
|
- q2 = bson.M{"state": state}
|
|
|
+ q2 = map[string]interface{}{"state": state}
|
|
|
} else {
|
|
|
q2["$or"] = []interface{}{
|
|
|
- bson.M{"state": Sp_state_3},
|
|
|
- bson.M{"state": Sp_state_5},
|
|
|
- bson.M{"state": Sp_state_6},
|
|
|
+ map[string]interface{}{"state": Sp_state_3},
|
|
|
+ map[string]interface{}{"state": Sp_state_5},
|
|
|
+ map[string]interface{}{"state": Sp_state_6},
|
|
|
}
|
|
|
}
|
|
|
- q3 := bson.M{}
|
|
|
+ q3 := map[string]interface{}{}
|
|
|
if event > -1 {
|
|
|
- q3 = bson.M{"event": event}
|
|
|
+ q3 = map[string]interface{}{"event": event}
|
|
|
+ }
|
|
|
+ q4 := map[string]interface{}{}
|
|
|
+ if platform != "-1" {
|
|
|
+ q4 = map[string]interface{}{"platform": platform}
|
|
|
}
|
|
|
if search != "" {
|
|
|
- query["$and"] = []interface{}{q1, q2, q3}
|
|
|
+ query["$and"] = []interface{}{q1, q2, q3, q4}
|
|
|
} else {
|
|
|
- query["$and"] = []interface{}{q2, q3}
|
|
|
+ query["$and"] = []interface{}{q2, q3, q4}
|
|
|
}
|
|
|
sort := `{"%s":%d}`
|
|
|
orderIndex := f.GetString("order[0][column]")
|
|
|
@@ -1251,9 +1247,10 @@ func (f *Front) LuaList() {
|
|
|
}
|
|
|
sort = fmt.Sprintf(sort, orderName, orderType)
|
|
|
page := start / 10
|
|
|
- luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
|
|
|
- count := mgdb.Count("luaconfig", query)
|
|
|
- for k, v := range luas {
|
|
|
+ qu.Debug("query:", query)
|
|
|
+ luas, _ := u.MgoE.Find("luaconfig", query, sort, list_fields, false, start, limit)
|
|
|
+ count := u.MgoE.Count("luaconfig", query)
|
|
|
+ for k, v := range *luas {
|
|
|
v["num"] = k + 1 + page*10
|
|
|
l_uploadtime := qu.Int64All(v["l_uploadtime"])
|
|
|
v["l_uploadtime"] = qu.FormatDateByInt64(&l_uploadtime, qu.Date_Full_Layout)
|
|
|
@@ -1301,7 +1298,7 @@ func (f *Front) Heart() {
|
|
|
searchStr := f.GetString("search[value]")
|
|
|
search := strings.TrimSpace(searchStr) //只能搜索code
|
|
|
//qu.Debug("search:", search, "start:", start, "limit:", limit, "draw:", draw)
|
|
|
- query := bson.M{
|
|
|
+ query := map[string]interface{}{
|
|
|
"del": false,
|
|
|
}
|
|
|
if event > -1 {
|
|
|
@@ -1319,24 +1316,41 @@ func (f *Front) Heart() {
|
|
|
}
|
|
|
sort = fmt.Sprintf(sort, orderName, orderType)
|
|
|
qu.Debug("query:", query, "sort:", sort)
|
|
|
- list := *mgu.Find("spider_heart", "spider", "spider", query, sort, nil, false, start, limit)
|
|
|
- count := mgu.Count("spider_heart", "spider", "spider", query)
|
|
|
- for _, l := range list {
|
|
|
+ list, _ := u.MgoS.Find("spider_heart", query, sort, nil, false, start, limit)
|
|
|
+ count := u.MgoS.Count("spider_heart", query)
|
|
|
+ for _, l := range *list {
|
|
|
code := qu.ObjToString(l["code"])
|
|
|
- d := *mgdb.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1})
|
|
|
- l["state"] = d["state"]
|
|
|
- l["param_common"] = d["param_common"]
|
|
|
+ d, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1, "str_list": 1, "type_list": 1})
|
|
|
+ l["state"] = (*d)["state"]
|
|
|
+ l["param_common"] = (*d)["param_common"]
|
|
|
if lt := qu.Int64All(l["list"]); lt != 0 {
|
|
|
l["list"] = qu.FormatDateByInt64(<, qu.Date_Full_Layout)
|
|
|
+ } else {
|
|
|
+ l["list"] = 0
|
|
|
}
|
|
|
if dt := qu.Int64All(l["detail"]); dt != 0 {
|
|
|
l["detail"] = qu.FormatDateByInt64(&dt, qu.Date_Full_Layout)
|
|
|
+ } else {
|
|
|
+ l["detail"] = 0
|
|
|
}
|
|
|
if det := qu.Int64All(l["detailexecute"]); det != 0 {
|
|
|
l["detailexecute"] = qu.FormatDateByInt64(&det, qu.Date_Full_Layout)
|
|
|
+ } else {
|
|
|
+ l["detailexecute"] = 0
|
|
|
+ }
|
|
|
+ if ft := qu.Int64All(l["findlist"]); ft != 0 {
|
|
|
+ l["findlist"] = qu.FormatDateByInt64(&ft, qu.Date_Full_Layout)
|
|
|
+ } else {
|
|
|
+ l["findlist"] = 0
|
|
|
}
|
|
|
ut := qu.Int64All(l["updatetime"])
|
|
|
l["updatetime"] = qu.FormatDateByInt64(&ut, qu.Date_Full_Layout)
|
|
|
+ l["isfindlist"] = "否"
|
|
|
+ typeList := qu.IntAll((*d)["type_list"])
|
|
|
+ strList := qu.ObjToString((*d)["str_list"])
|
|
|
+ if typeList == 1 && strings.Contains(strList, "findListHtml") {
|
|
|
+ l["isfindlist"] = "是"
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
f.ServeJson(map[string]interface{}{"draw": draw, "data": list, "recordsFiltered": count, "recordsTotal": count})
|
|
|
@@ -1409,43 +1423,41 @@ func SpiderModifyTask(data string) {
|
|
|
//查看是否有该任务
|
|
|
func checkTask(codes []string, num int) []string {
|
|
|
// var id string = ""
|
|
|
- query := bson.M{}
|
|
|
+ query := map[string]interface{}{}
|
|
|
var idArr []string
|
|
|
if len(codes) > 0 {
|
|
|
for _, v := range codes {
|
|
|
if num == 1 {
|
|
|
- query = bson.M{
|
|
|
+ query = map[string]interface{}{
|
|
|
"s_code": v,
|
|
|
- "i_state": bson.M{
|
|
|
+ "i_state": map[string]interface{}{
|
|
|
"$in": []int{1, 2, 5},
|
|
|
},
|
|
|
}
|
|
|
} else if num == 2 { //打回时查询待审核的任务
|
|
|
- query = bson.M{
|
|
|
+ query = map[string]interface{}{
|
|
|
"s_code": v,
|
|
|
"i_state": 3,
|
|
|
}
|
|
|
} else if num == 3 { //审核通过时查询待处理、处理中、待审核、未通过的任务
|
|
|
- query = bson.M{
|
|
|
+ query = map[string]interface{}{
|
|
|
"s_code": v,
|
|
|
- "i_state": bson.M{
|
|
|
+ "i_state": map[string]interface{}{
|
|
|
"$in": []int{1, 2, 3, 5},
|
|
|
},
|
|
|
}
|
|
|
} else if num == 7 {
|
|
|
- query = bson.M{
|
|
|
+ query = map[string]interface{}{
|
|
|
"s_code": v,
|
|
|
- "i_state": bson.M{
|
|
|
+ "i_state": map[string]interface{}{
|
|
|
"$in": []int{2, 5},
|
|
|
},
|
|
|
}
|
|
|
}
|
|
|
- //task := *mgdb.FindOne("task", query)
|
|
|
- task := *mgdb.Find("task", query, nil, nil, false, -1, -1)
|
|
|
+ task, _ := u.MgoE.Find("task", query, nil, nil, false, -1, -1)
|
|
|
if task != nil {
|
|
|
- for _, v := range task {
|
|
|
- //id = v["_id"].(bson.ObjectId).Hex()
|
|
|
- idArr = append(idArr, v["_id"].(bson.ObjectId).Hex())
|
|
|
+ for _, t := range *task {
|
|
|
+ idArr = append(idArr, mongodb.BsonIdToSId(t["_id"]))
|
|
|
}
|
|
|
}
|
|
|
return idArr
|
