spider_editor/src/front/spider.go

package front

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"mongodb"
	"sort"
	"spider"

	mu "mfw/util"
	qu "qfw/util"
	util "spiderutil"
	"strings"
	"time"
	u "util"
)

type Base struct {
	SpiderCode             string
	SpiderCodeOld          string
	SpiderName             string
	SpiderChannel          string
	SpiderDownDetailPage   bool
	SpiderStartPage        int
	SpiderMaxPage          int
	SpiderRunRate          int
	Spider2Collection      string
	SpiderPageEncoding     string
	SpiderStoreMode        int //1,2
	SpiderStoreToMsgEvent  int
	SpiderTargetChannelUrl string
	SpiderLastDownloadTime string
	SpiderIsHistoricalMend bool
	SpiderIsMustDownload   bool
}

type Step1 struct {
	Address        string
	ContentChooser string
	DateFormat     string
	Expert         string
	Types          int
}

type Step2 struct {
	Listadd        string
	Listadds       string
	BlockChooser   string
	AddressChooser string
	TitleChooser   string
	DateChooser    string
	DateFormat     string
	Expert         string
	Types          int
}

type Step3 struct {
	ContentChooser string
	ElementChooser string
	T_title        string
	T_href         string
	T_date         string
	Expert         string
	Types          int
}

type StepRe3 struct {
	Checked bool
	Expert  string
}

//加载某个爬虫
func (f *Front) LoadSpider(codeTaskIdReState string) error {
	tmpStr := strings.Split(codeTaskIdReState, "__")
	code := tmpStr[0]
	taskId := tmpStr[1]
	auth := qu.IntAll(f.GetSession("auth"))
	restate := -1
	if taskId == "restate=1" { //重采编辑
		restate = 1
	} else if taskId == "restate=2" {
		restate = 2
	} else if taskId == "restate=3" {
		restate = 3
	} else {
		if auth == role_dev && (f.GetSession(taskId) == nil || f.GetSession(taskId) == "") {
			xgTime := time.Unix(time.Now().Unix(), 0).Format("2006-01-02 15:04:05")
			f.SetSession(taskId, xgTime)
		}
	}
	copy := f.GetString("copy")
	if f.Method() == "GET" {
		code := util.Se.Decode4Hex(code)
		f.T["actiontext"] = "编辑"
		lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
		auth := qu.IntAll(f.GetSession("auth"))
		if qu.ObjToString((*lua)["createuserid"]) == f.GetSession("userid").(string) || auth >= 1 {
			if len(*lua) > 0 {
				if copy != "" {
					luacopy, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": copy})
					if len(*luacopy) > 0 {
						(*lua)["model"] = (*luacopy)["model"]
						common_copy := (*luacopy)["param_common"].([]interface{})
						common := (*lua)["param_common"].([]interface{})
						common_copy[0] = common[0]
						common_copy[1] = common[1]
						common_copy[2] = common[2]
						common_copy[11] = common[11]
						(*lua)["param_common"] = (*luacopy)["param_common"]
						(*lua)["param_time"] = (*luacopy)["param_time"]
						(*lua)["param_list"] = (*luacopy)["param_list"]
						(*lua)["param_content"] = (*luacopy)["param_content"]
						(*lua)["str_list"] = (*luacopy)["str_list"]
						(*lua)["str_time"] = (*luacopy)["str_time"]
						(*lua)["str_content"] = (*luacopy)["str_content"]
						(*lua)["Thref"] = (*luacopy)["Thref"]
						(*lua)["Tpublishtime"] = (*luacopy)["Tpublishtime"]
						(*lua)["Ttitle"] = (*luacopy)["Ttitle"]
						(*lua)["Tdate"] = (*luacopy)["Tdate"]
						(*lua)["type_content"] = (*luacopy)["type_content"]
						(*lua)["type_list"] = (*luacopy)["type_list"]
						(*lua)["type_time"] = (*luacopy)["type_time"]
					}
				}
				if (*lua)["listcheck"] != nil {
					listcheck := (*lua)["listcheck"].(string)
					listcheck = strings.Replace(listcheck, "\\n", "\n", -1)
					listcheck = strings.Replace(listcheck, "\\", "", -1)
					(*lua)["listcheck"] = listcheck
				}
				if (*lua)["contentcheck"] != nil {
					contentcheck := (*lua)["contentcheck"].(string)
					contentcheck = strings.Replace(contentcheck, "\\n", "\n", -1)
					contentcheck = strings.Replace(contentcheck, "\\", "", -1)
					(*lua)["contentcheck"] = contentcheck
				}
				js, _ := json.MarshalIndent((*lua)["model"], "", "  ")
				(*lua)["js"] = string(js)
				f.T["lua"] = lua
				f.T["taskId"] = taskId
				f.T["restate"] = restate
				f.T["isflow"] = (*lua)["isflow"]
				f.T["spidertype"] = (*lua)["spidertype"]
				f.T["spidermovevent"] = (*lua)["spidermovevent"]
				f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
				if (*lua)["oldlua"] != nil {
					return f.Render("oldedit.html", &f.T)
				}
				return f.Render("spideredit.html", &f.T)
			}
		} else {
			f.Write("您没有编辑他人脚本的权限")
		}
	}
	return nil
}

//查看某个爬虫
func (f *Front) ViewSpider(id string) error {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth >= 1 {
		if f.Method() == "GET" {
			code := util.Se.Decode4Hex(id)
			f.T["actiontext"] = "编辑"
			lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
			if len(*lua) > 0 {
				if (*lua)["listcheck"] != nil {
					listcheck := (*lua)["listcheck"].(string)
					listcheck = strings.Replace(listcheck, "\\n", "\n", -1)
					listcheck = strings.Replace(listcheck, "\\", "", -1)
					(*lua)["listcheck"] = listcheck
				}
				if (*lua)["contentcheck"] != nil {
					contentcheck := (*lua)["contentcheck"].(string)
					contentcheck = strings.Replace(contentcheck, "\\n", "\n", -1)
					contentcheck = strings.Replace(contentcheck, "\\", "", -1)
					(*lua)["contentcheck"] = contentcheck
				}
				js, _ := json.MarshalIndent((*lua)["model"], "", "  ")
				(*lua)["js"] = string(js)
				f.T["lua"] = lua
				f.T["isflow"] = (*lua)["isflow"]
				f.T["spidertype"] = (*lua)["spidertype"]
				f.T["spidermovevent"] = (*lua)["spidermovevent"]
				f.T["spiderhistorymaxpage"] = (*lua)["spiderhistorymaxpage"]
				if (*lua)["oldlua"] != nil {
					return f.Render("oldedit.html", &f.T)
				}
				return f.Render("spiderview.html", &f.T)
			} else {
				f.Write("没有对应记录！")
				return nil
			}
		}
		return f.Redirect("/center")
	} else {
		f.Write("您没有查看他人脚本的权限")
		return nil
	}
}

func (f *Front) LoadModel(id string) error {
	if f.Method() == "GET" {
		lua, _ := u.MgoE.Find("luaconfig", map[string]interface{}{"code": id}, nil, map[string]interface{}{"model": 1}, true, -1, -1)
		if len(*lua) > 0 {
			f.ServeJson((*lua)[0])
		}
	}
	return f.Redirect("/center")
}

func (f *Front) SaveStep() {
	userid, _ := f.GetSession("userid").(string)
	auth := qu.IntAll(f.GetSession("auth"))
	rep := map[string]interface{}{}
	if f.GetString("oldlua") != "" {
		id := f.GetString("code")
		one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
		id = (*one)["code"].(string)
		script := f.GetStringComm("script")
		if strings.Index(script, id) == -1 {
			rep["msg"] = "code/名称都不能更改"
			f.ServeJson(rep)
			return
		} else {
			upset := map[string]interface{}{"luacontent": script}
			upset["modifytime"] = time.Now().Unix()
			b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
			if b {
				rep["msg"] = "保存成功"
				rep["code"] = util.Se.Encode2Hex(id)
				f.ServeJson(rep)
				return
			}
		}
	} else {
		if f.Base.SpiderName != "" && f.Base.SpiderCode != "" {
			code := f.Base.SpiderCode
			one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": f.Base.SpiderCode})
			state := qu.IntAllDef((*one)["state"], 0)
			restate := qu.IntAll((*one)["restate"])
			comeintime := time.Now().Unix()
			if len((*one)) > 0 {
				comeintime = qu.Int64All((*one)["comeintime"])
				ouserid := (*one)["createuserid"].(string)
				if ouserid != userid && auth == role_dev {
					f.Write("权限不够，不能修改他人脚本")
					return
				} else {
					code = (*one)["code"].(string)
					f.Base.SpiderCode = (*one)["code"].(string)
					f.Base.SpiderName = ((*one)["param_common"].([]interface{}))[1].(string)
				}
			} else {
				if auth != role_admin {
					f.Write("不能新建爬虫，请联系管理员导入")
					return
				}
			}
			listcheck := f.GetString("listcheck")
			contentcheck := f.GetString("contentcheck")
			if auth == role_dev {
				//f.Base.SpiderStoreToMsgEvent = 4002
			}
			common := []interface{}{
				f.Base.SpiderCode,
				f.Base.SpiderName,
				f.Base.SpiderChannel,
				f.Base.SpiderDownDetailPage,
				f.Base.SpiderStartPage,
				f.Base.SpiderMaxPage,
				f.Base.SpiderRunRate,
				f.Base.Spider2Collection,
				f.Base.SpiderPageEncoding,
				f.Base.SpiderStoreMode,
				f.Base.SpiderStoreToMsgEvent,
				f.Base.SpiderTargetChannelUrl,
				f.Base.SpiderLastDownloadTime,
				f.Base.SpiderIsHistoricalMend,
				f.Base.SpiderIsMustDownload,
			}
			ptime := []interface{}{
				f.Step1.DateFormat,
				f.Step1.Address,
				f.Step1.ContentChooser,
			}
			list := []interface{}{
				f.Step2.Listadd,
				f.Step2.Listadds,
				f.Step2.BlockChooser,
				f.Step2.AddressChooser,
				f.Step2.TitleChooser,
				f.Step2.DateChooser,
				f.Step2.DateFormat,
			}
			content := []interface{}{
				f.Step3.ContentChooser,
				f.Step3.ElementChooser,
			}
			param := map[string]interface{}{}
			common[4] = 1
			param["param_common"] = common
			//向导模式
			param["param_time"] = ptime
			param["param_list"] = list
			param["param_content"] = content
			param["type_time"] = f.Step1.Types
			param["type_list"] = f.Step2.Types
			param["type_content"] = f.Step3.Types
			//专家模式
			param["str_time"] = f.Step1.Expert
			param["str_list"] = f.Step2.Expert
			param["str_content"] = f.Step3.Expert
			param["comeintime"] = comeintime
			listcheck = strings.Replace(listcheck, "\n", "\\\\n", -1)
			param["listcheck"] = strings.Replace(listcheck, "\"", "\\\\\"", -1)
			contentcheck = strings.Replace(contentcheck, "\n", "\\\\n", -1)
			param["contentcheck"] = strings.Replace(contentcheck, "\"", "\\\\\"", -1)
			//补充模型
			s_model := f.GetString("model")
			configModel := util.Config.Model[s_model]
			model := map[string]interface{}{}
			for k, _ := range configModel {
				model[k] = f.GetString(k)
			}
			model["model"] = s_model
			param["code"] = f.Base.SpiderCode
			param["model"] = model
			if len((*one)) > 0 {
				param["createuser"] = (*one)["createuser"]
				param["createuserid"] = (*one)["createuserid"]
				param["code"] = (*one)["code"]
				//开发员关联任务修改爬虫状态
				state = qu.IntAll((*one)["state"])
				if auth == role_dev && state >= Sp_state_3 && restate != 1 { //开发员修改,已经审核通过(不包含已上架),状态重置为待完成(restate!=1判断，重采修改保存爬虫时不修改爬虫状态)
					param["state"] = 0
				} else {
					param["state"] = state
				}

			} else {
				param["createuser"] = f.GetSession("loginuser")
				param["createuserid"] = f.GetSession("userid")
				param["createuseremail"] = f.GetSession("email")
				param["next"] = f.GetSession("email")
				param["state"] = 0
			}
			if qu.ObjToString((*one)["modifyuser"]) == "" {
				param["modifyuser"] = param["createuser"]
				param["modifyuserid"] = param["createuserid"]
			}
			param["modifytime"] = time.Now().Unix()
			param["Ttitle"] = f.Step3.T_title
			param["Thref"] = f.Step3.T_href
			param["Tdate"] = f.Step3.T_date
			//其他信息
			param["isflow"] = f.OtherBase.IsFlow
			param["spidertype"] = f.OtherBase.SpiderType
			param["spidermovevent"] = f.OtherBase.SpiderMoveEvent
			param["spiderhistorymaxpage"] = f.OtherBase.SpiderHistoryMaxPage
			if f.OtherBase.SpiderType == "history" { //爬虫类型是history的放到7000节点,并记录历史节点
				param["event"] = 7000
				if event := qu.IntAll((*one)["event"]); event != 7000 {
					param["historyevent"] = event
				}
			}
			if f.OtherBase.SpiderMoveEvent == "7700" {
				param["historyevent"] = 7700
			}
			//三级页复制
			param["str_recontent"] = f.StepRe3.Expert
			param["iscopycontent"] = f.StepRe3.Checked
			//
			param["listisfilter"] = ListFilterReg.MatchString(f.Step2.Expert) //列表页校验是否含“--关键词过滤”
			matchLua, msg := LuaTextCheck(f.Step2.Expert, f.Step3.Expert, f.Step2.Types)
			if !matchLua {
				issave := spider.SaveSpider(code, param) //保存脚本
				if issave {
					for k, v := range *one {
						if k != "_id" && param[k] == nil {
							param[k] = v
						}
					}
					Wlog(f.Base.SpiderName, f.Base.SpiderCode, f.GetSession("username").(string), f.GetSession("userid").(string), "修改", param)
					rep["msg"] = "保存成功"
				} else {
					rep["msg"] = "保存失败"
				}
			} else {
				rep["msg"] = "保存失败，" + msg
			}
			rep["code"] = util.Se.Encode2Hex(code)
			f.ServeJson(rep)
		}
	}
}

//检查列表页和三级页代码中是否含lua原生方法
func LuaTextCheck(list, detail string, type_list int) (b bool, msg string) {
	defer qu.Catch()
	if LuaReg.MatchString(list) || LuaReg.MatchString(detail) {
		b = true
		msg = "代码中含有lua原生方法；"
	}
	//if type_list != 0 && !strings.Contains(list, "sendListNum") { //列表页专家模式且不含sendListNum
	//	b = true
	//	msg = "代码中缺少sendListNum方法；" + msg
	//}
	return
}

func (f *Front) Assort() {
	state, _ := f.GetInteger("state")
	code := f.GetString("code")
	sql := map[string]interface{}{
		"$set": map[string]interface{}{
			"state":      state,
			"modifytime": time.Now().Unix(),
		},
	}
	queryT := map[string]interface{}{
		"code": code,
	}
	//下架爬虫
	lua, _ := u.MgoE.FindOne("luaconfig", queryT)
	upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
	qu.Debug("下架爬虫：", code, upresult, err)
	if upresult && err == nil {
		//更新爬虫
		u.MgoE.Update("luaconfig", queryT, sql, false, false)
		//关闭任务
		query := map[string]interface{}{
			"s_code": code,
		}
		sql = map[string]interface{}{
			"$set": map[string]interface{}{
				"i_state": 6,
			},
		}
		u.MgoE.Update("task", query, sql, false, true)
		//删除心跳
		DelSpiderHeart(code)
	}
	f.ServeJson(map[string]interface{}{"upresult": upresult})
}

//方法测试
func (f *Front) RunStep() {
	imodal, _ := f.GetInteger("imodal")
	script, _ := f.GetBool("script")
	listcheck := f.GetString("listcheck")
	contentcheck := f.GetString("contentcheck")
	downloadnode := f.GetString("downloadnode") //下载节点
	common := []interface{}{
		f.Base.SpiderCode,
		f.Base.SpiderName,
		f.Base.SpiderChannel,
		f.Base.SpiderDownDetailPage,
		f.Base.SpiderStartPage,
		f.Base.SpiderMaxPage,
		f.Base.SpiderRunRate,
		f.Base.Spider2Collection,
		f.Base.SpiderPageEncoding,
		f.Base.SpiderStoreMode,
		f.Base.SpiderStoreToMsgEvent,
		f.Base.SpiderTargetChannelUrl,
		f.Base.SpiderLastDownloadTime,
		f.Base.SpiderIsHistoricalMend,
		f.Base.SpiderIsMustDownload,
		"",
		"",
		"",
	}
	if f.Method() == "POST" {
		switch f.GetString("step") {
		case "step1":
			ptime := []interface{}{
				f.Step1.DateFormat,
				f.Step1.Address,
				f.Step1.ContentChooser,
			}
			if script {
				_, scripts := spider.GetLastPublishTime(common, ptime, f.Step1.Expert, downloadnode, imodal, 1)
				f.ServeJson(scripts)
				return
			}
			rs, err := spider.GetLastPublishTime(common, ptime, f.Step1.Expert, downloadnode, imodal)
			if err == nil {
				f.ServeJson(rs)
			}
		case "step2":
			addrs := strings.Split(f.Step2.Listadds, "\n")
			if len(addrs) > 0 {
				for k, v := range addrs {
					addrs[k] = "'" + v + "'"
				}
				f.Step2.Listadds = strings.Join(addrs, ",")
			} else if len(f.Step2.Listadds) > 5 {
				f.Step2.Listadds = "'" + f.Step2.Listadds + "'"
			} else {
				f.Step2.Listadds = ""
			}
			list := []interface{}{
				f.Step2.Listadd,
				f.Step2.Listadds,
				f.Step2.BlockChooser,
				f.Step2.AddressChooser,
				f.Step2.TitleChooser,
				f.Step2.DateChooser,
				f.Step2.DateFormat,
			}
			listcheck = strings.Replace(listcheck, "\n", "\\n", -1)
			listcheck = strings.Replace(listcheck, "\"", "\\\"", -1)
			s_model := f.GetString("model")
			configModel := util.Config.Model[s_model]
			model := map[string]interface{}{}
			for k, _ := range configModel {
				model[k] = f.GetString(k)
			}
			if script {
				_, script := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal, 1)
				f.ServeJson(script)
				return
			}
			rs, err := spider.GetPageList(common, list, model, listcheck, f.Step2.Expert, downloadnode, imodal)
			if err == nil {
				f.ServeJson(rs)
			} else if err.(error).Error() == "no" {
				f.ServeJson(rs[0])
			}
		case "step3":
			content := []interface{}{
				f.Step3.ContentChooser,
				f.Step3.ElementChooser,
			}

			contentcheck = strings.Replace(contentcheck, "\n", "\\n", -1)
			contentcheck = strings.Replace(contentcheck, "\"", "\\\"", -1)
			data := map[string]interface{}{}
			data["title"] = f.Step3.T_title
			data["href"] = f.Step3.T_href
			data["publishtime"] = f.Step3.T_date
			if script {
				_, script := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal, 1)
				f.ServeJson(script)
				return
			}
			rs, err := spider.GetContentInfo(common, content, data, contentcheck, f.Step3.Expert, downloadnode, imodal)
			if projectinfo, ok := rs["projectinfo"].(map[string]interface{}); ok && projectinfo != nil {
				if attachments, ok := projectinfo["attachments"].(map[string]interface{}); ok && attachments != nil {
					for _, tmp := range attachments {
						tmpMap := tmp.(map[string]interface{})
						if qu.ObjToString(tmpMap["filename"]) == "附件中含有乱码" {
							rs["msg"] = "附件中含有乱码"
						}
					}
				}
			}
			if err == nil {
				f.ServeJson(rs)
			} else {
				f.ServeJson(rs["no"])
			}
		}
	}
}

//爬虫测试数据json
func (f *Front) GetJson() {
	list, _ := f.GetSession("listInfo").([]map[string]interface{})
	data, _ := f.GetSession("dataInfo").(map[string]interface{})
	descript := f.GetSession("task_descript")
	remark := f.GetSession("task_remark")
	rateremark := f.GetSession("task_rateremark")
	reason := f.GetSession("reason")
	username := f.GetSession("username").(string)
	msg := f.GetSession(username + "_msg")
	if len(data) > 0 {
		data["contenthtml"] = ""
	}
	for k, v := range list {
		v["a_index"] = k + 1
	}

	f.T["list"] = list
	f.T["data"] = data
	f.T["descript"] = descript
	f.T["remark"] = remark
	f.T["rateremark"] = rateremark
	f.T["reason"] = reason
	f.T["msg"] = msg

	f.DelSession("listInfo")
	f.DelSession("dataInfo")
	f.DelSession("task_descript")
	f.DelSession("task_remark")
	f.DelSession("task_rateremark")
	f.DelSession("reason")

	f.Render("jsonInfo.html", &f.T)
}

//整体测试
func (f *Front) SpiderPass() {
	defer mu.Catch()
	list := []map[string]interface{}{}
	data := map[string]interface{}{}
	msg1 := ""
	code := f.GetString("code")
	downloadnode := f.GetString("node")
	//根据code查询待确认任务
	query := map[string]interface{}{
		"s_code":  code,
		"i_state": 3,
	}
	task, _ := u.MgoE.FindOne("task", query)
	descript := "null"
	remark := "null"
	remarktmp := []string{}
	rateremarktmp := []string{}
	if len(*task) > 0 {
		descript = (*task)["s_descript"].(string)
		if mrecord, ok := (*task)["a_mrecord"].([]interface{}); ok {
			for _, m := range mrecord {
				remarkInfo := m.(map[string]interface{})
				if remark := qu.ObjToString(remarkInfo["s_mrecord_remark"]); remark != "" {
					remarktmp = append(remarktmp, remark+";")
				}
				if rateremark := qu.ObjToString(remarkInfo["s_mrecord_rateremark"]); rateremark != "" {
					rateremarktmp = append(rateremarktmp, rateremark+";")
				}
			}
		}
	}
	if len(remarktmp) > 0 {
		remark = ""
		remark = strings.Join(remarktmp, "")
	}

	f.SetSession("task_remark", remark)
	f.SetSession("task_rateremark", rateremarktmp)
	f.SetSession("task_descript", descript)
	//基本信息、方法一、方法二、方法三、总请求次数、go方法一、go方法二、go方法三、列表页条数
	steps := []interface{}{false, false, false, false, 0, 0, 0, 0, 0}
	one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
	reason, _ := (*one)["reason"].(string)
	f.SetSession("reason", reason)
	if len(*one) > 0 && (*one)["oldlua"] == nil {
		common := (*one)["param_common"].([]interface{})
		if len(common) < 13 {
			f.ServeJson(steps)
			return
		} else {
			steps[0] = true
		}
	} else {
		steps[0] = true
	}
	script, liststr, contentstr := "", "", ""
	if (*one)["oldlua"] == nil {
		script, liststr, contentstr = spider.GetScript(code)
	} else {
		script = (*one)["luacontent"].(string)
	}
	if liststr != "" && contentstr != "" {
		msg1 = u.SpiderPassCheckLua(liststr, contentstr, (*one))
	}
	s := spider.CreateSpider(downloadnode, script)
	s.SpiderMaxPage = 1
	s.Timeout = 60
	time, timeerr := s.GetLastPublishTime()
	if timeerr == nil && len(time) > 4 {
		steps[1] = true
		list, _ = s.DownListPageItem()
		if len(list) > 0 {
			f.SetSession("listInfo", list)
			listone := list[0]
			if len(qu.ObjToString(listone["href"])) < 7 ||
				(qu.ObjToString(listone["publishtime"]) != "0" && len(qu.ObjToString(listone["publishtime"])) < 5) ||
				len(qu.ObjToString(listone["title"])) < 3 {
				f.ServeJson(steps)
				return
			} else {
				steps[2] = true
				if s.DownDetail {
					param := map[string]string{}
					index := 0
					if len(list) > 0 {
						steps[8] = len(list)
						index = len(list) / 2
						for k, v := range list[index] {
							param[k] = qu.ObjToString(v)
						}
						data = map[string]interface{}{}
						s.DownloadDetailPage(param, data)
						if len(data) > 0 {
							f.SetSession("dataInfo", data)
						} else {
							f.SetSession("dataInfo", "")
						}
						if len(data) == 0 || data["detail"].(string) == "" {
							steps[3] = false
						} else {
							steps[3] = true
						}
					}
				} else {
					steps[3] = true
				}
			}
		} else {
			f.SetSession("listInfo", "")
			f.SetSession("dataInfo", "")
		}
	}
	//关闭laustate
	s.L.Close()
	steps[4] = s.Test_luareqcount
	steps[5] = s.Test_goreqtime
	steps[6] = s.Test_goreqlist
	steps[7] = s.Test_goreqcon
	//校验
	msg := u.SpiderPassCheckListAndDetail(list, data)
	if msg1 != "" {
		msg = msg1 + "；" + msg
	}
	username := f.GetSession("username").(string)
	f.SetSession(username+"_msg", msg)
	f.ServeJson(steps)
}

func (f *Front) DownSpider(id string) {
	//auth := qu.IntAll(f.GetSession("auth"))
	//if auth > role_dev {
	one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
	createuserid := qu.ObjToString((*one)["createuserid"])
	script := ""
	filename := id + ".lua"
	if len(*one) > 0 {
		if (*one)["oldlua"] != nil {
			if (*one)["luacontent"] != nil {
				script = (*one)["luacontent"].(string)
			}
		} else {
			user, _ := u.MgoE.FindById("user", createuserid, nil)
			name := (*one)["createuser"]
			email := (*user)["s_email"]
			upload := time.Now().Format("2006-01-02 15:04:05")
			script, _, _ = spider.GetScript(id, name, email, upload)
		}
	}
	f.ResponseWriter.Header().Del("Content-Type")
	f.ResponseWriter.Header().Add("Content-Type", "application/x-download")
	f.ResponseWriter.Header().Add("Content-Disposition", "attachment;filename=spider_"+filename)
	f.WriteBytes([]byte(script))
	// } else {
	// 	f.Write("您没有权限")
	// }
}

//更新爬虫状态
func (f *Front) UpState() error {
	username := f.GetSession("username").(string)
	code := f.GetString("code")
	state, _ := f.GetInt("state")
	id := f.GetString("taskId")
	reason := f.GetString("reason")
	auth := qu.IntAll(f.GetSession("auth"))
	var codeArr = []string{code}
	var taskid []string
	//修改任务状态
	istotask := false
	res := map[string]interface{}{
		"istotask": istotask,
		"err":      "没有权限",
		"code":     util.Se.Encode2Hex(code),
		"taskid":   taskid,
	}
	var xgTime int64
	if f.GetSession(id) == nil || f.GetSession(id) == "" {
		xgTime = time.Now().Unix()
	} else {
		xgTimeStr := qu.ObjToString(f.GetSession(id))
		xgTimeTmp, _ := time.ParseInLocation("2006-01-02  15:04:05", xgTimeStr, time.Local)
		xgTime = xgTimeTmp.Unix()
	}
	f.DelSession(id)
	if IsHasUpState(auth, int(state)) {
		b, err := UpStateAndUpSpider(code, "", reason, username, int(state)) //更新爬虫状态
		if b && state == Sp_state_1 {                                        //提交审核
			//有对应任务跳转提交记录页
			taskid = checkTask(codeArr, 1)

			if len(taskid) > 0 {
				res["istotask"] = true
				res["taskid"] = taskid[0]
			}
		} else if b && state == Sp_state_2 { //打回
			taskid = checkTask(codeArr, 2)
			if len(taskid) > 0 {
				//UpTaskState([]string{taskid}, 2)     //修改状态
				UpTaskState(taskid, 2, "", int64(0)) //修改任务状态
				SaveRemark(taskid, reason, username) //保存记录信息
			}
		} else if b && state == Sp_state_3 { //审核通过
			taskid = checkTask(codeArr, 3)
			if len(taskid) > 0 {
				//UpTaskState([]string{taskid}, 3)
				UpTaskState(taskid, 3, "", int64(0))
				SaveRemark(taskid, "", username)
			}
		} else if b && state == Sp_state_6 { //下架
			//下架成功删除心跳数据
			flag := DelSpiderHeart(code)
			log.Println(code, "---下架删除download数据：", flag)
		} else if b && state == Sp_state_7 { //反馈
			taskid = checkTask(codeArr, 7)
			if len(taskid) > 0 {
				UpTaskState(taskid, 7, reason, xgTime)
			}
		}

		if err != nil {
			res["err"] = err.Error()
			f.ServeJson(res)
		} else {
			res["err"] = ""
			f.ServeJson(res)
		}
	} else {
		f.ServeJson(res)
	}
	return nil
}

//下架删除心跳
func DelSpiderHeart(code string) bool {
	return u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
}

//下架删除download数据
//func delDownloadData(code string) bool {
//	return mgu.Del("download", "spider", "spider", `{"code":"`+code+`"}`)
//}

//批量作废删除download数据
//func disableDelDownloadData(code []string) {
//	for _, v := range code {
//		flag := delDownloadData(v)
//		log.Println(code, "---批量删除download数据：", flag)
//	}
//}

//爬虫核对
func (f *Front) Checktime() {
	code := f.GetString("code")
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.ServeJson(false)
	} else {
		b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
			"l_checktime": time.Now().Unix(),
		}}, true, false)
		f.ServeJson(b)
	}
}

//批量作废
func (f *Front) Disables() error {
	auth := qu.IntAll(f.GetSession("auth"))
	names := strings.Split(f.GetString("names"), ",")
	ids := strings.Split(f.GetString("ids"), ",")
	codes := strings.Split(f.GetString("codes"), ",")
	disablereason := f.GetString("disablereason")
	res := ""
	if IsHasUpState(auth, Sp_state_4) {
		for k, id := range ids {
			b, err := UpStateAndUpSpider("", id, disablereason, "", Sp_state_4)
			if b { //作废成功
				//修改任务状态
				UpTaskState(codes, 4, "", int64(0))
				//删除download表数据
				//go disableDelDownloadData(codes)
				if err != nil {
					res = res + names[k] + ",ok" + qu.ObjToString(err.Error()) + ";"
				} else {
					res = res + names[k] + ",ok" + ";"
				}
			} else {
				res = res + names[k] + "," + qu.ObjToString(err.Error()) + ";"
			}
		}
	} else {
		res = "没有权限"
	}
	f.ServeJson(res)
	return nil
}

//批量上下架
func (f *Front) BatchShelves() {
	codes := strings.Split(f.GetString("codes"), ",")
	state, _ := f.GetInteger("state")
	auth := qu.IntAll(f.GetSession("auth"))
	errCode := []string{}
	var err error
	b := false
	if IsHasUpState(auth, Sp_state_5) {
		if state == 5 { //批量上架
			for _, code := range codes {
				_, err = UpStateAndUpSpider(code, "", "", "", Sp_state_5)
				if err != nil {
					errCode = append(errCode, code)
				}
			}
		} else { //批量下架
			for _, code := range codes {
				b, err = UpStateAndUpSpider(code, "", "", "", Sp_state_6)
				if !b || err != nil {
					errCode = append(errCode, code)
				}
				//下架删除download数据
				//if b {
				//	flag := delDownloadData(code)
				//	log.Println(code, "---删除download数据：", flag)
				//}
			}
		}
	} else {
		errCode = append(errCode, "没有权限")
	}
	f.ServeJson(errCode)
}

//更新爬虫状态，并判断是否更新节点爬虫
func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, error) {
	upresult := false
	var err error
	one := &map[string]interface{}{}
	if code != "" {
		one, _ = u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
	} else {
		one, _ = u.MgoE.FindById("luaconfig", id, nil)
		code = qu.ObjToString((*one)["code"])
	}
	if len(*one) > 0 {
		var event int
		if (*one)["event"] != nil {
			event = qu.IntAll((*one)["event"])
		} else {
			for k, _ := range util.Config.Uploadevents { //?
				event = qu.IntAll(k)
				break
			}
			//r := rand.New(rand.NewSource(time.Now().UnixNano()))
			//event = util.Config.Uploadevents[r.Intn(len(util.Config.Uploadevents))]
		}
		//oldstate := qu.IntAll(one["state"])
		switch state {
		case Sp_state_4, Sp_state_6: //作废、下架
			// if oldstate == Sp_state_5 {
			// 	upresult = false
			// 	err = errors.New("已上架不允许作废")
			// } else {
			// 	upresult = true
			// }
			upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event) //下架
			qu.Debug("下架:", upresult, code)
		case Sp_state_5: //上架(爬虫端在更新上架的时候为了更新内存中字段，采用先下架在上架)
			upresult, err = spider.UpdateSpiderByCodeState(code, "6", event)
			qu.Debug("下架:", upresult, code)
			if upresult && err == nil {
				upresult, err = spider.UpdateSpiderByCodeState(code, fmt.Sprint(state), event)
				qu.Debug("上架:", upresult, code)
			}
		default:
			upresult = true
			err = nil
		}
		if err != nil && strings.Contains(err.Error(), "timeout") {
			err = errors.New("连接节点" + fmt.Sprint(event) + "超时")
			upresult = true
		}

		if upresult && err == nil {
			upset := map[string]interface{}{"state": state} //修改状态
			if (*one)["oldlua"] != nil {                    //老脚本上传
				upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
			} else {
				if state == Sp_state_1 { //提交审核
					upset["l_complete"] = time.Now().Unix()
					upset["report"] = ""
				} else if state == Sp_state_3 { //审核通过
					if (*one)["event"] == nil {
						upset["event"] = event
						upset["modifytime"] = time.Now().Unix()
					}
					upset["frequencyerrtimes"] = 0 //爬虫审核通过，重置采集频率异常次数
					upset["l_uploadtime"] = time.Now().Unix()
				} else if state == Sp_state_2 { //打回原因
					upset["reason"] = reason
				} else if state == Sp_state_7 { //反馈问题
					upset["report"] = reason
					upset["state"] = 1 //反馈后爬虫改为待审核
				} else if state == Sp_state_5 { //上架，核对时间重置
					upset["l_checktime"] = 0
				} else if state == Sp_state_4 { //作废，作废原因
					upset["disablereason"] = reason
					upset["modifytime"] = time.Now().Unix()
				}
				upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
				qu.Debug("提交日志：", code, upset, upresult)
				if upresult && (state == Sp_state_2 || state == Sp_state_3) { //打回、审核记录日志
					types := "打回"
					if state == Sp_state_3 {
						types = "审核"
					}
					obj := map[string]interface{}{
						"code":       code,
						"auditor":    username,
						"types":      types,
						"comeintime": time.Now().Unix(),
						"reason":     reason,
						"spideruser": (*one)["createuser"],
						"modifytime": (*one)["modifytime"],
					}
					u.MgoE.Save("lua_logs_auditor", obj)
				}
			}
		}
	}
	return upresult, err
}

//保存记录信息
func SaveRemark(taskid []string, reason, username string) {
	timeNow := time.Now().Unix()
	if reason == "" {
		reason = "审核通过"
	}
	for _, id := range taskid {
		task, _ := u.MgoE.FindById("task", id, nil)
		if task != nil && len(*task) > 0 {
			checkData := (*task)["a_check"]
			var checkArr []map[string]interface{}
			newData := make(map[string]interface{})

			newData["s_check_checkUser"] = username
			newData["l_check_checkTime"] = timeNow
			newData["s_check_checkRemark"] = reason
			if checkData != nil {
				myArr := qu.ObjArrToMapArr(checkData.([]interface{}))
				if myArr != nil && len(myArr) > 0 {
					for _, v := range myArr {
						checkArr = append(checkArr, v)
					}
				}
			}
			checkArr = append(checkArr, newData)
			(*task)["a_check"] = checkArr
			u.MgoE.UpdateById("task", id, map[string]interface{}{"$set": &task})
		}
	}
}

//修改任务状态
func UpTaskState(code []string, num int, reason string, startTime int64) {
	query := map[string]interface{}{}
	update := map[string]interface{}{}
	for _, v := range code {
		if num == 1 || num == 2 || num == 3 || num == 7 { //id
			query = map[string]interface{}{
				"_id": mongodb.StringTOBsonId(v),
			}
		} else {
			query = map[string]interface{}{ //code
				"s_code": v,
			}
		}

		if num == 1 { //提交审核
			update = map[string]interface{}{
				"$set": map[string]interface{}{
					"i_state": 3,
				},
			}
		} else if num == 2 { //打回  -->未通过
			update = map[string]interface{}{
				"$set": map[string]interface{}{
					"i_state": 5,
				},
			}
		} else if num == 3 { //发布(审核通过)  -->审核通过
			update = map[string]interface{}{
				"$set": map[string]interface{}{
					"i_state": 4,
				},
			}
		} else if num == 4 { //批量作废  -->关闭
			update = map[string]interface{}{
				"$set": map[string]interface{}{
					"i_state":    6,
					"l_complete": time.Now().Unix(),
				},
			}
		} else if num == 7 { //反馈信息 -->待审核
			newData := map[string]interface{}{
				"l_mrecord_comeintime": startTime,
				"l_mrecord_complete":   time.Now().Unix(),
				"s_mrecord_remark":     reason,
			}
			mrecord := []interface{}{}
			mrecord = append(mrecord, newData)
			update = map[string]interface{}{
				"$set": map[string]interface{}{
					"i_state":    3,
					"l_complete": time.Now().Unix(),
					"a_mrecord":  mrecord,
				},
			}
		}
		flag := u.MgoE.Update("task", query, update, false, true)
		log.Println("codeOrId:", query, "	修改任务状态：", flag)
	}
}

//更新节点
func (f *Front) ChangeEvent() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.ServeJson("没有权限")
	}
	code := f.GetString("code")
	event, _ := f.GetInt("event")
	eventok := false
	for k, _ := range util.Config.Uploadevents {
		if event == qu.Int64All(k) {
			eventok = true
			break
		}
	}
	if !eventok {
		f.ServeJson("没有对应节点")
		return
	}
	info, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
	if len(*info) > 0 {
		oldevent := qu.IntAll((*info)["event"])
		if qu.IntAll((*info)["state"]) == Sp_state_5 {
			//源节点下架
			_, err := spider.UpdateSpiderByCodeState(code, fmt.Sprint(Sp_state_6), oldevent)
			set := map[string]interface{}{
				"$set": map[string]interface{}{
					"event": qu.IntAll(event),
					"state": Sp_state_6,
				},
			}
			u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
			if err != nil && strings.Contains(err.Error(), "timeout") {
				f.ServeJson("连接节点" + fmt.Sprint(oldevent) + "超时")
			} else {
				f.ServeJson(err.Error())
			}
		} else {
			set := map[string]interface{}{
				"$set": map[string]interface{}{
					"event": qu.IntAll(event),
				},
			}
			u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
		}
	} else {
		f.ServeJson("没有对应记录")
	}
}

//验证用户是否有更改状态权限
func IsHasUpState(auth, state int) bool {
	rep := false
	switch auth {
	case role_dev:
		if state == Sp_state_1 || state == Sp_state_7 {
			rep = true
		}
	case role_examine:
		if state == Sp_state_2 || state == Sp_state_3 {
			rep = true
		}
	case role_admin:
		rep = true
	default:
	}
	return rep
}

var list_fields = `{"_id":1,"code":1,"createuser":1,"modifyuser":1,"modifytime":1,"l_uploadtime":1,"l_checktime":1,"state":1,"param_common":1,"event":1,"urgency":1,"platform":1,"pendstate":1}`

//脚本管理，结合爬虫运行信息
func (f *Front) LuaList() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.ServeJson("没有权限！")
		return
	}
	if f.Method() == "POST" {
		state, _ := f.GetInteger("state")
		event, _ := f.GetInteger("event")
		start, _ := f.GetInteger("start")
		limit, _ := f.GetInteger("length")
		draw, _ := f.GetInteger("draw")
		searchStr := f.GetString("search[value]")
		//search := strings.Replace(searchStr, " ", "", -1)
		search := strings.TrimSpace(searchStr)
		platform := f.GetString("platform")
		query := map[string]interface{}{}

		q1 := map[string]interface{}{}
		q1["$or"] = []interface{}{
			map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
			map[string]interface{}{"createuser": map[string]interface{}{"$regex": search}},
			map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": search}},
		}
		q2 := map[string]interface{}{}
		if state > -1 {
			q2 = map[string]interface{}{"state": state}
		} else {
			q2["$or"] = []interface{}{
				map[string]interface{}{"state": Sp_state_3},
				map[string]interface{}{"state": Sp_state_5},
				map[string]interface{}{"state": Sp_state_6},
			}
		}
		q3 := map[string]interface{}{}
		if event > -1 {
			q3 = map[string]interface{}{"event": event}
		}
		q4 := map[string]interface{}{}
		if platform != "-1" {
			q4 = map[string]interface{}{"platform": platform}
		}
		if search != "" {
			query["$and"] = []interface{}{q1, q2, q3, q4}
		} else {
			query["$and"] = []interface{}{q2, q3, q4}
		}
		sort := `{"%s":%d}`
		orderIndex := f.GetString("order[0][column]")
		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
		orderType := 1
		if f.GetString("order[0][dir]") != "asc" {
			orderType = -1
		}
		sort = fmt.Sprintf(sort, orderName, orderType)
		page := start / 10
		qu.Debug("query:", query)
		luas, _ := u.MgoE.Find("luaconfig", query, sort, list_fields, false, start, limit)
		count := u.MgoE.Count("luaconfig", query)
		for k, v := range *luas {
			v["num"] = k + 1 + page*10
			l_uploadtime := qu.Int64All(v["l_uploadtime"])
			v["l_uploadtime"] = qu.FormatDateByInt64(&l_uploadtime, qu.Date_Full_Layout)
			l_checktime := qu.Int64All(v["l_checktime"])
			v["l_checktime"] = qu.FormatDateByInt64(&l_checktime, qu.Date_Full_Layout)
			if l_checktime > 0 { //核对
				v["is_check"] = true
			} else { //未核对
				v["is_check"] = false
			}
			if tmp, ok := spinfos.Load(v["code"]); ok {
				info := tmp.(*spinfo)
				v["modifytime"] = info.lastHeartbeat
				v["yesterday"] = fmt.Sprint(info.yesterdayDowncount) + "/" + fmt.Sprint(info.yestoDayRequestNum)
				v["terday"] = fmt.Sprint(info.todayDowncount) + "/" + fmt.Sprint(info.toDayRequestNum)
				v["lastdowncount"] = info.lastDowncount
				v["lstate"] = info.lstate
			} else {
				v["modifytime"] = ""
				v["yesterday"] = ""
				v["terday"] = ""
				v["lastdowncount"] = 0
				v["lstate"] = ""
			}
		}
		f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
	} else {
		events := []string{}
		for k, _ := range util.Config.Uploadevents {
			events = append(events, k)
		}
		sort.Strings(events)
		f.T["events"] = events
		f.Render("lualist.html", &f.T)
	}
}

//心跳监控
func (f *Front) Heart() {
	if f.Method() == "POST" {
		event, _ := f.GetInteger("event")
		start, _ := f.GetInteger("start")
		limit, _ := f.GetInteger("length")
		draw, _ := f.GetInteger("draw")
		searchStr := f.GetString("search[value]")
		search := strings.TrimSpace(searchStr) //只能搜索code
		//qu.Debug("search:", search, "start:", start, "limit:", limit, "draw:", draw)
		query := map[string]interface{}{
			"del": false,
		}
		if event > -1 {
			query["event"] = event
		}
		if search != "" {
			query["code"] = search
		}
		sort := `{"%s":%d}`
		orderIndex := f.GetString("order[0][column]")
		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
		orderType := 1
		if f.GetString("order[0][dir]") != "asc" {
			orderType = -1
		}
		sort = fmt.Sprintf(sort, orderName, orderType)
		qu.Debug("query:", query, "sort:", sort)
		list, _ := u.MgoS.Find("spider_heart", query, sort, nil, false, start, limit)
		count := u.MgoS.Count("spider_heart", query)
		for _, l := range *list {
			code := qu.ObjToString(l["code"])
			d, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1, "str_list": 1, "type_list": 1})
			l["state"] = (*d)["state"]
			l["param_common"] = (*d)["param_common"]
			if lt := qu.Int64All(l["list"]); lt != 0 {
				l["list"] = qu.FormatDateByInt64(&lt, qu.Date_Full_Layout)
			} else {
				l["list"] = 0
			}
			if dt := qu.Int64All(l["detail"]); dt != 0 {
				l["detail"] = qu.FormatDateByInt64(&dt, qu.Date_Full_Layout)
			} else {
				l["detail"] = 0
			}
			if det := qu.Int64All(l["detailexecute"]); det != 0 {
				l["detailexecute"] = qu.FormatDateByInt64(&det, qu.Date_Full_Layout)
			} else {
				l["detailexecute"] = 0
			}
			if ft := qu.Int64All(l["findlist"]); ft != 0 {
				l["findlist"] = qu.FormatDateByInt64(&ft, qu.Date_Full_Layout)
			} else {
				l["findlist"] = 0
			}
			ut := qu.Int64All(l["updatetime"])
			l["updatetime"] = qu.FormatDateByInt64(&ut, qu.Date_Full_Layout)
			//l["isfindlist"] = "否"
			//typeList := qu.IntAll((*d)["type_list"])
			//strList := qu.ObjToString((*d)["str_list"])
			//if typeList == 1 && strings.Contains(strList, "findListHtml") {
			//	l["isfindlist"] = "是"
			//}
		}

		f.ServeJson(map[string]interface{}{"draw": draw, "data": list, "recordsFiltered": count, "recordsTotal": count})
	} else {
		events := []string{}
		for k, _ := range util.Config.Uploadevents {
			events = append(events, k)
		}
		sort.Strings(events)
		f.T["events"] = events
		f.Render("heart.html", &f.T)
	}
}

//爬虫信息
type spinfo struct {
	code                                   string
	todayDowncount, toDayRequestNum        int
	yesterdayDowncount, yestoDayRequestNum int
	totalDowncount, totalRequestNum        int
	errorNum, roundCount, runRate          int
	lastDowncount                          int
	lastHeartbeat                          string
	lstate                                 string
}

//爬虫信息
func SpiderInfo(data string) {
	data = util.Se.DecodeString(data)
	infos := []map[string]interface{}{}
	err := json.Unmarshal([]byte(data), &infos)
	if err != nil {
		return
	}
	for _, tmp := range infos {
		lastHeartbeat := qu.Int64All(tmp["lastHeartbeat"])
		info := &spinfo{
			code:               fmt.Sprint(tmp["code"]),
			todayDowncount:     qu.IntAll(tmp["todayDowncount"]),
			toDayRequestNum:    qu.IntAll(tmp["toDayRequestNum"]),
			yesterdayDowncount: qu.IntAll(tmp["yesterdayDowncount"]),
			yestoDayRequestNum: qu.IntAll(tmp["yestoDayRequestNum"]),
			totalDowncount:     qu.IntAll(tmp["totalDowncount"]),
			totalRequestNum:    qu.IntAll(tmp["totalRequestNum"]),
			errorNum:           qu.IntAll(tmp["errorNum"]),
			roundCount:         qu.IntAll(tmp["roundCount"]),
			runRate:            qu.IntAll(tmp["runRate"]),
			lastHeartbeat:      qu.FormatDateByInt64(&lastHeartbeat, qu.Date_Full_Layout),
			lastDowncount:      qu.IntAll(tmp["lastDowncount"]),
			lstate:             fmt.Sprint(tmp["lstate"]),
		}
		spinfos.Store(info.code, info)
		//log.Println(info)
	}
}

//接受维护任务信息
func SpiderModifyTask(data string) {
	data = util.Se.DecodeString(data)
	mtasks := []map[string]interface{}{}
	err := json.Unmarshal([]byte(data), &mtasks)
	if err != nil {
		return
	}
	for k, tmp := range mtasks {
		log.Println(k, tmp)
	}
}

//查看是否有该任务
func checkTask(codes []string, num int) []string {
	//	var id string = ""
	query := map[string]interface{}{}
	var idArr []string
	if len(codes) > 0 {
		for _, v := range codes {
			if num == 1 {
				query = map[string]interface{}{
					"s_code": v,
					"i_state": map[string]interface{}{
						"$in": []int{1, 2, 5},
					},
				}
			} else if num == 2 { //打回时查询待审核的任务
				query = map[string]interface{}{
					"s_code":  v,
					"i_state": 3,
				}
			} else if num == 3 { //审核通过时查询待处理、处理中、待审核、未通过的任务
				query = map[string]interface{}{
					"s_code": v,
					"i_state": map[string]interface{}{
						"$in": []int{1, 2, 3, 5},
					},
				}
			} else if num == 7 {
				query = map[string]interface{}{
					"s_code": v,
					"i_state": map[string]interface{}{
						"$in": []int{2, 5},
					},
				}
			}
			task, _ := u.MgoE.Find("task", query, nil, nil, false, -1, -1)
			if task != nil {
				for _, t := range *task {
					idArr = append(idArr, mongodb.BsonIdToSId(t["_id"]))
				}
			}
			return idArr
		}
	}
	return idArr
}