spider_editor/src/front/front.go

// front
package front

import (
	"fmt"
	"github.com/go-xweb/httpsession"
	"github.com/go-xweb/xweb"
	"github.com/lauyoume/gopinyin"
	"github.com/tealeg/xlsx"
	"io/ioutil"
	"log"
	"mongodb"
	"os"
	qu "qfw/util"
	"regexp"
	"sort"
	"spider"
	util "spiderutil"
	"strconv"
	"strings"
	"sync"
	"time"
	u "util"
)

type Front struct {
	*xweb.Action
	login        xweb.Mapper `xweb:"/"`
	logout       xweb.Mapper `xweb:"/center/logout"`               //退出
	loadIndex    xweb.Mapper `xweb:"/center"`                      //控制中心
	spidernew    xweb.Mapper `xweb:"/center/spider"`               //爬虫新建
	reg          xweb.Mapper `xweb:"/center/reg"`                  //爬虫注册
	assign       xweb.Mapper `xweb:"/center/user/assign"`          //分配爬虫
	auditExport  xweb.Mapper `xweb:"/center/user/auditexport"`     //导出审核日志
	loadSpider   xweb.Mapper `xweb:"/center/spider/edit/(.*)"`     //爬虫加载
	viewSpider   xweb.Mapper `xweb:"/center/spider/view/(.*)"`     //爬虫查看
	downSpider   xweb.Mapper `xweb:"/center/spider/download/(.*)"` //爬虫下载
	upState      xweb.Mapper `xweb:"/center/spider/upstate"`       //爬虫状态更新
	assort       xweb.Mapper `xweb:"/center/spider/assort"`        //审核人员分类(无发布、需登录、无法处理、需删除)
	batchShelves xweb.Mapper `xweb:"/center/spider/batchShelves"`  //批量上下架
	checktime    xweb.Mapper `xweb:"/center/spider/checktime"`     //爬虫核对
	disables     xweb.Mapper `xweb:"/center/spider/disable"`       //批量作废
	changeEvent  xweb.Mapper `xweb:"/center/changeEvent"`          //节点更新
	getJson      xweb.Mapper `xweb:"/center/spider/json"`          //
	//delRedis        xweb.Mapper `xweb:"/center/spider/delRedis"`        //清理Redis
	updateESP       xweb.Mapper `xweb:"/center/spider/updateesp"`       //修改爬虫的节点/状态/平台
	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
	tagCode         xweb.Mapper `xweb:"/center/spider/tagcode"`         //标记爬虫
	spiderRemark    xweb.Mapper `xweb:"/center/spider/spiderremark"`    //标记爬虫

	spiderModel     xweb.Mapper `xweb:"/center/model"`            //获取补充模型
	runStep         xweb.Mapper `xweb:"/center/run"`              //方法测试
	spiderPass      xweb.Mapper `xweb:"/center/spider/pass"`      //整体测试
	runPinYin       xweb.Mapper `xweb:"/center/runpy"`            //获取拼音
	saveStep        xweb.Mapper `xweb:"/center/save"`             //爬虫保存
	saveJs          xweb.Mapper `xweb:"/center/save/js"`          //保存js
	loadModel       xweb.Mapper `xweb:"/center/gmodel/(.*)"`      //加载模型
	importdata      xweb.Mapper `xweb:"/center/importdata"`       //导入爬虫列表页面
	importLua       xweb.Mapper `xweb:"/center/importlua"`        //导入爬虫
	oldedit         xweb.Mapper `xweb:"/center/oldedit"`          //老文件编辑
	findName        xweb.Mapper `xweb:"/center/findname"`         //即时查询名称
	checkrepeat     xweb.Mapper `xweb:"/center/spider/isrepeat"`  //脚本代码判重
	heart           xweb.Mapper `xweb:"/center/heart"`            //心跳监控
	spiderCopy      xweb.Mapper `xweb:"/center/spider/copy"`      //补采复制爬虫
	spiderSplitCopy xweb.Mapper `xweb:"/center/spider/splitcopy"` //拆分复制爬虫
	spiderCloseErr  xweb.Mapper `xweb:"/center/spider/closeerr"`  //删除错误爬虫

	Base      Base
	OtherBase OtherBase
	Step1     Step1
	Step2     Step2
	Step3     Step3
	StepRe3   StepRe3
	U         U

	luaList       xweb.Mapper `xweb:"/center/lualist.html"`       //脚本管理
	user          xweb.Mapper `xweb:"/center/user.html"`          //用户管理
	delUser       xweb.Mapper `xweb:"/center/user/del"`           //删除用户
	updateUser    xweb.Mapper `xweb:"/center/user/updateUser"`    //修改用户信息
	checkUsenamer xweb.Mapper `xweb:"/center/user/checkUsenamer"` //校验用户名的唯一性
	checkEmail    xweb.Mapper `xweb:"/center/user/checkEmail"`    //校验邮箱的唯一性
	saveNewUser   xweb.Mapper `xweb:"/center/user/saveNewUser"`   //添加用户

	getCity xweb.Mapper `xweb:"/center/getCity"` //获取城市

	//补采信息
	supplementDayList  xweb.Mapper `xweb:"/center/supplement/daylist"`  //日补采列表
	supplementWeekList xweb.Mapper `xweb:"/center/supplement/weeklist"` //周补采列表

	//爬虫认领
	claimCode  xweb.Mapper `xweb:"/center/claim/claimcode"`  //爬虫认领
	returnCode xweb.Mapper `xweb:"/center/claim/returncode"` //爬虫归还
}

const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7无发布，8需登录，9转python，10已删除

var spinfos sync.Map = sync.Map{}
var SessMap map[string]*httpsession.Session
var AutoTpl map[string]interface{}

var Mails *util.Mail

// var Reg = regexp.MustCompile(`(http|https)://([\w]+\.)+[\w]+`)
var ProjectHrefReg = regexp.MustCompile("projecthref")
var Transfercode map[string]interface{}

// 爬虫列表
func (f *Front) LoadIndex() {
	auth := qu.IntAll(f.GetSession("auth"))
	if f.Method() == "POST" {
		start, _ := f.GetInteger("start")
		limit, _ := f.GetInteger("length")
		draw, _ := f.GetInteger("draw")
		searchStr := f.GetString("search[value]")
		event, _ := f.GetInteger("taskEvent") //节点
		search := strings.TrimSpace(searchStr)
		state, _ := f.GetInteger("state")
		platform := f.GetString("platform")
		modifyuser := f.GetString("modifyuser")
		infoformat, _ := f.GetInteger("infoformat") //招标类型
		priority := f.GetString("priority")         //优先级
		query := map[string]interface{}{}
		if event > -1 {
			query["event"] = event
		}
		if platform != "-1" {
			query["platform"] = platform
		}
		if modifyuser != "-1" {
			query["modifyuser"] = modifyuser
		}
		if priority != "-1" {
			priorityRange := strings.Split(priority, "-")
			gte := qu.IntAll(priorityRange[0])
			lt := qu.IntAll(priorityRange[1])
			q := map[string]interface{}{}
			if lt == 0 {
				q["$gte"] = gte
			} else {
				q = map[string]interface{}{
					"$gte": gte,
					"$lt":  lt,
				}
			}
			query["priority"] = q
		}
		if infoformat > -1 {
			query["infoformat"] = infoformat
		}
		if search != "" {
			query["$or"] = []interface{}{
				map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"createuser": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": search}},
			}
		}
		if auth == u.Role_Dev { //开发员
			if state > -1 {
				query["state"] = state
			}
			query["createuserid"] = f.GetSession("userid")
			query["claimtype"] = map[string]interface{}{ //爬虫开发只能搜索已认领和历史爬虫
				"$gte": CLAIMTYPECLAIMED,
			}
		} else {
			claimtype, _ := f.GetInteger("claimtype") //认领状态
			if claimtype > -1 {
				query["claimtype"] = claimtype
			}
			identity := qu.IntAll(f.GetSession("identity"))
			if identity == 0 { //外包管理员
				ids := []string{}
				if tmpIds := f.GetSession("ids"); tmpIds != nil {
					ids = qu.ObjArrToStringArr(tmpIds.([]interface{}))
				}
				query["createuserid"] = map[string]interface{}{
					"$in": ids,
				}
			}
			if state > -1 {
				query["state"] = state
			} else if auth == u.Role_Examine {
				query["state"] = Sp_state_1
			}
		}
		sort := `{"%s":%d}`
		orderIndex := f.GetString("order[0][column]")
		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
		orderType := 1
		if f.GetString("order[0][dir]") != "asc" {
			orderType = -1
		}
		if orderName == "param_common" {
			orderName = orderName + ".1"
		}
		sort = fmt.Sprintf(sort, orderName, orderType)
		page := start / 10
		//log.Println("sort", sort, orderName)
		qu.Debug("query:", query, "sort:", sort)
		//luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
		//count := mgdb.Count("luaconfig", query)
		luas, _ := u.MgoEB.Find("luaconfig", query, sort, map[string]interface{}{"str_list": 0, "str_content": 0}, false, start, limit)
		count := u.MgoEB.Count("luaconfig", query)
		for k, v := range *luas {
			v["num"] = k + 1 + page*10
			v["encode"] = util.Se.Encode2Hex(fmt.Sprint(v["code"]))
			if v["event"] == nil { //节点
				v["event"] = 0
			}
			user, _ := u.MgoEB.FindOne("user", map[string]interface{}{"s_name": qu.ObjToString(v["createuser"])})
			//v["state"] = LuaStateMap[qu.IntAll(v["state"])]
			v["i_scope"] = (*user)["i_scope"]
		}
		f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
	} else {
		events := []string{}
		for k, _ := range util.Config.Uploadevents {
			events = append(events, k)
		}
		identity := qu.IntAll(f.GetSession("identity"))
		ids := []string{}
		if identity == 0 && auth > u.Role_Dev { //外包管理员
			if tmpIds := f.GetSession("ids"); tmpIds != nil {
				ids = qu.ObjArrToStringArr(tmpIds.([]interface{}))
			}
		}
		f.T["identity"] = identity
		f.T["ids"] = ids
		f.T["modifyusers"] = u.GetModifyUsers()
		sort.Strings(events)
		f.T["events"] = events
		f.Render("index.html", &f.T)
	}
}

func (f *Front) Checkrepeat() {
	code := f.GetString("code")
	//one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
	one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
	if len(*one) > 0 {
		f.ServeJson("y")
	} else {
		f.ServeJson("n")
	}

}

// 新建
func (f *Front) Spidernew() error {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != u.Role_Admin {
		return nil
	}
	copy := f.GetString("copy")
	if copy != "" {
		//one := *mgdb.FindOne("luaconfig", bson.M{"code": copy})
		one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": copy})
		delete((*one), "_id")
		delete((*one), "code")
		base := (*one)["param_common"].([]interface{})
		base[0] = ""
		base[1] = ""
		(*one)["param_common"] = base
		f.T["lua"] = one
	}
	f.T["isflow"] = 1 //新建爬虫时，初始化isflow的值
	f.T["actiontext"] = "新建"
	f.T["restate"] = 4            //此处设置restate=4无意义，只为了页面不报错
	f.T["areas"] = u.Area         //
	f.T["citys"] = u.City         //
	f.T["provinces"] = u.Province //
	return f.Render("spideredit.html", &f.T)
}

// 得到模型
func (f *Front) SpiderModel() {
	f.ServeJson(util.Config.Model)
}

func (f *Front) RunPinYin() {
	word := f.GetString("word")
	str := gopinyin.Convert(word, true)
	f.Write(str)
}

type U struct {
	User string
	Name string
	Pwd  string
}

func (f *Front) Reg() {

}

func (f *Front) ImportLua() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != u.Role_Admin {
		f.ServeJson("没有权限")
		return
	}
	if f.Method() == "POST" {
		mf, _, err := f.GetFile("xlsx")
		errorinfo := map[string]interface{}{}
		if err == nil {
			binary, _ := ioutil.ReadAll(mf)
			xls, _ := xlsx.OpenBinary(binary)
			sheet := xls.Sheets[0]
			rows := sheet.Rows
			for k, v := range rows {
				if k != 0 {
					cells := v.Cells
					if cells[1].Value != "" {
						code := cells[1].Value
						code = u.SymbolReg.ReplaceAllString(code, "")
						query := map[string]interface{}{"code": cells[1].Value}
						rs, _ := u.MgoEB.FindOne("import", query)
						if len(*rs) > 0 {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫代码重复,请修改"
							continue
						}
						o := make(map[string]interface{})
						o["name"] = u.SymbolReg.ReplaceAllString(cells[0].Value, "")
						o["code"] = u.SymbolReg.ReplaceAllString(code, "")
						o["channel"] = cells[2].Value
						spiderremark := cells[3].Value
						if spiderremark == "" {

							spiderremark = `采集“` + cells[2].Value + `”栏目（含子栏目）`
						}
						o["spiderremark"] = spiderremark
						//重复域名的网站不再新增爬虫
						href := cells[4].Value
						one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"href": href})
						if len(*one) > 0 {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",该栏目已存在,请确认"
							//continue
						}
						o["channeladdr"] = href
						o["timestamp"] = time.Now().Unix()
						o["modifyuser"] = cells[5].Value
						o["event"] = cells[6].Value
						o["incrementevent"] = cells[7].Value
						if cells[8].Value == "是" {
							o["isflow"] = 1
						} else {
							o["isflow"] = 0
						}
						o["priority"], _ = cells[9].Int()
						o["platform"] = cells[10].Value
						o["area"] = cells[11].Value
						o["city"] = cells[12].Value
						o["district"] = cells[13].Value
						weigh, _ := cells[14].Int()
						if weigh == -1 {
							weigh = 1
						}
						o["weight"] = weigh
						//存储表
						coll := cells[15].Value
						if coll == "" {
							coll = "bidding"
						}
						//爬虫类型
						infoformat, _ := cells[16].Int()
						if infoformat < 1 {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",缺少爬虫类型信息"
							continue
						}
						o["infoformat"] = infoformat

						if strings.Contains(code, "bidding") {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行爬虫" + cells[1].Value + ",存储表错误"
							continue
						}
						o["coll"] = coll
						//table := cells[6].Value
						//o["table"] = table
						//o["transfercode"] = qu.IntAll(Transfercode[table])
						ok := saveLua(o) //保存爬虫
						if ok == false {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
						} else {
							o["status"] = 1
							o["author"] = o["modifyuser"]
							o["importuser"] = f.GetSession("username")
							u.MgoEB.Save("import", o)
						}
					}
				}
			}
			f.ServeJson(errorinfo)
		} else {
			f.ServeJson(false)
		}
	}
}

func saveLua(o map[string]interface{}) bool {
	param := map[string]interface{}{}
	AutoTpl["Base.SpiderName"] = o["name"]
	AutoTpl["Base.SpiderCode"] = o["code"]
	AutoTpl["Base.SpiderChannel"] = o["channel"]
	AutoTpl["Base.SpiderTargetChannelUrl"] = o["channeladdr"]
	modifyuser := o["modifyuser"].(string)
	priority := qu.IntAll(o["priority"])
	param["priority"] = o["priority"] //优先级
	platform := qu.ObjToString(o["platform"])
	var userid, email string
	claimLog := map[string]interface{}{}
	if modifyuser != "" { //指定维护人
		one, _ := u.MgoEB.FindOne("user", map[string]interface{}{"s_name": modifyuser, "i_auth": 1, "i_delete": 0})
		if len(*one) == 0 {
			return false
		}
		email = qu.ObjToString((*one)["s_email"])
		userid = mongodb.BsonIdToSId((*one)["_id"])
		if platform != "python" {
			now := time.Now().Unix()
			recovertime := now + 365*86400
			param["claimtime"] = now //认领时间
			param["claimtype"] = CLAIMTYPECLAIMED
			param["recovertime"] = recovertime
			claimLog = map[string]interface{}{
				"site":             o["name"],
				"code":             o["code"],
				"channel":          o["channel"],
				"modifyuser":       modifyuser,
				"priority":         priority,
				"stype":            "认领",
				"comeintime":       now,
				"claimtime":        now,
				"recovertime":      recovertime,
				"returntime":       int64(0),
				"important":        false,
				"returnreason":     "",
				"claimrecovertype": 0,
			}
		} else {
			param["recovertime"] = int64(0)       //回收时间
			param["claimtime"] = int64(0)         //认领时间
			param["claimtype"] = CLAIMTYPEHISTORY //爬虫认领状态（历史爬虫）
		}
	} else { //未指定人
		param["recovertime"] = int64(0)         //回收时间
		param["claimtime"] = int64(0)           //认领时间
		param["claimtype"] = CLAIMTYPEUNCLAIMED //爬虫认领状态（未认领）
	}
	common := []interface{}{
		AutoTpl["Base.SpiderCode"],
		AutoTpl["Base.SpiderName"],
		AutoTpl["Base.SpiderChannel"],
		AutoTpl["Base.SpiderDownDetailPage"],
		AutoTpl["Base.SpiderStartPage"],
		AutoTpl["Base.SpiderMaxPage"],
		AutoTpl["Base.SpiderRunRate"],
		//AutoTpl["Base.Spider2Collection"],
		//"bidding", //爬虫导入新建默认为bidding
		qu.ObjToString(o["coll"]),
		AutoTpl["Base.SpiderPageEncoding"],
		AutoTpl["Base.SpiderStoreMode"],
		AutoTpl["Base.SpiderStoreToMsgEvent"],
		AutoTpl["Base.SpiderTargetChannelUrl"],
		AutoTpl["Base.SpiderLastDownloadTime"],
		AutoTpl["Base.SpiderIsHistoricalMend"],
		AutoTpl["Base.SpiderIsMustDownload"],
	}
	ptime := []interface{}{
		AutoTpl["Step1.DateFormat"],
		AutoTpl["Step1.Address"],
		AutoTpl["Step1.ContentChooser"],
	}
	list := []interface{}{
		AutoTpl["Step2.Listadd"],
		AutoTpl["Step2.Listadds"],
		AutoTpl["Step2.BlockChooser"],
		AutoTpl["Step2.AddressChooser"],
		AutoTpl["Step2.TitleChooser"],
		AutoTpl["Step2.DateChooser"],
		AutoTpl["Step2.DateFormat"],
	}
	content := []interface{}{
		AutoTpl["Step3.ContentChooser"],
		AutoTpl["Step3.ElementChooser"],
	}
	param["param_common"] = common
	//向导模式
	param["param_time"] = ptime
	param["param_list"] = list
	param["param_content"] = content
	param["type_time"] = 0
	param["type_list"] = 0
	param["type_content"] = 0
	//专家模式
	param["str_time"] = ""
	param["str_list"] = ""
	param["str_content"] = ""
	param["comeintime"] = time.Now().Unix()
	param["code"] = o["code"]
	param["site"] = o["name"]
	param["href"] = o["channeladdr"]
	param["channel"] = o["channel"]
	param["createuser"] = modifyuser
	param["createuserid"] = userid
	param["createuseremail"] = email
	param["modifyuser"] = modifyuser
	param["modifyuserid"] = userid
	param["modifytime"] = time.Now().Unix()
	param["state"] = 0 //未完成
	if qu.IntAll(o["event"]) > 0 {
		param["event"] = qu.IntAll(o["event"])
	}
	s_model := "bid"
	configModel := util.Config.Model[s_model]
	model := map[string]interface{}{}
	for k, _ := range configModel {
		model[k] = qu.ObjToString(o[k])
	}
	param["model"] = model
	param["next"] = email
	param["urgency"] = o["urgency"]
	param["isflow"] = o["isflow"]
	param["spidertype"] = "history"
	param["spiderremark"] = o["spiderremark"]
	incrementevent := qu.ObjToString(o["incrementevent"])
	if movevent, ok := util.Config.Uploadevents[incrementevent].(string); ok && movevent != "" {
		param["spidermovevent"] = movevent
	}
	//} else {
	//	param["spidermovevent"] = "7700"
	//}
	param["incrementevent"] = qu.IntAll(o["incrementevent"])
	param["platform"] = platform
	param["weight"] = o["weight"]
	param["infoformat"] = o["infoformat"]
	infoformat := qu.IntAll(o["infoformat"])
	infotype := "招标"
	if infoformat == 2 {
		infotype = "拟建/审批"
	} else if infoformat == 3 {
		infotype = "产权"
	} else if infoformat == 4 {
		infotype = "舆情"
	}
	//默认字段
	param["spidercompete"] = true     //2021-11-20后爬虫加此字段（表示新爬虫，剑鱼网站不展示原文）
	param["spiderhistorymaxpage"] = 1 //历史最大页
	param["pendstate"] = 0            //
	param["grade"] = 0                //爬虫难易度（主要用于python爬虫使用）
	param["spiderimportant"] = false  //是否为重点网站爬虫
	param["urgency"] = 0
	//qu.Debug("param---", param)

	ok := spider.SaveSpider(o["code"].(string), param)
	if ok { //保存成功，校验新导入的爬虫对应站点是否存在，否则加站点记录
		site, _ := u.MgoEB.FindOneByField("site", map[string]interface{}{"site": o["name"]}, map[string]interface{}{"important": 1})
		if len(*site) == 0 {
			qu.Debug("补充站点信息：", o["name"])
			domain := u.DomainReg.FindString(qu.ObjToString(AutoTpl["Base.SpiderTargetChannelUrl"]))
			if domain != "" {
				domain = u.ReplaceReg.ReplaceAllString(domain, "")
			}
			siteInfo := map[string]interface{}{
				"site":           o["name"],
				"domain":         domain, //
				"another_name":   "",
				"area":           qu.ObjToString(model["area"]),
				"city":           qu.ObjToString(model["city"]),
				"district":       qu.ObjToString(model["district"]),
				"site_type":      "",
				"second_type":    "",
				"industry":       "",
				"p_site":         "",
				"s_site":         "",
				"remarktime":     time.Now().Unix(),
				"event":          incrementevent,
				"platform":       o["platform"],
				"spider_status":  "0/1",
				"updatetime":     time.Now().Unix(),
				"delete":         false,
				"comeintime":     time.Now().Unix(),
				"important":      0,
				"site_status":    1,
				"lasttime":       int64(0),
				"site_datanum":   0,
				"period":         float32(0),
				"infotype":       infotype,
				"sponsor":        "",
				"isneedregister": 0,
				"isregistered":   0,
				"special_type":   "",
				"account":        "",
				"password":       "",
				"f_area":         "",
				"f_city":         "",
				"f_district":     "",
				"site_subtype":   "",
				"site_toptype":   "",
				"type_plate":     "",
			}
			u.MgoEB.Save("site", siteInfo)
		} else if qu.IntAll((*site)["important"]) == 1 { //重点网站
			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{"spiderimportant": true}}, false, false)
			u.MgoEB.Update("site_code_baseinfo", map[string]interface{}{"spidercode": o["code"]}, map[string]interface{}{"$set": map[string]interface{}{
				"site":       o["name"],
				"channel":    o["channel"],
				"spidercode": o["code"],
				"platform":   o["platform"],
				"modifyuser": "",
				"state":      0,
			}}, true, false)
		}
		//生成认领日志
		if len(claimLog) > 0 {
			u.MgoEB.Save("lua_logs_claim", claimLog)
		}
	}
	return ok
}

func (f *Front) Importdata() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth == u.Role_Admin {
		if f.Method() == "GET" {
			f.Render("import.html")
		} else {
			query := map[string]interface{}{}
			searchStr := f.GetString("search[value]")
			search := strings.TrimSpace(searchStr)
			if search != "" {
				query["$or"] = []interface{}{
					map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
					map[string]interface{}{"name": map[string]interface{}{"$regex": search}},
				}
			}
			start, _ := f.GetInteger("start")
			limit, _ := f.GetInteger("length")
			draw, _ := f.GetInteger("draw")
			data, _ := u.MgoEB.Find("import", query, `{"timestamp": -1}`, nil, false, start, limit)
			count := u.MgoEB.Count("import", query)
			f.ServeJson(map[string]interface{}{
				"draw":            draw,
				"data":            data,
				"recordsFiltered": count,
				"recordsTotal":    count,
			})
		}
	} else {
		f.Write("您没有导入脚本的权限")
	}
}

func Wlog(name, code, man, manid, types string, content map[string]interface{}) {
	obj := map[string]interface{}{
		"name":    name,
		"code":    code,
		"man":     man,
		"manid":   manid,
		"types":   types,
		"time":    time.Now().Unix(),
		"content": content,
	}
	u.MgoEB.Save("lua_logs", obj)
}

func (f *Front) Oldedit() {
	if f.Method() == "GET" {
		f.Render("oldedit.html")
	} else {

	}
}

func (f *Front) FindName() {
	words := f.GetString("words")
	if words == "" {
		f.ServeJson(map[string]interface{}{"error": "null"})
	}
	//query := bson.M{"$or": []interface{}{
	//	bson.M{"param_common.0": bson.M{"$regex": words}},
	//	bson.M{"param_common.1": bson.M{"$regex": words}},
	//	bson.M{"createuser": bson.M{"$regex": words}},
	//}, "oldlua": bson.M{"$exists": false}}
	query := map[string]interface{}{
		"$or": []interface{}{
			map[string]interface{}{"param_common.0": map[string]interface{}{"$regex": words}},
			map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": words}},
			map[string]interface{}{"createuser": map[string]interface{}{"$regex": words}},
		},
		"oldlua": map[string]interface{}{
			"$exists": false,
		},
	}
	//rs := *mgdb.Find("luaconfig", query, bson.M{"modifytime": -1}, bson.M{"param_common": 1}, false, -1, -1)
	rs, _ := u.MgoEB.Find("luaconfig", query, map[string]interface{}{"modifytime": -1}, map[string]interface{}{"param_common": 1}, false, -1, -1)
	if len(*rs) > 0 {
		f.ServeJson(map[string]interface{}{"data": (*rs)})
	} else {
		f.ServeJson(map[string]interface{}{"error": "data"})
	}
}

// 分配爬虫
func (f *Front) Assign() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != u.Role_Admin {
		f.Write("n")
		return
	}
	codes := f.GetString("codes")
	userid := f.GetString("userid")       //分配给谁
	codesarr := strings.Split(codes, ",") //被分配的爬虫
	user, _ := u.MgoEB.FindById("user", userid, nil)
	if user != nil && len(*user) > 0 {
		platform := (*user)["s_platform"]
		name := qu.ObjToString((*user)["s_name"])
		query := map[string]interface{}{
			"code": map[string]interface{}{
				"$in": codesarr,
			},
		}
		luas, _ := u.MgoEB.Find("luaconfig", query, nil, ClaimQueryFields, false, -1, -1)
		update := [][]map[string]interface{}{}
		save := []map[string]interface{}{}
		for _, l := range *luas {
			set := map[string]interface{}{}
			now := time.Now().Unix()
			claimtype := qu.IntAll(l["claimtype"])
			priority := qu.IntAll(l["priority"])
			spiderimportant, _ := l["spiderimportant"].(bool)

			if claimtype <= CLAIMTYPECLAIMED { //非历史爬虫分配
				recovertime := CreateRecovertime(spiderimportant, priority)
				set = map[string]interface{}{
					"claimtype":       CLAIMTYPECLAIMED,
					"claimtime":       now,
					"recovertime":     recovertime,
					"createuseremail": (*user)["s_email"],
					"createuser":      name,
					"createuserid":    userid,
					"modifyuser":      name,
					"modifyuserid":    userid,
					"platform":        platform,
				}
				if claimtype == CLAIMTYPECLAIMED { //已认领的爬虫增加回收日志
					//回收日志
					recovelog := map[string]interface{}{
						"site":             l["site"],
						"code":             l["code"],
						"channel":          l["channel"],
						"modifyuser":       l["modifyuser"],
						"priority":         priority,
						"stype":            "回收",
						"comeintime":       now,
						"claimtime":        l["claimtime"],
						"recovertime":      l["recovertime"],
						"returntime":       int64(0),
						"important":        spiderimportant,
						"returnreason":     "",
						"claimrecovertype": 0,
					}
					save = append(save, recovelog)
				}
				//被重新分配的爬虫视为重新认领（注意：lua爬虫分配到除chrome外的平台，变为历史爬虫）
				if platform == "golua平台" { //爬虫分配到非lua平台，不视为认领
					//认领日志
					claimlog := map[string]interface{}{
						"site":             l["site"],
						"code":             l["code"],
						"channel":          l["channel"],
						"modifyuser":       name,
						"priority":         priority,
						"stype":            "认领",
						"comeintime":       now,
						"claimtime":        now,
						"recovertime":      recovertime,
						"returntime":       int64(0),
						"important":        spiderimportant,
						"returnreason":     "",
						"claimrecovertype": 0,
					}
					save = append(save, claimlog)
				} else if platform != "golua平台" && platform != "chrome" { //分配到其他平台改为历史爬虫
					set["claimtype"] = CLAIMTYPEHISTORY
					set["claimtime"] = 0
					set["recovertime"] = 0
				}
			} else { //历史爬虫
				set = map[string]interface{}{
					"createuserid":    userid,
					"createuser":      name,
					"createuseremail": (*user)["s_email"],
					"modifyuser":      name,
					"modifyuserid":    userid,
					"platform":        (*user)["s_platform"],
				}
			}
			//更新爬虫
			up := []map[string]interface{}{
				{"code": l["code"]},
				{"$set": set},
			}
			update = append(update, up)
		}
		qu.Debug(query)
		//更新爬虫信息
		b := u.MgoEB.UpdateBulk("luaconfig", update...)
		//保存认领日志
		if len(save) > 0 {
			u.MgoEB.SaveBulk("lua_logs_claim", save...)
		}
		if b {
			go editModify(codesarr, userid, name) //修改爬虫对应任务的维护人
			f.Write("y")
		} else {
			f.Write("n")
		}
	} else {
		f.Write("null")
	}
}

// 审核日志导出
func (f *Front) AuditExport() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth > u.Role_Admin {
		exportnames := f.GetString("exportnames")
		exporttype := f.GetString("exporttype")
		starttime := f.GetString("starttime")
		endtime := f.GetString("endtime")
		qu.Debug(exportnames, exporttype, starttime, endtime)
		file, err := xlsx.OpenFile("爬虫审核记录.xlsx")
		if err != nil {
			f.Write("文件打开失败！")
			return
		}
		sheet := file.Sheets[0]
		exportLogs(sheet, exportnames, exporttype, starttime, endtime) //导出数据
		fname := fmt.Sprintf("爬虫审核打回记录%d.xlsx", time.Now().Unix())
		err = file.Save(fname)
		if err != nil {
			qu.Debug("Save Excel" + fname + "Error")
			f.ServeJson("导出失败")
			return
		}
		f.ResponseWriter.Header().Add("Content-Disposition", "attachment;filename=爬虫审核打回记录.xlsx")
		f.ServeFile(fname)
		go func(path string) { //删除
			time.Sleep(time.Second * 30)
			os.Remove(path)
		}(fname)
	} else {
		f.Write("您没有权限")
	}
}
func exportLogs(sheet *xlsx.Sheet, exportnames, exporttype, starttime, endtime string) {
	st, _ := time.ParseInLocation(qu.Date_Short_Layout, starttime, time.Local)
	et, _ := time.ParseInLocation(qu.Date_Short_Layout, endtime, time.Local)
	query := map[string]interface{}{
		"comeintime": map[string]interface{}{
			"$gte": st.Unix(),
			"$lt":  et.Unix(),
		},
	}
	if exporttype == "repulse" { //打回
		query["types"] = "打回"
	} else if exporttype == "audit" { //审核
		query["types"] = "审核"
	}
	users := map[string]bool{}
	for _, user := range strings.Split(exportnames, ",") {
		users[user] = true
	}
	qu.Debug("audit log query:", query)
	sess := u.MgoEB.GetMgoConn()
	defer u.MgoEB.DestoryMongoConn(sess)
	lock := &sync.Mutex{}
	wg := &sync.WaitGroup{}
	ch := make(chan bool, 10)
	n := 0
	it := sess.DB(u.MgoEB.DbName).C("lua_logs_auditor").Find(&query).Iter()
	count, _ := sess.DB(u.MgoEB.DbName).C("lua_logs_auditor").Find(&query).Count()
	qu.Debug("count:", count)
	for tmp := map[string]interface{}{}; it.Next(tmp); n++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			id := mongodb.BsonIdToSId(tmp["_id"])
			auditor := qu.ObjToString(tmp["auditor"])
			code := qu.ObjToString(tmp["code"])
			comeintime := qu.Int64All(tmp["comeintime"])
			comeintime_z := qu.FormatDateByInt64(&comeintime, qu.Date_Full_Layout)
			event := qu.IntAll(tmp["event"])
			modifytime := qu.Int64All(tmp["modifytime"])
			modifytime_z := qu.FormatDateByInt64(&modifytime, qu.Date_Full_Layout)
			reason := qu.ObjToString(tmp["reason"])
			types := qu.ObjToString(tmp["types"])
			spideruser := qu.ObjToString(tmp["spideruser"])
			if !users[spideruser] {
				return
			}
			lock.Lock()
			row := sheet.AddRow()
			lock.Unlock()
			row.AddCell().SetValue(id)
			row.AddCell().SetValue(auditor)
			row.AddCell().SetValue(code)
			row.AddCell().SetValue(comeintime)
			row.AddCell().SetValue(comeintime_z)
			row.AddCell().SetValue(event)
			row.AddCell().SetValue(modifytime)
			row.AddCell().SetValue(modifytime_z)
			row.AddCell().SetValue(reason)
			row.AddCell().SetValue(spideruser)
			row.AddCell().SetValue(types)
		}(tmp)
		tmp = make(map[string]interface{})
	}
}

// 修改维护人
func editModify(codesarr []string, userid, name string) {
	qu.Debug("修改爬虫对应任务的维护人：", name, codesarr)
	//修改modifyid和modify
	query := map[string]interface{}{
		"s_code": map[string]interface{}{
			"$in": codesarr,
		},
		"i_state": map[string]interface{}{
			"$nin": []int{4, 6},
		},
	}
	set := map[string]interface{}{
		"$set": map[string]interface{}{
			"s_modify":   name,
			"s_modifyid": userid,
		},
	}
	b := u.MgoEB.Update("task", query, set, false, true)
	qu.Debug("任务重新分配成功", b)
}

func (f *Front) UpdatePendState() {
	success := false
	pendstate, _ := f.GetInteger("pendstate")
	code := f.GetString("code")
	stype := f.GetString("stype")
	qu.Debug(code, stype, pendstate)
	//success = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
	success = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate, "pendtime": time.Now().Unix()}}, false, false)
	if success {
		if stype == "code" { //爬虫挂起时，关闭所有任务
			u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$ne": 6}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6, "l_pendtime": time.Now().Unix()}}, false, true)
		} else if stype == "task" { //任务挂起任务时任务关闭
			id := f.GetString("id")
			u.MgoEB.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_pendstate": pendstate, "i_state": 6, "l_pendtime": time.Now().Unix()}})
		}
	}
	f.ServeJson(map[string]interface{}{"success": success})
}

func (f *Front) UpdateESP() {
	val := f.GetString("val")
	w := f.GetString("w")
	id := f.GetString("id")
	code := f.GetString("c")
	query := map[string]interface{}{
		"_id": mongodb.StringTOBsonId(id),
	}
	set := map[string]interface{}{}
	update := map[string]interface{}{
		"$set": set,
	}
	//one, _ := u.MgoE.FindById("luaconfig", id, nil)
	one, _ := u.MgoEB.FindById("luaconfig", id, nil)
	if len(*one) == 0 {
		f.Write("n")
		return
	}
	if w == "infoformat" {
		infoformat, _ := f.GetInteger("val")
		set["infoformat"] = infoformat
	} else if w == "urgency" { //修改紧急度
		urgency, _ := f.GetInteger("val")
		set["urgency"] = urgency
	} else if w == "state" { //无效爬虫改为待完成
		tmpEvent := qu.IntAll((*one)["event"])
		if one != nil && len(*one) > 0 {
			if (*one)["incrementevent"] == nil && tmpEvent != 7000 { //除7000节点外没有incrementevent的要重新设置
				set["incrementevent"] = tmpEvent
			}
		}
		set["state"] = 0
		set["event"] = 7000
		set["spidertype"] = "history"
		go ModifyLogs_UpdateCodeState(code)
	} else if w == "platform" {
		//常规字段更新
		set["platform"] = val
		set["comeintime"] = time.Now().Unix()
		if val != "golua平台" && val != "chrome" {
			b := u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
			qu.Debug("Del Heart:", b)
			b, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*one)["event"])) //下架
			if b && err == nil {
				//历史节点下架为了避免线上运行爬虫待完成时改为7000采集历史，但是又转到其他平台，导致原线上运行节点爬虫并未下线，心跳异常
				if incrementevent := (*one)["incrementevent"]; incrementevent != nil {
					b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll(incrementevent))
				}
				//关闭任务
				u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
				if b && err == nil {
					set["state"] = 0 //更新状态
					set["luauser"] = map[string]interface{}{
						"createuser":      (*one)["createuser"],
						"createuserid":    (*one)["createuserid"],
						"createuseremail": (*one)["createuseremail"],
						"modifyuser":      (*one)["modifyuser"],
						"modifyuserid":    (*one)["modifyuserid"],
					}
				} else {
					qu.Debug("历史节点下架失败")
					f.Write("n")
					return
				}
			} else {
				qu.Debug("增量节点下架失败")
				f.Write("n")
				return
			}
			//lua转python，爬虫认领状态更新
			UpdateLuaClaimtype(code)
			//换平台爬虫回收，保存日志
			recovelog := map[string]interface{}{
				"site":             (*one)["site"],
				"code":             (*one)["code"],
				"channel":          (*one)["channel"],
				"modifyuser":       (*one)["modifyuser"],
				"priority":         (*one)["priority"],
				"stype":            "回收",
				"comeintime":       time.Now().Unix(),
				"claimtime":        (*one)["claimtime"],
				"recovertime":      (*one)["recovertime"],
				"returntime":       int64(0),
				"important":        (*one)["spiderimportant"],
				"returnreason":     "转平台",
				"claimrecovertype": 0,
			}
			u.MgoEB.Save("lua_logs_claim", recovelog)
		}
	} else if w == "priority" { //调整优先级
		priority := qu.IntAll(val)
		if priority < 0 {
			priority = 0
		} else if priority > 1000 {
			priority = 1000
		}
		set["priority"] = priority
	} else { //修改节点
		event, _ := strconv.Atoi(val)
		set["event"] = event
		set["incrementevent"] = event
		set["state"] = 6
		b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
		qu.Debug("爬虫下架成功：", b)
		if !b || err != nil {
			f.Write("n")
			return
		}
	}
	if u.MgoEB.Update("luaconfig", query, update, false, false) {
		log.Println("Id:", id, "	Update", w, val, "Success")
		f.Write("y")
		return
	} else {
		log.Println("Id:", id, "	Update", w, val, "Failed")
		f.Write("n")
		return
	}
	f.Write("n")
}

func (f *Front) GetCity() {
	area := f.GetString("area")
	cityArr := []string{}
	cityArr = u.Province[area]
	f.ServeJson(cityArr)
}

func (f *Front) SpiderCopy() {
	defer qu.Catch()
	code := f.GetString("code")
	code = strings.TrimSpace(code)
	success := false
	text := ""
	encode := ""
	if code != "" {
		//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
		if len(*lua) > 0 {
			codeOld := qu.ObjToString((*lua)["code"])
			codeNew := codeOld + u.Bu
			//luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
			luaTmp, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
			if len(*luaTmp) > 0 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "已存在补漏爬虫"})
				return
			}
			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
			(*lua)["code"] = codeNew
			(*lua)["relatecode"] = codeOld
			if param_common := (*lua)["param_common"].([]interface{}); len(param_common) < 13 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "脚本异常，请联系管理员"})
				return
			} else {
				param_common = param_common[:13]
				param_common[0] = codeNew                  //code
				param_common = append(param_common, true)  //历史补漏
				param_common = append(param_common, false) //强制下载
				(*lua)["param_common"] = param_common
			}
			str_list := qu.ObjToString((*lua)["str_list"])
			(*lua)["str_list"] = strings.ReplaceAll(str_list, codeOld, codeNew)
			(*lua)["comeintime"] = time.Now().Unix()
			(*lua)["modifytime"] = time.Now().Unix()
			(*lua)["spidertype"] = "increment"
			(*lua)["event"] = 7000
			(*lua)["state"] = 6
			id := u.MgoEB.Save("luaconfig", (*lua))
			if id != "" {
				success = true
				encode = util.Se.Encode2Hex(codeNew)
			} else {
				text = "新爬虫保存失败"
			}
		} else {
			text = "未查询到复制原"
		}
	} else {
		text = "脚本不能为空"
	}
	f.ServeJson(map[string]interface{}{"success": success, "text": text, "encode": encode})
}

func (f *Front) SpiderSplitCopy() {
	defer qu.Catch()
	code := f.GetString("code")
	code = strings.TrimSpace(code)
	num, _ := f.GetInteger("num")
	text := ""
	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
	if len(*lua) > 0 {
		param_common := (*lua)["param_common"].([]interface{})
		str_list := qu.ObjToString((*lua)["str_list"])
		channelOld := qu.ObjToString((*lua)["channel"])
		copyNum := qu.IntAll((*lua)["copynum"]) //当前已经拆分了多少个爬虫
		delete(*lua, "relatecode")              //删除关联爬虫字段
		for i := 1; i <= num; i++ {
			copyNum++
			var codeSuffix string
			if copyNum < 10 {
				codeSuffix = fmt.Sprintf("0%s", fmt.Sprint(copyNum))

			} else {
				codeSuffix = fmt.Sprint(copyNum)
			}
			(*lua)["code"] = code + "_" + codeSuffix
			(*lua)["channel"] = channelOld + "_" + codeSuffix
			(*lua)["str_list"] = strings.ReplaceAll(str_list, code, code+"_"+codeSuffix)
			(*lua)["comeintime"] = time.Now().Unix()
			(*lua)["modifytime"] = time.Now().Unix()
			(*lua)["state"] = 0
			param_common[0] = code + "_" + codeSuffix
			param_common[2] = channelOld + "_" + codeSuffix
			(*lua)["param_common"] = param_common
			u.MgoEB.Save("luaconfig", (*lua))
		}
		u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"copynum": copyNum}}, false, false)
	} else {
		text = "未查询到复制原"
	}
	f.ServeJson(map[string]interface{}{"text": text})
}

/*// 清理Redis
func (f *Front) DelRedis() {
	hrefs := f.GetString("href")
	hrefsarr := strings.Split(hrefs, ",")
	auth := qu.IntAll(f.GetSession("auth"))
	err := []string{}
	if auth == u.Role_Admin { //权限控制
		if len(hrefsarr) > 0 {
			for k1, h := range hrefsarr {
				href := Reg.FindString(h)
				if href != "" {
					href = "url_repeat_" + href + "*"
					res := redis.GetKeysByPattern("title_repeat_judgement", href)
					if res != nil {
						for _, v := range res {
							hf := string(v.([]uint8))
							b := redis.Del("title_repeat_judgement", hf)
							if !b {
								err = append(err, "第"+strconv.Itoa(k1+1)+"个")
							}
						}
					}
				} else {
					err = append(err, "第"+strconv.Itoa(k1+1)+"个")
				}
			}
		}
	} else {
		err = append(err, "没有权限")
	}
	f.ServeJson(err)
}*/