spider_editor/src/front/front.go

// front
package front

import (
	"fmt"
	"io/ioutil"
	"log"
	qu "qfw/util"
	mgdb "qfw/util/mongodb"
	mgu "qfw/util/mongodbutil"
	"qfw/util/redis"
	"regexp"
	"sort"
	"spider"
	util "spiderutil"
	"strconv"
	"strings"
	"sync"
	"time"
	u "util"

	"github.com/go-xweb/httpsession"
	"github.com/go-xweb/xweb"
	"github.com/lauyoume/gopinyin"
	"github.com/tealeg/xlsx"
	"gopkg.in/mgo.v2/bson"
)

type Front struct {
	*xweb.Action
	login           xweb.Mapper `xweb:"/"`
	logout          xweb.Mapper `xweb:"/center/logout"`                 //退出
	loadIndex       xweb.Mapper `xweb:"/center"`                        //控制中心
	spidernew       xweb.Mapper `xweb:"/center/spider"`                 //爬虫新建
	reg             xweb.Mapper `xweb:"/center/reg"`                    //爬虫注册
	assign          xweb.Mapper `xweb:"/center/user/assign"`            //分配爬虫
	loadSpider      xweb.Mapper `xweb:"/center/spider/edit/(.*)"`       //爬虫加载
	viewSpider      xweb.Mapper `xweb:"/center/spider/view/(.*)"`       //爬虫查看
	downSpider      xweb.Mapper `xweb:"/center/spider/download/(.*)"`   //爬虫下载
	upState         xweb.Mapper `xweb:"/center/spider/upstate"`         //爬虫状态更新
	assort          xweb.Mapper `xweb:"/center/spider/assort"`          //审核人员分类
	batchShelves    xweb.Mapper `xweb:"/center/spider/batchShelves"`    //批量上下架
	checktime       xweb.Mapper `xweb:"/center/spider/checktime"`       //爬虫核对
	disables        xweb.Mapper `xweb:"/center/spider/disable"`         //批量作废
	changeEvent     xweb.Mapper `xweb:"/center/changeEvent"`            //节点更新
	getJson         xweb.Mapper `xweb:"/center/spider/json"`            //
	delRedis        xweb.Mapper `xweb:"/center/spider/delRedis"`        //清理Redis
	updateESP       xweb.Mapper `xweb:"/center/spider/updateesp"`       //修改爬虫的节点/状态/平台
	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //修改爬虫的节点/状态/平台

	spiderModel xweb.Mapper `xweb:"/center/model"`           //获取补充模型
	runStep     xweb.Mapper `xweb:"/center/run"`             //方法测试
	spiderPass  xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
	runPinYin   xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
	saveStep    xweb.Mapper `xweb:"/center/save"`            //脚本保存
	loadModel   xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
	importdata  xweb.Mapper `xweb:"/center/importdata"`      //导入脚本
	importfile  xweb.Mapper `xweb:"/center/importfile"`      //批量导入爬虫
	oldedit     xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
	findName    xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
	checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
	heart       xweb.Mapper `xweb:"/center/heart"`           //心跳监控
	spiderCopy  xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫

	Base      Base
	OtherBase OtherBase
	Step1     Step1
	Step2     Step2
	Step3     Step3
	StepRe3   StepRe3
	U         U

	luaList       xweb.Mapper `xweb:"/center/lualist.html"`       //脚本管理
	user          xweb.Mapper `xweb:"/center/user.html"`          //用户管理
	delUser       xweb.Mapper `xweb:"/center/user/del"`           //删除用户
	updateUser    xweb.Mapper `xweb:"/center/user/updateUser"`    //修改用户信息
	checkUsenamer xweb.Mapper `xweb:"/center/user/checkUsenamer"` //校验用户名的唯一性
	checkEmail    xweb.Mapper `xweb:"/center/user/checkEmail"`    //校验邮箱的唯一性
	saveNewUser   xweb.Mapper `xweb:"/center/user/saveNewUser"`   //添加用户

	getCity xweb.Mapper `xweb:"/center/getCity"` //获取城市
}

const role_admin, role_examine, role_dev = 3, 2, 1                                                                            //管理员，审核员，开发员
const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7 = 0, 1, 2, 3, 4, 5, 6, 7 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7其他

var LuaReg = regexp.MustCompile(`(:(find|match)|string\.|com\.getPureContent)`)
var spinfos sync.Map = sync.Map{}
var SessMap map[string]*httpsession.Session
var AutoTpl map[string]interface{}
var Mails *util.Mail
var Reg = regexp.MustCompile(`(http|https)://([\w]+\.)+[\w]+`)
var ListFilterReg = regexp.MustCompile("关键词过滤")
var Transfercode map[string]interface{}
var LuaStateMap = map[int]string{
	0:  "待完成",
	1:  "待审核",
	2:  "未通过",
	3:  "已通过",
	4:  "已作废",
	5:  "已上架",
	6:  "已下架",
	7:  "无发布",
	8:  "需登录",
	9:  "无法处理",
	10: "已删除",
}

func (f *Front) Login() error {
	username := f.GetString("username")
	password := f.GetString("password")
	f.SetSession("password", password)
	password = util.Se.EncodeString(password)
	query := bson.M{
		"s_name": username,
		"s_pass": password,
	}
	user := *mgdb.FindOne("user", query)
	if user != nil && user["i_delete"] == 0 {
		f.SetSession("userid", user["_id"].(bson.ObjectId).Hex())
		f.SetSession("username", user["s_fullname"])
		f.SetSession("loginuser", user["s_name"])
		f.SetSession("email", user["s_email"])
		f.SetSession("auth", user["i_auth"])
		f.SetSession("paltform", user["s_platform"])
		comeintime := time.Unix(user["l_comeintime"].(int64), 0).Format("2006-01-02")
		f.SetSession("comeintime", comeintime)
		if qu.IntAll(user["i_auth"]) > role_admin {
			return f.Redirect("/center/user.html")
		} else if qu.IntAll(user["i_auth"]) == role_dev {
			return f.Redirect("/center/mytask")
		} else {
			return f.Redirect("/center")
		}
	} else {
		if username != "" {
			f.T["fail"] = "fail"
		}
		return f.Render("login.html", &f.T)
	}
}

//用户管理
func (f *Front) User() {
	if f.Method() == "POST" {
		auth := qu.IntAll(f.GetSession("auth"))
		if auth > role_admin {
			start, _ := f.GetInteger("start")
			limit, _ := f.GetInteger("length")
			draw, _ := f.GetInteger("draw")
			query := bson.M{
				"i_delete": 0, //可用用户
				"i_auth": bson.M{
					"$lt": auth,
				},
			}
			user := *mgdb.Find("user", query, nil, nil, false, start, limit)
			count := mgdb.Count("user", query)
			page := start / 10
			for k, v := range user {
				v["num"] = k + 1 + page*10

				v["l_comeintime"] = time.Unix(v["l_comeintime"].(int64), 0).Format("2006-01-02")
				v["s_pass"] = util.Se.DecodeString(v["s_pass"].(string))
				v["userid"] = v["_id"].(bson.ObjectId).Hex()
			}
			f.ServeJson(map[string]interface{}{
				"draw":            draw,
				"data":            user,
				"recordsFiltered": count,
				"recordsTotal":    count,
			})
		}
	} else {
		f.Render("user.html")
	}

}

//删除用户
func (f *Front) DelUser() {
	userid := f.GetString("userid")
	auth := qu.IntAll(f.GetSession("auth"))
	if auth > role_admin {
		query := bson.M{
			"_id": bson.ObjectIdHex(userid),
		}

		update := bson.M{
			"$set": bson.M{
				"i_delete": 1,
			},
		}
		ok := mgdb.Update("user", query, update, false, false)
		if ok {
			f.ServeJson(map[string]interface{}{
				"status": "y",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "n",
			})
		}
	} else {
		f.ServeJson(map[string]interface{}{
			"status": "e",
		})
	}
}

//修改用户
func (f *Front) UpdateUser() {
	//	username := f.GetString("username")
	password := f.GetString("password")
	userid := f.GetString("userid")
	f.SetSession("password", password)
	userAuth := f.GetString("userAuth")
	auth := qu.IntAll(f.GetSession("auth"))
	self := f.GetString("self")
	//log.Println("userid----:", userid, "username----:", username, "password----:", password, "auth----:", auth, "self----:", self)
	query := bson.M{
		"_id": bson.ObjectIdHex(userid),
	}
	update := bson.M{}
	if "y" == self { //修改个人信息 只修改个人密码
		password = util.Se.EncodeString(password)
		update = bson.M{
			"$set": bson.M{
				"s_pass": password,
			},
		}
	} else if "y" != self && auth == 4 { //修改他人信息 只修改他人权限
		if userAuth == "开发员" || userAuth == "审核员" || userAuth == "管理员" {
			switch userAuth {
			case "开发员":
				update = bson.M{
					"$set": bson.M{
						"i_auth": 1,
					},
				}
			case "审核员":
				update = bson.M{
					"$set": bson.M{
						"i_auth": 2,
					},
				}
			case "管理员":
				update = bson.M{
					"$set": bson.M{
						"i_auth": 3,
					},
				}
			}
		}
	} else {
		f.ServeJson("没有权限！")
		return
	}
	ok := mgdb.Update("user", query, update, false, false)
	if ok {
		f.ServeJson(map[string]interface{}{
			"status": "y",
		})
	} else {
		f.ServeJson(map[string]interface{}{
			"status": "n",
		})
	}
}

func (f *Front) CheckUsenamer() {
	username := f.GetString("username")
	if username != "" {
		query := bson.M{
			"s_name": username,
		}
		user := *mgdb.FindOne("user", query)
		if user != nil {
			f.ServeJson(map[string]interface{}{
				"status": "hasUser",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "notHasUser",
			})
		}
	}

}

func (f *Front) CheckEmail() {
	email := f.GetString("email")
	//校验邮箱
	if email != "" {
		query := bson.M{
			"s_email": email,
		}
		user := *mgdb.FindOne("user", query)
		if user != nil {
			f.ServeJson(map[string]interface{}{
				"status": "hasEmail",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "notHasEmail",
			})
		}
	}
}

//新增用户
func (f *Front) SaveNewUser() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != 4 {
		f.ServeJson("没有权限！")
		return
	} else {
		i_auth := 1
		username := f.GetString("username")
		password := f.GetString("password")
		relname := f.GetString("relname")
		password = util.Se.EncodeString(password)
		email := f.GetString("email")
		userAuth := f.GetString("userAuth")
		if userAuth == "开发员" {
			i_auth = 1
		} else if userAuth == "审核员" {
			i_auth = 2
		} else {
			i_auth = 3
		}
		time := time.Now().Unix()
		save := bson.M{
			"s_name":       username,
			"s_fullname":   relname,
			"s_email":      email,
			"s_pass":       password,
			"i_auth":       i_auth,
			"i_delete":     0,
			"l_comeintime": time,
		}
		ok := mgdb.Save("user", save)
		if ok != "" {
			f.ServeJson(map[string]interface{}{
				"status": "y",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "n",
			})
		}
	}

}

func (f *Front) Logout() {
	email := f.GetSession("email").(string)
	f.DelSession("username")
	f.DelSession("userid")
	f.DelSession("email")
	f.DelSession("user")
	f.DelSession("loginuser")
	f.DelSession("auth")
	delete(SessMap, email)
	f.Redirect("/center")
}

//控制中心
func (f *Front) LoadIndex() {
	auth := qu.IntAll(f.GetSession("auth"))
	if f.Method() == "POST" {
		start, _ := f.GetInteger("start")
		limit, _ := f.GetInteger("length")
		draw, _ := f.GetInteger("draw")
		searchStr := f.GetString("search[value]")
		event, _ := f.GetInteger("taskEvent") //节点
		//searchN := strings.Replace(searchStr, " ", "", -1)
		//search := strings.Replace(searchN, "\n", "", -1)
		search := strings.TrimSpace(searchStr)
		state, _ := f.GetInteger("state")
		urgency, _ := f.GetInteger("urgency") //节点
		platform := f.GetString("platform")
		query := bson.M{}
		if event > -1 {
			query["event"] = event
		}
		if urgency > -1 {
			query["urgency"] = urgency
		}
		if platform != "-1" {
			query["platform"] = platform
		}
		if search != "" {
			query["$or"] = []interface{}{
				bson.M{"code": bson.M{"$regex": search}},
				bson.M{"createuser": bson.M{"$regex": search}},
				bson.M{"param_common.1": bson.M{"$regex": search}},
			}
		}
		if auth == role_examine { //审核员
			if state > -1 {
				query["state"] = state
			} else {
				query["state"] = Sp_state_1
			}

		} else if auth == role_dev { //开发员
			if state > -1 {
				query["state"] = state
			}
			query["createuserid"] = f.GetSession("userid")

		} else { //管理员
			if state > -1 {
				query["state"] = state
			}
		}
		sort := `{"%s":%d}`
		orderIndex := f.GetString("order[0][column]")
		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
		orderType := 1
		if f.GetString("order[0][dir]") != "asc" {
			orderType = -1
		}
		if orderName == "param_common" {
			orderName = orderName + ".1"
		}
		sort = fmt.Sprintf(sort, orderName, orderType)
		page := start / 10
		//log.Println("sort", sort, orderName)
		qu.Debug("query:", query, "sort:", sort)
		luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
		count := mgdb.Count("luaconfig", query)
		for k, v := range luas {
			v["num"] = k + 1 + page*10
			if v["modifytime"] != nil {
				v["modifytime"] = time.Unix(v["modifytime"].(int64), 0).Format("2006-01-02 15:04:05")
			} else {
				v["modifytime"] = "-"
			}
			if v["modifyuser"] == nil {
				v["modifytime"] = "-"
			}
			v["encode"] = util.Se.Encode2Hex(fmt.Sprint(v["code"]))
			if v["event"] == nil { //节点
				v["event"] = 0
			}
			//v["state"] = LuaStateMap[qu.IntAll(v["state"])]
		}
		f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
	} else {
		events := []string{}
		for k, _ := range util.Config.Uploadevents {
			events = append(events, k)
		}
		sort.Strings(events)
		f.T["events"] = events
		f.Render("index.html", &f.T)
	}
}

func (f *Front) Checkrepeat() {
	code := f.GetString("code")
	one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
	if len(one) > 0 {
		f.ServeJson("y")
	} else {
		f.ServeJson("n")
	}

}

//新建
func (f *Front) Spidernew() error {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		return nil
	}
	copy := f.GetString("copy")
	if copy != "" {
		one := *mgdb.FindOne("luaconfig", bson.M{"code": copy})
		delete(one, "_id")
		delete(one, "code")
		base := one["param_common"].([]interface{})
		base[0] = ""
		base[1] = ""
		one["param_common"] = base
		f.T["lua"] = one
	}
	f.T["isflow"] = 1 //新建爬虫时，初始化isflow的值
	f.T["actiontext"] = "新建"
	f.T["restate"] = 4 //此处设置restate=4无意义，只为了页面不报错
	return f.Render("spideredit.html", &f.T)
}

//得到模型
func (f *Front) SpiderModel() {
	f.ServeJson(util.Config.Model)
}

func (f *Front) RunPinYin() {
	word := f.GetString("word")
	str := gopinyin.Convert(word, true)
	f.Write(str)
}

type U struct {
	User string
	Name string
	Pwd  string
}

func (f *Front) Reg() {

}

func (f *Front) Importfile() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.ServeJson("没有权限")
		return
	}
	if f.Method() == "POST" {
		mf, _, err := f.GetFile("xlsx")
		errorinfo := map[string]interface{}{}
		if err == nil {
			binary, _ := ioutil.ReadAll(mf)
			xls, _ := xlsx.OpenBinary(binary)
			sheet := xls.Sheets[0]
			rows := sheet.Rows
			for k, v := range rows {
				if k != 0 {
					cells := v.Cells
					if cells[1].Value != "" {
						o := make(map[string]interface{})
						o["name"] = cells[0].Value
						o["code"] = cells[1].Value
						o["channel"] = cells[2].Value
						o["channeladdr"] = cells[3].Value
						o["author"] = cells[4].Value
						o["timestamp"] = time.Now().Unix()
						o["status"] = 1
						o["next"] = cells[4].Value
						o["event"] = cells[5].Value
						o["historyevent"] = cells[6].Value
						if cells[7].Value == "是" {
							o["isflow"] = 1
						} else {
							o["isflow"] = 0
						}
						if cells[8].Value == "紧急" {
							o["urgency"] = 1
						} else {
							o["urgency"] = 0
						}
						o["platform"] = cells[9].Value
						weigh, _ := cells[10].Int()
						estimate, _ := cells[11].Int()
						o["estimate"] = estimate
						o["weight"] = weigh
						//存储表
						o["coll"] = cells[12].Value
						//导入默认字段
						o["spidercompete"] = true
						o["pendstate"] = 0
						//table := cells[6].Value
						//o["table"] = table
						//o["transfercode"] = qu.IntAll(Transfercode[table])

						query := bson.M{"code": cells[1].Value}
						rs := *mgdb.FindOne("import", query)
						if len(rs) > 0 {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行重复,已经过滤"
						} else {
							ok, name := pf(o) //保存爬虫
							if ok == false {
								errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
							} else {
								o["author"] = name
								mgdb.Save("import", o)
							}
						}
					}
				}
			}
			f.ServeJson(errorinfo)
		} else {
			f.ServeJson(false)
		}
	}
}

func pf(o map[string]interface{}) (bool, string) {
	AutoTpl["Base.SpiderName"] = o["name"]
	AutoTpl["Base.SpiderCode"] = o["code"]
	AutoTpl["Base.SpiderChannel"] = o["channel"]
	AutoTpl["Base.SpiderTargetChannelUrl"] = o["channeladdr"]
	author := o["author"].(string)
	one := *mgdb.FindOne("user", bson.M{"s_email": author})
	id := one["_id"].(bson.ObjectId).Hex()
	if len(one) == 0 {
		return false, ""
	}
	common := []interface{}{
		AutoTpl["Base.SpiderCode"],
		AutoTpl["Base.SpiderName"],
		AutoTpl["Base.SpiderChannel"],
		AutoTpl["Base.SpiderDownDetailPage"],
		AutoTpl["Base.SpiderStartPage"],
		AutoTpl["Base.SpiderMaxPage"],
		AutoTpl["Base.SpiderRunRate"],
		//AutoTpl["Base.Spider2Collection"],
		//"bidding", //爬虫导入新建默认为bidding
		qu.ObjToString(o["coll"]),
		AutoTpl["Base.SpiderPageEncoding"],
		AutoTpl["Base.SpiderStoreMode"],
		AutoTpl["Base.SpiderStoreToMsgEvent"],
		AutoTpl["Base.SpiderTargetChannelUrl"],
		AutoTpl["Base.SpiderLastDownloadTime"],
		AutoTpl["Base.SpiderIsHistoricalMend"],
		AutoTpl["Base.SpiderIsMustDownload"],
	}
	ptime := []interface{}{
		AutoTpl["Step1.DateFormat"],
		AutoTpl["Step1.Address"],
		AutoTpl["Step1.ContentChooser"],
	}
	list := []interface{}{
		AutoTpl["Step2.Listadd"],
		AutoTpl["Step2.Listadds"],
		AutoTpl["Step2.BlockChooser"],
		AutoTpl["Step2.AddressChooser"],
		AutoTpl["Step2.TitleChooser"],
		AutoTpl["Step2.DateChooser"],
		AutoTpl["Step2.DateFormat"],
	}
	content := []interface{}{
		AutoTpl["Step3.ContentChooser"],
		AutoTpl["Step3.ElementChooser"],
	}
	param := map[string]interface{}{}
	param["param_common"] = common
	//向导模式
	param["param_time"] = ptime
	param["param_list"] = list
	param["param_content"] = content
	param["type_time"] = 0
	param["type_list"] = 0
	param["type_content"] = 0
	//专家模式
	param["str_time"] = ""
	param["str_list"] = ""
	param["str_content"] = ""
	param["comeintime"] = time.Now().Unix()
	param["code"] = o["code"]
	param["createuser"] = one["s_name"]
	param["createuserid"] = id
	param["createuseremail"] = one["s_email"]
	param["modifyuser"] = one["s_name"]
	param["modifyuserid"] = id
	param["modifytime"] = time.Now().Unix()
	param["state"] = 0 //未完成
	if qu.IntAll(o["event"]) > 0 {
		param["event"] = qu.IntAll(o["event"])
	}
	s_model := "bid"
	configModel := util.Config.Model[s_model]
	model := map[string]interface{}{}
	for k, _ := range configModel {
		model[k] = ""
	}
	param["model"] = model
	param["next"] = o["next"]
	param["urgency"] = o["urgency"]
	param["isflow"] = o["isflow"]
	param["spidertype"] = "history"
	historyevent := qu.ObjToString(o["historyevent"])
	if movevent, ok := util.Config.Uploadevents[historyevent].(string); ok && movevent != "" {
		param["spidermovevent"] = movevent
	} else {
		param["spidermovevent"] = "7700"
	}
	param["historyevent"] = qu.IntAll(o["historyevent"])
	param["spiderhistorymaxpage"] = 1
	param["platform"] = o["platform"]
	param["spidercompete"] = o["spidercompete"]
	param["estimate"] = o["estimate"]
	param["weight"] = o["weight"]

	//qu.Debug("param---", param)
	issave := spider.SaveSpider(o["code"].(string), param)
	return issave, one["s_name"].(string)
}

func (f *Front) Importdata() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth == role_admin {
		if f.Method() == "GET" {
			f.Render("import.html")
		} else {
			rss := *mgdb.Find("import", nil, `{"timestamp": -1}`, nil, false, -1, -1)
			f.ServeJson(map[string]interface{}{
				"data": rss,
			})
		}
	} else {
		f.Write("您没有导入脚本的权限")
	}
}

func Wlog(name, code, man, manid, types string, content map[string]interface{}) {
	obj := bson.M{
		"name":    name,
		"code":    code,
		"man":     man,
		"manid":   manid,
		"types":   types,
		"time":    time.Now().Unix(),
		"content": content,
	}
	mgdb.Save("lua_logs", obj)
}

func (f *Front) Oldedit() {
	if f.Method() == "GET" {
		f.Render("oldedit.html")
	} else {

	}
}

func (f *Front) FindName() {
	words := f.GetString("words")
	if words == "" {
		f.ServeJson(bson.M{"error": "null"})
	}
	query := bson.M{"$or": []interface{}{
		bson.M{"param_common.0": bson.M{"$regex": words}},
		bson.M{"param_common.1": bson.M{"$regex": words}},
		bson.M{"createuser": bson.M{"$regex": words}},
	}, "oldlua": bson.M{"$exists": false}}
	rs := *mgdb.Find("luaconfig", query, bson.M{"modifytime": -1}, bson.M{"param_common": 1}, false, -1, -1)
	if len(rs) > 0 {
		f.ServeJson(bson.M{"data": rs})
	} else {
		f.ServeJson(bson.M{"error": "data"})
	}
}

//分配爬虫
func (f *Front) Assign() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.Write("n")
		return
	}
	codes := f.GetString("codes")
	userid := f.GetString("userid")       //分配给谁
	codesarr := strings.Split(codes, ",") //被分配的爬虫
	user, _ := u.MgoE.FindById("user", userid, nil)
	if user != nil && len(*user) > 0 {
		name := qu.ObjToString((*user)["s_name"])
		query := map[string]interface{}{
			"code": map[string]interface{}{
				"$in": codesarr,
			},
		}
		qu.Debug(query)
		set := map[string]interface{}{
			"$set": map[string]interface{}{
				"createuserid":    userid,
				"createuser":      name,
				"createuseremail": (*user)["s_email"],
				"modifyuser":      name,
				"modifyuserid":    userid,
			},
		}
		b := u.MgoE.Update("luaconfig", query, set, false, true)
		if b {
			go editModify(codesarr, userid, name) //修改爬虫对应任务的维护人
			f.Write("y")
		} else {
			f.Write("n")
		}
	} else {
		f.Write("null")
	}

}

//修改维护人
func editModify(codesarr []string, userid, name string) {
	qu.Debug("修改爬虫对应任务的维护人：", name, codesarr)
	//修改modifyid和modify
	query := map[string]interface{}{
		"s_code": map[string]interface{}{
			"$in": codesarr,
		},
		"i_state": map[string]interface{}{
			"$nin": []int{4, 6},
		},
	}
	set := map[string]interface{}{
		"$set": map[string]interface{}{
			"s_modify":   name,
			"s_modifyid": userid,
		},
	}
	b := u.MgoE.Update("task", query, set, false, true)
	qu.Debug("任务重新分配成功", b)
}

//清理Redis
func (f *Front) DelRedis() {
	hrefs := f.GetString("href")
	hrefsarr := strings.Split(hrefs, ",")
	auth := qu.IntAll(f.GetSession("auth"))
	err := []string{}
	if auth == role_admin { //权限控制
		if len(hrefsarr) > 0 {
			for k1, h := range hrefsarr {
				href := Reg.FindString(h)
				if href != "" {
					href = "url_repeat_" + href + "*"
					res := redis.GetKeysByPattern("title_repeat_judgement", href)
					if res != nil {
						for _, v := range res {
							hf := string(v.([]uint8))
							b := redis.Del("title_repeat_judgement", hf)
							if !b {
								err = append(err, "第"+strconv.Itoa(k1+1)+"个")
							}
						}
					}
				} else {
					err = append(err, "第"+strconv.Itoa(k1+1)+"个")
				}
			}
		}
	} else {
		err = append(err, "没有权限")
	}
	f.ServeJson(err)
}

func (f *Front) UpdatePendState() {
	success := false
	pendstate, _ := f.GetInteger("pendstate")
	code := f.GetString("code")
	stype := f.GetString("stype")
	qu.Debug(code, stype, pendstate)
	success = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
	if success {
		if stype == "code" { //爬虫挂起时，关闭所有任务
			u.MgoE.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$ne": 6}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
		} else if stype == "task" { //任务挂起任务时任务关闭
			id := f.GetString("id")
			u.MgoE.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_pendstate": pendstate, "i_state": 6}})
		}
	}

	f.ServeJson(map[string]interface{}{"success": success})
}

func (f *Front) UpdateESP() {
	val := f.GetString("val")
	w := f.GetString("w")
	id := f.GetString("id")
	code := f.GetString("c")
	query := map[string]interface{}{
		"_id": qu.StringTOBsonId(id),
	}
	set := map[string]interface{}{}
	update := map[string]interface{}{
		"$set": set,
	}
	if w == "state" { //无效爬虫改为待完成
		one, _ := u.MgoE.FindById("luaconfig", id, map[string]interface{}{"historyevent": 1, "event": 1})
		tmpEvent := qu.IntAll((*one)["event"])
		if one != nil && len(*one) > 0 {
			if (*one)["historyevent"] == nil && tmpEvent != 7000 { //除7000节点外没有historyevent的要重新设置
				set["historyevent"] = tmpEvent
			}
		}
		set["state"] = 0
		set["event"] = 7000
		set["spidertype"] = "history"
	} else if w == "platform" {
		set["platform"] = val
		if val != "golua平台" { //由lua平台切换到其他平台，删除心跳;爬虫下架;修改爬虫状态
			set["state"] = 0
			b := mgu.Update("spider_heart", "spider", "spider", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
			qu.Debug("Del Heart:", b)
			b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
			qu.Debug("爬虫下架成功：", b)
			if !b || err != nil {
				f.Write("n")
				return
			}
		}
	} else { //修改节点
		event, _ := strconv.Atoi(val)
		set["event"] = event
		set["historyevent"] = event
		//state := f.GetString("s")
		//if state == "5" { //已上架状态改为下架
		code := f.GetString("c")
		set["state"] = 6
		b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
		qu.Debug("爬虫下架成功：", b)
		if !b || err != nil {
			f.Write("n")
			return
		}
		//}
	}
	if mgdb.Update("luaconfig", query, update, false, false) {
		log.Println("Id:", id, "	Update", w, val, "Success")
		f.Write("y")
	} else {
		log.Println("Id:", id, "	Update", w, val, "Failed")
		f.Write("n")
	}
	f.Write("n")
}

func (f *Front) GetCity() {
	area := f.GetString("area")
	cityArr := []string{}
	cityArr = u.Province[area]
	f.ServeJson(cityArr)
}

func (f *Front) SpiderCopy() {
	defer qu.Catch()
	code := f.GetString("code")
	code = strings.TrimSpace(code)
	success := false
	text := ""
	encode := ""
	if code != "" {
		lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
		if len(lua) > 0 {
			codeOld := qu.ObjToString(lua["code"])
			codeNew := codeOld + "_bu"
			luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
			if len(luaTmp) > 0 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "已存在补漏爬虫"})
				return
			}
			lua["code"] = codeNew
			if param_common := lua["param_common"].([]interface{}); len(param_common) < 13 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "脚本异常，请联系管理员"})
				return
			} else {
				param_common = param_common[:13]
				param_common[0] = codeNew                  //code
				param_common = append(param_common, true)  //历史补漏
				param_common = append(param_common, false) //强制下载
				lua["param_common"] = param_common
			}
			str_list := qu.ObjToString(lua["str_list"])
			lua["str_list"] = strings.ReplaceAll(str_list, codeOld, codeNew)
			lua["comeintime"] = time.Now().Unix()
			lua["modifytime"] = time.Now().Unix()
			lua["spidertype"] = "increment"
			lua["event"] = 7000
			lua["state"] = 6
			id := mgdb.Save("luaconfig", lua)
			if id != "" {
				success = true
				encode = util.Se.Encode2Hex(codeNew)
			} else {
				text = "新爬虫保存失败"
			}
		} else {
			text = "未查询到复制原"
		}
	} else {
		text = "脚本不能为空"
	}
	f.ServeJson(map[string]interface{}{"success": success, "text": text, "encode": encode})
}