spider_editor/src/front/front.go

// front
package front

import (
	"fmt"
	"io/ioutil"
	"log"
	"mongodb"
	qu "qfw/util"
	"qfw/util/redis"
	"regexp"
	"sort"
	"spider"
	util "spiderutil"
	"strconv"
	"strings"
	"sync"
	"time"
	u "util"

	"github.com/go-xweb/httpsession"
	"github.com/go-xweb/xweb"
	"github.com/lauyoume/gopinyin"
	"github.com/tealeg/xlsx"
)

type Front struct {
	*xweb.Action
	login           xweb.Mapper `xweb:"/"`
	logout          xweb.Mapper `xweb:"/center/logout"`                 //退出
	loadIndex       xweb.Mapper `xweb:"/center"`                        //控制中心
	spidernew       xweb.Mapper `xweb:"/center/spider"`                 //爬虫新建
	reg             xweb.Mapper `xweb:"/center/reg"`                    //爬虫注册
	assign          xweb.Mapper `xweb:"/center/user/assign"`            //分配爬虫
	loadSpider      xweb.Mapper `xweb:"/center/spider/edit/(.*)"`       //爬虫加载
	viewSpider      xweb.Mapper `xweb:"/center/spider/view/(.*)"`       //爬虫查看
	downSpider      xweb.Mapper `xweb:"/center/spider/download/(.*)"`   //爬虫下载
	upState         xweb.Mapper `xweb:"/center/spider/upstate"`         //爬虫状态更新
	assort          xweb.Mapper `xweb:"/center/spider/assort"`          //审核人员分类(无发布、需登录、无法处理、需删除)
	batchShelves    xweb.Mapper `xweb:"/center/spider/batchShelves"`    //批量上下架
	checktime       xweb.Mapper `xweb:"/center/spider/checktime"`       //爬虫核对
	disables        xweb.Mapper `xweb:"/center/spider/disable"`         //批量作废
	changeEvent     xweb.Mapper `xweb:"/center/changeEvent"`            //节点更新
	getJson         xweb.Mapper `xweb:"/center/spider/json"`            //
	delRedis        xweb.Mapper `xweb:"/center/spider/delRedis"`        //清理Redis
	updateESP       xweb.Mapper `xweb:"/center/spider/updateesp"`       //修改爬虫的节点/状态/平台
	updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
	tagCode         xweb.Mapper `xweb:"/center/spider/tagcode"`         //标记爬虫

	spiderModel xweb.Mapper `xweb:"/center/model"`           //获取补充模型
	runStep     xweb.Mapper `xweb:"/center/run"`             //方法测试
	spiderPass  xweb.Mapper `xweb:"/center/spider/pass"`     //整体测试
	runPinYin   xweb.Mapper `xweb:"/center/runpy"`           //获取拼音
	saveStep    xweb.Mapper `xweb:"/center/save"`            //脚本保存
	loadModel   xweb.Mapper `xweb:"/center/gmodel/(.*)"`     //加载模型
	importdata  xweb.Mapper `xweb:"/center/importdata"`      //导入脚本
	importfile  xweb.Mapper `xweb:"/center/importfile"`      //批量导入爬虫
	oldedit     xweb.Mapper `xweb:"/center/oldedit"`         //老文件编辑
	findName    xweb.Mapper `xweb:"/center/findname"`        //即时查询名称
	checkrepeat xweb.Mapper `xweb:"/center/spider/isrepeat"` //脚本代码判重
	heart       xweb.Mapper `xweb:"/center/heart"`           //心跳监控
	spiderCopy  xweb.Mapper `xweb:"/center/spider/copy"`     //复制爬虫

	Base      Base
	OtherBase OtherBase
	Step1     Step1
	Step2     Step2
	Step3     Step3
	StepRe3   StepRe3
	U         U

	luaList       xweb.Mapper `xweb:"/center/lualist.html"`       //脚本管理
	user          xweb.Mapper `xweb:"/center/user.html"`          //用户管理
	delUser       xweb.Mapper `xweb:"/center/user/del"`           //删除用户
	updateUser    xweb.Mapper `xweb:"/center/user/updateUser"`    //修改用户信息
	checkUsenamer xweb.Mapper `xweb:"/center/user/checkUsenamer"` //校验用户名的唯一性
	checkEmail    xweb.Mapper `xweb:"/center/user/checkEmail"`    //校验邮箱的唯一性
	saveNewUser   xweb.Mapper `xweb:"/center/user/saveNewUser"`   //添加用户

	getCity xweb.Mapper `xweb:"/center/getCity"` //获取城市
}

const role_admin, role_examine, role_dev = 3, 2, 1                                                                            //管理员，审核员，开发员
const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7 = 0, 1, 2, 3, 4, 5, 6, 7 //0待完成，1待审核，2打回，3发布，4作废，5已上架，6已下架，7其他

var LuaReg = regexp.MustCompile(`(:(find|match)|string\.|com\.getPureContent)`)
var spinfos sync.Map = sync.Map{}
var SessMap map[string]*httpsession.Session
var AutoTpl map[string]interface{}
var Mails *util.Mail
var Reg = regexp.MustCompile(`(http|https)://([\w]+\.)+[\w]+`)
var ListFilterReg = regexp.MustCompile("关键词过滤")
var Transfercode map[string]interface{}
var LuaStateMap = map[int]string{
	0:  "待完成",
	1:  "待审核",
	2:  "未通过",
	3:  "已通过",
	4:  "已作废",
	5:  "已上架",
	6:  "已下架",
	7:  "无发布",
	8:  "需登录",
	9:  "转python",
	10: "已删除",
}

func (f *Front) Login() error {
	username := f.GetString("username")
	password := f.GetString("password")
	f.SetSession("password", password)
	password = util.Se.EncodeString(password)
	query := map[string]interface{}{
		"s_name": username,
		"s_pass": password,
	}
	user, _ := u.MgoEB.FindOne("user", query)
	if user != nil && len(*user) > 0 && qu.IntAll((*user)["i_delete"]) == 0 {
		f.SetSession("userid", mongodb.BsonIdToSId((*user)["_id"]))
		f.SetSession("username", (*user)["s_fullname"])
		f.SetSession("loginuser", (*user)["s_name"])
		f.SetSession("email", (*user)["s_email"])
		f.SetSession("auth", (*user)["i_auth"])
		f.SetSession("platform", (*user)["s_platform"])
		comeintime := time.Unix((*user)["l_comeintime"].(int64), 0).Format("2006-01-02")
		f.SetSession("comeintime", comeintime)
		if qu.IntAll((*user)["i_auth"]) > role_admin {
			return f.Redirect("/center/user.html")
		} else if qu.IntAll((*user)["i_auth"]) == role_dev {
			return f.Redirect("/center/mytask")
		} else {
			return f.Redirect("/center")
		}
	} else {
		if username != "" {
			f.T["fail"] = "fail"
		}
		return f.Render("login.html", &f.T)
	}
}

//用户管理
func (f *Front) User() {
	if f.Method() == "POST" {
		auth := qu.IntAll(f.GetSession("auth"))
		if auth > role_admin {
			start, _ := f.GetInteger("start")
			limit, _ := f.GetInteger("length")
			draw, _ := f.GetInteger("draw")
			query := map[string]interface{}{
				"i_delete": 0, //可用用户
				"i_auth": map[string]interface{}{
					"$lt": auth,
				},
			}
			user, _ := u.MgoEB.Find("user", query, nil, nil, false, start, limit)
			count := u.MgoEB.Count("user", query)
			page := start / 10
			for k, v := range *user {
				v["num"] = k + 1 + page*10

				v["l_comeintime"] = time.Unix(v["l_comeintime"].(int64), 0).Format("2006-01-02")
				v["s_pass"] = util.Se.DecodeString(v["s_pass"].(string))
				v["userid"] = mongodb.BsonIdToSId(v["_id"])
			}
			f.ServeJson(map[string]interface{}{
				"draw":            draw,
				"data":            user,
				"recordsFiltered": count,
				"recordsTotal":    count,
			})
		}
	} else {
		f.Render("user.html")
	}

}

//删除用户
func (f *Front) DelUser() {
	userid := f.GetString("userid")
	auth := qu.IntAll(f.GetSession("auth"))
	if auth > role_admin {
		query := map[string]interface{}{
			"_id": mongodb.StringTOBsonId(userid),
		}

		update := map[string]interface{}{
			"$set": map[string]interface{}{
				"i_delete": 1,
			},
		}
		ok := u.MgoEB.Update("user", query, update, false, false)
		if ok {
			f.ServeJson(map[string]interface{}{
				"status": "y",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "n",
			})
		}
	} else {
		f.ServeJson(map[string]interface{}{
			"status": "e",
		})
	}
}

//修改用户
func (f *Front) UpdateUser() {
	//	username := f.GetString("username")
	password := f.GetString("password")
	userid := f.GetString("userid")
	f.SetSession("password", password)
	userAuth := f.GetString("userAuth")
	auth := qu.IntAll(f.GetSession("auth"))
	self := f.GetString("self")
	//log.Println("userid----:", userid, "username----:", username, "password----:", password, "auth----:", auth, "self----:", self)
	query := map[string]interface{}{
		"_id": mongodb.StringTOBsonId(userid),
	}
	update := map[string]interface{}{}
	if "y" == self { //修改个人信息 只修改个人密码
		password = util.Se.EncodeString(password)
		update = map[string]interface{}{
			"$set": map[string]interface{}{
				"s_pass": password,
			},
		}
	} else if "y" != self && auth == 4 { //修改他人信息 只修改他人权限
		if userAuth == "开发员" || userAuth == "审核员" || userAuth == "管理员" {
			switch userAuth {
			case "开发员":
				update = map[string]interface{}{
					"$set": map[string]interface{}{
						"i_auth": 1,
					},
				}
			case "审核员":
				update = map[string]interface{}{
					"$set": map[string]interface{}{
						"i_auth": 2,
					},
				}
			case "管理员":
				update = map[string]interface{}{
					"$set": map[string]interface{}{
						"i_auth": 3,
					},
				}
			}
		}
	} else {
		f.ServeJson("没有权限！")
		return
	}
	ok := u.MgoEB.Update("user", query, update, false, false)
	if ok {
		f.ServeJson(map[string]interface{}{
			"status": "y",
		})
	} else {
		f.ServeJson(map[string]interface{}{
			"status": "n",
		})
	}
}

func (f *Front) CheckUsenamer() {
	username := f.GetString("username")
	if username != "" {
		query := map[string]interface{}{
			"s_name": username,
		}
		user, _ := u.MgoEB.FindOne("user", query)
		if user != nil && len(*user) > 0 {
			f.ServeJson(map[string]interface{}{
				"status": "hasUser",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "notHasUser",
			})
		}
	}

}

func (f *Front) CheckEmail() {
	email := f.GetString("email")
	//校验邮箱
	if email != "" {
		query := map[string]interface{}{
			"s_email": email,
		}
		user, _ := u.MgoEB.FindOne("user", query)
		if user != nil && len(*user) > 0 {
			f.ServeJson(map[string]interface{}{
				"status": "hasEmail",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "notHasEmail",
			})
		}
	}
}

//新增用户
func (f *Front) SaveNewUser() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != 4 {
		f.ServeJson("没有权限！")
		return
	} else {
		i_auth := 1
		username := f.GetString("username")
		password := f.GetString("password")
		relname := f.GetString("relname")
		password = util.Se.EncodeString(password)
		email := f.GetString("email")
		userAuth := f.GetString("userAuth")
		if userAuth == "开发员" {
			i_auth = 1
		} else if userAuth == "审核员" {
			i_auth = 2
		} else {
			i_auth = 3
		}
		time := time.Now().Unix()
		save := map[string]interface{}{
			"s_name":       username,
			"s_fullname":   relname,
			"s_email":      email,
			"s_pass":       password,
			"i_auth":       i_auth,
			"i_delete":     0,
			"l_comeintime": time,
		}
		ok := u.MgoEB.Save("user", save)
		if ok != "" {
			f.ServeJson(map[string]interface{}{
				"status": "y",
			})
		} else {
			f.ServeJson(map[string]interface{}{
				"status": "n",
			})
		}
	}

}

func (f *Front) Logout() {
	email := f.GetSession("email").(string)
	f.DelSession("username")
	f.DelSession("userid")
	f.DelSession("email")
	f.DelSession("user")
	f.DelSession("loginuser")
	f.DelSession("auth")
	delete(SessMap, email)
	f.Redirect("/center")
}

//控制中心
func (f *Front) LoadIndex() {
	auth := qu.IntAll(f.GetSession("auth"))
	if f.Method() == "POST" {
		start, _ := f.GetInteger("start")
		limit, _ := f.GetInteger("length")
		draw, _ := f.GetInteger("draw")
		searchStr := f.GetString("search[value]")
		event, _ := f.GetInteger("taskEvent") //节点
		//searchN := strings.Replace(searchStr, " ", "", -1)
		//search := strings.Replace(searchN, "\n", "", -1)
		search := strings.TrimSpace(searchStr)
		state, _ := f.GetInteger("state")
		urgency, _ := f.GetInteger("urgency") //节点
		platform := f.GetString("platform")
		query := map[string]interface{}{}
		if event > -1 {
			query["event"] = event
		}
		if urgency > -1 {
			query["urgency"] = urgency
		}
		if platform != "-1" {
			query["platform"] = platform
		}
		if search != "" {
			query["$or"] = []interface{}{
				map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"createuser": map[string]interface{}{"$regex": search}},
				map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": search}},
			}
		}
		if auth == role_examine { //审核员
			if state > -1 {
				query["state"] = state
			} else {
				query["state"] = Sp_state_1
			}

		} else if auth == role_dev { //开发员
			if state > -1 {
				query["state"] = state
			}
			query["createuserid"] = f.GetSession("userid")

		} else { //管理员
			if state > -1 {
				query["state"] = state
			}
		}
		sort := `{"%s":%d}`
		orderIndex := f.GetString("order[0][column]")
		orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
		orderType := 1
		if f.GetString("order[0][dir]") != "asc" {
			orderType = -1
		}
		if orderName == "param_common" {
			orderName = orderName + ".1"
		}
		sort = fmt.Sprintf(sort, orderName, orderType)
		page := start / 10
		//log.Println("sort", sort, orderName)
		qu.Debug("query:", query, "sort:", sort)
		//luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
		//count := mgdb.Count("luaconfig", query)
		luas, _ := u.MgoEB.Find("luaconfig", query, sort, list_fields, false, start, limit)
		count := u.MgoEB.Count("luaconfig", query)
		for k, v := range *luas {
			v["num"] = k + 1 + page*10
			if v["modifytime"] != nil {
				v["modifytime"] = time.Unix(v["modifytime"].(int64), 0).Format("2006-01-02 15:04:05")
			} else {
				v["modifytime"] = "-"
			}
			if v["modifyuser"] == nil {
				v["modifytime"] = "-"
			}
			v["encode"] = util.Se.Encode2Hex(fmt.Sprint(v["code"]))
			if v["event"] == nil { //节点
				v["event"] = 0
			}
			user, _ := u.MgoEB.FindOne("user", map[string]interface{}{"s_name": qu.ObjToString(v["createuser"])})
			//v["state"] = LuaStateMap[qu.IntAll(v["state"])]
			v["i_scope"] = (*user)["i_scope"]
		}
		f.ServeJson(map[string]interface{}{"draw": draw, "data": luas, "recordsFiltered": count, "recordsTotal": count})
	} else {
		events := []string{}
		for k, _ := range util.Config.Uploadevents {
			events = append(events, k)
		}
		sort.Strings(events)
		f.T["events"] = events
		f.Render("index.html", &f.T)
	}
}

func (f *Front) Checkrepeat() {
	code := f.GetString("code")
	//one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
	one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
	if len(*one) > 0 {
		f.ServeJson("y")
	} else {
		f.ServeJson("n")
	}

}

//新建
func (f *Front) Spidernew() error {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		return nil
	}
	copy := f.GetString("copy")
	if copy != "" {
		//one := *mgdb.FindOne("luaconfig", bson.M{"code": copy})
		one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": copy})
		delete((*one), "_id")
		delete((*one), "code")
		base := (*one)["param_common"].([]interface{})
		base[0] = ""
		base[1] = ""
		(*one)["param_common"] = base
		f.T["lua"] = one
	}
	f.T["isflow"] = 1 //新建爬虫时，初始化isflow的值
	f.T["actiontext"] = "新建"
	f.T["restate"] = 4 //此处设置restate=4无意义，只为了页面不报错
	return f.Render("spideredit.html", &f.T)
}

//得到模型
func (f *Front) SpiderModel() {
	f.ServeJson(util.Config.Model)
}

func (f *Front) RunPinYin() {
	word := f.GetString("word")
	str := gopinyin.Convert(word, true)
	f.Write(str)
}

type U struct {
	User string
	Name string
	Pwd  string
}

func (f *Front) Reg() {

}

func (f *Front) Importfile() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.ServeJson("没有权限")
		return
	}
	if f.Method() == "POST" {
		mf, _, err := f.GetFile("xlsx")
		errorinfo := map[string]interface{}{}
		if err == nil {
			binary, _ := ioutil.ReadAll(mf)
			xls, _ := xlsx.OpenBinary(binary)
			sheet := xls.Sheets[0]
			rows := sheet.Rows
			for k, v := range rows {
				if k != 0 {
					cells := v.Cells
					if cells[1].Value != "" {
						o := make(map[string]interface{})
						o["name"] = cells[0].Value
						o["code"] = cells[1].Value
						o["channel"] = cells[2].Value
						o["channeladdr"] = cells[3].Value
						o["author"] = cells[4].Value
						o["timestamp"] = time.Now().Unix()
						o["status"] = 1
						o["next"] = cells[4].Value
						o["event"] = cells[5].Value
						o["historyevent"] = cells[6].Value
						if cells[7].Value == "是" {
							o["isflow"] = 1
						} else {
							o["isflow"] = 0
						}
						if cells[8].Value == "紧急" {
							o["urgency"] = 1
						} else {
							o["urgency"] = 0
						}
						o["platform"] = cells[9].Value
						o["area"] = cells[10].Value
						o["city"] = cells[11].Value
						o["district"] = cells[12].Value
						weigh, _ := cells[13].Int()
						o["weight"] = weigh
						//存储表
						o["coll"] = cells[14].Value
						//table := cells[6].Value
						//o["table"] = table
						//o["transfercode"] = qu.IntAll(Transfercode[table])

						query := map[string]interface{}{"code": cells[1].Value}
						rs, _ := u.MgoEB.FindOne("import", query)
						if len(*rs) > 0 {
							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行重复,已经过滤"
						} else {
							ok, name := savelua(o) //保存爬虫
							if ok == false {
								errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
							} else {
								o["author"] = name
								u.MgoEB.Save("import", o)
							}
						}
					}
				}
			}
			f.ServeJson(errorinfo)
		} else {
			f.ServeJson(false)
		}
	}
}

func savelua(o map[string]interface{}) (bool, string) {
	AutoTpl["Base.SpiderName"] = o["name"]
	AutoTpl["Base.SpiderCode"] = o["code"]
	AutoTpl["Base.SpiderChannel"] = o["channel"]
	AutoTpl["Base.SpiderTargetChannelUrl"] = o["channeladdr"]
	author := o["author"].(string)
	one, _ := u.MgoEB.FindOne("user", map[string]interface{}{"s_email": author})
	id := mongodb.BsonIdToSId((*one)["_id"])
	if len(*one) == 0 {
		return false, ""
	}
	common := []interface{}{
		AutoTpl["Base.SpiderCode"],
		AutoTpl["Base.SpiderName"],
		AutoTpl["Base.SpiderChannel"],
		AutoTpl["Base.SpiderDownDetailPage"],
		AutoTpl["Base.SpiderStartPage"],
		AutoTpl["Base.SpiderMaxPage"],
		AutoTpl["Base.SpiderRunRate"],
		//AutoTpl["Base.Spider2Collection"],
		//"bidding", //爬虫导入新建默认为bidding
		qu.ObjToString(o["coll"]),
		AutoTpl["Base.SpiderPageEncoding"],
		AutoTpl["Base.SpiderStoreMode"],
		AutoTpl["Base.SpiderStoreToMsgEvent"],
		AutoTpl["Base.SpiderTargetChannelUrl"],
		AutoTpl["Base.SpiderLastDownloadTime"],
		AutoTpl["Base.SpiderIsHistoricalMend"],
		AutoTpl["Base.SpiderIsMustDownload"],
	}
	ptime := []interface{}{
		AutoTpl["Step1.DateFormat"],
		AutoTpl["Step1.Address"],
		AutoTpl["Step1.ContentChooser"],
	}
	list := []interface{}{
		AutoTpl["Step2.Listadd"],
		AutoTpl["Step2.Listadds"],
		AutoTpl["Step2.BlockChooser"],
		AutoTpl["Step2.AddressChooser"],
		AutoTpl["Step2.TitleChooser"],
		AutoTpl["Step2.DateChooser"],
		AutoTpl["Step2.DateFormat"],
	}
	content := []interface{}{
		AutoTpl["Step3.ContentChooser"],
		AutoTpl["Step3.ElementChooser"],
	}
	param := map[string]interface{}{}
	param["param_common"] = common
	//向导模式
	param["param_time"] = ptime
	param["param_list"] = list
	param["param_content"] = content
	param["type_time"] = 0
	param["type_list"] = 0
	param["type_content"] = 0
	//专家模式
	param["str_time"] = ""
	param["str_list"] = ""
	param["str_content"] = ""
	param["comeintime"] = time.Now().Unix()
	param["code"] = o["code"]
	param["site"] = o["name"]
	param["channel"] = o["channel"]
	param["createuser"] = (*one)["s_name"]
	param["createuserid"] = id
	param["createuseremail"] = (*one)["s_email"]
	param["modifyuser"] = (*one)["s_name"]
	param["modifyuserid"] = id
	param["modifytime"] = time.Now().Unix()
	param["state"] = 0 //未完成
	if qu.IntAll(o["event"]) > 0 {
		param["event"] = qu.IntAll(o["event"])
	}
	s_model := "bid"
	configModel := util.Config.Model[s_model]
	model := map[string]interface{}{}
	for k, _ := range configModel {
		model[k] = qu.ObjToString(o[k])
	}
	param["model"] = model
	param["next"] = o["next"]
	param["urgency"] = o["urgency"]
	param["isflow"] = o["isflow"]
	param["spidertype"] = "history"
	historyevent := qu.ObjToString(o["historyevent"])
	if movevent, ok := util.Config.Uploadevents[historyevent].(string); ok && movevent != "" {
		param["spidermovevent"] = movevent
	} else {
		param["spidermovevent"] = "7700"
	}
	param["historyevent"] = qu.IntAll(o["historyevent"])
	param["platform"] = o["platform"]
	param["weight"] = o["weight"]
	//默认字段
	param["spidercompete"] = true     //2021-11-20后爬虫加此字段（表示新爬虫，剑鱼网站不展示原文）
	param["spiderhistorymaxpage"] = 1 //历史最大页
	param["pendstate"] = 0            //
	param["grade"] = 0                //爬虫难易度（主要用于python爬虫使用）
	//qu.Debug("param---", param)
	ok := spider.SaveSpider(o["code"].(string), param)
	if ok { //保存成功，校验新导入的爬虫对应站点是否存在，否则加站点记录
		if u.MgoEB.Count("site", map[string]interface{}{"site": o["name"]}) == 0 {
			qu.Debug("补充站点信息：", o["name"])
			domain := u.DomainReg.FindString(qu.ObjToString(AutoTpl["Base.SpiderTargetChannelUrl"]))
			if domain != "" {
				domain = u.ReplaceReg.ReplaceAllString(domain, "")
			}
			siteInfo := map[string]interface{}{
				"site":          o["name"],
				"domain":        domain, //
				"another_name":  "",
				"area":          qu.ObjToString(model["area"]),
				"city":          qu.ObjToString(model["city"]),
				"district":      qu.ObjToString(model["district"]),
				"site_type":     "",
				"industry":      "",
				"p_site":        "",
				"s_site":        "",
				"remarktime":    time.Now().Unix(),
				"event":         fmt.Sprint(historyevent),
				"platform":      o["platform"],
				"spider_status": "0/1",
				"updatetime":    time.Now().Unix(),
				"delete":        false,
				"comeintime":    time.Now().Unix(),
				"important":     0,
			}
			u.MgoEB.Save("site", siteInfo)
		}
	}
	return ok, (*one)["s_name"].(string)
}

func (f *Front) Importdata() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth == role_admin {
		if f.Method() == "GET" {
			f.Render("import.html")
		} else {
			rss, _ := u.MgoEB.Find("import", nil, `{"timestamp": -1}`, nil, false, -1, -1)
			f.ServeJson(map[string]interface{}{
				"data": *rss,
			})
		}
	} else {
		f.Write("您没有导入脚本的权限")
	}
}

func Wlog(name, code, man, manid, types string, content map[string]interface{}) {
	obj := map[string]interface{}{
		"name":    name,
		"code":    code,
		"man":     man,
		"manid":   manid,
		"types":   types,
		"time":    time.Now().Unix(),
		"content": content,
	}
	u.MgoEB.Save("lua_logs", obj)
}

func (f *Front) Oldedit() {
	if f.Method() == "GET" {
		f.Render("oldedit.html")
	} else {

	}
}

func (f *Front) FindName() {
	words := f.GetString("words")
	if words == "" {
		f.ServeJson(map[string]interface{}{"error": "null"})
	}
	//query := bson.M{"$or": []interface{}{
	//	bson.M{"param_common.0": bson.M{"$regex": words}},
	//	bson.M{"param_common.1": bson.M{"$regex": words}},
	//	bson.M{"createuser": bson.M{"$regex": words}},
	//}, "oldlua": bson.M{"$exists": false}}
	query := map[string]interface{}{
		"$or": []interface{}{
			map[string]interface{}{"param_common.0": map[string]interface{}{"$regex": words}},
			map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": words}},
			map[string]interface{}{"createuser": map[string]interface{}{"$regex": words}},
		},
		"oldlua": map[string]interface{}{
			"$exists": false,
		},
	}
	//rs := *mgdb.Find("luaconfig", query, bson.M{"modifytime": -1}, bson.M{"param_common": 1}, false, -1, -1)
	rs, _ := u.MgoEB.Find("luaconfig", query, map[string]interface{}{"modifytime": -1}, map[string]interface{}{"param_common": 1}, false, -1, -1)
	if len(*rs) > 0 {
		f.ServeJson(map[string]interface{}{"data": (*rs)})
	} else {
		f.ServeJson(map[string]interface{}{"error": "data"})
	}
}

//分配爬虫
func (f *Front) Assign() {
	auth := qu.IntAll(f.GetSession("auth"))
	if auth != role_admin {
		f.Write("n")
		return
	}
	codes := f.GetString("codes")
	userid := f.GetString("userid")       //分配给谁
	codesarr := strings.Split(codes, ",") //被分配的爬虫
	user, _ := u.MgoEB.FindById("user", userid, nil)
	if user != nil && len(*user) > 0 {
		name := qu.ObjToString((*user)["s_name"])
		query := map[string]interface{}{
			"code": map[string]interface{}{
				"$in": codesarr,
			},
		}
		qu.Debug(query)
		set := map[string]interface{}{
			"$set": map[string]interface{}{
				"createuserid":    userid,
				"createuser":      name,
				"createuseremail": (*user)["s_email"],
				"modifyuser":      name,
				"modifyuserid":    userid,
			},
		}
		//b := u.MgoE.Update("luaconfig", query, set, false, true)
		b := u.MgoEB.Update("luaconfig", query, set, false, true)
		if b {
			go editModify(codesarr, userid, name) //修改爬虫对应任务的维护人
			f.Write("y")
		} else {
			f.Write("n")
		}
	} else {
		f.Write("null")
	}

}

//修改维护人
func editModify(codesarr []string, userid, name string) {
	qu.Debug("修改爬虫对应任务的维护人：", name, codesarr)
	//修改modifyid和modify
	query := map[string]interface{}{
		"s_code": map[string]interface{}{
			"$in": codesarr,
		},
		"i_state": map[string]interface{}{
			"$nin": []int{4, 6},
		},
	}
	set := map[string]interface{}{
		"$set": map[string]interface{}{
			"s_modify":   name,
			"s_modifyid": userid,
		},
	}
	b := u.MgoEB.Update("task", query, set, false, true)
	qu.Debug("任务重新分配成功", b)
}

//清理Redis
func (f *Front) DelRedis() {
	hrefs := f.GetString("href")
	hrefsarr := strings.Split(hrefs, ",")
	auth := qu.IntAll(f.GetSession("auth"))
	err := []string{}
	if auth == role_admin { //权限控制
		if len(hrefsarr) > 0 {
			for k1, h := range hrefsarr {
				href := Reg.FindString(h)
				if href != "" {
					href = "url_repeat_" + href + "*"
					res := redis.GetKeysByPattern("title_repeat_judgement", href)
					if res != nil {
						for _, v := range res {
							hf := string(v.([]uint8))
							b := redis.Del("title_repeat_judgement", hf)
							if !b {
								err = append(err, "第"+strconv.Itoa(k1+1)+"个")
							}
						}
					}
				} else {
					err = append(err, "第"+strconv.Itoa(k1+1)+"个")
				}
			}
		}
	} else {
		err = append(err, "没有权限")
	}
	f.ServeJson(err)
}

func (f *Front) UpdatePendState() {
	success := false
	pendstate, _ := f.GetInteger("pendstate")
	code := f.GetString("code")
	stype := f.GetString("stype")
	qu.Debug(code, stype, pendstate)
	//success = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
	success = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate, "pendtime": time.Now().Unix()}}, false, false)
	if success {
		if stype == "code" { //爬虫挂起时，关闭所有任务
			u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$ne": 6}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6, "l_pendtime": time.Now().Unix()}}, false, true)
		} else if stype == "task" { //任务挂起任务时任务关闭
			id := f.GetString("id")
			u.MgoEB.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_pendstate": pendstate, "i_state": 6, "l_pendtime": time.Now().Unix()}})
		}
	}
	f.ServeJson(map[string]interface{}{"success": success})
}

func (f *Front) UpdateESP() {
	val := f.GetString("val")
	w := f.GetString("w")
	id := f.GetString("id")
	code := f.GetString("c")
	query := map[string]interface{}{
		"_id": mongodb.StringTOBsonId(id),
	}
	set := map[string]interface{}{}
	update := map[string]interface{}{
		"$set": set,
	}
	//one, _ := u.MgoE.FindById("luaconfig", id, nil)
	one, _ := u.MgoEB.FindById("luaconfig", id, nil)
	if len(*one) == 0 {
		f.Write("n")
		return
	}
	if w == "urgency" { //修改紧急度
		urgency, _ := f.GetInteger("val")
		set["urgency"] = urgency
	} else if w == "state" { //无效爬虫改为待完成
		tmpEvent := qu.IntAll((*one)["event"])
		if one != nil && len(*one) > 0 {
			if (*one)["historyevent"] == nil && tmpEvent != 7000 { //除7000节点外没有historyevent的要重新设置
				set["historyevent"] = tmpEvent
			}
		}
		set["state"] = 0
		set["event"] = 7000
		set["spidertype"] = "history"
		//更新、新建任务
		task, _ := u.MgoEB.FindOne("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}})
		text := "指定追加描述：------------------------------\n无发布转待完成\n"
		if len(*task) > 0 { //已有任务，更新
			u.MgoEB.UpdateById("task", (*task)["_id"],
				map[string]interface{}{
					"$set": map[string]interface{}{
						"i_state":      2,
						"s_descript":   qu.ObjToString((*task)["s_descript"]) + text,
						"l_updatetime": time.Now().Unix(),
					},
				})
		} else {
			save := map[string]interface{}{
				"s_channel":           (*one)["channel"],
				"i_event":             (*one)["event"],
				"i_frequencyerrtimes": 0,
				"i_state":             2,
				"s_descript":          text,
				"l_complete":          time.Now().Unix() + int64(24*3600),
				"s_urgency":           "4",
				"i_pendstate":         0,
				"s_modifyid":          (*one)["modifyuserid"],
				"s_source":            "人工",
				"i_times":             0,
				"l_comeintime":        time.Now().Unix(),
				"i_num":               0,
				"s_modify":            (*one)["modifyuser"],
				"s_code":              code,
				"s_site":              (*one)["site"],
				"s_type":              "0",
			}
			u.MgoEB.Save("task", save)
		}
	} else if w == "platform" {
		set["platform"] = val
		set["comeintime"] = time.Now().Unix()
		if val != "golua平台" { //由lua平台切换到其他平台，删除心跳;爬虫下架;修改爬虫状态
			b := u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
			qu.Debug("Del Heart:", b)
			b, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*one)["event"])) //下架
			if b && err == nil {
				//历史节点下架为了避免线上运行爬虫待完成时改为7000采集历史，但是又转到其他平台，导致原线上运行节点爬虫并未下线，心跳异常
				if historyevent := (*one)["historyevent"]; historyevent != nil {
					b, err = spider.UpdateSpiderByCodeState(code, "6", qu.IntAll(historyevent))
				}
				//关闭任务
				u.MgoEB.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$in": []int{0, 1, 2, 3, 5}}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
				if b && err == nil {
					set["state"] = 0 //更新状态
					set["luauser"] = map[string]interface{}{
						"createuser":      (*one)["createuser"],
						"createuserid":    (*one)["createuserid"],
						"createuseremail": (*one)["createuseremail"],
						"modifyuser":      (*one)["modifyuser"],
						"modifyuserid":    (*one)["modifyuserid"],
					}
				} else {
					qu.Debug("历史节点下架失败")
					f.Write("n")
					return
				}
			} else {
				qu.Debug("增量节点下架失败")
				f.Write("n")
				return
			}
			//qu.Debug("下架:", upresult, code)
			//b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
			//qu.Debug("爬虫下架成功：", b)
			//if !b || err != nil {
			//	f.Write("n")
			//	return
			//}
		}
	} else { //修改节点
		event, _ := strconv.Atoi(val)
		set["event"] = event
		set["historyevent"] = event
		//state := f.GetString("s")
		//if state == "5" { //已上架状态改为下架
		code := f.GetString("c")
		set["state"] = 6
		b, err := UpStateAndUpSpider(code, "", "", "", Sp_state_6) //线上爬虫下架
		qu.Debug("爬虫下架成功：", b)
		if !b || err != nil {
			f.Write("n")
			return
		}
		//}
	}
	//if mgdb.Update("luaconfig", query, update, false, false) {
	if u.MgoEB.Update("luaconfig", query, update, false, false) {
		log.Println("Id:", id, "	Update", w, val, "Success")
		f.Write("y")
		return
	} else {
		log.Println("Id:", id, "	Update", w, val, "Failed")
		f.Write("n")
		return
	}
	f.Write("n")
}

func (f *Front) GetCity() {
	area := f.GetString("area")
	cityArr := []string{}
	cityArr = u.Province[area]
	f.ServeJson(cityArr)
}

func (f *Front) SpiderCopy() {
	defer qu.Catch()
	code := f.GetString("code")
	code = strings.TrimSpace(code)
	success := false
	text := ""
	encode := ""
	if code != "" {
		//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
		if len(*lua) > 0 {
			codeOld := qu.ObjToString((*lua)["code"])
			codeNew := codeOld + "_bu"
			//luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
			luaTmp, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
			if len(*luaTmp) > 0 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "已存在补漏爬虫"})
				return
			}
			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
			(*lua)["code"] = codeNew
			(*lua)["relatecode"] = codeOld
			if param_common := (*lua)["param_common"].([]interface{}); len(param_common) < 13 {
				f.ServeJson(map[string]interface{}{"success": false, "text": "脚本异常，请联系管理员"})
				return
			} else {
				param_common = param_common[:13]
				param_common[0] = codeNew                  //code
				param_common = append(param_common, true)  //历史补漏
				param_common = append(param_common, false) //强制下载
				(*lua)["param_common"] = param_common
			}
			str_list := qu.ObjToString((*lua)["str_list"])
			(*lua)["str_list"] = strings.ReplaceAll(str_list, codeOld, codeNew)
			(*lua)["comeintime"] = time.Now().Unix()
			(*lua)["modifytime"] = time.Now().Unix()
			(*lua)["spidertype"] = "increment"
			(*lua)["event"] = 7000
			(*lua)["state"] = 6
			id := u.MgoEB.Save("luaconfig", (*lua))
			if id != "" {
				success = true
				encode = util.Se.Encode2Hex(codeNew)
			} else {
				text = "新爬虫保存失败"
			}
		} else {
			text = "未查询到复制原"
		}
	} else {
		text = "脚本不能为空"
	}
	f.ServeJson(map[string]interface{}{"success": success, "text": text, "encode": encode})
}