|
@@ -12,7 +12,6 @@ import (
|
|
|
"mongodb"
|
|
|
"os"
|
|
|
qu "qfw/util"
|
|
|
- "qfw/util/redis"
|
|
|
"regexp"
|
|
|
"sort"
|
|
|
"spider"
|
|
@@ -27,24 +26,24 @@ import (
|
|
|
|
|
|
type Front struct {
|
|
|
*xweb.Action
|
|
|
- login xweb.Mapper `xweb:"/"`
|
|
|
- logout xweb.Mapper `xweb:"/center/logout"` //退出
|
|
|
- loadIndex xweb.Mapper `xweb:"/center"` //控制中心
|
|
|
- spidernew xweb.Mapper `xweb:"/center/spider"` //爬虫新建
|
|
|
- reg xweb.Mapper `xweb:"/center/reg"` //爬虫注册
|
|
|
- assign xweb.Mapper `xweb:"/center/user/assign"` //分配爬虫
|
|
|
- auditExport xweb.Mapper `xweb:"/center/user/auditexport"` //导出审核日志
|
|
|
- loadSpider xweb.Mapper `xweb:"/center/spider/edit/(.*)"` //爬虫加载
|
|
|
- viewSpider xweb.Mapper `xweb:"/center/spider/view/(.*)"` //爬虫查看
|
|
|
- downSpider xweb.Mapper `xweb:"/center/spider/download/(.*)"` //爬虫下载
|
|
|
- upState xweb.Mapper `xweb:"/center/spider/upstate"` //爬虫状态更新
|
|
|
- assort xweb.Mapper `xweb:"/center/spider/assort"` //审核人员分类(无发布、需登录、无法处理、需删除)
|
|
|
- batchShelves xweb.Mapper `xweb:"/center/spider/batchShelves"` //批量上下架
|
|
|
- checktime xweb.Mapper `xweb:"/center/spider/checktime"` //爬虫核对
|
|
|
- disables xweb.Mapper `xweb:"/center/spider/disable"` //批量作废
|
|
|
- changeEvent xweb.Mapper `xweb:"/center/changeEvent"` //节点更新
|
|
|
- getJson xweb.Mapper `xweb:"/center/spider/json"` //
|
|
|
- delRedis xweb.Mapper `xweb:"/center/spider/delRedis"` //清理Redis
|
|
|
+ login xweb.Mapper `xweb:"/"`
|
|
|
+ logout xweb.Mapper `xweb:"/center/logout"` //退出
|
|
|
+ loadIndex xweb.Mapper `xweb:"/center"` //控制中心
|
|
|
+ spidernew xweb.Mapper `xweb:"/center/spider"` //爬虫新建
|
|
|
+ reg xweb.Mapper `xweb:"/center/reg"` //爬虫注册
|
|
|
+ assign xweb.Mapper `xweb:"/center/user/assign"` //分配爬虫
|
|
|
+ auditExport xweb.Mapper `xweb:"/center/user/auditexport"` //导出审核日志
|
|
|
+ loadSpider xweb.Mapper `xweb:"/center/spider/edit/(.*)"` //爬虫加载
|
|
|
+ viewSpider xweb.Mapper `xweb:"/center/spider/view/(.*)"` //爬虫查看
|
|
|
+ downSpider xweb.Mapper `xweb:"/center/spider/download/(.*)"` //爬虫下载
|
|
|
+ upState xweb.Mapper `xweb:"/center/spider/upstate"` //爬虫状态更新
|
|
|
+ assort xweb.Mapper `xweb:"/center/spider/assort"` //审核人员分类(无发布、需登录、无法处理、需删除)
|
|
|
+ batchShelves xweb.Mapper `xweb:"/center/spider/batchShelves"` //批量上下架
|
|
|
+ checktime xweb.Mapper `xweb:"/center/spider/checktime"` //爬虫核对
|
|
|
+ disables xweb.Mapper `xweb:"/center/spider/disable"` //批量作废
|
|
|
+ changeEvent xweb.Mapper `xweb:"/center/changeEvent"` //节点更新
|
|
|
+ getJson xweb.Mapper `xweb:"/center/spider/json"` //
|
|
|
+ //delRedis xweb.Mapper `xweb:"/center/spider/delRedis"` //清理Redis
|
|
|
updateESP xweb.Mapper `xweb:"/center/spider/updateesp"` //修改爬虫的节点/状态/平台
|
|
|
updatePendState xweb.Mapper `xweb:"/center/spider/updatePendState"` //更新爬虫挂起状态
|
|
|
tagCode xweb.Mapper `xweb:"/center/spider/tagcode"` //标记爬虫
|
|
@@ -84,6 +83,9 @@ type Front struct {
|
|
|
saveNewUser xweb.Mapper `xweb:"/center/user/saveNewUser"` //添加用户
|
|
|
|
|
|
getCity xweb.Mapper `xweb:"/center/getCity"` //获取城市
|
|
|
+
|
|
|
+ //补采信息
|
|
|
+ supplementList xweb.Mapper `xweb:"/center/supplement/list"` //脚本管理
|
|
|
}
|
|
|
|
|
|
const Sp_state_0, Sp_state_1, Sp_state_2, Sp_state_3, Sp_state_4, Sp_state_5, Sp_state_6, Sp_state_7, Sp_state_8, Sp_state_9, Sp_state_10 = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 //0待完成,1待审核,2打回,3发布,4作废,5已上架,6已下架,7无发布,8需登录,9转python,10已删除
|
|
@@ -98,282 +100,6 @@ var ListFilterReg = regexp.MustCompile("关键词过滤")
|
|
|
var ProjectHrefReg = regexp.MustCompile("projecthref")
|
|
|
var Transfercode map[string]interface{}
|
|
|
|
|
|
-func (f *Front) Login() error {
|
|
|
- username := f.GetString("username")
|
|
|
- password := f.GetString("password")
|
|
|
- f.SetSession("password", password)
|
|
|
- password = util.Se.EncodeString(password)
|
|
|
- query := map[string]interface{}{
|
|
|
- "s_name": username,
|
|
|
- "s_pass": password,
|
|
|
- }
|
|
|
- user, _ := u.MgoEB.FindOne("user", query)
|
|
|
- if user != nil && len(*user) > 0 && qu.IntAll((*user)["i_delete"]) == 0 {
|
|
|
- f.SetSession("userid", mongodb.BsonIdToSId((*user)["_id"]))
|
|
|
- f.SetSession("username", (*user)["s_fullname"])
|
|
|
- f.SetSession("loginuser", (*user)["s_name"])
|
|
|
- f.SetSession("email", (*user)["s_email"])
|
|
|
- f.SetSession("auth", (*user)["i_auth"])
|
|
|
- f.SetSession("platform", (*user)["s_platform"])
|
|
|
- f.SetSession("identity", (*user)["i_identity"])
|
|
|
- comeintime := time.Unix((*user)["l_comeintime"].(int64), 0).Format("2006-01-02")
|
|
|
- f.SetSession("comeintime", comeintime)
|
|
|
- if qu.IntAll((*user)["i_auth"]) > u.Role_Admin {
|
|
|
- return f.Redirect("/center/user.html")
|
|
|
- } else if qu.IntAll((*user)["i_auth"]) == u.Role_Dev {
|
|
|
- return f.Redirect("/center/mytask")
|
|
|
- } else {
|
|
|
- return f.Redirect("/center")
|
|
|
- }
|
|
|
- } else {
|
|
|
- if username != "" {
|
|
|
- f.T["fail"] = "fail"
|
|
|
- }
|
|
|
- return f.Render("login.html", &f.T)
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// 用户管理
|
|
|
-func (f *Front) User() {
|
|
|
- if f.Method() == "POST" {
|
|
|
- auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- if auth > u.Role_Admin {
|
|
|
- start, _ := f.GetInteger("start")
|
|
|
- limit, _ := f.GetInteger("length")
|
|
|
- draw, _ := f.GetInteger("draw")
|
|
|
- platform := f.GetString("s_platform")
|
|
|
- identity, _ := f.GetInteger("i_identity")
|
|
|
- userauth, _ := f.GetInteger("i_auth")
|
|
|
- searchStr := f.GetString("search[value]")
|
|
|
- search := strings.TrimSpace(searchStr)
|
|
|
- query := map[string]interface{}{
|
|
|
- "i_delete": 0, //可用用户
|
|
|
- "i_auth": map[string]interface{}{
|
|
|
- "$lt": 4,
|
|
|
- },
|
|
|
- }
|
|
|
- if platform != "-1" {
|
|
|
- query["s_platform"] = platform
|
|
|
- }
|
|
|
- if identity != -1 {
|
|
|
- query["i_identity"] = identity
|
|
|
- }
|
|
|
- if userauth != -1 {
|
|
|
- query["i_auth"] = userauth
|
|
|
- }
|
|
|
- if search != "" {
|
|
|
- query["$or"] = []interface{}{
|
|
|
- map[string]interface{}{"s_name": map[string]interface{}{"$regex": search}},
|
|
|
- map[string]interface{}{"s_fullname": map[string]interface{}{"$regex": search}},
|
|
|
- }
|
|
|
- }
|
|
|
- sort := `{"%s":%d}`
|
|
|
- orderIndex := f.GetString("order[0][column]")
|
|
|
- orderName := f.GetString(fmt.Sprintf("columns[%s][data]", orderIndex))
|
|
|
- orderType := 1
|
|
|
- if f.GetString("order[0][dir]") != "desc" {
|
|
|
- orderType = -1
|
|
|
- }
|
|
|
- sort = fmt.Sprintf(sort, orderName, orderType)
|
|
|
- qu.Debug(query, sort)
|
|
|
- user, _ := u.MgoEB.Find("user", query, sort, nil, false, start, limit)
|
|
|
- count := u.MgoEB.Count("user", query)
|
|
|
- page := start / 10
|
|
|
- for k, v := range *user {
|
|
|
- v["num"] = k + 1 + page*10
|
|
|
- v["s_pass"] = util.Se.DecodeString(v["s_pass"].(string))
|
|
|
- }
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "draw": draw,
|
|
|
- "data": user,
|
|
|
- "recordsFiltered": count,
|
|
|
- "recordsTotal": count,
|
|
|
- })
|
|
|
- }
|
|
|
- } else {
|
|
|
- f.Render("user.html")
|
|
|
- }
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-// 删除用户
|
|
|
-func (f *Front) DelUser() {
|
|
|
- userid := f.GetString("userid")
|
|
|
- auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- if auth > u.Role_Admin {
|
|
|
- query := map[string]interface{}{
|
|
|
- "_id": mongodb.StringTOBsonId(userid),
|
|
|
- }
|
|
|
-
|
|
|
- update := map[string]interface{}{
|
|
|
- "$set": map[string]interface{}{
|
|
|
- "i_delete": 1,
|
|
|
- },
|
|
|
- }
|
|
|
- ok := u.MgoEB.Update("user", query, update, false, false)
|
|
|
- if ok {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "y",
|
|
|
- })
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "n",
|
|
|
- })
|
|
|
- }
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "e",
|
|
|
- })
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// 修改用户
|
|
|
-func (f *Front) UpdateUser() {
|
|
|
- // username := f.GetString("username")
|
|
|
- password := f.GetString("password")
|
|
|
- userid := f.GetString("userid")
|
|
|
- f.SetSession("password", password)
|
|
|
- userAuth := f.GetString("userAuth")
|
|
|
- auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- self := f.GetString("self")
|
|
|
- identity, _ := f.GetInteger("identity")
|
|
|
- platform := f.GetString("platform")
|
|
|
- //log.Println("userid----:", userid, "platform----:", platform, "identity----:", identity, "password----:", password, "auth----:", auth, "self----:", self)
|
|
|
- query := map[string]interface{}{
|
|
|
- "_id": mongodb.StringTOBsonId(userid),
|
|
|
- }
|
|
|
- update := map[string]interface{}{}
|
|
|
- if "y" == self { //修改个人信息 只修改个人密码
|
|
|
- password = util.Se.EncodeString(password)
|
|
|
- update["s_pass"] = password
|
|
|
- } else if "y" != self && auth == 4 { //修改他人信息
|
|
|
- update["i_identity"] = identity
|
|
|
- update["s_platform"] = platform
|
|
|
- if userAuth == "开发员" {
|
|
|
- update["i_auth"] = 1
|
|
|
- } else if userAuth == "审核员" {
|
|
|
- update["i_auth"] = 2
|
|
|
- } else if userAuth == "管理员" {
|
|
|
- update["i_auth"] = 3
|
|
|
- }
|
|
|
- } else {
|
|
|
- f.ServeJson("没有权限!")
|
|
|
- return
|
|
|
- }
|
|
|
- ok := u.MgoEB.Update("user", query, map[string]interface{}{"$set": update}, false, false)
|
|
|
- if ok {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "y",
|
|
|
- })
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "n",
|
|
|
- })
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-func (f *Front) CheckUsenamer() {
|
|
|
- username := f.GetString("username")
|
|
|
- if username != "" {
|
|
|
- query := map[string]interface{}{
|
|
|
- "s_name": username,
|
|
|
- }
|
|
|
- user, _ := u.MgoEB.FindOne("user", query)
|
|
|
- if user != nil && len(*user) > 0 {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "hasUser",
|
|
|
- })
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "notHasUser",
|
|
|
- })
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-func (f *Front) CheckEmail() {
|
|
|
- email := f.GetString("email")
|
|
|
- //校验邮箱
|
|
|
- if email != "" {
|
|
|
- query := map[string]interface{}{
|
|
|
- "s_email": email,
|
|
|
- }
|
|
|
- user, _ := u.MgoEB.FindOne("user", query)
|
|
|
- if user != nil && len(*user) > 0 {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "hasEmail",
|
|
|
- })
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "notHasEmail",
|
|
|
- })
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-// 新增用户
|
|
|
-func (f *Front) SaveNewUser() {
|
|
|
- auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- if auth != 4 {
|
|
|
- f.ServeJson("没有权限!")
|
|
|
- return
|
|
|
- } else {
|
|
|
- i_auth := 1
|
|
|
- username := f.GetString("username")
|
|
|
- password := f.GetString("password")
|
|
|
- relname := f.GetString("relname")
|
|
|
- password = util.Se.EncodeString(password)
|
|
|
- email := f.GetString("email")
|
|
|
- userAuth := f.GetString("userAuth")
|
|
|
- platform := f.GetString("platform")
|
|
|
- identity, _ := f.GetInteger("identity")
|
|
|
- if userAuth == "开发员" {
|
|
|
- i_auth = 1
|
|
|
- } else if userAuth == "审核员" {
|
|
|
- i_auth = 2
|
|
|
- } else {
|
|
|
- i_auth = 3
|
|
|
- }
|
|
|
- time := time.Now().Unix()
|
|
|
- save := map[string]interface{}{
|
|
|
- "s_name": username,
|
|
|
- "s_fullname": relname,
|
|
|
- "s_email": email,
|
|
|
- "s_pass": password,
|
|
|
- "i_auth": i_auth,
|
|
|
- "i_delete": 0,
|
|
|
- "l_comeintime": time,
|
|
|
- "i_identity": identity,
|
|
|
- "i_scope": -1,
|
|
|
- "s_platform": platform,
|
|
|
- }
|
|
|
- ok := u.MgoEB.Save("user", save)
|
|
|
- if ok != "" {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "y",
|
|
|
- })
|
|
|
- } else {
|
|
|
- f.ServeJson(map[string]interface{}{
|
|
|
- "status": "n",
|
|
|
- })
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
-}
|
|
|
-
|
|
|
-func (f *Front) Logout() {
|
|
|
- email := f.GetSession("email").(string)
|
|
|
- f.DelSession("username")
|
|
|
- f.DelSession("userid")
|
|
|
- f.DelSession("email")
|
|
|
- f.DelSession("user")
|
|
|
- f.DelSession("loginuser")
|
|
|
- f.DelSession("auth")
|
|
|
- delete(SessMap, email)
|
|
|
- f.Redirect("/center")
|
|
|
-}
|
|
|
-
|
|
|
// 控制中心
|
|
|
func (f *Front) LoadIndex() {
|
|
|
auth := qu.IntAll(f.GetSession("auth"))
|
|
@@ -1054,39 +780,6 @@ func editModify(codesarr []string, userid, name string) {
|
|
|
qu.Debug("任务重新分配成功", b)
|
|
|
}
|
|
|
|
|
|
-// 清理Redis
|
|
|
-func (f *Front) DelRedis() {
|
|
|
- hrefs := f.GetString("href")
|
|
|
- hrefsarr := strings.Split(hrefs, ",")
|
|
|
- auth := qu.IntAll(f.GetSession("auth"))
|
|
|
- err := []string{}
|
|
|
- if auth == u.Role_Admin { //权限控制
|
|
|
- if len(hrefsarr) > 0 {
|
|
|
- for k1, h := range hrefsarr {
|
|
|
- href := Reg.FindString(h)
|
|
|
- if href != "" {
|
|
|
- href = "url_repeat_" + href + "*"
|
|
|
- res := redis.GetKeysByPattern("title_repeat_judgement", href)
|
|
|
- if res != nil {
|
|
|
- for _, v := range res {
|
|
|
- hf := string(v.([]uint8))
|
|
|
- b := redis.Del("title_repeat_judgement", hf)
|
|
|
- if !b {
|
|
|
- err = append(err, "第"+strconv.Itoa(k1+1)+"个")
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- } else {
|
|
|
- err = append(err, "第"+strconv.Itoa(k1+1)+"个")
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- } else {
|
|
|
- err = append(err, "没有权限")
|
|
|
- }
|
|
|
- f.ServeJson(err)
|
|
|
-}
|
|
|
-
|
|
|
func (f *Front) UpdatePendState() {
|
|
|
success := false
|
|
|
pendstate, _ := f.GetInteger("pendstate")
|
|
@@ -1347,3 +1040,36 @@ func (f *Front) SpiderSplitCopy() {
|
|
|
}
|
|
|
f.ServeJson(map[string]interface{}{"text": text})
|
|
|
}
|
|
|
+
|
|
|
+/*// 清理Redis
|
|
|
+func (f *Front) DelRedis() {
|
|
|
+ hrefs := f.GetString("href")
|
|
|
+ hrefsarr := strings.Split(hrefs, ",")
|
|
|
+ auth := qu.IntAll(f.GetSession("auth"))
|
|
|
+ err := []string{}
|
|
|
+ if auth == u.Role_Admin { //权限控制
|
|
|
+ if len(hrefsarr) > 0 {
|
|
|
+ for k1, h := range hrefsarr {
|
|
|
+ href := Reg.FindString(h)
|
|
|
+ if href != "" {
|
|
|
+ href = "url_repeat_" + href + "*"
|
|
|
+ res := redis.GetKeysByPattern("title_repeat_judgement", href)
|
|
|
+ if res != nil {
|
|
|
+ for _, v := range res {
|
|
|
+ hf := string(v.([]uint8))
|
|
|
+ b := redis.Del("title_repeat_judgement", hf)
|
|
|
+ if !b {
|
|
|
+ err = append(err, "第"+strconv.Itoa(k1+1)+"个")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ err = append(err, "第"+strconv.Itoa(k1+1)+"个")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ err = append(err, "没有权限")
|
|
|
+ }
|
|
|
+ f.ServeJson(err)
|
|
|
+}*/
|