luaconfig表迁移

src/autoimport.json

@@ -4,13 +4,13 @@
     "Base.SpiderChannel": "",
     "Base.SpiderDownDetailPage": true,
     "Base.SpiderStartPage": 1,
-    "Base.SpiderMaxPage": 1,
+    "Base.SpiderMaxPage": 2,
     "Base.Spider2Collection": "",
     "Base.SpiderPageEncoding": "utf8",
     "Base.spiderLastDownloadTime": "2016-09-01 00:00:00",
     "Base.SpiderStoreMode": 1,
     "Base.SpiderStoreToMsgEvent": 4002,
-    "Base.SpiderRunRate": 30,
+    "Base.SpiderRunRate": 10,
     "Base.SpiderTargetChannelUrl": "",
 	"Base.SpiderIsHistoricalMend": false,
 	"Base.SpiderIsMustDownload": false,

src/config.json

@@ -3,6 +3,13 @@
     "dbaddr": "192.168.3.207:27092",
     "dbname": "editor",
     "dbname2": "spider",
+    "bideditor": {
+        "addr": "192.168.3.207:27092",
+        "db": "editor",
+        "size": 5,
+        "username": "",
+        "password": ""
+    },
     "sitecoll": "site",
     "udport": 1499,
     "udpaddr": "127.0.0.1",
@@ -26,17 +33,18 @@
     "msgname":"editor",
     "uploadevents": {
         "7100": "bid",
-        "7110": "comm",
-		"7410": "bid",
-		"7200": "comm",
-		"7210": "comm",
-		"7300": "comm",
-	    "7310": "comm",
-		"7400": "bid",
-		"7000": "bid",
-		"7500": "comm",
-		"7510": "bid",
-		"7700": "comm"
+        "7110": "bid",
+        "7410": "bid",
+        "7200": "comm",
+        "7210": "comm",
+        "7300": "comm",
+        "7310": "comm",
+        "7400": "bid",
+        "7000": "comm",
+        "7500": "comm",
+        "7510": "comm",
+        "7520": "comm",
+        "7700": "comm"
     },
     "serveraddress": "127.0.0.1:8030",
     "word":{

src/front/front.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"mongodb"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
 	"qfw/util/redis"
 	"regexp"
 	"sort"
@@ -22,7 +22,6 @@ import (
 	"github.com/go-xweb/xweb"
 	"github.com/lauyoume/gopinyin"
 	"github.com/tealeg/xlsx"
-	"gopkg.in/mgo.v2/bson"
 )
 
 type Front struct {
@@ -37,7 +36,7 @@ type Front struct {
 	viewSpider      xweb.Mapper `xweb:"/center/spider/view/(.*)"`       //爬虫查看
 	downSpider      xweb.Mapper `xweb:"/center/spider/download/(.*)"`   //爬虫下载
 	upState         xweb.Mapper `xweb:"/center/spider/upstate"`         //爬虫状态更新
-	assort          xweb.Mapper `xweb:"/center/spider/assort"`          //审核人员分类
+	assort          xweb.Mapper `xweb:"/center/spider/assort"`          //审核人员分类(无发布、需登录、无法处理、需删除)
 	batchShelves    xweb.Mapper `xweb:"/center/spider/batchShelves"`    //批量上下架
 	checktime       xweb.Mapper `xweb:"/center/spider/checktime"`       //爬虫核对
 	disables        xweb.Mapper `xweb:"/center/spider/disable"`         //批量作废
@@ -101,7 +100,7 @@ var LuaStateMap = map[int]string{
 	6:  "已下架",
 	7:  "无发布",
 	8:  "需登录",
-	9:  "无法处理",
+	9:  "转python",
 	10: "已删除",
 }
 
@@ -110,23 +109,23 @@ func (f *Front) Login() error {
 	password := f.GetString("password")
 	f.SetSession("password", password)
 	password = util.Se.EncodeString(password)
-	query := bson.M{
+	query := map[string]interface{}{
 		"s_name": username,
 		"s_pass": password,
 	}
-	user := *mgdb.FindOne("user", query)
-	if user != nil && user["i_delete"] == 0 {
-		f.SetSession("userid", user["_id"].(bson.ObjectId).Hex())
-		f.SetSession("username", user["s_fullname"])
-		f.SetSession("loginuser", user["s_name"])
-		f.SetSession("email", user["s_email"])
-		f.SetSession("auth", user["i_auth"])
-		f.SetSession("paltform", user["s_platform"])
-		comeintime := time.Unix(user["l_comeintime"].(int64), 0).Format("2006-01-02")
+	user, _ := u.MgoE.FindOne("user", query)
+	if user != nil && len(*user) > 0 && qu.IntAll((*user)["i_delete"]) == 0 {
+		f.SetSession("userid", mongodb.BsonIdToSId((*user)["_id"]))
+		f.SetSession("username", (*user)["s_fullname"])
+		f.SetSession("loginuser", (*user)["s_name"])
+		f.SetSession("email", (*user)["s_email"])
+		f.SetSession("auth", (*user)["i_auth"])
+		f.SetSession("platform", (*user)["s_platform"])
+		comeintime := time.Unix((*user)["l_comeintime"].(int64), 0).Format("2006-01-02")
 		f.SetSession("comeintime", comeintime)
-		if qu.IntAll(user["i_auth"]) > role_admin {
+		if qu.IntAll((*user)["i_auth"]) > role_admin {
 			return f.Redirect("/center/user.html")
-		} else if qu.IntAll(user["i_auth"]) == role_dev {
+		} else if qu.IntAll((*user)["i_auth"]) == role_dev {
 			return f.Redirect("/center/mytask")
 		} else {
 			return f.Redirect("/center")
@@ -147,21 +146,21 @@ func (f *Front) User() {
 			start, _ := f.GetInteger("start")
 			limit, _ := f.GetInteger("length")
 			draw, _ := f.GetInteger("draw")
-			query := bson.M{
+			query := map[string]interface{}{
 				"i_delete": 0, //可用用户
-				"i_auth": bson.M{
+				"i_auth": map[string]interface{}{
 					"$lt": auth,
 				},
 			}
-			user := *mgdb.Find("user", query, nil, nil, false, start, limit)
-			count := mgdb.Count("user", query)
+			user, _ := u.MgoE.Find("user", query, nil, nil, false, start, limit)
+			count := u.MgoE.Count("user", query)
 			page := start / 10
-			for k, v := range user {
+			for k, v := range *user {
 				v["num"] = k + 1 + page*10
 
 				v["l_comeintime"] = time.Unix(v["l_comeintime"].(int64), 0).Format("2006-01-02")
 				v["s_pass"] = util.Se.DecodeString(v["s_pass"].(string))
-				v["userid"] = v["_id"].(bson.ObjectId).Hex()
+				v["userid"] = mongodb.BsonIdToSId(v["_id"])
 			}
 			f.ServeJson(map[string]interface{}{
 				"draw":            draw,
@@ -181,16 +180,16 @@ func (f *Front) DelUser() {
 	userid := f.GetString("userid")
 	auth := qu.IntAll(f.GetSession("auth"))
 	if auth > role_admin {
-		query := bson.M{
-			"_id": bson.ObjectIdHex(userid),
+		query := map[string]interface{}{
+			"_id": mongodb.StringTOBsonId(userid),
 		}
 
-		update := bson.M{
-			"$set": bson.M{
+		update := map[string]interface{}{
+			"$set": map[string]interface{}{
 				"i_delete": 1,
 			},
 		}
-		ok := mgdb.Update("user", query, update, false, false)
+		ok := u.MgoE.Update("user", query, update, false, false)
 		if ok {
 			f.ServeJson(map[string]interface{}{
 				"status": "y",
@@ -217,14 +216,14 @@ func (f *Front) UpdateUser() {
 	auth := qu.IntAll(f.GetSession("auth"))
 	self := f.GetString("self")
 	//log.Println("userid----:", userid, "username----:", username, "password----:", password, "auth----:", auth, "self----:", self)
-	query := bson.M{
-		"_id": bson.ObjectIdHex(userid),
+	query := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(userid),
 	}
-	update := bson.M{}
+	update := map[string]interface{}{}
 	if "y" == self { //修改个人信息 只修改个人密码
 		password = util.Se.EncodeString(password)
-		update = bson.M{
-			"$set": bson.M{
+		update = map[string]interface{}{
+			"$set": map[string]interface{}{
 				"s_pass": password,
 			},
 		}
@@ -232,20 +231,20 @@ func (f *Front) UpdateUser() {
 		if userAuth == "开发员" || userAuth == "审核员" || userAuth == "管理员" {
 			switch userAuth {
 			case "开发员":
-				update = bson.M{
-					"$set": bson.M{
+				update = map[string]interface{}{
+					"$set": map[string]interface{}{
 						"i_auth": 1,
 					},
 				}
 			case "审核员":
-				update = bson.M{
-					"$set": bson.M{
+				update = map[string]interface{}{
+					"$set": map[string]interface{}{
 						"i_auth": 2,
 					},
 				}
 			case "管理员":
-				update = bson.M{
-					"$set": bson.M{
+				update = map[string]interface{}{
+					"$set": map[string]interface{}{
 						"i_auth": 3,
 					},
 				}
@@ -255,7 +254,7 @@ func (f *Front) UpdateUser() {
 		f.ServeJson("没有权限！")
 		return
 	}
-	ok := mgdb.Update("user", query, update, false, false)
+	ok := u.MgoE.Update("user", query, update, false, false)
 	if ok {
 		f.ServeJson(map[string]interface{}{
 			"status": "y",
@@ -270,11 +269,11 @@ func (f *Front) UpdateUser() {
 func (f *Front) CheckUsenamer() {
 	username := f.GetString("username")
 	if username != "" {
-		query := bson.M{
+		query := map[string]interface{}{
 			"s_name": username,
 		}
-		user := *mgdb.FindOne("user", query)
-		if user != nil {
+		user, _ := u.MgoE.FindOne("user", query)
+		if user != nil && len(*user) > 0 {
 			f.ServeJson(map[string]interface{}{
 				"status": "hasUser",
 			})
@@ -291,11 +290,11 @@ func (f *Front) CheckEmail() {
 	email := f.GetString("email")
 	//校验邮箱
 	if email != "" {
-		query := bson.M{
+		query := map[string]interface{}{
 			"s_email": email,
 		}
-		user := *mgdb.FindOne("user", query)
-		if user != nil {
+		user, _ := u.MgoE.FindOne("user", query)
+		if user != nil && len(*user) > 0 {
 			f.ServeJson(map[string]interface{}{
 				"status": "hasEmail",
 			})
@@ -329,7 +328,7 @@ func (f *Front) SaveNewUser() {
 			i_auth = 3
 		}
 		time := time.Now().Unix()
-		save := bson.M{
+		save := map[string]interface{}{
 			"s_name":       username,
 			"s_fullname":   relname,
 			"s_email":      email,
@@ -338,7 +337,7 @@ func (f *Front) SaveNewUser() {
 			"i_delete":     0,
 			"l_comeintime": time,
 		}
-		ok := mgdb.Save("user", save)
+		ok := u.MgoE.Save("user", save)
 		if ok != "" {
 			f.ServeJson(map[string]interface{}{
 				"status": "y",
@@ -379,7 +378,7 @@ func (f *Front) LoadIndex() {
 		state, _ := f.GetInteger("state")
 		urgency, _ := f.GetInteger("urgency") //节点
 		platform := f.GetString("platform")
-		query := bson.M{}
+		query := map[string]interface{}{}
 		if event > -1 {
 			query["event"] = event
 		}
@@ -391,9 +390,9 @@ func (f *Front) LoadIndex() {
 		}
 		if search != "" {
 			query["$or"] = []interface{}{
-				bson.M{"code": bson.M{"$regex": search}},
-				bson.M{"createuser": bson.M{"$regex": search}},
-				bson.M{"param_common.1": bson.M{"$regex": search}},
+				map[string]interface{}{"code": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"createuser": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": search}},
 			}
 		}
 		if auth == role_examine { //审核员
@@ -428,9 +427,11 @@ func (f *Front) LoadIndex() {
 		page := start / 10
 		//log.Println("sort", sort, orderName)
 		qu.Debug("query:", query, "sort:", sort)
-		luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
-		count := mgdb.Count("luaconfig", query)
-		for k, v := range luas {
+		//luas := *mgdb.Find("luaconfig", query, sort, list_fields, false, start, limit)
+		//count := mgdb.Count("luaconfig", query)
+		luas, _ := u.MgoEB.Find("luaconfig", query, sort, list_fields, false, start, limit)
+		count := u.MgoEB.Count("luaconfig", query)
+		for k, v := range *luas {
 			v["num"] = k + 1 + page*10
 			if v["modifytime"] != nil {
 				v["modifytime"] = time.Unix(v["modifytime"].(int64), 0).Format("2006-01-02 15:04:05")
@@ -462,8 +463,9 @@ func (f *Front) LoadIndex() {
 
 func (f *Front) Checkrepeat() {
 	code := f.GetString("code")
-	one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
-	if len(one) > 0 {
+	//one := *mgdb.FindOne("luaconfig", bson.M{"code": code})
+	one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+	if len(*one) > 0 {
 		f.ServeJson("y")
 	} else {
 		f.ServeJson("n")
@@ -479,13 +481,14 @@ func (f *Front) Spidernew() error {
 	}
 	copy := f.GetString("copy")
 	if copy != "" {
-		one := *mgdb.FindOne("luaconfig", bson.M{"code": copy})
-		delete(one, "_id")
-		delete(one, "code")
-		base := one["param_common"].([]interface{})
+		//one := *mgdb.FindOne("luaconfig", bson.M{"code": copy})
+		one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": copy})
+		delete((*one), "_id")
+		delete((*one), "code")
+		base := (*one)["param_common"].([]interface{})
 		base[0] = ""
 		base[1] = ""
-		one["param_common"] = base
+		(*one)["param_common"] = base
 		f.T["lua"] = one
 	}
 	f.T["isflow"] = 1 //新建爬虫时，初始化isflow的值
@@ -562,16 +565,13 @@ func (f *Front) Importfile() {
 						o["weight"] = weigh
 						//存储表
 						o["coll"] = cells[14].Value
-						//导入默认字段
-						o["spidercompete"] = true
-						o["pendstate"] = 0
 						//table := cells[6].Value
 						//o["table"] = table
 						//o["transfercode"] = qu.IntAll(Transfercode[table])
 
-						query := bson.M{"code": cells[1].Value}
-						rs := *mgdb.FindOne("import", query)
-						if len(rs) > 0 {
+						query := map[string]interface{}{"code": cells[1].Value}
+						rs, _ := u.MgoE.FindOne("import", query)
+						if len(*rs) > 0 {
 							errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行重复,已经过滤"
 						} else {
 							ok, name := savelua(o) //保存爬虫
@@ -579,7 +579,7 @@ func (f *Front) Importfile() {
 								errorinfo[cells[1].Value] = "第" + strconv.Itoa(k) + "行找不到作者,已经过滤"
 							} else {
 								o["author"] = name
-								mgdb.Save("import", o)
+								u.MgoE.Save("import", o)
 							}
 						}
 					}
@@ -598,9 +598,9 @@ func savelua(o map[string]interface{}) (bool, string) {
 	AutoTpl["Base.SpiderChannel"] = o["channel"]
 	AutoTpl["Base.SpiderTargetChannelUrl"] = o["channeladdr"]
 	author := o["author"].(string)
-	one := *mgdb.FindOne("user", bson.M{"s_email": author})
-	id := one["_id"].(bson.ObjectId).Hex()
-	if len(one) == 0 {
+	one, _ := u.MgoE.FindOne("user", map[string]interface{}{"s_email": author})
+	id := mongodb.BsonIdToSId((*one)["_id"])
+	if len(*one) == 0 {
 		return false, ""
 	}
 	common := []interface{}{
@@ -657,10 +657,10 @@ func savelua(o map[string]interface{}) (bool, string) {
 	param["code"] = o["code"]
 	param["site"] = o["name"]
 	param["channel"] = o["channel"]
-	param["createuser"] = one["s_name"]
+	param["createuser"] = (*one)["s_name"]
 	param["createuserid"] = id
-	param["createuseremail"] = one["s_email"]
-	param["modifyuser"] = one["s_name"]
+	param["createuseremail"] = (*one)["s_email"]
+	param["modifyuser"] = (*one)["s_name"]
 	param["modifyuserid"] = id
 	param["modifytime"] = time.Now().Unix()
 	param["state"] = 0 //未完成
@@ -685,14 +685,46 @@ func savelua(o map[string]interface{}) (bool, string) {
 		param["spidermovevent"] = "7700"
 	}
 	param["historyevent"] = qu.IntAll(o["historyevent"])
-	param["spiderhistorymaxpage"] = 1
 	param["platform"] = o["platform"]
-	param["spidercompete"] = o["spidercompete"]
 	param["weight"] = o["weight"]
-	param["pendstate"] = o["pendstate"]
+	//默认字段
+	param["spidercompete"] = true     //2021-11-20后爬虫加此字段（表示新爬虫，剑鱼网站不展示原文）
+	param["spiderhistorymaxpage"] = 1 //历史最大页
+	param["pendstate"] = 0            //
+	param["grade"] = 0                //爬虫难易度（主要用于python爬虫使用）
 	//qu.Debug("param---", param)
-	issave := spider.SaveSpider(o["code"].(string), param)
-	return issave, one["s_name"].(string)
+	ok := spider.SaveSpider(o["code"].(string), param)
+	if ok { //保存成功，校验新导入的爬虫对应站点是否存在，否则加站点记录
+		if u.MgoE.Count("site", map[string]interface{}{"site": o["name"]}) == 0 {
+			qu.Debug("补充站点信息：", o["name"])
+			domain := u.DomainReg.FindString(qu.ObjToString(AutoTpl["Base.SpiderTargetChannelUrl"]))
+			if domain != "" {
+				domain = u.ReplaceReg.ReplaceAllString(domain, "")
+			}
+			siteInfo := map[string]interface{}{
+				"site":          o["name"],
+				"domain":        domain, //
+				"another_name":  "",
+				"area":          qu.ObjToString(model["area"]),
+				"city":          qu.ObjToString(model["city"]),
+				"district":      qu.ObjToString(model["district"]),
+				"site_type":     "",
+				"industry":      "",
+				"p_site":        "",
+				"s_site":        "",
+				"remarktime":    time.Now().Unix(),
+				"event":         fmt.Sprint(historyevent),
+				"platform":      o["platform"],
+				"spider_status": "0/1",
+				"updatetime":    time.Now().Unix(),
+				"delete":        false,
+				"comeintime":    time.Now().Unix(),
+				"important":     0,
+			}
+			u.MgoE.Save("site", siteInfo)
+		}
+	}
+	return ok, (*one)["s_name"].(string)
 }
 
 func (f *Front) Importdata() {
@@ -701,9 +733,9 @@ func (f *Front) Importdata() {
 		if f.Method() == "GET" {
 			f.Render("import.html")
 		} else {
-			rss := *mgdb.Find("import", nil, `{"timestamp": -1}`, nil, false, -1, -1)
+			rss, _ := u.MgoE.Find("import", nil, `{"timestamp": -1}`, nil, false, -1, -1)
 			f.ServeJson(map[string]interface{}{
-				"data": rss,
+				"data": *rss,
 			})
 		}
 	} else {
@@ -712,7 +744,7 @@ func (f *Front) Importdata() {
 }
 
 func Wlog(name, code, man, manid, types string, content map[string]interface{}) {
-	obj := bson.M{
+	obj := map[string]interface{}{
 		"name":    name,
 		"code":    code,
 		"man":     man,
@@ -721,7 +753,7 @@ func Wlog(name, code, man, manid, types string, content map[string]interface{})
 		"time":    time.Now().Unix(),
 		"content": content,
 	}
-	mgdb.Save("lua_logs", obj)
+	u.MgoE.Save("lua_logs", obj)
 }
 
 func (f *Front) Oldedit() {
@@ -735,18 +767,29 @@ func (f *Front) Oldedit() {
 func (f *Front) FindName() {
 	words := f.GetString("words")
 	if words == "" {
-		f.ServeJson(bson.M{"error": "null"})
-	}
-	query := bson.M{"$or": []interface{}{
-		bson.M{"param_common.0": bson.M{"$regex": words}},
-		bson.M{"param_common.1": bson.M{"$regex": words}},
-		bson.M{"createuser": bson.M{"$regex": words}},
-	}, "oldlua": bson.M{"$exists": false}}
-	rs := *mgdb.Find("luaconfig", query, bson.M{"modifytime": -1}, bson.M{"param_common": 1}, false, -1, -1)
-	if len(rs) > 0 {
-		f.ServeJson(bson.M{"data": rs})
+		f.ServeJson(map[string]interface{}{"error": "null"})
+	}
+	//query := bson.M{"$or": []interface{}{
+	//	bson.M{"param_common.0": bson.M{"$regex": words}},
+	//	bson.M{"param_common.1": bson.M{"$regex": words}},
+	//	bson.M{"createuser": bson.M{"$regex": words}},
+	//}, "oldlua": bson.M{"$exists": false}}
+	query := map[string]interface{}{
+		"$or": []interface{}{
+			map[string]interface{}{"param_common.0": map[string]interface{}{"$regex": words}},
+			map[string]interface{}{"param_common.1": map[string]interface{}{"$regex": words}},
+			map[string]interface{}{"createuser": map[string]interface{}{"$regex": words}},
+		},
+		"oldlua": map[string]interface{}{
+			"$exists": false,
+		},
+	}
+	//rs := *mgdb.Find("luaconfig", query, bson.M{"modifytime": -1}, bson.M{"param_common": 1}, false, -1, -1)
+	rs, _ := u.MgoEB.Find("luaconfig", query, map[string]interface{}{"modifytime": -1}, map[string]interface{}{"param_common": 1}, false, -1, -1)
+	if len(*rs) > 0 {
+		f.ServeJson(map[string]interface{}{"data": (*rs)})
 	} else {
-		f.ServeJson(bson.M{"error": "data"})
+		f.ServeJson(map[string]interface{}{"error": "data"})
 	}
 }
 
@@ -778,7 +821,8 @@ func (f *Front) Assign() {
 				"modifyuserid":    userid,
 			},
 		}
-		b := u.MgoE.Update("luaconfig", query, set, false, true)
+		//b := u.MgoE.Update("luaconfig", query, set, false, true)
+		b := u.MgoEB.Update("luaconfig", query, set, false, true)
 		if b {
 			go editModify(codesarr, userid, name) //修改爬虫对应任务的维护人
 			f.Write("y")
@@ -852,7 +896,8 @@ func (f *Front) UpdatePendState() {
 	code := f.GetString("code")
 	stype := f.GetString("stype")
 	qu.Debug(code, stype, pendstate)
-	success = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
+	//success = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
+	success = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"pendstate": pendstate}}, false, false)
 	if success {
 		if stype == "code" { //爬虫挂起时，关闭所有任务
 			u.MgoE.Update("task", map[string]interface{}{"s_code": code, "i_state": map[string]interface{}{"$ne": 6}}, map[string]interface{}{"$set": map[string]interface{}{"i_state": 6}}, false, true)
@@ -861,7 +906,6 @@ func (f *Front) UpdatePendState() {
 			u.MgoE.UpdateById("task", id, map[string]interface{}{"$set": map[string]interface{}{"i_pendstate": pendstate, "i_state": 6}})
 		}
 	}
-
 	f.ServeJson(map[string]interface{}{"success": success})
 }
 
@@ -871,18 +915,22 @@ func (f *Front) UpdateESP() {
 	id := f.GetString("id")
 	code := f.GetString("c")
 	query := map[string]interface{}{
-		"_id": qu.StringTOBsonId(id),
+		"_id": mongodb.StringTOBsonId(id),
 	}
 	set := map[string]interface{}{}
 	update := map[string]interface{}{
 		"$set": set,
 	}
-	one, _ := u.MgoE.FindById("luaconfig", id, nil)
+	//one, _ := u.MgoE.FindById("luaconfig", id, nil)
+	one, _ := u.MgoEB.FindById("luaconfig", id, nil)
 	if len(*one) == 0 {
 		f.Write("n")
 		return
 	}
-	if w == "state" { //无效爬虫改为待完成
+	if w == "urgency" { //修改紧急度
+		urgency, _ := f.GetInteger("val")
+		set["urgency"] = urgency
+	} else if w == "state" { //无效爬虫改为待完成
 		tmpEvent := qu.IntAll((*one)["event"])
 		if one != nil && len(*one) > 0 {
 			if (*one)["historyevent"] == nil && tmpEvent != 7000 { //除7000节点外没有historyevent的要重新设置
@@ -894,6 +942,7 @@ func (f *Front) UpdateESP() {
 		set["spidertype"] = "history"
 	} else if w == "platform" {
 		set["platform"] = val
+		set["comeintime"] = time.Now().Unix()
 		if val != "golua平台" { //由lua平台切换到其他平台，删除心跳;爬虫下架;修改爬虫状态
 			b := u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
 			qu.Debug("Del Heart:", b)
@@ -946,7 +995,8 @@ func (f *Front) UpdateESP() {
 		}
 		//}
 	}
-	if mgdb.Update("luaconfig", query, update, false, false) {
+	//if mgdb.Update("luaconfig", query, update, false, false) {
+	if u.MgoEB.Update("luaconfig", query, update, false, false) {
 		log.Println("Id:", id, "	Update", w, val, "Success")
 		f.Write("y")
 		return
@@ -973,19 +1023,22 @@ func (f *Front) SpiderCopy() {
 	text := ""
 	encode := ""
 	if code != "" {
-		lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-		if len(lua) > 0 {
-			codeOld := qu.ObjToString(lua["code"])
+		//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+		if len(*lua) > 0 {
+			codeOld := qu.ObjToString((*lua)["code"])
 			codeNew := codeOld + "_bu"
-			luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
-			if len(luaTmp) > 0 {
+			//luaTmp := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
+			luaTmp, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": codeNew})
+			if len(*luaTmp) > 0 {
 				f.ServeJson(map[string]interface{}{"success": false, "text": "已存在补漏爬虫"})
 				return
 			}
-			u.MgoE.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
-			lua["code"] = codeNew
-			lua["relatecode"] = codeOld
-			if param_common := lua["param_common"].([]interface{}); len(param_common) < 13 {
+			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
+			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": codeOld}, map[string]interface{}{"$set": map[string]interface{}{"relatecode": codeNew}}, false, false)
+			(*lua)["code"] = codeNew
+			(*lua)["relatecode"] = codeOld
+			if param_common := (*lua)["param_common"].([]interface{}); len(param_common) < 13 {
 				f.ServeJson(map[string]interface{}{"success": false, "text": "脚本异常，请联系管理员"})
 				return
 			} else {
@@ -993,16 +1046,16 @@ func (f *Front) SpiderCopy() {
 				param_common[0] = codeNew                  //code
 				param_common = append(param_common, true)  //历史补漏
 				param_common = append(param_common, false) //强制下载
-				lua["param_common"] = param_common
+				(*lua)["param_common"] = param_common
 			}
-			str_list := qu.ObjToString(lua["str_list"])
-			lua["str_list"] = strings.ReplaceAll(str_list, codeOld, codeNew)
-			lua["comeintime"] = time.Now().Unix()
-			lua["modifytime"] = time.Now().Unix()
-			lua["spidertype"] = "increment"
-			lua["event"] = 7000
-			lua["state"] = 6
-			id := mgdb.Save("luaconfig", lua)
+			str_list := qu.ObjToString((*lua)["str_list"])
+			(*lua)["str_list"] = strings.ReplaceAll(str_list, codeOld, codeNew)
+			(*lua)["comeintime"] = time.Now().Unix()
+			(*lua)["modifytime"] = time.Now().Unix()
+			(*lua)["spidertype"] = "increment"
+			(*lua)["event"] = 7000
+			(*lua)["state"] = 6
+			id := u.MgoEB.Save("luaconfig", (*lua))
 			if id != "" {
 				success = true
 				encode = util.Se.Encode2Hex(codeNew)

src/front/luamove.go

@@ -6,15 +6,14 @@ import (
 	"io/ioutil"
 	"net/http"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
 	"sort"
 	"spider"
 	util "spiderutil"
 	"strings"
 	"time"
+	u "util"
 
 	"github.com/go-xweb/xweb"
-	"gopkg.in/mgo.v2/bson"
 )
 
 type LuaMove struct {
@@ -46,12 +45,13 @@ func (lm *LuaMove) LuaMoveManager() {
 			lm.T["events"] = events
 			lm.Render("luamovesite.html")
 		} else {
-			query := bson.M{"lm_ismove": true}
-			lualist := *mgdb.Find("luaconfig", query, nil, nil, false, -1, -1)
+			query := map[string]interface{}{"lm_ismove": true}
+			//lualist := *mgdb.Find("luaconfig", query, nil, nil, false, -1, -1)
+			lualist, _ := u.MgoEB.Find("luaconfig", query, nil, nil, false, -1, -1)
 			data := []map[string]interface{}{}
 			siteMap := map[string][]map[string]interface{}{}
 
-			for _, l := range lualist {
+			for _, l := range *lualist {
 				pc := l["param_common"].([]interface{})
 				site := qu.ObjToString(pc[1])
 				href := ""
@@ -165,44 +165,47 @@ func (lm *LuaMove) UpdateEventBySite() {
 		info["lm_movevent"] = movevent
 	}
 	lm.SetSession("sitemap", sitemap)
-	query := bson.M{
-		"code": bson.M{
+	query := map[string]interface{}{
+		"code": map[string]interface{}{
 			"$in": codes,
 		},
 	}
-	set := bson.M{
-		"$set": bson.M{
+	set := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"lm_movevent": movevent,
 		},
 	}
-	mgdb.Update("luaconfig", query, set, false, true)
+	//mgdb.Update("luaconfig", query, set, false, true)
+	u.MgoEB.Update("luaconfig", query, set, false, true)
 }
 
 //爬虫迁移
 func SpiderMoveLua(codes []string) bool {
 	defer qu.Catch()
 	for _, code := range codes {
-		lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-		movevent := qu.IntAll(lua["lm_movevent"])
-		event := qu.IntAll(lua["event"])
-		state := qu.IntAll(lua["state"])
+		//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+		movevent := qu.IntAll((*lua)["lm_movevent"])
+		event := qu.IntAll((*lua)["event"])
+		state := qu.IntAll((*lua)["state"])
 		upresult := true
 		var err interface{}
-		if state < 7 || state == 4 { //无发布、需登录、无法处理、已删除状态无需下架
+		if state < 7 || state == 4 { //无发布、需登录、转python、已删除状态无需下架
 			//下架
 			upresult, err = spider.UpdateSpiderByCodeState(code, "6", event) //脚本下架
 		}
 		if upresult && err == nil { //下架成功，更新节点
 			//更新节点
-			query := bson.M{
+			query := map[string]interface{}{
 				"code": code,
 			}
-			set := bson.M{
-				"$set": bson.M{
+			set := map[string]interface{}{
+				"$set": map[string]interface{}{
 					"event": movevent,
 				},
 			}
-			mgdb.Update("luaconfig", query, set, false, false)
+			//mgdb.Update("luaconfig", query, set, false, false)
+			u.MgoEB.Update("luaconfig", query, set, false, false)
 			//上架
 			if state == 5 { //只有是已上架状态的爬虫上架
 				upresult, err = spider.UpdateSpiderByCodeState(code, "5", movevent) //脚本上架
@@ -225,13 +228,13 @@ func SpiderMoveLua(codes []string) bool {
 func (lm *LuaMove) SpiderCloseMoveLua(site string, codes []string, by string) (bool, bool) {
 	defer qu.Catch()
 	flush := false
-	query := bson.M{
-		"code": bson.M{
+	query := map[string]interface{}{
+		"code": map[string]interface{}{
 			"$in": codes,
 		},
 	}
-	set := bson.M{
-		"$set": bson.M{
+	set := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"lm_ismove": false,
 		},
 	}
@@ -260,7 +263,8 @@ func (lm *LuaMove) SpiderCloseMoveLua(site string, codes []string, by string) (b
 		}
 		lm.SetSession("sitemap", sitemap)
 	}
-	return mgdb.Update("luaconfig", query, set, false, true), flush
+	//return mgdb.Update("luaconfig", query, set, false, true), flush
+	return u.MgoEB.Update("luaconfig", query, set, false, true), flush
 }
 
 //
@@ -275,27 +279,31 @@ func SpiderMoveEvent(data string) {
 	}
 	code := qu.ObjToString(infos[0])
 	//迁移节点并上架
-	lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-	spidertype := qu.ObjToString(lua["spidertype"])
-	event := qu.IntAll(lua["event"])
+	//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+	spidertype := qu.ObjToString((*lua)["spidertype"])
+	event := qu.IntAll((*lua)["event"])
 	var upresult bool
 	set := map[string]interface{}{}
 	qu.Debug("lua move:", code, event)
 	if spidertype == "history" {
-		newevent := GetEvent(code, lua)
+		newevent := GetEvent(code, (*lua))
 		qu.Debug("new event:", newevent)
 		set["event"] = newevent
 		set["spidertype"] = "increment"
-		type_content, _ := lua["type_content"].(int)
-		iscopycontent, _ := lua["iscopycontent"].(bool)
-		str_content := qu.ObjToString(lua["str_content"])
-		str_recontent := qu.ObjToString(lua["str_recontent"])
+		type_content, _ := (*lua)["type_content"].(int)
+		iscopycontent, _ := (*lua)["iscopycontent"].(bool)
+		str_content := qu.ObjToString((*lua)["str_content"])
+		str_recontent := qu.ObjToString((*lua)["str_recontent"])
 		if type_content == 1 && iscopycontent && str_recontent != "" { //三级页是专家模式且有复制三级页代码
 			set["iscopycontent"] = false
 			set["str_content"] = str_recontent
 			set["str_recontent"] = str_content
 		}
-		if mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false) {
+		//if mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false) {
+		//	upresult, err = spider.UpdateSpiderByCodeState(code, "5", newevent) //脚本上架
+		//}
+		if u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false) {
 			upresult, err = spider.UpdateSpiderByCodeState(code, "5", newevent) //脚本上架
 		}
 	}
@@ -304,10 +312,11 @@ func SpiderMoveEvent(data string) {
 		ok = true
 		qu.Debug("Code:", code, "历史迁移到增量节点成功")
 	} else { //上架失败
-		mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"event": event, "state": 6}}, false, false)
+		//mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"event": event, "state": 6}}, false, false)
+		u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"event": event, "state": 6}}, false, false)
 		qu.Debug("Code:", code, "历史迁移到增量节点失败")
 	}
-	mgdb.Save("luamovelog", map[string]interface{}{
+	u.MgoE.Save("luamovelog", map[string]interface{}{
 		"code":       code,
 		"comeintime": time.Now().Unix(),
 		"type":       "movevent",
@@ -332,9 +341,10 @@ func GetEvent(code string, lua map[string]interface{}) int {
 		"param_common.1": site,
 		"state":          5,
 	}
-	tmp := *mgdb.FindOne("luaconfig", query)
-	if tmp != nil && len(tmp) > 0 {
-		return qu.IntAll(tmp["event"])
+	//tmp := *mgdb.FindOne("luaconfig", query)
+	tmp, _ := u.MgoEB.FindOne("luaconfig", query)
+	if tmp != nil && len(*tmp) > 0 {
+		return qu.IntAll((*tmp)["event"])
 	}
 	//3、7700
 	spidermovevent := qu.ObjToString(lua["spidermovevent"])
@@ -347,7 +357,8 @@ func GetEvent(code string, lua map[string]interface{}) int {
 	for k, t := range util.Config.Uploadevents {
 		if qu.ObjToString(t) == spidermovevent { //bid、comm
 			event := qu.IntAll(k)
-			count := mgdb.Count("luaconfig", map[string]interface{}{"state": 5, "event": event})
+			//count := mgdb.Count("luaconfig", map[string]interface{}{"state": 5, "event": event})
+			count := u.MgoEB.Count("luaconfig", map[string]interface{}{"state": 5, "event": event})
 			if num == 0 || count < num {
 				result = event
 				num = count

src/front/site.go

@@ -18,12 +18,13 @@ import (
 
 type Site struct {
 	*xweb.Action
-	siteList      xweb.Mapper `xweb:"/center/site/sitelist"`      //控制中心
-	saveSite      xweb.Mapper `xweb:"/center/site/savesite"`      //新增站点
-	importFile    xweb.Mapper `xweb:"/center/site/importfile"`    //导入站点
-	deleteSite    xweb.Mapper `xweb:"/center/site/delete"`        //删除站点
-	getLuas       xweb.Mapper `xweb:"/center/site/getluas/(.*)"`  //
-	getLuasBySite xweb.Mapper `xweb:"/center/site/getluasbysite"` //
+	siteList        xweb.Mapper `xweb:"/center/site/sitelist"`        //控制中心
+	saveSite        xweb.Mapper `xweb:"/center/site/savesite"`        //新增站点
+	importFile      xweb.Mapper `xweb:"/center/site/importfile"`      //导入站点
+	deleteSite      xweb.Mapper `xweb:"/center/site/delete"`          //删除站点
+	getLuas         xweb.Mapper `xweb:"/center/site/getluas/(.*)"`    //
+	getLuasBySite   xweb.Mapper `xweb:"/center/site/getluasbysite"`   //
+	updateImportant xweb.Mapper `xweb:"/center/site/updateimportant"` //更新是否为重点网站
 }
 
 func (s *Site) SiteList() {
@@ -255,10 +256,22 @@ func (s *Site) GetLuas(site string) {
 func (s *Site) GetLuasBySite() {
 	site := s.GetString("site")
 	qu.Debug(site)
-	luas, _ := util.MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, ``, false, -1, -1)
+	//luas, _ := util.MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, ``, false, -1, -1)
+	luas, _ := util.MgoEB.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, ``, false, -1, -1)
 	for _, l := range *luas {
 		modifytime := qu.Int64All(l["modifytime"])
 		l["modifytime"] = qu.FormatDateByInt64(&modifytime, qu.Date_Full_Layout)
 	}
 	s.ServeJson(map[string]interface{}{"data": luas})
 }
+
+func (s *Site) UpdateImportant() {
+	defer qu.Catch()
+	id := s.GetString("id")
+	important, _ := s.GetInteger("val")
+	if util.MgoE.UpdateById("site", id, map[string]interface{}{"$set": map[string]interface{}{"important": important}}) {
+		s.Write("y")
+		return
+	}
+	s.Write("n")
+}

src/front/spider.go

@@ -85,7 +85,7 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 	} else if taskId == "restate=3" {
 		restate = 3
 	} else {
-		if auth == role_dev && (f.GetSession(taskId) == nil || f.GetSession(taskId) == "") {
+		if auth == role_dev && qu.ObjToString(f.GetSession(taskId)) == "" {
 			xgTime := time.Unix(time.Now().Unix(), 0).Format("2006-01-02 15:04:05")
 			f.SetSession(taskId, xgTime)
 		}
@@ -94,12 +94,14 @@ func (f *Front) LoadSpider(codeTaskIdReState string) error {
 	if f.Method() == "GET" {
 		code := util.Se.Decode4Hex(code)
 		f.T["actiontext"] = "编辑"
-		lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 		auth := qu.IntAll(f.GetSession("auth"))
 		if qu.ObjToString((*lua)["createuserid"]) == f.GetSession("userid").(string) || auth >= 1 {
 			if len(*lua) > 0 {
 				if copy != "" {
-					luacopy, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": copy})
+					//luacopy, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": copy})
+					luacopy, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": copy})
 					if len(*luacopy) > 0 {
 						(*lua)["model"] = (*luacopy)["model"]
 						common_copy := (*luacopy)["param_common"].([]interface{})
@@ -164,7 +166,8 @@ func (f *Front) ViewSpider(id string) error {
 		if f.Method() == "GET" {
 			code := util.Se.Decode4Hex(id)
 			f.T["actiontext"] = "编辑"
-			lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+			//lua, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+			lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 			if len(*lua) > 0 {
 				if (*lua)["listcheck"] != nil {
 					listcheck := (*lua)["listcheck"].(string)
@@ -203,7 +206,8 @@ func (f *Front) ViewSpider(id string) error {
 
 func (f *Front) LoadModel(id string) error {
 	if f.Method() == "GET" {
-		lua, _ := u.MgoE.Find("luaconfig", map[string]interface{}{"code": id}, nil, map[string]interface{}{"model": 1}, true, -1, -1)
+		//lua, _ := u.MgoE.Find("luaconfig", map[string]interface{}{"code": id}, nil, map[string]interface{}{"model": 1}, true, -1, -1)
+		lua, _ := u.MgoEB.Find("luaconfig", map[string]interface{}{"code": id}, nil, map[string]interface{}{"model": 1}, true, -1, -1)
 		if len(*lua) > 0 {
 			f.ServeJson((*lua)[0])
 		}
@@ -217,8 +221,9 @@ func (f *Front) SaveStep() {
 	rep := map[string]interface{}{}
 	if f.GetString("oldlua") != "" {
 		id := f.GetString("code")
-		one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
-		id = (*one)["code"].(string)
+		//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": id})
+		one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": id})
+		id = qu.ObjToString((*one)["code"])
 		script := f.GetStringComm("script")
 		if strings.Index(script, id) == -1 {
 			rep["msg"] = "code/名称都不能更改"
@@ -227,7 +232,8 @@ func (f *Front) SaveStep() {
 		} else {
 			upset := map[string]interface{}{"luacontent": script}
 			upset["modifytime"] = time.Now().Unix()
-			b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
+			//b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
+			b := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": id}, map[string]interface{}{"$set": upset}, true, false)
 			if b {
 				rep["msg"] = "保存成功"
 				rep["code"] = util.Se.Encode2Hex(id)
@@ -238,19 +244,23 @@ func (f *Front) SaveStep() {
 	} else {
 		if f.Base.SpiderName != "" && f.Base.SpiderCode != "" {
 			code := f.Base.SpiderCode
-			one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": f.Base.SpiderCode})
+			//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": f.Base.SpiderCode})
+			one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": f.Base.SpiderCode})
+			//记录上架操作前的第一次保存时的爬虫历史
+			user := f.GetSession("username").(string)
+			LuaSaveLog(f.Base.SpiderCode, user, one, 0)
 			state := qu.IntAllDef((*one)["state"], 0)
 			restate := qu.IntAll((*one)["restate"])
 			comeintime := time.Now().Unix()
 			if len((*one)) > 0 {
 				comeintime = qu.Int64All((*one)["comeintime"])
-				ouserid := (*one)["createuserid"].(string)
+				ouserid := qu.ObjToString((*one)["createuserid"])
 				if ouserid != userid && auth == role_dev {
 					f.Write("权限不够，不能修改他人脚本")
 					return
 				} else {
-					code = (*one)["code"].(string)
-					f.Base.SpiderCode = (*one)["code"].(string)
+					code = qu.ObjToString((*one)["code"])
+					f.Base.SpiderCode = code
 					f.Base.SpiderName = ((*one)["param_common"].([]interface{}))[1].(string)
 				}
 			} else {
@@ -383,7 +393,7 @@ func (f *Front) SaveStep() {
 							param[k] = v
 						}
 					}
-					Wlog(f.Base.SpiderName, f.Base.SpiderCode, f.GetSession("username").(string), f.GetSession("userid").(string), "修改", param)
+					Wlog(f.Base.SpiderName, f.Base.SpiderCode, user, f.GetSession("userid").(string), "修改", param)
 					rep["msg"] = "保存成功"
 				} else {
 					rep["msg"] = "保存失败"
@@ -397,6 +407,53 @@ func (f *Front) SaveStep() {
 	}
 }
 
+func LuaSaveLog(code, user string, data *map[string]interface{}, stype int) {
+	saveOne, _ := u.MgoE.FindOne("luasavelog", map[string]interface{}{"state": 0, "code": code})
+	if stype == 0 { //保存记录
+		if len(*saveOne) == 0 && len(*data) > 0 { //重新记录
+			delete(*data, "_id")
+			save := map[string]interface{}{
+				"code":       code,
+				"state":      0,
+				"saveuser":   user,
+				"comeintime": time.Now().Unix(),
+				"luaold":     data,
+			}
+			u.MgoE.Save("luasavelog", save)
+		}
+	} else if stype == 1 { //对比
+		if len(*saveOne) > 0 {
+			tmp := (*saveOne)["luaold"].(map[string]interface{})
+			updateMap := map[string]interface{}{} //记录字段改变值
+			for k, v := range *data {
+				if k != "_id" && k != "state" && k != "modifytime" {
+					if tmpV := tmp[k]; tmpV != nil { //历史记录存在字段
+						tmpJson, _ := json.Marshal(tmpV)
+						dataJson, _ := json.Marshal(v)
+						if string(tmpJson) != string(dataJson) {
+							updateMap[k] = v
+						}
+						delete(tmp, k) //删除对比过的字段
+					} else { //历史记录不存在字段
+						updateMap[k] = v
+					}
+				}
+			}
+			if len(tmp) > 0 {
+				for k, _ := range tmp { //上架时爬虫较历史爬虫少的字段信息
+					updateMap[k] = nil
+				}
+			}
+			set := map[string]interface{}{"state": 1, "updatetime": time.Now().Unix(), "updateuser": user}
+			if len(updateMap) > 0 { //有字段改变
+				set["luaupdate"] = updateMap
+				set["lusnew"] = data
+			}
+			u.MgoE.UpdateById("luasavelog", (*saveOne)["_id"], map[string]interface{}{"$set": set})
+		}
+	}
+}
+
 //检查列表页和三级页代码中是否含lua原生方法
 func LuaTextCheck(list, detail string, type_list int) (b bool, msg string) {
 	defer qu.Catch()
@@ -415,50 +472,6 @@ func LuaTextCheck(list, detail string, type_list int) (b bool, msg string) {
 	return
 }
 
-func (f *Front) Assort() {
-	state, _ := f.GetInteger("state")
-	code := f.GetString("code")
-	codes := u.SymbolReg.Split(code, -1)
-	success := true
-	msg := ""
-	for _, code := range codes {
-		query := map[string]interface{}{
-			"code": code,
-		}
-		//下架爬虫
-		lua, _ := u.MgoE.FindOne("luaconfig", query)
-		upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
-		qu.Debug("下架爬虫：", code, upresult, err)
-		if upresult && err == nil {
-			//更新爬虫状态
-			update := map[string]interface{}{
-				"$set": map[string]interface{}{
-					"state":      state,
-					"modifytime": time.Now().Unix(),
-				},
-			}
-			u.MgoE.Update("luaconfig", query, update, false, false)
-			//关闭任务
-			query = map[string]interface{}{
-				"s_code": code,
-			}
-			update = map[string]interface{}{
-				"$set": map[string]interface{}{
-					"i_state": 6,
-				},
-			}
-			u.MgoE.Update("task", query, update, false, true)
-			//删除心跳
-			DelSpiderHeart(code)
-		} else {
-			success = false
-			msg += code + ";"
-		}
-	}
-
-	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
-}
-
 //方法测试
 func (f *Front) RunStep() {
 	imodal, _ := f.GetInteger("imodal")
@@ -657,7 +670,8 @@ func (f *Front) SpiderPass() {
 	f.SetSession("task_descript", descript)
 	//基本信息、方法一、方法二、方法三、总请求次数、go方法一、go方法二、go方法三、列表页条数
 	steps := []interface{}{false, false, false, false, 0, 0, 0, 0, 0}
-	one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+	//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+	one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	reason, _ := (*one)["reason"].(string)
 	f.SetSession("reason", reason)
 	if len(*one) > 0 && (*one)["oldlua"] == nil {
@@ -751,7 +765,8 @@ func (f *Front) DownSpider(code string) {
 	script := ""
 	if auth > role_dev {
 		success = true
-		one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		//one, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		one, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 		createuserid := qu.ObjToString((*one)["createuserid"])
 		filename := code + ".lua"
 		if len(*one) > 0 {
@@ -786,81 +801,6 @@ func (f *Front) DownSpider(code string) {
 	u.MgoE.Save("luadownlogs", downlogs)
 }
 
-//更新爬虫状态
-func (f *Front) UpState() error {
-	username := f.GetSession("username").(string)
-	code := f.GetString("code")
-	state, _ := f.GetInt("state")
-	id := f.GetString("taskId")
-	reason := f.GetString("reason")
-	auth := qu.IntAll(f.GetSession("auth"))
-	var codeArr = []string{code}
-	var taskid []string
-	//修改任务状态
-	istotask := false
-	res := map[string]interface{}{
-		"istotask": istotask,
-		"err":      "没有权限",
-		"code":     util.Se.Encode2Hex(code),
-		"taskid":   taskid,
-	}
-	var xgTime int64
-	if f.GetSession(id) == nil || f.GetSession(id) == "" {
-		xgTime = time.Now().Unix()
-	} else {
-		xgTimeStr := qu.ObjToString(f.GetSession(id))
-		xgTimeTmp, _ := time.ParseInLocation("2006-01-02  15:04:05", xgTimeStr, time.Local)
-		xgTime = xgTimeTmp.Unix()
-	}
-	f.DelSession(id)
-	if IsHasUpState(auth, int(state)) {
-		b, err := UpStateAndUpSpider(code, "", reason, username, int(state)) //更新爬虫状态
-		if b && state == Sp_state_1 {                                        //提交审核
-			//有对应任务跳转提交记录页
-			taskid = checkTask(codeArr, 1)
-
-			if len(taskid) > 0 {
-				res["istotask"] = true
-				res["taskid"] = taskid[0]
-			}
-		} else if b && state == Sp_state_2 { //打回
-			taskid = checkTask(codeArr, 2)
-			if len(taskid) > 0 {
-				//UpTaskState([]string{taskid}, 2)     //修改状态
-				UpTaskState(taskid, 2, "", int64(0)) //修改任务状态
-				SaveRemark(taskid, reason, username) //保存记录信息
-			}
-		} else if b && state == Sp_state_3 { //审核通过
-			taskid = checkTask(codeArr, 3)
-			if len(taskid) > 0 {
-				//UpTaskState([]string{taskid}, 3)
-				UpTaskState(taskid, 3, "", int64(0))
-				SaveRemark(taskid, "", username)
-			}
-		} else if b && state == Sp_state_6 { //下架
-			//下架成功删除心跳数据
-			flag := DelSpiderHeart(code)
-			log.Println(code, "---下架删除download数据：", flag)
-		} else if b && state == Sp_state_7 { //反馈
-			taskid = checkTask(codeArr, 7)
-			if len(taskid) > 0 {
-				UpTaskState(taskid, 7, reason, xgTime)
-			}
-		}
-
-		if err != nil {
-			res["err"] = err.Error()
-			f.ServeJson(res)
-		} else {
-			res["err"] = ""
-			f.ServeJson(res)
-		}
-	} else {
-		f.ServeJson(res)
-	}
-	return nil
-}
-
 //下架删除心跳
 func DelSpiderHeart(code string) bool {
 	return u.MgoS.Update("spider_heart", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"del": true}}, false, true)
@@ -886,7 +826,10 @@ func (f *Front) Checktime() {
 	if auth != role_admin {
 		f.ServeJson(false)
 	} else {
-		b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
+		//b := u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
+		//	"l_checktime": time.Now().Unix(),
+		//}}, true, false)
+		b := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{
 			"l_checktime": time.Now().Unix(),
 		}}, true, false)
 		f.ServeJson(b)
@@ -960,15 +903,138 @@ func (f *Front) BatchShelves() {
 	f.ServeJson(errCode)
 }
 
+//更新爬虫状态
+func (f *Front) UpState() error {
+	username := f.GetSession("username").(string)
+	code := f.GetString("code")
+	state, _ := f.GetInt("state")
+	id := f.GetString("taskId")
+	reason := f.GetString("reason")
+	auth := qu.IntAll(f.GetSession("auth"))
+	var codeArr = []string{code}
+	var taskid []string
+	//修改任务状态
+	istotask := false
+	res := map[string]interface{}{
+		"istotask": istotask,
+		"err":      "没有权限",
+		"code":     util.Se.Encode2Hex(code),
+		"taskid":   taskid,
+	}
+	var xgTime int64
+	if f.GetSession(id) == nil || f.GetSession(id) == "" {
+		xgTime = time.Now().Unix()
+	} else {
+		xgTimeStr := qu.ObjToString(f.GetSession(id))
+		xgTimeTmp, _ := time.ParseInLocation("2006-01-02  15:04:05", xgTimeStr, time.Local)
+		xgTime = xgTimeTmp.Unix()
+	}
+	f.DelSession(id)
+	if IsHasUpState(auth, int(state)) {
+		b, err := UpStateAndUpSpider(code, "", reason, username, int(state)) //更新爬虫状态
+		if b && state == Sp_state_1 {                                        //提交审核
+			//有对应任务跳转提交记录页
+			taskid = checkTask(codeArr, 1)
+
+			if len(taskid) > 0 {
+				res["istotask"] = true
+				res["taskid"] = taskid[0]
+			}
+		} else if b && state == Sp_state_2 { //打回
+			taskid = checkTask(codeArr, 2)
+			if len(taskid) > 0 {
+				//UpTaskState([]string{taskid}, 2)     //修改状态
+				UpTaskState(taskid, 2, "", int64(0)) //修改任务状态
+				SaveRemark(taskid, reason, username) //保存记录信息
+			}
+		} else if b && state == Sp_state_3 { //审核通过
+			taskid = checkTask(codeArr, 3)
+			if len(taskid) > 0 {
+				//UpTaskState([]string{taskid}, 3)
+				UpTaskState(taskid, 3, "", int64(0))
+				SaveRemark(taskid, "", username)
+			}
+		} else if b && state == Sp_state_6 { //下架
+			//下架成功删除心跳数据
+			flag := DelSpiderHeart(code)
+			log.Println(code, "---下架删除download数据：", flag)
+		} else if b && state == Sp_state_7 { //反馈
+			taskid = checkTask(codeArr, 7)
+			if len(taskid) > 0 {
+				UpTaskState(taskid, 7, reason, xgTime)
+			}
+		}
+
+		if err != nil {
+			res["err"] = err.Error()
+			f.ServeJson(res)
+		} else {
+			res["err"] = ""
+			f.ServeJson(res)
+		}
+	} else {
+		f.ServeJson(res)
+	}
+	return nil
+}
+func (f *Front) Assort() {
+	state, _ := f.GetInteger("state")
+	code := f.GetString("code")
+	codes := u.SymbolReg.Split(code, -1)
+	success := true
+	msg := ""
+	for _, code := range codes {
+		query := map[string]interface{}{
+			"code": code,
+		}
+		//下架爬虫
+		//lua, _ := u.MgoE.FindOne("luaconfig", query)
+		lua, _ := u.MgoEB.FindOne("luaconfig", query)
+		upresult, err := spider.UpdateSpiderByCodeState(code, "6", qu.IntAll((*lua)["event"]))
+		qu.Debug("下架爬虫：", code, upresult, err)
+		if upresult && err == nil {
+			//更新爬虫状态
+			update := map[string]interface{}{
+				"$set": map[string]interface{}{
+					"state": state,
+					//"modifytime":   time.Now().Unix(),
+					"l_uploadtime": time.Now().Unix(),
+				},
+			}
+			//u.MgoE.Update("luaconfig", query, update, false, false)
+			u.MgoEB.Update("luaconfig", query, update, false, false)
+			//关闭任务
+			query = map[string]interface{}{
+				"s_code": code,
+			}
+			update = map[string]interface{}{
+				"$set": map[string]interface{}{
+					"i_state": 6,
+				},
+			}
+			u.MgoE.Update("task", query, update, false, true)
+			//删除心跳
+			DelSpiderHeart(code)
+		} else {
+			success = false
+			msg += code + ";"
+		}
+	}
+
+	f.ServeJson(map[string]interface{}{"success": success, "msg": msg})
+}
+
 //更新爬虫状态，并判断是否更新节点爬虫
 func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, error) {
 	upresult := false
 	var err error
 	one := &map[string]interface{}{}
 	if code != "" {
-		one, _ = u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		//one, _ = u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+		one, _ = u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	} else {
-		one, _ = u.MgoE.FindById("luaconfig", id, nil)
+		//one, _ = u.MgoE.FindById("luaconfig", id, nil)
+		one, _ = u.MgoEB.FindById("luaconfig", id, nil)
 		code = qu.ObjToString((*one)["code"])
 	}
 	if len(*one) > 0 {
@@ -1013,30 +1079,35 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 		if upresult && err == nil {
 			upset := map[string]interface{}{"state": state} //修改状态
 			if (*one)["oldlua"] != nil {                    //老脚本上传
-				upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
+				//upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
+				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, true, false)
 			} else {
 				if state == Sp_state_1 { //提交审核
 					upset["l_complete"] = time.Now().Unix()
 					upset["report"] = ""
+				} else if state == Sp_state_7 { //反馈问题
+					upset["report"] = reason
+					upset["state"] = 1 //反馈后爬虫改为待审核
+					upset["l_complete"] = time.Now().Unix()
 				} else if state == Sp_state_3 { //审核通过
 					if (*one)["event"] == nil {
 						upset["event"] = event
-						upset["modifytime"] = time.Now().Unix()
+						//upset["modifytime"] = time.Now().Unix()
 					}
 					upset["frequencyerrtimes"] = 0 //爬虫审核通过，重置采集频率异常次数
 					upset["l_uploadtime"] = time.Now().Unix()
 				} else if state == Sp_state_2 { //打回原因
 					upset["reason"] = reason
-				} else if state == Sp_state_7 { //反馈问题
-					upset["report"] = reason
-					upset["state"] = 1 //反馈后爬虫改为待审核
 				} else if state == Sp_state_5 { //上架，核对时间重置
 					upset["l_checktime"] = 0
+					LuaSaveLog(code, username, one, 1)
 				} else if state == Sp_state_4 { //作废，作废原因
 					upset["disablereason"] = reason
-					upset["modifytime"] = time.Now().Unix()
+					//upset["modifytime"] = time.Now().Unix()
+					upset["l_uploadtime"] = time.Now().Unix() //l_complete爬虫完成时间
 				}
-				upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
+				//upresult = u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
+				upresult = u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": upset}, false, false)
 				qu.Debug("提交日志：", code, upset, upresult)
 				if upresult && (state == Sp_state_2 || state == Sp_state_3) { //打回、审核记录日志
 					types := "打回"
@@ -1051,8 +1122,16 @@ func UpStateAndUpSpider(code, id, reason, username string, state int) (bool, err
 						"reason":     reason,
 						"spideruser": (*one)["createuser"],
 						"modifytime": (*one)["modifytime"],
+						"event":      (*one)["event"],
+					}
+					//新爬虫审核记录表
+					if state == Sp_state_3 {
+						count := u.MgoE.Count("lua_logs_auditor", map[string]interface{}{"code": code, "types": types})
+						if count == 0 { //新爬虫审核记录
+							u.MgoE.Save("lua_logs_auditor_new", obj)
+						}
 					}
-					u.MgoE.Save("lua_logs_auditor", obj)
+					u.MgoE.Save("lua_logs_auditor", obj) //历史维护爬虫审核记录
 				}
 			}
 		}
@@ -1115,20 +1194,24 @@ func UpTaskState(code []string, num int, reason string, startTime int64) {
 		} else if num == 2 { //打回  -->未通过
 			update = map[string]interface{}{
 				"$set": map[string]interface{}{
-					"i_state": 5,
+					"i_state":      5,
+					"l_updatetime": time.Now().Unix(),
 				},
 			}
 		} else if num == 3 { //发布(审核通过)  -->审核通过
 			update = map[string]interface{}{
 				"$set": map[string]interface{}{
-					"i_state": 4,
+					"i_state":      4,
+					"l_updatetime": time.Now().Unix(),
+					"l_uploadtime": time.Now().Unix(),
 				},
 			}
 		} else if num == 4 { //批量作废  -->关闭
 			update = map[string]interface{}{
 				"$set": map[string]interface{}{
-					"i_state":    6,
-					"l_complete": time.Now().Unix(),
+					"i_state":      6,
+					"l_complete":   time.Now().Unix(),
+					"l_updatetime": time.Now().Unix(),
 				},
 			}
 		} else if num == 7 { //反馈信息 -->待审核
@@ -1141,9 +1224,10 @@ func UpTaskState(code []string, num int, reason string, startTime int64) {
 			mrecord = append(mrecord, newData)
 			update = map[string]interface{}{
 				"$set": map[string]interface{}{
-					"i_state":    3,
-					"l_complete": time.Now().Unix(),
-					"a_mrecord":  mrecord,
+					"i_state":      3,
+					"l_complete":   time.Now().Unix(),
+					"a_mrecord":    mrecord,
+					"l_updatetime": time.Now().Unix(),
 				},
 			}
 		}
@@ -1171,7 +1255,8 @@ func (f *Front) ChangeEvent() {
 		f.ServeJson("没有对应节点")
 		return
 	}
-	info, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+	//info, _ := u.MgoE.FindOne("luaconfig", map[string]interface{}{"code": code})
+	info, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	if len(*info) > 0 {
 		oldevent := qu.IntAll((*info)["event"])
 		if qu.IntAll((*info)["state"]) == Sp_state_5 {
@@ -1183,7 +1268,8 @@ func (f *Front) ChangeEvent() {
 					"state": Sp_state_6,
 				},
 			}
-			u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
+			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
+			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
 			if err != nil && strings.Contains(err.Error(), "timeout") {
 				f.ServeJson("连接节点" + fmt.Sprint(oldevent) + "超时")
 			} else {
@@ -1195,7 +1281,8 @@ func (f *Front) ChangeEvent() {
 					"event": qu.IntAll(event),
 				},
 			}
-			u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
+			//u.MgoE.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
+			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, set, true, false)
 		}
 	} else {
 		f.ServeJson("没有对应记录")
@@ -1281,8 +1368,10 @@ func (f *Front) LuaList() {
 		sort = fmt.Sprintf(sort, orderName, orderType)
 		page := start / 10
 		qu.Debug("query:", query)
-		luas, _ := u.MgoE.Find("luaconfig", query, sort, list_fields, false, start, limit)
-		count := u.MgoE.Count("luaconfig", query)
+		//luas, _ := u.MgoE.Find("luaconfig", query, sort, list_fields, false, start, limit)
+		//count := u.MgoE.Count("luaconfig", query)
+		luas, _ := u.MgoEB.Find("luaconfig", query, sort, list_fields, false, start, limit)
+		count := u.MgoEB.Count("luaconfig", query)
 		for k, v := range *luas {
 			v["num"] = k + 1 + page*10
 			l_uploadtime := qu.Int64All(v["l_uploadtime"])
@@ -1354,7 +1443,8 @@ func (f *Front) Heart() {
 		for _, l := range *list {
 			code := qu.ObjToString(l["code"])
 			qu.Debug(code)
-			d, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1, "str_list": 1, "type_list": 1})
+			//d, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1, "str_list": 1, "type_list": 1})
+			d, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"state": 1, "param_common": 1, "str_list": 1, "type_list": 1})
 			l["state"] = (*d)["state"]
 			l["param_common"] = (*d)["param_common"]
 			if lt := qu.Int64All(l["list"]); lt != 0 {

src/luaerrdata/errdata.go

@@ -2,17 +2,16 @@ package luaerrdata
 
 import (
 	"encoding/json"
+	"mongodb"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
-	mgu "qfw/util/mongodbutil"
 	"spider"
 	util "spiderutil"
 	"strings"
 	"time"
 	"udp"
+	u "util"
 
 	"github.com/go-xweb/xweb"
-	"gopkg.in/mgo.v2/bson"
 )
 
 const role_admin, role_examine, role_dev = 3, 2, 1 //管理员，审核员，开发员
@@ -56,48 +55,49 @@ func (ed *ErrorData) ErrorDataIndex() {
 			return
 		}
 		query := map[string]interface{}{
-			"state":      bson.M{"$lte": 3}, //查询state==3及修复成功的数据是为了在页面展示该数据的爬虫方便“更新上架”,“确认修复”
+			"state":      map[string]interface{}{"$lte": 3}, //查询state==3及修复成功的数据是为了在页面展示该数据的爬虫方便“更新上架”,“确认修复”
 			"from":       "lua",
-			"comeintime": bson.M{"$gte": startTime, "$lte": endTime},
+			"comeintime": map[string]interface{}{"$gte": startTime, "$lte": endTime},
 		}
 		if auth == 1 {
 			query["modifyuser"] = user
 		}
 		if search != "" {
-			query = bson.M{"spidercode": bson.M{"$regex": search}}
+			query = map[string]interface{}{"spidercode": map[string]interface{}{"$regex": search}}
 		}
 		qu.Debug("query:", query)
 		data := []map[string]interface{}{}
-		list := *mgu.Find("regatherdata", "spider", "spider", query, nil, nil, false, -1, -1)
-		if len(list) > 0 {
+		list, _ := u.MgoS.Find("regatherdata", query, nil, nil, false, -1, -1)
+		if len(*list) > 0 {
 			//alltmp := map[string]int{}
 			errtmp := map[string]int{}
 			rerrtmp := map[string]int{}
 			saverrtmp := map[string]int{}
 			successtmp := map[string]int{}
 			codeLua := map[string]*LuaInfo{}
-			for _, l := range list {
+			for _, l := range *list {
 				code := qu.ObjToString(l["spidercode"])
 				state := qu.IntAll(l["state"])
 				//alltmp[code] = alltmp[code] + 1
 				if codeLua[code] == nil {
-					data := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-					restate := qu.IntAll(data["restate"])
+					//data := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+					data, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+					restate := qu.IntAll((*data)["restate"])
 					info := &LuaInfo{
-						ModifyUser: qu.ObjToString(data["modifyuser"]),
+						ModifyUser: qu.ObjToString((*data)["modifyuser"]),
 						ReState:    restate,
-						State:      qu.IntAll(data["state"]),
-						Event:      qu.IntAll(data["event"]),
+						State:      qu.IntAll((*data)["state"]),
+						Event:      qu.IntAll((*data)["event"]),
 						StateTime:  int64(0),
 					}
 					if restate == 4 {
-						info.StateTime = qu.Int64All(data["updatetime"])
+						info.StateTime = qu.Int64All((*data)["updatetime"])
 					} else if restate == 3 {
-						info.StateTime = qu.Int64All(data["auditime"])
+						info.StateTime = qu.Int64All((*data)["auditime"])
 					} else if restate == 2 {
-						info.StateTime = qu.Int64All(data["repairtime"])
+						info.StateTime = qu.Int64All((*data)["repairtime"])
 					} else if restate == 1 {
-						info.StateTime = qu.Int64All(data["confirmtime"])
+						info.StateTime = qu.Int64All((*data)["confirmtime"])
 					}
 					codeLua[code] = info
 				}
@@ -240,24 +240,24 @@ func (ed *ErrorData) FindByCode() {
 		limit, _ := ed.GetInteger("length")
 		draw, _ := ed.GetInteger("draw")
 		state, _ := ed.GetInteger("state")
-		q_state := bson.M{}
+		q_state := map[string]interface{}{}
 		if state == -1 {
-			q_state = bson.M{"$lt": 3}
+			q_state = map[string]interface{}{"$lt": 3}
 		} else {
-			q_state = bson.M{"$eq": state}
+			q_state = map[string]interface{}{"$eq": state}
 		}
-		query := bson.M{
+		query := map[string]interface{}{
 			"from":       "lua",
 			"spidercode": code,
 			"state":      q_state,
-			"comeintime": bson.M{"$gte": startTime, "$lte": endTime},
+			"comeintime": map[string]interface{}{"$gte": startTime, "$lte": endTime},
 		}
 		qu.Debug("query:", query)
 		page := start / 10
-		data := *mgu.Find("regatherdata", "spider", "spider", query, `{"state":1}`, `{"href":1,"spidercode":1,"state":1,comeintime:1}`, false, start, limit)
-		count := mgu.Count("regatherdata", "spider", "spider", query)
-		if data != nil {
-			for k, d := range data {
+		data, _ := u.MgoS.Find("regatherdata", query, `{"state":1}`, `{"href":1,"spidercode":1,"state":1,comeintime:1}`, false, start, limit)
+		count := u.MgoS.Count("regatherdata", query)
+		if data != nil && len(*data) > 0 {
+			for k, d := range *data {
 				d["num"] = k + 1 + page*10
 				state := qu.IntAll(d["state"])
 				if state == 0 {
@@ -292,7 +292,7 @@ func (ed *ErrorData) RegatherData() {
 			"isrun":    false,
 			"type":     "code",
 		}
-		if id := mgu.Save("regatherudp", "spider", "spider", save); id != "" {
+		if id := u.MgoS.Save("regatherudp", save); id != "" {
 			qu.Debug("发送udp", id)
 			udpMap := map[string]interface{}{
 				"udpid": id,
@@ -307,9 +307,9 @@ func (ed *ErrorData) RegatherData() {
 					ed.ServeJson(info)
 				case <-time.After(time.Second * 10):
 					go func() {
-						q := map[string]interface{}{"_id": qu.StringTOBsonId(id)}
+						q := map[string]interface{}{"_id": mongodb.StringTOBsonId(id)}
 						s := map[string]interface{}{"$set": map[string]interface{}{"isrun": true, "remark": "Udp发送失败"}}
-						mgu.Update("regatherudp", "spider", "spider", q, s, false, false)
+						u.MgoS.Update("regatherudp", q, s, false, false)
 					}()
 					udp.IsSendUdp = false
 					ed.ServeJson("重新采集失败，请稍后重试")
@@ -345,7 +345,7 @@ func (ed *ErrorData) SingleRegather() {
 		save["codeorid"] = []string{code}
 		save["type"] = "code"
 	}
-	if id := mgu.Save("regatherudp", "spider", "spider", save); id != "" {
+	if id := u.MgoS.Save("regatherudp", save); id != "" {
 		qu.Debug("发送udp", id)
 		udpMap := map[string]interface{}{
 			"udpid": id,
@@ -360,9 +360,9 @@ func (ed *ErrorData) SingleRegather() {
 				ed.ServeJson(info)
 			case <-time.After(time.Second * 10):
 				go func() {
-					q := map[string]interface{}{"_id": qu.StringTOBsonId(id)}
+					q := map[string]interface{}{"_id": mongodb.StringTOBsonId(id)}
 					s := map[string]interface{}{"$set": map[string]interface{}{"isrun": true, "remark": "Udp发送失败"}}
-					mgu.Update("regatherudp", "spider", "spider", q, s, false, false)
+					u.MgoS.Update("regatherudp", q, s, false, false)
 				}()
 				udp.IsSendUdp = false
 				ed.ServeJson("重新采集失败，请稍后重试")
@@ -380,9 +380,11 @@ func (ed *ErrorData) ConfirmLua() {
 	codes := ed.GetString("codes")
 	state := true
 	for _, code := range strings.Split(codes, ",") {
-		lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-		if restate := qu.IntAll(lua["restate"]); restate == 0 || restate == 4 || restate == 3 { //0是未修复过的爬虫；4是已经修复过的爬虫
-			b := mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 1, "confirmtime": time.Now().Unix()}}, false, false)
+		//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+		lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+		if restate := qu.IntAll((*lua)["restate"]); restate == 0 || restate == 4 || restate == 3 { //0是未修复过的爬虫；4是已经修复过的爬虫
+			//b := mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 1, "confirmtime": time.Now().Unix()}}, false, false)
+			b := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 1, "confirmtime": time.Now().Unix()}}, false, false)
 			if !b {
 				state = false
 			}
@@ -395,8 +397,9 @@ func (ed *ErrorData) UpdateRestate() {
 	defer qu.Catch()
 	code := ed.GetString("code")
 	restateTmp, _ := ed.GetInteger("restate")
-	lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-	restate := qu.IntAll(lua["restate"])
+	//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+	restate := qu.IntAll((*lua)["restate"])
 	if restate == 0 {
 		ed.ServeJson("该爬虫未确认")
 		return
@@ -417,7 +420,8 @@ func (ed *ErrorData) UpdateRestate() {
 		set["restate"] = 3
 		set["auditime"] = time.Now().Unix()
 	}
-	b := mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
+	//b := mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
+	b := u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": set}, false, false)
 	text := ""
 	if restateTmp == 1 {
 		if b {
@@ -438,15 +442,17 @@ func (ed *ErrorData) UpdateRestate() {
 func (ed *ErrorData) UpdateOnlineLua() {
 	defer qu.Catch()
 	code := ed.GetString("code")
-	lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-	if qu.IntAll(lua["state"]) != 5 { //非上架爬虫，不能更新
+	//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+	if qu.IntAll((*lua)["state"]) != 5 { //非上架爬虫，不能更新
 		ed.ServeJson(map[string]interface{}{"state": false})
 		return
 	}
-	event := qu.IntAll(lua["event"])
+	event := qu.IntAll((*lua)["event"])
 	b, err := spider.UpdateSpiderByCodeState(code, "-1", event)
 	if b && err == nil {
-		mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false)
+		//mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false)
+		u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false)
 		ed.ServeJson(map[string]interface{}{"state": true})
 	} else {
 		ed.ServeJson(map[string]interface{}{"state": false})
@@ -461,7 +467,10 @@ func (ed *ErrorData) ConfirmRepair() {
 		if code == "" {
 			continue
 		}
-		if !mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false) {
+		//if !mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false) {
+		//	data = append(data, code)
+		//}
+		if !u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"restate": 4, "updatetime": time.Now().Unix()}}, false, false) {
 			data = append(data, code)
 		}
 	}

src/luaerrdata/errlua.go

@@ -1,12 +1,12 @@
 package luaerrdata
 
 import (
+	"mongodb"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
-	mgu "qfw/util/mongodbutil"
 	"spider"
 	util "spiderutil"
 	"time"
+	u "util"
 
 	"github.com/go-xweb/xweb"
 )
@@ -35,11 +35,11 @@ func (el *ErrorLua) ErrorLuaIndex() {
 			}
 		}
 		qu.Debug("query:", query)
-		list := *mgu.Find("spider_loadfail", "spider", "spider", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
-		for _, l := range list {
+		list, _ := u.MgoS.Find("spider_loadfail", query, map[string]interface{}{"_id": -1}, nil, false, -1, -1)
+		for _, l := range *list {
 			l["encode"] = util.Se.Encode2Hex(qu.ObjToString(l["code"]))
 		}
-		el.ServeJson(map[string]interface{}{"data": list})
+		el.ServeJson(map[string]interface{}{"data": *list})
 	} else {
 		el.Render("errlualist.html")
 	}
@@ -49,19 +49,20 @@ func (el *ErrorLua) ReUpSpider() {
 	defer qu.Catch()
 	code := el.GetString("code")
 	id := el.GetString("id")
-	lua := *mgdb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	//lua := *mgdb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	success := false
 	text := ""
-	if len(lua) > 0 {
-		event := qu.IntAll(lua["event"])
-		if qu.IntAll(lua["state"]) == 5 { //是上架状态
+	if len(*lua) > 0 {
+		event := qu.IntAll((*lua)["event"])
+		if qu.IntAll((*lua)["state"]) == 5 { //是上架状态
 			//重新上架
 			b, err := spider.UpdateSpiderByCodeState(code, "5", event)
 			if b && err == nil {
 				success = true
-				go mgu.Update("spider_loadfail", "spider", "spider",
+				go u.MgoS.Update("spider_loadfail",
 					map[string]interface{}{
-						"_id": qu.StringTOBsonId(id),
+						"_id": mongodb.StringTOBsonId(id),
 					},
 					map[string]interface{}{
 						"$set": map[string]interface{}{

src/main.go

@@ -6,6 +6,7 @@ import (
 	"quesManager"
 	"spider"
 	"vps"
+
 	//. "luaweb/task"
 	"luaerrdata"
 	"taskManager"
@@ -14,8 +15,6 @@ import (
 
 	"net/http"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
-	mgu "qfw/util/mongodbutil"
 	"qfw/util/redis"
 	u "util"
 
@@ -40,17 +39,9 @@ func init() {
 	redis.InitRedis(util.Config.Redisservers)
 	//验证码识别client
 	codegrpc.InitCodeGrpcClient()
-	//新建连接
-	conf := *new(mgu.PoolConfig)
-	conf.Addr = util.Config.Dbaddr
-	conf.Alias = util.Config.Dbname2
-	conf.DB = util.Config.Dbname2
-	conf.Size = 10
-	mgu.Config = append(mgu.Config, conf)
-	mgu.InitMongodbPool()
+	//mail
 	smtp := util.Config.Smtp
 	front.Mails = util.New(smtp["addr"], qu.IntAll(smtp["port"]), smtp["user"], smtp["pwd"])
-	mgdb.InitMongodbPool(20, util.Config.Dbaddr, util.Config.Dbname)
 	lua.Disablelib(util.Config.Luadisablelib)
 	//初始化区域信息
 	u.InitAreaCity()
@@ -102,8 +93,6 @@ func init() {
 
 //
 func main() {
-	//定时查询任务发送邮件
-	//go tomail.SendToMail()
 	//定时任务
 	go timetask.TimeTask()
 	//提供接口，接收其他数据

src/main_test.go

@@ -4,10 +4,8 @@ package main
 import (
 	"github.com/lauyoume/gopinyin"
 	"log"
-	mgdb "qfw/util/mongodb"
 	"spider"
 	"testing"
-	"time"
 )
 
 //模板测试
@@ -61,67 +59,67 @@ import (
 //}
 
 //保存更新爬虫
-func Test_saveSpider(t *testing.T) {
-	mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb")
-	param := map[string]interface{}{}
-	//通用变量配置
-	common := []interface{}{
-		"upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002,
-		"http://www.ccgp.gov.cn/zycg/zycgdt/index.htm",
-	}
-	//最新时间配置
-	ptime := []interface{}{
-		"yyyyMMddHHmm",
-		"http://www.ccgp.gov.cn/zycg/zycgdt/",
-		"ul li em:eq(0)",
-	}
-	param["type_time"] = 0 //0向导模式 1专家模式
-
-	//列表页配置
-	list := []interface{}{
-		"http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm",
-		"'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'",
-		"ul#main_list_lt_list li",
-		"a:eq(1):attr(href)",
-		"a:eq(1):attr(title)",
-		"em:eq(0)",
-		"yyyyMMddHHmm",
-	}
-	param["type_list"] = 0 //0向导模式 1专家模式
-
-	//三级页配置
-	content := []interface{}{
-		"div.TRS_Editor",
-		"div.TRS_Editor",
-	}
-	param["type_content"] = 0 //0向导模式 1专家模式
-
-	param["param_common"] = common
-	//向导模式
-	param["param_time"] = ptime
-	param["param_list"] = list
-	param["param_content"] = content
-	//专家模式
-	param["str_time"] = `function getLastPublishTime()
-							local content = download("href",{})
-							return lastpushtime
-						end`
-	param["str_list"] = `function downloadAndParseListPage(pageno)
-	 					end`
-	param["str_content"] = `function downloadDetailPage(data)
- 							end`
-
-	param["comeintime"] = time.Now().Unix()
-	param["model"] = map[string]interface{}{
-		"type":        "tender",
-		"area":        "HA",
-		"city":        "郑州",
-		"publishdept": "郑州市财政厅",
-	} //补充数据模型
-	param["createuser"] = "zjk" //姓名
-	param["upload"] = false     //是否上传
-	spider.SaveSpider("upload_test", param)
-}
+//func Test_saveSpider(t *testing.T) {
+//	mgdb.InitMongodbPool(2, "192.168.3.18:27080", "luaweb")
+//	param := map[string]interface{}{}
+//	//通用变量配置
+//	common := []interface{}{
+//		"upload_test", "中央采购网-上传测试", "测试脚本栏目", true, 1, 10, 30, "bidding", "utf8", 1, 4002,
+//		"http://www.ccgp.gov.cn/zycg/zycgdt/index.htm",
+//	}
+//	//最新时间配置
+//	ptime := []interface{}{
+//		"yyyyMMddHHmm",
+//		"http://www.ccgp.gov.cn/zycg/zycgdt/",
+//		"ul li em:eq(0)",
+//	}
+//	param["type_time"] = 0 //0向导模式 1专家模式
+//
+//	//列表页配置
+//	list := []interface{}{
+//		"http://www.ccgp.gov.cn/zycg/zycgdt/index#pageno#.htm",
+//		"'http://www.ccgp.gov.cn/zycg/zycgdt/index.htm','http://www.ccgp.gov.cn/zycg/zycgdt/index_1.htm'",
+//		"ul#main_list_lt_list li",
+//		"a:eq(1):attr(href)",
+//		"a:eq(1):attr(title)",
+//		"em:eq(0)",
+//		"yyyyMMddHHmm",
+//	}
+//	param["type_list"] = 0 //0向导模式 1专家模式
+//
+//	//三级页配置
+//	content := []interface{}{
+//		"div.TRS_Editor",
+//		"div.TRS_Editor",
+//	}
+//	param["type_content"] = 0 //0向导模式 1专家模式
+//
+//	param["param_common"] = common
+//	//向导模式
+//	param["param_time"] = ptime
+//	param["param_list"] = list
+//	param["param_content"] = content
+//	//专家模式
+//	param["str_time"] = `function getLastPublishTime()
+//							local content = download("href",{})
+//							return lastpushtime
+//						end`
+//	param["str_list"] = `function downloadAndParseListPage(pageno)
+//	 					end`
+//	param["str_content"] = `function downloadDetailPage(data)
+// 							end`
+//
+//	param["comeintime"] = time.Now().Unix()
+//	param["model"] = map[string]interface{}{
+//		"type":        "tender",
+//		"area":        "HA",
+//		"city":        "郑州",
+//		"publishdept": "郑州市财政厅",
+//	} //补充数据模型
+//	param["createuser"] = "zjk" //姓名
+//	param["upload"] = false     //是否上传
+//	spider.SaveSpider("upload_test", param)
+//}
 
 //生成lua文件测试
 //func Test_createFile(t *testing.T) {

src/quesManager/quesManager.go

@@ -4,16 +4,16 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"mongodb"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
+	"util"
 
 	"github.com/go-xweb/xweb"
 	"github.com/tealeg/xlsx"
-	"gopkg.in/mgo.v2/bson"
 )
 
 type QuesM struct {
@@ -39,26 +39,26 @@ func (q *QuesM) ManagerQues() {
 		state, _ := q.GetInteger("state")           //问题状态
 		introStage, _ := q.GetInteger("introStage") //引入阶段
 		//log.Println("start:", start, "	limit:", limit, "	draw:", draw, "	search:", search, "	state", state, "	introStage", introStage)
-		query := bson.M{}
+		query := map[string]interface{}{}
 		if state >= 0 && introStage >= 0 { //同时满足问题状态和引入阶段
-			query = bson.M{
+			query = map[string]interface{}{
 				"i_state":      state,
 				"i_introStage": introStage,
 			}
 		} else if state < 0 && introStage >= 0 { //仅仅搜索满足引入阶段，不搜索问题状态
-			query = bson.M{
+			query = map[string]interface{}{
 				"i_introStage": introStage,
 			}
 		} else if state >= 0 && introStage < 0 { //仅仅搜索满足问题状态，不搜索引入阶段
-			query = bson.M{
+			query = map[string]interface{}{
 				"i_state": state,
 			}
 		}
 		if search != "" {
 			query["$or"] = []interface{}{
-				bson.M{"s_title": bson.M{"$regex": search}},
-				bson.M{"s_id": bson.M{"$regex": search}},
-				bson.M{"s_errFiled": bson.M{"$regex": search}},
+				map[string]interface{}{"s_title": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_id": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_errFiled": map[string]interface{}{"$regex": search}},
 			}
 		}
 		sort := `{"%s":%d}`
@@ -69,14 +69,14 @@ func (q *QuesM) ManagerQues() {
 			orderType = -1
 		}
 		sort = fmt.Sprintf(sort, orderName, orderType)
-		ques := *mgdb.Find("question", query, sort, nil, false, start, limit)
-		count := mgdb.Count("question", query)
+		ques, _ := util.MgoE.Find("question", query, sort, nil, false, start, limit)
+		count := util.MgoE.Count("question", query)
 		page := start / 10
-		if ques != nil {
-			for k, v := range ques {
+		if ques != nil && len(*ques) > 0 {
+			for k, v := range *ques {
 				v["num"] = k + 1 + page*10
-				i_state := v["i_state"]
-				i_introStage := v["i_introStage"]
+				i_state := qu.IntAll(v["i_state"])
+				i_introStage := qu.IntAll(v["i_introStage"])
 				if i_state == 0 { //未处理
 					v["i_state"] = "未处理"
 				} else if i_state == 1 { //已处理
@@ -150,9 +150,9 @@ func (q *QuesM) SaveNewQues() {
 	newQues["s_sugesstion"] = ""          //意见反馈
 
 	//log.Println(title, id, addr, introStage, errFiled, name, descript, endTime)
-	s := mgdb.Save("question", newQues)
+	saveid := util.MgoE.Save("question", newQues)
 	state := "err"
-	if len(s) > 0 {
+	if saveid != "" {
 		state = "ok"
 	}
 	q.ServeJson(map[string]interface{}{
@@ -173,17 +173,17 @@ func (q *QuesM) SaveFeedbackSug() {
 	} else if state == "暂不处理" {
 		i_state = 2
 	}
-	query := bson.M{
-		"_id": bson.ObjectIdHex(_id),
+	query := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(_id),
 	}
-	update := bson.M{
-		"$set": bson.M{
+	update := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"i_state":      i_state,
 			"s_sugesstion": s_sugesstion,
 		},
 	}
 	//	log.Println(_id, s_sugesstion, state, query, update)
-	b := mgdb.Update("question", query, update, false, false)
+	b := util.MgoE.Update("question", query, update, false, false)
 	log.Println("Save:	", _id, "	反馈意见", b)
 	returnState := "false"
 	if b {
@@ -197,15 +197,15 @@ func (q *QuesM) SaveFeedbackSug() {
 //关闭问题
 func (q *QuesM) CloseQues() {
 	_id := q.GetString("id")
-	query := bson.M{
-		"_id": bson.ObjectIdHex(_id),
+	query := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(_id),
 	}
-	update := bson.M{
-		"$set": bson.M{
+	update := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"i_state": 4,
 		},
 	}
-	b := mgdb.Update("question", query, update, false, false)
+	b := util.MgoE.Update("question", query, update, false, false)
 	log.Println("Close :", _id, "	", b)
 	state := "false"
 	if b {
@@ -298,7 +298,7 @@ func arrangeData(title, id, addr, errField, descript, introStage, feedbackPerson
 	fileQues["i_time"] = time.Now().Unix()        //录入时间
 	fileQues["i_state"] = 0                       //问题状态(新建默认为未处理)
 	fileQues["s_sugesstion"] = ""                 //意见反馈
-	mgdb.Save("question", fileQues)
+	util.MgoE.Save("question", fileQues)
 	return ""
 
 }

src/spider/service.go

@@ -8,28 +8,27 @@ import (
 	mu "mfw/util"
 	"os"
 	qu "qfw/util"
-	mongodb "qfw/util/mongodb"
 	util "spiderutil"
 	"strings"
 	"time"
-
-	"gopkg.in/mgo.v2/bson"
+	u "util"
 )
 
 //获取脚本文件
 func GetScript(code string, str ...interface{}) (script, script_list, script_content string) {
 	defer mu.Catch()
 	//script := ""
-	luaconfig := *mongodb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	//luaconfig := *mongodb.FindOne("luaconfig", `{"code":"`+code+`"}`)
+	lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
 	//qu.Debug(code, "lua---", luaconfig)
-	if luaconfig["listcheck"] == nil {
-		luaconfig["listcheck"] = ""
+	if (*lua)["listcheck"] == nil {
+		(*lua)["listcheck"] = ""
 	}
-	if luaconfig["contentcheck"] == nil {
-		luaconfig["contentcheck"] = ""
+	if (*lua)["contentcheck"] == nil {
+		(*lua)["contentcheck"] = ""
 	}
-	if luaconfig != nil && len(luaconfig) > 0 {
-		common := luaconfig["param_common"].([]interface{})
+	if lua != nil && len(*lua) > 0 {
+		common := (*lua)["param_common"].([]interface{})
 		if len(str) > 0 {
 			if len(common) == 15 {
 				common = append(common, str[0], str[1], str[2])
@@ -48,20 +47,20 @@ func GetScript(code string, str ...interface{}) (script, script_list, script_con
 				common[k] = qu.IntAll(v)
 			}
 		}
-
 		script, _ = GetTmpModel(map[string][]interface{}{"common": common})
 		script_time := ""
-		if luaconfig["type_time"] == 0 {
-			time := luaconfig["param_time"].([]interface{})
+		if qu.IntAll((*lua)["type_time"]) == 0 {
+			time := (*lua)["param_time"].([]interface{})
 			script_time, _ = GetTmpModel(map[string][]interface{}{
 				"time": time,
 			})
 		} else {
-			script_time = luaconfig["str_time"].(string)
+			script_time = (*lua)["str_time"].(string)
 		}
 		//script_list := "" //列表页
-		if luaconfig["type_list"] == 0 {
-			list := luaconfig["param_list"].([]interface{})
+
+		if qu.IntAll((*lua)["type_list"]) == 0 {
+			list := (*lua)["param_list"].([]interface{})
 			addrs := strings.Split(list[1].(string), "\n")
 			if len(addrs) > 0 {
 				for k, v := range addrs {
@@ -73,34 +72,35 @@ func GetScript(code string, str ...interface{}) (script, script_list, script_con
 			}
 			script_list, _ = GetTmpModel(map[string][]interface{}{
 				"list":      list,
-				"listcheck": []interface{}{luaconfig["listcheck"]},
+				"listcheck": []interface{}{(*lua)["listcheck"]},
 			})
 		} else {
-			script_list = luaconfig["str_list"].(string)
+			script_list = (*lua)["str_list"].(string)
 		}
 		//script_content := "" //三级页
-		if luaconfig["type_content"] == 0 {
-			content := luaconfig["param_content"].([]interface{})
+		if qu.IntAll((*lua)["type_content"]) == 0 {
+			content := (*lua)["param_content"].([]interface{})
 			script_content, _ = GetTmpModel(map[string][]interface{}{
 				"content":      content,
-				"contentcheck": []interface{}{luaconfig["contentcheck"]},
+				"contentcheck": []interface{}{(*lua)["contentcheck"]},
 			})
 		} else {
-			script_content = luaconfig["str_content"].(string)
+			script_content = (*lua)["str_content"].(string)
 		}
-		script += fmt.Sprintf(util.Tmp_Other, luaconfig["spidertype"], luaconfig["spiderhistorymaxpage"], luaconfig["spidermovevent"], luaconfig["spidercompete"])
+		script += fmt.Sprintf(util.Tmp_Other, (*lua)["spidertype"], (*lua)["spiderhistorymaxpage"], (*lua)["spidermovevent"], (*lua)["spidercompete"])
 		script += ` 
 			` + script_time + `
 			` + script_list + `
 			` + script_content
-		script = ReplaceModel(script, common, luaconfig["model"].(map[string]interface{}))
+		script = ReplaceModel(script, common, (*lua)["model"].(map[string]interface{}))
 	}
 	return
 }
 
 //保存更新脚本
 func SaveSpider(code string, param map[string]interface{}) bool {
-	return mongodb.Update("luaconfig", bson.M{"code": code}, map[string]interface{}{"$set": param}, true, true)
+	//return mongodb.Update("luaconfig", bson.M{"code": code}, map[string]interface{}{"$set": param}, true, true)
+	return u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": param}, true, true)
 }
 
 /*获取最后发布时间

src/task/flush.go

@@ -2,12 +2,11 @@ package task
 
 import (
 	"fmt"
-	mgdb "qfw/util/mongodb"
+	"mongodb"
 	"time"
+	u "util"
 
 	"front"
-
-	"gopkg.in/mgo.v2/bson"
 )
 
 var timer *time.Ticker
@@ -28,30 +27,32 @@ func TimeTask() {
 }
 
 func FlushAuthor() {
-	query := bson.M{
-		"flush": bson.M{"$exists": false},
-		"next":  bson.M{"$exists": true},
-		"l_uploadtime": bson.M{
+	query := map[string]interface{}{
+		"flush": map[string]interface{}{"$exists": false},
+		"next":  map[string]interface{}{"$exists": true},
+		"l_uploadtime": map[string]interface{}{
 			"$lte": time.Now().AddDate(0, 0, -30).Unix(),
 		},
 		"$where": "this.createuseremail != this.next",
 	}
-	rets := *mgdb.Find("luaconfig", query, bson.M{}, bson.M{}, false, -1, -1)
-	for _, v := range rets {
+	//rets := *mgdb.Find("luaconfig", query, bson.M{}, bson.M{}, false, -1, -1)
+	rets, _ := u.MgoEB.Find("luaconfig", query, nil, nil, false, -1, -1)
+	for _, v := range *rets {
 		next, ok := v["next"].(string)
 		if ok {
-			one := *mgdb.FindOne("user", bson.M{"s_email": next})
-			if len(one) > 0 {
-				update := bson.M{
-					"$set": bson.M{
-						"createuser":      one["s_name"],
-						"createuserid":    one["_id"].(bson.ObjectId).Hex(),
+			one, _ := u.MgoE.FindOne("user", map[string]interface{}{"s_email": next})
+			if len(*one) > 0 {
+				update := map[string]interface{}{
+					"$set": map[string]interface{}{
+						"createuser":      (*one)["s_name"],
+						"createuserid":    mongodb.BsonIdToSId((*one)["_id"]),
 						"createuseremail": next,
 						"flush":           "ok",
 					},
 				}
-				front.Wlog("转移爬虫给", v["code"].(string), fmt.Sprint(one["s_name"]), next, "系统定时任务", nil)
-				mgdb.Update("luaconfig", bson.M{"_id": v["_id"]}, update, false, false)
+				front.Wlog("转移爬虫给", v["code"].(string), fmt.Sprint((*one)["s_name"]), next, "系统定时任务", nil)
+				//mgdb.Update("luaconfig", bson.M{"_id": v["_id"]}, update, false, false)
+				u.MgoEB.UpdateById("luaconfig", v["_id"], update)
 			}
 		}
 	}

src/taskManager/taskManager.go

@@ -5,9 +5,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"mongodb"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
-	mgu "qfw/util/mongodbutil"
 	"sort"
 	util "spiderutil"
 	"strconv"
@@ -17,7 +16,6 @@ import (
 
 	"github.com/go-xweb/xweb"
 	"github.com/tealeg/xlsx"
-	"gopkg.in/mgo.v2/bson"
 )
 
 type TaskM struct {
@@ -77,14 +75,14 @@ func (t *TaskM) Taskfile() {
 							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行未指定爬虫"
 							continue
 						}
-						query := bson.M{
+						query := map[string]interface{}{
 							"s_code": code,
-							"i_state": bson.M{
+							"i_state": map[string]interface{}{
 								"$in": []int{0, 1, 2, 3, 5},
 							},
 						}
-						task := *mgdb.FindOne("task", query)
-						if len(task) > 0 { //任务已存在
+						task, _ := u.MgoE.FindOne("task", query)
+						if len(*task) > 0 { //任务已存在
 							errorinfo[cells[0].Value] = "第" + fmt.Sprint(k+1) + "行爬虫任务已存在"
 							continue
 						}
@@ -124,16 +122,17 @@ func (t *TaskM) Taskfile() {
 							timeDate, _ := time.ParseInLocation(qu.Date_Full_Layout, completeTimeStr, time.Local)
 							completeTime = timeDate.Unix()
 						}
-						lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-						if len(lua) > 0 {
-							param := lua["param_common"]
+						//lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
+						lua, _ := u.MgoEB.FindOne("luaconfig", map[string]interface{}{"code": code})
+						if len(*lua) > 0 {
+							param := (*lua)["param_common"]
 							o["s_site"] = param.([]interface{})[1] //取数组中的某个值
 							o["s_channel"] = param.([]interface{})[2]
-							o["i_state"] = 1
-							o["s_modify"] = lua["createuser"]
-							o["s_modifyid"] = lua["createuserid"]
+							o["i_state"] = 2
+							o["s_modify"] = (*lua)["createuser"]
+							o["s_modifyid"] = (*lua)["createuserid"]
 							o["l_comeintime"] = time.Now().Unix()
-							o["i_event"] = lua["event"]
+							o["i_event"] = (*lua)["event"]
 							o["i_times"] = 0
 							o["s_date"] = time.Now().Format("2006-01-02")
 							o["s_source"] = "人工"
@@ -143,7 +142,8 @@ func (t *TaskM) Taskfile() {
 							o["s_type"] = stype
 							o["l_complete"] = completeTime
 							o["i_pendstate"] = 0
-							mgdb.Save("task", o)
+							o["l_checktime"] = time.Now().Unix()
+							u.MgoE.Save("task", o)
 							//清空map
 							o = map[string]interface{}{}
 						} else {
@@ -188,10 +188,10 @@ func (t *TaskM) ManagerTask() {
 			query := queryCriteria(userid, urgency, taskState, event, stype)
 			if search != "" {
 				query["$or"] = []interface{}{
-					bson.M{"s_code": bson.M{"$regex": search}},
-					bson.M{"s_modify": bson.M{"$regex": search}},
-					bson.M{"s_site": bson.M{"$regex": search}},
-					bson.M{"s_channel": bson.M{"$regex": search}},
+					map[string]interface{}{"s_code": map[string]interface{}{"$regex": search}},
+					map[string]interface{}{"s_modify": map[string]interface{}{"$regex": search}},
+					map[string]interface{}{"s_site": map[string]interface{}{"$regex": search}},
+					map[string]interface{}{"s_channel": map[string]interface{}{"$regex": search}},
 				}
 			}
 			sort := `{"%s":%d}`
@@ -208,21 +208,12 @@ func (t *TaskM) ManagerTask() {
 				//sort = `{"l_complete":1,` + sortb + `}`
 				sort = `{` + sortb + `,"l_complete":1}`
 			}
-			qu.Debug("query:", query)
-			task := *mgdb.Find("task", query, sort, nil, false, start, limit)
-			count := mgdb.Count("task", query)
+			qu.Debug("query:", query, sort)
+			task, _ := u.MgoE.Find("task", query, sort, nil, false, start, limit)
+			count := u.MgoE.Count("task", query)
 			page := start / 10
-			if len(task) > 0 {
-				for k, v := range task {
-					// if v["f_num"] == nil {
-					// 	v["f_num"] = 0
-					// }
-					// if v["f_min"] == nil {
-					// 	v["f_min"] = 0
-					// }
-					// if v["i_minNum"] == nil {
-					// 	v["i_minNum"] = 0
-					// }
+			if len(*task) > 0 {
+				for k, v := range *task {
 					v["num"] = k + 1 + page*10
 					v["encode"] = util.Se.Encode2Hex(fmt.Sprint(v["s_code"]))
 					s_urgency := qu.IntAll(v["s_urgency"])
@@ -235,36 +226,29 @@ func (t *TaskM) ManagerTask() {
 					} else if s_urgency == 4 {
 						v["s_urgency"] = "特别紧急"
 					}
-
-					if v["i_state"] == 0 {
+					state := qu.IntAll(v["i_state"])
+					if state == 0 {
 						v["i_state"] = "待确认"
-					} else if v["i_state"] == 1 {
+					} else if state == 1 {
 						v["i_state"] = "待处理"
-					} else if v["i_state"] == 2 {
+					} else if state == 2 {
 						v["i_state"] = "处理中"
-					} else if v["i_state"] == 3 {
+					} else if state == 3 {
 						v["i_state"] = "待审核"
-					} else if v["i_state"] == 4 {
+					} else if state == 4 {
 						v["i_state"] = "审核通过"
-					} else if v["i_state"] == 5 {
+					} else if state == 5 {
 						v["i_state"] = "未通过"
-					} else if v["i_state"] == 6 {
+					} else if state == 6 {
 						v["i_state"] = "关闭"
 					}
 					if v["i_event"] == nil { //节点
 						v["i_event"] = 0
 					}
-					//					if v["continueTimes"] == nil { //特别紧急任务连续报错次数
-					//						v["continueTimes"] = 0
-					//					} else {
-					//						for _, times := range v["continueTimes"].(map[string]interface{}) {
-					//							v["continueTimes"] = times
-					//						}
-					//					}
 					l_complete := qu.Int64All(v["l_complete"])
 					v["l_complete"] = qu.FormatDateByInt64(&l_complete, qu.Date_Full_Layout)
 					//v["l_complete"] = time.Unix(v["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
-					v["_id"] = v["_id"].(bson.ObjectId).Hex()
+					v["_id"] = mongodb.BsonIdToSId(v["_id"])
 					//根据code查询luaconfig
 					param := findLua(v["s_code"].(string))
 					if len(param) < 13 || param == nil {
@@ -282,8 +266,9 @@ func (t *TaskM) ManagerTask() {
 }
 
 func findLua(code string) []interface{} {
-	lua := *mgdb.FindOneByField("luaconfig", `{"code":"`+code+`"}`, `{"param_common":1}`)
-	param := lua["param_common"].([]interface{})
+	//lua := *mgdb.FindOneByField("luaconfig", `{"code":"`+code+`"}`, `{"param_common":1}`)
+	lua, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"param_common": 1})
+	param := (*lua)["param_common"].([]interface{})
 	return param
 }
 
@@ -295,7 +280,7 @@ func (t *TaskM) Mytask() {
 		return
 	}
 	userid := qu.ObjToString(t.GetSession("userid"))
-	query := bson.M{}
+	query := map[string]interface{}{}
 	if t.Method() == "POST" {
 		start, _ := t.GetInteger("start")
 		limit, _ := t.GetInteger("length")
@@ -312,9 +297,9 @@ func (t *TaskM) Mytask() {
 		query = queryCriteria(userid, urgency, taskState, event, stype)
 		if search != "" {
 			query["$or"] = []interface{}{
-				bson.M{"s_code": bson.M{"$regex": search}},
-				bson.M{"s_site": bson.M{"$regex": search}},
-				bson.M{"s_channel": bson.M{"$regex": search}},
+				map[string]interface{}{"s_code": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_site": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_channel": map[string]interface{}{"$regex": search}},
 			}
 		}
 		sort := `{"%s":%d}`
@@ -329,41 +314,44 @@ func (t *TaskM) Mytask() {
 		// sorta := strings.Replace(sort, "{", "", -1)
 		// sortb := strings.Replace(sorta, "}", "", -1)
 		// sortNew := `{"l_complete": 1,` + sortb + `}`
-		task := *mgdb.Find("task", query, sort, nil, false, start, limit)
-		count := mgdb.Count("task", query)
-		if task != nil {
-			for _, v := range task {
+		qu.Debug(query, sort)
+		task, _ := u.MgoE.Find("task", query, sort, nil, false, start, limit)
+		count := u.MgoE.Count("task", query)
+		if task != nil && len(*task) > 0 {
+			for _, v := range *task {
 				code := fmt.Sprint(v["s_code"])
 				relatecode := ""
-				lua, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code, "state": 5}, map[string]interface{}{"relatecode": 1})
+				//lua, _ := u.MgoE.FindOneByField("luaconfig", map[string]interface{}{"code": code, "state": 5}, map[string]interface{}{"relatecode": 1})
+				lua, _ := u.MgoEB.FindOneByField("luaconfig", map[string]interface{}{"code": code, "state": 5}, map[string]interface{}{"relatecode": 1})
 				if len(*lua) > 0 {
 					relatecode = qu.ObjToString((*lua)["relatecode"])
 				}
 				v["relatecode"] = relatecode
 				v["encode"] = util.Se.Encode2Hex(code)
-				if v["s_urgency"] == "1" {
+				urgency := qu.ObjToString(v["s_urgency"])
+				if urgency == "1" {
 					v["s_urgency"] = "普通"
-				} else if v["s_urgency"] == "2" {
+				} else if urgency == "2" {
 					v["s_urgency"] = "紧急"
-				} else if v["s_urgency"] == "3" {
+				} else if urgency == "3" {
 					v["s_urgency"] = "非常紧急"
-				} else if v["s_urgency"] == "4" {
+				} else if urgency == "4" {
 					v["s_urgency"] = "特别紧急"
 				}
-
-				if v["i_state"] == 0 {
+				state := qu.IntAll(v["i_state"])
+				if state == 0 {
 					v["i_state"] = "待确认"
-				} else if v["i_state"] == 1 {
+				} else if state == 1 {
 					v["i_state"] = "待处理"
-				} else if v["i_state"] == 2 {
+				} else if state == 2 {
 					v["i_state"] = "处理中"
-				} else if v["i_state"] == 3 {
+				} else if state == 3 {
 					v["i_state"] = "待审核"
-				} else if v["i_state"] == 4 {
+				} else if state == 4 {
 					v["i_state"] = "审核通过"
-				} else if v["i_state"] == 5 {
+				} else if state == 5 {
 					v["i_state"] = "未通过"
-				} else if v["i_state"] == 6 {
+				} else if state == 6 {
 					v["i_state"] = "关闭"
 				}
 				if v["i_event"] == nil { //节点
@@ -371,7 +359,7 @@ func (t *TaskM) Mytask() {
 				}
 
 				v["l_complete"] = time.Unix(v["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
-				v["_id"] = v["_id"].(bson.ObjectId).Hex()
+				v["_id"] = mongodb.BsonIdToSId(v["_id"])
 			}
 		}
 		t.ServeJson(map[string]interface{}{"draw": draw, "data": task, "recordsFiltered": count, "recordsTotal": count})
@@ -387,13 +375,13 @@ func (t *TaskM) Mytask() {
 		if SessionFailuer {
 			query["s_modifyid"] = userid
 			query["i_state"] = 5
-			task := *mgdb.Find("task", query, nil, nil, false, -1, -1)
-			failedtasknum = len(task)
+			task, _ := u.MgoE.Find("task", query, nil, nil, false, -1, -1)
+			failedtasknum = len(*task)
 			SessionFailuer = false
 		}
 		//查询是否有错误爬虫
 		modifyuser := qu.ObjToString(t.GetSession("loginuser"))
-		errluanum := mgu.Count("spider_loadfail", "spider", "spider",
+		errluanum := u.MgoS.Count("spider_loadfail",
 			map[string]interface{}{
 				"modifyuser": modifyuser,
 				"modifytime": map[string]interface{}{
@@ -411,29 +399,30 @@ func (t *TaskM) Mytask() {
 func (t *TaskM) CheckCode() {
 	code := t.GetString("code")
 	status := "notHasCode"
-	var lua map[string]interface{}
+	lua := &map[string]interface{}{}
 	if code != "" {
-		query := bson.M{
+		query := map[string]interface{}{
 			"s_code": code,
-			"i_state": bson.M{
+			"i_state": map[string]interface{}{
 				"$in": []int{0, 1, 2, 3, 5},
 			},
 		}
-		task := *mgdb.FindOne("task", query)
-		if task != nil {
-			task["l_complete"] = time.Unix(task["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
+		task, _ := u.MgoE.FindOne("task", query)
+		if task != nil && len(*task) > 0 {
+			(*task)["l_complete"] = time.Unix((*task)["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
 			status = "hasCode"
 
 		} else {
-			luaQuery := bson.M{
+			luaQuery := map[string]interface{}{
 				"code": code,
 			}
-			lua = *mgdb.FindOne("luaconfig", luaQuery)
+			//lua = *mgdb.FindOne("luaconfig", luaQuery)
+			lua, _ = u.MgoEB.FindOne("luaconfig", luaQuery)
 		}
 		t.ServeJson(map[string]interface{}{
 			"status": status,
 			"task":   task,
-			"lua":    lua,
+			"lua":    *lua,
 		})
 	}
 }
@@ -483,18 +472,20 @@ func (t *TaskM) SaveNewTask() {
 	newTask["s_site"] = site             //站点
 	newTask["s_channel"] = channel       //栏目
 	newTask["s_code"] = code             //代码
-	newTask["i_state"] = 1               //完成状态
+	newTask["i_state"] = 2               //完成状态
 	newTask["l_comeintime"] = comeintime //创建时间
 	newTask["s_date"] = time.Now().Format("2006-01-02")
 	newTask["i_times"] = 0
 	newTask["i_pendstate"] = 0
-	queryL := bson.M{
+	newTask["l_checktime"] = time.Now().Unix()
+	queryL := map[string]interface{}{
 		"code": code,
 	}
-	lua := *mgdb.FindOne("luaconfig", queryL)
+	//lua := *mgdb.FindOne("luaconfig", queryL)
+	lua, _ := u.MgoEB.FindOne("luaconfig", queryL)
 	if lua != nil {
-		newTask["s_modifyid"] = lua["createuserid"] //维护人员id
-		newTask["i_event"] = lua["event"]           //节点
+		newTask["s_modifyid"] = (*lua)["createuserid"] //维护人员id
+		newTask["i_event"] = (*lua)["event"]           //节点
 	}
 	newTask["s_modify"] = modify     //维护人员
 	newTask["s_descript"] = descript //描述
@@ -522,8 +513,8 @@ func (t *TaskM) SaveNewTask() {
 			newTask["l_complete"] = timeDate.Unix()
 		}
 	}
-	task := mgdb.Save("task", newTask)
-	if task != "" {
+	taskid := u.MgoE.Save("task", newTask)
+	if taskid != "" {
 		t.ServeJson(map[string]interface{}{
 			"state": "ok",
 		})
@@ -532,21 +523,21 @@ func (t *TaskM) SaveNewTask() {
 
 //编辑 查看任务
 func (t *TaskM) EditTask(ids string) error {
-	auth := t.GetSession("auth")
+	auth := qu.IntAll(t.GetSession("auth"))
 	//code := strings.Split(codes, "__")[0]
 	id := strings.Split(ids, "__")[0]
 	param := strings.Split(ids, "__")[1]
 	if t.Method() == "GET" {
-		query := bson.M{
+		query := map[string]interface{}{
 			//"s_code": util.Se.Decode4Hex(code),
-			"_id": bson.ObjectIdHex(id),
+			"_id": mongodb.StringTOBsonId(id),
 		}
-		task := *mgdb.FindOne("task", query)
-		if task != nil {
-			task["l_comeintime"] = time.Unix(task["l_comeintime"].(int64), 0).Format("2006-01-02 15:04:05")
-			task["l_complete"] = time.Unix(task["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
-			if task["a_mrecord"] != nil {
-				mrecord := qu.ObjArrToMapArr(task["a_mrecord"].([]interface{}))
+		task, _ := u.MgoE.FindOne("task", query)
+		if task != nil && len(*task) > 0 {
+			(*task)["l_comeintime"] = time.Unix((*task)["l_comeintime"].(int64), 0).Format("2006-01-02 15:04:05")
+			(*task)["l_complete"] = time.Unix((*task)["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
+			if (*task)["a_mrecord"] != nil {
+				mrecord := qu.ObjArrToMapArr((*task)["a_mrecord"].([]interface{}))
 				if mrecord != nil && len(mrecord) > 0 {
 					for _, v := range mrecord {
 						v["l_mrecord_comeintime"] = time.Unix(qu.Int64All(v["l_mrecord_comeintime"]), 0).Format("2006-01-02 15:04:05")
@@ -554,17 +545,17 @@ func (t *TaskM) EditTask(ids string) error {
 					}
 				}
 			}
-			if task["a_check"] != nil {
-				check := qu.ObjArrToMapArr(task["a_check"].([]interface{}))
+			if (*task)["a_check"] != nil {
+				check := qu.ObjArrToMapArr((*task)["a_check"].([]interface{}))
 				if check != nil && len(check) > 0 {
 					for _, v := range check {
 						v["l_check_checkTime"] = time.Unix(qu.Int64All(v["l_check_checkTime"]), 0).Format("2006-01-02 15:04:05")
 					}
 				}
 			}
-			t.T["encode"] = util.Se.Encode2Hex(fmt.Sprint(task["s_code"]))
+			t.T["encode"] = util.Se.Encode2Hex(fmt.Sprint((*task)["s_code"]))
 			t.T["id"] = id
-			t.T["task"] = task
+			t.T["task"] = *task
 			t.T["param"] = param
 			if t.GetSession(id) == "" || t.GetSession(id) == nil {
 				t.T["xgTime"] = time.Unix(time.Now().Unix(), 0).Format("2006-01-02 15:04:05")
@@ -593,10 +584,10 @@ func (t *TaskM) Del() {
 		t.ServeJson("没有权限")
 		return
 	}
-	del := bson.M{
-		"_id": bson.ObjectIdHex(id),
+	del := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(id),
 	}
-	ok := mgdb.Del("task", del)
+	ok := u.MgoE.Del("task", del)
 	if ok {
 		state = "ok"
 	}
@@ -645,23 +636,23 @@ func (t *TaskM) UpdateTask() {
 		l_complete = ft.CompleteTime(urgency)
 	}
 
-	queryU := bson.M{
+	queryU := map[string]interface{}{
 		"s_name": modify,
 	}
-	task := *mgdb.FindOne("user", queryU)
-	queryT := bson.M{
-		"_id": bson.ObjectIdHex(id),
+	task, _ := u.MgoE.FindOne("user", queryU)
+	queryT := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(id),
 	}
-	update := bson.M{
-		"$set": bson.M{
+	update := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"s_modify":   modify,
 			"s_descript": descript,
 			"s_urgency":  urgency,
 			"l_complete": l_complete,
-			"s_modifyid": task["_id"].(bson.ObjectId).Hex(),
+			"s_modifyid": mongodb.BsonIdToSId((*task)["_id"]),
 		},
 	}
-	ok := mgdb.Update("task", queryT, update, false, false)
+	ok := u.MgoE.Update("task", queryT, update, false, false)
 	if ok {
 		state = "ok"
 		t.SetSession("jumpMark", "y")
@@ -679,44 +670,23 @@ func (t *TaskM) UpdateTaskState() {
 	if t.GetSession(id) == nil {
 		t.SetSession(id, xgTime)
 	}
-	query := bson.M{
-		"_id": bson.ObjectIdHex(id),
+	query := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(id),
 	}
-	task := *mgdb.FindOne("task", query)
+	task, _ := u.MgoE.FindOne("task", query)
 	updateOk := false
-	if len(task) > 0 {
-		state := qu.IntAll(task["i_state"])
+	if len(*task) > 0 {
+		state := qu.IntAll((*task)["i_state"])
 		if state == 1 { //修改任务状态为待处理
-			update := bson.M{
-				"$set": bson.M{
+			update := map[string]interface{}{
+				"$set": map[string]interface{}{
 					"i_state": 2,
 				},
 			}
-			updateOk = mgdb.Update("task", query, update, false, false) //更新任务状态
-			code := qu.ObjToString(task["s_code"])
-			mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"ismodify": true}}, false, false)
-			// code := qu.ObjToString(task["s_code"])
-			// lua := *mgdb.FindOne("luaconfig", map[string]interface{}{"code": code})
-			// if len(lua) > 0 {
-			// 	lua_state := qu.IntAll(lua["state"])
-			// 	if lua_state == sp.Sp_state_5 {
-			// 		b, err := sp.UpStateAndUpSpider(code, "", "", "", sp.Sp_state_6) //点击修改爬虫时,线上爬虫下架
-			// 		if b && err == nil {                                             //下架成功
-			// 			log.Println("爬虫下架成功", code, ",任务id:", id)
-			// 			ok := mgdb.Update("task", query, update, false, false) //更新任务状态
-			// 			if ok {
-			// 				updateOk = true
-			// 			}
-			// 		} else {
-			// 			log.Println("修改任务失败", id)
-			// 		}
-			// 	} else {
-			// 		ok := mgdb.Update("task", query, update, false, false) //更新任务状态
-			// 		if ok {
-			// 			updateOk = true
-			// 		}
-			// 	}
-			// }
+			updateOk = u.MgoE.Update("task", query, update, false, false) //更新任务状态
+			code := qu.ObjToString((*task)["s_code"])
+			//mgdb.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"ismodify": true}}, false, false)
+			u.MgoEB.Update("luaconfig", map[string]interface{}{"code": code}, map[string]interface{}{"$set": map[string]interface{}{"ismodify": true}}, false, false)
 		} else {
 			updateOk = true
 		}
@@ -779,26 +749,26 @@ func (t *TaskM) Audit() {
 		//search := strings.Replace(searchStr, " ", "", -1)
 		search := strings.TrimSpace(searchStr)
 		//查询自己的任务
-		query := bson.M{}
-		query = bson.M{
-			"i_state": bson.M{
+		query := map[string]interface{}{}
+		query = map[string]interface{}{
+			"i_state": map[string]interface{}{
 				"$gte": 3,
 				"$lte": 5,
 			},
 		}
 		if state >= 0 && taskState >= 0 {
-			query = bson.M{
+			query = map[string]interface{}{
 				"s_urgency": strconv.Itoa(state),
 				"i_state":   taskState,
 			}
 		} else if state < 0 && taskState >= 0 {
-			query = bson.M{
+			query = map[string]interface{}{
 				"i_state": taskState,
 			}
 		} else if state >= 0 && taskState < 0 {
-			query = bson.M{
+			query = map[string]interface{}{
 				"s_urgency": strconv.Itoa(state),
-				"i_state": bson.M{
+				"i_state": map[string]interface{}{
 					"$gte": 3,
 					"$lte": 5,
 				},
@@ -806,9 +776,9 @@ func (t *TaskM) Audit() {
 		}
 		if search != "" {
 			query["$or"] = []interface{}{
-				bson.M{"s_code": bson.M{"$regex": search}},
-				bson.M{"s_site": bson.M{"$regex": search}},
-				bson.M{"s_channel": bson.M{"$regex": search}},
+				map[string]interface{}{"s_code": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_site": map[string]interface{}{"$regex": search}},
+				map[string]interface{}{"s_channel": map[string]interface{}{"$regex": search}},
 			}
 		}
 		sort := `{"%s":%d}`
@@ -823,39 +793,40 @@ func (t *TaskM) Audit() {
 		sorta := strings.Replace(sort, "{", "", -1)
 		sortb := strings.Replace(sorta, "}", "", -1)
 		sortNew := `{"l_complete": 1,` + sortb + `}`
-		task := *mgdb.Find("task", query, sortNew, nil, false, start, limit)
-		count := mgdb.Count("task", query)
-		if task != nil {
-			for _, v := range task {
+		task, _ := u.MgoE.Find("task", query, sortNew, nil, false, start, limit)
+		count := u.MgoE.Count("task", query)
+		if task != nil && len(*task) > 0 {
+			for _, v := range *task {
 				v["encode"] = util.Se.Encode2Hex(fmt.Sprint(v["s_code"]))
-				if v["s_urgency"] == "1" {
+				urgency := qu.ObjToString(v["s_urgency"])
+				if urgency == "1" {
 					v["s_urgency"] = "普通"
-				} else if v["s_urgency"] == "2" {
+				} else if urgency == "2" {
 					v["s_urgency"] = "紧急"
-				} else if v["s_urgency"] == "3" {
+				} else if urgency == "3" {
 					v["s_urgency"] = "非常紧急"
-				} else if v["s_urgency"] == "4" {
+				} else if urgency == "4" {
 					v["s_urgency"] = "特别紧急"
 				}
-
-				if v["i_state"] == 0 {
+				state := qu.IntAll(v["i_state"])
+				if state == 0 {
 					v["i_state"] = "待确认"
-				} else if v["i_state"] == 1 {
+				} else if state == 1 {
 					v["i_state"] = "待处理"
-				} else if v["i_state"] == 2 {
+				} else if state == 2 {
 					v["i_state"] = "处理中"
-				} else if v["i_state"] == 3 {
+				} else if state == 3 {
 					v["i_state"] = "待审核"
-				} else if v["i_state"] == 4 {
+				} else if state == 4 {
 					v["i_state"] = "审核通过"
-				} else if v["i_state"] == 5 {
+				} else if state == 5 {
 					v["i_state"] = "未通过"
-				} else if v["i_state"] == 6 {
+				} else if state == 6 {
 					v["i_state"] = "关闭"
 				}
 
 				v["l_complete"] = time.Unix(v["l_complete"].(int64), 0).Format("2006-01-02 15:04:05")
-				v["_id"] = v["_id"].(bson.ObjectId).Hex()
+				v["_id"] = mongodb.BsonIdToSId(v["_id"])
 			}
 		}
 
@@ -867,18 +838,18 @@ func (t *TaskM) Audit() {
 
 //更新task
 func UpdateOldTask(id, descript, urgency string, complete int64) {
-	queryT := bson.M{
-		"_id": bson.ObjectIdHex(id),
+	queryT := map[string]interface{}{
+		"_id": mongodb.StringTOBsonId(id),
 	}
-	update := bson.M{
-		"$set": bson.M{
+	update := map[string]interface{}{
+		"$set": map[string]interface{}{
 			"s_descript":   descript,
 			"s_urgency":    urgency,
 			"l_complete":   complete,
 			"l_comeintime": time.Now().Unix(),
 		},
 	}
-	ok := mgdb.Update("task", queryT, update, false, false)
+	ok := u.MgoE.Update("task", queryT, update, false, false)
 	if ok {
 		log.Println("更新已有任务成功")
 	} else {
@@ -895,31 +866,34 @@ func (t *TaskM) AssignChangeTaskState() {
 	auth := qu.IntAll(t.GetSession("auth"))
 	if auth == role_admin {
 		//先根据code查有没有相关任务，再根据id修改任务状态
-		query := bson.M{
+		query := map[string]interface{}{
 			"s_code": code,
-			"i_state": bson.M{
+			"i_state": map[string]interface{}{
 				"$in": []int{1, 2, 3, 5},
 			},
 		}
-		task := *mgdb.FindOne("task", query)
-		if len(task) > 0 { //有相关任务不能分发
+		task, _ := u.MgoE.FindOne("task", query)
+		if len(*task) > 0 { //有相关任务不能分发
 			t.ServeJson("e")
 			return
 		}
 		//没有相关任务，修改状态
-		query = bson.M{
-			"_id": bson.ObjectIdHex(id),
+		query = map[string]interface{}{
+			"_id": mongodb.StringTOBsonId(id),
+		}
+		set := map[string]interface{}{
+			"i_state":     1,
+			"l_checktime": time.Now().Unix(),
 		}
-		set := bson.M{"i_state": 1}
 		if reason != "" {
-			tmp := *mgdb.FindOne("task", query)
-			descript := qu.ObjToString(tmp["s_descript"]) + "审核人员追加描述：------------------------------\n" + reason + "\n"
+			tmp, _ := u.MgoE.FindOne("task", query)
+			descript := qu.ObjToString((*tmp)["s_descript"]) + "审核人员追加描述：------------------------------\n" + reason + "\n"
 			set["s_descript"] = descript
 		}
-		update := bson.M{
+		update := map[string]interface{}{
 			"$set": set,
 		}
-		flag := mgdb.Update("task", query, update, false, false)
+		flag := u.MgoE.Update("task", query, update, false, false)
 		if flag {
 			t.ServeJson("y")
 			t.SetSession("jumpMark", "y")
@@ -939,15 +913,16 @@ func (t *TaskM) CloseChangeTaskState() {
 	auth := qu.IntAll(t.GetSession("auth"))
 	if auth == role_admin {
 		//根据id关闭任务
-		query := bson.M{
-			"_id": bson.ObjectIdHex(id),
+		query := map[string]interface{}{
+			"_id": mongodb.StringTOBsonId(id),
 		}
-		update := bson.M{
-			"$set": bson.M{
-				"i_state": 6,
+		update := map[string]interface{}{
+			"$set": map[string]interface{}{
+				"i_state":     6,
+				"l_checktime": time.Now().Unix(),
 			},
 		}
-		flag := mgdb.Update("task", query, update, false, false)
+		flag := u.MgoE.Update("task", query, update, false, false)
 		if flag {
 			go updateClose(code) //更新closerate数据
 			t.ServeJson("y")
@@ -966,31 +941,32 @@ func (t *TaskM) BatchAssign() {
 	ids := strings.Split(t.GetString("ids"), ",")
 	codes := strings.Split(t.GetString("codes"), ",")
 	auth := qu.IntAll(t.GetSession("auth"))
-	query := bson.M{}
+	query := map[string]interface{}{}
 	var existCode []string
 	if auth == role_admin {
 		for k, code := range codes {
-			query = bson.M{
+			query = map[string]interface{}{
 				"s_code": code,
-				"i_state": bson.M{
+				"i_state": map[string]interface{}{
 					"$in": []int{1, 2, 3, 5},
 				},
 			}
-			task := *mgdb.FindOne("task", query)
+			task, _ := u.MgoE.FindOne("task", query)
 			//log.Println("task", task)
-			if len(task) > 0 { //任务已经存在
+			if len(*task) > 0 { //任务已经存在
 				existCode = append(existCode, code)
 			} else {
 				id := ids[k]
-				query = bson.M{
-					"_id": bson.ObjectIdHex(id),
+				query = map[string]interface{}{
+					"_id": mongodb.StringTOBsonId(id),
 				}
-				update := bson.M{
-					"$set": bson.M{
-						"i_state": 1,
+				update := map[string]interface{}{
+					"$set": map[string]interface{}{
+						"i_state":     1,
+						"l_checktime": time.Now().Unix(),
 					},
 				}
-				flag := mgdb.Update("task", query, update, false, false)
+				flag := u.MgoE.Update("task", query, update, false, false)
 				log.Println("任务id:", id, "	更新:", flag)
 			}
 		}
@@ -1009,15 +985,16 @@ func (t *TaskM) BatchClose() {
 	var falseCode []string
 	if auth == role_admin {
 		for k, id := range ids {
-			query := bson.M{
-				"_id": bson.ObjectIdHex(id),
+			query := map[string]interface{}{
+				"_id": mongodb.StringTOBsonId(id),
 			}
-			update := bson.M{
-				"$set": bson.M{
-					"i_state": 6,
+			update := map[string]interface{}{
+				"$set": map[string]interface{}{
+					"i_state":     6,
+					"l_checktime": time.Now().Unix(),
 				},
 			}
-			flag := mgdb.Update("task", query, update, false, false)
+			flag := u.MgoE.Update("task", query, update, false, false)
 			log.Println("任务id:", id, "	关闭:", flag)
 			if !flag {
 				falseCode = append(falseCode, codes[k])
@@ -1036,16 +1013,17 @@ func (t *TaskM) BatchDeal() {
 	ids := strings.Split(t.GetString("ids"), ",")
 	auth := qu.IntAll(t.GetSession("auth"))
 	if auth == role_admin {
-		update := bson.M{
-			"$set": bson.M{
-				"i_state": 2,
+		update := map[string]interface{}{
+			"$set": map[string]interface{}{
+				"i_state":     2,
+				"l_checktime": time.Now().Unix(),
 			},
 		}
 		for _, id := range ids {
-			query := bson.M{
-				"_id": bson.ObjectIdHex(id),
+			query := map[string]interface{}{
+				"_id": mongodb.StringTOBsonId(id),
 			}
-			flag := mgdb.Update("task", query, update, false, false)
+			flag := u.MgoE.Update("task", query, update, false, false)
 			log.Println("任务id:", id, "	更新为处理中:", flag)
 		}
 		t.ServeJson("处理成功")
@@ -1067,29 +1045,29 @@ func (t *TaskM) GetJumpMark() {
 //更新closerate
 func updateClose(code string) {
 	defer qu.Catch()
-	query := bson.M{
+	query := map[string]interface{}{
 		"s_code": code,
 	}
-	data := *mgdb.FindOne("closerate", query)
-	if data != nil && len(data) > 0 {
-		arr := qu.ObjArrToStringArr(data["timeClose"].([]interface{}))
+	data, _ := u.MgoE.FindOne("closerate", query)
+	if data != nil && len(*data) > 0 {
+		arr := qu.ObjArrToStringArr((*data)["timeClose"].([]interface{}))
 		last := arr[len(arr)-1] //更新最后一天的数据
 		timeAndClose := strings.Split(last, ":")
 		if len(timeAndClose) >= 2 {
 			arr[len(arr)-1] = timeAndClose[0] + ":3"
 		}
 		//更新
-		update := bson.M{
-			"$set": bson.M{
+		update := map[string]interface{}{
+			"$set": map[string]interface{}{
 				"timeClose": arr,
 			},
 		}
-		flag := mgdb.Update("closerate", query, update, false, false)
+		flag := u.MgoE.Update("closerate", query, update, false, false)
 		fmt.Println("closerate关闭任务：code	", flag)
 	}
 }
-func queryCriteria(userid string, urgency, taskState, event, stype int) (query bson.M) {
-	query = bson.M{}
+func queryCriteria(userid string, urgency, taskState, event, stype int) (query map[string]interface{}) {
+	query = map[string]interface{}{}
 	if userid != "" && userid != "-1" {
 		query["s_modifyid"] = userid
 	}
@@ -1107,36 +1085,36 @@ func queryCriteria(userid string, urgency, taskState, event, stype int) (query b
 	}
 
 	// if urgency >= 0 && taskState >= 0 && event >= 0 { //选择节点，状态和紧急度
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"s_urgency": strconv.Itoa(urgency),
 	// 		"i_state":   taskState,
 	// 		"i_event":   event,
 	// 	}
 	// } else if event >= 0 && urgency < 0 && taskState >= 0 { //选择节点和状态，未选择紧急度
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"i_state": taskState,
 	// 		"i_event": event,
 	// 	}
 	// } else if event >= 0 && urgency >= 0 && taskState < 0 { //选择节点和紧急度，未选择状态
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"i_event":   event,
 	// 		"s_urgency": strconv.Itoa(urgency),
 	// 	}
 	// } else if event >= 0 && urgency < 0 && taskState < 0 { //选择节点，未选择紧急度和状态
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"i_event": event,
 	// 	}
 	// } else if event < 0 && urgency >= 0 && taskState >= 0 { //未选择节点,选择紧急度和状态
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"s_urgency": strconv.Itoa(urgency),
 	// 		"i_state":   taskState,
 	// 	}
 	// } else if event < 0 && urgency < 0 && taskState >= 0 { //未选择节点和紧急度，选择状态
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"i_state": taskState,
 	// 	}
 	// } else if event < 0 && urgency >= 0 && taskState < 0 { //未选择节点和状态，选择紧急度
-	// 	query = bson.M{
+	// 	query = map[string]interface{}{
 	// 		"s_urgency": strconv.Itoa(urgency),
 	// 	}
 	// } else {

src/timetask/timetask.go

@@ -3,21 +3,14 @@ package timetask
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
-	"net/smtp"
 	qu "qfw/util"
-	mgdb "qfw/util/mongodb"
 	sp "spiderutil"
-	"strconv"
 	"strings"
 	"time"
 	"util"
 
 	"github.com/cron"
-
-	"gopkg.in/mgo.v2/bson"
-	. "gopkg.in/mgo.v2/bson"
 )
 
 var Mail map[string]interface{}
@@ -44,10 +37,10 @@ func CheckLuaMove() {
 		"ok": false,
 	}
 	qu.Debug("query:", query)
-	list := *mgdb.Find("luamovelog", query, nil, nil, false, -1, -1)
+	list, _ := util.MgoE.Find("luamovelog", query, nil, nil, false, -1, -1)
 	text := ""
-	if len(list) > 0 {
-		for _, l := range list {
+	if len(*list) > 0 {
+		for _, l := range *list {
 			stype := qu.ObjToString(l["type"])
 			code := qu.ObjToString(l["code"])
 			text += code + "：" + stype + "；"
@@ -76,9 +69,9 @@ func CheckCreateTask() {
 		},
 	}
 	codes := []string{}
-	list := *mgdb.Find("luacreatetaskerr", query, nil, nil, false, -1, -1)
-	if len(list) > 0 {
-		for _, l := range list {
+	list, _ := util.MgoE.Find("luacreatetaskerr", query, nil, nil, false, -1, -1)
+	if len(*list) > 0 {
+		for _, l := range *list {
 			code := qu.ObjToString(l["code"])
 			codes = append(codes, code)
 		}
@@ -96,59 +89,6 @@ func CheckCreateTask() {
 	}
 }
 
-func SendToMail() {
-	mailInfo := *(qu.ObjToMap(Mail["smtp"]))
-	host := mailInfo["host"].(string)
-	from := mailInfo["from"].(string)
-	pwd := mailInfo["password"].(string)
-	subject := mailInfo["subject"].(string)
-
-	hour := time.Now().Hour()
-	if hour == 8 {
-		//定时查询数据库 查询需要发邮件的人和相关信息
-		timeStr := time.Now().Format("2006-01-02")
-		the_time, _ := time.ParseInLocation("2006-01-02", timeStr, time.Local)
-		time_zero := the_time.Unix()         //当日凌晨的时间戳
-		time_twentyFour := time_zero + 86399 //当日24时的时间戳
-		//聚合查询数据
-		//mgdb.InitMongodbPool(5, "192.168.3.207:27080", "editor")
-		sess := mgdb.GetMgoConn()
-		defer mgdb.DestoryMongoConn(sess)
-		var res []M
-		sess.DB("editor").C("task").Pipe([]M{M{"$match": M{"l_complete": M{"$gte": time_zero, "$lte": time_twentyFour}, "i_state": M{"$gte": 1, "$lte": 2}}},
-			M{"$group": M{"_id": "$s_modifyid", "count": M{"$sum": 1}}}}).All(&res)
-		//遍历数据进行发邮件
-		for _, v := range res {
-			_id, ok := v["_id"].(string)
-			if ok {
-				query := bson.M{
-					"_id": bson.ObjectIdHex(_id),
-				}
-				user := *mgdb.FindOne("user", query)
-				if user["s_email"] == nil {
-					continue
-				}
-				num := strconv.Itoa(v["count"].(int))
-				body := `<html><body><h3>你有` + num + `条任务需要今天完成</h3></body></html>`
-				hp := strings.Split(host, ":")
-				auth := smtp.PlainAuth("", from, pwd, hp[0])
-				content_type := "Content-Type: text/html; charset=UTF-8"
-				msg := []byte("To: " + user["s_email"].(string) + "\r\nFrom: " + from + "\r\nSubject: " + subject + "\r\n" + content_type + "\r\n\r\n" + body)
-				send_to := strings.Split(user["s_email"].(string), ";")
-				err := smtp.SendMail(host, auth, from, send_to, msg)
-				if err == nil {
-					log.Println(user["s_email"].(string), "  sendMail   success")
-				} else {
-					log.Println(user["s_email"].(string), "  sendMail   fail")
-				}
-			}
-		}
-		time.Sleep(1 * time.Hour)
-	}
-	//time.AfterFunc(30*time.Minute, func() { SendToMail(to, num) })
-	time.AfterFunc(30*time.Minute, SendToMail)
-}
-
 //获取第day天凌晨的时间戳
 func GetTime(day int) int64 {
 	defer qu.Catch()
@@ -198,7 +138,8 @@ func UpdateCodeHeart() {
 			},
 		},
 	}
-	list, _ := util.MgoE.Find("luaconfig", query, nil, map[string]interface{}{"code": 1}, false, -1, -1)
+	//list, _ := util.MgoE.Find("luaconfig", query, nil, map[string]interface{}{"code": 1}, false, -1, -1)
+	list, _ := util.MgoEB.Find("luaconfig", query, nil, map[string]interface{}{"code": 1}, false, -1, -1)
 	qu.Debug("定时更新爬虫心跳信息个数：", len(*list))
 	for _, l := range *list {
 		util.MgoS.Update("spider_heart", map[string]interface{}{"code": l["code"]}, map[string]interface{}{

src/util/util.go

@@ -14,7 +14,8 @@ import (
 )
 
 var (
-	MgoE            *mgo.MongodbSim
+	MgoE            *mgo.MongodbSim //编辑器87
+	MgoEB           *mgo.MongodbSim //编辑器163
 	MgoS            *mgo.MongodbSim
 	Province        map[string][]string
 	DomainNameReg   = regexp.MustCompile(`(http|https)[:：]+`)
@@ -90,6 +91,14 @@ func InitMgo() {
 		Size:        10,
 	}
 	MgoE.InitPool()
+	MgoEB = &mgo.MongodbSim{
+		MongodbAddr: sp.Config.BidEditor.Addr,
+		DbName:      sp.Config.BidEditor.Db,
+		Size:        sp.Config.BidEditor.Size,
+		UserName:    sp.Config.BidEditor.Username,
+		Password:    sp.Config.BidEditor.Password,
+	}
+	MgoEB.InitPool()
 	MgoS = &mgo.MongodbSim{
 		MongodbAddr: sp.Config.Dbaddr,
 		DbName:      sp.Config.Dbname2,
@@ -137,17 +146,20 @@ func SpiderPassCheckLua(liststr, contentstr string, lua map[string]interface{})
 		}
 	}
 	if isHttps {
-		downLoadText := DownLoadReg.FindString(contentstr)
-		if downLoadText != "" {
-			textArr := strings.Split(downLoadText, ",")
-			if len(textArr) < 4 {
-				msg = append(msg, "download方法添加下载参数")
-			} else if len(textArr) == 4 {
-				if !CodeTypeReg.MatchString(textArr[0]) || textArr[1] != "true" {
-					msg = append(msg, "download方法添加下载参数")
+		for tmpStr, tmpText := range map[string]string{"列表页": liststr, "三级页": contentstr} {
+			downLoadText := DownLoadReg.FindString(tmpText)
+			if downLoadText != "" {
+				textArr := strings.Split(downLoadText, ",")
+				if len(textArr) < 4 {
+					msg = append(msg, tmpStr+"download方法添加下载参数")
+				} else if len(textArr) == 4 {
+					if !CodeTypeReg.MatchString(textArr[0]) || (textArr[1] != "true" && textArr[1] != "false") {
+						msg = append(msg, tmpStr+"download方法添加下载参数")
+					}
 				}
 			}
 		}
+
 	}
 	//3.检测title
 	if strings.Contains(liststr, `item["title"]="a"`) {
@@ -228,7 +240,8 @@ func GetLuasInfoBySite(site string) (domain, status, event, platform, area, city
 	cityDistrictMap := map[string]map[string]int{}
 	domainMap := map[string]int{}
 	remarktime = time.Now().Unix()
-	luas, _ := MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
+	//luas, _ := MgoE.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
+	luas, _ := MgoEB.Find("luaconfig", `{"param_common.1":"`+site+`"}`, ``, `{"model":1,"event":1,"state":1,"platform":1,"param_common":1,"comeintime":1}`, false, -1, -1)
 	for _, l := range *luas {
 		//remarktime
 		if comeintime := qu.Int64All(l["comeintime"]); comeintime != int64(0) && comeintime < remarktime {

src/web/templates/head.html

@@ -596,7 +596,7 @@
 			<li data="index"><a href="/center"><i class="glyphicon glyphicon-eye-open"></i> <span>爬虫列表</span></a></li>
 			<li data="index_rwlb"><a href="/center/mytask"><i class="glyphicon glyphicon-tasks"></i> <span>任务列表</span></a></li>
 		{{end}}
-		{{if eq (session "paltform") "golua平台"}}
+		{{if eq (session "platform") "golua平台"}}
      	<li data="index_heart"><a href="/center/heart"><i class="fa fa-heartbeat"></i> <span>心跳监控</span></a></li>
     	<li data="index_errlua"><a href="/center/errorLua"><i class="fa fa-warning"></i> <span>错误爬虫管理</span></a></li>
     	<li data="index_errdata"><a href="/center/errorData"><i class="glyphicon glyphicon-exclamation-sign"></i> <span>错误信息管理</span></a></li>

src/web/templates/index.html

@@ -170,6 +170,9 @@
           				<th>栏目名称</th>
           				<th>爬虫代码</th>
                   		<th>紧急度</th>
+						{{if eq (session "platform") "python"}}
+						<th>难易度</th>
+						{{end}}
           				<th>节点</th>
           				<th>作者</th>
           				<th>最后修改时间</th>
@@ -195,6 +198,28 @@ $(function(){
           },
 		"columnDefs": [
 			{ "orderable": false, "targets": [0,10,12] },
+			//更新紧急度
+			{"targets":[5], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
+					{{if gt (session "auth") 2}}
+					var aInput;
+					$(cell).click(function () {
+						$(this).html(updateUrgency(rowData._id,rowData.state,rowData.code));
+						var aInput = $(this).find(":input");
+						aInput.focus().val(cellData);
+					});
+					$(cell).on("click", ":input", function (e) {
+						e.stopPropagation();
+					});
+					$(cell).on("change", ":input", function () {
+						$(this).blur();
+					});
+					$(cell).on("blur", ":input", function () {
+						var text = $(this).find("option:selected").text();
+						ttable.cell(cell).data(text);
+						cellData = text;
+					});
+					{{end}}
+				}},
 			//更新节点
 			{"targets":[6], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
 				  {{if gt (session "auth") 2}}
@@ -245,7 +270,7 @@ $(function(){
 					  text = 7
 					}else if (text=="需登录"){
 					  text = 8
-					}else if (text=="无法处理"){
+					}else if (text=="转python"){
 					  text = 9
 					}else if (text=="已删除"){
 					  text = 10
@@ -274,21 +299,6 @@ $(function(){
 				  });
 				  $(cell).on("blur", ":input", function () {
 					var text = $(this).find("option:selected").text();
-					if(text=="待完成"){
-					  text = 0
-					}else if (text=="已作废"){
-					  text = 4
-					}else if (text=="无发布"){
-					  text = 7
-					}else if (text=="需登录"){
-					  text = 8
-					}else if (text=="无法处理"){
-					  text = 9
-					}else if (text=="已删除"){
-					  text = 10
-					}else if (text=="已上线"){
-					  text = 11
-					}
 					ttable.cell(cell).data(text);
 					  cellData = text;
 				  });
@@ -365,6 +375,18 @@ $(function(){
 				  return "普通"
 				}
 			}},
+			{{if eq (session "platform") "python"}}
+			{ "data": "grade",render:function(val){
+				console.log(val)
+				if(val == undefined||val == 0){
+					return "普通"
+				}else if (val == 1){
+					return "较难"
+				}else if (val == 2){
+					return "很难"
+				}
+			}},
+			{{end}}
 			{ "data": "event","width":"50px"},
 			{ "data": "createuser"},
 			{ "data": "modifytime"},
@@ -389,7 +411,7 @@ $(function(){
 				}else if(state==8){
 					val="<span class='text-danger text-bold'>需登录</span>"
 				}else if(state==9){
-					val="<span class='text-danger text-bold'>无法处理</span>"
+					val="<span class='text-danger text-bold'>转python</span>"
 				}else if(state==10){
 					val="<span class='text-danger text-bold'>已删除</span>"
 				}else if(state==11){
@@ -519,7 +541,7 @@ $(function(){
 			"<option value='6'>已下架</option>";
 		var opt4="<option value='7'>无发布</option>"+
 			"<option value='8'>需登录</option>"+
-			"<option value='9'>无法处理</option>"+
+			"<option value='9'>转python</option>"+
 			"<option value='10'>已删除</option>"+
       "<option value='11'>已上线</option>";
 		{{if eq (session "auth") 3}}
@@ -840,6 +862,11 @@ $(function(){
 			return;
 		}	
 	};
+	//修改紧急度
+	function updateUrgency(id,state,code){
+		var spiderUrgency="<option value=0>普通</option><option value=1>紧急</option>";
+		return "<select onchange='updateesp(this.value,\"urgency\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderUrgency+"</select>"
+	};
     //修改节点
     function createComboxEvent(id,state,code){
     	var events={{.T.events}};
@@ -847,10 +874,10 @@ $(function(){
 		for(k in events){
 			spiderEvent+="<option value='"+events[k]+"'>"+events[k]+"</option>"
 		}
-    	return "<select onchange='changeEventOrStateOrPlatform(this.value,\"event\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderEvent+"</select>"
+    	return "<select onchange='updateesp(this.value,\"event\",\""+state+"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+spiderEvent+"</select>"
     };
     function createComboxState(id){
-    return "<select id='task_state'  onchange='changeEventOrStateOrPlatform(this.value,\"state\",\"\",\"\",\""+id+"\")' class='form-control input-sm'>"+
+    return "<select id='task_state'  onchange='updateesp(this.value,\"state\",\"\",\"\",\""+id+"\")' class='form-control input-sm'>"+
             "<option value='0' class='text-info text-bold'>待完成</option>"+
       		  "<option disabled value='1'>待审核</option>"+
       			"<option disabled value='2'>未通过</option>"+
@@ -860,20 +887,20 @@ $(function(){
       			"<option disabled value='6'>已下架</option>"+
       		  "<option disabled value='7'>无发布</option>"+
       			"<option disabled value='8'>需登录</option>"+
-      			"<option disabled value='9'>无法处理</option>"+
+      			"<option disabled value='9'>转python</option>"+
       			"<option disabled value='10'>已删除</option>"+
             "<option disabled value='10'>已上线</option>"+
           +"</select>"
   };
     function createComboxPlatform(code,id){
-    	return "<select onchange='changeEventOrStateOrPlatform(this.value,\"platform\",\"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+
+    	return "<select onchange='updateesp(this.value,\"platform\",\"\",\""+code+"\",\""+id+"\")' class='form-control input-sm'>"+
     			  "<option value='golua平台'>golua平台</option>"+
     			  "<option value='python'>python</option>"+
     				"<option value='通用爬虫'>通用爬虫</option>"+
     				"<option value='chrome插件'>chrome插件</option>"+
           "</select>"
   };
-    function changeEventOrStateOrPlatform(val,w,s,c,id){
+    function updateesp(val,w,s,c,id){
     	$.ajax({
 			url:"/center/spider/updateesp",
 			type:"post",
@@ -884,11 +911,11 @@ $(function(){
 				"s":s,
 				"c":c
 			},
-       success:function(r){
-				if(r=="n"){
-          showTip("修改失败", 1000, function() {});
-        }
-        ttable.ajax.reload();
+		    success:function(r){
+			   if(r=="n"){
+				  showTip("修改失败", 1000, function() {});
+				}
+        		ttable.ajax.reload();
 			}
 		})
   }

src/web/templates/luamovecode.html

@@ -95,7 +95,7 @@
 				}else if(val==8){
 					val="<span class='text-danger text-bold'>需登录</span>"
 				}else if(val==9){
-					val="<span class='text-danger text-bold'>无法处理</span>"
+					val="<span class='text-danger text-bold'>转python</span>"
 				}else if(val==10){
 					val="<span class='text-danger text-bold'>已删除</span>"
 				}

src/web/templates/sitelist.html

@@ -200,6 +200,7 @@
                     <th><input type="checkbox" id="selrow" onclick="selectrow(this)"/></th>
           				  <th>编号</th>
                     <th>站点</th>
+                    <th>重点</th>
                     <th>别名</th>
                     <th>域名</th>
                     <th>省份</th>
@@ -236,7 +237,32 @@ $(function(){
               "url": "/js/dataTables.chinese.lang"
           },
 		"columnDefs": [
-		     { "orderable": false, "targets": [0,1,3,4,5,6,7,8,9,10,11,13,14] }
+		     { "orderable": false, "targets": [0,1,3,4,5,6,7,8,9,10,11,13,14] },
+            //爬虫挂起
+            {"targets":[3], createdCell: function (cell, cellData, rowData, rowIndex, colIndex) {
+                var aInput;
+                $(cell).click(function () {
+                    $(this).html(selectImportant(rowData._id));
+                    var aInput = $(this).find(":input");
+                    aInput.focus().val(cellData);
+                });
+                $(cell).on("click", ":input", function (e) {
+                    e.stopPropagation();
+                });
+                $(cell).on("change", ":input", function () {
+                    $(this).blur();
+                });
+                $(cell).on("blur", ":input", function () {
+                    var text = $(this).find("option:selected").text();
+                    if(text == "否"){
+                        text = 0
+                    }else if(text == "是"){
+                        text = 1
+                    }
+                    ttable.cell(cell).data(text);
+                    cellData = text;
+                });
+            }}
 		],
 		"order": [[12,"desc"]],
         "lengthChange":false,
@@ -245,7 +271,7 @@ $(function(){
 		"searching": true,
 		"ordering": true,
 		"info": false,
-		"autoWidth": true,
+		"autoWidth": false,
 		"ajax": {
 			"url": "/center/site/sitelist",
 			"type": "POST"
@@ -258,6 +284,13 @@ $(function(){
 				return row.num
 			}},
 	  { "data": "site"},
+      { "data": "important",render:function(val,a,row){
+          if(val == 1){
+              return "是"
+          }else{
+              return "否"
+          }
+      }},
       { "data": "another_name"},
       { "data": "domain",render:function(val,a,row){
           return "<a href='http://"+val+"' target='_blank' style='color: #333'>"+val+"</a>"
@@ -555,6 +588,30 @@ function selectrow(me){
 			$("#sitelist td input[type=checkbox]").prop("checked",false);
 		}
 	}
+
+//
+function selectImportant(id){
+    var siteImportant="<option value=1>是</option><option value=0>否</option>";
+    return "<select onchange='updateImportant(this.value,\""+id+"\")' class='form-control input-sm'>"+siteImportant+"</select>"
+};
+
+function updateImportant(val,id) {
+    $.ajax({
+        url:"/center/site/updateimportant",
+        type:"post",
+        data:{
+            "val":val,
+            "id":id
+        },
+        success:function(r){
+            if(r=="n"){
+                showTip("修改失败", 1000, function() {});
+            }
+            ttable.ajax.reload();
+        }
+    })
+}
+
 </script>
 </div>
 {{include "bottom.html"}}

src/web/templates/sitelualist.html

@@ -96,7 +96,7 @@
               }else if(state==8){
                 val="<span class='text-danger text-bold'>需登录</span>"
               }else if(state==9){
-                val="<span class='text-danger text-bold'>无法处理</span>"
+                val="<span class='text-danger text-bold'>转python</span>"
               }else if(state==10){
                 val="<span class='text-danger text-bold'>已删除</span>"
               }else if(state==11){

src/web/templates/spiderbase.html

@@ -95,11 +95,11 @@
 						value="{{if .T.spiderhistorymaxpage}}{{.T.spiderhistorymaxpage}}{{else}}1{{end}}"
 						/>
 					</div>
-          <label class="col-sm-1 control-label " style="margin-left: -30px;">
+          			<label class="col-sm-1 control-label " style="margin-left: -30px;">
 						页
 					</label>
-				</div>
-        <div class="form-group">
+					</div>
+        			<div class="form-group">
 					<label for="spiderMaxPage" class="col-sm-4 control-label">
 						最大页　
 					</label>
@@ -108,51 +108,77 @@
 						value="{{if .T.lua.param_common}}{{index .T.lua.param_common 5}}{{else}}1{{end}}"
 						/>
 					</div>
-          <label class="col-sm-1 control-label ">
+          				<label class="col-sm-1 control-label ">
 						页
-					</label>
-				</div>
-        <div class="form-group">
-					<label for="spiderRunRate" class="col-sm-4 control-label ">
-						间隔周期
-					</label>
+						</label>
+					</div>
+        			<div class="form-group">
+						<label for="spiderRunRate" class="col-sm-4 control-label ">
+							间隔周期
+						</label>
 					<div class="col-sm-4">
 						<input type="text" style="background:red;color:#fff;margin-left: 29px;" class="form-control" name="Base.SpiderRunRate" id="spiderRunRate"
 						value="{{if .T.lua.param_common}}{{index .T.lua.param_common 6}}{{else}}30{{end}}"
 						/>
 					</div>
-					<label class="col-sm-1 control-label ">
-						分
-					</label>
-				</div>
-        <div class="form-group" id="movevent">
-					<label for="spidermovevent" class="col-sm-4 control-label ">
-						增量模式
-					</label>
-					<div class="col-sm-7">
-						<select name="OtherBase.SpiderMoveEvent" id="spidermovevent" class="form-control">
-							<option value="bid" disabled style="color:#DADADA">
-								高性能模式
-							</option>
-							<option value="comm" disabled style="color:#DADADA">
-								低性能模式
-							</option>
-              <option value="7700">
-								7700
-							</option>
-						</select>
-						<script>
-              if ({{.T.spidermovevent}} == "bid") {
-							  $("#spidermovevent").get(0).selectedIndex = 0;
-							} else if({{.T.spidermovevent}} == "comm"){
-							  $("#spidermovevent").get(0).selectedIndex = 1;
-							}else{
-                $("#spidermovevent").get(0).selectedIndex = 2;
-              }
-						</script>
+						<label class="col-sm-1 control-label ">
+							分
+						</label>
+					</div>
+<!--        			<div class="form-group" id="movevent">-->
+<!--						<label for="spidermovevent" class="col-sm-4 control-label ">-->
+<!--							增量模式-->
+<!--						</label>-->
+<!--						<div class="col-sm-7">-->
+<!--							<select name="OtherBase.SpiderMoveEvent" id="spidermovevent" class="form-control">-->
+<!--								<option value="bid" disabled style="color:#DADADA">-->
+<!--									高性能模式-->
+<!--								</option>-->
+<!--								<option value="comm" disabled style="color:#DADADA">-->
+<!--									低性能模式-->
+<!--								</option>-->
+<!--								<option value="7700">-->
+<!--									7700-->
+<!--								</option>-->
+<!--							</select>-->
+<!--							<script>-->
+<!--							if ({{.T.spidermovevent}} == "bid") {-->
+<!--								  $("#spidermovevent").get(0).selectedIndex = 0;-->
+<!--								} else if({{.T.spidermovevent}} == "comm"){-->
+<!--								  $("#spidermovevent").get(0).selectedIndex = 1;-->
+<!--								}else{-->
+<!--								$("#spidermovevent").get(0).selectedIndex = 2;-->
+<!--							}-->
+<!--							</script>-->
+<!--						</div>-->
+<!--					</div>-->
+					<div class="form-group" id="spiderevent">
+						<label for="spiderevent" class="col-sm-4 control-label ">
+							增量节点
+						</label>
+						<div class="col-sm-7">
+							<select name="OtherBase.SpiderMoveEvent" id="spidermovevent" class="form-control">
+								<option value="bid" disabled style="color:#DADADA">
+									高性能模式
+								</option>
+								<option value="comm" disabled style="color:#DADADA">
+									低性能模式
+								</option>
+								<option value="7700">
+									7700
+								</option>
+							</select>
+							<script>
+								if ({{.T.spidermovevent}} == "bid") {
+									$("#spidermovevent").get(0).selectedIndex = 0;
+								} else if({{.T.spidermovevent}} == "comm"){
+									$("#spidermovevent").get(0).selectedIndex = 1;
+								}else{
+									$("#spidermovevent").get(0).selectedIndex = 2;
+								}
+							</script>
+						</div>
 					</div>
-				</div>
-        
 				<div class="form-group">
 					<label for="spidertype" class="col-sm-4 control-label ">
 						爬虫类型
@@ -167,15 +193,15 @@
 							</option>
 						</select>
 						<script>
-              if ({{.T.spidertype}} == "increment") {
-							    $("#spiderType").get(0).selectedIndex = 0;
-                  $("#historymaxpage").addClass("hide");
-                  $("#movevent").addClass("hide");
-							} else {
-							    $("#spiderType").get(0).selectedIndex = 1;
-                  $("#historymaxpage").removeClass("hide");
-                  $("#movevent").removeClass("hide");
-							}
+              			if ({{.T.spidertype}} == "increment") {
+							$("#spiderType").get(0).selectedIndex = 0;
+                  			$("#historymaxpage").addClass("hide");
+                 			$("#movevent").addClass("hide");
+						} else {
+							$("#spiderType").get(0).selectedIndex = 1;
+                  			$("#historymaxpage").removeClass("hide");
+                  			$("#movevent").removeClass("hide");
+						}
 						</script>
 					</div>
 				</div>

src/web/templates/spideredit.html

@@ -47,7 +47,7 @@
   					<select id="assort2"> 
   						<option value=7>无发布</option> 
   						<option value=8>需登录</option> 
-  						<option value=9>无法处理</option> 
+  						<option value=9>转python</option>
   						<option value=10>需删除</option>
               <option value=11>已上线</option> 
   					</select> 
@@ -68,7 +68,7 @@
   					<select id="assort2"> 
   						<option value=7>无发布</option> 
   						<option value=8>需登录</option> 
-  						<option value=9>无法处理</option> 
+  						<option value=9>转python</option>
   						<option value=10>需删除</option>
               <option value=11>已上线</option> 
   					</select>