Merge branch 'dev3.1.2' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.1.2

src/udpfileserver/config.json

@@ -1,15 +1,12 @@
 {
   "udpip": "127.0.0.1",
   "udpport": "8888",
-  "channelsize": "5",
+  "channelsize": "1",
   "dbsize": "5",
   "mongodb_one_ip": "127.0.0.1:27017",
   "mongodb_one_db": "spider",
-  "mongodb_one_c": "data_bak",
+  "mongodb_one_c": "bidding_file",
   "mongodb_one_filefiled": "projectinfo",
-  "mongodb_two_ip": "192.168.3.207:27082",
-  "mongodb_two_db": "spider",
-  "mongodb_two_c": "data_bak",
-  "mongodb_two_filefiled": "projectinfo",
-  "file2text": "192.168.3.207:1234"
+  "file2text": "192.168.3.207:1234",
+  "PageSize":5000
 }

src/udpfileserver/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"encoding/json"
-	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
 	"jy/mongodbutil"
 	"log"
 	mu "mfw/util"
@@ -11,26 +11,28 @@ import (
 	"path"
 	qu "qfw/util"
 	"strings"
-
-	"gopkg.in/mgo.v2/bson"
+	"time"
 )
 
 var udpclient mu.UdpClient //udp对象
 var Sysconfig map[string]interface{}
 var MgoIP, MgoDB, MgoC, MgoFileFiled string
 var ChanB chan bool
+var PageSize int
 
 func init() {
 	qu.ReadConfig(&Sysconfig)
 	MgoIP = qu.ObjToString(Sysconfig["mongodb_one_ip"])
 	MgoDB = qu.ObjToString(Sysconfig["mongodb_one_db"])
 	MgoC = qu.ObjToString(Sysconfig["mongodb_one_c"])
+	PageSize = qu.IntAllDef(Sysconfig["PageSize"],2000)
 	MgoFileFiled = qu.ObjToStringDef(Sysconfig["mongodb_one_filefiled"], "projectinfo")
-	if strings.TrimSpace(MgoIP) == "" || strings.TrimSpace(MgoDB) == "" || strings.TrimSpace(MgoC) == "" {
+	if strings.TrimSpace(MgoIP) == "" || strings.TrimSpace(MgoDB) == "" || strings.TrimSpace(MgoC) == "" ||PageSize <=0{
 		log.Println("获取配置文件参数失败", Sysconfig)
 		return
 	}
 	mongodbutil.Mgo = mongodbutil.MgoFactory(qu.IntAllDef(Sysconfig["dbsize"], 5), 10, 120, MgoIP, MgoDB)
+	log.Println(mongodbutil.Mgo.Get().Ping())
 	ChanB = make(chan bool, qu.IntAllDef(Sysconfig["channelsize"], 5))
 }
 
@@ -54,46 +56,31 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			return
 		}
 		log.Println(mapInfo)
+		stime :=time.Now()
 		gid := strings.TrimSpace(mapInfo["gtid"].(string))
 		lid := strings.TrimSpace(mapInfo["lteid"].(string))
 		if bson.IsObjectIdHex(gid) && bson.IsObjectIdHex(lid) {
-			MgoSession, err := mgo.Dial(MgoIP)
-			defer MgoSession.Close()
-			if err != nil {
-				log.Println("mongo err:",err)
-				return
+			var jsq int64
+			query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(gid),"$lte": bson.ObjectIdHex(lid),},MgoFileFiled: bson.M{"$ne": nil,},}
+			log.Println("query---:", query)
+			sum :=mongodbutil.Mgo.Count(MgoC,query)
+			log.Println("sum:", sum)
+			pageNum := (sum + PageSize - 1) / PageSize
+			limit := PageSize
+			if sum < PageSize {
+				limit = sum
 			}
+			for i := 0; i < pageNum; i++ {
+				query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(gid)}}
+				log.Println("page=", i+1,"query=", query)
+				list, b := mongodbutil.Mgo.Find(MgoC,query,nil,bson.M{"_id": 1,MgoFileFiled:1},false,0, limit)
+				if !b{
+					log.Println("查询失败")
+					continue
+				}
 
-			iter := MgoSession.DB(MgoDB).C(MgoC).Find(
-				bson.M{
-					"_id": bson.M{
-						"$gte": bson.ObjectIdHex(gid),
-						"$lte": bson.ObjectIdHex(lid),
-					},
-					MgoFileFiled: bson.M{
-						"$ne": nil,
-					},
-				},).Select(bson.M{"_id": 1,MgoFileFiled:1}).Iter()
-
-			//if findAll, b := mongodbutil.Mgo.Find(MgoC,
-			//	bson.M{
-			//		"_id": bson.M{
-			//			"$gte": bson.ObjectIdHex(gid),
-			//			"$lte": bson.ObjectIdHex(lid),
-			//		},
-			//		MgoFileFiled: bson.M{
-			//			"$ne": nil,
-			//		},
-			//	},
-			//	//if findAll, b := mongodbutil.Mgo.Find(MgoC, bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(gid), "$lte": bson.ObjectIdHex(lid)}},
-			//	nil, `{"_id":"1",`+MgoFileFiled+`:"1"}`, false, -1, -1); !b {
-			//	log.Println("查询数据失败 :", string(data))
-			//} else {
-			var result *map[string]interface{}
-			log.Println("处理查询数据...")
-			for iter.Next(&result){
-				//for _, v := range *result {
-					qmap := qu.ObjToMap(result)
+				for _,v:=range *list {
+					qmap := qu.ObjToMap(v)
 					mid := (*qmap)["_id"]
 					if v, ok := (*qmap)[MgoFileFiled].(map[string]interface{}); !ok {
 						mongodbutil.Mgo.UpdateById(MgoC, mid, bson.M{
@@ -122,43 +109,11 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 							}
 						}
 					}
-					//fileMap := *qu.ObjToMap(qmap["projectinfo"])
-					//fmt.Println(fileMap["attachments"])
+					gid = qu.BsonIdToSId(v["_id"])
+					jsq++
 				}
-			//}
-			defer iter.Close()
-			log.Println("处理查询数据结束...")
-			//fmt.Println(len(*findAll))
-				//if len(*findAll) <= 0 {
-				//	log.Println("查询数据为空 :", string(data))
-				//	return
-				//}
-				//for _, v := range *findAll {
-				//	qmap := *qu.ObjToMap(v)
-				//	mid := qmap["_id"]
-				//	if v, ok := qmap[MgoFileFiled].(map[string]interface{}); !ok {
-				//		log.Println(mid, "mgo 转换异常", MgoFileFiled)
-				//		continue
-				//	} else {
-				//		switch v["attachments"].(type) {
-				//		case map[string]interface{}:
-				//			att := v["attachments"].(map[string]interface{})
-				//			for _, vaatt := range att {
-				//				if fileinfo, ok := vaatt.(map[string]interface{}); !ok {
-				//					log.Println(mid, "mgo 结构体转换失败", vaatt)
-				//					continue
-				//				} else {
-				//					ChanB <- true
-				//					go save(mid, qmap, fileinfo)
-				//
-				//				}
-				//			}
-				//		}
-				//	}
-				//	//fileMap := *qu.ObjToMap(qmap["projectinfo"])
-				//	//fmt.Println(fileMap["attachments"])
-				//}
-			//}
+			}
+			log.Println("处理查询数据结束...",jsq,time.Now().Sub(stime))
 		} else {
 			log.Println("开始id或结束id参数错误:", string(data))
 		}
@@ -231,7 +186,7 @@ func save(mid interface{},attk string, qmap, fileinfo *map[string]interface{}) {
 		log.Println(mid, "rpc返回数据为空:", string(reply))
 		return
 	}
-	log.Println(mid, string(reply)[:23])
+	//log.Println(mid, string(reply))
 	rdata := make(map[string]interface{})
 	if err := json.Unmarshal(reply, &rdata); err != nil {
 		mongodbutil.Mgo.UpdateById(MgoC, mid, bson.M{
@@ -249,9 +204,10 @@ func save(mid interface{},attk string, qmap, fileinfo *map[string]interface{}) {
 		}
 		//log.Println((*fileinfo))
 
-		asdf := (*qmap)[MgoFileFiled].(map[string]interface{})
-		qwer := asdf["attachments"].(map[string]interface{})
-		qwer[attk] =*fileinfo
+		(*qmap)[MgoFileFiled].(map[string]interface{})["attachments"].(map[string]interface{})[attk]=*fileinfo
+		//asdf := (*qmap)[MgoFileFiled].(map[string]interface{})
+		//qwer := asdf["attachments"].(map[string]interface{})
+		//qwer[attk] =*fileinfo
 		//log.Println((*qmap)[MgoFileFiled])
 
 		updateBool := mongodbutil.Mgo.UpdateById(MgoC, mid, bson.M{

src/web/templates/admin/audit_classlist.html

@@ -63,7 +63,7 @@ $(function () {
         },
 		"columns": [
             { "data": "s_name"},
-			{ "data": "l_lasttime"},
+			{ "data": "l_createtime"},
 			{ "data": "s_user"},
 			{ "data": "_id",render:function(val,a,row){
 				return '<a class="btn btn-sm btn-info opr" opr="edit">编辑</a>'+

src/web/templates/admin/audit_recogfield.html

@@ -62,7 +62,7 @@ $(function () {
 		"columns": [
             { "data": "s_name"},
 			{ "data": "s_recogfield"},
-			{ "data": "l_lasttime"},
+			{ "data": "l_createtime"},
 			{ "data": "s_user"},
 			{ "data": "_id",render:function(val,a,row){
 				return '<a class="btn btn-sm btn-info opr" opr="edit">编辑</a>'+

src/web/templates/admin/audit_rulelist.html

@@ -65,7 +65,7 @@ $(function () {
         },
 		"columns": [
             { "data": "s_name"},
-			{ "data": "l_lasttime"},
+			{ "data": "l_createtime"},
 			{ "data": "s_user"},
 			{ "data": "_id",render:function(val,a,row,meta){
 				var udhtml = '&nbsp;&nbsp;<a class="btn btn-sm btn-success opr" opr="moveup" num="'+meta.row+'">上移</a>'+