Bläddra i källkod

Merge branch 'dev3.4' of http://39.105.157.10:10080/qmx/jy-data-extract into dev3.4

fengweiqiang 5 år sedan
förälder
incheckning
d839d3c17a

+ 8 - 9
fullproject/src_v1/load_data.go

@@ -30,7 +30,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 			case tmp := <-pool:
 				n++
 				if n%10000 == 0 {
-					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
+					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
 				}
 				if tmp != nil {
 					id := tmp.Id.Hex()
@@ -143,17 +143,16 @@ func (p *ProjectTask) loadSite() {
 }
 
 func saveFiled(p *ProjectTask, res map[string]interface{}, tmp *ProjectInfo) {
-	if jsonData, ok := res["jsondata"].(map[string]interface{}); ok {
-		proHref := util.ObjToString(jsonData["projecthref"])
-		if jsonData != nil && proHref != "" {
-			p.mapHrefLock.Lock()
-			p.mapHref[proHref] = tmp.Id.Hex()
-			p.mapHrefLock.Unlock()
-		}
+	proHref := util.ObjToString(res["projecthref"])
+	if proHref != "" {
+		p.mapHrefLock.Lock()
+		p.mapHref[proHref] = tmp.Id.Hex()
+		p.mapHrefLock.Unlock()
 	}
+
 	tmpMap := make(map[string]InfoField)
 	infoMap := res["infofield"].(map[string]interface{})
-	for _, v := range infoMap{
+	for _, v := range infoMap {
 		var field InfoField
 		b, _ := json.Marshal(v)
 		_ = json.Unmarshal(b, &field)

+ 228 - 0
src/jy/admin/audit/buyerinfo.go

@@ -0,0 +1,228 @@
+package audit
+
+import (
+	"github.com/gin-gonic/gin"
+	"gopkg.in/mgo.v2/bson"
+	. "jy/admin"
+	"jy/clear"
+	. "jy/mongodbutil"
+	"jy/util"
+	"log"
+	"qfw/util/elastic"
+	"strings"
+)
+
+/**
+企业库
+ */
+func init() {
+	//页面
+	Admin.GET("/audit/buyer_info", func(c *gin.Context) {
+		c.HTML(200, "buyerinfo.html", gin.H{})
+	})
+	//save
+	Admin.POST("/audit/buyer_info/save", func(c *gin.Context) {
+		_id := c.PostForm("_id")
+		company_name := c.PostForm("company_name")
+		alias := c.PostForm("alias")
+		province := c.PostForm("province")
+		city := c.PostForm("city")
+		district := c.PostForm("district")
+		capital := c.PostForm("capital")
+		company_address := c.PostForm("company_address")
+		if strings.TrimSpace(company_name) == "" {
+			c.JSON(200, gin.H{"rep": 400})
+			return
+		}
+		capitalfloat := clear.ObjToMoney([]interface{}{capital, ""})[0]
+		e := make(map[string]interface{})
+		e["company_name"] = company_name
+		e["alias"] = alias
+		e["province"] = province
+		e["city"] = city
+		e["district"] = district
+		e["capital"] = capitalfloat
+		e["company_address"] = company_address
+		var sid string
+		if strings.TrimSpace(_id) != "" && bson.IsObjectIdHex(_id) {
+			sid = _id
+			tmpb := Mgo.Update("enterprise_qyxy", bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{
+				"company_name":    company_name,
+				"alias":           alias,
+				"province":        province,
+				"city":            city,
+				"district":        district,
+				"capital":         capitalfloat,
+				"company_address": company_address,
+			}}, false, false)
+			if tmpb {
+				escon := elastic.GetEsConn()
+				defer elastic.DestoryEsConn(escon)
+				_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).
+					Id(sid).Doc(map[string]interface{}{
+					"company_name":    company_name,
+					"alias":           alias,
+					"province":        province,
+					"city":            city,
+					"district":        district,
+					"capital":         capitalfloat,
+					"company_address": company_address,
+				}).Refresh(true).Do()
+				if err != nil {
+					log.Println("update qyk err:", err)
+				}
+			}
+		} else {
+			sid = Mgo.Save("enterprise_qyxy", e)
+			delete(e, "_id")
+			escon := elastic.GetEsConn()
+			defer elastic.DestoryEsConn(escon)
+			_, err := escon.Index().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
+			if err != nil {
+				log.Println("save qyk err:", err)
+			}
+		}
+		c.JSON(200, gin.H{"rep": 200, "saveid": sid})
+	})
+	////列表查询
+	//Admin.POST("/audit/query_qyk/list", func(c *gin.Context) {
+	//	search, _ := c.GetPostForm("search[value]")
+	//	startstr, _ := c.GetPostForm("start")
+	//	limitstr, _ := c.GetPostForm("length")
+	//	start, _ := strconv.Atoi(startstr)
+	//	limit, _ := strconv.Atoi(limitstr)
+	//	if limit < 1 {
+	//		limit = 10
+	//	}
+	//	if search == "" {
+	//		c.JSON(200, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
+	//	} else {
+	//		//log.Println(util.ElasticClientIndex, util.ElasticClientType, search)
+	//		escon := elastic.GetEsConn()
+	//		defer elastic.DestoryEsConn(escon)
+	//		res, err := escon.Search(util.ElasticClientIndex).
+	//			Type(util.ElasticClientType).Source(`{"query": {"match_phrase":{"company_name":"`+search+`"}}}`).
+	//			//Query( elastic.NewMatchPhraseQuery("company_name", search)).
+	//			Size(limit).
+	//			From(start).
+	//			Do()
+	//		if err != nil {
+	//			log.Println(err)
+	//			c.JSON(500, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
+	//			return
+	//		}
+	//		if res.Hits != nil{
+	//			tmps := make([]map[string]interface{}, 0)
+	//			for _, v := range res.Hits.Hits {
+	//				tmp := make(map[string]interface{})
+	//				err := json.Unmarshal(*v.Source, &tmp)
+	//				if err != nil {
+	//					log.Println(err)
+	//					continue
+	//				}
+	//				tmp["_id"] = v.Id
+	//				//log.Println(tmp)
+	//				tmps = append(tmps, tmp)
+	//			}
+	//			//count := Mgo.Count("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}})
+	//			//data, _ := Mgo.Find("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}}, `{"_id":-1}`, nil, false, start, limit)
+	//			c.JSON(200, gin.H{"data": tmps, "recordsFiltered": res.Hits.TotalHits, "recordsTotal": res.Hits.TotalHits})
+	//		}else {
+	//			c.JSON(200, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
+	//		}
+	//	}
+	//})
+	////delete
+	//Admin.POST("/audit/buyer_info/deleteQyk", func(c *gin.Context) {
+	//	_id := c.PostForm("_id")
+	//	if bson.IsObjectIdHex(_id) {
+	//		delisok := Mgo.Del("enterprise_qyxy", bson.M{"_id": bson.ObjectIdHex(_id)})
+	//		escon := elastic.GetEsConn()
+	//		defer elastic.DestoryEsConn(escon)
+	//		_, err := escon.Delete().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Refresh(true).Do()
+	//		if err != nil {
+	//			log.Println("delete qyk err:", err)
+	//		}
+	//		c.JSON(200, gin.H{"rep": 200, "data": delisok})
+	//	} else {
+	//		c.JSON(200, gin.H{"rep": 400})
+	//	}
+	//})
+	////queryById
+	//Admin.POST("/audit/query_qyk/ById", func(c *gin.Context) {
+	//	_id := c.PostForm("_id")
+	//	q_field := c.PostForm("q_field")
+	//	if bson.IsObjectIdHex(_id) && strings.TrimSpace(q_field) != "" {
+	//		data, _ := Mgo.FindById(util.ElasticClientDB, _id, bson.M{q_field: 1})
+	//		c.JSON(200, gin.H{"rep": 200, "data": data})
+	//	} else {
+	//		c.JSON(200, gin.H{"rep": 400})
+	//	}
+	//})
+	////updateIndustrys
+	//Admin.POST("/audit/query_qyk/UpdateIndustrys", func(c *gin.Context) {
+	//	_id := c.PostForm("_id")
+	//	industrys := c.PostFormArray("industry")
+	//	//log.Println(_id,industrys)
+	//	if bson.IsObjectIdHex(_id) {
+	//		b := Mgo.Update(util.ElasticClientDB, bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"industry": industrys}}, false, false)
+	//		if b {
+	//			escon := elastic.GetEsConn()
+	//			defer elastic.DestoryEsConn(escon)
+	//			_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
+	//				"industry": industrys,
+	//			}).Refresh(true).Do()
+	//			if err != nil {
+	//				log.Println("update yqk industry err :", err)
+	//			}
+	//		}
+	//		c.JSON(200, gin.H{"rep": 200, "data": b})
+	//	} else {
+	//		c.JSON(200, gin.H{"rep": 400})
+	//	}
+	//})
+	////updateTels
+	//Admin.POST("/audit/query_qyk/UpdateTels", func(c *gin.Context) {
+	//	_id := c.PostForm("_id")
+	//	//log.Println(_id)
+	//	contact_persons := c.PostFormArray("contact_persons")
+	//	contact_types := c.PostFormArray("contact_types")
+	//	phones := c.PostFormArray("phones")
+	//	topscopeclasss := c.PostFormArray("topscopeclasss")
+	//	if bson.IsObjectIdHex(_id) && len(contact_persons) == len(contact_types) && len(phones) == len(topscopeclasss) && len(phones) == len(contact_persons) {
+	//		contacts := make([]map[string]interface{}, 0)
+	//		for _, v := range contact_persons {
+	//			tmp := make(map[string]interface{})
+	//			tmp["contact_person"] = v
+	//			contacts = append(contacts, tmp)
+	//		}
+	//		for i, v := range contact_types {
+	//			contacts[i]["contact_type"] = v
+	//		}
+	//		for i, v := range phones {
+	//			contacts[i]["phone"] = v
+	//		}
+	//		for i, v := range topscopeclasss {
+	//			contacts[i]["topscopeclass"] = v
+	//			contacts[i]["updatetime"] = time.Now().Unix()
+	//		}
+	//		//for k,v := range contacts{
+	//		//	log.Println(k,v)
+	//		//}
+	//		b := Mgo.Update(util.ElasticClientDB, bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"contact": contacts}}, false, false)
+	//		if b {
+	//			escon := elastic.GetEsConn()
+	//			defer elastic.DestoryEsConn(escon)
+	//			_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
+	//				"contact": contacts,
+	//			}).Refresh(true).Do()
+	//			if err != nil {
+	//				log.Println("update yqk contact err :", err)
+	//			}
+	//		}
+	//		c.JSON(200, gin.H{"rep": 200, "data": b})
+	//	} else {
+	//		c.JSON(200, gin.H{"rep": 400})
+	//	}
+	//})
+}

+ 15 - 16
src/jy/admin/audit/dataaudit.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	qu "qfw/util"
 	redis "qfw/util/redis"
-	"strconv"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -16,8 +15,10 @@ func init() {
 	//列表、编辑
 	Admin.GET("/audit/dataaudit", func(c *gin.Context) {
 		name := c.Query("name")
-		//s_coll := c.Query("s_coll")
-		c.HTML(200, "audit_auditone.html", gin.H{"name": name})
+		//lib := c.Query("lib")
+		coll := c.Query("coll")
+		auditattr := c.Query("auditattr")
+		c.HTML(200, "audit_auditone.html", gin.H{"name": name, "coll": coll, "auditattr": auditattr})
 	})
 	Admin.POST("/audit/auditonefield", AuditOneField)
 	Admin.POST("/audit/buyerclass", BuyerClass) //采购单位分类
@@ -47,21 +48,19 @@ func AuditData(c *gin.Context) {
 
 func AuditOneField(c *gin.Context) {
 	field, _ := c.GetPostForm("field")
-	limitstr, _ := c.GetPostForm("length")
-	startstr, _ := c.GetPostForm("start")
-	start, _ := strconv.Atoi(startstr)
-	limit, _ := strconv.Atoi(limitstr)
+	//lib, _ := c.GetPostForm("lib")
+	coll, _ := c.GetPostForm("coll")
+	auditattr, _ := c.GetPostForm("auditattr")
 	query := map[string]interface{}{
-		"field":     field,
-		"ispackage": false,
-	}
-	data, _ := Mgo.Find("audit_err", query, `{"_id":-1}`, nil, false, start, limit)
-	for _, d := range *data {
-		timeStr := time.Unix(d["l_createtime"].(int64), 0).Format(Date_Short_Layout)
-		d["l_createtime"] = timeStr
+		auditattr + "_" + field: 1,
 	}
-	count := Mgo.Count("audit_err", query)
-	c.JSON(200, gin.H{"data": data, "recordsFiltered": count, "recordsTotal": count})
+	data, _ := Mgo.Find(coll, query, `{"_id":1}`, nil, false, -1, -1)
+	//	for _, d := range *data {
+	//		timeStr := time.Unix(d["l_createtime"].(int64), 0).Format(Date_Short_Layout)
+	//		d["l_createtime"] = timeStr
+	//	}
+	//	count := Mgo.Count("audit_err", query)
+	c.JSON(200, gin.H{"data": data})
 }
 
 func BuyerClass(c *gin.Context) {

+ 1 - 0
src/jy/clear/clear.go

@@ -28,6 +28,7 @@ func init() {
 	BindFn("chiToEng", ChiToEng)                 //中文符号转英文
 	BindFn("clearBuyerPerson", ClearBuyerPerson) //处理较长采购联系人
 	BindFn("clearNumber", ClearNumber)           //一般用于处理抽取联系人后带有电话的情况
+	BindFn("clearEndSymblo", ClearEndSymblo)     //去除尾部特殊符号
 }
 
 //绑定清理方法

+ 12 - 0
src/jy/clear/cutspace.go

@@ -13,6 +13,7 @@ var (
 	separateSymbol *regexp.Regexp
 	placeReg       *regexp.Regexp
 	clearNum       *regexp.Regexp
+	endSymblo      *regexp.Regexp
 )
 
 var spaces = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
@@ -24,6 +25,7 @@ func init() {
 	separateSymbol, _ = regexp.Compile("[\\s\u3000\u2003\u00a0\\n,,、/。|]")
 	placeReg, _ = regexp.Compile("^.*(公司|学(校)?|集团|单位|机构|企业|厂|场|院|所|店|中心|市|局|站|城|处|行|部|队|联合(会|体)|工作室)$")
 	clearNum, _ = regexp.Compile("[\\d-]+")
+	endSymblo = regexp.MustCompile(`[/\\?]$`)
 }
 
 var LableStr = "&?(amp|nbsp|#8266);?|(&lt;).*?(&gt;?)"
@@ -92,6 +94,16 @@ func CutAllSpace(data []interface{}) []interface{} {
 	return data
 }
 
+//清理尾部符号
+func ClearEndSymblo(data []interface{}) []interface{} {
+	text := fmt.Sprint(data[0])
+	for i := 0; i <= 2; i++ {
+		text = endSymblo.ReplaceAllString(text, "")
+	}
+	data[0] = text
+	return data
+}
+
 //清理符号
 func CutSymbol(data []interface{}) []interface{} {
 	value := fmt.Sprint(CutSpace(data)[0])

+ 3 - 3
src/main_blocktest.go

@@ -26,7 +26,7 @@ func main1() {
 	one()
 }
 func all() {
-	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27082", "extract_kf")
+	m := mongodbutil.MgoFactory(3, 3, 120, "127.0.0.1:27092", "extract_kf")
 	sess := m.Get()
 	defer m.Close(sess)
 	it := sess.DB("extract_kf").C("bidding201901").Find(nil).Iter()
@@ -50,8 +50,8 @@ func all() {
 	time.Sleep(time.Hour)
 }
 func one() {
-	m := mongodbutil.MgoFactory(3, 3, 120, "192.168.3.207:27081", "qfw")
-	d, _ := m.FindById("bidding", "5e17dfcb50b5ea296ec93aea", extract.Fields)
+	m := mongodbutil.MgoFactory(3, 3, 120, "127.0.0.1:27092", "extract_kf")
+	d, _ := m.FindById("bidding_data", "5e34e8f550b5ea296ed0deeb", extract.Fields)
 	com(*d)
 }
 func com(doc map[string]interface{}) {

+ 38 - 2
src/res/fieldscore.json

@@ -320,7 +320,7 @@
         "negativewords": [
             {
                 "describe": "出现符号",
-                "regstr": "[*]",
+                "regstr": "[*a-z]",
                 "score": -10
             },
 			{
@@ -339,7 +339,43 @@
                 "range": [
                     0,
                     1,
-                    -5
+                    -10
+                ]
+            }
+        ]
+    },
+	"buyerperson": {
+        "type": "string",
+        "positivewords": [
+            {
+                "describe": "以*结尾",
+                "regstr": ".{2,100}(工|老师|经理|女士|先生|主任|科长)$",
+                "score": 3
+            }
+        ],
+        "negativewords": [
+            {
+                "describe": "出现符号",
+                "regstr": "[*a-z]",
+                "score": -10
+            },
+			{
+                "describe": "是数字",
+                "regstr": "^\\d*[×―—-\\-]*[\u3000\u2003\u00a0\\s]*\\d*$",
+                "score": -10
+            },  {
+                "describe": "包含负分",
+                "regstr": "(详见公告)",
+                "score": -10
+            }
+        ],
+        "length": [
+            {
+                "describe": "[gt,lte,score]",
+                "range": [
+                    0,
+                    1,
+                    -10
                 ]
             }
         ]

+ 10 - 0
src/web/templates/admin/agencyinfo.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+
+</body>
+</html>

+ 18 - 38
src/web/templates/admin/audit_auditone.html

@@ -8,16 +8,14 @@
 	<section class="content-header">
 		<h1>
 			<small>
-				{{if eq .name "buyer" "winner"}}
+				<!--{{if eq .name "buyer" "winner"}}
 				<button class="btn btn-primary" onclick="addaudit()">新增{{.name}}</button>
-				{{end}}
+				{{end}}-->
 			</small>
 		</h1>
 		<ol class="breadcrumb">
-		<li><a href="/admin/audit/recogfield"><i class="fa fa-dashboard"></i> 识别字段</a></li>
-			<li class="active"><a href="#">数据审核</a></li>
-		  	
-			 
+		<li><a href="/admin/audit/recogfield"><i class="fa fa-dashboard"></i> 字段审核</a></li>
+			<li class="active"><a href="#">数据列表</a></li>
 		</ol>
     </section>
   <!-- Main content -->
@@ -29,12 +27,11 @@
 		            <table id="dataauditTable" class="table table-bordered table-hover">
 		              <thead>
 		              <tr>
-		                <th>字段名称</th>
 						<th>字段值</th>
-						<th>ExtractID</th>
+						<!--<th>ExtractID</th>
 						<th>结果表</th>
 						<th>分包</th>
-						<th>时间</th>
+						<th>时间</th>-->
 						<th>操作</th>
 		              </tr>
 		              </thead>
@@ -49,7 +46,7 @@
 </div>
 
 <!--新增标签--> 
-<div class="modal fade" id="modal-info-auditdata">
+<!--<div class="modal fade" id="modal-info-auditdata">
   	<div class="modal-dialog">
 	    <form id="add-dataform" class="form-horizontal" role="form">
 		<div class="modal-content">
@@ -80,21 +77,14 @@
 		        <button type="button" class="btn btn-primary" onclick="saveaudit()">保存</button>
 	    	</div>
 		</div>
-	    <!-- /.modal-content -->
 	    </form>
   	</div>
-  <!-- /.modal-dialog -->
-</div>
+</div>-->
 <!-- /.modal -->
 {{template "dialog"}}
 {{template "footer"}}
 <script>
 menuActive("recogfield")
-var id = "";
-var eid = "";
-var classifymap = {};
-var bid = 1;
-var coll = "";
 var field = {{.name}}
 $(function () {
 	ttable=$('#dataauditTable').DataTable({
@@ -103,31 +93,21 @@ $(function () {
 		"searching"   : true,
 		"ordering"    : false,
 		"info"        : true,
-		"autoWidth"   : false,
-		"serverSide"  : true,
+		"autoWidth"   : true,
+		"serverSide"  : false,
 		"ajax": {
 			"url": "/admin/audit/auditonefield",
 			"type": "post",
-			"data": {"field":{{.name}}}
+			"data": {"field":{{.name}},"coll":{{.coll}},"auditattr":{{.auditattr}}}
 		},
 		"language": {
-            "url": "/res/dist/js/dataTables.chinese.lang"
-        },
+            	"url": "/res/dist/js/dataTables.chinese.lang"
+       	},
 		"columns": [
-            { "data": "field"},
-			{ "data": "val"},
-			{ "data": "extractid"},
-			{ "data": "extractcoll"},
-			{ "data": "ispackage",render:function(val,a,row){
-				if(val){
-					return "是"
-				}else{
-					return "否"
-				}
-			}},
-			{ "data": "l_createtime"},
+			{ "data": {{.name}}},
 			{ "data": "_id",render:function(val,a,row){
-					return '<a class="btn btn-sm btn-success" href="#" onclick="audit(\''+row.val+'\',\''+val+'\',\''+row.extractid+'\',\''+row.extractcoll+'\')">审核</a>'
+					//return '<a class="btn btn-sm btn-success" href="#" onclick="audit(\''+row.val+'\',\''+val+'\',\''+row.extractid+'\',\''+row.extractcoll+'\')">审核</a>'
+				return '<a class="btn btn-sm btn-success" href="#">审核<a>';
 			}}
        	]
 	});
@@ -233,7 +213,7 @@ function auditsave(parentclass,childclass){
 	});
 }
 
-$(document).ready(function(){
+/*$(document).ready(function(){
 	$("#parentclass").change(function(){
 	    var parentname = $(this).val();
 		var child = classifymap[parentname];
@@ -246,5 +226,5 @@ $(document).ready(function(){
 			}
 		}
 	});
-});
+});*/
 </script>

+ 5 - 1
src/web/templates/admin/audit_recogfield.html

@@ -66,7 +66,7 @@ $(function () {
 			{ "data": "s_user"},
 			{ "data": "_id","width":"30%",render:function(val,a,row){
 				 return	 '<a class="btn btn-sm btn-info opr" opr="edit">编辑</a>'+
-					'&nbsp;&nbsp;<a class="btn btn-sm btn-success" href="/admin/audit/dataaudit?name='+row["s_recogfield"]+'">数据审核</a>'+
+					'&nbsp;&nbsp;<a class="btn btn-sm btn-success" href="/admin/audit/dataaudit?name='+row["s_recogfield"]+'&coll='+row["s_coll"]+'&auditattr='+row["s_auditattr"]+'">数据审核</a>'+
 					'&nbsp;&nbsp;<a class="btn btn-sm btn-warning" href="/admin/rulemanager/getrulelist?id='+val+'&fname='+row["s_recogfield"]+'">编辑规则</a>'+
 					'&nbsp;&nbsp;<a class="btn btn-sm btn-danger" onclick="del(\''+val+'\')">删除</a>'
 			}}
@@ -84,6 +84,10 @@ $(function () {
 				tag=[
 					{label:"名称",s_label:"s_name",must:true},
 					{label:"审核字段",s_label:"s_recogfield",must:true},
+					//{label:"信息库",s_label:"s_lib",must:true},
+					{label:"信息表",s_label:"s_coll",must:true},
+					{label:"异常标记",s_label:"s_tagattr",must:true},
+					{label:"审核标记",s_label:"s_auditattr",type:"tpl_list_local",must:true,list:[{"s_name":"正确","_id":"ok"},{"s_name":"异常","_id":"err"}],default:"ok"},
 					{s_label:"_id",type:"tpl_hidden"},
 				]
 				if(n == "new"){

+ 3 - 3
src/web/templates/admin/audit_rulelist.html

@@ -71,9 +71,9 @@ $(function () {
 			{ "data": "s_username"},
 			{ "data": "s_descript"},
 			{ "data": "s_type",render:function(val){
-				if(val=="0"){
+				if(val=="ok"){
 					return "正确";
-				}else if(val == "1"){
+				}else if(val == "err"){
 					return "异常";
 				}
 			}},
@@ -156,7 +156,7 @@ $(function () {
 					{label:"名称",s_label:"s_name",must:true},
 					{label:"描述",s_label:"s_descript"},
 					{label:"启用",s_label:"isuse",type:"tpl_list_local",list:[{"s_name":"是","_id":true},{"s_name":"否","_id":false}],default:true},
-					{label:"类型",s_label:"s_type",type:"tpl_list_local",must:true,list:[{"s_name":"正确","_id":"0"},{"s_name":"异常","_id":"1"}],default:"0"},
+					{label:"类型",s_label:"s_type",type:"tpl_list_local",must:true,list:[{"s_name":"正确","_id":"ok"},{"s_name":"异常","_id":"err"}],default:"ok"},
 					{label:"正则",s_label:"s_rule",type:"tpl_text",rows:2,must:true},
 					{s_label:"_id",type:"tpl_hidden"},
 					{s_label:"s_fid",type:"tpl_hidden",val:fid},

+ 486 - 0
src/web/templates/admin/buyerinfo.html

@@ -0,0 +1,486 @@
+{{template "inc"}}
+<!-- Main Header -->
+{{template "header"}}
+<!-- Left side column. 权限菜单 -->
+{{template "memu"}}
+<link href="/res/other/css/other.css" rel="stylesheet" type="text/css">
+<style>
+    #dataTable_filter div {
+        padding: 0px 10px;
+    }
+
+    .operate a {
+        padding: 5px 10px;
+    }
+</style>
+<div class="content-wrapper">
+    <section class="content-header" style="padding: 49px 15px 0 15px;">
+        <h1>
+            <small><a class="btn btn-primary opraaa" opraaa="newqyk">新增buyer信息</a></small>
+        </h1>
+        <ol class="breadcrumb">
+            <li><a href="#"><i class="fa fa-dashboard"></i> buyer增删改查</a></li>
+        </ol>
+    </section>
+    <!-- Main content -->
+    <section class="content">
+        <div class="row">
+            <div class="col-xs-12">
+                <div class="box">
+                    <div class="box-body">
+                        <table id="dataTable" class="table table-bordered table-hover">
+                            <thead>
+                            <tr>
+                                <th>企业名称</th>
+                                <th>别名</th>
+                                <th>省份</th>
+                                <th>城市</th>
+                                <th>区县</th>
+                                <th>注册资本</th>
+                                <th>企业地址</th>
+                                <th>操作</th>
+                            </tr>
+                            </thead>
+                        </table>
+                    </div>
+                    <!-- /.box-body -->
+                </div>
+                <!-- /.box -->
+            </div>
+        </div>
+    </section>
+
+</div>
+<div class="modal fade" id="info_data">
+    <div class="modal-dialog">
+        <form id="update-dataform" class="form-horizontal" role="form">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                        <span aria-hidden="true">&times;</span></button>
+                    <h4 class="modal-title" id="info_title"></h4>
+                </div>
+                <div class="modal-body" id="infos">
+                </div>
+            </div>
+            <!-- /.modal-content -->
+        </form>
+    </div>
+    <!-- /.modal-dialog -->
+</div>
+{{template "dialog"}}
+{{template "footer"}}
+<script>
+    menuActive("qiyeku_info")
+    var ttable = {}
+    $(function () {
+        ttable = $('#dataTable').DataTable({
+            "paging": true,
+            "lengthChange": false,
+            "searching": true,
+            "ordering": false,
+            "info": true,
+            "autoWidth": false,
+            "serverSide": true,
+            "ajax": {
+                "url": "/admin/audit/query_qyk/list",
+                "type": "post",
+            },
+            "language": {
+                "url": "/res/dist/js/dataTables.chinese.lang"
+            },
+            "columns": [
+                {
+                    "data": "company_name", "width": "18%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "alias", "width": "11%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "province", "width": "4%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "city", "width": "4%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "district", "width": "4%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "capital", "width": "6%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "company_address", "width": "17%", render: function (val, a, row) {
+                        if (val == null) {
+                            return ""
+                        }
+                        return val
+                    }
+                },
+                {
+                    "data": "_id", "width": "24%", render: function (val, a, row) {
+                        var valueStr = JSON.stringify(row);
+                        return "<div class='operate'>" +
+                            "<a class='btn btn-primary' onclick='editdata(" + valueStr + ")'>编辑</a>&nbsp;" +
+                            "<a class='btn btn-default' onclick='showHylxs(\"" + val + "\")'>详细行业类型</a>&nbsp;" +
+                            "<a class='btn btn-sm btn-warning' onclick='showTels(\"" + val + "\")'>详细联系方式</a>&nbsp;" +
+                            "<a class='btn btn-sm btn-danger' onclick='deleteQyk(\"" + val + "\")'>删除</a>&nbsp;" +
+                            "</div>"
+                    }
+                }
+            ]
+        });
+        /*新增企业库*/
+        ttable.on('init.dt', function () {
+            $(".opraaa").click(function () {
+                var n = $(this).attr("opraaa");
+                var _tit = "", htmlObj = {}, obj, tag = [];
+                switch (n) {
+                    case "newqyk":
+                        tag = [
+                            {label: "企业名称:", s_label: "s_company_name", must: true},
+                            {label: "别名:", s_label: "s_alias"},
+                            {label: "省份:", s_label: "s_province"},
+                            {label: "城市:", s_label: "s_city"},
+                            {label: "区县:", s_label: "s_district"},
+                            {label: "注册资本:", s_label: "s_capital", placeholder: "示例:45678.98元;456.78万元;1234.25;"},
+                            {label: "企业地址:", s_label: "s_company_address"},
+                        ];
+                        //新增企业库按钮
+                        htmlObj = {
+                            title: "新增企业库信息",
+                            tag: tag,
+                            bts: [
+                                {
+                                    label: "Save", class: "btn-danger",
+                                    fun: function () {
+                                        var company_name = $("#s_company_name").val();
+                                        var alias = $("#s_alias").val();
+                                        var province = $("#s_province").val();
+                                        var city = $("#s_city").val();
+                                        var district = $("#s_district").val();
+                                        var capital = $("#s_capital").val();
+                                        var company_address = $("#s_company_address").val();
+                                        if (company_name === "") {
+                                            alert("红色标签的表单不能为空!");
+                                            return
+                                        }
+                                        //新增企业库api
+                                        $.post("/admin/audit/qiyeku_info/save", {
+                                            "company_name": company_name,
+                                            "alias": alias,
+                                            "province": province,
+                                            "city": city,
+                                            "district": district,
+                                            "capital": capital,
+                                            "company_address": company_address,
+                                        }, function (data) {
+                                            if (data && data.rep) {
+                                                $("#myModal").modal("hide");
+                                                alert("保存成功");
+                                                setTimeout(function () {
+                                                    ttable.search(company_name).draw();
+                                                    //    history.go(0) //刷新本页
+                                                }, 1500) //停1.5秒
+
+                                            } else {
+                                                alert("内部错误");
+                                            }
+                                        }, 'json')
+                                    }
+                                }
+                            ]
+                        };
+                        OpenDialog(htmlObj, obj);
+                        break;
+                }
+            });
+        })
+    });
+
+    //详细行业类型
+    function showHylxs(_id) {
+        $("#infos").html("");
+        $("#info_title").html("详细行业类型");
+        $.post("/admin/audit/query_qyk/ById", {
+            "_id": _id,
+            "q_field": "industry",
+        }, function (data) {
+            if (data && data.rep) {
+                // console.log(data);
+                var tmp = "<div class='row'>";
+                for (var i in data.data.industry) {
+                    tmp = tmp +
+                        "<div class=\"col-lg-10\">\n" +
+                        "<div class=\"input-group\">\n" +
+                        "   <input type='text' class='form-control' name='industry'  placeholder=\"请填写行业类型\" value='" + data.data.industry[i] + "'>" +
+                        "   <span class=\"input-group-btn\">" +
+                        '       <button class="btn btn-info " type="button" data-toggle="tooltip" title="删除" id="delCenterIpGrp"><span class="glyphicon glyphicon-minus"></span></button>' +
+                        "   </span> " +
+                        "  </div>" +
+                        "</div>";
+                    // console.log(i, data.data.industry[i]);
+                }
+                tmp = tmp +
+                    "<div class=\"col-lg-10\">\n" +
+                    "<button class='btn btn-info col-md-3' type='button'  id='addCenterIpGrpBtn' onclick='addCenterIpGrp(this)' >新增行业类型</button> " +
+                    "<button type='button' class='btn btn-sm btn-default col-md-offset-5 col-md-2'  data-dismiss='modal' aria-label='Close'>取消</button>" +
+                    "<button type='button' class='btn btn-sm btn-danger col-md-2' onclick='updateIndustrys(\"" + _id + "\")'>保存</button>" +
+                    "</div>" +
+                    "</div>";
+                $("#infos").html(tmp);
+
+
+            } else {
+                alert("内部错误");
+            }
+        }, 'json');
+        //展示详细行业类型
+        $("#info_data").modal("show");
+    }
+
+    //添加行业详情
+    function addCenterIpGrp(obj) {
+        html = '<div class="input-group centerIp">' +
+            '<input type="text" name="industry" class="form-control" id="ipInput" placeholder="请填写行业类型">' +
+            '<span class="input-group-btn">' +
+            '<button class="btn btn-info" type="button" data-toggle="tooltip" title="删除" id="delCenterIpGrp"><span class="glyphicon glyphicon-minus"></span></button>' +
+            '</span>' +
+            '</div>'
+        obj.insertAdjacentHTML('beforebegin', html);
+    }
+
+    //更新行业详情
+    function updateIndustrys(_id) {
+        if (_id === "") {
+            alert("参数id不存在");
+            return
+        }
+        var params = $("#update-dataform").serialize();
+        // console.log(_id, params);
+        $.ajax({
+            type: "POST",
+            url: "/admin/audit/query_qyk/UpdateIndustrys",
+            data: params + "&_id=" + _id,
+            success: function (msg) {
+                $("#info_data").modal("hide");
+                if (msg.data) {
+                    alert("success");
+                } else {
+                    alert("更新失败");
+                }
+            }
+        });
+    }
+
+    $(document).on('click', '#delCenterIpGrp', function () {
+        var el = this.parentNode.parentNode;
+        if (confirm('您确定要删除选中的命令?')) {
+            el.parentNode.removeChild(el);
+        }
+    });
+
+    //详细联系方式
+    function showTels(_id) {
+        $("#infos").html("");
+        $("#info_title").html("详细联系方式");
+        $.post("/admin/audit/query_qyk/ById", {
+            "_id": _id,
+            "q_field": "contact",
+        }, function (data) {
+            if (data && data.rep) {
+                // console.log(data);
+                // var tmp = "<div class='row'>";
+                var tmp = "";
+                for (var i in data.data.contact) {
+                    tmp += "<div class='row'>" +
+                        "<div class=\"col-lg-10\">" +
+                        '<h1><div class="input-group centerIp">' +
+                        '<input type="text" name="contact_persons" class="form-control" id="ipInput" placeholder="请填写联系人" value=' + data.data.contact[i].contact_person + '>' +
+                        '<select class="form-control" name="contact_types" value=' + data.data.contact[i].contact_type + '><option value ="法定代表人">法定代表人</option><option value ="项目联系人">项目联系人</option></select>' +
+                        '<input type="text" name="phones" class="form-control" id="ipInput" placeholder="请填写联系电话" value=' + data.data.contact[i].phone + '>' +
+                        '<select class="form-control" name="topscopeclasss" value=' + data.data.contact[i].topscopeclass + '><option value ="企业公示">企业公示</option><option value ="剑鱼标讯">剑鱼标讯</option></select>' +
+                        '<span class="input-group-btn">' +
+                        '<button class="btn btn-info" type="button" data-toggle="tooltip" title="删除" id="delCenterIpGrp"><span class="glyphicon glyphicon-minus"></span></button>' +
+                        '</span>' +
+                        '</div><h1>' +
+                        '</div></div>';
+                    // console.log(i, data.data.contact[i]);
+                }
+                tmp = tmp +
+                    "<div class='row'>" +
+                    "<div class=\"col-lg-10\">" +
+                    "<button class='btn btn-info col-md-3' type='button'  id='addCenterIpGrpBtn' onclick='addCenterIpGrpTels(this)' >新增联系方式</button> " +
+                    "<button type='button' class='btn btn-sm btn-default col-md-offset-5 col-md-2'  data-dismiss='modal' aria-label='Close'>取消</button>" +
+                    "<button type='button' class='btn btn-sm btn-danger col-md-2' onclick='updateTels(\"" + _id + "\")'>保存</button>" +
+                    "</div>" +
+                    "</div>";
+                $("#infos").html(tmp);
+            } else {
+                alert("内部错误");
+            }
+        }, 'json');
+        $("#info_data").modal("show");
+    }
+
+    //添加联系方式
+    function addCenterIpGrpTels(obj) {
+        html = '<h1><div class="input-group centerIp">' +
+            '<input type="text" name="contact_persons" class="form-control" id="ipInput" placeholder="请填写联系人">' +
+            '<select class="form-control" name="contact_types"><option value ="法定代表人">法定代表人</option><option value ="项目联系人">项目联系人</option></select>' +
+            '<input type="text" name="phones" class="form-control" id="ipInput" placeholder="请填写联系电话">' +
+            '<select class="form-control" name="topscopeclasss"><option value ="企业公示">企业公示</option><option value ="剑鱼标讯">剑鱼标讯</option></select>' +
+            '<span class="input-group-btn">' +
+            '<button class="btn btn-info" type="button" data-toggle="tooltip" title="删除" id="delCenterIpGrp"><span class="glyphicon glyphicon-minus"></span></button>' +
+            '</span>' +
+            '</div><h1>'
+        obj.insertAdjacentHTML('beforebegin', html);
+    }
+
+    //更新联系方式
+    function updateTels(_id) {
+        if (_id === "") {
+            alert("参数id不存在");
+            return
+        }
+        var params = $("#update-dataform").serialize();
+        console.log(params)
+        $.ajax({
+            type: "POST",
+            url: "/admin/audit/query_qyk/UpdateTels",
+            data: params + "&_id=" + _id,
+            // contentType: "application/json; charset=utf-8",
+            dataType: "json",
+            traditional: true,
+            success: function (msg) {
+                $("#info_data").modal("hide");
+                if (msg.data) {
+                    alert("success");
+                } else {
+                    alert("更新失败");
+                }
+            }
+        });
+    }
+
+    //删除
+    function deleteQyk(_id) {
+        if (_id === "") {
+            alert("参数id为空");
+            return
+        }
+        if (confirm("您确认要提交此操作?")) {
+            $.post("/admin/audit/qiyeku_info/deleteQyk", {
+                "_id": _id,
+            }, function (data) {
+                if (data && data.rep) {
+                    setTimeout(function () {
+                        alert("删除成功");
+                        ttable.search("").draw();
+                        //    history.go(0) //刷新本页
+                    }, 800) //停1.5秒
+
+                } else {
+                    alert("内部错误");
+                }
+            }, 'json');
+            ttable.draw();
+        }
+    }
+
+    function editdata(_obj) {
+        $("#infos").html("");
+        $("#info_title").html("编辑企业库信息");
+        com = "<div class=\"row form-group\">" +
+            "<label for=\"lastname\" class=\"col-sm-2 control-label\" style=\"color: red;\">企业名称:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_company_name\" must=\"true\" value='" + _obj.company_name + "'></div>" +
+            "</div><div class=\"row form-group\">" +
+            "<label for=\"lastname\" class=\"col-sm-2 control-label\">别名:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_alias\" value='" + _obj.alias + "'></div>" +
+            "</div><div class=\"row form-group\"><label for=\"lastname\" class=\"col-sm-2 control-label\">省份:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_province\" value='" + _obj.province + "'></div>" +
+            "</div><div class=\"row form-group\"><label for=\"lastname\" class=\"col-sm-2 control-label\">城市:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_city\" value='" + _obj.city + "'></div>" +
+            "</div><div class=\"row form-group\"><label for=\"lastname\" class=\"col-sm-2 control-label\">区县:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_district\" value='" + _obj.district + "'></div>" +
+            "</div><div class=\"row form-group\"><label for=\"lastname\" class=\"col-sm-2 control-label\">注册资本:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_capital\" value='" + _obj.capital + "'  placeholder=\"示例:45678.98元;456.78万元;1234.25;\"></div>" +
+            "</div><div class=\"row form-group\"><label for=\"lastname\" class=\"col-sm-2 control-label\">企业地址:</label>" +
+            "<div class=\"col-sm-10\"><input type=\"text\" class=\"form-control\" id=\"s_company_address\" value='" + _obj.company_address + "'></div>" +
+            "</div></div>" +
+            "<div class=\"row form-group\">" +
+            "<button type='button' class='btn btn-sm btn-default col-md-offset-5 col-md-2'  data-dismiss='modal' aria-label='Close'>取消</button>" +
+            "<button type='button' class='btn btn-sm btn-danger col-md-2' onclick='updatedate(\"" + _obj._id + "\")'>更新</button></div>";
+        $("#infos").html(com);
+        $("#info_data").modal("show");
+    }
+
+    function updatedate(_id) {
+        if (_id === "") {
+            alert("id不存在!");
+            return
+        }
+        var company_name = $("#s_company_name").val();
+        var alias = $("#s_alias").val();
+        var province = $("#s_province").val();
+        var city = $("#s_city").val();
+        var district = $("#s_district").val();
+        var capital = $("#s_capital").val();
+        var company_address = $("#s_company_address").val();
+        if (company_name === "") {
+            alert("红色标签的表单不能为空!");
+            return
+        }
+        //新增企业库api
+        $.post("/admin/audit/qiyeku_info/save", {
+            "_id": _id,
+            "company_name": company_name,
+            "alias": alias,
+            "province": province,
+            "city": city,
+            "district": district,
+            "capital": capital,
+            "company_address": company_address,
+        }, function (data) {
+            if (data && data.rep) {
+                alert("更新成功");
+                $("#info_data").modal("hide");
+                setTimeout(function () {
+                    ttable.search(company_name).draw();
+                    //    history.go(0) //刷新本页
+                }, 800) //停1.5秒
+            } else {
+                alert("内部错误");
+            }
+        }, 'json')
+    }
+</script>

+ 1 - 1
src/web/templates/admin/clear.html

@@ -102,7 +102,7 @@ menuActive("version")
 var field = {{.field}};
 var _id = "";
 //var clearArr = ["cutspace","cutallspace","cutSymbol","cutNotPrs","clearAllWord","clearMaxAmount","clearProjectName","toint","tofloat","totimestamp","tomoney","getcurrency","getrate","getPhone","rateToFloat"]; 
-var clearMap = {"中文符号转英文":"chiToEng","去除首尾空格":"cutspace","去除所有空格":"cutallspace","清理符号":"cutSymbol","清理不成对符号后面的内容":"cutNotPrs","清理全部是汉字或者特殊符号的情况":"clearAllWord","过滤大于1万亿":"clearMaxAmount","清理项目名称":"clearProjectName","转int":"toint","转float":"tofloat","转时间戳":"totimestamp","转换金额":"tomoney","获取币种":"getcurrency","获取汇率":"getrate","取手机号":"getPhone","清理数字":"clearNumber","费率转小数":"rateToFloat","处理较长采购联系人":"clearBuyerPerson"}
+var clearMap = {"中文符号转英文":"chiToEng","去除首尾空格":"cutspace","去除所有空格":"cutallspace","清理符号":"cutSymbol","清理不成对符号后面的内容":"cutNotPrs","清理全部是汉字或者特殊符号的情况":"clearAllWord","过滤大于1万亿":"clearMaxAmount","清理项目名称":"clearProjectName","转int":"toint","转float":"tofloat","转时间戳":"totimestamp","转换金额":"tomoney","获取币种":"getcurrency","获取汇率":"getrate","取手机号":"getPhone","清理数字":"clearNumber","费率转小数":"rateToFloat","处理较长采购联系人":"clearBuyerPerson","去除尾部特殊符号":"clearEndSymblo"}
 $(function () {
 	ttableclear=$('#clearTable').DataTable({
 		"lengthChange": false,

+ 20 - 0
udpcreateindex/src/projectindex.go

@@ -35,11 +35,30 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 	var n int
 	i := 0
 	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
+		//不生索引字段
 		delete(tmp, "package")
 		delete(tmp, "winnerorder")
 		delete(tmp, "infofield")
 		delete(tmp, "budgettag")
 		delete(tmp, "bidamounttag")
+		list := tmp["list"].([]interface{})
+		for _, m := range list {
+			tmpM := m.(map[string]interface{})
+			if bidamount, ok := tmpM["bidamount"].(string); ok && len(bidamount) > 0 { //bidamount为string类型,转成float
+				tmpB := util.Float64All(tmpM["bidamount"])
+				tmpM["bidamount"] = tmpB
+			}
+			//projectscope截断
+			listProjectscopeRune := []rune(util.ObjToString(tmpM["projectscope"]))
+			if len(listProjectscopeRune) > 1000 {
+				tmpM["projectscope"] = string(listProjectscopeRune[:1000])
+			}
+		}
+		//projectscope截断
+		projectscopeRune := []rune(util.ObjToString(tmp["projectscope"]))
+		if len(projectscopeRune) > 1000 {
+			tmp["projectscope"] = string(projectscopeRune[:1000])
+		}
 		//		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
 		//			tmp["budget"] = nil
 		//		}
@@ -66,6 +85,7 @@ func projectTask(data []byte, mapInfo map[string]interface{}) {
 		}
 		tmp = make(map[string]interface{})
 	}
+
 	if i > 0 {
 		elastic.BulkSave(index, itype, &arr, true)
 	}

+ 5 - 1
udpfilterdup/src/datamap.go

@@ -71,7 +71,11 @@ func NewDatamap(days int, lastid string) *datamap {
 	//初始化加载数据
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
-	it := sess.DB(mgo.DbName).C(extract).Find(mongodb.ObjToMQ(`{"_id":{"$lte":"`+lastid+`"}}`, true)).Sort("-_id").Iter()
+	query := map[string]interface{}{"_id": map[string]interface{}{
+		"$lte": StringTOBsonId(lastid),
+	}}
+	log.Println("query", query)
+	it := sess.DB(mgo.DbName).C(extract).Find(query).Sort("-_id").Iter()
 	now1 := int64(0)
 	n, continuSum := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {

+ 1 - 1
udpfilterdup/src/mgo.go

@@ -144,7 +144,7 @@ func (m *MongodbSim) InitPool() {
 	opts := options.Client()
 	opts.SetConnectTimeout(3 * time.Second)
 	opts.ApplyURI("mongodb://" + m.MongodbAddr)
-	opts.SetMaxPoolSize(uint16(m.Size))
+	opts.SetMaxPoolSize(uint64(m.Size))
 	m.pool = make(chan bool, m.Size)
 	opts.SetMaxConnIdleTime(2 * time.Hour)
 	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)