Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/init.go

@@ -294,14 +294,16 @@ type ProjectInfo struct {
 	Buyerclass  string                 `json:"buyerclass"`            //采购单位分类
 	Bidopentime int64                  `json:"bidopentime,omitempty"` //开标时间
 	//	Zbtime        int64                  `json:"zbtime"`        //招标时间
-	//	Jgtime        int64                  `json:"jgtime"`        //结果中标时间
+	Jgtime        int64              `json:"jgtime"`        //结果中标时间
 	Bidamount float64 `json:"bidamount,omitempty"` //中标金额
 	Budget    float64 `json:"budget,omitempty"`    //预算
 	//Winnerorder []string `json:"winnerorder"` //中标候选人
 	score         int
 	comStr        string
 	resVal, pjVal int
-	InfoFiled	map[string]InfoField	`json:"infofiled"`		//逻辑处理需要的info字段
+	InfoFiled	map[string]InfoField	`json:"infofiled"`			//逻辑处理需要的info字段
+	Budgettag		int					`json:"budgettag"`			//预算是否有效标记
+	Bidamounttag	int					`json:"bidamounttag"`		//中标金额是否有效标记
 }
 
 //存储部分招标信息字段，业务逻辑处理需要

fullproject/src_v1/project.go

@@ -3,13 +3,13 @@ package main
 import (
 	"encoding/json"
 	"log"
+	"time"
+
 	//	"log"
 	"math"
 	qu "qfw/util"
 	"sort"
 	"strings"
-	"time"
-
 	//"gopkg.in/mgo.v2/bson"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -456,37 +456,11 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			set[f] = tmp[f]
 		}
 	}
-	if tmp["budget"] != nil {
-		set["budget"] = thisinfo.Budget
-	}
-	if tmp["bidamount"] != nil {
-		set["bidamount"] = thisinfo.Bidamount
-	}
 	bidopentime := qu.Int64All(tmp["bidopentime"])
 	if bidopentime > 0 {
 		set["bidopentime"] = bidopentime
 	}
-	if thisinfo.ProjectName != "" {
-		set["s_projectname"] = tmp["projectname"]
-	}
-	now := time.Now().Unix()
-	set["createtime"] = now
-	set["sourceinfoid"] = thisinfo.Id
-	set["sourceinfourl"] = tmp["href"]
-	set["firsttime"] = tmp["publishtime"]
-	set["lasttime"] = tmp["publishtime"]
-	set["pici"] = p.pici
-	set["ids"] = []string{thisinfo.Id}
-	if thisinfo.TopType == "招标" {
-		set["zbtime"] = tmp["publishtime"]
-	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
-		set["jgtime"] = tmp["publishtime"]
-	}
-	//废标、流标   处理时间
-	if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-		set["zbtime"] = int64(0)
-		set["bidopentime"] = int64(0)
-	}
+
 	//异常标记
 	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
 		set["exception"] = 1
@@ -508,12 +482,14 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	set["bidtype"] = bidtype[bs]
 	if bt == "招标" {
 		set["projectscope"] = qu.ObjToString(tmp["projectscope"])
-		set["bidstatus"] = bs
+		set["bidstatus"] = bt
 	}else {
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
 		} else if tmp["infoformat"] == 2 {
 			set["bidstatus"] = "拟建"
+		}else if bs == "" {
+			set["bidstatus"] =  ""
 		} else {
 			set["bidstatus"] = "其它"
 		}
@@ -522,10 +498,45 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 
 	pkg := PackageFormat(thisinfo, nil)
 	p1 := p.NewCachePinfo(pId, thisinfo, bt, pkg)
+
+	now := time.Now().Unix()
+	set["createtime"] = now
+	set["sourceinfoid"] = thisinfo.Id
+	set["sourceinfourl"] = tmp["href"]
+	set["firsttime"] = tmp["publishtime"]
+	set["lasttime"] = tmp["publishtime"]
+	set["pici"] = p.pici
+	set["ids"] = []string{thisinfo.Id}
+	if thisinfo.TopType == "招标" {
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+			set["zbtime"] = tmp["publishtime"]
+		}
+	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		set["jgtime"] = tmp["publishtime"]
+		p1.Jgtime = thisinfo.Publishtime
+	}
+
 	if len(thisinfo.Subscopeclass) > 0 {
-		s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
-		set["s_subscopeclass"] = s_subscopeclass
+		set["s_subscopeclass"] = strings.Join(thisinfo.Subscopeclass, ",")
 	}
+
+	if tmp["budget"] != nil && tmp["budget"] != "" {
+		set["budget"] = thisinfo.Budget
+		p1.Budgettag = 0
+		set["budgettag"] = 0
+	}else {
+		p1.Budgettag = 1
+		set["budgettag"] = 1
+	}
+	if tmp["bidamount"] != nil && tmp["bidamount"] != "" {
+		set["bidamount"] = thisinfo.Bidamount
+		p1.Bidamounttag = 0
+		set["bidamounttag"] = 0
+	}else {
+		p1.Bidamounttag = 1
+		set["bidamounttag"] = 1
+	}
+
 	if len(thisinfo.Winners) > 0 {
 		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
 		p1.Winners = thisinfo.Winners
@@ -537,6 +548,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["multipackage"] = 0
 	}
 
+	if thisinfo.Buyer == "" {
+		set["buyerclass"] = ""
+	}
+
 	p1.InfoFiled = make(map[string]InfoField)
 	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
@@ -596,6 +611,7 @@ var INFOFIELDS = []string{
 	"buyertel",
 	"area",
 	"city",
+	"district",
 	"spidercode",
 	"site",
 }
@@ -658,22 +674,35 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	set := map[string]interface{}{}
 	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
-
-	//1--firsttime
-	if thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0 {
-		pInfo.FirstTime = thisinfo.Publishtime
-		set["firsttime"] = thisinfo.Publishtime
-		if thisinfo.TopType == "招标" {
-			set["zbtime"] = tmp["publishtime"]
-		}
+	if len(pInfo.Ids) > 30 {
+		//异常标记
+		set["listtag"] = 1
 	}
-	//2--lasttime
+
+	//zbtime、lasttime、jgtime
 	pInfo.LastTime = thisinfo.Publishtime
 	set["lasttime"] = thisinfo.Publishtime
 	if thisinfo.TopType == "招标" {
-		set["zbtime"] = tmp["publishtime"]
-	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
-		set["jgtime"] = tmp["publishtime"]
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+			set["zbtime"] = tmp["publishtime"]
+		}
+		if pInfo.Jgtime > 0 {
+			pInfo.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	}else if thisinfo.TopType == "结果" {
+		pInfo.Jgtime = thisinfo.Publishtime
+		set["jgtime"] = thisinfo.Publishtime
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Jgtime <= 0 {
+			set["jgtime"] = tmp["publishtime"]
+			pInfo.Jgtime = thisinfo.Publishtime
+		}
+	}
+
+	if thisinfo.Bidopentime > pInfo.Bidopentime {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = pInfo.Bidopentime
 	}
 
 	bt := qu.ObjToString(tmp["toptype"])
@@ -687,30 +716,25 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		if tmp["projectscope"] != nil {
 			set["projectscope"] = qu.ObjToString(tmp["projectscope"])
 		}
-		set["bidstatus"] = bs
+		set["bidstatus"] = bt
+		pInfo.Bidstatus = bt
 	}else {
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
+			pInfo.Bidstatus = thisinfo.SubType
 		} else if tmp["infoformat"] == 2 {
 			set["bidstatus"] = "拟建"
-		} else {
+			pInfo.Bidstatus = "拟建"
+		}else if bs == "" {
+			set["bidstatus"] =  ""
+			pInfo.Bidstatus = ""
+		}else {
 			set["bidstatus"] = "其它"
+			pInfo.Bidstatus =  "其它"
 		}
 	}
 	p.mapBidLock.Unlock()
 
-	//废标、流标   处理时间
-	if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-		pInfo.FirstTime = thisinfo.Publishtime
-		pInfo.Bidopentime = int64(0)
-		pInfo.LastTime = thisinfo.Publishtime
-
-		set["firsttime"] = thisinfo.Publishtime
-		set["zbtime"] = int64(0)
-		set["publishtime"] = thisinfo.Publishtime
-		set["bidopentime"] = int64(0)
-	}
-
 	//异常标记
 	if ex > 0 {
 		set["exception"] = ex
@@ -775,11 +799,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		set["buyertel"] = pInfo.Buyertel
 	}
 
-	if thisinfo.Bidopentime > pInfo.Bidopentime {
-		pInfo.Bidopentime = thisinfo.Bidopentime
-		set["bidopentime"] = pInfo.Bidopentime
-	}
-
 	if len(thisinfo.Topscopeclass) > 0 {
 		sort.Strings(pInfo.Topscopeclass)
 		for _, k := range thisinfo.Topscopeclass {
@@ -805,7 +824,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	//winner
 	if len(thisinfo.Winners) > 0 {
 		if len(pInfo.Winners) <= 0 {
-			set["winner"] = tmp["winner"].(string)
+			set["winner"] = qu.ObjToString(tmp["winner"])
 		}
 
 		sort.Strings(pInfo.Winners)
@@ -826,9 +845,20 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		set["multipackage"] = 0
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(pInfo, thisinfo)
-	set["budget"] = pInfo.Budget
-	set["bidamount"] = pInfo.Bidamount
+	CountAmount(pInfo, thisinfo, tmp)
+	if pInfo.Budget >= 0 && pInfo.Budgettag != 1 {
+		set["budget"] = pInfo.Budget
+		set["budgettag"] = 0
+	}else {
+		set["budgettag"] = 1
+	}
+	if pInfo.Bidamount >= 0 && pInfo.Bidamounttag != 1 {
+		set["bidamount"] = pInfo.Bidamount
+		set["bidamounttag"] = 0
+	}else {
+		set["bidamounttag"] = 1
+	}
+
 	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
 		Bidamount:    thisinfo.Bidamount,
@@ -887,7 +917,9 @@ func (p *ProjectTask) CompareStatus(project *ProjectInfo, info *Info) (bool, int
 	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
 		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
 			return false, 0
-		} else {
+		}else if project.Bidstatus == "废标" || project.Bidstatus == "流标" {
+			return false, 0
+		}else {
 			return true, 0
 		}
 	} else if info.TopType == "结果" {
@@ -926,10 +958,10 @@ func ComparePlace(project *ProjectInfo, info *Info) bool {
 		return false
 	}
 	if info.Area == project.Area {
-		if info.City == "" {
+		if info.City == "" || project.City == "" {
 			return false
 		} else if info.City == project.City {
-			if info.District == "" || info.District == project.District {
+			if project.District == "" || info.District == "" || info.District == project.District {
 				return false
 			} else {
 				return true
@@ -1013,28 +1045,34 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 }
 
 //计算预算(budget)、中标金额(bidamount)
-func CountAmount(project *ProjectInfo, info *Info) {
+func CountAmount(project *ProjectInfo, info *Info, tmp map[string]interface{}) {
 	if info.HasPackage {
 		budget := 0.0
 		for _, v := range project.Package{
 			v1, _ := v.([]map[string]interface{})
 			for _, v2 := range v1{
-				b1 := qu.Float64All(v2["budget"])
-				if b1 > 0 {
-					budget = budget + b1
-					break
+				if v2["budget"] != nil {
+					b1 := qu.Float64All(v2["budget"])
+					if b1 > 0 {
+						budget = budget + b1
+						break
+					}
+				}else {
+					project.Budgettag = 1
 				}
+
 			}
 		}
-		if budget == 0 && info.Budget > 0 {
-			budget = info.Budget
-		}
 		if budget > 0 {
 			project.Budget = budget
+			project.Budgettag = 0
+		}else if budget == 0 && info.Budget > 0 {
+			project.Budget = info.Budget
+			project.Budgettag = 0
 		}
 	}else {
 		//招标没有多包
-		k := KeyPackage.FindStringSubmatch(project.ProjectName)
+		k := KeyPackage.FindStringSubmatch(info.ProjectName)
 		if len(k) > 0 {
 			//招标是单包
 			if len(project.Package) > 0 {
@@ -1048,32 +1086,26 @@ func CountAmount(project *ProjectInfo, info *Info) {
 				}
 				if !flag {
 					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
 				}
 			}else {
 				//项目没有多包
 				if info.Budget > 0 {
 					project.Budget = project.Budget + info.Budget
+					project.Budgettag = 0
+				}else if info.Budget == 0 && tmp["budget"] != nil {
+					project.Budgettag = 0
+				}else {
+					project.Budgettag = 1
 				}
 			}
 		}else {
-			//招标不是单包
-			//flag := false
-			//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-			//	for _, res := range project.InfoFiled {
-			//		if res.ProjectName == info.ProjectName {
-			//			if res.Budget < info.Budget {
-			//				project.Budget = project.Budget - res.Budget + info.Budget
-			//			}
-			//			flag = true
-			//			break
-			//		}
-			//	}
-			//	if !flag {
-			//		project.Budget = project.Budget + info.Budget
-			//	}
-			//}
 			if project.Budget < info.Budget {
 				project.Budget = info.Budget
+				project.Budgettag = 0
+			}
+			if tmp["budget"] == nil {
+				project.Budgettag = 1
 			}
 		}
 	}
@@ -1083,17 +1115,27 @@ func CountAmount(project *ProjectInfo, info *Info) {
 			for _, v := range project.Package{
 				v1, _ := v.([]map[string]interface{})
 				for _, v2 := range v1{
-					b1 := qu.Float64All(v2["bidamount"])
-					if b1 > 0 {
-						bidamount = bidamount + b1
-						break
+					if tmp["bidamount"] != nil {
+						b1 := qu.Float64All(v2["bidamount"])
+						if b1 > 0 {
+							bidamount = bidamount + b1
+							break
+						}
+					}else {
+						project.Bidamount = 1
 					}
 				}
 			}
-			project.Bidamount = bidamount
+			if bidamount > 0 {
+				project.Bidamount = bidamount
+				project.Bidamounttag = 0
+			}else if bidamount == 0 && info.Budget > 0 {
+				project.Bidamount = info.Bidamount
+				project.Bidamounttag = 0
+			}
 		}else {
 			//招标没有多包
-			k := KeyPackage.FindStringSubmatch(project.ProjectName)
+			k := KeyPackage.FindStringSubmatch(info.ProjectName)
 			if len(k) > 0 {
 				//招标是单包
 				if len(project.Package) > 0 {
@@ -1101,49 +1143,30 @@ func CountAmount(project *ProjectInfo, info *Info) {
 					flag := false
 					for _, v := range project.Package{
 						v1, _ := v.([]map[string]interface{})
-						if len(v1) > 0 {
+						if len(v1) > 0 && v1[0]["name"] == info.ProjectName {
 							flag = true
 						}
 					}
 					if !flag {
 						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
 					}
 				}else {
 					//项目没有多包
 					if info.Bidamount > 0 {
 						project.Bidamount = project.Bidamount + info.Bidamount
+						project.Bidamounttag = 0
+					}else if info.Bidamount == 0 && tmp["bidamount"] != nil {
+						project.Bidamounttag = 0
+					}else {
+						project.Bidamounttag = 1
 					}
 				}
 			}else {
-				//招标不是单包
-				//flag := false
-				//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-				//	for _, res := range project.InfoFiled {
-				//		if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
-				//			if res.ContractCode == info.ContractCode {
-				//				if res.Bidamount < info.Bidamount {
-				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
-				//				}
-				//				flag = true
-				//				break
-				//			}
-				//		}else {
-				//			if res.ProjectName == info.ProjectName {
-				//				if res.Bidamount < info.Bidamount {
-				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
-				//				}
-				//				flag = true
-				//				break
-				//			}
-				//		}
-				//	}
-				//	if !flag {
-				//		project.Bidamount = project.Bidamount + info.Bidamount
-				//	}
-				//}
 				if info.SubType == "中标" || info.SubType == "成交" {
 					if project.Bidamount < info.Bidamount {
 						project.Bidamount = info.Bidamount
+						project.Bidamounttag = 0
 					}else {
 						flag := false
 						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
@@ -1159,17 +1182,25 @@ func CountAmount(project *ProjectInfo, info *Info) {
 							}
 							if !flag {
 								project.Bidamount = project.Bidamount + info.Bidamount
+								project.Bidamounttag = 0
 							}else {
 								if project.Bidamount < info.Bidamount {
 									project.Bidamount = info.Bidamount
+									project.Bidamounttag = 0
 								}
 							}
 						}
 					}
 				}
+				if tmp["bidamount"] == nil {
+					project.Budgettag = 1
+				}
 			}
 		}
+	}else {
+		project.Bidamounttag = 1
 	}
+
 }
 
 //结构体转map

fullproject/src_v1/task.go

@@ -718,6 +718,8 @@ func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
 					tmp["city"] = site.City
 					tmp["district"] = site.District
 					return
+				}else if site.City == "" {
+					return
 				}
 			}
 		} else {

fullproject/src_v1/update.go

@@ -63,7 +63,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 			//跟原项目list第0条信息作对比，可以合并
 			if pId == pInfoId {
 				//合并到当前项目
-				mergeProject(p, currentPro.P, info, tmpPro, index)
+				mergeProject(p, currentPro.P, info, tmpPro, tmp)
 				p.AllIdsMapLock.Lock()
 				p.AllIdsMap[currentPro.P.Id.Hex()].P = currentPro.P
 				p.AllIdsMapLock.Unlock()
@@ -263,13 +263,13 @@ func (p *ProjectTask) updateMerge(index int, info *Info, pInfoId string, tmp map
 //内部合并
 func (p *ProjectTask) innerMerge(pInfo *ProjectInfo, info *Info, tmp map[string]interface{}, tmpPro map[string]interface{}) {
 	infoList := []interface{}(tmpPro["list"].(primitive.A))
-	for index, info1 := range infoList{
+	for _, info1 := range infoList{
 		info2 := info1.(map[string]interface{})
 		if info2["infoid"] == info.Id {
 			deleteSlice1(infoList, info1)
 			continue
 		}
-		mergeProject(p, pInfo, info, tmpPro, index)
+		mergeProject(p, pInfo, info, tmpPro, tmp)
 	}
 	mapTmp, _ := tmpPro["infofiled"].(map[string]interface{})
 	delete(mapTmp, info.Id)
@@ -280,46 +280,43 @@ func (p *ProjectTask) innerMerge(pInfo *ProjectInfo, info *Info, tmp map[string]
 }
 
 //合并字段到project
-func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[string]interface{}, index int) map[string]interface{} {
-	//1--firsttime
-	if thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0 {
-		pInfo.FirstTime = thisinfo.Publishtime
-		set["firsttime"] = thisinfo.Publishtime
-		if thisinfo.TopType == "招标" {
-			set["zbtime"] = thisinfo.Publishtime
-		}
-	}
-	//2--lasttime
+func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[string]interface{}, tmp map[string]interface{}) map[string]interface{} {
 	pInfo.LastTime = thisinfo.Publishtime
 	set["lasttime"] = thisinfo.Publishtime
 	if thisinfo.TopType == "招标" {
-		set["zbtime"] = thisinfo.Publishtime
-	} else if thisinfo.TopType == "结果" || thisinfo.SubType == "合同" {
+		if thisinfo.SubType != "变更" && thisinfo.SubType != "其它" {
+			set["zbtime"] = thisinfo.Publishtime
+		}
+		if pInfo.Jgtime > 0 {
+			pInfo.Jgtime = int64(0)
+			set["jgtime"] = int64(0)
+		}
+	}else if thisinfo.TopType == "结果" {
+		pInfo.Jgtime = thisinfo.Publishtime
 		set["jgtime"] = thisinfo.Publishtime
+	} else if thisinfo.SubType == "合同" {
+		if pInfo.Jgtime <= 0 {
+			set["jgtime"] = thisinfo.Publishtime
+		}
 	}
-	//废标、流标   处理时间
-	if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
-		pInfo.FirstTime = thisinfo.Publishtime
-		pInfo.Bidopentime = int64(0)
-		pInfo.LastTime = thisinfo.Publishtime
 
-		set["firsttime"] = thisinfo.Publishtime
-		set["zbtime"] = int64(0)
-		set["publishtime"] = thisinfo.Publishtime
-		set["bidopentime"] = int64(0)
+	if thisinfo.Bidopentime > pInfo.Bidopentime {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = pInfo.Bidopentime
 	}
 
+
 	bt := thisinfo.TopType
 	bs := thisinfo.SubType
 	p.mapBidLock.Lock()
+	set["bidtype"] = bidtype[bs]
 	if bt == "招标" {
 		//招标状态，更新projectscope
 		if thisinfo.ProjectScope != "" {
 			set["projectscope"] = thisinfo.ProjectScope
 		}
-		set["bidtype"] = bidtype[bs]
+		set["bidstatus"] = bt
 	}else {
-		set["bidtype"] = bt
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
 		} else if thisinfo.Infoformat == 2 {
@@ -328,7 +325,6 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 			set["bidstatus"] = "其它"
 		}
 	}
-
 	p.mapBidLock.Unlock()
 
 	//3\4\5--省、市、县
@@ -373,6 +369,10 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 		set["buyerclass"] = ""
 	}
 
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = pInfo.ContractCode + ","+thisinfo.ContractCode
+	}
+
 	//8--代理机构
 	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
 		pInfo.Agency = thisinfo.Agency
@@ -437,7 +437,6 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 				sort.Strings(pInfo.Winners)
 			}
 		}
-		//set["winners"] = pInfo.Winners
 		set["s_winner"] = strings.Join(pInfo.Winners, ",")
 	}
 
@@ -449,7 +448,7 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 		set["multipackage"] = 0
 	}
 	//处理多包后，计算预算金额、中标金额
-	CountAmount(pInfo, thisinfo)
+	CountAmount(pInfo, thisinfo, tmp)
 	set["budget"] = pInfo.Budget
 	set["bidamount"] = pInfo.Bidamount
 

src/jy/extract/extpackage.go

@@ -129,13 +129,19 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 				if pkg != nil {
 					sonJobResult["origin"] = pkg.Origin
 					sonJobResult["text"] = pkg.Text
-					sonJobResult["budget"] = pkg.Budget
-					sonJobResult["bidamount"] = pkg.Bidamount
+					if pkg.IsTrueBudget{
+						sonJobResult["budget"] = pkg.Budget
+					}
+					if pkg.IsTrueBidamount{
+						sonJobResult["bidamount"] = pkg.Bidamount
+					}
 					if pkg.Winner == "" && len(j.Winnerorder) > 0 {
 						if sonJobResult["winnerorder"] == nil {
 							sonJobResult["winnerorder"] = j.Winnerorder
-							if sonJobResult["bidamount"].(float64) <= 0 {
-								sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+							if  sonJobResult["bidamount"] == nil ||sonJobResult["bidamount"].(float64) <= 0 {
+								if j.Winnerorder[0]["price"] != nil{
+									sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+								}
 							}
 							if sonJobResult["winner"] == "" {
 								sonJobResult["winner"] = j.Winnerorder[0]["entname"]
@@ -143,7 +149,9 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 						}
 					} else {
 						if len(j.Winnerorder) > 0 {
-							sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+							if j.Winnerorder[0]["price"] !=nil{
+								sonJobResult["bidamount"] = qu.Float64All(j.Winnerorder[0]["price"])
+							}
 							sonJobResult["winner"] = j.Winnerorder[0]["entname"]
 						}
 						if len(pkg.WinnerOrder) > 0 {
@@ -158,7 +166,7 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 
 					sonJobResult["type"] = pkg.Type
 					if len(tmpkeys) == 1 {
-						if qu.Float64All(sonJobResult["budget"]) == 0 {
+						if qu.Float64All(sonJobResult["budget"]) == 0 && pkg.IsTrueBudget {
 							for _, bv := range j.Block {
 								kvparse(bv.ColonKV, e, &sonJobResult, isSite, codeSite)
 								kvparse(bv.TableKV, e, &sonJobResult, isSite, codeSite)
@@ -271,8 +279,10 @@ func PackageDetail(j *ju.Job, e *ExtractTask, isSite bool, codeSite string) {
 					if qu.ObjToString(sonJobResult["winner"]) == "" || (!pkg.Accuracy && qu.ObjToString(firstWinnerOrder["entname"]) != "" && qu.Int64All(firstWinnerOrder["sort"]) == 1) {
 						sonJobResult["winner"] = firstWinnerOrder["entname"]
 					}
-					if qu.Float64All(sonJobResult["bidamount"]) == 0 || (!pkg.Accuracy && qu.Float64All(firstWinnerOrder["price"]) > 0 && qu.Int64All(firstWinnerOrder["sort"]) == 1) {
-						sonJobResult["bidamount"] = firstWinnerOrder["price"]
+					if (qu.Float64All(sonJobResult["bidamount"]) == 0 && pkg.IsTrueBidamount)|| (!pkg.Accuracy && qu.Float64All(firstWinnerOrder["price"]) > 0 && qu.Int64All(firstWinnerOrder["sort"]) == 1) {
+						if firstWinnerOrder["price"] != nil{
+							sonJobResult["bidamount"] = firstWinnerOrder["price"]
+						}
 					}
 				}
 				//log.Println(pkName, sonJobResult)

src/jy/extract/extract.go

@@ -27,12 +27,12 @@ import (
 var (
 	lock, lockrule, lockclear, locktag, blocktag sync.RWMutex
 
-	cut           = ju.NewCut()                          //获取正文并清理
-	ExtLogs       map[*TaskInfo][]map[string]interface{} //抽取日志
-	TaskList      map[string]*ExtractTask                //任务列表
-	ClearTaskList map[string]*ClearTask                  //清理任务列表
-	saveLimit     = 100                                  //抽取日志批量保存
-	PageSize      = 5000                                 //查询分页
+	cut     = ju.NewCut()                          //获取正文并清理
+	ExtLogs map[*TaskInfo][]map[string]interface{} //抽取日志
+	TaskList      map[string]*ExtractTask          //任务列表
+	ClearTaskList map[string]*ClearTask            //清理任务列表
+	saveLimit     = 100                            //抽取日志批量保存
+	PageSize      = 5000                           //查询分页
 	Fields        = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"bidstatus":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1,"href":1}`
 	Fields2       = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
 )
@@ -319,6 +319,9 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 		BuyerAddr: qu.ObjToString(doc["buyeraddr"]),
 		RuleBlock: e.RuleBlock,
 	}
+	if (j.Jsondata != nil || (*j.Jsondata) != nil) && (*j.Jsondata)["jsoncontent"] != nil {
+		delete((*j.Jsondata), "jsoncontent")
+	}
 	if isextFile {
 		jf = &ju.Job{
 			SourceMid:  qu.BsonIdToSId(doc["_id"]),
@@ -336,6 +339,9 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 			RuleBlock:  e.RuleBlock,
 			IsFile:     isextFile,
 		}
+		if (jf.Jsondata != nil || (*jf.Jsondata) != nil) && (*jf.Jsondata)["jsoncontent"] != nil {
+			delete((*jf.Jsondata), "jsoncontent")
+		}
 	}
 	codeSite := j.SpiderCode
 	//是否启用站点
@@ -968,14 +974,20 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 								cfn := e.ClearFn[in.Field]
 								lock.Unlock()
 								data := clear.DoClearFn(cfn, []interface{}{strings.TrimSpace(rep[in.Field+"_"+fmt.Sprint(i)]), j.Content})
-								j.BlockPackage[k].Budget = qu.Float64All(data[0])
+								if data[len(data)-1].(bool) {
+									j.BlockPackage[k].Budget = qu.Float64All(data[0])
+									j.BlockPackage[k].IsTrueBudget = true
+								}
 								break
 							} else if in.Field == "bidamount" && vbpkg.Bidamount <= 0 {
 								lock.Lock()
 								cfn := e.ClearFn[in.Field]
 								lock.Unlock()
 								data := clear.DoClearFn(cfn, []interface{}{strings.TrimSpace(rep[in.Field+"_"+fmt.Sprint(i)]), j.Content})
-								j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+								if data[len(data)-1].(bool) {
+									j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+									j.BlockPackage[k].IsTrueBidamount = true
+								}
 								break
 							} else if in.Field == "winner" {
 								if j.BlockPackage[k].Winner == "" {
@@ -1014,7 +1026,10 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 						cfn := e.ClearFn[in.Field]
 						lock.Unlock()
 						data := clear.DoClearFn(cfn, []interface{}{val, j.Content})
-						j.BlockPackage[k].Budget = qu.Float64All(data[0])
+						if data[len(data)-1].(bool) {
+							j.BlockPackage[k].Budget = qu.Float64All(data[0])
+							j.BlockPackage[k].IsTrueBudget = true
+						}
 						break
 					}
 					if in.Field == "bidamount" && vbpkg.Bidamount <= 0 {
@@ -1022,7 +1037,10 @@ func ExtRuleCoreByPkgReg(j *ju.Job, in *RegLuaInfo, e *ExtractTask) {
 						cfn := e.ClearFn[in.Field]
 						lock.Unlock()
 						data := clear.DoClearFn(cfn, []interface{}{val, j.Content})
-						j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+						if data[len(data)-1].(bool) {
+							j.BlockPackage[k].Bidamount = qu.Float64All(data[0])
+							j.BlockPackage[k].IsTrueBidamount = true
+						}
 						break
 					} else if in.Field == "bidstatus" {
 						if j.BlockPackage[k].BidStatus == "" {
@@ -1053,7 +1071,7 @@ func getKvByLuaFields(vc *RuleCore, j *ju.Job, et *ExtractTask) (map[string][]ma
 				kvmap[vc.Field] = append(kvmap[vc.Field], map[string]interface{}{
 					"code":        "winnerorder",
 					"field":       vc.Field,
-					"ruletext":    "中标候选人_"+ v["sortstr"].(string),
+					"ruletext":    "中标候选人_" + v["sortstr"].(string),
 					"extfrom":     v["sortstr"],
 					"sourcevalue": v["price"],
 					"value":       v["price"],
@@ -1579,6 +1597,8 @@ type FieldValue struct {
 	Count int
 }
 
+var clearWinnerReg = regexp.MustCompile("名称|施工|拟定供应商名称|：|:")
+
 //分析抽取结果并保存
 func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 	qu.Try(func() {
@@ -1611,20 +1631,27 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 				}
 			}
 		}
+		if len(j.PackageInfo) > 10 {
+			for k, v := range j.PackageInfo {
+				j.PackageInfo = map[string]map[string]interface{}{}
+				j.PackageInfo[k] = v
+				break
+			}
+		}
 		if len(j.PackageInfo) > 0 { //分包信息
 			tmp["package"] = j.PackageInfo
 			//包预算，中标金额合并大于抽取就覆盖
 			var tmpBidamount, tmpBudget float64
 			//s_winner逗号分隔拼接，分包中标人
-			var tmpstr,savewinner []string
-
+			var tmpstr, savewinner []string
+			//按包排序
 			for b, v := range j.PackageInfo {
-				if v["winner"]!= nil && v["winner"]!=""{
-					tmpstr = append(tmpstr,b)
+				if v["winner"] != nil && v["winner"] != "" {
+					tmpstr = append(tmpstr, b)
 				}
 			}
 			//包预算，中标金额合并大于抽取就覆盖
-			if len(j.PackageInfo) >1{
+			if len(j.PackageInfo) > 1 {
 				//包数大于1累加
 				for _, v := range j.PackageInfo {
 					if v["budget"] != nil {
@@ -1640,10 +1667,10 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 				if qu.Float64All(tmp["bidamount"]) < tmpBidamount {
 					tmp["bidamount"] = tmpBidamount
 				}
-			}else {
+			} else {
 				//包数等于1,tmp没有值取包里的值
 				if tmp["budget"] == nil || tmp["budget"] == 0 {
-					for _,v := range j.PackageInfo {
+					for _, v := range j.PackageInfo {
 						if v["budget"] != nil {
 							tmp["budget"] = v["budget"]
 						}
@@ -1651,7 +1678,7 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 
 				}
 				if tmp["bidamount"] == nil || tmp["bidamount"] == 0 {
-					for _,v := range j.PackageInfo {
+					for _, v := range j.PackageInfo {
 						if v["bidamount"] != nil {
 							tmp["bidamount"] = v["bidamount"]
 						}
@@ -1660,12 +1687,21 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 			}
 			//s_winner逗号分隔拼接，分包中标人
 			sort.Strings(tmpstr)
-			for _,v := range tmpstr{
-				savewinner = append(savewinner,qu.ObjToString(j.PackageInfo[v]["winner"]))
+			for _, v := range tmpstr {
+				svvvv := qu.ObjToString(j.PackageInfo[v]["winner"])
+				savevvv := clearWinnerReg.ReplaceAllString(svvvv, "")
+				if savevvv == "" {
+					continue
+				}
+				savewinner = append(savewinner, savevvv)
+			}
+			if (savewinner == nil || len(savewinner) == 0) && tmp["winner"] != nil {
+				tmp["s_winner"] = tmp["winner"]
+			} else if savewinner != nil {
+				tmp["s_winner"] = strings.Join(savewinner, ",")
 			}
-			tmp["s_winner"] = strings.Join(savewinner,",")
 
-		}else if tmp["winner"]!= nil && tmp["winner"]!=""{
+		} else if tmp["winner"] != nil && tmp["winner"] != "" {
 			//没有分包取winner
 			tmp["s_winner"] = tmp["winner"]
 		}
@@ -1825,8 +1861,12 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 		} else { //测试结果
 			delete(tmp, "_id")
 			if len(j.BlockPackage) > 0 { //分包详情
-				bs, _ := json.Marshal(j.BlockPackage)
-				tmp["epackage"] = string(bs)
+				if len(j.BlockPackage) >10{
+					tmp["epackage"] = "分包异常"
+				}else {
+					bs, _ := json.Marshal(j.BlockPackage)
+					tmp["epackage"] = string(bs)
+				}
 			}
 			tmp["result"] = result
 			tmp["resultf"] = resultf
@@ -2018,7 +2058,7 @@ func (e *ExtractTask) QualityAudit(resulttmp map[string]interface{}) {
 func (e *ExtractTask) RedisMatch(field, fv string, val map[string]interface{}) {
 	defer qu.Catch()
 	i := redis.GetInt(field, field+"_"+fv) //查找redis
-	if i == 0 {                            //reids未找到，执行规则匹配
+	if i == 0 { //reids未找到，执行规则匹配
 		val[field+"_isredis"] = false
 		e.RuleMatch(field, fv, val) //规则匹配
 	} else { //redis找到，打标识存库
@@ -2110,20 +2150,20 @@ func resetWinnerorder(j *ju.Job) {
 	if maxlen > 0 {
 		winners = append(winners, &ju.ExtField{Code: "winnerorder", Field: "winner", ExtFrom: "j.Winnerorder", Value: j.Winnerorder[0]["entname"], Score: 0.5})
 		if j.Winnerorder[0]["price"] != nil {
-			tmpPrice := clear.ObjToMoney([]interface{}{j.Winnerorder[0]["price"],""})
-			if tmpPrice[len(tmpPrice)-1].(bool){
-				bidamounts = append(bidamounts, &ju.ExtField{Code: "winnerorder", Field: "bidamount", ExtFrom: "j.Winnerorder",SourceValue:j.Winnerorder[0]["price"], Value: tmpPrice[0], Score: 0.5})
+			tmpPrice := clear.ObjToMoney([]interface{}{j.Winnerorder[0]["price"], ""})
+			if tmpPrice[len(tmpPrice)-1].(bool) {
+				bidamounts = append(bidamounts, &ju.ExtField{Code: "winnerorder", Field: "bidamount", ExtFrom: "j.Winnerorder", SourceValue: j.Winnerorder[0]["price"], Value: tmpPrice[0], Score: 0.5})
 			}
 		}
 	}
 	if j.Result["winner"] == nil && len(winners) > 0 {
 		j.Result["winner"] = winners
-	} else {
+	} else if len(winners) > 0 {
 		j.Result["winner"] = append(j.Result["winner"], winners...)
 	}
 	if j.Result["bidamount"] == nil && len(bidamounts) > 0 {
 		j.Result["bidamount"] = bidamounts
-	} else {
+	} else if len(bidamounts) > 0 {
 		j.Result["bidamount"] = append(j.Result["bidamount"], bidamounts...)
 	}
 	//j.Result["winner"] = winners

src/jy/extract/extractInit.go

@@ -1409,8 +1409,7 @@ func (e *ExtractTask) ResultSave(init bool) {
 						log.Debug(err)
 					})
 				}
-
-				time.Sleep(3 * time.Second)
+				time.Sleep(2 * time.Second)
 			}
 		}()
 	} else {

src/jy/extract/score_jsondata.go

@@ -12,7 +12,7 @@ import (
 )
 
 func JsonDataMergeProcessing(j *util.Job, e *ExtractTask) map[string][]*util.ExtField {
-	if len(j.Result) <= 0 || j.Jsondata == nil || len(*j.Jsondata) <= 0 {
+	if len(j.Result) <= 0 || j.Jsondata == nil || len(*j.Jsondata) <= 0 || j.Site =="中国政府采购网"{
 		return j.Result
 	}
 	jdextweight := util2.IntAll((*j.Jsondata)["extweight"])

src/jy/pretreated/analystep.go

@@ -351,6 +351,14 @@ func processTableResult(tabres *TableResult, block *util.Block, job *util.Job, i
 			if tv.WinnerOrder != nil && len(tv.WinnerOrder) > 0 {
 				bv.WinnerOrder = tv.WinnerOrder
 			}
+			if tv.Bidamount > 0 && bv.Bidamount == 0 {
+				bv.Bidamount = tv.Bidamount
+				bv.IsTrueBidamount = tv.IsTrueBidamount
+			}
+			if tv.Budget >0 && bv.Bidamount == 0{
+				bv.Budget = tv.Budget
+				bv.IsTrueBudget = tv.IsTrueBudget
+			}
 		}
 		for k, v := range pkgMap {
 			job.BlockPackage[k] = v

src/jy/pretreated/analytable.go

@@ -2,7 +2,9 @@ package pretreated
 
 import (
 	"fmt"
+	"jy/clear"
 	u "jy/util"
+
 	//"log"
 	qutil "qfw/util"
 	"regexp"
@@ -46,7 +48,7 @@ var (
 	FindVal_1  = regexp.MustCompile("[第]?([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)((子|合同|分|施工|监理)?(标段?|包|合同段|标包))|((子|合同|分|施工|监理)?(标|包)(段|号)?)[ 　\u3000\u2003\u00a0]*([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+)")
 	FindVal2_1 = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ\\-]+)|^(设计|施工|监理|验收)[分子]?[标包]?[段号]?$")
 	//判断分包前排除
-	excludeKey = regexp.MustCompile("(涉及包号|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)") //编号|划分
+	excludeKey = regexp.MustCompile("(涉及包号|分包数量|包件号?|项目标号|规格|型号|招标范围|业绩|废标)|(^编号$)|([^包段标]编号)") //编号|划分
 	//-------------
 
 	cut = u.NewCut()
@@ -86,7 +88,7 @@ var (
 	//简单判断
 	filter_zbdw_jd = regexp.MustCompile("(投标|成交|中标|合同)(供应商|单位|人|名称).{0,4}$")
 	//且不包含以下字眼
-	filter_zbdw_kn = regexp.MustCompile("第[2二3三4四5五]|得分|地址")
+	filter_zbdw_kn = regexp.MustCompile("第[2二3三4四5五]|得分|地址|询价保证金")
 	//且值包含以下字眼
 	filter_zbdw_v = regexp.MustCompile("(公司|集团|研究院|设计院|局|厂|部|站|城|店|市|所|处)$|([^购]中心|办公|用品)")
 	//且值包含以下字眼
@@ -197,9 +199,9 @@ func CommonDataAnaly(k, tabletag, tabledesc string, v interface{}, isSite bool,
 			if filter_tag_zb.MatchString(tabletag) || filter_tag_zb.MatchString(tabledesc) {
 				//u.Debug(v1, k, "-----", filter_zbdw_jd.MatchString(k), filter_zbdw_v.MatchString(v1))
 				if filter_zbje_jd.MatchString(k) && !filter_zbje_kn.MatchString(k) && filter_zbje_v.MatchString(v1) {
-					if filter_ysje_jd.MatchString(k){
+					if filter_ysje_jd.MatchString(k) {
 						kvTags["预算金额"] = append(kvTags["预算金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
-					}else {
+					} else if !filter_zbdw_kn.MatchString(k) {
 						kvTags["中标金额"] = append(kvTags["中标金额"], &u.Tag{Key: k, Value: v1, Weight: -100})
 					}
 
@@ -2128,6 +2130,44 @@ func (tn *Table) manyPackageProcessByIndex(index []string, standIndex_pos []int,
 			//}
 			for _, vcgdw := range k1tags {
 				if vcgdw.Value == "采购单位" {
+				} else if vcgdw.Value == "预算" && len(val) == len(index) {
+					for bi, bv := range index {
+						if tn.BlockPackage.Map[bv].(*u.BlockPackage).Budget == 0 {
+							moneys := clear.ObjToMoney([]interface{}{val[bi], ""})
+							if len(moneys) > 0 {
+								if vf, ok := moneys[0].(float64); ok {
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).Budget = vf
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).IsTrueBudget = moneys[len(moneys)-1].(bool)
+								} else if vi, ok := moneys[0].(int); ok {
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).Budget = float64(vi)
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).IsTrueBudget = moneys[len(moneys)-1].(bool)
+								}
+							}
+						}
+					}
+				} else if vcgdw.Value == "中标金额" && len(val) == len(index){
+					for bi, bv := range index {
+						if tn.BlockPackage.Map[bv].(*u.BlockPackage).Bidamount == 0 {
+							moneys := clear.ObjToMoney([]interface{}{val[bi], ""})
+							if len(moneys) > 0 {
+								if vf, ok := moneys[0].(float64); ok {
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).Bidamount = vf
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).IsTrueBidamount = moneys[len(moneys)-1].(bool)
+								} else if vi, ok := moneys[0].(int); ok {
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).Bidamount = float64(vi)
+									tn.BlockPackage.Map[bv].(*u.BlockPackage).IsTrueBidamount = moneys[len(moneys)-1].(bool)
+								}
+							}
+						}
+					}
+				} else if vcgdw.Value == "中标单位" {
+					for _, bv := range index {
+						if tn.BlockPackage.Map[bv].(*u.BlockPackage).WinnerOrder == nil || len(tn.BlockPackage.Map[bv].(*u.BlockPackage).WinnerOrder) == 0 {
+							continue
+						} else {
+							tn.BlockPackage.Map[bv].(*u.BlockPackage).Winner = qutil.ObjToString(tn.BlockPackage.Map[bv].(*u.BlockPackage).WinnerOrder[0]["entname"])
+						}
+					}
 				}
 			}
 		} else if val, bvs := v1.(string); bvs && len(index) == 1 {
@@ -3280,7 +3320,7 @@ func initLineMapLineMapArr(table *Table) (lineMapArr map[string]*SortMap, lineMa
 	for _, key := range table.SortKV.Keys { //遍历table.SortKV.Keys而不是直接遍历table.SortKV.Map是为了得到table头的顺序
 		val := table.SortKV.Map[key]
 		key = regReplAllSpace.ReplaceAllString(key, "")
-		key = strings.Replace(key, "", "", -1)    //处理一个特殊的采购量 经上层处理空格后未处理掉
+		key = strings.Replace(key, "", "", -1) //处理一个特殊的采购量 经上层处理空格后未处理掉
 		if realTypeVal, ok := val.([]string); ok { //val为数组 {"数量":["1","2","3"]}
 			/*
 				{

src/jy/pretreated/tablev2.go

@@ -494,6 +494,9 @@ func (tr *TR) AddTD(td *TD) {
 		tr.TDs[len(tr.TDs)-1].RightNode = td
 	}
 	**/
+	if tr==nil|| tr.TDs == nil{
+		return
+	}
 	td.ColPos = len(tr.TDs)
 	tr.TDs = append(tr.TDs, td)
 }

src/jy/util/article.go

@@ -128,20 +128,22 @@ type Segment struct {
 
 //包
 type BlockPackage struct {
-	Origin      string                   //包的原始值
-	Name        string                   //标段（包）名称
-	Text        string                   //包文 (包对应的正文)
-	Budget      float64                  //标段（包）预算
-	Winner      string                   //标段（包）中标单位
-	Bidamount   float64                  //标段（包）中标价
-	Index       string                   //序号 (转换后编号，只有数字或字母)
-	Type        string                   //类型 (匹配后面的标段、包之类的词)
-	ColonKV     *JobKv                   //冒号kv (分出的对应的KV值)
-	TableKV     *JobKv                   //table kv (分出的对应的KV值)
-	SpaceKV     *JobKv                   //空格 kv (分出的对应的KV值)
-	BidStatus   string                   //成交状态
-	WinnerOrder []map[string]interface{} //中标人排序
-	Accuracy    bool                     //包里面抽取字段的准确性，如果能打上块标签的话，就不用中标候选人中的值覆盖包里面的值
+	Origin          string                   //包的原始值
+	Name            string                   //标段（包）名称
+	Text            string                   //包文 (包对应的正文)
+	Budget          float64                  //标段（包）预算
+	IsTrueBudget    bool                     //标段（包）预算0是否有效
+	Winner          string                   //标段（包）中标单位
+	Bidamount       float64                  //标段（包）中标价
+	IsTrueBidamount bool                     //标段（包）中标价 0是否有效
+	Index           string                   //序号 (转换后编号，只有数字或字母)
+	Type            string                   //类型 (匹配后面的标段、包之类的词)
+	ColonKV         *JobKv                   //冒号kv (分出的对应的KV值)
+	TableKV         *JobKv                   //table kv (分出的对应的KV值)
+	SpaceKV         *JobKv                   //空格 kv (分出的对应的KV值)
+	BidStatus       string                   //成交状态
+	WinnerOrder     []map[string]interface{} //中标人排序
+	Accuracy        bool                     //包里面抽取字段的准确性，如果能打上块标签的话，就不用中标候选人中的值覆盖包里面的值
 }
 
 //联系人

src/main.go

@@ -44,7 +44,7 @@ func init() {
 	//	util.ElasticClient = eClient
 	util.ElasticClientIndex = qu.ObjToString(util.Config["elasticsearch_index"])
 	util.ElasticClientType = qu.ObjToString(util.Config["elasticsearch_type"])
-	util.ElasticClientDB = qu.ObjToString(util.Config["winner_enterprise"])
+	util.ElasticClientDB = qu.ObjToString(util.Config["elasticsearch_db"])
 	//}
 }
 

src/res/fieldscore.json

@@ -147,7 +147,7 @@
         "positivewords": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|中心|分校|办公室|学校|幼儿园|动物园|管理站|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|初中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司)$",
+                "regstr": ".{2,100}(委员会|中心|分校|办公室|学校|幼儿园|动物园|管理站|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|初中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司|监狱|监测站|血站|检查站)$",
                 "score": 3
             }
         ],
@@ -207,7 +207,7 @@
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(附件|否决原因|候选|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(定标|通知|异议|要求|详细|意见|原因|具体|结果|负责|付款|附件|候选|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -10
             },
 			{
@@ -259,8 +259,8 @@
         ],
         "negativewords": [
             {
-                "describe": "不在展示",
-                "regstr": "(详见公告)",
+                "describe": "包含负分",
+                "regstr": "(详见公告|原因|未知|收费|标注|负责人)",
                 "score": -10
             }
         ],
@@ -291,6 +291,18 @@
             }
         ]
     },
+    "agencyaddr": {
+        "type": "string",
+        "positivewords": [
+        ],
+        "negativewords": [
+            {
+                "describe": "包含负分",
+                "regstr": "(详见|公告|原因|未知|收费|标注)",
+                "score": -10
+            }
+        ]
+    },
     "buyerperson": {
         "type": "string",
         "positivewords": [
@@ -389,7 +401,7 @@
             },
 			{
                 "describe": "包含负分",
-                "regstr": "(详见公告)",
+                "regstr": "(详见|公告)",
                 "score": -10
             }
         ],

udp_winner/config.json

@@ -3,8 +3,10 @@
   "elasticsearch_index": "winner_enterprise",
   "elasticsearch_type": "winnerent",
   "udpport": "127.0.0.1:12311",
+  "port": "12311",
   "mgoinit": "192.168.3.207:27081",
   "mgodb_bidding": "qfw",
+  "mgodb_mgoinit_c": "bidding",
   "mgodb_enterprise": "enterprise",
   "mgodb_enterprise_c": "qyxy",
   "mgourl2": "192.168.3.207:27092",

udp_winner/main.go

@@ -9,11 +9,14 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	es "gopkg.in/olivere/elastic.v1"
 	"log"
 	mu "mfw/util"
 	"net"
-	elastic "qfw/common/src/qfw/util/elastic"
+	"qfw/common/src/qfw/util/elastic"
 	"qfw/util"
+	"regexp"
+	"strconv"
 
 	"sort"
 	"strings"
@@ -29,6 +32,11 @@ var
 	Addrs                                 = make(map[string]interface{}, 0) //省市县
 	udpclient                             mu.UdpClient                      //udp对象
 	ElasticClientIndex, ElasticClientType string
+	Reg_xing                              = regexp.MustCompile(`\*{1,}`)
+	Reg_person                            = regexp.MustCompile("[\u4E00-\u9FA5\\s]+")
+	Reg_tel                               = regexp.MustCompile(`^[0-9\-\s]*$`)
+	EsConn                                *es.Client
+	Updport                               int
 )
 /**
 新增
@@ -40,38 +48,40 @@ func init() {
 	log.Println(Config)
 	Fields = []string{"_id", "contact", "partners", "business_scope", "company_address", "capital",
 		"establish_date", "legal_person", "company_type", "district", "city", "province", "area_code", "credit_no",
-		"company_name", "history_name", "topscopeclass",  "wechat_accounts", "alias","website","report_websites"}
+		"company_name", "history_name", "topscopeclass", "wechat_accounts", "alias", "website", "report_websites"}
 	var err error
 	//mongo init
-	SourceClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgoinit"]).SetMaxPoolSize(20))
+	SourceClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgoinit"]).SetMaxPoolSize(20).SetMaxConnIdleTime(time.Hour * 24))
 	if err != nil {
 		log.Fatalln(err)
 	}
-	c1 := context.Background()
+	c1, _ := context.WithTimeout(context.Background(), 9999*time.Hour)
+
 	err = SourceClient.Connect(c1)
 	//defer SourceClient.Disconnect(c1)
 	if err != nil {
 		log.Fatalln(err)
 	}
-	FClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgourl"]).SetMaxPoolSize(20))
+	FClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgourl"]).SetMaxPoolSize(20).SetMaxConnIdleTime(time.Hour * 999999))
 	if err != nil {
 		log.Fatalln(err)
 	}
-	cc := context.Background()
-	err = FClient.Connect(cc)
+	err = FClient.Connect(c1)
 	//defer FClient.Disconnect(cc)
 	if err != nil {
 		log.Fatalln(err)
 	}
 	//加载省市县代码
-	cursor2, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("address").Find(cc, bson.M{},
+	cursor2, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("address").Find(c1, bson.M{},
 		options.Find().SetProjection(bson.M{"province": 1, "code": 1, "city": 1, "district": 1}))
-	defer cursor2.Close(cc)
-	defer FClient.Connect(cc)
+	//defer FClient.Connect(cc)
 	if err != nil {
 		log.Fatalln(err)
 	}
-	for cursor2.Next(cc) {
+	for cursor2.Next(context.TODO()) {
+		if err := cursor2.Err(); err != nil {
+			log.Println("cursor.Err();", err)
+		}
 		tmp := make(map[string]interface{})
 		if err := cursor2.Decode(&tmp); err != nil {
 			log.Println(err)
@@ -83,27 +93,17 @@ func init() {
 			}
 		}
 	}
+	defer cursor2.Close(context.TODO())
 	log.Println(len(Addrs))
 	//es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
 	//es init
-	elastic.InitElasticSize(Config["elasticsearch"],10)
-	//esConn := elastic.GetEsConn()
-	//defer elastic.DestoryEsConn(esConn)
-	//log.Println(esConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id("123").BodyJson(map[string]interface{}{"testname":"六盘水市钟山开发区亿农科贸有限公司"}).Refresh(true).Do())
-	//log.Println(esConn.Delete().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id("123").Refresh(true).Do())
-
-	//if ESclient, err = elastic.NewClient(elastic.SetURL(Config["elasticsearch"]), elastic.SetHealthcheckTimeout(time.Minute)); err != nil {
-	//	log.Println(Config["elasticsearch"])
-	//	log.Fatalln("ElasticClient err:", err)
-	//} else {
-	//	ElasticClientIndex = Config["elasticsearch_index"]
-	//	ElasticClientType = Config["elasticsearch_type"]
-	//}
-
+	elastic.InitElasticSize(Config["elasticsearch"], 50)
+	EsConn = elastic.GetEsConn()
+	defer elastic.DestoryEsConn(EsConn)
 	//redis
 	RedisPool = redis.Pool{
-		MaxIdle:     10,
-		IdleTimeout: 240 * time.Second,
+		MaxIdle:     50,
+		IdleTimeout: 10 * time.Second,
 		Dial: func() (redis.Conn, error) {
 			conn, e := redis.Dial("tcp", Config["redis"])
 			if e != nil {
@@ -117,17 +117,19 @@ func init() {
 		},}
 	c := RedisPool.Get()
 	if _, err := c.Do("PING"); err != nil {
-		log.Fatalln(err)
+		log.Fatalln("redis err:", err)
 	}
-	defer c.Close()
+	c.Close()
 }
 
 func main() {
 	//udp
 	updport := Config["udpport"]
+	Updport, _ = strconv.Atoi(Config["port"])
 	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
 	udpclient.Listen(processUdpMsg)
 	log.Println("Udp服务监听", updport)
+	log.Println("发送端口port：",Updport)
 	go TimedTask() //定时任务
 	c := make(chan int, 1)
 	<-c
@@ -170,18 +172,23 @@ func task(mapinfo *map[string]interface{}) {
 	// topscopeclass项目类型-industry行业类型&&topscopeclass联系人项目类型
 	// (area地区-province省份 city城市-city城市 district区县-district区县)
 	// winneraddr-company_address企业地址
-	cursor, err := SourceClient.Database(Config["mgodb_bidding"]).Collection("bidding").Find(context.TODO(), bson.M{
+	cursor, err := SourceClient.Database(Config["mgodb_bidding"]).Collection(Config["mgodb_mgoinit_c"]).Find(context.Background(), bson.M{
 		"_id": bson.M{
 			"$gte": GId,
 			"$lte": LtId,
 		},
 	}, options.Find().SetProjection(bson.M{"winner": 1, "winnertel": 1, "winnerperson": 1,
-		"topscopeclass": 1, "winneraddr": 1}))
+		"topscopeclass": 1, "winneraddr": 1}).SetSort(bson.M{"_id": 1}).SetBatchSize(60000000).SetMaxTime(time.Hour*24))
 	if err != nil {
 		log.Println(err)
 		return
 	}
-	for cursor.Next(context.TODO()) {
+	overid := ""
+	c2, _ := context.WithTimeout(context.Background(), 9999*time.Hour)
+	for cursor.Next(c2) {
+		if err := cursor.Err(); err != nil {
+			log.Println("cursor.Err();", err)
+		}
 		tmp := map[string]interface{}{}
 		if err := cursor.Decode(&tmp); err == nil {
 			if tmp["winner"] == nil || tmp["winner"] == "" {
@@ -189,21 +196,32 @@ func task(mapinfo *map[string]interface{}) {
 			}
 			//redis查询是否存在
 			rdb := RedisPool.Get()
-			defer rdb.Close()
 			if reply, err := redis.String(rdb.Do("GET", tmp["winner"])); err != nil {
 				//redis不存在存到临时表,定时任务处理
-				FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").InsertOne(context.TODO(), tmp)
-				//log.Println(tmp, err)
+				if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").InsertOne(context.TODO(), tmp); err != nil {
+					log.Println(err, tmp)
+				}
+				//log.Println("get redis id err:定时任务处理", err, tmp)
+				if err := rdb.Close(); err != nil {
+					log.Println(err)
+				}
 				continue
 			} else {
+				log.Println("redis get :", reply)
 				//redis存在
 				//log.Println(reply)
 				//reply = "5e0316b998a9abaf6535df3d"
 				id, err := primitive.ObjectIDFromHex(reply)
 				if err != nil {
-					log.Println("get redis id err:", err, tmp)
+					log.Println("get redis id  Hex err:", err, tmp)
+					if err := rdb.Close(); err != nil {
+						log.Println(err)
+					}
 					continue
 				}
+				if err := rdb.Close(); err != nil {
+					log.Println(err)
+				}
 				//拿到合并后的qyk
 				oldTmp := make(map[string]interface{})
 				err = FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
@@ -217,6 +235,7 @@ func task(mapinfo *map[string]interface{}) {
 				tmpTopscopeclass := []string{}
 				tmpTopscopeclassMap := make(map[string]bool)
 				log.Println(tmp["_id"])
+				overid = tmp["_id"].(primitive.ObjectID).Hex()
 				if oldTmp["industry"] == nil {
 					//log.Println(reflect.ValueOf(tmp["topscopeclass"]))
 					if v, ok := tmp["topscopeclass"].(primitive.A); ok {
@@ -251,24 +270,35 @@ func task(mapinfo *map[string]interface{}) {
 				sort.Strings(tmpTopscopeclass)
 				oldTmp["industry"] = tmpTopscopeclass
 				esId := oldTmp["_id"].(primitive.ObjectID).Hex()
-				//联系方式合并
-				if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" {
-					oldTmp["updatatime"] =time.Now().Unix()
+				//更新行业类型
+				if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
+					oldTmp["updatatime"] = time.Now().Unix()
 					//mongo更新
-					FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
-						UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp})
+					if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+						UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp}); err != nil {
+						log.Println("mongo更新err:", err)
+					}
 					//es更新
 					delete(oldTmp, "_id")
-					esConn := elastic.GetEsConn()
-					defer elastic.DestoryEsConn(esConn)
-					esConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do()
+					//esConn := elastic.GetEsConn()
+					//defer elastic.DestoryEsConn(esConn)
+					if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
+						log.Println("update es err:", err)
+					}
 					//log.Println( err2,err3)
 					continue
 				}
+				//联系方式合并
 				var tmpperson, winnertel string
 				tmpperson = tmp["winnerperson"].(string)
-				if tmp["winnertel"] == nil {
+				if tmp["winnertel"] == nil || tmp["winnertel"] == "" {
 					winnertel = ""
+				} else {
+					if Reg_xing.MatchString(util.ObjToString(tmp["winnertel"])) || !Reg_tel.MatchString(util.ObjToString(tmp["winnertel"])) {
+						winnertel = ""
+					} else {
+						winnertel = util.ObjToString(tmp["winnertel"])
+					}
 				}
 				contactMaps := make([]interface{}, 0)
 				if oldTmp["contact"] == nil {
@@ -306,14 +336,18 @@ func task(mapinfo *map[string]interface{}) {
 				}
 				oldTmp["contact"] = contactMaps
 				//mongo更新
-				oldTmp["updatatime"] =time.Now().Unix()
-				FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
-					UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp})
+				oldTmp["updatatime"] = time.Now().Unix()
+				if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+					UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp}); err != nil {
+					log.Println("mongo更新 err :", err)
+				}
 				//es更新
 				delete(oldTmp, "_id")
-				esConn := elastic.GetEsConn()
-				defer elastic.DestoryEsConn(esConn)
-				esConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do()
+				//esConn := elastic.GetEsConn()
+				//defer elastic.DestoryEsConn(esConn)
+				if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
+					log.Println("EsConn err :", err)
+				}
 				//log.Println( err2,err3)
 			}
 		} else {
@@ -321,149 +355,22 @@ func task(mapinfo *map[string]interface{}) {
 			continue
 		}
 	}
-	defer cursor.Close(context.TODO())
-	//tmps := make([]interface{}, 0)
-	//num, snum := 0, 0
-	//for k := range keys {
-	//	//if num == 6 {
-	//	//	return
-	//	//}
-	//	tmp := make(map[string]interface{})
-	//	err := Client.Database("enterprise").Collection("qyxy").FindOne(context.TODO(), bson.M{"company_name": k}).Decode(&tmp)
-	//	if err != nil {
-	//		//log.Println(k, err)
-	//		continue
-	//	}
-	//	if tmp["credit_no"] != nil {
-	//		if credit_no, ok := tmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
-	//			len(strings.TrimSpace(credit_no)) > 8 {
-	//			dataNo := strings.TrimSpace(credit_no)[2:8]
-	//			if Addrs[dataNo] != nil {
-	//				if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
-	//					if tmp["province"] == nil || tmp["province"] == "" {
-	//						tmp["province"] = v["province"]
-	//					}
-	//					tmp["city"] = v["city"]
-	//					tmp["district"] = v["district"]
-	//
-	//				}
-	//			}
-	//		}
-	//	}
-	//	contacts := make([]map[string]interface{}, 0)
-	//	contact := make(map[string]interface{}, 0)
-	//	if tmp["legal_person"] != nil {
-	//		contact["contact_person"] = tmp["legal_person"] //联系人
-	//	} else {
-	//		contact["contact_person"] = "" //联系人
-	//	}
-	//	contact["contact_type"] = "法定代表人" //法定代表人
-	//	//log.Println(1)
-	//	if tmp["annual_reports"] != nil {
-	//		bytes, err := json.Marshal(tmp["annual_reports"])
-	//		if err != nil {
-	//			log.Println("annual_reports err:", err)
-	//		}
-	//		//log.Println(2, string(bytes))
-	//		phonetmp := make([]map[string]interface{}, 0)
-	//		err = json.Unmarshal(bytes, &phonetmp)
-	//		if err != nil {
-	//			log.Println("Unmarshal err:", err)
-	//		}
-	//		//log.Println(44, err)
-	//		for _, vv := range phonetmp {
-	//			if vv["company_phone"] != nil {
-	//				if vv["company_phone"] == "" {
-	//					continue
-	//				} else {
-	//					contact["phone"] = vv["company_phone"] //联系电话
-	//					break
-	//				}
-	//			} else {
-	//				contact["phone"] = "" //联系电话
-	//			}
-	//
-	//		}
-	//	}
-	//	//log.Println(k, contact["phone"], tmp["_id"])
-	//	//time.Sleep(10 * time.Second)
-	//	if contact["phone"] == nil {
-	//		contact["phone"] = "" //联系电话
-	//	}
-	//	contact["topscopeclass"] = "企业公示"         //项目类型
-	//	contact["updatetime"] = time.Now().Unix() //更新时间
-	//	contacts = append(contacts, contact)
-	//	tmp["contact"] = contacts
-	//
-	//	savetmp := make(map[string]interface{}, 0)
-	//	//字段处理
-	//	for _, sk := range Fields {
-	//		if sk == "establish_date" { //成立日期
-	//			if tmp[sk] != nil {
-	//				savetmp[sk] = tmp[sk].(primitive.DateTime).Time().UTC().Unix()
-	//				continue
-	//			}
-	//		} else if sk == "capital" { //注册资本
-	//			//log.Println(sk, tmp[sk])
-	//			savetmp[sk] = ObjToMoney([]interface{}{tmp[sk], ""})[0]
-	//			continue
-	//		} else if sk == "partners" { //股东及出资信息
-	//			//log.Println(sk, tmp[sk], )
-	//			//fmt.Println(reflect.TypeOf(tmp[sk]))
-	//			if tmp[sk] != nil {
-	//				if ppms, ok := tmp[sk].(primitive.A); ok {
-	//					for i, _ := range ppms {
-	//						if ppms[i].(map[string]interface{})["stock_type"] != nil {
-	//							ppms[i].(map[string]interface{})["stock_type"] = "企业公示"
-	//						}
-	//						delete(ppms[i].(map[string]interface{}), "identify_type")
-	//					}
-	//					savetmp[sk] = ppms
-	//					continue
-	//				}
-	//			}
-	//		} else if sk == "_id" { //_id备份企业库
-	//			savetmp["tmp"+sk] = tmp[sk]
-	//			continue
-	//		}
-	//		if tmp[sk] == nil && sk != "history_name" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "wechat_accounts" {
-	//			savetmp[sk] = ""
-	//		} else {
-	//			if sk == "wechat_accounts" { //微信公众号
-	//				if savetmp[sk] == nil {
-	//					//TODO 微信公众号取值未确认
-	//					savetmp[sk] = []string{}
-	//				}
-	//				continue
-	//			} else if sk == "website" { //网址
-	//				//TODO 网址取值未确认
-	//				continue
-	//			}
-	//			savetmp[sk] = tmp[sk]
-	//		}
-	//	}
-	//	savetmp["alias"] = "" //别名
-	//	tmps = append(tmps, savetmp)
-	//	num++
-	//	snum++
-	//	if snum >= 300 {
-	//		_, err := Client.Database("extract_v3").Collection("enterprise_qyxy").InsertMany(context.TODO(), tmps)
-	//		if err != nil {
-	//			log.Println("save:", err)
-	//			continue
-	//		} else {
-	//			log.Println(num)
-	//			tmps = []interface{}{}
-	//			snum = 0
-	//		}
-	//	}
-	//}
-	//if len(tmps) > 0 {
-	//	result, err := Client.Database("extract_v3").Collection("enterprise_qyxy").InsertMany(context.TODO(), tmps)
-	//	if err != nil {
-	//		log.Println("save over:", err)
-	//	} else {
-	//		log.Println("last save num:", len(result.InsertedIDs))
-	//	}
-	//}
+	defer cursor.Close(c2)
+	log.Println("合并执行完成", gtid, lteid ,overid)
+	if gtid != lteid{
+		by, _ := json.Marshal(map[string]interface{}{
+			"gtid":  overid,
+			"lteid": lteid,
+			"stype": "",
+		})
+		if e := udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
+			IP:   net.ParseIP("127.0.0.1"),
+			Port: Updport,
+		});e != nil{
+			log.Println(e)
+		}
+		log.Println("重新发送udp:",string(by))
+	}
+	log.Println("合并执行完成 ok", gtid, lteid,overid)
+
 }

udp_winner/timedTask.go

@@ -4,14 +4,16 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	elastic "qfw/common/src/qfw/util/elastic"
+	"github.com/garyburd/redigo/redis"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"gopkg.in/mgo.v2/bson"
 	"log"
+	"net"
 	"sort"
 	"strings"
 	"time"
+	mu "mfw/util"
 )
 
 //定时任务
@@ -21,7 +23,7 @@ func TimedTask() {
 	t2 := time.NewTimer(time.Second * 5)
 	for range t2.C {
 		tmpLast := map[string]interface{}{}
-		if err := FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").FindOne(context.TODO(), bson.M{}, options.FindOne().SetSort(bson.M{"_id": -1})).Decode(&tmpLast); err != nil {
+		if err := FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").FindOne(context.TODO(), bson.M{}, options.FindOne().SetSort(bson.M{"_id": -1})).Decode(&tmpLast); err != nil {
 			//临时表无数据
 			log.Println("临时表无数据:", err)
 			t2.Reset(time.Minute * 5)
@@ -29,7 +31,7 @@ func TimedTask() {
 		} else {
 			//临时表有数据
 			log.Println("临时表有数据:", tmpLast)
-			cursor, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").Find(context.TODO(), bson.M{
+			cursor, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").Find(context.TODO(), bson.M{
 				"_id": bson.M{
 					"$lte": tmpLast["_id"],
 				},
@@ -41,15 +43,54 @@ func TimedTask() {
 			}
 			//遍历临时表数据，匹配不到原始库存入异常表
 			for cursor.Next(context.TODO()) {
+				if err := cursor.Err(); err != nil {
+					log.Println("cursor.Err();", err)
+				}
 				tmp := make(map[string]interface{})
 				if err := cursor.Decode(&tmp); err == nil {
+					//再重新查找redis,存在发udp处理，不存在走新增合并
+					rdb := RedisPool.Get()
+					if reply, err := redis.String(rdb.Do("GET", tmp["winner"])); err == nil {
+						//{"gtid":"57d7ad2f61a0721f152d2ad5","lteid":"5e20968d85a9271abf0ad6c2","stype":""}
+						//redis存在发送udp进行处理
+						go func(reply string) {
+							by, _ := json.Marshal(map[string]interface{}{
+								"gtid":  reply,
+								"lteid": reply,
+								"stype": "",
+							})
+							if e := udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{
+								IP:   net.ParseIP("127.0.0.1"),
+								Port: Updport,
+							});e != nil{
+								log.Println(e)
+							}
+						}(reply)
+						//存在的话删除tmp mongo表
+						if _, err = FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").DeleteOne(context.TODO(), tmp);err != nil{
+							log.Println("删除临时表err:",err)
+						}
+						if err := rdb.Close(); err != nil {
+							log.Println(err)
+						}
+						continue
+					} else {
+						if err = rdb.Close(); err != nil {
+							log.Println(err)
+						}
+					}
+					//查询redis不存在新增
 					resulttmp := make(map[string]interface{})
 					r := FClient.Database(Config["mgodb_enterprise"]).Collection(Config["mgodb_enterprise_c"]).FindOne(context.TODO(), bson.M{"company_name": tmp["winner"]}).Decode(&resulttmp)
 					if r != nil {
 						//log.Println(r)
 						//匹配不到原始库,存入异常表删除临时表
-						FClient.Database(Config["mgodb_extract_kf"]).Collection("err_winner_qyk").InsertOne(context.TODO(), tmp)
-						FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").DeleteOne(context.TODO(), tmp)
+						if _, err = FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_err").InsertOne(context.TODO(), tmp); err != nil {
+							log.Println(err)
+						}
+						if _, err = FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").DeleteOne(context.TODO(), tmp); err != nil {
+							log.Println(err)
+						}
 						continue
 					} else {
 						//log.Println(123)
@@ -84,13 +125,11 @@ func TimedTask() {
 							if err != nil {
 								log.Println("annual_reports err:", err)
 							}
-							//log.Println(2, string(bytes))
 							phonetmp := make([]map[string]interface{}, 0)
 							err = json.Unmarshal(bytes, &phonetmp)
 							if err != nil {
 								log.Println("Unmarshal err:", err)
 							}
-							//log.Println(44, err)
 							for _, vv := range phonetmp {
 								if vv["company_phone"] != nil {
 									if vv["company_phone"] == "" {
@@ -140,7 +179,7 @@ func TimedTask() {
 										savetmp[sk] = ppms
 
 									}
-								}else {
+								} else {
 									savetmp[sk] = []interface{}{}
 								}
 								continue
@@ -174,37 +213,45 @@ func TimedTask() {
 								savetmp[sk] = []interface{}{}
 								continue
 							}
-							if resulttmp[sk] == nil && sk != "history_name" &&sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
+							if resulttmp[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
 								savetmp[sk] = ""
 							} else {
 								savetmp[sk] = resulttmp[sk]
 							}
 						}
 						//tmps = append(tmps, savetmp)
-						savetmp["updatatime"] =time.Now().Unix()
+						savetmp["updatatime"] = time.Now().Unix()
 						//保存mongo
 						result, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
 							InsertOne(context.TODO(), savetmp)
 						if err == nil {
 							//保存redis
 							rc := RedisPool.Get()
-							defer rc.Close()
 							var _id string
 							if v, ok := result.InsertedID.(primitive.ObjectID); ok {
 								_id = v.Hex()
 							}
-							if _, err := rc.Do("SET", savetmp["company_name"], result.InsertedID.(primitive.ObjectID).Hex()); err != nil {
+							if _, err := rc.Do("SET", savetmp["company_name"], _id); err != nil {
 								log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["company_name"], err)
+								if err := rc.Close(); err != nil {
+									log.Println(err)
+								}
 							} else {
 								//保存es
 								delete(savetmp, "_id")
-								esConn := elastic.GetEsConn()
-								defer elastic.DestoryEsConn(esConn)
-								if _, err := esConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(_id).BodyJson(savetmp).Refresh(true).Do(); err != nil {
+								if err := rc.Close(); err != nil {
+									log.Println(err)
+								}
+
+								//esConn := elastic.GetEsConn()
+								//defer elastic.DestoryEsConn(esConn)
+								if _, err := EsConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(_id).BodyJson(savetmp).Refresh(true).Do(); err != nil {
 									log.Println("save es err :", tmp["_id"], savetmp["_id"], err)
 								} else {
 									//删除临时表
-									FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").DeleteOne(context.TODO(), tmp)
+									if _, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("winner_new").DeleteOne(context.TODO(), tmp); err != nil {
+										log.Println(err)
+									}
 								}
 							}
 						} else {

udpfilterdup/src/config.json

@@ -3,11 +3,11 @@
     "dupdays": 5,
     "mongodb": {
         "addr": "192.168.3.207:27092",
-        "pool": 5,
+        "pool": 10,
         "db": "extract_kf",
         "extract": "zk",
         "site": {
-            "dbname": "zhaolongyue",
+            "dbname": "extract_kf",
             "coll": "site"
         }
     },
@@ -18,6 +18,7 @@
     "nextNode": [],
     "isMerger": false,
     "threads": 1,
+    "isSort":true,
     "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
     "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批)",
     "specialtitle_2": "项目（[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]）",

udpfilterdup/src/datamap.go

@@ -11,29 +11,33 @@ import (
 )
 
 type Info struct {
-	id          string	//id
-	title       string  //标题
-	area        string  //省份
-	city        string  //城市
-	subtype     string  //信息类型
-	buyer       string  //采购单位
-	agency      string  //代理机构
-	winner      string  //中标单位
-	budget      float64 //预算金额
-	bidamount   float64 //中标金额
-	projectname string  //项目名称
-	projectcode string  //项目编号
-	contractnumber string //合同编号
-	publishtime int64   //发布时间
-	bidopentime int64   //开标时间
-	agencyaddr  string  //开标地点
-	site        string  //站点
-	href        string  //正文的url
-
-	repeatid    string  //重复id
+	id    string //id
+	title string //标题
+
+	area           string  //省份
+	city           string  //城市
+	subtype        string  //信息类型
+	buyer          string  //采购单位
+	agency         string  //代理机构
+	winner         string  //中标单位
+	budget         float64 //预算金额
+	bidamount      float64 //中标金额
+	projectname    string  //项目名称
+	projectcode    string  //项目编号
+	contractnumber string  //合同编号
+	publishtime    int64   //发布时间
+	comeintime     int64   //入库时间
+	bidopentime    int64   //开标时间
+	agencyaddr     string  //开标地点
+
+	site string //站点
+	href string //正文的url
+
+	repeatid         string                 //重复id
 	titleSpecialWord bool                   //标题特殊词
 	specialWord      bool                   //再次判断的特殊词
 	mergemap         map[string]interface{} //合并记录
+	is_site          bool                   //是否站点城市
 
 }
 
@@ -74,9 +78,12 @@ func NewDatamap(days int, lastid string) *datamap {
 		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.IntAll(tmp["repeat"]) == -1 {
 			continuSum++
 		} else {
-			pt:=tmp["publishtime"]
-			pt_time:=qutil.Int64All(pt)
-			if pt_time<=0 {
+			pt := tmp["comeintime"]
+			if Is_Sort {
+				pt = tmp["publishtime"]
+			}
+			pt_time := qutil.Int64All(pt)
+			if pt_time <= 0 {
 				continue
 			}
 			if now1 == 0 {
@@ -120,9 +127,12 @@ func NewHistorymap(startid string, lastid string, startTime int64, lastTime int6
 		true)).Sort("-_id").Iter()
 	m, n := 0, 0
 	for tmp_start := make(map[string]interface{}); it_start.Next(&tmp_start); {
-		pt_s:=tmp_start["publishtime"]
-		pt_time:=qutil.Int64All(pt_s)
-		if pt_time<=0 {
+		pt_s := tmp_start["comeintime"]
+		if Is_Sort {
+			pt_s = tmp_start["publishtime"]
+		}
+		pt_time := qutil.Int64All(pt_s)
+		if pt_time <= 0 {
 			continue
 		}
 		if qutil.Float64All(startTime-pt_time) <= datelimit {
@@ -151,9 +161,12 @@ func NewHistorymap(startid string, lastid string, startTime int64, lastTime int6
 		true)).Sort("_id").Iter()
 
 	for tmp_last := make(map[string]interface{}); it_last.Next(&tmp_last); {
-		pt_l:=tmp_last["publishtime"]
-		pt_time:=qutil.Int64All(pt_l)
-		if pt_time<=0 {
+		pt_l := tmp_last["comeintime"]
+		if Is_Sort {
+			pt_l = tmp_last["publishtime"]
+		}
+		pt_time := qutil.Int64All(pt_l)
+		if pt_time <= 0 {
 			continue
 		}
 		if qutil.Float64All(pt_time-lastTime) <= datelimit {
@@ -185,7 +198,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 		area = "全国"
 	}
 	info := &Info{}
-	info.id = qutil.BsonIdToSId(tmp["_id"])
+	info.id = BsonTOStringId(tmp["_id"])
 	info.title = qutil.ObjToString(tmp["title"])
 	info.area = area
 	info.subtype = subtype
@@ -199,6 +212,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.budget = qutil.Float64All(tmp["budget"])
 	info.bidamount = qutil.Float64All(tmp["bidamount"])
 	info.publishtime = qutil.Int64All(tmp["publishtime"])
+	info.comeintime = qutil.Int64All(tmp["comeintime"])
 	info.bidopentime = qutil.Int64All(tmp["bidopentime"])
 	info.agencyaddr = qutil.ObjToString(tmp["agencyaddr"])
 	info.site = qutil.ObjToString(tmp["site"])
@@ -211,12 +225,16 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	if info.mergemap == nil {
 		info.mergemap = make(map[string]interface{}, 0)
 	}
+
+	info.is_site = false
+
 	return info
 }
+
 //判重方法
 func (d *datamap) check(info *Info) (b bool, source *Info, reasons string) {
 
-	reason:=""
+	reason := ""
 	keys := []string{}
 	d.lock.Lock()
 	for k, _ := range d.keys { //不同时间段
@@ -244,10 +262,12 @@ L:
 						sitelock.Unlock()
 						if dict != nil {
 							if info.area == "全国" && dict["area"] != "" {
+								info.is_site = true
 								info.area = qutil.ObjToString(dict["area"])
 								info.city = qutil.ObjToString(dict["city"])
 							} else {
 								if info.city == "" && dict["city"] != "" {
+									info.is_site = true
 									info.area = qutil.ObjToString(dict["area"])
 									info.city = qutil.ObjToString(dict["city"])
 								}
@@ -281,9 +301,9 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = reason+"标题关键词且包含关系"
+							reason = reason + "标题关键词且包含关系"
 							//继续二级金额判断
-							if !againRepeat(v,info){
+							if !againRepeat(v, info) {
 								b = true
 								source = v
 								reasons = reason
@@ -331,7 +351,10 @@ L:
 
 	//往预存数据 d 添加
 	if !b {
-		ct := info.publishtime
+		ct := info.comeintime
+		if Is_Sort {
+			ct = info.publishtime
+		}
 		dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
 		k := fmt.Sprintf("%s_%s_%s", dkey, info.subtype, info.area)
 		d.lock.Lock()
@@ -354,7 +377,7 @@ L:
 }
 
 func (h *historymap) checkHistory(info *Info) (b bool, source *Info, reasons string) {
-	reason:=""
+	reason := ""
 	keys := []string{}
 	h.lock.Lock()
 	for k, _ := range h.keys { //不同时间段
@@ -420,9 +443,9 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = reason+"标题关键词且包含关系"
+							reason = reason + "标题关键词且包含关系"
 							//继续二级金额判断
-							if !againRepeat(v,info){
+							if !againRepeat(v, info) {
 								b = true
 								source = v
 								reasons = reason
@@ -472,11 +495,11 @@ L:
 	//
 	if b {
 		if info.repeatid == source.id {
-			b = false//重复-无变化-不处理
+			b = false //重复-无变化-不处理
 		}
 	} else {
 		if source != nil {
-			if source.repeatid != "" {//未判重-有变化--记录
+			if source.repeatid != "" { //未判重-有变化--记录
 				b = true
 				reason = "未判重记录"
 				reasons = reason
@@ -485,7 +508,10 @@ L:
 	}
 	//往预存数据 d 添加
 	if !b {
-		ct := info.publishtime
+		ct := info.comeintime
+		if Is_Sort {
+			ct = info.publishtime
+		}
 		dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
 		k := fmt.Sprintf("%s_%s_%s", dkey, info.subtype, info.area)
 		data := h.data[k]
@@ -506,7 +532,10 @@ L:
 
 //替换原始数据池
 func (d *datamap) replaceSourceData(replaceData *Info, replaceId string) {
-	ct := replaceData.publishtime
+	ct := replaceData.comeintime
+	if Is_Sort {
+		ct = replaceData.publishtime
+	}
 	dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
 	k := fmt.Sprintf("%s_%s_%s", dkey, replaceData.subtype, replaceData.area)
 	d.lock.Lock()
@@ -531,7 +560,10 @@ func (d *datamap) replaceSourceData(replaceData *Info, replaceId string) {
 }
 
 func (h *historymap) replaceSourceData(replaceData *Info, replaceId string) {
-	ct := replaceData.publishtime
+	ct := replaceData.comeintime
+	if Is_Sort {
+		ct = replaceData.publishtime
+	}
 	dkey := qutil.FormatDateByInt64(&ct, qutil.Date_yyyyMMdd)
 	k := fmt.Sprintf("%s_%s_%s", dkey, replaceData.subtype, replaceData.area)
 	h.lock.Lock()
@@ -555,8 +587,6 @@ func (h *historymap) replaceSourceData(replaceData *Info, replaceId string) {
 	h.lock.Unlock()
 }
 
-
-
 func (d *datamap) update(t int64) {
 	//每天0点清除历史数据
 	d.keymap = d.GetLatelyFiveDay(t)
@@ -592,21 +622,15 @@ func (d *datamap) GetLatelyFiveDay(t int64) []string {
 	return array
 }
 
-
-
-
-
-
-
 /*
 **************************
 ******* 以下为判重 ********
 **************************
-*/
+ */
 //判重方法1
 func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {
 
-	isMeet:=false
+	isMeet := false
 	if info.subtype == "招标" || info.subtype == "邀标" || info.subtype == "询价" ||
 		info.subtype == "竞谈" || info.subtype == "单一" || info.subtype == "竞价" ||
 		info.subtype == "变更" || info.subtype == "其他" {
@@ -624,7 +648,7 @@ func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {
 
 	} else if info.subtype == "中标" || info.subtype == "成交" || info.subtype == "废标" || info.subtype == "流标" {
 		//中标结果
-		if isMeet, reason = winningRepeat_A(v, info, reason);isMeet {
+		if isMeet, reason = winningRepeat_A(v, info, reason); isMeet {
 			if winningRepeat_C(v, info) {
 				return false, reason
 			} else {
@@ -637,7 +661,7 @@ func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {
 
 	} else if info.subtype == "合同" || info.subtype == "验收" || info.subtype == "违规" {
 		//合同
-		if isMeet, reason = contractRepeat_A(v, info, reason);isMeet {
+		if isMeet, reason = contractRepeat_A(v, info, reason); isMeet {
 			if contractRepeat_C(v, info) {
 				return false, reason
 			} else {
@@ -649,7 +673,7 @@ func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {
 		}
 	} else {
 		//招标结果
-		if isMeet, reason = tenderRepeat_A(v, info, reason);isMeet {
+		if isMeet, reason = tenderRepeat_A(v, info, reason); isMeet {
 			if tenderRepeat_C(v, info) {
 				return false, reason
 			} else {
@@ -666,13 +690,13 @@ func quickHeavyMethodOne(v *Info, info *Info, reason string) (bool, string) {
 
 //判重方法2
 func quickHeavyMethodTwo(v *Info, info *Info, reason string) (bool, string) {
-	isMeet:=false
+	isMeet := false
 	if v.agency == info.agency && v.agency != "" && info.agency != "" {
 		if info.subtype == "招标" || info.subtype == "邀标" || info.subtype == "询价" ||
 			info.subtype == "竞谈" || info.subtype == "单一" || info.subtype == "竞价" ||
 			info.subtype == "变更" || info.subtype == "其他" {
 			//招标结果
-			if isMeet, reason =  tenderRepeat_B(v, info, reason);isMeet {
+			if isMeet, reason = tenderRepeat_B(v, info, reason); isMeet {
 				if tenderRepeat_C(v, info) { //有不同
 					return false, reason
 				} else {
@@ -685,7 +709,7 @@ func quickHeavyMethodTwo(v *Info, info *Info, reason string) (bool, string) {
 
 		} else if info.subtype == "中标" || info.subtype == "成交" || info.subtype == "废标" || info.subtype == "流标" {
 			//中标结果
-			if isMeet, reason =  winningRepeat_B(v, info, reason);isMeet {
+			if isMeet, reason = winningRepeat_B(v, info, reason); isMeet {
 				if winningRepeat_C(v, info) { //有不同
 					return false, reason
 				} else {
@@ -698,7 +722,7 @@ func quickHeavyMethodTwo(v *Info, info *Info, reason string) (bool, string) {
 
 		} else if info.subtype == "合同" || info.subtype == "验收" || info.subtype == "违规" {
 			//合同
-			if isMeet, reason = contractRepeat_B(v, info, reason);isMeet {
+			if isMeet, reason = contractRepeat_B(v, info, reason); isMeet {
 				if contractRepeat_C(v, info) { //有不同
 					return false, reason
 				} else {
@@ -710,7 +734,7 @@ func quickHeavyMethodTwo(v *Info, info *Info, reason string) (bool, string) {
 			}
 		} else {
 			//招标结果
-			if isMeet, reason = tenderRepeat_B(v, info, reason);isMeet {
+			if isMeet, reason = tenderRepeat_B(v, info, reason); isMeet {
 				if tenderRepeat_C(v, info) { //有不同
 					return false, reason
 				} else {
@@ -742,7 +766,7 @@ func quickHeavyMethodTwo(v *Info, info *Info, reason string) (bool, string) {
 }
 
 //招标_A
-func tenderRepeat_A(v *Info, info *Info, reason string) (bool ,string) {
+func tenderRepeat_A(v *Info, info *Info, reason string) (bool, string) {
 
 	var ss string
 	p1, p2, p3, p4, p9, p10, p11 := false, false, false, false, false, false, false
@@ -754,8 +778,8 @@ func tenderRepeat_A(v *Info, info *Info, reason string) (bool ,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
-		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode) >= 5) ||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber) >= 5) {
 		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
@@ -785,13 +809,13 @@ func tenderRepeat_A(v *Info, info *Info, reason string) (bool ,string) {
 		(p3 && p4 && p9) || (p3 && p4 && p10) || (p3 && p4 && p11) ||
 		(p4 && p9 && p10) || (p4 && p9 && p11) || (p9 && p10 && p11) {
 		reason = reason + "满足招标A，3要素组合-" + ss + ","
-		return true,reason
+		return true, reason
 	}
-	return false,reason
+	return false, reason
 }
 
 //招标_B
-func tenderRepeat_B(v *Info, info *Info, reason string) (bool,string) {
+func tenderRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 
 	m, n := 0, 0
 	if v.projectname != "" && v.projectname == info.projectname {
@@ -801,8 +825,8 @@ func tenderRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
-		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode) >= 5) ||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber) >= 5) {
 		m++
 	}
 	if v.budget != 0 && v.budget == info.budget {
@@ -821,13 +845,13 @@ func tenderRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	}
 	if m >= 2 {
 		if n == 2 && m == 2 {
-			return false,reason
+			return false, reason
 		} else {
 			reason = reason + "满足招标B,七选二,"
-			return true,reason
+			return true, reason
 		}
 	}
-	return false,reason
+	return false, reason
 }
 
 //招标_C
@@ -838,7 +862,9 @@ func tenderRepeat_C(v *Info, info *Info) bool {
 		return true
 	}
 	//原始地址...
-
+	if v.buyer != "" && info.buyer != "" && v.buyer != info.buyer {
+		return true
+	}
 	if v.bidopentime != 0 && info.bidopentime != 0 && v.bidopentime != info.bidopentime {
 		return true
 	}
@@ -850,7 +876,7 @@ func tenderRepeat_C(v *Info, info *Info) bool {
 }
 
 //中标_A
-func winningRepeat_A(v *Info, info *Info, reason string) (bool,string) {
+func winningRepeat_A(v *Info, info *Info, reason string) (bool, string) {
 
 	var ss string
 	p1, p2, p3, p5, p6, p11 := false, false, false, false, false, false
@@ -862,8 +888,8 @@ func winningRepeat_A(v *Info, info *Info, reason string) (bool,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
-		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode) >= 5) ||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber) >= 5) {
 		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
@@ -888,14 +914,14 @@ func winningRepeat_A(v *Info, info *Info, reason string) (bool,string) {
 		(p3 && p5 && p6) || (p3 && p5 && p11) || (p3 && p6 && p11) ||
 		(p5 && p6 && p11) {
 		reason = reason + "满足中标A,3要素组合-" + ss + ","
-		return true,reason
+		return true, reason
 	}
 
-	return false,reason
+	return false, reason
 }
 
 //中标_B
-func winningRepeat_B(v *Info, info *Info, reason string) (bool,string) {
+func winningRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 
 	m, n := 0, 0
 	if v.projectname != "" && v.projectname == info.projectname {
@@ -905,8 +931,8 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
-		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode) >= 5) ||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber) >= 5) {
 		m++
 	}
 	if v.bidamount != 0 && v.bidamount == info.bidamount {
@@ -922,13 +948,13 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	}
 	if m >= 2 {
 		if n == 2 && m == 2 {
-			return false,reason
+			return false, reason
 		} else {
 			reason = reason + "满足中标B.六选二,"
-			return true,reason
+			return true, reason
 		}
 	}
-	return false,reason
+	return false, reason
 }
 
 //中标_C
@@ -941,37 +967,39 @@ func winningRepeat_C(v *Info, info *Info) bool {
 		return true
 	}
 	//原始地址...
-
+	if v.buyer != "" && info.buyer != "" && v.buyer != info.buyer {
+		return true
+	}
 	return false
 }
 
 //合同_A
-func contractRepeat_A(v *Info, info *Info, reason string) (bool,string) {
+func contractRepeat_A(v *Info, info *Info, reason string) (bool, string) {
 
 	isMeet_1 := false
-	if isMeet_1, reason = tenderRepeat_A(v, info, reason);isMeet_1 {
-		return true,reason
+	if isMeet_1, reason = tenderRepeat_A(v, info, reason); isMeet_1 {
+		return true, reason
 	}
 
 	isMeet_2 := false
-	if isMeet_2, reason = winningRepeat_A(v, info, reason);isMeet_2 {
-		return true,reason
+	if isMeet_2, reason = winningRepeat_A(v, info, reason); isMeet_2 {
+		return true, reason
 	}
-	return false,reason
+	return false, reason
 }
 
 //合同_B
-func contractRepeat_B(v *Info, info *Info, reason string) (bool,string) {
+func contractRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 
 	isMeet_1 := false
-	if isMeet_1, reason = tenderRepeat_B(v, info, reason);isMeet_1 {
-		return true,reason
+	if isMeet_1, reason = tenderRepeat_B(v, info, reason); isMeet_1 {
+		return true, reason
 	}
 	isMeet_2 := false
-	if isMeet_2, reason = winningRepeat_B(v, info, reason);isMeet_2 {
-		return true,reason
+	if isMeet_2, reason = winningRepeat_B(v, info, reason); isMeet_2 {
+		return true, reason
 	}
-	return false,reason
+	return false, reason
 }
 
 //合同_C
@@ -986,40 +1014,28 @@ func contractRepeat_C(v *Info, info *Info) bool {
 	return false
 }
 
-
-func againRepeat(v *Info ,info *Info) bool {
+func againRepeat(v *Info, info *Info) bool {
 	//相同采购单位下
-	if info.buyer != "" &&v.buyer == info.buyer {
-		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
-			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
-			info.subtype=="其他"||info.subtype=="变更" {
+	if info.buyer != "" && v.buyer == info.buyer {
+		if info.subtype == "招标" || info.subtype == "邀标" || info.subtype == "询价" ||
+			info.subtype == "竞谈" || info.subtype == "单一" || info.subtype == "竞价" ||
+			info.subtype == "其他" || info.subtype == "变更" {
 			//预算金额满足条件
-			if v.budget!=info.budget&&v.budget!=0&&info.budget!=0 {
+			if v.budget != info.budget && v.budget != 0 && info.budget != 0 {
 				return true
 			}
-		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||
-			info.subtype=="流标"||info.subtype=="合同"||info.subtype=="验收"||
-			info.subtype=="违规"{
+		} else if info.subtype == "中标" || info.subtype == "成交" || info.subtype == "废标" ||
+			info.subtype == "流标" || info.subtype == "合同" || info.subtype == "验收" ||
+			info.subtype == "违规" {
 			//中标金额单位满足条件
-			if (v.bidamount!=info.bidamount&&v.bidamount!=0&&info.bidamount!=0)||
-				(v.winner!=info.winner&&v.winner!=""&&info.winner!=""){
+			if (v.bidamount != info.bidamount && v.bidamount != 0 && info.bidamount != 0) ||
+				(v.winner != info.winner && v.winner != "" && info.winner != "") {
 				return true
 			}
-		}else {
+		} else {
 
 		}
 	}
 
 	return false
 }
-
-
-
-
-
-
-
-
-
-
-

udpfilterdup/src/datamap.go.bak

@@ -1,205 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"math"
-	qutil "qfw/util"
-	"strings"
-	"sync"
-	"time"
-)
-
-type Info struct {
-	id                 string
-	title              string
-	area               string
-	city               string
-	subtype            string
-	buyer              string
-	agency             string //代理机构
-	winner             string //中标单位
-	projectname        string
-	projectcode        string
-	publishtime        int64
-	ContainSpecialWord bool
-}
-
-var datelimit = float64(432000)
-
-type datamap struct {
-	lock   sync.Mutex //锁
-	days   int        //保留几天数据
-	data   map[string][]*Info
-	keymap []string
-}
-
-func NewDatamap(days int) *datamap {
-	datelimit = qutil.Float64All(days * 86400)
-	dm := &datamap{sync.Mutex{}, days, map[string][]*Info{}, []string{}}
-	dm.keymap = dm.GetLatelyFiveDay()
-	//初始化加载数据
-	sess := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(sess)
-	it := sess.DB(mgo.DbName).C(extract).Find(nil).Sort("-_id").Iter()
-	now1 := time.Now().Unix()
-	n, continuSum := 0, 0
-	for tmp := make(map[string]interface{}); it.Next(tmp); n++ {
-		//
-		if qutil.IntAll(tmp["repeat"]) == 1 || qutil.ObjToString(tmp["subtype"]) == "变更" {
-			continuSum++
-		} else {
-			cm := tmp["comeintime"]
-			comeintime := qutil.Int64All(cm)
-			if qutil.Float64All(now1-comeintime) < datelimit {
-				info := NewInfo(tmp)
-				k := fmt.Sprintf("%s_%s_%s", qutil.FormatDateWithObj(&cm, qutil.Date_yyyyMMdd), info.subtype, info.area)
-				data := dm.data[k]
-				if data == nil {
-					data = []*Info{}
-					//log.Println(k)
-				}
-				data = append(data, info)
-				dm.data[k] = data
-			} else {
-				break
-			}
-		}
-		if n%5000 == 0 {
-			log.Println("current n:", n, continuSum)
-		}
-		tmp = make(map[string]interface{})
-	}
-	log.Println("load data:", n)
-	//启动定时任务
-	now := time.Now()
-	t2 := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.Local)
-	go time.AfterFunc(time.Duration(int64(t2.Unix()-now.Unix()))*time.Second, func() {
-		//go time.AfterFunc(time.Duration(10)*time.Second, func() {
-		dm.update()
-	})
-	return dm
-}
-func NewInfo(tmp map[string]interface{}) *Info {
-	subtype := qutil.ObjToString(tmp["subtype"])
-	area := qutil.ObjToString(tmp["area"])
-	if area == "A" {
-		area = "全国"
-	}
-	info := &Info{}
-	info.id = qutil.BsonIdToSId(tmp["_id"])
-	info.title = qutil.ObjToString(tmp["title"])
-	info.area = area
-	info.subtype = subtype
-	info.buyer = qutil.ObjToString(tmp["buyer"])
-	info.projectname = qutil.ObjToString(tmp["projectname"])
-	info.ContainSpecialWord = FilterRegexp.MatchString(info.projectname) || FilterRegexp.MatchString(info.title)
-	info.projectcode = qutil.ObjToString(tmp["projectcode"])
-	info.city = qutil.ObjToString(tmp["city"])
-	info.agency = qutil.ObjToString(tmp["agency"])
-	//info.winner = qutil.ObjToString(tmp["winner"])
-	info.publishtime = qutil.Int64All(tmp["publishtime"])
-	return info
-}
-
-func (d *datamap) check(info *Info) (b bool, id string) {
-	d.lock.Lock()
-	defer d.lock.Unlock()
-	keys := []string{}
-	for _, k := range d.keymap {
-		keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, info.area))
-		if info.area != "全国" { //这个后续可以不要
-			keys = append(keys, fmt.Sprintf("%s_%s_%s", k, info.subtype, "全国"))
-		}
-	}
-L:
-	for _, k := range keys {
-		data := d.data[k]
-		if len(data) > 1 { //对比
-			for _, v := range data {
-				if math.Abs(qutil.Float64All(v.publishtime-info.publishtime)) > datelimit {
-					continue
-				}
-				if v.agency != "" && info.agency != "" && v.agency != info.agency {
-					continue
-				}
-				n := 0
-				if v.buyer != "" && v.buyer == info.buyer {
-					n++
-				}
-				if v.projectname != "" && v.projectname == info.projectname {
-					n++
-				}
-				if !info.ContainSpecialWord && n > 1 {
-					b = true
-					id = v.id
-					break L
-				} else if v.projectcode != "" && v.projectcode == info.projectcode {
-					n++
-				}
-				if !info.ContainSpecialWord && n > 1 || n > 2 {
-					b = true
-					id = v.id
-					break L
-				}
-				//标题长度大于10且相等即为重复
-				//				if len([]rune(info.title)) > 10 && v.title == info.title {
-				//					b = true
-				//					id = v.id
-				//					break L
-				//				}
-				//标题长度大于10且包含关系+buyer/projectname/projectcode/city（全国/A的只判断包含关系即可）相等即为重复
-				if len([]rune(info.title)) > 10 && len([]rune(v.title)) > 10 && (strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title)) {
-					if info.area == "全国" || n > 0 || info.city == v.city {
-						b = true
-						id = v.id
-						break L
-					}
-				}
-			}
-		}
-	}
-	if !b {
-		k := fmt.Sprintf("%s_%s_%s", time.Now().Format(qutil.Date_yyyyMMdd), info.subtype, info.area)
-		data := d.data[k]
-		if data == nil {
-			data = []*Info{info}
-		} else {
-			data = append(data, info)
-		}
-		d.data[k] = data
-	}
-	return
-}
-
-func (d *datamap) update() {
-	//每天0点清除历史数据
-	d.lock.Lock()
-	now, now1 := time.Now(), time.Now()
-	t2 := time.Date(now1.Year(), now1.Month(), now1.Day()+1, 0, 0, 0, 0, time.Local)
-	date := now.AddDate(0, 0, -d.days).Format(qutil.Date_yyyyMMdd)
-	all, all1 := 0, 0
-	for k, v := range d.data {
-		all += len(v)
-		if strings.HasPrefix(k, date) {
-			delete(d.data, k)
-		}
-	}
-	for _, v := range d.data {
-		all1 += len(v)
-	}
-	log.Println("更新前后数据:", all, all1)
-	d.keymap = d.GetLatelyFiveDay()
-	d.lock.Unlock()
-	time.AfterFunc(time.Duration(int64(t2.Unix()-now1.Unix()))*time.Second, d.update)
-}
-
-func (d *datamap) GetLatelyFiveDay() []string {
-	array := make([]string, d.days)
-	now := time.Now()
-	for i := 0; i < d.days; i++ {
-		array[i] = now.Format(qutil.Date_yyyyMMdd)
-		now = now.AddDate(0, 0, -1)
-	}
-	return array
-}

udpfilterdup/src/main.go

@@ -13,7 +13,6 @@ import (
 	"net"
 	"os"
 	"qfw/util"
-	"qfw/util/mongodb"
 	"regexp"
 	"sync"
 	"time"
@@ -22,7 +21,7 @@ import (
 var (
 	Sysconfig map[string]interface{} //配置文件
 	mconf     map[string]interface{} //mongodb配置信息
-	mgo       *mongodb.MongodbSim    //mongodb操作对象
+	mgo       *MongodbSim            //mongodb操作对象
 	extract   string
 	udpclient mu.UdpClient             //udp对象
 	nextNode  []map[string]interface{} //下节点数组
@@ -30,22 +29,20 @@ var (
 	DM        *datamap                 //
 	HM        *historymap              //判重数据
 	lastid    = ""
-	/*
-		5da3f31aa5cb26b9b798d3aa
-	*/
+
 	//正则筛选相关
 	FilterRegTitle   = regexp.MustCompile("^_$")
 	FilterRegTitle_1 = regexp.MustCompile("^_$")
 	FilterRegTitle_2 = regexp.MustCompile("^_$")
 
-	isMerger bool                              //是否合并
-	threadNum int								   //线程数量
-	SiteMap  map[string]map[string]interface{} //站点map
-	idtype, sid, eid string //测试人员判重使用
+	isMerger         bool                              //是否合并
+	Is_Sort          bool                              //是否排序
+	threadNum        int                               //线程数量
+	SiteMap          map[string]map[string]interface{} //站点map
+	idtype, sid, eid string                            //测试人员判重使用
 )
 
 func init() {
-
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
@@ -55,14 +52,13 @@ func init() {
 	util.ReadConfig(&Sysconfig)
 	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
 	mconf = Sysconfig["mongodb"].(map[string]interface{})
-	mgo = &mongodb.MongodbSim{
+	mgo = &MongodbSim{
 		MongodbAddr: mconf["addr"].(string),
 		DbName:      mconf["db"].(string),
 		Size:        util.IntAllDef(mconf["pool"], 10),
 	}
-	extract = mconf["extract"].(string)
 	mgo.InitPool()
-
+	extract = mconf["extract"].(string)
 	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
 	//加载数据
 	DM = NewDatamap(dupdays, lastid)
@@ -70,6 +66,7 @@ func init() {
 	FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
 	FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
 	isMerger = Sysconfig["isMerger"].(bool)
+	Is_Sort = Sysconfig["isSort"].(bool)
 	threadNum = util.IntAllDef(Sysconfig["threads"], 1)
 
 	//站点配置
@@ -77,8 +74,8 @@ func init() {
 	SiteMap = make(map[string]map[string]interface{}, 0)
 	start := int(time.Now().Unix())
 	sess_site := mgo.GetMgoConn()
-	defer sess_site.Close()
-	res_site := sess_site.DB(site["dbname"].(string)).C(site["coll"].(string)).Find(nil).Sort("_id").Iter()
+	defer mgo.DestoryMongoConn(sess_site)
+	res_site := sess_site.DB(site["dbname"].(string)).C(site["coll"].(string)).Find(map[string]interface{}{}).Sort("_id").Iter()
 	for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
 		data_map := map[string]interface{}{
 			"area":     util.ObjToString(site_dict["area"]),
@@ -104,14 +101,14 @@ func main() {
 //测试组人员使用
 func mainT() {
 	/*
-	ObjectId("5da3f31aa5cb26b9b798d3aa")
-	ObjectId("5da418c4a5cb26b9b7e3e9a6")
-	ObjectId("5df5071ce9d1f601e495fa54")
-	ObjectId("5e09c05f0cf41612e0626abc")
+		ObjectId("5da3f31aa5cb26b9b798d3aa")
+		ObjectId("5da418c4a5cb26b9b7e3e9a6")
+		ObjectId("5df5071ce9d1f601e495fa54")
+		ObjectId("5e09c05f0cf41612e0626abc")
 	*/
-	//sid = "5df5071ce9d1f601e495fa50"
-	//eid = "5e09c05f0cf41612e0626abc"
-
+	log.Println("测试开始")
+	sid = "5da3f31aa5cb26b9b798d3aa"
+	eid = "5da418c4a5cb26b9b7e3e9a6"
 	mapinfo := map[string]interface{}{}
 	if sid == "" || eid == "" {
 		log.Println("sid,eid参数不能为空")
@@ -120,7 +117,7 @@ func mainT() {
 	mapinfo["gtid"] = sid
 	mapinfo["lteid"] = eid
 	mapinfo["stop"] = "true"
-	task([]byte{}, mapinfo)
+	historyTask([]byte{}, mapinfo)
 	time.Sleep(10 * time.Second)
 }
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
@@ -166,9 +163,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 	fmt.Println("开始数据判重")
 	defer util.Catch()
 	//区间id
-	sess := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(sess)
-	var q map[string]interface{}
+	q := map[string]interface{}{}
 	if idtype == "1" {
 		q = map[string]interface{}{
 			"_id": map[string]interface{}{
@@ -179,17 +174,24 @@ func task(data []byte, mapInfo map[string]interface{}) {
 	} else {
 		q = map[string]interface{}{
 			"_id": map[string]interface{}{
-				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
-				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
 			},
 		}
 	}
-	log.Println(mgo.DbName,extract,q)
-	it := sess.DB(mgo.DbName).C(extract).Find(&q).Sort("publishtime").Iter()
+	log.Println(mgo.DbName, extract, q)
+	sess := mgo.GetMgoConn()
+	defer mgo.DestoryMongoConn(sess)
+
+	//是否排序
+	it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
+	if Is_Sort {
+		it = sess.DB(mgo.DbName).C(extract).Find(&q).Sort("publishtime").Iter()
+	}
+	//it = sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
 	updateExtract := [][]map[string]interface{}{}
-	log.Println("线程数：",threadNum)
+	log.Println("线程数：", threadNum)
 	pool := make(chan bool, threadNum)
-
 	wg := &sync.WaitGroup{}
 	//mapLock := &sync.Mutex{}
 	n, repeateN := 0, 0
@@ -206,7 +208,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			}()
 			info := NewInfo(tmp)
 			//是否为无效数据
-			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
+			if invalidData(info.buyer, info.projectname, info.projectcode, info.contractnumber) {
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
 						"_id": tmp["_id"],
@@ -218,45 +220,45 @@ func task(data []byte, mapInfo map[string]interface{}) {
 					},
 				})
 				if len(updateExtract) > 500 {
-					mgo.UpdateBulk(extract, updateExtract...)
+					mgo.UpSertBulk(extract, updateExtract...)
 					updateExtract = [][]map[string]interface{}{}
 				}
 			} else {
 				b, source, reason := DM.check(info)
 				if b { //有重复,生成更新语句，更新抽取和更新招标
 					repeateN++
-					var is_replace  = false
-					var mergeArr = []int64{} //更改合并数组记录
-					var newData = &Info{}    //更换新的数据池数据
+					var is_replace = false
+					var mergeArr = []int64{}                    //更改合并数组记录
+					var newData = &Info{}                       //更换新的数据池数据
 					var repeat_idMap = map[string]interface{}{} //记录判重的
-					var merge_idMap = map[string]interface{}{} //记录合并的
-					if idtype == "1" { //先临时决定一个id
+					var merge_idMap = map[string]interface{}{}  //记录合并的
+					if idtype == "1" {                          //先临时决定一个id
 						repeat_idMap["_id"] = info.id
 						merge_idMap["_id"] = source.id
 					} else {
-						repeat_idMap["_id"] = util.StringTOBsonId(info.id)
-						merge_idMap["_id"] = util.StringTOBsonId(source.id)
+						repeat_idMap["_id"] = StringTOBsonId(info.id)
+						merge_idMap["_id"] = StringTOBsonId(source.id)
 					}
-					repeat_id:=source.id
+					repeat_id := source.id
 					//以下合并相关
 					if isMerger {
 						basic_bool := basicDataScore(source, info)
 						if basic_bool {
 							//已原始数据为标准 - 对比数据打判重标签-
-							newData, mergeArr,is_replace = mergeDataFields(source, info)
+							newData, mergeArr, is_replace = mergeDataFields(source, info)
 							DM.replaceSourceData(newData, source.id) //替换
 							//对比数据打重复标签的id,原始数据id的记录
 							if idtype == "1" {
 								repeat_idMap["_id"] = info.id
 								merge_idMap["_id"] = source.id
 							} else {
-								repeat_idMap["_id"] = util.StringTOBsonId(info.id)
-								merge_idMap["_id"] = util.StringTOBsonId(source.id)
+								repeat_idMap["_id"] = StringTOBsonId(info.id)
+								merge_idMap["_id"] = StringTOBsonId(source.id)
 							}
 							repeat_id = source.id
 						} else {
 							//已对比数据为标准 ，数据池的数据打判重标签
-							newData, mergeArr,is_replace = mergeDataFields(info, source)
+							newData, mergeArr, is_replace = mergeDataFields(info, source)
 							DM.replaceSourceData(newData, source.id) //替换
 
 							//原始数据打重复标签的id,   对比数据id的记录
@@ -264,25 +266,27 @@ func task(data []byte, mapInfo map[string]interface{}) {
 								repeat_idMap["_id"] = source.id
 								merge_idMap["_id"] = info.id
 							} else {
-								repeat_idMap["_id"] = util.StringTOBsonId(source.id)
-								merge_idMap["_id"] = util.StringTOBsonId(info.id)
+								repeat_idMap["_id"] = StringTOBsonId(source.id)
+								merge_idMap["_id"] = StringTOBsonId(info.id)
 							}
 							repeat_id = info.id
 						}
 
-
-						merge_map := make(map[string]interface{},0)
-						if is_replace {//有过合并-更新数据
+						merge_map := make(map[string]interface{}, 0)
+						if is_replace { //有过合并-更新数据
 
 							merge_map = map[string]interface{}{
 								"$set": map[string]interface{}{
-									"merge":newData.mergemap,
+									"merge": newData.mergemap,
 								},
 							}
 
 							//更新合并后的数据
 							for _, value := range mergeArr {
-								if value == 1 {
+								if value == 0 {
+									merge_map["$set"].(map[string]interface{})["area"] = newData.area
+									merge_map["$set"].(map[string]interface{})["city"] = newData.city
+								} else if value == 1 {
 									merge_map["$set"].(map[string]interface{})["area"] = newData.area
 									merge_map["$set"].(map[string]interface{})["city"] = newData.city
 								} else if value == 2 {
@@ -301,11 +305,11 @@ func task(data []byte, mapInfo map[string]interface{}) {
 									merge_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
 								} else if value == 9 {
 									merge_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
-								}else {
-								}
-
-								if value==0 {
-
+								} else if value == 10 {
+									merge_map["$set"].(map[string]interface{})["publishtime"] = newData.publishtime
+								} else if value == 11 {
+									merge_map["$set"].(map[string]interface{})["agency"] = newData.agency
+								} else {
 								}
 							}
 							//模板数据更新
@@ -316,15 +320,14 @@ func task(data []byte, mapInfo map[string]interface{}) {
 						}
 					}
 
-
 					//重复数据打标签
 					updateExtract = append(updateExtract, []map[string]interface{}{
 						repeat_idMap,
 						map[string]interface{}{
 							"$set": map[string]interface{}{
-								"repeat": 1,
+								"repeat":        1,
 								"repeat_reason": reason,
-								"repeat_id":repeat_id,
+								"repeat_id":     repeat_id,
 							},
 						},
 					})
@@ -333,14 +336,14 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			}
 		}(tmp)
 		if len(updateExtract) > 500 {
-			mgo.UpdateBulk(extract, updateExtract...)
+			mgo.UpSertBulk(extract, updateExtract...)
 			updateExtract = [][]map[string]interface{}{}
 		}
 		tmp = make(map[string]interface{})
 	}
 	wg.Wait()
 	if len(updateExtract) > 0 {
-		mgo.UpdateBulk(extract, updateExtract...)
+		mgo.UpSertBulk(extract, updateExtract...)
 		//mgo.UpdateBulk(bidding, updateBidding...)
 	}
 	log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])
@@ -387,8 +390,8 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 	} else {
 		q = map[string]interface{}{
 			"_id": map[string]interface{}{
-				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
-				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
 			},
 		}
 	}
@@ -397,11 +400,15 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 	minTime, maxTime := int64(0), int64(0)
 	for tmp := make(map[string]interface{}); it.Next(&tmp); {
 		//取出最大最小时间
-		if minTime == 0 || maxTime == 0 &&util.Int64All(tmp["publishtime"])!=0{
-			minTime = util.Int64All(tmp["publishtime"])
-			maxTime = util.Int64All(tmp["publishtime"])
+		info_time:=tmp["comeintime"]
+		if Is_Sort {
+			info_time = tmp["publishtime"]
+		}
+		if minTime == 0 || maxTime == 0 && util.Int64All(info_time) != 0 {
+			minTime = util.Int64All(info_time)
+			maxTime = util.Int64All(info_time)
 		} else {
-			t := util.Int64All(tmp["publishtime"])
+			t := util.Int64All(info_time)
 			if t < minTime && t != 0 {
 				minTime = t
 			}
@@ -411,19 +418,19 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 		}
 	}
 	//时间不正确时
-	if minTime==0&&maxTime==0 {
-		log.Println("段数据区间 publishtime不符合")
+	if minTime == 0 && maxTime == 0 {
+		log.Println("段数据区间 不符合")
 		return
 	}
 	fmt.Println("最小时间==", minTime, "最大时间==", maxTime)
-	gtid,lteid:= util.BsonIdToSId(mapInfo["gtid"].(string)), util.BsonIdToSId(mapInfo["lteid"].(string))
-	fmt.Println(gtid,lteid)
-	HM = NewHistorymap(gtid,lteid, minTime, maxTime)
+	gtid, lteid := util.BsonIdToSId(mapInfo["gtid"].(string)), util.BsonIdToSId(mapInfo["lteid"].(string))
+	fmt.Println(gtid, lteid)
+	HM = NewHistorymap(gtid, lteid, minTime, maxTime)
 	fmt.Println("开始历史数据判重")
 
 	defer util.Catch()
 	//区间id
-	sess_history:= mgo.GetMgoConn()
+	sess_history := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess_history)
 	var q_history map[string]interface{}
 	if idtype == "1" {
@@ -436,15 +443,20 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 	} else {
 		q_history = map[string]interface{}{
 			"_id": map[string]interface{}{
-				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
-				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
+				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
+				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
 			},
 		}
 	}
-	log.Println(mgo.DbName,extract,q_history)
-	it_history := sess_history.DB(mgo.DbName).C(extract).Find(&q_history).Sort("publishtime").Iter()
+	log.Println(mgo.DbName, extract, q_history)
+
+	//是否排序
+	it_history := sess_history.DB(mgo.DbName).C(extract).Find(&q_history).Iter()
+	if Is_Sort {
+		it_history = sess_history.DB(mgo.DbName).C(extract).Find(&q_history).Sort("publishtime").Iter()
+	}
 	updateExtract := [][]map[string]interface{}{}
-	log.Println("线程数：",threadNum)
+	log.Println("线程数：", threadNum)
 	pool := make(chan bool, threadNum)
 	wg := &sync.WaitGroup{}
 	//mapLock := &sync.Mutex{}
@@ -461,7 +473,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 				wg.Done()
 			}()
 			info := NewInfo(tmp)
-			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
+			if invalidData(info.buyer, info.projectname, info.projectcode, info.contractnumber) {
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
 						"_id": tmp["_id"],
@@ -473,7 +485,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 					},
 				})
 				if len(updateExtract) > 500 {
-					mgo.UpdateBulk(extract, updateExtract...)
+					mgo.UpSertBulk(extract, updateExtract...)
 					updateExtract = [][]map[string]interface{}{}
 				}
 			} else {
@@ -496,38 +508,38 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 						})
 					} else {
 						repeateN++
-						var is_replace  = false
-						var mergeArr = []int64{} //更改合并数组记录
-						var newData = &Info{}    //更换新的数据池数据
+						var is_replace = false
+						var mergeArr = []int64{}                    //更改合并数组记录
+						var newData = &Info{}                       //更换新的数据池数据
 						var repeat_idMap = map[string]interface{}{} //记录判重的
-						var merge_idMap = map[string]interface{}{} //记录合并的
-						if idtype == "1" { //先临时决定一个id
+						var merge_idMap = map[string]interface{}{}  //记录合并的
+						if idtype == "1" {                          //先临时决定一个id
 							repeat_idMap["_id"] = info.id
 							merge_idMap["_id"] = source.id
 						} else {
-							repeat_idMap["_id"] = util.StringTOBsonId(info.id)
-							merge_idMap["_id"] = util.StringTOBsonId(source.id)
+							repeat_idMap["_id"] = StringTOBsonId(info.id)
+							merge_idMap["_id"] = StringTOBsonId(source.id)
 						}
-						repeat_id:=source.id
+						repeat_id := source.id
 						//以下合并相关
 						if isMerger {
 							basic_bool := basicDataScore(source, info)
 							if basic_bool {
 								//已原始数据为标准 - 对比数据打判重标签-
-								newData, mergeArr,is_replace = mergeDataFields(source, info)
+								newData, mergeArr, is_replace = mergeDataFields(source, info)
 								DM.replaceSourceData(newData, source.id) //替换
 								//对比数据打重复标签的id,原始数据id的记录
 								if idtype == "1" {
 									repeat_idMap["_id"] = info.id
 									merge_idMap["_id"] = source.id
 								} else {
-									repeat_idMap["_id"] = util.StringTOBsonId(info.id)
-									merge_idMap["_id"] = util.StringTOBsonId(source.id)
+									repeat_idMap["_id"] = StringTOBsonId(info.id)
+									merge_idMap["_id"] = StringTOBsonId(source.id)
 								}
 								repeat_id = source.id
 							} else {
 								//已对比数据为标准 ，数据池的数据打判重标签
-								newData, mergeArr,is_replace = mergeDataFields(info, source)
+								newData, mergeArr, is_replace = mergeDataFields(info, source)
 								DM.replaceSourceData(newData, source.id) //替换
 
 								//原始数据打重复标签的id,   对比数据id的记录
@@ -535,25 +547,27 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 									repeat_idMap["_id"] = source.id
 									merge_idMap["_id"] = info.id
 								} else {
-									repeat_idMap["_id"] = util.StringTOBsonId(source.id)
-									merge_idMap["_id"] = util.StringTOBsonId(info.id)
+									repeat_idMap["_id"] = StringTOBsonId(source.id)
+									merge_idMap["_id"] = StringTOBsonId(info.id)
 								}
 								repeat_id = info.id
 							}
 
-
-							merge_map := make(map[string]interface{},0)
-							if is_replace {//有过合并-更新数据
+							merge_map := make(map[string]interface{}, 0)
+							if is_replace { //有过合并-更新数据
 
 								merge_map = map[string]interface{}{
 									"$set": map[string]interface{}{
-										"merge":newData.mergemap,
+										"merge": newData.mergemap,
 									},
 								}
 
 								//更新合并后的数据
 								for _, value := range mergeArr {
-									if value == 1 {
+									if value == 0 {
+										merge_map["$set"].(map[string]interface{})["area"] = newData.area
+										merge_map["$set"].(map[string]interface{})["city"] = newData.city
+									} else if value == 1 {
 										merge_map["$set"].(map[string]interface{})["area"] = newData.area
 										merge_map["$set"].(map[string]interface{})["city"] = newData.city
 									} else if value == 2 {
@@ -572,11 +586,11 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 										merge_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
 									} else if value == 9 {
 										merge_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
-									}else {
-									}
-
-									if value==0 {
-
+									} else if value == 10 {
+										merge_map["$set"].(map[string]interface{})["publishtime"] = newData.publishtime
+									} else if value == 11 {
+										merge_map["$set"].(map[string]interface{})["agency"] = newData.agency
+									} else {
 									}
 								}
 								//模板数据更新
@@ -587,15 +601,14 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 							}
 						}
 
-
 						//重复数据打标签
 						updateExtract = append(updateExtract, []map[string]interface{}{
 							repeat_idMap,
 							map[string]interface{}{
 								"$set": map[string]interface{}{
-									"repeat": 1,
+									"repeat":        1,
 									"repeat_reason": reason,
-									"repeat_id":repeat_id,
+									"repeat_id":     repeat_id,
 								},
 							},
 						})
@@ -605,14 +618,14 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 			}
 		}(tmp)
 		if len(updateExtract) > 500 {
-			mgo.UpdateBulk(extract, updateExtract...)
+			mgo.UpSertBulk(extract, updateExtract...)
 			updateExtract = [][]map[string]interface{}{}
 		}
 		tmp = make(map[string]interface{})
 	}
 	wg.Wait()
 	if len(updateExtract) > 0 {
-		mgo.UpdateBulk(extract, updateExtract...)
+		mgo.UpSertBulk(extract, updateExtract...)
 		//mgo.UpdateBulk(bidding, updateBidding...)
 	}
 	log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])
@@ -641,21 +654,34 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 }
 
 //合并字段-并更新merge字段的值
-func mergeDataFields(source *Info, info *Info) (*Info, []int64,bool) {
+func mergeDataFields(source *Info, info *Info) (*Info, []int64, bool) {
 
-	//定义一个新的map[string]interface{}{}
-	merge_recordMap := make(map[string]interface{},0)
+	merge_recordMap := make(map[string]interface{}, 0)
 	mergeArr := make([]int64, 0)
 	//是否替换数据了-记录原始的数据
-	is_replace :=false
+	is_replace := false
 	//1、城市
-	if (source.area == "" || source.area == "全国") && info.area != "全国" && info.area != "" {
-		merge_recordMap["area"] = source.area
-		merge_recordMap["city"] = source.city
-		source.area = info.area
-		source.city = info.city
-		mergeArr = append(mergeArr, 1)
-		is_replace = true
+	if source.area == "" || source.area == "全国" {
+		//为空
+		if info.area != "全国" && info.area != "" {
+			merge_recordMap["area"] = source.area
+			merge_recordMap["city"] = source.city
+			source.area = info.area
+			source.city = info.city
+			mergeArr = append(mergeArr, 1)
+			is_replace = true
+		}
+	} else {
+		//不为空-查看站点相关-有值必替换
+		if source.is_site {
+			//是站点替换的城市
+			merge_recordMap["site_area"] = source.area
+			merge_recordMap["site_city"] = source.city
+			mergeArr = append(mergeArr, 0)
+			is_replace = true
+			source.is_site = false
+
+		}
 	}
 	//2、项目名称
 	if source.projectname == "" && info.projectname != "" {
@@ -715,18 +741,32 @@ func mergeDataFields(source *Info, info *Info) (*Info, []int64,bool) {
 		is_replace = true
 	}
 
-	if is_replace {//有过替换更新
+	//10、发布时间
+	if source.publishtime == 0 && info.publishtime != 0 {
+		merge_recordMap["publishtime"] = source.publishtime
+		source.publishtime = info.publishtime
+		mergeArr = append(mergeArr, 10)
+		is_replace = true
+	}
+	//11、代理机构
+	if source.agency == "" && info.agency != "" {
+		merge_recordMap["agency"] = source.agency
+		source.agency = info.agency
+		mergeArr = append(mergeArr, 11)
+		is_replace = true
+	}
+
+	if is_replace { //有过替换更新
 		//总次数+1
-		source.mergemap["total_num"] = util.Int64All(source.mergemap["total_num"])+1
+		source.mergemap["total_num"] = util.Int64All(source.mergemap["total_num"]) + 1
 		merge_recordMap["num"] = util.Int64All(source.mergemap["total_num"])
 		//和哪一个数据id进行非空替换的-记录
-		key:=info.id
+		key := info.id
 		source.mergemap[key] = merge_recordMap
 	}
 
-
 	//以上合并过于简单，待进一步优化
-	return source, mergeArr,is_replace
+	return source, mergeArr, is_replace
 }
 
 //权重评估
@@ -830,7 +870,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if v.buyer != "" {
 		m++
 	}
-	if v.projectcode != ""||v.contractnumber != "" {
+	if v.projectcode != "" || v.contractnumber != "" {
 		m++
 	}
 	if v.budget != 0 {
@@ -861,7 +901,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if info.buyer != "" {
 		n++
 	}
-	if info.projectcode != "" || info.contractnumber != ""{
+	if info.projectcode != "" || info.contractnumber != "" {
 		n++
 	}
 	if info.budget != 0 {
@@ -914,7 +954,7 @@ func invalidData(d1 string, d2 string, d3 string, d4 string) bool {
 	if d4 != "" {
 		n++
 	}
- 	if n == 0 {
+	if n == 0 {
 		return true
 	}
 	return false

udpfilterdup/src/mgo.go

@@ -0,0 +1,315 @@
+package main
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type MgoSess struct {
+	Db     string
+	Coll   string
+	Query  interface{}
+	Sorts  []string
+	fields interface{}
+	limit  int64
+	skip   int64
+	M      *MongodbSim
+}
+
+type MgoIter struct {
+	Cursor *mongo.Cursor
+}
+
+func (mt *MgoIter) Next(result interface{}) bool {
+	if mt.Cursor != nil {
+		if mt.Cursor.Next(nil) {
+			err := mt.Cursor.Decode(result)
+			if err != nil {
+				log.Println("mgo cur err", err.Error())
+				mt.Cursor.Close(nil)
+				return false
+			}
+			return true
+		} else {
+			mt.Cursor.Close(nil)
+			return false
+		}
+	} else {
+		return false
+	}
+
+}
+
+func (ms *MgoSess) DB(name string) *MgoSess {
+	ms.Db = name
+	return ms
+}
+
+func (ms *MgoSess) C(name string) *MgoSess {
+	ms.Coll = name
+	return ms
+}
+
+func (ms *MgoSess) Find(q interface{}) *MgoSess {
+	ms.Query = q
+	return ms
+}
+
+func (ms *MgoSess) Select(fields interface{}) *MgoSess {
+	ms.fields = fields
+	return ms
+}
+
+func (ms *MgoSess) Limit(limit int64) *MgoSess {
+	ms.limit = limit
+	return ms
+}
+func (ms *MgoSess) Skip(skip int64) *MgoSess {
+	ms.skip = skip
+	return ms
+}
+
+func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
+	ms.Sorts = sorts
+	return ms
+}
+
+func (ms *MgoSess) Iter() *MgoIter {
+	it := &MgoIter{}
+	find := options.Find()
+	if ms.skip > 0 {
+		find.SetSkip(ms.skip)
+	}
+	if ms.limit > 0 {
+		find.SetLimit(ms.limit)
+	}
+	find.SetBatchSize(100)
+	if len(ms.Sorts) > 0 {
+		sort := bson.M{}
+		for _, k := range ms.Sorts {
+			switch k[:1] {
+			case "-":
+				sort[k[1:]] = -1
+			case "+":
+				sort[k[1:]] = 1
+			default:
+				sort[k] = 1
+			}
+		}
+		find.SetSort(sort)
+	}
+	if ms.fields != nil {
+		find.SetProjection(ms.fields)
+	}
+	cur, err := ms.M.C.Database(ms.Db).Collection(ms.Coll).Find(ms.M.Ctx, ms.Query, find)
+	if err != nil {
+		log.Println("mgo find err", err.Error())
+	} else {
+		it.Cursor = cur
+	}
+	return it
+}
+
+type MongodbSim struct {
+	MongodbAddr string
+	Size        int
+	//	MinSize     int
+	DbName   string
+	C        *mongo.Client
+	Ctx      context.Context
+	ShortCtx context.Context
+	pool     chan bool
+}
+
+func (m *MongodbSim) GetMgoConn() *MgoSess {
+	//m.Open()
+	ms := &MgoSess{}
+	ms.M = m
+	return ms
+}
+
+func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
+	//m.Close()
+	ms.M = nil
+	ms = nil
+}
+
+func (m *MongodbSim) InitPool() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint16(m.Size))
+	m.pool = make(chan bool, m.Size)
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
+func (m *MongodbSim) Open() {
+	m.pool <- true
+}
+func (m *MongodbSim) Close() {
+	<-m.pool
+}
+
+//批量插入
+func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewUpdateOneModel()
+		write.SetFilter(d[0])
+		write.SetUpdate(d[1])
+		write.SetUpsert(true)
+		writes = append(writes, write)
+	}
+	r, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo upsert error:", e.Error())
+		return nil, false
+	}
+	//	else {
+	//		if r.UpsertedCount != int64(len(doc)) {
+	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
+	//		}
+	//		return true
+	//	}
+	return r.UpsertedIDs, true
+}
+
+//批量插入
+func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	_, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo savebulk error:", e.Error())
+		return false
+	}
+	return true
+}
+
+//保存
+func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.InsertOne(m.Ctx, doc)
+	if err != nil {
+		return nil
+	}
+	return r.InsertedID
+}
+
+//更新by Id
+func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+//删除by id
+func (m *MongodbSim) DeleteById(c, id string) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//通过条件删除
+func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteMany(m.Ctx, query)
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//findbyid
+func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//findone
+func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, query)
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//find
+func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+//创建_id
+func NewObjectId() primitive.ObjectID {
+	return primitive.NewObjectID()
+}
+
+func StringTOBsonId(id string) primitive.ObjectID {
+	objectId, _ := primitive.ObjectIDFromHex(id)
+	return objectId
+}
+
+func BsonTOStringId(id interface{}) string {
+	return id.(primitive.ObjectID).Hex()
+}

udps/main.go

@@ -30,7 +30,7 @@ ObjectId("5da418c4a5cb26b9b7e3e9a6")
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
 	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
-	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
+	flag.StringVar(&endDate, "end", "2019-11-10", "结束日期2006-01-02")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")
 	flag.StringVar(&port, "port", "1488", "dup端口")
 	flag.StringVar(&stype, "stype", "", "stype")