winnertel winnerperson抽取

fullproject/src_v1/config.json

@@ -1,6 +1,7 @@
 {
     "loadStart": 0,
 	"validdays":150,
+    "statusdays": 7,
 	"mongodbServers": "192.168.3.207:27082",
     "mongodbPoolSize": 10,
     "mongodbName": "extract_kf",
@@ -15,12 +16,7 @@
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"
     },
-    "udpport": ":1482",
+    "udpport": ":1182",
     "nextNode": [
-        {
-            "addr": "172.17.145.163",
-            "port": 1483,
-            "memo": "创建项目索引new"
-        }
     ]
 }

fullproject/src_v1/init.go

@@ -5,14 +5,11 @@ import (
 	"math"
 	mu "mfw/util"
 	"qfw/util"
-	//"qfw/util/mongodb"
 	"regexp"
 	"sort"
 	"strings"
 	"sync"
 
-	//"gopkg.in/mgo.v2/bson"
-	//"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
@@ -299,6 +296,7 @@ type ProjectInfo struct {
 	score         int
 	comStr        string
 	resVal, pjVal int
+	IdStatusInfo  map[string]map[string]interface{}
 }
 
 type Site struct {

fullproject/src_v1/project.go

@@ -2,8 +2,6 @@ package main
 
 import (
 	"log"
-	"strconv"
-
 	//	"log"
 	"math"
 	qu "qfw/util"
@@ -177,8 +175,8 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 
 			ex := 0
 			resArr := []*ProjectInfo{}
-			for _, res := range resN{
-				choose, e := CompareStatus(resN[0], info)
+			for _, res := range resN {
+				choose, e := p.CompareStatus(resN[0], info)
 				if !choose {
 					ex = e
 					resArr = append(resArr, res)
@@ -220,7 +218,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 					}
 				}
 				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
-			}else {
+			} else {
 				bFindProject = false
 				findPid = ""
 			}
@@ -473,7 +471,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["bidopentime"] = bidopentime
 	}
 	if thisinfo.ProjectName != "" {
-		set["s_projectname"] = tmp["projectname"] //兼容老版本
+		set["s_projectname"] = tmp["projectname"]
 	}
 	now := time.Now().Unix()
 	set["createtime"] = now
@@ -500,7 +498,9 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 
 	//招标类型
+	p.mapBidLock.Lock()
 	bt := bidtype[thisinfo.SubType]
+	p.mapBidLock.Unlock()
 	if bt == "" {
 		bt = "招标"
 	}
@@ -508,6 +508,18 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	set["bidstatus"] = thisinfo.SubType
 
 	p1, pkg := p.NewCachePinfo(pId, thisinfo, bt)
+	//招标信息是中标或者成交，保存bidstatus、budget、bidamount
+	if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" {
+		p1.IdStatusInfo = map[string]map[string]interface{}{
+			thisinfo.Id: {
+				"projectname": thisinfo.ProjectName,
+				"bidstatus": thisinfo.SubType,
+				"budget": thisinfo.Budget,
+				"bidamount": thisinfo.Bidamount,
+			},
+		}
+	}
+
 	if len(thisinfo.Subscopeclass) > 0 {
 		s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
 		set["s_subscopeclass"] = s_subscopeclass
@@ -581,7 +593,7 @@ func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bs
 func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype string) (ProjectInfo, map[string]interface{}) {
 	pkg := map[string]interface{}{}
 	if thisinfo.HasPackage {
-		pkg, _, _ = PackageFormat(thisinfo, nil)
+		pkg = PackageFormat(thisinfo, nil)
 	}
 	p1 := ProjectInfo{
 		Id:            id,
@@ -603,10 +615,10 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 		FirstTime:     thisinfo.Publishtime,
 		LastTime:      thisinfo.Publishtime,
 		Budget:        thisinfo.Budget,
-		Package: 	   pkg,
+		Package:       pkg,
 		Bidamount:     thisinfo.Bidamount,
-		Bidstatus:	   thisinfo.SubType,
-		Bidtype:	   bidtype,
+		Bidstatus:     thisinfo.SubType,
+		Bidtype:       bidtype,
 	}
 	if thisinfo.LenPTC > 5 {
 		p1.MPC = append(p1.MPC, thisinfo.PTC)
@@ -637,6 +649,8 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.Publishtime > pInfo.LastTime {
 		pInfo.LastTime = thisinfo.Publishtime
 		set["lasttime"] = thisinfo.Publishtime
+		p.mapBidLock.Lock()
+		defer p.mapBidLock.Unlock()
 		bt := bidtype[thisinfo.SubType]
 		if bt != "" {
 			set["bidtype"] = bt
@@ -647,14 +661,27 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			if bidstatus[bs] != "预告" && bidstatus[bs] != "合同" {
 				set["jgtime"] = tmp["publishtime"]
 			}
-		}else if tmp["infoformat"] == 2 {
+		} else if tmp["infoformat"] == 2 {
 			set["bidstatus"] = "拟建"
-		}else if tmp["subytpe"] == "招标" {
+		} else if tmp["subytpe"] == "招标" {
 			set["bidstatus"] = thisinfo.TopType
-		}else {
+		} else {
 			set["bidstatus"] = "其它"
 		}
 	}
+
+	//废标、流标   处理时间
+	if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+		pInfo.FirstTime = thisinfo.Publishtime
+		pInfo.Bidopentime = int64(0)
+		pInfo.LastTime = thisinfo.Publishtime
+
+		set["firsttime"] = thisinfo.Publishtime
+		set["zbtime"] = int64(0)
+		set["publishtime"] = thisinfo.Publishtime
+		set["bidopentime"] = int64(0)
+	}
+
 	//异常标记
 	if ex > 0 {
 		set["exception"] = ex
@@ -700,11 +727,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if pInfo.Buyer == "" {
 		set["buyerclass"] = ""
 	}
-	//if thisinfo.Buyerclass != "" && pInfo.Buyerclass == "" {
-	//	pInfo.Buyerclass = thisinfo.Buyerclass
-	//	set["buyerclass"] = pInfo.Buyerclass
-	//}
-
 
 	//8--代理机构
 	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
@@ -726,15 +748,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		pInfo.Bidopentime = thisinfo.Bidopentime
 		set["bidopentime"] = pInfo.Bidopentime
 	}
-	if thisinfo.Bidamount > 0 {
-		pInfo.Bidamount = pInfo.Bidamount + thisinfo.Bidamount
-		set["bidamount"] = pInfo.Bidamount
-	}
-
-	if thisinfo.Budget > 0 {
-		pInfo.Budget = pInfo.Budget + thisinfo.Budget
-		set["budget"] = pInfo.Budget
-	}
 
 	if len(thisinfo.Topscopeclass) > 0 {
 		sort.Strings(pInfo.Topscopeclass)
@@ -773,12 +786,17 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 
 	if thisinfo.HasPackage { //多包处理
 		set["multipackage"] = 1
-		pkg, _, _:= PackageFormat(thisinfo, pInfo)
+		pkg := PackageFormat(thisinfo, pInfo)
 		pInfo.Package = pkg
 		set["package"] = pInfo.Package
-	}else {
+	} else {
 		set["multipackage"] = 0
 	}
+	//处理多包后，计算预算金额、中标金额
+	CountAmount(pInfo, thisinfo)
+	set["budget"] = pInfo.Budget
+	set["bidamount"] = pInfo.Bidamount
+	//
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
@@ -799,7 +817,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	if len(update) > 0 {
 		updateInfo := []map[string]interface{}{
-			map[string]interface{}{
+			{
 				"_id": pInfo.Id,
 			},
 			update,
@@ -808,7 +826,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 }
 
-
 /**
  *	更新项目时，项目状态的处理
  *	返回是否新增项目，异常标记
@@ -817,33 +834,33 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
  *	3、项目合并时，项目状态是”流标“/”废标“，招标信息状态不是”招标“		异常：2
  *	4、项目合并时，项目状态是”合同“/”其它“，招标信息类型是”结果“		异常：3
  */
-func CompareStatus(project *ProjectInfo, info *Info) (bool, int) {
+func (p *ProjectTask) CompareStatus(project *ProjectInfo, info *Info) (bool, int) {
 	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
 		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
 			return false, 0
-		}else {
+		} else {
 			return true, 0
 		}
-	}else if info.TopType == "结果" {
+	} else if info.TopType == "结果" {
 		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
 			return false, 0
-		}else if project.Bidstatus == info.SubType {
-			key := titleGetPc.FindStringSubmatch(info.ProjectName)
-			if key != nil {
-				return false, 0
-			}else {
+		}  else if project.Bidstatus == info.SubType {
+			//状态一样，根据发布时间判断是否合并
+			if (info.Publishtime - project.FirstTime) > p.statusTime {
+				return true, 0
+			} else {
 				return true, 0
 			}
-		}else if project.Bidstatus == "成交" && info.SubType == "中标" {
+		} else if project.Bidstatus == "成交" && info.SubType == "中标" {
 			return true, 0
-		}else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
+		} else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
 			return false, 2
-		}else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
+		} else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
 			return false, 3
-		}else {
+		} else {
 			return false, 0
 		}
-	}else {
+	} else {
 		return false, 0
 	}
 }
@@ -862,16 +879,16 @@ func ComparePlace(project *ProjectInfo, info *Info) bool {
 	if info.Area == project.Area {
 		if info.City == "" {
 			return false
-		}else if info.City == project.City {
+		} else if info.City == project.City {
 			if info.District == "" || info.District == project.District {
 				return false
-			}else {
+			} else {
 				return true
 			}
-		}else {
+		} else {
 			return true
 		}
-	}else {
+	} else {
 		return true
 	}
 }
@@ -888,75 +905,132 @@ var PackageEle = []string{
 	"bidstatus",
 }
 
-func packageEle(map1 map[string]interface{}) (map[string]interface{}, float64, float64) {
-	budget := 0.0
-	bidamount := 0.0
+func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 	p2 := map[string]interface{}{}
-	for _, k := range PackageEle{
+	for _, k := range PackageEle {
 		if map1[k] != nil {
 			p2[k] = map1[k]
 		}
-		if p2["budget"] != nil {
-			budget = budget + p2["budget"].(float64)
-		}
-		if p2["bidamount"] != nil {
-			bidamount = bidamount + p2["bidamount"].(float64)
+		infoid := p2["infoid"]
+		if infoid == nil {
+			p2["infoid"] = id
 		}
 	}
-	return p2, budget, bidamount
+	return p2
 }
 
-func PackageFormat(info *Info, project *ProjectInfo) (map[string]interface{}, float64, float64) {
-	budget := 0.0
-	bidamount := 0.0
+func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
+	budget := 0
+	bidamount := 0
 	p1 := map[string]interface{}{}
 	if project != nil && project.Package != nil && len(project.Package) > 0 {
-		budget = project.Budget
-		bidamount = project.Bidamount
 		p1 = project.Package
-		ids := p1["ids"].([]string)
-		ids = append(ids, info.Id)
-		p1["ids"] = ids
-		for _, v := range info.Package {
+		for k, v := range info.Package {
 			if v1, ok := v.(map[string]interface{}); ok {
 				v2 := map[string]interface{}{}
-				v2, budget, bidamount = packageEle(v1)
+				v2 = packageEle(v1, info.Id)
 				if v2["bidstatus"] == nil {
 					v2["bidstatus"] = info.SubType
 				}
-				n := 0
+				if isCount(project, v2[""]) {
+					
+				}
+				if v2["budget"] != nil {
+					budget = budget + v2["budget"].(float64)
+				}
+				if v2["bidamount"] != nil {
+					bidamount = bidamount + v2["bidamount"].(float64)
+				}
 				addFlag := false
-				for k, v3 := range p1 {
-					if k == "ids" {
-						continue
-					}
+				for k1, v3 := range p1 {
 					if v4, ok := v3.([]map[string]interface{}); ok {
-						if v4[0]["origin"].(string) == v2["origin"].(string) && v4[0]["name"].(string) == v2["name"].(string) {
+						if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
 							v4 = append(v4, v2)
-							p1[k] = v4
+							p1[k1] = v4
 							addFlag = true
 							break
 						}
 					}
 				}
 				if !addFlag {
-					n++
-					m := len(ids) + n
-					p1[strconv.Itoa(m)] = []map[string]interface{}{v2}
+					p1[k] = []map[string]interface{}{v2}
 				}
 			}
 		}
-	}else {
-		p1["ids"] = []string{info.Id}
+	} else {
 		for k, v := range info.Package {
-			v1 := v.(map[string]interface{})
+			v1, _ := v.(map[string]interface{})
 			p2 := map[string]interface{}{}
-			p2, budget, bidamount = packageEle(v1)
+			p2 = packageEle(v1, info.Id)
 			if p2["bidstatus"] == nil {
 				p2["bidstatus"] = info.SubType
 			}
+			if p2["budget"] != nil {
+				budget = p2["budget"].(float64)
+			}
+			if p2["bidamount"] != nil {
+				bidamount = p2["bidamount"].(float64)
+			}
 			p1[k] = []map[string]interface{}{p2}
 		}
 	}
-	return p1, budget, bidamount
-}
+	info.Budget = budget
+	info.Bidamount = bidamount
+	return p1
+}
+
+//计算预算(budget)、中标金额(bidamount)
+func CountAmount(project *ProjectInfo, info *Info) {
+	//if project!= nil && project.Package != nil && len(project.Package) > 0 {
+	////暂时未考虑太多情况，简单处理
+	//}
+
+	if info.Budget > 0 {
+		//项目中第一条招标信息是否是包/段项目
+		key := titleGetPc.FindStringSubmatch(project.ProjectName)
+		if len(key) > 0 {
+			//判断项目中是否已经计算过该包/段的预算
+			if !isCount(project, info.ProjectName) {
+				project.Budget = project.Budget + info.Budget
+			}
+		}else {
+			if project.Budget < info.Budget {
+				project.Budget = info.Budget
+			}
+		}
+	}
+	//中标、成交、合同     处理中标金额(bidamount)
+	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+		if info.Bidamount > 0 {
+			key := titleGetPc.FindStringSubmatch(project.ProjectName)
+			if len(key) > 0 {
+				if !isCount(project, info.ProjectName) {
+					project.Bidamount = project.Bidamount + info.Bidamount
+				}
+			}else {
+				if project.Bidamount < project.Bidamount {
+					project.Bidamount = info.Bidamount
+				}
+			}
+		}
+	}
+
+	//保存信息到IdStatusInfo
+	project.IdStatusInfo[info.Id] = map[string]interface{}{
+		"projectname": info.ProjectName,
+		"bidstatus": info.SubType,
+		"budget": info.Budget,
+		"bidamount": info.Bidamount,
+	}
+}
+
+func isCount(project *ProjectInfo, infoName string) bool {
+	if project.IdStatusInfo != nil && len(project.IdStatusInfo) > 0 {
+		for _, v := range project.IdStatusInfo{
+			if v["projectname"] == infoName {
+				return true
+			}
+		}
+	}
+	return false
+}

fullproject/src_v1/task.go

@@ -38,11 +38,13 @@ type ProjectTask struct {
 	//采购单位、项目名称、项目编号
 	mapPb, mapPn, mapPc map[string]*Key
 	//流程数据  字段相同，直接合并
-	mapHref map[string]string
+	mapHref     map[string]string
 	mapHrefLock sync.Mutex
 	//站点
-	mapSite map[string]*Site
+	mapSite     map[string]*Site
 	mapSiteLock sync.Mutex
+	//bidtype、bidstatus 锁
+	mapBidLock sync.Mutex
 	//更新或新增通道
 	updatePool chan []map[string]interface{}
 	//savePool   chan map[string]interface{}
@@ -59,6 +61,7 @@ type ProjectTask struct {
 	saveSize  int
 	pici      int64
 	validTime int64
+	statusTime int64
 	//	LockPool     chan *sync.Mutex
 	//	LockPoolLock sync.Mutex
 	//	m1, m23, m4  map[int]int
@@ -86,6 +89,7 @@ func NewPT() *ProjectTask {
 		//updateSign: make(chan bool, 1),
 		coll:      ProjectColl,
 		validTime: int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
+		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 7) * 86400),
 	}
 	return p
 }
@@ -202,7 +206,7 @@ func (p *ProjectTask) clearMem() {
 							}
 						}
 					}
-					for kHref, pid := range p.mapHref{
+					for kHref, pid := range p.mapHref {
 						if pid == k {
 							delete(p.mapHref, kHref)
 						}
@@ -395,11 +399,11 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						if p.currentType == "updateInfo" {
 							//招标信息更改合并
 							p.updateJudge(tmp, info)
-						}else {
+						} else {
 							//普通合并
 							p.CommonMerge(tmp, info)
 						}
-					}else {
+					} else {
 						//信息错误,进行更新
 					}
 				}(tmp)
@@ -465,9 +469,9 @@ var (
 func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
 	if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
 		if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
-			if jsonData != nil && jsonData["projecthref"] != nil {
+			proHref := util.ObjToString(jsonData["projecthref"])
+			if jsonData != nil && proHref != "" {
 				//projectHref字段合并
-				proHref := jsonData["projecthref"].(string)
 				tmp["projecthref"] = proHref
 				p.mapHrefLock.Lock()
 				pid := p.mapHref[proHref]
@@ -487,11 +491,11 @@ func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
 					p.AllIdsMap[id] = &ID{Id: id, P: p1}
 					p.AllIdsMapLock.Unlock()
 				}
-			}else {
+			} else {
 				//项目合并
 				p.startProjectMerge(info, tmp)
 			}
-		}else {
+		} else {
 			//项目合并
 			p.startProjectMerge(info, tmp)
 		}
@@ -593,13 +597,13 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	return thisinfo
 }
 
-func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info)  {
+func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info) {
 	index := -1
 	pInfoId := ""
 	p.AllIdsMapLock.Lock()
-	F:
+F:
 	for k, ID := range p.AllIdsMap {
-		for i, id := range ID.P.Ids{
+		for i, id := range ID.P.Ids {
 			if info.Id == id {
 				pInfoId = k
 				index = i
@@ -614,7 +618,7 @@ func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info)  {
 			p.currentTime = info.Publishtime
 			p.startProjectMerge(info, tmp)
 		}
-	}else {
+	} else {
 		tmpPro := MongoTool.FindById(ProjectColl, pInfoId)
 		infoList := []interface{}(tmpPro["list"].(primitive.A))
 		infoMap := infoList[index].(map[string]interface{})
@@ -627,7 +631,7 @@ func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info)  {
 			p.mapHrefLock.Lock()
 			pid := p.mapHref[proHref]
 			p.mapHrefLock.Unlock()
-			if  pid == pInfoId {
+			if pid == pInfoId {
 				p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
 				return
 			}
@@ -637,7 +641,7 @@ func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info)  {
 			//合并、修改
 			log.Println("合并修改更新", "----------------------------")
 			p.mergeAndModify(pInfoId, index, info, tmp, tmpPro, modifyMap)
-		}else {
+		} else {
 			//修改
 			log.Println("修改更新", "----------------------------")
 			p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
@@ -659,7 +663,7 @@ var Elements = []string{
 
 /**
 判断修改的字段是否是影响合并流程的要素字段
- */
+*/
 func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
 	modifyMap := map[string]interface{}{}
 	for k, _ := range tmpPro {
@@ -671,7 +675,7 @@ func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[s
 		}
 	}
 	for k, _ := range modifyMap {
-		for _, str := range Elements{
+		for _, str := range Elements {
 			if k == str {
 				return modifyMap, true
 			}
@@ -682,7 +686,7 @@ func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[s
 }
 
 //补全位置信息
-func (p *ProjectTask) fillInPlace(tmp map[string]interface{})  {
+func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
 	area := util.ObjToString(tmp["area"])
 	city := util.ObjToString(tmp["city"])
 	district := util.ObjToString(tmp["district"])
@@ -690,7 +694,10 @@ func (p *ProjectTask) fillInPlace(tmp map[string]interface{})  {
 		return
 	}
 
-	tmpSite := tmp["site"].(string)
+	tmpSite := util.ObjToString(tmp["site"])
+	if tmpSite == "" {
+		return
+	}
 	p.mapSiteLock.Lock()
 	defer p.mapSiteLock.Unlock()
 	site := p.mapSite[tmpSite]
@@ -704,26 +711,25 @@ func (p *ProjectTask) fillInPlace(tmp map[string]interface{})  {
 			}
 			if area != site.Area {
 				return
-			}else {
+			} else {
 				if city == site.City {
 					if district == "" {
 						tmp["district"] = site.District
 						return
 					}
-				}else if city == "" {
+				} else if city == "" {
 					tmp["city"] = site.City
 					tmp["district"] = site.District
 					return
 				}
 			}
-		}else {
+		} else {
 			tmp["area"] = site.Area
 			tmp["city"] = site.City
 			tmp["district"] = site.District
 			return
 		}
 	}
-
 }
 
 //从数组中删除元素
@@ -734,4 +740,4 @@ func deleteSlice(arr []string, v string) []string {
 		}
 	}
 	return arr
-}
+}

fullproject/src_v1/update.go

@@ -270,7 +270,9 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 	if thisinfo.Publishtime > pInfo.LastTime {
 		pInfo.LastTime = thisinfo.Publishtime
 		set["lasttime"] = thisinfo.Publishtime
+		p.mapBidLock.Lock()
 		bt := bidtype[thisinfo.SubType]
+		p.mapBidLock.Unlock()
 		if bt != "" {
 			set["bidtype"] = bt
 		}
@@ -356,7 +358,7 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 		set["bidamount"] = pInfo.Bidamount
 	}
 
-	if thisinfo.Budget > 0 && pInfo.Budget < 1 { //多包的会有问题，没有进行合计。
+	if thisinfo.Budget > 0 && pInfo.Budget < 1 {
 		pInfo.Budget = thisinfo.Budget
 		set["budget"] = pInfo.Budget
 	}
@@ -398,18 +400,16 @@ func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[st
 
 	if thisinfo.HasPackage {
 		pkg, _, _ := PackageFormat(thisinfo, pInfo)
+		set["multipackage"] = 1
 		pInfo.Package = pkg
+	}else {
+		set["multipackage"] = 0
 	}
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
 	set["pici"] = p.pici
 
-	if thisinfo.HasPackage {
-		set["multipackage"] = 1
-	} else {
-		set["multipackage"] = 0
-	}
 	return set
 }
 

src/jy/extract/extract.go

@@ -282,7 +282,8 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 		subtype = "all"
 	}
 	toMap := qu.ObjToMap(doc["jsondata"])
-	if toMap != nil && len(*toMap) > 0 {
+	//log.Debug("toMap", toMap)
+	if toMap != nil {
 		if (*toMap)["extweight"] == nil {
 			(*toMap)["extweight"] = ju.Config["jsondata_extweight"]
 		}
@@ -1096,6 +1097,7 @@ func extractFromKv(field, fieldname string, blocks []*ju.Block, vc *RuleCore, kv
 						"blocktag":    bl.Classify,
 						"weight":      vv.Weight,
 					})
+					break //暂定取第一个
 				}
 			}
 		}

src/res/fieldscore.json

@@ -200,7 +200,7 @@
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(附件|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(附件|否决原因|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -10
             },
 			{

udpfilterdup/src/config.json

@@ -4,9 +4,9 @@
     "mongodb": {
         "addr": "192.168.3.207:27082",
         "pool": 15,
-        "db": "extract_kf",
-        "extract": "bidding_20190910_01",
-        "extract_copy": "a_testbidding_copy",
+        "db": "zhaolongyue",
+        "extract": "aliyun_0102",
+        "extract_copy": "a_testbidding",
         "bidding": "bidding_126"
     },
     "jkmail": {
@@ -27,6 +27,7 @@
             "memo": "创建招标数据索引"
         }
     ],
+    "isMerger":false,
     "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
     "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包)",
     "specialtitle_2": "项目（[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]）",

udpfilterdup/src/datamap.go

@@ -279,31 +279,23 @@ L:
 				if info.subtype==v.subtype {
 					//站点配置--
 					if info.site!="" {
-						dict := SiteMap[info.site].(map[string]string)
+						dict := SiteMap[info.site]
+
 						if dict!=nil{
 							//临时改变--具体值
 							if info.area=="全国" &&dict["area"]!="" {
-								info.area = dict["area"]
-								info.city = dict["city"]
+								info.area = qutil.ObjToString(dict["area"])
+								info.city = qutil.ObjToString(dict["city"])
 							}else {
 								if info.city=="" &&dict["city"]!="" {
-									info.area = dict["area"]
-									info.city = dict["city"]
+									info.area = qutil.ObjToString(dict["area"])
+									info.city = qutil.ObjToString(dict["city"])
 								}
 							}
 						}
 					}
 
-					//前置条件2个不重复  一个重复
-					if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-						continue
-					}
-					if info.buyer != "" &&v.buyer == info.buyer {
-						//满足标题
-						if len([]rune(v.title)) >= 10 && len([]rune(info.title)) >= 10 && v.title != info.title && (info.specialWord || v.specialWord) {
-							continue
-						}
-					}
+					//前置条件1  	站点相关
 					if info.site!=""&&info.site==v.site{
 						if info.href!=""&&info.href==v.href {
 							reason = "href相同"
@@ -312,7 +304,34 @@ L:
 							reasons = reason
 							break L
 						}
+						if info.href!=""&&info.href!=v.href {
+							continue
+						}
+					}
+
+					//前置条件2  标题相关 - 有且一个关键词
+					if ((info.titleSpecialWord&&!v.titleSpecialWord)||(info.specialWord&&!v.specialWord))&&
+						info.title!=v.title&&v.title!=""&&info.title!="" {
+						continue
+					}
+
+					//前置条件3 	标题相关 - 均含有关键词
+					if ((info.titleSpecialWord&&v.titleSpecialWord)||(info.specialWord&&v.specialWord))&&
+						len([]rune(v.title))>10 && len([]rune(info.title))>10&&v.title!=""&&info.title!=""{
+						if !(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+							continue //无包含关系
+						}
+						if strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title) {
+							reason = "标题关键词且包含关系"
+							b = true
+							source = v
+							reasons = reason
+							break L
+						}
 					}
+
+
+
 					//代理机构相同-非空相等
 					if v.agency != "" && info.agency != "" && v.agency == info.agency {
 						reason = reason + "同机构-"
@@ -398,31 +417,23 @@ L:
 				if info.subtype==v.subtype {
 					//站点配置--
 					if info.site!="" {
-						dict := SiteMap[info.site].(map[string]string)
+						dict := SiteMap[info.site]
+
 						if dict!=nil{
 							//临时改变--具体值
 							if info.area=="全国" &&dict["area"]!="" {
-								info.area = dict["area"]
-								info.city = dict["city"]
+								info.area = qutil.ObjToString(dict["area"])
+								info.city = qutil.ObjToString(dict["city"])
 							}else {
 								if info.city=="" &&dict["city"]!="" {
-									info.area = dict["area"]
-									info.city = dict["city"]
+									info.area = qutil.ObjToString(dict["area"])
+									info.city = qutil.ObjToString(dict["city"])
 								}
 							}
 						}
 					}
 
-					//前置条件2个不重复  一个重复
-					if info.titleSpecialWord&&info.title!=v.title&&v.title!="" {
-						continue
-					}
-					if info.buyer != "" &&v.buyer == info.buyer {
-						//满足标题
-						if len([]rune(v.title)) >= 10 && len([]rune(info.title)) >= 10 && v.title != info.title && (info.specialWord || v.specialWord) {
-							continue
-						}
-					}
+					//前置条件1  	站点相关
 					if info.site!=""&&info.site==v.site{
 						if info.href!=""&&info.href==v.href {
 							reason = "href相同"
@@ -431,7 +442,34 @@ L:
 							reasons = reason
 							break L
 						}
+						if info.href!=""&&info.href!=v.href {
+							continue
+						}
 					}
+
+					//前置条件2  标题相关 - 有且一个关键词
+					if ((info.titleSpecialWord&&!v.titleSpecialWord)||(info.specialWord&&!v.specialWord))&&
+						info.title!=v.title&&v.title!=""&&info.title!="" {
+						continue
+					}
+
+					//前置条件3 	标题相关 - 均含有关键词
+					if ((info.titleSpecialWord&&v.titleSpecialWord)||(info.specialWord&&v.specialWord))&&
+						len([]rune(v.title))>10 && len([]rune(info.title))>10&&v.title!=""&&info.title!=""{
+						if !(strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title)) {
+							continue //无包含关系
+						}
+						if strings.Contains(v.title, info.title)||strings.Contains(info.title, v.title) {
+							reason = "标题关键词且包含关系"
+							b = true
+							source = v
+							reasons = reason
+							break L
+						}
+					}
+
+
+
 					//代理机构相同-非空相等
 					if v.agency != "" && info.agency != "" && v.agency == info.agency {
 						reason = reason + "同机构-"
@@ -610,7 +648,17 @@ func quickHeavyMethodOne(v *Info ,info *Info) bool {
 			return false
 		}
 	}else {
-
+		//招标结果
+		if tenderRepeat_A(v,info) {
+			if tenderRepeat_C(v,info) {
+				return false
+			}else {
+				reason = reason+"---类别空-招标类"
+				return true
+			}
+		}else {
+			return false
+		}
 	}
 
 	return false
@@ -662,7 +710,17 @@ func quickHeavyMethodTwo(v *Info ,info *Info) bool {
 				return false
 			}
 		}else {
-
+			//招标结果
+			if tenderRepeat_B(v,info) {
+				if tenderRepeat_C(v,info) { //有不同
+					return false
+				}else {
+					reason = reason+"---类别空-招标类"
+					return true
+				}
+			}else{
+				return false
+			}
 		}
 	}
 
@@ -679,6 +737,10 @@ func quickHeavyMethodTwo(v *Info ,info *Info) bool {
 			return false
 		}
 	}
+
+
+
+
 	return false
 }
 
@@ -720,7 +782,7 @@ func tenderRepeat_A(v *Info ,info *Info) bool {
 	}
 
 	if (p1&&p2&&p3)||(p1&&p2&&p4)||(p1&&p2&&p9)||
-		(p1&&p2&&p10)||(p1&&p3&&p9)||(p1&&p3&&p10)||
+		(p1&&p2&&p10)||(p1&&p2&&p11)||(p1&&p3&&p9)||(p1&&p3&&p10)||
 		(p1&&p4&&p9)||(p1&&p4&&p10)||(p2&&p3&&p4)||
 		(p2&&p3&&p9)||(p2&&p3&&p10)||(p2&&p3&&p11)||
 		(p2&&p4&&p9)||(p2&&p4&&p10)||(p2&&p4&&p11)||
@@ -785,11 +847,6 @@ func tenderRepeat_C(v *Info ,info *Info) bool {
 	if v.agencyaddr!=""&&info.agencyaddr!=""&&v.agencyaddr!=info.agencyaddr {
 		return true
 	}
-	if info.specialWord||v.specialWord||info.titleSpecialWord||v.titleSpecialWord{
-		return true
-	}
-
-
 
 	return false
 }
@@ -800,7 +857,7 @@ func winningRepeat_A(v *Info ,info *Info) bool {
 	var ss string
 	p1,p2,p3,p5,p6,p11 := false,false,false,false,false,false
 	if v.projectname!=""&&v.projectname==info.projectname {
-		ss = ss+"p1(标题)-"
+		ss = ss+"p1(项目名称)-"
 		p1 = true
 	}
 	if v.buyer!=""&&v.buyer==info.buyer {
@@ -884,10 +941,6 @@ func winningRepeat_C(v *Info ,info *Info) bool {
 	}
 	//原始地址...
 
-	if info.specialWord||v.specialWord||info.titleSpecialWord||v.titleSpecialWord{
-		return true
-	}
-
 	return false
 }
 

udpfilterdup/src/main.go

@@ -6,8 +6,8 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
-	"gopkg.in/mgo.v2/bson"
 	"log"
 	mu "mfw/util"
 	"net"
@@ -25,9 +25,7 @@ var (
 	Sysconfig    map[string]interface{} //配置文件
 	mconf        map[string]interface{} //mongodb配置信息
 	mgo          *mongodb.MongodbSim    //mongodb操作对象
-	siteMgo             *mongodb.MongodbSim
-	//mgoTest          *mongodb.MongodbSim    //mongodb操作对象
-
+	//siteMgo      *mongodb.MongodbSim
 	extract      string
 	extract_copy string
 	bidding      string
@@ -36,27 +34,23 @@ var (
 	dupdays      = 5                      //初始化判重范围
 	DM           *datamap                 //
 	HM           *historymap                 //判重数据
-	lastid       = "5d767728a5cb26b9b7748868"
-	//ObjectId("5d767728a5cb26b9b7748868")
+	lastid       = ""
 	//正则筛选相关
 	FilterRegTitle = regexp.MustCompile("^_$")
 	FilterRegTitle_1 = regexp.MustCompile("^_$")
 	FilterRegTitle_2 = regexp.MustCompile("^_$")
 
-
-
-
-	SiteMap  map[string]interface{} //站点map
+	isMerger bool //是否合并
+	SiteMap  map[string]map[string]interface{} //站点map
 )
 
 func init() {
-	//flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
-	//flag.Parse()
+	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
+	flag.Parse()
 	//172.17.145.163:27080
 	util.ReadConfig(&Sysconfig)
 	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
 	mconf = Sysconfig["mongodb"].(map[string]interface{})
-
 	mgo = &mongodb.MongodbSim{
 		MongodbAddr: mconf["addr"].(string),
 		DbName:      mconf["db"].(string),
@@ -64,184 +58,43 @@ func init() {
 	}
 	extract = mconf["extract"].(string)
 	extract_copy = mconf["extract_copy"].(string)
-	//bidding = mconf["bidding"].(string)
 	mgo.InitPool()
 
 
-	//测试需临时注释
+	//测试可以临时注释
 	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
 	//加载数据
 	DM = NewDatamap(dupdays, lastid)
 	FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
 	FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
 	FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
+	isMerger = Sysconfig["isMerger"].(bool)
 
-	//站点相关数据库
-	mongodb.InitMongodbPool(5, "192.168.3.207:27082", "")
-
-	siteMgo = &mongodb.MongodbSim{
-		MongodbAddr: "192.168.3.207:27082",
-		Size:        5,
-		DbName:      "zhaolongyue",
-	}
-	siteMgo.InitPool()
-
-
-	SiteMap = make(map[string]interface{},0)
 
+	//配置站点Map
+	SiteMap = make(map[string]map[string]interface{},0)
 	start := int(time.Now().Unix())
 	//站点配置
-	sess_site := siteMgo.GetMgoConn()
+	sess_site := mgo.GetMgoConn()
 	defer sess_site.Close()
 	res_site := sess_site.DB("zhaolongyue").C("site").Find(nil).Sort("_id").Iter()
 	for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
-			data_map := map[string]string{
+			data_map := map[string]interface{}{
 				"area":util.ObjToString(site_dict["area"]),
 				"city":util.ObjToString(site_dict["city"]),
 				"district":util.ObjToString(site_dict["district"]),
+				"sitetype":util.ObjToString(site_dict["sitetype"]),
+				"level":util.ObjToString(site_dict["level"]),
 			}
-		SiteMap[site_dict["site"].(string)]= data_map
+		SiteMap[util.ObjToString(site_dict["site"])]= data_map
 	}
-	
 	fmt.Printf("用时：%d秒,%d个",int(time.Now().Unix())-start,len(SiteMap))
 
 
 }
 
-//新增一个方法 判断
-func mainTest()  {
-
-	//log.Println("1")
-	//代码copy数据
-	//sessTest :=mgoTest.GetMgoConn()
-	//defer sessTest.Close()
-	//
-	//sess := mgo.GetMgoConn()
-	//defer sess.Close()
-	//
-	////var arr []map[string]interface{}
-	//
-	//res_test := sessTest.DB("qfw").C("bidding").Find(mongodb.ObjToMQ(`{"comeintime":{"$gte": 1571025600, "$lte": 1571976000}}`, true)).Iter()
-	//res :=sess.DB("extract_kf").C("a_testbidding")
-	//5
-	//
-	//
-	//
-	//
-	//i:=0
-	//for dict := make(map[string]interface{}); res_test.Next(&dict); i++{
-	//
-	//	//插入
-	//	if i%2000==0 {
-	//		log.Println("当前：",i)
-	//	}
-	//	res.Insert(dict)
-	//	//if len(arr)>=500 {
-	//	//	arr = make([]map[string]interface{},0)
-	//	//}else {
-	//	//	arr = append(arr,dict)
-	//	//}
-	//}
-
-
-
-	sess := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(sess)
-	res_copy := sess.DB("extract_kf").C(extract_copy).Find(nil).Iter()
-
-	m1 :=map[string]int{} //老版本
-	m2 :=map[string]int{} //新版本
-
-	i:=0
-	j:=0
-	for v1 := make(map[string]interface{}); res_copy.Next(&v1); i++{
-		if i%2000==0 {
-			log.Println("当前i：",i)
-		}
-		m1[(v1["_id"].(bson.ObjectId).Hex())]= util.IntAll(v1["repeat"])
-	}
-
-	sesss := mgo.GetMgoConn()
-	defer mgo.DestoryMongoConn(sesss)
-	res := sesss.DB("extract_kf").C(extract).Find(nil).Iter()
-
-
-	for v2 := make(map[string]interface{}); res.Next(&v2); j++{
-		if j%2000==0 {
-			log.Println("当前j：",j)
-		}
-		m2[(v2["_id"].(bson.ObjectId).Hex())]= util.IntAll(v2["repeat"])
-	}
-
-	fmt.Println(len(m1),len(m2))
-	n1:=0
-	n2:=0
-	n3:=0
-	n4:=0
-	n5:=0
-	n6:=0
-
-	var arr1 []string
-	var arr2 []string
-	for k,v:=range m1{
-
-		if m2[k]==1&&v==0{//0:1
-			n1++
-			arr2 = append(arr2,fmt.Sprintf("目标_id:%s",k))
-		}
-		if m2[k]==0&&v==1{ //1:0
-			n2++
-			arr1 = append(arr1,fmt.Sprintf("目标_id:%s",k))
-		}
-		if m2[k]==0&&v==0{ //0:0
-			n3++
-		}
-		if m2[k]==1&&v==1{//1:1
-			n4++
-		}
-		if m2[k]==-1&&v==0{ //0:-1
-			n5++
-		}
-		if m2[k]==-1&&v==1{//1:-1
-			n6++
-		}
-
-	}
-	//打印 1：0情况    66989；
-	mm:=0
-	for _,v:=range arr1 {
-		mm++
-		if mm%222==0 {
-			log.Println(v)
-		}
-	}
-
-	log.Println("分割线---------------")
-	log.Println("分割线---------------")
-
-
-	//打印 0：1情况  8729
-	nn:=0
-	for _,v:=range arr2 {
-		nn++
-		if nn%30==0 {
-			log.Println(v)
-		}
-	}
-
-	log.Println("V1 0:1---",n1)
-	log.Println("V1 1:0---",n2)
-	log.Println("V1 0:0---",n3)
-	log.Println("V1 1:1---",n4)
-	log.Println("V1 0:-1---",n5)
-	log.Println("V1 1:-1---",n6)
-
-}
-
-
 
 func main() {
-
 	go checkMapJob()
 
 	updport := Sysconfig["udpport"].(string)
@@ -264,12 +117,17 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
 		} else if mapInfo != nil {
 
-			//更新流程
-			go historyTask(data,mapInfo)
-
-
-			//判重流程
-			//go task(data, mapInfo)
+			taskType:= util.ObjToString(mapInfo["stype"])
+			if taskType == "historyTask" {
+				//更新流程
+				go historyTask(data,mapInfo)
+			}else if taskType == "normalTask" {
+				//判重流程
+				go task(data, mapInfo)
+			}else {
+				//其他
+				go task(data, mapInfo)
+			}
 
 			key, _ := mapInfo["key"].(string)
 			if key == "" {
@@ -330,7 +188,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 					},
 					map[string]interface{}{
 						"$set": map[string]interface{}{
-							"repeat":   -1,
+							"repeat":-1,
 						},
 					},
 				})
@@ -349,59 +207,69 @@ func task(data []byte, mapInfo map[string]interface{}) {
 					var mergeArr  = []int64{} 	//更改合并数组记录
 					var newData  = &Info{}		//更换新的数据池数据
 					var id_map  = map[string]interface{}{}
-					repeat_id := ""
-
-					//合并操作--评功权重打分-合并完替换原始数据池
-					basic_bool := basicDataScore(source,info)
-					if basic_bool {
-						//已原始数据为标准-对比数据打判重标签
-						newData,mergeArr= mergeDataFields(source,info)
-						DM.replaceSourceData(newData,source.id) //替换
-						id_map["_id"]= util.StringTOBsonId(source.id)
-						repeat_id = source.id
-					}else {
-						//已对比数据为标准 ，数据池的数据打判重标签
-						newData,mergeArr= mergeDataFields(info,source)
-						DM.replaceSourceData(newData,source.id)//替换
-						id_map["_id"]= util.StringTOBsonId(info.id)
-						repeat_id = info.id
+					repeat_id := source.id
+					id_map["_id"]= util.StringTOBsonId(info.id)
+
+					if isMerger{
+						//需要合并相关操作
+						//合并操作--评功权重打分-合并完替换原始数据池
+						basic_bool := basicDataScore(source,info)
+						if basic_bool {
+							//已原始数据为标准-对比数据打判重标签
+							newData,mergeArr= mergeDataFields(source,info)
+							DM.replaceSourceData(newData,source.id) //替换
+							id_map["_id"]= util.StringTOBsonId(source.id)
+							repeat_id = source.id
+						}else {
+							//已对比数据为标准 ，数据池的数据打判重标签
+							newData,mergeArr= mergeDataFields(info,source)
+							DM.replaceSourceData(newData,source.id)//替换
+							id_map["_id"]= util.StringTOBsonId(info.id)
+							repeat_id = info.id
+						}
 					}
 
+
+
 					var update_map  = map[string]interface{}{
 						"$set": map[string]interface{}{
-							"reason":reason,
-							"repeat":"1",
+							"repeat_reason":reason,
+							"repeat":1,
 							"repeatid":repeat_id,
 						},
 					}
 
-					//合并记录
-					if len(newData.mergemap)>0 {
-						update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
-					}
+					if isMerger {
+						//合并记录
+						if len(newData.mergemap)>0 {
+							update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
+							//fmt.Println("合并长度：",len(newData.mergemap))
+						}
 
-					//更新合并后的数据
-					for _,value :=range mergeArr {
-						if value==1 {
-							update_map["$set"].(map[string]interface{})["area"] = newData.area
-							update_map["$set"].(map[string]interface{})["city"] = newData.city
-						}else if value==2 {
-							update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
-						}else if value==3 {
-							update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
-						}else if value==4 {
-							update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
-						}else if value==5 {
-							update_map["$set"].(map[string]interface{})["budget"] = newData.budget
-						}else if value==6 {
-							update_map["$set"].(map[string]interface{})["winner"] = newData.winner
-						}else if value==7 {
-							update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
-						}else if value==8 {
-							update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-						}else {
+						//更新合并后的数据
+						for _,value :=range mergeArr {
+							if value==1 {
+								update_map["$set"].(map[string]interface{})["area"] = newData.area
+								update_map["$set"].(map[string]interface{})["city"] = newData.city
+							}else if value==2 {
+								update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
+							}else if value==3 {
+								update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
+							}else if value==4 {
+								update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
+							}else if value==5 {
+								update_map["$set"].(map[string]interface{})["budget"] = newData.budget
+							}else if value==6 {
+								update_map["$set"].(map[string]interface{})["winner"] = newData.winner
+							}else if value==7 {
+								update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
+							}else if value==8 {
+								update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
+							}else {
 
+							}
 						}
+
 					}
 
 					//构建数据库更新用到的
@@ -552,7 +420,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 							map[string]interface{}{
 								"$set": map[string]interface{}{
 									"repeat":   0,
-									"repeatid": "-1",
+									"repeatid": -2,
 								},
 							},
 						})
@@ -566,60 +434,71 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 						var mergeArr  = []int64{} 	//更改合并数组记录
 						var newData  = &Info{}		//更换新的数据池数据
 						var id_map  = map[string]interface{}{}
-						repeat_id := ""
+						repeat_id := source.id
+						id_map["_id"]= util.StringTOBsonId(info.id)
 
-						//合并操作--评功权重打分-合并完替换原始数据池
-						basic_bool := basicDataScore(source,info)
-						if basic_bool {
-							//已原始数据为标准-对比数据打判重标签
-							newData,mergeArr= mergeDataFields(source,info)
-							DM.replaceSourceData(newData,source.id) //替换。
-							id_map["_id"]= util.StringTOBsonId(source.id)
-							repeat_id = source.id
-						}else {
-							//已对比数据为标准 ，数据池的数据打判重标签
-							newData,mergeArr= mergeDataFields(info,source)
-							DM.replaceSourceData(newData,source.id)//替换
-							id_map["_id"]= util.StringTOBsonId(info.id)
-							repeat_id = info.id
+						if isMerger{
+							//需要合并相关操作
+							//合并操作--评功权重打分-合并完替换原始数据池
+							basic_bool := basicDataScore(source,info)
+							if basic_bool {
+								//已原始数据为标准-对比数据打判重标签
+								newData,mergeArr= mergeDataFields(source,info)
+								DM.replaceSourceData(newData,source.id) //替换
+								id_map["_id"]= util.StringTOBsonId(source.id)
+								repeat_id = source.id
+							}else {
+								//已对比数据为标准 ，数据池的数据打判重标签
+								newData,mergeArr= mergeDataFields(info,source)
+								DM.replaceSourceData(newData,source.id)//替换
+								id_map["_id"]= util.StringTOBsonId(info.id)
+								repeat_id = info.id
+							}
 						}
 
+
+
 						var update_map  = map[string]interface{}{
 							"$set": map[string]interface{}{
-								"reason":reason,
-								"repeat":"1",
+								"repeat_reason":reason,
+								"repeat":1,
 								"repeatid":repeat_id,
 							},
 						}
 
-						//合并记录
-						if len(newData.mergemap)>0 {
-							update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
-						}
-
-						//更新合并后的数据
-						for _,value :=range mergeArr {
-							if value==1 {
-								update_map["$set"].(map[string]interface{})["area"] = newData.area
-								update_map["$set"].(map[string]interface{})["city"] = newData.city
-							}else if value==2 {
-								update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
-							}else if value==3 {
-								update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
-							}else if value==4 {
-								update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
-							}else if value==5 {
-								update_map["$set"].(map[string]interface{})["budget"] = newData.budget
-							}else if value==6 {
-								update_map["$set"].(map[string]interface{})["winner"] = newData.winner
-							}else if value==7 {
-								update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
-							}else if value==8 {
-								update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-							}else {
+						if isMerger {
+							//合并记录
+							if len(newData.mergemap)>0 {
+								update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
+								//fmt.Println("合并长度：",len(newData.mergemap))
+							}
 
+							//更新合并后的数据
+							for _,value :=range mergeArr {
+								if value==1 {
+									update_map["$set"].(map[string]interface{})["area"] = newData.area
+									update_map["$set"].(map[string]interface{})["city"] = newData.city
+								}else if value==2 {
+									update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
+								}else if value==3 {
+									update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
+								}else if value==4 {
+									update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
+								}else if value==5 {
+									update_map["$set"].(map[string]interface{})["budget"] = newData.budget
+								}else if value==6 {
+									update_map["$set"].(map[string]interface{})["winner"] = newData.winner
+								}else if value==7 {
+									update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
+								}else if value==8 {
+									update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
+								}else {
+
+								}
 							}
+
 						}
+
 						//构建数据库更新用到的
 						updateExtract = append(updateExtract, []map[string]interface{}{
 							id_map,
@@ -706,6 +585,8 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 		source.area = info.area
 		source.city = info.city
 		mergeArr = append(mergeArr,1)
+
+		//fmt.Println("合并-城市")
 	}
 	//2、项目名称
 	if source.projectname==""&&info.projectname!=""{
@@ -720,6 +601,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.projectname = info.projectname
 		mergeArr = append(mergeArr,2)
+		//fmt.Println("合并-项目名称")
 	}
 	//3、项目编号
 	if source.projectcode==""&&info.projectcode!=""{
@@ -734,6 +616,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.projectcode = info.projectcode
 		mergeArr = append(mergeArr,3)
+		//fmt.Println("合并-项目标号")
 	}
 	//4、采购单位
 	if source.buyer==""&&info.buyer!=""{
@@ -748,6 +631,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.buyer = info.buyer
 		mergeArr = append(mergeArr,4)
+		//fmt.Println("合并-采购单位")
 	}
 	//5、预算
 	if source.budget==0&&info.budget!=0{
@@ -762,6 +646,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.budget = info.budget
 		mergeArr = append(mergeArr,5)
+		//fmt.Println("合并-预算")
 	}
 	//6、中标单位
 	if source.winner==""&&info.winner!=""{
@@ -776,6 +661,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.winner = info.winner
 		mergeArr = append(mergeArr,6)
+		//fmt.Println("合并-中标单位")
 	}
 	//7、中标金额
 	if source.bidamount==0&&info.bidamount!=0{
@@ -790,6 +676,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.bidamount = info.bidamount
 		mergeArr = append(mergeArr,7)
+		//fmt.Println("合并-中标金额")
 	}
 	//8、开天时间-地点
 	if source.bidopentime==0&&info.bidopentime!=0{
@@ -804,6 +691,7 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 		source.bidopentime = info.bidopentime
 		mergeArr = append(mergeArr,8)
+		//fmt.Println("合并-开标时间")
 	}
 
 	//以上合并过于简单，待进一步优化
@@ -813,6 +701,102 @@ func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
 
 //权重评估
 func basicDataScore(v *Info, info *Info) bool  {
+
+	//权重评估
+	/*
+	网站优先级判定规则：
+    1、中央>省>市>县区
+    2、政府采购>公共资源>采购单位官网>招标代理公司/平台
+	*/
+
+
+	v_score,info_score :=-1,-1
+	dict_v := SiteMap[v.site]
+	dict_info := SiteMap[info.site]
+	//先判断level
+	if dict_v !=nil {
+		v_level := util.ObjToString(dict_v["level"])
+		if v_level =="中央" {
+			v_score = 4
+		}else if v_level =="省级" {
+			v_score = 3
+		}else if v_level =="市级" {
+			v_score = 2
+		}else if v_level =="县区" {
+			v_score = 1
+		}else if v_level =="" {
+		}else {
+			v_score = 0
+		}
+	}
+
+	if dict_info !=nil {
+		info_level := util.ObjToString(dict_info["level"])
+		if info_level =="中央" {
+			info_score = 4
+		}else if info_level =="省级" {
+			info_score = 3
+		}else if info_level =="市级" {
+			info_score = 2
+		}else if info_level =="县区" {
+			info_score = 1
+		}else if info_level == ""{
+
+		}else {
+			v_score = 0
+		}
+	}
+
+	if v_score>info_score{
+		return true
+	}
+	if v_score<info_score{
+		return false
+	}
+
+	//判断sitetype
+	if dict_v !=nil {
+		v_sitetype := util.ObjToString(dict_v["sitetype"])
+		if v_sitetype =="政府采购"||v_sitetype=="政府门户" {
+			v_score = 4
+		}else if v_sitetype =="公共资源" {
+			v_score = 3
+		}else if v_sitetype =="官方网站" {
+			v_score = 2
+		}else if v_sitetype =="社会公共招标平台"||v_sitetype =="企业招标平台" {
+			v_score = 1
+		}else if v_sitetype =="" {
+		}else {
+			v_score = 0
+		}
+	}
+
+	if dict_info !=nil {
+		info_sitetype := util.ObjToString(dict_info["sitetype"])
+		if info_sitetype =="政府采购"||info_sitetype=="政府门户" {
+			info_score = 4
+		}else if info_sitetype =="公共资源" {
+			info_score = 3
+		}else if info_sitetype =="官方网站" {
+			info_score = 2
+		}else if info_sitetype =="社会公共招标平台"||info_sitetype =="企业招标平台" {
+			info_score = 1
+		}else if info_sitetype =="" {
+		}else {
+			info_score = 0
+		}
+	}
+
+	if v_score>info_score{
+		return true
+	}
+	if v_score<info_score{
+		return false
+	}
+
+
+
+	//网站评估
 	m,n:=0,0
 	if v.projectname!="" {m++}
 	if v.buyer!="" {m++}

udps/main.go

@@ -23,23 +23,12 @@ func main() {
 	//2017-06-01,2018-06-01
 	//2018-06-01,2019-02-20
 
-	/*
-	5da3f2c5a5cb26b9b79847fc
-	5db2735ba5cb26b9b7c99c6f   76万
-	*/
 
 	/*
-		9W
-	5d767728a5cb26b9b7748868
-	ObjectId("5d77c881a5cb26b9b7de209d")
-
 
-	//历史中间一段数据
-	ObjectId("5d771e90a5cb26b9b7be7976")
-	ObjectId("5d775be4a5cb26b9b759b5eb")
 	*/
-	flag.StringVar(&sid, "sid", "5d771e90a5cb26b9b7be7976", "开始id")
-	flag.StringVar(&eid, "eid", "5d775be4a5cb26b9b759b5eb", "结束id")
+	flag.StringVar(&sid, "sid", "", "开始id")
+	flag.StringVar(&eid, "eid", "", "结束id")
 	flag.StringVar(&startDate, "start", "", "开始日期2006-01-02")
 	flag.StringVar(&endDate, "end", "", "结束日期2006-01-02")
 	flag.StringVar(&ip, "ip", "127.0.0.1", "ip")