Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/config.json

@@ -1,22 +1,22 @@
 {
-    "loadStart":-1,
+    "loadStart": 0,
 	"validdays":150,
+    "statusdays": 7,
 	"mongodbServers": "192.168.3.207:27082",
     "mongodbPoolSize": 10,
-    "mongodbName": "cesuo",
-	"hints":"_id_1_publishtime_1",
-    "extractColl": "key1_biddingall",
-    "projectColl": "projectset_zjk",
+    "mongodbName": "extract_kf",
+	"hints":"publishtime_1",
+    "extractColl": "jh_info",
+    "projectColl": "jh_project",
+    "backupFlag": true,
+    "backupColl": "jh_project1",
+    "siteColl": "site",
+
     "jkmail": {
-        "to": "zhangjinkun@topnet.net.cn",
+        "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"
     },
-    "udpport": ":1482",
+    "udpport": ":1182",
     "nextNode": [
-        {
-            "addr": "172.17.145.163",
-            "port": 1483,
-            "memo": "创建项目索引new"
-        }
     ]
 }

fullproject/src_v1/init.go

@@ -5,14 +5,11 @@ import (
 	"math"
 	mu "mfw/util"
 	"qfw/util"
-	//"qfw/util/mongodb"
 	"regexp"
 	"sort"
 	"strings"
 	"sync"
 
-	//"gopkg.in/mgo.v2/bson"
-	//"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
@@ -23,7 +20,7 @@ const (
 var (
 	Sysconfig                map[string]interface{} //读取配置文件
 	MongoTool                *MongodbSim            //mongodb连接
-	ExtractColl, ProjectColl string                 //抽取表、项目表
+	ExtractColl, ProjectColl, BackupColl, SiteColl string	//抽取表、项目表、项目快照表、站点表
 	//NextNode                 []interface{}
 )
 
@@ -65,6 +62,8 @@ func init() {
 
 	ExtractColl = Sysconfig["extractColl"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
+	BackupColl = Sysconfig["backupColl"].(string)
+	SiteColl = Sysconfig["siteColl"].(string)
 	//NextNode = Sysconfig["nextNode"].([]interface{})
 	udpport, _ := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
@@ -201,7 +200,6 @@ type KeyMap struct {
 type ID struct {
 	Id   string
 	Lock sync.Mutex
-	pos  int
 	P    *ProjectInfo
 }
 type Key struct {
@@ -225,6 +223,7 @@ type Info struct {
 	Id          string                 `json:"_id"`
 	Href        string                 `json:"href"` //源地址
 	Publishtime int64                  `json:"publishtime"`
+	Comeintime  int64				   `json:"comeintime"`
 	Title       string                 `json:"title"`
 	TopType     string                 `json:"toptype"`
 	SubType     string                 `json:"subtype"`
@@ -237,6 +236,7 @@ type Info struct {
 	Area        string                 `json:"area"`
 	City        string                 `json:"city"`
 	District    string                 `json:"district"`
+	Infoformat  int					   `json:"infoformat"`
 	HasPackage  bool                   // `json:"haspackage"`
 	Package     map[string]interface{} `json:"package"`
 	//PNum          string                 `json:"pnum"`
@@ -282,6 +282,8 @@ type ProjectInfo struct {
 	Area          string             `json:"area"`                    //地区
 	City          string             `json:"city"`                    //地市
 	District      string             `json:"district"`                //区县
+	Bidstatus     string			 `json:"bidstatus"`				  //
+	Bidtype		  string			 `json:"bidtype"`				  //
 	//HasPackage    bool                   `json:"haspackage"`              //是否有分包
 	Package     map[string]interface{} `json:"package,omitempty"`     //分包的对比对象
 	Buyerclass  string                 `json:"buyerclass"`            //采购单位分类
@@ -294,6 +296,16 @@ type ProjectInfo struct {
 	score         int
 	comStr        string
 	resVal, pjVal int
+	IdStatusInfo  map[string]map[string]interface{}
+}
+
+type Site struct {
+	Id			string				`json:"_id"`
+	Site		string				`json:"site"`			//站点名字
+	Area 		string				`json:"area"`			//省
+	City		string				`json:"city"`			//市
+	District	string				`json:"district"`		//区、县
+	Domain		string				`json:"domain"`			//地址
 }
 
 //二分字符串查找

fullproject/src_v1/load_data.go

@@ -9,7 +9,7 @@ import (
 //初始加载数据，默认加载最近6个月的数据
 
 func (p *ProjectTask) loadData(starttime int64) {
-	log.Println("load start..", starttime)
+	log.Println("load project start..", starttime)
 	p.findLock.Lock()
 	defer p.findLock.Unlock()
 	p.AllIdsMapLock.Lock()
@@ -23,7 +23,6 @@ func (p *ProjectTask) loadData(starttime int64) {
 		"list": 0,
 	}).Iter()
 	n := 0
-	//	tmp := &ProjectInfo{}
 	pool := make(chan *ProjectInfo, 100)
 	over := make(chan bool)
 	go func() {
@@ -86,7 +85,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 			go func(res map[string]interface{}) {
 				bys, _ := json.Marshal(result)
 				var tmp *ProjectInfo
-				json.Unmarshal(bys, &tmp)
+				_ = json.Unmarshal(bys, &tmp)
 				pool <- tmp
 			}(result)
 		} else {
@@ -95,5 +94,50 @@ func (p *ProjectTask) loadData(starttime int64) {
 	}
 	time.Sleep(2 * time.Second)
 	over <- true
-	log.Println("load over..", n)
+	log.Println("load project over..", n)
+}
+
+func (p *ProjectTask) loadSite() {
+	log.Println("load site start..")
+	p.findLock.Lock()
+	defer p.findLock.Unlock()
+	p.mapSiteLock.Lock()
+	defer p.mapSiteLock.Unlock()
+	sess := MongoTool.GetMgoConn()
+	defer MongoTool.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(MongoTool.DbName).C(SiteColl).Find(&q).Iter()
+	n := 0
+	pool := make(chan *Site, 100)
+	over := make(chan bool)
+	go func() {
+		for {
+			select {
+			case tmp := <-pool:
+				n++
+				if tmp != nil {
+					p.mapSite[tmp.Site] = tmp
+				}
+			case <-over:
+				return
+			}
+		}
+	}()
+	for {
+		result := make(map[string]interface{})
+		if it.Next(&result) {
+			go func(res map[string]interface{}) {
+				bys, _ := json.Marshal(result)
+				var tmp *Site
+				_ = json.Unmarshal(bys, &tmp)
+				pool <- tmp
+			}(result)
+		} else {
+			break
+		}
+	}
+	time.Sleep(2 * time.Second)
+	over <- true
+	log.Println("load site over..", n)
+
 }

fullproject/src_v1/main.go

@@ -68,7 +68,8 @@ func main() {
 			P_QL.loadData(loadStart)
 		}
 	}
-	go checkMapJob()
+	P_QL.loadSite()
+	//go checkMapJob()
 	time.Sleep(99999 * time.Hour)
 }
 
@@ -80,7 +81,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 		err := json.Unmarshal(data, &mapInfo)
 		log.Println("err:", err, "mapInfo:", mapInfo)
 		if err != nil {
-			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+			_ = udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
 		} else if mapInfo != nil {
 			key, _ := mapInfo["key"].(string)
 			if key == "" {
@@ -110,7 +111,15 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					P_QL.pici = time.Now().Unix()
 					P_QL.taskZl(mapInfo)
 				}()
-
+			case "updateInfo":		//招标字段变更
+				go func() {
+					defer func() {
+						<-SingleThread
+					}()
+					P_QL.currentType = tasktype
+					P_QL.pici = time.Now().Unix()
+					P_QL.taskUpdateInfo(mapInfo)
+				}()
 			case "history": //历史数据合并,暂时不写
 				go func() {
 					defer func() {

fullproject/src_v1/mgotool.go

@@ -247,3 +247,13 @@ func (m *MongodbSim) Delete(c, id string) int64 {
 	}
 	return r.DeletedCount
 }
+
+func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	result := map[string]interface{}{}
+	_ = r.Decode(&result)
+	return result
+}

fullproject/src_v1/project.go

@@ -124,7 +124,16 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 		//问题出地LastTime!!!!!
 		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
 		if diffTime <= p.validTime {
-			//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
+
+			//代理机构完全不相同，直接新建项目
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+			//地区(省、市、区)不同，直接新建项目
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+
 			info.PNBH = 0
 			info.PCBH = 0
 			info.PTCBH = 0
@@ -142,72 +151,7 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 				compareProject.pjVal = pjVal
 				compareProject.resVal = resVal
 				//log.Println(compareProject.comStr)
-				eqV := 0
-				switch resVal {
-				case 3:
-					if pjVal == 3 && comStr[3:] != "CCCDCCC" {
-						eqV = 1
-					} else if compareBuyer < "C" {
-						if pjVal > 1 {
-							eqV = 1
-						} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
-							eqV = 2
-						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-					} else {
-						if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-					}
-				case 2:
-					if compareBuyer < "C" {
-						if pjVal > 1 {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
-							eqV = 3
-						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
-							eqV = 3
-						}
-
-					} else {
-						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
-							eqV = 3
-						}
-					}
-				case 1:
-					if compareBuyer < "C" {
-						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
-							eqV = 3
-						}
-					} else if compareBuyer == "D" {
-						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
-							eqV = 2
-						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
-							eqV = 3
-						}
-					} else {
-						if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
-							eqV = 3
-						}
-					}
-				}
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
 				if eqV == 1 {
 					comRes1 = append(comRes1, compareProject)
 				} else if eqV == 2 {
@@ -215,14 +159,11 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 				} else if eqV == 3 {
 					comRes3 = append(comRes3, compareProject)
 				}
-				//				else if resVal == 3 || pjVal > 1 {
-				//					log.Println("===", resVal, pjVal, comStr, info.ProjectCode, compareProject.ProjectCode,
-				//						info.ProjectName, compareProject.ProjectName, info.Buyer, compareProject.Buyer, info.Id, compareProject.Id.Hex())
-				//				}
 			}
 		}
 	}
 	//--------------------------------对比完成-----------------------
+
 	//更新数组、更新项目
 	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
 		if len(resN) > 0 {
@@ -232,47 +173,60 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 				})
 			}
 
-			bFindProject = true
-			findPid = resN[0].Id.Hex()
-			for k2, bv := range []int{bpn, bpc, bptc, bpb} {
-				if bv > -1 {
-					pids[bv].Arr = append(pids[bv].Arr, findPid)
-					if k2 == 0 {
-						if resN[0].ProjectName == "" {
-							resN[0].ProjectName = info.ProjectName
-						} else {
-							if resN[0].MPN == nil {
-								resN[0].MPN = []string{info.ProjectName}
+			ex := 0
+			resArr := []*ProjectInfo{}
+			for _, res := range resN {
+				choose, e := p.CompareStatus(resN[0], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
 							} else {
-								resN[0].MPN = append(resN[0].MPN, info.ProjectName)
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
 							}
-						}
 
-					} else if k2 < 3 {
-						if resN[0].ProjectCode == "" {
-							resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
-						} else {
-							if resN[0].MPC == nil {
-								resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
 							} else {
-								resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
 							}
-						}
 
-					} else {
-						if resN[0].Buyer == "" {
-							resN[0].Buyer = info.Buyer
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
 						}
 					}
 				}
+				p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+			} else {
+				bFindProject = false
+				findPid = ""
 			}
-			p.UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
 			break
 		}
 	}
 
 	if !bFindProject {
-		//没有找到
 		id, p1 := p.NewProject(tmp, info)
 		p.AllIdsMapLock.Lock()
 		p.AllIdsMap[id] = &ID{Id: id, P: p1}
@@ -281,7 +235,6 @@ func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{})
 			m.Arr = append(m.Arr, id)
 		}
 	}
-
 }
 
 func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectInfo, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
@@ -384,6 +337,76 @@ func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectInfo, diffTime int64,
 	return
 }
 
+func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
+	eqV := 0
+	switch resVal {
+	case 3:
+		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
+			eqV = 1
+		} else if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 1
+			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
+				eqV = 2
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		} else {
+			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+		}
+	case 2:
+		if compareBuyer < "C" {
+			if pjVal > 1 {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
+				eqV = 3
+			}
+
+		} else {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		}
+	case 1:
+		if compareBuyer < "C" {
+			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else if compareBuyer == "D" {
+			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
+				eqV = 2
+			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
+				eqV = 3
+			}
+		} else {
+			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
+				eqV = 3
+			}
+		}
+	}
+	return eqV
+}
+
 var FIELDS = []string{
 	"area",
 	"city",
@@ -409,33 +432,27 @@ var FIELDS = []string{
 
 var bidtype = map[string]string{
 	"招标": "招标",
-	"邀标": "邀标",
 	"询价": "询价",
 	"竞谈": "竞谈",
 	"单一": "单一",
 	"竞价": "竞价",
+	"变更": "变更",
+	"邀标": "邀标",
 }
 
-//var bidstatus = map[string]string{
-//	"中标": "中标",
-//	"成交": "成交",
-//	"废标": "废标",
-//	"流标": "流标",
-//	"合同": "合同",
-//}
 var bidstatus = map[string]string{
 	"预告": "预告",
 	"中标": "中标",
 	"成交": "成交",
 	"废标": "废标",
 	"流标": "流标",
+	"合同": "合同",
 }
 
 //招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime
 
 func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
 	pId := primitive.NewObjectID() //NewObjectId()
-	p1 := p.NewCachePinfo(pId, thisinfo)
 	set := map[string]interface{}{}
 	set["_id"] = pId
 	for _, f := range FIELDS {
@@ -454,7 +471,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["bidopentime"] = bidopentime
 	}
 	if thisinfo.ProjectName != "" {
-		set["s_projectname"] = tmp["projectname"] //兼容老版本
+		set["s_projectname"] = tmp["projectname"]
 	}
 	now := time.Now().Unix()
 	set["createtime"] = now
@@ -469,19 +486,40 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	} else if thisinfo.TopType == "结果" {
 		set["jgtime"] = tmp["publishtime"]
 	}
+	//异常标记
+	if thisinfo.TopType != "招标" && thisinfo.TopType != "拟建" && thisinfo.TopType != "预告" {
+		set["exception"] = 1
+	}
+	//projecthref保存
+	if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+		if jsonData != nil && jsonData["projecthref"] != "" {
+			set["projecthref"] = jsonData["projecthref"]
+		}
+	}
+
 	//招标类型
+	p.mapBidLock.Lock()
 	bt := bidtype[thisinfo.SubType]
+	p.mapBidLock.Unlock()
 	if bt == "" {
 		bt = "招标"
 	}
 	set["bidtype"] = bt
-	bs, _ := tmp["bidstatus"].(string)
-	if bidstatus[bs] != "" {
-		set["bidstatus"] = bs
-	}
-	if set["bidstatus"] == nil && thisinfo.TopType == "结果" {
-		set["bidstatus"] = thisinfo.SubType
+	set["bidstatus"] = thisinfo.SubType
+
+	p1, pkg := p.NewCachePinfo(pId, thisinfo, bt)
+	//招标信息是中标或者成交，保存bidstatus、budget、bidamount
+	if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" {
+		p1.IdStatusInfo = map[string]map[string]interface{}{
+			thisinfo.Id: {
+				"projectname": thisinfo.ProjectName,
+				"bidstatus": thisinfo.SubType,
+				"budget": thisinfo.Budget,
+				"bidamount": thisinfo.Bidamount,
+			},
+		}
 	}
+
 	if len(thisinfo.Subscopeclass) > 0 {
 		s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
 		set["s_subscopeclass"] = s_subscopeclass
@@ -492,6 +530,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	if thisinfo.HasPackage {
 		set["multipackage"] = 1
+		set["package"] = pkg
 	} else {
 		set["multipackage"] = 0
 	}
@@ -501,10 +540,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	}
 	//p.savePool <- set
 	p.updatePool <- []map[string]interface{}{
-		map[string]interface{}{
+		{
 			"_id": pId,
 		},
-		map[string]interface{}{
+		{
 			"$set": set,
 		},
 	}
@@ -534,6 +573,7 @@ var INFOFIELDS = []string{
 	"buyertel",
 	"area",
 	"city",
+	"list",
 }
 
 //项目中list的信息
@@ -550,7 +590,11 @@ func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bs
 }
 
 //生成存放在内存中的对象
-func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info) ProjectInfo {
+func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype string) (ProjectInfo, map[string]interface{}) {
+	pkg := map[string]interface{}{}
+	if thisinfo.HasPackage {
+		pkg = PackageFormat(thisinfo, nil)
+	}
 	p1 := ProjectInfo{
 		Id:            id,
 		Ids:           []string{thisinfo.Id},
@@ -571,18 +615,20 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info) Proje
 		FirstTime:     thisinfo.Publishtime,
 		LastTime:      thisinfo.Publishtime,
 		Budget:        thisinfo.Budget,
-		Package:       thisinfo.Package,
+		Package:       pkg,
 		Bidamount:     thisinfo.Bidamount,
+		Bidstatus:     thisinfo.SubType,
+		Bidtype:       bidtype,
 	}
 	if thisinfo.LenPTC > 5 {
 		p1.MPC = append(p1.MPC, thisinfo.PTC)
 	}
-	return p1
+	return p1, pkg
 }
 
 //更新项目
-func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) {
-	if p.currentType != "ql" {
+func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, ex int) {
+	if p.currentType != "ql" && p.currentType != "updateInfo" {
 		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
 			log.Println("repeat", thisinfo.Id)
 			return
@@ -603,33 +649,45 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if thisinfo.Publishtime > pInfo.LastTime {
 		pInfo.LastTime = thisinfo.Publishtime
 		set["lasttime"] = thisinfo.Publishtime
+		p.mapBidLock.Lock()
+		defer p.mapBidLock.Unlock()
 		bt := bidtype[thisinfo.SubType]
 		if bt != "" {
 			set["bidtype"] = bt
 		}
-		//if thisinfo.TopType == "结果" {
-		//	set["bidstatus"] = thisinfo.SubType
-		//	set["jgtime"] = tmp["publishtime"]
-		//}
-		bs, _ := tmp["bidstatus"].(string)
+		bs, _ := tmp["subtype"].(string)
 		if bidstatus[bs] != "" {
 			set["bidstatus"] = thisinfo.SubType
-			if bidstatus[bs] != "预告" {
+			if bidstatus[bs] != "预告" && bidstatus[bs] != "合同" {
 				set["jgtime"] = tmp["publishtime"]
 			}
-		}else if tmp["infoformat"] == 2 {
+		} else if tmp["infoformat"] == 2 {
 			set["bidstatus"] = "拟建"
-		}else if tmp["bidstatus"] == "招标" {
+		} else if tmp["subytpe"] == "招标" {
 			set["bidstatus"] = thisinfo.TopType
-		}else {
-			set["bidstatus"] = thisinfo.SubType
+		} else {
+			set["bidstatus"] = "其它"
 		}
+	}
 
+	//废标、流标   处理时间
+	if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
+		pInfo.FirstTime = thisinfo.Publishtime
+		pInfo.Bidopentime = int64(0)
+		pInfo.LastTime = thisinfo.Publishtime
+
+		set["firsttime"] = thisinfo.Publishtime
+		set["zbtime"] = int64(0)
+		set["publishtime"] = thisinfo.Publishtime
+		set["bidopentime"] = int64(0)
 	}
 
+	//异常标记
+	if ex > 0 {
+		set["exception"] = ex
+	}
 	//3\4\5--省、市、县
 	if thisinfo.Area != "全国" {
-		//xt := true
 		if pInfo.Area == "全国" {
 			pInfo.Area = thisinfo.Area
 			set["area"] = thisinfo.Area
@@ -646,10 +704,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 			pInfo.District = thisinfo.District
 			set["district"] = thisinfo.District
 		}
-		//省市县有不相同的
-		//		if !xt {
-		//			log.Println(pInfo.Area, pInfo.City, thisinfo.Area, thisinfo.District)
-		//		}
 	}
 	//6--项目名称
 	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
@@ -673,11 +727,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	if pInfo.Buyer == "" {
 		set["buyerclass"] = ""
 	}
-	//if thisinfo.Buyerclass != "" && pInfo.Buyerclass == "" {
-	//	pInfo.Buyerclass = thisinfo.Buyerclass
-	//	set["buyerclass"] = pInfo.Buyerclass
-	//}
-
 
 	//8--代理机构
 	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
@@ -699,15 +748,6 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		pInfo.Bidopentime = thisinfo.Bidopentime
 		set["bidopentime"] = pInfo.Bidopentime
 	}
-	if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 {
-		pInfo.Bidamount = thisinfo.Bidamount
-		set["bidamount"] = pInfo.Bidamount
-	}
-
-	if thisinfo.Budget > 0 && pInfo.Budget < 1 { //多包的会有问题，没有进行合计。
-		pInfo.Budget = thisinfo.Budget
-		set["budget"] = pInfo.Budget
-	}
 
 	if len(thisinfo.Topscopeclass) > 0 {
 		sort.Strings(pInfo.Topscopeclass)
@@ -745,42 +785,23 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 
 	if thisinfo.HasPackage { //多包处理
-		p2, _ := tmp["package"].(map[string]interface{})
-		if p2 != nil {
-			if pInfo.Package != nil {
-				for pk2, pv2 := range p2 {
-					if pInfo.Package[pk2] != nil { //合并
-						item1, _ := pInfo.Package[pk2].(map[string]interface{})
-						item2, _ := pv2.(map[string]interface{})
-						if item1 != nil && item2 != nil { //原始项
-							for ik1, iv1 := range item2 {
-								if item1[ik1] == nil {
-									item1[ik1] = iv1
-								}
-							}
-						}
-						pInfo.Package[pk2] = item1
-					} else {
-						pInfo.Package[pk2] = pv2
-					}
-				}
-			} else {
-				pInfo.Package = p2
-			}
-		}
+		set["multipackage"] = 1
+		pkg := PackageFormat(thisinfo, pInfo)
+		pInfo.Package = pkg
 		set["package"] = pInfo.Package
+	} else {
+		set["multipackage"] = 0
 	}
+	//处理多包后，计算预算金额、中标金额
+	CountAmount(pInfo, thisinfo)
+	set["budget"] = pInfo.Budget
+	set["bidamount"] = pInfo.Bidamount
+	//
 
 	set["mpn"] = pInfo.MPN
 	set["mpc"] = pInfo.MPC
 	set["pici"] = p.pici
 
-	if thisinfo.HasPackage {
-		set["multipackage"] = 1
-	} else {
-		set["multipackage"] = 0
-	}
-
 	update := map[string]interface{}{}
 	if len(set) > 0 {
 		update["$set"] = set
@@ -796,7 +817,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	}
 	if len(update) > 0 {
 		updateInfo := []map[string]interface{}{
-			map[string]interface{}{
+			{
 				"_id": pInfo.Id,
 			},
 			update,
@@ -804,3 +825,212 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		p.updatePool <- updateInfo
 	}
 }
+
+/**
+ *	更新项目时，项目状态的处理
+ *	返回是否新增项目，异常标记
+ *	1、项目时，新项目时，招标信息的状态(toptype)不是招标、拟建、预告	异常：1
+ *	   异常1是在项目新建的时候才会产生
+ *	3、项目合并时，项目状态是”流标“/”废标“，招标信息状态不是”招标“		异常：2
+ *	4、项目合并时，项目状态是”合同“/”其它“，招标信息类型是”结果“		异常：3
+ */
+func (p *ProjectTask) CompareStatus(project *ProjectInfo, info *Info) (bool, int) {
+	if info.TopType == "拟建" || info.TopType == "预告" || info.TopType == "招标" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		} else {
+			return true, 0
+		}
+	} else if info.TopType == "结果" {
+		if project.Bidstatus == "拟建" || project.Bidstatus == "预告" || project.Bidstatus == "招标" {
+			return false, 0
+		}  else if project.Bidstatus == info.SubType {
+			//状态一样，根据发布时间判断是否合并
+			if (info.Publishtime - project.FirstTime) > p.statusTime {
+				return true, 0
+			} else {
+				return true, 0
+			}
+		} else if project.Bidstatus == "成交" && info.SubType == "中标" {
+			return true, 0
+		} else if project.Bidstatus == "流标" || project.Bidstatus == "废标" {
+			return false, 2
+		} else if project.Bidstatus == "合同" || project.Bidstatus == "其它" {
+			return false, 3
+		} else {
+			return false, 0
+		}
+	} else {
+		return false, 0
+	}
+}
+
+/*
+ *	对比地区(省、市、区)，存在且不同，不能合并
+ *	返回是否新建项目
+ */
+func ComparePlace(project *ProjectInfo, info *Info) bool {
+	if info.Area == "全国" || info.Area == "" {
+		return false
+	}
+	if project.Area == "全国" || project.Area == "" {
+		return false
+	}
+	if info.Area == project.Area {
+		if info.City == "" {
+			return false
+		} else if info.City == project.City {
+			if info.District == "" || info.District == project.District {
+				return false
+			} else {
+				return true
+			}
+		} else {
+			return true
+		}
+	} else {
+		return true
+	}
+}
+
+var PackageEle = []string{
+	"origin",
+	"name",
+	"text",
+	"budget",
+	"winner",
+	"bidamount",
+	"bidamounttype",
+	"currency",
+	"bidstatus",
+}
+
+func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
+	p2 := map[string]interface{}{}
+	for _, k := range PackageEle {
+		if map1[k] != nil {
+			p2[k] = map1[k]
+		}
+		infoid := p2["infoid"]
+		if infoid == nil {
+			p2["infoid"] = id
+		}
+	}
+	return p2
+}
+
+func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
+	budget := 0
+	bidamount := 0
+	p1 := map[string]interface{}{}
+	if project != nil && project.Package != nil && len(project.Package) > 0 {
+		p1 = project.Package
+		for k, v := range info.Package {
+			if v1, ok := v.(map[string]interface{}); ok {
+				v2 := map[string]interface{}{}
+				v2 = packageEle(v1, info.Id)
+				if v2["bidstatus"] == nil {
+					v2["bidstatus"] = info.SubType
+				}
+				if isCount(project, v2[""]) {
+					
+				}
+				if v2["budget"] != nil {
+					budget = budget + v2["budget"].(float64)
+				}
+				if v2["bidamount"] != nil {
+					bidamount = bidamount + v2["bidamount"].(float64)
+				}
+				addFlag := false
+				for k1, v3 := range p1 {
+					if v4, ok := v3.([]map[string]interface{}); ok {
+						if qu.ObjToString(v4[0]["origin"]) == qu.ObjToString(v2["origin"]) && qu.ObjToString(v4[0]["name"]) == qu.ObjToString(v2["name"]) {
+							v4 = append(v4, v2)
+							p1[k1] = v4
+							addFlag = true
+							break
+						}
+					}
+				}
+				if !addFlag {
+					p1[k] = []map[string]interface{}{v2}
+				}
+			}
+		}
+	} else {
+		for k, v := range info.Package {
+			v1, _ := v.(map[string]interface{})
+			p2 := map[string]interface{}{}
+			p2 = packageEle(v1, info.Id)
+			if p2["bidstatus"] == nil {
+				p2["bidstatus"] = info.SubType
+			}
+			if p2["budget"] != nil {
+				budget = p2["budget"].(float64)
+			}
+			if p2["bidamount"] != nil {
+				bidamount = p2["bidamount"].(float64)
+			}
+			p1[k] = []map[string]interface{}{p2}
+		}
+	}
+	info.Budget = budget
+	info.Bidamount = bidamount
+	return p1
+}
+
+//计算预算(budget)、中标金额(bidamount)
+func CountAmount(project *ProjectInfo, info *Info) {
+	//if project!= nil && project.Package != nil && len(project.Package) > 0 {
+	////暂时未考虑太多情况，简单处理
+	//}
+
+	if info.Budget > 0 {
+		//项目中第一条招标信息是否是包/段项目
+		key := titleGetPc.FindStringSubmatch(project.ProjectName)
+		if len(key) > 0 {
+			//判断项目中是否已经计算过该包/段的预算
+			if !isCount(project, info.ProjectName) {
+				project.Budget = project.Budget + info.Budget
+			}
+		}else {
+			if project.Budget < info.Budget {
+				project.Budget = info.Budget
+			}
+		}
+	}
+	//中标、成交、合同     处理中标金额(bidamount)
+	if info.SubType == "中标" || info.SubType == "成交" || info.SubType == "合同" {
+		if info.Bidamount > 0 {
+			key := titleGetPc.FindStringSubmatch(project.ProjectName)
+			if len(key) > 0 {
+				if !isCount(project, info.ProjectName) {
+					project.Bidamount = project.Bidamount + info.Bidamount
+				}
+			}else {
+				if project.Bidamount < project.Bidamount {
+					project.Bidamount = info.Bidamount
+				}
+			}
+		}
+	}
+
+	//保存信息到IdStatusInfo
+	project.IdStatusInfo[info.Id] = map[string]interface{}{
+		"projectname": info.ProjectName,
+		"bidstatus": info.SubType,
+		"budget": info.Budget,
+		"bidamount": info.Bidamount,
+	}
+}
+
+func isCount(project *ProjectInfo, infoName string) bool {
+	if project.IdStatusInfo != nil && len(project.IdStatusInfo) > 0 {
+		for _, v := range project.IdStatusInfo{
+			if v["projectname"] == infoName {
+				return true
+			}
+		}
+	}
+	return false
+}

fullproject/src_v1/task.go

@@ -37,7 +37,14 @@ type ProjectTask struct {
 	AllIdsMap map[string]*ID
 	//采购单位、项目名称、项目编号
 	mapPb, mapPn, mapPc map[string]*Key
-	//	mapPbLock, mapPnLock, mapPcLock sync.Mutex
+	//流程数据  字段相同，直接合并
+	mapHref     map[string]string
+	mapHrefLock sync.Mutex
+	//站点
+	mapSite     map[string]*Site
+	mapSiteLock sync.Mutex
+	//bidtype、bidstatus 锁
+	mapBidLock sync.Mutex
 	//更新或新增通道
 	updatePool chan []map[string]interface{}
 	//savePool   chan map[string]interface{}
@@ -54,6 +61,7 @@ type ProjectTask struct {
 	saveSize  int
 	pici      int64
 	validTime int64
+	statusTime int64
 	//	LockPool     chan *sync.Mutex
 	//	LockPoolLock sync.Mutex
 	//	m1, m23, m4  map[int]int
@@ -73,12 +81,15 @@ func NewPT() *ProjectTask {
 		mapPb:     make(map[string]*Key, 1500000),
 		mapPn:     make(map[string]*Key, 5000000),
 		mapPc:     make(map[string]*Key, 5000000),
+		mapHref:   make(map[string]string, 1500000),
+		mapSite:   make(map[string]*Site, 1000000),
 		saveSize:  400,
 
 		//saveSign:   make(chan bool, 1),
 		//updateSign: make(chan bool, 1),
 		coll:      ProjectColl,
 		validTime: int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
+		statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 7) * 86400),
 	}
 	return p
 }
@@ -134,7 +145,7 @@ func (p *ProjectTask) clearMem() {
 	c := cron.New()
 	//在内存中保留最近6个月的信息
 	//跑全量时每4分钟跑一次，跑增量时400分钟跑一次
-	c.AddFunc("50 0/15 * * * *", func() {
+	_ = c.AddFunc("50 0/15 * * * *", func() {
 		if p.currentType == "ql" || p.clearContimes >= 60 {
 			//跳过的次数清零
 			p.clearContimes = 0
@@ -146,6 +157,7 @@ func (p *ProjectTask) clearMem() {
 			//遍历id
 			//所有内存中的项目信息
 			p.AllIdsMapLock.Lock()
+			p.mapHrefLock.Lock()
 
 			//清除计数
 			clearNum := 0
@@ -194,12 +206,18 @@ func (p *ProjectTask) clearMem() {
 							}
 						}
 					}
+					for kHref, pid := range p.mapHref {
+						if pid == k {
+							delete(p.mapHref, kHref)
+						}
+					}
 					v = nil
 				}
 			}
+			p.mapHrefLock.Unlock()
 			p.AllIdsMapLock.Unlock()
 			p.findLock.Unlock()
-			log.Println("清除完成:", clearNum, len(p.AllIdsMap), len(p.mapPn), len(p.mapPc), len(p.mapPb))
+			log.Println("清除完成:", clearNum, len(p.AllIdsMap), len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref))
 		} else {
 			p.clearContimes++
 		}
@@ -296,6 +314,37 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 	}
 }
 
+//招标字段更新
+func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
+	defer util.Catch()
+	db, _ := udpInfo["db"].(string)
+	if db == "" {
+		db = MongoTool.DbName
+	}
+	coll, _ := udpInfo["coll"].(string)
+	if coll == "" {
+		coll = ExtractColl
+	}
+	thread := util.IntAllDef(udpInfo["thread"], 4)
+	if thread > 0 {
+		p.thread = thread
+	}
+	q, _ := udpInfo["query"].(map[string]interface{})
+	gtid := udpInfo["gtid"].(string)
+	lteid := udpInfo["lteid"].(string)
+	if q == nil {
+		q = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt":  StringTOBsonId(gtid),
+				"$lte": StringTOBsonId(lteid),
+			},
+			"is_m": 1,
+		}
+	}
+	log.Println("查询语句：", q)
+	p.enter(db, coll, q)
+}
+
 func StringTOBsonId(id string) primitive.ObjectID {
 	objectId, _ := primitive.ObjectIDFromHex(id)
 	return objectId
@@ -313,7 +362,7 @@ func nextNode(mapInfo map[string]interface{}, pici int64) {
 		datas, _ := json.Marshal(mapInfo)
 		node := &udpNode{datas, to, time.Now().Unix(), 0}
 		udptaskmap.Store(key, node)
-		udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, to)
+		_ = udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, to)
 	}
 }
 
@@ -344,10 +393,15 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 						<-pool
 					}()
 					if util.IntAll(tmp["repeat"]) == 0 {
+						p.fillInPlace(tmp)
 						info := ParseInfo(tmp)
-						if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
-							p.currentTime = info.Publishtime
-							p.startProjectMerge(info, tmp)
+						p.currentTime = info.Publishtime
+						if p.currentType == "updateInfo" {
+							//招标信息更改合并
+							p.updateJudge(tmp, info)
+						} else {
+							//普通合并
+							p.CommonMerge(tmp, info)
 						}
 					} else {
 						//信息错误,进行更新
@@ -364,7 +418,6 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 		ms.Hint(Sysconfig["hints"])
 	}
 	query := ms.Iter()
-	//
 	var lastid interface{}
 L:
 	for {
@@ -378,7 +431,7 @@ L:
 			tmp := make(map[string]interface{})
 			if query.Next(&tmp) {
 				lastid = tmp["_id"]
-				if count%2000 == 0 {
+				if count%10000 == 0 {
 					log.Println("current", count, lastid)
 				}
 				infoPool <- tmp
@@ -409,12 +462,50 @@ var (
 	StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
 	//纯数字或纯字母
 	StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
+	//含分包词，招标未识别分包  合并到一个项目
+	KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
 )
 
+func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
+	if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
+		if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
+			proHref := util.ObjToString(jsonData["projecthref"])
+			if jsonData != nil && proHref != "" {
+				//projectHref字段合并
+				tmp["projecthref"] = proHref
+				p.mapHrefLock.Lock()
+				pid := p.mapHref[proHref]
+				p.mapHrefLock.Unlock()
+				if pid != "" {
+					p.AllIdsMapLock.Lock()
+					comparePro := p.AllIdsMap[pid].P
+					p.AllIdsMapLock.Unlock()
+					_, ex := CompareStatus(comparePro, info)
+					p.UpdateProject(tmp, info, comparePro, -1, "AAAAAAAAAA", ex)
+				} else {
+					id, p1 := p.NewProject(tmp, info)
+					p.mapHrefLock.Lock()
+					p.mapHref[proHref] = id
+					p.mapHrefLock.Unlock()
+					p.AllIdsMapLock.Lock()
+					p.AllIdsMap[id] = &ID{Id: id, P: p1}
+					p.AllIdsMapLock.Unlock()
+				}
+			} else {
+				//项目合并
+				p.startProjectMerge(info, tmp)
+			}
+		} else {
+			//项目合并
+			p.startProjectMerge(info, tmp)
+		}
+	}
+}
+
 func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	bys, _ := json.Marshal(tmp)
 	var thisinfo *Info
-	json.Unmarshal(bys, &thisinfo)
+	_ = json.Unmarshal(bys, &thisinfo)
 	if thisinfo == nil {
 		return nil
 	}
@@ -425,6 +516,11 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 		thisinfo.Subscopeclass = []string{}
 	}
 
+	if thisinfo.Publishtime == 0 {
+		thisinfo.Publishtime = thisinfo.Comeintime
+		tmp["publishtime"] = thisinfo.Comeintime
+	}
+
 	//从标题中查找项目编号
 	res := titleGetPc.FindStringSubmatch(thisinfo.Title)
 	if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
@@ -501,6 +597,141 @@ func ParseInfo(tmp map[string]interface{}) (info *Info) {
 	return thisinfo
 }
 
+func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info) {
+	index := -1
+	pInfoId := ""
+	p.AllIdsMapLock.Lock()
+F:
+	for k, ID := range p.AllIdsMap {
+		for i, id := range ID.P.Ids {
+			if info.Id == id {
+				pInfoId = k
+				index = i
+				break F
+			}
+		}
+	}
+	p.AllIdsMapLock.Unlock()
+	//未找到招标信息
+	if index == -1 {
+		if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
+			p.currentTime = info.Publishtime
+			p.startProjectMerge(info, tmp)
+		}
+	} else {
+		tmpPro := MongoTool.FindById(ProjectColl, pInfoId)
+		infoList := []interface{}(tmpPro["list"].(primitive.A))
+		infoMap := infoList[index].(map[string]interface{})
+		modifyMap, f := modifyEle(infoMap, tmp)
+		//projecthref字段
+		jsonData := tmp["jsondata"].(map[string]interface{})
+		if jsonData != nil && jsonData["projecthref"] != nil {
+			proHref := jsonData["projecthref"].(string)
+			tmp["projecthref"] = proHref
+			p.mapHrefLock.Lock()
+			pid := p.mapHref[proHref]
+			p.mapHrefLock.Unlock()
+			if pid == pInfoId {
+				p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
+				return
+			}
+		}
+
+		if f {
+			//合并、修改
+			log.Println("合并修改更新", "----------------------------")
+			p.mergeAndModify(pInfoId, index, info, tmp, tmpPro, modifyMap)
+		} else {
+			//修改
+			log.Println("修改更新", "----------------------------")
+			p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
+		}
+	}
+}
+
+var Elements = []string{
+	"projectname",
+	"projectcode",
+	"agency",
+	"budget",
+	"bidamount",
+	"buyerperson",
+	"area",
+	"city",
+	"publishtime",
+}
+
+/**
+判断修改的字段是否是影响合并流程的要素字段
+*/
+func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
+	modifyMap := map[string]interface{}{}
+	for k, _ := range tmpPro {
+		for k1, _ := range tmp {
+			if k == k1 && tmpPro[k] != tmp[k1] {
+				modifyMap[k] = tmp[k1]
+				break
+			}
+		}
+	}
+	for k, _ := range modifyMap {
+		for _, str := range Elements {
+			if k == str {
+				return modifyMap, true
+			}
+		}
+	}
+	delete(modifyMap, "_id")
+	return modifyMap, false
+}
+
+//补全位置信息
+func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
+	area := util.ObjToString(tmp["area"])
+	city := util.ObjToString(tmp["city"])
+	district := util.ObjToString(tmp["district"])
+	if area != "" && city != "" && district != "" {
+		return
+	}
+
+	tmpSite := util.ObjToString(tmp["site"])
+	if tmpSite == "" {
+		return
+	}
+	p.mapSiteLock.Lock()
+	defer p.mapSiteLock.Unlock()
+	site := p.mapSite[tmpSite]
+	if site != nil {
+		if area != "" {
+			if area == "全国" {
+				tmp["area"] = site.Area
+				tmp["city"] = site.City
+				tmp["district"] = site.District
+				return
+			}
+			if area != site.Area {
+				return
+			} else {
+				if city == site.City {
+					if district == "" {
+						tmp["district"] = site.District
+						return
+					}
+				} else if city == "" {
+					tmp["city"] = site.City
+					tmp["district"] = site.District
+					return
+				}
+			}
+		} else {
+			tmp["area"] = site.Area
+			tmp["city"] = site.City
+			tmp["district"] = site.District
+			return
+		}
+	}
+}
+
 //从数组中删除元素
 func deleteSlice(arr []string, v string) []string {
 	for k, v1 := range arr {
@@ -510,154 +741,3 @@ func deleteSlice(arr []string, v string) []string {
 	}
 	return arr
 }
-
-//			if taskcount > 0 && taskcount%50000 == 0 { //歇歇
-//				log.Println("pause start..", taskcount)
-//				for n := 0; n < p.thread; n++ {
-//					pool <- true
-//				}
-//				for n := 0; n < p.thread; n++ {
-//					<-pool
-//				}
-//				log.Println("pause over..")
-//			}
-//lastid = tmp["_id"]
-//tmp = make(map[string]interface{})
-//		if count > 40000 {
-//			query.Close()
-//			break
-//		}
-//over++
-//func (p *ProjectTask) saveQueue() {
-//	arr := make([]map[string]interface{}, p.saveSize)
-//	indexs := 0
-//	for {
-//		select {
-//		case <-p.saveSign:
-//			if indexs > 0 {
-//				MongoTool.SaveBulk(p.coll, arr[:indexs]...)
-//				arr = make([]map[string]interface{}, p.saveSize)
-//				indexs = 0
-//			}
-//			p.updateSign <- true
-//		case v := <-p.savePool:
-//			arr[indexs] = v
-//			indexs++
-//			if indexs == p.saveSize {
-//				MongoTool.SaveBulk(p.coll, arr...)
-//				arr = make([]map[string]interface{}, p.saveSize)
-//				indexs = 0
-//			}
-//		case <-time.After(100 * time.Millisecond):
-//			if indexs > 0 {
-//				MongoTool.SaveBulk(p.coll, arr[:indexs]...)
-//				arr = make([]map[string]interface{}, p.saveSize)
-//				indexs = 0
-//			}
-//		}
-//	}
-//}
-
-////项目保存和更新通道
-//func (p *ProjectTask) updateQueue() {
-//	arru := make([][]map[string]interface{}, p.saveSize)
-//	indexu := 0
-//	for {
-//		select {
-//		case v := <-p.updatePool:
-//			arru[indexu] = v
-//			indexu++
-//			if indexu == p.saveSize {
-//				//更新之前先保存
-//				p.saveSign <- true
-//				<-p.updateSign
-//				MongoTool.UpdateBulk(p.coll, arru...)
-//				arru = make([][]map[string]interface{}, p.saveSize)
-//				indexu = 0
-//			}
-//		case <-time.After(100 * time.Millisecond):
-//			if indexu > 0 {
-//				p.saveSign <- true
-//				<-p.updateSign
-//				MongoTool.UpdateBulk(p.coll, arru[:indexu]...)
-//				arru = make([][]map[string]interface{}, p.saveSize)
-//				indexu = 0
-//			}
-//		}
-//	}
-//}
-//func (p *ProjectTask) ConCurrentLock(n1, n2, n3, n4 int) {
-//	var lock *sync.Mutex
-//	p.LockPoolLock.Lock()
-//	if p.m1[n1] > 0 || p.m23[n2] > 0 || p.m23[n3] > 0 || p.m4[n4] > 0 {
-//		if p.l1[n1] != nil {
-//			lock = p.l1[n1]
-//		} else if p.l23[n2] != nil {
-//			lock = p.l23[n2]
-//		} else if p.l23[n3] != nil {
-//			lock = p.l23[n3]
-//		} else if p.l4[n4] != nil {
-//			lock = p.l4[n4]
-//		}
-//	} else {
-//		lock = <-p.LockPool
-//	}
-//	if n1 > 0 {
-//		p.m1[n1]++
-//		p.l1[n1] = lock
-//	}
-//	if n2 > 0 {
-//		p.m23[n2]++
-//		p.l23[n2] = lock
-//	}
-//	if n3 > 0 {
-//		p.m23[n3]++
-//		p.l23[n3] = lock
-//	}
-//	if n4 > 0 {
-//		p.m4[n4]++
-//		p.l4[n4] = lock
-//	}
-//	p.LockPoolLock.Unlock()
-//	lock.Lock()
-//}
-
-//func (p *ProjectTask) ConCurrentUnLock(n1, n2, n3, n4 int) {
-//	var lock1 *sync.Mutex
-//	p.LockPoolLock.Lock()
-//	if p.l1[n1] != nil {
-//		lock1 = p.l1[n1]
-//	} else if p.l23[n2] != nil {
-//		lock1 = p.l23[n2]
-//	} else if p.l23[n3] != nil {
-//		lock1 = p.l23[n3]
-//	} else if p.l4[n4] != nil {
-//		lock1 = p.l4[n4]
-//	}
-//	if p.m1[n1] > 0 {
-//		p.m1[n1]--
-//		if p.m1[n1] == 0 {
-//			p.l1[n1] = nil
-//		}
-//	}
-//	if p.m23[n2] > 0 {
-//		p.m23[n2]--
-//		if p.m23[n2] == 0 {
-//			p.l23[n2] = nil
-//		}
-//	}
-//	if p.m23[n3] > 0 {
-//		p.m23[n3]--
-//		if p.m23[n3] == 0 {
-//			p.l23[n3] = nil
-//		}
-//	}
-//	if p.m4[n4] > 0 {
-//		p.m4[n4]--
-//		if p.m4[n4] == 0 {
-//			p.l4[n4] = nil
-//		}
-//	}
-//	p.LockPoolLock.Unlock()
-//	lock1.Unlock()
-//}

fullproject/src_v1/update.go

@@ -0,0 +1,448 @@
+package main
+
+import (
+	"github.com/go-xweb/log"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"math"
+	qu "qfw/util"
+	"reflect"
+	"sort"
+	"strings"
+)
+
+func (p *ProjectTask) modifyUpdate(pInfoId string, index int, info *Info, tmp map[string]interface{}, tmpPro map[string]interface{}, modifyMap map[string]interface{}) {
+	backupPro(tmpPro)
+	infoList := []interface{}(tmpPro["list"].(primitive.A))
+	infoMap := infoList[index].(map[string]interface{})
+	infoList[index] = updateValue(infoMap, modifyMap)
+	set := map[string]interface{}{
+		"$set": tmpPro,
+	}
+	_ = MongoTool.UpdateById(ProjectColl, pInfoId, set)
+}
+
+func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp map[string]interface{}, tmpPro map[string]interface{}, modifyMap map[string]interface{}) {
+	p.AllIdsMapLock.Lock()
+	currentPro := p.AllIdsMap[pInfoId]
+	p.AllIdsMapLock.Unlock()
+	merge, pId, i, comStr, mergePro := p.updateMerge(index, info, pInfoId, tmp, tmpPro)
+	//招标信息在list中第1个
+	if index == 0 {
+		log.Println("第1条合并", "------1------")
+		//快照
+		backupPro(tmpPro)
+		p.AllIdsMapLock.Lock()
+		delete(p.AllIdsMap, pInfoId)
+		p.AllIdsMapLock.Unlock()
+		count := MongoTool.Delete(ProjectColl, pInfoId)
+		log.Println(count)
+		for _, infoId := range currentPro.P.Ids{
+			if infoId == info.Id {
+				continue
+			}
+			tmp1 := MongoTool.FindById(ExtractColl, infoId)
+			info1 := ParseInfo(tmp1)
+			p.CommonMerge(tmp1, info1)
+		}
+	}else {
+		log.Println("第n条合并", "------n------")
+		//快照    招标信息所在的项目
+		backupPro(tmpPro)
+		if merge {
+			//跟原项目list第0条信息作对比，可以合并
+			if pId == pInfoId {
+				//合并到当前项目
+				mergeProject(p, currentPro.P, info, tmpPro, index)
+				p.AllIdsMapLock.Lock()
+				p.AllIdsMap[currentPro.P.Id.Hex()].P = currentPro.P
+				p.AllIdsMapLock.Unlock()
+				infoList := []interface{}(tmpPro["list"].(primitive.A))
+				infoMap := infoList[index].(map[string]interface{})
+				infoList[index] = updateValue(infoMap, modifyMap)
+				set := map[string]interface{}{
+					"$set": tmpPro,
+				}
+				MongoTool.UpdateById(ProjectColl, pInfoId, set)
+			}else {
+				//合并到的其它项目里边
+				//更新其它的项目
+				pro := MongoTool.FindById(ProjectColl, mergePro.Id.Hex())
+				backupPro(pro, )
+				choose, ex := CompareStatus(mergePro, info)
+				if !choose {
+					p.UpdateProject(tmp, info, mergePro, i, comStr, ex)
+				}else {
+					id, p1 := p.NewProject(tmp, info)
+					p.AllIdsMapLock.Lock()
+					p.AllIdsMap[id] = &ID{Id: id, P: p1}
+					p.AllIdsMapLock.Unlock()
+				}
+				//内部合并
+				p.innerMerge(currentPro.P, info, tmp, tmpPro)
+			}
+		}else {
+			//合并到新建的项目里边，原有项目内部合并
+			p.innerMerge(currentPro.P, info, tmp, tmpPro)
+		}
+	}
+}
+
+func (p *ProjectTask) updateMerge(index int, info *Info, pInfoId string, tmp map[string]interface{}, tmpPro map[string]interface{}) (bool, string, int, string, *ProjectInfo) {
+	bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
+	defer p.wg.Done()
+
+	for _, m := range pids {
+		defer m.Lock.Unlock()
+	}
+	for _, id := range IDArr {
+		defer id.Lock.Unlock()
+	}
+
+	bFindProject := false
+	findPid := ""
+	//获取完id，进行计算
+	//定义两组
+	comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
+	comRes2 := []*ProjectInfo{} //优化级其次
+	comRes3 := []*ProjectInfo{}
+	for _, v := range IDArr {
+		if index == 0 && pInfoId == v.P.Id.Hex() {
+			//招标条目信息在原项目中是第0条时，忽略
+			continue
+		}
+		comStr := ""
+		compareProject := v.P
+		compareProject.score = 0
+		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
+		if diffTime <= p.validTime {
+
+			//代理机构完全不相同，直接新建项目
+			if CheckContain(compareProject.Agency, info.Agency) == 3 {
+				continue
+			}
+
+			//地区(省、市、区)不同，直接新建项目
+			if ComparePlace(compareProject, info) {
+				continue
+			}
+
+			info.PNBH = 0
+			info.PCBH = 0
+			info.PTCBH = 0
+			compareStr, score := comparePNC(info, compareProject)
+			resVal, pjVal := Select(compareStr, info, compareProject)
+			if resVal > 0 {
+				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score)
+
+				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
+				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
+				compareProject.comStr = comStr
+				compareProject.pjVal = pjVal
+				compareProject.resVal = resVal
+				eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
+				if eqV == 1 {
+					comRes1 = append(comRes1, compareProject)
+				} else if eqV == 2 {
+					comRes2 = append(comRes2, compareProject)
+				} else if eqV == 3 {
+					comRes3 = append(comRes3, compareProject)
+				}
+			}
+		}
+	}
+	//--------------------------------对比完成-----------------------
+	k := -1
+	comStr := ""
+	mergePro := &ProjectInfo{}
+	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
+		if len(resN) > 0 {
+			if len(resN) > 1 {
+				sort.Slice(resN, func(i, j int) bool {
+					return resN[i].score > resN[j].score
+				})
+			}
+
+			ex := 0
+			resArr := []*ProjectInfo{}
+			for _, res := range resN{
+				choose, e := CompareStatus(resN[0], info)
+				if !choose {
+					ex = e
+					resArr = append(resArr, res)
+				}
+			}
+			if len(resArr) > 0 {
+				bFindProject = true
+				findPid = resArr[0].Id.Hex()
+				for k2, bv := range []int{bpn, bpc, bptc, bpb} {
+					if bv > -1 {
+						pids[bv].Arr = append(pids[bv].Arr, findPid)
+						if k2 == 0 {
+							if resArr[0].ProjectName == "" {
+								resArr[0].ProjectName = info.ProjectName
+							} else {
+								if resArr[0].MPN == nil {
+									resArr[0].MPN = []string{info.ProjectName}
+								} else {
+									resArr[0].MPN = append(resArr[0].MPN, info.ProjectName)
+								}
+							}
+
+						} else if k2 < 3 {
+							if resArr[0].ProjectCode == "" {
+								resArr[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
+							} else {
+								if resArr[0].MPC == nil {
+									resArr[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
+								} else {
+									resArr[0].MPC = append(resArr[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
+								}
+							}
+
+						} else {
+							if resArr[0].Buyer == "" {
+								resArr[0].Buyer = info.Buyer
+							}
+						}
+					}
+				}
+				if index != 0 {
+					//招标条目信息在原项目中是第非0条时，返回对比结果
+					k = kv+1
+					comStr = resArr[0].comStr
+					mergePro = resArr[0]
+					bFindProject = false
+					findPid = ""
+				}else {
+					mergePro = resArr[0]
+					//如果有合并，合并之前备份(快照)
+					backupPro(tmpPro)
+					p.UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr, ex)
+				}
+			}else {
+				bFindProject = false
+				findPid = ""
+			}
+			break
+		}
+	}
+
+	if !bFindProject {
+		id, p1 := p.NewProject(tmp, info)
+		p.AllIdsMapLock.Lock()
+		p.AllIdsMap[id] = &ID{Id: id, P: p1}
+		p.AllIdsMapLock.Unlock()
+		for _, m := range pids {
+			m.Arr = append(m.Arr, id)
+		}
+	}
+	return bFindProject, findPid, k, comStr, mergePro
+}
+
+//内部合并
+func (p *ProjectTask) innerMerge(pInfo *ProjectInfo, info *Info, tmp map[string]interface{}, tmpPro map[string]interface{}) {
+	infoList := []interface{}(tmpPro["list"].(primitive.A))
+	for index, info1 := range infoList{
+		info2 := info1.(map[string]interface{})
+		if info2["infoid"] == info.Id {
+			deleteSlice1(infoList, info1)
+			continue
+		}
+		mergeProject(p, pInfo, info, tmpPro, index)
+	}
+	set := map[string]interface{}{
+		"$set": tmpPro,
+	}
+	MongoTool.UpdateById(ProjectColl, pInfo.Id.Hex(), set)
+}
+
+//合并字段到project
+func mergeProject(p *ProjectTask, pInfo *ProjectInfo, thisinfo *Info, set map[string]interface{}, index int) map[string]interface{} {
+	//1--firsttime
+	if thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0 {
+		pInfo.FirstTime = thisinfo.Publishtime
+		set["firsttime"] = thisinfo.Publishtime
+		if thisinfo.TopType == "招标" {
+			set["zbtime"] = thisinfo.Publishtime
+		}
+	}
+	//2--lasttime
+	if thisinfo.Publishtime > pInfo.LastTime {
+		pInfo.LastTime = thisinfo.Publishtime
+		set["lasttime"] = thisinfo.Publishtime
+		p.mapBidLock.Lock()
+		bt := bidtype[thisinfo.SubType]
+		p.mapBidLock.Unlock()
+		if bt != "" {
+			set["bidtype"] = bt
+		}
+		if thisinfo.SubType != "" {
+			set["bidstatus"] = thisinfo.SubType
+			if thisinfo.SubType != "预告" {
+				set["jgtime"] = thisinfo.Publishtime
+			}
+		}else if thisinfo.Infoformat == 2 {
+			set["bidstatus"] = "拟建"
+		}else if thisinfo.SubType == "招标" {
+			set["bidstatus"] = thisinfo.TopType
+		}else {
+			set["bidstatus"] = thisinfo.SubType
+		}
+
+	}
+
+	//3\4\5--省、市、县
+	if thisinfo.Area != "全国" {
+		//xt := true
+		if pInfo.Area == "全国" {
+			pInfo.Area = thisinfo.Area
+			set["area"] = thisinfo.Area
+		} else if pInfo.Area != thisinfo.Area {
+			//xt = false
+		}
+		if pInfo.City == "" && thisinfo.City != "" {
+			pInfo.City = thisinfo.City
+			set["city"] = thisinfo.City
+		} else if pInfo.City != thisinfo.City {
+			//xt = false
+		}
+		if thisinfo.District != "" && pInfo.District == "" {
+			pInfo.District = thisinfo.District
+			set["district"] = thisinfo.District
+		}
+	}
+	//6--项目名称
+	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
+		pInfo.ProjectName = thisinfo.ProjectName
+		set["projectname"] = thisinfo.ProjectName
+	}
+	//7--项目编号
+	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
+		pInfo.ProjectCode = thisinfo.ProjectCode
+		set["projectcode"] = thisinfo.ProjectCode
+	}
+	//7--采购单位
+	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
+		pInfo.Buyer = thisinfo.Buyer
+		set["buyer"] = thisinfo.Buyer
+
+		pInfo.Buyerclass = thisinfo.Buyerclass
+		set["buyerclass"] = thisinfo.Buyerclass
+	}
+	if pInfo.Buyer == "" {
+		set["buyerclass"] = ""
+	}
+
+	//8--代理机构
+	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
+		pInfo.Agency = thisinfo.Agency
+		set["agency"] = thisinfo.Agency
+	}
+	//9--采购单位联系人
+	if thisinfo.Buyerperson != "" && strings.Index(pInfo.Buyerperson, thisinfo.Buyerperson) < 0 {
+		pInfo.Buyerperson = thisinfo.Buyerperson
+		set["buyerperson"] = pInfo.Buyerperson
+	}
+	//10--采购单位電話
+	if thisinfo.Buyertel != "" && strings.Index(pInfo.Buyertel, thisinfo.Buyertel) < 0 {
+		pInfo.Buyertel = thisinfo.Buyertel
+		set["buyertel"] = pInfo.Buyertel
+	}
+
+	if thisinfo.Bidopentime > pInfo.Bidopentime {
+		pInfo.Bidopentime = thisinfo.Bidopentime
+		set["bidopentime"] = pInfo.Bidopentime
+	}
+	if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 {
+		pInfo.Bidamount = thisinfo.Bidamount
+		set["bidamount"] = pInfo.Bidamount
+	}
+
+	if thisinfo.Budget > 0 && pInfo.Budget < 1 {
+		pInfo.Budget = thisinfo.Budget
+		set["budget"] = pInfo.Budget
+	}
+
+	if len(thisinfo.Topscopeclass) > 0 {
+		sort.Strings(pInfo.Topscopeclass)
+		for _, k := range thisinfo.Topscopeclass {
+			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
+				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
+				sort.Strings(pInfo.Topscopeclass)
+			}
+		}
+		set["topscopeclass"] = pInfo.Topscopeclass
+	}
+
+	if len(thisinfo.Subscopeclass) > 0 {
+		sort.Strings(pInfo.Subscopeclass)
+		for _, k := range thisinfo.Subscopeclass {
+			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
+				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
+				sort.Strings(pInfo.Subscopeclass)
+			}
+		}
+		set["subscopeclass"] = pInfo.Subscopeclass
+		set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",")
+	}
+	//winner
+	if len(thisinfo.Winners) > 0 {
+		sort.Strings(pInfo.Winners)
+		for _, k := range thisinfo.Winners {
+			if BinarySearch(pInfo.Winners, k) == -1 {
+				pInfo.Winners = append(pInfo.Winners, k)
+				sort.Strings(pInfo.Winners)
+			}
+		}
+		//set["winners"] = pInfo.Winners
+		set["s_winner"] = strings.Join(pInfo.Winners, ",")
+	}
+
+	if thisinfo.HasPackage {
+		pkg, _, _ := PackageFormat(thisinfo, pInfo)
+		set["multipackage"] = 1
+		pInfo.Package = pkg
+	}else {
+		set["multipackage"] = 0
+	}
+
+	set["mpn"] = pInfo.MPN
+	set["mpc"] = pInfo.MPC
+	set["pici"] = p.pici
+
+	return set
+}
+
+func deleteSlice1(arr []interface{}, v interface{}) []interface{} {
+	for k, v1 := range arr {
+		if reflect.DeepEqual(v1, v) {
+			return append(arr[:k], arr[k+1:]...)
+		}
+	}
+	return arr
+}
+
+func updateValue(info map[string]interface{}, tmp map[string]interface{}) map[string]interface{} {
+	for k := range info{
+		for k1 := range tmp {
+			if k == k1 && info[k] != tmp[k1] {
+				info[k] = tmp[k]
+				break
+			}
+		}
+	}
+	return info
+}
+
+//备份(快照)
+func backupPro(tmp map[string]interface{}) {
+	tmp1 := make(map[string]interface{})
+	for k, v := range tmp{
+		tmp1[k] = v
+	}
+	if Sysconfig["backupFlag"].(bool) {
+		tmp1["sourceprojectid"] = tmp1["_id"]
+		delete(tmp1, "_id")
+		MongoTool.Save(BackupColl, tmp1)
+	}
+}

src/jy/extract/extract.go

@@ -26,12 +26,12 @@ import (
 var (
 	lock, lockrule, lockclear, locktag, blocktag sync.RWMutex
 
-	cut     = ju.NewCut()                          //获取正文并清理
-	ExtLogs map[*TaskInfo][]map[string]interface{} //抽取日志
-	TaskList      map[string]*ExtractTask          //任务列表
-	ClearTaskList map[string]*ClearTask            //清理任务列表
-	saveLimit     = 100                            //抽取日志批量保存
-	PageSize      = 5000                           //查询分页
+	cut           = ju.NewCut()                          //获取正文并清理
+	ExtLogs       map[*TaskInfo][]map[string]interface{} //抽取日志
+	TaskList      map[string]*ExtractTask                //任务列表
+	ClearTaskList map[string]*ClearTask                  //清理任务列表
+	saveLimit     = 100                                  //抽取日志批量保存
+	PageSize      = 5000                                 //查询分页
 	Fields        = `{"title":1,"summary":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1,"projectinfo":1,"jsondata":1}`
 	Fields2       = `{"budget":1,"bidamount":1,"title":1,"projectname":1,"winner":1}`
 )
@@ -290,11 +290,12 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 		subtype = "all"
 	}
 	toMap := qu.ObjToMap(doc["jsondata"])
-	if toMap != nil {
-		if (*toMap)["extweight"] == nil {
-			(*toMap)["extweight"] = ju.Config["jsondata_extweight"]
-		}
-	}
+	//	log.Debug("toMap", toMap)
+	//	if toMap != nil {
+	//		if (*toMap)["extweight"] == nil {
+	//			(*toMap)["extweight"] = ju.Config["jsondata_extweight"]
+	//		}
+	//	}
 	j = &ju.Job{
 		SourceMid:      qu.BsonIdToSId(doc["_id"]),
 		Category:       toptype,
@@ -1104,6 +1105,7 @@ func extractFromKv(field, fieldname string, blocks []*ju.Block, vc *RuleCore, kv
 						"blocktag":    bl.Classify,
 						"weight":      vv.Weight,
 					})
+					break //暂定取第一个
 				}
 			}
 		}
@@ -1870,7 +1872,7 @@ func (e *ExtractTask) QualityAudit(resulttmp map[string]interface{}) {
 func (e *ExtractTask) RedisMatch(field, fv string, val map[string]interface{}) {
 	defer qu.Catch()
 	i := redis.GetInt(field, field+"_"+fv) //查找redis
-	if i == 0 { //reids未找到，执行规则匹配
+	if i == 0 {                            //reids未找到，执行规则匹配
 		val[field+"_isredis"] = false
 		e.RuleMatch(field, fv, val) //规则匹配
 	} else { //redis找到，打标识存库

src/res/fieldscore.json

@@ -200,7 +200,7 @@
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(附件|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(附件|否决原因|招标失败|注册表|交易中心|序号内容|不足|公告|变更|采购|招标|废标|废止|流标|中标|投标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -10
             },
 			{