jy-data-extract/fullproject/src_dev1/merge.go

package main

import (
	"encoding/json"
	"log"
	"math"
	qu "qfw/util"
	"qfw/util/redis"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"gopkg.in/mgo.v2/bson"
)

func startProjectMerge(info *Info, tmp map[string]interface{}) {
	//只有或没有采购单位的无法合并
	if info.pnbval < 2 && info.Buyer != "" {
		//extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"]))
		log.Println("invalid", info.pnbval, info.Buyer, info.Id)
	} else {
		bpn, bpc, bptc, bpb, pids := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
		repeatId := map[string]bool{}
		idArr := []string{} //项目id
		IDArr := []*ID{}    //项目信息
		//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock
		for _, m := range pids {
			defer m.Lock.Unlock()
			for _, id := range m.Arr {
				if !repeatId[id] {
					repeatId[id] = true
					_, _ = strconv.ParseInt(id[0:8], 16, 64)
					AllIdsMapLock.Lock()
					//Id := AllIdsMap[int(t)%idsMapSize][id]
					Id := AllIdsMap2[id]
					AllIdsMapLock.Unlock()
					Id.Lock.Lock()
					defer Id.Lock.Unlock()
					idArr = append(idArr, id)
					IDArr = append(IDArr, Id)
				}
			}
		}

		bFindProject := false
		findPid := ""
		//获取完id，进行计算
		if len(idArr) > 0 && len(idArr) == len(IDArr) {
			res := redis.Mget(REDISIDS, idArr)
			//定义两组
			comRes1 := []*ProjectInfo{}
			comRes2 := []*ProjectInfo{}
			comRes3 := []*ProjectInfo{}
			if len(res) == len(idArr) {
				for _, v := range res {
					var cp ProjectInfo
					comStr := ""
					var compareProject *ProjectInfo
					err := json.Unmarshal(v.([]byte), &cp)
					compareProject = &cp
					diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
					if err != nil {
						log.Println("从redis取id信息出错！！！", err)
					} else if diffTime < 360*86400 {
						//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
						compareStr, score := comparePNCB(info, compareProject)
						//if info.ProjectCode == "ZWJLZXQ-2016-0508" {
						//}
						resVal := Select(compareStr, info, compareProject)
						//---------------------------------------
						if resVal > 0 {
							compareBuyer := ""
							if info.Buyer != "" {
								if info.Buyer == compareProject.Buyer {
									compareBuyer += "A"
								} else if compareProject.Buyer != "" {
									if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
										compareBuyer += "B"
									} else {
										compareBuyer += "C"
									}
								} else {
									compareBuyer += "D"
								}
							} else {
								compareBuyer += "D"
							}

							//---------------------------------------
							compareCity := ""
							if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
								compareCity += "A"
								score += 2
							} else if info.Area == "全国" || compareProject.Area == "全国" {
								compareCity += "B"
								score += 1
							} else {
								compareCity += "C"
							}

							if info.City != "" && info.City == compareProject.City {
								compareCity += "A"
								score += 2
							} else {
								if info.Area == "全国" || compareProject.Area == "全国" {
									compareCity += "B"
								} else {
									compareCity += "C"
								}
							}
							compareTime := ""
							//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
							if diffTime < 45*86400 {
								compareTime += "A"
								score += 2
							} else if diffTime < 90*86400 {
								compareTime += "B"
								score += 1
							} else {
								compareTime += "D"
							}
							compareAgency := ""
							if info.Agency != "" {
								if info.Agency == compareProject.Agency {
									compareAgency += "A"
									score += 2
								} else if compareProject.Agency != "" {
									if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
										compareAgency += "B"
										score += 1
									} else {
										compareAgency += "C"
									}
								} else {
									compareAgency += "D"
								}
							} else {
								compareAgency += "D"
							}

							//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构

							comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency
							compareProject.comStr = comStr
							//							log.Println(resVal, info.ProjectName, info.ProjectCode, info.Buyer, compareProject.Id, compareProject.ProjectName, compareProject.ProjectCode, compareProject.Buyer, compareProject.MPC, compareStr)
							//							log.Println(compareBuyer, compareCity, compareAgency, compareTime)
							switch resVal {
							case 3:
								if compareBuyer == "A" || compareBuyer == "B" {
									comRes1 = append(comRes1, compareProject)
								} else if compareBuyer == "D" {
									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
										comRes1 = append(comRes1, compareProject)

									} else {
										comRes2 = append(comRes2, compareProject)

									}
								} else {
									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
										comRes2 = append(comRes2, compareProject)

									} else if compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D" {
										comRes3 = append(comRes3, compareProject)
									}
								}
							case 1, 2:

								if compareBuyer == "A" {
									comRes1 = append(comRes1, compareProject)
								} else if compareBuyer == "B" {
									comRes2 = append(comRes2, compareProject)

								} else if compareBuyer == "D" {
									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
										comRes2 = append(comRes2, compareProject)
									} else {
										comRes3 = append(comRes3, compareProject)
									}
								} else {
									if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
										comRes3 = append(comRes3, compareProject)
									}
								}
							}
						}
					}
				}
				//--------------------------------对比完成-----------------------
				for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
					if len(resN) > 0 {
						if len(resN) > 1 {
							sort.Slice(comRes1, func(i, j int) bool {
								return comRes1[i].score > comRes1[j].score
							})
						}

						bFindProject = true
						findPid = resN[0].Id
						for k2, bv := range []int{bpn, bpc, bptc, bpb} {
							if bv > -1 {
								pids[bv].Arr = append(pids[bv].Arr, findPid)
								if k2 == 0 {
									if resN[0].ProjectName == "" {
										resN[0].ProjectName = info.ProjectName
									} else {
										if resN[0].MPN == nil {
											resN[0].MPN = []string{info.ProjectName}
										} else {
											resN[0].MPN = append(resN[0].MPN, info.ProjectName)
										}
									}

								} else if k2 < 3 {
									if resN[0].ProjectCode == "" {
										resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
									} else {
										if resN[0].MPC == nil {
											resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
										} else {
											resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
										}
									}

								} else {
									if resN[0].Buyer == "" {
										resN[0].Buyer = info.Buyer
									}
								}
							}
						}
						UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
						//更新AllIdsMao中的时间
						redis.Put(REDISIDS, findPid, resN[0], 0)
						break
					}
				}
			} else {
				log.Println("redis记录缺失！！！")
			}
		}

		if !bFindProject {
			//没有找到
			id := NewProject(tmp, info)
			_, _ = strconv.ParseInt(id[0:8], 16, 64)
			AllIdsMapLock.Lock()
			//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
			AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime}
			AllIdsMapLock.Unlock()
			for _, m := range pids {
				m.Arr = append(m.Arr, id)
			}
		}
	}
}

func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) {
	//log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr)
	//	updateid := pInfo.Id
	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
		return //updateid
	}
	set := map[string]interface{}{}
	res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`)
	EqInfoUpdate(thisinfo, pInfo)
	if bres && res != nil && *res != nil {
		set["topscopeclass"] = pInfo.Topscopeclass
		set["subscopeclass"] = pInfo.Subscopeclass
		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
		if len(s_subscopeclass) > 0 {
			s_subscopeclass = "," + s_subscopeclass + ","
		}
		set["s_subscopeclass"] = s_subscopeclass
		s_winner := strings.Join(pInfo.Winners, ",")
		if len(s_winner) > 0 {
			s_winner = "," + s_winner + ","
		}
		set["s_winner"] = s_winner
		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
			set["buyerperson"] = pInfo.Buyerperson
			set["buyertel"] = pInfo.Buyertel
		}
		if pInfo.Buyerclass != "" {
			set["buyerclass"] = pInfo.Buyerclass
		}
		if pInfo.District != "" {
			set["district"] = pInfo.District
		}
		if pInfo.Bidopentime > 0 {
			set["bidopentime"] = pInfo.Bidopentime
		}
		if len(pInfo.Winnerorder) > 0 {
			set["winnerorder"] = pInfo.Winnerorder
		}
		if thisinfo.HasPackage {
			set["multipackage"] = 1
		} else {
			set["multipackage"] = 0
		}

		if pInfo.ProjectName != "" {
			set["projectname"] = pInfo.ProjectName
		}

		if pInfo.ProjectCode != "" {
			set["projectcode"] = pInfo.ProjectCode
		}

		if pInfo.Buyer != "" {
			set["buyer"] = pInfo.Buyer
		}

		//预算、中标价
		if qu.Float64All(tmp["bidamount"]) > 0 {
			if qu.Float64All((*res)["bidamount"]) == 0 {
				set["bidamount"] = tmp["bidamount"]
			}
		}

		if qu.Float64All(tmp["budget"]) > 0 {
			if qu.Float64All((*res)["budget"]) == 0 {
				set["budget"] = tmp["budget"]
			}
		}

		set["mpn"] = pInfo.MPN
		set["mpc"] = pInfo.MPC
		set["area"] = pInfo.Area
		set["city"] = pInfo.City

		//e := InitEL(util.ObjToString((*res)["extractpos"]))
		if thisinfo.dealtype == 1 {
			var sonpackage map[string]interface{}
			for _, obj := range tmp["package"].(map[string]interface{}) {
				sonpackage, _ = obj.(map[string]interface{})
			}
			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
				if sonpackage[v2] != nil {
					tmp[v2] = sonpackage[v2]
				}
			}
		}
		//e.fieldpriority(&tmp, res, &set)
		//set["extractpos"] = e.GetVal()
		if thisinfo.HasPackage { //多包处理
			p1, _ := (*res)["package"].(map[string]interface{})
			p2, _ := tmp["package"].(map[string]interface{})
			if p2 != nil {
				if p1 != nil {
					for pk2, pv2 := range p2 {
						if p1[pk2] != nil { //合并
							item1, _ := p1[pk2].(map[string]interface{})
							item2, _ := pv2.(map[string]interface{})
							if item1 != nil && item2 != nil { //原始项
								for ik1, iv1 := range item2 {
									if item1[ik1] == nil {
										item1[ik1] = iv1
									}
								}
							}
						} else {
							p1[pk2] = pv2
						}
					}
				} else {
					p1 = p2
				}
			}
			set["package"] = p1
		}
		//中标候选人合并

		update := map[string]interface{}{}
		if len(set) > 0 {
			update["$set"] = set
		}
		//保留原数据吧
		push := NewPushInfo(tmp)
		push["compareStr"] = comStr
		update["$push"] = map[string]interface{}{
			"list": push,
		}
		if len(update) > 0 {
			MQFW.Update(projectColl, map[string]interface{}{
				"_id": qu.StringTOBsonId(pInfo.Id),
			}, &update, false, false)
		}
	}
	//再往redis中放 index
	//往队列中增加时间 -------------->start
}

func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
	var tk int
	bf1 := false
	for _k, tv := range pInfo.Publistime {
		tk = _k
		if tv > thisinfo.Publishtime {
			bf1 = true
			break
		}
	}
	if bf1 {
		pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...)
		//pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...)
		//pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...)
	} else {
		pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime)
		//pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType})
	}
	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
	//增加发布时间结束----------------->end

	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
		pInfo.Buyer = thisinfo.Buyer
	}
	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
		pInfo.Agency = thisinfo.Agency
	}
	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
		pInfo.ProjectCode = thisinfo.ProjectCode
	}

	if pInfo.Area == "全国" && thisinfo.Area != "全国" {
		pInfo.Area = thisinfo.Area
		pInfo.City = thisinfo.City
	}
	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
		pInfo.Buyerperson = thisinfo.Buyerperson
		pInfo.Buyertel = thisinfo.Buyertel
	}
	if thisinfo.Buyerclass != "" {
		pInfo.Buyerclass = thisinfo.Buyerclass
	}
	if thisinfo.District != "" {
		pInfo.District = thisinfo.District
	}
	if thisinfo.Bidopentime > 0 {
		pInfo.Bidopentime = thisinfo.Bidopentime
	}
	if len(thisinfo.Topscopeclass) > 0 {
		sort.Strings(pInfo.Topscopeclass)
		for _, k := range thisinfo.Topscopeclass {
			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
				sort.Strings(pInfo.Topscopeclass)
			}
		}
	}

	if len(thisinfo.Subscopeclass) > 0 {
		sort.Strings(pInfo.Subscopeclass)
		for _, k := range thisinfo.Subscopeclass {
			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
				sort.Strings(pInfo.Subscopeclass)
			}
		}
	}
	//winner
	if len(thisinfo.Winners) > 0 {
		sort.Strings(pInfo.Winners)
		for _, k := range thisinfo.Winners {
			if BinarySearch(pInfo.Winners, k) == -1 {
				pInfo.Winners = append(pInfo.Winners, k)
				sort.Strings(pInfo.Winners)
			}
		}
	}
	//winnerorder
	if len(thisinfo.Winnerorder) > 0 {
		sort.Strings(pInfo.Winnerorder)
		for _, k := range thisinfo.Winnerorder {
			if BinarySearch(pInfo.Winnerorder, k) == -1 {
				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
				sort.Strings(pInfo.Winnerorder)
			}
		}
	}
}

//二分字符串查找
func BinarySearch(s []string, k string) int {
	sort.Strings(s)
	lo, hi := 0, len(s)-1
	for lo <= hi {
		m := (lo + hi) >> 1
		if s[m] < k {
			lo = m + 1
		} else if s[m] > k {
			hi = m - 1
		} else {
			return m
		}
	}
	return -1
}

var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}

func NewProject(tmp map[string]interface{}, thisinfo *Info) string {

	set := map[string]interface{}{}
	for _, f := range FIELDS {
		if tmp[f] != nil {
			set[f] = tmp[f]
		}
	}
	set["s_projectname"] = thisinfo.ProjectName
	set["createtime"] = time.Now().Unix()
	set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"])
	set["sourceinfourl"] = tmp["href"]
	set["topscopeclass"] = thisinfo.Topscopeclass
	set["subscopeclass"] = thisinfo.Subscopeclass
	if thisinfo.Buyerperson != "" {
		set["buyerperson"] = thisinfo.Buyerperson
	}
	if thisinfo.Buyertel != "" {
		set["buyertel"] = thisinfo.Buyertel
	}
	if thisinfo.Buyerclass != "" {
		set["buyertel"] = thisinfo.Buyerclass
	}
	if thisinfo.Bidopentime > 0 {
		set["bidopentime"] = thisinfo.Bidopentime
	}
	if len(thisinfo.Winnerorder) > 0 {
		set["winnerorder"] = thisinfo.Winnerorder
	}
	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
	set["s_subscopeclass"] = s_subscopeclass
	s_winner := strings.Join(thisinfo.Winners, ",")
	set["s_winner"] = s_winner
	if tmp["package"] != nil {
		set["package"] = tmp["package"] //没定义优先级
	}
	push := NewPushInfo(tmp)
	set["list"] = []bson.M{
		push,
	}

	id := MQFW.Save(projectColl, set)
	p1 := NewPinfo(id, thisinfo)
	redis.PutCKV(REDISIDS, id, p1)

	return id
}

//生成存放在redis数组中的对象
func NewPinfo(id string, thisinfo *Info) ProjectInfo {
	p1 := ProjectInfo{
		Publistime:    []int64{thisinfo.Publishtime},
		InfoType:      [][]string{[]string{thisinfo.TopType, thisinfo.SubType}},
		Id:            id,
		Ids:           []string{thisinfo.Id},
		Topscopeclass: thisinfo.Topscopeclass,
		Subscopeclass: thisinfo.Subscopeclass,
		Winners:       thisinfo.Winners,
		ProjectName:   thisinfo.ProjectName,
		ProjectCode:   thisinfo.ProjectCode,
		Buyer:         thisinfo.Buyer,
		Agency:        thisinfo.Agency,
		Area:          thisinfo.Area,
		City:          thisinfo.City,
		District:      thisinfo.District,
		MPN:           []string{},
		MPC:           []string{},
		HasPackage:    thisinfo.HasPackage,
		Package:       map[string]interface{}{},
		Buyerclass:    thisinfo.Buyerclass,
		Bidopentime:   thisinfo.Bidopentime,
		Winnerorder:   thisinfo.Winnerorder,
	}
	if thisinfo.LenPTC > 5 {
		p1.MPC = append(p1.MPC, thisinfo.PTC)
	}
	for k4, _ := range thisinfo.Package {
		p1.Package[k4] = ""
	}
	return p1
}

type IdsMapAndIndex struct {
	Key   *Key
	Index int
}

var sm = sync.Map{}

var ALock = sync.Mutex{}

var mapPn = map[string]*Key{}
var mapPc = map[string]*Key{}
var mapPb = map[string]*Key{}

//var mapPnLock, mapPcLock, mapPbLock = sync.Mutex{}, sync.Mutex{}, sync.Mutex{}

//从对应map中获取对比的项目id
func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key) {
	ALock.Lock()
	defer ALock.Unlock()
	res = []*Key{}
	bpn, bpc, bptc, bpb = -1, -1, -1, -1
	if pn != "" {
		//mapPnLock.Lock()
		ids := mapPn[pn]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPn[pn] = ids
			bpn = 0
		}
		ids.Lock.Lock()
		res = append(res, ids)
		//mapPnLock.Unlock()

		//		index := len(pn) % size
		//		m := AllPNMap[index]
		//		m.Lock.Lock()
		//		ids := m.Map[pn]
		//		//log.Println("PN", ids, pn, index)
		//		if ids == nil {
		//			ids = &Key{Arr: []string{}}
		//			m.Map[pn] = ids
		//			bpn = 0
		//		}
		//		ids.Lock.Lock()
		//		res = append(res, ids)
		//		m.Lock.Unlock()
	}

	if pc != "" {
		//mapPcLock.Lock()
		ids := mapPc[pc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPc[pc] = ids
			bpc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
		//mapPcLock.Unlock()
		//		index := len(pc) % size
		//		m := AllPCMap[index]
		//		m.Lock.Lock()
		//		ids := m.Map[pc]
		//		//log.Println("PC", ids, pc, index)
		//		if ids == nil {
		//			ids = &Key{Arr: []string{}}
		//			m.Map[pc] = ids
		//			bpc = len(res)
		//		}
		//		ids.Lock.Lock()
		//		res = append(res, ids)
		//		m.Lock.Unlock()
	}

	if ptc != "" {
		ids := mapPc[ptc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPc[ptc] = ids
			bptc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
		//		index := len(ptc) % size
		//		m := AllPTCMap[index]
		//		m.Lock.Lock()
		//		ids := m.Map[ptc]
		//		//log.Println("PTC", ids, ptc, index)
		//		if ids == nil {
		//			ids = &Key{Arr: []string{}}
		//			m.Map[ptc] = ids
		//			bptc = len(res)
		//		}
		//		ids.Lock.Lock()
		//		res = append(res, ids)
		//		m.Lock.Unlock()
	}

	if pb != "" {
		ids := mapPb[pb]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPb[pb] = ids
			bpb = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
		//		index := len(pb) % size
		//		m := AllPBMap[index]
		//		m.Lock.Lock()
		//		ids := m.Map[pb]
		//		//log.Println("PB", ids, pb, index)
		//		if ids == nil {
		//			ids = &Key{Arr: []string{}}
		//			m.Map[pb] = ids
		//			bpb = len(res)
		//		}
		//		ids.Lock.Lock()
		//		res = append(res, ids)
		//		m.Lock.Unlock()
	}

	return
}

//抽取信息打标记
func extInfoTag(sflag, id string) {
	MQFW.UpdateById(extractColl, id,
		map[string]interface{}{
			"$set": map[string]interface{}{
				"meger_sflag": sflag,
			},
		})
}