fieldproject/tieta_data/project.go

package main

import (
	"context"
	"github.com/google/uuid"
	"go.mongodb.org/mongo-driver/bson"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"math"
	"sort"
	"strings"
	"sync"
	"tieta_data/config"
)

var (
	wg sync.WaitGroup
)

// 从对应map中获取对比的项目id
func getCompareIds(pn, pc, pb string) (bpn, bpc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
	//	p.ConCurrentLock(n1, n2, n3, n4)
	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
	wg.Add(1)
	//查找到id数组
	res = []*Key{}
	//是否查找到，并标识位置。-1代表值为空或已经存在。
	bpn, bpc, bpb = -1, -1, -1
	if pn != "" {
		ids := mapPn[pn]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPn[pn] = ids
			bpn = 0
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pc != "" {
		ids := mapPc[pc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPc[pc] = ids
			bpc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pb != "" {
		ids := mapPb[pb]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPb[pb] = ids
			bpb = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}
	repeatId := map[string]bool{}
	idArr = []string{} //项目id
	IDArr = []*ID{}    //项目信息
	for _, m := range res {
		for _, id := range m.Arr {
			if !repeatId[id] {
				repeatId[id] = true
				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
				AllIdsMapLock.Lock()
				Id := AllIdsMap[id]
				AllIdsMapLock.Unlock()
				if Id != nil {
					Id.Lock.Lock()
					idArr = append(idArr, id)
					IDArr = append(IDArr, Id)
				}
			}
		}
	}
	return
}

func startProjectMerge(info *Info, tmp map[string]interface{}) {
	findLock.Lock()
	defer findLock.Unlock()
	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
	//pids 是项目id数组集合
	//IDArr，是单个项目ID对象集合
	bpn, bpc, bpb, pids, _, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer)))
	defer wg.Done()
	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock

	for _, m := range pids {
		defer m.Lock.Unlock()
	}
	for _, id := range IDArr {
		defer id.Lock.Unlock()
	}

	bFindProject := false
	findPid := ""
	//获取完id，进行计算
	//定义两组
	comRes1 := []*Project{} //优先级最高的对比结果数组
	comRes2 := []*Project{} //优化级其次
	comRes3 := []*Project{}
	for _, v := range IDArr {
		comStr := ""
		compareProject := v.P
		compareProject.score = 0
		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
		//代理机构完全不相同，直接新建项目
		if CheckContain(compareProject.Agency, info.Agency) == 3 {
			continue
		}
		//地区(省、市、区)不同，直接新建项目
		if ComparePlace(compareProject, info) {
			continue
		}
		// 多包 处理
		if ComparePkg(compareProject, info) {
			continue
		}

		compareStr, score := comparePNC(info, compareProject)

		resVal, pjVal := Select(compareStr, info, compareProject)
		//---------------------------------------
		if resVal > 0 {
			compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := compareBCTABB(info, compareProject, diffTime, score)

			//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构
			comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
			compareProject.comStr = comStr
			compareProject.pjVal = pjVal
			compareProject.resVal = resVal
			//log.Println(compareProject.comStr)
			eqV := compareResult(resVal, pjVal, score2, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount)
			if eqV == 1 {
				comRes1 = append(comRes1, compareProject)
			} else if eqV == 2 {
				comRes2 = append(comRes2, compareProject)
			} else if eqV == 3 {
				comRes3 = append(comRes3, compareProject)
			}
		}
	}
	//--------------------------------对比完成-----------------------

	//更新数组、更新项目
	for kv, resN := range [][]*Project{comRes1, comRes2, comRes3} {
		if len(resN) > 0 {
			if len(resN) > 1 {
				sort.Slice(resN, func(i, j int) bool {
					return resN[i].score > resN[j].score
				})
			}

			resArr := []*Project{}
			for _, res := range resN {
				resArr = append(resArr, res)
			}
			if len(resArr) > 0 {
				bFindProject = true
				findPid = resArr[0].ProjectId
				UpdateProject(tmp, info, resArr[0], kv+1, resArr[0].comStr)
				for k2, bv := range []int{bpn, bpc, bpb} {
					if bv > -1 {
						pids[bv].Arr = append(pids[bv].Arr, findPid)
						if k2 == 0 {
							if resArr[0].ProjectName == "" {
								resArr[0].ProjectName = info.ProjectName
							}
						} else if k2 < 3 {
							if resArr[0].ProjectCode == "" {
								resArr[0].ProjectCode = info.ProjectCode
							}
						} else {
							if resArr[0].Buyer == "" {
								resArr[0].Buyer = info.Buyer
							}
						}
					}
				}
			} else {
				bFindProject = false
				findPid = ""
			}
			break
		}
	}

	if !bFindProject {
		id, p1 := NewProject(tmp, info)
		AllIdsMapLock.Lock()
		AllIdsMap[id] = &ID{Id: id, P: p1}
		AllIdsMapLock.Unlock()
		for _, m := range pids {
			m.Arr = append(m.Arr, id)
		}
	}
}

func compareBCTABB(info *Info, cp *Project, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
	compareBuyer = "D"
	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
		v := CheckContain(info.Buyer, cp.Buyer)
		if v == 1 {
			compareBuyer = "A"
			score += 3
		} else {
			//v1 := CosineSimilar(info.Buyer, cp.Buyer)
			if v == 2 {
				compareBuyer = "B"
				score += 1
			} else {
				compareBuyer = "C"
			}
		}
	}
	//---------------------------------------

	compareCity = ""
	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
		compareCity += "A"
		score += 2
	} else if info.Area == "全国" || cp.Area == "全国" {
		compareCity += "B"
		score += 1
	} else {
		compareCity += "C"
	}
	if compareCity != "C" {
		if info.City != "" && info.City == cp.City {
			compareCity += "A"
			score += 2
		} else {
			if info.Area == "全国" || cp.Area == "全国" {
				compareCity += "B"
			} else if info.City == compareCity {
				compareCity += "B"
			} else {
				compareCity += "C"
			}
		}
	} else {
		compareCity += "C"
	}
	score2 = 0
	if compareCity == "AA" {
		if info.District != "" && info.District == cp.District {
			score2 = 1
		}
	}

	compareTime = "D"
	if diffTime < 45*86400 {
		compareTime = "A"
		score += 2
	} else if diffTime < 90*86400 {
		compareTime = "B"
		score += 1
	}

	compareAgency = "D"
	if info.Agency != "" {
		if info.Agency == cp.Agency {
			compareAgency = "A"
			score += 2
			score2 += 1
		} else if cp.Agency != "" {
			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
				compareAgency = "B"
				score += 1
				score2 += 1
			} else {
				compareAgency = "C"
			}
		}
	}
	compareBudget = "C"
	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.15*info.Budget))) {
		compareBudget = "A"
		score += 1
		score2 += 1
	}
	//	else if info.Budget == 0 && cp.Budget == 0 {
	//		compareBudget = "B"
	//	}
	compareBidmount = "C"
	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.15*cp.Budget)) {
		compareBidmount = "A"
		score += 1
		score2 += 1
	}
	//	else if info.Bidamount == 0 && cp.Bidamount == 0 {
	//		compareBidmount = "B"
	//	}

	cp.score = score
	return
}

func compareResult(resVal, pjVal, score2 int, comStr, compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string) int {
	eqV := 0
	switch resVal {
	case 3:
		if pjVal == 3 && comStr[3:] != "CCCDCCC" {
			eqV = 1
		} else if compareBuyer < "C" {
			if pjVal > 1 {
				eqV = 1
			} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
				eqV = 2
			}
		} else if compareBuyer == "D" {
			if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
				eqV = 3
			}
		} else {
			if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
				eqV = 2
			} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
				eqV = 3
			} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
				eqV = 3
			}
		}
	case 2:
		if compareBuyer < "C" {
			if pjVal > 1 {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
				eqV = 3
			}
		} else if compareBuyer == "D" {
			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
				eqV = 3
			}

		} else {
			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
				eqV = 3
			}
		}
	case 1:
		if compareBuyer < "C" {
			if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
				eqV = 3
			}
		} else if compareBuyer == "D" {
			if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
				eqV = 2
			} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 0) && (compareBudget == "A" || compareBidmount == "A") {
				eqV = 3
			}
		} else {
			if pjVal > 1 && compareTime == "A" && score2 > 0 && (compareBudget == "A" || compareBidmount == "A") && compareCity[1:1] != "C" {
				eqV = 3
			}
		}
	}
	return eqV
}

func ComparePlace(project *Project, info *Info) bool {
	if info.Area == "全国" || info.Area == "" {
		return false
	}
	if project.Area == "全国" || project.Area == "" {
		return false
	}
	if info.Area == project.Area {
		if info.City == "" || project.City == "" {
			return false
		} else if info.City == project.City {
			return false
		}
	} else {
		return true
	}
	return true
}

func ComparePkg(project *Project, info *Info) bool {
	if project.MultiPackage == 1 && project.MultiPackageId == info.MultiPackageId {
		return false
	}
	return true
}

func NewCachePinfo(thisinfo *Info, pid string) Project {
	p1 := Project{
		ProjectId:      pid,
		ProjectName:    thisinfo.ProjectName,
		ProjectCode:    thisinfo.ProjectCode,
		Buyer:          thisinfo.Buyer,
		Agency:         thisinfo.Agency,
		Area:           thisinfo.Area,
		City:           thisinfo.City,
		District:       thisinfo.District,
		FirstTime:      thisinfo.Publishtime,
		LastTime:       thisinfo.Publishtime,
		Budget:         thisinfo.Budget,
		Bidamount:      thisinfo.Bidamount,
		MultiPackage:   thisinfo.MultiPackage,
		MultiPackageId: thisinfo.MultiPackageId,
	}
	return p1
}

func NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *Project) {
	pId := uuid.New().String()
	pId = strings.ReplaceAll(pId, "-", "")
	set := map[string]interface{}{}
	set["projectId"] = pId
	set["sourceinfoid"] = thisinfo.Id
	for _, f := range FIELDS {
		if tmp[f] != nil && tmp[f] != "" {
			set[f] = tmp[f]
		}
	}
	if util.ObjToString(tmp["projectname"]) == "" {
		set["projectname"] = thisinfo.ProjectName
	}
	p1 := NewCachePinfo(thisinfo, pId)
	if thisinfo.TopType == "招标" {
		p1.Zbtime = util.Int64All(tmp["publishtime"])
		set["zb_href"] = thisinfo.Href
		set["zb_jybxhref"] = thisinfo.JybxHref
		set["zb_updatetime"] = tmp["publishtime"]
		set["zb_infoid"] = thisinfo.Id
		set["zb_toptype"] = thisinfo.TopType
		set["zb_subtype"] = thisinfo.SubType
	} else if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "合同" || thisinfo.SubType == "验收" ||
		thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
		p1.Jgtime = thisinfo.Publishtime
		set["href"] = thisinfo.Href
		set["jybxhref"] = thisinfo.JybxHref
		set["updatetime"] = tmp["publishtime"]
		set["infoid"] = thisinfo.Id
		set["toptype"] = thisinfo.TopType
		set["subtype"] = thisinfo.SubType
	}
	if len(thisinfo.Winners) > 0 {
		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
		p1.Winners = strings.Join(thisinfo.Winners, ",")

		if util.ObjToString(tmp["winnertel"]) != "" {
			set["winnertel"] = tmp["winnertel"]
		}
		if util.ObjToString(tmp["winnerperson"]) != "" {
			set["winnerperson"] = tmp["winnerperson"]
		}
	}
	if util.ObjToString(tmp["tagname"]) != "" {
		p1.TagName = util.ObjToString(tmp["tagname"])
		set["tagname"] = util.ObjToString(tmp["tagname"])
	}
	if util.ObjToString(tmp["tagname2"]) != "" {
		p1.TagName2 = util.ObjToString(tmp["tagname2"])
		set["tagname2"] = util.ObjToString(tmp["tagname2"])
	}
	if util.ObjToString(tmp["tagname3"]) != "" {
		p1.TagName3 = util.ObjToString(tmp["tagname3"])
		set["tagname3"] = util.ObjToString(tmp["tagname3"])
	}
	if thisinfo.MultiPackage == 0 {
		set["multipackage"] = 0
	} else {
		set["multipackage"] = 1
		set["multipackageid"] = thisinfo.MultiPackageId
	}

	id := MysqlTool.Insert(config.Conf.DB.Mysql.Pcoll, set)
	if id > 0 {
		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"msg_id": thisinfo.MsgId}, bson.M{"projectId": pId})
		client := Es.GetEsConn()
		defer Es.DestoryEsConn(client)
		client.Update().Index(config.Conf.DB.Es.Index).Id(thisinfo.Nid).
			Doc(map[string]interface{}{"projectId": pId}).Refresh("true").Do(context.TODO())
	}
	return pId, &p1
}

func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *Project, weight int, comStr string) {
	set := map[string]interface{}{}
	//相同城市的公告才会合并到一起(全国列外)
	if thisinfo.Area != "全国" {
		pInfo.Area = thisinfo.Area
		set["area"] = thisinfo.Area
		if thisinfo.City != "" {
			pInfo.City = thisinfo.City
			set["city"] = thisinfo.City
		}
		if thisinfo.District != "" {
			pInfo.District = thisinfo.District
			set["district"] = thisinfo.District
		}
	}
	//项目名称
	if pInfo.ProjectName == "" && thisinfo.ProjectName != "" {
		pInfo.ProjectName = thisinfo.ProjectName
		set["projectname"] = thisinfo.ProjectName
	}
	if pInfo.ProjectCode == "" && thisinfo.ProjectCode != "" {
		pInfo.ProjectCode = thisinfo.ProjectCode
		set["projectcode"] = thisinfo.ProjectCode
	}
	//7--采购单位
	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
		pInfo.Buyer = thisinfo.Buyer
		set["buyer"] = thisinfo.Buyer

		pInfo.Buyerclass = thisinfo.Buyerclass
		set["buyerclass"] = thisinfo.Buyerclass
	}
	//采购单位联系人
	if thisinfo.Buyerperson != "" {
		pInfo.Buyerperson = thisinfo.Buyerperson
		set["buyerperson"] = thisinfo.Buyerperson
	}
	//采购单位電話
	if thisinfo.Buyertel != "" {
		pInfo.Buyertel = thisinfo.Buyertel
		set["buyertel"] = pInfo.Buyertel
	}

	//8--代理机构
	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
		pInfo.Agency = thisinfo.Agency
		set["agency"] = thisinfo.Agency
	}
	if util.ObjToString(tmp["agencyperson"]) != "" {
		set["agencyperson"] = tmp["agencyperson"]
	}
	if util.ObjToString(tmp["agencytel"]) != "" {
		set["agencytel"] = tmp["agencytel"]
	}
	if len(thisinfo.Winners) > 0 {
		var winners []string
		if pInfo.Winners == "" {
			for _, k := range thisinfo.Winners {
				if k == "" {
					continue
				}
				if BinarySearch(winners, k) == -1 {
					winners = append(winners, k)
				}
			}
		} else {
			winners = strings.Split(pInfo.Winners, ",")
			for _, k := range thisinfo.Winners {
				if k == "" {
					continue
				}
				if thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
					if BinarySearch(winners, k) != -1 {
						deleteSlice(winners, k, "")
					}
				} else {
					if BinarySearch(winners, k) == -1 {
						winners = append(winners, k)
					}
				}
			}
		}
		set["s_winner"] = strings.Join(util.ObjArrToStringArr(Duplicate(winners)), ",")
		if util.ObjToString(tmp["winnertel"]) != "" {
			set["winnertel"] = tmp["winnertel"]
		}
		if util.ObjToString(tmp["winnerperson"]) != "" {
			set["winnerperson"] = tmp["winnerperson"]
		}
	}
	if thisinfo.TopType == "招标" {
		set["zb_href"] = thisinfo.Href
		set["zb_jybxhref"] = thisinfo.JybxHref
		set["zb_infoid"] = thisinfo.Id
		set["zb_updatetime"] = tmp["publishtime"]
		set["zb_toptype"] = thisinfo.TopType
		set["zb_subtype"] = thisinfo.SubType
	} else if thisinfo.SubType == "中标" || thisinfo.SubType == "成交" || thisinfo.SubType == "合同" || thisinfo.SubType == "验收" ||
		thisinfo.SubType == "流标" || thisinfo.SubType == "废标" {
		set["href"] = thisinfo.Href
		set["jybxhref"] = thisinfo.JybxHref
		set["updatetime"] = tmp["publishtime"]
		set["infoid"] = thisinfo.Id
		set["toptype"] = thisinfo.TopType
		set["subtype"] = thisinfo.SubType
	}

	if util.ObjToString(tmp["tagname"]) != "" && pInfo.TagName != "" {
		pInfo.TagName = util.ObjToString(tmp["tagname"])
		set["tagname"] = util.ObjToString(tmp["tagname"])
	}
	if util.ObjToString(tmp["tagname2"]) != "" && pInfo.TagName2 != "" {
		pInfo.TagName2 = util.ObjToString(tmp["tagname2"])
		set["tagname2"] = util.ObjToString(tmp["tagname2"])
	}
	if util.ObjToString(tmp["tagname3"]) != "" && pInfo.TagName3 != "" {
		pInfo.TagName3 = util.ObjToString(tmp["tagname3"])
		set["tagname3"] = util.ObjToString(tmp["tagname3"])
	}

	if thisinfo.Budget > 0 {
		set["budget"] = thisinfo.Budget
		pInfo.Budget = thisinfo.Budget
	}
	if thisinfo.Bidamount > 0 {
		set["bidamount"] = thisinfo.Bidamount
		pInfo.Bidamount = thisinfo.Bidamount
	}

	b := MysqlTool.Update(config.Conf.DB.Mysql.Pcoll, bson.M{"projectId": pInfo.ProjectId}, set)
	if b {
		MysqlTool.Update(config.Conf.DB.Mysql.Coll, bson.M{"msg_id": thisinfo.MsgId}, bson.M{"projectId": pInfo.ProjectId})
		client := Es.GetEsConn()
		defer Es.DestoryEsConn(client)
		client.Update().Index(config.Conf.DB.Es.Index).Id(thisinfo.Nid).
			Doc(map[string]interface{}{"projectId": pInfo.ProjectId}).Refresh("true").Do(context.TODO())
	}
}