jy-data-extract/fullproject/src_dev2/merge.go

package main

import (
	"encoding/json"
	"log"
	"math"
	qu "qfw/util"
	"qfw/util/redis"
	"sort"
	//"strconv"
	"strings"
	"sync"
)

//单线程控制查找
var findLock = sync.Mutex{}

//从对应map中获取对比的项目id
func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
	findLock.Lock()
	defer findLock.Unlock()
	res = []*Key{}
	//是否查找到，并标识位置。-1代表未查找到。
	bpn, bpc, bptc, bpb = -1, -1, -1, -1
	if pn != "" {
		ids := mapPn[pn]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPn[pn] = ids
			bpn = 0
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pc != "" {
		ids := mapPc[pc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPc[pc] = ids
			bpc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if ptc != "" {
		ids := mapPc[ptc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPc[ptc] = ids
			bptc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pb != "" {
		ids := mapPb[pb]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			mapPb[pb] = ids
			bpb = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}
	repeatId := map[string]bool{}
	idArr = []string{} //项目id
	IDArr = []*ID{}    //项目信息
	for _, m := range res {
		for _, id := range m.Arr {
			if !repeatId[id] {
				repeatId[id] = true
				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
				AllIdsMapLock.Lock()
				Id := AllIdsMap2[id]
				AllIdsMapLock.Unlock()
				Id.Lock.Lock()
				idArr = append(idArr, id)
				IDArr = append(IDArr, Id)
			}
		}
	}
	return
}

//项目合并入口
func startProjectMerge(info *Info, tmp map[string]interface{}) {
	//只有或没有采购单位的无法合并
	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
	//pids 是项目id数组集合
	//IDArr，是单个项目ID对象集合
	bpn, bpc, bptc, bpb, pids, idArr, IDArr := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock

	for _, m := range pids {
		defer m.Lock.Unlock()
	}
	for _, id := range IDArr {
		defer id.Lock.Unlock()
	}

	bFindProject := false
	findPid := ""
	//获取完id，进行计算
	if len(idArr) > 0 && len(idArr) == len(IDArr) {
		res := redis.Mget(ProjectCache, idArr)
		//定义两组
		comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
		comRes2 := []*ProjectInfo{} //优化级其次
		comRes3 := []*ProjectInfo{}
		if len(res) == len(idArr) {
			for _, v := range res {
				var cp ProjectInfo
				comStr := ""
				var compareProject *ProjectInfo
				err := json.Unmarshal(v.([]byte), &cp)
				compareProject = &cp
				//问题出地LastTime!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
				diffTime := math.Abs(float64(info.Publishtime - compareProject.LastTime))
				if err != nil {
					log.Println("从redis取id信息出错！！！", err)
				} else if diffTime < 360*86400 {
					//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
					compareStr, score := comparePNCB(info, compareProject)
					//if info.ProjectCode == "ZWJLZXQ-2016-0508" {
					//log.Println(compareStr)
					//}
					resVal := Select(compareStr, info, compareProject)
					//---------------------------------------
					if resVal > 0 {
						compareBuyer := "D"
						if info.Buyer != "" {
							if info.Buyer == compareProject.Buyer {
								compareBuyer = "A"
								score += 3
							} else if compareProject.Buyer != "" {
								if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) {
									compareBuyer = "B"
									score += 1
								} else {
									compareBuyer = "C"
								}
							}
						}

						//---------------------------------------

						compareCity := ""
						if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area {
							compareCity += "A"
							score += 2
						} else if info.Area == "全国" || compareProject.Area == "全国" {
							compareCity += "B"
							score += 1
						} else {
							compareCity += "C"
						}
						if compareCity != "C" {
							if info.City != "" && info.City == compareProject.City {
								compareCity += "A"
								score += 2
							} else {
								if info.Area == "全国" || compareProject.Area == "全国" {
									compareCity += "B"
								} else if info.City == compareCity {
									compareCity += "B"
								} else {
									compareCity += "C"
								}
							}
						} else {
							compareCity += "C"
						}

						compareTime := "D"
						//diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1]))
						if diffTime < 45*86400 {
							compareTime = "A"
							score += 2
						} else if diffTime < 90*86400 {
							compareTime = "B"
							score += 1
						}
						score2 := 0
						compareAgency := "D"
						if info.Agency != "" {
							if info.Agency == compareProject.Agency {
								compareAgency = "A"
								score += 2
								score2 += 1
							} else if compareProject.Agency != "" {
								if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) {
									compareAgency = "B"
									score += 1
									score2 += 1
								} else {
									compareAgency = "C"
								}
							}
						}
						compareBudget := "C"
						if info.budget > 0 && (info.budget == compareProject.Budget || (compareProject.Bidamount > 0 && info.budget > compareProject.Bidamount && (info.budget-compareProject.Bidamount) < (0.1*info.budget))) {
							compareBudget = "A"
							score += 1
							score2 += 1
						}
						compareBidmount := "C"
						if info.bidamount > 0 && (info.bidamount == compareProject.Bidamount || (compareProject.Budget > 0 && compareProject.Budget > info.bidamount && (compareProject.Budget-info.bidamount) < 0.1*compareProject.Budget)) {
							compareBidmount = "A"
							score += 1
							score2 += 1
						}

						//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构

						comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
						compareProject.comStr = comStr
						eqV := 0
						switch resVal {
						case 3:
							if compareBuyer < "C" {
								eqV = 1
							} else if compareBuyer == "D" {
								if "AA" == compareCity && compareAgency != "C" && compareTime != "D" && score2 > 0 {
									eqV = 2
								} else if compareCity[1:1] != "C" && score2 > 0 && compareTime == "A" {
									eqV = 3
								} else if compareTime != "D" && compareAgency != "C" && score2 > 1 {
									eqV = 3
								}
							} else {
								if "AA" == compareCity && compareAgency != "C" && compareTime != "D" {
									eqV = 2
								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
									eqV = 3
								}
							}

						case 2:
							if compareBuyer < "C" {
								if compareTime != "D" && "AA" == compareCity && score2 > 0 {
									eqV = 2
								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
									eqV = 3
								}
							} else if compareBuyer == "D" {
								if "AA" == compareCity && compareTime != "D" && score2 > 1 {
									eqV = 3
								} else if score2 > 2 && compareTime == "A" {
									eqV = 3
								}
							} else {
								if "AA" == compareCity && (compareAgency == "A" || score2 > 1) && compareTime == "A" {
									eqV = 3
								}
							}
						case 1:
							if compareBuyer < "C" {
								if compareTime != "D" && "AA" == compareCity && score2 > 1 {
									eqV = 2
								} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 1 {
									eqV = 3
								}
							} else if compareBuyer == "D" {
								if "AA" == compareCity && compareTime != "D" && score2 > 2 {
									eqV = 3
								} else if compareCity[1:1] != "C" && score2 > 2 && compareTime == "A" {
									eqV = 3
								}
							} else {
								if "AA" == compareCity && score2 > 2 && compareTime == "A" {
									eqV = 3
								}
							}
						}
						if eqV == 1 {
							comRes1 = append(comRes1, compareProject)
						} else if eqV == 2 {
							comRes2 = append(comRes2, compareProject)
						} else if eqV == 3 {
							comRes3 = append(comRes3, compareProject)
						} else {
							log.Println("+++++++++++", resVal, comStr, info, compareProject)
						}
					}
				}
			}
			//--------------------------------对比完成-----------------------
			//更新数组、更新项目
			for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
				if len(resN) > 0 {
					if len(resN) > 1 {
						sort.Slice(resN, func(i, j int) bool {
							return resN[i].score > resN[j].score
						})
					}

					bFindProject = true
					findPid = resN[0].Id.Hex()
					for k2, bv := range []int{bpn, bpc, bptc, bpb} {
						if bv > -1 {
							pids[bv].Arr = append(pids[bv].Arr, findPid)
							if k2 == 0 {
								if resN[0].ProjectName == "" {
									resN[0].ProjectName = info.ProjectName
								} else {
									if resN[0].MPN == nil {
										resN[0].MPN = []string{info.ProjectName}
									} else {
										resN[0].MPN = append(resN[0].MPN, info.ProjectName)
									}
								}

							} else if k2 < 3 {
								if resN[0].ProjectCode == "" {
									resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
								} else {
									if resN[0].MPC == nil {
										resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
									} else {
										resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
									}
								}

							} else {
								if resN[0].Buyer == "" {
									resN[0].Buyer = info.Buyer
								}
							}
						}
					}
					UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr)
					//更新AllIdsMao中的时间
					redis.Put(ProjectCache, findPid, resN[0], 0)
					break
				}
			}
		} else {
			log.Println("redis记录缺失！！！")
		}
	}

	if !bFindProject {
		//没有找到
		id := NewProject(tmp, info)
		//_, _ = strconv.ParseInt(id[0:8], 16, 64)
		AllIdsMapLock.Lock()
		//AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime}
		AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime}
		AllIdsMapLock.Unlock()
		for _, m := range pids {
			m.Arr = append(m.Arr, id)
		}
	}

}

//二分字符串查找
func BinarySearch(s []string, k string) int {
	sort.Strings(s)
	lo, hi := 0, len(s)-1
	for lo <= hi {
		m := (lo + hi) >> 1
		if s[m] < k {
			lo = m + 1
		} else if s[m] > k {
			hi = m - 1
		} else {
			return m
		}
	}
	return -1
}

var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"}