jy-data-extract/udpprojectset/src/projectmeger.go

// projectmeger
package main

import (
	"encoding/json"
	"fmt"
	du "jy/util"
	"log"
	qu "qfw/util"
	"qfw/util/redis"
	"sort"
	"strings"
	"sync"
	"time"
)

//有效值三选一、三选二
var ThreeToTow, ThreeToOne map[string]bool
var ThreeToLock = &sync.Mutex{}

func init() {
	ThreeToTow = map[string]bool{}
	ThreeToOne = map[string]bool{
		"AAA": true,
		"AAB": true,
	}
	tows := [][][]string{
		[][]string{
			[]string{"A", "B"},
			[]string{"A", "B"},
			[]string{"A", "B"},
		},
		[][]string{
			[]string{"A", "B"},
			[]string{"D", "E"},
			[]string{"A", "B"},
		},
		[][]string{
			[]string{"A", "B"},
			[]string{"A", "B"},
			[]string{"D", "E"},
		},
		[][]string{
			[]string{"D", "E"},
			[]string{"A", "B"},
			[]string{"A", "B"},
		},
	}
	for _, tow := range tows {
		for _, v1 := range tow[0] {
			for _, v2 := range tow[1] {
				for _, v3 := range tow[2] {
					key := fmt.Sprintf("%s%s%s", v1, v2, v3)
					ThreeToTow[key] = true
				}
			}
		}
	}
}

func startProjectMerge(thisinfo *Info, tmp map[string]interface{}) {
	bNormalScore := false
	pcbv := PCBVal(tmp)
	if checkInfoAlter(tmp) { //进入变更信息流程
		if pcbv.Val > 1 { //判断三项至少包含两项
			bNormalScore = true
		} else {
			extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"])) //无效信息，打标记
			//go IS.Add("invalid")                              //数据统计使用
			return
		}
	} else {
		bNormalScore = true
	}
	//合并流程
	if bNormalScore {
		if pcbv.Buyer { //有采购单位
			hasBuyer(pcbv, thisinfo, tmp)
		} else { //无采购单位
			noBuyer(pcbv, thisinfo, tmp)
		}
	}
}

//判断信息是否是变更
func checkInfoAlter(tmp map[string]interface{} /*新信息*/) bool {
	toptype := qu.ObjToString(tmp["toptype"])
	subtype := qu.ObjToString(tmp["subtype"])
	title := qu.ObjToString(tmp["title"])
	if subtype == "变更" || strings.Index(title, "变更公告") > -1 || strings.Index(title, "更正公告") > -1 {
		//当信息类型是变更或标题中含变更时
		if toptype == "招标" {
			//招标的变更公告，不作处理
		} else if toptype == "结果" {
			subtype = "变更"
		}
	}
	return subtype == "变更"
}

//有采购单位
func hasBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) {
	var pncb []*CompareInfo
	var res []interface{}
	sflag := ""
	pici := time.Now().Unix()
	if p.Pname || p.Pcode { //有项目名称或项目编号
		//获取对比项目数组
		res, pncb = getComeperProjects(p, thisinfo)
		//三选二打分
		scores := score3Select2(p, thisinfo, tmp, res, pncb)
		//项目合并
		sflag = mergeProject(tmp, thisinfo, scores, pncb)
	} else {
		//生成项目，不参与后续对比
		sflag = "alone"
		mess := map[string]interface{}{
			"meger_mess":  "有采购单位,不满足三选二",
			"meger_sflag": sflag,
		}
		newProject(tmp, mess, pici, thisinfo)
	}
	extInfoTag(sflag, thisinfo.Id)
	go IS.Add(sflag) //数据统计使用
}

//无采购单位
func noBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) {
	var pncb []*CompareInfo
	var res []interface{}
	sflag := ""
	pici := time.Now().Unix()
	if p.Pname { //有项目名称
		//获取对比项目数组
		res, pncb = getComeperProjects(p, thisinfo)
		if p.Pcode { //有项目编号
			//三选二打分
			scores := score3Select2(p, thisinfo, tmp, res, pncb)
			//项目合并
			sflag = mergeProject(tmp, thisinfo, scores, pncb)
		} else { //无项目编号
			if p.PnameLen > MegerFieldsLen.ProjectNamelen {
				if p.Area && p.City && p.Agency {
					//三选一打分
					scores := score3Select1(p, thisinfo, tmp, res, pncb)
					//项目合并
					sflag = mergeProject(tmp, thisinfo, scores, pncb)
				} else {
					sflag = "alone"
					mess := map[string]interface{}{
						"meger_mess":  "无采购单位,不满足三选一",
						"meger_sflag": sflag,
					}
					newProject(tmp, mess, pici, thisinfo)
				}
			} else {
				//生成项目，不参与后续对比
				sflag = "alone"
				mess := map[string]interface{}{
					"meger_mess":  "无采购单位,不满足三选一",
					"meger_sflag": sflag,
				}
				newProject(tmp, mess, pici, thisinfo)
			}
		}
	} else { //无项目名称
		if p.Pcode {
			//获取对比项目数组
			res, pncb = getComeperProjects(p, thisinfo)
			if p.PcodeLen > MegerFieldsLen.ProjectCodelen {
				if p.Area && p.City && p.Agency { //有省市代理机构
					//三选一打分
					scores := score3Select1(p, thisinfo, tmp, res, pncb)
					//项目合并
					sflag = mergeProject(tmp, thisinfo, scores, pncb)
				} else {
					//生成项目，不参与后续对比
					sflag = "alone"
					mess := map[string]interface{}{
						"meger_mess":  "无采购单位,不满足三选一",
						"meger_sflag": sflag,
					}
					newProject(tmp, mess, pici, thisinfo)
				}
			} else {
				//生成项目，不参与后续对比
				sflag = "alone"
				mess := map[string]interface{}{
					"meger_mess":  "无采购单位,不满足三选一",
					"meger_sflag": sflag,
				}
				newProject(tmp, mess, pici, thisinfo)
			}
		} else {
			//信息无效,打标记
			sflag = "invalid"
		}
	}
	extInfoTag(sflag, thisinfo.Id)
	//go IS.Add(sflag) //数据统计使用
}

//3选2打分
func score3Select2(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) {
	defer qu.Catch()
	if len(res) > 0 {
		//对比打分
		for k, tmps := range res { //遍历对象数组
			if tmp, ok := tmps.([]interface{}); ok {
				for _, b1 := range tmp {
					if b1 != nil {
						var info ProjectInfo
						err := json.Unmarshal(b1.([]byte), &info)
						if err != nil {
							log.Println(err)
						}
						cone := &CompareOne{
							Parent: pncb[k],
							Pinfo:  &info,
						}
						cone.BuyerType, cone.Score = fieldPCBScore(thisinfo.Buyer, info.Buyer, cone.BuyerType, cone.Score)
						if p.Buyer {
							cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
							cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
						} else { //无采购单位，打分考虑长度
							if len([]rune(thisinfo.ProjectName)) > MegerFieldsLen.ProjectNamelen {
								cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score)
							} else {
								cone.ProjectNameType = "D"
							}
							if len(thisinfo.ProjectCode) > MegerFieldsLen.ProjectCodelen {
								cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score)
							} else {
								cone.ProjectCodeType = "D"
							}
						}
						//省市打分
						if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
							if thisinfo.Area == info.Area && thisinfo.City == info.City {
								cone.Score += 2
							} else {
								cone.Score -= 1
							}
						} else {
							cone.Score += 1
						}
						//代理机构打分
						if len([]rune(info.Agency)) > 0 {
							if thisinfo.Agency == info.Agency { //A
								cone.Score += 2
							} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
								cone.Score += 1
							} else {
								if len(thisinfo.Agency) < 1 { //E
									cone.Score -= 1
								} else { //C
									cone.Score -= 2
								}
							}
						} else { //D不计分
							//
						}
						skey := fmt.Sprintf("%s%s%s", cone.BuyerType, cone.ProjectNameType, cone.ProjectCodeType)
						cone.Cresult = skey
						ThreeToLock.Lock()
						if ThreeToTow[skey] {
							scores = append(scores, cone)
						}
						ThreeToLock.Unlock()
					}
				}
			}
		}
	}
	return
}

//3选1打分
func score3Select1(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) {
	defer qu.Catch()
	//对比打分
	if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时
		for k, tmps := range res { //遍历对象数组
			if tmp, ok := tmps.([]interface{}); ok {
				for _, b1 := range tmp {
					if b1 != nil {
						var info ProjectInfo
						err := json.Unmarshal(b1.([]byte), &info)
						if err != nil {
							log.Println(err)
						}
						cone := &CompareOne{
							Parent: pncb[k],
							Pinfo:  &info,
						}
						if pncb[k].Field == "pn" { //比较字段项目名称
							if len(info.ProjectName) > 0 {
								if thisinfo.ProjectName == info.ProjectName { //A
									cone.Score += 2
									cone.ProjectNameType = "A"
								} else if strings.Index(info.ProjectName, thisinfo.ProjectName) > -1 || strings.Index(thisinfo.ProjectName, info.ProjectName) > -1 { //B
									cone.Score += 1
									cone.ProjectNameType = "B"
								} else { //C
									cone.Score -= 2
									cone.ProjectNameType = "C"
								}
							} else { //D不计分
								cone.ProjectNameType = "D"
							}
						}
						if pncb[k].Field == "pc" { //比较字段项目编号
							if len(info.ProjectCode) > 0 {
								if thisinfo.ProjectCode == info.ProjectCode { //A
									cone.Score += 2
									cone.ProjectCodeType = "A"
								} else if strings.Index(info.ProjectCode, thisinfo.ProjectCode) > -1 || strings.Index(thisinfo.ProjectCode, info.ProjectCode) > -1 { //B
									cone.Score += 1
									cone.ProjectCodeType = "B"
								} else { //C
									cone.Score -= 2
									cone.ProjectCodeType = "C"
								}
							} else { //D不计分
								cone.ProjectCodeType = "D"
							}
						}
						if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" {
							if thisinfo.Area == info.Area && thisinfo.City == info.City {
								cone.Score += 2
								cone.AreaType = "A"
							} else {
								cone.Score -= 1
								cone.AreaType = "C"
							}
						} else {
							cone.Score += 1
							cone.AreaType = "B"
						}
						if len([]rune(info.Agency)) > 0 {
							if thisinfo.Agency == info.Agency { //A
								cone.Score += 2
								cone.AgencyType = "A"
							} else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B
								cone.Score += 1
								cone.AgencyType = "B"
							} else {
								if len(thisinfo.Agency) < 1 { //E
									cone.Score -= 1
									cone.AgencyType = "E"
								} else { //C
									cone.Score -= 2
									cone.AgencyType = "C"
								}
							}
						} else { //D不计分
							cone.AgencyType = "D"
						}
						skey := fmt.Sprintf("%s%s%s%s", cone.ProjectNameType, cone.ProjectCodeType, cone.AreaType, cone.AgencyType)
						cone.Cresult = skey
						ThreeToLock.Lock()
						if ThreeToOne[skey] {
							scores = append(scores, cone)
						}
						ThreeToLock.Unlock()
					}
				}
			}
		}
	}
	return
}

//获取对比项目数组
func getComeperProjects(p PCBV, thisinfo *Info) (res []interface{}, pncb []*CompareInfo) {
	if p.PnameLen > 0 {
		pn := NewCompareInfo("pn", thisinfo.PNKey, PNKey)
		pncb = append(pncb, pn)
	}
	if p.PcodeLen > 0 {
		pc := NewCompareInfo("pc", thisinfo.PCKey, PCKey)
		pncb = append(pncb, pc)
	}
	if p.BuyerLen > 0 {
		pb := NewCompareInfo("pb", thisinfo.PBKey, PBKey)
		pncb = append(pncb, pb)
	}
	repeatId := map[string]bool{}
	//IdLock.Lock() //此处加id锁，会引进多线程的死锁，对比三个大map数组，找到key相同的项目id数组,并去重
	for _, pv := range pncb {
		pv.KeyMap.Lock.Lock()
		K := pv.KeyMap.Map[pv.Key]
		if K == nil {
			K = &Key{&[]string{}, &sync.Mutex{}}
			pv.KeyMap.Map[pv.Key] = K
		}
		pv.K = K
		pv.K.Lock.Lock()
		pv.KeyMap.Lock.Unlock()
		defer pv.K.Lock.Unlock()
		newarr := []string{}
		for _, id := range *K.Arr {
			if !repeatId[id] {
				newarr = append(newarr, id)
				repeatId[id] = true
			}
		}
		pv.IdArr = newarr
	}
	//IdLock.Unlock()
	for _, pv := range pncb {
		if len(pv.IdArr) > 0 {
			res = append(res, redis.Mget(REDISIDS, pv.IdArr))
		}
	}
	return
}

//合并项目
func mergeProject(tmp map[string]interface{}, thisinfo *Info, scores []*CompareOne, pncb []*CompareInfo) (sflag string) {
	var id = ""
	if len(scores) > 0 {
		//分值排序
		sort.Slice(scores, func(i, j int) bool {
			return scores[i].Score > scores[j].Score
		})
		max := scores[0]
		if max.Score > 0 {
			sflag = "normal"
			max.Parent.Bfind = true
			tmp["cresult"] = max.Cresult
			tmp["score"] = max.Score
			id = updateinfo(thisinfo, tmp, max.Pinfo, time.Now().Unix())
			//log.Println(sflag, id, max.Cresult)
			if len(scores) > 1 && (scores[0].Score == scores[1].Score) {
				//通知检查,异常数据存库
				checkNotice(thisinfo.Id, id, scores)
			}
		} else {
			sflag = "invalidscore"
		}
	} else {
		//生成项目
		sflag = "normal"
		mess := map[string]interface{}{
			"meger_sflag": sflag,
		}
		id = newProject(tmp, mess, time.Now().Unix(), thisinfo)
	}
	if id != "" { //更新REDISKEYS redis
		put := []interface{}{}
		for _, pv := range pncb {
			if pv != nil && !pv.Bfind {
				if BinarySearch(*pv.K.Arr, id) == -1 {
					*(pv.K.Arr) = append(*(pv.K.Arr), id)
					put = append(put, []interface{}{pv.Key, *(pv.K.Arr)})
				}
			}
		}
		if len(put) > 0 {
			redis.BulkPut(REDISKEYS, 0, put...)
		}
	}
	return sflag
}

//新增项目
func newProject(tmp, mess map[string]interface{}, pipc int64, thisinfo *Info) (id string) {
	id = InsertProject(thisinfo.NewPNKey, tmp, mess, pipc, thisinfo)
	sflag := qu.ObjToString(mess["meger_sflag"])
	if sflag != "alone" {
		p1 := NewPinfo(id, thisinfo)
		redis.PutCKV(REDISIDS, id, p1)
	} else {
		du.Debug("新增项目，不参与对比", id)
	}
	return id
}

//抽取信息打标记
func extInfoTag(sflag, id string) {
	MQFW.UpdateById(extractColl, id,
		map[string]interface{}{
			"$set": map[string]interface{}{
				"meger_sflag": sflag,
			},
		})
}

//检查通知
func checkNotice(infoid, pid string, scores []*CompareOne) {
	MQFW.Update("project_unusual", `{"project_id":"`+pid+`"}`,
		map[string]interface{}{
			"$set": map[string]interface{}{
				"project_id": pid,
				"pici":       time.Now().Unix(),
			},
			"$push": map[string]interface{}{
				"list": map[string]interface{}{
					"infoid": infoid,
					"pinfo1": scores[0].Pinfo.Id,
					"pinfo2": scores[1].Pinfo.Id,
				},
			},
		}, true, false)
	du.Debug("通知检查,异常数据存库", infoid, pid)
}

//属性判断组
func PCBVal(tmp map[string]interface{}) PCBV {
	pcbv := PCBV{}
	projectName := qu.ObjToString(tmp["projectname"])
	if len([]rune(projectName)) > 1 {
		pcbv.Val += 1
		pcbv.Pname = true
		pcbv.PnameLen = len([]rune(projectName))
	}
	projectCode := qu.ObjToString(tmp["projectcode"])
	if len([]rune(projectCode)) > 1 {
		pcbv.Val += 1
		pcbv.Pcode = true
		pcbv.PcodeLen = len(projectCode)
	}
	buyer := qu.ObjToString(tmp["buyer"])
	if len([]rune(buyer)) > 1 {
		pcbv.Val += 1
		pcbv.Buyer = true
		pcbv.BuyerLen = len([]rune(buyer))
	}
	//省市代理机构
	if qu.ObjToString(tmp["area"]) != "" {
		pcbv.Area = true
	}
	if qu.ObjToString(tmp["city"]) != "" {
		pcbv.City = true
	}
	if qu.ObjToString(tmp["agency"]) != "" {
		pcbv.Agency = true
	}
	return pcbv
}

//项目名称、项目编号、采购单位打分
func fieldPCBScore(this, info, ctype string, score int) (string, int) {
	if len(info) > 0 {
		if this == info { //A
			score += 5
			ctype = "A"
		} else if strings.Index(info, this) > -1 || strings.Index(this, info) > -1 { //B
			score += 2
			ctype = "B"
		} else {
			if len(this) < 1 { //E
				score -= 1
				ctype = "E"
			} else { //C
				score -= 2
				ctype = "C"
			}
		}
	} else { //D不计分
		ctype = "D"
	}
	return ctype, score
}

//pncbMap加锁
func lockPNCBMap(thisinfo *Info) {
	for { //等待其他任务完成
		ok := true
		if len(thisinfo.PNKey) > 3 {
			if _, b := PNKeyMap.Load(thisinfo.PNKey); b {
				ok = false
			}
		}
		if len(thisinfo.PCKey) > 3 {
			if _, b := PCKeyMap.Load(thisinfo.PCKey); b {
				ok = false
			}
		}
		if len(thisinfo.PBKey) > 3 {
			if _, b := PBKeyMap.Load(thisinfo.PBKey); b {
				ok = false
			}
		}
		if ok {
			break
		} else {
			//log.Println("has key store")
			time.Sleep(100 * time.Millisecond)
		}
	}
}

//pncbMap解锁
func unlockPNCBMap(thisinfo *Info) {
	//log.Println("del key store", thisinfo.PNKey)
	//if len(thisinfo.PNKey) > 3 {
	PNKeyMap.Delete(thisinfo.PNKey)
	//}
	//if len(thisinfo.PCKey) > 3 {
	PCKeyMap.Delete(thisinfo.PCKey)
	//}
	//if len(thisinfo.PBKey) > 3 {
	PBKeyMap.Delete(thisinfo.PBKey)
	//}
}

//store lock
func storeLock(thisinfo *Info) {
	PncbMayLock.Lock()
	PNKeyMap.Store(thisinfo.PNKey, true)
	PBKeyMap.Store(thisinfo.PBKey, true)
	PCKeyMap.Store(thisinfo.PCKey, true)
	PncbMayLock.Unlock()
}