jy-data-extract/fullproject/src_dev5/project.go

package main

import (
	"log"
	//	"log"
	"math"
	qu "qfw/util"
	"sort"
	"strings"
	"time"

	"gopkg.in/mgo.v2/bson"
)

/**
项目合并，对比，计算，合并，生成项目
**/

//从对应map中获取对比的项目id
func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
	p.findLock.Lock()
	defer p.findLock.Unlock()
	p.wg.Add(1)
	//查找到id数组
	res = []*Key{}
	//是否查找到，并标识位置。-1代表值为空或已经存在。
	bpn, bpc, bptc, bpb = -1, -1, -1, -1
	if pn != "" {
		ids := p.mapPn[pn]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			p.mapPn[pn] = ids
			bpn = 0
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pc != "" {
		ids := p.mapPc[pc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			p.mapPc[pc] = ids
			bpc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if ptc != "" {
		ids := p.mapPc[ptc]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			p.mapPc[ptc] = ids
			bptc = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}

	if pb != "" {
		ids := p.mapPb[pb]
		if ids == nil {
			ids = &Key{Arr: []string{}}
			p.mapPb[pb] = ids
			bpb = len(res)
		}
		ids.Lock.Lock()
		res = append(res, ids)
	}
	repeatId := map[string]bool{}
	idArr = []string{} //项目id
	IDArr = []*ID{}    //项目信息
	for _, m := range res {
		for _, id := range m.Arr {
			if !repeatId[id] {
				repeatId[id] = true
				//_, _ = strconv.ParseInt(id[0:8], 16, 64)
				p.AllIdsMapLock.Lock()
				Id := p.AllIdsMap[id]
				p.AllIdsMapLock.Unlock()
				if Id != nil {
					Id.Lock.Lock()
					idArr = append(idArr, id)
					IDArr = append(IDArr, Id)
				}
			}
		}
	}
	return
}

func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{}) {
	//只有或没有采购单位的无法合并
	//bpn, bpc, bptc, bpb 是否查找到，并标识位置。-1代表未查找到。
	//pids 是项目id数组集合
	//IDArr，是单个项目ID对象集合
	bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer)
	defer p.wg.Done()
	//map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock

	for _, m := range pids {
		defer m.Lock.Unlock()
	}
	for _, id := range IDArr {
		defer id.Lock.Unlock()
	}

	bFindProject := false
	findPid := ""
	//获取完id，进行计算
	//定义两组
	comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组
	comRes2 := []*ProjectInfo{} //优化级其次
	comRes3 := []*ProjectInfo{}
	resVal, pjVal := 0, 0
	for _, v := range IDArr {
		comStr := ""
		compareProject := v.P
		compareProject.score = 0
		//问题出地LastTime!!!!!
		diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime)))
		if diffTime < 185*86400 {
			//"A 相等 	B 被包含 	C 不相等	 	D不存在  E被包含
			info.PNBH = 0
			info.PCBH = 0
			info.PTCBH = 0
			compareStr, score := comparePNC(info, compareProject)

			resVal, pjVal = Select(compareStr, info, compareProject)
			//---------------------------------------
			//log.Println(resVal, pjVal, compareProject)
			if resVal > 0 {

				compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score)

				//项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构

				comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount
				compareProject.comStr = comStr
				//log.Println(compareProject.comStr)
				eqV := 0
				switch resVal {
				case 3:
					if pjVal == 3 && comStr[3:] != "CCCDCCC" {
						eqV = 1
					} else if compareBuyer < "C" {
						if pjVal > 1 {
							eqV = 1
						} else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0
							eqV = 2
						}
					} else if compareBuyer == "D" {
						if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
							eqV = 3
						}
					} else {
						if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") {
							eqV = 2
						} else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
							eqV = 3
						} else if compareCity == "AA" && compareTime == "A" && score2 > 0 {
							eqV = 3
						}
					}
				case 2:
					if compareBuyer < "C" {
						if pjVal > 1 {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 {
							eqV = 3
						}
					} else if compareBuyer == "D" {
						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 {
							eqV = 3
						}

					} else {
						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
							eqV = 3
						}
					}
				case 1:
					if compareBuyer < "C" {
						if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
							eqV = 3
						}
					} else if compareBuyer == "D" {
						if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") {
							eqV = 2
						} else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) {
							eqV = 3
						}
					} else {
						if pjVal > 1 && compareTime == "A" && score2 > 1 && compareCity[1:1] != "C" {
							eqV = 3
						}
					}
				}
				if eqV == 1 {
					comRes1 = append(comRes1, compareProject)
				} else if eqV == 2 {
					comRes2 = append(comRes2, compareProject)
				} else if eqV == 3 {
					comRes3 = append(comRes3, compareProject)
				}
				//				else if resVal == 3 || pjVal > 1 {
				//					log.Println("===", resVal, pjVal, comStr, info.ProjectCode, compareProject.ProjectCode,
				//						info.ProjectName, compareProject.ProjectName, info.Buyer, compareProject.Buyer, info.Id, compareProject.Id.Hex())
				//				}
			}
		}
	}
	//--------------------------------对比完成-----------------------
	//更新数组、更新项目
	for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} {
		if len(resN) > 0 {
			if len(resN) > 1 {
				sort.Slice(resN, func(i, j int) bool {
					return resN[i].score > resN[j].score
				})
			}

			bFindProject = true
			findPid = resN[0].Id.Hex()
			for k2, bv := range []int{bpn, bpc, bptc, bpb} {
				if bv > -1 {
					pids[bv].Arr = append(pids[bv].Arr, findPid)
					if k2 == 0 {
						if resN[0].ProjectName == "" {
							resN[0].ProjectName = info.ProjectName
						} else {
							if resN[0].MPN == nil {
								resN[0].MPN = []string{info.ProjectName}
							} else {
								resN[0].MPN = append(resN[0].MPN, info.ProjectName)
							}
						}

					} else if k2 < 3 {
						if resN[0].ProjectCode == "" {
							resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)
						} else {
							if resN[0].MPC == nil {
								resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)}
							} else {
								resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string))
							}
						}

					} else {
						if resN[0].Buyer == "" {
							resN[0].Buyer = info.Buyer
						}
					}
				}
			}
			p.UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr, resVal, pjVal)
			break
		}
	}

	if !bFindProject {
		//没有找到
		id, p1 := p.NewProject(tmp, info)
		p.AllIdsMapLock.Lock()
		p.AllIdsMap[id] = &ID{Id: id, P: p1}
		p.AllIdsMapLock.Unlock()
		for _, m := range pids {
			m.Arr = append(m.Arr, id)
		}
	}

}

func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectInfo, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) {
	compareBuyer = "D"
	if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 {
		v := CheckContain(info.Buyer, cp.Buyer)
		if v == 1 {
			compareBuyer = "A"
			score += 3
		} else {
			v1 := CosineSimilar(info.Buyer, cp.Buyer)
			if v == 2 || v1 > 0.8 {
				compareBuyer = "B"
				score += 1
			} else {
				compareBuyer = "C"
			}
		}
	}
	//---------------------------------------

	compareCity = ""
	if info.Area != "全国" && info.Area != "" && info.Area == cp.Area {
		compareCity += "A"
		score += 2
	} else if info.Area == "全国" || cp.Area == "全国" {
		compareCity += "B"
		score += 1
	} else {
		compareCity += "C"
	}
	if compareCity != "C" {
		if info.City != "" && info.City == cp.City {
			compareCity += "A"
			score += 2
		} else {
			if info.Area == "全国" || cp.Area == "全国" {
				compareCity += "B"
			} else if info.City == compareCity {
				compareCity += "B"
			} else {
				compareCity += "C"
			}
		}
	} else {
		compareCity += "C"
	}
	score2 = 0
	if compareCity == "AA" {
		if info.District != "" && info.District == cp.District {
			score2 = 1
		}
	}

	compareTime = "D"
	if diffTime < 45*86400 {
		compareTime = "A"
		score += 2
	} else if diffTime < 90*86400 {
		compareTime = "B"
		score += 1
	}

	compareAgency = "D"
	if info.Agency != "" {
		if info.Agency == cp.Agency {
			compareAgency = "A"
			score += 2
			score2 += 1
		} else if cp.Agency != "" {
			if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) {
				compareAgency = "B"
				score += 1
				score2 += 1
			} else {
				compareAgency = "C"
			}
		}
	}
	compareBudget = "C"
	if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.1*info.Budget))) {
		compareBudget = "A"
		score += 1
		score2 += 1
	} else if info.Budget == 0 && cp.Budget == 0 {
		compareBudget = "B"
	}
	compareBidmount = "C"
	if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.1*cp.Budget)) {
		compareBidmount = "A"
		score += 1
		score2 += 1
	} else if info.Bidamount == 0 && cp.Bidamount == 0 {
		compareBidmount = "B"
	}

	cp.score = score
	return
}

var FIELDS = []string{
	"area",
	"city",
	"district",
	"projectname",
	"projectcode",
	"buyer",
	"buyerclass",
	"buyerperson",
	"buyertel",
	"winner",
	"budget",
	"bidamount",
	"bidstatus",
	"agency",
	"projectscope",
	"bidopentime",
	"topscopeclass",
	"subscopeclass",
	"winnerorder",
	"package",
}

var bidtype = map[string]string{
	"招标": "招标",
	"邀标": "邀标",
	"询价": "询价",
	"竞谈": "竞谈",
	"单一": "单一",
	"竞价": "竞价",
}

//招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime

func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) {
	pId := bson.NewObjectId()
	p1 := p.NewCachePinfo(pId, thisinfo)
	set := map[string]interface{}{}
	set["_id"] = pId
	for _, f := range FIELDS {
		if tmp[f] != nil {
			set[f] = tmp[f]
		}
	}
	if thisinfo.ProjectName != "" {
		set["s_projectname"] = tmp["projectname"] //兼容老版本
	}
	now := time.Now().Unix()
	set["createtime"] = now
	set["sourceinfoid"] = thisinfo.Id
	set["sourceinfourl"] = tmp["href"]
	set["firsttime"] = tmp["publishtime"]
	set["lasttime"] = tmp["publishtime"]
	set["pici"] = now
	set["ids"] = []string{thisinfo.Id}
	if thisinfo.TopType == "招标" {
		set["zbtime"] = tmp["publishtime"]
	} else if thisinfo.TopType == "结果" {
		set["jgtime"] = tmp["publishtime"]
	}
	//招标类型
	bt := bidtype[thisinfo.SubType]
	if bt == "" {
		bt = "招标"
	}
	set["bidtype"] = bt
	if set["bidstatus"] == nil && thisinfo.TopType == "结果" {
		set["bidstatus"] = thisinfo.SubType
	}
	if len(thisinfo.Subscopeclass) > 0 {
		s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
		set["s_subscopeclass"] = s_subscopeclass
	}
	if len(thisinfo.Winners) > 0 {
		set["s_winner"] = strings.Join(thisinfo.Winners, ",")
		p1.Winners = thisinfo.Winners
	}
	push := p.PushListInfo(tmp)
	set["list"] = []bson.M{
		push,
	}
	p.updatePool <- []map[string]interface{}{
		map[string]interface{}{
			"_id": pId,
		},
		map[string]interface{}{
			"$set": set,
		},
	}
	return pId.Hex(), &p1
}

var INFOFIELDS = []string{
	"projectname",
	"projectcode",
	"title",
	"href",
	"publishtime",
	"comeintime",
	"bidopentime",
	"toptype",
	"subtype",
	"buyer",
	"buyerclass",
	"agency",
	"winner",
	"budget",
	"bidamount",
	"topscopeclass",
	"subscopclass",
	"infoformat",
}

//项目中list的信息
func (p *ProjectTask) PushListInfo(tmp map[string]interface{}) bson.M {
	res := bson.M{

		"infoid": qu.BsonIdToSId(tmp["_id"]),
	}
	for _, k := range INFOFIELDS {
		if tmp[k] != nil {
			res[k] = tmp[k]
		}
	}
	return res
}

//生成存放在内存中的对象
func (p *ProjectTask) NewCachePinfo(id bson.ObjectId, thisinfo *Info) ProjectInfo {
	p1 := ProjectInfo{
		Id:            id,
		Ids:           []string{thisinfo.Id},
		ProjectName:   thisinfo.ProjectName,
		ProjectCode:   thisinfo.ProjectCode,
		Buyer:         thisinfo.Buyer,
		Buyerclass:    thisinfo.Buyerclass,
		Buyerperson:   thisinfo.Buyerperson,
		Buyertel:      thisinfo.Buyertel,
		Topscopeclass: thisinfo.Topscopeclass,
		Subscopeclass: thisinfo.Subscopeclass,
		Agency:        thisinfo.Agency,
		Area:          thisinfo.Area,
		City:          thisinfo.City,
		District:      thisinfo.District,
		MPN:           []string{},
		MPC:           []string{},
		HasPackage:    thisinfo.HasPackage,
		FirstTime:     thisinfo.Publishtime,
		LastTime:      thisinfo.Publishtime,
		Budget:        thisinfo.Budget,
		Bidamount:     thisinfo.Bidamount,
	}
	if thisinfo.LenPTC > 5 {
		p1.MPC = append(p1.MPC, thisinfo.PTC)
	}
	return p1
}

//更新项目
func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string, resVal, pjVal int) {
	if p.currentType != "ql" {
		if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
			log.Println("repeat", thisinfo.Id)
			return
		}
	}
	set := map[string]interface{}{}
	pInfo.Ids = append(pInfo.Ids, thisinfo.Id)

	//1--firsttime
	if thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0 {
		pInfo.FirstTime = thisinfo.Publishtime
		set["firsttime"] = thisinfo.Publishtime
		if thisinfo.TopType == "招标" {
			set["zbtime"] = tmp["publishtime"]
		}
	}
	//2--lasttime
	if thisinfo.Publishtime > pInfo.LastTime {
		pInfo.LastTime = thisinfo.Publishtime
		set["lasttime"] = thisinfo.Publishtime
		bt := bidtype[thisinfo.SubType]
		if bt != "" {
			set["bidtype"] = bt
		}
		if thisinfo.TopType == "结果" {
			set["bidstatus"] = thisinfo.SubType
			set["jgtime"] = tmp["publishtime"]
		}
	}
	//3\4\5--省、市、县
	if thisinfo.Area != "全国" {
		//xt := true
		if pInfo.Area == "全国" {
			pInfo.Area = thisinfo.Area
			set["area"] = thisinfo.Area
		} else if pInfo.Area != thisinfo.Area {
			//xt = false
		}
		if pInfo.City == "" && thisinfo.City != "" {
			pInfo.City = thisinfo.City
			set["city"] = thisinfo.City
		} else if pInfo.City != thisinfo.City {
			//xt = false
		}
		if thisinfo.District != "" && pInfo.District == "" {
			pInfo.District = thisinfo.District
			set["district"] = thisinfo.District
		}
		//省市县有不相同的
		//		if !xt {
		//			log.Println(pInfo.Area, pInfo.City, thisinfo.Area, thisinfo.District)
		//		}
	}
	//6--项目名称
	if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) {
		pInfo.ProjectName = thisinfo.ProjectName
		set["projectname"] = thisinfo.ProjectName
	}
	//7--项目编号
	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
		pInfo.ProjectCode = thisinfo.ProjectCode
		set["projectcode"] = thisinfo.ProjectCode
	}
	//7--采购单位
	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
		pInfo.Buyer = thisinfo.Buyer
		set["buyer"] = thisinfo.Buyer
	}
	//8--代理机构
	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
		pInfo.Agency = thisinfo.Agency
		set["agency"] = thisinfo.Agency
	}
	//9--采购单位联系人
	if thisinfo.Buyerperson != "" && strings.Index(pInfo.Buyerperson, thisinfo.Buyerperson) < 0 {
		pInfo.Buyerperson += thisinfo.Buyerperson
		set["buyerperson"] = pInfo.Buyerperson
	}
	//10--采购单位電話
	if thisinfo.Buyertel != "" && strings.Index(pInfo.Buyertel, thisinfo.Buyertel) < 0 {
		pInfo.Buyertel += thisinfo.Buyertel
		set["buyertel"] = pInfo.Buyertel
	}

	if thisinfo.Buyerclass != "" && pInfo.Buyerclass == "" {
		pInfo.Buyerclass = thisinfo.Buyerclass
		set["buyerclass"] = pInfo.Buyerclass
	}
	if thisinfo.Bidopentime > pInfo.Bidopentime {
		pInfo.Bidopentime = thisinfo.Bidopentime
		set["bidopentime"] = pInfo.Bidopentime
	}
	if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 {
		pInfo.Bidamount = thisinfo.Bidamount
		set["bidamount"] = pInfo.Bidamount
	}

	if thisinfo.Budget > 0 && pInfo.Budget < 1 {
		pInfo.Budget = thisinfo.Budget
		set["budget"] = pInfo.Budget
	}

	if len(thisinfo.Topscopeclass) > 0 {
		sort.Strings(pInfo.Topscopeclass)
		for _, k := range thisinfo.Topscopeclass {
			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
				sort.Strings(pInfo.Topscopeclass)
			}
		}
		set["topscopeclass"] = pInfo.Topscopeclass
	}

	if len(thisinfo.Subscopeclass) > 0 {
		sort.Strings(pInfo.Subscopeclass)
		for _, k := range thisinfo.Subscopeclass {
			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
				sort.Strings(pInfo.Subscopeclass)
			}
		}
		set["subscopeclass"] = pInfo.Subscopeclass
		set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",")
	}
	//winner
	if len(thisinfo.Winners) > 0 {
		sort.Strings(pInfo.Winners)
		for _, k := range thisinfo.Winners {
			if BinarySearch(pInfo.Winners, k) == -1 {
				pInfo.Winners = append(pInfo.Winners, k)
				sort.Strings(pInfo.Winners)
			}
		}
		set["winners"] = pInfo.Winners
		set["s_winner"] = strings.Join(pInfo.Winners, ",")
	}

	set["mpn"] = pInfo.MPN
	set["mpc"] = pInfo.MPC

	if thisinfo.HasPackage {
		set["multipackage"] = 1
	} else {
		set["multipackage"] = 0
	}

	update := map[string]interface{}{}
	if len(set) > 0 {
		update["$set"] = set
	}
	//保留原数据吧
	push := p.PushListInfo(tmp)
	push["compareStr"] = comStr
	push["resVal"] = resVal
	push["pjVal"] = pjVal
	update["$push"] = map[string]interface{}{
		"list": push,
		"ids":  thisinfo.Id,
	}
	if len(update) > 0 {
		updateInfo := []map[string]interface{}{
			map[string]interface{}{
				"_id": pInfo.Id,
			},
			update,
		}
		p.updatePool <- updateInfo
	}
}