package main import ( "log" // "log" "math" qu "qfw/util" "sort" "strings" "time" "gopkg.in/mgo.v2/bson" ) /** 项目合并,对比,计算,合并,生成项目 **/ //从对应map中获取对比的项目id func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) { p.findLock.Lock() defer p.findLock.Unlock() // p.ConCurrentLock(n1, n2, n3, n4) // defer p.ConCurrentUnLock(n1, n2, n3, n4) p.wg.Add(1) //查找到id数组 res = []*Key{} //是否查找到,并标识位置。-1代表值为空或已经存在。 bpn, bpc, bptc, bpb = -1, -1, -1, -1 if pn != "" { ids := p.mapPn[pn] if ids == nil { ids = &Key{Arr: []string{}} p.mapPn[pn] = ids bpn = 0 } ids.Lock.Lock() res = append(res, ids) } if pc != "" { ids := p.mapPc[pc] if ids == nil { ids = &Key{Arr: []string{}} p.mapPc[pc] = ids bpc = len(res) } ids.Lock.Lock() res = append(res, ids) } if ptc != "" { ids := p.mapPc[ptc] if ids == nil { ids = &Key{Arr: []string{}} p.mapPc[ptc] = ids bptc = len(res) } ids.Lock.Lock() res = append(res, ids) } if pb != "" { ids := p.mapPb[pb] if ids == nil { ids = &Key{Arr: []string{}} p.mapPb[pb] = ids bpb = len(res) } ids.Lock.Lock() res = append(res, ids) } repeatId := map[string]bool{} idArr = []string{} //项目id IDArr = []*ID{} //项目信息 for _, m := range res { for _, id := range m.Arr { if !repeatId[id] { repeatId[id] = true //_, _ = strconv.ParseInt(id[0:8], 16, 64) p.AllIdsMapLock.Lock() Id := p.AllIdsMap[id] p.AllIdsMapLock.Unlock() if Id != nil { Id.Lock.Lock() idArr = append(idArr, id) IDArr = append(IDArr, Id) } } } } return } func (p *ProjectTask) startProjectMerge(info *Info, tmp map[string]interface{}) { //只有或没有采购单位的无法合并 //bpn, bpc, bptc, bpb 是否查找到,并标识位置。-1代表未查找到。 //pids 是项目id数组集合 //IDArr,是单个项目ID对象集合 bpn, bpc, bptc, bpb, pids, _, IDArr := p.getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) //, info.LenPN, info.LenPC, info.LenPTC, len([]rune(info.Buyer))) defer p.wg.Done() //map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock for _, m := range pids { defer m.Lock.Unlock() } for _, id := range IDArr { defer id.Lock.Unlock() } bFindProject := false findPid := "" //获取完id,进行计算 //定义两组 comRes1 := []*ProjectInfo{} //优先级最高的对比结果数组 comRes2 := []*ProjectInfo{} //优化级其次 comRes3 := []*ProjectInfo{} for _, v := range IDArr { comStr := "" compareProject := v.P compareProject.score = 0 //问题出地LastTime!!!!! diffTime := int64(math.Abs(float64(info.Publishtime - compareProject.LastTime))) if diffTime <= p.validTime { //"A 相等 B 被包含 C 不相等 D不存在 E被包含 info.PNBH = 0 info.PCBH = 0 info.PTCBH = 0 compareStr, score := comparePNC(info, compareProject) resVal, pjVal := Select(compareStr, info, compareProject) //--------------------------------------- //log.Println(resVal, pjVal, compareProject) if resVal > 0 { compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount, score2 := p.compareBCTABB(info, compareProject, diffTime, score) //项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构 comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency + compareBudget + compareBidmount compareProject.comStr = comStr compareProject.pjVal = pjVal compareProject.resVal = resVal //log.Println(compareProject.comStr) eqV := 0 switch resVal { case 3: if pjVal == 3 && comStr[3:] != "CCCDCCC" { eqV = 1 } else if compareBuyer < "C" { if pjVal > 1 { eqV = 1 } else { //if (compareCity[1:1] != "C" || compareTime != "D") && score2 > 0 eqV = 2 } } else if compareBuyer == "D" { if pjVal > 1 && (compareCity[1:1] != "C" || score2 > 0) { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 { eqV = 3 } } else { if pjVal == 3 && (score2 > 0 || compareCity[1:1] != "C") { eqV = 2 } else if pjVal == 2 && compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 { eqV = 3 } else if compareCity == "AA" && compareTime == "A" && score2 > 0 { eqV = 3 } } case 2: if compareBuyer < "C" { if pjVal > 1 { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" || score2 > 0 { eqV = 3 } } else if compareBuyer == "D" { if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" && score2 > 0 { eqV = 3 } } else { if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) { eqV = 3 } } case 1: if compareBuyer < "C" { if pjVal > 1 && (score2 > 0 || compareCity[1:1] != "C") { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) { eqV = 3 } } else if compareBuyer == "D" { if pjVal > 1 && compareTime == "A" && (score2 > 0 || compareCity[1:1] != "C") { eqV = 2 } else if compareCity[1:1] != "C" && compareTime == "A" && (compareAgency == "A" || score2 > 1) { eqV = 3 } } else { if pjVal > 1 && compareTime == "A" && score2 > 1 && compareCity[1:1] != "C" { eqV = 3 } } } if eqV == 1 { comRes1 = append(comRes1, compareProject) } else if eqV == 2 { comRes2 = append(comRes2, compareProject) } else if eqV == 3 { comRes3 = append(comRes3, compareProject) } // else if resVal == 3 || pjVal > 1 { // log.Println("===", resVal, pjVal, comStr, info.ProjectCode, compareProject.ProjectCode, // info.ProjectName, compareProject.ProjectName, info.Buyer, compareProject.Buyer, info.Id, compareProject.Id.Hex()) // } } } } //--------------------------------对比完成----------------------- //更新数组、更新项目 for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} { if len(resN) > 0 { if len(resN) > 1 { sort.Slice(resN, func(i, j int) bool { return resN[i].score > resN[j].score }) } bFindProject = true findPid = resN[0].Id.Hex() for k2, bv := range []int{bpn, bpc, bptc, bpb} { if bv > -1 { pids[bv].Arr = append(pids[bv].Arr, findPid) if k2 == 0 { if resN[0].ProjectName == "" { resN[0].ProjectName = info.ProjectName } else { if resN[0].MPN == nil { resN[0].MPN = []string{info.ProjectName} } else { resN[0].MPN = append(resN[0].MPN, info.ProjectName) } } } else if k2 < 3 { if resN[0].ProjectCode == "" { resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string) } else { if resN[0].MPC == nil { resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)} } else { resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)) } } } else { if resN[0].Buyer == "" { resN[0].Buyer = info.Buyer } } } } p.UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr) break } } if !bFindProject { //没有找到 id, p1 := p.NewProject(tmp, info) p.AllIdsMapLock.Lock() p.AllIdsMap[id] = &ID{Id: id, P: p1} p.AllIdsMapLock.Unlock() for _, m := range pids { m.Arr = append(m.Arr, id) } } } func (p *ProjectTask) compareBCTABB(info *Info, cp *ProjectInfo, diffTime int64, score int) (compareBuyer, compareCity, compareTime, compareAgency, compareBudget, compareBidmount string, score2 int) { compareBuyer = "D" if len([]rune(info.Buyer)) > 3 && len([]rune(cp.Buyer)) > 3 { v := CheckContain(info.Buyer, cp.Buyer) if v == 1 { compareBuyer = "A" score += 3 } else { v1 := CosineSimilar(info.Buyer, cp.Buyer) if v == 2 || v1 > 0.8 { compareBuyer = "B" score += 1 } else { compareBuyer = "C" } } } //--------------------------------------- compareCity = "" if info.Area != "全国" && info.Area != "" && info.Area == cp.Area { compareCity += "A" score += 2 } else if info.Area == "全国" || cp.Area == "全国" { compareCity += "B" score += 1 } else { compareCity += "C" } if compareCity != "C" { if info.City != "" && info.City == cp.City { compareCity += "A" score += 2 } else { if info.Area == "全国" || cp.Area == "全国" { compareCity += "B" } else if info.City == compareCity { compareCity += "B" } else { compareCity += "C" } } } else { compareCity += "C" } score2 = 0 if compareCity == "AA" { if info.District != "" && info.District == cp.District { score2 = 1 } } compareTime = "D" if diffTime < 45*86400 { compareTime = "A" score += 2 } else if diffTime < 90*86400 { compareTime = "B" score += 1 } compareAgency = "D" if info.Agency != "" { if info.Agency == cp.Agency { compareAgency = "A" score += 2 score2 += 1 } else if cp.Agency != "" { if strings.Contains(info.Agency, cp.Agency) || strings.Contains(cp.Agency, info.Agency) { compareAgency = "B" score += 1 score2 += 1 } else { compareAgency = "C" } } } compareBudget = "C" if info.Budget > 0 && (info.Budget == cp.Budget || (cp.Bidamount > 0 && info.Budget > cp.Bidamount && (info.Budget-cp.Bidamount) < (0.1*info.Budget))) { compareBudget = "A" score += 1 score2 += 1 } // else if info.Budget == 0 && cp.Budget == 0 { // compareBudget = "B" // } compareBidmount = "C" if info.Bidamount > 0 && (info.Bidamount == cp.Bidamount || (cp.Budget > 0 && cp.Budget > info.Bidamount && (cp.Budget-info.Bidamount) < 0.1*cp.Budget)) { compareBidmount = "A" score += 1 score2 += 1 } // else if info.Bidamount == 0 && cp.Bidamount == 0 { // compareBidmount = "B" // } cp.score = score return } var FIELDS = []string{ "area", "city", "district", "projectname", "projectcode", "buyer", "buyerclass", "buyerperson", "buyertel", "winner", "budget", "bidamount", //"bidstatus", "agency", "projectscope", "bidopentime", "topscopeclass", "subscopeclass", "winnerorder", "package", } var bidtype = map[string]string{ "招标": "招标", "邀标": "邀标", "询价": "询价", "竞谈": "竞谈", "单一": "单一", "竞价": "竞价", } var bidstatus = map[string]string{ "中标": "中标", "成交": "成交", "废标": "废标", "流标": "流标", "合同": "合同", } //招标时间zbtime、中标时间jgtime、项目状态bidstatus、招标类型bidtype、最后发布时间lasttime、首次发布时间firsttime func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (string, *ProjectInfo) { pId := bson.NewObjectId() p1 := p.NewCachePinfo(pId, thisinfo) set := map[string]interface{}{} set["_id"] = pId for _, f := range FIELDS { if tmp[f] != nil { set[f] = tmp[f] } } if thisinfo.ProjectName != "" { set["s_projectname"] = tmp["projectname"] //兼容老版本 } now := time.Now().Unix() set["createtime"] = now set["sourceinfoid"] = thisinfo.Id set["sourceinfourl"] = tmp["href"] set["firsttime"] = tmp["publishtime"] set["lasttime"] = tmp["publishtime"] set["pici"] = p.pici set["ids"] = []string{thisinfo.Id} if thisinfo.TopType == "招标" { set["zbtime"] = tmp["publishtime"] } else if thisinfo.TopType == "结果" { set["jgtime"] = tmp["publishtime"] } //招标类型 bt := bidtype[thisinfo.SubType] if bt == "" { bt = "招标" } set["bidtype"] = bt bs, _ := tmp["bidstatus"].(string) if bidstatus[bs] != "" { set["bidstatus"] = bs } if set["bidstatus"] == nil && thisinfo.TopType == "结果" { set["bidstatus"] = thisinfo.SubType } if len(thisinfo.Subscopeclass) > 0 { s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",") set["s_subscopeclass"] = s_subscopeclass } if len(thisinfo.Winners) > 0 { set["s_winner"] = strings.Join(thisinfo.Winners, ",") p1.Winners = thisinfo.Winners } push := p.PushListInfo(tmp) set["list"] = []bson.M{ push, } //p.savePool <- set p.updatePool <- []map[string]interface{}{ map[string]interface{}{ "_id": pId, }, map[string]interface{}{ "$set": set, }, } return pId.Hex(), &p1 } var INFOFIELDS = []string{ "projectname", "projectcode", "title", "href", "publishtime", "comeintime", "bidopentime", "toptype", "subtype", "buyer", "buyerclass", "agency", "winner", "budget", "bidamount", "topscopeclass", "subscopclass", "infoformat", "buyerperson", "buyertel", } //项目中list的信息 func (p *ProjectTask) PushListInfo(tmp map[string]interface{}) bson.M { res := bson.M{ "infoid": qu.BsonIdToSId(tmp["_id"]), } for _, k := range INFOFIELDS { if tmp[k] != nil { res[k] = tmp[k] } } return res } //生成存放在内存中的对象 func (p *ProjectTask) NewCachePinfo(id bson.ObjectId, thisinfo *Info) ProjectInfo { p1 := ProjectInfo{ Id: id, Ids: []string{thisinfo.Id}, ProjectName: thisinfo.ProjectName, ProjectCode: thisinfo.ProjectCode, Buyer: thisinfo.Buyer, Buyerclass: thisinfo.Buyerclass, Buyerperson: thisinfo.Buyerperson, Buyertel: thisinfo.Buyertel, Topscopeclass: thisinfo.Topscopeclass, Subscopeclass: thisinfo.Subscopeclass, Agency: thisinfo.Agency, Area: thisinfo.Area, City: thisinfo.City, District: thisinfo.District, MPN: []string{}, MPC: []string{}, HasPackage: thisinfo.HasPackage, FirstTime: thisinfo.Publishtime, LastTime: thisinfo.Publishtime, Budget: thisinfo.Budget, Bidamount: thisinfo.Bidamount, } if thisinfo.LenPTC > 5 { p1.MPC = append(p1.MPC, thisinfo.PTC) } return p1 } //更新项目 func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) { if p.currentType != "ql" { if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 { log.Println("repeat", thisinfo.Id) return } } set := map[string]interface{}{} pInfo.Ids = append(pInfo.Ids, thisinfo.Id) //1--firsttime if thisinfo.Publishtime < pInfo.FirstTime && thisinfo.Publishtime > 0 { pInfo.FirstTime = thisinfo.Publishtime set["firsttime"] = thisinfo.Publishtime if thisinfo.TopType == "招标" { set["zbtime"] = tmp["publishtime"] } } //2--lasttime if thisinfo.Publishtime > pInfo.LastTime { pInfo.LastTime = thisinfo.Publishtime set["lasttime"] = thisinfo.Publishtime bt := bidtype[thisinfo.SubType] if bt != "" { set["bidtype"] = bt } if thisinfo.TopType == "结果" { set["bidstatus"] = thisinfo.SubType set["jgtime"] = tmp["publishtime"] } } //3\4\5--省、市、县 if thisinfo.Area != "全国" { //xt := true if pInfo.Area == "全国" { pInfo.Area = thisinfo.Area set["area"] = thisinfo.Area } else if pInfo.Area != thisinfo.Area { //xt = false } if pInfo.City == "" && thisinfo.City != "" { pInfo.City = thisinfo.City set["city"] = thisinfo.City } else if pInfo.City != thisinfo.City { //xt = false } if thisinfo.District != "" && pInfo.District == "" { pInfo.District = thisinfo.District set["district"] = thisinfo.District } //省市县有不相同的 // if !xt { // log.Println(pInfo.Area, pInfo.City, thisinfo.Area, thisinfo.District) // } } //6--项目名称 if (thisinfo.ProjectName != "" && pInfo.ProjectName == "") || (len([]rune(pInfo.ProjectName)) < 6 && thisinfo.LenPN > 6) { pInfo.ProjectName = thisinfo.ProjectName set["projectname"] = thisinfo.ProjectName } //7--项目编号 if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) { pInfo.ProjectCode = thisinfo.ProjectCode set["projectcode"] = thisinfo.ProjectCode } //7--采购单位 if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) { pInfo.Buyer = thisinfo.Buyer set["buyer"] = thisinfo.Buyer } //8--代理机构 if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) { pInfo.Agency = thisinfo.Agency set["agency"] = thisinfo.Agency } //9--采购单位联系人 if thisinfo.Buyerperson != "" && strings.Index(pInfo.Buyerperson, thisinfo.Buyerperson) < 0 { pInfo.Buyerperson = thisinfo.Buyerperson set["buyerperson"] = pInfo.Buyerperson } //10--采购单位電話 if thisinfo.Buyertel != "" && strings.Index(pInfo.Buyertel, thisinfo.Buyertel) < 0 { pInfo.Buyertel = thisinfo.Buyertel set["buyertel"] = pInfo.Buyertel } if thisinfo.Buyerclass != "" && pInfo.Buyerclass == "" { pInfo.Buyerclass = thisinfo.Buyerclass set["buyerclass"] = pInfo.Buyerclass } if thisinfo.Bidopentime > pInfo.Bidopentime { pInfo.Bidopentime = thisinfo.Bidopentime set["bidopentime"] = pInfo.Bidopentime } if thisinfo.Bidamount > 0 && pInfo.Bidamount < 1 { pInfo.Bidamount = thisinfo.Bidamount set["bidamount"] = pInfo.Bidamount } if thisinfo.Budget > 0 && pInfo.Budget < 1 { //多包的会有问题,没有进行合计。 pInfo.Budget = thisinfo.Budget set["budget"] = pInfo.Budget } if len(thisinfo.Topscopeclass) > 0 { sort.Strings(pInfo.Topscopeclass) for _, k := range thisinfo.Topscopeclass { if BinarySearch(pInfo.Topscopeclass, k) == -1 { pInfo.Topscopeclass = append(pInfo.Topscopeclass, k) sort.Strings(pInfo.Topscopeclass) } } set["topscopeclass"] = pInfo.Topscopeclass } if len(thisinfo.Subscopeclass) > 0 { sort.Strings(pInfo.Subscopeclass) for _, k := range thisinfo.Subscopeclass { if BinarySearch(pInfo.Subscopeclass, k) == -1 { pInfo.Subscopeclass = append(pInfo.Subscopeclass, k) sort.Strings(pInfo.Subscopeclass) } } set["subscopeclass"] = pInfo.Subscopeclass set["s_subscopeclass"] = strings.Join(pInfo.Subscopeclass, ",") } //winner if len(thisinfo.Winners) > 0 { sort.Strings(pInfo.Winners) for _, k := range thisinfo.Winners { if BinarySearch(pInfo.Winners, k) == -1 { pInfo.Winners = append(pInfo.Winners, k) sort.Strings(pInfo.Winners) } } //set["winners"] = pInfo.Winners set["s_winner"] = strings.Join(pInfo.Winners, ",") } set["mpn"] = pInfo.MPN set["mpc"] = pInfo.MPC set["pici"] = p.pici if thisinfo.HasPackage { set["multipackage"] = 1 } else { set["multipackage"] = 0 } update := map[string]interface{}{} if len(set) > 0 { update["$set"] = set } //保留原数据吧 push := p.PushListInfo(tmp) push["compareStr"] = comStr push["resVal"] = pInfo.resVal push["pjVal"] = pInfo.pjVal update["$push"] = map[string]interface{}{ "list": push, "ids": thisinfo.Id, } if len(update) > 0 { updateInfo := []map[string]interface{}{ map[string]interface{}{ "_id": pInfo.Id, }, update, } p.updatePool <- updateInfo } }