package main import ( "encoding/json" "log" "math" qu "qfw/util" "qfw/util/redis" "sort" "strconv" "strings" "sync" "time" "gopkg.in/mgo.v2/bson" ) func startProjectMerge(info *Info, tmp map[string]interface{}) { //只有或没有采购单位的无法合并 if info.pnbval < 2 && info.Buyer != "" { //extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"])) log.Println("invalid", info.pnbval, info.Buyer, info.Id) } else { bpn, bpc, bptc, bpb, pids := getCompareIds(info.ProjectName, info.ProjectCode, info.PTC, info.Buyer) repeatId := map[string]bool{} idArr := []string{} //项目id IDArr := []*ID{} //项目信息 //map--k为pn,ptn,pc,ptc,buyer值 v为Id数组和lock for _, m := range pids { defer m.Lock.Unlock() for _, id := range m.Arr { if !repeatId[id] { repeatId[id] = true _, _ = strconv.ParseInt(id[0:8], 16, 64) AllIdsMapLock.Lock() //Id := AllIdsMap[int(t)%idsMapSize][id] Id := AllIdsMap2[id] AllIdsMapLock.Unlock() Id.Lock.Lock() defer Id.Lock.Unlock() idArr = append(idArr, id) IDArr = append(IDArr, Id) } } } bFindProject := false findPid := "" //获取完id,进行计算 if len(idArr) > 0 && len(idArr) == len(IDArr) { res := redis.Mget(REDISIDS, idArr) //定义两组 comRes1 := []*ProjectInfo{} comRes2 := []*ProjectInfo{} comRes3 := []*ProjectInfo{} if len(res) == len(idArr) { for _, v := range res { var cp ProjectInfo comStr := "" var compareProject *ProjectInfo err := json.Unmarshal(v.([]byte), &cp) compareProject = &cp diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1])) if err != nil { log.Println("从redis取id信息出错!!!", err) } else if diffTime < 360*86400 { //"A 相等 B 被包含 C 不相等 D不存在 E被包含 compareStr, score := comparePNCB(info, compareProject) //if info.ProjectCode == "ZWJLZXQ-2016-0508" { //} resVal := Select(compareStr, info, compareProject) //--------------------------------------- if resVal > 0 { compareBuyer := "" if info.Buyer != "" { if info.Buyer == compareProject.Buyer { compareBuyer += "A" } else if compareProject.Buyer != "" { if strings.Contains(info.Buyer, compareProject.Buyer) || strings.Contains(compareProject.Buyer, info.Buyer) { compareBuyer += "B" } else { compareBuyer += "C" } } else { compareBuyer += "D" } } else { compareBuyer += "D" } //--------------------------------------- compareCity := "" if info.Area != "全国" && info.Area != "" && info.Area == compareProject.Area { compareCity += "A" score += 2 } else if info.Area == "全国" || compareProject.Area == "全国" { compareCity += "B" score += 1 } else { compareCity += "C" } if info.City != "" && info.City == compareProject.City { compareCity += "A" score += 2 } else { if info.Area == "全国" || compareProject.Area == "全国" { compareCity += "B" } else { compareCity += "C" } } compareTime := "" //diffTime := math.Abs(float64(info.Publishtime - compareProject.Publistime[len(compareProject.Publistime)-1])) if diffTime < 45*86400 { compareTime += "A" score += 2 } else if diffTime < 90*86400 { compareTime += "B" score += 1 } else { compareTime += "D" } compareAgency := "" if info.Agency != "" { if info.Agency == compareProject.Agency { compareAgency += "A" score += 2 } else if compareProject.Agency != "" { if strings.Contains(info.Agency, compareProject.Agency) || strings.Contains(compareProject.Agency, info.Agency) { compareAgency += "B" score += 1 } else { compareAgency += "C" } } else { compareAgency += "D" } } else { compareAgency += "D" } //项目名称、项目编号、标题项目编号、采购单位、省、市、发布时间、代理机构 comStr = compareStr + compareBuyer + compareCity + compareTime + compareAgency compareProject.comStr = comStr // log.Println(resVal, info.ProjectName, info.ProjectCode, info.Buyer, compareProject.Id, compareProject.ProjectName, compareProject.ProjectCode, compareProject.Buyer, compareProject.MPC, compareStr) // log.Println(compareBuyer, compareCity, compareAgency, compareTime) switch resVal { case 3: if compareBuyer == "A" || compareBuyer == "B" { comRes1 = append(comRes1, compareProject) } else if compareBuyer == "D" { if "AA" == compareCity && compareAgency != "C" && compareTime != "D" { comRes1 = append(comRes1, compareProject) } else { comRes2 = append(comRes2, compareProject) } } else { if "AA" == compareCity && compareAgency != "C" && compareTime != "D" { comRes2 = append(comRes2, compareProject) } else if compareCity[1:1] != "C" && compareAgency != "C" && compareTime != "D" { comRes3 = append(comRes3, compareProject) } } case 1, 2: if compareBuyer == "A" { comRes1 = append(comRes1, compareProject) } else if compareBuyer == "B" { comRes2 = append(comRes2, compareProject) } else if compareBuyer == "D" { if "AA" == compareCity && compareAgency != "C" && compareTime != "D" { comRes2 = append(comRes2, compareProject) } else { comRes3 = append(comRes3, compareProject) } } else { if "AA" == compareCity && compareAgency != "C" && compareTime != "D" { comRes3 = append(comRes3, compareProject) } } } } } } //--------------------------------对比完成----------------------- for kv, resN := range [][]*ProjectInfo{comRes1, comRes2, comRes3} { if len(resN) > 0 { if len(resN) > 1 { sort.Slice(comRes1, func(i, j int) bool { return comRes1[i].score > comRes1[j].score }) } bFindProject = true findPid = resN[0].Id for k2, bv := range []int{bpn, bpc, bptc, bpb} { if bv > -1 { pids[bv].Arr = append(pids[bv].Arr, findPid) if k2 == 0 { if resN[0].ProjectName == "" { resN[0].ProjectName = info.ProjectName } else { if resN[0].MPN == nil { resN[0].MPN = []string{info.ProjectName} } else { resN[0].MPN = append(resN[0].MPN, info.ProjectName) } } } else if k2 < 3 { if resN[0].ProjectCode == "" { resN[0].ProjectCode = qu.If(k2 == 1, info.ProjectCode, info.PTC).(string) } else { if resN[0].MPC == nil { resN[0].MPC = []string{qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)} } else { resN[0].MPC = append(resN[0].MPC, qu.If(k2 == 1, info.ProjectCode, info.PTC).(string)) } } } else { if resN[0].Buyer == "" { resN[0].Buyer = info.Buyer } } } } UpdateProject(tmp, info, resN[0], kv+1, resN[0].comStr) //更新AllIdsMao中的时间 redis.Put(REDISIDS, findPid, resN[0], 0) break } } } else { log.Println("redis记录缺失!!!") } } if !bFindProject { //没有找到 id := NewProject(tmp, info) _, _ = strconv.ParseInt(id[0:8], 16, 64) AllIdsMapLock.Lock() //AllIdsMap[int(t)%idsMapSize][id] = &ID{Id: id, lastTime: info.Publishtime} AllIdsMap2[id] = &ID{Id: id, lastTime: info.Publishtime} AllIdsMapLock.Unlock() for _, m := range pids { m.Arr = append(m.Arr, id) } } } } func UpdateProject(tmp map[string]interface{}, thisinfo *Info, pInfo *ProjectInfo, weight int, comStr string) { //log.Println(thisinfo.ProjectName, pInfo.ProjectName, comStr) // updateid := pInfo.Id if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 { return //updateid } set := map[string]interface{}{} res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`) EqInfoUpdate(thisinfo, pInfo) if bres && res != nil && *res != nil { set["topscopeclass"] = pInfo.Topscopeclass set["subscopeclass"] = pInfo.Subscopeclass s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",") if len(s_subscopeclass) > 0 { s_subscopeclass = "," + s_subscopeclass + "," } set["s_subscopeclass"] = s_subscopeclass s_winner := strings.Join(pInfo.Winners, ",") if len(s_winner) > 0 { s_winner = "," + s_winner + "," } set["s_winner"] = s_winner if pInfo.Buyerperson != "" && pInfo.Buyertel != "" { set["buyerperson"] = pInfo.Buyerperson set["buyertel"] = pInfo.Buyertel } if pInfo.Buyerclass != "" { set["buyerclass"] = pInfo.Buyerclass } if pInfo.District != "" { set["district"] = pInfo.District } if pInfo.Bidopentime > 0 { set["bidopentime"] = pInfo.Bidopentime } if len(pInfo.Winnerorder) > 0 { set["winnerorder"] = pInfo.Winnerorder } if thisinfo.HasPackage { set["multipackage"] = 1 } else { set["multipackage"] = 0 } if pInfo.ProjectName != "" { set["projectname"] = pInfo.ProjectName } if pInfo.ProjectCode != "" { set["projectcode"] = pInfo.ProjectCode } if pInfo.Buyer != "" { set["buyer"] = pInfo.Buyer } //预算、中标价 if qu.Float64All(tmp["bidamount"]) > 0 { if qu.Float64All((*res)["bidamount"]) == 0 { set["bidamount"] = tmp["bidamount"] } } if qu.Float64All(tmp["budget"]) > 0 { if qu.Float64All((*res)["budget"]) == 0 { set["budget"] = tmp["budget"] } } set["mpn"] = pInfo.MPN set["mpc"] = pInfo.MPC set["area"] = pInfo.Area set["city"] = pInfo.City //e := InitEL(util.ObjToString((*res)["extractpos"])) if thisinfo.dealtype == 1 { var sonpackage map[string]interface{} for _, obj := range tmp["package"].(map[string]interface{}) { sonpackage, _ = obj.(map[string]interface{}) } for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} { if sonpackage[v2] != nil { tmp[v2] = sonpackage[v2] } } } //e.fieldpriority(&tmp, res, &set) //set["extractpos"] = e.GetVal() if thisinfo.HasPackage { //多包处理 p1, _ := (*res)["package"].(map[string]interface{}) p2, _ := tmp["package"].(map[string]interface{}) if p2 != nil { if p1 != nil { for pk2, pv2 := range p2 { if p1[pk2] != nil { //合并 item1, _ := p1[pk2].(map[string]interface{}) item2, _ := pv2.(map[string]interface{}) if item1 != nil && item2 != nil { //原始项 for ik1, iv1 := range item2 { if item1[ik1] == nil { item1[ik1] = iv1 } } } } else { p1[pk2] = pv2 } } } else { p1 = p2 } } set["package"] = p1 } //中标候选人合并 update := map[string]interface{}{} if len(set) > 0 { update["$set"] = set } //保留原数据吧 push := NewPushInfo(tmp) push["compareStr"] = comStr update["$push"] = map[string]interface{}{ "list": push, } if len(update) > 0 { MQFW.Update(projectColl, map[string]interface{}{ "_id": qu.StringTOBsonId(pInfo.Id), }, &update, false, false) } } //再往redis中放 index //往队列中增加时间 -------------->start } func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) { var tk int bf1 := false for _k, tv := range pInfo.Publistime { tk = _k if tv > thisinfo.Publishtime { bf1 = true break } } if bf1 { pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...) //pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...) //pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...) } else { pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime) //pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType}) } pInfo.Ids = append(pInfo.Ids, thisinfo.Id) //增加发布时间结束----------------->end if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) { pInfo.Buyer = thisinfo.Buyer } if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) { pInfo.Agency = thisinfo.Agency } if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) { pInfo.ProjectCode = thisinfo.ProjectCode } if pInfo.Area == "全国" && thisinfo.Area != "全国" { pInfo.Area = thisinfo.Area pInfo.City = thisinfo.City } if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 { pInfo.Buyerperson = thisinfo.Buyerperson pInfo.Buyertel = thisinfo.Buyertel } if thisinfo.Buyerclass != "" { pInfo.Buyerclass = thisinfo.Buyerclass } if thisinfo.District != "" { pInfo.District = thisinfo.District } if thisinfo.Bidopentime > 0 { pInfo.Bidopentime = thisinfo.Bidopentime } if len(thisinfo.Topscopeclass) > 0 { sort.Strings(pInfo.Topscopeclass) for _, k := range thisinfo.Topscopeclass { if BinarySearch(pInfo.Topscopeclass, k) == -1 { pInfo.Topscopeclass = append(pInfo.Topscopeclass, k) sort.Strings(pInfo.Topscopeclass) } } } if len(thisinfo.Subscopeclass) > 0 { sort.Strings(pInfo.Subscopeclass) for _, k := range thisinfo.Subscopeclass { if BinarySearch(pInfo.Subscopeclass, k) == -1 { pInfo.Subscopeclass = append(pInfo.Subscopeclass, k) sort.Strings(pInfo.Subscopeclass) } } } //winner if len(thisinfo.Winners) > 0 { sort.Strings(pInfo.Winners) for _, k := range thisinfo.Winners { if BinarySearch(pInfo.Winners, k) == -1 { pInfo.Winners = append(pInfo.Winners, k) sort.Strings(pInfo.Winners) } } } //winnerorder if len(thisinfo.Winnerorder) > 0 { sort.Strings(pInfo.Winnerorder) for _, k := range thisinfo.Winnerorder { if BinarySearch(pInfo.Winnerorder, k) == -1 { pInfo.Winnerorder = append(pInfo.Winnerorder, k) sort.Strings(pInfo.Winnerorder) } } } } //二分字符串查找 func BinarySearch(s []string, k string) int { sort.Strings(s) lo, hi := 0, len(s)-1 for lo <= hi { m := (lo + hi) >> 1 if s[m] < k { lo = m + 1 } else if s[m] > k { hi = m - 1 } else { return m } } return -1 } var FIELDS = []string{"area", "city", "district", "projectname", "projectcode", "buyer", "winner", "budget", "bidamount", "bidstatus", "agency", "projectscope"} func NewProject(tmp map[string]interface{}, thisinfo *Info) string { set := map[string]interface{}{} for _, f := range FIELDS { if tmp[f] != nil { set[f] = tmp[f] } } set["s_projectname"] = thisinfo.ProjectName set["createtime"] = time.Now().Unix() set["sourceinfoid"] = qu.BsonIdToSId(tmp["_id"]) set["sourceinfourl"] = tmp["href"] set["topscopeclass"] = thisinfo.Topscopeclass set["subscopeclass"] = thisinfo.Subscopeclass if thisinfo.Buyerperson != "" { set["buyerperson"] = thisinfo.Buyerperson } if thisinfo.Buyertel != "" { set["buyertel"] = thisinfo.Buyertel } if thisinfo.Buyerclass != "" { set["buyertel"] = thisinfo.Buyerclass } if thisinfo.Bidopentime > 0 { set["bidopentime"] = thisinfo.Bidopentime } if len(thisinfo.Winnerorder) > 0 { set["winnerorder"] = thisinfo.Winnerorder } s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",") set["s_subscopeclass"] = s_subscopeclass s_winner := strings.Join(thisinfo.Winners, ",") set["s_winner"] = s_winner if tmp["package"] != nil { set["package"] = tmp["package"] //没定义优先级 } push := NewPushInfo(tmp) set["list"] = []bson.M{ push, } id := MQFW.Save(projectColl, set) p1 := NewPinfo(id, thisinfo) redis.PutCKV(REDISIDS, id, p1) return id } //生成存放在redis数组中的对象 func NewPinfo(id string, thisinfo *Info) ProjectInfo { p1 := ProjectInfo{ Publistime: []int64{thisinfo.Publishtime}, InfoType: [][]string{[]string{thisinfo.TopType, thisinfo.SubType}}, Id: id, Ids: []string{thisinfo.Id}, Topscopeclass: thisinfo.Topscopeclass, Subscopeclass: thisinfo.Subscopeclass, Winners: thisinfo.Winners, ProjectName: thisinfo.ProjectName, ProjectCode: thisinfo.ProjectCode, Buyer: thisinfo.Buyer, Agency: thisinfo.Agency, Area: thisinfo.Area, City: thisinfo.City, District: thisinfo.District, MPN: []string{}, MPC: []string{}, HasPackage: thisinfo.HasPackage, Package: map[string]interface{}{}, Buyerclass: thisinfo.Buyerclass, Bidopentime: thisinfo.Bidopentime, Winnerorder: thisinfo.Winnerorder, } if thisinfo.LenPTC > 5 { p1.MPC = append(p1.MPC, thisinfo.PTC) } for k4, _ := range thisinfo.Package { p1.Package[k4] = "" } return p1 } type IdsMapAndIndex struct { Key *Key Index int } var sm = sync.Map{} var ALock = sync.Mutex{} var mapPn = map[string]*Key{} var mapPc = map[string]*Key{} var mapPb = map[string]*Key{} //var mapPnLock, mapPcLock, mapPbLock = sync.Mutex{}, sync.Mutex{}, sync.Mutex{} //从对应map中获取对比的项目id func getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key) { ALock.Lock() defer ALock.Unlock() res = []*Key{} bpn, bpc, bptc, bpb = -1, -1, -1, -1 if pn != "" { //mapPnLock.Lock() ids := mapPn[pn] if ids == nil { ids = &Key{Arr: []string{}} mapPn[pn] = ids bpn = 0 } ids.Lock.Lock() res = append(res, ids) //mapPnLock.Unlock() // index := len(pn) % size // m := AllPNMap[index] // m.Lock.Lock() // ids := m.Map[pn] // //log.Println("PN", ids, pn, index) // if ids == nil { // ids = &Key{Arr: []string{}} // m.Map[pn] = ids // bpn = 0 // } // ids.Lock.Lock() // res = append(res, ids) // m.Lock.Unlock() } if pc != "" { //mapPcLock.Lock() ids := mapPc[pc] if ids == nil { ids = &Key{Arr: []string{}} mapPc[pc] = ids bpc = len(res) } ids.Lock.Lock() res = append(res, ids) //mapPcLock.Unlock() // index := len(pc) % size // m := AllPCMap[index] // m.Lock.Lock() // ids := m.Map[pc] // //log.Println("PC", ids, pc, index) // if ids == nil { // ids = &Key{Arr: []string{}} // m.Map[pc] = ids // bpc = len(res) // } // ids.Lock.Lock() // res = append(res, ids) // m.Lock.Unlock() } if ptc != "" { ids := mapPc[ptc] if ids == nil { ids = &Key{Arr: []string{}} mapPc[ptc] = ids bptc = len(res) } ids.Lock.Lock() res = append(res, ids) // index := len(ptc) % size // m := AllPTCMap[index] // m.Lock.Lock() // ids := m.Map[ptc] // //log.Println("PTC", ids, ptc, index) // if ids == nil { // ids = &Key{Arr: []string{}} // m.Map[ptc] = ids // bptc = len(res) // } // ids.Lock.Lock() // res = append(res, ids) // m.Lock.Unlock() } if pb != "" { ids := mapPb[pb] if ids == nil { ids = &Key{Arr: []string{}} mapPb[pb] = ids bpb = len(res) } ids.Lock.Lock() res = append(res, ids) // index := len(pb) % size // m := AllPBMap[index] // m.Lock.Lock() // ids := m.Map[pb] // //log.Println("PB", ids, pb, index) // if ids == nil { // ids = &Key{Arr: []string{}} // m.Map[pb] = ids // bpb = len(res) // } // ids.Lock.Lock() // res = append(res, ids) // m.Lock.Unlock() } return } //抽取信息打标记 func extInfoTag(sflag, id string) { MQFW.UpdateById(extractColl, id, map[string]interface{}{ "$set": map[string]interface{}{ "meger_sflag": sflag, }, }) }