// projectmeger package main import ( "encoding/json" "fmt" du "jy/util" "log" qu "qfw/util" "qfw/util/redis" "sort" "strings" "sync" "time" ) //有效值三选一、三选二 var ThreeToTow, ThreeToOne map[string]bool var ThreeToLock = &sync.Mutex{} func init() { ThreeToTow = map[string]bool{} ThreeToOne = map[string]bool{ "AAA": true, "AAB": true, } tows := [][][]string{ [][]string{ []string{"A", "B"}, []string{"A", "B"}, []string{"A", "B"}, }, [][]string{ []string{"A", "B"}, []string{"D", "E"}, []string{"A", "B"}, }, [][]string{ []string{"A", "B"}, []string{"A", "B"}, []string{"D", "E"}, }, [][]string{ []string{"D", "E"}, []string{"A", "B"}, []string{"A", "B"}, }, } for _, tow := range tows { for _, v1 := range tow[0] { for _, v2 := range tow[1] { for _, v3 := range tow[2] { key := fmt.Sprintf("%s%s%s", v1, v2, v3) ThreeToTow[key] = true } } } } } func startProjectMerge(thisinfo *Info, tmp map[string]interface{}) { bNormalScore := false pcbv := PCBVal(tmp) if checkInfoAlter(tmp) { //进入变更信息流程 if pcbv.Val > 1 { //判断三项至少包含两项 bNormalScore = true } else { extInfoTag("invalid", qu.BsonIdToSId(tmp["_id"])) //无效信息,打标记 //go IS.Add("invalid") //数据统计使用 return } } else { bNormalScore = true } //合并流程 if bNormalScore { if pcbv.Buyer { //有采购单位 hasBuyer(pcbv, thisinfo, tmp) } else { //无采购单位 noBuyer(pcbv, thisinfo, tmp) } } } //判断信息是否是变更 func checkInfoAlter(tmp map[string]interface{} /*新信息*/) bool { toptype := qu.ObjToString(tmp["toptype"]) subtype := qu.ObjToString(tmp["subtype"]) title := qu.ObjToString(tmp["title"]) if subtype == "变更" || strings.Index(title, "变更公告") > -1 || strings.Index(title, "更正公告") > -1 { //当信息类型是变更或标题中含变更时 if toptype == "招标" { //招标的变更公告,不作处理 } else if toptype == "结果" { subtype = "变更" } } return subtype == "变更" } //有采购单位 func hasBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) { var pncb []*CompareInfo var res []interface{} sflag := "" pici := time.Now().Unix() if p.Pname || p.Pcode { //有项目名称或项目编号 //获取对比项目数组 res, pncb = getComeperProjects(p, thisinfo) //三选二打分 scores := score3Select2(p, thisinfo, tmp, res, pncb) //项目合并 sflag = mergeProject(tmp, thisinfo, scores, pncb) } else { //生成项目,不参与后续对比 sflag = "alone" mess := map[string]interface{}{ "meger_mess": "有采购单位,不满足三选二", "meger_sflag": sflag, } newProject(tmp, mess, pici, thisinfo) } extInfoTag(sflag, thisinfo.Id) go IS.Add(sflag) //数据统计使用 } //无采购单位 func noBuyer(p PCBV, thisinfo *Info, tmp map[string]interface{}) { var pncb []*CompareInfo var res []interface{} sflag := "" pici := time.Now().Unix() if p.Pname { //有项目名称 //获取对比项目数组 res, pncb = getComeperProjects(p, thisinfo) if p.Pcode { //有项目编号 //三选二打分 scores := score3Select2(p, thisinfo, tmp, res, pncb) //项目合并 sflag = mergeProject(tmp, thisinfo, scores, pncb) } else { //无项目编号 if p.PnameLen > MegerFieldsLen.ProjectNamelen { if p.Area && p.City && p.Agency { //三选一打分 scores := score3Select1(p, thisinfo, tmp, res, pncb) //项目合并 sflag = mergeProject(tmp, thisinfo, scores, pncb) } else { sflag = "alone" mess := map[string]interface{}{ "meger_mess": "无采购单位,不满足三选一", "meger_sflag": sflag, } newProject(tmp, mess, pici, thisinfo) } } else { //生成项目,不参与后续对比 sflag = "alone" mess := map[string]interface{}{ "meger_mess": "无采购单位,不满足三选一", "meger_sflag": sflag, } newProject(tmp, mess, pici, thisinfo) } } } else { //无项目名称 if p.Pcode { //获取对比项目数组 res, pncb = getComeperProjects(p, thisinfo) if p.PcodeLen > MegerFieldsLen.ProjectCodelen { if p.Area && p.City && p.Agency { //有省市代理机构 //三选一打分 scores := score3Select1(p, thisinfo, tmp, res, pncb) //项目合并 sflag = mergeProject(tmp, thisinfo, scores, pncb) } else { //生成项目,不参与后续对比 sflag = "alone" mess := map[string]interface{}{ "meger_mess": "无采购单位,不满足三选一", "meger_sflag": sflag, } newProject(tmp, mess, pici, thisinfo) } } else { //生成项目,不参与后续对比 sflag = "alone" mess := map[string]interface{}{ "meger_mess": "无采购单位,不满足三选一", "meger_sflag": sflag, } newProject(tmp, mess, pici, thisinfo) } } else { //信息无效,打标记 sflag = "invalid" } } extInfoTag(sflag, thisinfo.Id) //go IS.Add(sflag) //数据统计使用 } //3选2打分 func score3Select2(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) { defer qu.Catch() if len(res) > 0 { //对比打分 for k, tmps := range res { //遍历对象数组 if tmp, ok := tmps.([]interface{}); ok { for _, b1 := range tmp { if b1 != nil { var info ProjectInfo err := json.Unmarshal(b1.([]byte), &info) if err != nil { log.Println(err) } cone := &CompareOne{ Parent: pncb[k], Pinfo: &info, } cone.BuyerType, cone.Score = fieldPCBScore(thisinfo.Buyer, info.Buyer, cone.BuyerType, cone.Score) if p.Buyer { cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score) cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score) } else { //无采购单位,打分考虑长度 if len([]rune(thisinfo.ProjectName)) > MegerFieldsLen.ProjectNamelen { cone.ProjectNameType, cone.Score = fieldPCBScore(thisinfo.ProjectName, info.ProjectName, cone.ProjectNameType, cone.Score) } else { cone.ProjectNameType = "D" } if len(thisinfo.ProjectCode) > MegerFieldsLen.ProjectCodelen { cone.ProjectCodeType, cone.Score = fieldPCBScore(thisinfo.ProjectCode, info.ProjectCode, cone.ProjectCodeType, cone.Score) } else { cone.ProjectCodeType = "D" } } //省市打分 if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" { if thisinfo.Area == info.Area && thisinfo.City == info.City { cone.Score += 2 } else { cone.Score -= 1 } } else { cone.Score += 1 } //代理机构打分 if len([]rune(info.Agency)) > 0 { if thisinfo.Agency == info.Agency { //A cone.Score += 2 } else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B cone.Score += 1 } else { if len(thisinfo.Agency) < 1 { //E cone.Score -= 1 } else { //C cone.Score -= 2 } } } else { //D不计分 // } skey := fmt.Sprintf("%s%s%s", cone.BuyerType, cone.ProjectNameType, cone.ProjectCodeType) cone.Cresult = skey ThreeToLock.Lock() if ThreeToTow[skey] { scores = append(scores, cone) } ThreeToLock.Unlock() } } } } } return } //3选1打分 func score3Select1(p PCBV, thisinfo *Info, tmp map[string]interface{}, res []interface{}, pncb []*CompareInfo) (scores []*CompareOne) { defer qu.Catch() //对比打分 if len(res) > 0 { //找到项目名称、项目编号或采购单位相同时 for k, tmps := range res { //遍历对象数组 if tmp, ok := tmps.([]interface{}); ok { for _, b1 := range tmp { if b1 != nil { var info ProjectInfo err := json.Unmarshal(b1.([]byte), &info) if err != nil { log.Println(err) } cone := &CompareOne{ Parent: pncb[k], Pinfo: &info, } if pncb[k].Field == "pn" { //比较字段项目名称 if len(info.ProjectName) > 0 { if thisinfo.ProjectName == info.ProjectName { //A cone.Score += 2 cone.ProjectNameType = "A" } else if strings.Index(info.ProjectName, thisinfo.ProjectName) > -1 || strings.Index(thisinfo.ProjectName, info.ProjectName) > -1 { //B cone.Score += 1 cone.ProjectNameType = "B" } else { //C cone.Score -= 2 cone.ProjectNameType = "C" } } else { //D不计分 cone.ProjectNameType = "D" } } if pncb[k].Field == "pc" { //比较字段项目编号 if len(info.ProjectCode) > 0 { if thisinfo.ProjectCode == info.ProjectCode { //A cone.Score += 2 cone.ProjectCodeType = "A" } else if strings.Index(info.ProjectCode, thisinfo.ProjectCode) > -1 || strings.Index(thisinfo.ProjectCode, info.ProjectCode) > -1 { //B cone.Score += 1 cone.ProjectCodeType = "B" } else { //C cone.Score -= 2 cone.ProjectCodeType = "C" } } else { //D不计分 cone.ProjectCodeType = "D" } } if thisinfo.Area != "A" && thisinfo.Area != "全国" && info.Area != "A" && info.Area != "全国" { if thisinfo.Area == info.Area && thisinfo.City == info.City { cone.Score += 2 cone.AreaType = "A" } else { cone.Score -= 1 cone.AreaType = "C" } } else { cone.Score += 1 cone.AreaType = "B" } if len([]rune(info.Agency)) > 0 { if thisinfo.Agency == info.Agency { //A cone.Score += 2 cone.AgencyType = "A" } else if strings.Index(info.Agency, thisinfo.Agency) > -1 || strings.Index(thisinfo.Agency, info.Agency) > -1 { //B cone.Score += 1 cone.AgencyType = "B" } else { if len(thisinfo.Agency) < 1 { //E cone.Score -= 1 cone.AgencyType = "E" } else { //C cone.Score -= 2 cone.AgencyType = "C" } } } else { //D不计分 cone.AgencyType = "D" } skey := fmt.Sprintf("%s%s%s%s", cone.ProjectNameType, cone.ProjectCodeType, cone.AreaType, cone.AgencyType) cone.Cresult = skey ThreeToLock.Lock() if ThreeToOne[skey] { scores = append(scores, cone) } ThreeToLock.Unlock() } } } } } return } //获取对比项目数组 func getComeperProjects(p PCBV, thisinfo *Info) (res []interface{}, pncb []*CompareInfo) { if p.PnameLen > 0 { pn := NewCompareInfo("pn", thisinfo.PNKey, PNKey) pncb = append(pncb, pn) } if p.PcodeLen > 0 { pc := NewCompareInfo("pc", thisinfo.PCKey, PCKey) pncb = append(pncb, pc) } if p.BuyerLen > 0 { pb := NewCompareInfo("pb", thisinfo.PBKey, PBKey) pncb = append(pncb, pb) } repeatId := map[string]bool{} //IdLock.Lock() //此处加id锁,会引进多线程的死锁,对比三个大map数组,找到key相同的项目id数组,并去重 for _, pv := range pncb { pv.KeyMap.Lock.Lock() K := pv.KeyMap.Map[pv.Key] if K == nil { K = &Key{&[]string{}, &sync.Mutex{}} pv.KeyMap.Map[pv.Key] = K } pv.K = K pv.K.Lock.Lock() pv.KeyMap.Lock.Unlock() defer pv.K.Lock.Unlock() newarr := []string{} for _, id := range *K.Arr { if !repeatId[id] { newarr = append(newarr, id) repeatId[id] = true } } pv.IdArr = newarr } //IdLock.Unlock() for _, pv := range pncb { if len(pv.IdArr) > 0 { res = append(res, redis.Mget(REDISIDS, pv.IdArr)) } } return } //合并项目 func mergeProject(tmp map[string]interface{}, thisinfo *Info, scores []*CompareOne, pncb []*CompareInfo) (sflag string) { var id = "" if len(scores) > 0 { //分值排序 sort.Slice(scores, func(i, j int) bool { return scores[i].Score > scores[j].Score }) max := scores[0] if max.Score > 0 { sflag = "normal" max.Parent.Bfind = true tmp["cresult"] = max.Cresult tmp["score"] = max.Score id = updateinfo(thisinfo, tmp, max.Pinfo, time.Now().Unix()) //log.Println(sflag, id, max.Cresult) if len(scores) > 1 && (scores[0].Score == scores[1].Score) { //通知检查,异常数据存库 checkNotice(thisinfo.Id, id, scores) } } else { sflag = "invalidscore" } } else { //生成项目 sflag = "normal" mess := map[string]interface{}{ "meger_sflag": sflag, } id = newProject(tmp, mess, time.Now().Unix(), thisinfo) } if id != "" { //更新REDISKEYS redis put := []interface{}{} for _, pv := range pncb { if pv != nil && !pv.Bfind { if BinarySearch(*pv.K.Arr, id) == -1 { *(pv.K.Arr) = append(*(pv.K.Arr), id) put = append(put, []interface{}{pv.Key, *(pv.K.Arr)}) } } } if len(put) > 0 { redis.BulkPut(REDISKEYS, 0, put...) } } return sflag } //新增项目 func newProject(tmp, mess map[string]interface{}, pipc int64, thisinfo *Info) (id string) { id = InsertProject(thisinfo.NewPNKey, tmp, mess, pipc, thisinfo) sflag := qu.ObjToString(mess["meger_sflag"]) if sflag != "alone" { p1 := NewPinfo(id, thisinfo) redis.PutCKV(REDISIDS, id, p1) } else { du.Debug("新增项目,不参与对比", id) } return id } //抽取信息打标记 func extInfoTag(sflag, id string) { MQFW.UpdateById(extractColl, id, map[string]interface{}{ "$set": map[string]interface{}{ "meger_sflag": sflag, }, }) } //检查通知 func checkNotice(infoid, pid string, scores []*CompareOne) { MQFW.Update("project_unusual", `{"project_id":"`+pid+`"}`, map[string]interface{}{ "$set": map[string]interface{}{ "project_id": pid, "pici": time.Now().Unix(), }, "$push": map[string]interface{}{ "list": map[string]interface{}{ "infoid": infoid, "pinfo1": scores[0].Pinfo.Id, "pinfo2": scores[1].Pinfo.Id, }, }, }, true, false) du.Debug("通知检查,异常数据存库", infoid, pid) } //属性判断组 func PCBVal(tmp map[string]interface{}) PCBV { pcbv := PCBV{} projectName := qu.ObjToString(tmp["projectname"]) if len([]rune(projectName)) > 1 { pcbv.Val += 1 pcbv.Pname = true pcbv.PnameLen = len([]rune(projectName)) } projectCode := qu.ObjToString(tmp["projectcode"]) if len([]rune(projectCode)) > 1 { pcbv.Val += 1 pcbv.Pcode = true pcbv.PcodeLen = len(projectCode) } buyer := qu.ObjToString(tmp["buyer"]) if len([]rune(buyer)) > 1 { pcbv.Val += 1 pcbv.Buyer = true pcbv.BuyerLen = len([]rune(buyer)) } //省市代理机构 if qu.ObjToString(tmp["area"]) != "" { pcbv.Area = true } if qu.ObjToString(tmp["city"]) != "" { pcbv.City = true } if qu.ObjToString(tmp["agency"]) != "" { pcbv.Agency = true } return pcbv } //项目名称、项目编号、采购单位打分 func fieldPCBScore(this, info, ctype string, score int) (string, int) { if len(info) > 0 { if this == info { //A score += 5 ctype = "A" } else if strings.Index(info, this) > -1 || strings.Index(this, info) > -1 { //B score += 2 ctype = "B" } else { if len(this) < 1 { //E score -= 1 ctype = "E" } else { //C score -= 2 ctype = "C" } } } else { //D不计分 ctype = "D" } return ctype, score } //pncbMap加锁 func lockPNCBMap(thisinfo *Info) { for { //等待其他任务完成 ok := true if len(thisinfo.PNKey) > 3 { if _, b := PNKeyMap.Load(thisinfo.PNKey); b { ok = false } } if len(thisinfo.PCKey) > 3 { if _, b := PCKeyMap.Load(thisinfo.PCKey); b { ok = false } } if len(thisinfo.PBKey) > 3 { if _, b := PBKeyMap.Load(thisinfo.PBKey); b { ok = false } } if ok { break } else { //log.Println("has key store") time.Sleep(100 * time.Millisecond) } } } //pncbMap解锁 func unlockPNCBMap(thisinfo *Info) { //log.Println("del key store", thisinfo.PNKey) //if len(thisinfo.PNKey) > 3 { PNKeyMap.Delete(thisinfo.PNKey) //} //if len(thisinfo.PCKey) > 3 { PCKeyMap.Delete(thisinfo.PCKey) //} //if len(thisinfo.PBKey) > 3 { PBKeyMap.Delete(thisinfo.PBKey) //} } //store lock func storeLock(thisinfo *Info) { PncbMayLock.Lock() PNKeyMap.Store(thisinfo.PNKey, true) PBKeyMap.Store(thisinfo.PBKey, true) PCKeyMap.Store(thisinfo.PCKey, true) PncbMayLock.Unlock() }