package main import ( "encoding/json" // "fmt" du "jy/util" qu "qfw/util" "regexp" "strings" "sync" "time" "gopkg.in/mgo.v2/bson" ) //抽取信息映射实体类 type Info struct { Id string `json:"_id"` Href string `json:"href"` Publishtime int64 `json:"publishtime"` Title string `json:"title"` TopType string `json:"toptype"` SubType string `json:"subtype"` ProjectName string `json:"projectname"` ProjectCode string `json:"projectcode"` Buyer string `json:"buyer"` Buyerperson string `json:"buyerperson"` Buyertel string `json:"buyertel"` Agency string `json:"agency"` Area string `json:"area"` City string `json:"city"` HasPackage bool `json:"haspackage"` Package map[string]interface{} `json:"package"` PNum string `json:"pnum"` Topscopeclass []string `json:"topscopeclass"` Subscopeclass []string `json:"subscopeclass"` Winners []string dealtype int Buyerclass string `json:"buyerclass"` Bidopentime int64 `json:"bidopentime"` District string `json:"district"` Winnerorder []string PTC string pnbval int LenPC int LenPN int LenPTC int } var ( PNKeyMap, PCKeyMap, PBKeyMap = sync.Map{}, sync.Map{}, sync.Map{} pnreg = regexp.MustCompile("^(及编号[：:])|(项目|采购|招标|中标|成交|结果|[_]|公告)$") titleGetPn = regexp.MustCompile("^([\\[【(（]?.?(资格预审|中标|招标|延期|成交|结果|合同|失败|询价|关于对?)(公告)?[\\]】)）]?([:：]|关于对?)?)?(.{4,70}?(采购|工程)?(项目)?)([(（【]?(第?[一二三四五六七八九1-9再]次|重新|重招|公开|[预拟]).{0,3}?[)）】]?)?(招标|采购|采购计划|发包|结果|变更|更正|成交|网上(竞价)?|电子化|电子反拍|询比?价|比价|竞争性(谈判|磋商)|流标|废标|邀请|合同|验收|违规|资格|预审|中标(结果)?|延期|澄清|暂停|补遗|终止|文件|标前|征求|报建|征集|论证|谈判|拟实施|中止|需求|比选|评标(过程)?及?|磋商|未入围|进口|投标|答疑|抽签|异常|质疑|答复|回复|应答|遴选|最高|拦标|推迟|开标|取消|延迟|撤销|控制价|场外|作废|候选人|采用|实施|预|不良记录|竞买|反拍|修正|调整|简称|小型)?(公告|记录|公示|预告|通知[函书]?|意见[函书]?|[函书])?([(（【].*?[)）】])?$") titleGetPc = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)") titleGetPc1 = regexp.MustCompile("[\\[【(（](.{0,6}(编号|编码|项号|包号|代码|标段?号)[:：为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()（）][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】)）]") titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$") numreg = regexp.MustCompile("^[0-9]$") numreg2 = regexp.MustCompile("^[0-9]+$") numCheckPc = regexp.MustCompile("^[0-9-]{1,10}$") TitleReg = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+(子|合同|分|施工|监理|标)?[包标段][号段]?[、]?)+|((子|合同|分|施工|监理|标)?[包标段][号段]?[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+[、]?)+|(子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+[\\-~－至、](子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+") ) type ProjectInfo struct { Id string `json:"id"` Publistime []int64 `json:"publistime"` //多条信息的发布时间、跨度 InfoType [][]string `json:"infotype"` //多条信息内的 toptype、subtype Ids []string `json:"ids"` Topscopeclass []string `json:"topscopeclass"` Subscopeclass []string `json:"subscopeclass"` Winners []string `json:"winners"` ProjectName string `json:"projectname"` ProjectCode string `json:"projectcode"` //项目代码唯一（纯数字的权重低） Buyer string `json:"buyer"` //采购单位唯一 MPN []string `json:"mpn"` //合并后多余的项目名称 MPC []string `json:"mpc"` //合并后多余的项目编号 Buyerperson string `json:"buyerperson"` Buyertel string `json:"buyertel"` Agency string `json:"agency"` //代理机构唯一 Area string `json:"area"` //地区唯一 City string `json:"city"` //地市 District string `json:"district"` //区县 HasPackage bool `json:"haspackage"` //是否有分包 Package map[string]interface{} `json:"package"` //分包的对比对象 Buyerclass string `json:"buyerclass"` //采购单位分类 Bidopentime int64 `json:"bidopentime"` //开标时间 Winnerorder []string //中标候选人 score int comStr string } type KeyMap struct { Lock sync.Mutex Map map[string]*Key } type Key struct { Arr []string Lock sync.Mutex } type IdAndLock struct { Id string Lock sync.Mutex } func NewKeyMap() *KeyMap { return &KeyMap{ Map: map[string]*Key{}, Lock: sync.Mutex{}, } } var size, idsMapSize = 30, 100 var AllPNMap = make([]*KeyMap, size) //存储 项目名称,值为id数组 var AllPCMap = make([]*KeyMap, size) //存储 项目编号,值为id数组 var AllPTCMap = make([]*KeyMap, size) //存储 项目编号,值为id数组 var AllPBMap = make([]*KeyMap, size) //存储 采购单位,值为id数组 type ID struct { Id string Lock sync.Mutex lastTime int64 pos int } //所有项目id对象，加锁，删除等用 var AllIdsMap = make([]map[string]*ID, idsMapSize) var AllIdsMap2 = map[string]*ID{} var AllIdsMapLock = sync.Mutex{} //预定义字符串 [ABCD][ABCD][ABCD] 项目名称/编号/标题编号 var compareNoPass = map[string]bool{} var compareAB = map[string]bool{} var compareAB2D = map[string]bool{} var compareABD = map[string]bool{} var compareAB2CD = map[string]bool{} var compareABCD = map[string]bool{} func init() { for i := 0; i < size; i++ { AllPNMap[i] = NewKeyMap() AllPCMap[i] = NewKeyMap() AllPTCMap[i] = NewKeyMap() AllPBMap[i] = NewKeyMap() } for i := 0; i < idsMapSize; i++ { AllIdsMap[i] = map[string]*ID{} } //---不能通过 vm := []string{"C", "D"} for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { for k := 0; k < 2; k++ { key := vm[i] + vm[j] + vm[k] compareNoPass[key] = true //fmt.Println(key) } } } //fmt.Println("-------------------") //三个元素一致 [AB][AB][AB]，分值最高 vm = []string{"A", "B"} for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { for k := 0; k < 2; k++ { key := vm[i] + vm[j] + vm[k] compareAB[key] = true //fmt.Println(key) } } } //fmt.Println("-------------------", len(compareAB)) //---至少两个一致，其他可能不存在 //[AB][AB][ABD] //[AB][ABD][AB] vm = []string{"A", "B"} vm2 := []string{"A", "B", "D"} for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { for k := 0; k < 3; k++ { key := vm[i] + vm[j] + vm2[k] if !compareAB[key] { compareAB2D[key] = true //fmt.Println(key) } } } } for i := 0; i < 2; i++ { for j := 0; j < 3; j++ { for k := 0; k < 2; k++ { key := vm[i] + vm2[j] + vm[k] if !compareAB[key] { compareAB2D[key] = true //fmt.Println(key) } } } } //fmt.Println("-------------------", len(compareAB2D)) //---至少一个一致，其他可能不存在 //[ABD][ABD][ABD] //已经删除DDD vm = []string{"A", "B", "D"} for i := 0; i < 3; i++ { for j := 0; j < 3; j++ { for k := 0; k < 3; k++ { key := vm[i] + vm[j] + vm[k] if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] { compareABD[key] = true //fmt.Println(key) } } } } //fmt.Println("-------------------", len(compareABD)) //[AB][ABCD][AB] //[AB][AB][ABCD] vm = []string{"A", "B"} vm2 = []string{"A", "B", "C", "D"} for i := 0; i < 2; i++ { for j := 0; j < 4; j++ { for k := 0; k < 2; k++ { key := vm[i] + vm2[j] + vm[k] if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] { compareAB2CD[key] = true //fmt.Println(key) } } } } for i := 0; i < 2; i++ { for j := 0; j < 2; j++ { for k := 0; k < 4; k++ { key := vm[i] + vm[j] + vm2[k] if !compareAB[key] && !compareAB2D[key] && !compareNoPass[key] && !compareABD[key] { compareAB2CD[key] = true //fmt.Println(key) } } } } //fmt.Println("-------------------", len(compareAB2CD)) //[ABECD][ABECD][ABECD] //已经删除[CD][CD][CD] //这个要重点讨论 vm = []string{"A", "B", "C", "D"} for i := 0; i < 4; i++ { for j := 0; j < 4; j++ { for k := 0; k < 4; k++ { key := vm[i] + vm[j] + vm[k] if !compareAB[key] && !compareAB2D[key] && !compareABD[key] && !compareNoPass[key] && !compareAB2CD[key] { compareABCD[key] = true //fmt.Println(key) } } } } //fmt.Println("-------------------", len(compareABCD)) } //预处理信息 func PreThisInfo(tmp map[string]interface{}) *Info { bys, _ := json.Marshal(tmp) var thisinfo *Info json.Unmarshal(bys, &thisinfo) if thisinfo == nil { return nil } if len(thisinfo.Topscopeclass) == 0 { thisinfo.Topscopeclass = []string{} } if len(thisinfo.Subscopeclass) == 0 { thisinfo.Subscopeclass = []string{} } //去重 thisinfo.Subscopeclass = RemoveDup(thisinfo.Subscopeclass) if len(thisinfo.Package) > 0 { //信息是否分包 thisinfo.HasPackage = true } else if thisinfo.TopType == "结果" && TitleReg.MatchString(thisinfo.Title) { //当信息类型是结果时，并且标题中包含分包字样，找到包号，用以后面比较打分 res := TitleReg.FindAllStringSubmatch(thisinfo.Title, -1) pnum := du.PackageNumberConvert(res[0][0]) //du.Debug(pnum, res) thisinfo.PNum = pnum } // if checkInfoAlter(tmp) { // thisinfo.SubType = "变更" // } //计算中标人 winner, _ := tmp["winner"].(string) m1 := map[string]bool{} winners := []string{} if winner != "" { m1[winner] = true winners = append(winners, winner) } if thisinfo.HasPackage { packageM, _ := tmp["package"].(bson.M) for _, p := range packageM { pm, _ := p.(map[string]interface{}) pw, _ := pm["winner"].(string) if pw != "" { m1[pw] = true winners = append(winners, pw) } } } thisinfo.Winners = winners m1 = nil //中标候选人 winnerorder := []string{} if winorders, ok := tmp["winnerorder"].([]interface{}); ok { for _, wins := range winorders { if win, ok := wins.(map[string]interface{}); ok { entname := qu.ObjToString(win["entname"]) if entname != "" && len([]rune(entname)) > 6 { winnerorder = append(winnerorder, entname) } } } } thisinfo.Winnerorder = winnerorder res := titleGetPc.FindStringSubmatch(thisinfo.Title) if len(res) > 1 && len(res[1]) > 8 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) { thisinfo.PTC = res[1] thisinfo.pnbval++ } else { res = titleGetPc1.FindStringSubmatch(thisinfo.Title) if len(res) > 3 && len(res[3]) > 8 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) { thisinfo.PTC = res[3] thisinfo.pnbval++ } else { res = titleGetPc2.FindStringSubmatch(thisinfo.Title) if len(res) > 1 && len(res[1]) > 8 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) { thisinfo.PTC = res[1] thisinfo.pnbval++ } } } if thisinfo.ProjectName != "" { thisinfo.pnbval++ } if thisinfo.ProjectCode != "" && !_zimureg1.MatchString(thisinfo.ProjectCode) { thisinfo.pnbval++ } else { thisinfo.ProjectCode = "" } if thisinfo.Buyer != "" { thisinfo.pnbval++ } thisinfo.LenPC = len([]rune(thisinfo.ProjectCode)) thisinfo.LenPTC = len([]rune(thisinfo.PTC)) thisinfo.LenPN = len([]rune(thisinfo.ProjectName)) return thisinfo } //移除数组中重复的元素 func RemoveDup(arr []string) (newarr []string) { m1 := map[string]bool{} newarr = []string{} for _, k := range arr { if !m1[k] { m1[k] = true newarr = append(newarr, k) } } return } //阻塞同名的pb、pc、pv并发 func lockPNCBMap(thisinfo *Info) { for { //等待其他任务完成 ok := true if thisinfo.LenPN > 0 { if _, b := PNKeyMap.Load(thisinfo.ProjectName); b { ok = false } } if thisinfo.LenPC > 0 { if _, b := PCKeyMap.Load(thisinfo.ProjectCode); b { ok = false } } if thisinfo.LenPTC > 0 { if _, b := PCKeyMap.Load(thisinfo.PTC); b { ok = false } } if len(thisinfo.Buyer) > 0 { if _, b := PBKeyMap.Load(thisinfo.Buyer); b { ok = false } } if ok { break } else { time.Sleep(30 * time.Millisecond) } } } //放行 func unlockPNCBMap(thisinfo *Info) { PNKeyMap.Delete(thisinfo.ProjectName) PCKeyMap.Delete(thisinfo.ProjectCode) PCKeyMap.Delete(thisinfo.PTC) PBKeyMap.Delete(thisinfo.Buyer) } //判断信息是否是变更 func checkInfoAlter(tmp map[string]interface{} /*新信息*/) bool { toptype := qu.ObjToString(tmp["toptype"]) subtype := qu.ObjToString(tmp["subtype"]) title := qu.ObjToString(tmp["title"]) if subtype == "变更" || strings.Index(title, "变更公告") > -1 || strings.Index(title, "更正公告") > -1 { //当信息类型是变更或标题中含变更时 if toptype == "招标" { //招标的变更公告，不作处理 } else if toptype == "结果" { subtype = "变更" } } return subtype == "变更" }