jy-data-extract/udpprojectset/src/compare.go

package main

import (
	"qfw/util"
	"qfw/util/redis"
	"regexp"
	"sort"
	"strings"
	"time"

	"gopkg.in/mgo.v2/bson"
)

/**
发布时间（结合信息类型-->）
信息类型（拟建、招标(变更)、结果(成交、中标、流标、废标、变更)、其他(合同、验收、违规、变更)）
项目代码（长度）
采购单位（结合省份）
招标机构
所属省份
信息标题（含有二次、标段、包的情况，结合分包情况处理）
是否分包：
**/
type ProjectInfo struct {
	Id            string                 `json:"id"`
	IdInc         string                 `json:"_id"`        //补漏使用
	Publistime    []int64                `json:"publistime"` //多条信息的发布时间、跨度
	InfoType      [][]string             `json:"infotype"`   //多条信息内的 toptype、subtype
	Ids           []string               `json:"ids"`
	Topscopeclass []string               `json:"topscopeclass"`
	Subscopeclass []string               `json:"subscopeclass"`
	Winners       []string               `json:"winners"`
	ProjectName   string                 `json:"projectname"`
	ProjectCode   string                 `json:"projectcode"` //项目代码唯一（纯数字的权重低）
	Buyer         string                 `json:"buyer"`       //采购单位唯一
	Buyerperson   string                 `json:"buyerperson"`
	Buyertel      string                 `json:"buyertel"`
	Agency        string                 `json:"agency"`     //代理机构唯一
	Area          string                 `json:"area"`       //地区唯一
	City          string                 `json:"city"`       //地市
	HasPackage    bool                   `json:"haspackage"` //是否有分包
	Package       map[string]interface{} `json:"package"`    //分包的对比对象

	Buyerclass  string   `json:"buyerclass"`  //采购单位分类
	Bidopentime int64    `json:"bidopentime"` //开标时间
	District    string   `json:"district"`    //区县
	Winnerorder []string //中标候选人
}

type CompareOne struct {
	ProjectNameType string //项目名称对比结果分类 A相等 B包含 C不相等 Dthis存在对比不存在 Ethis不存在
	ProjectCodeType string //同上
	BuyerType       string //同上
	AreaType        string //同上
	AgencyType      string //同上
	PublistimeType  int    //1在时间范围  2不在
	PackageType     int    //1都是多包 2都不是多包 3招标 4新信息是结果
	Score           int
	Parent          *CompareInfo
	Pinfo           *ProjectInfo
	Pos             int

	Cresult string //对比结果
}

type CompareInfo struct {
	Field  string        //对比属性 pn/pc/pb
	Key    string        //存放rediskey
	Scores []*CompareOne //对比分值 pinfo索引对应分值
	Bfind  bool          //是否查找到
	IdArr  []string
	K      *Key
	KeyMap *KeyMap
}

func NewCompareInfo(field, key string, KeyMap *KeyMap) *CompareInfo {
	return &CompareInfo{
		Field:  field,
		Key:    key,
		Scores: []*CompareOne{},
		KeyMap: KeyMap,
	}

}

var numreg2 = regexp.MustCompile("^[0-9]+$")
var TitleReg = regexp.MustCompile("([一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+(子|合同|分|施工|监理|标)?[包标段][号段]?[、]?)+|((子|合同|分|施工|监理|标)?[包标段][号段]?[一二三四五六七八九十0-9A-Za-zⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ、\\-~至]+[、]?)+|(子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+[\\-~－至、](子|合同|分|施工|监理|标)?[包标段][号段]?[a-zA-Z0-9]+")

//没有在redis中查找到，新增项目
func InsertProject(new_pn string, tmp, mess map[string]interface{}, pici int64, thisinfo *Info) string {
	e := InitEL("")
	set := map[string]interface{}{
		"pici": pici,
	}
	set["s_projectname"] = new_pn
	e.fieldpriority(&tmp, nil, &set)
	set["extractpos"] = e.GetVal()
	set["createtime"] = time.Now().Unix()
	set["sourceinfoid"] = util.BsonIdToSId(tmp["_id"])
	set["sourceinfourl"] = tmp["href"]
	set["topscopeclass"] = thisinfo.Topscopeclass
	set["subscopeclass"] = thisinfo.Subscopeclass
	if thisinfo.Buyerperson != "" {
		set["buyerperson"] = thisinfo.Buyerperson
	}
	if thisinfo.Buyertel != "" {
		set["buyertel"] = thisinfo.Buyertel
	}
	if thisinfo.Buyerclass != "" {
		set["buyertel"] = thisinfo.Buyerclass
	}
	if thisinfo.District != "" {
		set["district"] = thisinfo.District
	}
	if thisinfo.Bidopentime > 0 {
		set["bidopentime"] = thisinfo.Bidopentime
	}
	if len(thisinfo.Winnerorder) > 0 {
		set["winnerorder"] = thisinfo.Winnerorder
	}
	s_subscopeclass := strings.Join(thisinfo.Subscopeclass, ",")
	set["s_subscopeclass"] = s_subscopeclass
	s_winner := strings.Join(thisinfo.Winners, ",")
	set["s_winner"] = s_winner
	if tmp["package"] != nil {
		set["package"] = tmp["package"] //没定义优先级
	}
	push := NewPushInfo(tmp)
	for tkey, _ := range extractpos {
		if tmp[tkey] != nil {
			push[tkey] = tmp[tkey]
		}
	}
	set["list"] = []bson.M{
		push,
	}
	if mess != nil { //项目标识
		for k, v := range mess {
			set[k] = v
		}
	}
	return MQFW.Save(projectColl, set)
}

//生成存放在redis数组中的对象
func NewPinfo(id string, thisinfo *Info) ProjectInfo {
	p1 := ProjectInfo{
		Publistime:    []int64{thisinfo.Publishtime},
		InfoType:      [][]string{[]string{thisinfo.TopType, thisinfo.SubType}},
		Id:            id,
		Ids:           []string{thisinfo.Id},
		Topscopeclass: thisinfo.Topscopeclass,
		Subscopeclass: thisinfo.Subscopeclass,
		Winners:       thisinfo.Winners,
		ProjectName:   thisinfo.ProjectName,
		ProjectCode:   thisinfo.ProjectCode,
		Buyer:         thisinfo.Buyer,
		Agency:        thisinfo.Agency,
		Area:          thisinfo.Area,
		HasPackage:    thisinfo.HasPackage,
		Package:       map[string]interface{}{},
		Buyerclass:    thisinfo.Buyerclass,
		Bidopentime:   thisinfo.Bidopentime,
		District:      thisinfo.District,
		Winnerorder:   thisinfo.Winnerorder,
	}
	for k4, _ := range thisinfo.Package {
		p1.Package[k4] = ""
	}
	return p1
}

//每一个字段的打分map,获取最高分
func GetMaxScore(scoremap map[int]int) (find bool, index int) {
	ks := []int{}
	vs := []int{}
	for k, v := range scoremap {
		mk, mv := -1, -100
		bf := false
		for mk, mv = range vs {
			if mv > v {
				bf = true
				break
			}
		}
		if mk == -1 || !bf {
			vs = append(vs, v)
			ks = append(ks, k)
		} else {
			vs = append(append(vs[:mk], v), vs[mk:]...)
			ks = append(append(ks[:mk], k), ks[mk:]...)
		}
	}
	if len(ks) > 0 {
		index = ks[len(ks)-1]
		find = true
	}
	return
}

//往对应redis中更新信息
func updateinfo(thisinfo *Info, tmp map[string]interface{}, pInfo *ProjectInfo, pici int64) string {
	updateid := pInfo.Id
	if BinarySearch(pInfo.Ids, thisinfo.Id) > -1 {
		return updateid
	}
	set := map[string]interface{}{
		"pici": pici,
	}
	res, bres := MQFW.FindById(projectColl, pInfo.Id, `{"list":0}`)
	EqInfoUpdate(thisinfo, pInfo)
	if bres && res != nil && *res != nil {
		set["topscopeclass"] = pInfo.Topscopeclass
		set["subscopeclass"] = pInfo.Subscopeclass
		s_subscopeclass := strings.Join(pInfo.Subscopeclass, ",")
		if len(s_subscopeclass) > 0 {
			s_subscopeclass = "," + s_subscopeclass + ","
		}
		set["s_subscopeclass"] = s_subscopeclass
		s_winner := strings.Join(pInfo.Winners, ",")
		if len(s_winner) > 0 {
			s_winner = "," + s_winner + ","
		}
		set["s_winner"] = s_winner
		if pInfo.Buyerperson != "" && pInfo.Buyertel != "" {
			set["buyerperson"] = pInfo.Buyerperson
			set["buyertel"] = pInfo.Buyertel
		}
		if pInfo.Buyerclass != "" {
			set["buyerclass"] = pInfo.Buyerclass
		}
		if pInfo.District != "" {
			set["district"] = pInfo.District
		}
		if pInfo.Bidopentime > 0 {
			set["bidopentime"] = pInfo.Bidopentime
		}
		if len(pInfo.Winnerorder) > 0 {
			set["winnerorder"] = pInfo.Winnerorder
		}
		if thisinfo.HasPackage {
			set["multipackage"] = 1
		} else {
			set["multipackage"] = 0
		}
		e := InitEL(util.ObjToString((*res)["extractpos"]))
		if thisinfo.dealtype == 1 {
			var sonpackage map[string]interface{}
			for _, obj := range tmp["package"].(map[string]interface{}) {
				sonpackage, _ = obj.(map[string]interface{})
			}
			for _, v2 := range []string{"budget", "budget_w", "winner", "winner_w", "bidstatus", "bidstatus_w"} {
				if sonpackage[v2] != nil {
					tmp[v2] = sonpackage[v2]
				}
			}
		}
		e.fieldpriority(&tmp, res, &set)
		set["extractpos"] = e.GetVal()
		if thisinfo.HasPackage { //多包处理
			p1, _ := (*res)["package"].(map[string]interface{})
			p2, _ := tmp["package"].(map[string]interface{})
			if p2 != nil {
				if p1 != nil {
					for pk2, pv2 := range p2 {
						if p1[pk2] != nil { //合并
							item1, _ := p1[pk2].(map[string]interface{})
							item2, _ := pv2.(map[string]interface{})
							if item1 != nil && item2 != nil { //原始项
								for ik1, iv1 := range item2 {
									if item1[ik1] == nil {
										item1[ik1] = iv1
									}
								}
							}
						} else {
							p1[pk2] = pv2
						}
					}
				} else {
					p1 = p2
				}
			}
			set["package"] = p1
		}
		//中标候选人合并

		update := map[string]interface{}{}
		if len(set) > 0 {
			update["$set"] = set
		}
		//保留原数据吧
		push := NewPushInfo(tmp)
		for tkey, _ := range extractpos {
			if tmp[tkey] != nil {
				push[tkey] = tmp[tkey]
			}
		}
		update["$push"] = map[string]interface{}{
			"list": push,
		}
		if len(update) > 0 {
			MQFW.Update(projectColl, map[string]interface{}{
				"_id": util.StringTOBsonId(pInfo.Id),
			}, &update, false, false)
		}
	}
	//再往redis中放 index
	//往队列中增加时间 -------------->start
	redis.Put(REDISIDS, updateid, pInfo, 0)
	return updateid
}

func EqInfoUpdate(thisinfo *Info, pInfo *ProjectInfo) {
	var tk int
	bf1 := false
	for _k, tv := range pInfo.Publistime {
		tk = _k
		if tv > thisinfo.Publishtime {
			bf1 = true
			break
		}
	}
	if bf1 {
		pInfo.Publistime = append(append(pInfo.Publistime[:tk], thisinfo.Publishtime), pInfo.Publistime[tk:]...)
		pInfo.InfoType = append(append(pInfo.InfoType[:tk], []string{thisinfo.TopType, thisinfo.SubType}), pInfo.InfoType[tk:]...)
		pInfo.Ids = append(append(pInfo.Ids[:tk], thisinfo.Id), pInfo.Ids[tk:]...)
	} else {
		pInfo.Publistime = append(pInfo.Publistime, thisinfo.Publishtime)
		pInfo.InfoType = append(pInfo.InfoType, []string{thisinfo.TopType, thisinfo.SubType})
		pInfo.Ids = append(pInfo.Ids, thisinfo.Id)
	}
	//增加发布时间结束----------------->end

	if (pInfo.Buyer == "" && thisinfo.Buyer != "") || (len([]rune(pInfo.Buyer)) < 5 && len([]rune(thisinfo.Buyer)) > 5) {
		pInfo.Buyer = thisinfo.Buyer
	}
	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
		pInfo.Agency = thisinfo.Agency
	}
	if (pInfo.ProjectCode == "" && thisinfo.ProjectCode != "") || (len([]rune(pInfo.ProjectCode)) < 6 && len([]rune(thisinfo.ProjectCode)) > 6) {
		pInfo.ProjectCode = thisinfo.ProjectCode
	}

	if thisinfo.Buyerperson != "" && thisinfo.Buyertel != "" && len([]rune(thisinfo.Buyertel)) > 6 {
		pInfo.Buyerperson = thisinfo.Buyerperson
		pInfo.Buyertel = thisinfo.Buyertel
	}
	if thisinfo.Buyerclass != "" {
		pInfo.Buyerclass = thisinfo.Buyerclass
	}
	if thisinfo.District != "" {
		pInfo.District = thisinfo.District
	}
	if thisinfo.Bidopentime > 0 {
		pInfo.Bidopentime = thisinfo.Bidopentime
	}
	if len(thisinfo.Topscopeclass) > 0 {
		sort.Strings(pInfo.Topscopeclass)
		for _, k := range thisinfo.Topscopeclass {
			if BinarySearch(pInfo.Topscopeclass, k) == -1 {
				pInfo.Topscopeclass = append(pInfo.Topscopeclass, k)
				sort.Strings(pInfo.Topscopeclass)
			}
		}
	}

	if len(thisinfo.Subscopeclass) > 0 {
		sort.Strings(pInfo.Subscopeclass)
		for _, k := range thisinfo.Subscopeclass {
			if BinarySearch(pInfo.Subscopeclass, k) == -1 {
				pInfo.Subscopeclass = append(pInfo.Subscopeclass, k)
				sort.Strings(pInfo.Subscopeclass)
			}
		}
	}
	//winner
	if len(thisinfo.Winners) > 0 {
		sort.Strings(pInfo.Winners)
		for _, k := range thisinfo.Winners {
			if BinarySearch(pInfo.Winners, k) == -1 {
				pInfo.Winners = append(pInfo.Winners, k)
				sort.Strings(pInfo.Winners)
			}
		}
	}
	//winnerorder
	if len(thisinfo.Winnerorder) > 0 {
		sort.Strings(pInfo.Winnerorder)
		for _, k := range thisinfo.Winnerorder {
			if BinarySearch(pInfo.Winnerorder, k) == -1 {
				pInfo.Winnerorder = append(pInfo.Winnerorder, k)
				sort.Strings(pInfo.Winnerorder)
			}
		}
	}
	//return pInfo
}

//二分字符串查找
func BinarySearch(s []string, k string) int {
	sort.Strings(s)
	lo, hi := 0, len(s)-1
	for lo <= hi {
		m := (lo + hi) >> 1
		if s[m] < k {
			lo = m + 1
		} else if s[m] > k {
			hi = m - 1
		} else {
			return m
		}
	}
	return -1
}

//移除数组中重复的元素
func RemoveDup(arr []string) (newarr []string) {
	m1 := map[string]bool{}
	newarr = []string{}
	for _, k := range arr {
		if !m1[k] {
			m1[k] = true
			newarr = append(newarr, k)
		}
	}
	return
}