jy-data-extract/udpfilterdup/src/main.go

package main

/**
招标信息判重
**/

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	mu "mfw/util"
	"net"
	"os"
	"qfw/util"
	"regexp"
	"sync"
	"time"
)

var (
	Sysconfig map[string]interface{} //配置文件
	mconf     map[string]interface{} //mongodb配置信息
	mgo       *MongodbSim            //mongodb操作对象
	extract   string
	udpclient mu.UdpClient             //udp对象
	nextNode  []map[string]interface{} //下节点数组
	dupdays   = 5                      //初始化判重范围
	DM        *datamap                 //
	HM        *historymap              //判重数据
	lastid    = ""

	//正则筛选相关
	FilterRegTitle   = regexp.MustCompile("^_$")
	FilterRegTitle_1 = regexp.MustCompile("^_$")
	FilterRegTitle_2 = regexp.MustCompile("^_$")

	isMerger         bool                              //是否合并
	Is_Sort          bool                              //是否排序
	threadNum        int                               //线程数量
	SiteMap          map[string]map[string]interface{} //站点map
	idtype, sid, eid string                            //测试人员判重使用
)

func init() {
	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
	flag.StringVar(&sid, "sid", "", "开始id")
	flag.StringVar(&eid, "eid", "", "结束id")
	flag.StringVar(&idtype, "idtype", "", "id类型，默认ObjectId:0,String:1")
	flag.Parse()
	//172.17.145.163:27080
	util.ReadConfig(&Sysconfig)
	nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
	mconf = Sysconfig["mongodb"].(map[string]interface{})
	mgo = &MongodbSim{
		MongodbAddr: mconf["addr"].(string),
		DbName:      mconf["db"].(string),
		Size:        util.IntAllDef(mconf["pool"], 10),
	}
	mgo.InitPool()
	extract = mconf["extract"].(string)
	dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
	//加载数据
	DM = NewDatamap(dupdays, lastid)
	FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
	FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
	FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
	isMerger = Sysconfig["isMerger"].(bool)
	Is_Sort = Sysconfig["isSort"].(bool)
	threadNum = util.IntAllDef(Sysconfig["threads"], 1)

	//站点配置
	site := mconf["site"].(map[string]interface{})
	SiteMap = make(map[string]map[string]interface{}, 0)
	start := int(time.Now().Unix())
	sess_site := mgo.GetMgoConn()
	defer mgo.DestoryMongoConn(sess_site)
	res_site := sess_site.DB(site["dbname"].(string)).C(site["coll"].(string)).Find(map[string]interface{}{}).Sort("_id").Iter()
	for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
		data_map := map[string]interface{}{
			"area":     util.ObjToString(site_dict["area"]),
			"city":     util.ObjToString(site_dict["city"]),
			"district": util.ObjToString(site_dict["district"]),
			"sitetype": util.ObjToString(site_dict["sitetype"]),
			"level":    util.ObjToString(site_dict["level"]),
		}
		SiteMap[util.ObjToString(site_dict["site"])] = data_map
	}
	log.Printf("站点加载用时：%d秒,%d个\n", int(time.Now().Unix())-start, len(SiteMap))
}

func main() {
	go checkMapJob()
	updport := Sysconfig["udpport"].(string)
	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
	udpclient.Listen(processUdpMsg)
	log.Println("Udp服务监听", updport)
	time.Sleep(99999 * time.Hour)
}

//测试组人员使用
func mainT() {
	/*
		ObjectId("5da3f31aa5cb26b9b798d3aa")
		ObjectId("5da418c4a5cb26b9b7e3e9a6")
		ObjectId("5df5071ce9d1f601e495fa54")
		ObjectId("5e09c05f0cf41612e0626abc")
	*/
	log.Println("测试开始")
	sid = "5da3f31aa5cb26b9b798d3aa"
	eid = "5da418c4a5cb26b9b7e3e9a6"
	mapinfo := map[string]interface{}{}
	if sid == "" || eid == "" {
		log.Println("sid,eid参数不能为空")
		os.Exit(0)
	}
	mapinfo["gtid"] = sid
	mapinfo["lteid"] = eid
	mapinfo["stop"] = "true"
	task([]byte{}, mapinfo)
	time.Sleep(10 * time.Second)
}
func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
	fmt.Println("接受的段数据")
	switch act {
	case mu.OP_TYPE_DATA: //上个节点的数据
		//从表中开始处理
		var mapInfo map[string]interface{}
		err := json.Unmarshal(data, &mapInfo)
		log.Println("err:", err, "mapInfo:", mapInfo)
		if err != nil {
			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
		} else if mapInfo != nil {
			taskType := util.ObjToString(mapInfo["stype"])
			if taskType == "historyTask" {
				//更新流程
				go historyTask(data, mapInfo)
			} else if taskType == "normalTask" {
				//判重流程
				go task(data, mapInfo)
			} else {
				//其他
				go task(data, mapInfo)
			}

			key, _ := mapInfo["key"].(string)
			if key == "" {
				key = "udpok"
			}
			udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
		}
	case mu.OP_NOOP: //下个节点回应
		ok := string(data)
		if ok != "" {
			log.Println("ok:", ok)
			udptaskmap.Delete(ok)
		}
	}
}

//开始判重程序
func task(data []byte, mapInfo map[string]interface{}) {
	fmt.Println("开始数据判重")
	defer util.Catch()
	//区间id
	q := map[string]interface{}{}
	if idtype == "1" {
		q = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  mapInfo["gtid"].(string),
				"$lte": mapInfo["lteid"].(string),
			},
		}
	} else {
		q = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
			},
		}
	}
	log.Println(mgo.DbName, extract, q)
	sess := mgo.GetMgoConn()
	defer mgo.DestoryMongoConn(sess)

	//是否排序
	it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
	if Is_Sort {
		it = sess.DB(mgo.DbName).C(extract).Find(&q).Sort("publishtime").Iter()
	}
	//it = sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
	updateExtract := [][]map[string]interface{}{}
	log.Println("线程数：", threadNum)
	pool := make(chan bool, threadNum)
	wg := &sync.WaitGroup{}
	//mapLock := &sync.Mutex{}
	n, repeateN := 0, 0
	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
		if n%10000 == 0 {
			log.Println("current:", n, tmp["_id"], "repeateN:", repeateN)
		}
		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()
			info := NewInfo(tmp)
			//是否为无效数据
			if invalidData(info.buyer, info.projectname, info.projectcode, info.contractnumber) {
				updateExtract = append(updateExtract, []map[string]interface{}{
					map[string]interface{}{
						"_id": tmp["_id"],
					},
					map[string]interface{}{
						"$set": map[string]interface{}{
							"repeat": -1,
						},
					},
				})
				if len(updateExtract) > 500 {
					mgo.UpSertBulk(extract, updateExtract...)
					updateExtract = [][]map[string]interface{}{}
				}
			} else {
				b, source, reason := DM.check(info)
				if b { //有重复,生成更新语句，更新抽取和更新招标
					repeateN++
					var is_replace = false
					var mergeArr = []int64{}                    //更改合并数组记录
					var newData = &Info{}                       //更换新的数据池数据
					var repeat_idMap = map[string]interface{}{} //记录判重的
					var merge_idMap = map[string]interface{}{}  //记录合并的
					if idtype == "1" {                          //先临时决定一个id
						repeat_idMap["_id"] = info.id
						merge_idMap["_id"] = source.id
					} else {
						repeat_idMap["_id"] = StringTOBsonId(info.id)
						merge_idMap["_id"] = StringTOBsonId(source.id)
					}
					repeat_id := source.id
					//以下合并相关
					if isMerger {
						basic_bool := basicDataScore(source, info)
						if basic_bool {
							//已原始数据为标准 - 对比数据打判重标签-
							newData, mergeArr, is_replace = mergeDataFields(source, info)
							DM.replaceSourceData(newData, source.id) //替换
							//对比数据打重复标签的id,原始数据id的记录
							if idtype == "1" {
								repeat_idMap["_id"] = info.id
								merge_idMap["_id"] = source.id
							} else {
								repeat_idMap["_id"] = StringTOBsonId(info.id)
								merge_idMap["_id"] = StringTOBsonId(source.id)
							}
							repeat_id = source.id
						} else {
							//已对比数据为标准 ，数据池的数据打判重标签
							newData, mergeArr, is_replace = mergeDataFields(info, source)
							DM.replaceSourceData(newData, source.id) //替换

							//原始数据打重复标签的id,   对比数据id的记录
							if idtype == "1" {
								repeat_idMap["_id"] = source.id
								merge_idMap["_id"] = info.id
							} else {
								repeat_idMap["_id"] = StringTOBsonId(source.id)
								merge_idMap["_id"] = StringTOBsonId(info.id)
							}
							repeat_id = info.id
						}

						merge_map := make(map[string]interface{}, 0)
						if is_replace { //有过合并-更新数据

							merge_map = map[string]interface{}{
								"$set": map[string]interface{}{
									"merge": newData.mergemap,
								},
							}

							//更新合并后的数据
							for _, value := range mergeArr {
								if value == 0 {
									merge_map["$set"].(map[string]interface{})["area"] = newData.area
									merge_map["$set"].(map[string]interface{})["city"] = newData.city
								} else if value == 1 {
									merge_map["$set"].(map[string]interface{})["area"] = newData.area
									merge_map["$set"].(map[string]interface{})["city"] = newData.city
								} else if value == 2 {
									merge_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
								} else if value == 3 {
									merge_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
								} else if value == 4 {
									merge_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
								} else if value == 5 {
									merge_map["$set"].(map[string]interface{})["budget"] = newData.budget
								} else if value == 6 {
									merge_map["$set"].(map[string]interface{})["winner"] = newData.winner
								} else if value == 7 {
									merge_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
								} else if value == 8 {
									merge_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
								} else if value == 9 {
									merge_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
								} else if value == 10 {
									merge_map["$set"].(map[string]interface{})["publishtime"] = newData.publishtime
								} else if value == 11 {
									merge_map["$set"].(map[string]interface{})["agency"] = newData.agency
								} else {
								}
							}
							//模板数据更新
							updateExtract = append(updateExtract, []map[string]interface{}{
								merge_idMap,
								merge_map,
							})
						}
					}

					//重复数据打标签
					updateExtract = append(updateExtract, []map[string]interface{}{
						repeat_idMap,
						map[string]interface{}{
							"$set": map[string]interface{}{
								"repeat":        1,
								"repeat_reason": reason,
								"repeat_id":     repeat_id,
							},
						},
					})

				}
			}
		}(tmp)
		if len(updateExtract) > 500 {
			mgo.UpSertBulk(extract, updateExtract...)
			updateExtract = [][]map[string]interface{}{}
		}
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	if len(updateExtract) > 0 {
		mgo.UpSertBulk(extract, updateExtract...)
		//mgo.UpdateBulk(bidding, updateBidding...)
	}
	log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])

	//任务完成，开始发送广播通知下面节点
	if n > repeateN && mapInfo["stop"] == nil {
		for _, to := range nextNode {
			sid, _ := mapInfo["gtid"].(string)
			eid, _ := mapInfo["lteid"].(string)
			key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
			by, _ := json.Marshal(map[string]interface{}{
				"gtid":  sid,
				"lteid": eid,
				"stype": util.ObjToString(to["stype"]),
				"key":   key,
			})
			addr := &net.UDPAddr{
				IP:   net.ParseIP(to["addr"].(string)),
				Port: util.IntAll(to["port"]),
			}
			node := &udpNode{by, addr, time.Now().Unix(), 0}
			udptaskmap.Store(key, node)
			udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
		}
	}
}

//支持历史更新
func historyTask(data []byte, mapInfo map[string]interface{}) {

	fmt.Println("开始取历史时间段")
	defer util.Catch()
	sess := mgo.GetMgoConn()
	defer mgo.DestoryMongoConn(sess)

	var q map[string]interface{}
	if idtype == "1" {
		q = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  mapInfo["gtid"].(string),
				"$lte": mapInfo["lteid"].(string),
			},
		}
	} else {
		q = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
			},
		}
	}

	it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
	minTime, maxTime := int64(0), int64(0)
	for tmp := make(map[string]interface{}); it.Next(&tmp); {
		//取出最大最小时间
		info_time:=tmp["comeintime"]
		if Is_Sort {
			info_time = tmp["publishtime"]
		}
		if minTime == 0 || maxTime == 0 && util.Int64All(info_time) != 0 {
			minTime = util.Int64All(info_time)
			maxTime = util.Int64All(info_time)
		} else {
			t := util.Int64All(info_time)
			if t < minTime && t != 0 {
				minTime = t
			}
			if t > maxTime && t != 0 {
				maxTime = t
			}
		}
	}
	//时间不正确时
	if minTime == 0 && maxTime == 0 {
		log.Println("段数据区间 不符合")
		return
	}
	fmt.Println("最小时间==", minTime, "最大时间==", maxTime)
	gtid, lteid := util.BsonIdToSId(mapInfo["gtid"].(string)), util.BsonIdToSId(mapInfo["lteid"].(string))
	fmt.Println(gtid, lteid)
	HM = NewHistorymap(gtid, lteid, minTime, maxTime)
	fmt.Println("开始历史数据判重")

	defer util.Catch()
	//区间id
	sess_history := mgo.GetMgoConn()
	defer mgo.DestoryMongoConn(sess_history)
	var q_history map[string]interface{}
	if idtype == "1" {
		q_history = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  mapInfo["gtid"].(string),
				"$lte": mapInfo["lteid"].(string),
			},
		}
	} else {
		q_history = map[string]interface{}{
			"_id": map[string]interface{}{
				"$gt":  StringTOBsonId(mapInfo["gtid"].(string)),
				"$lte": StringTOBsonId(mapInfo["lteid"].(string)),
			},
		}
	}
	log.Println(mgo.DbName, extract, q_history)

	//是否排序
	it_history := sess_history.DB(mgo.DbName).C(extract).Find(&q_history).Iter()
	if Is_Sort {
		it_history = sess_history.DB(mgo.DbName).C(extract).Find(&q_history).Sort("publishtime").Iter()
	}
	updateExtract := [][]map[string]interface{}{}
	log.Println("线程数：", threadNum)
	pool := make(chan bool, threadNum)
	wg := &sync.WaitGroup{}
	//mapLock := &sync.Mutex{}
	n, repeateN := 0, 0
	for tmp := make(map[string]interface{}); it_history.Next(&tmp); n++ {
		if n%10000 == 0 {
			log.Println("current:", n, tmp["_id"], "repeateN:", repeateN)
		}
		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()
			info := NewInfo(tmp)
			if invalidData(info.buyer, info.projectname, info.projectcode, info.contractnumber) {
				updateExtract = append(updateExtract, []map[string]interface{}{
					map[string]interface{}{
						"_id": tmp["_id"],
					},
					map[string]interface{}{
						"$set": map[string]interface{}{
							"repeat": -1,
						},
					},
				})
				if len(updateExtract) > 500 {
					mgo.UpSertBulk(extract, updateExtract...)
					updateExtract = [][]map[string]interface{}{}
				}
			} else {
				b, source, reason := HM.checkHistory(info)
				if b { //有重复,生成更新语句，更新抽取和更新招标
					if reason == "未判重记录" {
						fmt.Println("未判重记录")
						//把info的数据判重的标签更换，并新增字段
						DM.replaceSourceData(info, info.id) //替换即添加
						updateExtract = append(updateExtract, []map[string]interface{}{
							map[string]interface{}{
								"_id": tmp["_id"],
							},
							map[string]interface{}{
								"$set": map[string]interface{}{
									"repeat":   0,
									"repeatid": -2,
								},
							},
						})
					} else {
						repeateN++
						var is_replace = false
						var mergeArr = []int64{}                    //更改合并数组记录
						var newData = &Info{}                       //更换新的数据池数据
						var repeat_idMap = map[string]interface{}{} //记录判重的
						var merge_idMap = map[string]interface{}{}  //记录合并的
						if idtype == "1" {                          //先临时决定一个id
							repeat_idMap["_id"] = info.id
							merge_idMap["_id"] = source.id
						} else {
							repeat_idMap["_id"] = StringTOBsonId(info.id)
							merge_idMap["_id"] = StringTOBsonId(source.id)
						}
						repeat_id := source.id
						//以下合并相关
						if isMerger {
							basic_bool := basicDataScore(source, info)
							if basic_bool {
								//已原始数据为标准 - 对比数据打判重标签-
								newData, mergeArr, is_replace = mergeDataFields(source, info)
								DM.replaceSourceData(newData, source.id) //替换
								//对比数据打重复标签的id,原始数据id的记录
								if idtype == "1" {
									repeat_idMap["_id"] = info.id
									merge_idMap["_id"] = source.id
								} else {
									repeat_idMap["_id"] = StringTOBsonId(info.id)
									merge_idMap["_id"] = StringTOBsonId(source.id)
								}
								repeat_id = source.id
							} else {
								//已对比数据为标准 ，数据池的数据打判重标签
								newData, mergeArr, is_replace = mergeDataFields(info, source)
								DM.replaceSourceData(newData, source.id) //替换

								//原始数据打重复标签的id,   对比数据id的记录
								if idtype == "1" {
									repeat_idMap["_id"] = source.id
									merge_idMap["_id"] = info.id
								} else {
									repeat_idMap["_id"] = StringTOBsonId(source.id)
									merge_idMap["_id"] = StringTOBsonId(info.id)
								}
								repeat_id = info.id
							}

							merge_map := make(map[string]interface{}, 0)
							if is_replace { //有过合并-更新数据

								merge_map = map[string]interface{}{
									"$set": map[string]interface{}{
										"merge": newData.mergemap,
									},
								}

								//更新合并后的数据
								for _, value := range mergeArr {
									if value == 0 {
										merge_map["$set"].(map[string]interface{})["area"] = newData.area
										merge_map["$set"].(map[string]interface{})["city"] = newData.city
									} else if value == 1 {
										merge_map["$set"].(map[string]interface{})["area"] = newData.area
										merge_map["$set"].(map[string]interface{})["city"] = newData.city
									} else if value == 2 {
										merge_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
									} else if value == 3 {
										merge_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
									} else if value == 4 {
										merge_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
									} else if value == 5 {
										merge_map["$set"].(map[string]interface{})["budget"] = newData.budget
									} else if value == 6 {
										merge_map["$set"].(map[string]interface{})["winner"] = newData.winner
									} else if value == 7 {
										merge_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
									} else if value == 8 {
										merge_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
									} else if value == 9 {
										merge_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
									} else if value == 10 {
										merge_map["$set"].(map[string]interface{})["publishtime"] = newData.publishtime
									} else if value == 11 {
										merge_map["$set"].(map[string]interface{})["agency"] = newData.agency
									} else {
									}
								}
								//模板数据更新
								updateExtract = append(updateExtract, []map[string]interface{}{
									merge_idMap,
									merge_map,
								})
							}
						}

						//重复数据打标签
						updateExtract = append(updateExtract, []map[string]interface{}{
							repeat_idMap,
							map[string]interface{}{
								"$set": map[string]interface{}{
									"repeat":        1,
									"repeat_reason": reason,
									"repeat_id":     repeat_id,
								},
							},
						})

					}
				}
			}
		}(tmp)
		if len(updateExtract) > 500 {
			mgo.UpSertBulk(extract, updateExtract...)
			updateExtract = [][]map[string]interface{}{}
		}
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	if len(updateExtract) > 0 {
		mgo.UpSertBulk(extract, updateExtract...)
		//mgo.UpdateBulk(bidding, updateBidding...)
	}
	log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])

	//任务完成，开始发送广播通知下面节点
	if n > repeateN && mapInfo["stop"] == nil {
		for _, to := range nextNode {
			sid, _ := mapInfo["gtid"].(string)
			eid, _ := mapInfo["lteid"].(string)
			key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
			by, _ := json.Marshal(map[string]interface{}{
				"gtid":  sid,
				"lteid": eid,
				"stype": util.ObjToString(to["stype"]),
				"key":   key,
			})
			addr := &net.UDPAddr{
				IP:   net.ParseIP(to["addr"].(string)),
				Port: util.IntAll(to["port"]),
			}
			node := &udpNode{by, addr, time.Now().Unix(), 0}
			udptaskmap.Store(key, node)
			udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
		}
	}
}

//合并字段-并更新merge字段的值
func mergeDataFields(source *Info, info *Info) (*Info, []int64, bool) {

	merge_recordMap := make(map[string]interface{}, 0)
	mergeArr := make([]int64, 0)
	//是否替换数据了-记录原始的数据
	is_replace := false
	//1、城市
	if source.area == "" || source.area == "全国" {
		//为空
		if info.area != "全国" && info.area != "" {
			merge_recordMap["area"] = source.area
			merge_recordMap["city"] = source.city
			source.area = info.area
			source.city = info.city
			mergeArr = append(mergeArr, 1)
			is_replace = true
		}
	} else {
		//不为空-查看站点相关-有值必替换
		if source.is_site {
			//是站点替换的城市
			merge_recordMap["site_area"] = source.area
			merge_recordMap["site_city"] = source.city
			mergeArr = append(mergeArr, 0)
			is_replace = true
			source.is_site = false

		}
	}
	//2、项目名称
	if source.projectname == "" && info.projectname != "" {
		merge_recordMap["projectname"] = source.projectname
		source.projectname = info.projectname
		mergeArr = append(mergeArr, 2)
		is_replace = true
	}
	//3、项目编号
	if source.projectcode == "" && info.projectcode != "" {
		merge_recordMap["projectcode"] = source.projectcode
		source.projectcode = info.projectcode
		mergeArr = append(mergeArr, 3)
		is_replace = true
	}
	//4、采购单位
	if source.buyer == "" && info.buyer != "" {
		merge_recordMap["buyer"] = source.buyer
		source.buyer = info.buyer
		mergeArr = append(mergeArr, 4)
		is_replace = true
	}
	//5、预算
	if source.budget == 0 && info.budget != 0 {
		merge_recordMap["budget"] = source.budget
		source.budget = info.budget
		mergeArr = append(mergeArr, 5)
		is_replace = true
	}
	//6、中标单位
	if source.winner == "" && info.winner != "" {
		merge_recordMap["winner"] = source.winner
		source.winner = info.winner
		mergeArr = append(mergeArr, 6)
		is_replace = true
	}
	//7、中标金额
	if source.bidamount == 0 && info.bidamount != 0 {
		merge_recordMap["bidamount"] = source.bidamount
		source.bidamount = info.bidamount
		mergeArr = append(mergeArr, 7)
		is_replace = true
	}
	//8、开标时间-地点
	if source.bidopentime == 0 && info.bidopentime != 0 {
		merge_recordMap["bidopentime"] = source.bidopentime
		source.bidopentime = info.bidopentime
		mergeArr = append(mergeArr, 8)
		is_replace = true
	}

	//9、合同编号
	if source.contractnumber == "" && info.contractnumber != "" {
		merge_recordMap["contractnumber"] = source.contractnumber
		source.contractnumber = info.contractnumber
		mergeArr = append(mergeArr, 9)
		is_replace = true
	}

	//10、发布时间
	if source.publishtime == 0 && info.publishtime != 0 {
		merge_recordMap["publishtime"] = source.publishtime
		source.publishtime = info.publishtime
		mergeArr = append(mergeArr, 10)
		is_replace = true
	}
	//11、代理机构
	if source.agency == "" && info.agency != "" {
		merge_recordMap["agency"] = source.agency
		source.agency = info.agency
		mergeArr = append(mergeArr, 11)
		is_replace = true
	}

	if is_replace { //有过替换更新
		//总次数+1
		source.mergemap["total_num"] = util.Int64All(source.mergemap["total_num"]) + 1
		merge_recordMap["num"] = util.Int64All(source.mergemap["total_num"])
		//和哪一个数据id进行非空替换的-记录
		key := info.id
		source.mergemap[key] = merge_recordMap
	}

	//以上合并过于简单，待进一步优化
	return source, mergeArr, is_replace
}

//权重评估
func basicDataScore(v *Info, info *Info) bool {

	/*
	  权重评估
	  网站优先级判定规则：
	  1、中央>省>市>县区
	  2、政府采购>公共资源>采购单位官网>招标代理公司/平台
	*/
	v_score, info_score := -1, -1
	dict_v := SiteMap[v.site]
	dict_info := SiteMap[info.site]
	//先判断level
	if dict_v != nil {
		v_level := util.ObjToString(dict_v["level"])
		if v_level == "中央" {
			v_score = 4
		} else if v_level == "省级" {
			v_score = 3
		} else if v_level == "市级" {
			v_score = 2
		} else if v_level == "县区" {
			v_score = 1
		} else if v_level == "" {
		} else {
			v_score = 0
		}
	}

	if dict_info != nil {
		info_level := util.ObjToString(dict_info["level"])
		if info_level == "中央" {
			info_score = 4
		} else if info_level == "省级" {
			info_score = 3
		} else if info_level == "市级" {
			info_score = 2
		} else if info_level == "县区" {
			info_score = 1
		} else if info_level == "" {

		} else {
			v_score = 0
		}
	}

	if v_score > info_score {
		return true
	}
	if v_score < info_score {
		return false
	}

	//判断sitetype
	if dict_v != nil {
		v_sitetype := util.ObjToString(dict_v["sitetype"])
		if v_sitetype == "政府采购" || v_sitetype == "政府门户" {
			v_score = 4
		} else if v_sitetype == "公共资源" {
			v_score = 3
		} else if v_sitetype == "官方网站" {
			v_score = 2
		} else if v_sitetype == "社会公共招标平台" || v_sitetype == "企业招标平台" {
			v_score = 1
		} else if v_sitetype == "" {
		} else {
			v_score = 0
		}
	}

	if dict_info != nil {
		info_sitetype := util.ObjToString(dict_info["sitetype"])
		if info_sitetype == "政府采购" || info_sitetype == "政府门户" {
			info_score = 4
		} else if info_sitetype == "公共资源" {
			info_score = 3
		} else if info_sitetype == "官方网站" {
			info_score = 2
		} else if info_sitetype == "社会公共招标平台" || info_sitetype == "企业招标平台" {
			info_score = 1
		} else if info_sitetype == "" {
		} else {
			info_score = 0
		}
	}

	if v_score > info_score {
		return true
	}
	if v_score < info_score {
		return false
	}

	//网站评估
	m, n := 0, 0
	if v.projectname != "" {
		m++
	}
	if v.buyer != "" {
		m++
	}
	if v.projectcode != "" || v.contractnumber != "" {
		m++
	}
	if v.budget != 0 {
		m++
	}
	if v.bidamount != 0 {
		m++
	}
	if v.winner != "" {
		m++
	}
	if v.bidopentime != 0 {
		m++
	}
	if v.agencyaddr != "" {
		m++
	}
	if v.agency != "" {
		m = m + 2
	}
	if v.city != "" {
		m = m + 2
	}

	if info.projectname != "" {
		n++
	}
	if info.buyer != "" {
		n++
	}
	if info.projectcode != "" || info.contractnumber != "" {
		n++
	}
	if info.budget != 0 {
		n++
	}
	if info.bidamount != 0 {
		n++
	}
	if info.winner != "" {
		n++
	}
	if info.bidopentime != 0 {
		n++
	}
	if info.agencyaddr != "" {
		n++
	}
	if info.agency != "" {
		n = n + 2
	}
	if info.city != "" {
		n = n + 2
	}

	if m > n {
		return true
	} else if m == n {
		if v.publishtime >= info.publishtime {
			return true
		} else {
			return false
		}
	} else {
		return false
	}
}

//无效数据
func invalidData(d1 string, d2 string, d3 string, d4 string) bool {
	var n int
	if d1 != "" {
		n++
	}
	if d2 != "" {
		n++
	}
	if d3 != "" {
		n++
	}
	if d4 != "" {
		n++
	}
	if n == 0 {
		return true
	}
	return false
}