Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/config.json

@@ -7,11 +7,10 @@
     "mongodbName": "extract_kf",
 	"hints":"publishtime_1",
     "extractColl": "ceshi_info",
-    "projectColl": "ceshi_project",
+    "projectColl": "jh_project",
     "backupFlag": false,
-    "backupColl": "jh_project1",
     "siteColl": "site",
-
+    "thread": 1,
     "jkmail": {
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"

fullproject/src_v1/deepcopy.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"reflect"
+	"time"
+)
+
+// Interface for delegating copy process to type
+type Interface interface {
+	DeepCopy() interface{}
+}
+
+// Iface is an alias to Copy; this exists for backwards compatibility reasons.
+func Iface(iface interface{}) interface{} {
+	return Copy(iface)
+}
+
+// Copy creates a deep copy of whatever is passed to it and returns the copy
+// in an interface{}.  The returned value will need to be asserted to the
+// correct type.
+func Copy(src interface{}) interface{} {
+	if src == nil {
+		return nil
+	}
+
+	// Make the interface a reflect.Value
+	original := reflect.ValueOf(src)
+
+	// Make a copy of the same type as the original.
+	cpy := reflect.New(original.Type()).Elem()
+
+	// Recursively copy the original.
+	copyRecursive(original, cpy)
+
+	// Return the copy as an interface.
+	return cpy.Interface()
+}
+
+// copyRecursive does the actual copying of the interface. It currently has
+// limited support for what it can handle. Add as needed.
+func copyRecursive(original, cpy reflect.Value) {
+	// check for implement deepcopy.Interface
+	if original.CanInterface() {
+		if copier, ok := original.Interface().(Interface); ok {
+			cpy.Set(reflect.ValueOf(copier.DeepCopy()))
+			return
+		}
+	}
+
+	// handle according to original's Kind
+	switch original.Kind() {
+	case reflect.Ptr:
+		// Get the actual value being pointed to.
+		originalValue := original.Elem()
+
+		// if  it isn't valid, return.
+		if !originalValue.IsValid() {
+			return
+		}
+		cpy.Set(reflect.New(originalValue.Type()))
+		copyRecursive(originalValue, cpy.Elem())
+
+	case reflect.Interface:
+		// If this is a nil, don't do anything
+		if original.IsNil() {
+			return
+		}
+		// Get the value for the interface, not the pointer.
+		originalValue := original.Elem()
+
+		// Get the value by calling Elem().
+		copyValue := reflect.New(originalValue.Type()).Elem()
+		copyRecursive(originalValue, copyValue)
+		cpy.Set(copyValue)
+
+	case reflect.Struct:
+		t, ok := original.Interface().(time.Time)
+		if ok {
+			cpy.Set(reflect.ValueOf(t))
+			return
+		}
+		// Go through each field of the struct and copy it.
+		for i := 0; i < original.NumField(); i++ {
+			// The Type's StructField for a given field is checked to see if StructField.PkgPath
+			// is set to determine if the field is exported or not because CanSet() returns false
+			// for settable fields.  I'm not sure why.  -mohae
+			if original.Type().Field(i).PkgPath != "" {
+				continue
+			}
+			copyRecursive(original.Field(i), cpy.Field(i))
+		}
+
+	case reflect.Slice:
+		if original.IsNil() {
+			return
+		}
+		// Make a new slice and copy each element.
+		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
+		for i := 0; i < original.Len(); i++ {
+			copyRecursive(original.Index(i), cpy.Index(i))
+		}
+
+	case reflect.Map:
+		if original.IsNil() {
+			return
+		}
+		cpy.Set(reflect.MakeMap(original.Type()))
+		for _, key := range original.MapKeys() {
+			originalValue := original.MapIndex(key)
+			copyValue := reflect.New(originalValue.Type()).Elem()
+			copyRecursive(originalValue, copyValue)
+			copyKey := Copy(key.Interface())
+			cpy.SetMapIndex(reflect.ValueOf(copyKey), copyValue)
+		}
+
+	default:
+		cpy.Set(original)
+	}
+}

fullproject/src_v1/init.go

@@ -21,6 +21,7 @@ var (
 	Sysconfig                map[string]interface{} //读取配置文件
 	MongoTool                *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string	//抽取表、项目表、项目快照表、站点表
+	Thread					 int				    //配置项线程数
 	//NextNode                 []interface{}
 )
 
@@ -62,8 +63,10 @@ func init() {
 
 	ExtractColl = Sysconfig["extractColl"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
-	BackupColl = Sysconfig["backupColl"].(string)
+	BackupColl = Sysconfig["projectColl"].(string)+"_back"
+	log.Println(BackupColl)
 	SiteColl = Sysconfig["siteColl"].(string)
+	Thread = util.IntAll(Sysconfig["thread"])
 	//NextNode = Sysconfig["nextNode"].([]interface{})
 	udpport, _ := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
@@ -275,6 +278,7 @@ type ProjectInfo struct {
 	Winners       []string           `json:"winners,omitempty"`       //中标人
 	ProjectName   string             `json:"projectname,omitempty"`   //项目名称
 	ProjectCode   string             `json:"projectcode,omitempty"`   //项目代码唯一（纯数字的权重低）
+	ContractCode  string			 `json:"contractcode,omitempty"`  //项目编号
 	Buyer         string             `json:"buyer,omitempty"`         //采购单位唯一
 	MPN           []string           `json:"mpn,omitempty"`           //合并后多余的项目名称
 	MPC           []string           `json:"mpc,omitempty"`           //合并后多余的项目编号
@@ -298,11 +302,11 @@ type ProjectInfo struct {
 	score         int
 	comStr        string
 	resVal, pjVal int
-	InfoFiled	map[string]InfoFiled	`json:"infofiled"`		//逻辑处理需要的info字段
+	InfoFiled	map[string]InfoField	`json:"infofiled"`		//逻辑处理需要的info字段
 }
 
 //存储部分招标信息字段，业务逻辑处理需要
-type InfoFiled struct {
+type InfoField struct {
 	Budget			float64				`json:"budget"`
 	Bidamount		float64				`json:"bidamount"`
 	ContractCode	string				`json:"contractcode"`

fullproject/src_v1/load_data.go

@@ -84,6 +84,15 @@ func (p *ProjectTask) loadData(starttime int64) {
 				bys, _ := json.Marshal(result)
 				var tmp *ProjectInfo
 				_ = json.Unmarshal(bys, &tmp)
+				tmpMap := make(map[string]InfoField)
+				infoMap := result["infofield"].(map[string]interface{})
+				for _, v := range infoMap{
+					var field InfoField
+					b, _ := json.Marshal(v)
+					_ = json.Unmarshal(b, &field)
+					tmpMap[tmp.Id.Hex()] = field
+				}
+				tmp.InfoFiled = tmpMap
 				pool <- tmp
 			}(result)
 		} else {

fullproject/src_v1/main.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"log"
 	mu "mfw/util"
 	"net"
 	"os"
 	"os/signal"
 	"qfw/util"
+	qu "qfw/util"
 	"syscall"
 	"time"
 )
@@ -17,6 +19,8 @@ var (
 	SingleThread = make(chan bool, 1)
 	toaddr       = []*net.UDPAddr{} //下节点对象
 	ChSign       = make(chan os.Signal)
+
+	sid, eid string 	//测试使用
 )
 
 func init() {
@@ -55,7 +59,7 @@ func DealSign() {
 	}
 }
 
-func main() {
+func mainT() {
 	//udp跑增量  id段   project
 	//udp跑全量			ql
 	//udp跑历史数据  信息id1,id2/或id段  ls
@@ -69,10 +73,82 @@ func main() {
 		}
 	}
 	P_QL.loadSite()
-	//go checkMapJob()
+	go checkMapJob()
 	time.Sleep(99999 * time.Hour)
 }
 
+//测试组人员使用
+func main() {
+	//sid = "5dfbd43ce9d1f601e43fa402"
+	//eid = "5e0954b30cf41612e061d0c8"
+
+	flag.StringVar(&sid, "sid", "", "开始id")
+	flag.StringVar(&eid, "eid", "", "结束id")
+	flag.Parse()
+
+	mapinfo := map[string]interface{}{}
+	if sid == "" || eid == "" {
+		log.Println("sid, eid参数不能为空")
+		os.Exit(0)
+	}
+	mapinfo["gtid"] = qu.StringTOBsonId(sid)
+	mapinfo["lteid"] = qu.StringTOBsonId(eid)
+	mapinfo["stype"] = "ql"
+	mapinfo["ip"] = "127.0.0.1"
+	mapinfo["port"] = Sysconfig["udpport"]
+	if Sysconfig["loadStart"] != nil {
+		loadStart := util.Int64All(Sysconfig["loadStart"])
+		if loadStart > -1 {
+			P_QL.loadData(loadStart)
+		}
+	}
+	P_QL.loadSite()
+	task(mapinfo)
+	time.Sleep(20 * time.Second)
+}
+
+func task(mapInfo map[string]interface{})  {
+	SingleThread <- true
+	tasktype, _ := mapInfo["stype"].(string)
+	log.Println("tasktype:", tasktype)
+	switch tasktype {
+	case "ql": //全量合并
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskQl(mapInfo)
+		}()
+	case "project": //增量合并，未抽取到项目名称或项目编号的不合并  bidding中mergestatus 1已合并 2字段问题不合并 3历史待合并
+		//合同、验收公告在6个月内查询不到可扩展到两年
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskZl(mapInfo)
+		}()
+	case "updateInfo":		//招标字段变更
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskUpdateInfo(mapInfo)
+		}()
+	case "history": //历史数据合并,暂时不写
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+		}()
+	}
+}
+
 //udp调用信号
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	switch act {

fullproject/src_v1/project.go

@@ -498,6 +498,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			set["projecthref"] = jsonData["projecthref"]
 		}
 	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
 
 	bt := qu.ObjToString(tmp["toptype"])
 	bs := qu.ObjToString(tmp["subtype"])
@@ -534,8 +538,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["multipackage"] = 0
 	}
 
-	p1.InfoFiled = make(map[string]InfoFiled)
-	infofiled := InfoFiled{
+	p1.InfoFiled = make(map[string]InfoField)
+	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
 		Bidamount:    thisinfo.Bidamount,
 		ContractCode: thisinfo.ContractCode,
@@ -620,6 +624,7 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 		Ids:           []string{thisinfo.Id},
 		ProjectName:   thisinfo.ProjectName,
 		ProjectCode:   thisinfo.ProjectCode,
+		ContractCode:  thisinfo.ContractCode,
 		Buyer:         thisinfo.Buyer,
 		Buyerclass:    thisinfo.Buyerclass,
 		Buyerperson:   thisinfo.Buyerperson,
@@ -750,10 +755,12 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		pInfo.Buyerclass = thisinfo.Buyerclass
 		set["buyerclass"] = thisinfo.Buyerclass
 	}
-
 	if pInfo.Buyer == "" {
 		set["buyerclass"] = ""
 	}
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = pInfo.ContractCode + ","+thisinfo.ContractCode
+	}
 
 	//8--代理机构
 	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
@@ -821,7 +828,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	CountAmount(pInfo, thisinfo)
 	set["budget"] = pInfo.Budget
 	set["bidamount"] = pInfo.Bidamount
-	infofiled := InfoFiled{
+	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
 		Bidamount:    thisinfo.Bidamount,
 		ContractCode: thisinfo.ContractCode,
@@ -829,12 +836,14 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		ProjectCode:  thisinfo.ProjectCode,
 		Bidstatus:    bs,
 	}
-	pInfo.InfoFiled[thisinfo.Id] = infofiled
+	copyMap := Copy(pInfo.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofiled
 	tmpMap := make(map[string]interface{})
-	for k, v := range pInfo.InfoFiled{
+	for k, v := range copyMap{
 		tmpMap[k] = StructToMap(v)
 	}
 	tmpMap[thisinfo.Id] = StructToMap(infofiled)
+	pInfo.InfoFiled = copyMap
 	set["infofield"] = tmpMap
 
 	set["mpn"] = pInfo.MPN
@@ -960,8 +969,12 @@ func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 
 func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 	p1 := map[string]interface{}{}
-	if project != nil && project.Package != nil && len(project.Package) > 0 {
-		p1 = project.Package
+	packageCopy := make(map[string]interface{})
+	if project != nil && project.Package != nil {
+		packageCopy = Copy(project.Package).(map[string]interface{})
+	}
+	if packageCopy != nil && len(packageCopy) > 0 {
+		p1 = packageCopy
 		for k, v := range info.Package {
 			if v1, ok := v.(map[string]interface{}); ok {
 				v2 := map[string]interface{}{}
@@ -986,7 +999,7 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 			}
 		}
 	} else {
-		for k, v := range info.Package {
+		for k, v := range packageCopy {
 			v1, _ := v.(map[string]interface{})
 			p2 := map[string]interface{}{}
 			p2 = packageEle(v1, info.Id)
@@ -996,6 +1009,7 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 			p1[k] = []map[string]interface{}{p2}
 		}
 	}
+	p1 = packageCopy
 	return p1
 }
 
@@ -1039,20 +1053,23 @@ func CountAmount(project *ProjectInfo, info *Info) {
 			}
 		}else {
 			//招标不是单包
-			flag := false
-			if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-				for _, res := range project.InfoFiled {
-					if res.ProjectName == info.ProjectName {
-						if res.Budget < info.Budget {
-							project.Budget = project.Budget - res.Budget + info.Budget
-						}
-						flag = true
-						break
-					}
-				}
-				if !flag {
-					project.Budget = project.Budget + info.Budget
-				}
+			//flag := false
+			//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+			//	for _, res := range project.InfoFiled {
+			//		if res.ProjectName == info.ProjectName {
+			//			if res.Budget < info.Budget {
+			//				project.Budget = project.Budget - res.Budget + info.Budget
+			//			}
+			//			flag = true
+			//			break
+			//		}
+			//	}
+			//	if !flag {
+			//		project.Budget = project.Budget + info.Budget
+			//	}
+			//}
+			if project.Budget < info.Budget {
+				project.Budget = info.Budget
 			}
 		}
 	}
@@ -1095,30 +1112,55 @@ func CountAmount(project *ProjectInfo, info *Info) {
 				}
 			}else {
 				//招标不是单包
-				flag := false
-				if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-					for _, res := range project.InfoFiled {
-						if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
-							if res.ContractCode == info.ContractCode {
-								if res.Bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//flag := false
+				//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+				//	for _, res := range project.InfoFiled {
+				//		if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
+				//			if res.ContractCode == info.ContractCode {
+				//				if res.Bidamount < info.Bidamount {
+				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//				}
+				//				flag = true
+				//				break
+				//			}
+				//		}else {
+				//			if res.ProjectName == info.ProjectName {
+				//				if res.Bidamount < info.Bidamount {
+				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//				}
+				//				flag = true
+				//				break
+				//			}
+				//		}
+				//	}
+				//	if !flag {
+				//		project.Bidamount = project.Bidamount + info.Bidamount
+				//	}
+				//}
+				if info.SubType == "中标" || info.SubType == "成交" {
+					if project.Bidamount < info.Bidamount {
+						project.Bidamount = info.Bidamount
+					}else {
+						flag := false
+						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+							for _, res := range project.InfoFiled {
+								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+									flag = true
+									break
+								}
+								if res.Bidamount == project.Bidamount {
+									flag = true
+									break
 								}
-								flag = true
-								break
 							}
-						}else {
-							if res.ProjectName == info.ProjectName {
-								if res.Bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+							if !flag {
+								project.Bidamount = project.Bidamount + info.Bidamount
+							}else {
+								if project.Bidamount < info.Bidamount {
+									project.Bidamount = info.Bidamount
 								}
-								flag = true
-								break
 							}
 						}
-
-					}
-					if !flag {
-						project.Bidamount = project.Bidamount + info.Bidamount
 					}
 				}
 			}
@@ -1127,7 +1169,7 @@ func CountAmount(project *ProjectInfo, info *Info) {
 }
 
 //结构体转map
-func StructToMap(filed InfoFiled) map[string]interface{} {
+func StructToMap(filed InfoField) map[string]interface{} {
 	//先转json
 	result, err := json.Marshal(filed)
 	if err != nil {

fullproject/src_v1/task.go

@@ -237,7 +237,7 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -256,7 +256,7 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 			if idmap == nil {
 				idmap = map[string]interface{}{}
 			}
-			idmap["$gt"] = StringTOBsonId(gtid)
+			idmap["$gte"] = StringTOBsonId(gtid)
 		}
 		if idmap != nil {
 			q["_id"] = idmap
@@ -280,7 +280,7 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -324,7 +324,7 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -334,7 +334,7 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	if q == nil {
 		q = map[string]interface{}{
 			"_id": map[string]interface{}{
-				"$gt":  StringTOBsonId(gtid),
+				"$gte":  StringTOBsonId(gtid),
 				"$lte": StringTOBsonId(lteid),
 			},
 			"is_m": 1,

fullproject/src_v1/update.go

@@ -15,7 +15,7 @@ func (p *ProjectTask) modifyUpdate(pInfoId string, index int, info *Info, tmp ma
 	infoList := []interface{}(tmpPro["list"].(primitive.A))
 	infoMap := infoList[index].(map[string]interface{})
 	infoList[index] = updateValue(infoMap, modifyMap)
-	infofiled := InfoFiled{
+	infofiled := InfoField{
 		Budget:       info.Budget,
 		Bidamount:    info.Bidamount,
 		ContractCode: info.ContractCode,
@@ -70,7 +70,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 				infoList := []interface{}(tmpPro["list"].(primitive.A))
 				infoMap := infoList[index].(map[string]interface{})
 				infoList[index] = updateValue(infoMap, modifyMap)
-				infofiled := InfoFiled{
+				infofiled := InfoField{
 					Budget:       info.Budget,
 					Bidamount:    info.Bidamount,
 					ContractCode: info.ContractCode,

src/res/fieldscore.json

@@ -90,7 +90,7 @@
                 "range": [
                     0,
                     3,
-                    -2
+                    -5
                 ]
             },
             {
@@ -98,7 +98,7 @@
                 "range": [
                     3,
                     5,
-                    0
+                    -2
                 ]
             },
             {

udpfilterdup/src/config.json

@@ -5,7 +5,7 @@
         "addr": "192.168.3.207:27092",
         "pool": 5,
         "db": "extract_kf",
-        "extract": "zheng_test1_jd1",
+        "extract": "zk",
         "site": {
             "dbname": "zhaolongyue",
             "coll": "site"
@@ -19,7 +19,7 @@
     "isMerger": false,
     "threads": 1,
     "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
-    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包)",
+    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批)",
     "specialtitle_2": "项目（[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]）",
     "beifen": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
 }

udpfilterdup/src/datamap.go

@@ -23,6 +23,7 @@ type Info struct {
 	bidamount   float64 //中标金额
 	projectname string  //项目名称
 	projectcode string  //项目编号
+	contractnumber string //合同编号
 	publishtime int64   //发布时间
 	bidopentime int64   //开标时间
 	agencyaddr  string  //开标地点
@@ -190,6 +191,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.subtype = subtype
 	info.buyer = qutil.ObjToString(tmp["buyer"])
 	info.projectname = qutil.ObjToString(tmp["projectname"])
+	info.contractnumber = qutil.ObjToString(tmp["contractnumber"])
 	info.projectcode = qutil.ObjToString(tmp["projectcode"])
 	info.city = qutil.ObjToString(tmp["city"])
 	info.agency = qutil.ObjToString(tmp["agency"])
@@ -276,10 +278,13 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = "标题关键词且包含关系"
-							b = true
-							source = v
-							break L
+							reason = reason+"标题关键词且包含关系"
+							//继续二级金额判断
+							if !againRepeat(v,info){
+								b = true
+								source = v
+								break
+							}
 						}
 					}
 
@@ -406,10 +411,14 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = "标题关键词且包含关系"
-							b = true
-							source = v
-							break L
+							reason = reason+"标题关键词且包含关系"
+							//继续二级金额判断
+							if !againRepeat(v,info){
+								b = true
+								source = v
+								break
+							}
+
 						}
 					}
 
@@ -735,8 +744,9 @@ func tenderRepeat_A(v *Info, info *Info, reason string) (bool ,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
-		ss = ss + "p3(编号)-"
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
 	if v.budget != 0 && v.budget == info.budget {
@@ -781,7 +791,8 @@ func tenderRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
 		m++
 	}
 	if v.budget != 0 && v.budget == info.budget {
@@ -841,8 +852,9 @@ func winningRepeat_A(v *Info, info *Info, reason string) (bool,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
-		ss = ss + "p3(编号)-"
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
 	if v.bidamount != 0 && v.bidamount == info.bidamount {
@@ -883,7 +895,8 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
 		m++
 	}
 	if v.bidamount != 0 && v.bidamount == info.bidamount {
@@ -961,6 +974,32 @@ func contractRepeat_C(v *Info, info *Info) bool {
 }
 
 
+func againRepeat(v *Info ,info *Info) bool {
+	//相同采购单位下
+	if info.buyer != "" &&v.buyer == info.buyer {
+		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+			info.subtype=="其他"||info.subtype=="变更" {
+			//预算金额满足条件
+			if v.budget!=info.budget&&v.budget!=0&&info.budget!=0 {
+				return true
+			}
+		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||
+			info.subtype=="流标"||info.subtype=="合同"||info.subtype=="验收"||
+			info.subtype=="违规"{
+			//中标金额单位满足条件
+			if (v.bidamount!=info.bidamount&&v.bidamount!=0&&info.bidamount!=0)||
+				(v.winner!=info.winner&&v.winner!=""&&info.winner!=""){
+				return true
+			}
+		}else {
+
+		}
+	}
+
+	return false
+}
+
 
 
 

udpfilterdup/src/main.go

@@ -38,7 +38,6 @@ var (
 	FilterRegTitle_1 = regexp.MustCompile("^_$")
 	FilterRegTitle_2 = regexp.MustCompile("^_$")
 
-
 	isMerger bool                              //是否合并
 	threadNum int								   //线程数量
 	SiteMap  map[string]map[string]interface{} //站点map
@@ -46,6 +45,7 @@ var (
 )
 
 func init() {
+
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
@@ -104,8 +104,8 @@ func main() {
 
 //测试组人员使用
 func mainT() {
-	//sid = "5dfbd43ce9d1f601e43fa402"
-	//eid = "5e0954b30cf41612e061d0c8"
+	//sid = "5da3f31aa5cb26b9b798d3aa"
+	//eid = "5da422fba5cb26b9b706984b"
 	mapinfo := map[string]interface{}{}
 	if sid == "" || eid == "" {
 		log.Println("sid,eid参数不能为空")
@@ -115,7 +115,7 @@ func mainT() {
 	mapinfo["lteid"] = eid
 	mapinfo["stop"] = "true"
 	task([]byte{}, mapinfo)
-	time.Sleep(5 * time.Second)
+	time.Sleep(10 * time.Second)
 }
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	fmt.Println("接受的段数据")
@@ -200,7 +200,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			}()
 			info := NewInfo(tmp)
 			//是否为无效数据
-			if invalidData(info.buyer, info.projectname, info.projectcode) {
+			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
 						"_id": tmp["_id"],
@@ -286,7 +286,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 								update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
 							} else if value == 8 {
 								update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-							} else {
+							} else if value == 9 {
+								update_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
+							}else {
 							}
 						}
 					}
@@ -427,7 +429,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 				wg.Done()
 			}()
 			info := NewInfo(tmp)
-			if invalidData(info.buyer, info.projectname, info.projectcode) {
+			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
 				//mapLock.Lock()
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
@@ -529,7 +531,9 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 									update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
 								} else if value == 8 {
 									update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-								} else {
+								} else if value == 9 {
+									update_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
+								}else {
 
 								}
 							}
@@ -707,6 +711,21 @@ func mergeDataFields(source *Info, info *Info) (*Info, []int64) {
 		mergeArr = append(mergeArr, 8)
 	}
 
+	//9、合同编号
+	if source.contractnumber == "" && info.contractnumber != "" {
+		var arr []string
+		if source.mergemap["contractnumber"] == nil {
+			arr = make([]string, 0)
+		} else {
+			arr = source.mergemap["contractnumber"].([]string)
+		}
+		arr = append(arr, source.contractnumber)
+		source.mergemap["contractnumber"] = arr
+
+		source.contractnumber = info.contractnumber
+		mergeArr = append(mergeArr, 9)
+	}
+
 	//以上合并过于简单，待进一步优化
 	return source, mergeArr
 }
@@ -812,7 +831,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if v.buyer != "" {
 		m++
 	}
-	if v.projectcode != "" {
+	if v.projectcode != ""||v.contractnumber != "" {
 		m++
 	}
 	if v.budget != 0 {
@@ -843,7 +862,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if info.buyer != "" {
 		n++
 	}
-	if info.projectcode != "" {
+	if info.projectcode != "" || info.contractnumber != ""{
 		n++
 	}
 	if info.budget != 0 {
@@ -882,7 +901,7 @@ func basicDataScore(v *Info, info *Info) bool {
 }
 
 //无效数据
-func invalidData(d1 string, d2 string, d3 string) bool {
+func invalidData(d1 string, d2 string, d3 string, d4 string) bool {
 	var n int
 	if d1 != "" {
 		n++
@@ -893,7 +912,10 @@ func invalidData(d1 string, d2 string, d3 string) bool {
 	if d3 != "" {
 		n++
 	}
-	if n == 0 {
+	if d4 != "" {
+		n++
+	}
+ 	if n == 0 {
 		return true
 	}
 	return false

versioncomparison/config.json

@@ -21,6 +21,7 @@
         "buyeraddr",
         "agencyperson",
         "agencytel",
-        "agencyaddr"
+        "agencyaddr",
+        "package"
     ]
 }

versioncomparison/main.go

@@ -163,6 +163,7 @@ func getVersionData() {
 				rd[_id] = tmp
 			} else {
 				rd[_id] = &Data{
+					Id:     qu.BsonIdToSId(_id),
 					NewVal: fmt.Sprint(v[field]),
 				}
 			}