Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

fullproject/src_v1/config.json

@@ -7,11 +7,10 @@
     "mongodbName": "extract_kf",
 	"hints":"publishtime_1",
     "extractColl": "ceshi_info",
-    "projectColl": "ceshi_project",
+    "projectColl": "jh_project",
     "backupFlag": false,
-    "backupColl": "jh_project1",
     "siteColl": "site",
-
+    "thread": 1,
     "jkmail": {
         "to": "wangjianghan@topnet.net.cn",
         "api": "http://10.171.112.160:19281/_send/_mail"

fullproject/src_v1/deepcopy.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"reflect"
+	"time"
+)
+
+// Interface for delegating copy process to type
+type Interface interface {
+	DeepCopy() interface{}
+}
+
+// Iface is an alias to Copy; this exists for backwards compatibility reasons.
+func Iface(iface interface{}) interface{} {
+	return Copy(iface)
+}
+
+// Copy creates a deep copy of whatever is passed to it and returns the copy
+// in an interface{}.  The returned value will need to be asserted to the
+// correct type.
+func Copy(src interface{}) interface{} {
+	if src == nil {
+		return nil
+	}
+
+	// Make the interface a reflect.Value
+	original := reflect.ValueOf(src)
+
+	// Make a copy of the same type as the original.
+	cpy := reflect.New(original.Type()).Elem()
+
+	// Recursively copy the original.
+	copyRecursive(original, cpy)
+
+	// Return the copy as an interface.
+	return cpy.Interface()
+}
+
+// copyRecursive does the actual copying of the interface. It currently has
+// limited support for what it can handle. Add as needed.
+func copyRecursive(original, cpy reflect.Value) {
+	// check for implement deepcopy.Interface
+	if original.CanInterface() {
+		if copier, ok := original.Interface().(Interface); ok {
+			cpy.Set(reflect.ValueOf(copier.DeepCopy()))
+			return
+		}
+	}
+
+	// handle according to original's Kind
+	switch original.Kind() {
+	case reflect.Ptr:
+		// Get the actual value being pointed to.
+		originalValue := original.Elem()
+
+		// if  it isn't valid, return.
+		if !originalValue.IsValid() {
+			return
+		}
+		cpy.Set(reflect.New(originalValue.Type()))
+		copyRecursive(originalValue, cpy.Elem())
+
+	case reflect.Interface:
+		// If this is a nil, don't do anything
+		if original.IsNil() {
+			return
+		}
+		// Get the value for the interface, not the pointer.
+		originalValue := original.Elem()
+
+		// Get the value by calling Elem().
+		copyValue := reflect.New(originalValue.Type()).Elem()
+		copyRecursive(originalValue, copyValue)
+		cpy.Set(copyValue)
+
+	case reflect.Struct:
+		t, ok := original.Interface().(time.Time)
+		if ok {
+			cpy.Set(reflect.ValueOf(t))
+			return
+		}
+		// Go through each field of the struct and copy it.
+		for i := 0; i < original.NumField(); i++ {
+			// The Type's StructField for a given field is checked to see if StructField.PkgPath
+			// is set to determine if the field is exported or not because CanSet() returns false
+			// for settable fields.  I'm not sure why.  -mohae
+			if original.Type().Field(i).PkgPath != "" {
+				continue
+			}
+			copyRecursive(original.Field(i), cpy.Field(i))
+		}
+
+	case reflect.Slice:
+		if original.IsNil() {
+			return
+		}
+		// Make a new slice and copy each element.
+		cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
+		for i := 0; i < original.Len(); i++ {
+			copyRecursive(original.Index(i), cpy.Index(i))
+		}
+
+	case reflect.Map:
+		if original.IsNil() {
+			return
+		}
+		cpy.Set(reflect.MakeMap(original.Type()))
+		for _, key := range original.MapKeys() {
+			originalValue := original.MapIndex(key)
+			copyValue := reflect.New(originalValue.Type()).Elem()
+			copyRecursive(originalValue, copyValue)
+			copyKey := Copy(key.Interface())
+			cpy.SetMapIndex(reflect.ValueOf(copyKey), copyValue)
+		}
+
+	default:
+		cpy.Set(original)
+	}
+}

fullproject/src_v1/init.go

@@ -21,6 +21,7 @@ var (
 	Sysconfig                map[string]interface{} //读取配置文件
 	MongoTool                *MongodbSim            //mongodb连接
 	ExtractColl, ProjectColl, BackupColl, SiteColl string	//抽取表、项目表、项目快照表、站点表
+	Thread					 int				    //配置项线程数
 	//NextNode                 []interface{}
 )
 
@@ -62,8 +63,10 @@ func init() {
 
 	ExtractColl = Sysconfig["extractColl"].(string)
 	ProjectColl = Sysconfig["projectColl"].(string)
-	BackupColl = Sysconfig["backupColl"].(string)
+	BackupColl = Sysconfig["projectColl"].(string)+"_back"
+	log.Println(BackupColl)
 	SiteColl = Sysconfig["siteColl"].(string)
+	Thread = util.IntAll(Sysconfig["thread"])
 	//NextNode = Sysconfig["nextNode"].([]interface{})
 	udpport, _ := Sysconfig["udpport"].(string)
 	udpclient = mu.UdpClient{Local: udpport, BufSize: 1024}
@@ -275,6 +278,7 @@ type ProjectInfo struct {
 	Winners       []string           `json:"winners,omitempty"`       //中标人
 	ProjectName   string             `json:"projectname,omitempty"`   //项目名称
 	ProjectCode   string             `json:"projectcode,omitempty"`   //项目代码唯一（纯数字的权重低）
+	ContractCode  string			 `json:"contractcode,omitempty"`  //项目编号
 	Buyer         string             `json:"buyer,omitempty"`         //采购单位唯一
 	MPN           []string           `json:"mpn,omitempty"`           //合并后多余的项目名称
 	MPC           []string           `json:"mpc,omitempty"`           //合并后多余的项目编号
@@ -298,11 +302,11 @@ type ProjectInfo struct {
 	score         int
 	comStr        string
 	resVal, pjVal int
-	InfoFiled	map[string]InfoFiled	`json:"infofiled"`		//逻辑处理需要的info字段
+	InfoFiled	map[string]InfoField	`json:"infofiled"`		//逻辑处理需要的info字段
 }
 
 //存储部分招标信息字段，业务逻辑处理需要
-type InfoFiled struct {
+type InfoField struct {
 	Budget			float64				`json:"budget"`
 	Bidamount		float64				`json:"bidamount"`
 	ContractCode	string				`json:"contractcode"`

fullproject/src_v1/load_data.go

@@ -84,6 +84,15 @@ func (p *ProjectTask) loadData(starttime int64) {
 				bys, _ := json.Marshal(result)
 				var tmp *ProjectInfo
 				_ = json.Unmarshal(bys, &tmp)
+				tmpMap := make(map[string]InfoField)
+				infoMap := result["infofield"].(map[string]interface{})
+				for _, v := range infoMap{
+					var field InfoField
+					b, _ := json.Marshal(v)
+					_ = json.Unmarshal(b, &field)
+					tmpMap[tmp.Id.Hex()] = field
+				}
+				tmp.InfoFiled = tmpMap
 				pool <- tmp
 			}(result)
 		} else {

fullproject/src_v1/main.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"log"
 	mu "mfw/util"
 	"net"
 	"os"
 	"os/signal"
 	"qfw/util"
+	qu "qfw/util"
 	"syscall"
 	"time"
 )
@@ -17,6 +19,8 @@ var (
 	SingleThread = make(chan bool, 1)
 	toaddr       = []*net.UDPAddr{} //下节点对象
 	ChSign       = make(chan os.Signal)
+
+	sid, eid string //测试使用
 )
 
 func init() {
@@ -69,10 +73,83 @@ func main() {
 		}
 	}
 	P_QL.loadSite()
-	//go checkMapJob()
+	go checkMapJob()
 	time.Sleep(99999 * time.Hour)
 }
 
+//测试组人员使用
+func mainT() {
+	//sid = "5dfbd43ce9d1f601e43fa402"
+	//eid = "5e0954b30cf41612e061d0c8"
+
+	flag.StringVar(&sid, "sid", "", "开始id")
+	flag.StringVar(&eid, "eid", "", "结束id")
+	flag.Parse()
+
+	mapinfo := map[string]interface{}{}
+	if sid == "" || eid == "" {
+		log.Println("sid, eid参数不能为空")
+		os.Exit(0)
+	}
+	mapinfo["gtid"] = qu.StringTOBsonId(sid)
+	mapinfo["lteid"] = qu.StringTOBsonId(eid)
+	mapinfo["stype"] = "ql"
+	mapinfo["ip"] = "127.0.0.1"
+	mapinfo["port"] = Sysconfig["udpport"]
+	if Sysconfig["loadStart"] != nil {
+		loadStart := util.Int64All(Sysconfig["loadStart"])
+		if loadStart > -1 {
+			P_QL.loadData(loadStart)
+		}
+	}
+	P_QL.loadSite()
+	task(mapinfo)
+	SingleThread <- true
+	time.Sleep(20 * time.Second)
+}
+
+func task(mapInfo map[string]interface{}) {
+	SingleThread <- true
+	tasktype, _ := mapInfo["stype"].(string)
+	log.Println("tasktype:", tasktype)
+	switch tasktype {
+	case "ql": //全量合并
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskQl(mapInfo)
+		}()
+	case "project": //增量合并，未抽取到项目名称或项目编号的不合并  bidding中mergestatus 1已合并 2字段问题不合并 3历史待合并
+		//合同、验收公告在6个月内查询不到可扩展到两年
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskZl(mapInfo)
+		}()
+	case "updateInfo": //招标字段变更
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+			P_QL.currentType = tasktype
+			P_QL.pici = time.Now().Unix()
+			P_QL.taskUpdateInfo(mapInfo)
+		}()
+	case "history": //历史数据合并,暂时不写
+		go func() {
+			defer func() {
+				<-SingleThread
+			}()
+		}()
+	}
+}
+
 //udp调用信号
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	switch act {
@@ -111,7 +188,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					P_QL.pici = time.Now().Unix()
 					P_QL.taskZl(mapInfo)
 				}()
-			case "updateInfo":		//招标字段变更
+			case "updateInfo": //招标字段变更
 				go func() {
 					defer func() {
 						<-SingleThread

fullproject/src_v1/project.go

@@ -498,6 +498,10 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 			set["projecthref"] = jsonData["projecthref"]
 		}
 	}
+	//合同编号
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = thisinfo.ContractCode
+	}
 
 	bt := qu.ObjToString(tmp["toptype"])
 	bs := qu.ObjToString(tmp["subtype"])
@@ -534,8 +538,8 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 		set["multipackage"] = 0
 	}
 
-	p1.InfoFiled = make(map[string]InfoFiled)
-	infofiled := InfoFiled{
+	p1.InfoFiled = make(map[string]InfoField)
+	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
 		Bidamount:    thisinfo.Bidamount,
 		ContractCode: thisinfo.ContractCode,
@@ -620,6 +624,7 @@ func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidty
 		Ids:           []string{thisinfo.Id},
 		ProjectName:   thisinfo.ProjectName,
 		ProjectCode:   thisinfo.ProjectCode,
+		ContractCode:  thisinfo.ContractCode,
 		Buyer:         thisinfo.Buyer,
 		Buyerclass:    thisinfo.Buyerclass,
 		Buyerperson:   thisinfo.Buyerperson,
@@ -750,10 +755,12 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		pInfo.Buyerclass = thisinfo.Buyerclass
 		set["buyerclass"] = thisinfo.Buyerclass
 	}
-
 	if pInfo.Buyer == "" {
 		set["buyerclass"] = ""
 	}
+	if thisinfo.ContractCode != "" {
+		set["contractcode"] = pInfo.ContractCode + ","+thisinfo.ContractCode
+	}
 
 	//8--代理机构
 	if (pInfo.Agency == "" && thisinfo.Agency != "") || (len([]rune(pInfo.Agency)) < 5 && len([]rune(thisinfo.Agency)) > 5) {
@@ -821,7 +828,7 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 	CountAmount(pInfo, thisinfo)
 	set["budget"] = pInfo.Budget
 	set["bidamount"] = pInfo.Bidamount
-	infofiled := InfoFiled{
+	infofiled := InfoField{
 		Budget:       thisinfo.Budget,
 		Bidamount:    thisinfo.Bidamount,
 		ContractCode: thisinfo.ContractCode,
@@ -829,12 +836,14 @@ func (p *ProjectTask) UpdateProject(tmp map[string]interface{}, thisinfo *Info,
 		ProjectCode:  thisinfo.ProjectCode,
 		Bidstatus:    bs,
 	}
-	pInfo.InfoFiled[thisinfo.Id] = infofiled
+	copyMap := Copy(pInfo.InfoFiled).(map[string]InfoField)
+	copyMap[thisinfo.Id] = infofiled
 	tmpMap := make(map[string]interface{})
-	for k, v := range pInfo.InfoFiled{
+	for k, v := range copyMap{
 		tmpMap[k] = StructToMap(v)
 	}
 	tmpMap[thisinfo.Id] = StructToMap(infofiled)
+	pInfo.InfoFiled = copyMap
 	set["infofield"] = tmpMap
 
 	set["mpn"] = pInfo.MPN
@@ -960,8 +969,12 @@ func packageEle(map1 map[string]interface{}, id string) map[string]interface{} {
 
 func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 	p1 := map[string]interface{}{}
-	if project != nil && project.Package != nil && len(project.Package) > 0 {
-		p1 = project.Package
+	packageCopy := make(map[string]interface{})
+	if project != nil && project.Package != nil {
+		packageCopy = Copy(project.Package).(map[string]interface{})
+	}
+	if packageCopy != nil && len(packageCopy) > 0 {
+		p1 = packageCopy
 		for k, v := range info.Package {
 			if v1, ok := v.(map[string]interface{}); ok {
 				v2 := map[string]interface{}{}
@@ -986,7 +999,7 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 			}
 		}
 	} else {
-		for k, v := range info.Package {
+		for k, v := range packageCopy {
 			v1, _ := v.(map[string]interface{})
 			p2 := map[string]interface{}{}
 			p2 = packageEle(v1, info.Id)
@@ -996,6 +1009,7 @@ func PackageFormat(info *Info, project *ProjectInfo) map[string]interface{} {
 			p1[k] = []map[string]interface{}{p2}
 		}
 	}
+	p1 = packageCopy
 	return p1
 }
 
@@ -1039,20 +1053,23 @@ func CountAmount(project *ProjectInfo, info *Info) {
 			}
 		}else {
 			//招标不是单包
-			flag := false
-			if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-				for _, res := range project.InfoFiled {
-					if res.ProjectName == info.ProjectName {
-						if res.Budget < info.Budget {
-							project.Budget = project.Budget - res.Budget + info.Budget
-						}
-						flag = true
-						break
-					}
-				}
-				if !flag {
-					project.Budget = project.Budget + info.Budget
-				}
+			//flag := false
+			//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+			//	for _, res := range project.InfoFiled {
+			//		if res.ProjectName == info.ProjectName {
+			//			if res.Budget < info.Budget {
+			//				project.Budget = project.Budget - res.Budget + info.Budget
+			//			}
+			//			flag = true
+			//			break
+			//		}
+			//	}
+			//	if !flag {
+			//		project.Budget = project.Budget + info.Budget
+			//	}
+			//}
+			if project.Budget < info.Budget {
+				project.Budget = info.Budget
 			}
 		}
 	}
@@ -1095,30 +1112,55 @@ func CountAmount(project *ProjectInfo, info *Info) {
 				}
 			}else {
 				//招标不是单包
-				flag := false
-				if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
-					for _, res := range project.InfoFiled {
-						if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
-							if res.ContractCode == info.ContractCode {
-								if res.Bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//flag := false
+				//if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+				//	for _, res := range project.InfoFiled {
+				//		if res.Bidstatus == "合同" && res.ContractCode != "" && info.SubType == "合同" && info.ContractCode != "" {
+				//			if res.ContractCode == info.ContractCode {
+				//				if res.Bidamount < info.Bidamount {
+				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//				}
+				//				flag = true
+				//				break
+				//			}
+				//		}else {
+				//			if res.ProjectName == info.ProjectName {
+				//				if res.Bidamount < info.Bidamount {
+				//					project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+				//				}
+				//				flag = true
+				//				break
+				//			}
+				//		}
+				//	}
+				//	if !flag {
+				//		project.Bidamount = project.Bidamount + info.Bidamount
+				//	}
+				//}
+				if info.SubType == "中标" || info.SubType == "成交" {
+					if project.Bidamount < info.Bidamount {
+						project.Bidamount = info.Bidamount
+					}else {
+						flag := false
+						if project.InfoFiled != nil && len(project.InfoFiled) > 0 {
+							for _, res := range project.InfoFiled {
+								if res.ContractCode != "" && res.ContractCode == info.ContractCode {
+									flag = true
+									break
+								}
+								if res.Bidamount == project.Bidamount {
+									flag = true
+									break
 								}
-								flag = true
-								break
 							}
-						}else {
-							if res.ProjectName == info.ProjectName {
-								if res.Bidamount < info.Bidamount {
-									project.Bidamount = project.Bidamount - res.Bidamount + info.Bidamount
+							if !flag {
+								project.Bidamount = project.Bidamount + info.Bidamount
+							}else {
+								if project.Bidamount < info.Bidamount {
+									project.Bidamount = info.Bidamount
 								}
-								flag = true
-								break
 							}
 						}
-
-					}
-					if !flag {
-						project.Bidamount = project.Bidamount + info.Bidamount
 					}
 				}
 			}
@@ -1127,7 +1169,7 @@ func CountAmount(project *ProjectInfo, info *Info) {
 }
 
 //结构体转map
-func StructToMap(filed InfoFiled) map[string]interface{} {
+func StructToMap(filed InfoField) map[string]interface{} {
 	//先转json
 	result, err := json.Marshal(filed)
 	if err != nil {

fullproject/src_v1/task.go

@@ -237,7 +237,7 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -256,7 +256,7 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 			if idmap == nil {
 				idmap = map[string]interface{}{}
 			}
-			idmap["$gt"] = StringTOBsonId(gtid)
+			idmap["$gte"] = StringTOBsonId(gtid)
 		}
 		if idmap != nil {
 			q["_id"] = idmap
@@ -280,7 +280,7 @@ func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -324,7 +324,7 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	if coll == "" {
 		coll = ExtractColl
 	}
-	thread := util.IntAllDef(udpInfo["thread"], 4)
+	thread := util.IntAllDef(Thread, 4)
 	if thread > 0 {
 		p.thread = thread
 	}
@@ -334,7 +334,7 @@ func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
 	if q == nil {
 		q = map[string]interface{}{
 			"_id": map[string]interface{}{
-				"$gt":  StringTOBsonId(gtid),
+				"$gte":  StringTOBsonId(gtid),
 				"$lte": StringTOBsonId(lteid),
 			},
 			"is_m": 1,

fullproject/src_v1/update.go

@@ -15,7 +15,7 @@ func (p *ProjectTask) modifyUpdate(pInfoId string, index int, info *Info, tmp ma
 	infoList := []interface{}(tmpPro["list"].(primitive.A))
 	infoMap := infoList[index].(map[string]interface{})
 	infoList[index] = updateValue(infoMap, modifyMap)
-	infofiled := InfoFiled{
+	infofiled := InfoField{
 		Budget:       info.Budget,
 		Bidamount:    info.Bidamount,
 		ContractCode: info.ContractCode,
@@ -70,7 +70,7 @@ func (p *ProjectTask) mergeAndModify(pInfoId string, index int, info *Info, tmp
 				infoList := []interface{}(tmpPro["list"].(primitive.A))
 				infoMap := infoList[index].(map[string]interface{})
 				infoList[index] = updateValue(infoMap, modifyMap)
-				infofiled := InfoFiled{
+				infofiled := InfoField{
 					Budget:       info.Budget,
 					Bidamount:    info.Bidamount,
 					ContractCode: info.ContractCode,

src/config.json

@@ -3,15 +3,16 @@
     "mgodb": "192.168.3.207:27092",
     "dbsize": 10,
     "dbname": "extract_dev32",
-    "redis": "buyer=192.168.3.207:1679,winner=192.168.3.207:1679,agency=192.168.3.207:1679",
-    "elasticsearch": "http://127.0.0.1:9200",
-    "elasticsearch_index": "extract_kf",
-    "elasticsearch_type": "enterprise_qyxy",
-    "elasticPoolSize": 30,
+    "redis": "buyer=192.168.3.205:6379,winner=192.168.3.205:6379,agency=192.168.3.205:6379",
+    "elasticsearch": "http://127.0.0.1:9800",
+    "elasticsearch_index": "winner_enterprise",
+    "elasticsearch_type": "winnerent",
+    "elasticsearch_db": "winner_enterprise",
+    "elasticPoolSize": 10,
     "mergetable": "projectset",
     "mergetablealias": "projectset_v1",
-    "saveresult": false,
-    "fieldsfind":false,
+    "saveresult": true,
+    "fieldsfind": false,
     "qualityaudit": false,
     "saveblock": false,
     "filelength": 100000,
@@ -59,9 +60,9 @@
             }
         ]
     },
-	"isSaveTag":false,
+    "isSaveTag": false,
     "tomail": "zhangjinkun@topnet.net.cn,chenmingzhu@topnet.net.cn,zhaolongyue@topnet.net.cn",
     "api": "http://10.171.112.160:19281/_send/_mail",
-    "deleteInstanceTimeHour":1,
+    "deleteInstanceTimeHour": 1,
     "jsondata_extweight": 1
 } 

src/jy/admin/audit/qiyeku.go

@@ -1,16 +1,15 @@
 package audit
 
 import (
-	"context"
 	"encoding/json"
 	"github.com/gin-gonic/gin"
 	"gopkg.in/mgo.v2/bson"
-	"gopkg.in/olivere/elastic.v5"
 	. "jy/admin"
 	"jy/clear"
 	. "jy/mongodbutil"
 	"jy/util"
 	"log"
+	"qfw/util/elastic"
 	"strconv"
 	"strings"
 	"time"
@@ -38,7 +37,7 @@ func init() {
 			c.JSON(200, gin.H{"rep": 400})
 			return
 		}
-		capitalfloat :=clear.ObjToMoney([]interface{}{capital, ""})[0]
+		capitalfloat := clear.ObjToMoney([]interface{}{capital, ""})[0]
 		e := make(map[string]interface{})
 		e["company_name"] = company_name
 		e["alias"] = alias
@@ -59,8 +58,10 @@ func init() {
 				"capital":         capitalfloat,
 				"company_address": company_address,
 			}}, false, false)
-			if tmpb{
-				_,err := util.ElasticClient.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).
+			if tmpb {
+				escon := elastic.GetEsConn()
+				defer elastic.DestoryEsConn(escon)
+				_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).
 					Id(sid).Doc(map[string]interface{}{
 					"company_name":    company_name,
 					"alias":           alias,
@@ -69,17 +70,19 @@ func init() {
 					"district":        district,
 					"capital":         capitalfloat,
 					"company_address": company_address,
-				}).Refresh("true").Do(context.TODO())
-				if err != nil{
-					log.Println("update qyk err:",err)
+				}).Refresh(true).Do()
+				if err != nil {
+					log.Println("update qyk err:", err)
 				}
 			}
 		} else {
 			sid = Mgo.Save("enterprise_qyxy", e)
-			delete(e,"_id")
-			_,err :=util.ElasticClient.Index().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(sid).BodyJson(e).Refresh("true").Do(context.TODO())
-			if err != nil{
-				log.Println("save qyk err:",err)
+			delete(e, "_id")
+			escon := elastic.GetEsConn()
+			defer elastic.DestoryEsConn(escon)
+			_, err := escon.Index().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(sid).BodyJson(e).Refresh(true).Do()
+			if err != nil {
+				log.Println("save qyk err:", err)
 			}
 		}
 		c.JSON(200, gin.H{"rep": 200, "saveid": sid})
@@ -98,32 +101,38 @@ func init() {
 			c.JSON(200, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
 		} else {
 			//log.Println(util.ElasticClientIndex, util.ElasticClientType, search)
-			res, err := util.ElasticClient.Search(util.ElasticClientIndex).
-				Type(util.ElasticClientType).
-				Query( elastic.NewMatchPhraseQuery("company_name", search)).
+			escon := elastic.GetEsConn()
+			defer elastic.DestoryEsConn(escon)
+			res, err := escon.Search(util.ElasticClientIndex).
+				Type(util.ElasticClientType).Source(`{"query": {"match_phrase":{"company_name":"`+search+`"}}}`).
+				//Query( elastic.NewMatchPhraseQuery("company_name", search)).
 				Size(limit).
 				From(start).
-				Do(context.TODO())
+				Do()
 			if err != nil {
 				log.Println(err)
 				c.JSON(500, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
 				return
 			}
-			tmps := make([]map[string]interface{}, 0)
-			for _, v := range res.Hits.Hits {
-				tmp := make(map[string]interface{})
-				err := json.Unmarshal(*v.Source, &tmp)
-				if err != nil {
-					log.Println(err)
-					continue
+			if res.Hits != nil{
+				tmps := make([]map[string]interface{}, 0)
+				for _, v := range res.Hits.Hits {
+					tmp := make(map[string]interface{})
+					err := json.Unmarshal(*v.Source, &tmp)
+					if err != nil {
+						log.Println(err)
+						continue
+					}
+					tmp["_id"] = v.Id
+					//log.Println(tmp)
+					tmps = append(tmps, tmp)
 				}
-				tmp["_id"] = v.Id
-				//log.Println(tmp)
-				tmps = append(tmps, tmp)
+				//count := Mgo.Count("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}})
+				//data, _ := Mgo.Find("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}}, `{"_id":-1}`, nil, false, start, limit)
+				c.JSON(200, gin.H{"data": tmps, "recordsFiltered": res.Hits.TotalHits, "recordsTotal": res.Hits.TotalHits})
+			}else {
+				c.JSON(200, gin.H{"data": []map[string]interface{}{}, "recordsFiltered": 0, "recordsTotal": 0})
 			}
-			//count := Mgo.Count("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}})
-			//data, _ := Mgo.Find("enterprise_qyxy", bson.M{"company_name": bson.M{"$regex": bson.RegEx{search, "i"}}}, `{"_id":-1}`, nil, false, start, limit)
-			c.JSON(200, gin.H{"data": tmps, "recordsFiltered": res.Hits.TotalHits, "recordsTotal": res.Hits.TotalHits})
 		}
 	})
 	//delete
@@ -131,9 +140,11 @@ func init() {
 		_id := c.PostForm("_id")
 		if bson.IsObjectIdHex(_id) {
 			delisok := Mgo.Del("enterprise_qyxy", bson.M{"_id": bson.ObjectIdHex(_id)})
-			_,err := util.ElasticClient.Delete().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Refresh("true").Do(context.TODO())
-			if err != nil{
-				log.Println("delete qyk err:",err)
+			escon := elastic.GetEsConn()
+			defer elastic.DestoryEsConn(escon)
+			_, err := escon.Delete().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Refresh(true).Do()
+			if err != nil {
+				log.Println("delete qyk err:", err)
 			}
 			c.JSON(200, gin.H{"rep": 200, "data": delisok})
 		} else {
@@ -145,7 +156,7 @@ func init() {
 		_id := c.PostForm("_id")
 		q_field := c.PostForm("q_field")
 		if bson.IsObjectIdHex(_id) && strings.TrimSpace(q_field) != "" {
-			data, _ := Mgo.FindById("enterprise_qyxy", _id, bson.M{q_field: 1})
+			data, _ := Mgo.FindById(util.ElasticClientDB, _id, bson.M{q_field: 1})
 			c.JSON(200, gin.H{"rep": 200, "data": data})
 		} else {
 			c.JSON(200, gin.H{"rep": 400})
@@ -157,13 +168,15 @@ func init() {
 		industrys := c.PostFormArray("industry")
 		//log.Println(_id,industrys)
 		if bson.IsObjectIdHex(_id) {
-			b := Mgo.Update("enterprise_qyxy", bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"industry": industrys}}, false, false)
+			b := Mgo.Update(util.ElasticClientDB, bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"industry": industrys}}, false, false)
 			if b {
-				_,err := util.ElasticClient.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
-					"industry":industrys,
-				}).Refresh("true").Do(context.TODO())
-				if err != nil{
-					log.Println("update yqk industry err :",err)
+				escon := elastic.GetEsConn()
+				defer elastic.DestoryEsConn(escon)
+				_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
+					"industry": industrys,
+				}).Refresh(true).Do()
+				if err != nil {
+					log.Println("update yqk industry err :", err)
 				}
 			}
 			c.JSON(200, gin.H{"rep": 200, "data": b})
@@ -199,13 +212,15 @@ func init() {
 			//for k,v := range contacts{
 			//	log.Println(k,v)
 			//}
-			b := Mgo.Update("enterprise_qyxy", bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"contact": contacts}}, false, false)
+			b := Mgo.Update(util.ElasticClientDB, bson.M{"_id": bson.ObjectIdHex(_id)}, bson.M{"$set": bson.M{"contact": contacts}}, false, false)
 			if b {
-				_,err := util.ElasticClient.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
-					"contact":contacts,
-				}).Refresh("true").Do(context.TODO())
-				if err != nil{
-					log.Println("update yqk contact err :",err)
+				escon := elastic.GetEsConn()
+				defer elastic.DestoryEsConn(escon)
+				_, err := escon.Update().Index(util.ElasticClientIndex).Type(util.ElasticClientType).Id(_id).Doc(map[string]interface{}{
+					"contact": contacts,
+				}).Refresh(true).Do()
+				if err != nil {
+					log.Println("update yqk contact err :", err)
 				}
 			}
 			c.JSON(200, gin.H{"rep": 200, "data": b})

src/jy/extract/extract.go

@@ -289,6 +289,10 @@ func (e *ExtractTask) PreInfo(doc map[string]interface{}) (j, jf *ju.Job, isSite
 	if subtype == "" {
 		subtype = "all"
 	}
+	if toptype == "其它" || subtype == "其它" || subtype == "其他" || subtype == "结果变更" {
+		toptype = "all"
+		subtype = "all"
+	}
 	toMap := qu.ObjToMap(doc["jsondata"])
 	//log.Debug("toMap", toMap)
 	if (*toMap) != nil {
@@ -1139,9 +1143,9 @@ func extractFromKv(field, fieldname string, blocks []*ju.Block, vc *RuleCore, kv
 						"blocktag":    bl.Classify,
 						"weight":      vv.Weight,
 					})
-					if field != "winnertel" && field != "winnerperson" {
-						break //暂定取第一个
-					}
+					//if field != "winnertel" && field != "winnerperson" {
+					//	//break //暂定取第一个
+					//}
 				}
 			}
 		}
@@ -1687,19 +1691,19 @@ func AnalysisSaveResult(j, jf *ju.Job, e *ExtractTask) {
 		//tmp["extract_content"] = j.Content
 		if e.TaskInfo.TestColl == "" {
 			if len(tmp) > 0 { //保存抽取结果
-				if len(e.SiteFields) <= 0 {
-					for field, _ := range e.Fields {
-						if tmp[field] == nil {
-							tmp[field] = "" //覆盖之前版本数据
-						}
-					}
+			/*	if len(e.SiteFields) <= 0 {
+					//for field, _ := range e.Fields {
+					//	if tmp[field] == nil &&  {
+					//		tmp[field] = "" //覆盖之前版本数据
+					//	}
+					//}
 				} else {
-					for field, _ := range e.SiteFields {
-						if tmp[field] == nil {
-							tmp[field] = "" //覆盖之前版本数据
-						}
-					}
-				}
+					//for field, _ := range e.SiteFields {
+					//	if tmp[field] == nil &&{
+					//		tmp[field] = "" //覆盖之前版本数据
+					//	}
+					//}
+				}*/
 				tmp["repeat"] = 0
 				tmparr := []map[string]interface{}{
 					map[string]interface{}{

src/jy/extract/score.go

@@ -339,9 +339,10 @@ func projectWeightClear(tmps []*ju.ExtField) []*ju.ExtField {
 			tmpWeight = v.Weight
 			tmpIndex = i
 		} else if v.Weight == tmpWeight {
-			if utf8.RuneCountInString(qu.ObjToString(v.Value)) >= 5 && utf8.RuneCountInString(qu.ObjToString(v.Value)) <= 38 && v.Value != tmps[tmpIndex].Value {
-				vmap[qu.ObjToString(v.Value)] = i
-			}
+			//if utf8.RuneCountInString(qu.ObjToString(v.Value)) >= 5 && utf8.RuneCountInString(qu.ObjToString(v.Value)) <= 38 && v.Value != tmps[tmpIndex].Value {
+			//	vmap[qu.ObjToString(v.Value)] = i
+			//}
+			newList = append(newList, v)
 		}
 	}
 	if tmpIndex != -999 {

src/jy/pretreated/analytable.go

@@ -96,7 +96,7 @@ var (
 	projectnameReg = regexp.MustCompile("((公开)?招标)*[(（第]*[一二三四五六七八九十a-zA-Z0-9]+(标段|包|标|段)[)）]*$")
 	MhSpilt        = regexp.MustCompile("[:：]")
 	//识别采购单位联系人、联系电话、代理机构联系人、联系电话
-	ContactInfoVagueReg = regexp.MustCompile("邮政编码|邮编|(征求意见|报名审核购买)?((联系人?(及|和)?|办公|单位)?(((联系)?(电话|方式)([／/]传真|及手机)?|手机)(号码)?|邮箱(地址)?|(详细)?(地(址|点)))|(联系|收料)(人(姓名)?|方式)|传真|电子邮件|(主要负责|项目(负责|联系)|经办)人)|采购方代表")
+	ContactInfoVagueReg = regexp.MustCompile("邮政编码|邮编|名称|(征求意见|报名审核购买)?((联系人?(及|和)?|办公|单位)?((电话([／/]传真|及手机)?|手机)(号码)?|邮箱(地址)?|(地(址|点)))|(联系|收料)(人(姓名)?|方式)|传真|电子邮件|(主要负责|项目(负责|联系)|经办)人)|采购方代表")
 	ContactInfoMustReg  = regexp.MustCompile("^(" + ContactInfoVagueReg.String() + ")$")
 	ContactType         = map[string]*regexp.Regexp{
 		"采购单位": regexp.MustCompile("(采购(项目.{2}|服务)?|比选|询价|招标(服务)?|甲|建设|委托|发包|业主|使用|谈判|本招标项目经办|征求意见联系|项目实施)(人|单位|部门|机构|机关|(执行)?方$)|(项目|建(库|设))单位|招标人信息|采购中心(地址)?|业主|收料人|采购部"),
@@ -244,6 +244,13 @@ func (table *Table) KVFilter(isSite bool, codeSite string) {
 			MergeKvTags(table.StandKV, kvTags)
 		} else {
 			//u.Debug(k, v, "---------")
+			if strings.Contains(k,"总价"){
+				if vvvv,ok := v.([]string);ok && len(vvvv)>0{
+					as.RemoveKey("报价")
+					as.AddKey(k,vvvv[len(vvvv)-1])
+					continue
+				}
+			}
 			as.AddKey(k, v)
 		}
 	}
@@ -470,7 +477,17 @@ func (table *Table) sortKVArr(as *SortMap, isSite bool, codeSite string) {
 							tmp.Weight = vv[0].Weight
 							tmp.Key = vv[0].Key
 							tmp.IsInvalid = vv[0].IsInvalid
-							tmp.Value = vvvvvv
+							if kk == "单品报价"||kk == "中标金额"||kk == "预算"{
+								if strings.Contains(k,"万"){
+									tmp.Value = vvvvvv+"万"
+								}else if strings.Contains(k,"亿"){
+									tmp.Value = vvvvvv+"亿"
+								}else {
+									tmp.Value = vvvvvv
+								}
+							}else {
+								tmp.Value = vvvvvv
+							}
 							table.StandKV[kk] = append(table.StandKV[kk], &tmp)
 						}
 					}
@@ -2591,8 +2608,6 @@ func (tn *Table) TdContactFormat(contactFormat *u.ContactFormat, isSite bool, co
 		}
 	}
 	//////
-	//qutil.Debug("indexMap-------------------------", indexMap)
-	//indexMap = map[int]string{}
 L:
 	for tr_index, tr := range tn.TRs {
 		thisTrHasMatch := false

src/jy/pretreated/tablev2.go

@@ -219,6 +219,9 @@ func (td *TD) tdHasTable(bsontable *bool, tr *TR,isSite bool,codeSite string) {
 				stag = str
 			}
 		}
+		if strings.Contains(stag,"开标记录"){
+			return
+		}
 		for _, tv := range tabs {
 			if IsHide(tv) {
 				continue

src/jy/util/elastic_client.go

@@ -1,11 +1,11 @@
 package util
 
-import (
-	"gopkg.in/olivere/elastic.v5"
-)
-
-var ElasticClient *elastic.Client
-var ElasticClientIndex,ElasticClientType string
+//import (
+//	"gopkg.in/olivere/elastic.v5"
+//)
+//
+//var ElasticClient *elastic.Client
+var ElasticClientIndex,ElasticClientType,ElasticClientDB string
 
 //func init() {
 //	if eClient, err := elastic.NewClient(elastic.SetURL(util.ObjToString(Config["elasticsearch"])));err != nil{

src/main.go

@@ -14,10 +14,9 @@ import (
 	qu "qfw/util"
 
 	log "github.com/donnie4w/go-logger/logger"
-	//"qfw/util/elastic"
+	"qfw/util/elastic"
 	"qfw/util/redis"
 
-	"gopkg.in/olivere/elastic.v5"
 )
 
 func init() {
@@ -37,15 +36,16 @@ func init() {
 	redis.InitRedisBySize(qu.ObjToString(util.Config["redis"]), 50, 30, 240)
 	//初始化elastic连接
 	//"winner=172.17.145.179:2710,buyer=172.17.145.179:2711"
-	//elastic.InitElasticSize(qu.ObjToString(util.Config["elasticsearch"]), qu.IntAllDef(util.Config["elasticPoolSize"], 30))
-	if eClient, err := elastic.NewClient(elastic.SetURL(qu.ObjToString(util.Config["elasticsearch"]))); err != nil {
-		log.Debug(util.Config["elasticsearch"])
-		log.Fatal("ElasticClient err:", err)
-	} else {
-		util.ElasticClient = eClient
+	elastic.InitElasticSize(qu.ObjToString(util.Config["elasticsearch"]), qu.IntAllDef(3, 30))
+	//if eClient, err := elastic.NewClient(elastic.SetURL(qu.ObjToString(util.Config["elasticsearch"]))); err != nil {
+	//	log.Debug(util.Config["elasticsearch"])
+	//	log.Fatal("ElasticClient err:", err)
+	//} else {
+	//	util.ElasticClient = eClient
 		util.ElasticClientIndex = qu.ObjToString(util.Config["elasticsearch_index"])
-		util.ElasticClientType = qu.ObjToString(util.Config["elasticsearch_type"])
-	}
+	util.ElasticClientType = qu.ObjToString(util.Config["elasticsearch_type"])
+	util.ElasticClientDB = qu.ObjToString(util.Config["winner_enterprise"])
+	//}
 }
 
 func main() {

src/res/fieldscore.json

@@ -90,7 +90,7 @@
                 "range": [
                     0,
                     3,
-                    -2
+                    -5
                 ]
             },
             {
@@ -98,7 +98,7 @@
                 "range": [
                     3,
                     5,
-                    0
+                    -2
                 ]
             },
             {
@@ -140,14 +140,14 @@
         "positivewords": [
             {
                 "describe": "以*结尾",
-                "regstr": ".{2,100}(委员会|办公室|学校|幼儿园|动物园|管理站|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|初中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司)$",
+                "regstr": ".{2,100}(委员会|分校|办公室|学校|幼儿园|动物园|管理站|图书馆|殡仪馆|博物馆|基地|青年宫|少年宫|艺术宫|电视台|协会|政府|初中|集团|银行|[大中小]学|院|厂|店|段|场|社|室|部|厅|局|处|所|队|公司)$",
                 "score": 3
             }
         ],
         "negativewords": [
             {
                 "describe": "包含负分",
-                "regstr": "(标人|附件|委托|代理|咨询|顾问|管理有限公司|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
+                "regstr": "(标人|附件|委托|认证|代理|咨询|顾问|管理有限公司|管理顾问|招标失败|交易中心|不足|公告|变更|招标|废标|废止|流标|中标|评标|开标|供应商|金额|万元|元整|预算|报价|单价|第(\\d|一|二|三|四|五)(名|包)|排名|候选|确定|标段|(标|一|二|三|四|五)包|中选|成交|包号|(A|B|C|D|E|F|G)包|地址|详情|要求|推荐|名称|评审|得分|合同|平方米|公示期|结果|备注|说明|单位|代表|委托|工作日|营业(执|期)|通过|代码|电话|联系|条件|合理|费率|以上|以下|拟定|为|注：|\\d[\\s]{0,10}(\\.|元|包|米|平米|平方米|吨|辆|千克|克|毫克|毫升|公升|套|件|瓶|箱|只|台|年|月|日|天|号)|(:|：|；|;|？|￥|\\*|%)|^[a-zA-Z0-9-]{5,100}|^[a-zA-Z0-9-]{1,100}$|[a-zA-Z0-9-]{10,100})",
                 "score": -5
             },
             {

udp_winner/config.json

@@ -0,0 +1,15 @@
+{
+  "elasticsearch": "http://127.0.0.1:9800",
+  "elasticsearch_index": "winner_enterprise",
+  "elasticsearch_type": "winnerent",
+  "udpport": "127.0.0.1:12311",
+  "mgoinit": "192.168.3.207:27081",
+  "mgodb_bidding": "qfw",
+  "mgodb_enterprise": "enterprise",
+  "mgodb_enterprise_c": "qyxy",
+  "mgourl2": "192.168.3.207:27092",
+  "mgourl": "127.0.0.1:27017",
+  "mgodb_extract_kf": "extract_kf",
+  "mgo_qyk_c": "enterprise_qyxy",
+  "redis": "127.0.0.1:6379"
+}

udp_winner/main.go

@@ -0,0 +1,469 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/garyburd/redigo/redis"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"log"
+	mu "mfw/util"
+	"net"
+	elastic "qfw/common/src/qfw/util/elastic"
+	"qfw/util"
+
+	"sort"
+	"strings"
+	"time"
+)
+
+var
+(
+	Config                                = make(map[string]string)
+	Fields                                []string
+	SourceClient, FClient                 *mongo.Client
+	RedisPool                             redis.Pool
+	Addrs                                 = make(map[string]interface{}, 0) //省市县
+	udpclient                             mu.UdpClient                      //udp对象
+	ElasticClientIndex, ElasticClientType string
+)
+/**
+新增
+初始化
+ */
+func init() {
+	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
+	util.ReadConfig(&Config)
+	log.Println(Config)
+	Fields = []string{"_id", "contact", "partners", "business_scope", "company_address", "capital",
+		"establish_date", "legal_person", "company_type", "district", "city", "province", "area_code", "credit_no",
+		"company_name", "history_name", "topscopeclass",  "wechat_accounts", "alias","website","report_websites"}
+	var err error
+	//mongo init
+	SourceClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgoinit"]).SetMaxPoolSize(20))
+	if err != nil {
+		log.Fatalln(err)
+	}
+	c1 := context.Background()
+	err = SourceClient.Connect(c1)
+	//defer SourceClient.Disconnect(c1)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	FClient, err = mongo.NewClient(options.Client().ApplyURI("mongodb://" + Config["mgourl"]).SetMaxPoolSize(20))
+	if err != nil {
+		log.Fatalln(err)
+	}
+	cc := context.Background()
+	err = FClient.Connect(cc)
+	//defer FClient.Disconnect(cc)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	//加载省市县代码
+	cursor2, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("address").Find(cc, bson.M{},
+		options.Find().SetProjection(bson.M{"province": 1, "code": 1, "city": 1, "district": 1}))
+	defer cursor2.Close(cc)
+	defer FClient.Connect(cc)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	for cursor2.Next(cc) {
+		tmp := make(map[string]interface{})
+		if err := cursor2.Decode(&tmp); err != nil {
+			log.Println(err)
+			continue
+		} else {
+			code := tmp["code"]
+			if code != nil && strings.TrimSpace(code.(string)) != "" {
+				Addrs[fmt.Sprint(code)] = tmp
+			}
+		}
+	}
+	log.Println(len(Addrs))
+	//es.NewClient(es.SetURL(addrs...), es.SetMaxRetries(2), es.SetSniff(false))
+	//es init
+	elastic.InitElasticSize(Config["elasticsearch"],10)
+	//esConn := elastic.GetEsConn()
+	//defer elastic.DestoryEsConn(esConn)
+	//log.Println(esConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id("123").BodyJson(map[string]interface{}{"testname":"六盘水市钟山开发区亿农科贸有限公司"}).Refresh(true).Do())
+	//log.Println(esConn.Delete().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id("123").Refresh(true).Do())
+
+	//if ESclient, err = elastic.NewClient(elastic.SetURL(Config["elasticsearch"]), elastic.SetHealthcheckTimeout(time.Minute)); err != nil {
+	//	log.Println(Config["elasticsearch"])
+	//	log.Fatalln("ElasticClient err:", err)
+	//} else {
+	//	ElasticClientIndex = Config["elasticsearch_index"]
+	//	ElasticClientType = Config["elasticsearch_type"]
+	//}
+
+	//redis
+	RedisPool = redis.Pool{
+		MaxIdle:     10,
+		IdleTimeout: 240 * time.Second,
+		Dial: func() (redis.Conn, error) {
+			conn, e := redis.Dial("tcp", Config["redis"])
+			if e != nil {
+				return conn, e
+			}
+			_, err = conn.Do("SELECT", "1")
+			if err != nil {
+				return nil, err
+			}
+			return conn, nil
+		},}
+	c := RedisPool.Get()
+	if _, err := c.Do("PING"); err != nil {
+		log.Fatalln(err)
+	}
+	defer c.Close()
+}
+
+func main() {
+	//udp
+	updport := Config["udpport"]
+	udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Println("Udp服务监听", updport)
+	go TimedTask() //定时任务
+	c := make(chan int, 1)
+	<-c
+
+}
+
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	log.Println(act, string(data), ra)
+	switch act {
+	case mu.OP_TYPE_DATA: //上个节点的数据
+		//从表中开始处理生成企业数据
+		tmp := new(map[string]interface{})
+		err := json.Unmarshal(data, &tmp)
+		if err != nil {
+			log.Println("err:", err)
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+			return
+		} else if tmp != nil {
+			udpclient.WriteUdp([]byte("ok,run"), mu.OP_NOOP, ra)
+			go task(tmp)
+		}
+	case mu.OP_NOOP: //下个节点回应
+		log.Println("发送成功", string(data))
+	}
+}
+func task(mapinfo *map[string]interface{}) {
+	defer util.Catch()
+	gtid, lteid := util.ObjToString((*mapinfo)["gtid"]), util.ObjToString((*mapinfo)["lteid"])
+	if gtid == "" || lteid == "" {
+		log.Println(gtid, lteid, "参数错误")
+		return
+	}
+	GId, err := primitive.ObjectIDFromHex(gtid)
+	LtId, err2 := primitive.ObjectIDFromHex(lteid)
+	if err != nil || err2 != nil {
+		log.Println(gtid, lteid, "转换_id错误")
+		return
+	}
+	//udp的id区间查询bidding  中标人 中标联系人 中标联系电话
+	// topscopeclass项目类型-industry行业类型&&topscopeclass联系人项目类型
+	// (area地区-province省份 city城市-city城市 district区县-district区县)
+	// winneraddr-company_address企业地址
+	cursor, err := SourceClient.Database(Config["mgodb_bidding"]).Collection("bidding").Find(context.TODO(), bson.M{
+		"_id": bson.M{
+			"$gte": GId,
+			"$lte": LtId,
+		},
+	}, options.Find().SetProjection(bson.M{"winner": 1, "winnertel": 1, "winnerperson": 1,
+		"topscopeclass": 1, "winneraddr": 1}))
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	for cursor.Next(context.TODO()) {
+		tmp := map[string]interface{}{}
+		if err := cursor.Decode(&tmp); err == nil {
+			if tmp["winner"] == nil || tmp["winner"] == "" {
+				continue
+			}
+			//redis查询是否存在
+			rdb := RedisPool.Get()
+			defer rdb.Close()
+			if reply, err := redis.String(rdb.Do("GET", tmp["winner"])); err != nil {
+				//redis不存在存到临时表,定时任务处理
+				FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").InsertOne(context.TODO(), tmp)
+				//log.Println(tmp, err)
+				continue
+			} else {
+				//redis存在
+				//log.Println(reply)
+				//reply = "5e0316b998a9abaf6535df3d"
+				id, err := primitive.ObjectIDFromHex(reply)
+				if err != nil {
+					log.Println("get redis id err:", err, tmp)
+					continue
+				}
+				//拿到合并后的qyk
+				oldTmp := make(map[string]interface{})
+				err = FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+					FindOne(context.TODO(), bson.M{"_id": id}).Decode(&oldTmp)
+				if err != nil {
+					log.Println("qyk id err:", err, id)
+					continue
+				}
+				//比较合并
+				//行业类型
+				tmpTopscopeclass := []string{}
+				tmpTopscopeclassMap := make(map[string]bool)
+				log.Println(tmp["_id"])
+				if oldTmp["industry"] == nil {
+					//log.Println(reflect.ValueOf(tmp["topscopeclass"]))
+					if v, ok := tmp["topscopeclass"].(primitive.A); ok {
+						for _, vv := range v {
+							if vvv, ok := vv.(string); ok && len(vvv) > 1 {
+								tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
+							}
+						}
+						for k := range tmpTopscopeclassMap {
+							tmpTopscopeclass = append(tmpTopscopeclass, k)
+						}
+					}
+				} else {
+					if v, ok := oldTmp["industry"].(primitive.A); ok {
+						for _, vv := range v {
+							if vvv, ok := vv.(string); ok {
+								tmpTopscopeclassMap[vvv] = true
+							}
+						}
+					}
+					if v, ok := tmp["topscopeclass"].(primitive.A); ok {
+						for _, vv := range v {
+							if vvv, ok := vv.(string); ok && len(vvv) > 1 {
+								tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
+							}
+						}
+						for k := range tmpTopscopeclassMap {
+							tmpTopscopeclass = append(tmpTopscopeclass, k)
+						}
+					}
+				}
+				sort.Strings(tmpTopscopeclass)
+				oldTmp["industry"] = tmpTopscopeclass
+				esId := oldTmp["_id"].(primitive.ObjectID).Hex()
+				//联系方式合并
+				if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" {
+					oldTmp["updatatime"] =time.Now().Unix()
+					//mongo更新
+					FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+						UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp})
+					//es更新
+					delete(oldTmp, "_id")
+					esConn := elastic.GetEsConn()
+					defer elastic.DestoryEsConn(esConn)
+					esConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do()
+					//log.Println( err2,err3)
+					continue
+				}
+				var tmpperson, winnertel string
+				tmpperson = tmp["winnerperson"].(string)
+				if tmp["winnertel"] == nil {
+					winnertel = ""
+				}
+				contactMaps := make([]interface{}, 0)
+				if oldTmp["contact"] == nil {
+					tmpContact := make(map[string]interface{})
+					tmpContact["contact_person"] = tmpperson
+					tmpContact["contact_type"] = "项目联系人"
+					tmpContact["phone"] = winnertel
+					tmpContact["topscopeclass"] = strings.Join(tmpTopscopeclass, ";")
+					tmpContact["updatetime"] = time.Now().Unix()
+					contactMaps = append(contactMaps, tmpContact)
+				} else {
+					//对比前四项,相等丢弃
+					if v, ok := oldTmp["contact"].(primitive.A); ok {
+						var isNotUpdate bool
+						for _, vv := range v {
+							if vvv, ok := vv.(map[string]interface{}); ok {
+								if vvv["contact_person"] == tmpperson && vvv["contact_type"] == "项目联系人" &&
+									vvv["phone"] == winnertel && vvv["topscopeclass"] == strings.Join(tmpTopscopeclass, ";") {
+									isNotUpdate = true
+									vvv["updatetime"] = time.Now().Unix()
+								}
+								contactMaps = append(contactMaps, vvv)
+							}
+						}
+						if !isNotUpdate {
+							vvv := make(map[string]interface{})
+							vvv["contact_person"] = tmp["winnerperson"]
+							vvv["contact_type"] = "项目联系人"
+							vvv["phone"] = winnertel
+							vvv["topscopeclass"] = strings.Join(tmpTopscopeclass, ";")
+							vvv["updatetime"] = time.Now().Unix()
+							contactMaps = append(contactMaps, vvv)
+						}
+					}
+				}
+				oldTmp["contact"] = contactMaps
+				//mongo更新
+				oldTmp["updatatime"] =time.Now().Unix()
+				FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+					UpdateOne(context.TODO(), bson.M{"_id": oldTmp["_id"]}, bson.M{"$set": oldTmp})
+				//es更新
+				delete(oldTmp, "_id")
+				esConn := elastic.GetEsConn()
+				defer elastic.DestoryEsConn(esConn)
+				esConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do()
+				//log.Println( err2,err3)
+			}
+		} else {
+			log.Println(tmp)
+			continue
+		}
+	}
+	defer cursor.Close(context.TODO())
+	//tmps := make([]interface{}, 0)
+	//num, snum := 0, 0
+	//for k := range keys {
+	//	//if num == 6 {
+	//	//	return
+	//	//}
+	//	tmp := make(map[string]interface{})
+	//	err := Client.Database("enterprise").Collection("qyxy").FindOne(context.TODO(), bson.M{"company_name": k}).Decode(&tmp)
+	//	if err != nil {
+	//		//log.Println(k, err)
+	//		continue
+	//	}
+	//	if tmp["credit_no"] != nil {
+	//		if credit_no, ok := tmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
+	//			len(strings.TrimSpace(credit_no)) > 8 {
+	//			dataNo := strings.TrimSpace(credit_no)[2:8]
+	//			if Addrs[dataNo] != nil {
+	//				if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
+	//					if tmp["province"] == nil || tmp["province"] == "" {
+	//						tmp["province"] = v["province"]
+	//					}
+	//					tmp["city"] = v["city"]
+	//					tmp["district"] = v["district"]
+	//
+	//				}
+	//			}
+	//		}
+	//	}
+	//	contacts := make([]map[string]interface{}, 0)
+	//	contact := make(map[string]interface{}, 0)
+	//	if tmp["legal_person"] != nil {
+	//		contact["contact_person"] = tmp["legal_person"] //联系人
+	//	} else {
+	//		contact["contact_person"] = "" //联系人
+	//	}
+	//	contact["contact_type"] = "法定代表人" //法定代表人
+	//	//log.Println(1)
+	//	if tmp["annual_reports"] != nil {
+	//		bytes, err := json.Marshal(tmp["annual_reports"])
+	//		if err != nil {
+	//			log.Println("annual_reports err:", err)
+	//		}
+	//		//log.Println(2, string(bytes))
+	//		phonetmp := make([]map[string]interface{}, 0)
+	//		err = json.Unmarshal(bytes, &phonetmp)
+	//		if err != nil {
+	//			log.Println("Unmarshal err:", err)
+	//		}
+	//		//log.Println(44, err)
+	//		for _, vv := range phonetmp {
+	//			if vv["company_phone"] != nil {
+	//				if vv["company_phone"] == "" {
+	//					continue
+	//				} else {
+	//					contact["phone"] = vv["company_phone"] //联系电话
+	//					break
+	//				}
+	//			} else {
+	//				contact["phone"] = "" //联系电话
+	//			}
+	//
+	//		}
+	//	}
+	//	//log.Println(k, contact["phone"], tmp["_id"])
+	//	//time.Sleep(10 * time.Second)
+	//	if contact["phone"] == nil {
+	//		contact["phone"] = "" //联系电话
+	//	}
+	//	contact["topscopeclass"] = "企业公示"         //项目类型
+	//	contact["updatetime"] = time.Now().Unix() //更新时间
+	//	contacts = append(contacts, contact)
+	//	tmp["contact"] = contacts
+	//
+	//	savetmp := make(map[string]interface{}, 0)
+	//	//字段处理
+	//	for _, sk := range Fields {
+	//		if sk == "establish_date" { //成立日期
+	//			if tmp[sk] != nil {
+	//				savetmp[sk] = tmp[sk].(primitive.DateTime).Time().UTC().Unix()
+	//				continue
+	//			}
+	//		} else if sk == "capital" { //注册资本
+	//			//log.Println(sk, tmp[sk])
+	//			savetmp[sk] = ObjToMoney([]interface{}{tmp[sk], ""})[0]
+	//			continue
+	//		} else if sk == "partners" { //股东及出资信息
+	//			//log.Println(sk, tmp[sk], )
+	//			//fmt.Println(reflect.TypeOf(tmp[sk]))
+	//			if tmp[sk] != nil {
+	//				if ppms, ok := tmp[sk].(primitive.A); ok {
+	//					for i, _ := range ppms {
+	//						if ppms[i].(map[string]interface{})["stock_type"] != nil {
+	//							ppms[i].(map[string]interface{})["stock_type"] = "企业公示"
+	//						}
+	//						delete(ppms[i].(map[string]interface{}), "identify_type")
+	//					}
+	//					savetmp[sk] = ppms
+	//					continue
+	//				}
+	//			}
+	//		} else if sk == "_id" { //_id备份企业库
+	//			savetmp["tmp"+sk] = tmp[sk]
+	//			continue
+	//		}
+	//		if tmp[sk] == nil && sk != "history_name" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "wechat_accounts" {
+	//			savetmp[sk] = ""
+	//		} else {
+	//			if sk == "wechat_accounts" { //微信公众号
+	//				if savetmp[sk] == nil {
+	//					//TODO 微信公众号取值未确认
+	//					savetmp[sk] = []string{}
+	//				}
+	//				continue
+	//			} else if sk == "website" { //网址
+	//				//TODO 网址取值未确认
+	//				continue
+	//			}
+	//			savetmp[sk] = tmp[sk]
+	//		}
+	//	}
+	//	savetmp["alias"] = "" //别名
+	//	tmps = append(tmps, savetmp)
+	//	num++
+	//	snum++
+	//	if snum >= 300 {
+	//		_, err := Client.Database("extract_v3").Collection("enterprise_qyxy").InsertMany(context.TODO(), tmps)
+	//		if err != nil {
+	//			log.Println("save:", err)
+	//			continue
+	//		} else {
+	//			log.Println(num)
+	//			tmps = []interface{}{}
+	//			snum = 0
+	//		}
+	//	}
+	//}
+	//if len(tmps) > 0 {
+	//	result, err := Client.Database("extract_v3").Collection("enterprise_qyxy").InsertMany(context.TODO(), tmps)
+	//	if err != nil {
+	//		log.Println("save over:", err)
+	//	} else {
+	//		log.Println("last save num:", len(result.InsertedIDs))
+	//	}
+	//}
+}

udp_winner/text.md

@@ -0,0 +1,19 @@
+{
+  "elasticsearch": "http://172.17.145.170:9800",
+  "elasticsearch_index": "winner_enterprise",
+  "elasticsearch_type": "winnerent",
+  "udpport": "127.0.0.1:1486",
+  "mgoinit": "10.30.94.175:27081,10.81.232.246:27082,10.172.242.243:27080",
+  "mgodb_bidding": "qfw",
+  "mgodb_enterprise": "enterprise",
+  "mgodb_enterprise_c": "qyxy",
+  "mgourl": "172.17.145.163:27082",
+  "mgodb_extract_kf": "extract_v3",
+  "mgo_qyk_c": "winner_enterprise",
+  "redis": "172.17.148.44:1479"
+}
+
+中标单位企业库
+/file/udpwinnerent_1486
+172.17.145.163 
+JyTopnet@ZY#@!2018

udp_winner/timedTask.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	elastic "qfw/common/src/qfw/util/elastic"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"gopkg.in/mgo.v2/bson"
+	"log"
+	"sort"
+	"strings"
+	"time"
+)
+
+//定时任务
+//1.存异常表
+//2.合并原始库新增
+func TimedTask() {
+	t2 := time.NewTimer(time.Second * 5)
+	for range t2.C {
+		tmpLast := map[string]interface{}{}
+		if err := FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").FindOne(context.TODO(), bson.M{}, options.FindOne().SetSort(bson.M{"_id": -1})).Decode(&tmpLast); err != nil {
+			//临时表无数据
+			log.Println("临时表无数据:", err)
+			t2.Reset(time.Minute * 5)
+			continue
+		} else {
+			//临时表有数据
+			log.Println("临时表有数据:", tmpLast)
+			cursor, err := FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").Find(context.TODO(), bson.M{
+				"_id": bson.M{
+					"$lte": tmpLast["_id"],
+				},
+			}, options.Find().SetSort(bson.M{"_id": 1}))
+			if err != nil {
+				log.Println(err)
+				t2.Reset(time.Second * 5)
+				continue
+			}
+			//遍历临时表数据，匹配不到原始库存入异常表
+			for cursor.Next(context.TODO()) {
+				tmp := make(map[string]interface{})
+				if err := cursor.Decode(&tmp); err == nil {
+					resulttmp := make(map[string]interface{})
+					r := FClient.Database(Config["mgodb_enterprise"]).Collection(Config["mgodb_enterprise_c"]).FindOne(context.TODO(), bson.M{"company_name": tmp["winner"]}).Decode(&resulttmp)
+					if r != nil {
+						//log.Println(r)
+						//匹配不到原始库,存入异常表删除临时表
+						FClient.Database(Config["mgodb_extract_kf"]).Collection("err_winner_qyk").InsertOne(context.TODO(), tmp)
+						FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").DeleteOne(context.TODO(), tmp)
+						continue
+					} else {
+						//log.Println(123)
+						//匹配到原始库,新增 resulttmp
+						if resulttmp["credit_no"] != nil {
+							if credit_no, ok := resulttmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
+								len(strings.TrimSpace(credit_no)) > 8 {
+								dataNo := strings.TrimSpace(credit_no)[2:8]
+								if Addrs[dataNo] != nil {
+									if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
+										if resulttmp["province"] == nil || resulttmp["province"] == "" {
+											resulttmp["province"] = v["province"]
+										}
+										resulttmp["city"] = v["city"]
+										resulttmp["district"] = v["district"]
+
+									}
+								}
+							}
+						}
+						contacts := make([]map[string]interface{}, 0)
+						contact := make(map[string]interface{}, 0)
+						if resulttmp["legal_person"] != nil {
+							contact["contact_person"] = resulttmp["legal_person"] //联系人
+						} else {
+							contact["contact_person"] = "" //联系人
+						}
+						contact["contact_type"] = "法定代表人" //法定代表人
+						//log.Println(1)
+						if resulttmp["annual_reports"] != nil {
+							bytes, err := json.Marshal(resulttmp["annual_reports"])
+							if err != nil {
+								log.Println("annual_reports err:", err)
+							}
+							//log.Println(2, string(bytes))
+							phonetmp := make([]map[string]interface{}, 0)
+							err = json.Unmarshal(bytes, &phonetmp)
+							if err != nil {
+								log.Println("Unmarshal err:", err)
+							}
+							//log.Println(44, err)
+							for _, vv := range phonetmp {
+								if vv["company_phone"] != nil {
+									if vv["company_phone"] == "" {
+										continue
+									} else {
+										contact["phone"] = vv["company_phone"] //联系电话
+										break
+									}
+								} else {
+									contact["phone"] = "" //联系电话
+								}
+
+							}
+						}
+						//log.Println(k, contact["phone"], resulttmp["_id"])
+						//time.Sleep(10 * time.Second)
+						if contact["phone"] == nil {
+							contact["phone"] = "" //联系电话
+						}
+						contact["topscopeclass"] = "企业公示"         //项目类型
+						contact["updatetime"] = time.Now().Unix() //更新时间
+						contacts = append(contacts, contact)
+						resulttmp["contact"] = contacts
+
+						savetmp := make(map[string]interface{}, 0)
+						for _, sk := range Fields {
+							if sk == "establish_date" {
+								if resulttmp[sk] != nil {
+									savetmp[sk] = resulttmp[sk].(primitive.DateTime).Time().UTC().Unix()
+									continue
+								}
+							} else if sk == "capital" {
+								//log.Println(sk, resulttmp[sk])
+								savetmp[sk] = ObjToMoney([]interface{}{resulttmp[sk], ""})[0]
+								continue
+							} else if sk == "partners" {
+								//log.Println(sk, resulttmp[sk], )
+								//fmt.Println(reflect.TypeOf(resulttmp[sk]))
+								if resulttmp[sk] != nil {
+									if ppms, ok := resulttmp[sk].(primitive.A); ok {
+										for i, _ := range ppms {
+											if ppms[i].(map[string]interface{})["stock_type"] != nil {
+												ppms[i].(map[string]interface{})["stock_type"] = "企业公示"
+											}
+											delete(ppms[i].(map[string]interface{}), "identify_type")
+										}
+										savetmp[sk] = ppms
+
+									}
+								}else {
+									savetmp[sk] = []interface{}{}
+								}
+								continue
+							} else if sk == "_id" {
+								savetmp["tmp"+sk] = resulttmp[sk]
+								continue
+							} else if sk == "area_code" {
+								//行政区划代码
+								savetmp[sk] = fmt.Sprint(resulttmp[sk])
+								continue
+							} else if sk == "report_websites" {
+								//网址
+								if resulttmp["report_websites"] == nil {
+									savetmp["website"] = ""
+								} else {
+									report_websitesArr := []string{}
+									if ppms, ok := resulttmp[sk].(primitive.A); ok {
+										for _, v := range ppms {
+											if vvv, ok := v.(map[string]interface{}); ok {
+												if rv, ok := vvv["website_url"].(string); ok {
+													report_websitesArr = append(report_websitesArr, rv)
+												}
+											}
+										}
+									}
+									sort.Strings(report_websitesArr)
+									savetmp["website"] = strings.Join(report_websitesArr, ";")
+								}
+								continue
+							} else if sk == "wechat_accounts" {
+								savetmp[sk] = []interface{}{}
+								continue
+							}
+							if resulttmp[sk] == nil && sk != "history_name" &&sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
+								savetmp[sk] = ""
+							} else {
+								savetmp[sk] = resulttmp[sk]
+							}
+						}
+						//tmps = append(tmps, savetmp)
+						savetmp["updatatime"] =time.Now().Unix()
+						//保存mongo
+						result, err := FClient.Database(Config["mgodb_extract_kf"]).Collection(Config["mgo_qyk_c"]).
+							InsertOne(context.TODO(), savetmp)
+						if err == nil {
+							//保存redis
+							rc := RedisPool.Get()
+							defer rc.Close()
+							var _id string
+							if v, ok := result.InsertedID.(primitive.ObjectID); ok {
+								_id = v.Hex()
+							}
+							if _, err := rc.Do("SET", savetmp["company_name"], result.InsertedID.(primitive.ObjectID).Hex()); err != nil {
+								log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["company_name"], err)
+							} else {
+								//保存es
+								delete(savetmp, "_id")
+								esConn := elastic.GetEsConn()
+								defer elastic.DestoryEsConn(esConn)
+								if _, err := esConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(_id).BodyJson(savetmp).Refresh(true).Do(); err != nil {
+									log.Println("save es err :", tmp["_id"], savetmp["_id"], err)
+								} else {
+									//删除临时表
+									FClient.Database(Config["mgodb_extract_kf"]).Collection("tmp_winner_qyk").DeleteOne(context.TODO(), tmp)
+								}
+							}
+						} else {
+							log.Println("save mongo err:", err, tmp["_id"])
+						}
+					}
+				}
+
+			}
+			defer cursor.Close(context.TODO())
+		}
+		t2.Reset(time.Minute)
+	}
+}

udp_winner/util.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	regOperator, _ = regexp.Compile(`[*|+|)*)]`)
+	regNumFloat, _ = regexp.Compile(`([1-9]\d*|0)(\.\d+)?`)
+	regStrUnit, _  = regexp.Compile(`[元|万|亿]`)
+
+	regStrChar      = `[〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]`
+	moneyRegChar, _ = regexp.Compile(regStrChar)
+	contentUnit, _  = regexp.Compile(`(万元|单位/万)`)
+	numCapitals, _  = regexp.Compile(`([〇|零|点|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|百|佰|千|仟|万|亿|億|元|圆|角|分|整|正]{4,40})`)
+
+	regQianw, _ = regexp.Compile(`\d{1,2}千万`)
+	moneyChar   = map[string]interface{}{ //"〇": "0", "零": "0",
+		"一": float64(1), "壹": float64(1), "二": float64(2), "贰": float64(2), "三": float64(3), "叁": float64(3), "四": float64(4), "肆": float64(4), "五": float64(5), "伍": float64(5),
+		"六": float64(6), "陆": float64(6), "七": float64(7), "柒": float64(7), "八": float64(8), "捌": float64(8), "九": float64(9), "玖": float64(9), "十": float64(10), "拾": float64(10),
+		"百": float64(100), "佰": float64(100), "千": float64(1000), "仟": float64(1000), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000),
+		"零": float64(0), "点": ".", "角": float64(0.1), "分": float64(0.01),
+	}
+	moneyUnit = map[string]float64{
+		"元": float64(1), "万": float64(10000), "亿": float64(100000000), "億": float64(100000000), //单位
+	}
+	cutAllSpace, _ = regexp.Compile(`\s*`)
+	spaces         = []string{"\u3000", "\u2003", "\u00a0", "\t", "\r", "\n"}
+)
+//大写数子金额转换
+func capitalMoney(data []interface{}) []interface{} {
+	nodes := []float64{}
+	node := float64(0)
+	tmp := float64(0)
+	decimals := 0.0
+	ishaspoint := false //是否含小数点
+	fnum := float64(0)
+	end := false
+	str := fmt.Sprint(data[0])
+	//提取第一个大写信息
+	strmatch := numCapitals.FindAllStringSubmatch(str, -1)
+	if len(strmatch) > 0 {
+		str = strmatch[0][0]
+	}
+	suffixUnit := float64(1)
+	if strings.HasSuffix(str, "万") || strings.HasSuffix(str, "万元") || strings.HasSuffix(str, "万元整") {
+		index := strings.LastIndex(str, "万")
+		str = str[0:index]
+		suffixUnit = float64(10000)
+	}
+	moneyRegChar.ReplaceAllStringFunc(str, func(key string) string {
+		if key == "元" || key == "圆" || key == "点" {
+			ishaspoint = true
+		}
+		if v, ok := moneyChar[key].(float64); ok && !end {
+			if ishaspoint && v > 10 { //排除后面有其他的单位
+				return ""
+			}
+			//fmt.Println(key, v, fnum)
+			if v < 10 && v >= 0 {
+				if ishaspoint { //小数部分
+					if v >= 1 {
+						fnum = v
+					} else if v < 1 && v > 0 {
+						decimals += fnum * v
+					}
+				} else {
+					if tmp != float64(0) {
+						node += tmp
+					}
+					tmp = float64(v)
+				}
+			} else if v == 10000 || v == 100000000 { //单位万、亿
+				if tmp != float64(0) {
+					node += tmp
+					tmp = float64(0)
+				}
+				nodes = append(nodes, node*float64(v))
+				node = float64(0)
+			} else {
+				if v == 10 && tmp == 0 {
+					tmp = 1
+				}
+				tmp = tmp * float64(v)
+				node += tmp
+				tmp = float64(0)
+			}
+		}
+		if key == "整" || key == "正" || key == "分" {
+			end = true
+		}
+		return ""
+	})
+	nodes = append(nodes, node, tmp)
+	ret := float64(0)
+	for _, v := range nodes {
+		ret += v
+	}
+	return []interface{}{(ret + decimals) * suffixUnit, data[1]}
+}
+
+//金额转换
+func ObjToMoney(data []interface{}) []interface{} {
+	isfindUnit := true
+	ret := capitalMoney(data)[0]
+	if ret.(float64) < float64(10000) || ret.(float64) > float64(50000000000) {
+		ret2, b := numMoney(data)
+		isfindUnit = b
+		if ret2[0].(float64) > ret.(float64) {
+			ret = ret2[0]
+		}
+	}
+	f, _ := strconv.ParseFloat(strconv.FormatFloat(ret.(float64), 'f', 4, 64), 64)
+	if f < 1 {
+		f = 0
+	}
+	//若果金额小于50，全文检索单位：万
+	if f < 50 && f > 0 && isfindUnit {
+		rep := contentUnit.FindAllStringIndex(fmt.Sprint(data[1]), -1)
+		if len(rep) > 0 {
+			f = f * 10000
+		}
+	}
+	data[0] = f
+	return data
+}
+
+//符号替换
+func replaceString(con string, ret, rep []string) string {
+	for k, v := range ret {
+		if len(rep) > k {
+			con = strings.Replace(con, v, rep[k], -1)
+		}
+	}
+	return con
+}
+
+//清理所有空白符
+func CutAllSpace(data []interface{}) []interface{} {
+	tmp := cutAllSpace.ReplaceAllString(fmt.Sprint(data[0]), "")
+	tmp = replaceSymbol(tmp, spaces)
+	data[0] = tmp
+	return data
+}
+
+//数字金额转换
+func numMoney(data []interface{}) ([]interface{}, bool) {
+	tmp := fmt.Sprintf("%f", data[0])
+	repUnit := float64(1)
+	if regQianw.MatchString(tmp) {
+		tmp = strings.Replace(tmp, "千万", "万", -1)
+		repUnit = float64(1000)
+	}
+	tmp = replaceSymbol(tmp, []string{",", "，", "（", "）", "(", ")", "：", "\n"})
+	tmp = replaceString(tmp, []string{"万元", "亿元", "．"}, []string{"万", "亿", "."})
+	tmp = fmt.Sprint(CutAllSpace([]interface{}{tmp, data[1]})[0])
+	rets := regNumFloat.FindAllString(tmp, -1)
+	fnums := []float64{}
+	unitstrs := []string{}
+	if len(rets) > 0 {
+		pindex := 0 //单位前置
+		for k, v := range rets {
+			f, err := strconv.ParseFloat(v, 64)
+			if err == nil {
+				fnums = append(fnums, f)
+				index := strings.Index(tmp, v)
+				//单位后置
+				start := index + len(v)
+				end := start + 3
+				//log.Println("vvv", tmp, v, pindex, index, start)
+				if k > 0 {
+					if start >= pindex+3 {
+						pstart := pindex + 3
+						if pstart >= index {
+							pstart = index
+						}
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[pstart:index]+tmp[start:])
+						}
+					} else {
+						if len(tmp) > end {
+							unitstrs = append(unitstrs, tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				} else {
+					if len(tmp) > end {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:end])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:end])
+						}
+					} else {
+						if index-3 >= 0 {
+							unitstrs = append(unitstrs, tmp[index-3:index]+tmp[start:])
+						} else {
+							unitstrs = append(unitstrs, tmp[start:])
+						}
+					}
+				}
+				pindex = start
+			}
+		}
+	}
+	//log.Println("unitstrs", fnums, unitstrs)
+	unit := float64(0)
+	fnum := float64(0)
+	for k, v := range fnums {
+		fnum = v
+		units := regStrUnit.FindAllString(unitstrs[k], -1)
+		for _, v := range units {
+			if moneyUnit[v] != 0 {
+				unit = moneyUnit[v]
+				break
+			}
+		}
+		if unit != float64(0) { //取第一个
+			break
+		}
+	}
+	fnum = fnum * repUnit
+	if unit == float64(0) {
+		data[0] = fnum
+	} else {
+		data[0] = fnum * unit
+	}
+	if unit == 10000 {
+		return data, false
+	} else {
+		return data, true
+	}
+}
+
+//过滤符号
+func replaceSymbol(con string, rep []string) string {
+	for _, v := range rep {
+		con = strings.Replace(con, v, "", -1)
+	}
+	return con
+}
+

udpfilterdup/src/config.json

@@ -5,7 +5,7 @@
         "addr": "192.168.3.207:27092",
         "pool": 5,
         "db": "extract_kf",
-        "extract": "zheng_test1_jd1",
+        "extract": "zk",
         "site": {
             "dbname": "zhaolongyue",
             "coll": "site"
@@ -19,7 +19,7 @@
     "isMerger": false,
     "threads": 1,
     "specialwords": "(重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研)",
-    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包)",
+    "specialtitle_1": "[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９](次|包|标段|标包|批)",
     "specialtitle_2": "项目（[0-9a-zA-Z一二三四五六七八九十零１２３４５６７８９]）",
     "beifen": "[(（]?[0-9一二三四五六七八九十零１２３４５６７８９再][)）]?[子分]?[次批标包]|重招|重新招标|勘察|设计|施工|监理|总承包|土石方|可研"
 }

udpfilterdup/src/datamap.go

@@ -23,6 +23,7 @@ type Info struct {
 	bidamount   float64 //中标金额
 	projectname string  //项目名称
 	projectcode string  //项目编号
+	contractnumber string //合同编号
 	publishtime int64   //发布时间
 	bidopentime int64   //开标时间
 	agencyaddr  string  //开标地点
@@ -190,6 +191,7 @@ func NewInfo(tmp map[string]interface{}) *Info {
 	info.subtype = subtype
 	info.buyer = qutil.ObjToString(tmp["buyer"])
 	info.projectname = qutil.ObjToString(tmp["projectname"])
+	info.contractnumber = qutil.ObjToString(tmp["contractnumber"])
 	info.projectcode = qutil.ObjToString(tmp["projectcode"])
 	info.city = qutil.ObjToString(tmp["city"])
 	info.agency = qutil.ObjToString(tmp["agency"])
@@ -276,10 +278,13 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = "标题关键词且包含关系"
-							b = true
-							source = v
-							break L
+							reason = reason+"标题关键词且包含关系"
+							//继续二级金额判断
+							if !againRepeat(v,info){
+								b = true
+								source = v
+								break
+							}
 						}
 					}
 
@@ -406,10 +411,14 @@ L:
 							continue //无包含关系
 						}
 						if strings.Contains(v.title, info.title) || strings.Contains(info.title, v.title) {
-							reason = "标题关键词且包含关系"
-							b = true
-							source = v
-							break L
+							reason = reason+"标题关键词且包含关系"
+							//继续二级金额判断
+							if !againRepeat(v,info){
+								b = true
+								source = v
+								break
+							}
+
 						}
 					}
 
@@ -735,8 +744,9 @@ func tenderRepeat_A(v *Info, info *Info, reason string) (bool ,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
-		ss = ss + "p3(编号)-"
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
 	if v.budget != 0 && v.budget == info.budget {
@@ -781,7 +791,8 @@ func tenderRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
 		m++
 	}
 	if v.budget != 0 && v.budget == info.budget {
@@ -841,8 +852,9 @@ func winningRepeat_A(v *Info, info *Info, reason string) (bool,string) {
 		ss = ss + "p2(单位)-"
 		p2 = true
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
-		ss = ss + "p3(编号)-"
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
+		ss = ss + "p3(编号组)-"
 		p3 = true
 	}
 	if v.bidamount != 0 && v.bidamount == info.bidamount {
@@ -883,7 +895,8 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool,string) {
 	if v.buyer != "" && v.buyer == info.buyer {
 		m++
 	}
-	if v.projectcode != "" && v.projectcode == info.projectcode {
+	if (v.projectcode != "" && v.projectcode == info.projectcode && len(v.projectcode)>=5)||
+		(v.contractnumber != "" && v.contractnumber == info.contractnumber && len(v.contractnumber)>=5){
 		m++
 	}
 	if v.bidamount != 0 && v.bidamount == info.bidamount {
@@ -961,6 +974,32 @@ func contractRepeat_C(v *Info, info *Info) bool {
 }
 
 
+func againRepeat(v *Info ,info *Info) bool {
+	//相同采购单位下
+	if info.buyer != "" &&v.buyer == info.buyer {
+		if info.subtype=="招标"||info.subtype=="邀标"||info.subtype=="询价"||
+			info.subtype=="竞谈"||info.subtype=="单一"||info.subtype=="竞价"||
+			info.subtype=="其他"||info.subtype=="变更" {
+			//预算金额满足条件
+			if v.budget!=info.budget&&v.budget!=0&&info.budget!=0 {
+				return true
+			}
+		}else if info.subtype=="中标"||info.subtype=="成交"||info.subtype=="废标"||
+			info.subtype=="流标"||info.subtype=="合同"||info.subtype=="验收"||
+			info.subtype=="违规"{
+			//中标金额单位满足条件
+			if (v.bidamount!=info.bidamount&&v.bidamount!=0&&info.bidamount!=0)||
+				(v.winner!=info.winner&&v.winner!=""&&info.winner!=""){
+				return true
+			}
+		}else {
+
+		}
+	}
+
+	return false
+}
+
 
 
 

udpfilterdup/src/main.go

@@ -38,7 +38,6 @@ var (
 	FilterRegTitle_1 = regexp.MustCompile("^_$")
 	FilterRegTitle_2 = regexp.MustCompile("^_$")
 
-
 	isMerger bool                              //是否合并
 	threadNum int								   //线程数量
 	SiteMap  map[string]map[string]interface{} //站点map
@@ -46,6 +45,7 @@ var (
 )
 
 func init() {
+
 	flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
 	flag.StringVar(&sid, "sid", "", "开始id")
 	flag.StringVar(&eid, "eid", "", "结束id")
@@ -104,8 +104,8 @@ func main() {
 
 //测试组人员使用
 func mainT() {
-	//sid = "5dfbd43ce9d1f601e43fa402"
-	//eid = "5e0954b30cf41612e061d0c8"
+	//sid = "5da3f31aa5cb26b9b798d3aa"
+	//eid = "5da422fba5cb26b9b706984b"
 	mapinfo := map[string]interface{}{}
 	if sid == "" || eid == "" {
 		log.Println("sid,eid参数不能为空")
@@ -115,7 +115,7 @@ func mainT() {
 	mapinfo["lteid"] = eid
 	mapinfo["stop"] = "true"
 	task([]byte{}, mapinfo)
-	time.Sleep(5 * time.Second)
+	time.Sleep(10 * time.Second)
 }
 func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 	fmt.Println("接受的段数据")
@@ -200,7 +200,7 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			}()
 			info := NewInfo(tmp)
 			//是否为无效数据
-			if invalidData(info.buyer, info.projectname, info.projectcode) {
+			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
 						"_id": tmp["_id"],
@@ -286,7 +286,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 								update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
 							} else if value == 8 {
 								update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-							} else {
+							} else if value == 9 {
+								update_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
+							}else {
 							}
 						}
 					}
@@ -427,7 +429,7 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 				wg.Done()
 			}()
 			info := NewInfo(tmp)
-			if invalidData(info.buyer, info.projectname, info.projectcode) {
+			if invalidData(info.buyer, info.projectname, info.projectcode,info.contractnumber) {
 				//mapLock.Lock()
 				updateExtract = append(updateExtract, []map[string]interface{}{
 					map[string]interface{}{
@@ -529,7 +531,9 @@ func historyTask(data []byte, mapInfo map[string]interface{}) {
 									update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
 								} else if value == 8 {
 									update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
-								} else {
+								} else if value == 9 {
+									update_map["$set"].(map[string]interface{})["contractnumber"] = newData.contractnumber
+								}else {
 
 								}
 							}
@@ -707,6 +711,21 @@ func mergeDataFields(source *Info, info *Info) (*Info, []int64) {
 		mergeArr = append(mergeArr, 8)
 	}
 
+	//9、合同编号
+	if source.contractnumber == "" && info.contractnumber != "" {
+		var arr []string
+		if source.mergemap["contractnumber"] == nil {
+			arr = make([]string, 0)
+		} else {
+			arr = source.mergemap["contractnumber"].([]string)
+		}
+		arr = append(arr, source.contractnumber)
+		source.mergemap["contractnumber"] = arr
+
+		source.contractnumber = info.contractnumber
+		mergeArr = append(mergeArr, 9)
+	}
+
 	//以上合并过于简单，待进一步优化
 	return source, mergeArr
 }
@@ -812,7 +831,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if v.buyer != "" {
 		m++
 	}
-	if v.projectcode != "" {
+	if v.projectcode != ""||v.contractnumber != "" {
 		m++
 	}
 	if v.budget != 0 {
@@ -843,7 +862,7 @@ func basicDataScore(v *Info, info *Info) bool {
 	if info.buyer != "" {
 		n++
 	}
-	if info.projectcode != "" {
+	if info.projectcode != "" || info.contractnumber != ""{
 		n++
 	}
 	if info.budget != 0 {
@@ -882,7 +901,7 @@ func basicDataScore(v *Info, info *Info) bool {
 }
 
 //无效数据
-func invalidData(d1 string, d2 string, d3 string) bool {
+func invalidData(d1 string, d2 string, d3 string, d4 string) bool {
 	var n int
 	if d1 != "" {
 		n++
@@ -893,7 +912,10 @@ func invalidData(d1 string, d2 string, d3 string) bool {
 	if d3 != "" {
 		n++
 	}
-	if n == 0 {
+	if d4 != "" {
+		n++
+	}
+ 	if n == 0 {
 		return true
 	}
 	return false

versioncomparison/config.json

@@ -1,23 +1,27 @@
 {
-    "premgo": "192.168.3.207:27082",
+    "premgo": "192.168.3.207:27092",
     "predb": "extract_kf",
-    "prec": "result_v3",
-    "newmgo": "192.168.3.207:27082",
+    "prec": "datainfo_dev3.2",
+    "newmgo": "192.168.3.207:27092",
     "newdb": "extract_kf",
-    "newc": "result_data",
+    "newc": "datainfo_dev3.4",
     "fields": [
         "projectname",
         "projectcode",
+        "contractcode",
         "buyer",
         "bidamount",
         "budget",
         "winner",
+        "winnertel",
+        "winnerperson",
         "agency",
         "buyerperson",
         "buyertel",
         "buyeraddr",
         "agencyperson",
         "agencytel",
-        "agencyaddr"
+        "agencyaddr",
+        "package"
     ]
 }

versioncomparison/main.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"jy/mongodbutil"
 	"log"
-	"qfw/common/src/qfw/util"
 	qu "qfw/util"
 	"strings"
 
@@ -41,8 +40,8 @@ type Data struct {
 }
 
 func init() {
-	flag.StringVar(&Sid, "sid", "5cc87219a5cb26b9b719a6be", "开始id")
-	flag.StringVar(&Eid, "eid", "5d0c876aa5cb26b9b7078df8", "结束id")
+	flag.StringVar(&Sid, "sid", "5e17deb150b5ea296ec939d3", "开始id")
+	flag.StringVar(&Eid, "eid", "5e17e1e685a9271abf08616d", "结束id")
 	flag.Parse()
 	qu.ReadConfig(&SysConfig)
 	Premgo = mongodbutil.MgoFactory(1, 3, 120, qu.ObjToString(SysConfig["premgo"]), qu.ObjToString(SysConfig["predb"]))
@@ -83,12 +82,12 @@ func createXlsx() {
 				(*row).Cells[2].SetStyle(style)
 				(*row).Cells[3].SetStyle(style)
 				(*row).Cells[4].SetStyle(style)
-				if Compares[field] == nil{
+				if Compares[field] == nil {
 					(*row).Cells[1].SetInt(0)
 					(*row).Cells[2].SetInt(0)
 					(*row).Cells[3].SetInt(0)
 					(*row).Cells[4].SetInt(0)
-				}else {
+				} else {
 					(*row).Cells[1].SetInt(Compares[field].PreExtNum)
 					(*row).Cells[2].SetInt(Compares[field].NewExtNum)
 					(*row).Cells[3].SetInt(Compares[field].EqNum)
@@ -135,7 +134,7 @@ func getVersionData() {
 			if rd == nil {
 				rd = map[string]*Data{}
 			}
-			if v[key] == nil && strings.TrimSpace(util.ObjToString(v[key])) == ""{
+			if v[key] == nil && strings.TrimSpace(qu.ObjToString(v[key])) == "" {
 				continue
 			}
 			rd[qu.BsonIdToSId(v["_id"])] = &Data{
@@ -154,7 +153,7 @@ func getVersionData() {
 			if rd == nil {
 				rd = map[string]*Data{}
 			}
-			if v[field] == nil && strings.TrimSpace(util.ObjToString(v[field])) == ""{
+			if v[field] == nil && strings.TrimSpace(qu.ObjToString(v[field])) == "" {
 				continue
 			}
 			_id := qu.BsonIdToSId(v["_id"])
@@ -164,6 +163,7 @@ func getVersionData() {
 				rd[_id] = tmp
 			} else {
 				rd[_id] = &Data{
+					Id:     qu.BsonIdToSId(_id),
 					NewVal: fmt.Sprint(v[field]),
 				}
 			}