cm

data_fusion/common.toml

@@ -1,7 +1,7 @@
 [serve]
 thread = 10
 fields = ["agency", "agencytel", "agencyperson", "bidendtime", "budget", "bidamount", "buyer", "buyerclass", "buyerperson", "buyertel", "buyeraddr", "bidway",
-    "district", "projectcode", "projectscope", "s_winner", "winnertel", "winerperson", "publishtime", "signaturedate", "docstarttime", "attachments",
+    "area", "city", "district", "projectcode", "projectscope", "s_winner", "winnertel", "winerperson", "publishtime", "signaturedate", "docstarttime", "attachments",
     "project_timeunit", "project_completedate", "project_duration", "project_startdate", "winnerorder", "purchasinglist", "qualifies", "contractcode", "bidopentime",
     "topscopeclass", "subscopeclass", "toptype", "subtype"]
 
@@ -31,15 +31,15 @@ winnerorder = 1
 
 [db]
 [db.mongo]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 coll = "bidding"
-record = ""
+record = "bidding_record"
 size = 15
 user = ""
 password = ""
 [db.mongoS]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 coll = "spider_compete"
 size = 15

data_fusion/data_fusion

@@ -0,0 +1 @@
+ELF

data_fusion/go.mod

@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.1
-	go.mongodb.org/mongo-driver v1.12.0 // indirect
+	go.mongodb.org/mongo-driver v1.12.0
 	go.uber.org/zap v1.24.0
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
 )

data_fusion/go.sum

@@ -86,15 +86,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
 github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
-github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
@@ -102,7 +99,6 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
 go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
 go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
 go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
@@ -142,7 +138,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -150,7 +145,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -171,7 +165,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=

data_fusion/main.go

@@ -331,6 +331,20 @@ func mergeTmp(tmp map[string]interface{}, tmp1 map[string]interface{}) (map[stri
 					}
 				}
 			}
+		} else if v == "city" {
+			if util.ObjToString(tmp["area"]) == util.ObjToString(tmp1["area"]) {
+				if tmp[v] == nil && tmp1[v] != nil {
+					update[v] = util.ObjToString(tmp1[v])
+					fs = append(fs, v)
+				}
+			}
+		} else if v == "district" {
+			if util.ObjToString(tmp["city"]) == util.ObjToString(tmp1["city"]) {
+				if tmp[v] == nil && tmp1[v] != nil {
+					update[v] = util.ObjToString(tmp1[v])
+					fs = append(fs, v)
+				}
+			}
 		} else {
 			if tmp[v] == nil && tmp1[v] != nil {
 				if reflect.TypeOf(tmp1[v]).String() == "string" && util.ObjToString(tmp1[v]) != "" {

data_project/common.toml

@@ -11,7 +11,7 @@ siteColl = "site"
 [db.mongoB]
 addr = "192.168.3.71:29099"
 dbname = "wjh"
-coll = "wjh_bidding_0614_1"
+coll = "bidding"
 size = 15
 user = ""
 password = ""
@@ -39,6 +39,8 @@ addr = "http://192.168.3.241:9205"
 index = "projectset"
 itype = "projectset"
 pool = 10
+user = "es_all"
+password = "TopJkO2E_d1x"
 
 [nsq]
 addr = "192.168.3.166:4150"

data_project/main.go

@@ -86,8 +86,8 @@ func main() {
 	if config.Conf.Serve.LoadStart > 0 {
 		P_QL.loadData(config.Conf.Serve.LoadStart)
 	}
-	go checkMapJob()
-	go P_QL.nsqMethod()
+	//go checkMapJob()
+	//go P_QL.nsqMethod()
 
 	for {
 		mapinfo, ok := <-UdpChan
@@ -168,8 +168,8 @@ func main() {
 }
 
 func mainT() {
-	sid = "56614b76af5374118a002100"
-	eid = "64884e4d8aea8786d11c99f9"
+	sid = "5c24a939a5cb26b9b75446e6"
+	eid = "5c24a939a5cb26b9b75446e7"
 	//flag.StringVar(&sid, "sid", "", "开始id")
 	//flag.StringVar(&eid, "eid", "", "结束id")
 	//flag.Parse()

data_project/project.go

@@ -19,7 +19,7 @@ import (
 项目合并，对比，计算，合并，生成项目
 **/
 
-//从对应map中获取对比的项目id
+// 从对应map中获取对比的项目id
 func (p *ProjectTask) getCompareIds(pn, pc, ptc, pb string) (bpn, bpc, bptc, bpb int, res []*Key, idArr []string, IDArr []*ID) {
 	//	p.ConCurrentLock(n1, n2, n3, n4)
 	//	defer p.ConCurrentUnLock(n1, n2, n3, n4)
@@ -643,7 +643,7 @@ func (p *ProjectTask) NewProject(tmp map[string]interface{}, thisinfo *Info) (st
 	return pId.Hex(), &p1
 }
 
-//项目中list的信息
+// 项目中list的信息
 func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bson.M {
 	res := bson.M{
 		"infoid": infoid,
@@ -656,7 +656,7 @@ func (p *ProjectTask) PushListInfo(tmp map[string]interface{}, infoid string) bs
 	return res
 }
 
-//生成存放在内存中的对象
+// 生成存放在内存中的对象
 func (p *ProjectTask) NewCachePinfo(id primitive.ObjectID, thisinfo *Info, bidtype, bidstatus string) ProjectCache {
 	p1 := ProjectCache{
 		Id:          id,
@@ -1435,7 +1435,7 @@ func CountAmount(project *Project, info *Info, tmp map[string]interface{}) {
 
 }
 
-//结构体转map
+// 结构体转map
 func StructToMap(filed InfoField) map[string]interface{} {
 	//先转json
 	result, err := json.Marshal(filed)

data_project/project_tool.go

@@ -2,7 +2,7 @@ package main
 
 var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
 
-//项目中的字段
+// 项目中的字段
 var FIELDS = []string{
 	"area",
 	"city",
@@ -17,10 +17,9 @@ var FIELDS = []string{
 	"agency",
 	"topscopeclass",
 	"subscopeclass",
-	"package",
 }
 
-//招标信息字段
+// 招标信息字段
 var INFOFIELDS = []string{
 	"projectname",
 	"projectcode",

data_project/task.go

@@ -283,7 +283,7 @@ func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
 	if util.ObjToString(udpInfo["coll"]) != "" {
 		BiddingColl = "bidding_back"
 	} else {
-		BiddingColl = "wjh_bidding_0614_1"
+		BiddingColl = config.Conf.DB.MongoB.Coll
 	}
 	//生成查询语句执行
 	p.enter(MgoB.DbName, BiddingColl, q)
@@ -493,7 +493,7 @@ func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
 		c, _ := sess.DB(db).C(coll).Find(q).Count()
 		log.Info(fmt.Sprintf("共查询: %d条", c))
 	}
-	ms := sess.DB(db).C(coll).Find(q).Select(fields)
+	ms := sess.DB(db).C(coll).Find(q).Select(fields).Sort("publishtime")
 	query := ms.Iter()
 	var lastid interface{}
 L:

field_py/common.toml

@@ -9,6 +9,7 @@ thread = 50
 addr = "192.168.3.207:27092"
 dbname = "wjh"
 size = 5
+coll = ""
 user = ""
 password = ""
 

field_py/config/conf.go

@@ -72,6 +72,7 @@ type db struct {
 type mgo struct {
 	Addr     string
 	Dbname   string
+	Coll     string
 	Size     int
 	User     string
 	Password string

field_py/go.mod

@@ -4,10 +4,10 @@ go 1.16
 
 require (
 	github.com/BurntSushi/toml v1.2.0
-	go.mongodb.org/mongo-driver v1.11.7
+	go.mongodb.org/mongo-driver v1.11.7 // indirect
 	go.uber.org/zap v1.22.0
 	google.golang.org/grpc v1.56.0
 	google.golang.org/protobuf v1.30.0
 	jygit.jydev.jianyu360.cn/BP/servicerd v0.0.0-20230621085048-b7e09df0c982
-	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3
 )

field_py/go.sum

@@ -1578,8 +1578,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 jygit.jydev.jianyu360.cn/BP/servicerd v0.0.0-20230621085048-b7e09df0c982 h1:SEAsV0LOUck33TAHfCKHTNlPOTac0vSwLJLtb9pubM4=
 jygit.jydev.jianyu360.cn/BP/servicerd v0.0.0-20230621085048-b7e09df0c982/go.mod h1:vpR5YGF/LLRp5H13fASW7fKx/UExpU7WMbkZ1MmNS8U=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1 h1:MbtiS3BwDnB7uRjHFMCBe7/oW+aLaGMeQNiQa3qvd4c=
-jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230621011928-cdaea81cabe1/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3 h1:kgtSaRR/hRunxM6Kxi66REk7f2PqN1u56j/V+8FfPW8=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20230712115659-b418d6181de3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=

field_py/main.go

@@ -200,7 +200,7 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoB.UpdateBulk("bidding", arru...)
+					MgoB.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0
@@ -212,7 +212,7 @@ func updateMethod() {
 					defer func() {
 						<-updateSp
 					}()
-					MgoB.UpdateBulk("bidding", arru...)
+					MgoB.UpdateBulk(config.Conf.DB.Mongo.Coll, arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 500)
 				indexu = 0

field_py/task.go

@@ -8,7 +8,6 @@ import (
 	"field-dispose/config"
 	"field-dispose/proto"
 	"fmt"
-	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
@@ -26,14 +25,14 @@ import (
 
 var IpDialErrNum, IpGetErrNum, ExtractDialErrNum = int64(0), int64(0), int64(0)
 
-//@Description 处理字段procurementlist、review_experts
+// @Description 处理字段procurementlist、review_experts
 // @Author J 2022/8/31 14:57
 func getIntention(mapinfo map[string]interface{}) {
 	defer util.Catch()
 	gtid, _ := mapinfo["gtid"].(string)
 	lteid, _ := mapinfo["lteid"].(string)
 
-	MgoB.Update("bidding_processing_ids", bson.M{"gtid": gtid}, bson.M{"$set": bson.M{"dataprocess": 2, "updatetime": time.Now().Unix()}}, false, false)
+	//MgoB.Update("bidding_processing_ids", bson.M{"gtid": gtid}, bson.M{"$set": bson.M{"dataprocess": 2, "updatetime": time.Now().Unix()}}, false, false)
 
 	sess := MgoB.GetMgoConn()
 	defer MgoB.DestoryMongoConn(sess)
@@ -53,7 +52,7 @@ func getIntention(mapinfo map[string]interface{}) {
 		"detail":      1,
 	}
 	log.Info(fmt.Sprintf("count --- %d", MgoB.Count("bidding", query)))
-	it := sess.DB(config.Conf.DB.Mongo.Dbname).C("bidding").Find(&query).Select(&field).Iter()
+	it := sess.DB(config.Conf.DB.Mongo.Dbname).C(config.Conf.DB.Mongo.Coll).Find(&query).Select(&field).Iter()
 	count := 0
 	for tmp := make(map[string]interface{}); it.Next(tmp); count++ {
 		if count%200 == 0 {
@@ -78,6 +77,7 @@ func getIntention(mapinfo map[string]interface{}) {
 			if result1 != nil && len(result1) > 0 {
 				if result1["purchasinglist"] != nil && len(result1["purchasinglist"].([]interface{})) > 0 {
 					update["purchasinglist"] = result1["purchasinglist"]
+					update["purchasing"] = result1["purchasing"]
 				}
 				if result1["procurementlist"] != nil {
 					update["procurementlist"] = result1["procurementlist"]
@@ -339,7 +339,7 @@ func rpcGetFieldR(reqStr, id string) (map[string]interface{}, error) {
 	return result, nil
 }
 
-//定时检测异常数据量，发告警
+// 定时检测异常数据量，发告警
 func CheckErrorNum() {
 	defer util.Catch()
 	for {
@@ -371,7 +371,7 @@ var TextModel = `{
     }
 }`
 
-//告警信息
+// 告警信息
 func SendWarnInfo(content string) {
 	defer util.Catch()
 	toUserMsg := fmt.Sprintf(TextModel, content, "13373929153,15090279371") //李俊亮、王江含
@@ -391,7 +391,7 @@ func SendWarnInfo(content string) {
 var SP = make(chan bool, 5)
 var ErrorInfoCache = make(chan map[string]interface{}, 1000) //异常信息集合
 
-//批量保存异常信息
+// 批量保存异常信息
 func SaveErrorInfo() {
 	log.Info("SaveErrorInfo 异常信息...")
 	savearr := make([]map[string]interface{}, 200)

field_sync/common.toml

@@ -9,7 +9,7 @@ fields = ["buyerzipcode", "winnertel", "winnerperson", "contractcode", "winnerad
     "funds", "review_experts", "bidmethod", "bidendtime", "bidopenaddress", "docamount", "bidway", "agencyrate", "agencyfee", "getdocmethod", "purchasing_tag",
     "package", "history_updatetime", "total_investment", "owner", "projecttype", "project_person", "project_phone", "approvedept", "construction_area",
     "floor_area", "bidstarttime", "docendtime", "docstarttime", "signendtime", "signstarttime", "issue_quota", "bidopen_shape", "quote_mode", "is_acquire_tender",
-    "is_payment_deposit", "is_joint_bidding", "est_purchase_time", "est_purchase_amount"
+    "is_payment_deposit", "is_joint_bidding", "est_purchase_time", "est_purchase_amount", "s_project_startdate", "s_project_completedate"
 ]
 
 [udp]
@@ -45,14 +45,14 @@ concurrent = 1
 [db]
 
 [db.mongoB]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 coll = "bidding"
 size = 5
 user = ""
 password = ""
 [db.mongoE]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "wjh"
 size = 5
 coll = "extract"
@@ -60,14 +60,14 @@ coll1 = ""
 user = ""
 password = ""
 [db.mongoQ]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "mixdata"
 size = 5
 coll = "qyxy_std"
 user = ""
 password = ""
 [db.mongoP]
-addr = "192.168.3.207:27092"
+addr = "192.168.3.71:29099"
 dbname = "mixdata"
 size = 5
 coll = "company_history_name"

field_sync/task.go

@@ -165,7 +165,7 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 								continue
 							}
 							del[k] = 1
-							//util.Debug("抽取结果没有值,bidding有值：field--", k, "val--", v2)
+							//util.Debug("抽取结果没有值,bidding有值：field--", k, del)
 						}
 					}
 					if util.IntAll(compare["repeat"]) == 1 {
@@ -256,6 +256,7 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 	for n, tmp := range infos {
 		tid := mongodb.BsonIdToSId(tmp["_id"])
 		update := map[string]interface{}{} //要更新的mongo数据
+		del := map[string]interface{}{}
 		//对比方法----------------
 		if eMap[tid] != nil {
 			compare = eMap[tid]
@@ -286,8 +287,14 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				} else if v2 != nil && v1 != nil {
 					update[k] = v1
 				} else if v2 != nil && v1 == nil {
-					if k == "city" || k == "district" {
+					if k == "s_subscopeclass" && del["subscopeclass"] == nil {
+						continue
+					} else if k == "s_topscopeclass" && del["topscopeclass"] == nil {
+						continue
+					} else if k == "city" || k == "district" {
 						update[k] = ""
+					} else {
+						del[k] = 1
 					}
 				}
 			}
@@ -321,9 +328,9 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		//------------------对比结束
 
 		//处理key descript
-		if bkey == "" {
-			DealInfo(&tmp, &update)
-		}
+		//if bkey == "" {
+		//	DealInfo(&tmp, &update)
+		//}
 		// entidlist
 		extractMap := make(map[string]interface{})
 		if update["s_winner"] != "" {
@@ -366,12 +373,19 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 			}
 		}
 		if len(update) > 0 {
-
-			bidUpdate = append(bidUpdate, []map[string]interface{}{{
-				"_id": tmp["_id"],
-			},
-				{"$set": update},
-			})
+			if len(del) > 0 {
+				bidUpdate = append(bidUpdate, []map[string]interface{}{{
+					"_id": tmp["_id"],
+				},
+					{"$set": update, "$unset": del},
+				})
+			} else {
+				bidUpdate = append(bidUpdate, []map[string]interface{}{{
+					"_id": tmp["_id"],
+				},
+					{"$set": update},
+				})
+			}
 			if len(bidUpdate) >= MgoBulkSize {
 				tmps := bidUpdate
 				MgoB.UpdateBulk(config.Conf.DB.MongoB.Coll, tmps...)