Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

udpcreateindex/src/biddingall.go

@@ -100,11 +100,11 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 				tid := qutil.BsonIdToSId(tmp["_id"])
 				if cid == tid {
 					bnil = false
-					//更新bidding表，生成索引;bidding表中modifyinfo中的字段不更新
+					//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
 					modifyinfo := make(map[string]bool)
 					if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
-						for k, v := range tmpmodifyinfo {
-							modifyinfo[k] = v.(bool)
+						for k, _ := range tmpmodifyinfo {
+							modifyinfo[k] = true
 						}
 					}
 					for _, k := range fields { //fields更新到mongo的字段

udpcreateindex/src/biddingindex.go

@@ -158,13 +158,20 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 				continue
 			}
 			delete(eMap, tid)
+			//更新bidding表，生成索引;bidding表modifyinfo中的字段不更新
+			modifyinfo := make(map[string]bool)
+			if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
+				for k, _ := range tmpmodifyinfo {
+					modifyinfo[k] = true
+				}
+			}
 			//更新bidding表，生成索引
 			for _, k := range fields {
 				v1 := compare[k] //extract
 				v2 := tmp[k]     //bidding
-				if v2 == nil && v1 != nil {
+				if v2 == nil && v1 != nil && !modifyinfo[k] {
 					update[k] = v1
-				} else if v2 != nil && v1 != nil {
+				} else if v2 != nil && v1 != nil && !modifyinfo[k] {
 					//update[k+"_b"] = v2
 					update[k] = v1
 				} else if v2 != nil && v1 == nil {

udpcreateindex/src/config.json

@@ -30,7 +30,7 @@
     },
     "bidding": {
         "db": "mxs",
-        "collect": "test1",
+        "collect": "test",
         "index": "bidding_v1",
         "type": "bidding",
         "extractdb": "mxs",

udpfilterdup/src/config.json

@@ -5,8 +5,8 @@
         "addr": "192.168.3.207:27092",
         "pool": 5,
         "db": "extract_kf",
-        "extract": "zk_test",
-        "extract_back": "zk_test",
+        "extract": "zk_zk_test",
+        "extract_back": "zk_zk_test",
         "site": {
             "dbname": "zhaolongyue",
             "coll": "site"

udpfilterdup/src/dataMethod.go

@@ -161,10 +161,9 @@ func leadingElementSame(v *Info, info *Info) bool {
 	if info.title != "" && v.title == info.title {
 		isok++
 	}
-	if v.agency == info.agency {
+	if v.agency == info.agency &&info.agency != "" {
 		isok++
 	}
-
 	if v.winner == info.winner&&info.winner != "" {
 		isok++
 	}

udpfilterdup/src/dataMethodHeavy.go

@@ -240,18 +240,9 @@ func tenderRepeat_C(v *Info, info *Info) bool {
 	if v.budget != 0 && info.budget != 0 && v.budget != info.budget {
 		return true
 	}
-	//原始地址...
-	//if v.buyer != "" && info.buyer != "" && v.buyer != info.buyer {
-	//	return true
-	//}
-
 	if v.bidopentime != 0 && info.bidopentime != 0 && isBidopentimeInterval(info.bidopentime,v.bidopentime) {
 		return true
 	}
-	//if v.bidopenaddress != "" && info.bidopenaddress != "" && v.bidopenaddress != info.bidopenaddress {
-	//	return true
-	//}
-
 	return false
 }
 
@@ -344,7 +335,7 @@ func winningRepeat_B(v *Info, info *Info, reason string) (bool, string) {
 func winningRepeat_C(v *Info, info *Info) bool {
 
 	if v.bidamount != 0 && info.bidamount != 0 && isBidWinningAmount(v.bidamount,info.bidamount) {
-		//避免抽错金额- title+name+winner
+		//避免抽错金额-
 		if ((v.projectcode!=""&&info.projectcode!=""&&v.projectcode==info.projectcode)||
 			(v.contractnumber!=""&&info.contractnumber!=""&&v.contractnumber==info.contractnumber)) &&
 			(v.winner!=""&&info.winner!=""&&v.winner==info.winner) {

udpfilterdup/src/main.go

@@ -128,8 +128,8 @@ func mainT() {
 		time.Sleep(99999 * time.Hour)
 	} else {
 		//IdType = true  //打开id字符串模式
-		sid = "5da3f2c5a5cb26b9b79847f0"
-		eid = "5da40bdaa5cb26b9b7bea480"
+		sid = "5da3f3a2a5cb26b9b799aa65"
+		eid = "5da40b26a5cb26b9b7bde10b"
 		log.Println("正常判重测试开始")
 		log.Println(sid, "---", eid)
 		mapinfo := map[string]interface{}{}
@@ -295,9 +295,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 					newData, update_map ,isReplace := mergeDataFields(source, info)
 					if isReplace {//替换-数据池
 						fmt.Println("替换更新：",source.id)
-						//数据池
-						DM.replacePoolData(newData) //替换
-						//mongo更新 - 具体字段
+						//数据池 - 替换
+						DM.replacePoolData(newData)
+						//mongo更新 - 具体字段 - merge
 						mgo.UpdateById(extract,source.id,update_map)
 						//发udp  ids：更新索引
 						if len(ids)>=9 {

udpfilterdup/src1/main.go

@@ -105,7 +105,7 @@ func init() {
 }
 
 
-func main() {
+func mainT() {
 
 	go checkMapJob()
 	updport := Sysconfig["udpport"].(string)
@@ -121,7 +121,8 @@ func main() {
 
 
 //测试组人员使用
-func mainT() {
+func main() {
+
 	if TimingTask {
 		log.Println("定时任务测试开始")
 		go timedTaskDay()
@@ -212,7 +213,9 @@ func task(data []byte, mapInfo map[string]interface{}) {
 			log.Println("current:", n, tmp["_id"], "repeateN:", repeateN)
 		}
 		source := util.ObjToMap(tmp["jsondata"]) //前置-jsondata判重
-		if util.IntAll((*source)["sourcewebsite"]) == 1 {
+
+
+		if util.IntAll((*source)["sourcewebsite"]) == 1 && tmp["attach_text"]!=nil{
 			repeateN++
 			updateExtract = append(updateExtract, []map[string]interface{}{
 				map[string]interface{}{