阳光采购优化

field_sync/main.go

@@ -15,7 +15,6 @@ import (
 	"log"
 
 	elastic "app.yhyue.com/moapp/jybase/es"
-	"go.uber.org/zap"
 	"gopkg.in/mgo.v2/bson"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
@@ -335,7 +334,7 @@ func checkMapJob() {
 							log.Println("send mail ... r:", string(read), "err:", err)
 						}
 					} else {
-						log.Println("udp重发", zap.Any("k:", k))
+						log.Println("udp重发", k)
 						//UdpClient.WriteUdp(node.data, udp.OP_TYPE_DATA, node.addr)
 					}
 				} else if now-node.timestamp > 10 {

field_sync/task.go

@@ -373,23 +373,37 @@ func doIndex(infos []map[string]interface{}, bkey, stype string) int {
 		spidercode := util.ObjToString(tmp["spidercode"])
 		site := util.ObjToString(tmp["site"])
 		infoAttribute := util.ObjToString(tmp["infoattribute"])
-		if infoAttribute != "" {
+		if infoAttribute == "zc_cgxx" {
 			update["public_type"] = "用户发布"
 			update["extracttype"] = -1
+			update["dataprocess"] = 7
 			MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "user"})
+			log.Println("阳光采购用户发布", tid)
 		} else {
-			for _, v := range ygsiteArr {
-				spidercodes := util.ObjToString(v["spidercode"])
-				sites := util.ObjToString(v["site"])
-				datatype := util.ObjToString(v["datatype"])
-				if datatype == "1" && spidercodes == spidercode {
-					update["infoattribute"] = "zc_cgxx"
-					update["public_type"] = "平台发布"
-					MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "spidercode"})
-				} else if datatype == "2" && site == sites {
-					update["infoattribute"] = "zc_cgxx"
-					update["public_type"] = "平台发布"
-					MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "site"})
+			if util.IntAll(compare["repeat"]) != 1 {
+				for _, v := range ygsiteArr {
+					spidercodes := util.ObjToString(v["spidercode"])
+					sites := util.ObjToString(v["site"])
+					datatype := util.ObjToString(v["datatype"])
+					if datatype == "1" && spidercodes == spidercode {
+						update["infoattribute"] = "zc_cgxx"
+						update["public_type"] = "平台发布"
+						MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "spidercode"})
+						log.Println("阳光采购平台发布爬虫", tid)
+					} else if datatype == "2" && site == sites {
+						update["infoattribute"] = "zc_cgxx"
+						update["public_type"] = "平台发布"
+						MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "site"})
+						log.Println("阳光采购平台发布站点", tid)
+					} else if datatype == "3" && spidercodes == spidercode {
+						update["infoattribute"] = "zc_cgxx"
+						update["public_type"] = "平台发布"
+						update["is_yg_new"] = 1
+						update["extracttype"] = -1
+						update["dataprocess"] = 7
+						MgoE.Save("bidding_yg", map[string]interface{}{"id": tid, "source": "is_yg_new"})
+						log.Println("阳光采购平台新采集爬虫", tid)
+					}
 				}
 			}
 		}