Merge branch 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract into dev3.4

* 'dev3.4' of http://192.168.3.207:10080/qmx/jy-data-extract:
  xg
  优化
  channel
  xg
  优化

src/jy/extract/newextractcity.go

@@ -77,12 +77,14 @@ func (e *ExtractTask) NewExtractCity(j *ju.Job, resulttmp map[string]interface{}
 	projectname, _ := resulttmp["projectname"].(string)
 	buyer, _ := resulttmp["buyer"].(string)
 	projectaddr, _ := resulttmp["projectaddr"].(string)
+	bidopenaddress, _ := resulttmp["bidopenaddress"].(string)
 	//qu.Debug("buyeraddr--", buyeraddr, "--buyer--", buyer, "--title--", title, "--projectname--", projectname)
 	sm.AddKey("buyeraddr", buyeraddr)
 	sm.AddKey("buyer", buyer)
 	sm.AddKey("title", title)
 	sm.AddKey("projectname", projectname)
 	sm.AddKey("projectaddr",projectaddr)
+	sm.AddKey("bidopenaddress",bidopenaddress)
 	//7.buyeraddr buyer title projectname抽取
 	e.NewGetCityByOthers(j, sm, &pscore, &cscore, &dscore)
 	//qu.Debug("全称打分后结果---", j.FullAreaScore, j.FullCityScore, j.FullDistrictScore)

udp_city/src/config.json

@@ -4,8 +4,9 @@
   "dbsize": 3,
   "dbname": "enterprise",
   "mgodb_bidding": "192.168.3.207:27092",
-  "mgodb_bidding_xs": "10.172.242.243:27080",
+  "mgodb_bidding_xs": "172.17.4.187:27083",
   "dbname_bidding": "qfw",
+  "findDb": "result_20200714",
   "udpport": "1485",
   "nextNode": [
     {

udp_city/src/main.go

@@ -18,6 +18,7 @@ var PageSize = 5000 //查询分页
 var biddingFields = `{"buyer":1,"modifyinfo":1}`
 var qyxyFields = `{"company_code":1,"province":1,"city":1,"district":1}`
 var findDb string
+var cc chan bool = make(chan bool, 5)
 
 func init() {
 	qu.ReadConfig(&Config)
@@ -28,6 +29,7 @@ func init() {
 	initCap := qu.IntAll(Config["dbsize"])
 	addr := qu.ObjToString(Config["mgodb"])
 	dbname := qu.ObjToString(Config["dbname"])
+	cc = make(chan bool, 3)
 	mgo.Mgo = mgo.MgoFactory(initCap, initCap*3, 120, addr, dbname)
 	mgo.Mgo_Bidding = mgo.MgoFactory(initCap, initCap*3, 120, qu.ObjToString(Config["mgodb_bidding"]), qu.ObjToString(Config["dbname_bidding"]))
 	nextNodes = qu.ObjArrToMapArr(Config["nextNode"].([]interface{}))
@@ -78,7 +80,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 
 func getCity(sid, eid, rep string) {
 	index := 0
-	unum := 0
+	var unum int64
 	query := bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
 	count := mgo.Mgo_Bidding.Count(findDb, query)
 	log.Println("查询条件为:", query, "查询条数:", count)
@@ -92,51 +94,51 @@ func getCity(sid, eid, rep string) {
 		query = bson.M{"_id": bson.M{"$gte": bson.ObjectIdHex(sid), "$lte": bson.ObjectIdHex(eid)}}
 		log.Printf("page=%d,query=%v,db=%v\n", i+1, query, table)
 		list, _ := mgo.Mgo_Bidding.Find(table, query, nil, biddingFields, false, 0, limit)
-		processingCity(sid, eid, list, index, table, i,unum)
-	}
-}
+		for _, v := range *list {
+			if qu.ObjToString(v["district"]) != "" && qu.ObjToString(v["city"]) != "" && qu.ObjToString(v["area"]) != "" {
+				index++
+				continue
+			}
+			if qu.ObjToString(v["buyer"]) == "" {
+				index++
+				continue
+			}
 
-func processingCity(sid string, eid string, list *[]map[string]interface{}, index int, table string, i,unum int) {
-	for _, v := range *list {
-		if qu.ObjToString(v["district"]) != "" && qu.ObjToString(v["city"]) != "" && qu.ObjToString(v["area"]) != "" {
-			index++
-			continue
-		}
-		if qu.ObjToString(v["buyer"]) == "" {
+			_id := qu.BsonIdToSId(v["_id"])
+			cc <- true
+			go func(v map[string]interface{}) {
+				rdata := cityMarshal(v)
+				if len(rdata) > 0 {
+					umap := make(map[string]interface{})
+					if v["modifyinfo"] == nil {
+						umap["modifyinfo"] = make(map[string]interface{})
+					} else {
+						umap["modifyinfo"] = v["modifyinfo"]
+					}
+					for rk, rv := range rdata {
+						umap[rk] = rv
+						umap["modifyinfo"].(map[string]interface{})[rk] = "企业信息"
+					}
+					unum++
+					log.Println(unum, ",id：", _id, umap)
+					mgo.Mgo_Bidding.UpdateById(table, v["_id"], map[string]interface{}{
+						"$set": umap,
+					})
+				}
+				<-cc
+			}(v)
 			index++
-			continue
-		}
-
-		_id := qu.BsonIdToSId(v["_id"])
-		rdata := cityMarshal(v)
-		if len(rdata) > 0 {
-			umap := make(map[string]interface{})
-			if v["modifyinfo"] == nil {
-				umap["modifyinfo"] = make(map[string]interface{})
-			} else {
-				umap["modifyinfo"] = v["modifyinfo"]
+			if index%1000 == 0 {
+				log.Println("index：", index, ",页码：", i+1, ",_id：", _id)
 			}
-			for rk, rv := range rdata {
-				umap[rk] = rv
-				umap["modifyinfo"].(map[string]interface{})[rk] = "企业信息"
+			sid = _id
+			if sid >= eid {
+				break
 			}
-			unum++
-			log.Println(unum,",id：",_id)
-			mgo.Mgo_Bidding.UpdateById(table, v["_id"], map[string]interface{}{
-				"$set": umap,
-			})
-
-		}
-		index++
-		if index%1000 == 0 {
-			log.Println("index：", index, ",页码：", i+1, ",_id：", _id)
-		}
-		sid = _id
-		if sid >= eid {
-			break
 		}
 	}
 }
+
 func cityMarshal(data map[string]interface{}) map[string]string {
 	rdata := make(map[string]string)
 	buyer := qu.ObjToString(data["buyer"])
@@ -148,29 +150,57 @@ func cityMarshal(data map[string]interface{}) map[string]string {
 	if len(company_code) > 5 {
 		province_city_district, _ := mgo.Mgo.FindOne("address", `{"code":"`+company_code[:6]+`"}`)
 		if province_city_district != nil && (*province_city_district) != nil {
-			if province := qu.ObjToString((*province_city_district)["province"]); province != "" {
-				rdata["area"] = province
-			}
-			if city := qu.ObjToString((*province_city_district)["city"]); city != "" && !strings.Contains(city,rdata["area"]) {
-				rdata["city"] = city
-			}
-			if district := qu.ObjToString((*province_city_district)["district"]); district != "" && !strings.Contains(district,rdata["area"]){
-				rdata["district"] = district
+			if qu.ObjToString(data["area"]) == "" {
+				if province := qu.ObjToString((*province_city_district)["province"]); province != "" {
+					rdata["area"] = province
+				}
+				if city := qu.ObjToString((*province_city_district)["city"]); city != "" && !strings.Contains(city, rdata["area"]) {
+					rdata["city"] = city
+				}
+				if district := qu.ObjToString((*province_city_district)["district"]); district != "" && !strings.Contains(district, rdata["area"]) {
+					rdata["district"] = district
+				}
+			} else if qu.ObjToString(data["city"]) == "" && qu.ObjToString((*province_city_district)["province"]) != "" && qu.ObjToString((*province_city_district)["province"]) == qu.ObjToString(data["area"]) {
+				if city := qu.ObjToString((*province_city_district)["city"]); city != "" && !strings.Contains(city, rdata["area"]) {
+					rdata["city"] = city
+				}
+				if district := qu.ObjToString((*province_city_district)["district"]); district != "" && !strings.Contains(district, rdata["area"]) {
+					rdata["district"] = district
+				}
+			} else if qu.ObjToString(data["district"]) == "" && qu.ObjToString((*province_city_district)["city"]) != "" && qu.ObjToString((*province_city_district)["city"]) == qu.ObjToString(data["city"]) {
+				if district := qu.ObjToString((*province_city_district)["district"]); district != "" && !strings.Contains(district, rdata["area"]) {
+					rdata["district"] = district
+				}
 			}
 			return rdata
 		}
 	}
-	if province := qu.ObjToString((*tmp)["province"]); province != "" {
-		province = strings.TrimRight(province,"省")
-		province = strings.TrimRight(province,"市")
-		rdata["area"] = province
-	}
-	if city := qu.ObjToString((*tmp)["city"]); city != "" {
-		rdata["city"] = city
-	}
-	if district := qu.ObjToString((*tmp)["district"]); district != "" {
-		rdata["district"] = district
+	var province string
+	if qu.ObjToString(data["area"]) == "" {
+		if province = qu.ObjToString((*tmp)["province"]); province != "" {
+			province = strings.TrimRight(province, "省")
+			province = strings.TrimRight(province, "市")
+			rdata["area"] = province
+		}
+		if city := qu.ObjToString((*tmp)["city"]); city != "" {
+			rdata["city"] = city
+		}
+		if district := qu.ObjToString((*tmp)["district"]); district != "" {
+			rdata["district"] = district
+		}
+	} else if qu.ObjToString(data["city"]) == "" && province != "" && qu.ObjToString(data["area"]) == province {
+		if city := qu.ObjToString((*tmp)["city"]); city != "" {
+			rdata["city"] = city
+		}
+		if district := qu.ObjToString((*tmp)["district"]); district != "" {
+			rdata["district"] = district
+		}
+	} else if qu.ObjToString(data["district"]) == "" && qu.ObjToString((*tmp)["city"]) != "" && qu.ObjToString((*tmp)["city"]) == qu.ObjToString(data["city"]) {
+		if district := qu.ObjToString((*tmp)["district"]); district != "" {
+			rdata["district"] = district
+		}
 	}
+
 	return rdata
 }
 func main() {

udpcreateindex/src/biddingall.go

@@ -245,6 +245,10 @@ func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
 			if district, ok := tmp["district"].(string); ok && district == "" { //city为空
 				delete(tmp, "district")
 			}
+			//栏目channel
+			if channel, ok := tmp["channel"].(string); ok && channel == "" { //channel为空
+				delete(tmp, "channel")
+			}
 			//预算和中标金额
 			//			if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
 			//				tmp["budget"] = nil

udpcreateindex/src/biddingindex.go

@@ -287,6 +287,10 @@ func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interfac
 		if district, ok := tmp["district"].(string); ok && district == "" { //city为空
 			delete(tmp, "district")
 		}
+		//栏目channel
+		if channel, ok := tmp["channel"].(string); ok && channel == "" { //channel为空
+			delete(tmp, "channel")
+		}
 		UpdatesLock.Lock()
 		//		for k1, _ := range tmp {
 		//			if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {

udpcreateindex/src/util/ossclient.go

@@ -12,8 +12,8 @@ import (
 
 var (
 	ossEndpoint        = "http://oss-cn-beijing-internal.aliyuncs.com" //正式环境用：oss-cn-beijing-internal.aliyuncs.com 测试：oss-cn-beijing.aliyuncs.com
-	ossAccessKeyId     = "LTAI4FvLSWN3Wz9F6dUxQGMR"
-	ossAccessKeySecret = "WnQpnNVEiRfZsz5hIqFSr0phayMo3U"
+	ossAccessKeyId     = "LTAI4G5x9aoZx8dDamQ7vfZi"
+	ossAccessKeySecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
 	ossBucketName      = "topjy"
 	ossclient          *oss.Client
 )