package main import ( "go.mongodb.org/mongo-driver/bson" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "log" "strings" "sync" "time" "unicode/utf8" ) // dealXmData 处理中标企业注册地为厦门的数据;陈伟铭 需求 /** 中标方企业注册地为厦门的企业数量一共是多少,可以分别看下22年度,23年度的两个数据总量 */ func dealXmData() { Mgo := &mongodb.MongodbSim{ MongodbAddr: "172.17.189.140:27080", //MongodbAddr: "127.0.0.1:27083", DbName: "qfw", Size: 10, UserName: "SJZY_RWbid_ES", Password: "SJZY@B4i4D5e6S", //Direct: true, } Mgo.InitPool() MgoC := &mongodb.MongodbSim{ MongodbAddr: "172.17.189.140:27080", //MongodbAddr: "127.0.0.1:27083", DbName: "mixdata", Size: 10, UserName: "SJZY_RWbid_ES", Password: "SJZY@B4i4D5e6S", //Direct: true, } MgoC.InitPool() defer util.Catch() sess := Mgo.GetMgoConn() defer Mgo.DestoryMongoConn(sess) pool := make(chan bool, 10) //处理协程 wg := &sync.WaitGroup{} // 设置查询条件 filter := bson.D{ {"publishtime", bson.M{"$gte": 1640966400, "$lt": 1704038400}}, {"subtype", bson.M{"$in": []string{"中标", "单一", "成交", "合同"}}}, } selected := map[string]interface{}{ "contenthtml": 0, // 0表示不返回该字段 "attach_text": 0, // 0表示不返回该字段 "detail": 0, // 0表示不返回该字段 "purchasingsource": 0, // 0表示不返回该字段 "jsondata": 0, // 0表示不返回该字段 "package": 0, // 0表示不返回该字段 } it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Iter() count := 0 for tmp := make(map[string]interface{}); it.Next(&tmp); count++ { if count%10000 == 0 { log.Println("current :", count) } pool <- true wg.Add(1) go func(tmp map[string]interface{}) { defer func() { <-pool wg.Done() }() if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词,不生索引 tmp = make(map[string]interface{}) return } // 针对存量数据,重复数据不进索引 if util.IntAll(tmp["extracttype"]) == -1 { return } swinner := util.ObjToString(tmp["s_winner"]) if swinner == "" { return } if utf8.RuneCountInString(swinner) < 4 { return } publishtime := util.Int64All(tmp["publishtime"]) projectName := util.ObjToString(tmp["projectname"]) if strings.Contains(swinner, ",") { winners := strings.Split(swinner, ",") for _, v := range winners { if utf8.RuneCountInString(v) < 4 { continue } else { da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v}) if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" { continue } city := util.ObjToString((*da)["company_city"]) if city != "厦门市" { continue } insert := map[string]interface{}{ "winner": v, "credit_no": (*da)["credit_no"], "bidding_id": mongodb.BsonIdToSId(tmp["_id"]), "projectname": projectName, "company_type": (*da)["company_type"], "company_status": (*da)["company_status"], "company_area": (*da)["company_area"], "company_city": (*da)["company_city"], "year": time.Unix(publishtime, 0).Year(), } Mgo.Save("wcc_xiamen_winner", insert) } } } else { da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner}) if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" { return } city := util.ObjToString((*da)["company_city"]) if city != "厦门市" { return } insert := map[string]interface{}{ "winner": swinner, "credit_no": (*da)["credit_no"], "bidding_id": mongodb.BsonIdToSId(tmp["_id"]), "projectname": projectName, "company_type": (*da)["company_type"], "company_status": (*da)["company_status"], "company_area": (*da)["company_area"], "company_city": (*da)["company_city"], "year": time.Unix(publishtime, 0).Year(), } Mgo.Save("wcc_xiamen_winner", insert) } }(tmp) tmp = make(map[string]interface{}) } wg.Wait() log.Println("结束") } // CountBidamount 统计厦门中标单位,中标金额总数 func CountBidamount() { Mgo := &mongodb.MongodbSim{ MongodbAddr: "172.17.189.140:27080", //MongodbAddr: "127.0.0.1:27083", DbName: "qfw", Size: 10, UserName: "SJZY_RWbid_ES", Password: "SJZY@B4i4D5e6S", //Direct: true, } Mgo.InitPool() defer util.Catch() sess := Mgo.GetMgoConn() defer Mgo.DestoryMongoConn(sess) var BidMap = make(map[string]bool) it := sess.DB("qfw").C("wcc_xiamen_winner").Find(nil).Select(nil).Iter() count := 0 var total = float64(0) for tmp := make(map[string]interface{}); it.Next(&tmp); count++ { biddingID := util.ObjToString(tmp["bidding_id"]) if BidMap[biddingID] { continue } data, _ := Mgo.FindById("bidding", biddingID, nil) BidMap[biddingID] = true bid := util.Float64All((*data)["bidamount"]) total += bid tmp = make(map[string]interface{}) } log.Println("total", total) }