package main import ( "encoding/json" "flag" "mongodb" "qfw/util" "qfw/util/redis" "sort" "strings" "sync" ) var ( MongoTool *mongodb.MongodbSim PnameLength int PurchasingLength int ) func init() { MongoTool = &mongodb.MongodbSim{ MongodbAddr: "172.17.4.85:27080", // 172.17.4.85:27080 Size: 10, DbName: "qfw", } MongoTool.InitPool() //MongoTool = &mongodb.MongodbSim{ // MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083", // Size: 10, // DbName: "mixdata", // UserName: "SJZY_RWMIX_Other", // Password: "SJZY@M34I6x7D9ata", //} //MongoTool.InitPool() //redis.InitRedis1("qyxy_id=172.17.4.189:8379", 2) // 企业company_id(中标单位) redis.InitRedis1("qyxy=172.17.4.189:8379", 3) // 剑鱼企业搜索搜索 PnameLength = 500 PurchasingLength = 500 } var name string func main1() { flag.StringVar(&name, "n", "", "企业名称") flag.Parse() if name == "" { flag.PrintDefaults() return } text := redis.GetStr("qyxy_id", name) //maps := make(map[string]interface{}) //err1 := json.Unmarshal([]byte(text), &maps) //if err1 != nil { // util.Debug("-----map解析异常") //} util.Debug("company_id=", text) } func main() { sess := MongoTool.GetMgoConn() defer MongoTool.DestoryMongoConn(sess) ch := make(chan bool, 3) wg := &sync.WaitGroup{} //q := map[string]interface{}{ // "_id": map[string]interface{}{ // "$gte": mongodb.StringTOBsonId("5c2a3d000000000000000000"), // "$lte": mongodb.StringTOBsonId("5e0a1f000000000000000000"), // }, //} field := map[string]interface{}{"agency": 1, "buyer": 1, "s_winner": 1, "projectname": 1, "purchasing": 1, "area": 1} //field := map[string]interface{}{"use_flag": 1, "company_type": 1, "company_name": 1} query := sess.DB("qfw").C("projectset_20200918").Find(nil).Select(field).Iter() count := 0 for tmp := make(map[string]interface{}); query.Next(tmp); count++ { if count%5000 == 0 { util.Debug("current ---", count) } ch <- true wg.Add(1) go func(tmp map[string]interface{}) { defer func() { <-ch wg.Done() }() //redis.PutCKV("qyxy_id", util.ObjToString(tmp["company_name"]), util.ObjToString(tmp["_id"])) taskinfo1(tmp) }(tmp) tmp = make(map[string]interface{}) } wg.Wait() util.Debug("over ---", count) } func taskinfo1(tmp map[string]interface{}) { if util.ObjToString(tmp["s_winner"]) != "" { winners := strings.Split(util.ObjToString(tmp["s_winner"]), ",") for _, v := range winners { winnermap := make(map[string]interface{}) if b, err := redis.Exists("qyxy", v); err == nil && b { text := redis.GetStr("qyxy", v) err1 := json.Unmarshal([]byte(text), &winnermap) if err1 != nil { util.Debug(v, "winner-----map解析异常") } // bid_projectname pname := util.ObjToString(tmp["projectname"]) if pname != "" { if winnermap["bid_projectname"] != nil { arr := winnermap["bid_projectname"].([]interface{}) pnameArr := util.ObjArrToStringArr(arr) if BinarySearch(pnameArr, util.ObjToString(tmp["projectname"])) == -1 { pnameArr = append(pnameArr, util.ObjToString(tmp["projectname"])) } if len(pnameArr) > PnameLength { pnameArr1 := pnameArr[len(pnameArr)-PnameLength:] winnermap["bid_projectname"] = pnameArr1 } else { winnermap["bid_projectname"] = pnameArr } } else { pname := []string{util.ObjToString(tmp["projectname"])} winnermap["bid_projectname"] = pname } } // bid_purchasing if util.ObjToString(tmp["purchasing"]) != "" { if winnermap["bid_purchasing"] != nil { arr := winnermap["bid_purchasing"].([]interface{}) purs := util.ObjArrToStringArr(arr) if BinarySearch(purs, util.ObjToString(tmp["purchasing"])) == -1 { purs = append(purs, util.ObjToString(tmp["purchasing"])) } if len(purs) > PurchasingLength { purs1 := purs[len(purs)-PurchasingLength:] winnermap["bid_purchasing"] = purs1 } else { winnermap["bid_purchasing"] = purs } } else { purs := []string{util.ObjToString(tmp["purchasing"])} winnermap["bid_purchasing"] = purs } } // bid_area if winnermap["bid_area"] != nil { arr := winnermap["bid_area"].([]interface{}) areas := util.ObjArrToStringArr(arr) if BinarySearch(areas, util.ObjToString(tmp["area"])) == -1 { areas = append(areas, util.ObjToString(tmp["area"])) } winnermap["bid_area"] = areas } else { areas := []string{util.ObjToString(tmp["area"])} winnermap["bid_area"] = areas } // bid_unittype if winnermap["bid_unittype"] != nil { arr := winnermap["bid_unittype"].([]interface{}) types := util.ObjArrToStringArr(arr) if BinarySearch(types, "中标单位") == -1 { types = append(types, "中标单位") } winnermap["bid_unittype"] = types } else { types := []string{"中标单位"} winnermap["bid_unittype"] = types } } else { // bid_projectname if tmp["projectname"] != nil && tmp["projectname"] != "" { winnermap["bid_projectname"] = []string{util.ObjToString(tmp["projectname"])} } // bid_purchasing if tmp["purchasing"] != nil && tmp["purchasing"] != "" { winnermap["bid_purchasing"] = strings.Split(util.ObjToString(tmp["purchasing"]), ",") } // bid_area winnermap["bid_area"] = []string{util.ObjToString(tmp["area"])} // bid_unittype winnermap["bid_unittype"] = []string{"中标单位"} } if len(winnermap) > 0 { data, _ := json.Marshal(winnermap) redis.Put("qyxy", v, string(data), 0) } } } if buyer := util.ObjToString(tmp["buyer"]); buyer != "" { buyermap := make(map[string]interface{}) if b, err := redis.Exists("qyxy", buyer); err == nil && b { text := redis.GetStr("qyxy", buyer) err1 := json.Unmarshal([]byte(text), &buyermap) if err1 != nil { util.Debug(buyer, "buyer-----map解析异常") } // bid_unittype if buyermap["bid_unittype"] != nil { arr := buyermap["bid_unittype"].([]interface{}) types := util.ObjArrToStringArr(arr) if BinarySearch(types, "采购单位") == -1 { types = append(types, "采购单位") } buyermap["bid_unittype"] = types } else { types := []string{"采购单位"} buyermap["bid_unittype"] = types } } else { buyermap["bid_unittype"] = []string{"采购单位"} } if len(buyermap) > 0 { data, _ := json.Marshal(buyermap) redis.Put("qyxy", buyer, string(data), 0) } } if agency := util.ObjToString(tmp["agency"]); agency != "" { agencymap := make(map[string]interface{}) if b, err := redis.Exists("qyxy", agency); err == nil && b { text := redis.GetStr("qyxy", agency) err1 := json.Unmarshal([]byte(text), &agencymap) if err1 != nil { util.Debug(agency, "agency----map解析异常") } // bid_unittype if agencymap["bid_unittype"] != nil { arr := agencymap["bid_unittype"].([]interface{}) types := util.ObjArrToStringArr(arr) if BinarySearch(types, "代理机构") == -1 { types = append(types, "代理机构") } agencymap["bid_unittype"] = types } else { types := []string{"代理机构"} agencymap["bid_unittype"] = types } } else { agencymap["bid_unittype"] = []string{"代理机构"} } if len(agencymap) > 0 { data, _ := json.Marshal(agencymap) redis.Put("qyxy", agency, string(data), 0) } } } func BinarySearch(s []string, k string) int { sort.Strings(s) lo, hi := 0, len(s)-1 for lo <= hi { m := (lo + hi) >> 1 if s[m] < k { lo = m + 1 } else if s[m] > k { hi = m - 1 } else { return m } } return -1 }