package main import ( "data_tidb/config" "fmt" "go.uber.org/zap" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/log" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "jygit.jydev.jianyu360.cn/data_processing/common_utils/redis" "strings" "sync" "time" ) func taskP() { sess := MongoP.GetMgoConn() defer MongoP.DestoryMongoConn(sess) ch := make(chan bool, 20) wg := &sync.WaitGroup{} //q := map[string]interface{}{"_id": mongodb.StringTOBsonId("5c4044d4a5cb26b9b7b963cc")} query := sess.DB(config.Conf.DB.MongoP.Dbname).C("projectset_20230407").Find(nil).Sort("_id").Iter() count := 0 for tmp := make(map[string]interface{}); query.Next(tmp); count++ { if count%20000 == 0 { log.Info(fmt.Sprintf("current --- %d", count)) } ch <- true wg.Add(1) go func(tmp map[string]interface{}) { defer func() { <-ch wg.Done() }() //taskPro(tmp) //taskBusiness(tmp) //taskProTag(tmp) taskRelation(tmp) }(tmp) tmp = make(map[string]interface{}) } wg.Wait() log.Info(fmt.Sprintf("over --- %d", count)) } var BidStatus = map[string]int{ "预告": 0, "拟建": 1, "招标": 2, "中标": 3, "成交": 4, "废标": 5, "流标": 6, "合同": 7, "其它": 8, } var BidType = map[string]int{ "招标": 0, "邀标": 1, "单一": 2, "竞价": 3, "竞谈": 4, "询价": 5, } // @Description 基础信息 // @Author J 2022/9/22 18:32 func taskPro(tmp map[string]interface{}) { saveM := make(map[string]interface{}) for _, f := range ProField { if f == "projectid" { saveM[f] = mongodb.BsonIdToSId(tmp["_id"]) } else if f == "area_code" { if tmp["area"] != nil { saveM[f] = AreaCode[util.ObjToString(tmp["area"])] } } else if f == "city_code" { if tmp["area"] != nil && tmp["city"] != nil { c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) saveM[f] = AreaCode[c] } } else if f == "district_code" { if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil { c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"]) saveM[f] = AreaCode[c] } } else if f == "updatetime" { saveM[f] = time.Now().Format(util.Date_Full_Layout) } else if f == "buyerclass_code" { if obj := util.ObjToString(tmp["buyerclass"]); obj != "" { saveM[f] = BuyerCode[obj] } } else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" { if tmp[f] != nil && util.Int64All(tmp[f]) > 0 { t := util.Int64All(tmp[f]) saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout) } } else if f == "bidstatus" { if b := util.ObjToString(tmp[f]); b != "" { tmp[f] = BidStatus[b] } } else if f == "bidtype" { if b := util.ObjToString(tmp[f]); b != "" { tmp[f] = BidType[b] } } else if f == "multipackage" { if tmp[f] == nil { saveM[f] = 0 } else { saveM[f] = tmp[f] } } else if f == "buyer_id" { if b := util.ObjToString(tmp["buyer"]); b != "" { if code := redis.GetStr("qyxy_id", b); code != "" { saveM[f] = code } } } else if f == "agency_id" { if b := util.ObjToString(tmp["agency"]); b != "" { if code := redis.GetStr("qyxy_id", b); code != "" { saveM[f] = code } } } else { if tmp[f] != nil { if ProVMap[f] != nil { saveM[f], _ = verifyF(f, tmp[f], ProVMap[f]) } else { saveM[f] = tmp[f] } } } } saveProPool <- saveM } // @Description 项目业务表 // @Author J 2022/9/30 13:40 func taskBusiness(tmp map[string]interface{}) { warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",") if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 { warr = append(warr, util.ObjToString(tmp["winner"])) } for _, s := range warr { saveM := make(map[string]interface{}) for _, f := range ProBusField { if f == "projectid" { saveM[f] = mongodb.BsonIdToSId(tmp["_id"]) } else if f == "area_code" { if tmp["area"] != nil { saveM[f] = AreaCode[util.ObjToString(tmp["area"])] } } else if f == "city_code" { if tmp["area"] != nil && tmp["city"] != nil { c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) saveM[f] = AreaCode[c] } } else if f == "district_code" { if tmp["area"] != nil && tmp["city"] != nil && tmp["district"] != nil { c := util.ObjToString(tmp["area"]) + "," + util.ObjToString(tmp["city"]) + "," + util.ObjToString(tmp["district"]) saveM[f] = AreaCode[c] } } else if f == "updatetime" { saveM[f] = time.Now().Format(util.Date_Full_Layout) } else if f == "buyerclass_code" { if obj := util.ObjToString(tmp["buyerclass"]); obj != "" { saveM[f] = BuyerCode[obj] } } else if f == "firsttime" || f == "zbtime" || f == "jgtime" || f == "lasttime" || f == "bidopentime" || f == "createtime" { if tmp[f] != nil && util.Int64All(tmp[f]) > 0 { t := util.Int64All(tmp[f]) saveM[f] = util.FormatDateByInt64(&t, util.Date_Full_Layout) } } else if f == "bidstatus" { if b := util.ObjToString(tmp[f]); b != "" { tmp[f] = BidStatus[b] } } else if f == "bidtype" { if b := util.ObjToString(tmp[f]); b != "" { tmp[f] = BidType[b] } } else if f == "buyer_id" { if b := util.ObjToString(tmp["buyer"]); b != "" { saveM["buyer"] = b if code := redis.GetStr("qyxy_id", b); code != "" { saveM[f] = code } } } else if f == "agency_id" { if b := util.ObjToString(tmp["agency"]); b != "" { saveM["agency"] = b if code := redis.GetStr("qyxy_id", b); code != "" { saveM[f] = code } } } else if f == "winner_id" { if s != "" { saveM["winner"] = s if code := redis.GetStr("qyxy_id", s); code != "" { saveM[f] = code } } } else { if tmp[f] != nil { if ProBusVMap[f] != nil { saveM[f], _ = verifyF(f, tmp[f], ProBusVMap[f]) } else { saveM[f] = tmp[f] } } } } saveProbPool <- saveM } } // @Description 项目信息标签 // @Author J 2022/9/30 13:54 func taskProTag(tmp map[string]interface{}) { id := mongodb.BsonIdToSId(tmp["_id"]) if topArr, ok := tmp["topscopeclass"].([]interface{}); ok { for _, i2 := range topArr { tclass := regLetter.ReplaceAllString(util.ObjToString(i2), "") // 去除字母 code := TopScopeCode[tclass] saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "1", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)} } } if subArr, ok := tmp["subscopeclass"].([]interface{}); ok { for _, i2 := range subArr { sc := strings.Split(util.ObjToString(i2), "_") code := SubScopeCode[sc[1]] saveProTagPool <- map[string]interface{}{"projectid": id, "labelcode": "2", "labelvalues": code, "labelweight": 1, "createtime": time.Now().Format(util.Date_Full_Layout)} } } } // @Description 关系表 // @Author J 2022/9/30 13:56 func taskRelation(tmp map[string]interface{}) { pid := mongodb.BsonIdToSId(tmp["_id"]) if tmp["ids"] == nil { log.Info("taskRelation ids err", zap.Any("id", pid)) return } ids := util.ObjArrToStringArr(tmp["ids"].([]interface{})) lid := ids[len(ids)-1] if b := util.ObjToString(tmp["buyer"]); b != "" { saveM := make(map[string]interface{}) for _, f := range RelationField { if f == "projectid" { saveM[f] = pid } else if f == "infoid" { saveM[f] = lid } else if f == "name_id" { if code := redis.GetStr("qyxy_id", b); code != "" { saveM[f] = code } } else if f == "contact_id" { // todo } else if f == "identity_type" { saveM[f] = 1 // 001 } else if f == "createtime" { saveM[f] = time.Now().Format(util.Date_Full_Layout) } } saveRelationPool <- saveM } if a := util.ObjToString(tmp["agency"]); a != "" { saveM := make(map[string]interface{}) for _, f := range RelationField { if f == "projectid" { saveM[f] = pid } else if f == "infoid" { saveM[f] = lid } else if f == "name_id" { if code := redis.GetStr("qyxy_id", a); code != "" { saveM[f] = code } } else if f == "contact_id" { } else if f == "identity_type" { saveM[f] = 4 // 100 } else if f == "createtime" { saveM[f] = time.Now().Format(util.Date_Full_Layout) } } saveRelationPool <- saveM } warr := strings.Split(util.ObjToString(tmp["s_winner"]), ",") if BinarySearch(warr, util.ObjToString(tmp["winner"])) == -1 { warr = append(warr, util.ObjToString(tmp["winner"])) } for _, ws := range warr { saveM := make(map[string]interface{}) for _, f := range RelationField { if f == "projectid" { saveM[f] = pid } else if f == "infoid" { saveM[f] = lid } else if f == "name_id" { if code := redis.GetStr("qyxy_id", ws); code != "" { saveM[f] = code } } else if f == "contact_id" { } else if f == "identity_type" { saveM[f] = 2 // 010 } else if f == "createtime" { saveM[f] = time.Now().Format(util.Date_Full_Layout) } } saveRelationPool <- saveM } }