package tool import ( "data_ai/extract" "data_ai/ul" log "github.com/donnie4w/go-logger/logger" "github.com/google/uuid" qu "jygit.jydev.jianyu360.cn/data_processing/common_utils" "strings" "sync" ) // 工具修正程序 func StartToolInfo() { log.Debug("工具开始大模型修正数据······") q := map[string]interface{}{} pool_mgo := make(chan bool, ul.Reading) wg_mgo := &sync.WaitGroup{} sess := ul.SourceMgo.GetMgoConn() defer ul.SourceMgo.DestoryMongoConn(sess) total, isok := 0, 0 it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter() for tmp := make(map[string]interface{}); it.Next(&tmp); total++ { if total%100 == 0 { log.Debug("cur index ", total) } isok++ pool_mgo <- true wg_mgo.Add(1) go func(tmp map[string]interface{}) { defer func() { <-pool_mgo wg_mgo.Done() }() u_id := ul.BsonTOStringId(tmp["_id"]) data := extract.ResolveInfo(tmp) if len(data) > 0 || u_id == "" { tmp["ai_zhipu"] = data update_info := make(map[string]interface{}, 0) is_unset := ul.ChooseCheckDataAI(tmp, &update_info) if update_info["com_package"] == nil { //构建单包信息··· com_package := CreatSingleFieldInfo(tmp, update_info) update_info["com_package"] = com_package } update_info["ai_zhipu"] = data //清洗与记录 if len(update_info) > 0 { //$set ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{ "$set": update_info, }) } if is_unset { //"$unset" ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{ "$unset": ul.Unset_Check, }) } } }(tmp) tmp = make(map[string]interface{}) } wg_mgo.Wait() log.Debug("ai is over ...") } // 工具更新程序 //func StartToolUpdateInfo() { // log.Debug("工具开始大模型修正数据······") // q := map[string]interface{}{} // pool_mgo := make(chan bool, ul.Reading) // wg_mgo := &sync.WaitGroup{} // sess := ul.SourceMgo.GetMgoConn() // defer ul.SourceMgo.DestoryMongoConn(sess) // total, isok := 0, 0 // it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter() // for tmp := make(map[string]interface{}); it.Next(&tmp); total++ { // if total%100 == 0 { // log.Debug("cur index ", total) // } // isok++ // pool_mgo <- true // wg_mgo.Add(1) // go func(tmp map[string]interface{}) { // defer func() { // <-pool_mgo // wg_mgo.Done() // }() // u_id := ul.BsonTOStringId(tmp["_id"]) // b_id := qu.ObjToString(tmp["id"]) // data := ul.BidMgo.FindById("bidding", b_id) // if len(data) > 0 && u_id != "" && b_id != "" && data["ai_zhipu"] != nil { // ai_zhipu := qu.ObjToMap(data["ai_zhipu"]) // tmp["ai_zhipu"] = ai_zhipu // update_info := make(map[string]interface{}, 0) // is_unset := ul.ChooseCheckDataAI(tmp, &update_info) // update_info["ai_zhipu"] = ai_zhipu // //清洗与记录 // if len(update_info) > 0 { //$set // ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{ // "$set": update_info, // }) // } // if is_unset { //"$unset" // ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{ // "$unset": ul.Unset_Check, // }) // } // } // }(tmp) // tmp = make(map[string]interface{}) // } // wg_mgo.Wait() // log.Debug("ai update is over ...") //} func CreatSingleFieldInfo(tmp map[string]interface{}, update_info map[string]interface{}) []map[string]interface{} { pkgArr := []map[string]interface{}{} com_package := map[string]interface{}{} package_id := uuid.New().String() package_id = strings.ReplaceAll(package_id, "-", "") com_package["package_id"] = package_id com_package["name"] = qu.ObjToString(tmp["projectname"]) if update_info["budget"] != nil { com_package["budget"] = update_info["budget"] } else { if tmp["budget"] != nil { com_package["budget"] = tmp["budget"] } } subtype := qu.ObjToString(update_info["subtype"]) if subtype == "" { subtype = qu.ObjToString(tmp["subtype"]) } if subtype == "单一" || subtype == "中标" || subtype == "成交" || subtype == "合同" { if update_info["bidamount"] != nil { com_package["bidamount"] = update_info["bidamount"] } else { if tmp["bidamount"] != nil { com_package["bidamount"] = tmp["bidamount"] } } if update_info["winner"] != nil { com_package["winner"] = update_info["winner"] } else { if tmp["winner"] != nil { com_package["winner"] = tmp["winner"] } } } pkgArr = append(pkgArr, com_package) return pkgArr }