package extract import ( "data_ai/prompt" "data_ai/ul" "fmt" log "github.com/donnie4w/go-logger/logger" new_xlsx "github.com/tealeg/xlsx/v3" qu "jygit.jydev.jianyu360.cn/data_processing/common_utils" "sync" "time" "unicode/utf8" ) // 验证单条数据··· func TestSingleFieldInfo(name string, tmpid string) { log.Debug("测试单条大模型数据···") now := time.Now().Unix() tmp := ul.BidMgo.FindById(name, tmpid) if len(tmp) == 0 || tmp == nil { log.Debug("未查询到数据...", tmpid) return } data := ResolveInfo(tmp) //最终结果... if data != nil { log.Debug(data["s_area"], "~", data["s_city"]) } log.Debug("耗时···", time.Now().Unix()-now) } // 验证单条分包数据··· func TestSinglePurchasingInfo(name string, tmpid string) { now := time.Now().Unix() tmp := ul.BidMgo.FindById(name, tmpid) if len(tmp) == 0 || tmp == nil { log.Debug("未查询到数据...", tmpid) return } detail := getDetailText(tmp, tmpid) //获取正文文本 p_list := getPurList(tmp, detail, map[string]interface{}{}) log.Debug(p_list) //最终结果... for k, v := range p_list { log.Debug(k, "~", v) } log.Debug("耗时···", time.Now().Unix()-now) } // 验证单条分包数据··· func TestSinglePackageInfo(name string, tmpid string) { now := time.Now().Unix() tmp := ul.BidMgo.FindById(name, tmpid) if len(tmp) == 0 || tmp == nil { log.Debug("未查询到数据...", tmpid) return } detail := qu.ObjToString(tmp["detail"]) filetext := qu.ObjToString(tmp["filetext"]) //此处为附件信息··· if utf8.RuneCountInString(detail) < 100 { detail = filetext } detail = ul.HttpConvertToMarkdown(detail) pkg := prompt.AcquireNewMultiplePackageInfo(detail, false) //最终结果... for k, v := range pkg { log.Debug(k, "~", v) } log.Debug("耗时···", time.Now().Unix()-now) } // 新分包数据··· func TestNewPackageInfo1010() { dataArr, _ := ul.SourceMgo.Find("zktest_info_0930", map[string]interface{}{}, nil, nil) log.Debug("数量···", len(dataArr)) pool_mgo := make(chan bool, 50) wg_mgo := &sync.WaitGroup{} for k, v := range dataArr { if k%50 == 0 { log.Debug("cur index ", k) } pool_mgo <- true wg_mgo.Add(1) go func(v map[string]interface{}) { defer func() { <-pool_mgo wg_mgo.Done() }() new_v := v data := ResolveInfo(v) if len(data) > 0 { new_v["ai_zhipu"] = data } ul.SourceMgo.Save("zktest_info_0930_new", new_v) }(v) } wg_mgo.Wait() log.Debug("is over ...") } // 更新链接 func TestUpdateJyhref(name string) { dataArr, _ := ul.SourceMgo.Find(name, map[string]interface{}{}, nil, map[string]interface{}{"_id": 1}) for _, v := range dataArr { tmpid := ul.BsonTOStringId(v["_id"]) jyhref := fmt.Sprintf(ul.Url, qu.CommonEncodeArticle("content", tmpid)) ul.SourceMgo.UpdateById(name, tmpid, map[string]interface{}{ "$set": map[string]interface{}{ "jyhref": jyhref, }, }) } log.Debug("is over ...") } func writeRow(row *new_xlsx.Row, arr []string) { for _, v := range arr { row.AddCell().Value = v } }