package extract import ( "data_ai/prompt" "data_ai/ul" "fmt" log "github.com/donnie4w/go-logger/logger" new_xlsx "github.com/tealeg/xlsx/v3" qu "jygit.jydev.jianyu360.cn/data_processing/common_utils" "os" "sync" "unicode/utf8" ) func TestSingleFieldInfo(name string, tmpid string) { tmp := ul.SourceMgo.FindById(name, tmpid) data := ResolveInfo(tmp) //最终结果... for k, v := range data { log.Debug(k, "~", v) } } func TestPackageInfo() { query := map[string]interface{}{ "new_pkg": map[string]interface{}{ "$exists": 1, }, } dataArr, _ := ul.SourceMgo.Find("ai_41411_zhipu", query, nil, map[string]interface{}{}) log.Debug("查询数量...", len(dataArr)) os.Remove("test.xlsx") f := new_xlsx.NewFile() sheet, _ := f.AddSheet("数据信息") row := sheet.AddRow() writeRow(row, []string{"唯一标识", "站点", "剑鱼链接", "子包名称", "子包单位", "子包金额"}) for _, v := range dataArr { tmpid := ul.BsonTOStringId(v["_id"]) ttt := ul.SourceMgo.FindById("ai_41411", tmpid) site := qu.ObjToString(ttt["site"]) jyhref := fmt.Sprintf(ul.Url, qu.CommonEncodeArticle("content", tmpid)) p_info := *qu.ObjToMap(v["new_pkg"]) p_arr := ul.IsMarkInterfaceMap(p_info["分包信息"]) for _, v1 := range p_arr { row = sheet.AddRow() arr := []string{} arr = append(arr, tmpid) arr = append(arr, site) arr = append(arr, jyhref) arr = append(arr, qu.ObjToString(v1["包项目名称"])) arr = append(arr, qu.ObjToString(v1["中标单位"])) arr = append(arr, qu.ObjToString(v1["中标金额"])) writeRow(row, arr) } } if err := f.Save("test.xlsx"); err != nil { fmt.Println("保存xlsx失败:", err) } else { fmt.Println("保存xlsx成功:", err) } log.Debug("is over ...") return //分包判断,获取信息 pool_mgo := make(chan bool, 80) wg_mgo := &sync.WaitGroup{} for k, v := range dataArr { if k%10 == 0 { log.Debug(k, "~", v["_id"]) } pool_mgo <- true wg_mgo.Add(1) go func(v map[string]interface{}) { defer func() { <-pool_mgo wg_mgo.Done() }() tmpid := ul.BsonTOStringId(v["_id"]) data := ul.SourceMgo.FindById("ai_41411", tmpid) if detail := qu.ObjToString(data["detail"]); utf8.RuneCountInString(detail) > 100 { pkg := prompt.AcquireMultiplePackageInfo(detail) //最终结果... ul.SourceMgo.UpdateById("ai_41411_zhipu", tmpid, map[string]interface{}{ "$set": map[string]interface{}{ "new_pkg": pkg, }, }) } }(v) } wg_mgo.Wait() } // 更新链接 func TestUpdateJyhref(name string) { dataArr, _ := ul.SourceMgo.Find(name, map[string]interface{}{}, nil, map[string]interface{}{"_id": 1}) for _, v := range dataArr { tmpid := ul.BsonTOStringId(v["_id"]) jyhref := fmt.Sprintf(ul.Url, qu.CommonEncodeArticle("content", tmpid)) ul.SourceMgo.UpdateById(name, tmpid, map[string]interface{}{ "$set": map[string]interface{}{ "jyhref": jyhref, }, }) } log.Debug("is over ...") } func writeRow(row *new_xlsx.Row, arr []string) { for _, v := range arr { row.AddCell().Value = v } }