data_ai/extract/test.go

package extract

import (
	"data_ai/prompt"
	"data_ai/ul"
	"fmt"
	log "github.com/donnie4w/go-logger/logger"
	new_xlsx "github.com/tealeg/xlsx/v3"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"sync"
	"time"
	"unicode/utf8"
)

// 验证单条数据···
func TestSingleFieldInfo(name string, tmpid string) {
	log.Debug("测试单条大模型数据···")
	now := time.Now().Unix()
	tmp := ul.BidMgo.FindById(name, tmpid)
	if len(tmp) == 0 || tmp == nil {
		log.Debug("未查询到数据...", tmpid)
		return
	}
	data := ResolveInfo(tmp)
	//最终结果...
	if data != nil {
		log.Debug(data["s_area"], "~", data["s_city"])
	}
	log.Debug("耗时···", time.Now().Unix()-now)
}

// 验证单条分包数据···
func TestSinglePurchasingInfo(name string, tmpid string) {
	now := time.Now().Unix()
	tmp := ul.BidMgo.FindById(name, tmpid)
	if len(tmp) == 0 || tmp == nil {
		log.Debug("未查询到数据...", tmpid)
		return
	}
	detail := getDetailText(tmp, tmpid) //获取正文文本
	p_list := getPurList(tmp, detail, map[string]interface{}{})
	log.Debug(p_list)

	//最终结果...
	for k, v := range p_list {
		log.Debug(k, "~", v)
	}

	log.Debug("耗时···", time.Now().Unix()-now)
}

// 验证单条分包数据···
func TestSinglePackageInfo(name string, tmpid string) {
	now := time.Now().Unix()
	tmp := ul.BidMgo.FindById(name, tmpid)
	if len(tmp) == 0 || tmp == nil {
		log.Debug("未查询到数据...", tmpid)
		return
	}
	detail := qu.ObjToString(tmp["detail"])
	filetext := qu.ObjToString(tmp["filetext"]) //此处为附件信息···
	if utf8.RuneCountInString(detail) < 100 {
		detail = filetext
	}
	detail = ul.HttpConvertToMarkdown(detail)
	pkg := prompt.AcquireNewMultiplePackageInfo(detail, false)
	//最终结果...
	for k, v := range pkg {
		log.Debug(k, "~", v)
	}
	log.Debug("耗时···", time.Now().Unix()-now)
}

// 新分包数据···
func TestNewPackageInfo1010() {
	dataArr, _ := ul.SourceMgo.Find("zktest_info_0930", map[string]interface{}{}, nil, nil)
	log.Debug("数量···", len(dataArr))
	pool_mgo := make(chan bool, 50)
	wg_mgo := &sync.WaitGroup{}
	for k, v := range dataArr {
		if k%50 == 0 {
			log.Debug("cur index ", k)
		}
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(v map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			new_v := v
			data := ResolveInfo(v)
			if len(data) > 0 {
				new_v["ai_zhipu"] = data
			}
			ul.SourceMgo.Save("zktest_info_0930_new", new_v)
		}(v)
	}
	wg_mgo.Wait()
	log.Debug("is over ...")
}

// 更新链接
func TestUpdateJyhref(name string) {
	dataArr, _ := ul.SourceMgo.Find(name, map[string]interface{}{}, nil, map[string]interface{}{"_id": 1})
	for _, v := range dataArr {
		tmpid := ul.BsonTOStringId(v["_id"])
		jyhref := fmt.Sprintf(ul.Url, qu.CommonEncodeArticle("content", tmpid))
		ul.SourceMgo.UpdateById(name, tmpid, map[string]interface{}{
			"$set": map[string]interface{}{
				"jyhref": jyhref,
			},
		})
	}
	log.Debug("is over ...")
}

func writeRow(row *new_xlsx.Row, arr []string) {
	for _, v := range arr {
		row.AddCell().Value = v
	}
}