data_ai/tool/tool.go

package tool

import (
	"data_ai/extract"
	"data_ai/ul"
	log "github.com/donnie4w/go-logger/logger"
	"sync"
)

// 工具修正程序
func StartToolInfo() {
	log.Debug("工具开始大模型修正数据······")
	q := map[string]interface{}{}
	pool_mgo := make(chan bool, ul.Reading)
	wg_mgo := &sync.WaitGroup{}
	sess := ul.SourceMgo.GetMgoConn()
	defer ul.SourceMgo.DestoryMongoConn(sess)
	total, isok := 0, 0
	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%100 == 0 {
			log.Debug("cur index ", total)
		}
		isok++
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			u_id := ul.BsonTOStringId(tmp["_id"])
			data := extract.ResolveInfo(tmp)
			if len(data) > 0 || u_id == "" {
				tmp["ai_zhipu"] = data
				update_info := make(map[string]interface{}, 0)
				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
				update_info["ai_zhipu"] = data
				//清洗与记录
				if len(update_info) > 0 {
					//$set
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$set": update_info,
					})
				}
				if is_unset {
					//"$unset"
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$unset": ul.Unset_Check,
					})
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg_mgo.Wait()
	log.Debug("ai is over ...")
}