data_ai/tool/tool.go

package tool

import (
	"data_ai/extract"
	"data_ai/ul"
	log "github.com/donnie4w/go-logger/logger"
	"github.com/google/uuid"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"strings"
	"sync"
)

// 工具修正程序
func StartToolInfo() {
	log.Debug("工具开始大模型修正数据······")
	q := map[string]interface{}{}
	pool_mgo := make(chan bool, ul.Reading)
	wg_mgo := &sync.WaitGroup{}
	sess := ul.SourceMgo.GetMgoConn()
	defer ul.SourceMgo.DestoryMongoConn(sess)
	total, isok := 0, 0
	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%100 == 0 {
			log.Debug("cur index ", total)
		}
		isok++
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			u_id := ul.BsonTOStringId(tmp["_id"])
			data := extract.ResolveInfo(tmp)
			if len(data) > 0 || u_id == "" {
				tmp["ai_zhipu"] = data
				update_info := make(map[string]interface{}, 0)
				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
				if update_info["com_package"] == nil { //构建单包信息···
					com_package := CreatSingleFieldInfo(tmp, update_info)
					update_info["com_package"] = com_package
				}
				update_info["ai_zhipu"] = data
				//清洗与记录
				if len(update_info) > 0 {
					//$set
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$set": update_info,
					})
				}
				if is_unset {
					//"$unset"
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$unset": ul.Unset_Check,
					})
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg_mgo.Wait()
	log.Debug("ai is over ...")
}

// 工具更新程序
func StartToolUpdateInfo() {
	log.Debug("工具开始大模型修正数据······")
	q := map[string]interface{}{}
	pool_mgo := make(chan bool, ul.Reading)
	wg_mgo := &sync.WaitGroup{}
	sess := ul.SourceMgo.GetMgoConn()
	defer ul.SourceMgo.DestoryMongoConn(sess)
	total, isok := 0, 0
	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%100 == 0 {
			log.Debug("cur index ", total)
		}
		isok++
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			u_id := ul.BsonTOStringId(tmp["_id"])
			b_id := qu.ObjToString(tmp["id"])
			data := ul.BidMgo.FindById("bidding", b_id)
			if len(data) > 0 && u_id != "" && b_id != "" && data["ai_zhipu"] != nil {
				ai_zhipu := qu.ObjToMap(data["ai_zhipu"])
				tmp["ai_zhipu"] = ai_zhipu
				update_info := make(map[string]interface{}, 0)
				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
				update_info["ai_zhipu"] = ai_zhipu
				//清洗与记录
				if len(update_info) > 0 { //$set
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$set": update_info,
					})
				}
				if is_unset { //"$unset"
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$unset": ul.Unset_Check,
					})
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg_mgo.Wait()
	log.Debug("ai update is over ...")
}

func CreatSingleFieldInfo(tmp map[string]interface{}, update_info map[string]interface{}) []map[string]interface{} {
	pkgArr := []map[string]interface{}{}
	com_package := map[string]interface{}{}
	package_id := uuid.New().String()
	package_id = strings.ReplaceAll(package_id, "-", "")
	com_package["package_id"] = package_id
	com_package["name"] = qu.ObjToString(tmp["projectname"])

	if update_info["budget"] != nil {
		com_package["budget"] = update_info["budget"]
	} else {
		if tmp["budget"] != nil {
			com_package["budget"] = tmp["budget"]
		}
	}
	subtype := qu.ObjToString(update_info["subtype"])
	if subtype == "" {
		subtype = qu.ObjToString(tmp["subtype"])
	}
	if subtype == "单一" || subtype == "中标" || subtype == "成交" || subtype == "合同" {
		if update_info["bidamount"] != nil {
			com_package["bidamount"] = update_info["bidamount"]
		} else {
			if tmp["bidamount"] != nil {
				com_package["bidamount"] = tmp["bidamount"]
			}
		}

		if update_info["winner"] != nil {
			com_package["winner"] = update_info["winner"]
		} else {
			if tmp["winner"] != nil {
				com_package["winner"] = tmp["winner"]
			}
		}
	}
	pkgArr = append(pkgArr, com_package)
	return pkgArr
}