data_ai/tool/tool.go

package tool

import (
	"data_ai/extract"
	"data_ai/ul"
	log "github.com/donnie4w/go-logger/logger"
	"github.com/google/uuid"
	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"strings"
	"sync"
	"time"
)

// 工具修正程序
func StartToolInfo() {
	log.Debug("工具开始大模型修正数据······")
	q := map[string]interface{}{}
	pool_mgo := make(chan bool, ul.Reading)
	wg_mgo := &sync.WaitGroup{}
	sess := ul.SourceMgo.GetMgoConn()
	defer ul.SourceMgo.DestoryMongoConn(sess)
	total, isok := 0, 0
	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
		if total%100 == 0 {
			log.Debug("cur index ", total)
		}
		isok++
		pool_mgo <- true
		wg_mgo.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool_mgo
				wg_mgo.Done()
			}()
			u_id := ul.BsonTOStringId(tmp["_id"])
			data := extract.ResolveInfo(tmp)
			if len(data) > 0 || u_id == "" {
				tmp["ai_zhipu"] = data
				update_info := make(map[string]interface{}, 0)
				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
				if update_info["com_package"] == nil { //构建单包信息···
					com_package := CreatSingleFieldInfo(tmp, update_info)
					update_info["com_package"] = com_package
				}
				update_info["ai_zhipu"] = data
				//清洗与记录
				if len(update_info) > 0 {
					//$set
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$set": update_info,
					})
				}
				if is_unset {
					//"$unset"
					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
						"$unset": ul.Unset_Check,
					})
				}
			}
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg_mgo.Wait()
	log.Debug("ai is over ...")
}

// 测试验证单条工具更新···
func TestToolSingleInfo(name string, tmpid string) {
	log.Debug("测试单条数据数据···")
	now := time.Now().Unix()
	tmp := ul.BidMgo.FindById(name, tmpid)
	if len(tmp) == 0 || tmp == nil {
		log.Debug("未查询到数据...", tmpid)
		return
	}
	u_id := ul.BsonTOStringId(tmp["_id"])
	data := extract.ResolveInfo(tmp)
	if len(data) > 0 || u_id == "" {
		tmp["ai_zhipu"] = data
		update_info := make(map[string]interface{}, 0)
		is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
		if update_info["com_package"] == nil { //构建单包信息···
			com_package := CreatSingleFieldInfo(tmp, update_info)
			update_info["com_package"] = com_package
		}
		update_info["ai_zhipu"] = data
		//清洗与记录
		if len(update_info) > 0 {

		}
		if is_unset {

		}
		log.Debug(update_info)
	}
	log.Debug("耗时···", time.Now().Unix()-now)
}

// 工具更新程序
//func StartToolUpdateInfo() {
//	log.Debug("工具开始大模型修正数据······")
//	q := map[string]interface{}{}
//	pool_mgo := make(chan bool, ul.Reading)
//	wg_mgo := &sync.WaitGroup{}
//	sess := ul.SourceMgo.GetMgoConn()
//	defer ul.SourceMgo.DestoryMongoConn(sess)
//	total, isok := 0, 0
//	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
//	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
//		if total%100 == 0 {
//			log.Debug("cur index ", total)
//		}
//		isok++
//		pool_mgo <- true
//		wg_mgo.Add(1)
//		go func(tmp map[string]interface{}) {
//			defer func() {
//				<-pool_mgo
//				wg_mgo.Done()
//			}()
//			u_id := ul.BsonTOStringId(tmp["_id"])
//			b_id := qu.ObjToString(tmp["id"])
//			data := ul.BidMgo.FindById("bidding", b_id)
//			if len(data) > 0 && u_id != "" && b_id != "" && data["ai_zhipu"] != nil {
//				ai_zhipu := qu.ObjToMap(data["ai_zhipu"])
//				tmp["ai_zhipu"] = ai_zhipu
//				update_info := make(map[string]interface{}, 0)
//				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
//				update_info["ai_zhipu"] = ai_zhipu
//				//清洗与记录
//				if len(update_info) > 0 { //$set
//					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
//						"$set": update_info,
//					})
//				}
//				if is_unset { //"$unset"
//					ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
//						"$unset": ul.Unset_Check,
//					})
//				}
//			}
//		}(tmp)
//		tmp = make(map[string]interface{})
//	}
//	wg_mgo.Wait()
//	log.Debug("ai update is over ...")
//}

func CreatSingleFieldInfo(tmp map[string]interface{}, update_info map[string]interface{}) []map[string]interface{} {
	pkgArr := []map[string]interface{}{}
	com_package := map[string]interface{}{}
	package_id := uuid.New().String()
	package_id = strings.ReplaceAll(package_id, "-", "")
	com_package["package_id"] = package_id
	com_package["name"] = qu.ObjToString(tmp["projectname"])
	com_package["projectcode"] = qu.ObjToString(tmp["projectcode"])
	com_package["packagecode"] = qu.ObjToString(tmp["packagecode"])
	com_package["contractcode"] = qu.ObjToString(tmp["contractcode"])
	if update_info["budget"] != nil {
		com_package["budget"] = update_info["budget"]
	} else {
		if tmp["budget"] != nil {
			com_package["budget"] = tmp["budget"]
		}
	}
	toptype := qu.ObjToString(update_info["toptype"])
	subtype := qu.ObjToString(update_info["subtype"])
	if toptype == "" {
		toptype = qu.ObjToString(tmp["toptype"])
	}
	if subtype == "" {
		subtype = qu.ObjToString(tmp["subtype"])
	}
	//中标类
	if toptype == "结果" || toptype == "其它" || subtype == "单一" {
		if update_info["bidamount"] != nil {
			com_package["bidamount"] = update_info["bidamount"]
		} else {
			if tmp["bidamount"] != nil {
				com_package["bidamount"] = tmp["bidamount"]
			}
		}

		if update_info["winner"] != nil {
			com_package["winner"] = update_info["winner"]
		} else {
			if tmp["winner"] != nil {
				com_package["winner"] = tmp["winner"]
			}
		}
	}
	pkgArr = append(pkgArr, com_package)
	return pkgArr
}