适配最新大模型数据

extract/extract.go

@@ -23,6 +23,7 @@ func ExtractFieldInfo(sid string, eid string) {
 	dict := ConfrimExtractInfo(q)
 	log.Debug("查询语句...", q, "~", len(dict))
 	if len(dict) >= ul.MaxUdp { //根据数量限制使用具体模型
+		log.Debug("数量超过限制临时使用：glm-4-flashx")
 		ul.FlashModel = "glm-4-flashx"
 	} else {
 		ul.FlashModel = "glm-4-flash"
@@ -85,6 +86,7 @@ func ResolveInfo(v map[string]interface{}) map[string]interface{} {
 	if utf8.RuneCountInString(detail) < 100 {
 		return f_data
 	}
+	//文本格式转换
 	detail = ul.HttpConvertToMarkdown(detail)
 	//获取外围字段数据
 	f_info := prompt.AcquireExtractFieldInfo(detail)
@@ -93,14 +95,6 @@ func ResolveInfo(v map[string]interface{}) map[string]interface{} {
 	if len(pkg) > 0 {
 		f_info["s_pkg"] = pkg
 	}
-	//分包判断-获取分包方法舍弃
-	//ispkg, pkg := false, map[string]interface{}{}
-	//if ispkg = prompt.AcquireIsPackageInfo(detail); ispkg {
-	//	f_info["ispkg"] = ispkg
-	//	if pkg = prompt.AcquireMultiplePackageInfo(detail); len(pkg) > 0 {
-	//		f_info["s_pkg"] = pkg
-	//	}
-	//}
 
 	//获取分类字段数据
 	s_toptype, s_subtype := "", ""

extract/test.go

@@ -14,6 +14,7 @@ import (
 
 // 验证单条数据···
 func TestSingleFieldInfo(name string, tmpid string) {
+	log.Debug("测试单条数据数据···")
 	now := time.Now().Unix()
 	tmp := ul.BidMgo.FindById(name, tmpid)
 	if len(tmp) == 0 || tmp == nil {

main.go

@@ -6,13 +6,11 @@ import (
 	"data_ai/udp"
 	"data_ai/ul"
 	log "github.com/donnie4w/go-logger/logger"
-	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
-	"sync"
-	"unicode/utf8"
 )
 
 func init() {
-	ul.IsTool = false
+	//增量设置全部false
+	ul.IsTool = true
 	ul.IsFull = false
 	ul.IsLocal = true
 	if ul.IsTool {
@@ -35,48 +33,48 @@ func main() {
 		return
 	} else {
 		if !ul.IsFull {
-			//extract.TestSinglePackageInfo("bidding", "66e38634b25c3e1debf23a00")
-			extract.TestSinglePackageInfo("bidding", "67050280b25c3e1deb667d38")
+			extract.TestSingleFieldInfo("bidding", "670cc446b25c3e1deb887d52")
+			return
 		}
 	}
 	lock := make(chan bool)
 	<-lock
 }
 
-func test() {
-	log.Debug("开始大模型验证内存数据···")
-	q := map[string]interface{}{}
-	pool_mgo := make(chan bool, ul.Reading)
-	wg_mgo := &sync.WaitGroup{}
-	sess := ul.SourceMgo.GetMgoConn()
-	defer ul.SourceMgo.DestoryMongoConn(sess)
-	total, isok := 0, 0
-	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%100 == 0 {
-			log.Debug("cur index ", total)
-		}
-		isok++
-		pool_mgo <- true
-		wg_mgo.Add(1)
-		go func(tmp map[string]interface{}) {
-			defer func() {
-				<-pool_mgo
-				wg_mgo.Done()
-			}()
-			detail := qu.ObjToString(tmp["detail"])
-			filetext := qu.ObjToString(tmp["filetext"]) //此处为附件信息···
-			if utf8.RuneCountInString(detail) < 100 {
-				detail = filetext
-			}
-			detail = ul.HttpConvertToMarkdown(detail)
-			//最终结果...
-			if detail != "" {
-
-			}
-		}(tmp)
-		tmp = make(map[string]interface{})
-	}
-	wg_mgo.Wait()
-	log.Debug("ai is over ...")
-}
+//func test() {
+//	log.Debug("开始大模型验证内存数据···")
+//	q := map[string]interface{}{}
+//	pool_mgo := make(chan bool, ul.Reading)
+//	wg_mgo := &sync.WaitGroup{}
+//	sess := ul.SourceMgo.GetMgoConn()
+//	defer ul.SourceMgo.DestoryMongoConn(sess)
+//	total, isok := 0, 0
+//	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
+//	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+//		if total%100 == 0 {
+//			log.Debug("cur index ", total)
+//		}
+//		isok++
+//		pool_mgo <- true
+//		wg_mgo.Add(1)
+//		go func(tmp map[string]interface{}) {
+//			defer func() {
+//				<-pool_mgo
+//				wg_mgo.Done()
+//			}()
+//			detail := qu.ObjToString(tmp["detail"])
+//			filetext := qu.ObjToString(tmp["filetext"]) //此处为附件信息···
+//			if utf8.RuneCountInString(detail) < 100 {
+//				detail = filetext
+//			}
+//			detail = ul.HttpConvertToMarkdown(detail)
+//			//最终结果...
+//			if detail != "" {
+//
+//			}
+//		}(tmp)
+//		tmp = make(map[string]interface{})
+//	}
+//	wg_mgo.Wait()
+//	log.Debug("ai is over ...")
+//}

tool.json

@@ -1,28 +1,28 @@
 {
   "reading": 50,
-  "ext_name": "zktest_package_1011",
+  "ext_name": "20241015Hs_5_area",
   "s_mgo": {
-    "local": true,
-    "l_addr": "127.0.0.1:12005",
-    "addr": "127.0.0.1:12005",
-    "dbname" : "qfw",
-    "username": "zhengkun",
-    "password": "zk@123123"
+    "local": false,
+    "l_addr": "192.168.3.166:27082",
+    "addr": "192.168.3.166:27082",
+    "dbname" : "zhaoxiuzhen",
+    "username": "",
+    "password": ""
   },
   "b_mgo": {
     "local": true,
     "l_addr": "127.0.0.1:12005",
     "addr": "127.0.0.1:12005",
-    "dbname" : "qfw",
+    "dbname" : "qfw_ai",
     "username": "zhengkun",
     "password": "zk@123123"
   },
   "qy_mgo": {
-    "local": true,
-    "l_addr": "127.0.0.1:12005",
-    "addr": "127.0.0.1:12005",
-    "dbname" : "mixdata",
-    "username": "zhengkun",
-    "password": "zk@123123"
+    "local": false,
+    "l_addr": "192.168.3.166:27082",
+    "addr": "192.168.3.166:27082",
+    "dbname" : "zhengkun",
+    "username": "",
+    "password": ""
   }
 }

tool/tool.go

@@ -4,6 +4,9 @@ import (
 	"data_ai/extract"
 	"data_ai/ul"
 	log "github.com/donnie4w/go-logger/logger"
+	"github.com/google/uuid"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"strings"
 	"sync"
 )
 
@@ -35,6 +38,10 @@ func StartToolInfo() {
 				tmp["ai_zhipu"] = data
 				update_info := make(map[string]interface{}, 0)
 				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
+				if update_info["com_package"] == nil { //构建单包信息···
+					com_package := CreatSingleFieldInfo(tmp, update_info)
+					update_info["com_package"] = com_package
+				}
 				update_info["ai_zhipu"] = data
 				//清洗与记录
 				if len(update_info) > 0 {
@@ -56,3 +63,96 @@ func StartToolInfo() {
 	wg_mgo.Wait()
 	log.Debug("ai is over ...")
 }
+
+// 工具更新程序
+func StartToolUpdateInfo() {
+	log.Debug("工具开始大模型修正数据······")
+	q := map[string]interface{}{}
+	pool_mgo := make(chan bool, ul.Reading)
+	wg_mgo := &sync.WaitGroup{}
+	sess := ul.SourceMgo.GetMgoConn()
+	defer ul.SourceMgo.DestoryMongoConn(sess)
+	total, isok := 0, 0
+	it := sess.DB(ul.SourceMgo.DbName).C(ul.Ext_Name).Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%100 == 0 {
+			log.Debug("cur index ", total)
+		}
+		isok++
+		pool_mgo <- true
+		wg_mgo.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool_mgo
+				wg_mgo.Done()
+			}()
+			u_id := ul.BsonTOStringId(tmp["_id"])
+			b_id := qu.ObjToString(tmp["id"])
+			data := ul.BidMgo.FindById("bidding", b_id)
+			if len(data) > 0 && u_id != "" && b_id != "" && data["ai_zhipu"] != nil {
+				ai_zhipu := qu.ObjToMap(data["ai_zhipu"])
+				tmp["ai_zhipu"] = ai_zhipu
+				update_info := make(map[string]interface{}, 0)
+				is_unset := ul.ChooseCheckDataAI(tmp, &update_info)
+
+				update_info["ai_zhipu"] = ai_zhipu
+				//清洗与记录
+				if len(update_info) > 0 {
+					//$set
+					//ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
+					//	"$set": update_info,
+					//})
+				}
+				if is_unset {
+					//"$unset"
+					//ul.SourceMgo.UpdateById(ul.Ext_Name, u_id, map[string]interface{}{
+					//	"$unset": ul.Unset_Check,
+					//})
+				}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg_mgo.Wait()
+	log.Debug("ai update is over ...")
+}
+
+func CreatSingleFieldInfo(tmp map[string]interface{}, update_info map[string]interface{}) []map[string]interface{} {
+	pkgArr := []map[string]interface{}{}
+	com_package := map[string]interface{}{}
+	package_id := uuid.New().String()
+	package_id = strings.ReplaceAll(package_id, "-", "")
+	com_package["package_id"] = package_id
+	com_package["name"] = qu.ObjToString(tmp["projectname"])
+
+	if update_info["budget"] != nil {
+		com_package["budget"] = update_info["budget"]
+	} else {
+		if tmp["budget"] != nil {
+			com_package["budget"] = tmp["budget"]
+		}
+	}
+	subtype := qu.ObjToString(update_info["subtype"])
+	if subtype == "" {
+		subtype = qu.ObjToString(tmp["subtype"])
+	}
+	if subtype == "单一" || subtype == "中标" || subtype == "成交" || subtype == "合同" {
+		if update_info["bidamount"] != nil {
+			com_package["bidamount"] = update_info["bidamount"]
+		} else {
+			if tmp["bidamount"] != nil {
+				com_package["bidamount"] = tmp["bidamount"]
+			}
+		}
+
+		if update_info["winner"] != nil {
+			com_package["winner"] = update_info["winner"]
+		} else {
+			if tmp["winner"] != nil {
+				com_package["winner"] = tmp["winner"]
+			}
+		}
+	}
+	pkgArr = append(pkgArr, com_package)
+	return pkgArr
+}

ul/global.go

@@ -19,8 +19,10 @@ func ChooseCheckDataAI(tmp map[string]interface{}, update_info *map[string]inter
 	s_toptype, s_subtype := ChooseTheBestClassField(ai_zhipu, tmp, update_info, &ext_ai_record)
 	//选取字段
 	ChooseTheBestCoreField(ai_zhipu, s_toptype, s_subtype, tmp, update_info, &ext_ai_record)
+
 	//选取分包
 	ChooseTheBestPackageField(ai_zhipu, s_toptype, s_subtype, tmp, update_info, &ext_ai_record)
+
 	//最终金额修正与选取
 	ChooseTheBestAmountField(tmp, update_info)
 

ul/init.go

@@ -259,7 +259,7 @@ func IsMarkInterfaceMap(t interface{}) []map[string]interface{} {
 }
 
 func PostMarkDownText(html string) string {
-	url := "http://172.17.0.11:8888/md"
+	url := "http://172.17.162.35:18811/md"
 	if IsLocal {
 		url = "http://172.17.0.11:8888/md"
 	}