dataIdentify/main_test.go

package main

import (
	"app.yhyue.com/moapp/jybase/encrypt"
	. "app.yhyue.com/moapp/jybase/mongodb"
	. "dataIdentify/db"
	. "dataIdentify/service"
	"fmt"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/os/gctx"
	"github.com/gogf/gf/v2/util/gconv"
	"github.com/xuri/excelize/v2"
	"log"
	"net/rpc"
	"sync"
	"testing"
)

//	{
//		$unset: {
//		"bid_commonwealth" : "",
//		"quote_mode" : "",
//		"model_dataidentify" : ""
//		}
//	}
func TestAddField(t *testing.T) {
	pool := make(chan bool, 5)
	wait := &sync.WaitGroup{}
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	it := sess.DB("qfw_data").C("wcj_bidding_20250708").Find(map[string]interface{}{}).Select(nil).Sort("-_id").Iter()
	index := 0
	for tm := make(map[string]interface{}); it.Next(tm); {
		index++
		if index%100 == 0 {
			log.Println("index", index)
		}
		pool <- true
		wait.Add(1)
		go func(m map[string]interface{}) {
			defer func() {
				<-pool
				wait.Done()
			}()
			href := "https://www.jianyu360.com/nologin/content/" + encrypt.CommonEncodeArticle("content", BsonIdToSId(m["_id"])) + ".html"
			Mgo_Main.Update("wcj_bidding_20250708", map[string]interface{}{
				"_id": m["_id"],
			}, map[string]interface{}{"$set": map[string]interface{}{
				"href": href,
			}}, false, false)
		}(tm)
		tm = make(map[string]interface{})
	}
	wait.Wait()
	log.Println("over...", index)
}

// 示例测试
func TestRule(t *testing.T) {
	Start("686bc7add5d8e4081f8b8766")
}
func TestAll(t *testing.T) {
	pool := make(chan bool, g.Config().MustGet(gctx.New(), "poolSize").Int())
	wait := &sync.WaitGroup{}
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	collection := "wcj_bidding_20250708"
	it := sess.DB("qfw_data").C(collection).Find(nil).Select(SelectField).Sort("-_id").Iter()
	index := 0
	for tm := make(map[string]interface{}); it.Next(tm); {
		index++
		if index%100 == 0 {
			log.Println("index", index)
		}
		pool <- true
		wait.Add(1)
		go func(m map[string]interface{}) {
			defer func() {
				<-pool
				wait.Done()
			}()
			_id := BsonIdToSId(m["_id"])
			flag, quoteMode, _, bidCommonwealth, _ := Pretreatment(_id, m, 0)
			if !flag {
				return
			}
			set := map[string]interface{}{}
			if quoteMode != "" {
				set["quote_mode"] = quoteMode
			}
			if bidCommonwealth != -1 {
				set["bid_commonwealth"] = bidCommonwealth
			}
			if len(set) > 0 {
				Mgo_Main.UpdateById(collection, m["_id"], map[string]interface{}{"$set": set})
			}
		}(tm)
		tm = make(map[string]interface{})
	}
	wait.Wait()
	log.Println("over...", index)
}

/* 返回结果：
 * map[中标联合体:否 报价模式:其他]
 * 中标联合体: 是/否
 * 报价模式：单价/费率/折扣率/正常报价/其他
 */
func TestRpc(t *testing.T) {
	conn, err := rpc.DialHTTP("tcp", "172.31.31.203:8811")
	if err != nil {
		log.Println(err)
		return
	}
	defer conn.Close()
	var reply map[string]string
	err = conn.Call("DataIdentify.Execute", "5b0dfeb4a5cb26b9b79c1330", &reply)
	if err != nil {
		log.Println(err)
	}
	log.Println(reply)
}
func TestDd(t *testing.T) {
	of, err := excelize.OpenFile("./报价模式+中标联合体验证.xlsx")
	if err != nil {
		log.Fatalln(err)
		return
	}
	// 读取指定工作表中的所有行。
	f := excelize.NewFile()
	sheetName := of.GetSheetList()[1]
	newSheetIndex, _ := f.NewSheet(sheetName)
	f.SetActiveSheet(newSheetIndex)
	rows, err := of.GetRows(sheetName)
	k := 0
	for ck, row := range rows {
		if ck != 0 {
			data, _ := Mgo_Main.FindById("bidding_hasdetail", row[0], `{"subtype":1}`)
			if data == nil || len(*data) == 0 {
				continue
			} else if subtype := gconv.String((*data)["subtype"]); subtype != "中标" && subtype != "成交" && subtype != "合同" {
				continue
			}
		}
		k++
		for kk, vv := range row {
			cell, _ := excelize.ColumnNumberToName(kk + 1)
			f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, k), vv)
		}
	}
	f.SaveAs("./新_报价模式+中标联合体验证.xlsx")
}

// 总共 300 报价模式不一致 86 中标联合体不一致 2
func TestCompare(t *testing.T) {
	// 打开一个已存在的Excel文件。
	of, err := excelize.OpenFile("./报价模式+中标联合体验证第二轮.xlsx")
	if err != nil {
		log.Fatalln(err)
		return
	}

	// 获取工作表的名称列表。
	sheets := of.GetSheetList()
	// 读取指定工作表中的所有行。
	f := excelize.NewFile()
	sheetName := "对比结果"
	newSheetIndex, _ := f.NewSheet(sheetName)
	f.SetActiveSheet(newSheetIndex)
	// 设置工作表的标题行
	headers := []string{"_id", "链接", "测试-报价模式", "抽取-报价模式", "测试-中标联合体", "抽取-中标联合体"}
	for colNum, header := range headers {
		cell, _ := excelize.ColumnNumberToName(colNum + 1)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, 1), header)
	}
	rowsTemp, err := of.GetRows(sheets[1])
	if err != nil {
		log.Fatalln(err)
	}
	//rows := [][]string{rowsTemp[0], rowsTemp[1]}
	rows := rowsTemp
	result := map[string]map[string]interface{}{}
	lock := &sync.Mutex{}
	pool := make(chan bool, 5)
	wait := &sync.WaitGroup{}
	var aEq, bEq float64
	oneFieldMap := map[string]float64{}
	oneFieldEqMap := map[string]float64{}
	var aBidModel, bBigModel int64
	for k, r := range rows {
		if k == 0 {
			continue
		}
		pool <- true
		wait.Add(1)
		go func(kk int, row []string) {
			defer func() {
				<-pool
				wait.Done()
			}()
			_, a, aa, b, bb := Start(row[0])
			if a == QuoteMode_Other {
				a = QuoteMode_Whole
			}
			bv := ""
			if b == 1 {
				bv = "是"
			} else if b == -1 {
				bv = "否"
			}
			lock.Lock()
			result[row[0]] = map[string]interface{}{
				"报价模式":      a,
				"报价模式_大模型":  aa,
				"中标联合体":     bv,
				"中标联合体_大模型": bb,
			}
			lock.Unlock()
		}(k, r)
	}
	wait.Wait()
	rowNum := 1
	for k, row := range rows {
		if k == 0 {
			continue
		}
		if row[4] == "" {
			row[4] = QuoteMode_Whole
		}
		obj := result[row[0]]
		a, b := obj["报价模式"], obj["中标联合体"]
		if gconv.Bool(obj["报价模式_大模型"]) {
			aBidModel++
		}
		if gconv.Bool(obj["中标联合体_大模型"]) {
			bBigModel++
		}
		oneFieldMap[row[4]]++
		if a == row[4] {
			aEq++
			oneFieldEqMap[row[4]]++
		}
		if b == row[6] {
			bEq++
		}
		if a == row[4] && b == row[6] {
			continue
		}
		rowNum++
		cell, _ := excelize.ColumnNumberToName(1)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[0])
		cell, _ = excelize.ColumnNumberToName(2)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[1])
		cell, _ = excelize.ColumnNumberToName(3)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[4])
		cell, _ = excelize.ColumnNumberToName(4)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), a)
		cell, _ = excelize.ColumnNumberToName(5)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[6])
		cell, _ = excelize.ColumnNumberToName(6)
		f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), b)
	}
	f.SaveAs("./对比结果.xlsx")
	countRows := float64(len(rows) - 1)
	oneFieldMsg := ""
	for k, v := range oneFieldMap {
		oneFieldMsg += fmt.Sprintf("报价模式-%s，总数：%.f，正确：%.f，正确率：%.2f%%", k, v, oneFieldEqMap[k], oneFieldEqMap[k]/v*100) + "\n"
	}
	log.Println("总数", countRows, "\n",
		fmt.Sprintf("报价模式，正确：%.f，正确率：%.2f%%", aEq, aEq/countRows*100), "\n",
		oneFieldMsg,
		fmt.Sprintf("中标联合，正确：%.f，正确率：%.2f%%", bEq, bEq/countRows*100), "\n",
		"报价模式大模型抽取", aBidModel, "中标联合体大模型抽取", bBigModel)
}