main_test.go 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274
  1. package main
  2. import (
  3. "app.yhyue.com/moapp/jybase/encrypt"
  4. . "app.yhyue.com/moapp/jybase/mongodb"
  5. . "dataIdentify/db"
  6. . "dataIdentify/service"
  7. "fmt"
  8. "github.com/gogf/gf/v2/frame/g"
  9. "github.com/gogf/gf/v2/os/gctx"
  10. "github.com/gogf/gf/v2/util/gconv"
  11. "github.com/xuri/excelize/v2"
  12. "log"
  13. "net/rpc"
  14. "sync"
  15. "testing"
  16. )
  17. // {
  18. // $unset: {
  19. // "bid_commonwealth" : "",
  20. // "quote_mode" : "",
  21. // "model_dataidentify" : ""
  22. // }
  23. // }
  24. func TestAddField(t *testing.T) {
  25. pool := make(chan bool, 5)
  26. wait := &sync.WaitGroup{}
  27. sess := Mgo_Main.GetMgoConn()
  28. defer Mgo_Main.DestoryMongoConn(sess)
  29. it := sess.DB("qfw_data").C("wcj_bidding_20250708").Find(map[string]interface{}{}).Select(nil).Sort("-_id").Iter()
  30. index := 0
  31. for tm := make(map[string]interface{}); it.Next(tm); {
  32. index++
  33. if index%100 == 0 {
  34. log.Println("index", index)
  35. }
  36. pool <- true
  37. wait.Add(1)
  38. go func(m map[string]interface{}) {
  39. defer func() {
  40. <-pool
  41. wait.Done()
  42. }()
  43. href := "https://www.jianyu360.com/nologin/content/" + encrypt.CommonEncodeArticle("content", BsonIdToSId(m["_id"])) + ".html"
  44. Mgo_Main.Update("wcj_bidding_20250708", map[string]interface{}{
  45. "_id": m["_id"],
  46. }, map[string]interface{}{"$set": map[string]interface{}{
  47. "href": href,
  48. }}, false, false)
  49. }(tm)
  50. tm = make(map[string]interface{})
  51. }
  52. wait.Wait()
  53. log.Println("over...", index)
  54. }
  55. // 示例测试
  56. func TestRule(t *testing.T) {
  57. Start("686bc7add5d8e4081f8b8766")
  58. }
  59. func TestAll(t *testing.T) {
  60. pool := make(chan bool, g.Config().MustGet(gctx.New(), "poolSize").Int())
  61. wait := &sync.WaitGroup{}
  62. sess := Mgo_Main.GetMgoConn()
  63. defer Mgo_Main.DestoryMongoConn(sess)
  64. collection := "wcj_bidding_20250708"
  65. it := sess.DB("qfw_data").C(collection).Find(nil).Select(SelectField).Sort("-_id").Iter()
  66. index := 0
  67. for tm := make(map[string]interface{}); it.Next(tm); {
  68. index++
  69. if index%100 == 0 {
  70. log.Println("index", index)
  71. }
  72. pool <- true
  73. wait.Add(1)
  74. go func(m map[string]interface{}) {
  75. defer func() {
  76. <-pool
  77. wait.Done()
  78. }()
  79. _id := BsonIdToSId(m["_id"])
  80. flag, quoteMode, _, bidCommonwealth, _ := Pretreatment(_id, m, 0)
  81. if !flag {
  82. return
  83. }
  84. set := map[string]interface{}{}
  85. if quoteMode != "" {
  86. set["quote_mode"] = quoteMode
  87. }
  88. if bidCommonwealth != -1 {
  89. set["bid_commonwealth"] = bidCommonwealth
  90. }
  91. if len(set) > 0 {
  92. Mgo_Main.UpdateById(collection, m["_id"], map[string]interface{}{"$set": set})
  93. }
  94. }(tm)
  95. tm = make(map[string]interface{})
  96. }
  97. wait.Wait()
  98. log.Println("over...", index)
  99. }
  100. /* 返回结果:
  101. * map[中标联合体:否 报价模式:其他]
  102. * 中标联合体: 是/否
  103. * 报价模式:单价/费率/折扣率/正常报价/其他
  104. */
  105. func TestRpc(t *testing.T) {
  106. conn, err := rpc.DialHTTP("tcp", "172.31.31.203:8811")
  107. if err != nil {
  108. log.Println(err)
  109. return
  110. }
  111. defer conn.Close()
  112. var reply map[string]string
  113. err = conn.Call("DataIdentify.Execute", "5b0dfeb4a5cb26b9b79c1330", &reply)
  114. if err != nil {
  115. log.Println(err)
  116. }
  117. log.Println(reply)
  118. }
  119. func TestDd(t *testing.T) {
  120. of, err := excelize.OpenFile("./报价模式+中标联合体验证.xlsx")
  121. if err != nil {
  122. log.Fatalln(err)
  123. return
  124. }
  125. // 读取指定工作表中的所有行。
  126. f := excelize.NewFile()
  127. sheetName := of.GetSheetList()[1]
  128. newSheetIndex, _ := f.NewSheet(sheetName)
  129. f.SetActiveSheet(newSheetIndex)
  130. rows, err := of.GetRows(sheetName)
  131. k := 0
  132. for ck, row := range rows {
  133. if ck != 0 {
  134. data, _ := Mgo_Main.FindById("bidding_hasdetail", row[0], `{"subtype":1}`)
  135. if data == nil || len(*data) == 0 {
  136. continue
  137. } else if subtype := gconv.String((*data)["subtype"]); subtype != "中标" && subtype != "成交" && subtype != "合同" {
  138. continue
  139. }
  140. }
  141. k++
  142. for kk, vv := range row {
  143. cell, _ := excelize.ColumnNumberToName(kk + 1)
  144. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, k), vv)
  145. }
  146. }
  147. f.SaveAs("./新_报价模式+中标联合体验证.xlsx")
  148. }
  149. // 总共 300 报价模式不一致 86 中标联合体不一致 2
  150. func TestCompare(t *testing.T) {
  151. // 打开一个已存在的Excel文件。
  152. of, err := excelize.OpenFile("./报价模式+中标联合体验证第二轮.xlsx")
  153. if err != nil {
  154. log.Fatalln(err)
  155. return
  156. }
  157. // 获取工作表的名称列表。
  158. sheets := of.GetSheetList()
  159. // 读取指定工作表中的所有行。
  160. f := excelize.NewFile()
  161. sheetName := "对比结果"
  162. newSheetIndex, _ := f.NewSheet(sheetName)
  163. f.SetActiveSheet(newSheetIndex)
  164. // 设置工作表的标题行
  165. headers := []string{"_id", "链接", "测试-报价模式", "抽取-报价模式", "测试-中标联合体", "抽取-中标联合体"}
  166. for colNum, header := range headers {
  167. cell, _ := excelize.ColumnNumberToName(colNum + 1)
  168. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, 1), header)
  169. }
  170. rowsTemp, err := of.GetRows(sheets[1])
  171. if err != nil {
  172. log.Fatalln(err)
  173. }
  174. //rows := [][]string{rowsTemp[0], rowsTemp[1]}
  175. rows := rowsTemp
  176. result := map[string]map[string]interface{}{}
  177. lock := &sync.Mutex{}
  178. pool := make(chan bool, 5)
  179. wait := &sync.WaitGroup{}
  180. var aEq, bEq float64
  181. oneFieldMap := map[string]float64{}
  182. oneFieldEqMap := map[string]float64{}
  183. var aBidModel, bBigModel int64
  184. for k, r := range rows {
  185. if k == 0 {
  186. continue
  187. }
  188. pool <- true
  189. wait.Add(1)
  190. go func(kk int, row []string) {
  191. defer func() {
  192. <-pool
  193. wait.Done()
  194. }()
  195. _, a, aa, b, bb := Start(row[0])
  196. if a == QuoteMode_Other {
  197. a = QuoteMode_Whole
  198. }
  199. bv := ""
  200. if b == 1 {
  201. bv = "是"
  202. } else if b == -1 {
  203. bv = "否"
  204. }
  205. lock.Lock()
  206. result[row[0]] = map[string]interface{}{
  207. "报价模式": a,
  208. "报价模式_大模型": aa,
  209. "中标联合体": bv,
  210. "中标联合体_大模型": bb,
  211. }
  212. lock.Unlock()
  213. }(k, r)
  214. }
  215. wait.Wait()
  216. rowNum := 1
  217. for k, row := range rows {
  218. if k == 0 {
  219. continue
  220. }
  221. if row[4] == "" {
  222. row[4] = QuoteMode_Whole
  223. }
  224. obj := result[row[0]]
  225. a, b := obj["报价模式"], obj["中标联合体"]
  226. if gconv.Bool(obj["报价模式_大模型"]) {
  227. aBidModel++
  228. }
  229. if gconv.Bool(obj["中标联合体_大模型"]) {
  230. bBigModel++
  231. }
  232. oneFieldMap[row[4]]++
  233. if a == row[4] {
  234. aEq++
  235. oneFieldEqMap[row[4]]++
  236. }
  237. if b == row[6] {
  238. bEq++
  239. }
  240. if a == row[4] && b == row[6] {
  241. continue
  242. }
  243. rowNum++
  244. cell, _ := excelize.ColumnNumberToName(1)
  245. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[0])
  246. cell, _ = excelize.ColumnNumberToName(2)
  247. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[1])
  248. cell, _ = excelize.ColumnNumberToName(3)
  249. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[4])
  250. cell, _ = excelize.ColumnNumberToName(4)
  251. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), a)
  252. cell, _ = excelize.ColumnNumberToName(5)
  253. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), row[6])
  254. cell, _ = excelize.ColumnNumberToName(6)
  255. f.SetCellValue(sheetName, fmt.Sprintf("%s%d", cell, rowNum), b)
  256. }
  257. f.SaveAs("./对比结果.xlsx")
  258. countRows := float64(len(rows) - 1)
  259. oneFieldMsg := ""
  260. for k, v := range oneFieldMap {
  261. oneFieldMsg += fmt.Sprintf("报价模式-%s,总数:%.f,正确:%.f,正确率:%.2f%%", k, v, oneFieldEqMap[k], oneFieldEqMap[k]/v*100) + "\n"
  262. }
  263. log.Println("总数", countRows, "\n",
  264. fmt.Sprintf("报价模式,正确:%.f,正确率:%.2f%%", aEq, aEq/countRows*100), "\n",
  265. oneFieldMsg,
  266. fmt.Sprintf("中标联合,正确:%.f,正确率:%.2f%%", bEq, bEq/countRows*100), "\n",
  267. "报价模式大模型抽取", aBidModel, "中标联合体大模型抽取", bBigModel)
  268. }