biddingindex.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729
  1. package main
  2. import (
  3. "encoding/json"
  4. "go.mongodb.org/mongo-driver/bson"
  5. "mongodb"
  6. "qfw/util/redis"
  7. "reflect"
  8. //"fmt"
  9. "log"
  10. mu "mfw/util"
  11. "net"
  12. qutil "qfw/util"
  13. elastic "qfw/util/elastic"
  14. "regexp"
  15. "strings"
  16. "sync"
  17. "time"
  18. )
  19. var date1 = regexp.MustCompile("20[0-2][0-9][年|\\-\\/|.][0-9]{1,2}[月|\\-|\\/|.][0-9]{1,2}[日]?")
  20. //对字段处理 bidamount budget
  21. //招标数据表和抽取表一一对应开始更新
  22. func biddingTask(data []byte, mapInfo map[string]interface{}, tasktype string) {
  23. defer qutil.Catch()
  24. q, _ := mapInfo["query"].(map[string]interface{})
  25. bkey, _ := mapInfo["bkey"].(string)
  26. if q == nil {
  27. q = map[string]interface{}{
  28. "_id": map[string]interface{}{
  29. "$gt": mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
  30. "$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
  31. },
  32. }
  33. }
  34. //logger.SetRollingDaily("./logs", "id.log")
  35. //连接信息
  36. c, _ := bidding["collect"].(string)
  37. extractc, _ := bidding["extractcollect"].(string)
  38. db, _ := bidding["db"].(string)
  39. extractdb, _ := bidding["extractdb"].(string)
  40. index, _ := bidding["index"].(string)
  41. itype, _ := bidding["type"].(string)
  42. //extract库
  43. extractsession := extractmgo.GetMgoConn()
  44. defer extractmgo.DestoryMongoConn(extractsession)
  45. extractquery := extractsession.DB(extractdb).C(extractc).Find(q).Sort("_id").Iter()
  46. eMap := map[string]map[string]interface{}{}
  47. extCount, repeatCount := 0, 0
  48. for tmp := make(map[string]interface{}); extractquery.Next(tmp); extCount++ {
  49. if qutil.IntAll(tmp["repeat"]) == 1 {
  50. repeatCount++
  51. }
  52. tid := mongodb.BsonIdToSId(tmp["_id"])
  53. eMap[tid] = tmp
  54. tmp = make(map[string]interface{})
  55. }
  56. //bidding库
  57. session := mgo.GetMgoConn()
  58. count, _ := session.DB(db).C(c).Find(&q).Count()
  59. log.Println("抽取表 重复数据量:", extCount, repeatCount)
  60. log.Println("查询语句:", q, "同步总数:", count, "elastic库:", index)
  61. n1, n2 := 0, 0
  62. if count < 200000 {
  63. var res []map[string]interface{}
  64. //res := make([]map[string]interface{}, 1)
  65. result := session.DB(db).C(c).Find(q).Select(map[string]interface{}{
  66. "projectinfo.attachment": 0,
  67. "contenthtml": 0,
  68. }).Iter()
  69. for tmp := make(map[string]interface{}); result.Next(tmp); {
  70. res = append(res, tmp)
  71. tmp = make(map[string]interface{})
  72. }
  73. mgo.DestoryMongoConn(session)
  74. log.Println("查询结果", "bidding:", count, "抽取:", extCount)
  75. if int64(len(res)) != count {
  76. time.Sleep(20 * time.Second)
  77. toadd := &net.UDPAddr{
  78. IP: net.ParseIP("127.0.0.1"),
  79. Port: qutil.IntAll(Sysconfig["udpport"]),
  80. }
  81. udpclient.WriteUdp(data, mu.OP_TYPE_DATA, toadd)
  82. }
  83. n1, n2 = doIndex(res, eMap, index, itype, db, c, bkey, tasktype)
  84. //if int64(n1 + n2) != count {
  85. // log.Println("任务错误,结果不一致")
  86. //}
  87. } else {
  88. log.Println("数据量太大,放弃!", count)
  89. mgo.DestoryMongoConn(session)
  90. }
  91. log.Println(mapInfo, "create bidding index...over", "all:", count, "bidding size:", n1, ",es size:", n2)
  92. if tasktype == "bidding_history" {
  93. qutil.Debug(tasktype)
  94. // 历史判重id段结束之后 生全量数据索引
  95. biddingDataTask(data, mapInfo)
  96. }
  97. }
  98. func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, index, itype, db, c, bkey, tasktype string) (int, int) {
  99. n1, n2 := 0, 0 //bidding数量,索引数量
  100. //线程池
  101. UpdatesLock := sync.Mutex{}
  102. fields := strings.Split(bidding["fields"].(string), ",")
  103. //更新数组
  104. arr := [][]map[string]interface{}{}
  105. arrEs := []map[string]interface{}{}
  106. //对比两张表数据,减少查询次数
  107. var compare map[string]interface{}
  108. log.Println("开始迭代..")
  109. for n, tmp := range infos {
  110. n1++
  111. if sensitive := qutil.ObjToString(tmp["sensitive"]); sensitive == "测试" || sensitive == "异常" { //bidding中有敏感词,不生索引
  112. tmp = make(map[string]interface{})
  113. continue
  114. }
  115. tid := mongodb.BsonIdToSId(tmp["_id"])
  116. //loginfo := make(map[string]interface{}) // 日志
  117. update := map[string]interface{}{} //要更新的mongo数据
  118. //对比方法----------------
  119. if eMap[tid] != nil {
  120. compare = eMap[tid]
  121. if tasktype == "bidding" {
  122. // 增量id段 正常数据
  123. if num := qutil.IntAll(compare["dataging"]); num == 1 { //extract中dataging=1跳过
  124. tmp = make(map[string]interface{})
  125. compare = nil
  126. continue
  127. }
  128. delete(eMap, tid)
  129. }
  130. if tasktype == "bidding_history" {
  131. //增量id段 历史数据
  132. if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
  133. tmp = make(map[string]interface{})
  134. compare = nil
  135. continue
  136. }
  137. delete(eMap, tid)
  138. }
  139. //更新bidding表,生成索引;bidding表modifyinfo中的字段不更新
  140. modifyinfo := make(map[string]bool)
  141. if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
  142. for k, _ := range tmpmodifyinfo {
  143. modifyinfo[k] = true
  144. }
  145. }
  146. //更新bidding表,生成索引
  147. for _, k := range fields {
  148. v1 := compare[k] //extract
  149. v2 := tmp[k] //bidding
  150. if v2 == nil && v1 != nil && !modifyinfo[k] {
  151. update[k] = v1
  152. } else if v2 != nil && v1 != nil && !modifyinfo[k] {
  153. //update[k+"_b"] = v2
  154. update[k] = v1
  155. } else if v2 != nil && v1 == nil {
  156. //update[k+"_b"] = v2
  157. if k == "area" || k == "city" || k == "district" {
  158. update[k] = ""
  159. }
  160. }
  161. }
  162. if qutil.IntAll(compare["repeat"]) == 1 {
  163. update["extracttype"] = -1
  164. } else {
  165. update["extracttype"] = 1
  166. }
  167. } else {
  168. compare = nil
  169. if qutil.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
  170. update["dataging"] = 0
  171. }
  172. }
  173. //下面可以多线程跑的--->
  174. //处理分类
  175. if compare != nil { //extract
  176. subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
  177. if subscopeclass != nil {
  178. //str := ","
  179. m1 := map[string]bool{}
  180. newclass := []string{}
  181. for _, sc := range subscopeclass {
  182. sclass, _ := sc.(string)
  183. if !m1[sclass] {
  184. m1[sclass] = true
  185. //str += sclass + ","
  186. newclass = append(newclass, sclass)
  187. }
  188. }
  189. update["s_subscopeclass"] = strings.Join(newclass, ",")
  190. update["subscopeclass"] = newclass
  191. }
  192. topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
  193. if topscopeclass != nil {
  194. m2 := map[string]bool{}
  195. newclass := []string{}
  196. for _, tc := range topscopeclass {
  197. tclass, _ := tc.(string)
  198. tclass = reg_letter.ReplaceAllString(tclass, "") // 去除字母
  199. if !m2[tclass] {
  200. m2[tclass] = true
  201. newclass = append(newclass, tclass)
  202. }
  203. }
  204. update["s_topscopeclass"] = strings.Join(newclass, ",")
  205. }
  206. if package1 := compare["package"]; package1 != nil {
  207. packageM, _ := package1.(map[string]interface{})
  208. for _, p := range packageM {
  209. pm, _ := p.(map[string]interface{})
  210. if qutil.ObjToString(pm["winner"]) != "" || qutil.Float64All(pm["budget"]) > 0 ||
  211. qutil.Float64All(pm["bidamount"]) > 0 {
  212. update["multipackage"] = 1
  213. break
  214. }
  215. }
  216. } else {
  217. update["multipackage"] = 0
  218. }
  219. compare = nil
  220. } else {
  221. area := qutil.ObjToString(tmp["area"])
  222. city := qutil.ObjToString(tmp["city"])
  223. district := qutil.ObjToString(tmp["district"])
  224. rdata := standardCheckCity(area, city, district)
  225. if len(rdata) > 0 {
  226. for k, v := range rdata {
  227. update[k] = v
  228. }
  229. }
  230. }
  231. //------------------对比结束
  232. //处理key descript
  233. if bkey == "" {
  234. DealInfo(&tmp, &update)
  235. }
  236. //同时保存到elastic
  237. for tk, tv := range update {
  238. tmp[tk] = tv
  239. }
  240. if tmp["s_winner"] != "" {
  241. sWinnerarr := strings.Split(qutil.ObjToString(tmp["s_winner"]), ",")
  242. var cid []string
  243. for _, w := range sWinnerarr {
  244. if w != "" {
  245. id := redis.GetStr("qyxy_id", w)
  246. if id == "" {
  247. ents, _ := mgostandard.Find("qyxy_std", bson.M{"company_name": w}, bson.M{"updatetime": -1}, nil, false, -1, -1)
  248. if len(*ents) > 0 {
  249. id = qutil.ObjToString((*ents)[0]["_id"])
  250. redis.PutCKV("qyxy_id", w, id)
  251. } else {
  252. ent, _ := qyxydb.FindOne("company_history_name", bson.M{"history_name": w})
  253. if len(*ent) > 0 {
  254. id = qutil.ObjToString((*ent)["company_id"])
  255. redis.PutCKV("qyxy_id", w, id)
  256. }
  257. }
  258. }
  259. if id == "" {
  260. id = "-"
  261. }
  262. cid = append(cid, id)
  263. //ent, _ := mgostandard.FindOne("qyxy_historyname", map[string]interface{}{"company_name": w})
  264. //if len(*ent) > 0 {
  265. // cid = append(cid, qutil.ObjToString((*ent)["company_id"]))
  266. //}else {
  267. // ent, _ = mgostandard.FindOne("qyxy_std", map[string]interface{}{"company_name": w})
  268. // if len(*ent) > 0 {
  269. // cid = append(cid, qutil.ObjToString((*ent)["_id"]))
  270. // }
  271. //}
  272. }
  273. }
  274. if len(cid) > 0 {
  275. tmp["entidlist"] = cid
  276. update["entidlist"] = cid
  277. tmp_up := []map[string]interface{}{}
  278. tmp_up = append(tmp_up, map[string]interface{}{"_id": tmp["_id"]})
  279. tmp_up = append(tmp_up, map[string]interface{}{"$set": map[string]interface{}{"entidlist": cid}})
  280. UpdataMgoCache <- tmp_up
  281. }
  282. }
  283. //对projectscope字段的索引处理
  284. ps, _ := tmp["projectscope"].(string)
  285. if len(ps) > ESLEN {
  286. tmp["projectscope"] = string(([]rune(ps))[:4000])
  287. }
  288. //对标的物为空处理
  289. if filetext := getFileText(tmp); len(filetext) > 10 { //attach_text
  290. // if site, _ := tmp["site"].(string); site == "中国招标投标公共服务平台" { //site:中国招标投标公共服务平台 detail替换成filetext 并加入标记filedetail=1
  291. // tmp["detail"] = filetext //更新es中detail
  292. // update["detail"] = filetext //更新mongo中detail
  293. // update["filedetail"] = 1 //mongo中打标记
  294. // }
  295. tmp["filetext"] = filetext
  296. }
  297. if purchasing, ok := tmp["purchasing"].(string); ok && purchasing == "" {
  298. delete(tmp, "purchasing")
  299. }
  300. if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok && len(purchasinglist) == 0 {
  301. delete(tmp, "purchasinglist")
  302. }
  303. //数据为空处理
  304. for _, f := range []string{"bidstatus", "city", "district", "channel"} {
  305. if fVal, ok := tmp[f].(string); ok && fVal == "" {
  306. delete(tmp, f)
  307. }
  308. }
  309. UpdatesLock.Lock()
  310. // for k1, _ := range tmp {
  311. // if strings.HasSuffix(k1, "_b") || k1 == "contenthtml" {
  312. // delete(tmp, k1)
  313. // }
  314. // }
  315. go IS.Add("bidding")
  316. if qutil.IntAll(update["extracttype"]) != -1 {
  317. n2++
  318. newTmp := map[string]interface{}{} //最终生索引的数据
  319. for field, ftype := range biddingIndexFieldsMap { //
  320. if tmp[field] != nil { //
  321. if field == "projectinfo" {
  322. mp, _ := tmp[field].(map[string]interface{})
  323. if mp != nil {
  324. newmap := map[string]interface{}{}
  325. for k, ktype := range projectinfoFieldsMap {
  326. mpv := mp[k]
  327. if mpv != nil && reflect.TypeOf(mpv).String() == ktype {
  328. newmap[k] = mp[k]
  329. }
  330. }
  331. if len(newmap) > 0 {
  332. newTmp[field] = newmap
  333. }
  334. }
  335. } else if field == "purchasinglist" { //标的物处理
  336. purchasinglist_new := []map[string]interface{}{}
  337. if pcl, _ := tmp[field].([]interface{}); len(pcl) > 0 {
  338. for _, ls := range pcl {
  339. lsm_new := make(map[string]interface{})
  340. lsm := ls.(map[string]interface{})
  341. for pf, pftype := range purchasinglistFieldsMap {
  342. lsmv := lsm[pf]
  343. if lsmv != nil && reflect.TypeOf(lsmv).String() == pftype {
  344. lsm_new[pf] = lsm[pf]
  345. }
  346. }
  347. if lsm_new != nil && len(lsm_new) > 0 {
  348. purchasinglist_new = append(purchasinglist_new, lsm_new)
  349. }
  350. }
  351. }
  352. if len(purchasinglist_new) > 0 {
  353. newTmp[field] = purchasinglist_new
  354. }
  355. } else if field == "winnerorder" { //中标候选
  356. winnerorder_new := []map[string]interface{}{}
  357. if winnerorder, _ := tmp[field].([]interface{}); len(winnerorder) > 0 {
  358. for _, win := range winnerorder {
  359. winMap_new := make(map[string]interface{})
  360. winMap := win.(map[string]interface{})
  361. for wf, wftype := range winnerorderlistFieldsMap {
  362. wfv := winMap[wf]
  363. if wfv != nil && reflect.TypeOf(wfv).String() == wftype {
  364. if wf == "sort" && qutil.Int64All(wfv) > 100 {
  365. continue
  366. }
  367. winMap_new[wf] = winMap[wf]
  368. }
  369. }
  370. if winMap_new != nil && len(winMap_new) > 0 {
  371. winnerorder_new = append(winnerorder_new, winMap_new)
  372. }
  373. }
  374. }
  375. if len(winnerorder_new) > 0 {
  376. newTmp[field] = winnerorder_new
  377. }
  378. } else if field == "qualifies" {
  379. //项目资质
  380. qs := []string{}
  381. if q, _ := tmp[field].([]interface{}); len(q) > 0 {
  382. for _, v := range q {
  383. v1 := v.(map[string]interface{})
  384. qs = append(qs, qutil.ObjToString(v1["key"]))
  385. }
  386. }
  387. if len(qs) > 0 {
  388. newTmp[field] = strings.Join(qs, ",")
  389. }
  390. } else if field == "detail" { //过滤
  391. detail, _ := tmp[field].(string)
  392. if len([]rune(detail)) > detailLength {
  393. detail = detail[:detailLength]
  394. }
  395. newTmp[field] = qutil.ObjToString(tmp["title"]) + " " + FilterDetail(detail)
  396. } else if field == "_id" || field == "topscopeclass" { //不做处理
  397. newTmp[field] = tmp[field]
  398. } else if field == "publishtime" || field == "comeintime" {
  399. //字段类型不正确,特别处理
  400. if tmp[field] != nil && qutil.Int64All(tmp[field]) > 0 {
  401. newTmp[field] = qutil.Int64All(tmp[field])
  402. }
  403. } else if field == "review_experts" {
  404. // 评审专家
  405. if arr, ok := tmp["review_experts"].([]interface{}); ok && len(arr) > 0 {
  406. arr1 := qutil.ObjArrToStringArr(arr)
  407. newTmp[field] = strings.Join(arr1, ",")
  408. }
  409. } else if field == "entidlist" {
  410. newTmp[field] = tmp[field]
  411. } else if field == "bidopentime" {
  412. if tmp[field] != nil && tmp["bidendtime"] == nil {
  413. newTmp["bidendtime"] = tmp[field]
  414. newTmp[field] = tmp[field]
  415. } else if tmp[field] == nil && tmp["bidendtime"] != nil {
  416. newTmp["bidendtime"] = tmp["bidendtime"]
  417. newTmp[field] = tmp["bidendtime"]
  418. } else {
  419. if tmp["bidopentime"] != nil {
  420. newTmp[field] = tmp["bidopentime"]
  421. }
  422. }
  423. } else { //其它字段判断数据类型,不正确舍弃
  424. if fieldval := tmp[field]; reflect.TypeOf(fieldval).String() != ftype {
  425. continue
  426. } else {
  427. if fieldval != "" {
  428. newTmp[field] = fieldval
  429. }
  430. }
  431. }
  432. }
  433. }
  434. YuceEndtime(newTmp) // 预测结果时间
  435. newTmp["createtime"] = time.Now().Unix() // es库数据创建时间,只有增量数据有
  436. if qutil.ObjToString(newTmp["spidercode"]) == "a_jyxxfbpt_gg" {
  437. // 剑鱼信息发布数据 通过udp通知信息发布程序
  438. go UdpMethod(mongodb.BsonIdToSId(newTmp["_id"]))
  439. }
  440. arrEs = append(arrEs, newTmp)
  441. }
  442. if len(update) > 0 {
  443. delete(update, "winnerorder") //winnerorder不需要更新到bindding表,删除
  444. arr = append(arr, []map[string]interface{}{
  445. {"_id": tmp["_id"]},
  446. {"$set": update},
  447. })
  448. }
  449. if len(arr) >= BulkSize-1 {
  450. mgo.UpdateBulkAll(db, c, arr...)
  451. arr = [][]map[string]interface{}{}
  452. }
  453. if len(arrEs) >= BulkSize-1 {
  454. tmps := arrEs
  455. if StopFlag {
  456. qutil.Debug("es队列紧张,暂停10s执行")
  457. time.Sleep(time.Second * 10)
  458. }
  459. elastic.BulkSave(index, itype, &tmps, true)
  460. if other_index != "" && other_itype != "" {
  461. elastic.BulkSave(other_index, other_itype, &tmps, true)
  462. }
  463. if len(multiIndex) == 2 {
  464. elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
  465. }
  466. arrEs = []map[string]interface{}{}
  467. }
  468. UpdatesLock.Unlock()
  469. if n%100 == 0 {
  470. log.Println("current:", n)
  471. }
  472. tmp = make(map[string]interface{})
  473. }
  474. UpdatesLock.Lock()
  475. if len(arr) > 0 {
  476. mgo.UpdateBulkAll(db, c, arr...)
  477. }
  478. if len(arrEs) > 0 {
  479. tmps := arrEs
  480. if StopFlag {
  481. qutil.Debug("es队列紧张,暂停10s执行")
  482. time.Sleep(time.Second * 10)
  483. }
  484. elastic.BulkSave(index, itype, &tmps, true)
  485. if other_index != "" && other_itype != "" {
  486. bidding_other_es.BulkSave(other_index, other_itype, &tmps, true)
  487. }
  488. if len(multiIndex) == 2 {
  489. elastic.BulkSave(multiIndex[0], multiIndex[1], &tmps, true)
  490. }
  491. }
  492. UpdatesLock.Unlock()
  493. return n1, n2
  494. }
  495. var client *mu.Client
  496. var reg = regexp.MustCompile("^[0-9a-zA-Z-.]+$")
  497. var reg_space = regexp.MustCompile("(?ism)(<style.*?>.*?</style>)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|( +)|(" + string(rune(160)) + "+)")
  498. var reg_row = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+")
  499. var reg_dh = regexp.MustCompile("[,]+")
  500. var reg_newdb = regexp.MustCompile("([:,、:,。.;])[,]")
  501. var reg_no = regexp.MustCompile("^[0-9]*$")
  502. var reg_letter = regexp.MustCompile("[a-z]*")
  503. var MSG_SERVER = "123.56.236.148:7070"
  504. var DesLen = 120
  505. func inits() {
  506. ser := qutil.ObjToString(Sysconfig["msg_server"])
  507. if ser != "" {
  508. MSG_SERVER = ser
  509. }
  510. cf := &mu.ClientConfig{
  511. ClientName: "剑鱼抽关键词",
  512. EventHandler: func(p *mu.Packet) {},
  513. MsgServerAddr: MSG_SERVER,
  514. CanHandleEvents: []int{},
  515. OnConnectSuccess: func() {
  516. log.Println("c.")
  517. },
  518. ReadBufferSize: 10,
  519. WriteBufferSize: 10,
  520. }
  521. client, _ = mu.NewClient(cf)
  522. }
  523. //var clientlock = &sync.Mutex{}
  524. var keypool = make(chan bool, 1)
  525. func DealInfo(obj, update *map[string]interface{}) {
  526. defer qutil.Catch()
  527. if (*obj)["keywords"] != nil && (*obj)["description"] != nil {
  528. return
  529. } else {
  530. (*update)["keywords"] = ""
  531. (*update)["description"] = ""
  532. }
  533. title := qutil.ObjToString((*obj)["title"])
  534. var m [][]string
  535. select {
  536. case <-func() <-chan bool {
  537. ch := make(chan bool, 1)
  538. go func(chan bool) {
  539. select {
  540. case keypool <- true:
  541. defer func() {
  542. <-keypool
  543. }()
  544. ret, _ := client.Call("", mu.UUID(8), 4010, mu.SENDTO_TYPE_RAND_RECIVER, title, 1)
  545. json.Unmarshal(ret, &m)
  546. case <-time.After(10 * time.Millisecond):
  547. }
  548. ch <- true
  549. }(ch)
  550. return ch
  551. }():
  552. case <-time.After(40 * time.Millisecond):
  553. }
  554. arr := []string{}
  555. keyword := []string{}
  556. keywordnew := []string{}
  557. for _, tmp := range m {
  558. if reg.MatchString(tmp[0]) {
  559. arr = append(arr, tmp[0])
  560. } else {
  561. if len(arr) > 0 {
  562. str := strings.Join(arr, "")
  563. keyword = append(keyword, str)
  564. arr = []string{}
  565. }
  566. if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) {
  567. keyword = append(keyword, tmp[0])
  568. }
  569. }
  570. }
  571. for _, v := range keyword {
  572. v = reg_no.ReplaceAllString(v, "")
  573. if len(v) > 0 {
  574. keywordnew = append(keywordnew, v)
  575. }
  576. }
  577. keywords := strings.Join(keywordnew, ",")
  578. (*update)["keywords"] = keywords
  579. content := ""
  580. if (*obj)["detail_bak"] != nil {
  581. content = qutil.ObjToString((*obj)["detail_bak"])
  582. } else {
  583. content = qutil.ObjToString((*obj)["detail"])
  584. }
  585. //内容替换
  586. content = strings.Replace(content, " ", "", -1)
  587. content = reg_space.ReplaceAllString(content, "")
  588. content = reg_row.ReplaceAllString(content, ",")
  589. content = reg_dh.ReplaceAllString(content, ",")
  590. content = reg_newdb.ReplaceAllString(content, "$1")
  591. if strings.HasPrefix(content, ",") {
  592. content = content[1:]
  593. }
  594. //log.Println(content)
  595. tc := []rune(content)
  596. ltc := len(tc)
  597. description := content
  598. if ltc > DesLen {
  599. description = string(tc[:DesLen])
  600. }
  601. (*update)["description"] = description
  602. //保存到数据库
  603. return
  604. }
  605. // 预测结果时间
  606. func YuceEndtime(tmp map[string]interface{}) {
  607. flag := true
  608. scope := []string{"服务采购_法律咨询", "服务采购_会计", "服务采购_物业", "服务采购_审计", "服务采购_安保", "服务采购_仓储物流",
  609. "服务采购_广告宣传印刷"}
  610. subscopeclass := qutil.ObjToString(tmp["s_subscopeclass"])
  611. for _, v := range scope {
  612. if strings.Contains(subscopeclass, v) {
  613. flag = false
  614. break
  615. }
  616. }
  617. if flag {
  618. return
  619. }
  620. subtype := qutil.ObjToString(tmp["subtype"])
  621. if subtype == "成交" || subtype == "合同" {
  622. // yucestarttime、yuceendtime
  623. yucestarttime, yuceendtime := int64(0), int64(0)
  624. // 项目周期中
  625. if qutil.ObjToString(tmp["projectperiod"]) != "" {
  626. dateStr := date1.FindStringSubmatch(qutil.ObjToString(tmp["projectperiod"]))
  627. if len(dateStr) == 2 {
  628. sdate := FormatDateStr(dateStr[0])
  629. edate := FormatDateStr(dateStr[1])
  630. if sdate < edate && sdate != 0 && edate != 0 {
  631. yucestarttime = sdate
  632. yuceendtime = edate
  633. }
  634. }
  635. }
  636. if yucestarttime > 0 && yuceendtime > yucestarttime {
  637. tmp["yuceendtime"] = yuceendtime
  638. return
  639. }
  640. // 预测开始时间 合同签订日期
  641. if yucestarttime == 0 {
  642. if qutil.IntAll(tmp["signaturedate"]) <= 0 {
  643. if qutil.IntAll(tmp["publishtime"]) <= 0 {
  644. return
  645. } else {
  646. yucestarttime = qutil.Int64All(tmp["publishtime"])
  647. }
  648. } else {
  649. yucestarttime = qutil.Int64All(tmp["signaturedate"])
  650. }
  651. }
  652. // 预测结束时间
  653. if yucestarttime > 0 && yuceendtime == 0 {
  654. if qutil.IntAll(tmp["project_duration"]) > 0 && qutil.ObjToString(tmp["project_timeunit"]) != "" {
  655. yuceendtime = YcEndTime(yucestarttime, qutil.IntAll(tmp["project_duration"]), qutil.ObjToString(tmp["project_timeunit"]))
  656. tmp["yuceendtime"] = yuceendtime
  657. }
  658. }
  659. }
  660. }
  661. func YcEndTime(starttime int64, num int, unit string) int64 {
  662. yuceendtime := int64(0)
  663. if unit == "日历天" || unit == "天" || unit == "日" {
  664. yuceendtime = starttime + int64(num*86400)
  665. } else if unit == "周" {
  666. yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num*7).Unix()
  667. } else if unit == "月" {
  668. yuceendtime = time.Unix(starttime, 0).AddDate(0, num, 0).Unix()
  669. } else if unit == "年" {
  670. yuceendtime = time.Unix(starttime, 0).AddDate(num, 0, 0).Unix()
  671. } else if unit == "工作日" {
  672. n := num / 7 * 2
  673. yuceendtime = time.Unix(starttime, 0).AddDate(0, 0, num+n).Unix()
  674. }
  675. return yuceendtime
  676. }
  677. func FormatDateStr(ds string) int64 {
  678. ds = strings.Replace(ds, "年", "-", -1)
  679. ds = strings.Replace(ds, "月", "-", -1)
  680. ds = strings.Replace(ds, "日", "", -1)
  681. ds = strings.Replace(ds, "/", "-", -1)
  682. ds = strings.Replace(ds, ".", "-", -1)
  683. location, err := time.ParseInLocation(qutil.Date_Short_Layout, ds, time.Local)
  684. if err != nil {
  685. qutil.Debug(err)
  686. return 0
  687. } else {
  688. return location.Unix()
  689. }
  690. }
  691. type Request struct {
  692. InfoId string
  693. }
  694. type Response struct {
  695. Rep []map[string]interface{}
  696. }
  697. // @Description rpc调用信息发布程序接口
  698. // @Author J 2022/4/13 9:13 AM
  699. func UdpMethod(id string) {
  700. mapinfo := map[string]interface{}{
  701. "infoid": id,
  702. "stype": "jyfb_data_over",
  703. }
  704. datas, _ := json.Marshal(mapinfo)
  705. qutil.Debug(JyUdpAddr, string(datas))
  706. _ = udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, JyUdpAddr)
  707. }