task.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. package main
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "log"
  6. mu "mfw/util"
  7. "qfw/util"
  8. "regexp"
  9. "strings"
  10. "sync"
  11. "time"
  12. "github.com/robfig/cron"
  13. "go.mongodb.org/mongo-driver/bson/primitive"
  14. )
  15. /**
  16. 任务入口
  17. 全量、增量合并
  18. 更新、插入,内存清理
  19. 转换成info对象
  20. **/
  21. //项目合并对象
  22. type ProjectTask struct {
  23. InitMinTime int64 //最小时间,小于0的处理一次
  24. name string
  25. thread int //线程数
  26. //查找锁
  27. findLock sync.Mutex
  28. wg sync.WaitGroup
  29. //map锁
  30. AllIdsMapLock sync.Mutex
  31. //对应的id
  32. AllIdsMap map[string]*ID
  33. //采购单位、项目名称、项目编号
  34. mapPb, mapPn, mapPc map[string]*Key
  35. //流程数据 字段相同,直接合并
  36. mapHref map[string]string
  37. mapHrefLock sync.Mutex
  38. //站点
  39. mapSite map[string]*Site
  40. mapSiteLock sync.Mutex
  41. //bidtype、bidstatus 锁
  42. mapBidLock sync.Mutex
  43. //更新或新增通道
  44. updatePool chan []map[string]interface{}
  45. //savePool chan map[string]interface{}
  46. //saveSign, updateSign chan bool
  47. //表名
  48. coll string
  49. //当前状态是全量还是增量
  50. currentType string //当前是跑全量还是跑增量
  51. //
  52. clearContimes int
  53. //当前时间
  54. currentTime int64
  55. //保存长度
  56. saveSize int
  57. pici int64
  58. validTime int64
  59. statusTime int64
  60. // LockPool chan *sync.Mutex
  61. // LockPoolLock sync.Mutex
  62. // m1, m23, m4 map[int]int
  63. // l1, l23, l4 map[int]*sync.Mutex
  64. Brun bool
  65. }
  66. func NewPT() *ProjectTask {
  67. p := &ProjectTask{
  68. InitMinTime: int64(1325347200),
  69. name: "全/增量对象",
  70. thread: 4,
  71. updatePool: make(chan []map[string]interface{}, 5000),
  72. //savePool: make(chan map[string]interface{}, 2000),
  73. wg: sync.WaitGroup{},
  74. AllIdsMap: make(map[string]*ID, 5000000),
  75. mapPb: make(map[string]*Key, 1500000),
  76. mapPn: make(map[string]*Key, 5000000),
  77. mapPc: make(map[string]*Key, 5000000),
  78. mapHref: make(map[string]string, 1500000),
  79. mapSite: make(map[string]*Site, 1000000),
  80. saveSize: 100,
  81. //saveSign: make(chan bool, 1),
  82. //updateSign: make(chan bool, 1),
  83. coll: ProjectColl,
  84. validTime: int64(util.IntAllDef(Sysconfig["validdays"], 150) * 86400),
  85. statusTime: int64(util.IntAllDef(Sysconfig["statusdays"], 7) * 86400),
  86. }
  87. return p
  88. }
  89. var P_QL *ProjectTask
  90. var sp = make(chan bool, 5)
  91. //初始化全量合并对象
  92. func init() {
  93. P_QL = NewPT()
  94. log.Println(len(P_QL.updatePool))
  95. go P_QL.updateAllQueue()
  96. go P_QL.clearMem()
  97. }
  98. func (p *ProjectTask) updateAllQueue() {
  99. arru := make([][]map[string]interface{}, p.saveSize)
  100. indexu := 0
  101. for {
  102. select {
  103. case v := <-p.updatePool:
  104. arru[indexu] = v
  105. indexu++
  106. if indexu == p.saveSize {
  107. sp <- true
  108. go func(arru [][]map[string]interface{}) {
  109. defer func() {
  110. <-sp
  111. }()
  112. MongoTool.UpSertBulk(p.coll, arru...)
  113. }(arru)
  114. arru = make([][]map[string]interface{}, p.saveSize)
  115. indexu = 0
  116. }
  117. case <-time.After(1000 * time.Millisecond):
  118. if indexu > 0 {
  119. sp <- true
  120. go func(arru [][]map[string]interface{}) {
  121. defer func() {
  122. <-sp
  123. }()
  124. MongoTool.UpSertBulk(p.coll, arru...)
  125. }(arru[:indexu])
  126. arru = make([][]map[string]interface{}, p.saveSize)
  127. indexu = 0
  128. }
  129. }
  130. }
  131. }
  132. //项目合并内存更新
  133. func (p *ProjectTask) clearMem() {
  134. c := cron.New()
  135. //在内存中保留最近6个月的信息
  136. //跑全量时每5分钟跑一次,跑增量时400分钟跑一次
  137. _ = c.AddFunc("50 0/5 * * * *", func() {
  138. if (p.currentType == "ql" && SingleClear == 0) || p.clearContimes >= 80 {
  139. SingleClear = 1
  140. //跳过的次数清零
  141. p.clearContimes = 0
  142. //信息进入查找对比全局锁
  143. p.findLock.Lock()
  144. //defer p.findLock.Unlock()
  145. //合并进行的任务都完成
  146. p.wg.Wait()
  147. //遍历id
  148. //所有内存中的项目信息
  149. p.AllIdsMapLock.Lock()
  150. p.mapHrefLock.Lock()
  151. log.Println("清除开始")
  152. //清除计数
  153. clearNum := 0
  154. for kHref, pid := range p.mapHref { //删除mapHref,p.AllIdsMap删除之前执行
  155. v := p.AllIdsMap[pid]
  156. if p.currentTime-v.P.LastTime > p.validTime {
  157. delete(p.mapHref, kHref)
  158. }
  159. }
  160. for k, v := range p.AllIdsMap {
  161. if p.currentTime-v.P.LastTime > p.validTime {
  162. clearNum++
  163. //删除id的map
  164. delete(p.AllIdsMap, k)
  165. //删除pb
  166. if v.P.Buyer != "" {
  167. ids := p.mapPb[v.P.Buyer]
  168. if ids != nil {
  169. ids.Lock.Lock()
  170. ids.Arr = deleteSlice(ids.Arr, k, "pb")
  171. if len(ids.Arr) == 0 {
  172. delete(p.mapPb, v.P.Buyer)
  173. }
  174. ids.Lock.Unlock()
  175. }
  176. }
  177. //删除mapPn
  178. for _, vn := range append([]string{v.P.ProjectName}, v.P.MPN...) {
  179. if vn != "" {
  180. ids := p.mapPn[vn]
  181. if ids != nil {
  182. ids.Lock.Lock()
  183. ids.Arr = deleteSlice(ids.Arr, k, "pn")
  184. if len(ids.Arr) == 0 {
  185. delete(p.mapPn, vn)
  186. }
  187. ids.Lock.Unlock()
  188. }
  189. }
  190. }
  191. //删除mapPc
  192. for _, vn := range append([]string{v.P.ProjectCode}, v.P.MPC...) {
  193. if vn != "" {
  194. ids := p.mapPc[vn]
  195. if ids != nil {
  196. ids.Lock.Lock()
  197. ids.Arr = deleteSlice(ids.Arr, k, "pc")
  198. if len(ids.Arr) == 0 {
  199. delete(p.mapPc, vn)
  200. }
  201. ids.Lock.Unlock()
  202. }
  203. }
  204. }
  205. v = nil
  206. }
  207. }
  208. p.mapHrefLock.Unlock()
  209. p.AllIdsMapLock.Unlock()
  210. p.findLock.Unlock()
  211. SingleClear = 0
  212. log.Println("清除完成:", clearNum, len(p.AllIdsMap), len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref))
  213. } else {
  214. p.clearContimes++
  215. }
  216. })
  217. c.Start()
  218. select {}
  219. }
  220. //全量合并
  221. func (p *ProjectTask) taskQl(udpInfo map[string]interface{}) {
  222. defer util.Catch()
  223. //1、检查pubilshtime索引
  224. db, _ := udpInfo["db"].(string)
  225. if db == "" {
  226. db = MongoTool.DbName
  227. }
  228. coll, _ := udpInfo["coll"].(string)
  229. if coll == "" {
  230. coll = ExtractColl
  231. }
  232. thread := util.IntAllDef(Thread, 4)
  233. if thread > 0 {
  234. p.thread = thread
  235. }
  236. q, _ := udpInfo["query"].(map[string]interface{})
  237. if q == nil {
  238. q = map[string]interface{}{}
  239. lteid, _ := udpInfo["lteid"].(string)
  240. var idmap map[string]interface{}
  241. if len(lteid) > 15 {
  242. idmap = map[string]interface{}{
  243. "$lte": StringTOBsonId(lteid),
  244. }
  245. }
  246. gtid, _ := udpInfo["gtid"].(string)
  247. if len(gtid) > 15 {
  248. if idmap == nil {
  249. idmap = map[string]interface{}{}
  250. }
  251. idmap["$gte"] = StringTOBsonId(gtid)
  252. }
  253. if idmap != nil {
  254. q["_id"] = idmap
  255. }
  256. }
  257. //生成查询语句执行
  258. log.Println("查询语句:", q)
  259. p.enter(db, coll, q)
  260. }
  261. //增量合并
  262. func (p *ProjectTask) taskZl(udpInfo map[string]interface{}) {
  263. defer util.Catch()
  264. //1、检查pubilshtime索引
  265. db, _ := udpInfo["db"].(string)
  266. if db == "" {
  267. db = MongoTool.DbName
  268. }
  269. coll, _ := udpInfo["coll"].(string)
  270. if coll == "" {
  271. coll = ExtractColl
  272. }
  273. thread := util.IntAllDef(Thread, 4)
  274. if thread > 0 {
  275. p.thread = thread
  276. }
  277. //开始id和结束id
  278. q, _ := udpInfo["query"].(map[string]interface{})
  279. gtid := udpInfo["gtid"].(string)
  280. lteid := udpInfo["lteid"].(string)
  281. if q == nil {
  282. q = map[string]interface{}{
  283. "_id": map[string]interface{}{
  284. "$gt": StringTOBsonId(gtid),
  285. "$lte": StringTOBsonId(lteid),
  286. },
  287. }
  288. }
  289. if q != nil {
  290. //生成查询语句执行
  291. p.enter(db, coll, q)
  292. }
  293. if udpInfo["stop"] == nil {
  294. for i := 0; i < 5; i++ {
  295. sp <- true
  296. }
  297. for i := 0; i < 5; i++ {
  298. <-sp
  299. }
  300. log.Println("保存完成,生索引", p.pici)
  301. time.Sleep(5 * time.Second)
  302. nextNode(udpInfo, p.pici)
  303. }
  304. }
  305. //招标字段更新
  306. func (p *ProjectTask) taskUpdateInfo(udpInfo map[string]interface{}) {
  307. defer util.Catch()
  308. db, _ := udpInfo["db"].(string)
  309. if db == "" {
  310. db = MongoTool.DbName
  311. }
  312. coll, _ := udpInfo["coll"].(string)
  313. if coll == "" {
  314. coll = ExtractColl
  315. }
  316. thread := util.IntAllDef(Thread, 4)
  317. if thread > 0 {
  318. p.thread = thread
  319. }
  320. q, _ := udpInfo["query"].(map[string]interface{})
  321. gtid := udpInfo["gtid"].(string)
  322. lteid := udpInfo["lteid"].(string)
  323. if q == nil {
  324. q = map[string]interface{}{
  325. "_id": map[string]interface{}{
  326. "$gte": StringTOBsonId(gtid),
  327. "$lte": StringTOBsonId(lteid),
  328. },
  329. "is_m": 1,
  330. }
  331. }
  332. log.Println("查询语句:", q)
  333. p.enter(db, coll, q)
  334. }
  335. func StringTOBsonId(id string) primitive.ObjectID {
  336. objectId, _ := primitive.ObjectIDFromHex(id)
  337. return objectId
  338. }
  339. //通知下个节点nextNode
  340. func nextNode(mapInfo map[string]interface{}, pici int64) {
  341. mapInfo["stype"] = "project"
  342. mapInfo["query"] = map[string]interface{}{
  343. "pici": pici,
  344. }
  345. for n, to := range toaddr {
  346. key := fmt.Sprintf("%d-%s-%d", pici, "project", n)
  347. mapInfo["key"] = key
  348. datas, _ := json.Marshal(mapInfo)
  349. node := &udpNode{datas, to, time.Now().Unix(), 0}
  350. udptaskmap.Store(key, node)
  351. _ = udpclient.WriteUdp(datas, mu.OP_TYPE_DATA, to)
  352. }
  353. }
  354. func (p *ProjectTask) enter(db, coll string, q map[string]interface{}) {
  355. defer util.Catch()
  356. defer func() {
  357. p.Brun = false
  358. }()
  359. p.Brun = true
  360. count, taskcount := 0, 0
  361. pool := make(chan bool, p.thread)
  362. log.Println("start project", q)
  363. sess := MongoTool.GetMgoConn()
  364. defer MongoTool.DestoryMongoConn(sess)
  365. infoPool := make(chan map[string]interface{}, 2000)
  366. over := make(chan bool)
  367. go func() {
  368. L:
  369. for {
  370. select {
  371. case tmp := <-infoPool:
  372. pool <- true
  373. taskcount++
  374. go func(tmp map[string]interface{}) {
  375. defer func() {
  376. <-pool
  377. }()
  378. if util.IntAll(tmp["repeat"]) == 0 {
  379. p.fillInPlace(tmp)
  380. info := ParseInfo(tmp)
  381. p.currentTime = info.Publishtime
  382. if p.currentType == "updateInfo" {
  383. //招标信息更改合并
  384. p.updateJudge(tmp, info)
  385. } else {
  386. //普通合并
  387. p.CommonMerge(tmp, info)
  388. }
  389. } else {
  390. //信息错误,进行更新
  391. }
  392. }(tmp)
  393. case <-over:
  394. break L
  395. }
  396. }
  397. }()
  398. ms := sess.DB(db).C(coll).Find(q).Sort("publishtime")
  399. if Sysconfig["hints"] != nil {
  400. ms.Hint(Sysconfig["hints"])
  401. }
  402. query := ms.Iter()
  403. //query := sess.DB(db).C(coll).Find(q).Sort("publishtime").Iter()
  404. var lastid interface{}
  405. L:
  406. for {
  407. select {
  408. case <-queryClose:
  409. log.Println("receive interrupt sign")
  410. log.Println("close iter..", lastid, query.Cursor.Close(nil))
  411. queryCloseOver <- true
  412. break L
  413. default:
  414. tmp := make(map[string]interface{})
  415. if query.Next(&tmp) {
  416. lastid = tmp["_id"]
  417. if count%10000 == 0 {
  418. log.Println("current", count, lastid)
  419. }
  420. infoPool <- tmp
  421. count++
  422. } else {
  423. break L
  424. }
  425. }
  426. }
  427. time.Sleep(5 * time.Second)
  428. over <- true
  429. //阻塞
  430. for n := 0; n < p.thread; n++ {
  431. pool <- true
  432. }
  433. log.Println("所有线程执行完成...", count, taskcount)
  434. }
  435. var (
  436. //从标题获取项目编号
  437. titleGetPc = regexp.MustCompile("^([-0-9a-zA-Z第号采招政询电审竞#]{8,}[-0-9a-zA-Z#]+)")
  438. titleGetPc1 = regexp.MustCompile("[\\[【((](.{0,6}(编号|编码|项号|包号|代码|标段?号)[::为])?([-0-9a-zA-Z第号采招政询电审竞#]{5,}([\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+[\\[\\]()()][-0-9a-zA-Z第号采招审竞#]+)?)[\\]】))]")
  439. titleGetPc2 = regexp.MustCompile("([-0-9a-zA-Z第号采政招询电审竞#]{8,}[-0-9a-zA-Z#]+)(.{0,5}公告)?$")
  440. //项目编号过滤
  441. pcReplace = regexp.MustCompile("([\\[【((〖〔《{﹝{](重|第?[二三四再]次.{0,4})[\\]】))〗〕》}﹞}])$|[\\[\\]【】()()〖〗〔〕《》{}﹝﹞-;{}–  ]+|(号|重|第?[二三四五再]次(招标)?)$|[ __]+|((采购)?项目|采购(项目)?)$")
  442. //项目编号只是数字或只是字母4个以下
  443. StrOrNum = regexp.MustCompile("^[0-9_-]{1,4}$|^[a-zA-Z_-]{1,4}$")
  444. //纯数字或纯字母
  445. StrOrNum2 = regexp.MustCompile("^[0-9_-]+$|^[a-zA-Z_-]+$")
  446. //含分包词,招标未识别分包 合并到一个项目
  447. KeyPackage = regexp.MustCompile("[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}(包|段)|(包|段)[0-9a-zA-Z一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩ]+.{0,2}")
  448. )
  449. func (p *ProjectTask) CommonMerge(tmp map[string]interface{}, info *Info) {
  450. if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
  451. if jsonData, ok := tmp["jsondata"].(map[string]interface{}); ok {
  452. proHref := util.ObjToString(jsonData["projecthref"])
  453. if jsonData != nil && proHref != "" {
  454. //projectHref字段合并
  455. tmp["projecthref"] = proHref
  456. p.mapHrefLock.Lock()
  457. pid := p.mapHref[proHref]
  458. p.mapHrefLock.Unlock()
  459. if pid != "" {
  460. p.AllIdsMapLock.Lock()
  461. comparePro := p.AllIdsMap[pid].P
  462. p.AllIdsMapLock.Unlock()
  463. _, ex := p.CompareStatus(comparePro, info)
  464. p.UpdateProject(tmp, info, comparePro, -1, "AAAAAAAAAA", ex)
  465. } else {
  466. id, p1 := p.NewProject(tmp, info)
  467. p.mapHrefLock.Lock()
  468. p.mapHref[proHref] = id
  469. p.mapHrefLock.Unlock()
  470. p.AllIdsMapLock.Lock()
  471. p.AllIdsMap[id] = &ID{Id: id, P: p1}
  472. p.AllIdsMapLock.Unlock()
  473. }
  474. } else {
  475. //项目合并
  476. p.startProjectMerge(info, tmp)
  477. }
  478. } else {
  479. //项目合并
  480. p.startProjectMerge(info, tmp)
  481. }
  482. }
  483. }
  484. func ParseInfo(tmp map[string]interface{}) (info *Info) {
  485. bys, _ := json.Marshal(tmp)
  486. var thisinfo *Info
  487. _ = json.Unmarshal(bys, &thisinfo)
  488. if thisinfo == nil {
  489. return nil
  490. }
  491. if len(thisinfo.Topscopeclass) == 0 {
  492. thisinfo.Topscopeclass = []string{}
  493. }
  494. if len(thisinfo.Subscopeclass) == 0 {
  495. thisinfo.Subscopeclass = []string{}
  496. }
  497. if thisinfo.SubType == "" {
  498. thisinfo.SubType = util.ObjToString(tmp["bidstatus"])
  499. }
  500. //从标题中查找项目编号
  501. res := titleGetPc.FindStringSubmatch(thisinfo.Title)
  502. if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
  503. thisinfo.PTC = res[1]
  504. } else {
  505. res = titleGetPc1.FindStringSubmatch(thisinfo.Title)
  506. if len(res) > 3 && len(res[3]) > 6 && thisinfo.ProjectCode != res[3] && !numCheckPc.MatchString(res[3]) && !_zimureg1.MatchString(res[3]) {
  507. thisinfo.PTC = res[3]
  508. } else {
  509. res = titleGetPc2.FindStringSubmatch(thisinfo.Title)
  510. if len(res) > 1 && len(res[1]) > 6 && thisinfo.ProjectCode != res[1] && !numCheckPc.MatchString(res[1]) && !_zimureg1.MatchString(res[1]) {
  511. thisinfo.PTC = res[1]
  512. }
  513. }
  514. }
  515. if thisinfo.ProjectName != "" && len([]rune(thisinfo.ProjectName)) > 0 {
  516. thisinfo.ProjectName = pcReplace.ReplaceAllString(thisinfo.ProjectName, "")
  517. if thisinfo.ProjectName != "" {
  518. thisinfo.pnbval++
  519. }
  520. }
  521. if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
  522. if thisinfo.ProjectCode != "" {
  523. thisinfo.ProjectCode = pcReplace.ReplaceAllString(thisinfo.ProjectCode, "")
  524. if thisinfo.pnbval == 0 && len([]rune(thisinfo.ProjectCode)) < 5 {
  525. thisinfo.ProjectCode = StrOrNum.ReplaceAllString(thisinfo.ProjectCode, "")
  526. }
  527. } else {
  528. thisinfo.PTC = pcReplace.ReplaceAllString(thisinfo.PTC, "")
  529. if thisinfo.pnbval == 0 && len([]rune(thisinfo.PTC)) < 5 {
  530. thisinfo.PTC = StrOrNum.ReplaceAllString(thisinfo.PTC, "")
  531. }
  532. }
  533. if thisinfo.ProjectCode != "" || thisinfo.PTC != "" {
  534. thisinfo.pnbval++
  535. }
  536. }
  537. if thisinfo.ProjectCode == thisinfo.PTC || strings.Index(thisinfo.ProjectCode, thisinfo.PTC) > -1 {
  538. thisinfo.PTC = ""
  539. }
  540. if thisinfo.Buyer != "" && len([]rune(thisinfo.Buyer)) > 2 {
  541. thisinfo.pnbval++
  542. } else {
  543. thisinfo.Buyer = ""
  544. }
  545. //winners整理
  546. winner, _ := tmp["winner"].(string)
  547. m1 := map[string]bool{}
  548. winners := []string{}
  549. if winner != "" {
  550. m1[winner] = true
  551. winners = append(winners, winner)
  552. }
  553. packageM, _ := tmp["package"].(map[string]interface{})
  554. if packageM != nil {
  555. thisinfo.HasPackage = true
  556. for _, p := range packageM {
  557. pm, _ := p.(map[string]interface{})
  558. pw, _ := pm["winner"].(string)
  559. if pw != "" && !m1[pw] {
  560. m1[pw] = true
  561. winners = append(winners, pw)
  562. }
  563. }
  564. }
  565. thisinfo.Winners = winners
  566. thisinfo.LenPC = len([]rune(thisinfo.ProjectCode))
  567. thisinfo.LenPTC = len([]rune(thisinfo.PTC))
  568. thisinfo.LenPN = len([]rune(thisinfo.ProjectName))
  569. //处理分包中数据异常问题
  570. for k, tmp := range thisinfo.Package {
  571. if ps, ok := tmp.([]map[string]interface{}); ok {
  572. for i, p := range ps {
  573. name, _ := p["name"].(string)
  574. if len([]rune(name)) > 100 {
  575. p["name"] = fmt.Sprint([]rune(name[:100]))
  576. }
  577. ps[i] = p
  578. }
  579. thisinfo.Package[k] = ps
  580. }
  581. }
  582. return thisinfo
  583. }
  584. func (p *ProjectTask) updateJudge(tmp map[string]interface{}, info *Info) {
  585. index := -1
  586. pInfoId := ""
  587. p.AllIdsMapLock.Lock()
  588. F:
  589. for k, ID := range p.AllIdsMap {
  590. for i, id := range ID.P.Ids {
  591. if info.Id == id {
  592. pInfoId = k
  593. index = i
  594. break F
  595. }
  596. }
  597. }
  598. p.AllIdsMapLock.Unlock()
  599. //未找到招标信息
  600. if index == -1 {
  601. if info != nil && !((info.pnbval == 1 && info.Buyer != "") || info.pnbval == 0) {
  602. p.currentTime = info.Publishtime
  603. p.startProjectMerge(info, tmp)
  604. }
  605. } else {
  606. tmpPro := MongoTool.FindById(ProjectColl, pInfoId)
  607. infoList := []interface{}(tmpPro["list"].(primitive.A))
  608. infoMap := infoList[index].(map[string]interface{})
  609. modifyMap, f := modifyEle(infoMap, tmp)
  610. //projecthref字段
  611. jsonData := tmp["jsondata"].(map[string]interface{})
  612. if jsonData != nil && jsonData["projecthref"] != nil {
  613. proHref := jsonData["projecthref"].(string)
  614. tmp["projecthref"] = proHref
  615. p.mapHrefLock.Lock()
  616. pid := p.mapHref[proHref]
  617. p.mapHrefLock.Unlock()
  618. if pid == pInfoId {
  619. p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
  620. return
  621. }
  622. }
  623. if f {
  624. //合并、修改
  625. log.Println("合并修改更新", "----------------------------")
  626. p.mergeAndModify(pInfoId, index, info, tmp, tmpPro, modifyMap)
  627. } else {
  628. //修改
  629. log.Println("修改更新", "----------------------------")
  630. p.modifyUpdate(pInfoId, index, info, tmp, tmpPro, modifyMap)
  631. }
  632. }
  633. }
  634. var Elements = []string{
  635. "projectname",
  636. "projectcode",
  637. "agency",
  638. "budget",
  639. "bidamount",
  640. "buyerperson",
  641. "area",
  642. "city",
  643. "publishtime",
  644. }
  645. /**
  646. 判断修改的字段是否是影响合并流程的要素字段
  647. */
  648. func modifyEle(tmpPro map[string]interface{}, tmp map[string]interface{}) (map[string]interface{}, bool) {
  649. modifyMap := map[string]interface{}{}
  650. for k := range tmpPro {
  651. for k1 := range tmp {
  652. if k == k1 && tmpPro[k] != tmp[k1] {
  653. modifyMap[k] = tmp[k1]
  654. break
  655. }
  656. }
  657. }
  658. for k := range modifyMap {
  659. for _, str := range Elements {
  660. if k == str {
  661. return modifyMap, true
  662. }
  663. }
  664. }
  665. delete(modifyMap, "_id")
  666. return modifyMap, false
  667. }
  668. //补全位置信息
  669. func (p *ProjectTask) fillInPlace(tmp map[string]interface{}) {
  670. area := util.ObjToString(tmp["area"])
  671. city := util.ObjToString(tmp["city"])
  672. district := util.ObjToString(tmp["district"])
  673. if area != "" && city != "" && district != "" {
  674. return
  675. }
  676. tmpSite := util.ObjToString(tmp["site"])
  677. if tmpSite == "" {
  678. return
  679. }
  680. p.mapSiteLock.Lock()
  681. defer p.mapSiteLock.Unlock()
  682. site := p.mapSite[tmpSite]
  683. if site != nil {
  684. if area != "" {
  685. if area == "全国" {
  686. tmp["area"] = site.Area
  687. tmp["city"] = site.City
  688. tmp["district"] = site.District
  689. return
  690. }
  691. if area != site.Area {
  692. return
  693. } else {
  694. if city == site.City {
  695. if district == "" {
  696. tmp["district"] = site.District
  697. return
  698. }
  699. } else if city == "" {
  700. tmp["city"] = site.City
  701. tmp["district"] = site.District
  702. return
  703. } else if site.City == "" {
  704. return
  705. }
  706. }
  707. } else {
  708. tmp["area"] = site.Area
  709. tmp["city"] = site.City
  710. tmp["district"] = site.District
  711. return
  712. }
  713. }
  714. }
  715. //从数组中删除元素
  716. func deleteSlice(arr []string, v, stype string) []string {
  717. for k, v1 := range arr {
  718. if v1 == v {
  719. ts := time.Now().Unix()
  720. arr = append(arr[:k], arr[k+1:]...)
  721. rt := time.Now().Unix() - ts
  722. if rt > 0 {
  723. log.Println("deleteSlice", stype, rt, v, len(arr))
  724. }
  725. return arr
  726. }
  727. }
  728. return arr
  729. }