task.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732
  1. package main
  2. import (
  3. "encoding/json"
  4. "field_sync/config"
  5. "field_sync/oss"
  6. "fmt"
  7. "go.mongodb.org/mongo-driver/bson"
  8. "go.uber.org/zap"
  9. util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
  10. "jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
  11. "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
  12. "jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
  13. "jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
  14. "net"
  15. "reflect"
  16. "regexp"
  17. "sort"
  18. "strings"
  19. "time"
  20. )
  21. var (
  22. regLetter = regexp.MustCompile("[a-z]*")
  23. cityEndReg = regexp.MustCompile("(区|县|市)$")
  24. )
  25. func biddingTask(data []byte, mapInfo map[string]interface{}) {
  26. defer util.Catch()
  27. stype := util.ObjToString(mapInfo["stype"])
  28. if stype == "bidding" {
  29. uq := bson.M{"gtid": bson.M{"$gte": util.ObjToString(mapInfo["gtid"])},
  30. "lteid": bson.M{"$lte": util.ObjToString(mapInfo["lteid"])}}
  31. MgoB.Update("bidding_processing_ids", uq, bson.M{"$set": bson.M{"dataprocess": 7, "updatetime": time.Now().Unix()}}, false, true)
  32. }
  33. // 领域标签处理的数据 id段
  34. if stype == "bidding_history" {
  35. MgoB.Save("field_data_record", map[string]interface{}{"gtid": mapInfo["gtid"], "lteid": mapInfo["lteid"], "status": 0})
  36. }
  37. q, _ := mapInfo["query"].(map[string]interface{})
  38. bkey, _ := mapInfo["bkey"].(string)
  39. if q == nil {
  40. q = map[string]interface{}{
  41. "_id": map[string]interface{}{
  42. "$gt": mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
  43. "$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
  44. },
  45. }
  46. }
  47. //extract库
  48. extractConn := MgoE.GetMgoConn()
  49. defer MgoE.DestoryMongoConn(extractConn)
  50. extractResult := extractConn.DB(MgoE.DbName).C(config.Conf.DB.MongoE.Coll).Find(q).Select(map[string]interface{}{
  51. "field_source": 0,
  52. "kvtext": 0,
  53. }).Sort("_id").Iter()
  54. eMap := map[string]map[string]interface{}{}
  55. extCount, repeatCount := 0, 0
  56. for tmp := make(map[string]interface{}); extractResult.Next(tmp); extCount++ {
  57. if util.IntAll(tmp["repeat"]) == 1 {
  58. repeatCount++
  59. }
  60. tid := mongodb.BsonIdToSId(tmp["_id"])
  61. eMap[tid] = tmp
  62. tmp = make(map[string]interface{})
  63. }
  64. log.Info("抽取表", zap.Int("数据量", extCount), zap.Int("重复数据量", repeatCount))
  65. //bidding库
  66. biddingConn := MgoB.GetMgoConn()
  67. count, _ := biddingConn.DB(MgoB.DbName).C(config.Conf.DB.MongoB.Coll).Find(&q).Count()
  68. log.Info("bidding表", zap.Int64("同步总数:", count))
  69. c := 0
  70. if count < 500000 {
  71. var res []map[string]interface{}
  72. result := biddingConn.DB(MgoB.DbName).C(config.Conf.DB.MongoB.Coll).Find(q).Select(map[string]interface{}{
  73. "contenthtml": 0,
  74. }).Iter()
  75. for tmp := make(map[string]interface{}); result.Next(tmp); {
  76. res = append(res, tmp)
  77. tmp = make(map[string]interface{})
  78. }
  79. MgoB.DestoryMongoConn(biddingConn)
  80. log.Info("查询结果", zap.Int64("bidding", count), zap.Int("抽取:", extCount))
  81. c = doIndex(res, eMap, bkey, stype)
  82. } else {
  83. log.Info("查询结果", zap.Int64("数据量太大,放弃", count))
  84. MgoB.DestoryMongoConn(biddingConn)
  85. }
  86. log.Info("bidding sync...over", zap.Int64("all", count), zap.Int("extract sync", c))
  87. NextNode(mapInfo, stype)
  88. NextNodePro(mapInfo, stype)
  89. NextNodeTidb(mapInfo, stype)
  90. if stype == "bidding_history" {
  91. NextNodeBidData(mapInfo) // bidding-data数据
  92. NextNodeTidbQyxy(mapInfo) // tidb-企业数据
  93. NextNodeHn(mapInfo)
  94. }
  95. }
  96. func biddingAllTask(data []byte, mapInfo map[string]interface{}) {
  97. defer util.Catch()
  98. q, _ := mapInfo["query"].(map[string]interface{})
  99. if q == nil {
  100. q = map[string]interface{}{
  101. "_id": map[string]interface{}{
  102. "$gt": mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
  103. "$lte": mongodb.StringTOBsonId(mapInfo["lteid"].(string)),
  104. },
  105. }
  106. }
  107. //extract库
  108. extractConn := MgoE.GetMgoConn()
  109. defer MgoE.DestoryMongoConn(extractConn)
  110. extractResult := extractConn.DB(MgoE.DbName).C(config.Conf.DB.MongoE.Coll).Find(q).Select(map[string]interface{}{
  111. "field_source": 0,
  112. "kvtext": 0,
  113. }).Sort("-_id").Iter()
  114. //bidding库
  115. biddingConn := MgoB.GetMgoConn()
  116. defer MgoB.DestoryMongoConn(biddingConn)
  117. count := 0
  118. var compare map[string]interface{}
  119. result := biddingConn.DB(MgoB.DbName).C(config.Conf.DB.MongoB.Coll).Find(q).Select(map[string]interface{}{
  120. "contenthtml": 0,
  121. "field_source": 0,
  122. }).Sort("-_id").Iter()
  123. for tmp := make(map[string]interface{}); result.Next(tmp); count++ {
  124. update := map[string]interface{}{}
  125. del := map[string]interface{}{} //记录extract没有值而bidding中有值的字段
  126. //对比方法----------------
  127. for {
  128. if compare == nil {
  129. compare = make(map[string]interface{})
  130. if !extractResult.Next(compare) {
  131. break
  132. }
  133. }
  134. if compare != nil {
  135. cid := mongodb.BsonIdToSId(compare["_id"])
  136. tid := mongodb.BsonIdToSId(tmp["_id"])
  137. if cid == tid {
  138. //更新bidding表;bidding表modifyinfo中的字段不更新
  139. modifyinfo := make(map[string]bool)
  140. if tmpmodifyinfo, ok := tmp["modifyinfo"].(map[string]interface{}); ok && tmpmodifyinfo != nil {
  141. for k := range tmpmodifyinfo {
  142. modifyinfo[k] = true
  143. }
  144. }
  145. for _, k := range config.Conf.Serve.FieldS {
  146. v1 := compare[k] //extract
  147. v2 := tmp[k] //bidding
  148. if v2 == nil && v1 != nil {
  149. update[k] = v1
  150. } else if v2 != nil && v1 != nil && !modifyinfo[k] {
  151. update[k] = v1
  152. } else if v2 != nil && v1 == nil && !modifyinfo[k] {
  153. if k == "s_subscopeclass" && del["subscopeclass"] == nil {
  154. continue
  155. } else if k == "s_topscopeclass" && del["topscopeclass"] == nil {
  156. continue
  157. }
  158. del[k] = 1
  159. //util.Debug("抽取结果没有值,bidding有值:field--", k, "val--", v2)
  160. }
  161. }
  162. if util.IntAll(compare["repeat"]) == 1 {
  163. update["extracttype"] = -1
  164. update["dataprocess"] = 7
  165. update["repeat_id"] = compare["repeat_id"]
  166. } else {
  167. update["extracttype"] = 1
  168. update["dataprocess"] = 8
  169. }
  170. break
  171. } else {
  172. if cid < tid {
  173. compare = nil
  174. continue
  175. } else {
  176. break
  177. }
  178. }
  179. } else {
  180. break
  181. }
  182. }
  183. //------------------对比结束
  184. //处理分类
  185. if compare != nil { //extract
  186. fieldFun(compare, update)
  187. compare = nil
  188. }
  189. // entidlist
  190. extractMap := make(map[string]interface{})
  191. if update["s_winner"] != "" {
  192. cid := companyFun(update)
  193. if len(cid) > 0 {
  194. update["entidlist"] = cid
  195. extractMap["entidlist"] = cid
  196. }
  197. }
  198. if len(extractMap) > 0 {
  199. updateExtPool <- []map[string]interface{}{
  200. {"_id": tmp["_id"]},
  201. {"$set": extractMap},
  202. }
  203. }
  204. // 附件有效字段
  205. if i := validFile(tmp); i != 0 {
  206. if i == -1 {
  207. update["isValidFile"] = false
  208. } else {
  209. update["isValidFile"] = true
  210. }
  211. }
  212. if len(update) > 0 {
  213. if len(del) > 0 { //删除的字段
  214. updateBidPool <- []map[string]interface{}{{
  215. "_id": tmp["_id"],
  216. },
  217. {"$set": update, "$unset": del},
  218. }
  219. } else {
  220. updateBidPool <- []map[string]interface{}{{
  221. "_id": tmp["_id"],
  222. },
  223. {"$set": update},
  224. }
  225. }
  226. }
  227. if count%50000 == 0 {
  228. log.Info("biddingTask", zap.Int("current", count))
  229. }
  230. tmp = make(map[string]interface{})
  231. }
  232. log.Info("biddingAll sync...over", zap.Int("all", count))
  233. }
  234. func doIndex(infos []map[string]interface{}, eMap map[string]map[string]interface{}, bkey, stype string) int {
  235. syncNo := 0 //抽取表数据同步数量
  236. //对比两张表数据,减少查询次数
  237. var compare map[string]interface{}
  238. var bidUpdate [][]map[string]interface{}
  239. var extUpdate [][]map[string]interface{}
  240. //SaveEsLock := &sync.Mutex{}
  241. log.Info("start ...")
  242. for n, tmp := range infos {
  243. tid := mongodb.BsonIdToSId(tmp["_id"])
  244. update := map[string]interface{}{} //要更新的mongo数据
  245. //对比方法----------------
  246. if eMap[tid] != nil {
  247. compare = eMap[tid]
  248. if stype == "bidding" {
  249. // 增量id段 正常数据
  250. if dg := util.IntAll(compare["dataging"]); dg == 1 { //extract中dataging=1跳过
  251. tmp = make(map[string]interface{})
  252. compare = nil
  253. continue
  254. }
  255. delete(eMap, tid)
  256. }
  257. if stype == "bidding_history" {
  258. //增量id段 历史数据
  259. if compare["history_updatetime"] == nil { //extract中history_updatetime不存在跳过
  260. tmp = make(map[string]interface{})
  261. compare = nil
  262. continue
  263. }
  264. delete(eMap, tid)
  265. }
  266. syncNo++
  267. for _, k := range config.Conf.Serve.FieldS {
  268. v1 := compare[k] //extract
  269. v2 := tmp[k] //bidding
  270. if v2 == nil && v1 != nil {
  271. update[k] = v1
  272. } else if v2 != nil && v1 != nil {
  273. update[k] = v1
  274. } else if v2 != nil && v1 == nil {
  275. if k == "city" || k == "district" {
  276. update[k] = ""
  277. }
  278. }
  279. }
  280. if util.IntAll(compare["repeat"]) == 1 {
  281. update["extracttype"] = -1
  282. update["dataprocess"] = 7
  283. update["repeat_id"] = compare["repeat_id"]
  284. } else {
  285. update["extracttype"] = 1
  286. update["dataprocess"] = 8
  287. }
  288. } else {
  289. compare = nil
  290. if util.IntAll(tmp["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
  291. update["dataging"] = 0
  292. }
  293. update["dataprocess"] = 8
  294. }
  295. //下面可以多线程跑的--->
  296. //处理分类
  297. if compare != nil { //extract
  298. fieldFun(compare, update)
  299. // publishtime 20230523
  300. if util.IntAll(tmp["publishtime"]) == -1 {
  301. if pb := methodPb(compare); pb > 0 {
  302. update["publishtime"] = pb
  303. }
  304. }
  305. compare = nil
  306. }
  307. //------------------对比结束
  308. //处理key descript
  309. if bkey == "" {
  310. DealInfo(&tmp, &update)
  311. }
  312. // entidlist
  313. extractMap := make(map[string]interface{})
  314. if update["s_winner"] != "" {
  315. cid := companyFun(update)
  316. if len(cid) > 0 {
  317. tmp["entidlist"] = cid
  318. update["entidlist"] = cid
  319. extractMap["entidlist"] = cid
  320. }
  321. }
  322. // 6.10 剑鱼发布信息分类处理, 写在这里是为了修改抽取表
  323. typeFunc(tmp, update, extractMap)
  324. if len(extractMap) > 0 {
  325. if extractMap["toptype"] != nil && extractMap["subtype"] == nil {
  326. extUpdate = append(extUpdate, []map[string]interface{}{
  327. {"_id": tmp["_id"]},
  328. {"$set": extractMap, "$unset": map[string]interface{}{"subtype": ""}},
  329. })
  330. } else {
  331. extUpdate = append(extUpdate, []map[string]interface{}{
  332. {"_id": tmp["_id"]},
  333. {"$set": extractMap},
  334. })
  335. }
  336. if len(extUpdate) >= MgoBulkSize {
  337. tmps := extUpdate
  338. MgoE.UpdateBulk(config.Conf.DB.MongoE.Coll, tmps...)
  339. extUpdate = [][]map[string]interface{}{}
  340. }
  341. }
  342. // 附件有效字段
  343. if i := validFile(tmp); i != 0 {
  344. if i == -1 {
  345. tmp["isValidFile"] = false
  346. update["isValidFile"] = false
  347. } else {
  348. tmp["isValidFile"] = true
  349. update["isValidFile"] = true
  350. }
  351. }
  352. if len(update) > 0 {
  353. bidUpdate = append(bidUpdate, []map[string]interface{}{{
  354. "_id": tmp["_id"],
  355. },
  356. {"$set": update},
  357. })
  358. if len(bidUpdate) >= MgoBulkSize {
  359. tmps := bidUpdate
  360. MgoB.UpdateBulk(config.Conf.DB.MongoB.Coll, tmps...)
  361. bidUpdate = [][]map[string]interface{}{}
  362. }
  363. }
  364. if n%500 == 0 {
  365. log.Info("biddingTask", zap.Int("current", n))
  366. }
  367. tmp = make(map[string]interface{})
  368. }
  369. //SaveEsLock.Lock()
  370. if len(bidUpdate) > 0 {
  371. tmps := bidUpdate
  372. MgoB.UpdateBulk(config.Conf.DB.MongoB.Coll, tmps...)
  373. bidUpdate = [][]map[string]interface{}{}
  374. }
  375. if len(extUpdate) > 0 {
  376. tmps := extUpdate
  377. MgoE.UpdateBulk(config.Conf.DB.MongoE.Coll, tmps...)
  378. extUpdate = [][]map[string]interface{}{}
  379. }
  380. //SaveEsLock.Unlock()
  381. return syncNo
  382. }
  383. // @Description subscopeclass、topscopeclass、package
  384. // 20230523 多包处理 subpackage = 1
  385. // @Author J 2022/6/7 5:54 PM
  386. func fieldFun(compare, update map[string]interface{}) {
  387. subscopeclass, _ := compare["subscopeclass"].([]interface{}) //subscopeclass
  388. if subscopeclass != nil {
  389. m1 := map[string]bool{}
  390. newclass := []string{}
  391. for _, sc := range subscopeclass {
  392. sclass, _ := sc.(string)
  393. if !m1[sclass] {
  394. m1[sclass] = true
  395. newclass = append(newclass, sclass)
  396. }
  397. }
  398. update["s_subscopeclass"] = strings.Join(newclass, ",")
  399. update["subscopeclass"] = newclass
  400. }
  401. topscopeclass, _ := compare["topscopeclass"].([]interface{}) //topscopeclass
  402. if topscopeclass != nil {
  403. m2 := map[string]bool{}
  404. newclass := []string{}
  405. for _, tc := range topscopeclass {
  406. tclass, _ := tc.(string)
  407. tclass = regLetter.ReplaceAllString(tclass, "") // 去除字母
  408. if !m2[tclass] {
  409. m2[tclass] = true
  410. newclass = append(newclass, tclass)
  411. }
  412. }
  413. update["topscopeclass"] = topscopeclass
  414. update["s_topscopeclass"] = strings.Join(newclass, ",")
  415. }
  416. if package1 := compare["package"]; package1 != nil {
  417. packageM, _ := package1.(map[string]interface{})
  418. update["package"] = packageM
  419. for _, p := range packageM {
  420. pm, _ := p.(map[string]interface{})
  421. if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 ||
  422. util.Float64All(pm["bidamount"]) > 0 {
  423. update["multipackage"] = 1
  424. break
  425. }
  426. }
  427. } else {
  428. update["multipackage"] = 0
  429. }
  430. // subpackage
  431. if compare["package"] != nil && compare["s_winner"] != nil && compare["bidamount"] != nil {
  432. pg := compare["package"].(map[string]interface{})
  433. if len(pg) > 1 {
  434. var bmt []float64
  435. var swn []string
  436. for _, p := range pg {
  437. p1 := p.(map[string]interface{})
  438. if p1["bidamount"] != nil {
  439. bmt = append(bmt, util.Float64All(p1["bidamount"]))
  440. }
  441. if w := util.ObjToString(p1["winner"]); w != "" {
  442. swn = append(swn)
  443. }
  444. }
  445. if len(bmt) > 1 && len(swn) > 1 {
  446. sn := strings.Split(util.ObjToString(compare["s_winner"]), ",")
  447. sort.Strings(sn)
  448. sort.Strings(swn)
  449. swn1 := util.ObjArrToStringArr(Duplicate(swn)) // 去重
  450. if strings.Join(swn1, ",") == strings.Join(sn, ",") {
  451. bidamount := 0.0
  452. for _, f := range bmt {
  453. bidamount += f
  454. }
  455. if bidamount == util.Float64All(compare["bidamount"]) {
  456. update["subpackage"] = 1
  457. }
  458. }
  459. }
  460. }
  461. }
  462. }
  463. // @Description entidlist
  464. // @Author J 2022/6/7 2:36 PM
  465. func companyFun(tmp map[string]interface{}) (cid []string) {
  466. sWinnerarr := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
  467. for _, w := range sWinnerarr {
  468. if w != "" {
  469. id := redis.GetStr("qyxy_id", w)
  470. if id == "" {
  471. ents, _ := MgoQ.Find(config.Conf.DB.MongoQ.Coll, map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{"company_name": 1}, false, -1, -1)
  472. if len(*ents) > 0 {
  473. id = util.ObjToString((*ents)[0]["_id"])
  474. redis.PutCKV("qyxy_id", w, id)
  475. } else {
  476. ent, _ := MgoP.FindOne(config.Conf.DB.MongoP.Coll, map[string]interface{}{"history_name": w})
  477. if len(*ent) > 0 {
  478. id = util.ObjToString((*ent)["company_id"])
  479. redis.PutCKV("qyxy_id", w, id)
  480. }
  481. }
  482. }
  483. if id == "" {
  484. id = "-"
  485. }
  486. cid = append(cid, id)
  487. }
  488. }
  489. return cid
  490. }
  491. // @Description update 修改bidding表,extractM修改抽取表
  492. // @Author J 2022/6/10 10:29 AM
  493. func typeFunc(tmp, update, extractM map[string]interface{}) {
  494. if jyData, ok := tmp["jyfb_data"].(map[string]interface{}); ok {
  495. if t := util.ObjToString(jyData["type"]); t != "" {
  496. switch t {
  497. //case "采购信息":
  498. case "招标公告":
  499. if util.ObjToString(tmp["toptype"]) != "招标" {
  500. update["toptype"] = "招标"
  501. extractM["toptype"] = "招标"
  502. delete(update, "subtype")
  503. }
  504. case "采购意向":
  505. if util.ObjToString(tmp["toptype"]) != "采购意向" {
  506. update["toptype"] = "采购意向"
  507. update["subtype"] = "采购意向"
  508. extractM["toptype"] = "采购意向"
  509. extractM["subtype"] = "采购意向"
  510. }
  511. case "招标预告":
  512. if util.ObjToString(tmp["toptype"]) != "预告" {
  513. update["toptype"] = "预告"
  514. extractM["toptype"] = "预告"
  515. delete(update, "subtype")
  516. }
  517. case "招标结果":
  518. if util.ObjToString(tmp["toptype"]) != "结果" {
  519. update["toptype"] = "结果"
  520. extractM["toptype"] = "结果"
  521. delete(update, "subtype")
  522. }
  523. }
  524. }
  525. }
  526. }
  527. // @Description 附件有效字段(isValidFile)
  528. // @Author J 2022/7/8 14:41
  529. func validFile(tmp map[string]interface{}) int {
  530. isContinue := false
  531. if pinfo, o := tmp["projectinfo"].(map[string]interface{}); o {
  532. if atts, o1 := pinfo["attachments"].(map[string]interface{}); o1 {
  533. for _, att := range atts {
  534. if att == nil {
  535. continue
  536. }
  537. if reflect.TypeOf(att).String() == "string" {
  538. continue
  539. }
  540. att1 := att.(map[string]interface{})
  541. if fid := util.ObjToString(att1["fid"]); fid != "" {
  542. isContinue = true
  543. break
  544. }
  545. }
  546. if isContinue {
  547. if attachTxt, o := tmp["attach_text"].(map[string]interface{}); o {
  548. if len(attachTxt) > 0 {
  549. for _, at := range attachTxt {
  550. at1 := at.(map[string]interface{})
  551. if len(at1) > 0 {
  552. for k, _ := range at1 {
  553. if reflect.TypeOf(at1[k]).String() == "string" {
  554. continue
  555. }
  556. at2 := at1[k].(map[string]interface{})
  557. s := strings.ToLower(util.ObjToString(at2["file_name"]))
  558. if !strings.Contains(s, "jpg") || !strings.Contains(s, "jpeg") != strings.Contains(s, "png") ||
  559. strings.Contains(s, "pdf") {
  560. if strings.Contains(s, "swf") || strings.Contains(s, "html") {
  561. return -1
  562. } else if AnalysisFile(oss.OssGetObject(util.ObjToString(at2["attach_url"]))) {
  563. return 1
  564. }
  565. }
  566. }
  567. break
  568. } else {
  569. break
  570. }
  571. }
  572. }
  573. }
  574. flag := false
  575. for _, att := range atts {
  576. if att == nil {
  577. continue
  578. }
  579. if reflect.TypeOf(att).String() == "string" {
  580. continue
  581. }
  582. att1 := att.(map[string]interface{})
  583. if fid := util.ObjToString(att1["fid"]); fid != "" {
  584. ftype := strings.ToLower(util.ObjToString(tmp["ftype"]))
  585. if ftype != "swf" && ftype != "html" && oss.OssObjExists("jy-datafile", fid) {
  586. return 1
  587. } else {
  588. flag = true
  589. }
  590. }
  591. }
  592. if flag {
  593. return -1
  594. }
  595. }
  596. }
  597. }
  598. return 0
  599. }
  600. // @Description id不变,内容变化 重新索引数据
  601. // @Author J 2022/8/10 13:29
  602. func taskinfo(id string) {
  603. tmp, _ := MgoB.FindById("bidding", id, nil)
  604. if tmp == nil || len(*tmp) == 0 {
  605. log.Info(fmt.Sprintf("taskinfo bidding id=%s 未查询到数据", id))
  606. return
  607. }
  608. extractM, _ := MgoE.FindById(config.Conf.DB.MongoE.Coll, id, nil)
  609. if extractM == nil || len(*extractM) == 0 {
  610. extractM, _ = MgoE.FindById(config.Conf.DB.MongoE.Coll1, id, nil)
  611. if extractM == nil || len(*extractM) == 0 {
  612. log.Info(fmt.Sprintf("taskinfo extract id=%s 未查询到数据", id))
  613. return
  614. }
  615. }
  616. update := map[string]interface{}{} //要更新的mongo数据
  617. //更新bidding表字段
  618. for _, k := range config.Conf.Serve.FieldS {
  619. v1 := (*extractM)[k] //extract
  620. v2 := (*tmp)[k] //bidding
  621. if v2 == nil && v1 != nil {
  622. update[k] = v1
  623. } else if v2 != nil && v1 != nil {
  624. update[k] = v1
  625. } else if v2 != nil && v1 == nil {
  626. if k == "city" || k == "district" {
  627. update[k] = ""
  628. }
  629. }
  630. }
  631. if util.IntAll((*extractM)["repeat"]) == 1 {
  632. update["extracttype"] = -1
  633. update["dataprocess"] = 7
  634. update["repeat_id"] = (*extractM)["repeat_id"]
  635. } else {
  636. update["extracttype"] = 1
  637. update["dataprocess"] = 8
  638. }
  639. //处理分类
  640. fieldFun(*extractM, update)
  641. extractMap := make(map[string]interface{})
  642. if util.ObjToString((*tmp)["s_winner"]) != "" {
  643. cid := companyFun(*tmp)
  644. if len(cid) > 0 {
  645. update["entidlist"] = cid
  646. extractMap["entidlist"] = cid
  647. }
  648. MgoE.UpdateById(config.Conf.DB.MongoE.Coll, id, map[string]interface{}{"$set": extractMap})
  649. }
  650. // 附件有效字段
  651. if i := validFile(*tmp); i != 0 {
  652. if i == -1 {
  653. update["isValidFile"] = false
  654. } else {
  655. update["isValidFile"] = true
  656. }
  657. }
  658. if len(update) > 0 {
  659. MgoB.UpdateById(config.Conf.DB.MongoB.Coll, id, map[string]interface{}{"$set": update})
  660. }
  661. mapinfo := map[string]interface{}{
  662. "infoid": id,
  663. "stype": "index-by-id",
  664. }
  665. datas, _ := json.Marshal(mapinfo)
  666. var next = &net.UDPAddr{
  667. IP: net.ParseIP(config.Conf.Udp.Next.Addr),
  668. Port: util.IntAll(config.Conf.Udp.Next.Port),
  669. }
  670. log.Info("nsq data over", zap.Any("es", next), zap.String("mapinfo", string(datas)))
  671. _ = UdpClient.WriteUdp(datas, udp.OP_TYPE_DATA, next)
  672. }
  673. var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
  674. // @Description 发布时间处理
  675. // @Author J 2023/5/23 14:32
  676. func methodPb(tmp map[string]interface{}) int64 {
  677. if tmp["ext_publishtime"] != nil {
  678. if newPb := util.Int64All(tmp["ext_publishtime"]); newPb < time.Now().Unix() && newPb > 1420041600 {
  679. return newPb
  680. }
  681. }
  682. for _, d := range DateTimeSelect {
  683. if tmp[d] != nil && util.Int64All(tmp[d]) < time.Now().Unix() {
  684. return util.Int64All(tmp[d])
  685. }
  686. }
  687. return 0
  688. }
  689. // Duplicate
  690. // @Description 去重
  691. // @Author J 2023/5/24 09:53
  692. func Duplicate(a interface{}) (ret []interface{}) {
  693. va := reflect.ValueOf(a)
  694. for i := 0; i < va.Len(); i++ {
  695. if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
  696. continue
  697. }
  698. ret = append(ret, va.Index(i).Interface())
  699. }
  700. return ret
  701. }