updateBidding.go 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. package main
  2. import (
  3. util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
  4. "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
  5. "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
  6. "log"
  7. )
  8. // updateBidding 更新bidding数据
  9. func updateBidding() {
  10. //mongodb
  11. Mgo := &mongodb.MongodbSim{
  12. MongodbAddr: "172.17.189.140:27080",
  13. //MongodbAddr: "127.0.0.1:27083",
  14. DbName: "qfw",
  15. Size: 10,
  16. UserName: "SJZY_RWbid_ES",
  17. Password: "SJZY@B4i4D5e6S",
  18. //Direct: true,
  19. }
  20. Mgo.InitPool()
  21. //es
  22. Es := &elastic.Elastic{
  23. //S_esurl: "http://127.0.0.1:19805",
  24. S_esurl: "http://172.17.4.184:19805",
  25. I_size: 5,
  26. Username: "es_all",
  27. Password: "TopJkO2E_d1x",
  28. }
  29. Es.InitElasticSize()
  30. //es 新集群
  31. EsNew := &elastic.Elastic{
  32. //S_esurl: "http://127.0.0.1:19905",
  33. S_esurl: "http://172.17.4.184:19905",
  34. I_size: 5,
  35. Username: "jybid",
  36. Password: "Top2023_JEB01i@31",
  37. }
  38. EsNew.InitElasticSize()
  39. //
  40. defer util.Catch()
  41. sess := Mgo.GetMgoConn()
  42. defer Mgo.DestoryMongoConn(sess)
  43. selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
  44. it := sess.DB("qfw").C("bidding").Find(nil).Select(&selected).Iter()
  45. count := 0
  46. log.Println("开始")
  47. // 拟建 爬虫代码
  48. njCodes := []string{"a_qgjsxmhjxxgspt_ysbggs", "a_qgjsxmhjxxgspt_hpspqgs", "a_qgjsxmhjxxgspt_hpbggs"}
  49. //产权 爬虫代码
  50. cqCodes := []string{"sh_shsggzyjyzx_nyys", "sh_shsggzyjyzx_sfpm", "gd_dgsncjtzcglw_djyxx_lbfbzc", "gd_dgsncjtzcglw_djyxx_djyzc", "sc_scsggzyjyfwpt_jygk_qt", "qg_zhrmghgzrzyb_tdsccrgg", "ln_lnsggzyjypt_gycq", "a_gjggzyjypt_qt"}
  51. for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
  52. id := mongodb.BsonIdToSId(tmp["_id"])
  53. spidercode := util.ObjToString(tmp["spidercode"])
  54. if count%10000 == 0 {
  55. log.Println("current:", count, id)
  56. }
  57. if IsInStringArray(spidercode, njCodes) {
  58. //拟建数据
  59. update := map[string]interface{}{
  60. "toptype": "拟建",
  61. "subtype": "拟建",
  62. }
  63. Mgo.UpdateById("bidding", id, map[string]interface{}{"$set": update})
  64. err := Es.UpdateDocument("bidding", id, update)
  65. if err != nil {
  66. log.Println("es 更新失败", id, err)
  67. }
  68. err = EsNew.UpdateDocument("bidding", id, update)
  69. if err != nil {
  70. log.Println("EsNew 更新失败", id, err)
  71. }
  72. //产权数据
  73. } else if IsInStringArray(spidercode, cqCodes) {
  74. update := map[string]interface{}{
  75. "toptype": "产权",
  76. "subtype": "产权",
  77. }
  78. Mgo.UpdateById("bidding", id, map[string]interface{}{"$set": update})
  79. err := Es.DeleteByID("bidding", id)
  80. if err != nil {
  81. log.Println("es delete ", id, err)
  82. }
  83. err = EsNew.DeleteByID("bidding", id)
  84. if err != nil {
  85. log.Println("EsNew delete ", id, err)
  86. }
  87. }
  88. }
  89. log.Println("结束 ---------- over ")
  90. }
  91. // updateTop 更新一级分类
  92. func updateTop() {
  93. Mgo := &mongodb.MongodbSim{
  94. MongodbAddr: "172.17.189.140:27080",
  95. //MongodbAddr: "127.0.0.1:27083",
  96. Size: 10,
  97. DbName: "qfw",
  98. UserName: "SJZY_RWbid_ES",
  99. Password: "SJZY@B4i4D5e6S",
  100. //Direct: true,
  101. }
  102. Mgo.InitPool()
  103. sess := Mgo.GetMgoConn()
  104. defer Mgo.DestoryMongoConn(sess)
  105. query := sess.DB("qfw").C("wcc_bidding_20240103_subtype_exists").Find(nil).Select(nil).Iter()
  106. count := 0
  107. for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
  108. if count%10000 == 0 {
  109. log.Println("current:", count)
  110. }
  111. id := mongodb.BsonIdToSId(tmp["_id"])
  112. bidding, _ := Mgo.FindById("bidding", id, nil)
  113. update := map[string]interface{}{
  114. "toptype": (*bidding)["toptype"],
  115. "subtype": (*bidding)["subtype"],
  116. "infoformat": (*bidding)["infoformat"],
  117. }
  118. Mgo.UpdateById("wcc_bidding_20240103_subtype_exists", id, map[string]interface{}{"$set": update})
  119. tmp = make(map[string]interface{})
  120. }
  121. log.Println("over")
  122. }