timedTaskBuyer.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700
  1. package main
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "github.com/garyburd/redigo/redis"
  6. "gopkg.in/mgo.v2/bson"
  7. "log"
  8. util2 "mfw/util"
  9. "net"
  10. "qfw/util"
  11. "sort"
  12. "strings"
  13. "time"
  14. "unicode/utf8"
  15. )
  16. //之前main方法,只更新
  17. func TaskBuyer(mapinfo *map[string]interface{}) {
  18. defer util.Catch()
  19. gtid, lteid := util.ObjToString((*mapinfo)["gtid"]), util.ObjToString((*mapinfo)["lteid"])
  20. if gtid == "" || lteid == "" {
  21. log.Println(gtid, lteid, "参数错误")
  22. return
  23. }
  24. var GId, LtId bson.ObjectId
  25. if bson.IsObjectIdHex(gtid) && bson.IsObjectIdHex(lteid) {
  26. GId = bson.ObjectIdHex(gtid)
  27. LtId = bson.ObjectIdHex(lteid)
  28. } else {
  29. log.Println(gtid, lteid, "不是Objectid,转换_id错误", gtid, lteid)
  30. return
  31. }
  32. //timenow:=time.Now().Unix()
  33. //udp的id区间查询bidding 中标人 中标联系人 中标联系电话
  34. // topscopeclass项目类型-industry行业类型&&topscopeclass联系人项目类型
  35. // (area地区-province省份 city城市-city城市 district区县-district区县)
  36. // buyeraddr-company_address企业地址
  37. SourceClientcc := SourceClient.GetMgoConn(8640000)
  38. cursor := SourceClientcc.DB(Config["mgodb_bidding"]).C(Config["mgodb_mgoinit_c"]).Find(bson.M{
  39. "_id": bson.M{
  40. "$gte": GId,
  41. "$lte": LtId,
  42. },
  43. }).Select(bson.M{"buyer": 1, "buyertel": 1, "buyerperson": 1, "topscopeclass": 1,
  44. "buyeraddr": 1, "buyerclass": 1}).Iter()
  45. if cursor.Err() != nil {
  46. SourceClient.DestoryMongoConn(SourceClientcc)
  47. log.Println(cursor.Err())
  48. return
  49. }
  50. //判断是否是存量,是存量走Redis遍历
  51. if v, ok := (*mapinfo)["data_info"].(string); ok && v == "save" {
  52. //存量处理
  53. conn := HisRedisPool.Conn()
  54. defer conn.Close()
  55. //选择redis db
  56. conn.Select(redis_buyer_db)
  57. //遍历bidding表保存到redis
  58. //key:企业名 value:json结构体{"buyer": 1, "buyertel": 1, "buyerperson": 1,"topscopeclass": 1, "buyeraddr": 1,"_id":1}
  59. tmp := make(map[string]interface{})
  60. var num int
  61. var tmpRangeId string
  62. for cursor.Next(&tmp) {
  63. num++
  64. if num%10000==0 &&num>0{
  65. log.Println("当前采购单位数量:",num)
  66. }
  67. mgoId := tmp["_id"].(bson.ObjectId).Hex()
  68. tmpRangeId = mgoId
  69. buyer, ok := tmp["buyer"].(string)
  70. if !ok || utf8.RuneCountInString(buyer) < 4 {
  71. continue
  72. }
  73. //判断redis key是否存在
  74. e_num := conn.Exists(buyer).Val()
  75. //获取字符串_id
  76. //替换_id
  77. tmp["_id"] = mgoId
  78. //创建value数组
  79. tmps := make([]map[string]interface{}, 0)
  80. if e_num > 0 {
  81. //存量redis的key存在,累加更新
  82. bytes, _ := conn.Get(buyer).Bytes()
  83. json.Unmarshal(bytes, &tmps)
  84. }
  85. tmps = append(tmps, tmp)
  86. bytes, _ := json.Marshal(tmps)
  87. //存量redis的key不存在,新增 key :企业名 val :[]map
  88. if err := conn.Set(buyer, string(bytes), 0).Err(); err != nil {
  89. log.Println(err)
  90. }
  91. }
  92. log.Println("存量buyer mongo遍历完成:",num)
  93. if tmpRangeId != lteid{
  94. by, _ := json.Marshal(map[string]interface{}{
  95. "gtid": tmpRangeId,
  96. "lteid": lteid,
  97. "data_info":"save",
  98. "stype": "buyer",
  99. })
  100. if e := udpclient.WriteUdp(by, util2.OP_TYPE_DATA, &net.UDPAddr{
  101. IP: net.ParseIP("127.0.0.1"),
  102. Port: Updport,
  103. }); e != nil {
  104. log.Println(e)
  105. }
  106. SourceClient.DestoryMongoConn(SourceClientcc)
  107. return
  108. }
  109. SourceClient.DestoryMongoConn(SourceClientcc)
  110. //遍历redis
  111. if scan := conn.Scan(0, "", 100); scan.Err() != nil {
  112. log.Println(scan.Err())
  113. return
  114. } else {
  115. iterator := scan.Iterator()
  116. for iterator.Next() {
  117. redisCName := iterator.Val() //redis key 企业名
  118. redisvalueBytes, _ := conn.Get(redisCName).Bytes() //redis val []数组
  119. rValuesMaps := make([]map[string]interface{}, 0)
  120. json.Unmarshal(redisvalueBytes, &rValuesMaps)
  121. //redis查询是否存在
  122. rdb := RedisPool.Get()
  123. rdb.Do("SELECT", redis_buyer_db)
  124. if reply, err := redis.String(rdb.Do("GET", redisCName)); err != nil {
  125. //redis不存在,存到临时表,定时任务处理
  126. for _, vmap := range rValuesMaps {
  127. vmap["_id"] = bson.ObjectIdHex(vmap["_id"].(string))
  128. if errb := FClient.SaveByOriID(Config["mgo_qyk_c_b_new"], vmap); !errb {
  129. log.Println("存量 FClient.Save err", errb, vmap)
  130. }
  131. }
  132. //log.Println("get redis id err:定时任务处理", err, tmp)
  133. if err := rdb.Close(); err != nil {
  134. log.Println("存量", err)
  135. }
  136. continue
  137. } else {
  138. //redis存在更新合并
  139. if err := rdb.Close(); err != nil {
  140. log.Println(err)
  141. }
  142. //拿到合并后的qyk
  143. FClient.DbName = Config["mgodb_extract_kf"]
  144. oldTmp, b := FClient.FindById(Config["mgo_qyk_buyer"], reply, nil)
  145. if !b || (*oldTmp) == nil|| reply==""||(*oldTmp)["_id"]==nil{
  146. log.Println(redisCName, "存量 redis id 不存在", reply,"数据:",oldTmp)
  147. continue
  148. }
  149. tmpTopscopeclass := []string{}
  150. tmpTopscopeclassMap := make(map[string]bool)
  151. for _, rvaluemaps := range rValuesMaps {
  152. if tclasss, ok := rvaluemaps["topscopeclass"].([]string); ok {
  153. for _, vv := range tclasss {
  154. if len(vv) > 1 {
  155. tmpTopscopeclassMap[vv[:len(vv)-1]] = true
  156. }
  157. }
  158. }
  159. }
  160. for k := range tmpTopscopeclassMap {
  161. tmpTopscopeclass = append(tmpTopscopeclass, k)
  162. }
  163. sort.Strings(tmpTopscopeclass)
  164. esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
  165. //更新buyerclass合并
  166. if tmp["buyerclass"] == nil || tmp["buyerclass"] == "" {
  167. //无值,不更新
  168. } else {
  169. var buyerclass_new, buyerclass_old string
  170. buyerclass_new = tmp["buyerclass"].(string)
  171. buyerclass_old = (*oldTmp)["buyerclass"].(string)
  172. if buyerclass_old == "" {
  173. (*oldTmp)["buyerclass"] = buyerclass_new
  174. } else {
  175. if buyerclass_new != buyerclass_old {
  176. if !strings.Contains(buyerclass_old, buyerclass_new) {
  177. (*oldTmp)["buyerclass"] = buyerclass_old + "," + buyerclass_new //采购单位类型
  178. }
  179. }
  180. }
  181. }
  182. //联系方式合并
  183. contactMaps := make([]interface{}, 0)
  184. if (*oldTmp)["contact"] != nil {
  185. //直接添加联系人,不再判断
  186. if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
  187. contactMaps = append(contactMaps, v...)
  188. }
  189. }
  190. //遍历redis value联系人
  191. for _, rvmap := range rValuesMaps {
  192. var tmpperson, buyertel string
  193. if rvmapperson, ok := rvmap["buyerperson"].(string); ok && utf8.RuneCountInString(rvmapperson)>=2 && rvmapperson != "" {
  194. tmpperson = rvmapperson
  195. } else {
  196. continue
  197. }
  198. if rvmapwintel, ok := rvmap["buyertel"].(string); ok {
  199. buyertel = rvmapwintel
  200. } else {
  201. buyertel = ""
  202. }
  203. if Reg_xing.MatchString(buyertel) || !Reg_tel.MatchString(buyertel) {
  204. buyertel = ""
  205. }
  206. tmpContact := make(map[string]interface{})
  207. tmpContact["infoid"] = rvmap["_id"]
  208. tmpContact["contact_person"] = tmpperson
  209. tmpContact["contact_type"] = "项目联系人"
  210. tmpContact["phone"] = buyertel
  211. tmpclass := make([]string, 0)
  212. if tclasss, ok := rvmap["topscopeclass"].([]string); ok {
  213. for _, vv := range tclasss {
  214. if len(vv) > 1 {
  215. tmpclass = append(tmpclass, vv[:len(vv)-1])
  216. }
  217. }
  218. }
  219. tmpContact["topscopeclass"] = strings.Join(tmpclass, ";")
  220. tmpContact["updatetime"] = time.Now().Unix()
  221. contactMaps = append(contactMaps, tmpContact)
  222. }
  223. (*oldTmp)["contact"] = contactMaps
  224. //mongo更新
  225. (*oldTmp)["updatatime"] = time.Now().Unix()
  226. if !FClient.UpdateById(Config["mgo_qyk_buyer"], esId, bson.M{"$set": oldTmp}) {
  227. log.Println("存量 mongo更新 err", esId, oldTmp)
  228. }
  229. //es更新
  230. delete((*oldTmp), "_id")
  231. }
  232. }
  233. }
  234. log.Println("存量历史合并执行完成 ok", gtid, lteid)
  235. } else {
  236. //增量处理
  237. overid := gtid
  238. tmp := map[string]interface{}{}
  239. for cursor.Next(&tmp) {
  240. overid = AddBuyer(overid, tmp)
  241. }
  242. SourceClient.DestoryMongoConn(SourceClientcc)
  243. log.Println("增量合并执行完成 ok", gtid, lteid, overid)
  244. //发送udp 更新es段
  245. //nextNode("buyerent",timenow)
  246. }
  247. }
  248. //增量
  249. func AddBuyer(overid string, tmp map[string]interface{}) string {
  250. overid = tmp["_id"].(bson.ObjectId).Hex()
  251. buyer, ok := tmp["buyer"].(string)
  252. if !ok || utf8.RuneCountInString(buyer) < 4 {
  253. return overid
  254. }
  255. //redis查询是否存在
  256. rdb := RedisPool.Get()
  257. rdb.Do("SELECT", redis_buyer_db)
  258. if reply, err := redis.String(rdb.Do("GET", buyer)); err != nil {
  259. //redis不存在存到临时表,定时任务处理
  260. if errb := FClient.SaveByOriID(Config["mgo_qyk_c_b_new"], tmp); !errb {
  261. log.Println("FClient.Save err", errb, tmp)
  262. }
  263. //log.Println("get redis id err:定时任务处理", err, tmp)
  264. if err := rdb.Close(); err != nil {
  265. log.Println(err)
  266. }
  267. return overid
  268. } else {
  269. if err := rdb.Close(); err != nil {
  270. log.Println(err)
  271. }
  272. //拿到合并后的qyk
  273. oldTmp, b := FClient.FindById(Config["mgo_qyk_buyer"], reply, bson.M{})
  274. if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
  275. log.Println("redis id 不存在", reply)
  276. return overid
  277. }
  278. //比较合并 行业类型
  279. tmpTopscopeclass := []string{}
  280. tmpConTopscopeclass := []string{}
  281. tmpTopscopeclassMap := make(map[string]bool)
  282. if v, ok := tmp["topscopeclass"].([]interface{}); ok {
  283. for _, vv := range v {
  284. if vvv, ok := vv.(string); ok && len(vvv) > 1 {
  285. tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
  286. tmpConTopscopeclass = append(tmpConTopscopeclass, vvv[:len(vvv)-1])
  287. }
  288. }
  289. }
  290. for k := range tmpTopscopeclassMap {
  291. tmpTopscopeclass = append(tmpTopscopeclass, k)
  292. }
  293. sort.Strings(tmpTopscopeclass)
  294. esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
  295. //更新buyerclass合并
  296. if tmp["buyerclass"] == nil || tmp["buyerclass"] == "" {
  297. //无值,不更新
  298. } else {
  299. var buyerclass_new, buyerclass_old string
  300. buyerclass_new = tmp["buyerclass"].(string)
  301. buyerclass_old = (*oldTmp)["buyerclass"].(string)
  302. if buyerclass_old == "" {
  303. (*oldTmp)["buyerclass"] = buyerclass_new
  304. } else {
  305. if buyerclass_new != buyerclass_old {
  306. if !strings.Contains(buyerclass_old, buyerclass_new) {
  307. (*oldTmp)["buyerclass"] = buyerclass_old + "," + buyerclass_new //采购单位类型
  308. }
  309. }
  310. }
  311. }
  312. //更新行业类型
  313. if tmp["buyerperson"] == nil || tmp["buyerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["buyerperson"])) {
  314. (*oldTmp)["updatatime"] = time.Now().Unix()
  315. //mongo更新
  316. if !FClient.UpdateById(Config["mgo_qyk_buyer"], esId, bson.M{"$set": oldTmp}) {
  317. log.Println("mongo更新err", esId)
  318. }
  319. //es更新
  320. delete((*oldTmp), "_id")
  321. return overid
  322. }
  323. //联系方式合并
  324. contactMaps := make([]map[string]interface{}, 0)
  325. if (*oldTmp)["contact"] != nil {
  326. //直接添加联系人,不再判断
  327. if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
  328. for _, vv := range v {
  329. contactMaps = append(contactMaps, vv.(map[string]interface{}))
  330. }
  331. }
  332. }
  333. var tmpperson, buyertel string
  334. if tmppersona, ok := tmp["buyerperson"].(string); ok && utf8.RuneCountInString(tmppersona)>=2 && tmppersona != "" && Reg_person.MatchString(tmppersona) && !Reg_xing.MatchString(tmppersona) {
  335. tmpperson = tmppersona
  336. }
  337. if tmpperson != "" {
  338. if buyerteltmp, ok := tmp["buyertel"].(string); ok {
  339. buyertel = buyerteltmp
  340. }
  341. if Reg_xing.MatchString(buyertel) || !Reg_tel.MatchString(buyertel) {
  342. buyertel = ""
  343. } else {
  344. buyertel = buyertel
  345. }
  346. vvv := make(map[string]interface{})
  347. vvv["infoid"] = overid
  348. vvv["contact_person"] = tmpperson
  349. vvv["contact_type"] = "项目联系人"
  350. vvv["phone"] = buyertel
  351. vvv["topscopeclass"] = strings.Join(tmpConTopscopeclass, ";")
  352. vvv["updatetime"] = time.Now().Unix()
  353. contactMaps = append(contactMaps, vvv)
  354. }
  355. //分包处理
  356. if tmp["package"] != nil {
  357. PackageDealWithBuyer(oldTmp, tmp, buyer)
  358. }
  359. (*oldTmp)["contact"] = contactMaps
  360. //mongo更新
  361. (*oldTmp)["updatatime"] = time.Now().Unix()
  362. if !FClient.UpdateById(Config["mgo_qyk_buyer"], esId, bson.M{"$set": oldTmp}) {
  363. log.Println("mongo更新 err", esId, oldTmp)
  364. }
  365. //es更新
  366. delete((*oldTmp), "_id")
  367. }
  368. return overid
  369. }
  370. //定时任务 新增
  371. //1.存异常表
  372. //2.合并原始库新增
  373. func TimedTaskBuyer() {
  374. //time.Sleep(time.Hour*70)
  375. t2 := time.NewTimer(time.Second * 5)
  376. for range t2.C {
  377. //timenow := time.Now().Unix()
  378. Fcconn := FClient.GetMgoConn(86400)
  379. tmpLast := map[string]interface{}{}
  380. if iter := Fcconn.DB(Config["mgodb_extract_kf"]).C(Config["mgo_qyk_c_b_new"]).Find(bson.M{}).Sort("-_id").Limit(1).Iter(); iter != nil {
  381. if !iter.Next(&tmpLast) {
  382. //临时表无数据
  383. log.Println("临时表无数据:")
  384. t2.Reset(time.Minute * 5)
  385. FClient.DestoryMongoConn(Fcconn)
  386. continue
  387. } else {
  388. log.Println("临时表有数据:", tmpLast["_id"])
  389. fconn := FClient.GetMgoConn(86400)
  390. cursor := fconn.DB(Config["mgodb_extract_kf"]).C(Config["mgo_qyk_c_b_new"]).Find(bson.M{
  391. "_id": bson.M{
  392. "$lte": tmpLast["_id"],
  393. },
  394. }).Sort("_id").Iter()
  395. if cursor == nil {
  396. log.Println("查询失败")
  397. t2.Reset(time.Second * 5)
  398. FClient.DestoryMongoConn(fconn)
  399. continue
  400. }
  401. //遍历临时表数据,匹配不到原始库存入异常表
  402. tmp := make(map[string]interface{})
  403. for cursor.Next(&tmp) {
  404. tmpId := tmp["_id"].(bson.ObjectId).Hex()
  405. errbuyer, ok := tmp["buyer"].(string)
  406. if !ok || errbuyer == "" {
  407. continue
  408. }
  409. //再重新查找redis,存在发udp处理,不存在走新增合并
  410. rdb := RedisPool.Get()
  411. rdb.Do("SELECT", redis_buyer_db)
  412. if _, err := redis.String(rdb.Do("GET", errbuyer)); err == nil {
  413. //增量合并
  414. AddBuyer(tmpId, tmp)
  415. //存在的话删除tmp mongo表
  416. if DeletedCount := FClient.Del(Config["mgo_qyk_c_b_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !DeletedCount {
  417. log.Println("删除临时表err:", DeletedCount)
  418. }
  419. if err := rdb.Close(); err != nil {
  420. log.Println(err)
  421. }
  422. continue
  423. } else {
  424. if err = rdb.Close(); err != nil {
  425. log.Println(err)
  426. }
  427. }
  428. //查询redis不存在新增
  429. sessionfinone := FClient.GetMgoConn()
  430. resulttmp := make(map[string]interface{})
  431. err := sessionfinone.DB(Config["mgodb_enterprise"]).C(Config["mgodb_enterprise_c"]).Find(bson.M{"company_name": errbuyer}).One(&resulttmp)
  432. FClient.DestoryMongoConn(sessionfinone)
  433. if err != nil || resulttmp["_id"] == nil {
  434. //log.Println(r)
  435. //人工审核正则
  436. var isok bool
  437. //先遍历ok
  438. for _, v := range BuyerRegOk {
  439. isok = v.MatchString(errbuyer)
  440. if isok {
  441. //匹配ok完,匹配err
  442. for _, vRegErr := range BuyerRegErr {
  443. isok = vRegErr.MatchString(errbuyer)
  444. //匹配到ok 也匹配到err 按err算
  445. if isok {
  446. tmp["buyer_err"] = 1
  447. break
  448. }
  449. }
  450. //匹配ok,没匹配err 按ok算
  451. if tmp["buyer_err"] == nil {
  452. tmp["buyer_ok"] = 1
  453. break
  454. }
  455. }
  456. }
  457. //都没匹配
  458. if tmp["buyer_ok"] == nil && tmp["bnuyer_err"] == nil {
  459. tmp["buyer_err"] = 1
  460. }
  461. //匹配不到原始库,存入异常表删除临时表
  462. if errb := FClient.SaveByOriID(Config["mgo_qyk_c_b_err"], tmp); !errb{
  463. log.Println("存入异常表错误", errb, tmp)
  464. }
  465. if deleteNum := FClient.Del(Config["mgo_qyk_c_b_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
  466. log.Println("删除临时表错误", deleteNum)
  467. }
  468. continue
  469. } else {
  470. //log.Println(123)
  471. //匹配到原始库,新增 resulttmp buyer
  472. if resulttmp["credit_no"] != nil {
  473. if credit_no, ok := resulttmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
  474. len(strings.TrimSpace(credit_no)) > 8 {
  475. dataNo := strings.TrimSpace(credit_no)[2:8]
  476. if Addrs[dataNo] != nil {
  477. if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
  478. if resulttmp["province"] == nil || resulttmp["province"] == "" {
  479. resulttmp["province"] = v["province"]
  480. }
  481. resulttmp["city"] = v["city"]
  482. resulttmp["district"] = v["district"]
  483. }
  484. }
  485. }
  486. }
  487. //行业类型
  488. tmpclass := make([]string, 0)
  489. if tclasss, ok := tmp["topscopeclass"].([]interface{}); ok {
  490. for _, vv := range tclasss {
  491. if vvv, ok := vv.(string); ok {
  492. if len(vvv) > 1 {
  493. tmpclass = append(tmpclass, vvv[:len(vvv)-1])
  494. }
  495. }
  496. }
  497. }
  498. contacts := make([]map[string]interface{}, 0)
  499. if legal_person, ok := resulttmp["legal_person"].(string); ok && utf8.RuneCountInString(legal_person)>=2 &&legal_person != "" && !Reg_xing.MatchString(legal_person) && Reg_person.MatchString(legal_person) {
  500. contact := make(map[string]interface{}, 0)
  501. contact["contact_person"] = legal_person //联系人
  502. contact["contact_type"] = "法定代表人" //法定代表人
  503. if resulttmp["annual_reports"] != nil {
  504. bytes, err := json.Marshal(resulttmp["annual_reports"])
  505. if err != nil {
  506. log.Println("annual_reports err:", err)
  507. }
  508. phonetmp := make([]map[string]interface{}, 0)
  509. err = json.Unmarshal(bytes, &phonetmp)
  510. if err != nil {
  511. log.Println("Unmarshal err:", err)
  512. }
  513. for _, vv := range phonetmp {
  514. if vv["company_phone"] != nil {
  515. if vv["company_phone"] == "" {
  516. continue
  517. } else {
  518. contact["phone"] = vv["company_phone"] //联系电话
  519. break
  520. }
  521. } else {
  522. contact["phone"] = "" //联系电话
  523. }
  524. }
  525. }
  526. //log.Println(k, contact["phone"], resulttmp["_id"])
  527. //time.Sleep(10 * time.Second)
  528. if phone, ok := contact["phone"].(string); ok && phone != "" {
  529. if Reg_xing.MatchString(phone) || !Reg_tel.MatchString(phone) {
  530. contact["phone"] = "" //联系电话
  531. }
  532. } else {
  533. contact["phone"] = "" //联系电话
  534. }
  535. contact["topscopeclass"] = "企业公示" //项目类型
  536. contact["updatetime"] = time.Now().Unix() //更新时间
  537. contact["infoid"] = "" //招标信息id
  538. contacts = append(contacts, contact)
  539. }
  540. //添加临时表匹配到的联系人
  541. if buyerperson, ok := tmp["buyerperson"].(string); ok &&utf8.RuneCountInString(buyerperson)>=2 && buyerperson != "" &&
  542. !Reg_xing.MatchString(buyerperson) && Reg_person.MatchString(buyerperson) {
  543. vvv := make(map[string]interface{})
  544. vvv["infoid"] = tmp["_id"].(bson.ObjectId).Hex()
  545. vvv["contact_person"] = buyerperson
  546. vvv["contact_type"] = "项目联系人"
  547. if buyertel, ok := tmp["buyertel"].(string); ok && !Reg_xing.MatchString(buyertel) && Reg_tel.MatchString(buyertel) {
  548. vvv["phone"] = buyertel
  549. } else {
  550. vvv["phone"] = ""
  551. }
  552. vvv["topscopeclass"] = strings.Join(tmpclass, ";")
  553. vvv["updatetime"] = time.Now().Unix()
  554. contacts = append(contacts, vvv)
  555. }
  556. savetmp := make(map[string]interface{}, 0)
  557. for _, sk := range BuyerFields {
  558. if sk == "_id" {
  559. savetmp["tmp"+sk] = resulttmp[sk]
  560. continue
  561. } else if sk == "area_code" {
  562. //行政区划代码
  563. savetmp[sk] = fmt.Sprint(resulttmp[sk])
  564. continue
  565. } else if sk == "report_websites" {
  566. //网址
  567. if resulttmp["report_websites"] == nil {
  568. savetmp["website"] = ""
  569. } else {
  570. report_websitesArr := []string{}
  571. if ppms, ok := resulttmp[sk].([]interface{}); ok {
  572. for _, v := range ppms {
  573. if vvv, ok := v.(map[string]interface{}); ok {
  574. if rv, ok := vvv["website_url"].(string); ok {
  575. report_websitesArr = append(report_websitesArr, rv)
  576. }
  577. }
  578. }
  579. }
  580. sort.Strings(report_websitesArr)
  581. savetmp["website"] = strings.Join(report_websitesArr, ";")
  582. }
  583. continue
  584. } else if sk == "wechat_accounts" {
  585. savetmp[sk] = []interface{}{}
  586. continue
  587. } else if sk == "buyer_name" {
  588. if resulttmp["company_name"] == nil {
  589. savetmp[sk] = ""
  590. } else {
  591. savetmp[sk] = resulttmp["company_name"]
  592. }
  593. continue
  594. } else if sk == "address" {
  595. if resulttmp["company_address"] == nil {
  596. savetmp[sk] = ""
  597. } else {
  598. savetmp[sk] = resulttmp["company_address"]
  599. }
  600. continue
  601. }else if sk == "buyerclass" {
  602. if tmp["buyerclass"]==nil {
  603. savetmp[sk] = ""
  604. }else {
  605. savetmp[sk] = tmp["buyerclass"]
  606. }
  607. continue
  608. }
  609. if resulttmp[sk] == nil && sk != "history_name" && sk != "wechat_accounts" &&
  610. sk != "buyer_name" && sk != "address" && sk != "buyerclass"&&
  611. sk != "contact" && sk != "report_websites" {
  612. savetmp[sk] = ""
  613. } else {
  614. savetmp[sk] = resulttmp[sk]
  615. }
  616. }
  617. //判断分包
  618. if tmp["package"] != nil {
  619. PackageDealWithBuyer(&savetmp, tmp, errbuyer)
  620. }
  621. //tmps = append(tmps, savetmp)
  622. savetmp["comeintime"] = time.Now().Unix()
  623. savetmp["updatatime"] = time.Now().Unix()
  624. //保存mongo
  625. saveid := FClient.Save(Config["mgo_qyk_buyer"], savetmp)
  626. if saveid != "" {
  627. //保存redis
  628. rc := RedisPool.Get()
  629. rc.Do("SELECT", redis_buyer_db)
  630. if _, err := rc.Do("SET", savetmp["buyer_name"], saveid); err != nil {
  631. log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["buyer_name"], err)
  632. } else {
  633. //删除临时表
  634. if deleteNum := FClient.Del(Config["mgo_qyk_c_b_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
  635. log.Println("删除临时表失败", deleteNum)
  636. }
  637. }
  638. if err := rc.Close(); err != nil {
  639. log.Println(err)
  640. }
  641. } else {
  642. log.Println("save mongo err:", saveid, tmp["_id"])
  643. }
  644. }
  645. }
  646. FClient.DestoryMongoConn(fconn)
  647. log.Println("buyer_new,遍历完成")
  648. }
  649. }
  650. FClient.DestoryMongoConn(Fcconn)
  651. t2.Reset(time.Minute)
  652. //nextNode("buyerent",timenow)
  653. }
  654. }
  655. //分包处理
  656. func PackageDealWithBuyer(contactMap *map[string]interface{}, tmp map[string]interface{}, comName string) []interface{} {
  657. util.Catch()
  658. //if v, ok := tmp["package"].(map[string]interface{}); ok {
  659. //for i, pv := range v {
  660. // log.Println(i, pv)
  661. //}
  662. //}
  663. return nil
  664. }