main.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853
  1. package main
  2. /**
  3. 招标信息判重
  4. **/
  5. import (
  6. "encoding/json"
  7. "flag"
  8. "fmt"
  9. "log"
  10. mu "mfw/util"
  11. "net"
  12. "qfw/util"
  13. "qfw/util/mongodb"
  14. "regexp"
  15. "sync"
  16. "time"
  17. )
  18. var (
  19. Sysconfig map[string]interface{} //配置文件
  20. mconf map[string]interface{} //mongodb配置信息
  21. mgo *mongodb.MongodbSim //mongodb操作对象
  22. //siteMgo *mongodb.MongodbSim
  23. extract string
  24. extract_copy string
  25. bidding string
  26. udpclient mu.UdpClient //udp对象
  27. nextNode []map[string]interface{} //下节点数组
  28. dupdays = 5 //初始化判重范围
  29. DM *datamap //
  30. HM *historymap //判重数据
  31. lastid = ""
  32. //正则筛选相关
  33. FilterRegTitle = regexp.MustCompile("^_$")
  34. FilterRegTitle_1 = regexp.MustCompile("^_$")
  35. FilterRegTitle_2 = regexp.MustCompile("^_$")
  36. isMerger bool //是否合并
  37. SiteMap map[string]map[string]interface{} //站点map
  38. )
  39. func init() {
  40. flag.StringVar(&lastid, "id", "", "最后加载id") //以小于等于此id开始加载最近几天的数据
  41. flag.Parse()
  42. //172.17.145.163:27080
  43. util.ReadConfig(&Sysconfig)
  44. nextNode = util.ObjArrToMapArr(Sysconfig["nextNode"].([]interface{}))
  45. mconf = Sysconfig["mongodb"].(map[string]interface{})
  46. mgo = &mongodb.MongodbSim{
  47. MongodbAddr: mconf["addr"].(string),
  48. DbName: mconf["db"].(string),
  49. Size: util.IntAllDef(mconf["pool"], 10),
  50. }
  51. extract = mconf["extract"].(string)
  52. extract_copy = mconf["extract_copy"].(string)
  53. mgo.InitPool()
  54. //测试可以临时注释
  55. dupdays = util.IntAllDef(Sysconfig["dupdays"], 3)
  56. //加载数据
  57. DM = NewDatamap(dupdays, lastid)
  58. FilterRegTitle = regexp.MustCompile(util.ObjToString(Sysconfig["specialwords"]))
  59. FilterRegTitle_1 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_1"]))
  60. FilterRegTitle_2 = regexp.MustCompile(util.ObjToString(Sysconfig["specialtitle_2"]))
  61. isMerger = Sysconfig["isMerger"].(bool)
  62. //配置站点Map
  63. SiteMap = make(map[string]map[string]interface{},0)
  64. start := int(time.Now().Unix())
  65. //站点配置
  66. sess_site := mgo.GetMgoConn()
  67. defer sess_site.Close()
  68. res_site := sess_site.DB("zhaolongyue").C("site").Find(nil).Sort("_id").Iter()
  69. for site_dict := make(map[string]interface{}); res_site.Next(&site_dict); {
  70. data_map := map[string]interface{}{
  71. "area":util.ObjToString(site_dict["area"]),
  72. "city":util.ObjToString(site_dict["city"]),
  73. "district":util.ObjToString(site_dict["district"]),
  74. "sitetype":util.ObjToString(site_dict["sitetype"]),
  75. "level":util.ObjToString(site_dict["level"]),
  76. }
  77. SiteMap[util.ObjToString(site_dict["site"])]= data_map
  78. }
  79. fmt.Printf("用时:%d秒,%d个",int(time.Now().Unix())-start,len(SiteMap))
  80. }
  81. func main() {
  82. go checkMapJob()
  83. updport := Sysconfig["udpport"].(string)
  84. udpclient = mu.UdpClient{Local: updport, BufSize: 1024}
  85. udpclient.Listen(processUdpMsg)
  86. log.Println("Udp服务监听", updport)
  87. time.Sleep(99999 * time.Hour)
  88. }
  89. func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
  90. fmt.Println("接受的段数据")
  91. switch act {
  92. case mu.OP_TYPE_DATA: //上个节点的数据
  93. //从表中开始处理
  94. var mapInfo map[string]interface{}
  95. err := json.Unmarshal(data, &mapInfo)
  96. log.Println("err:", err, "mapInfo:", mapInfo)
  97. if err != nil {
  98. udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
  99. } else if mapInfo != nil {
  100. taskType:= util.ObjToString(mapInfo["stype"])
  101. if taskType == "historyTask" {
  102. //更新流程
  103. go historyTask(data,mapInfo)
  104. }else if taskType == "normalTask" {
  105. //判重流程
  106. go task(data, mapInfo)
  107. }else {
  108. //其他
  109. go task(data, mapInfo)
  110. }
  111. key, _ := mapInfo["key"].(string)
  112. if key == "" {
  113. key = "udpok"
  114. }
  115. udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
  116. }
  117. case mu.OP_NOOP: //下个节点回应
  118. ok := string(data)
  119. if ok != "" {
  120. log.Println("ok:", ok)
  121. udptaskmap.Delete(ok)
  122. }
  123. }
  124. }
  125. //开始判重程序
  126. func task(data []byte, mapInfo map[string]interface{}) {
  127. fmt.Println("开始数据判重")
  128. defer util.Catch()
  129. //区间id
  130. sess := mgo.GetMgoConn()
  131. defer mgo.DestoryMongoConn(sess)
  132. q := map[string]interface{}{
  133. "_id": map[string]interface{}{
  134. "$gt": util.StringTOBsonId(mapInfo["gtid"].(string)),
  135. "$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
  136. },
  137. }
  138. it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
  139. updateExtract := [][]map[string]interface{}{}
  140. pool := make(chan bool, 16)
  141. wg := &sync.WaitGroup{}
  142. mapLock := &sync.Mutex{}
  143. n, repeateN := 0, 0
  144. for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {
  145. if n%10000 == 0 {
  146. log.Println("current:", n, tmp["_id"],"repeateN:",repeateN)
  147. }
  148. pool <- true
  149. wg.Add(1)
  150. go func(tmp map[string]interface{}) {
  151. defer func() {
  152. <-pool
  153. wg.Done()
  154. }()
  155. info := NewInfo(tmp)
  156. //是否为无效数据
  157. if invalidData(info.buyer,info.projectname,info.projectcode) {
  158. mapLock.Lock()
  159. updateExtract = append(updateExtract, []map[string]interface{}{
  160. map[string]interface{}{
  161. "_id": tmp["_id"],
  162. },
  163. map[string]interface{}{
  164. "$set": map[string]interface{}{
  165. "repeat":-1,
  166. },
  167. },
  168. })
  169. if len(updateExtract) > 500 {
  170. mgo.UpdateBulk(extract, updateExtract...)
  171. updateExtract = [][]map[string]interface{}{}
  172. }
  173. mapLock.Unlock()
  174. }else {
  175. //判重原因 reason tmp["_id"] 对比id id原始id
  176. mapLock.Lock()
  177. b, source,reason := DM.check(info)
  178. if b { //有重复,生成更新语句,更新抽取和更新招标
  179. repeateN++
  180. var mergeArr = []int64{} //更改合并数组记录
  181. var newData = &Info{} //更换新的数据池数据
  182. var id_map = map[string]interface{}{}
  183. repeat_id := source.id
  184. id_map["_id"]= util.StringTOBsonId(info.id)
  185. if isMerger{
  186. //需要合并相关操作
  187. //合并操作--评功权重打分-合并完替换原始数据池
  188. basic_bool := basicDataScore(source,info)
  189. if basic_bool {
  190. //已原始数据为标准-对比数据打判重标签
  191. newData,mergeArr= mergeDataFields(source,info)
  192. DM.replaceSourceData(newData,source.id) //替换
  193. id_map["_id"]= util.StringTOBsonId(source.id)
  194. repeat_id = source.id
  195. }else {
  196. //已对比数据为标准 ,数据池的数据打判重标签
  197. newData,mergeArr= mergeDataFields(info,source)
  198. DM.replaceSourceData(newData,source.id)//替换
  199. id_map["_id"]= util.StringTOBsonId(info.id)
  200. repeat_id = info.id
  201. }
  202. }
  203. var update_map = map[string]interface{}{
  204. "$set": map[string]interface{}{
  205. "repeat_reason":reason,
  206. "repeat":1,
  207. "repeatid":repeat_id,
  208. },
  209. }
  210. if isMerger {
  211. //合并记录
  212. if len(newData.mergemap)>0 {
  213. update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
  214. //fmt.Println("合并长度:",len(newData.mergemap))
  215. }
  216. //更新合并后的数据
  217. for _,value :=range mergeArr {
  218. if value==1 {
  219. update_map["$set"].(map[string]interface{})["area"] = newData.area
  220. update_map["$set"].(map[string]interface{})["city"] = newData.city
  221. }else if value==2 {
  222. update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
  223. }else if value==3 {
  224. update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
  225. }else if value==4 {
  226. update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
  227. }else if value==5 {
  228. update_map["$set"].(map[string]interface{})["budget"] = newData.budget
  229. }else if value==6 {
  230. update_map["$set"].(map[string]interface{})["winner"] = newData.winner
  231. }else if value==7 {
  232. update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
  233. }else if value==8 {
  234. update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
  235. }else {
  236. }
  237. }
  238. }
  239. //构建数据库更新用到的
  240. updateExtract = append(updateExtract, []map[string]interface{}{
  241. id_map,
  242. update_map,
  243. })
  244. if len(updateExtract) > 500 {
  245. mgo.UpdateBulk(extract, updateExtract...)
  246. updateExtract = [][]map[string]interface{}{}
  247. }
  248. mapLock.Unlock()
  249. } else {
  250. mapLock.Unlock()
  251. }
  252. }
  253. }(tmp)
  254. tmp = make(map[string]interface{})
  255. }
  256. wg.Wait()
  257. if len(updateExtract) > 0 {
  258. mgo.UpdateBulk(extract, updateExtract...)
  259. //mgo.UpdateBulk(bidding, updateBidding...)
  260. }
  261. log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])
  262. //任务完成,开始发送广播通知下面节点
  263. if n > repeateN && mapInfo["stop"] == nil {
  264. for _, to := range nextNode {
  265. sid, _ := mapInfo["gtid"].(string)
  266. eid, _ := mapInfo["lteid"].(string)
  267. key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
  268. by, _ := json.Marshal(map[string]interface{}{
  269. "gtid": sid,
  270. "lteid": eid,
  271. "stype": util.ObjToString(to["stype"]),
  272. "key": key,
  273. })
  274. addr := &net.UDPAddr{
  275. IP: net.ParseIP(to["addr"].(string)),
  276. Port: util.IntAll(to["port"]),
  277. }
  278. node := &udpNode{by, addr, time.Now().Unix(), 0}
  279. udptaskmap.Store(key, node)
  280. udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
  281. }
  282. }
  283. }
  284. //支持历史更新
  285. func historyTask(data []byte, mapInfo map[string]interface{}) {
  286. fmt.Println("开始取历史时间段")
  287. defer util.Catch()
  288. sess := mgo.GetMgoConn()
  289. defer mgo.DestoryMongoConn(sess)
  290. q := map[string]interface{}{
  291. "_id": map[string]interface{}{
  292. "$gt": util.StringTOBsonId(mapInfo["gtid"].(string)),
  293. "$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
  294. },
  295. }
  296. it := sess.DB(mgo.DbName).C(extract).Find(&q).Iter()
  297. minTime,maxTime:=int64(0),int64(0)
  298. for tmp := make(map[string]interface{}); it.Next(&tmp);{
  299. //取出最大最小时间
  300. if minTime==0||maxTime ==0 {
  301. minTime = util.Int64All(tmp["comeintime"])
  302. maxTime = util.Int64All(tmp["comeintime"])
  303. }else {
  304. t := util.Int64All(tmp["comeintime"])
  305. if t<minTime&&t!=0 {
  306. minTime = t
  307. }
  308. if t>maxTime&&t!=0 {
  309. maxTime = t
  310. }
  311. }
  312. }
  313. fmt.Println("最小时间==",minTime,"最大时间==",maxTime)
  314. //最小时间== 1568087634 最大时间== 1568103381
  315. HM = NewHistorymap(util.ObjToString(mapInfo["gtid"]),
  316. util.ObjToString(mapInfo["lteid"]),minTime,maxTime)
  317. //return
  318. //开始判重...
  319. defer util.Catch()
  320. sess_task := mgo.GetMgoConn()
  321. defer mgo.DestoryMongoConn(sess_task)
  322. q_task := map[string]interface{}{
  323. "_id": map[string]interface{}{
  324. "$gt": util.StringTOBsonId(mapInfo["gtid"].(string)),
  325. "$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
  326. },
  327. }
  328. it_task := sess.DB(mgo.DbName).C(extract).Find(&q_task).Iter()
  329. updateExtract := [][]map[string]interface{}{}
  330. pool := make(chan bool, 16)
  331. wg := &sync.WaitGroup{}
  332. mapLock := &sync.Mutex{}
  333. n, repeateN := 0, 0
  334. for tmp := make(map[string]interface{}); it_task.Next(&tmp); n++ {
  335. if n%10000 == 0 {
  336. log.Println("current:", n, tmp["_id"],"repeateN:",repeateN)
  337. }
  338. pool <- true
  339. wg.Add(1)
  340. go func(tmp map[string]interface{}) {
  341. defer func() {
  342. <-pool
  343. wg.Done()
  344. }()
  345. info := NewInfo(tmp)
  346. //是否为无效数据
  347. if invalidData(info.buyer,info.projectname,info.projectcode) {
  348. mapLock.Lock()
  349. updateExtract = append(updateExtract, []map[string]interface{}{
  350. map[string]interface{}{
  351. "_id": tmp["_id"],
  352. },
  353. map[string]interface{}{
  354. "$set": map[string]interface{}{
  355. "repeat": -1,
  356. },
  357. },
  358. })
  359. if len(updateExtract) > 500 {
  360. mgo.UpdateBulk(extract, updateExtract...)
  361. updateExtract = [][]map[string]interface{}{}
  362. }
  363. mapLock.Unlock()
  364. }else {
  365. mapLock.Lock()
  366. b, source,reason := HM.checkHistory(info)
  367. if b { //有重复,生成更新语句,更新抽取和更新招标
  368. if reason == "未判重记录" {
  369. fmt.Println("未判重记录")
  370. //把info的数据判重的标签更换,并新增字段
  371. DM.replaceSourceData(info, info.id) //替换即添加
  372. updateExtract = append(updateExtract, []map[string]interface{}{
  373. map[string]interface{}{
  374. "_id": tmp["_id"],
  375. },
  376. map[string]interface{}{
  377. "$set": map[string]interface{}{
  378. "repeat": 0,
  379. "repeatid": -2,
  380. },
  381. },
  382. })
  383. if len(updateExtract) > 500 {
  384. mgo.UpdateBulk(extract, updateExtract...)
  385. updateExtract = [][]map[string]interface{}{}
  386. }
  387. mapLock.Unlock()
  388. }else {
  389. repeateN++
  390. var mergeArr = []int64{} //更改合并数组记录
  391. var newData = &Info{} //更换新的数据池数据
  392. var id_map = map[string]interface{}{}
  393. repeat_id := source.id
  394. id_map["_id"]= util.StringTOBsonId(info.id)
  395. if isMerger{
  396. //需要合并相关操作
  397. //合并操作--评功权重打分-合并完替换原始数据池
  398. basic_bool := basicDataScore(source,info)
  399. if basic_bool {
  400. //已原始数据为标准-对比数据打判重标签
  401. newData,mergeArr= mergeDataFields(source,info)
  402. DM.replaceSourceData(newData,source.id) //替换
  403. id_map["_id"]= util.StringTOBsonId(source.id)
  404. repeat_id = source.id
  405. }else {
  406. //已对比数据为标准 ,数据池的数据打判重标签
  407. newData,mergeArr= mergeDataFields(info,source)
  408. DM.replaceSourceData(newData,source.id)//替换
  409. id_map["_id"]= util.StringTOBsonId(info.id)
  410. repeat_id = info.id
  411. }
  412. }
  413. var update_map = map[string]interface{}{
  414. "$set": map[string]interface{}{
  415. "repeat_reason":reason,
  416. "repeat":1,
  417. "repeatid":repeat_id,
  418. },
  419. }
  420. if isMerger {
  421. //合并记录
  422. if len(newData.mergemap)>0 {
  423. update_map["$set"].(map[string]interface{})["merge"] = newData.mergemap
  424. //fmt.Println("合并长度:",len(newData.mergemap))
  425. }
  426. //更新合并后的数据
  427. for _,value :=range mergeArr {
  428. if value==1 {
  429. update_map["$set"].(map[string]interface{})["area"] = newData.area
  430. update_map["$set"].(map[string]interface{})["city"] = newData.city
  431. }else if value==2 {
  432. update_map["$set"].(map[string]interface{})["projectname"] = newData.projectname
  433. }else if value==3 {
  434. update_map["$set"].(map[string]interface{})["projectcode"] = newData.projectcode
  435. }else if value==4 {
  436. update_map["$set"].(map[string]interface{})["buyer"] = newData.buyer
  437. }else if value==5 {
  438. update_map["$set"].(map[string]interface{})["budget"] = newData.budget
  439. }else if value==6 {
  440. update_map["$set"].(map[string]interface{})["winner"] = newData.winner
  441. }else if value==7 {
  442. update_map["$set"].(map[string]interface{})["bidamount"] = newData.bidamount
  443. }else if value==8 {
  444. update_map["$set"].(map[string]interface{})["bidopentime"] = newData.bidopentime
  445. }else {
  446. }
  447. }
  448. }
  449. //构建数据库更新用到的
  450. updateExtract = append(updateExtract, []map[string]interface{}{
  451. id_map,
  452. update_map,
  453. })
  454. if len(updateExtract) > 500 {
  455. mgo.UpdateBulk(extract, updateExtract...)
  456. updateExtract = [][]map[string]interface{}{}
  457. }
  458. mapLock.Unlock()
  459. }
  460. }else {
  461. mapLock.Unlock()
  462. }
  463. }
  464. }(tmp)
  465. tmp = make(map[string]interface{})
  466. }
  467. wg.Wait()
  468. if len(updateExtract) > 0 {
  469. mgo.UpdateBulk(extract, updateExtract...)
  470. //mgo.UpdateBulk(bidding, updateBidding...)
  471. }
  472. log.Println("this task over.", n, "repeateN:", repeateN, mapInfo["stop"])
  473. //任务完成,开始发送广播通知下面节点
  474. if n > repeateN &&mapInfo["stop"] == nil {
  475. for _, to := range nextNode {
  476. sid, _ := mapInfo["gtid"].(string)
  477. eid, _ := mapInfo["lteid"].(string)
  478. key := sid + "-" + eid + "-" + util.ObjToString(to["stype"])
  479. by, _ := json.Marshal(map[string]interface{}{
  480. "gtid": sid,
  481. "lteid": eid,
  482. "stype": util.ObjToString(to["stype"]),
  483. "key": key,
  484. })
  485. addr := &net.UDPAddr{
  486. IP: net.ParseIP(to["addr"].(string)),
  487. Port: util.IntAll(to["port"]),
  488. }
  489. node := &udpNode{by, addr, time.Now().Unix(), 0}
  490. udptaskmap.Store(key, node)
  491. udpclient.WriteUdp(by, mu.OP_TYPE_DATA, addr)
  492. }
  493. }
  494. }
  495. //合并字段
  496. func mergeDataFields(source *Info, info *Info) (*Info,[]int64){
  497. var mergeArr []int64
  498. mergeArr = make([]int64,0)
  499. //1、城市
  500. if (source.area==""||source.area=="全国")&&info.area!="全国"&&info.area!=""{
  501. var arrA []string
  502. if source.mergemap["area"]==nil {
  503. arrA = make([]string, 0)
  504. }else {
  505. arrA = source.mergemap["area"].([]string)
  506. }
  507. arrA = append(arrA,source.area)
  508. source.mergemap["area"] = arrA
  509. var arrC []string
  510. if source.mergemap["city"]==nil {
  511. arrC = make([]string, 0)
  512. }else {
  513. arrC = source.mergemap["city"].([]string)
  514. }
  515. arrC = append(arrC,source.city)
  516. source.mergemap["city"] = arrC
  517. source.area = info.area
  518. source.city = info.city
  519. mergeArr = append(mergeArr,1)
  520. //fmt.Println("合并-城市")
  521. }
  522. //2、项目名称
  523. if source.projectname==""&&info.projectname!=""{
  524. var arr []string
  525. if source.mergemap["projectname"]==nil {
  526. arr = make([]string, 0)
  527. }else {
  528. arr = source.mergemap["projectname"].([]string)
  529. }
  530. arr = append(arr,source.projectname)
  531. source.mergemap["projectname"] = arr
  532. source.projectname = info.projectname
  533. mergeArr = append(mergeArr,2)
  534. //fmt.Println("合并-项目名称")
  535. }
  536. //3、项目编号
  537. if source.projectcode==""&&info.projectcode!=""{
  538. var arr []string
  539. if source.mergemap["projectcode"]==nil {
  540. arr = make([]string, 0)
  541. }else {
  542. arr = source.mergemap["projectcode"].([]string)
  543. }
  544. arr = append(arr,source.projectcode)
  545. source.mergemap["projectcode"] = arr
  546. source.projectcode = info.projectcode
  547. mergeArr = append(mergeArr,3)
  548. //fmt.Println("合并-项目标号")
  549. }
  550. //4、采购单位
  551. if source.buyer==""&&info.buyer!=""{
  552. var arr []string
  553. if source.mergemap["buyer"]==nil {
  554. arr = make([]string, 0)
  555. }else {
  556. arr = source.mergemap["buyer"].([]string)
  557. }
  558. arr = append(arr,source.buyer)
  559. source.mergemap["buyer"] = arr
  560. source.buyer = info.buyer
  561. mergeArr = append(mergeArr,4)
  562. //fmt.Println("合并-采购单位")
  563. }
  564. //5、预算
  565. if source.budget==0&&info.budget!=0{
  566. var arr []float64
  567. if source.mergemap["budget"]==nil {
  568. arr = make([]float64, 0)
  569. }else {
  570. arr = source.mergemap["budget"].([]float64)
  571. }
  572. arr = append(arr,source.budget)
  573. source.mergemap["budget"] = arr
  574. source.budget = info.budget
  575. mergeArr = append(mergeArr,5)
  576. //fmt.Println("合并-预算")
  577. }
  578. //6、中标单位
  579. if source.winner==""&&info.winner!=""{
  580. var arr []string
  581. if source.mergemap["winner"]==nil {
  582. arr = make([]string, 0)
  583. }else {
  584. arr = source.mergemap["winner"].([]string)
  585. }
  586. arr = append(arr,source.winner)
  587. source.mergemap["winner"] = arr
  588. source.winner = info.winner
  589. mergeArr = append(mergeArr,6)
  590. //fmt.Println("合并-中标单位")
  591. }
  592. //7、中标金额
  593. if source.bidamount==0&&info.bidamount!=0{
  594. var arr []float64
  595. if source.mergemap["bidamount"]==nil {
  596. arr = make([]float64, 0)
  597. }else {
  598. arr = source.mergemap["bidamount"].([]float64)
  599. }
  600. arr = append(arr,source.bidamount)
  601. source.mergemap["bidamount"] = arr
  602. source.bidamount = info.bidamount
  603. mergeArr = append(mergeArr,7)
  604. //fmt.Println("合并-中标金额")
  605. }
  606. //8、开天时间-地点
  607. if source.bidopentime==0&&info.bidopentime!=0{
  608. var arr []int64
  609. if source.mergemap["bidopentime"]==nil {
  610. arr = make([]int64, 0)
  611. }else {
  612. arr = source.mergemap["bidopentime"].([]int64)
  613. }
  614. arr = append(arr,source.bidopentime)
  615. source.mergemap["bidopentime"] = arr
  616. source.bidopentime = info.bidopentime
  617. mergeArr = append(mergeArr,8)
  618. //fmt.Println("合并-开标时间")
  619. }
  620. //以上合并过于简单,待进一步优化
  621. return source,mergeArr
  622. }
  623. //权重评估
  624. func basicDataScore(v *Info, info *Info) bool {
  625. //权重评估
  626. /*
  627. 网站优先级判定规则:
  628. 1、中央>省>市>县区
  629. 2、政府采购>公共资源>采购单位官网>招标代理公司/平台
  630. */
  631. v_score,info_score :=-1,-1
  632. dict_v := SiteMap[v.site]
  633. dict_info := SiteMap[info.site]
  634. //先判断level
  635. if dict_v !=nil {
  636. v_level := util.ObjToString(dict_v["level"])
  637. if v_level =="中央" {
  638. v_score = 4
  639. }else if v_level =="省级" {
  640. v_score = 3
  641. }else if v_level =="市级" {
  642. v_score = 2
  643. }else if v_level =="县区" {
  644. v_score = 1
  645. }else if v_level =="" {
  646. }else {
  647. v_score = 0
  648. }
  649. }
  650. if dict_info !=nil {
  651. info_level := util.ObjToString(dict_info["level"])
  652. if info_level =="中央" {
  653. info_score = 4
  654. }else if info_level =="省级" {
  655. info_score = 3
  656. }else if info_level =="市级" {
  657. info_score = 2
  658. }else if info_level =="县区" {
  659. info_score = 1
  660. }else if info_level == ""{
  661. }else {
  662. v_score = 0
  663. }
  664. }
  665. if v_score>info_score{
  666. return true
  667. }
  668. if v_score<info_score{
  669. return false
  670. }
  671. //判断sitetype
  672. if dict_v !=nil {
  673. v_sitetype := util.ObjToString(dict_v["sitetype"])
  674. if v_sitetype =="政府采购"||v_sitetype=="政府门户" {
  675. v_score = 4
  676. }else if v_sitetype =="公共资源" {
  677. v_score = 3
  678. }else if v_sitetype =="官方网站" {
  679. v_score = 2
  680. }else if v_sitetype =="社会公共招标平台"||v_sitetype =="企业招标平台" {
  681. v_score = 1
  682. }else if v_sitetype =="" {
  683. }else {
  684. v_score = 0
  685. }
  686. }
  687. if dict_info !=nil {
  688. info_sitetype := util.ObjToString(dict_info["sitetype"])
  689. if info_sitetype =="政府采购"||info_sitetype=="政府门户" {
  690. info_score = 4
  691. }else if info_sitetype =="公共资源" {
  692. info_score = 3
  693. }else if info_sitetype =="官方网站" {
  694. info_score = 2
  695. }else if info_sitetype =="社会公共招标平台"||info_sitetype =="企业招标平台" {
  696. info_score = 1
  697. }else if info_sitetype =="" {
  698. }else {
  699. info_score = 0
  700. }
  701. }
  702. if v_score>info_score{
  703. return true
  704. }
  705. if v_score<info_score{
  706. return false
  707. }
  708. //网站评估
  709. m,n:=0,0
  710. if v.projectname!="" {m++}
  711. if v.buyer!="" {m++}
  712. if v.projectcode!="" {m++}
  713. if v.budget!=0 {m++}
  714. if v.bidamount!=0 {m++}
  715. if v.winner!="" {m++}
  716. if v.bidopentime!=0 {m++}
  717. if v.agencyaddr!="" {m++}
  718. if v.agency!="" {m=m+2}
  719. if v.city!="" {m=m+2}
  720. if info.projectname!="" {n++}
  721. if info.buyer!="" {n++}
  722. if info.projectcode!="" {n++}
  723. if info.budget!=0 {n++}
  724. if info.bidamount!=0 {n++}
  725. if info.winner!="" {n++}
  726. if info.bidopentime!=0 {n++}
  727. if info.agencyaddr!="" {n++}
  728. if info.agency!="" {n=m+2}
  729. if info.city!="" {n=m+2}
  730. if m>n {
  731. return true
  732. }else if m==n {
  733. if v.comeintime>=info.comeintime {
  734. return true
  735. }else {
  736. return false
  737. }
  738. }else {
  739. return false
  740. }
  741. }
  742. //无效数据
  743. func invalidData(d1 string,d2 string,d3 string) bool{
  744. var n int
  745. if d1 != "" {
  746. n++
  747. }
  748. if d2 != "" {
  749. n++
  750. }
  751. if d3 != "" {
  752. n++
  753. }
  754. if n==0 {
  755. return true
  756. }
  757. return false
  758. }