|
|
@@ -4,9 +4,10 @@ import (
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"github.com/garyburd/redigo/redis"
|
|
|
- "gopkg.in/mgo.v2"
|
|
|
"gopkg.in/mgo.v2/bson"
|
|
|
"log"
|
|
|
+ util2 "mfw/util"
|
|
|
+ "net"
|
|
|
"qfw/util"
|
|
|
"sort"
|
|
|
"strings"
|
|
|
@@ -30,11 +31,12 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
log.Println(gtid, lteid, "不是Objectid，转换_id错误", gtid, lteid)
|
|
|
return
|
|
|
}
|
|
|
+ //timenow := time.Now().Unix()
|
|
|
//udp的id区间查询bidding 中标人 中标联系人 中标联系电话
|
|
|
// topscopeclass项目类型-industry行业类型&&topscopeclass联系人项目类型
|
|
|
// (area地区-province省份 city城市-city城市 district区县-district区县)
|
|
|
// winneraddr-company_address企业地址
|
|
|
- SourceClientcc := SourceClient.GetMgoConn(86400)
|
|
|
+ SourceClientcc := SourceClient.GetMgoConn(8640000)
|
|
|
cursor := SourceClientcc.DB(Config["mgodb_bidding"]).C(Config["mgodb_mgoinit_c"]).Find(bson.M{
|
|
|
"_id": bson.M{
|
|
|
"$gte": GId,
|
|
|
@@ -57,7 +59,12 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
//遍历bidding表保存到redis
|
|
|
//key：企业名 value:json结构体{"winner": 1, "winnertel": 1, "winnerperson": 1,"topscopeclass": 1, "winneraddr": 1,"_id":1}
|
|
|
tmp := make(map[string]interface{})
|
|
|
+ var num int
|
|
|
+ var tmpRangeId string
|
|
|
for cursor.Next(&tmp) {
|
|
|
+ num++
|
|
|
+ mgoId := tmp["_id"].(bson.ObjectId).Hex()
|
|
|
+ tmpRangeId = mgoId
|
|
|
winner, ok := tmp["winner"].(string)
|
|
|
if !ok || utf8.RuneCountInString(winner) < 4 {
|
|
|
continue
|
|
|
@@ -65,11 +72,57 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
//判断redis key是否存在
|
|
|
e_num := conn.Exists(winner).Val()
|
|
|
//获取字符串_id
|
|
|
- mgoId := tmp["_id"].(bson.ObjectId).Hex()
|
|
|
//替换_id
|
|
|
tmp["_id"] = mgoId
|
|
|
//创建value数组
|
|
|
tmps := make([]map[string]interface{}, 0)
|
|
|
+ //存量删除分包
|
|
|
+ if v, ok := tmp["package"].(map[string]interface{}); ok {
|
|
|
+ for i, vv := range v {
|
|
|
+ if vvv, ok2 := vv.(map[string]interface{}); ok2 {
|
|
|
+ if pwinner, ok := vvv["winner"].(string); ok && pwinner != "" {
|
|
|
+ if pwinper, ok4 := vvv["winnerperson"].(string); ok4 && pwinper != "" {
|
|
|
+ ptmp := make(map[string]interface{})
|
|
|
+ ptmp["p_id"] = mgoId + "_pkg_" + i
|
|
|
+ ptmp["_id"] = bson.NewObjectId().Hex()
|
|
|
+ ptmp["winner"] = winner
|
|
|
+ ptmp["winnerperson"] = pwinper
|
|
|
+ if pkgtel, ok6 := vvv["winnertel"].(string); ok6 {
|
|
|
+ ptmp["winnertel"] = pkgtel
|
|
|
+ }
|
|
|
+ if tmp["topscopeclass"] != nil {
|
|
|
+ ptmp["topscopeclass"] = tmp["topscopeclass"]
|
|
|
+ }
|
|
|
+ //分包里中标单位和企业名一样
|
|
|
+ if pwinner == winner {
|
|
|
+ tmps = append(tmps, ptmp)
|
|
|
+ } else if conn.Exists(pwinner).Val() > 0 {
|
|
|
+ //分包里中标单位和企业名不一样，存量里匹配上了
|
|
|
+ bytes, _ := conn.Get(pwinner).Bytes()
|
|
|
+ cltmps := make([]map[string]interface{}, 0)
|
|
|
+ json.Unmarshal(bytes, &cltmps)
|
|
|
+ cltmps = append(cltmps, ptmp)
|
|
|
+ bytes2, _ := json.Marshal(cltmps)
|
|
|
+ //存量redis的key存在，合并 key :企业名 val :[]map
|
|
|
+ if err := conn.Set(pwinner, string(bytes2), 0).Err(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ //分包和企业名不一样，而且存量没匹配上
|
|
|
+ cltmps := make([]map[string]interface{}, 0)
|
|
|
+ cltmps = append(cltmps, ptmp)
|
|
|
+ bytes2, _ := json.Marshal(cltmps)
|
|
|
+ //存量redis的key新增，新增 key :企业名 val :[]map
|
|
|
+ if err := conn.Set(pwinner, string(bytes2), 0).Err(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ delete(tmp, "package")
|
|
|
+ }
|
|
|
if e_num > 0 {
|
|
|
//存量redis的key存在，累加更新
|
|
|
bytes, _ := conn.Get(winner).Bytes()
|
|
|
@@ -82,6 +135,23 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
log.Println(err)
|
|
|
}
|
|
|
}
|
|
|
+ log.Println("存量 winner mongo遍历完成:", num)
|
|
|
+ if tmpRangeId != lteid{
|
|
|
+ by, _ := json.Marshal(map[string]interface{}{
|
|
|
+ "gtid": tmpRangeId,
|
|
|
+ "lteid": lteid,
|
|
|
+ "data_info":"save",
|
|
|
+ "stype": "winner",
|
|
|
+ })
|
|
|
+ if e := udpclient.WriteUdp(by, util2.OP_TYPE_DATA, &net.UDPAddr{
|
|
|
+ IP: net.ParseIP("127.0.0.1"),
|
|
|
+ Port: Updport,
|
|
|
+ }); e != nil {
|
|
|
+ log.Println(e)
|
|
|
+ }
|
|
|
+ SourceClient.DestoryMongoConn(SourceClientcc)
|
|
|
+ return
|
|
|
+ }
|
|
|
SourceClient.DestoryMongoConn(SourceClientcc)
|
|
|
//遍历redis
|
|
|
if scan := conn.Scan(0, "", 100); scan.Err() != nil {
|
|
|
@@ -99,14 +169,10 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
rdb.Do("SELECT", redis_winner_db)
|
|
|
if reply, err := redis.String(rdb.Do("GET", redisCName)); err != nil {
|
|
|
//redis不存在,存到临时表,定时任务处理
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- //if tmpid := FClient.Save("winner_new", tmps); tmpid == nil {
|
|
|
- // log.Println("存量 FClient.Save err", tmpid)
|
|
|
- //}
|
|
|
for _, vmap := range rValuesMaps {
|
|
|
vmap["_id"] = bson.ObjectIdHex(vmap["_id"].(string))
|
|
|
- if err = FClient.SaveForOld("winner_new", vmap); err != nil {
|
|
|
- log.Println("存量 FClient.Save err", err, vmap)
|
|
|
+ if errb := FClient.SaveByOriID(Config["mgo_qyk_c_w_new"], vmap); !errb {
|
|
|
+ log.Println("存量 FClient.Save err", errb, vmap)
|
|
|
}
|
|
|
}
|
|
|
//log.Println("get redis id err:定时任务处理", err, tmp)
|
|
|
@@ -120,7 +186,6 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
log.Println(err)
|
|
|
}
|
|
|
//拿到合并后的qyk
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, nil)
|
|
|
if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
log.Println(redisCName, "存量 redis id 不存在", reply)
|
|
|
@@ -197,116 +262,118 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
(*oldTmp)["contact"] = contactMaps
|
|
|
//mongo更新
|
|
|
(*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
log.Println("存量 mongo更新 err", esId, oldTmp)
|
|
|
}
|
|
|
//es更新
|
|
|
delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("存量 EsConn err :", err)
|
|
|
- }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
log.Println("存量历史合并执行完成 ok", gtid, lteid)
|
|
|
+ //发送udp 更新es段
|
|
|
|
|
|
} else {
|
|
|
- //增量
|
|
|
- overid := addfunc(gtid, cursor)
|
|
|
+ //增量处理
|
|
|
+ overid := gtid
|
|
|
+ tmp := map[string]interface{}{}
|
|
|
+ for cursor.Next(&tmp) {
|
|
|
+ overid = Add(overid, tmp)
|
|
|
+ }
|
|
|
SourceClient.DestoryMongoConn(SourceClientcc)
|
|
|
log.Println("增量合并执行完成 ok", gtid, lteid, overid)
|
|
|
+ //发送udp 更新es段
|
|
|
+ //nextNode("winnerent", timenow)
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
//增量
|
|
|
-func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
- //增量处理
|
|
|
- overid := gtid
|
|
|
- tmp := map[string]interface{}{}
|
|
|
- for cursor.Next(&tmp) {
|
|
|
- overid = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
- //log.Println(tmp["_id"])
|
|
|
- winner, ok := tmp["winner"].(string)
|
|
|
- if !ok || utf8.RuneCountInString(winner) < 4 {
|
|
|
- continue
|
|
|
+func Add(overid string, tmp map[string]interface{}) string {
|
|
|
+ overid = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
+ winner, ok := tmp["winner"].(string)
|
|
|
+ if !ok || utf8.RuneCountInString(winner) < 4 {
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //redis查询是否存在
|
|
|
+ rdb := RedisPool.Get()
|
|
|
+ rdb.Do("SELECT", redis_winner_db)
|
|
|
+ if reply, err := redis.String(rdb.Do("GET", winner)); err != nil {
|
|
|
+ //redis不存在存到临时表,定时任务处理
|
|
|
+ if errb := FClient.SaveByOriID(Config["mgo_qyk_c_w_new"], tmp); !errb {
|
|
|
+ log.Println("FClient.Save err", errb, tmp)
|
|
|
}
|
|
|
- //redis查询是否存在
|
|
|
- rdb := RedisPool.Get()
|
|
|
- rdb.Do("SELECT", redis_winner_db)
|
|
|
- if reply, err := redis.String(rdb.Do("GET", winner)); err != nil {
|
|
|
- //redis不存在存到临时表,定时任务处理
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if err := FClient.SaveForOld("winner_new", tmp); err != nil {
|
|
|
- log.Println("FClient.Save err", err, tmp)
|
|
|
- }
|
|
|
- //log.Println("get redis id err:定时任务处理", err, tmp)
|
|
|
- if err := rdb.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
- continue
|
|
|
- } else {
|
|
|
- if err := rdb.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
- //拿到合并后的qyk
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, bson.M{})
|
|
|
- if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
- log.Println("redis id 不存在")
|
|
|
- continue
|
|
|
- }
|
|
|
- //比较合并 行业类型
|
|
|
- tmpTopscopeclass := []string{}
|
|
|
- tmpConTopscopeclass := []string{}
|
|
|
- tmpTopscopeclassMap := make(map[string]bool)
|
|
|
+ //log.Println("get redis id err:定时任务处理", err, tmp)
|
|
|
+ if err := rdb.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ return overid
|
|
|
+ } else {
|
|
|
+ if err := rdb.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ //拿到合并后的qyk
|
|
|
+ oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, bson.M{})
|
|
|
+ if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
+ log.Println("redis id 不存在", reply)
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //比较合并 行业类型
|
|
|
+ tmpTopscopeclass := []string{}
|
|
|
+ tmpConTopscopeclass := []string{}
|
|
|
+ tmpTopscopeclassMap := make(map[string]bool)
|
|
|
|
|
|
- if (*oldTmp)["industry"] != nil {
|
|
|
- if v, ok := (*oldTmp)["industry"].([]interface{}); ok {
|
|
|
- for _, vv := range v {
|
|
|
- if vvv, ok := vv.(string); ok {
|
|
|
- tmpTopscopeclassMap[vvv] = true
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if v, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ if (*oldTmp)["industry"] != nil {
|
|
|
+ if v, ok := (*oldTmp)["industry"].([]interface{}); ok {
|
|
|
for _, vv := range v {
|
|
|
- if vvv, ok := vv.(string); ok && len(vvv) > 1 {
|
|
|
- tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
|
|
|
- tmpConTopscopeclass = append(tmpConTopscopeclass, vvv[:len(vvv)-1])
|
|
|
+ if vvv, ok := vv.(string); ok {
|
|
|
+ tmpTopscopeclassMap[vvv] = true
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- for k := range tmpTopscopeclassMap {
|
|
|
- tmpTopscopeclass = append(tmpTopscopeclass, k)
|
|
|
+ }
|
|
|
+ if v, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ for _, vv := range v {
|
|
|
+ if vvv, ok := vv.(string); ok && len(vvv) > 1 {
|
|
|
+ tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
|
|
|
+ tmpConTopscopeclass = append(tmpConTopscopeclass, vvv[:len(vvv)-1])
|
|
|
+ }
|
|
|
}
|
|
|
- sort.Strings(tmpTopscopeclass)
|
|
|
- (*oldTmp)["industry"] = tmpTopscopeclass
|
|
|
+ }
|
|
|
+ for k := range tmpTopscopeclassMap {
|
|
|
+ tmpTopscopeclass = append(tmpTopscopeclass, k)
|
|
|
+ }
|
|
|
+ sort.Strings(tmpTopscopeclass)
|
|
|
+ (*oldTmp)["industry"] = tmpTopscopeclass
|
|
|
|
|
|
- esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
|
|
|
- //更新行业类型
|
|
|
- if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
|
|
|
- (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- //mongo更新
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
- log.Println("mongo更新err", esId)
|
|
|
- }
|
|
|
+ esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
|
|
|
+ //更新行业类型
|
|
|
+ if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
|
|
|
+ (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
+ //mongo更新
|
|
|
+ if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
+ log.Println("mongo更新err", esId)
|
|
|
+ }
|
|
|
|
|
|
- //es更新
|
|
|
- delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("update es err:", err)
|
|
|
+ //es更新
|
|
|
+ delete((*oldTmp), "_id")
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //联系方式合并
|
|
|
+ contactMaps := make([]map[string]interface{}, 0)
|
|
|
+ if (*oldTmp)["contact"] != nil {
|
|
|
+ //直接添加联系人，不再判断
|
|
|
+ if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
|
|
|
+ for _, vv := range v {
|
|
|
+ contactMaps = append(contactMaps, vv.(map[string]interface{}))
|
|
|
}
|
|
|
- continue
|
|
|
- }
|
|
|
- //联系方式合并
|
|
|
- var tmpperson, winnertel string
|
|
|
- if tmppersona, ok := tmp["winnerperson"].(string); ok {
|
|
|
- tmpperson = tmppersona
|
|
|
}
|
|
|
+ }
|
|
|
+ var tmpperson, winnertel string
|
|
|
+ if tmppersona, ok := tmp["winnerperson"].(string); ok && tmppersona != "" && Reg_person.MatchString(tmppersona) && !Reg_xing.MatchString(tmppersona) {
|
|
|
+ tmpperson = tmppersona
|
|
|
+ }
|
|
|
+ if tmpperson != "" {
|
|
|
if winnerteltmp, ok := tmp["winnertel"].(string); ok {
|
|
|
winnertel = winnerteltmp
|
|
|
}
|
|
|
@@ -315,13 +382,7 @@ func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
} else {
|
|
|
winnertel = winnertel
|
|
|
}
|
|
|
- contactMaps := make([]interface{}, 0)
|
|
|
- if (*oldTmp)["contact"] != nil {
|
|
|
- //直接添加联系人，不再判断
|
|
|
- if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
|
|
|
- contactMaps = append(contactMaps, v...)
|
|
|
- }
|
|
|
- }
|
|
|
+
|
|
|
vvv := make(map[string]interface{})
|
|
|
vvv["infoid"] = overid
|
|
|
vvv["contact_person"] = tmpperson
|
|
|
@@ -330,23 +391,22 @@ func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
vvv["topscopeclass"] = strings.Join(tmpConTopscopeclass, ";")
|
|
|
vvv["updatetime"] = time.Now().Unix()
|
|
|
contactMaps = append(contactMaps, vvv)
|
|
|
- //分包处理
|
|
|
- PackageDealWith(&contactMaps, tmp)
|
|
|
- (*oldTmp)["contact"] = contactMaps
|
|
|
- //mongo更新
|
|
|
- (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
- log.Println("mongo更新 err", esId, oldTmp)
|
|
|
- }
|
|
|
- //es更新
|
|
|
- delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("EsConn err :", err)
|
|
|
- }
|
|
|
}
|
|
|
+ //分包处理
|
|
|
+ if tmp["package"] != nil {
|
|
|
+ PackageDealWith(tmp, tmpConTopscopeclass, overid)
|
|
|
+ }
|
|
|
+ (*oldTmp)["contact"] = contactMaps
|
|
|
+ //mongo更新
|
|
|
+ (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
+ if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
+ log.Println("mongo更新 err", esId, oldTmp)
|
|
|
+ }
|
|
|
+ //es更新
|
|
|
+ delete((*oldTmp), "_id")
|
|
|
}
|
|
|
return overid
|
|
|
+
|
|
|
}
|
|
|
|
|
|
//定时任务 新增
|
|
|
@@ -356,9 +416,10 @@ func TimedTaskWinner() {
|
|
|
//time.Sleep(time.Hour*70)
|
|
|
t2 := time.NewTimer(time.Second * 5)
|
|
|
for range t2.C {
|
|
|
+ //timenow:=time.Now().Unix()
|
|
|
Fcconn := FClient.GetMgoConn(86400)
|
|
|
tmpLast := map[string]interface{}{}
|
|
|
- if iter := Fcconn.DB(Config["mgodb_extract_kf"]).C("winner_new").Find(bson.M{}).Sort("-_id").Limit(1).Iter(); iter != nil {
|
|
|
+ if iter := Fcconn.DB(Config["mgodb_extract_kf"]).C(Config["mgo_qyk_c_w_new"]).Find(bson.M{}).Sort("-_id").Limit(1).Iter(); iter != nil {
|
|
|
if !iter.Next(&tmpLast) {
|
|
|
//临时表无数据
|
|
|
log.Println("临时表无数据:")
|
|
|
@@ -366,9 +427,9 @@ func TimedTaskWinner() {
|
|
|
FClient.DestoryMongoConn(Fcconn)
|
|
|
continue
|
|
|
} else {
|
|
|
- log.Println("临时表有数据:", tmpLast)
|
|
|
+ log.Println("临时表有数据:", tmpLast["_id"])
|
|
|
fconn := FClient.GetMgoConn(86400)
|
|
|
- cursor := fconn.DB(Config["mgodb_extract_kf"]).C("winner_new").Find(bson.M{
|
|
|
+ cursor := fconn.DB(Config["mgodb_extract_kf"]).C(Config["mgo_qyk_c_w_new"]).Find(bson.M{
|
|
|
"_id": bson.M{
|
|
|
"$lte": tmpLast["_id"],
|
|
|
},
|
|
|
@@ -391,14 +452,10 @@ func TimedTaskWinner() {
|
|
|
rdb := RedisPool.Get()
|
|
|
rdb.Do("SELECT", redis_winner_db)
|
|
|
if _, err := redis.String(rdb.Do("GET", errwinner)); err == nil {
|
|
|
- //redis存在发送udp进行处理
|
|
|
- TaskWinner(&map[string]interface{}{
|
|
|
- "gtid": tmpId,
|
|
|
- "lteid": tmpId,
|
|
|
- })
|
|
|
+ //增量合并
|
|
|
+ Add(tmpId, tmp)
|
|
|
//存在的话删除tmp mongo表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if DeletedCount := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !DeletedCount {
|
|
|
+ if DeletedCount := FClient.Del(Config["mgo_qyk_c_w_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !DeletedCount {
|
|
|
log.Println("删除临时表err:", DeletedCount)
|
|
|
}
|
|
|
if err := rdb.Close(); err != nil {
|
|
|
@@ -411,10 +468,11 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
}
|
|
|
//查询redis不存在新增
|
|
|
- FClient.DbName = Config["mgodb_enterprise"]
|
|
|
-
|
|
|
- resulttmp, b := FClient.FindOne(Config["mgodb_enterprise_c"], bson.M{"company_name": errwinner})
|
|
|
- if !b || (*resulttmp)["_id"] == nil {
|
|
|
+ sessionfinone := FClient.GetMgoConn()
|
|
|
+ resulttmp := make(map[string]interface{})
|
|
|
+ err := sessionfinone.DB(Config["mgodb_enterprise"]).C(Config["mgodb_enterprise_c"]).Find(bson.M{"company_name": errwinner}).One(&resulttmp)
|
|
|
+ FClient.DestoryMongoConn(sessionfinone)
|
|
|
+ if err != nil || resulttmp["_id"] == nil {
|
|
|
//log.Println(r)
|
|
|
//人工审核正则
|
|
|
var isok bool
|
|
|
@@ -439,122 +497,126 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
}
|
|
|
//都没匹配
|
|
|
- if tmp["winner_ok"] == nil && tmp["winner_err"] == nil{
|
|
|
+ if tmp["winner_ok"] == nil && tmp["winner_err"] == nil {
|
|
|
tmp["winner_err"] = 1
|
|
|
}
|
|
|
//匹配不到原始库,存入异常表删除临时表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if err := FClient.SaveForOld("winner_err", tmp); err != nil {
|
|
|
- log.Println("存入异常表错误", err, tmp)
|
|
|
+ if errb := FClient.SaveByOriID(Config["mgo_qyk_c_w_err"], tmp); !errb {
|
|
|
+ log.Println("存入异常表错误", errb, tmp)
|
|
|
}
|
|
|
- if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !b {
|
|
|
+ if deleteNum := FClient.Del(Config["mgo_qyk_c_w_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
log.Println("删除临时表错误", deleteNum)
|
|
|
}
|
|
|
continue
|
|
|
} else {
|
|
|
//log.Println(123)
|
|
|
//匹配到原始库,新增 resulttmp
|
|
|
- if (*resulttmp)["credit_no"] != nil {
|
|
|
- if credit_no, ok := (*resulttmp)["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
|
|
|
+ if resulttmp["credit_no"] != nil {
|
|
|
+ if credit_no, ok := resulttmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
|
|
|
len(strings.TrimSpace(credit_no)) > 8 {
|
|
|
dataNo := strings.TrimSpace(credit_no)[2:8]
|
|
|
if Addrs[dataNo] != nil {
|
|
|
if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
|
|
|
- if (*resulttmp)["province"] == nil || (*resulttmp)["province"] == "" {
|
|
|
- (*resulttmp)["province"] = v["province"]
|
|
|
+ if resulttmp["province"] == nil || resulttmp["province"] == "" {
|
|
|
+ resulttmp["province"] = v["province"]
|
|
|
}
|
|
|
- (*resulttmp)["city"] = v["city"]
|
|
|
- (*resulttmp)["district"] = v["district"]
|
|
|
+ resulttmp["city"] = v["city"]
|
|
|
+ resulttmp["district"] = v["district"]
|
|
|
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- contacts := make([]map[string]interface{}, 0)
|
|
|
- contact := make(map[string]interface{}, 0)
|
|
|
- if (*resulttmp)["legal_person"] != nil {
|
|
|
- contact["contact_person"] = (*resulttmp)["legal_person"] //联系人
|
|
|
- } else {
|
|
|
- contact["contact_person"] = "" //联系人
|
|
|
- }
|
|
|
- contact["contact_type"] = "法定代表人" //法定代表人
|
|
|
- //log.Println(1)
|
|
|
- if (*resulttmp)["annual_reports"] != nil {
|
|
|
- bytes, err := json.Marshal((*resulttmp)["annual_reports"])
|
|
|
- if err != nil {
|
|
|
- log.Println("annual_reports err:", err)
|
|
|
- }
|
|
|
- phonetmp := make([]map[string]interface{}, 0)
|
|
|
- err = json.Unmarshal(bytes, &phonetmp)
|
|
|
- if err != nil {
|
|
|
- log.Println("Unmarshal err:", err)
|
|
|
+ //行业类型
|
|
|
+ tmpclass := make([]string, 0)
|
|
|
+ if tclasss, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ for _, vv := range tclasss {
|
|
|
+ if vvv, ok := vv.(string); ok {
|
|
|
+ if len(vvv) > 1 {
|
|
|
+ tmpclass = append(tmpclass, vvv[:len(vvv)-1])
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
- for _, vv := range phonetmp {
|
|
|
- if vv["company_phone"] != nil {
|
|
|
- if vv["company_phone"] == "" {
|
|
|
- continue
|
|
|
+ }
|
|
|
+ contacts := make([]map[string]interface{}, 0)
|
|
|
+ if legal_person, ok := resulttmp["legal_person"].(string); ok && legal_person != "" && !Reg_xing.MatchString(legal_person) && Reg_person.MatchString(legal_person) {
|
|
|
+ contact := make(map[string]interface{}, 0)
|
|
|
+ contact["contact_person"] = legal_person //联系人
|
|
|
+ contact["contact_type"] = "法定代表人" //法定代表人
|
|
|
+ if resulttmp["annual_reports"] != nil {
|
|
|
+ bytes, err := json.Marshal(resulttmp["annual_reports"])
|
|
|
+ if err != nil {
|
|
|
+ log.Println("annual_reports err:", err)
|
|
|
+ }
|
|
|
+ phonetmp := make([]map[string]interface{}, 0)
|
|
|
+ err = json.Unmarshal(bytes, &phonetmp)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("Unmarshal err:", err)
|
|
|
+ }
|
|
|
+ for _, vv := range phonetmp {
|
|
|
+ if vv["company_phone"] != nil {
|
|
|
+ if vv["company_phone"] == "" {
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ contact["phone"] = vv["company_phone"] //联系电话
|
|
|
+ break
|
|
|
+ }
|
|
|
} else {
|
|
|
- contact["phone"] = vv["company_phone"] //联系电话
|
|
|
- break
|
|
|
+ contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
- } else {
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //log.Println(k, contact["phone"], resulttmp["_id"])
|
|
|
+ //time.Sleep(10 * time.Second)
|
|
|
+ if phone, ok := contact["phone"].(string); ok && phone != "" {
|
|
|
+ if Reg_xing.MatchString(phone) || !Reg_tel.MatchString(phone) {
|
|
|
contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
-
|
|
|
+ } else {
|
|
|
+ contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
+ contact["topscopeclass"] = "企业公示" //项目类型
|
|
|
+ contact["updatetime"] = time.Now().Unix() //更新时间
|
|
|
+ contact["infoid"] = "" //招标信息id
|
|
|
+ contacts = append(contacts, contact)
|
|
|
}
|
|
|
- //log.Println(k, contact["phone"], resulttmp["_id"])
|
|
|
- //time.Sleep(10 * time.Second)
|
|
|
- if contact["phone"] == nil {
|
|
|
- contact["phone"] = "" //联系电话
|
|
|
- }
|
|
|
- contact["topscopeclass"] = "企业公示" //项目类型
|
|
|
- contact["updatetime"] = time.Now().Unix() //更新时间
|
|
|
- contact["infoid"] = "" //招标信息id
|
|
|
- contacts = append(contacts, contact)
|
|
|
//添加临时表匹配到的联系人
|
|
|
- vvv := make(map[string]interface{})
|
|
|
- vvv["infoid"] = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
- if tmp["winnerperson"] != nil {
|
|
|
- vvv["contact_person"] = tmp["winnerperson"]
|
|
|
- } else {
|
|
|
- vvv["contact_person"] = ""
|
|
|
- }
|
|
|
- vvv["contact_type"] = "项目联系人"
|
|
|
- // "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1
|
|
|
- if tmp["winnertel"] != nil {
|
|
|
- vvv["phone"] = tmp["winnertel"]
|
|
|
- } else {
|
|
|
- vvv["phone"] = ""
|
|
|
- }
|
|
|
- tmpclass := make([]string, 0)
|
|
|
- if tclasss, ok := tmp["topscopeclass"].([]string); ok {
|
|
|
- for _, vv := range tclasss {
|
|
|
- if len(vv) > 1 {
|
|
|
- tmpclass = append(tmpclass, vv[:len(vv)-1])
|
|
|
- }
|
|
|
+ if winnerperson, ok := tmp["winnerperson"].(string); ok && winnerperson != "" &&
|
|
|
+ !Reg_xing.MatchString(winnerperson) && Reg_person.MatchString(winnerperson) {
|
|
|
+ vvv := make(map[string]interface{})
|
|
|
+ vvv["infoid"] = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
+ vvv["contact_person"] = winnerperson
|
|
|
+ vvv["contact_type"] = "项目联系人"
|
|
|
+ // "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1
|
|
|
+ if winnertel, ok := tmp["winnertel"].(string); ok && !Reg_xing.MatchString(winnertel) && Reg_tel.MatchString(winnertel) {
|
|
|
+ vvv["phone"] = winnertel
|
|
|
+ } else {
|
|
|
+ vvv["phone"] = ""
|
|
|
}
|
|
|
+
|
|
|
+ vvv["topscopeclass"] = strings.Join(tmpclass, ";")
|
|
|
+ vvv["updatetime"] = time.Now().Unix()
|
|
|
+ contacts = append(contacts, vvv)
|
|
|
}
|
|
|
- vvv["topscopeclass"] = strings.Join(tmpclass, ";")
|
|
|
- vvv["updatetime"] = time.Now().Unix()
|
|
|
- contacts = append(contacts, vvv)
|
|
|
- (*resulttmp)["contact"] = contacts
|
|
|
+
|
|
|
+ resulttmp["contact"] = contacts
|
|
|
|
|
|
savetmp := make(map[string]interface{}, 0)
|
|
|
for _, sk := range Fields {
|
|
|
if sk == "establish_date" {
|
|
|
- if (*resulttmp)[sk] != nil {
|
|
|
- savetmp[sk] = (*resulttmp)[sk].(time.Time).UTC().Unix()
|
|
|
+ if resulttmp[sk] != nil {
|
|
|
+ savetmp[sk] = resulttmp[sk].(time.Time).UTC().Unix()
|
|
|
continue
|
|
|
}
|
|
|
} else if sk == "capital" {
|
|
|
//log.Println(sk, resulttmp[sk])
|
|
|
- savetmp[sk] = ObjToMoney([]interface{}{(*resulttmp)[sk], ""})[0]
|
|
|
+ savetmp[sk] = ObjToMoney([]interface{}{resulttmp[sk], ""})[0]
|
|
|
continue
|
|
|
} else if sk == "partners" {
|
|
|
//log.Println(sk, resulttmp[sk], )
|
|
|
- if (*resulttmp)[sk] != nil {
|
|
|
- if ppms, ok := (*resulttmp)[sk].([]interface{}); ok {
|
|
|
+ if resulttmp[sk] != nil {
|
|
|
+ if ppms, ok := resulttmp[sk].([]interface{}); ok {
|
|
|
for i, _ := range ppms {
|
|
|
if ppms[i].(map[string]interface{})["stock_type"] != nil {
|
|
|
ppms[i].(map[string]interface{})["stock_type"] = "企业公示"
|
|
|
@@ -569,19 +631,19 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
continue
|
|
|
} else if sk == "_id" {
|
|
|
- savetmp["tmp"+sk] = (*resulttmp)[sk]
|
|
|
+ savetmp["tmp"+sk] = resulttmp[sk]
|
|
|
continue
|
|
|
} else if sk == "area_code" {
|
|
|
//行政区划代码
|
|
|
- savetmp[sk] = fmt.Sprint((*resulttmp)[sk])
|
|
|
+ savetmp[sk] = fmt.Sprint(resulttmp[sk])
|
|
|
continue
|
|
|
} else if sk == "report_websites" {
|
|
|
//网址
|
|
|
- if (*resulttmp)["report_websites"] == nil {
|
|
|
+ if resulttmp["report_websites"] == nil {
|
|
|
savetmp["website"] = ""
|
|
|
} else {
|
|
|
report_websitesArr := []string{}
|
|
|
- if ppms, ok := (*resulttmp)[sk].([]interface{}); ok {
|
|
|
+ if ppms, ok := resulttmp[sk].([]interface{}); ok {
|
|
|
for _, v := range ppms {
|
|
|
if vvv, ok := v.(map[string]interface{}); ok {
|
|
|
if rv, ok := vvv["website_url"].(string); ok {
|
|
|
@@ -610,50 +672,36 @@ func TimedTaskWinner() {
|
|
|
savetmp[sk] = tmpTopscopeclass
|
|
|
continue
|
|
|
}
|
|
|
- if (*resulttmp)[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
|
|
|
+ if resulttmp[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
|
|
|
savetmp[sk] = ""
|
|
|
} else {
|
|
|
- savetmp[sk] = (*resulttmp)[sk]
|
|
|
+ savetmp[sk] = resulttmp[sk]
|
|
|
}
|
|
|
}
|
|
|
+ //判断分包
|
|
|
+ if tmp["package"] != nil {
|
|
|
+ PackageDealWith(tmp,tmpclass, tmpId)
|
|
|
+ }
|
|
|
//tmps = append(tmps, savetmp)
|
|
|
+ savetmp["comeintime"] = time.Now().Unix()
|
|
|
savetmp["updatatime"] = time.Now().Unix()
|
|
|
//保存mongo
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
-
|
|
|
saveid := FClient.Save(Config["mgo_qyk_c"], savetmp)
|
|
|
if saveid != "" {
|
|
|
//保存redis
|
|
|
rc := RedisPool.Get()
|
|
|
rc.Do("SELECT", redis_winner_db)
|
|
|
- //var _id string
|
|
|
- //if v, ok := saveid.(primitive.ObjectID); ok {
|
|
|
- // _id = v.Hex()
|
|
|
- //}
|
|
|
if _, err := rc.Do("SET", savetmp["company_name"], saveid); err != nil {
|
|
|
log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["company_name"], err)
|
|
|
- if err := rc.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
} else {
|
|
|
- //保存es
|
|
|
- delete(savetmp, "_id")
|
|
|
- if err := rc.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
-
|
|
|
- //esConn := elastic.GetEsConn()
|
|
|
- //defer elastic.DestoryEsConn(esConn)
|
|
|
- if _, err := EsConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(saveid).BodyJson(savetmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("save es err :", tmp["_id"], savetmp["_id"], err)
|
|
|
- } else {
|
|
|
- //删除临时表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
- log.Println("删除临时表失败", deleteNum)
|
|
|
- }
|
|
|
+ //删除临时表
|
|
|
+ if deleteNum := FClient.Del(Config["mgo_qyk_c_w_new"], bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
+ log.Println("删除临时表失败", deleteNum)
|
|
|
}
|
|
|
}
|
|
|
+ if err := rc.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
} else {
|
|
|
log.Println("save mongo err:", saveid, tmp["_id"])
|
|
|
}
|
|
|
@@ -665,10 +713,34 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
FClient.DestoryMongoConn(Fcconn)
|
|
|
t2.Reset(time.Minute)
|
|
|
+ //nextNode("winnerent", timenow)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
//分包处理
|
|
|
-func PackageDealWith(contactMaps *[]interface{}, tmp map[string]interface{}) {
|
|
|
-
|
|
|
+func PackageDealWith(tmp map[string]interface{}, tmpclass []string,pkgid string) {
|
|
|
+ util.Catch()
|
|
|
+ if v, ok := tmp["package"].(map[string]interface{}); ok {
|
|
|
+ for i, pv := range v {
|
|
|
+ if ppv, ok2 := pv.(map[string]interface{}); ok2 {
|
|
|
+ pkgwinner, ok3 := ppv["winner"].(string)
|
|
|
+ if !ok3 || utf8.RuneCountInString(pkgwinner) < 4 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ //创建中标单位放入临时表
|
|
|
+ vvv := make(map[string]interface{})
|
|
|
+ vvv["winner"] = pkgwinner
|
|
|
+ vvv["topscopeclass"] = tmpclass
|
|
|
+ vvv["pkg_id"] =pkgid+"_"+i
|
|
|
+ if winnerperson, ok := tmp["winnerperson"].(string); ok && winnerperson != "" &&
|
|
|
+ !Reg_xing.MatchString(winnerperson) && Reg_person.MatchString(winnerperson) {
|
|
|
+ vvv["winnerperson"] = winnerperson
|
|
|
+ if winnertel, ok := tmp["winnertel"].(string); ok && !Reg_xing.MatchString(winnertel) && Reg_tel.MatchString(winnertel) {
|
|
|
+ vvv["winnertel"] = winnertel
|
|
|
+ }
|
|
|
+ }
|
|
|
+ FClient.Save(Config["mgo_qyk_c_w_new"],vvv)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
