|
|
@@ -4,7 +4,6 @@ import (
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"github.com/garyburd/redigo/redis"
|
|
|
- "gopkg.in/mgo.v2"
|
|
|
"gopkg.in/mgo.v2/bson"
|
|
|
"log"
|
|
|
"qfw/util"
|
|
|
@@ -99,10 +98,6 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
rdb.Do("SELECT", redis_winner_db)
|
|
|
if reply, err := redis.String(rdb.Do("GET", redisCName)); err != nil {
|
|
|
//redis不存在,存到临时表,定时任务处理
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- //if tmpid := FClient.Save("winner_new", tmps); tmpid == nil {
|
|
|
- // log.Println("存量 FClient.Save err", tmpid)
|
|
|
- //}
|
|
|
for _, vmap := range rValuesMaps {
|
|
|
vmap["_id"] = bson.ObjectIdHex(vmap["_id"].(string))
|
|
|
if err = FClient.SaveForOld("winner_new", vmap); err != nil {
|
|
|
@@ -120,7 +115,6 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
log.Println(err)
|
|
|
}
|
|
|
//拿到合并后的qyk
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, nil)
|
|
|
if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
log.Println(redisCName, "存量 redis id 不存在", reply)
|
|
|
@@ -197,116 +191,117 @@ func TaskWinner(mapinfo *map[string]interface{}) {
|
|
|
(*oldTmp)["contact"] = contactMaps
|
|
|
//mongo更新
|
|
|
(*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
log.Println("存量 mongo更新 err", esId, oldTmp)
|
|
|
}
|
|
|
//es更新
|
|
|
delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("存量 EsConn err :", err)
|
|
|
- }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
log.Println("存量历史合并执行完成 ok", gtid, lteid)
|
|
|
+ //发送udp 更新es段
|
|
|
|
|
|
} else {
|
|
|
- //增量
|
|
|
- overid := addfunc(gtid, cursor)
|
|
|
+ //增量处理
|
|
|
+ overid := gtid
|
|
|
+ tmp := map[string]interface{}{}
|
|
|
+ for cursor.Next(&tmp) {
|
|
|
+ overid = Add(overid, tmp)
|
|
|
+ }
|
|
|
SourceClient.DestoryMongoConn(SourceClientcc)
|
|
|
log.Println("增量合并执行完成 ok", gtid, lteid, overid)
|
|
|
+ //发送udp 更新es段
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
//增量
|
|
|
-func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
- //增量处理
|
|
|
- overid := gtid
|
|
|
- tmp := map[string]interface{}{}
|
|
|
- for cursor.Next(&tmp) {
|
|
|
- overid = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
- //log.Println(tmp["_id"])
|
|
|
- winner, ok := tmp["winner"].(string)
|
|
|
- if !ok || utf8.RuneCountInString(winner) < 4 {
|
|
|
- continue
|
|
|
+func Add(overid string, tmp map[string]interface{}) string {
|
|
|
+ overid = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
+ winner, ok := tmp["winner"].(string)
|
|
|
+ if !ok || utf8.RuneCountInString(winner) < 4 {
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //redis查询是否存在
|
|
|
+ rdb := RedisPool.Get()
|
|
|
+ rdb.Do("SELECT", redis_winner_db)
|
|
|
+ if reply, err := redis.String(rdb.Do("GET", winner)); err != nil {
|
|
|
+ //redis不存在存到临时表,定时任务处理
|
|
|
+ if err := FClient.SaveForOld("winner_new", tmp); err != nil {
|
|
|
+ log.Println("FClient.Save err", err, tmp)
|
|
|
}
|
|
|
- //redis查询是否存在
|
|
|
- rdb := RedisPool.Get()
|
|
|
- rdb.Do("SELECT", redis_winner_db)
|
|
|
- if reply, err := redis.String(rdb.Do("GET", winner)); err != nil {
|
|
|
- //redis不存在存到临时表,定时任务处理
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if err := FClient.SaveForOld("winner_new", tmp); err != nil {
|
|
|
- log.Println("FClient.Save err", err, tmp)
|
|
|
- }
|
|
|
- //log.Println("get redis id err:定时任务处理", err, tmp)
|
|
|
- if err := rdb.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
- continue
|
|
|
- } else {
|
|
|
- if err := rdb.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
- //拿到合并后的qyk
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, bson.M{})
|
|
|
- if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
- log.Println("redis id 不存在")
|
|
|
- continue
|
|
|
- }
|
|
|
- //比较合并 行业类型
|
|
|
- tmpTopscopeclass := []string{}
|
|
|
- tmpConTopscopeclass := []string{}
|
|
|
- tmpTopscopeclassMap := make(map[string]bool)
|
|
|
+ //log.Println("get redis id err:定时任务处理", err, tmp)
|
|
|
+ if err := rdb.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ return overid
|
|
|
+ } else {
|
|
|
+ if err := rdb.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
+ //拿到合并后的qyk
|
|
|
+ oldTmp, b := FClient.FindById(Config["mgo_qyk_c"], reply, bson.M{})
|
|
|
+ if !b || (*oldTmp) == nil || reply == "" || (*oldTmp)["_id"] == nil {
|
|
|
+ log.Println("redis id 不存在", reply)
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //比较合并 行业类型
|
|
|
+ tmpTopscopeclass := []string{}
|
|
|
+ tmpConTopscopeclass := []string{}
|
|
|
+ tmpTopscopeclassMap := make(map[string]bool)
|
|
|
|
|
|
- if (*oldTmp)["industry"] != nil {
|
|
|
- if v, ok := (*oldTmp)["industry"].([]interface{}); ok {
|
|
|
- for _, vv := range v {
|
|
|
- if vvv, ok := vv.(string); ok {
|
|
|
- tmpTopscopeclassMap[vvv] = true
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if v, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ if (*oldTmp)["industry"] != nil {
|
|
|
+ if v, ok := (*oldTmp)["industry"].([]interface{}); ok {
|
|
|
for _, vv := range v {
|
|
|
- if vvv, ok := vv.(string); ok && len(vvv) > 1 {
|
|
|
- tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
|
|
|
- tmpConTopscopeclass = append(tmpConTopscopeclass, vvv[:len(vvv)-1])
|
|
|
+ if vvv, ok := vv.(string); ok {
|
|
|
+ tmpTopscopeclassMap[vvv] = true
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- for k := range tmpTopscopeclassMap {
|
|
|
- tmpTopscopeclass = append(tmpTopscopeclass, k)
|
|
|
+ }
|
|
|
+ if v, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ for _, vv := range v {
|
|
|
+ if vvv, ok := vv.(string); ok && len(vvv) > 1 {
|
|
|
+ tmpTopscopeclassMap[vvv[:len(vvv)-1]] = true
|
|
|
+ tmpConTopscopeclass = append(tmpConTopscopeclass, vvv[:len(vvv)-1])
|
|
|
+ }
|
|
|
}
|
|
|
- sort.Strings(tmpTopscopeclass)
|
|
|
- (*oldTmp)["industry"] = tmpTopscopeclass
|
|
|
+ }
|
|
|
+ for k := range tmpTopscopeclassMap {
|
|
|
+ tmpTopscopeclass = append(tmpTopscopeclass, k)
|
|
|
+ }
|
|
|
+ sort.Strings(tmpTopscopeclass)
|
|
|
+ (*oldTmp)["industry"] = tmpTopscopeclass
|
|
|
|
|
|
- esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
|
|
|
- //更新行业类型
|
|
|
- if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
|
|
|
- (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- //mongo更新
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
- log.Println("mongo更新err", esId)
|
|
|
- }
|
|
|
+ esId := (*oldTmp)["_id"].(bson.ObjectId).Hex()
|
|
|
+ //更新行业类型
|
|
|
+ if tmp["winnerperson"] == nil || tmp["winnerperson"] == "" || Reg_xing.MatchString(util.ObjToString(tmp["winnerperson"])) {
|
|
|
+ (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
+ //mongo更新
|
|
|
+ if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
+ log.Println("mongo更新err", esId)
|
|
|
+ }
|
|
|
|
|
|
- //es更新
|
|
|
- delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("update es err:", err)
|
|
|
+ //es更新
|
|
|
+ delete((*oldTmp), "_id")
|
|
|
+ return overid
|
|
|
+ }
|
|
|
+ //联系方式合并
|
|
|
+ contactMaps := make([]map[string]interface{}, 0)
|
|
|
+ if (*oldTmp)["contact"] != nil {
|
|
|
+ //直接添加联系人，不再判断
|
|
|
+ if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
|
|
|
+ for _, vv := range v {
|
|
|
+ contactMaps = append(contactMaps, vv.(map[string]interface{}))
|
|
|
}
|
|
|
- continue
|
|
|
- }
|
|
|
- //联系方式合并
|
|
|
- var tmpperson, winnertel string
|
|
|
- if tmppersona, ok := tmp["winnerperson"].(string); ok {
|
|
|
- tmpperson = tmppersona
|
|
|
}
|
|
|
+ }
|
|
|
+ var tmpperson, winnertel string
|
|
|
+ if tmppersona, ok := tmp["winnerperson"].(string); ok && tmpperson != "" && Reg_person.MatchString(tmpperson) && !Reg_xing.MatchString(tmpperson) {
|
|
|
+ tmpperson = tmppersona
|
|
|
+ }
|
|
|
+ if tmpperson != "" {
|
|
|
if winnerteltmp, ok := tmp["winnertel"].(string); ok {
|
|
|
winnertel = winnerteltmp
|
|
|
}
|
|
|
@@ -315,13 +310,7 @@ func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
} else {
|
|
|
winnertel = winnertel
|
|
|
}
|
|
|
- contactMaps := make([]interface{}, 0)
|
|
|
- if (*oldTmp)["contact"] != nil {
|
|
|
- //直接添加联系人，不再判断
|
|
|
- if v, ok := (*oldTmp)["contact"].([]interface{}); ok {
|
|
|
- contactMaps = append(contactMaps, v...)
|
|
|
- }
|
|
|
- }
|
|
|
+
|
|
|
vvv := make(map[string]interface{})
|
|
|
vvv["infoid"] = overid
|
|
|
vvv["contact_person"] = tmpperson
|
|
|
@@ -330,23 +319,22 @@ func addfunc(gtid string, cursor *mgo.Iter) string {
|
|
|
vvv["topscopeclass"] = strings.Join(tmpConTopscopeclass, ";")
|
|
|
vvv["updatetime"] = time.Now().Unix()
|
|
|
contactMaps = append(contactMaps, vvv)
|
|
|
- //分包处理
|
|
|
- PackageDealWith(&contactMaps, tmp)
|
|
|
- (*oldTmp)["contact"] = contactMaps
|
|
|
- //mongo更新
|
|
|
- (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
- log.Println("mongo更新 err", esId, oldTmp)
|
|
|
- }
|
|
|
- //es更新
|
|
|
- delete((*oldTmp), "_id")
|
|
|
- if _, err := EsConn.Update().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(esId).Doc(oldTmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("EsConn err :", err)
|
|
|
- }
|
|
|
}
|
|
|
+ //分包处理
|
|
|
+ if tmp["package"] != nil {
|
|
|
+ PackageDealWith(oldTmp, tmp, winner)
|
|
|
+ }
|
|
|
+ (*oldTmp)["contact"] = contactMaps
|
|
|
+ //mongo更新
|
|
|
+ (*oldTmp)["updatatime"] = time.Now().Unix()
|
|
|
+ if !FClient.UpdateById(Config["mgo_qyk_c"], esId, bson.M{"$set": oldTmp}) {
|
|
|
+ log.Println("mongo更新 err", esId, oldTmp)
|
|
|
+ }
|
|
|
+ //es更新
|
|
|
+ delete((*oldTmp), "_id")
|
|
|
}
|
|
|
return overid
|
|
|
+
|
|
|
}
|
|
|
|
|
|
//定时任务 新增
|
|
|
@@ -362,11 +350,11 @@ func TimedTaskWinner() {
|
|
|
if !iter.Next(&tmpLast) {
|
|
|
//临时表无数据
|
|
|
log.Println("临时表无数据:")
|
|
|
- t2.Reset(time.Minute * 5)
|
|
|
+ t2.Reset(time.Minute * 1)
|
|
|
FClient.DestoryMongoConn(Fcconn)
|
|
|
continue
|
|
|
} else {
|
|
|
- log.Println("临时表有数据:", tmpLast)
|
|
|
+ log.Println("临时表有数据:", tmpLast["_id"])
|
|
|
fconn := FClient.GetMgoConn(86400)
|
|
|
cursor := fconn.DB(Config["mgodb_extract_kf"]).C("winner_new").Find(bson.M{
|
|
|
"_id": bson.M{
|
|
|
@@ -391,13 +379,9 @@ func TimedTaskWinner() {
|
|
|
rdb := RedisPool.Get()
|
|
|
rdb.Do("SELECT", redis_winner_db)
|
|
|
if _, err := redis.String(rdb.Do("GET", errwinner)); err == nil {
|
|
|
- //redis存在发送udp进行处理
|
|
|
- TaskWinner(&map[string]interface{}{
|
|
|
- "gtid": tmpId,
|
|
|
- "lteid": tmpId,
|
|
|
- })
|
|
|
+ //增量合并
|
|
|
+ Add(tmpId, tmp)
|
|
|
//存在的话删除tmp mongo表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
if DeletedCount := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !DeletedCount {
|
|
|
log.Println("删除临时表err:", DeletedCount)
|
|
|
}
|
|
|
@@ -411,10 +395,11 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
}
|
|
|
//查询redis不存在新增
|
|
|
- FClient.DbName = Config["mgodb_enterprise"]
|
|
|
-
|
|
|
- resulttmp, b := FClient.FindOne(Config["mgodb_enterprise_c"], bson.M{"company_name": errwinner})
|
|
|
- if !b || (*resulttmp)["_id"] == nil {
|
|
|
+ sessionfinone := FClient.GetMgoConn()
|
|
|
+ resulttmp := make(map[string]interface{})
|
|
|
+ err := sessionfinone.DB(Config["mgodb_enterprise"]).C(Config["mgodb_enterprise_c"]).Find(bson.M{"company_name": errwinner}).One(&resulttmp)
|
|
|
+ FClient.DestoryMongoConn(sessionfinone)
|
|
|
+ if err != nil || resulttmp["_id"] == nil {
|
|
|
//log.Println(r)
|
|
|
//人工审核正则
|
|
|
var isok bool
|
|
|
@@ -439,122 +424,126 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
}
|
|
|
//都没匹配
|
|
|
- if tmp["winner_ok"] == nil && tmp["winner_err"] == nil{
|
|
|
+ if tmp["winner_ok"] == nil && tmp["winner_err"] == nil {
|
|
|
tmp["winner_err"] = 1
|
|
|
}
|
|
|
//匹配不到原始库,存入异常表删除临时表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
if err := FClient.SaveForOld("winner_err", tmp); err != nil {
|
|
|
log.Println("存入异常表错误", err, tmp)
|
|
|
}
|
|
|
- if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !b {
|
|
|
+ if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
log.Println("删除临时表错误", deleteNum)
|
|
|
}
|
|
|
continue
|
|
|
} else {
|
|
|
//log.Println(123)
|
|
|
//匹配到原始库,新增 resulttmp
|
|
|
- if (*resulttmp)["credit_no"] != nil {
|
|
|
- if credit_no, ok := (*resulttmp)["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
|
|
|
+ if resulttmp["credit_no"] != nil {
|
|
|
+ if credit_no, ok := resulttmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" &&
|
|
|
len(strings.TrimSpace(credit_no)) > 8 {
|
|
|
dataNo := strings.TrimSpace(credit_no)[2:8]
|
|
|
if Addrs[dataNo] != nil {
|
|
|
if v, ok := Addrs[dataNo].(map[string]interface{}); ok {
|
|
|
- if (*resulttmp)["province"] == nil || (*resulttmp)["province"] == "" {
|
|
|
- (*resulttmp)["province"] = v["province"]
|
|
|
+ if resulttmp["province"] == nil || resulttmp["province"] == "" {
|
|
|
+ resulttmp["province"] = v["province"]
|
|
|
}
|
|
|
- (*resulttmp)["city"] = v["city"]
|
|
|
- (*resulttmp)["district"] = v["district"]
|
|
|
+ resulttmp["city"] = v["city"]
|
|
|
+ resulttmp["district"] = v["district"]
|
|
|
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- contacts := make([]map[string]interface{}, 0)
|
|
|
- contact := make(map[string]interface{}, 0)
|
|
|
- if (*resulttmp)["legal_person"] != nil {
|
|
|
- contact["contact_person"] = (*resulttmp)["legal_person"] //联系人
|
|
|
- } else {
|
|
|
- contact["contact_person"] = "" //联系人
|
|
|
- }
|
|
|
- contact["contact_type"] = "法定代表人" //法定代表人
|
|
|
- //log.Println(1)
|
|
|
- if (*resulttmp)["annual_reports"] != nil {
|
|
|
- bytes, err := json.Marshal((*resulttmp)["annual_reports"])
|
|
|
- if err != nil {
|
|
|
- log.Println("annual_reports err:", err)
|
|
|
- }
|
|
|
- phonetmp := make([]map[string]interface{}, 0)
|
|
|
- err = json.Unmarshal(bytes, &phonetmp)
|
|
|
- if err != nil {
|
|
|
- log.Println("Unmarshal err:", err)
|
|
|
+ //行业类型
|
|
|
+ tmpclass := make([]string, 0)
|
|
|
+ if tclasss, ok := tmp["topscopeclass"].([]interface{}); ok {
|
|
|
+ for _, vv := range tclasss {
|
|
|
+ if vvv, ok := vv.(string); ok {
|
|
|
+ if len(vvv) > 1 {
|
|
|
+ tmpclass = append(tmpclass, vvv[:len(vvv)-1])
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
- for _, vv := range phonetmp {
|
|
|
- if vv["company_phone"] != nil {
|
|
|
- if vv["company_phone"] == "" {
|
|
|
- continue
|
|
|
+ }
|
|
|
+ contacts := make([]map[string]interface{}, 0)
|
|
|
+ if legal_person, ok := resulttmp["legal_person"].(string); ok && legal_person != "" && !Reg_xing.MatchString(legal_person) && Reg_person.MatchString(legal_person) {
|
|
|
+ contact := make(map[string]interface{}, 0)
|
|
|
+ contact["contact_person"] = legal_person //联系人
|
|
|
+ contact["contact_type"] = "法定代表人" //法定代表人
|
|
|
+ if resulttmp["annual_reports"] != nil {
|
|
|
+ bytes, err := json.Marshal(resulttmp["annual_reports"])
|
|
|
+ if err != nil {
|
|
|
+ log.Println("annual_reports err:", err)
|
|
|
+ }
|
|
|
+ phonetmp := make([]map[string]interface{}, 0)
|
|
|
+ err = json.Unmarshal(bytes, &phonetmp)
|
|
|
+ if err != nil {
|
|
|
+ log.Println("Unmarshal err:", err)
|
|
|
+ }
|
|
|
+ for _, vv := range phonetmp {
|
|
|
+ if vv["company_phone"] != nil {
|
|
|
+ if vv["company_phone"] == "" {
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ contact["phone"] = vv["company_phone"] //联系电话
|
|
|
+ break
|
|
|
+ }
|
|
|
} else {
|
|
|
- contact["phone"] = vv["company_phone"] //联系电话
|
|
|
- break
|
|
|
+ contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
- } else {
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //log.Println(k, contact["phone"], resulttmp["_id"])
|
|
|
+ //time.Sleep(10 * time.Second)
|
|
|
+ if phone, ok := contact["phone"].(string); ok && phone != "" {
|
|
|
+ if Reg_xing.MatchString(phone) || !Reg_tel.MatchString(phone) {
|
|
|
contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
-
|
|
|
+ } else {
|
|
|
+ contact["phone"] = "" //联系电话
|
|
|
}
|
|
|
+ contact["topscopeclass"] = "企业公示" //项目类型
|
|
|
+ contact["updatetime"] = time.Now().Unix() //更新时间
|
|
|
+ contact["infoid"] = "" //招标信息id
|
|
|
+ contacts = append(contacts, contact)
|
|
|
}
|
|
|
- //log.Println(k, contact["phone"], resulttmp["_id"])
|
|
|
- //time.Sleep(10 * time.Second)
|
|
|
- if contact["phone"] == nil {
|
|
|
- contact["phone"] = "" //联系电话
|
|
|
- }
|
|
|
- contact["topscopeclass"] = "企业公示" //项目类型
|
|
|
- contact["updatetime"] = time.Now().Unix() //更新时间
|
|
|
- contact["infoid"] = "" //招标信息id
|
|
|
- contacts = append(contacts, contact)
|
|
|
//添加临时表匹配到的联系人
|
|
|
- vvv := make(map[string]interface{})
|
|
|
- vvv["infoid"] = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
- if tmp["winnerperson"] != nil {
|
|
|
- vvv["contact_person"] = tmp["winnerperson"]
|
|
|
- } else {
|
|
|
- vvv["contact_person"] = ""
|
|
|
- }
|
|
|
- vvv["contact_type"] = "项目联系人"
|
|
|
- // "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1
|
|
|
- if tmp["winnertel"] != nil {
|
|
|
- vvv["phone"] = tmp["winnertel"]
|
|
|
- } else {
|
|
|
- vvv["phone"] = ""
|
|
|
- }
|
|
|
- tmpclass := make([]string, 0)
|
|
|
- if tclasss, ok := tmp["topscopeclass"].([]string); ok {
|
|
|
- for _, vv := range tclasss {
|
|
|
- if len(vv) > 1 {
|
|
|
- tmpclass = append(tmpclass, vv[:len(vv)-1])
|
|
|
- }
|
|
|
+ if winnerperson, ok := tmp["winnerperson"].(string); ok && winnerperson != "" &&
|
|
|
+ !Reg_xing.MatchString(winnerperson) && Reg_person.MatchString(winnerperson) {
|
|
|
+ vvv := make(map[string]interface{})
|
|
|
+ vvv["infoid"] = tmp["_id"].(bson.ObjectId).Hex()
|
|
|
+ vvv["contact_person"] = winnerperson
|
|
|
+ vvv["contact_type"] = "项目联系人"
|
|
|
+ // "winner": 1, "winnertel": 1, "winnerperson": 1, "topscopeclass": 1
|
|
|
+ if winnertel, ok := tmp["winnertel"].(string); ok && !Reg_xing.MatchString(winnertel) && Reg_tel.MatchString(winnertel) {
|
|
|
+ vvv["phone"] = winnertel
|
|
|
+ } else {
|
|
|
+ vvv["phone"] = ""
|
|
|
}
|
|
|
+
|
|
|
+ vvv["topscopeclass"] = strings.Join(tmpclass, ";")
|
|
|
+ vvv["updatetime"] = time.Now().Unix()
|
|
|
+ contacts = append(contacts, vvv)
|
|
|
}
|
|
|
- vvv["topscopeclass"] = strings.Join(tmpclass, ";")
|
|
|
- vvv["updatetime"] = time.Now().Unix()
|
|
|
- contacts = append(contacts, vvv)
|
|
|
- (*resulttmp)["contact"] = contacts
|
|
|
+
|
|
|
+ resulttmp["contact"] = contacts
|
|
|
|
|
|
savetmp := make(map[string]interface{}, 0)
|
|
|
for _, sk := range Fields {
|
|
|
if sk == "establish_date" {
|
|
|
- if (*resulttmp)[sk] != nil {
|
|
|
- savetmp[sk] = (*resulttmp)[sk].(time.Time).UTC().Unix()
|
|
|
+ if resulttmp[sk] != nil {
|
|
|
+ savetmp[sk] = resulttmp[sk].(time.Time).UTC().Unix()
|
|
|
continue
|
|
|
}
|
|
|
} else if sk == "capital" {
|
|
|
//log.Println(sk, resulttmp[sk])
|
|
|
- savetmp[sk] = ObjToMoney([]interface{}{(*resulttmp)[sk], ""})[0]
|
|
|
+ savetmp[sk] = ObjToMoney([]interface{}{resulttmp[sk], ""})[0]
|
|
|
continue
|
|
|
} else if sk == "partners" {
|
|
|
//log.Println(sk, resulttmp[sk], )
|
|
|
- if (*resulttmp)[sk] != nil {
|
|
|
- if ppms, ok := (*resulttmp)[sk].([]interface{}); ok {
|
|
|
+ if resulttmp[sk] != nil {
|
|
|
+ if ppms, ok := resulttmp[sk].([]interface{}); ok {
|
|
|
for i, _ := range ppms {
|
|
|
if ppms[i].(map[string]interface{})["stock_type"] != nil {
|
|
|
ppms[i].(map[string]interface{})["stock_type"] = "企业公示"
|
|
|
@@ -569,19 +558,19 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
continue
|
|
|
} else if sk == "_id" {
|
|
|
- savetmp["tmp"+sk] = (*resulttmp)[sk]
|
|
|
+ savetmp["tmp"+sk] = resulttmp[sk]
|
|
|
continue
|
|
|
} else if sk == "area_code" {
|
|
|
//行政区划代码
|
|
|
- savetmp[sk] = fmt.Sprint((*resulttmp)[sk])
|
|
|
+ savetmp[sk] = fmt.Sprint(resulttmp[sk])
|
|
|
continue
|
|
|
} else if sk == "report_websites" {
|
|
|
//网址
|
|
|
- if (*resulttmp)["report_websites"] == nil {
|
|
|
+ if resulttmp["report_websites"] == nil {
|
|
|
savetmp["website"] = ""
|
|
|
} else {
|
|
|
report_websitesArr := []string{}
|
|
|
- if ppms, ok := (*resulttmp)[sk].([]interface{}); ok {
|
|
|
+ if ppms, ok := resulttmp[sk].([]interface{}); ok {
|
|
|
for _, v := range ppms {
|
|
|
if vvv, ok := v.(map[string]interface{}); ok {
|
|
|
if rv, ok := vvv["website_url"].(string); ok {
|
|
|
@@ -610,50 +599,36 @@ func TimedTaskWinner() {
|
|
|
savetmp[sk] = tmpTopscopeclass
|
|
|
continue
|
|
|
}
|
|
|
- if (*resulttmp)[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
|
|
|
+ if resulttmp[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" {
|
|
|
savetmp[sk] = ""
|
|
|
} else {
|
|
|
- savetmp[sk] = (*resulttmp)[sk]
|
|
|
+ savetmp[sk] = resulttmp[sk]
|
|
|
}
|
|
|
}
|
|
|
+ //判断分包
|
|
|
+ if tmp["package"] != nil {
|
|
|
+ PackageDealWith(&savetmp, tmp, errwinner)
|
|
|
+ }
|
|
|
//tmps = append(tmps, savetmp)
|
|
|
+ savetmp["comeintime"] = time.Now().Unix()
|
|
|
savetmp["updatatime"] = time.Now().Unix()
|
|
|
//保存mongo
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
-
|
|
|
saveid := FClient.Save(Config["mgo_qyk_c"], savetmp)
|
|
|
if saveid != "" {
|
|
|
//保存redis
|
|
|
rc := RedisPool.Get()
|
|
|
rc.Do("SELECT", redis_winner_db)
|
|
|
- //var _id string
|
|
|
- //if v, ok := saveid.(primitive.ObjectID); ok {
|
|
|
- // _id = v.Hex()
|
|
|
- //}
|
|
|
if _, err := rc.Do("SET", savetmp["company_name"], saveid); err != nil {
|
|
|
log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["company_name"], err)
|
|
|
- if err := rc.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
} else {
|
|
|
- //保存es
|
|
|
- delete(savetmp, "_id")
|
|
|
- if err := rc.Close(); err != nil {
|
|
|
- log.Println(err)
|
|
|
- }
|
|
|
-
|
|
|
- //esConn := elastic.GetEsConn()
|
|
|
- //defer elastic.DestoryEsConn(esConn)
|
|
|
- if _, err := EsConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(saveid).BodyJson(savetmp).Refresh(true).Do(); err != nil {
|
|
|
- log.Println("save es err :", tmp["_id"], savetmp["_id"], err)
|
|
|
- } else {
|
|
|
- //删除临时表
|
|
|
- FClient.DbName = Config["mgodb_extract_kf"]
|
|
|
- if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
- log.Println("删除临时表失败", deleteNum)
|
|
|
- }
|
|
|
+ //删除临时表
|
|
|
+ if deleteNum := FClient.Del("winner_new", bson.M{"_id": bson.ObjectIdHex(tmpId)}); !deleteNum {
|
|
|
+ log.Println("删除临时表失败", deleteNum)
|
|
|
}
|
|
|
}
|
|
|
+ if err := rc.Close(); err != nil {
|
|
|
+ log.Println(err)
|
|
|
+ }
|
|
|
} else {
|
|
|
log.Println("save mongo err:", saveid, tmp["_id"])
|
|
|
}
|
|
|
@@ -669,6 +644,12 @@ func TimedTaskWinner() {
|
|
|
}
|
|
|
|
|
|
//分包处理
|
|
|
-func PackageDealWith(contactMaps *[]interface{}, tmp map[string]interface{}) {
|
|
|
-
|
|
|
+func PackageDealWith(contactMap *map[string]interface{}, tmp map[string]interface{}, comName string) []interface{} {
|
|
|
+ util.Catch()
|
|
|
+ //if v, ok := tmp["package"].(map[string]interface{}); ok {
|
|
|
+ //for i, pv := range v {
|
|
|
+ // log.Println(i, pv)
|
|
|
+ //}
|
|
|
+ //}
|
|
|
+ return nil
|
|
|
}
|
