|
@@ -2,15 +2,11 @@ package subject_repair
|
|
|
|
|
|
import (
|
|
|
log "github.com/donnie4w/go-logger/logger"
|
|
|
- "github.com/uuid"
|
|
|
qu "qfw/util"
|
|
|
- "strings"
|
|
|
"subject"
|
|
|
su "subject_util"
|
|
|
"sync"
|
|
|
- "time"
|
|
|
"unicode/utf8"
|
|
|
- ul "util"
|
|
|
)
|
|
|
|
|
|
var (
|
|
@@ -18,15 +14,22 @@ var (
|
|
|
numLock sync.Mutex
|
|
|
)
|
|
|
|
|
|
-var RepairFields = map[string]interface{}{
|
|
|
+var RepairContactFields = map[string]interface{}{
|
|
|
+ "extracttype": 1, "buyerclass": 1, "publishtime": 1,
|
|
|
+ "buyer": 1, "agency": 1, "winner": 1, "s_winner": 1,
|
|
|
+ "agencytel": 1, "winnertel": 1, "buyertel": 1,
|
|
|
+ "agencyperson": 1, "winnerperson": 1, "buyerperson": 1,
|
|
|
+}
|
|
|
+
|
|
|
+//主体owner~相关
|
|
|
+var RepairOwnerFields = map[string]interface{}{
|
|
|
"extracttype": 1, "publishtime": 1,
|
|
|
- "winner": 1, "s_winner": 1,
|
|
|
- "winnertel": 1, "winnerperson": 1,
|
|
|
+ "owner": 1, "project_person": 1, "project_phone": 1,
|
|
|
}
|
|
|
|
|
|
-//全量数据准备
|
|
|
-func RunRepairFullDataInfo(lteid string) {
|
|
|
- log.Debug("开始修复全量主体~~~")
|
|
|
+//修复全量业主单位信息
|
|
|
+func RunRepairFullOwnerContactInfo(lteid string) {
|
|
|
+ log.Debug("开始修复~业主单位~联系人数据~")
|
|
|
sess := su.SourceMgo.GetMgoConn()
|
|
|
defer su.SourceMgo.DestoryMongoConn(sess)
|
|
|
q := map[string]interface{}{
|
|
@@ -35,13 +38,13 @@ func RunRepairFullDataInfo(lteid string) {
|
|
|
},
|
|
|
}
|
|
|
log.Debug("查询语句 ~ ", q)
|
|
|
- it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(RepairFields).Iter()
|
|
|
- pool := make(chan bool, 16)
|
|
|
+ it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(RepairOwnerFields).Iter()
|
|
|
+ pool := make(chan bool, 6)
|
|
|
wg := &sync.WaitGroup{}
|
|
|
total, isok := 0, 0
|
|
|
for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
|
|
|
- if total%10000 == 0 {
|
|
|
- log.Debug("cur index ", total)
|
|
|
+ if total%100000 == 0 {
|
|
|
+ log.Debug("cur index ", total, "~", isok)
|
|
|
}
|
|
|
pool <- true
|
|
|
wg.Add(1)
|
|
@@ -53,10 +56,66 @@ func RunRepairFullDataInfo(lteid string) {
|
|
|
if qu.IntAll(tmp["extracttype"]) != 1 {
|
|
|
return
|
|
|
}
|
|
|
- if qu.ObjToString(tmp["winner"]) == "" && qu.ObjToString(tmp["s_winner"]) == "" {
|
|
|
+ if qu.ObjToString(tmp["owner"]) == "" {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ createOwnerContactBaseInfo(tmp) //处理数据
|
|
|
+ }(tmp)
|
|
|
+ tmp = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ log.Debug("is over ~ ", total, "~", isok)
|
|
|
+}
|
|
|
+
|
|
|
+func createOwnerContactBaseInfo(tmp map[string]interface{}) {
|
|
|
+ owner := qu.ObjToString(tmp["owner"])
|
|
|
+ o_per := qu.ObjToString(tmp["project_person"])
|
|
|
+ o_tel := qu.ObjToString(tmp["project_phone"])
|
|
|
+ if utf8.RuneCountInString(o_tel) > 60 {
|
|
|
+ o_tel = ""
|
|
|
+ }
|
|
|
+ publishtime := qu.Int64All(tmp["publishtime"])
|
|
|
+ tmpid := su.BsonTOStringId(tmp["_id"])
|
|
|
+ dataLock.Lock()
|
|
|
+ if owner != "" && utf8.RuneCountInString(owner) < 30 {
|
|
|
+ contact := &subject.Contact{o_per, o_tel, false, false, false, true, publishtime}
|
|
|
+ dealWithRepairContactModel(owner, "", contact, "1000", tmpid)
|
|
|
+ }
|
|
|
+ dataLock.Unlock()
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+
|
|
|
+//全量数据修复通讯录~支持空联系人
|
|
|
+func RunRepairFullEmptyPersonContactInfo(lteid string) {
|
|
|
+ log.Debug("开始修复全量空联系人数据~~~")
|
|
|
+ sess := su.SourceMgo.GetMgoConn()
|
|
|
+ defer su.SourceMgo.DestoryMongoConn(sess)
|
|
|
+ q := map[string]interface{}{
|
|
|
+ "_id": map[string]interface{}{
|
|
|
+ "$lte": su.StringTOBsonId(lteid),
|
|
|
+ },
|
|
|
+ }
|
|
|
+ log.Debug("查询语句 ~ ", q)
|
|
|
+ it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(RepairContactFields).Iter()
|
|
|
+ pool := make(chan bool, 4)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+ total, isok := 0, 0
|
|
|
+ for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
|
|
|
+ if total%10000 == 0 {
|
|
|
+ log.Debug("cur index ", total, "~", isok)
|
|
|
+ }
|
|
|
+ pool <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-pool
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ if qu.IntAll(tmp["extracttype"]) != 1 {
|
|
|
return
|
|
|
}
|
|
|
- b := createRepairBaseInfo(tmp) //处理数据
|
|
|
+ b := createRepairContactBaseInfo(tmp) //处理数据
|
|
|
if b {
|
|
|
numLock.Lock()
|
|
|
isok++
|
|
@@ -69,162 +128,62 @@ func RunRepairFullDataInfo(lteid string) {
|
|
|
log.Debug("is over ~ ", total, "~", isok)
|
|
|
|
|
|
}
|
|
|
-
|
|
|
-func createRepairBaseInfo(tmp map[string]interface{}) bool {
|
|
|
+func createRepairContactBaseInfo(tmp map[string]interface{}) bool {
|
|
|
+ empty := false
|
|
|
+ buyer := qu.ObjToString(tmp["buyer"])
|
|
|
+ agency := qu.ObjToString(tmp["agency"])
|
|
|
winner := qu.ObjToString(tmp["winner"])
|
|
|
s_winner := qu.ObjToString(tmp["s_winner"])
|
|
|
+ b_per := qu.ObjToString(tmp["buyerperson"])
|
|
|
+ b_tel := qu.ObjToString(tmp["buyertel"])
|
|
|
+ if utf8.RuneCountInString(b_tel) > 60 {
|
|
|
+ b_tel = ""
|
|
|
+ }
|
|
|
+ a_per := qu.ObjToString(tmp["agencyperson"])
|
|
|
+ a_tel := qu.ObjToString(tmp["agencytel"])
|
|
|
+ if utf8.RuneCountInString(a_tel) > 60 {
|
|
|
+ a_tel = ""
|
|
|
+ }
|
|
|
w_per := qu.ObjToString(tmp["winnerperson"])
|
|
|
w_tel := qu.ObjToString(tmp["winnertel"])
|
|
|
-
|
|
|
if utf8.RuneCountInString(w_tel) > 60 {
|
|
|
w_tel = ""
|
|
|
}
|
|
|
+ buyerclass := qu.ObjToString(tmp["buyerclass"])
|
|
|
publishtime := qu.Int64All(tmp["publishtime"])
|
|
|
tmpid := su.BsonTOStringId(tmp["_id"])
|
|
|
winner_arr, winner_bool := subject.SegmentationEntName(winner, s_winner)
|
|
|
- if len(winner_arr) <= 1 {
|
|
|
- return false
|
|
|
+
|
|
|
+ dataLock.Lock()
|
|
|
+ if buyer != "" && utf8.RuneCountInString(buyer) < 30 {
|
|
|
+ if b_per == "" && b_tel != "" {
|
|
|
+ empty = true
|
|
|
+ dealWithRepairContactModel(buyer, buyerclass, &subject.Contact{b_per, b_tel, true, false, false, false, publishtime}, "001", tmpid)
|
|
|
+ }
|
|
|
}
|
|
|
- dataLock.Lock() //处理
|
|
|
+ //中标单位
|
|
|
for k, v := range winner_arr {
|
|
|
b := winner_bool[k]
|
|
|
- c := subject.Contact{"", "", false, false, true, publishtime}
|
|
|
+ c := subject.Contact{"", "", false, false, true, false, publishtime}
|
|
|
if b {
|
|
|
c.Per = w_per
|
|
|
c.Tel = w_tel
|
|
|
}
|
|
|
- dealWithRepairSubjectContact(v, "", &c, "010", tmpid)
|
|
|
+ if c.Per == "" && c.Tel != "" {
|
|
|
+ empty = true
|
|
|
+ dealWithRepairContactModel(v, "", &c, "010", tmpid)
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
+ if agency != "" && utf8.RuneCountInString(agency) < 30 {
|
|
|
+ if a_per == "" && a_tel != "" {
|
|
|
+ empty = true
|
|
|
+ dealWithRepairContactModel(agency, "", &subject.Contact{a_per, a_tel, false, true, false, false, publishtime}, "100", tmpid)
|
|
|
+ }
|
|
|
+ }
|
|
|
dataLock.Unlock()
|
|
|
|
|
|
- return true
|
|
|
+ return empty
|
|
|
}
|
|
|
|
|
|
-func dealWithRepairSubjectContact(name string, buyerclass string, contact *subject.Contact, identity string, tmpid string) {
|
|
|
- info := su.MysqlGlobalTool.FindOne(su.G_Units_Baseinfo, map[string]interface{}{"name": name}, "", "-id")
|
|
|
- if info == nil { //新增主体信息~企业补充~年报补充
|
|
|
- base_info := map[string]interface{}{}
|
|
|
- base_info["name"] = name
|
|
|
- qyxy_info := subject.CreateQyxyInfo(name)
|
|
|
- //通讯录数据结构
|
|
|
- contact_arr := []map[string]interface{}{}
|
|
|
- area, city, district := "", "", ""
|
|
|
- name_id := uuid.New().String()
|
|
|
- name_id = strings.ReplaceAll(name_id, "-", "")
|
|
|
- base_info["name_id"] = name_id
|
|
|
- base_info["identity_type"] = 2
|
|
|
- base_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
|
|
|
- //创建通讯录信息
|
|
|
- if contact.Per != "" && contact.Tel != "" {
|
|
|
- date := ""
|
|
|
- if contact.Publishtime > 0 {
|
|
|
- date = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
|
|
|
- }
|
|
|
- contact_arr = append(contact_arr, map[string]interface{}{
|
|
|
- "person": contact.Per,
|
|
|
- "tel": contact.Tel,
|
|
|
- "date": date,
|
|
|
- "email": "",
|
|
|
- "type": 1,
|
|
|
- "identity": base_info["identity_type"],
|
|
|
- })
|
|
|
- }
|
|
|
- if len(qyxy_info) > 0 { //含企业信息
|
|
|
- info_id := qu.ObjToString(qyxy_info["_id"])
|
|
|
- base_info["company_id"] = info_id
|
|
|
- address := qu.ObjToString(qyxy_info["company_address"])
|
|
|
- if utf8.RuneCountInString(address) > 300 {
|
|
|
- address = ""
|
|
|
- }
|
|
|
- base_info["address"] = address
|
|
|
- area = qu.ObjToString(qyxy_info["company_area"])
|
|
|
- city = qu.ObjToString(qyxy_info["company_city"])
|
|
|
- district = qu.ObjToString(qyxy_info["company_district"])
|
|
|
- legal_person := qu.ObjToString(qyxy_info["legal_person"])
|
|
|
- if legal_person != "" { //年报信息
|
|
|
- subject.CreateAnnualInfo(info_id, legal_person, &contact_arr)
|
|
|
- }
|
|
|
- }
|
|
|
- base_info["area_code"], base_info["city_code"], base_info["district_code"] = subject.CalculateRegionCode(area, city, district)
|
|
|
- //新增主体信息表
|
|
|
- b := su.InsertMysqlData(su.G_Units_Baseinfo, base_info, tmpid)
|
|
|
- if b > 0 { //存通讯录
|
|
|
- for _, v := range contact_arr {
|
|
|
- info_book := map[string]interface{}{}
|
|
|
- info_book["name_id"] = name_id
|
|
|
- info_book["contact_name"] = qu.ObjToString(v["person"])
|
|
|
- info_book["contact_tel"] = qu.ObjToString(v["tel"])
|
|
|
- info_book["contact_email"] = qu.ObjToString(v["email"])
|
|
|
- date := qu.ObjToString(v["date"])
|
|
|
- if date != "" {
|
|
|
- info_book["publishtime"] = date
|
|
|
- }
|
|
|
- info_book["source_type"] = qu.IntAll(v["type"])
|
|
|
- info_book["identity_type"] = qu.IntAll(v["identity"])
|
|
|
- info_book["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
|
|
|
- su.InsertMysqlData(su.G_Units_Contact, info_book, tmpid)
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
- }
|
|
|
- //更新~主体基本信息表
|
|
|
- name_id := qu.ObjToString((*info)["name_id"])
|
|
|
- update_info := map[string]interface{}{}
|
|
|
- info_identity := subject.ConvertToBin(subject.Str2DEC(qu.ObjToString((*info)["identity_type"])))
|
|
|
- info_identity = subject.SupplementIdentityType(info_identity)
|
|
|
- if info_identity != identity {
|
|
|
- new_identity := subject.CalculateIdentityType(info_identity, identity)
|
|
|
- if new_identity != info_identity {
|
|
|
- update_info["identity_type"] = qu.IntAll(subject.Str2DEC(new_identity))
|
|
|
- update_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
|
|
|
- su.MysqlGlobalTool.Update(su.G_Units_Baseinfo, map[string]interface{}{"name_id": name_id}, update_info)
|
|
|
- }
|
|
|
- }
|
|
|
- if contact.Per != "" && contact.Tel != "" {
|
|
|
- contact_datas := su.MysqlGlobalTool.Find(su.G_Units_Contact, map[string]interface{}{"name_id": name_id}, "", "", -1, -1)
|
|
|
- isExists := false
|
|
|
- for _, v := range *contact_datas {
|
|
|
- update_id := qu.IntAll(v["id"])
|
|
|
- person := qu.ObjToString(v["contact_name"])
|
|
|
- tel := qu.ObjToString(v["contact_tel"])
|
|
|
- pt_str := qu.ObjToString(v["publishtime"])
|
|
|
- pt := int64(0)
|
|
|
- if pt_str != "" {
|
|
|
- t, _ := time.ParseInLocation(su.TimeLayout, pt_str, time.Local)
|
|
|
- pt = t.Unix()
|
|
|
- }
|
|
|
- if person+"~"+tel == contact.Per+"~"+contact.Tel {
|
|
|
- isExists = true
|
|
|
- old_contact_identity := subject.ConvertToBin(subject.Str2DEC(qu.ObjToString(v["identity_type"])))
|
|
|
- old_contact_identity = subject.SupplementIdentityType(old_contact_identity)
|
|
|
- if old_contact_identity != identity || qu.IntAll(v["source_type"]) == 2 || (contact.Publishtime > pt && contact.Publishtime > 0) {
|
|
|
- identity = subject.CalculateIdentityType(old_contact_identity, identity)
|
|
|
- update_contact := map[string]interface{}{}
|
|
|
- update_contact["source_type"] = 1
|
|
|
- update_contact["identity_type"] = qu.IntAll(subject.Str2DEC(identity))
|
|
|
- update_contact["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
|
|
|
- if contact.Publishtime > pt {
|
|
|
- update_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
|
|
|
- }
|
|
|
- //更新通讯录
|
|
|
- su.MysqlGlobalTool.Update(su.G_Units_Contact, map[string]interface{}{"id": update_id}, update_contact)
|
|
|
- }
|
|
|
- break
|
|
|
- }
|
|
|
- }
|
|
|
- if !isExists {
|
|
|
- add_contact := map[string]interface{}{}
|
|
|
- add_contact["name_id"] = name_id
|
|
|
- add_contact["contact_name"] = contact.Per
|
|
|
- add_contact["contact_tel"] = contact.Tel
|
|
|
- add_contact["contact_email"] = ""
|
|
|
- add_contact["source_type"] = 1
|
|
|
- add_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
|
|
|
- add_contact["identity_type"] = 2
|
|
|
- add_contact["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
|
|
|
- //新增~通讯录表
|
|
|
- su.InsertMysqlData(su.G_Units_Contact, add_contact, tmpid)
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
+*/
|