主题库相关调整

fieldproject_medical/data_subject/src/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"subject_sql"
+	"subject"
 	su "subject_util"
 )
 
@@ -12,9 +12,15 @@ func init() {
 }
 
 func main() {
-	//subject.RunSubjectFullDataInfo("632892000000000000000000")
+	//subject.RunSubjectFullDataInfo("5a4af64340d2d9bbe8b0859d")
 	//subject_heal.HealSubjectInfoData()
-	subject_sql.ExportSqlInfoData()
+	//subject_sql.ExportSqlInfoData()
+
+	//修复数据
+	//subject_repair.RunRepairFullDataInfo("632892000000000000000000")
+
+	//增量~更新数据
+	subject.RunSubjectAddDataInfo("632892000000000000000000", "63346f800000000000000000")
 
 	lock := make(chan bool)
 	<-lock

fieldproject_medical/data_subject/src/mark

@@ -0,0 +1,37 @@
+全量准备
+632892000000000000000000
+
+63346f800000000000000000
+
+
+
+错误记录
+
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d17be4c014f37148fc0ce
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d180c4c014f3714944e54
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18104c014f3714947fd7
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d18114c014f3714948f14
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d19a14c014f3714abe0a2
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1a804c014f3714b8f605
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1a804c014f3714b8f605
+Data too long for column 'name' at row 1 插入数据异常...dws_f_ents_baseinfo~632d1a854c014f3714b941a9
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1ab04c014f3714bbbd07
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3
+Data too long for column 'contact_tel' at row 1 插入数据异常...dws_f_ents_contact~632d1bac4c014f3714ca8ae3

fieldproject_medical/data_subject/src/subject/info.go

@@ -1,14 +1,15 @@
 package subject
 
 type Info struct {
-	identity   string
-	buyerclass string
-	contact    map[string]*Contact
+	Identity   string
+	Buyerclass string
+	Contact    map[string]*Contact
 }
 type Contact struct {
-	per    string
-	tel    string
-	buyer  bool
-	agency bool
-	winner bool
+	Per         string
+	Tel         string
+	Buyer       bool
+	Agency      bool
+	Winner      bool
+	Publishtime int64
 }

fieldproject_medical/data_subject/src/subject/mem.go

@@ -1,28 +0,0 @@
-package subject
-
-import (
-	"fmt"
-	"runtime"
-)
-
-func toMegaBytes(bytes uint64) float64 {
-	return float64(bytes) / 1024 / 1024
-}
-
-func traceMemStats() {
-	var ms runtime.MemStats
-	runtime.ReadMemStats(&ms)
-	var result = make([]float64, 7)
-	result[0] = float64(ms.HeapObjects)
-	result[1] = toMegaBytes(ms.HeapAlloc)
-	result[2] = toMegaBytes(ms.TotalAlloc)
-	result[3] = toMegaBytes(ms.HeapSys)
-	result[4] = toMegaBytes(ms.HeapIdle)
-	result[5] = toMegaBytes(ms.HeapReleased)
-	result[6] = toMegaBytes(ms.HeapIdle - ms.HeapReleased)
-
-	for _, v := range result {
-		fmt.Printf("%.2f\t", v)
-	}
-	fmt.Printf("\n")
-}

fieldproject_medical/data_subject/src/subject/method.go

@@ -0,0 +1,176 @@
+package subject
+
+import (
+	"fmt"
+	"runtime"
+	"strconv"
+	"strings"
+	su "subject_util"
+	"unicode/utf8"
+)
+
+//转二进制
+func Str2DEC(s string) (num int) {
+	l := len(s)
+	for i := l - 1; i >= 0; i-- {
+		num += (int(s[l-i-1]) & 0xf) << uint8(i)
+	}
+	return
+}
+
+//
+func ConvertToBin(num int) string {
+	s := ""
+	if num == 0 {
+		return "0"
+	}
+	for ; num > 0; num /= 2 {
+		lsb := num % 2
+		s = strconv.Itoa(lsb) + s
+	}
+	return s
+}
+
+//补充身份类型字符串~
+func SupplementIdentityType(identity string) string {
+	new_str := identity
+	l := utf8.RuneCountInString(identity)
+	if l >= 3 {
+		return new_str[:3]
+	}
+	for i := 0; i < 3-l; i++ {
+		new_str = "0" + new_str
+	}
+	return new_str
+}
+
+//计算身份类型
+func CalculateIdentityType(identity string, new_identity string) string {
+	a := identity[:1]
+	b := identity[1:2]
+	c := identity[2:3]
+	if new_identity == "001" {
+		c = "1"
+	} else if new_identity == "010" {
+		b = "1"
+	} else if new_identity == "100" {
+		a = "1"
+	} else {
+
+	}
+	return a + b + c
+}
+
+//计算身份类型
+func CreateIdentityBool(buyer bool, agency bool, winner bool) string {
+	new_identity := ""
+	if agency {
+		new_identity += "1"
+	} else {
+		new_identity += "0"
+	}
+	if winner {
+		new_identity += "1"
+	} else {
+		new_identity += "0"
+	}
+	if buyer {
+		new_identity += "1"
+	} else {
+		new_identity += "0"
+	}
+	return new_identity
+}
+
+//处理地域代码
+func CalculateRegionCode(area string, city string, district string) (area_code string, city_code string, district_code string) {
+	area_code, city_code, district_code = "000000", "", ""
+	if district != "" {
+		key := area + "~" + city + "~" + district + "~"
+		code := su.RegionCodeData[key]
+		if code != "" {
+			district_code = code
+			city_code = code[:4] + "00"
+			area_code = code[:2] + "0000"
+			return
+		}
+	}
+	if city != "" {
+		key := area + "~" + city + "~" + "" + "~"
+		code := su.RegionCodeData[key]
+		if code != "" {
+			city_code = code
+			area_code = city_code[:2] + "0000"
+			return
+		}
+	}
+	if area != "" {
+		key := area + "~" + "" + "~" + "" + "~"
+		code := su.RegionCodeData[key]
+		if code != "" {
+			area_code = code
+			return
+		}
+	}
+	return
+}
+
+//单位分割~去重
+func SegmentationEntName(name_1 string, name_2 string) ([]string, []bool) {
+	new_arr := []string{}
+	new_bool := []bool{}
+	key_str := map[string]string{}
+
+	if name_1 != "" {
+		arr_1 := strings.Split(name_1, ",")
+		if len(arr_1) == 1 {
+			arr_1 = strings.Split(name_1, "，")
+		}
+		for _, v := range arr_1 {
+			if v != "" && utf8.RuneCountInString(v) > 2 && utf8.RuneCountInString(v) < 30 {
+				if key_str[v] == "" {
+					new_arr = append(new_arr, v)
+					key_str[v] = v
+					new_bool = append(new_bool, true)
+				}
+			}
+		}
+	}
+	if name_2 != "" {
+		arr_2 := strings.Split(name_2, ",")
+		if len(arr_2) == 1 {
+			arr_2 = strings.Split(name_2, "，")
+		}
+		for _, v := range arr_2 {
+			if v != "" && utf8.RuneCountInString(v) > 2 && utf8.RuneCountInString(v) < 30 {
+				if key_str[v] == "" {
+					new_arr = append(new_arr, v)
+					key_str[v] = v
+					new_bool = append(new_bool, false)
+				}
+			}
+		}
+	}
+	return new_arr, new_bool
+}
+
+func toMegaBytes(bytes uint64) float64 {
+	return float64(bytes) / 1024 / 1024
+}
+func traceMemStats() {
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	var result = make([]float64, 7)
+	result[0] = float64(ms.HeapObjects)
+	result[1] = toMegaBytes(ms.HeapAlloc)
+	result[2] = toMegaBytes(ms.TotalAlloc)
+	result[3] = toMegaBytes(ms.HeapSys)
+	result[4] = toMegaBytes(ms.HeapIdle)
+	result[5] = toMegaBytes(ms.HeapReleased)
+	result[6] = toMegaBytes(ms.HeapIdle - ms.HeapReleased)
+
+	for _, v := range result {
+		fmt.Printf("%.2f\t", v)
+	}
+	fmt.Printf("\n")
+}

fieldproject_medical/data_subject/src/subject/subject_add.go

@@ -0,0 +1,352 @@
+package subject
+
+import (
+	log "github.com/donnie4w/go-logger/logger"
+	"github.com/uuid"
+	qu "qfw/util"
+	"strings"
+	su "subject_util"
+	"sync"
+	"time"
+	"unicode/utf8"
+	ul "util"
+)
+
+var (
+	updateLock sync.Mutex
+)
+
+//正产增量主体数据服务
+func RunSubjectAddDataInfo(gtid string, lteid string) {
+	log.Debug("增量~~~")
+	sess := su.SourceMgo.GetMgoConn()
+	defer su.SourceMgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  su.StringTOBsonId(gtid),
+			"$lte": su.StringTOBsonId(lteid),
+		},
+	}
+	log.Debug("查询语句 ~ ", q)
+	it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(BidFields).Iter()
+	pool := make(chan bool, 8)
+	wg := &sync.WaitGroup{}
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			if qu.IntAll(tmp["extracttype"]) != 1 {
+				return
+			}
+			if qu.ObjToString(tmp["buyer"]) == "" &&
+				qu.ObjToString(tmp["agency"]) == "" &&
+				qu.ObjToString(tmp["winner"]) == "" {
+				return
+			}
+			dealWithAddSubjectInfo(tmp)
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Debug("is over ~ ", total)
+}
+
+func dealWithAddSubjectInfo(tmp map[string]interface{}) {
+	buyer := qu.ObjToString(tmp["buyer"])
+	agency := qu.ObjToString(tmp["agency"])
+	winner := qu.ObjToString(tmp["winner"])
+	s_winner := qu.ObjToString(tmp["s_winner"])
+	b_per := qu.ObjToString(tmp["buyerperson"])
+	b_tel := qu.ObjToString(tmp["buyertel"])
+	if utf8.RuneCountInString(b_tel) > 60 {
+		b_tel = ""
+	}
+	a_per := qu.ObjToString(tmp["agencyperson"])
+	a_tel := qu.ObjToString(tmp["agencytel"])
+	if utf8.RuneCountInString(a_tel) > 60 {
+		a_tel = ""
+	}
+	w_per := qu.ObjToString(tmp["winnerperson"])
+	w_tel := qu.ObjToString(tmp["winnertel"])
+	if utf8.RuneCountInString(w_tel) > 60 {
+		w_tel = ""
+	}
+	buyerclass := qu.ObjToString(tmp["buyerclass"])
+	publishtime := qu.Int64All(tmp["publishtime"])
+	tmpid := su.BsonTOStringId(tmp["_id"])
+	winner_arr, winner_bool := SegmentationEntName(winner, s_winner)
+
+	updateLock.Lock()
+	if buyer != "" && utf8.RuneCountInString(buyer) < 30 {
+		dealWithUpdateContact(buyer, buyerclass, &Contact{b_per, b_tel, true, false, false, publishtime}, "001", tmpid)
+	}
+
+	//中标单位
+	for k, v := range winner_arr {
+		b := winner_bool[k]
+		c := Contact{"", "", false, false, true, publishtime}
+		if b {
+			c.Per = w_per
+			c.Tel = w_tel
+		}
+		dealWithUpdateContact(v, "", &c, "010", tmpid)
+	}
+
+	if agency != "" && utf8.RuneCountInString(agency) < 30 {
+		dealWithUpdateContact(agency, "", &Contact{a_per, a_tel, false, true, false, publishtime}, "100", tmpid)
+	}
+	updateLock.Unlock()
+}
+
+func dealWithUpdateContact(name string, buyerclass string, contact *Contact, identity string, tmpid string) {
+	info := su.MysqlGlobalTool.FindOne(su.G_Units_Baseinfo, map[string]interface{}{"name": name}, "", "")
+	isNewEnt := false
+	isNewEntInfo := map[string]interface{}{}
+	if info == nil {
+	} else { //判断企业库最新的company_id是否与当前保持一致
+		qyxy_info := CreateQyxyInfo(name)
+		if len(qyxy_info) > 0 {
+			new_company_id := qu.ObjToString(qyxy_info["_id"])
+			old_company_id := qu.ObjToString((*info)["company_id"])
+			if old_company_id != "" && new_company_id != "" && old_company_id != new_company_id {
+				isNewEnt = true
+				isNewEntInfo["name"] = name
+				isNewEntInfo["name_id"] = qu.ObjToString((*info)["name_id"])
+				isNewEntInfo["company_id"] = old_company_id
+			}
+		}
+	}
+	if info == nil || isNewEnt { //新增主体信息~企业补充~年报补充
+		base_info := map[string]interface{}{}
+		base_info["name"] = name
+		qyxy_info := CreateQyxyInfo(name)
+		//通讯录数据结构
+		contact_arr := []map[string]interface{}{}
+		area, city, district := "", "", ""
+		name_id := uuid.New().String()
+		name_id = strings.ReplaceAll(name_id, "-", "")
+		base_info["name_id"] = name_id
+		base_info["identity_type"] = qu.IntAll(Str2DEC(identity))
+		base_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+		//创建通讯录信息
+		if contact.Per != "" && contact.Tel != "" {
+			date := ""
+			if contact.Publishtime > 0 {
+				date = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+			}
+			contact_arr = append(contact_arr, map[string]interface{}{
+				"person":   contact.Per,
+				"tel":      contact.Tel,
+				"date":     date,
+				"email":    "",
+				"type":     1,
+				"identity": base_info["identity_type"],
+			})
+		}
+		if len(qyxy_info) > 0 { //含企业信息
+			info_id := qu.ObjToString(qyxy_info["_id"])
+			base_info["company_id"] = info_id
+			address := qu.ObjToString(qyxy_info["company_address"])
+			if utf8.RuneCountInString(address) > 300 {
+				address = ""
+			}
+			base_info["address"] = address
+			area = qu.ObjToString(qyxy_info["company_area"])
+			city = qu.ObjToString(qyxy_info["company_city"])
+			district = qu.ObjToString(qyxy_info["company_district"])
+			legal_person := qu.ObjToString(qyxy_info["legal_person"])
+			if legal_person != "" { //年报信息
+				CreateAnnualInfo(info_id, legal_person, &contact_arr)
+			}
+		}
+		base_info["area_code"], base_info["city_code"], base_info["district_code"] = CalculateRegionCode(area, city, district)
+		//新增主体信息表
+		b := su.InsertMysqlData(su.G_Units_Baseinfo, base_info, tmpid)
+		if b > 0 { //存通讯录
+			for _, v := range contact_arr {
+				info_book := map[string]interface{}{}
+				info_book["name_id"] = name_id
+				info_book["contact_name"] = qu.ObjToString(v["person"])
+				info_book["contact_tel"] = qu.ObjToString(v["tel"])
+				info_book["contact_email"] = qu.ObjToString(v["email"])
+				date := qu.ObjToString(v["date"])
+				if date != "" {
+					info_book["publishtime"] = date
+				}
+				info_book["source_type"] = qu.IntAll(v["type"])
+				info_book["identity_type"] = qu.IntAll(v["identity"])
+				info_book["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+				su.InsertMysqlData(su.G_Units_Contact, info_book, tmpid)
+			}
+			//存主体标签记录~暂时只进采购单位的主体
+			if qu.IntAll(identity)%10 == 1 {
+				info_tag := map[string]interface{}{}
+				info_tag["name_id"] = name_id
+				info_tag["labelcode"] = "1"
+				labelvalues := "00"
+				if su.BuyerClassData[buyerclass] != "" {
+					labelvalues = su.BuyerClassData[buyerclass]
+				}
+				info_tag["labelvalues"] = labelvalues //代码表
+				info_tag["identity_type"] = 1
+				info_tag["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+				su.InsertMysqlData(su.G_Units_Tags, info_tag, tmpid)
+			}
+			if isNewEnt {
+				isNewEntInfo["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+				su.InsertMysqlData(su.G_Units_Warning, isNewEntInfo, tmpid)
+			}
+		}
+		return
+	}
+	//更新~主体基本信息表
+	name_id := qu.ObjToString((*info)["name_id"])
+	update_info := map[string]interface{}{}
+	info_identity := ConvertToBin(Str2DEC(qu.ObjToString((*info)["identity_type"])))
+	info_identity = SupplementIdentityType(info_identity)
+	if info_identity != identity {
+		new_identity := CalculateIdentityType(info_identity, identity)
+		if new_identity != info_identity {
+			update_info["identity_type"] = qu.IntAll(Str2DEC(new_identity))
+			update_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			su.MysqlGlobalTool.Update(su.G_Units_Baseinfo, map[string]interface{}{"name_id": name_id}, update_info)
+		}
+	}
+	//更新~主体标签记录表
+	if buyerclass != "" && contact.Buyer {
+		info_tag := su.MysqlGlobalTool.FindOne(su.G_Units_Tags, map[string]interface{}{"name_id": name_id}, "", "")
+		if info_tag != nil {
+			update_tag := map[string]interface{}{}
+			cur_code := "00"
+			if su.BuyerClassData[buyerclass] != "" {
+				cur_code = su.BuyerClassData[buyerclass]
+			}
+			old_code := qu.ObjToString((*info_tag)["labelvalues"])
+			if cur_code != old_code {
+				update_tag["labelvalues"] = cur_code
+				update_tag["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+				su.MysqlGlobalTool.Update(su.G_Units_Tags, map[string]interface{}{"name_id": name_id}, update_tag)
+			}
+		}
+	}
+	if contact.Per != "" && contact.Tel != "" {
+		contact_datas := su.MysqlGlobalTool.Find(su.G_Units_Contact, map[string]interface{}{"name_id": name_id}, "", "", -1, -1)
+		isExists := false
+		for _, v := range *contact_datas {
+			update_id := qu.IntAll(v["id"])
+			person := qu.ObjToString(v["contact_name"])
+			tel := qu.ObjToString(v["contact_tel"])
+			pt_str := qu.ObjToString(v["publishtime"])
+			pt := int64(0)
+			if pt_str != "" {
+				t, _ := time.ParseInLocation(su.TimeLayout, pt_str, time.Local)
+				pt = t.Unix()
+			}
+			if person+"~"+tel == contact.Per+"~"+contact.Tel {
+				isExists = true
+				old_contact_identity := ConvertToBin(Str2DEC(qu.ObjToString(v["identity_type"])))
+				old_contact_identity = SupplementIdentityType(old_contact_identity)
+				if old_contact_identity != identity || qu.IntAll(v["source_type"]) == 2 || (contact.Publishtime > pt && contact.Publishtime > 0) {
+					new_identity := CalculateIdentityType(old_contact_identity, identity)
+					update_contact := map[string]interface{}{}
+					update_contact["source_type"] = 1
+					update_contact["identity_type"] = qu.IntAll(Str2DEC(new_identity))
+					update_contact["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+					if contact.Publishtime > pt {
+						update_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+					}
+					//更新通讯录
+					su.MysqlGlobalTool.Update(su.G_Units_Contact, map[string]interface{}{"id": update_id}, update_contact)
+				}
+				break
+			}
+		}
+		if !isExists {
+			add_contact := map[string]interface{}{}
+			add_contact["name_id"] = name_id
+			add_contact["contact_name"] = contact.Per
+			add_contact["contact_tel"] = contact.Tel
+			add_contact["contact_email"] = ""
+			add_contact["source_type"] = 1
+			if contact.Publishtime > 0 {
+				add_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+			}
+			add_contact["identity_type"] = qu.IntAll(Str2DEC(identity))
+			add_contact["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			//新增~通讯录表
+			su.InsertMysqlData(su.G_Units_Contact, add_contact, tmpid)
+		}
+	}
+
+}
+
+//创建年报字段~与现有的通讯录
+func CreateAnnualInfo(company_id string, legal_person string, contact_arr *[]map[string]interface{}) {
+	keys := map[string]string{}
+	for _, v := range *contact_arr {
+		key := qu.ObjToString(v["person"]) + "~" + qu.ObjToString(v["tel"])
+		keys[key] = key
+	}
+	dataArr, _ := su.SpiMgo.Find("annual_report_base", map[string]interface{}{"company_id": company_id}, nil, map[string]interface{}{
+		"company_phone": 1,
+		"company_email": 1,
+		"report_date":   1,
+	})
+	for _, v := range dataArr {
+		company_phone := qu.ObjToString(v["company_phone"])
+		company_email := qu.ObjToString(v["company_email"])
+		if company_phone != "" {
+			report_date := qu.ObjToString(v["report_date"])
+			key := legal_person + "~" + company_phone
+			if keys[key] == "" {
+				dict := map[string]interface{}{
+					"person":   legal_person,
+					"tel":      company_phone,
+					"date":     report_date,
+					"email":    company_email,
+					"type":     2,
+					"identity": 0,
+				}
+				(*contact_arr) = append((*contact_arr), dict)
+				keys[key] = key
+			}
+		}
+	}
+}
+
+//创建企业信息
+func CreateQyxyInfo(name string) map[string]interface{} {
+	//查询企业库
+	qyxy_info := map[string]interface{}{}
+	dataArr, _ := su.QyxyMgo.Find("qyxy_std", map[string]interface{}{"company_name": name}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{
+		"_id":              1,
+		"company_name":     1,
+		"company_address":  1,
+		"company_area":     1,
+		"company_city":     1,
+		"company_district": 1,
+		"legal_person":     1,
+		"company_phone":    1,
+		"company_email":    1,
+	})
+	if len(dataArr) > 0 {
+		qyxy_info = dataArr[0] //补充企业信息
+	} else {
+		data := su.SpiMgo.FindOne("company_history_name", map[string]interface{}{
+			"history_name": name,
+		})
+		if len(data) > 0 {
+			company_id := qu.ObjToString(data["company_id"])
+			qyxy_info = su.QyxyMgo.FindOne("qyxy_std", map[string]interface{}{"_id": company_id})
+		}
+	}
+	return qyxy_info
+}

fieldproject_medical/data_subject/src/subject/subject.go → fieldproject_medical/data_subject/src/subject/subject_full.go

@@ -5,6 +5,7 @@ import (
 	qu "qfw/util"
 	su "subject_util"
 	"sync"
+	"unicode/utf8"
 )
 
 var (
@@ -13,8 +14,8 @@ var (
 )
 
 var BidFields = map[string]interface{}{
-	"extracttype": 1, "buyerclass": 1,
-	"buyer": 1, "agency": 1, "winner": 1,
+	"extracttype": 1, "buyerclass": 1, "publishtime": 1,
+	"buyer": 1, "agency": 1, "winner": 1, "s_winner": 1,
 	"agencytel": 1, "winnertel": 1, "buyertel": 1,
 	"agencyperson": 1, "winnerperson": 1, "buyerperson": 1,
 }
@@ -22,15 +23,15 @@ var BidFields = map[string]interface{}{
 //全量数据准备
 func RunSubjectFullDataInfo(lteid string) {
 	log.Debug("开始准备全量主体~~~")
-	sess := su.SaveMgo.GetMgoConn()
-	defer su.SaveMgo.DestoryMongoConn(sess)
+	sess := su.SourceMgo.GetMgoConn()
+	defer su.SourceMgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
 			"$lte": su.StringTOBsonId(lteid),
 		},
 	}
 	log.Debug("查询语句 ~ ", q)
-	it := sess.DB(su.SaveMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(BidFields).Iter()
+	it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(BidFields).Iter()
 	pool := make(chan bool, 8)
 	wg := &sync.WaitGroup{}
 	total := 0
@@ -82,20 +83,21 @@ func saveMgoInfo() {
 			}()
 			data := map[string]interface{}{}
 			data["name"] = k
-			data["buyerclass"] = v.buyerclass
-			data["identity"] = qu.IntAll(v.identity)
+			data["buyerclass"] = v.Buyerclass
+			data["identity"] = qu.IntAll(v.Identity)
 			arr := []map[string]interface{}{}
-			for _, v1 := range v.contact {
+			for _, v1 := range v.Contact {
 				arr = append(arr, map[string]interface{}{
-					"person": v1.per,
-					"tel":    v1.tel,
-					"buyer":  v1.buyer,
-					"agency": v1.agency,
-					"winner": v1.winner,
+					"person":      v1.Per,
+					"tel":         v1.Tel,
+					"buyer":       v1.Buyer,
+					"agency":      v1.Agency,
+					"winner":      v1.Winner,
+					"publishtime": v1.Publishtime,
 				})
 			}
 			data["contact"] = arr
-			su.SaveMgo.Save(su.O_Coll_Name, data)
+			su.SourceMgo.Save(su.O_Coll_Name, data)
 		}(k, v)
 	}
 	wg.Wait()
@@ -108,20 +110,32 @@ func createBaseInfo(tmp map[string]interface{}) {
 	winner := qu.ObjToString(tmp["winner"])
 	b_per := qu.ObjToString(tmp["buyerperson"])
 	b_tel := qu.ObjToString(tmp["buyertel"])
+	if utf8.RuneCountInString(b_tel) > 60 {
+		b_tel = ""
+	}
 	a_per := qu.ObjToString(tmp["agencyperson"])
 	a_tel := qu.ObjToString(tmp["agencytel"])
+	if utf8.RuneCountInString(a_tel) > 60 {
+		a_tel = ""
+	}
 	w_per := qu.ObjToString(tmp["winnerperson"])
 	w_tel := qu.ObjToString(tmp["winnertel"])
+	if utf8.RuneCountInString(w_tel) > 60 {
+		w_tel = ""
+	}
 	buyerclass := qu.ObjToString(tmp["buyerclass"])
+	publishtime := qu.Int64All(tmp["publishtime"])
+
 	dataLock.Lock()
 	if buyer != "" {
-		dealWithContact(buyer, buyerclass, &Contact{b_per, b_tel, true, false, false}, "001")
+		dealWithContact(buyer, buyerclass, &Contact{b_per, b_tel, true, false, false, publishtime}, "001")
 	}
 	if winner != "" {
-		dealWithContact(winner, "", &Contact{w_per, w_tel, false, false, true}, "010")
+		//拆s_winner 需要拆
+		dealWithContact(winner, "", &Contact{w_per, w_tel, false, false, true, publishtime}, "010")
 	}
 	if agency != "" {
-		dealWithContact(agency, "", &Contact{a_per, a_tel, false, true, false}, "100")
+		dealWithContact(agency, "", &Contact{a_per, a_tel, false, true, false, publishtime}, "100")
 	}
 	dataLock.Unlock()
 }
@@ -129,51 +143,37 @@ func createBaseInfo(tmp map[string]interface{}) {
 func dealWithContact(name string, buyerclass string, contact *Contact, identity string) {
 	info := SubjectInfos[name]
 	if info != nil {
-		if info.identity != identity {
-			info.identity = calculateIdentityType(info.identity, identity)
+		if info.Identity != identity {
+			info.Identity = CalculateIdentityType(info.Identity, identity)
 		}
 		if buyerclass != "" {
-			info.buyerclass = buyerclass
+			info.Buyerclass = buyerclass
 		}
-		if contact.per != "" && contact.tel != "" {
-			key := contact.per + "~" + contact.tel
-			old_contact := info.contact[key]
+		if contact.Per != "" && contact.Tel != "" {
+			key := contact.Per + "~" + contact.Tel
+			old_contact := info.Contact[key]
 			if old_contact != nil {
-				if old_contact.buyer {
-					contact.buyer = true
+				if old_contact.Buyer {
+					contact.Buyer = true
+				}
+				if old_contact.Agency {
+					contact.Agency = true
 				}
-				if old_contact.agency {
-					contact.agency = true
+				if old_contact.Winner {
+					contact.Winner = true
 				}
-				if old_contact.winner {
-					contact.winner = true
+				if old_contact.Publishtime > contact.Publishtime {
+					contact.Publishtime = old_contact.Publishtime
 				}
 			}
-			info.contact[key] = contact
+			info.Contact[key] = contact
 		}
 	} else {
 		data := map[string]*Contact{}
-		if contact.per != "" && contact.tel != "" {
-			key := contact.per + "~" + contact.tel
+		if contact.Per != "" && contact.Tel != "" {
+			key := contact.Per + "~" + contact.Tel
 			data[key] = contact
 		}
 		SubjectInfos[name] = &Info{identity, buyerclass, data}
 	}
 }
-
-//计算身份类型
-func calculateIdentityType(identity string, new_identity string) string {
-	a := identity[:1]
-	b := identity[1:2]
-	c := identity[2:3]
-	if new_identity == "001" {
-		c = "1"
-	} else if new_identity == "010" {
-		b = "1"
-	} else if new_identity == "100" {
-		a = "1"
-	} else {
-
-	}
-	return a + b + c
-}

fieldproject_medical/data_subject/src/subject_heal/heal.go

@@ -14,16 +14,16 @@ var (
 //治愈数据
 func HealSubjectInfoData() {
 	log.Debug("开始治愈数据~~~企业相关信息~~~")
-	sess := su.SaveMgo.GetMgoConn()
-	defer su.SaveMgo.DestoryMongoConn(sess)
+	sess := su.SourceMgo.GetMgoConn()
+	defer su.SourceMgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
 	log.Debug("查询语句 ~ ", q)
-	it := sess.DB(su.SaveMgo.DbName).C(su.O_Coll_Name).Find(&q).Sort("_id").Iter()
+	it := sess.DB(su.SourceMgo.DbName).C(su.O_Coll_Name).Find(&q).Sort("_id").Iter()
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 	total, isok := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
+		if total%5000 == 0 {
 			log.Debug("cur index ", total, "~", isok, "~", tmp["_id"])
 		}
 		pool <- true
@@ -50,7 +50,7 @@ func updateInfoData(tmp map[string]interface{}) bool {
 	update := map[string]interface{}{}
 	name := qu.ObjToString(tmp["name"])
 	contact := su.IsMarkInterfaceMap(tmp["contact"])
-	dataArr, _ := su.SaveMgo.Find("qyxy_std", map[string]interface{}{"company_name": name}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{
+	dataArr, _ := su.QyxyMgo.Find("qyxy_std", map[string]interface{}{"company_name": name}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{
 		"_id":              1,
 		"company_name":     1,
 		"company_address":  1,
@@ -59,10 +59,20 @@ func updateInfoData(tmp map[string]interface{}) bool {
 		"company_district": 1,
 		"legal_person":     1,
 		"company_phone":    1,
+		"company_email":    1,
 	})
 	if len(dataArr) > 0 {
-		info := dataArr[0]
-		dealWithUpdateInfo(contact, info, &update)
+		info := dataArr[0] //补充企业信息
+		updateQyxyInfo(info, &update)
+		//构建年报通讯录数据
+		info_id := qu.ObjToString(info["_id"])
+		legal_person := qu.ObjToString(info["legal_person"])
+		if legal_person != "" {
+			annualArr := updateAnnualInfo(info_id, contact, legal_person)
+			if len(annualArr) > 0 {
+				update["annual"] = annualArr
+			}
+		}
 	} else {
 		data := su.SpiMgo.FindOne("company_history_name", map[string]interface{}{
 			"history_name": name,
@@ -70,16 +80,24 @@ func updateInfoData(tmp map[string]interface{}) bool {
 		if len(data) > 0 {
 			company_id := qu.ObjToString(data["company_id"])
 			if company_id != "" {
-				info := su.SaveMgo.FindOne("qyxy_std", map[string]interface{}{"_id": company_id})
+				info := su.QyxyMgo.FindOne("qyxy_std", map[string]interface{}{"_id": company_id})
 				if len(info) > 0 && info != nil {
-					dealWithUpdateInfo(contact, info, &update)
+					updateQyxyInfo(info, &update)
+					//构建年报通讯录数据
+					legal_person := qu.ObjToString(info["legal_person"])
+					if legal_person != "" {
+						annualArr := updateAnnualInfo(company_id, contact, legal_person)
+						if len(annualArr) > 0 {
+							update["annual"] = annualArr
+						}
+					}
 				}
 			}
 		}
 	}
 	if len(update) > 0 { //更新操作
 		tmpid := su.BsonTOStringId(tmp["_id"])
-		su.SaveMgo.UpdateById(su.O_Coll_Name, tmpid, map[string]interface{}{
+		su.SourceMgo.UpdateById(su.O_Coll_Name, tmpid, map[string]interface{}{
 			"$set": update,
 		})
 		return true
@@ -87,28 +105,45 @@ func updateInfoData(tmp map[string]interface{}) bool {
 	return false
 }
 
-//处理待更新信息
-func dealWithUpdateInfo(contact []map[string]interface{}, info map[string]interface{}, update *map[string]interface{}) {
+//更新企业信息
+func updateQyxyInfo(info map[string]interface{}, update *map[string]interface{}) {
 	(*update)["company_address"] = qu.ObjToString(info["company_address"])
 	(*update)["company_id"] = qu.ObjToString(info["_id"])
 	(*update)["area"] = qu.ObjToString(info["company_area"])
 	(*update)["city"] = qu.ObjToString(info["company_city"])
 	(*update)["district"] = qu.ObjToString(info["company_district"])
-	legal_person := qu.ObjToString(info["legal_person"])
-	company_phone := qu.ObjToString(info["company_phone"])
-	if legal_person != "" && company_phone != "" {
-		is_R := false
-		for _, v := range contact {
-			person := qu.ObjToString(v["person"])
-			tel := qu.ObjToString(v["tel"])
-			if person == legal_person && company_phone == tel {
-				is_R = true
-				break
+}
+
+func updateAnnualInfo(company_id string, contact []map[string]interface{}, legal_person string) []map[string]interface{} {
+	annual := []map[string]interface{}{}
+	keys := map[string]string{}
+	for _, v := range contact {
+		key := qu.ObjToString(v["person"]) + "~" + qu.ObjToString(v["tel"])
+		keys[key] = key
+	}
+	//对联系人电话进行去重~
+	dataArr, _ := su.SpiMgo.Find("annual_report_base", map[string]interface{}{"company_id": company_id}, nil, map[string]interface{}{
+		"company_phone": 1,
+		"company_email": 1,
+		"report_date":   1,
+	})
+	for _, v := range dataArr {
+		company_phone := qu.ObjToString(v["company_phone"])
+		company_email := qu.ObjToString(v["company_email"])
+		if company_phone != "" {
+			report_date := qu.ObjToString(v["report_date"])
+			key := legal_person + "~" + company_phone
+			if keys[key] == "" {
+				dict := map[string]interface{}{
+					"person": legal_person,
+					"tel":    company_phone,
+					"date":   report_date,
+					"email":  company_email,
+				}
+				annual = append(annual, dict)
+				keys[key] = key
 			}
 		}
-		if !is_R {
-			(*update)["legal_person"] = legal_person
-			(*update)["legal_tel"] = company_phone
-		}
 	}
+	return annual
 }

fieldproject_medical/data_subject/src/subject_repair/repair.go

@@ -0,0 +1,230 @@
+package subject_repair
+
+import (
+	log "github.com/donnie4w/go-logger/logger"
+	"github.com/uuid"
+	qu "qfw/util"
+	"strings"
+	"subject"
+	su "subject_util"
+	"sync"
+	"time"
+	"unicode/utf8"
+	ul "util"
+)
+
+var (
+	dataLock sync.Mutex
+	numLock  sync.Mutex
+)
+
+var RepairFields = map[string]interface{}{
+	"extracttype": 1, "publishtime": 1,
+	"winner": 1, "s_winner": 1,
+	"winnertel": 1, "winnerperson": 1,
+}
+
+//全量数据准备
+func RunRepairFullDataInfo(lteid string) {
+	log.Debug("开始修复全量主体~~~")
+	sess := su.SourceMgo.GetMgoConn()
+	defer su.SourceMgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$lte": su.StringTOBsonId(lteid),
+		},
+	}
+	log.Debug("查询语句 ~ ", q)
+	it := sess.DB(su.SourceMgo.DbName).C(su.S_Coll_Name).Find(&q).Sort("_id").Select(RepairFields).Iter()
+	pool := make(chan bool, 16)
+	wg := &sync.WaitGroup{}
+	total, isok := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			if qu.IntAll(tmp["extracttype"]) != 1 {
+				return
+			}
+			if qu.ObjToString(tmp["winner"]) == "" && qu.ObjToString(tmp["s_winner"]) == "" {
+				return
+			}
+			b := createRepairBaseInfo(tmp) //处理数据
+			if b {
+				numLock.Lock()
+				isok++
+				numLock.Unlock()
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Debug("is over ~ ", total, "~", isok)
+
+}
+
+func createRepairBaseInfo(tmp map[string]interface{}) bool {
+	winner := qu.ObjToString(tmp["winner"])
+	s_winner := qu.ObjToString(tmp["s_winner"])
+	w_per := qu.ObjToString(tmp["winnerperson"])
+	w_tel := qu.ObjToString(tmp["winnertel"])
+
+	if utf8.RuneCountInString(w_tel) > 60 {
+		w_tel = ""
+	}
+	publishtime := qu.Int64All(tmp["publishtime"])
+	tmpid := su.BsonTOStringId(tmp["_id"])
+	winner_arr, winner_bool := subject.SegmentationEntName(winner, s_winner)
+	if len(winner_arr) <= 1 {
+		return false
+	}
+	dataLock.Lock() //处理
+	for k, v := range winner_arr {
+		b := winner_bool[k]
+		c := subject.Contact{"", "", false, false, true, publishtime}
+		if b {
+			c.Per = w_per
+			c.Tel = w_tel
+		}
+		dealWithRepairSubjectContact(v, "", &c, "010", tmpid)
+	}
+
+	dataLock.Unlock()
+
+	return true
+}
+
+func dealWithRepairSubjectContact(name string, buyerclass string, contact *subject.Contact, identity string, tmpid string) {
+	info := su.MysqlGlobalTool.FindOne(su.G_Units_Baseinfo, map[string]interface{}{"name": name}, "", "")
+	if info == nil { //新增主体信息~企业补充~年报补充
+		base_info := map[string]interface{}{}
+		base_info["name"] = name
+		qyxy_info := subject.CreateQyxyInfo(name)
+		//通讯录数据结构
+		contact_arr := []map[string]interface{}{}
+		area, city, district := "", "", ""
+		name_id := uuid.New().String()
+		name_id = strings.ReplaceAll(name_id, "-", "")
+		base_info["name_id"] = name_id
+		base_info["identity_type"] = 2
+		base_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+		//创建通讯录信息
+		if contact.Per != "" && contact.Tel != "" {
+			date := ""
+			if contact.Publishtime > 0 {
+				date = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+			}
+			contact_arr = append(contact_arr, map[string]interface{}{
+				"person":   contact.Per,
+				"tel":      contact.Tel,
+				"date":     date,
+				"email":    "",
+				"type":     1,
+				"identity": base_info["identity_type"],
+			})
+		}
+		if len(qyxy_info) > 0 { //含企业信息
+			info_id := qu.ObjToString(qyxy_info["_id"])
+			base_info["company_id"] = info_id
+			address := qu.ObjToString(qyxy_info["company_address"])
+			if utf8.RuneCountInString(address) > 300 {
+				address = ""
+			}
+			base_info["address"] = address
+			area = qu.ObjToString(qyxy_info["company_area"])
+			city = qu.ObjToString(qyxy_info["company_city"])
+			district = qu.ObjToString(qyxy_info["company_district"])
+			legal_person := qu.ObjToString(qyxy_info["legal_person"])
+			if legal_person != "" { //年报信息
+				subject.CreateAnnualInfo(info_id, legal_person, &contact_arr)
+			}
+		}
+		base_info["area_code"], base_info["city_code"], base_info["district_code"] = subject.CalculateRegionCode(area, city, district)
+		//新增主体信息表
+		b := su.InsertMysqlData(su.G_Units_Baseinfo, base_info, tmpid)
+		if b > 0 { //存通讯录
+			for _, v := range contact_arr {
+				info_book := map[string]interface{}{}
+				info_book["name_id"] = name_id
+				info_book["contact_name"] = qu.ObjToString(v["person"])
+				info_book["contact_tel"] = qu.ObjToString(v["tel"])
+				info_book["contact_email"] = qu.ObjToString(v["email"])
+				date := qu.ObjToString(v["date"])
+				if date != "" {
+					info_book["publishtime"] = date
+				}
+				info_book["source_type"] = qu.IntAll(v["type"])
+				info_book["identity_type"] = qu.IntAll(v["identity"])
+				info_book["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+				su.InsertMysqlData(su.G_Units_Contact, info_book, tmpid)
+			}
+		}
+		return
+	}
+	//更新~主体基本信息表
+	name_id := qu.ObjToString((*info)["name_id"])
+	update_info := map[string]interface{}{}
+	info_identity := subject.ConvertToBin(subject.Str2DEC(qu.ObjToString((*info)["identity_type"])))
+	info_identity = subject.SupplementIdentityType(info_identity)
+	if info_identity != identity {
+		new_identity := subject.CalculateIdentityType(info_identity, identity)
+		if new_identity != info_identity {
+			update_info["identity_type"] = qu.IntAll(subject.Str2DEC(new_identity))
+			update_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			su.MysqlGlobalTool.Update(su.G_Units_Baseinfo, map[string]interface{}{"name_id": name_id}, update_info)
+		}
+	}
+	if contact.Per != "" && contact.Tel != "" {
+		contact_datas := su.MysqlGlobalTool.Find(su.G_Units_Contact, map[string]interface{}{"name_id": name_id}, "", "", -1, -1)
+		isExists := false
+		for _, v := range *contact_datas {
+			update_id := qu.IntAll(v["id"])
+			person := qu.ObjToString(v["contact_name"])
+			tel := qu.ObjToString(v["contact_tel"])
+			pt_str := qu.ObjToString(v["publishtime"])
+			pt := int64(0)
+			if pt_str != "" {
+				t, _ := time.ParseInLocation(su.TimeLayout, pt_str, time.Local)
+				pt = t.Unix()
+			}
+			if person+"~"+tel == contact.Per+"~"+contact.Tel {
+				isExists = true
+				old_contact_identity := subject.ConvertToBin(subject.Str2DEC(qu.ObjToString(v["identity_type"])))
+				old_contact_identity = subject.SupplementIdentityType(old_contact_identity)
+				if old_contact_identity != identity || qu.IntAll(v["source_type"]) == 2 || (contact.Publishtime > pt && contact.Publishtime > 0) {
+					identity = subject.CalculateIdentityType(old_contact_identity, identity)
+					update_contact := map[string]interface{}{}
+					update_contact["source_type"] = 1
+					update_contact["identity_type"] = qu.IntAll(subject.Str2DEC(identity))
+					update_contact["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+					if contact.Publishtime > pt {
+						update_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+					}
+					//更新通讯录
+					su.MysqlGlobalTool.Update(su.G_Units_Contact, map[string]interface{}{"id": update_id}, update_contact)
+				}
+				break
+			}
+		}
+		if !isExists {
+			add_contact := map[string]interface{}{}
+			add_contact["name_id"] = name_id
+			add_contact["contact_name"] = contact.Per
+			add_contact["contact_tel"] = contact.Tel
+			add_contact["contact_email"] = ""
+			add_contact["source_type"] = 1
+			add_contact["publishtime"] = time.Unix(contact.Publishtime, 0).Format(ul.TimeLayout)
+			add_contact["identity_type"] = 2
+			add_contact["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			//新增~通讯录表
+			su.InsertMysqlData(su.G_Units_Contact, add_contact, tmpid)
+		}
+	}
+}

fieldproject_medical/data_subject/src/subject_sql/savesql.go

@@ -1,26 +1,33 @@
 package subject_sql
 
 import (
+	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
+	"github.com/uuid"
 	qu "qfw/util"
+	"strings"
+	"subject"
 	su "subject_util"
 	"sync"
+	"time"
+	"unicode/utf8"
+	ul "util"
 )
 
 //导出sql数据
 func ExportSqlInfoData() {
-	log.Debug("开始治愈数据~~~企业相关信息~~~")
-	sess := su.SaveMgo.GetMgoConn()
-	defer su.SaveMgo.DestoryMongoConn(sess)
+	log.Debug("开始导入数据~~~tidb~~~")
+	sess := su.SourceMgo.GetMgoConn()
+	defer su.SourceMgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
 	log.Debug("查询语句 ~ ", q)
-	it := sess.DB(su.SaveMgo.DbName).C(su.O_Coll_Name).Find(&q).Sort("_id").Iter()
-	pool := make(chan bool, 10)
+	it := sess.DB(su.SourceMgo.DbName).C(su.O_Coll_Name).Find(&q).Sort("_id").Iter()
+	pool := make(chan bool, 20)
 	wg := &sync.WaitGroup{}
-	total, isok := 0, 0
+	total := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%1000 == 0 {
-			log.Debug("cur index ", total, "~", isok, "~", tmp["_id"])
+			log.Debug("cur index ", total, "~", tmp["_id"])
 		}
 		pool <- true
 		wg.Add(1)
@@ -29,20 +36,88 @@ func ExportSqlInfoData() {
 				<-pool
 				wg.Done()
 			}()
-
 			//处理信息
 			dealWithSqlInfo(tmp)
-
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	wg.Wait()
-	log.Debug("is over ~ ", total, "~", isok)
+	log.Debug("is over ~ ", total)
 }
 
 //处理信息
 func dealWithSqlInfo(tmp map[string]interface{}) {
 	info := map[string]interface{}{}
 	info["name"] = qu.ObjToString(tmp["name"])
+	info["company_id"] = qu.ObjToString(tmp["company_id"])
+	address := qu.ObjToString(tmp["company_address"])
+	if utf8.RuneCountInString(address) > 300 {
+		address = ""
+	}
+	info["address"] = address
+	identity := fmt.Sprintf("%d", qu.IntAll(tmp["identity"]))
+	info["identity_type"] = qu.IntAll(subject.Str2DEC(identity))
+
+	info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+	info_id := uuid.New().String()
+	info_id = strings.ReplaceAll(info_id, "-", "")
+	info["name_id"] = info_id
+	//省市区代码
+	area := qu.ObjToString(tmp["area"])
+	city := qu.ObjToString(tmp["city"])
+	district := qu.ObjToString(tmp["district"])
+	info["area_code"], info["city_code"], info["district_code"] = subject.CalculateRegionCode(area, city, district)
+	b := su.InsertMysqlData(su.G_Units_Baseinfo, info, su.BsonTOStringId(tmp["_id"]))
+	if b > 0 { //存通讯录
+		contact := su.IsMarkInterfaceMap(tmp["contact"])
+		for _, v := range contact {
+			info_book := map[string]interface{}{}
+			info_book["name_id"] = info_id
+			info_book["contact_name"] = qu.ObjToString(v["person"])
+			info_book["contact_tel"] = qu.ObjToString(v["tel"])
+			info_book["contact_email"] = ""
+			info_book["source_type"] = 1
+			identity_book := subject.CreateIdentityBool(v["buyer"].(bool), v["agency"].(bool), v["winner"].(bool))
+			publishtime := qu.Int64All(v["publishtime"])
+			if publishtime > 0 {
+				info_book["publishtime"] = time.Unix(publishtime, 0).Format(ul.TimeLayout)
+			}
+			info_book["identity_type"] = qu.IntAll(subject.Str2DEC(identity_book))
+			info_book["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			su.InsertMysqlData(su.G_Units_Contact, info_book, su.BsonTOStringId(tmp["_id"]))
+		}
 
+		annual := su.IsMarkInterfaceMap(tmp["annual"])
+		for _, v := range annual {
+			info_book := map[string]interface{}{}
+			info_book["name_id"] = info_id
+			info_book["contact_name"] = qu.ObjToString(v["person"])
+			info_book["contact_tel"] = qu.ObjToString(v["tel"])
+			info_book["contact_email"] = qu.ObjToString(v["email"])
+			info_book["source_type"] = 2
+			info_book["identity_type"] = 0
+			date := qu.ObjToString(v["date"])
+			if date != "" {
+				info_book["publishtime"] = qu.ObjToString(v["date"])
+			}
+			info_book["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			su.InsertMysqlData(su.G_Units_Contact, info_book, su.BsonTOStringId(tmp["_id"]))
+		}
+
+		//存主体标签记录~暂时只进采购单位的主体
+		if qu.IntAll(tmp["identity"])%10 == 1 {
+			info_tag := map[string]interface{}{}
+			info_tag["name_id"] = info_id
+			info_tag["labelcode"] = "1"
+			labelvalues := "00"
+			buyerclass := qu.ObjToString(tmp["buyerclass"])
+			if su.BuyerClassData[buyerclass] != "" {
+				labelvalues = su.BuyerClassData[buyerclass]
+			}
+			info_tag["labelvalues"] = labelvalues //代码表
+			info_tag["identity_type"] = 1
+			info_tag["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+			su.InsertMysqlData(su.G_Units_Tags, info_tag, su.BsonTOStringId(tmp["_id"]))
+		}
+	}
 }

fieldproject_medical/data_subject/src/subject_util/initcfg.go

@@ -2,33 +2,53 @@ package subject_util
 
 import (
 	log "github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
 )
 
 var (
-	SaveMgo, SpiMgo *MongodbSim
-	TimeLayout      = "2006-01-02 15:04:05"
-	MysqlTool       *Mysql
-	IsLocal         bool
-	S_Coll_Name     = "bidding"
-	O_Coll_Name     = "zktest_subject_data"
+	SourceMgo, QyxyMgo, SpiMgo *MongodbSim
+	TimeLayout                 = "2006-01-02 15:04:05"
+	MysqlGlobalTool            *Mysql
+	BuyerClassData             = map[string]string{}
+	RegionCodeData             = map[string]string{}
+	IsLocal                    bool
+)
+
+const (
+	S_Coll_Name      = "bidding"
+	O_Coll_Name      = "zktest_subject_data"
+	G_Units_Baseinfo = "dws_f_ent_baseinfo"
+	G_Units_Contact  = "dws_f_ent_contact"
+	G_Units_Tags     = "dws_f_ent_tags"
+	G_Units_Warning  = "dws_f_ent_warning"
 )
 
 func InitClass() {
 	initMgo()
 	initMysql()
+	initVCode()
 }
 
 //初始化mgo
 func initMgo() {
 	if IsLocal {
-		SaveMgo = &MongodbSim{
+		SourceMgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:27017",
+			DbName:      "mixdata",
+			Size:        10,
+			UserName:    "",
+			Password:    "",
+		}
+		SourceMgo.InitPool()
+
+		QyxyMgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
 			DbName:      "mixdata",
 			Size:        10,
 			UserName:    "",
 			Password:    "",
 		}
-		SaveMgo.InitPool()
+		QyxyMgo.InitPool()
 
 		SpiMgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
@@ -39,14 +59,23 @@ func initMgo() {
 		}
 		SpiMgo.InitPool()
 	} else {
-		SaveMgo = &MongodbSim{
+		SourceMgo = &MongodbSim{
+			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+			DbName:      "qfw",
+			Size:        10,
+			UserName:    "zhengkun",
+			Password:    "zk@123123",
+		}
+		SourceMgo.InitPool()
+
+		QyxyMgo = &MongodbSim{
 			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 			DbName:      "mixdata",
 			Size:        10,
 			UserName:    "zhengkun",
 			Password:    "zk@123123",
 		}
-		SaveMgo.InitPool()
+		QyxyMgo.InitPool()
 
 		SpiMgo = &MongodbSim{
 			MongodbAddr: "172.17.4.181:27001",
@@ -67,18 +96,43 @@ func initMysql() {
 		password = "Zk#20220824"
 		address = "172.17.4.242:4000"
 	}
-	MysqlTool = &Mysql{
+	MysqlGlobalTool = &Mysql{
 		Address:  address,
 		UserName: username,
 		PassWord: password,
 		DBName:   "global_common_data",
 	}
-	MysqlTool.Init()
+	MysqlGlobalTool.Init()
+}
+
+//加载代码表
+func initVCode() {
+	data_types := MysqlGlobalTool.Find("code_buyerclass", nil, "", "", -1, -1)
+	//先构建所有一级数据
+	for _, v := range *data_types {
+		name := qu.ObjToString(v["name"])
+		code := qu.ObjToString(v["code"])
+		BuyerClassData[name] = code
+	}
+	BuyerClassData["其它"] = "00"
+	log.Debug("招标行业分类表~", len(BuyerClassData))
+
+	data_regions := MysqlGlobalTool.Find("code_area", nil, "", "", -1, -1)
+	//先构建所有一级数据
+	for _, v := range *data_regions {
+		area := qu.ObjToString(v["area"])
+		city := qu.ObjToString(v["city"])
+		district := qu.ObjToString(v["district"])
+		code := qu.ObjToString(v["code"])
+		key := area + "~" + city + "~" + district + "~"
+		RegionCodeData[key] = code
+	}
+	log.Debug("地域信息数量~", len(RegionCodeData))
 }
 
 //插入数据
 func InsertMysqlData(name string, data map[string]interface{}, mark string) int64 {
-	inb := MysqlTool.Insert(name, data)
+	inb := MysqlGlobalTool.Insert(name, data)
 	if inb == -1 {
 		log.Debug("插入数据异常...", name, "~", mark)
 	}