提交备份

ent_contact/contact_add.go

@@ -0,0 +1,5 @@
+package ent_contact
+
+func aaabbb() {
+
+}

ent_contact/contact_full.go

@@ -0,0 +1,58 @@
+package ent_contact
+
+func aaaa(tmp map[string]interface{}) {
+	//buyer := qu.ObjToString(tmp["buyer"])
+	//agency := qu.ObjToString(tmp["agency"])
+	//winner := qu.ObjToString(tmp["winner"])
+	//owner := qu.ObjToString(tmp["owner"])
+	//s_winner := qu.ObjToString(tmp["s_winner"])
+	//b_per := qu.ObjToString(tmp["buyerperson"])
+	//b_tel := qu.ObjToString(tmp["buyertel"])
+	//if utf8.RuneCountInString(b_tel) > 60 {
+	//	b_tel = ""
+	//}
+	//a_per := qu.ObjToString(tmp["agencyperson"])
+	//a_tel := qu.ObjToString(tmp["agencytel"])
+	//if utf8.RuneCountInString(a_tel) > 60 {
+	//	a_tel = ""
+	//}
+	//w_per := qu.ObjToString(tmp["winnerperson"])
+	//w_tel := qu.ObjToString(tmp["winnertel"])
+	//if utf8.RuneCountInString(w_tel) > 60 {
+	//	w_tel = ""
+	//}
+	//o_per := qu.ObjToString(tmp["project_person"])
+	//o_tel := qu.ObjToString(tmp["project_phone"])
+	//if utf8.RuneCountInString(o_tel) > 60 {
+	//	o_tel = ""
+	//}
+	//
+	//buyerclass := qu.ObjToString(tmp["buyerclass"])
+	//publishtime := qu.Int64All(tmp["publishtime"])
+	//tmpid := ul.BsonTOStringId(tmp["_id"])
+	//winner_arr, winner_bool := ul.SegmentationEntName(winner, s_winner)
+	//
+	//updateLock.Lock()
+	//if buyer != "" && utf8.RuneCountInString(buyer) < 30 {
+	//	dealWithUpdateContact(buyer, buyerclass, &ul.Contact{b_per, b_tel, true, false, false, false, publishtime}, "0001", tmpid)
+	//}
+	////中标单位
+	//for k, v := range winner_arr {
+	//	b := winner_bool[k]
+	//	c := ul.Contact{"", "", false, false, true, false, publishtime}
+	//	if b {
+	//		c.Per = w_per
+	//		c.Tel = w_tel
+	//	}
+	//	dealWithUpdateContact(v, "", &c, "0010", tmpid)
+	//}
+	//
+	//if agency != "" && utf8.RuneCountInString(agency) < 30 {
+	//	dealWithUpdateContact(agency, "", &ul.Contact{a_per, a_tel, false, true, false, false, publishtime}, "0100", tmpid)
+	//}
+	//
+	//if owner != "" && utf8.RuneCountInString(owner) < 30 {
+	//	dealWithUpdateContact(owner, "", &ul.Contact{o_per, o_tel, false, false, false, true, publishtime}, "1000", tmpid)
+	//}
+	//updateLock.Unlock()
+}

ent_legal/legal_add.go

@@ -0,0 +1,127 @@
+package ent_legal
+
+import (
+	ul "data_ent_wuye/ent_util"
+	log "github.com/donnie4w/go-logger/logger"
+	"github.com/google/uuid"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"strings"
+	"sync"
+	"time"
+)
+
+var BidFields = map[string]interface{}{
+	"extracttype": 1, "buyerclass": 1, "publishtime": 1,
+	"buyer": 1, "agency": 1, "winner": 1, "s_winner": 1,
+	"agencytel": 1, "winnertel": 1, "buyertel": 1,
+	"agencyperson": 1, "winnerperson": 1, "buyerperson": 1,
+	"owner": 1, "project_person": 1, "project_phone": 1,
+}
+
+// 正常标讯增量信息····
+func RunLegalInfo(gtid string, lteid string) {
+	log.Debug("增量~~~")
+	sess := ul.SourceMgo.GetMgoConn()
+	defer ul.SourceMgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt":  ul.StringTOBsonId(gtid),
+			"$lte": ul.StringTOBsonId(lteid),
+		},
+	}
+	log.Debug("查询语句 ~ ", q)
+	it := sess.DB(ul.SourceMgo.DbName).C(ul.S_Coll_Name).Find(&q).Sort("_id").Select(BidFields).Iter()
+	pool := make(chan bool, 10)
+	wg := &sync.WaitGroup{}
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			if qu.IntAll(tmp["extracttype"]) != 1 {
+				return
+			}
+			if qu.ObjToString(tmp["buyer"]) == "" &&
+				qu.ObjToString(tmp["agency"]) == "" &&
+				qu.ObjToString(tmp["winner"]) == "" &&
+				qu.ObjToString(tmp["owner"]) == "" {
+				return
+			}
+			//处理构建法人信息
+			CreateLegalInfo(tmp)
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Debug("is over ~ ", total)
+}
+
+func CreateLegalInfo(tmp map[string]interface{}) {
+	//构建法人信息...
+	方法名字ABC("徐氏企业")
+	//winner_arr, winner_bool := ul.SegmentationEntName(winner, s_winner)
+}
+
+func 方法名字ABC(name string) {
+	//法人是否存在更新？
+	//......
+	info := map[string]interface{}{}
+	if info == nil { //新增法人信息
+		base_info := map[string]interface{}{}
+		//法人信息所需要的大部分企业信息
+		qyxy_info := GetQyxyInfo(name)
+		base_info["name"] = name
+		area := qu.ObjToString(qyxy_info["company_area"])
+		city := qu.ObjToString(qyxy_info["company_city"])
+		district := qu.ObjToString(qyxy_info["company_district"])
+		base_info["area_code"], base_info["city_code"], base_info["district_code"] = ul.CalculateRegionCode(area, city, district)
+		name_id := uuid.New().String()
+		name_id = strings.ReplaceAll(name_id, "-", "")
+		base_info["name_id"] = name_id
+		base_info["createtime"] = time.Unix(time.Now().Unix(), 0).Format(ul.TimeLayout)
+		/*
+			···
+			···
+		*/
+		//告知通讯录...
+	} else {
+		//告知通讯录...
+	}
+}
+
+// 创建企业信息
+func GetQyxyInfo(name string) map[string]interface{} {
+	qyxy_info := map[string]interface{}{}
+	dataArr, _ := ul.QyxyMgo.Find("qyxy_std", map[string]interface{}{"company_name": name}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{
+		"_id":              1,
+		"company_name":     1,
+		"company_address":  1,
+		"company_area":     1,
+		"company_city":     1,
+		"company_district": 1,
+		"legal_person":     1,
+		"company_phone":    1,
+		"company_email":    1,
+		"company_type_old": 1,
+		"字段很多":             1,
+	})
+	if len(dataArr) > 0 {
+		qyxy_info = dataArr[0] //补充企业信息
+	} else {
+		data := ul.SpiMgo.FindOne("company_history_name", map[string]interface{}{
+			"history_name": name,
+		})
+		if len(data) > 0 {
+			company_id := qu.ObjToString(data["company_id"])
+			qyxy_info = ul.QyxyMgo.FindOne("qyxy_std", map[string]interface{}{"_id": company_id})
+		}
+	}
+	return qyxy_info
+}

ent_legal/legal_full.go

@@ -0,0 +1,10 @@
+package ent_legal
+
+func 全量方法ABC() {
+	//从现有主题库...直接获取
+
+	//凭安全量企业···查到数据告知通讯录
+
+	//马克信息···
+
+}

ent_util/info.go

@@ -0,0 +1,152 @@
+package ent_util
+
+import (
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"runtime"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+type Info struct {
+	Contact map[string]*Contact
+}
+type Contact struct {
+	Per    string
+	Tel    string
+	Buyer  bool
+	Agency bool
+	Winner bool
+	Owner  bool
+}
+
+// 处理地域代码
+func CalculateRegionCode(area string, city string, district string) (area_code string, city_code string, district_code string) {
+	area_code, city_code, district_code = "000000", "", ""
+	if district != "" {
+		key := area + "~" + city + "~" + district + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			district_code = code
+			city_code = code[:4] + "00"
+			area_code = code[:2] + "0000"
+			return
+		}
+	}
+	if city != "" {
+		key := area + "~" + city + "~" + "" + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			city_code = code
+			area_code = city_code[:2] + "0000"
+			return
+		}
+	}
+	if area != "" {
+		key := area + "~" + "" + "~" + "" + "~"
+		code := RegionCodeData[key]
+		if code != "" {
+			area_code = code
+			return
+		}
+	}
+	return
+}
+
+// 单位分割~去重 中标单位
+func SegmentationEntName(name_1 string, name_2 string) ([]string, []bool) {
+	new_arr := []string{}
+	new_bool := []bool{}
+	key_str := map[string]string{}
+
+	if name_1 != "" {
+		arr_1 := strings.Split(name_1, ",")
+		if len(arr_1) == 1 {
+			arr_1 = strings.Split(name_1, "，")
+		}
+		for _, v := range arr_1 {
+			if v != "" && utf8.RuneCountInString(v) > 2 && utf8.RuneCountInString(v) < 30 {
+				if key_str[v] == "" {
+					new_arr = append(new_arr, v)
+					key_str[v] = v
+					new_bool = append(new_bool, true)
+				}
+			}
+		}
+	}
+	if name_2 != "" {
+		arr_2 := strings.Split(name_2, ",")
+		if len(arr_2) == 1 {
+			arr_2 = strings.Split(name_2, "，")
+		}
+		for _, v := range arr_2 {
+			if v != "" && utf8.RuneCountInString(v) > 2 && utf8.RuneCountInString(v) < 30 {
+				if key_str[v] == "" {
+					new_arr = append(new_arr, v)
+					key_str[v] = v
+					new_bool = append(new_bool, false)
+				}
+			}
+		}
+	}
+	return new_arr, new_bool
+}
+
+// 转二进制
+func Str2DEC(s string) (num int) {
+	l := len(s)
+	for i := l - 1; i >= 0; i-- {
+		num += (int(s[l-i-1]) & 0xf) << uint8(i)
+	}
+	return
+}
+func ConvertToBin(num int) string {
+	s := ""
+	if num == 0 {
+		return "0"
+	}
+	for ; num > 0; num /= 2 {
+		lsb := num % 2
+		s = strconv.Itoa(lsb) + s
+	}
+	return s
+}
+
+func toMegaBytes(bytes uint64) float64 {
+	return float64(bytes) / 1024 / 1024
+}
+func traceMemStats() {
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	var result = make([]float64, 7)
+	result[0] = float64(ms.HeapObjects)
+	result[1] = toMegaBytes(ms.HeapAlloc)
+	result[2] = toMegaBytes(ms.TotalAlloc)
+	result[3] = toMegaBytes(ms.HeapSys)
+	result[4] = toMegaBytes(ms.HeapIdle)
+	result[5] = toMegaBytes(ms.HeapReleased)
+	result[6] = toMegaBytes(ms.HeapIdle - ms.HeapReleased)
+
+	for _, v := range result {
+		fmt.Printf("%.2f\t", v)
+	}
+	fmt.Printf("\n")
+}
+
+func IsMarkInterfaceArr(t interface{}) []string {
+	sub_list := []string{}
+	if list_3, ok_3 := t.([]string); ok_3 {
+		sub_list = list_3
+		return sub_list
+	}
+	if list_1, ok_1 := t.(primitive.A); ok_1 {
+		sub_list = qu.ObjArrToStringArr(list_1)
+	} else {
+		if list_2, ok_2 := t.([]interface{}); ok_2 {
+			sub_list = qu.ObjArrToStringArr(list_2)
+		}
+	}
+	return sub_list
+}

init.go → ent_util/init.go

@@ -1,4 +1,4 @@
-package main
+package ent_util
 
 import (
 	"context"
@@ -6,6 +6,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	log "github.com/donnie4w/go-logger/logger"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"runtime"
 	"time"
@@ -18,15 +19,26 @@ var (
 	IsLocal, IsDev     bool
 	ClickHouseConn     driver.Conn
 	EsClinet           *elastic.Elastic
+	BuyerClassData     = map[string]string{}
+	RegionCodeData     = map[string]string{}
 	TimeLayout         = "2006年01月02日"
 	TimeLayout_New     = "2006-01-02 15:04:05"
 	Url                = "https://www.jianyu360.cn/article/content/%s.html"
 )
 
+const (
+	S_Coll_Name      = "bidding"
+	G_Units_Baseinfo = "dws_f_ent_baseinfo"
+	G_Units_Contact  = "dws_f_ent_contact"
+	G_Units_Tags     = "dws_f_ent_tags"
+	G_Units_Warning  = "dws_f_ent_warning"
+)
+
 func InitGlobalVar() {
 	IsLocal = true
 	initMgo()
 	//initMysql()
+	//initVCode()
 	//initClickHouse()
 	//initEs()
 }
@@ -49,7 +61,6 @@ func initClickHouse() {
 	ClickHouseConn, _ = connectClickhouse()
 }
 
-// 初始化mgo
 // 初始化mgo
 func initMgo() {
 	if IsLocal {
@@ -147,6 +158,32 @@ func initMysql() {
 	MysqlGlobalTool.Init()
 }
 
+// 加载代码表
+func initVCode() {
+	data_types := MysqlGlobalTool.Find("code_buyerclass", nil, "", "", -1, -1)
+	//先构建所有一级数据
+	for _, v := range *data_types {
+
+		name := qu.ObjToString(v["name"])
+		code := qu.ObjToString(v["code"])
+		BuyerClassData[name] = code
+	}
+	BuyerClassData["其它"] = "00"
+	log.Debug("招标行业分类表~", len(BuyerClassData))
+
+	data_regions := MysqlGlobalTool.Find("code_area", nil, "", "", -1, -1)
+	//先构建所有一级数据
+	for _, v := range *data_regions {
+		area := qu.ObjToString(v["area"])
+		city := qu.ObjToString(v["city"])
+		district := qu.ObjToString(v["district"])
+		code := qu.ObjToString(v["code"])
+		key := area + "~" + city + "~" + district + "~"
+		RegionCodeData[key] = code
+	}
+	log.Debug("地域信息数量~", len(RegionCodeData))
+}
+
 // 创建clickhouse连接
 func connectClickhouse() (driver.Conn, error) {
 	var (
@@ -183,7 +220,7 @@ func connectClickhouse() (driver.Conn, error) {
 	return conn, nil
 }
 
-// 暂时无用...
+// ...
 func GetNewInfo(index, query string) map[string]interface{} {
 	//log.Println("query  -- ", query)
 	client := EsClinet.GetEsConn()

mgo.go → ent_util/mgo.go

@@ -1,4 +1,4 @@
-package main
+package ent_util
 
 import (
 	"context"

mysql.go → ent_util/mysql.go

@@ -1,4 +1,4 @@
-package main
+package ent_util
 
 import (
 	"bytes"

main.go

@@ -2,12 +2,13 @@ package main
 
 import (
 	"context"
+	"data_ent_wuye/ent_util"
 	log "github.com/donnie4w/go-logger/logger"
 	_ "github.com/gogf/gf/contrib/drivers/clickhouse/v2"
 )
 
 func init() {
-	InitGlobalVar()
+	ent_util.InitGlobalVar()
 }
 
 func main() {
@@ -20,7 +21,7 @@ func main() {
 // 以下测试...
 func test1() {
 	query := `SELECT id FROM information.information WHERE id = 'c2e076d8631741e9b85432ff060f2151'`
-	rows, err := ClickHouseConn.Query(context.Background(), query)
+	rows, err := ent_util.ClickHouseConn.Query(context.Background(), query)
 	if err != nil {
 		log.Debug(err)
 	}
@@ -35,7 +36,7 @@ func test1() {
 			log.Fatal(err)
 		}
 		log.Debug(id)
-		err := ClickHouseConn.Exec(context.Background(), "alter table information.information UPDATE endtime = 1738252800 where id = ?", id)
+		err := ent_util.ClickHouseConn.Exec(context.Background(), "alter table information.information UPDATE endtime = 1738252800 where id = ?", id)
 		if err != nil {
 			log.Debug("update err:", err)
 		}
@@ -48,11 +49,11 @@ func test() {
 
 	query := `ALTER TABLE information.information DELETE WHERE id='676470119ae64a18bab5d1fdb5f06bb3' `
 	query = `TRUNCATE TABLE information.information_copy;`
-	err := ClickHouseConn.Exec(context.Background(), query)
+	err := ent_util.ClickHouseConn.Exec(context.Background(), query)
 	if err != nil {
 		log.Debug(err)
 	}
-	rows, err := ClickHouseConn.Query(context.Background(), "select bitmapToArray(tag_bitmap) tag_bitmap from information.information limit 10")
+	rows, err := ent_util.ClickHouseConn.Query(context.Background(), "select bitmapToArray(tag_bitmap) tag_bitmap from information.information limit 10")
 	if err != nil {
 		log.Debug(err)
 	}