马克数据...

ent_contact/contact_add.go

@@ -1,5 +1,58 @@
 package ent_contact
 
 func aaabbb() {
-
+	//buyer := qu.ObjToString(tmp["buyer"])
+	//agency := qu.ObjToString(tmp["agency"])
+	//winner := qu.ObjToString(tmp["winner"])
+	//owner := qu.ObjToString(tmp["owner"])
+	//s_winner := qu.ObjToString(tmp["s_winner"])
+	//b_per := qu.ObjToString(tmp["buyerperson"])
+	//b_tel := qu.ObjToString(tmp["buyertel"])
+	//if utf8.RuneCountInString(b_tel) > 60 {
+	//	b_tel = ""
+	//}
+	//a_per := qu.ObjToString(tmp["agencyperson"])
+	//a_tel := qu.ObjToString(tmp["agencytel"])
+	//if utf8.RuneCountInString(a_tel) > 60 {
+	//	a_tel = ""
+	//}
+	//w_per := qu.ObjToString(tmp["winnerperson"])
+	//w_tel := qu.ObjToString(tmp["winnertel"])
+	//if utf8.RuneCountInString(w_tel) > 60 {
+	//	w_tel = ""
+	//}
+	//o_per := qu.ObjToString(tmp["project_person"])
+	//o_tel := qu.ObjToString(tmp["project_phone"])
+	//if utf8.RuneCountInString(o_tel) > 60 {
+	//	o_tel = ""
+	//}
+	//
+	//buyerclass := qu.ObjToString(tmp["buyerclass"])
+	//publishtime := qu.Int64All(tmp["publishtime"])
+	//tmpid := ul.BsonTOStringId(tmp["_id"])
+	//winner_arr, winner_bool := ul.SegmentationEntName(winner, s_winner)
+	//
+	//updateLock.Lock()
+	//if buyer != "" && utf8.RuneCountInString(buyer) < 30 {
+	//	dealWithUpdateContact(buyer, buyerclass, &ul.Contact{b_per, b_tel, true, false, false, false, publishtime}, "0001", tmpid)
+	//}
+	////中标单位
+	//for k, v := range winner_arr {
+	//	b := winner_bool[k]
+	//	c := ul.Contact{"", "", false, false, true, false, publishtime}
+	//	if b {
+	//		c.Per = w_per
+	//		c.Tel = w_tel
+	//	}
+	//	dealWithUpdateContact(v, "", &c, "0010", tmpid)
+	//}
+	//
+	//if agency != "" && utf8.RuneCountInString(agency) < 30 {
+	//	dealWithUpdateContact(agency, "", &ul.Contact{a_per, a_tel, false, true, false, false, publishtime}, "0100", tmpid)
+	//}
+	//
+	//if owner != "" && utf8.RuneCountInString(owner) < 30 {
+	//	dealWithUpdateContact(owner, "", &ul.Contact{o_per, o_tel, false, false, false, true, publishtime}, "1000", tmpid)
+	//}
+	//updateLock.Unlock()
 }

ent_contact/contact_full.go

@@ -1,58 +1,110 @@
 package ent_contact
 
-func aaaa(tmp map[string]interface{}) {
-	//buyer := qu.ObjToString(tmp["buyer"])
-	//agency := qu.ObjToString(tmp["agency"])
-	//winner := qu.ObjToString(tmp["winner"])
-	//owner := qu.ObjToString(tmp["owner"])
-	//s_winner := qu.ObjToString(tmp["s_winner"])
-	//b_per := qu.ObjToString(tmp["buyerperson"])
-	//b_tel := qu.ObjToString(tmp["buyertel"])
-	//if utf8.RuneCountInString(b_tel) > 60 {
-	//	b_tel = ""
-	//}
-	//a_per := qu.ObjToString(tmp["agencyperson"])
-	//a_tel := qu.ObjToString(tmp["agencytel"])
-	//if utf8.RuneCountInString(a_tel) > 60 {
-	//	a_tel = ""
-	//}
-	//w_per := qu.ObjToString(tmp["winnerperson"])
-	//w_tel := qu.ObjToString(tmp["winnertel"])
-	//if utf8.RuneCountInString(w_tel) > 60 {
-	//	w_tel = ""
-	//}
-	//o_per := qu.ObjToString(tmp["project_person"])
-	//o_tel := qu.ObjToString(tmp["project_phone"])
-	//if utf8.RuneCountInString(o_tel) > 60 {
-	//	o_tel = ""
-	//}
-	//
-	//buyerclass := qu.ObjToString(tmp["buyerclass"])
-	//publishtime := qu.Int64All(tmp["publishtime"])
-	//tmpid := ul.BsonTOStringId(tmp["_id"])
-	//winner_arr, winner_bool := ul.SegmentationEntName(winner, s_winner)
-	//
-	//updateLock.Lock()
-	//if buyer != "" && utf8.RuneCountInString(buyer) < 30 {
-	//	dealWithUpdateContact(buyer, buyerclass, &ul.Contact{b_per, b_tel, true, false, false, false, publishtime}, "0001", tmpid)
-	//}
-	////中标单位
-	//for k, v := range winner_arr {
-	//	b := winner_bool[k]
-	//	c := ul.Contact{"", "", false, false, true, false, publishtime}
-	//	if b {
-	//		c.Per = w_per
-	//		c.Tel = w_tel
+import (
+	"context"
+	ul "data_ent_wuye/ent_util"
+	log "github.com/donnie4w/go-logger/logger"
+	qu "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"time"
+)
+
+// from tidb
+func InjectContactTidbInfo(tmp map[string]interface{}) {
+	index, total := 0, 0
+	query := map[string]interface{}{}
+L:
+	for {
+		dataArr := ul.MysqlGlobalTool.Find(ul.G_Units_Baseinfo, query, "", "id", index*50000, 50000)
+		if dataArr != nil {
+			if len(*dataArr) == 0 {
+				break
+			}
+			//for _, v := range *dataArr {
+			//	tel :=
+			//}
+		} else {
+			break L
+		}
+		index++
+		total += len(*dataArr)
+		if index%5 == 0 {
+			log.Debug("cur idx ", total)
+		}
+	}
+}
+
+// from 凭安库
+func InjectContactPingAnInfo(tmp map[string]interface{}) {
+	//name := qu.ObjToString(tmp["company_name"])
+	//phone := qu.ObjToString(tmp["company_phone"])
+	//email := qu.ObjToString(tmp["company_email"])
+	//createtime := time.Now().Format(ul.TimeLayout_New)
+	////查询当前通讯录是否有数据
+	//if phone != "" {
+	//	if !QueryingEntContact(uid,name,phone) {
+	//		query := "INSERT INTO information.ent_contact(id,phone,name,email,create_time,update_time) VALUES(?,?,?,?,?,?)"
+	//		//插入数据
+	//		err := ul.ClickHouseConn.Exec(context.Background(), query, uid, phone, name, email, createtime, createtime)
+	//		if err != nil {
+	//			log.Debug(err)
+	//		}
 	//	}
-	//	dealWithUpdateContact(v, "", &c, "0010", tmpid)
-	//}
-	//
-	//if agency != "" && utf8.RuneCountInString(agency) < 30 {
-	//	dealWithUpdateContact(agency, "", &ul.Contact{a_per, a_tel, false, true, false, false, publishtime}, "0100", tmpid)
 	//}
-	//
-	//if owner != "" && utf8.RuneCountInString(owner) < 30 {
-	//	dealWithUpdateContact(owner, "", &ul.Contact{o_per, o_tel, false, false, false, true, publishtime}, "1000", tmpid)
-	//}
-	//updateLock.Unlock()
+}
+
+// from 马克
+func InjectContactMaKeInfo(uid string, tmp map[string]interface{}) {
+	name := qu.ObjToString(tmp["company_name"])
+	phone := qu.ObjToString(tmp["company_phone"])
+	email := qu.ObjToString(tmp["company_email"])
+	source_type := 2
+	createtime := time.Now().Unix()
+	//查询当前通讯录是否有数据
+	if phone != "" {
+		msgs := GetEntContacts(uid)
+		if !QueryingExists(msgs, name, phone) {
+			query := "INSERT INTO information.ent_contact(id,phone,name,email,source_type,create_time,update_time) VALUES(?,?,?,?,?,?,?)"
+			//插入数据
+			err := ul.ClickHouseConn.Exec(context.Background(), query, uid, phone, name, email, source_type, createtime, createtime)
+			if err != nil {
+				log.Debug(err)
+			}
+		}
+	}
+}
+
+func GetEntContacts(uid string) []Msg {
+	query := `SELECT id,name,phone FROM information.ent_contact WHERE id = ` + "'" + uid + "'"
+	rows, err := ul.ClickHouseConn.Query(context.Background(), query)
+	if err != nil {
+		log.Debug(err)
+		return []Msg{}
+	}
+	msgs := make([]Msg, 0)
+	for rows.Next() {
+		var msg Msg
+		rows.Scan(
+			&msg.id,
+			&msg.name,
+			&msg.phone,
+		)
+		msgs = append(msgs, msg)
+	}
+	return msgs
+}
+
+// 是否存在
+func QueryingExists(ects []Msg, name string, phone string) bool {
+	for _, msg := range ects {
+		if name == msg.name && phone == msg.phone {
+			return true
+		}
+	}
+	return false
+}
+
+type Msg struct {
+	id    string
+	name  string
+	phone string
 }

ent_legal/legal_add.go

@@ -30,7 +30,7 @@ func RunLegalInfo(gtid string, lteid string) {
 		},
 	}
 	log.Debug("查询语句 ~ ", q)
-	it := sess.DB(ul.SourceMgo.DbName).C(ul.S_Coll_Name).Find(&q).Sort("_id").Select(BidFields).Iter()
+	it := sess.DB(ul.SourceMgo.DbName).C("").Find(&q).Sort("_id").Select(BidFields).Iter()
 	pool := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 	total := 0

ent_util/info.go

@@ -150,3 +150,36 @@ func IsMarkInterfaceArr(t interface{}) []string {
 	}
 	return sub_list
 }
+
+// 获取最优企业
+func GetOneBaseCompany(arr []map[string]interface{}) int {
+	index := 0
+	flag := 100
+	if len(arr) > 1 {
+		for k, tmp := range arr {
+			use_flag := qu.IntAll(tmp["use_flag"])
+			if use_flag == 0 {
+				index = k
+				break
+			} else {
+				if use_flag < flag {
+					flag = use_flag
+					index = k
+				}
+			}
+		}
+	}
+	return index
+}
+
+// 清洗日期~
+func cleanErrDateTime(datetime string) bool {
+	arr := strings.Split(datetime, "-")
+	if len(arr) > 0 {
+		year := qu.IntAll(arr[0])
+		if year < 1900 || year > 2200 {
+			return true
+		}
+	}
+	return false
+}

ent_util/init.go

@@ -27,7 +27,6 @@ var (
 )
 
 const (
-	S_Coll_Name      = "bidding"
 	G_Units_Baseinfo = "dws_f_ent_baseinfo"
 	G_Units_Contact  = "dws_f_ent_contact"
 	G_Units_Tags     = "dws_f_ent_tags"
@@ -39,7 +38,7 @@ func InitGlobalVar() {
 	initMgo()
 	//initMysql()
 	//initVCode()
-	//initClickHouse()
+	initClickHouse()
 	//initEs()
 }
 
@@ -194,19 +193,34 @@ func connectClickhouse() (driver.Conn, error) {
 		*/
 
 		conn, err = clickhouse.Open(&clickhouse.Options{
-			Addr:         []string{"cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"},
+			Addr:         []string{"192.168.3.207:19000"},
 			DialTimeout:  10 * time.Second,
 			MaxIdleConns: 3,
 			MaxOpenConns: 30,
 			Auth: clickhouse.Auth{
 				Database: "information",
-				Username: "biservice",
-				Password: "Bi_top95215#",
+				Username: "jytop",
+				Password: "pwdTopJy123",
 			},
 			Debugf: func(format string, v ...interface{}) {
 				fmt.Printf(format, v)
 			},
 		})
+
+		//conn, err = clickhouse.Open(&clickhouse.Options{
+		//	Addr:         []string{"cc-2ze9tv451wov14w9e.public.clickhouse.ads.aliyuncs.com:9000"},
+		//	DialTimeout:  10 * time.Second,
+		//	MaxIdleConns: 3,
+		//	MaxOpenConns: 30,
+		//	Auth: clickhouse.Auth{
+		//		Database: "information",
+		//		Username: "biservice",
+		//		Password: "Bi_top95215#",
+		//	},
+		//	Debugf: func(format string, v ...interface{}) {
+		//		fmt.Printf(format, v)
+		//	},
+		//})
 	)
 	if err != nil {
 		return nil, err

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"data_ent_wuye/ent_contact"
 	"data_ent_wuye/ent_util"
 	log "github.com/donnie4w/go-logger/logger"
 	_ "github.com/gogf/gf/contrib/drivers/clickhouse/v2"
@@ -13,33 +14,38 @@ func init() {
 
 func main() {
 	log.Debug("main...")
+	ent_contact.InjectContactMaKeInfo("12345", map[string]interface{}{"company_phone": "158···"})
 
+	log.Debug("......")
 	lock := make(chan bool)
 	<-lock
 }
 
 // 以下测试...
 func test1() {
-	query := `SELECT id FROM information.information WHERE id = 'c2e076d8631741e9b85432ff060f2151'`
+	query := `SELECT id,title FROM information.information WHERE id = '000fcf377e334bcc9380b921df93c268'`
 	rows, err := ent_util.ClickHouseConn.Query(context.Background(), query)
 	if err != nil {
 		log.Debug(err)
 	}
 	isok := 0
+	type TTT struct {
+		title string
+		id    string
+	}
 	for rows.Next() {
-		var (
-			id string
-		)
+		var T TTT
 		if err := rows.Scan(
-			&id,
+			&T.id,
+			&T.title,
 		); err != nil {
 			log.Fatal(err)
 		}
-		log.Debug(id)
-		err := ent_util.ClickHouseConn.Exec(context.Background(), "alter table information.information UPDATE endtime = 1738252800 where id = ?", id)
-		if err != nil {
-			log.Debug("update err:", err)
-		}
+		log.Debug(T)
+		//err := ent_util.ClickHouseConn.Exec(context.Background(), "alter table information.information UPDATE endtime = 1738252800 where id = ?", id)
+		//if err != nil {
+		//	log.Debug("update err:", err)
+		//}
 		isok++
 	}
 	log.Debug("总计数量", isok)
@@ -48,7 +54,7 @@ func test1() {
 func test() {
 
 	query := `ALTER TABLE information.information DELETE WHERE id='676470119ae64a18bab5d1fdb5f06bb3' `
-	query = `TRUNCATE TABLE information.information_copy;`
+	query = `-- TRUNCATE TABLE information.information_copy;`
 	err := ent_util.ClickHouseConn.Exec(context.Background(), query)
 	if err != nil {
 		log.Debug(err)