xuzhiheng 5 жил өмнө
parent
commit
d836e1fb5c

+ 5 - 1
.gitignore

@@ -10,6 +10,10 @@ bin
 .idea/
 customerdata/.DS_Store
 customerdata/src/.DS_Store
+<<<<<<< HEAD
 customerdata/.idea/
 customerdata/pkg/
-sqlutil/pkg
+sqlutil/pkg
+=======
+.idea
+>>>>>>> dev-ent-sxf

+ 7 - 5
customerdata/src/config.json

@@ -1,7 +1,7 @@
 {
 	"mgodb": "192.168.3.207:27092",
 	"dbsize": 10,
-	"dbname": "datatag",
+	"dbname": "datatag_ent",
 	"tasktime": 1,
 	"latestid": "56388138af53745d9a000000",
 	"buyerinfo": {
@@ -21,12 +21,14 @@
 		"addr": "192.168.3.207:27092",
 		"size": 15,
 		"db": "mxs",
-		"coll": "usermail"
+		"coll": "usermail_0806_03",
+		"coll_project": "usermail_projectid_0806_03"
 	},
 	"es": {
-		"addr": "http://192.168.3.128:9800",
-		"index": "bidding_v1",
+		"addr": "http://127.0.0.1:9800",
+		"index": "bidding_v2",
 		"itype": "bidding",
 		"pool": 15
-	}
+	},
+	"project_appid": "jyOh1XQgUJBQ5bTUlKCyZ1"
 }

+ 4 - 3
customerdata/src/historytask.go

@@ -12,6 +12,7 @@ func HistoryTask(name string) {
 
 	//加载一个客户
 	customer, _ := MgoTag.Find("euser", map[string]interface{}{"s_name": name, "i_push": 1, "b_delete": false}, nil, nil)
+	log.Println("data", customer)
 	if len(customer) == 1 {
 		c := customer[0]
 		customerId := mgoutil.BsonTOStringId(c["_id"])
@@ -35,12 +36,12 @@ func HistoryTask(name string) {
 			}
 		}
 
-		cus.GetTagRules()    //获取客户打标签规则
+		cus.GetTagRules()             //获取客户打标签规则
 		cus.GetDepartments("history") //获取客户信息
 		//PrintLog(cus)        //打印查看初始化的信息
 		qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.Departments))
-		cus.GetData("history")             //获取数据
-		cus.RemoveRepeatData()    //数据去重
+		cus.GetData("history") //获取数据
+		cus.RemoveRepeatData() //数据去重
 
 		cus.AssembelAndSaveData() //组装、保存数据
 	} else {

+ 20 - 5
customerdata/src/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"flag"
+	"log"
 	qu "qfw/util"
 	"qfw/util/elastic"
+	"sync"
+	"time"
 	"util/mgodb"
 
 	"github.com/donnie4w/go-logger/logger"
@@ -11,7 +14,7 @@ import (
 
 var (
 	Sysconfig    map[string]interface{}
-	LatestId     string            //起始id
+	LatestId     string              //起始id
 	MgoTag       *mongodb.MongodbSim //标签库连接
 	MgoSave      *mongodb.MongodbSim //数据保存库连接
 	MgoBuyer     *mongodb.MongodbSim //医院等级信息
@@ -25,12 +28,14 @@ var (
 	Itype        string
 	TaskTime     int64 //定时任务时间
 	//历史数据
-	SId          string
-	EId          string
-	CustomerName string
+	SId             string
+	EId             string
+	CustomerName    string
+	SaveCollProject string
 )
 var MgoSaveCache = make(chan map[string]interface{}, 1000)
 var SP = make(chan bool, 5)
+var projectIdMap sync.Map
 
 func init() {
 	qu.ReadConfig(&Sysconfig)
@@ -45,6 +50,7 @@ func init() {
 	//mgo save
 	save := Sysconfig["save"].(map[string]interface{})
 	SaveColl = qu.ObjToString(save["coll"])
+	SaveCollProject = qu.ObjToString(save["coll_project"])
 	MgoSave = &mongodb.MongodbSim{
 		MongodbAddr: qu.ObjToString(save["addr"]),
 		DbName:      qu.ObjToString(save["db"]),
@@ -93,13 +99,22 @@ func init() {
 }
 func main() {
 	go SaveMgo()
-	go TimeTask() //定时任务
 	//flag.StringVar(&SId, "sid", "", "起始id")
 	//flag.StringVar(&EId, "eid", "", "结束id")
 	flag.StringVar(&CustomerName, "customer", "", "客户名称")
 	flag.Parse()
+	log.Println("customer", CustomerName)
 	if CustomerName != "" {
+		if CustomerName == "徐志恒的" {
+			start := time.Now().Unix()
+			log.Println("加载projectId---开始")
+			InitProjectId()
+			end := time.Now().Unix()
+			log.Println("加载projectId---结束,耗时", end-start, "秒")
+		}
 		go HistoryTask(CustomerName)
+	} else {
+		go TimeTask() //定时任务
 	}
 	ch := make(chan bool, 1)
 	<-ch

+ 28 - 7
customerdata/src/task.go

@@ -3,15 +3,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"gopkg.in/mgo.v2/bson"
 	"log"
 	qu "qfw/util"
 	"regexp"
 	. "sqlmodel"
 	"strings"
 	"sync"
+	"time"
 	mgoutil "util/mgodb"
 
+	"gopkg.in/mgo.v2/bson"
+
 	"github.com/cron"
 
 	"github.com/donnie4w/go-logger/logger"
@@ -65,12 +67,22 @@ func GetCustomerData() {
 				cus.IsSearchEnps = true
 			}
 		}
-
+		//
+		if appId == qu.ObjToString(Sysconfig["project_appid"]) {
+			start := time.Now().Unix()
+			log.Println("加载projectId---开始")
+			InitProjectId()
+			end := time.Now().Unix()
+			log.Println("加载projectId---结束,耗时", end-start, "秒")
+		} else {
+			projectIdMap = sync.Map{}
+		}
+		//
 		cus.GetTagRules()      //获取客户打标签规则
 		cus.GetDepartments("") //获取客户信息
 		//PrintLog(cus)        //打印查看初始化的信息
 		//qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.Departments))
-		cus.GetData("")             //获取数据
+		cus.GetData("")           //获取数据
 		cus.RemoveRepeatData()    //数据去重
 		cus.AssembelAndSaveData() //组装、保存数据
 	}
@@ -261,7 +273,7 @@ func (c *Customer) GetData(stype string) {
 								/*
 									到此已经匹配完数据
 								*/
-								//qu.Debug("---------------------", id, IsMatch, matchKey)
+								qu.Debug("---------------------", id, IsMatch, matchKey)
 								if IsMatch { //匹配成功,数据上新增规则id,matchKey,item并临时保存数据
 									tmpMatchKey := MapDataToArr(matchKey)
 									tmpMatchKeyType := MapDataToArr(matchKeyType)
@@ -411,7 +423,10 @@ func (c *Customer) AssembelAndSaveData() {
 					<-ch
 					wg.Done()
 				}()
-				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+				if !ok {
+					n--
+				}
 			}(tmp)
 			n++
 			if n%500 == 0 {
@@ -430,7 +445,10 @@ func (c *Customer) AssembelAndSaveData() {
 							<-ch
 							wg.Done()
 						}()
-						AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+						ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+						if !ok {
+							n--
+						}
 					}(tmp)
 					n++
 					if n%500 == 0 {
@@ -449,7 +467,10 @@ func (c *Customer) AssembelAndSaveData() {
 					<-ch
 					wg.Done()
 				}()
-				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+				if !ok {
+					n--
+				}
 			}(tmp)
 			n++
 			if n%500 == 0 {

+ 58 - 6
customerdata/src/util.go

@@ -1,14 +1,16 @@
 package main
 
 import (
-	"github.com/donnie4w/go-logger/logger"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"gopkg.in/mgo.v2/bson"
+	"fmt"
 	"log"
 	qu "qfw/util"
 	"regexp"
 	"strings"
 	"time"
+
+	"github.com/donnie4w/go-logger/logger"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
 )
 
 var LetterCase = regexp.MustCompile("[A-Za-z]")
@@ -339,7 +341,7 @@ func RegMatch(fieldText map[string]interface{}, matchType []string, matchReg *Re
 				for n, arc := range ar.AndRegKid { //ar.AndRegKid若有值必不小于2
 					if indexArr := arc.CrVal.FindAllStringIndex(text, -1); len(indexArr) < 1 { //匹配失败(ar.AndRegKid中任意一个未匹配则失败)
 						break
-					} else {                                                         //匹配成功,判断字母
+					} else { //匹配成功,判断字母
 						if arc.IsLetter && !CheckLetter(text, arc.CrVal, indexArr) { //reg有字母,判断是否是包含关系(AAAIBBB or AI)
 							break
 						}
@@ -438,7 +440,7 @@ func MergeField(str1, str2 string) string {
 }
 
 //补充信息并保存
-func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool) {
+func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, appid string) bool {
 	if IsSearchHosp { //医院信息
 		SearchHospInfo(tmp)
 	}
@@ -454,7 +456,21 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool) {
 	if bidopentime, ok := tmp["bidopentime"].(float64); ok && bidopentime > 0 {
 		tmp["bidopentime"] = qu.Int64All(bidopentime)
 	}
+	//
+	if appid == qu.ObjToString(Sysconfig["project_appid"]) {
+		projectId := GetProjectId(id)
+		if projectId != "" {
+			tmp["projectId"] = projectId
+			if _, ok := projectIdMap.Load(projectId); ok {
+				MgoSave.Save(SaveCollProject, tmp)
+				return false
+			}
+			projectIdMap.Store(projectId, true)
+		}
+	}
+	//
 	MgoSaveCache <- tmp
+	return true
 }
 
 //查询第三方医院等级信息和招标客户的社会征信代码
@@ -722,7 +738,7 @@ func RegMatchTest(fieldText map[string]interface{}, matchType []string, matchReg
 					qu.Debug("nnnn---", n, arc.CrVal, arc.IsLetter)
 					if indexArr := arc.CrVal.FindAllStringIndex(text, -1); len(indexArr) < 1 { //匹配失败(ar.AndRegKid中任意一个未匹配则失败)
 						break
-					} else {                                                         //匹配成功,判断字母
+					} else { //匹配成功,判断字母
 						if arc.IsLetter && !CheckLetter(text, arc.CrVal, indexArr) { //reg有字母,判断是否是包含关系(AAAIBBB or AI)
 							break
 						}
@@ -758,3 +774,39 @@ func SkipData(tmp map[string]interface{}) bool {
 	}
 	return false
 }
+
+func GetProjectId(id string) string {
+	query := `{"query": {"bool": {"must": [{"term": {"projectset.ids": "%s"}}],"must_not": [],"should": []}},"size": 1}`
+	querys := fmt.Sprintf(query, id)
+	projectId := ""
+	data := Es.Get("projectset", "projectset", querys)
+	if data != nil && *data != nil && len(*data) > 0 {
+		projectId = qu.ObjToString((*data)[0]["_id"])
+		log.Println("查找到projectId", projectId)
+	}
+	return projectId
+}
+
+func InitProjectId() {
+	session := MgoSave.GetMgoConn()
+	count := 0
+	defer func() {
+		MgoSave.DestoryMongoConn(session)
+		log.Printf("本次共取到%d个projectId\n", count)
+	}()
+	save := Sysconfig["save"].(map[string]interface{})
+	log.Println(Sysconfig["project_appid"])
+	query := map[string]interface{}{"appid": Sysconfig["project_appid"]}
+	field := map[string]interface{}{"projectId": 1}
+	iter := session.DB(qu.ObjToString(save["db"])).C(SaveColl).Find(query).Select(field).Sort("_id").Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		projectId := qu.ObjToString(thisData["projectId"])
+		projectIdMap.Store(projectId, true)
+		count++
+		thisData = map[string]interface{}{}
+	}
+}