深信服医疗事业部

customerdata/src/config.json

@@ -1,7 +1,7 @@
 {
 	"mgodb": "192.168.3.207:27092",
 	"dbsize": 10,
-	"dbname": "datatag",
+	"dbname": "datatag_ent",
 	"tasktime": 1,
 	"latestid": "56388138af53745d9a000000",
 	"buyerinfo": {
@@ -21,11 +21,12 @@
 		"addr": "192.168.3.207:27092",
 		"size": 15,
 		"db": "mxs",
-		"coll": "usermail"
+		"coll": "usermail",
+		"coll_project": "usermail_projectid"
 	},
 	"es": {
 		"addr": "http://192.168.3.128:9800",
-		"index": "bidding_v1",
+		"index": "bidding",
 		"itype": "bidding",
 		"pool": 15
 	}

customerdata/src/historytask.go

@@ -12,6 +12,7 @@ func HistoryTask(name string) {
 
 	//加载一个客户
 	customer, _ := MgoTag.Find("euser", map[string]interface{}{"s_name": name, "i_push": 1, "b_delete": false}, nil, nil)
+	log.Println("data", customer)
 	if len(customer) == 1 {
 		c := customer[0]
 		customerId := mgoutil.BsonTOStringId(c["_id"])
@@ -35,12 +36,12 @@ func HistoryTask(name string) {
 			}
 		}
 
-		cus.GetTagRules()    //获取客户打标签规则
+		cus.GetTagRules()             //获取客户打标签规则
 		cus.GetDepartments("history") //获取客户信息
 		//PrintLog(cus)        //打印查看初始化的信息
 		qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.Departments))
-		cus.GetData("history")             //获取数据
-		cus.RemoveRepeatData()    //数据去重
+		cus.GetData("history") //获取数据
+		cus.RemoveRepeatData() //数据去重
 
 		cus.AssembelAndSaveData() //组装、保存数据
 	} else {

customerdata/src/main.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"flag"
+	"log"
 	qu "qfw/util"
 	"qfw/util/elastic"
+	"sync"
+	"time"
 	"util/mgodb"
 
 	"github.com/donnie4w/go-logger/logger"
@@ -11,7 +14,7 @@ import (
 
 var (
 	Sysconfig    map[string]interface{}
-	LatestId     string            //起始id
+	LatestId     string              //起始id
 	MgoTag       *mongodb.MongodbSim //标签库连接
 	MgoSave      *mongodb.MongodbSim //数据保存库连接
 	MgoBuyer     *mongodb.MongodbSim //医院等级信息
@@ -25,12 +28,14 @@ var (
 	Itype        string
 	TaskTime     int64 //定时任务时间
 	//历史数据
-	SId          string
-	EId          string
-	CustomerName string
+	SId             string
+	EId             string
+	CustomerName    string
+	SaveCollProject string
 )
 var MgoSaveCache = make(chan map[string]interface{}, 1000)
 var SP = make(chan bool, 5)
+var projectIdMap *sync.Map
 
 func init() {
 	qu.ReadConfig(&Sysconfig)
@@ -45,6 +50,7 @@ func init() {
 	//mgo save
 	save := Sysconfig["save"].(map[string]interface{})
 	SaveColl = qu.ObjToString(save["coll"])
+	SaveCollProject = qu.ObjToString(save["coll_project"])
 	MgoSave = &mongodb.MongodbSim{
 		MongodbAddr: qu.ObjToString(save["addr"]),
 		DbName:      qu.ObjToString(save["db"]),
@@ -98,7 +104,15 @@ func main() {
 	//flag.StringVar(&EId, "eid", "", "结束id")
 	flag.StringVar(&CustomerName, "customer", "", "客户名称")
 	flag.Parse()
+	log.Println("customer", CustomerName)
 	if CustomerName != "" {
+		if CustomerName == "深信服医疗事业部" {
+			start := time.Now().Unix()
+			log.Println("加载projectId---开始")
+			InitProjectId()
+			end := time.Now().Unix()
+			log.Println("加载projectId---结束,耗时", end-start, "秒")
+		}
 		go HistoryTask(CustomerName)
 	}
 	ch := make(chan bool, 1)

customerdata/src/task.go

@@ -3,7 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"gopkg.in/mgo.v2/bson"
 	"log"
 	qu "qfw/util"
 	"regexp"
@@ -12,6 +11,8 @@ import (
 	"sync"
 	mgoutil "util/mgodb"
 
+	"gopkg.in/mgo.v2/bson"
+
 	"github.com/cron"
 
 	"github.com/donnie4w/go-logger/logger"
@@ -65,12 +66,22 @@ func GetCustomerData() {
 				cus.IsSearchEnps = true
 			}
 		}
-
+		//
+		if appId == "jyOh1XQgUJBQ5bTUlKCyZ1" {
+			start := time.Now().Unix()
+			log.Println("加载projectId---开始")
+			InitProjectId()
+			end := time.Now().Unix()
+			log.Println("加载projectId---结束,耗时", end-start, "秒")
+		} else {
+			projectIdMap = *sync.Map{}
+		}
+		//
 		cus.GetTagRules()      //获取客户打标签规则
 		cus.GetDepartments("") //获取客户信息
 		//PrintLog(cus)        //打印查看初始化的信息
 		//qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.Departments))
-		cus.GetData("")             //获取数据
+		cus.GetData("")           //获取数据
 		cus.RemoveRepeatData()    //数据去重
 		cus.AssembelAndSaveData() //组装、保存数据
 	}
@@ -411,7 +422,10 @@ func (c *Customer) AssembelAndSaveData() {
 					<-ch
 					wg.Done()
 				}()
-				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+				if !ok {
+					n--
+				}
 			}(tmp)
 			n++
 			if n%500 == 0 {
@@ -430,7 +444,10 @@ func (c *Customer) AssembelAndSaveData() {
 							<-ch
 							wg.Done()
 						}()
-						AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+						ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+						if !ok {
+							n--
+						}
 					}(tmp)
 					n++
 					if n%500 == 0 {
@@ -449,7 +466,10 @@ func (c *Customer) AssembelAndSaveData() {
 					<-ch
 					wg.Done()
 				}()
-				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps)
+				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, c.AppId)
+				if !ok {
+					n--
+				}
 			}(tmp)
 			n++
 			if n%500 == 0 {

customerdata/src/util.go

@@ -1,14 +1,17 @@
 package main
 
 import (
-	"github.com/donnie4w/go-logger/logger"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"gopkg.in/mgo.v2/bson"
+	"fmt"
 	"log"
 	qu "qfw/util"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
+
+	"github.com/donnie4w/go-logger/logger"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/mgo.v2/bson"
 )
 
 var LetterCase = regexp.MustCompile("[A-Za-z]")
@@ -438,7 +441,7 @@ func MergeField(str1, str2 string) string {
 }
 
 //补充信息并保存
-func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool) {
+func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, appid string) bool {
 	if IsSearchHosp { //医院信息
 		SearchHospInfo(tmp)
 	}
@@ -454,7 +457,21 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool) {
 	if bidopentime, ok := tmp["bidopentime"].(float64); ok && bidopentime > 0 {
 		tmp["bidopentime"] = qu.Int64All(bidopentime)
 	}
+	//
+	if appid == "jyOh1XQgUJBQ5bTUlKCyZ1" {
+		projectId := GetProjectId(id)
+		if projectId != "" {
+			tmp["projectId"] = projectId
+			if _, ok := projectIdMap.Load(projectId); ok {
+				MgoSave.Save(SaveCollProject, tmp)
+				return false
+			}
+			projectIdMap.Store(projectId, true)
+		}
+	}
+	//
 	MgoSaveCache <- tmp
+	return true
 }
 
 //查询第三方医院等级信息和招标客户的社会征信代码
@@ -753,8 +770,42 @@ func RegMatchTest(fieldText map[string]interface{}, matchType []string, matchReg
 //发布时间不在范围内(7天)不要这条数据
 func SkipData(tmp map[string]interface{}) bool {
 	comeIn := qu.Int64All(tmp["comeintime"])
-	if qu.Int64All(tmp["publishtime"]) > (comeIn - 7*24*60*60 ) {
+	if qu.Int64All(tmp["publishtime"]) > (comeIn - 7*24*60*60) {
 		return true
 	}
 	return false
-}
+}
+
+func GetProjectId(id string) string {
+	query := `{"query": {"bool": {"must": [{"term": {"projectset.ids": "%s"}}],"must_not": [],"should": []}},"size": 1}`
+	querys := fmt.Sprintf(query, id)
+	projectId := ""
+	data := Es.Get("projectset", "projectset", querys)
+	if data != nil && *data != nil {
+		projectId = qu.ObjToString((*data)[0]["id"])
+	}
+	return projectId
+}
+
+func InitProjectId() {
+	session := MgoSave.GetMgoConn()
+	count := 0
+	defer func() {
+		MgoSave.DestoryMongoConn(session)
+		log.Printf("本次共取到%d个projectId\n", count)
+	}()
+	save := Sysconfig["save"].(map[string]interface{})
+	query := map[string]interface{}{"appid": "jyOh1XQgUJBQ5bTUlKCyZ1"}
+	field := map[string]interface{}{"projectId": 1}
+	iter := session.DB(qu.ObjToString(save["db"])).C(SaveColl).Find(query).Select(field).Sort("_id").Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		projectId := qu.ObjToString(thisData["projectId"])
+		projectIdMap.Store(projectId, true)
+		count++
+		thisData = map[string]interface{}{}
+	}
+}