提交

customerdata/src/datamodel.go

@@ -19,7 +19,7 @@ type Customer struct {
 	IsSearchEnps bool                              //是否查询第三方企业信息
 	TagRules     []*TagRule                        //打标签规则集合
 	SaveDataMap  map[string]map[string]interface{} //最终入库数据(全局模式)
-	SaveDataArr  []map[string]interface{}          //最终入库数据(规则模式)
+	SaveDataArr  map[string]map[string]interface{} //最终入库数据(规则模式)
 	IdRange      bson.M                            //由当前时间生成是id查询区间
 }
 
@@ -30,12 +30,13 @@ type Department struct {
 	CustomerID     string                              //所属客户id
 	Rules          []*SearchRule                       //客户的多个规则
 	DepartmentData map[string][]map[string]interface{} //部门所有数据 key:ruleid value:[{k:dataid v:data},{k:dataid v:data}]
-	DataLock       *sync.Mutex                         //数据锁
-	SaveDataMap    map[string]map[string]interface{}   //最终入库数据(部门内部去重)
+	DataLock    *sync.Mutex                            //数据锁
+	SaveDataMap map[string]map[string]interface{}      //最终入库数据(部门内部去重)
 }
 
 //查询规则模型
-type SearchRule struct { //一条查询规则
+type SearchRule struct {
+	//一条查询规则
 	ID           string
 	Name         string
 	ExtFieldType int                    //规则1、标准字段包，2、高级字段包
@@ -51,7 +52,8 @@ type SearchRule struct { //一条查询规则
 }
 
 //打标签规则模型
-type TagRule struct { //一条打标签规则
+type TagRule struct {
+	//一条打标签规则
 	ID            string
 	Name          string
 	CustomerId    string

customerdata/src/historytask.go

@@ -24,7 +24,7 @@ func HistoryTask(name string) {
 
 		cus := &Customer{}
 		cus.SaveDataMap = map[string]map[string]interface{}{}
-		cus.SaveDataArr = []map[string]interface{}{}
+		cus.SaveDataArr = map[string]map[string]interface{}{}
 		cus.ID = customerId
 		cus.Name = customer
 		cus.PushModel = pushModel

customerdata/src/mainT.go

@@ -7,7 +7,7 @@ import (
 func main1() {
 	cus := &Customer{}
 	cus.IsTagRule = true                                  //是否合并tagname
-	cus.SaveDataArr = []map[string]interface{}{}          //4
+	cus.SaveDataArr = map[string]map[string]interface{}{}          //4
 	cus.SaveDataMap = map[string]map[string]interface{}{} //0
 
 	dm := &Department{}

customerdata/src/task.go

@@ -54,7 +54,7 @@ func GetCustomerData() {
 		log.Println("当前客户 ", customer)
 		cus := &Customer{}
 		cus.SaveDataMap = map[string]map[string]interface{}{}
-		cus.SaveDataArr = []map[string]interface{}{}
+		cus.SaveDataArr = map[string]map[string]interface{}{}
 		cus.IdRange = idRange
 		cus.ID = customerId
 		cus.Name = customer
@@ -373,7 +373,7 @@ func (c *Customer) GetData(stype string) {
 										tagIdArr := MapDataToArr(tagIdMap)
 										if len(tagNameArr) > 0 {
 											tmp["tagname"] = strings.Join(tagNameArr, ",")
-											if DisPackageAppidMap[c.AppId]{
+											if DisPackageAppidMap[c.AppId] {
 												tmp["buyer_type"] = strings.Join(tagNameArr, ",")
 											}
 											if c.PushModel == 2 {
@@ -442,19 +442,35 @@ func (c *Customer) RemoveRepeatData() {
 							c.SaveDataMap[dataId] = tmp
 						} else { //数据重复
 							cus_history := c.SaveDataMap[dataId]
-							MergeData(cus_history, tmp, c.IsTagRule, true) //合并字段
+							MergeData(cus_history, tmp, c.IsTagRule, true, c.PushModel) //合并字段
 							//c.SaveDataMap[dataId] = cus_history
 						}
-					} else if c.PushModel == 1 || c.PushModel == 2 || c.PushModel == 3 { //部门内部去重
+					} else if c.PushModel == 2 || c.PushModel == 3 { //部门内部去重
 						if dm.SaveDataMap[dataId] == nil {
 							dm.SaveDataMap[dataId] = tmp
 						} else { //数据重复
 							dm_history := dm.SaveDataMap[dataId]
-							MergeData(dm_history, tmp, c.IsTagRule, false) //合并字段
+							MergeData(dm_history, tmp, c.IsTagRule, false, c.PushModel) //合并字段
 							//dm.SaveDataMap[dataId] = dm_history
 						}
 					} else if c.PushModel == 4 { //规则模式不去重
-						c.SaveDataArr = append(c.SaveDataArr, tmp)
+						//c.SaveDataArr = append(c.SaveDataArr, tmp)
+						if c.SaveDataArr[dataId] == nil {
+							tmp["itemdist"] = map[string]interface{}{qu.ObjToString(tmp["item"]): qu.ObjToString(tmp["matchkey"])}
+							c.SaveDataArr[dataId] = tmp
+						} else { //数据重复
+							dm_history := c.SaveDataArr[dataId]
+							MergeDatas(dm_history, tmp, c.IsTagRule, false) //合并字段
+							//dm.SaveDataMap[dataId] = dm_history
+						}
+					} else if c.PushModel == 1 {
+						if c.SaveDataMap[dataId] == nil {
+							tmp["itemdist"] = map[string]interface{}{qu.ObjToString(tmp["item"]): qu.ObjToString(tmp["matchkey"])}
+							c.SaveDataMap[dataId] = tmp
+						} else { //数据重复
+							dm_history := c.SaveDataMap[dataId]
+							MergeData(dm_history, tmp, c.IsTagRule, true, c.PushModel) //合并字段
+						}
 					}
 				}
 			}
@@ -471,7 +487,7 @@ func (c *Customer) AssembelAndSaveData() {
 	ch := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
 	n := 0
-	if c.PushModel == 0 && len(c.SaveDataMap) > 0 {
+	if (c.PushModel == 0 || c.PushModel == 1) && len(c.SaveDataMap) > 0 {
 		for _, tmp := range c.SaveDataMap {
 			wg.Add(1)
 			ch <- true
@@ -491,7 +507,7 @@ func (c *Customer) AssembelAndSaveData() {
 			}
 		}
 		wg.Wait()
-	} else if c.PushModel == 1 || c.PushModel == 2 || c.PushModel == 3 {
+	} else if c.PushModel == 2 || c.PushModel == 3 {
 		for _, dm := range c.Departments {
 			if len(dm.SaveDataMap) > 0 {
 				for _, tmp := range dm.SaveDataMap {

customerdata/src/util.go

@@ -401,7 +401,7 @@ func MapDataToArr(tmpMap map[string]bool) (tmpArr []string) {
 }
 
 //合并数据
-func MergeData(history, tmp map[string]interface{}, isTagRule, isDepartRmvRep bool) {
+func MergeData(history, tmp map[string]interface{}, isTagRule, isDepartRmvRep bool, pushModel int) {
 	//matchkey、matchtype、ruleid均非空
 	matchkey1 := qu.ObjToString(history["matchkey"])
 	matchkey2 := qu.ObjToString(tmp["matchkey"])
@@ -419,6 +419,59 @@ func MergeData(history, tmp map[string]interface{}, isTagRule, isDepartRmvRep bo
 	rulename2 := qu.ObjToString(tmp["rulename"])
 	history["rulename"] = MergeField(rulename1, rulename2)
 
+	if isTagRule { //标签模式 tagname、tagid合并
+		tagname1 := qu.ObjToString(history["tagname"])
+		tagname2 := qu.ObjToString(tmp["tagname"])
+		if tagNameResult := MergeField(tagname1, tagname2); tagNameResult != "" {
+			history["tagname"] = tagNameResult
+		}
+
+		tagid1 := qu.ObjToString(history["tagid"])
+		tagid2 := qu.ObjToString(tmp["tagid"])
+		if tagIdResult := MergeField(tagid1, tagid2); tagIdResult != "" {
+			history["tagid"] = tagIdResult
+		}
+	}
+	if isDepartRmvRep { //全局模式 部门合并
+		departname1 := qu.ObjToString(history["departname"])
+		departname2 := qu.ObjToString(tmp["departname"])
+		history["departname"] = MergeField(departname1, departname2)
+		departid1 := qu.ObjToString(history["departid"])
+		departid2 := qu.ObjToString(tmp["departid"])
+		history["departid"] = MergeField(departid1, departid2)
+		if pushModel == 1 {
+			item1 := qu.ObjToString(history["item"])
+			item2 := qu.ObjToString(tmp["item"])
+			history["item"] = MergeField(item1, item2)
+			history["itemdist"].(map[string]interface{})[departname2] = matchkey2
+		}
+	}
+}
+
+func MergeDatas(history, tmp map[string]interface{}, isTagRule, isDepartRmvRep bool) {
+	//matchkey、matchtype、ruleid均非空
+	matchkey1 := qu.ObjToString(history["matchkey"])
+	matchkey2 := qu.ObjToString(tmp["matchkey"])
+	history["matchkey"] = MergeField(matchkey1, matchkey2)
+
+	matchkeytype1 := qu.ObjToString(history["matchtype"])
+	matchkeytype2 := qu.ObjToString(tmp["matchtype"])
+	history["matchtype"] = MergeField(matchkeytype1, matchkeytype2)
+
+	ruleid1 := qu.ObjToString(history["ruleid"])
+	ruleid2 := qu.ObjToString(tmp["ruleid"])
+	history["ruleid"] = MergeField(ruleid1, ruleid2)
+
+	rulename1 := qu.ObjToString(history["rulename"])
+	rulename2 := qu.ObjToString(tmp["rulename"])
+	history["rulename"] = MergeField(rulename1, rulename2)
+
+	item1 := qu.ObjToString(history["item"])
+	item2 := qu.ObjToString(tmp["item"])
+	history["item"] = MergeField(item1, item2)
+	history["itemdist"].(map[string]interface{})[item2] = matchkey2
+	qu.Debug("组装后的item：", history["item"], history["itemdist"])
+
 	if isTagRule { //标签模式 tagname、tagid合并
 		tagname1 := qu.ObjToString(history["tagname"])
 		tagname2 := qu.ObjToString(tmp["tagname"])
@@ -521,7 +574,7 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, a
 						if qu.ObjToString(tmp["toptype"]) == "结果" || qu.ObjToString(tmp["toptype"]) == "其它" || qu.ObjToString(tmp["subtype"]) == "单一" {
 							if vs["winner"] != nil && qu.ObjToString(vs["winner"]) != "" {
 								winner := strings.Split(qu.ObjToString(vs["winner"]), ",")
-								
+
 								for _, w := range winner {
 									// if MactchWinner(w) {
 									if index > 1 {
@@ -529,11 +582,11 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, a
 										for ks, vks := range tmp {
 											s_data[ks] = vks
 										}
-										delete(s_data,"legal_person")
-										delete(s_data,"company_address")
-										delete(s_data,"winner_credit_no")
-										delete(s_data,"company_email")
-										delete(s_data,"company_phone")
+										delete(s_data, "legal_person")
+										delete(s_data, "company_address")
+										delete(s_data, "winner_credit_no")
+										delete(s_data, "company_email")
+										delete(s_data, "company_phone")
 										s_data["s_winner"] = w
 										SearchEnterpriseInfos(s_data, w)
 										GetWinnerType(w, s_data)