Merge branch 'dev2.11' into dev2.1

* dev2.11:
  xg
  Revert "Revert "xg""
  Revert "xg"

CMPlatform/config.json

@@ -225,7 +225,7 @@
   "need_projectid_appid": [
     "jyFApXQQIEAw5TTUZOMBpD"
   ],
-  "redis_addrs": "datag=192.168.3.11:1712,session=192.168.3.11:1712,other=192.168.3.11:1712",
+  "redis_addrs": "datag=192.168.3.11:1712,session=192.168.3.11:1712,other=192.168.3.11:1712,ent=192.168.3.11:1712",
   "jyPushMysql": {
     "username": "root",
     "password": "Topnet123",

CMPlatform/history/datamodel.go

@@ -49,6 +49,16 @@ type Customer struct {
 	IdRange      bson.M                            //由当前时间生成是id查询区间
 	DataSave     string                            //临时存储
 	Exact        int                               //精准筛选服务开关
+
+	tempChan       chan *tempData
+	tempChanOver   chan bool
+	saveBeforeChan chan *tempData
+}
+
+type tempData struct {
+	dm   *Department
+	rule *SearchRule
+	data map[string]interface{}
 }
 
 // 部门模型
@@ -76,7 +86,8 @@ type SearchRule struct {
 	KW           []*KeyWord             //多组关键词(与附加词一一对应)
 	AW           []*AddWord             //多组附加词(与关键词一一对应)
 	Maths        []map[string]string
-	ExactRule    string //精准筛选规则
+	ExactRule    string   //精准筛选规则
+	NameArr      []string // 精准筛选规则
 }
 
 // 打标签规则模型

CMPlatform/history/historytask.go

@@ -83,6 +83,10 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		cus := &Customer{}
 		cus.SaveDataMap = map[string]map[string]interface{}{}
 		cus.SaveDataArr = []map[string]interface{}{}
+		cus.tempChan = make(chan *tempData, 2000)
+		cus.tempChanOver = make(chan bool)
+		cus.saveBeforeChan = make(chan *tempData, 2000)
+
 		cus.ID = customerId
 		cus.Name = customer_name
 		cus.PushModel = pushModel
@@ -153,9 +157,10 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		cus.GetTagRules(tag_rules, tag_rules2, tag_rules3) //获取客户打标签规则
 		cus.GetDepartments("history", dep_rules)           //获取客户信息
 		log.Debug(fmt.Sprintf("customer: %s, %s, %d, %s, %t, %t, %t, %t, %t, %d, %d, %d, %d", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsTagRule2, cus.IsTagRule3, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.TagRules2), len(cus.TagRules3), len(cus.Departments)))
-		cus.GetData("history", i_dataSource)                                                              //获取数据
-		cus.RemoveRepeatData()                                                                            //数据去重
-		cus.AssembelAndSaveData(history_id, isFilter, noticeFilter, dataTable, entId, iContact, xlsxData) //组装、保存数据
+		cus.GetData("history", i_dataSource) //获取数据
+		//cus.RemoveRepeatData()               //数据去重
+		cus.saveBeforeData(history_id, isFilter, noticeFilter, dataTable, entId, iContact, xlsxData)
+		//cus.AssembelAndSaveData(history_id, isFilter, noticeFilter, dataTable, entId, iContact, xlsxData) //组装、保存数据
 		if dataTable == 2 {
 			//使用pc商机管理的企业，数据存入mysql表，扣除数据量
 			if xlsxData.xlsxCount > 0 {

CMPlatform/history/task.go

@@ -20,99 +20,72 @@ import (
 	"time"
 )
 
+type mergeData struct {
+	stype string // new/update
+
+	data map[string]interface{}
+}
+
 // 获取客户打标签规则
 func (c *Customer) GetTagRules(tagRules, tagRules2, tagRules3 []map[string]interface{}) {
 	log.Debug("开始加载标签规则...")
 	defer common.Catch()
 	if len(tagRules) > 0 {
-		c.IsTagRule = true //查到打标签规则，表示打标签
-		for _, tr := range tagRules {
-			TR := &TagRule{}
-			TR.Fields = make(map[string]interface{})
-			TR.DepartRuleIds = make(map[string]bool)
-			id := mongodb.BsonIdToSId(tr["_id"])
-			name := common.ObjToString(tr["s_name"])
-			TR.ID = id
-			TR.Name = name
-			TR.CustomerId = c.ID
-			//部门规则id组
-			if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
-				for _, drid := range strings.Split(departRuleIds, ",") {
-					TR.DepartRuleIds[drid] = true
-				}
-			}
-
-			o_list, ok := tr["o_list"].([]interface{})
-			if !ok {
-				o_list = []interface{}{}
-			}
-			o_lists := common.ObjArrToMapArr(o_list)
-			//规则
-			if len(o_list) > 0 {
-				TR.GetKeyAddNotKeyWord(o_lists)
-			}
-			c.TagRules = append(c.TagRules, TR)
-		}
+		IsTagRule(c, 1, tagRules)
 	}
 	if len(tagRules2) > 0 {
-		c.IsTagRule2 = true //查到打标签规则，表示打标签
-		for _, tr := range tagRules2 {
-			TR := &TagRule{}
-			TR.Fields = make(map[string]interface{})
-			TR.DepartRuleIds = make(map[string]bool)
-			id := mongodb.BsonIdToSId(tr["_id"])
-			name := common.ObjToString(tr["s_name"])
-			TR.ID = id
-			TR.Name = name
-			TR.CustomerId = c.ID
-			//部门规则id组
-			if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
-				for _, drid := range strings.Split(departRuleIds, ",") {
-					TR.DepartRuleIds[drid] = true
-				}
-			}
-
-			o_list, ok := tr["o_list"].([]interface{})
-			if !ok {
-				o_list = []interface{}{}
-			}
-			o_lists := common.ObjArrToMapArr(o_list)
-			//规则
-			if len(o_list) > 0 {
-				TR.GetKeyAddNotKeyWord(o_lists)
-			}
-			c.TagRules2 = append(c.TagRules2, TR)
-		}
+		IsTagRule(c, 2, tagRules2)
 	}
 	if len(tagRules3) > 0 {
-		c.IsTagRule3 = true //查到打标签规则，表示打标签
-		for _, tr := range tagRules3 {
-			TR := &TagRule{}
-			TR.Fields = make(map[string]interface{})
-			TR.DepartRuleIds = make(map[string]bool)
-			id := mongodb.BsonIdToSId(tr["_id"])
-			name := common.ObjToString(tr["s_name"])
-			TR.ID = id
-			TR.Name = name
-			TR.CustomerId = c.ID
-			//部门规则id组
-			if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
-				for _, drid := range strings.Split(departRuleIds, ",") {
-					TR.DepartRuleIds[drid] = true
-				}
-			}
+		IsTagRule(c, 3, tagRules3)
+	}
+}
 
-			o_list, ok := tr["o_list"].([]interface{})
-			if !ok {
-				o_list = []interface{}{}
-			}
-			o_lists := common.ObjArrToMapArr(o_list)
-			//规则
-			if len(o_list) > 0 {
-				TR.GetKeyAddNotKeyWord(o_lists)
+func IsTagRule(c *Customer, tag int, tagRules []map[string]interface{}) {
+	//查到打标签规则，表示打标签
+	switch tag {
+	case 1:
+		c.IsTagRule = true
+	case 2:
+		c.IsTagRule2 = true
+	case 3:
+		c.IsTagRule3 = true
+	}
+	for _, tr := range tagRules {
+		TR := &TagRule{}
+		TR.Fields = make(map[string]interface{})
+		TR.DepartRuleIds = make(map[string]bool)
+		id := mongodb.BsonIdToSId(tr["_id"])
+		name := common.ObjToString(tr["s_name"])
+		TR.ID = id
+		TR.Name = name
+		TR.CustomerId = c.ID
+		//部门规则id组
+		if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
+			for _, drid := range strings.Split(departRuleIds, ",") {
+				TR.DepartRuleIds[drid] = true
 			}
+		}
+
+		o_list, ok := tr["o_list"].([]interface{})
+		if !ok {
+			o_list = []interface{}{}
+		}
+		o_lists := common.ObjArrToMapArr(o_list)
+		//规则
+		if len(o_list) > 0 {
+			TR.GetKeyAddNotKeyWord(o_lists)
+		}
+		// 标签规则
+		switch tag {
+		case 1:
+			c.TagRules = append(c.TagRules, TR)
+		case 2:
+			c.TagRules2 = append(c.TagRules2, TR)
+		case 3:
 			c.TagRules3 = append(c.TagRules3, TR)
 		}
+
 	}
 }
 
@@ -187,6 +160,9 @@ func (c *Customer) GetData(stype string, dataSource int) {
 }
 
 func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.EsV7) {
+
+	go c.processedData()
+
 	client := esCon.GetEsConn()
 	defer esCon.DestoryEsConn(client)
 	ctx, _ := context.WithTimeout(context.Background(), 3*time.Hour)
@@ -204,6 +180,18 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 			if escount == 0 {
 				continue
 			}
+			//精准筛选规则2022-10-19
+			if c.Exact == 1 && sr.ExactRule != "" {
+				nameArr := []string{}
+				data, ok := util.Mgo.Find("groups", map[string]interface{}{"ruleId": sr.ID}, nil, nil, false, -1, -1)
+				if ok && data != nil && len(*data) > 0 {
+					for _, v := range *data {
+						nameArr = append(nameArr, common.ObjToString(v["name"]))
+					}
+				}
+				sr.NameArr = nameArr
+			}
+
 			//查询条件类型转换
 			// var q esV7.Query
 			// tmpQuery := esV7.BoolQuery{
@@ -257,360 +245,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 										return
 									}
 								}
-								id := common.ObjToString(tmp["id"])
-								//亚信
-								if CheckBidOpenAppidMap[c.AppId] {
-									if tmp["bidopentime"] != nil {
-										bidopentime := common.Int64All(tmp["bidopentime"])
-										comeintime := common.Int64All(tmp["comeintime"])
-										if bidopentime-comeintime <= 7*24*60*60 {
-											log.Debug(fmt.Sprintf("跳过该条数据，开标时间-入库时间<=7天，%s", id))
-											return
-										}
-									}
-								}
-								//河南移动，过滤掉中国移动采购网招标数据
-								if CheckBidHrefRuleIdMap[dm.ID] {
-									if strings.Contains(common.ObjToString(tmp["href"]), "b2b.10086.cn") {
-										log.Debug(fmt.Sprintf("跳过该条数据，公告原网址中包含 b2b.10086.cn，%s", id))
-										return
-									}
-								}
-								//
-								tmp["id"] = id //记录数据原有id
-								delete(tmp, "_id")
-								if sr.ExtFieldType == 2 {
-									findwinner := ""
-									s_winner := strings.Split(common.ObjToString(tmp["s_winner"]), ",")
-									if len(s_winner) > 0 {
-										for i := 0; i < len(s_winner); i++ {
-											findwinners := strings.TrimSpace(s_winner[i])
-											if findwinners != "" {
-												for _, v := range util.Sysconfig["s_winner_filter"].([]interface{}) {
-													strings.ReplaceAll(findwinners, v.(string), "")
-												}
-												if findwinners != "" {
-													findwinner = findwinners
-													break
-												}
-											}
-										}
-									}
-									// findwinner := strings.TrimSpace(common.ObjToString(tmp["winner"]))
-									if findwinner != "" {
-										finddata, _ := util.MgoEnps.FindOne(util.EnpsColl, bson.M{"company_name": findwinner})
-										if finddata != nil {
-											if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
-												tmp["legal_person"] = legal_person
-											}
-											if email := common.ObjToString((*finddata)["company_email"]); email != "" {
-												tmp["company_email"] = email
-											}
-											if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
-												tmp["company_phone"] = phone
-											}
-										}
-									}
-								}
-								matchKey := map[string]bool{}     //记录所有匹配上的关键词
-								matchKeyType := map[string]bool{} //记录关键词对应的匹配方式
-								//先获取用到的所有字段值
-								fieldText := map[string]interface{}{}
-								for field, _ := range sr.Fields {
-									text := common.ObjToString(tmp[field])
-									text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
-									fieldText[field] = text
-								}
-								//清理词清理
-								for _, cwm := range sr.GCW.MatchType {
-									if text := common.ObjToString(fieldText[cwm]); text != "" {
-										for _, gcw_reg := range sr.GCW.KeyReg {
-											text = gcw_reg.ReplaceAllString(text, "")
-										}
-										fieldText[cwm] = text
-									}
-								}
-								//精准筛选规则2022-10-19
-								if c.Exact == 1 && sr.ExactRule != "" {
-									nameArr := []string{}
-									data, ok := util.Mgo.Find("groups", map[string]interface{}{"ruleId": sr.ID}, nil, nil, false, -1, -1)
-									if ok && data != nil && len(*data) > 0 {
-										for _, v := range *data {
-											nameArr = append(nameArr, common.ObjToString(v["name"]))
-										}
-									}
-									exactResult := exactMatchs(sr.ExactRule, common.ObjToString(tmp["title"]), common.ObjToString(tmp["detail"]), sr.Maths, nameArr)
-									log.Debug("-------------------精准匹配", zap.String("id", id), zap.Any("exactResult", exactResult))
-									if !exactResult {
-										return
-									}
-								}
-								//
-								/*
-									因为要记录所有匹配上的关键词，所有优先匹配附加词，在匹配关键词
-								*/
-								//1.附加词匹配
-								IsMatch := false
-								// common.Debug("sr.AW---", len(sr.AW))
-								for i, aw := range sr.AW {
-									// common.Debug("-------------------------开始附加词匹配--------------------------")
-									IsMatchAddKey := RegMatch(fieldText, aw.MatchType, aw.KeyReg, nil, nil, false, true)
-									// common.Debug(IsMatchAddKey, "------------------------------------------------------------")
-
-									//2.关键词匹配
-									if IsMatchAddKey {
-										kw := sr.KW[i]
-										// common.Debug("-------------------------开始关键词匹配--------------------------")
-										IsMatchKey := RegMatch(fieldText, kw.MatchType, kw.KeyReg, matchKey, matchKeyType, true, false)
-										// common.Debug(IsMatchKey, "------------------------------------------------------------")
-										if IsMatchKey {
-											IsMatch = true
-										}
-									}
-								}
-								if len(sr.AW) == 0 {
-									IsMatch = true
-								}
-								/*
-									到此已经匹配完数据
-								*/
-								log.Debug("---------------------", zap.String("id", id), zap.Any("IsMatch", IsMatch))
-								if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
-									// tmpMatchKey := MapDataToArr(matchKey)
-									tmpMatchKeyType := MapDataToArr(matchKeyType)
-									tmp["matchkey"] = GetMactchKeys(sr.Maths, tmp)
-									tmp["matchtype"] = strings.Join(tmpMatchKeyType, ",")
-									tmp["ruleid"] = sr.ID
-									tmp["rulename"] = sr.Name
-									tmpBuyerClass := common.ObjToString(tmp["buyerclass"])
-
-									//开始打标签
-									//common.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
-									if c.IsTagRule {
-										tagNameMap := map[string]bool{}
-										tagIdMap := map[string]bool{}
-										for _, tr := range c.TagRules {
-											if tr.DepartRuleIds[sr.ID] {
-												//先获取用到的所有字段值
-												for field, _ := range tr.Fields {
-													if fieldText[field] == nil { //补充fieldText
-														text := common.ObjToString(tmp[field])
-														text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
-														fieldText[field] = text
-													}
-												}
-												matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
-												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
-												//common.Debug("-------------------------开始排除词匹配--------------------------")
-												for j, tag_nw := range tr.NW { //排除词匹配
-													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
-													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
-														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
-															tagname := tr.TagNames[j]
-															tagBuyerClass := tr.BuyerClass[j]
-															if tagBuyerClass != "" {
-																if strings.Contains(tagBuyerClass, tmpBuyerClass) {
-																	if tagname == "" {
-																		tempList := []string{}
-																		for k, _ := range matchKeyTag {
-																			tempList = append(tempList, k)
-																		}
-																		tagname = strings.Join(tempList, ",")
-																		log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称%s", tagname))
-																	}
-																	tagNameMap[tagname] = true
-																	tagIdMap[tr.ID] = true
-																}
-															} else {
-																if tagname == "" {
-																	tempList := []string{}
-																	for k, _ := range matchKeyTag {
-																		tempList = append(tempList, k)
-																	}
-																	tagname = strings.Join(tempList, ",")
-																	log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称", tagname))
-																}
-																tagNameMap[tagname] = true
-																tagIdMap[tr.ID] = true
-															}
-														}
-													}
-												}
-											}
-										}
-										//tagname
-										tagNameArr := MapDataToArr(tagNameMap)
-										tagIdArr := MapDataToArr(tagIdMap)
-										if len(tagNameArr) > 0 {
-											tmp["tagname"] = strings.Join(tagNameArr, ",")
-											if DisPackageAppidMap[c.AppId] {
-												tmp["buyer_type"] = strings.Join(tagNameArr, ",")
-											}
-											if c.PushModel == 2 {
-												tmp["item"] = strings.Join(tagNameArr, ",")
-											}
-											tmp["tagid"] = strings.Join(tagIdArr, ",")
-										}
-									}
-									if c.IsTagRule2 {
-										tagNameMap := map[string]bool{}
-										tagIdMap := map[string]bool{}
-										for _, tr := range c.TagRules2 {
-											if tr.DepartRuleIds[sr.ID] {
-												//先获取用到的所有字段值
-												for field, _ := range tr.Fields {
-													if fieldText[field] == nil { //补充fieldText
-														text := common.ObjToString(tmp[field])
-														text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
-														fieldText[field] = text
-													}
-												}
-												matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
-												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
-												//common.Debug("-------------------------开始排除词匹配--------------------------")
-												for j, tag_nw := range tr.NW { //排除词匹配
-													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
-													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
-														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
-															tagname := tr.TagNames[j]
-															tagBuyerClass := tr.BuyerClass[j]
-															if tagBuyerClass != "" {
-																if strings.Contains(tagBuyerClass, tmpBuyerClass) {
-																	if tagname == "" {
-																		tempList := []string{}
-																		for k, _ := range matchKeyTag {
-																			tempList = append(tempList, k)
-																		}
-																		tagname = strings.Join(tempList, ",")
-																		log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称%s", tagname))
-																	}
-																	tagNameMap[tagname] = true
-																	tagIdMap[tr.ID] = true
-																}
-															} else {
-																if tagname == "" {
-																	tempList := []string{}
-																	for k, _ := range matchKeyTag {
-																		tempList = append(tempList, k)
-																	}
-																	tagname = strings.Join(tempList, ",")
-																	log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称%s", tagname))
-																}
-																tagNameMap[tagname] = true
-																tagIdMap[tr.ID] = true
-															}
-														}
-													}
-												}
-											}
-										}
-										//tagname
-										tagNameArr := MapDataToArr(tagNameMap)
-										tagIdArr := MapDataToArr(tagIdMap)
-										if len(tagNameArr) > 0 {
-											tmp["tagname2"] = strings.Join(tagNameArr, ",")
-											if DisPackageAppidMap[c.AppId] {
-												tmp["buyer_type2"] = strings.Join(tagNameArr, ",")
-											}
-											if c.PushModel == 2 {
-												tmp["item2"] = strings.Join(tagNameArr, ",")
-											}
-											tmp["tagid2"] = strings.Join(tagIdArr, ",")
-										}
-									}
-									if c.IsTagRule3 {
-										tagNameMap := map[string]bool{}
-										tagIdMap := map[string]bool{}
-										for _, tr := range c.TagRules3 {
-											if tr.DepartRuleIds[sr.ID] {
-												//先获取用到的所有字段值
-												for field, _ := range tr.Fields {
-													if fieldText[field] == nil { //补充fieldText
-														text := common.ObjToString(tmp[field])
-														text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
-														fieldText[field] = text
-													}
-												}
-												matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
-												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
-												//common.Debug("-------------------------开始排除词匹配--------------------------")
-												for j, tag_nw := range tr.NW { //排除词匹配
-													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
-													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
-														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
-															tagname := tr.TagNames[j]
-															tagBuyerClass := tr.BuyerClass[j]
-															if tagBuyerClass != "" {
-																if strings.Contains(tagBuyerClass, tmpBuyerClass) {
-																	if tagname == "" {
-																		tempList := []string{}
-																		for k, _ := range matchKeyTag {
-																			tempList = append(tempList, k)
-																		}
-																		tagname = strings.Join(tempList, ",")
-																		log.Debug("=====tagname为空取匹配词为标签名称", zap.String("tagname", tagname))
-																	}
-																	tagNameMap[tagname] = true
-																	tagIdMap[tr.ID] = true
-																}
-															} else {
-																if tagname == "" {
-																	tempList := []string{}
-																	for k, _ := range matchKeyTag {
-																		tempList = append(tempList, k)
-																	}
-																	tagname = strings.Join(tempList, ",")
-																	log.Debug("=====tagname为空取匹配词为标签名称", zap.String("tagname", tagname))
-																}
-																tagNameMap[tagname] = true
-																tagIdMap[tr.ID] = true
-															}
-														}
-													}
-												}
-											}
-										}
-										//tagname
-										tagNameArr := MapDataToArr(tagNameMap)
-										tagIdArr := MapDataToArr(tagIdMap)
-										if len(tagNameArr) > 0 {
-											tmp["tagname3"] = strings.Join(tagNameArr, ",")
-											if DisPackageAppidMap[c.AppId] {
-												tmp["buyer_type3"] = strings.Join(tagNameArr, ",")
-											}
-											if c.PushModel == 2 {
-												tmp["item3"] = strings.Join(tagNameArr, ",")
-											}
-											tmp["tagid3"] = strings.Join(tagIdArr, ",")
-										}
-									}
-									//item
-									switch c.PushModel {
-									case 0:
-										tmp["item"] = "数据"
-									case 1:
-										tmp["item"] = dm.Name
-									case 2:
-										//tmp["item"] = sr.Name
-									case 3:
-										tmp["item"] = dm.Name + "_" + sr.Name
-									case 4:
-										tmp["item"] = sr.Name
-									}
-									//appid
-									tmp["appid"] = c.AppId
-									//部门名称
-									tmp["departname"] = dm.Name
-									tmp["departid"] = dm.ID
-									//存储数据
-									dm.DataLock.Lock()
-									//common.Debug("tmp---", tmp)
-									tmpMap := map[string]interface{}{id: tmp}
-									dm.DepartmentData[sr.ID] = append(dm.DepartmentData[sr.ID], tmpMap)
-									dm.DataLock.Unlock()
-								} else {
-									// common.Debug("------------", id, IsMatch)
-								}
+								c.tempChan <- &tempData{dm: dm, rule: sr, data: tmp}
 							}
 						}(hit)
 						numDocs += 1
@@ -623,6 +258,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 				}
 				wg.Wait()
 				client.ClearScroll().ScrollId(scrollId).Do(ctx) //清理游标
+				c.tempChanOver <- true
 				log.Debug(fmt.Sprintf("SearchRule ID：%s, Result Data Count：%d", sr.ID, numDocs))
 			} else {
 				log.Debug(fmt.Sprintf("Customer: %s, Departmnet: %s, TagName: %s, Es Search Data Error,Tag ID: %s", c.Name, dm.Name, sr.Name, sr.ID))
@@ -631,6 +267,364 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 	}
 }
 
+func (c *Customer) processedData() {
+	ch := make(chan bool, 10)
+L:
+	for {
+		select {
+		case tmp := <-c.tempChan:
+			ch <- true
+			go func(tmp *tempData) {
+				defer func() {
+					<-ch
+				}()
+				processedData_A(c, tmp)
+			}(tmp)
+		default:
+			select {
+			case tmp := <-c.tempChan:
+				ch <- true
+				go func(tmp *tempData) {
+					defer func() {
+						<-ch
+					}()
+					processedData_A(c, tmp)
+				}(tmp)
+			case <-c.tempChanOver:
+				break L
+			}
+		}
+	}
+}
+
+func processedData_A(c *Customer, data *tempData) {
+	dm := data.dm   // 部门信息
+	sr := data.rule // 规则
+	tmp := data.data
+	id := common.ObjToString(tmp["id"])
+
+	//亚信
+	if CheckBidOpenAppidMap[c.AppId] {
+		if tmp["bidopentime"] != nil {
+			bidopentime := common.Int64All(tmp["bidopentime"])
+			comeintime := common.Int64All(tmp["comeintime"])
+			if bidopentime-comeintime <= 7*24*60*60 {
+				log.Debug(fmt.Sprintf("跳过该条数据，开标时间-入库时间<=7天，%s", id))
+				return
+			}
+		}
+	}
+	//河南移动，过滤掉中国移动采购网招标数据
+	if CheckBidHrefRuleIdMap[dm.ID] {
+		if strings.Contains(common.ObjToString(tmp["href"]), "b2b.10086.cn") {
+			log.Debug(fmt.Sprintf("跳过该条数据，公告原网址中包含 b2b.10086.cn，%s", id))
+			return
+		}
+	}
+	//精准筛选规则2022-10-19
+	if len(sr.NameArr) > 0 {
+		exactResult := exactMatchs(sr.ExactRule, common.ObjToString(tmp["title"]), common.ObjToString(tmp["detail"]), sr.Maths, sr.NameArr)
+		log.Debug("-------------------精准匹配", zap.String("id", id), zap.Any("exactResult", exactResult))
+		if !exactResult {
+			return
+		}
+	}
+	//
+	tmp["id"] = id //记录数据原有id
+	delete(tmp, "_id")
+	if sr.ExtFieldType == 2 {
+		findwinner := ""
+		s_winner := strings.Split(common.ObjToString(tmp["s_winner"]), ",")
+		if len(s_winner) > 0 {
+			for i := 0; i < len(s_winner); i++ {
+				findwinners := strings.TrimSpace(s_winner[i])
+				if findwinners != "" {
+					for _, v := range util.Sysconfig["s_winner_filter"].([]interface{}) {
+						strings.ReplaceAll(findwinners, v.(string), "")
+					}
+					if findwinners != "" {
+						findwinner = findwinners
+						break
+					}
+				}
+			}
+		}
+		if findwinner != "" {
+			EntInfo(findwinner, tmp)
+		}
+	}
+	matchKey := map[string]bool{}     //记录所有匹配上的关键词
+	matchKeyType := map[string]bool{} //记录关键词对应的匹配方式
+	//先获取用到的所有字段值
+	fieldText := map[string]interface{}{}
+	for field, _ := range sr.Fields {
+		text := common.ObjToString(tmp[field])
+		text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+		fieldText[field] = text
+	}
+	//清理词清理
+	for _, cwm := range sr.GCW.MatchType {
+		if text := common.ObjToString(fieldText[cwm]); text != "" {
+			for _, gcw_reg := range sr.GCW.KeyReg {
+				text = gcw_reg.ReplaceAllString(text, "")
+			}
+			fieldText[cwm] = text
+		}
+	}
+
+	//
+	/*
+		因为要记录所有匹配上的关键词，所有优先匹配附加词，在匹配关键词
+	*/
+	//1.附加词匹配
+	IsMatch := false
+	// common.Debug("sr.AW---", len(sr.AW))
+	for i, aw := range sr.AW {
+		// common.Debug("-------------------------开始附加词匹配--------------------------")
+		IsMatchAddKey := RegMatch(fieldText, aw.MatchType, aw.KeyReg, nil, nil, false, true)
+		// common.Debug(IsMatchAddKey, "------------------------------------------------------------")
+
+		//2.关键词匹配
+		if IsMatchAddKey {
+			kw := sr.KW[i]
+			// common.Debug("-------------------------开始关键词匹配--------------------------")
+			IsMatchKey := RegMatch(fieldText, kw.MatchType, kw.KeyReg, matchKey, matchKeyType, true, false)
+			// common.Debug(IsMatchKey, "------------------------------------------------------------")
+			if IsMatchKey {
+				IsMatch = true
+			}
+		}
+	}
+	if len(sr.AW) == 0 {
+		IsMatch = true
+	}
+	/*
+		到此已经匹配完数据
+	*/
+	log.Debug("---------------------", zap.String("id", id), zap.Any("IsMatch", IsMatch))
+	if IsMatch {
+		//匹配成功，数据上新增规则id,matchKey,item并临时保存数据
+		// tmpMatchKey := MapDataToArr(matchKey)
+		tmpMatchKeyType := MapDataToArr(matchKeyType)
+		tmp["matchkey"] = GetMactchKeys(sr.Maths, tmp)
+		tmp["matchtype"] = strings.Join(tmpMatchKeyType, ",")
+		tmp["ruleid"] = sr.ID
+		tmp["rulename"] = sr.Name
+
+		//开始打标签
+		//common.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
+		if c.IsTagRule {
+			TagRuleFuc(c.AppId, c.PushModel, 1, c.TagRules, sr, tmp, fieldText)
+		}
+		if c.IsTagRule2 {
+			TagRuleFuc(c.AppId, c.PushModel, 2, c.TagRules2, sr, tmp, fieldText)
+		}
+		if c.IsTagRule3 {
+			TagRuleFuc(c.AppId, c.PushModel, 3, c.TagRules3, sr, tmp, fieldText)
+		}
+		//item
+		switch c.PushModel {
+		case 0:
+			tmp["item"] = "数据"
+		case 1:
+			tmp["item"] = dm.Name
+		case 2:
+			//tmp["item"] = sr.Name
+		case 3:
+			tmp["item"] = dm.Name + "_" + sr.Name
+		case 4:
+			tmp["item"] = sr.Name
+		}
+		//appid
+		tmp["appid"] = c.AppId
+		//部门名称
+		tmp["departname"] = dm.Name
+		tmp["departid"] = dm.ID
+		//存储数据
+		//dm.DataLock.Lock()
+		//tmpMap := map[string]interface{}{id: tmp}
+		//dm.DepartmentData[sr.ID] = append(dm.DepartmentData[sr.ID], tmpMap)
+		//dm.DataLock.Unlock()
+		c.saveBeforeChan <- &tempData{dm: dm, rule: sr, data: tmp}
+	} else {
+		// common.Debug("------------", id, IsMatch)
+	}
+}
+
+// legal_person,company_email,company_phone
+func EntInfo(name string, tmp map[string]interface{}) {
+	if ss := redis.GetStr("ent", fmt.Sprintf("datatag_%s", name)); ss != "" {
+		if ss != "no" {
+			// legal_person,company_email,company_phone
+			s1 := strings.Split(ss, ",")
+			if s1[0] != "-" {
+				tmp["legal_person"] = s1[0]
+			}
+			if s1[1] != "-" {
+				tmp["legal_person"] = s1[1]
+			}
+			if s1[2] != "-" {
+				tmp["legal_person"] = s1[2]
+			}
+		}
+	} else {
+		finddata, _ := util.MgoEnps.FindOneByField(util.EnpsColl, bson.M{"company_name": name}, bson.M{"legal_person": 1, "company_email": 1, "company_phone": 1})
+		if finddata != nil {
+			str := ""
+			if legal_person := common.ObjToString((*finddata)["legal_person"]); legal_person != "" {
+				tmp["legal_person"] = legal_person
+				str = legal_person
+			} else {
+				str = "-"
+			}
+			if email := common.ObjToString((*finddata)["company_email"]); email != "" {
+				tmp["company_email"] = email
+				str += ","
+				str += email
+			} else {
+				str += ","
+				str += "-"
+			}
+			if phone := common.ObjToString((*finddata)["company_phone"]); phone != "" {
+				tmp["company_phone"] = phone
+				str += ","
+				str += phone
+			} else {
+				str += ","
+				str += "-"
+			}
+			redis.Put("ent", fmt.Sprintf("datatag_%s", name), str, 7*24*60*60)
+		} else {
+			redis.Put("ent", fmt.Sprintf("datatag_%s", name), "no", 10*60*60)
+		}
+	}
+}
+
+// 标签
+func TagRuleFuc(appid string, pushModel, tag int, rules []*TagRule, sr *SearchRule, tmp, fieldText map[string]interface{}) {
+	tmpBuyerClass := common.ObjToString(tmp["buyerclass"])
+	tagNameMap := map[string]bool{}
+	tagIdMap := map[string]bool{}
+	for _, tr := range rules {
+		if tr.DepartRuleIds[sr.ID] {
+			//先获取用到的所有字段值
+			for field, _ := range tr.Fields {
+				if fieldText[field] == nil { //补充fieldText
+					text := common.ObjToString(tmp[field])
+					text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+					fieldText[field] = text
+				}
+			}
+			matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
+			matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
+			//common.Debug("-------------------------开始排除词匹配--------------------------")
+			for j, tag_nw := range tr.NW { //排除词匹配
+				IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
+				if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
+					if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
+						tagname := tr.TagNames[j]
+						tagBuyerClass := tr.BuyerClass[j]
+						if tagBuyerClass != "" {
+							if strings.Contains(tagBuyerClass, tmpBuyerClass) {
+								if tagname == "" {
+									tempList := []string{}
+									for k, _ := range matchKeyTag {
+										tempList = append(tempList, k)
+									}
+									tagname = strings.Join(tempList, ",")
+									log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称%s", tagname))
+								}
+								tagNameMap[tagname] = true
+								tagIdMap[tr.ID] = true
+							}
+						} else {
+							if tagname == "" {
+								tempList := []string{}
+								for k, _ := range matchKeyTag {
+									tempList = append(tempList, k)
+								}
+								tagname = strings.Join(tempList, ",")
+								log.Debug(fmt.Sprintf("=====tagname为空取匹配词为标签名称", tagname))
+							}
+							tagNameMap[tagname] = true
+							tagIdMap[tr.ID] = true
+						}
+					}
+				}
+			}
+		}
+	}
+	//tagname
+	tagNameArr := MapDataToArr(tagNameMap)
+	tagIdArr := MapDataToArr(tagIdMap)
+	if len(tagNameArr) > 0 {
+		switch tag {
+		case 1:
+			tmp["tagname"] = strings.Join(tagNameArr, ",")
+			if DisPackageAppidMap[appid] {
+				tmp["buyer_type"] = strings.Join(tagNameArr, ",")
+			}
+			if pushModel == 2 {
+				tmp["item"] = strings.Join(tagNameArr, ",")
+			}
+			tmp["tagid"] = strings.Join(tagIdArr, ",")
+		case 2:
+			if len(tagNameArr) > 0 {
+				tmp["tagname2"] = strings.Join(tagNameArr, ",")
+				if DisPackageAppidMap[appid] {
+					tmp["buyer_type2"] = strings.Join(tagNameArr, ",")
+				}
+				if pushModel == 2 {
+					tmp["item2"] = strings.Join(tagNameArr, ",")
+				}
+				tmp["tagid2"] = strings.Join(tagIdArr, ",")
+			}
+		case 3:
+			if len(tagNameArr) > 0 {
+				tmp["tagname3"] = strings.Join(tagNameArr, ",")
+				if DisPackageAppidMap[appid] {
+					tmp["buyer_type3"] = strings.Join(tagNameArr, ",")
+				}
+				if pushModel == 2 {
+					tmp["item3"] = strings.Join(tagNameArr, ",")
+				}
+				tmp["tagid3"] = strings.Join(tagIdArr, ",")
+			}
+		}
+	}
+}
+
+func (c *Customer) saveBeforeData(historyId string, isFilter, noticeFilter, dataTable, entId, i_contact int, xlsxData *XlsxData) {
+	log.Debug("保存数据前置处理...")
+	ch := make(chan bool, 10)
+L:
+	for {
+		select {
+		case tmp := <-c.saveBeforeChan:
+			ch <- true
+			go func(tmp *tempData) {
+				defer func() {
+					<-ch
+				}()
+				AssembelSave(tmp.data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId, isFilter, noticeFilter, dataTable, entId, i_contact, xlsxData, c.DataSave)
+			}(tmp)
+		default:
+			select {
+			case tmp := <-c.saveBeforeChan:
+				ch <- true
+				go func(tmp *tempData) {
+					defer func() {
+						<-ch
+					}()
+					AssembelSave(tmp.data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId, isFilter, noticeFilter, dataTable, entId, i_contact, xlsxData, c.DataSave)
+				}(tmp)
+			case <-c.tempChanOver:
+				break L
+			}
+		}
+	}
+}
+
 // 数据去重
 func (c *Customer) RemoveRepeatData() {
 	log.Debug("开始数据去重...")

CMPlatform/history/util_history.go

@@ -761,6 +761,9 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, h
 	}
 	//匹配公告附件
 	info, _ := util.MgoBidding.FindOne(util.BiddingColl, map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
+	if info == nil || len(*info) == 0 {
+		info, _ = util.MgoBidding.FindOne("bidding_back", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
+	}
 	if info != nil && len(*info) > 0 {
 		if (*info)["projectinfo"] != nil {
 			projectInfo := common.ObjToMap((*info)["projectinfo"])
@@ -816,64 +819,6 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, h
 		tmp["approvestatus"] = common.ObjToString((*info)["approvestatus"])
 		tmp["approvetime"] = common.ObjToString((*info)["approvetime"])
 		tmp["project_completedate"] = common.Int64All((*info)["project_completedate"])
-	} else {
-		infos, _ := util.MgoBidding.FindOne("bidding_back", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
-		if infos != nil && len(*infos) > 0 {
-			if (*infos)["projectinfo"] != nil {
-				projectInfo := common.ObjToMap((*infos)["projectinfo"])
-				if projectInfo != nil && len(*projectInfo) > 0 {
-					if (*projectInfo)["attachments"] != nil {
-						filesArr := []map[string]interface{}{}
-						fileUrl := common.ObjToString(util.Sysconfig["file_downloadurl"])
-						if attachments, oks := (*projectInfo)["attachments"].(map[string]interface{}); oks {
-							for _, v := range attachments {
-								if atta, ok := v.(map[string]interface{}); ok {
-									if atta["fid"] != nil && common.ObjToString(atta["fid"]) != "" {
-										fileMap := map[string]interface{}{
-											"filename": atta["filename"],
-											"url":      fileUrl + common.ObjToString(atta["fid"]),
-										}
-										filesArr = append(filesArr, fileMap)
-									} else {
-										if atta["org_url"] != nil {
-											fileMap := map[string]interface{}{
-												"filename": atta["filename"],
-												"url":      common.ObjToString(atta["org_url"]),
-											}
-											filesArr = append(filesArr, fileMap)
-										}
-									}
-								}
-							}
-						}
-						log.Debug("查询附件结果：" + id + "，附件数量：" + fmt.Sprint(len(filesArr)))
-						if len(filesArr) > 0 {
-							if FilterFilehrefAppidMap[appid] {
-								filesArrs := []map[string]interface{}{}
-								for _, v := range filesArr {
-									url := common.ObjToString(v["url"])
-									if strings.Contains(url, "jy-datafile.oss-cn-beijing.aliyuncs.com") {
-										filesArrs = append(filesArrs, v)
-									}
-								}
-								if len(filesArrs) > 0 {
-									tmp["filehref"] = filesArrs
-								}
-							} else {
-								tmp["filehref"] = filesArr
-							}
-						}
-					}
-				}
-			}
-			tmp["details"] = common.ObjToString((*info)["detail"])
-			tmp["bidtype"] = common.ObjToString((*info)["bidtype"])
-			tmp["owner"] = common.ObjToString((*info)["owner"])
-			tmp["total_investment"] = common.ObjToString((*info)["total_investment"])
-			tmp["approvestatus"] = common.ObjToString((*info)["approvestatus"])
-			tmp["approvetime"] = common.Int64All((*info)["approvetime"])
-			tmp["project_completedate"] = common.Int64All((*info)["project_completedate"])
-		}
 	}
 	//祥和不锈钢
 	if FilterFileAndDetailMap[appid] {