升级es7.*

customerdata/src/main.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	elastic "es"
 	"flag"
 	"log"
 	qu "qfw/util"
-	"qfw/util/elastic"
+
+	// "qfw/util/elastic"
 	"qfw/util/redis"
 	"sync"
 	"util/mgodb"
@@ -60,7 +62,7 @@ var (
 	HospColl     string
 	BuyerEntColl string
 	EnpsColl     string
-	Es           *elastic.Elastic //es
+	Es           elastic.Es //es
 	Index        string
 	Itype        string
 	TaskTime     int64 //定时任务时间
@@ -158,11 +160,12 @@ func init() {
 
 	//es
 	es := Sysconfig.Es
-	Es = &elastic.Elastic{
-		S_esurl: qu.ObjToString(es["addr"]),
-		I_size:  qu.IntAllDef(es["pool"], 15),
-	}
-	Es.InitElasticSize()
+	// Es = &elastic.Elastic{
+	// 	S_esurl: qu.ObjToString(es["addr"]),
+	// 	I_size:  qu.IntAllDef(es["pool"], 15),
+	// }
+	// Es.InitElasticSize()
+	Es = elastic.NewEs(qu.ObjToString(es["version"]), qu.ObjToString(es["addr"]), qu.IntAllDef(es["pool"], 15), qu.ObjToString(es["userName"]), qu.ObjToString(es["password"]))
 	redis.InitRedis(Sysconfig.RedisAddrs)
 	Index = qu.ObjToString(es["index"])
 	Itype = qu.ObjToString(es["itype"])

customerdata/src/task.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -17,9 +18,13 @@ import (
 
 	"github.com/cron"
 
+	esv "es"
+
 	"github.com/antonmedv/expr"
 	"github.com/donnie4w/go-logger/logger"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+
+	esV7 "github.com/olivere/elastic"
 	es "gopkg.in/olivere/elastic.v1"
 )
 
@@ -154,8 +159,21 @@ func (c *Customer) GetDepartments(stype string) {
 func (c *Customer) GetData(stype string) {
 	log.Println("开始匹配数据...")
 	defer qu.Catch()
-	client := Es.GetEsConn()
-	defer Es.DestoryEsConn(client)
+	esConfig := Sysconfig.Es
+	esversion := qu.ObjToString(esConfig["version"])
+	if esversion == "v1" {
+		esCon := esv.VarEs.(*esv.EsV1)
+		c.EsConGetDataV1(stype, esCon)
+	} else {
+		esCon := esv.VarEs.(*esv.EsV7)
+		c.EsConGetDataV7(stype, esCon)
+	}
+
+}
+
+func (c *Customer) EsConGetDataV1(stype string, esCon *esv.EsV1) {
+	client := esCon.GetEsConn()
+	defer esCon.DestoryEsConn(client)
 	for _, dm := range c.Departments {
 		for _, sr := range dm.Rules {
 
@@ -480,6 +498,359 @@ func (c *Customer) GetData(stype string) {
 	}
 }
 
+type MySource struct {
+	Querys string
+}
+
+func (m *MySource) Source() (interface{}, error) {
+	mp := make(map[string]interface{})
+	json.Unmarshal([]byte(m.Querys), &mp)
+	return mp["query"], nil
+}
+
+func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
+	client := esCon.GetEsConn()
+	defer esCon.DestoryEsConn(client)
+	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
+	for _, dm := range c.Departments {
+		for _, sr := range dm.Rules {
+
+			//测试
+			// MgoDataTest(sr, dm, c)
+			// return
+
+			ch := make(chan bool, 10)
+			wg := &sync.WaitGroup{}
+			escount := Es.Count(Index, Itype, sr.EsQuery)
+			log.Println("查询总数:", escount, "规则ID:", sr.ID, "EsQuery:", sr.EsQuery)
+			if escount == 0 {
+				continue
+			}
+			//查询条件类型转换
+			// var q esV7.Query
+			//sr.EsQuery = `{"query":{"filtered":{"filter":{"bool":{"must":[{"bool":{"should":[{"terms":{"city":["上海市"]}}]}},{"terms":{"toptype":["招标","结果"]}},{"range":{"publishtime":{"gte":1588262400,"lt":1608825600}}}]}},"query":{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"bool":{"should":[{"bool":{"must":[{"terms":{"buyer":["上海城建职业学院","上海市第一人民医院","中国银联股份有限公司","上海立信会计金融学院法规处","中国东方航空股份有限公司","上海外国语大学","上海小昆山环卫服务有限公司","国家税务总局上海市税务局","中国浦东干部学院","上海市浦东新区老港镇人民政府","咪咕视讯科技有限公司","上海交通大学医学院附属新华医院","交通运输部上海打捞局","松江区体育局","复旦大学附属中山医院","上海交通大学医学院附属瑞金医院","中国科学院上海应用物理研究所"]}}]}}]}}]}}]}}],"must_not":[{"constant_score":{"filter":{"missing":{"field":"filetext"}}}}]}}}}}`
+			cc := &MySource{
+				Querys: sr.EsQuery,
+			}
+			//游标查询，index不支持别名，只能写索引库的名称
+			res, err := client.Scroll(Index).Query(cc).Size(200).Do(ctx) //查询一条获取游标
+			if err == nil {
+				numDocs := 0
+				scrollId := res.ScrollId
+				count := 1
+				for {
+					if scrollId == "" {
+						log.Println("ScrollId Is Error")
+						break
+					}
+					var searchResult *esV7.SearchResult
+					var err error
+					if count == 1 {
+						searchResult = res
+					} else {
+						searchResult, err = client.Scroll(Index).Size(200).ScrollId(scrollId).Do(ctx) //查询
+						if err != nil {
+							if err.Error() == "EOS" { //迭代完毕
+								log.Println("Es Search Data Over：", err)
+							} else {
+								log.Println("Es Search Data Error：", err)
+							}
+							break
+						}
+					}
+					log.Println("此次处理条数 ", len(searchResult.Hits.Hits))
+					if err != nil {
+						if err.Error() == "EOS" { //迭代完毕
+							log.Println("Es Search Data Over：", err)
+						} else {
+							log.Println("Es Search Data Error：", err)
+						}
+						break
+					}
+					for _, hit := range searchResult.Hits.Hits {
+						//开始处理数据
+						wg.Add(1)
+						ch <- true
+						go func(tmpHit *esV7.SearchHit) {
+							defer func() {
+								<-ch
+								wg.Done()
+							}()
+							tmp := make(map[string]interface{})
+							if json.Unmarshal(tmpHit.Source, &tmp) == nil {
+								if stype != "history" {
+									if !SkipData(tmp) {
+										qu.Debug("跳过该条数据，发布时间在入库时间7天之前，", qu.ObjToString(tmp["_id"]))
+										return
+									}
+								}
+								id := qu.ObjToString(tmp["id"])
+								//亚信
+								if CheckBidOpenAppidMap[c.AppId] {
+									if tmp["bidopentime"] != nil {
+										bidopentime := qu.Int64All(tmp["bidopentime"])
+										comeintime := qu.Int64All(tmp["comeintime"])
+										if bidopentime-comeintime <= 7*24*60*60 {
+											qu.Debug("跳过该条数据，开标时间-入库时间<=7天，", id)
+											return
+										}
+									}
+								}
+								//河南移动，过滤掉中国移动采购网招标数据
+								if CheckBidHrefRuleIdMap[dm.ID] {
+									if strings.Contains(qu.ObjToString(tmp["href"]), "b2b.10086.cn") {
+										qu.Debug("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
+										return
+									}
+								}
+								isExists, err := redis.Exists("datag", c.AppId+"_"+id)
+								if err != nil {
+									log.Println("redis信息id判重出错 ", err)
+								} else if isExists {
+									log.Println("信息id重复 ", id)
+									return
+								}
+								tmp["id"] = id //记录数据原有id
+								delete(tmp, "_id")
+								if sr.ExtFieldType == 2 {
+									findwinner := ""
+									s_winner := strings.Split(qu.ObjToString(tmp["s_winner"]), ",")
+									if len(s_winner) > 0 {
+										for i := 0; i < len(s_winner); i++ {
+											findwinners := strings.TrimSpace(s_winner[i])
+											if findwinners != "" {
+												for _, v := range Sysconfig.SWinnerFilter {
+													strings.ReplaceAll(findwinners, v, "")
+												}
+												if findwinners != "" {
+													findwinner = findwinners
+													break
+												}
+											}
+										}
+									}
+									// findwinner := strings.TrimSpace(qu.ObjToString(tmp["winner"]))
+									if findwinner != "" {
+										finddata := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
+										if finddata != nil {
+											if legal_person := qu.ObjToString(finddata["legal_person"]); legal_person != "" {
+												tmp["legal_person"] = legal_person
+											}
+											if email := qu.ObjToString(finddata["company_email"]); email != "" {
+												tmp["company_email"] = email
+											}
+											if phone := qu.ObjToString(finddata["company_phone"]); phone != "" {
+												tmp["company_phone"] = phone
+											}
+											//从最新年报中获取 中标单位联系电话、中标单位邮箱
+											// if annual_reports, ok := finddata["annual_reports"].(primitive.A); ok && len(annual_reports) > 0 {
+											// 	anreport := Sort_year_report(annual_reports)
+											// 	if len(anreport) > 0 {
+											// 		if email := qu.ObjToString(anreport["company_email"]); email != "" {
+											// 			tmp["company_email"] = email
+											// 		}
+											// 		if phone := qu.ObjToString(anreport["company_phone"]); phone != "" {
+											// 			tmp["company_phone"] = phone
+											// 		}
+											// 	}
+											// }
+										}
+									}
+								}
+								matchKey := map[string]bool{}     //记录所有匹配上的关键词
+								matchKeyType := map[string]bool{} //记录关键词对应的匹配方式
+								//先获取用到的所有字段值
+								fieldText := map[string]interface{}{}
+								for field, _ := range sr.Fields {
+									text := qu.ObjToString(tmp[field])
+									text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+									fieldText[field] = text
+								}
+								//清理词清理
+								for _, cwm := range sr.GCW.MatchType {
+									if text := qu.ObjToString(fieldText[cwm]); text != "" {
+										for _, gcw_reg := range sr.GCW.KeyReg {
+											text = gcw_reg.ReplaceAllString(text, "")
+										}
+										fieldText[cwm] = text
+									}
+								}
+								//精准筛选规则2022-10-19
+								if c.Exact == 1 && sr.ExactRule != "" {
+									nameArr := []string{}
+									data, _ := MgoTag.Find("groups", map[string]interface{}{"ruleId": sr.ID}, nil, nil)
+									if data != nil && len(data) > 0 {
+										for _, v := range data {
+											nameArr = append(nameArr, qu.ObjToString(v["name"]))
+										}
+									}
+									exactResult := exactMatchs(sr.ExactRule, qu.ObjToString(tmp["title"]), qu.ObjToString(tmp["detail"]), sr.Maths, nameArr)
+									qu.Debug("-------------------精准匹配", id, exactResult)
+									if !exactResult {
+										return
+									}
+								}
+								/*
+									因为要记录所有匹配上的关键词，所有优先匹配附加词，在匹配关键词
+								*/
+								//1.附加词匹配
+								IsMatch := false
+								//qu.Debug("sr.AW---", len(sr.AW))
+								for i, aw := range sr.AW {
+									//qu.Debug("-------------------------开始附加词匹配--------------------------")
+									IsMatchAddKey := RegMatch(fieldText, aw.MatchType, aw.KeyReg, nil, nil, false, true)
+									//qu.Debug(IsMatchAddKey, "------------------------------------------------------------")
+
+									//2.关键词匹配
+									if IsMatchAddKey {
+										kw := sr.KW[i]
+										//qu.Debug("-------------------------开始关键词匹配--------------------------")
+										IsMatchKey := RegMatch(fieldText, kw.MatchType, kw.KeyReg, matchKey, matchKeyType, true, false)
+										//qu.Debug(IsMatchKey, "------------------------------------------------------------")
+										if IsMatchKey {
+											IsMatch = true
+										}
+									}
+								}
+								if len(sr.AW) == 0 {
+									IsMatch = true
+								}
+								/*
+									到此已经匹配完数据
+								*/
+								qu.Debug("---------------------", id, IsMatch, matchKey)
+								if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
+									// tmpMatchKey := MapDataToArr(matchKey)
+									tmpMatchKeyType := MapDataToArr(matchKeyType)
+									tmp["matchkey"] = GetMactchKeys(sr.Maths, tmp)
+									tmp["matchtype"] = strings.Join(tmpMatchKeyType, ",")
+									tmp["ruleid"] = sr.ID
+									tmp["rulename"] = sr.Name
+									tmpBuyerClass := qu.ObjToString(tmp["buyerclass"])
+
+									//开始打标签
+									//qu.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
+									if c.IsTagRule {
+										tagNameMap := map[string]bool{}
+										tagIdMap := map[string]bool{}
+										//qu.Debug("c.TagRules---", len(c.TagRules))
+										//log.Println(c.TagRules,"=========",)
+										for _, tr := range c.TagRules {
+											if tr.DepartRuleIds[sr.ID] {
+												// log.Println(tr.TagNames, "===========打标签")
+												//先获取用到的所有字段值
+												for field, _ := range tr.Fields {
+													if fieldText[field] == nil { //补充fieldText
+														text := qu.ObjToString(tmp[field])
+														text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+														fieldText[field] = text
+													}
+												}
+												//qu.Debug("-------------------------开始排除词匹配--------------------------")
+												//qu.Debug("tr.NW---", len(tr.NW))
+												matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
+												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
+												for j, tag_nw := range tr.NW {       //排除词匹配
+													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
+													//qu.Debug(IsMatchNotKey, "------------------------------------------------------------")
+													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
+														// log.Println(j, tr.TagNames[j])
+														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
+															tagname := tr.TagNames[j]
+															tagBuyerClass := tr.BuyerClass[j]
+															if tagBuyerClass != "" {
+																if strings.Contains(tagBuyerClass, tmpBuyerClass) {
+																	if tagname == "" {
+																		tempList := []string{}
+																		for k, _ := range matchKeyTag {
+																			tempList = append(tempList, k)
+																		}
+																		tagname = strings.Join(tempList, ",")
+																		log.Println("=====tagname为空取匹配词为标签名称", tagname)
+																	}
+																	//qu.Debug("tagname-----", tagname)
+																	tagNameMap[tagname] = true
+																	tagIdMap[tr.ID] = true
+																}
+															} else {
+																if tagname == "" {
+																	tempList := []string{}
+																	for k, _ := range matchKeyTag {
+																		tempList = append(tempList, k)
+																	}
+																	tagname = strings.Join(tempList, ",")
+																	log.Println("=====tagname为空取匹配词为标签名称", tagname)
+																}
+																//qu.Debug("tagname-----", tagname)
+																tagNameMap[tagname] = true
+																tagIdMap[tr.ID] = true
+															}
+														}
+													}
+												}
+											}
+										}
+										//tagname
+										tagNameArr := MapDataToArr(tagNameMap)
+										tagIdArr := MapDataToArr(tagIdMap)
+										if len(tagNameArr) > 0 {
+											tmp["tagname"] = strings.Join(tagNameArr, ",")
+											if DisPackageAppidMap[c.AppId] {
+												tmp["buyer_type"] = strings.Join(tagNameArr, ",")
+											}
+											if c.PushModel == 2 {
+												tmp["item"] = strings.Join(tagNameArr, ",")
+											}
+											tmp["tagid"] = strings.Join(tagIdArr, ",")
+										}
+									}
+									//item
+									switch c.PushModel {
+									case 0:
+										tmp["item"] = "数据"
+									case 1:
+										tmp["item"] = dm.Name
+									case 2:
+										//tmp["item"] = sr.Name
+									case 3:
+										tmp["item"] = dm.Name + "_" + sr.Name
+									case 4:
+										tmp["item"] = sr.Name
+									}
+									//appid
+									tmp["appid"] = c.AppId
+									//部门名称
+									tmp["departname"] = dm.Name
+									tmp["departid"] = dm.ID
+									//存储数据
+									dm.DataLock.Lock()
+									//qu.Debug("tmp---", tmp)
+									tmpMap := map[string]interface{}{id: tmp}
+									dm.DepartmentData[sr.ID] = append(dm.DepartmentData[sr.ID], tmpMap)
+									dm.DataLock.Unlock()
+								} else {
+									qu.Debug("------------", id, IsMatch)
+								}
+							}
+						}(hit)
+						numDocs += 1
+						if numDocs%500 == 0 {
+							log.Println("Current：", numDocs)
+						}
+					}
+					scrollId = searchResult.ScrollId
+					count++
+				}
+				wg.Wait()
+				client.ClearScroll().ScrollId(scrollId).Do(ctx) //清理游标
+				log.Println("SearchRule ID", sr.ID, "Result Data Count：", numDocs)
+			} else {
+				log.Println("Customer:", c.Name, "Departmnet", dm.Name, "TagName", sr.Name, "Es Search Data Error,Tag ID:", sr.ID)
+			}
+		}
+	}
+}
+
 // 数据去重
 func (c *Customer) RemoveRepeatData() {
 	log.Println("开始数据去重...")
@@ -662,10 +1033,10 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange, tmpRanges bson
 	query := map[string]*QueryObjecct{}
 	if json.Unmarshal([]byte(esquery), &query) == nil {
 		qb := query["query"]
-		filter := qb.Filtered.Filter
-		if filter != nil && filter.Bool != nil { //有filter
+		filter := qb.Bool
+		if filter != nil { //有filter
 			index := -1 //记录range的位置
-			for i, m := range filter.Bool.Must {
+			for i, m := range filter.Must {
 				mMap := m.(map[string]interface{})
 				if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
 					if esRange["publishtime"] != nil {
@@ -675,12 +1046,12 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange, tmpRanges bson
 				}
 			}
 			if index > -1 {
-				filter.Bool.Must[index] = tmpRange
+				filter.Must[index] = tmpRange
 			} else {
-				filter.Bool.Must = append(filter.Bool.Must, tmpRange)
+				filter.Must = append(filter.Must, tmpRange)
 			}
 			if len(tmpRanges) > 0 {
-				filter.Bool.Must = append(filter.Bool.Must, tmpRanges)
+				filter.Must = append(filter.Must, tmpRanges)
 			}
 		} else { //无filter则添加
 			bo := &BoolObject{}
@@ -688,10 +1059,10 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange, tmpRanges bson
 			if len(tmpRanges) > 0 {
 				bo.Must = append(bo.Must, tmpRanges)
 			}
-			tmpFilter := &Filter{
-				Bool: bo,
-			}
-			qb.Filtered.Filter = tmpFilter
+			// tmpFilter := &Filter{
+			// 	Bool: bo,
+			// }
+			qb.Bool = bo
 		}
 		strquery, err := json.Marshal(query)
 		if err == nil {

customerdata/src/util.go

@@ -211,23 +211,23 @@ func GetIdRange() (bson.M, bson.M, bool, int64) {
 	// 	}}, true
 	// }
 	idQuery, endTime := "", int64(0)
-	esquery := `{"query":{"filtered":{"filter":{"bool":{"must":{"range":{"createtime":{"gt":"%d"}}}}}}},"_source":["_id","createtime"],"sort":{"createtime":"desc"},"from":0,"size":1}`
+	esquery := `{"query":{"bool":{"must":{"range":{"pici":{"gt":"%d"}}}}},"_source":["id","pici"],"sort":{"pici":"desc"},"from":0,"size":1}`
 	if LatestId == "" {
 		idQuery = strings.Replace(fmt.Sprintf(esquery, LatestTime), `"gt"`, `"gte"`, -1)
 	} else {
-		esquerys := `{"query":{"filtered":{"filter":{"bool":{"must":{"range":{"id":{"gt":"%s"}}}}}}},"_source":["_id","createtime"],"sort":{"createtime":"desc"},"from":0,"size":1}`
+		esquerys := `{"query":{"bool":{"must":{"range":{"id":{"gt":"%s"}}}}},"_source":["id","pici"],"sort":{"pici":"desc"},"from":0,"size":1}`
 		idQuery = fmt.Sprintf(esquerys, LatestId)
 	}
 	resId := Es.Get(Index, Itype, idQuery)
 	if resId != nil && *resId != nil && len(*resId) == 1 {
-		endTime = qu.Int64All((*resId)[0]["createtime"]) - 600
+		endTime = qu.Int64All((*resId)[0]["pici"]) - 600
 	} else {
 		logger.Debug("本次任务未查找到数据...", idQuery)
 		return bson.M{}, bson.M{}, false, endTime
 	}
 	tmpRange1 := bson.M{
 		"range": bson.M{
-			"createtime": bson.M{
+			"pici": bson.M{
 				"lte": endTime,
 				"gt":  LatestTime,
 			},
@@ -235,7 +235,7 @@ func GetIdRange() (bson.M, bson.M, bool, int64) {
 	}
 	tmpRange2 := bson.M{
 		"range": bson.M{
-			"createtime": bson.M{
+			"pici": bson.M{
 				"lte": endTime,
 			},
 		},
@@ -769,7 +769,7 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, a
 			tmp["owner"] = qu.ObjToString(info["owner"])
 			tmp["total_investment"] = qu.ObjToString(info["total_investment"])
 			tmp["approvestatus"] = qu.ObjToString(info["approvestatus"])
-			tmp["approvetime"] = qu.Int64All(info["approvetime"])
+			tmp["approvetime"] = qu.ObjToString(info["approvetime"])
 			tmp["project_completedate"] = qu.Int64All(info["project_completedate"])
 		}
 	}

sqlutil/src/sqlmodel/rules_es.go

@@ -1,10 +1,11 @@
 package sqlmodel
 
 type QueryObjecct struct {
-	Filtered struct {
-		Filter *Filter `json:"filter,omitempty"`
-		Query  *Query  `json:"query,omitempty"`
-	} `json:"filtered,omitempty"`
+	// Filtered struct {
+	// Filter *Filter `json:"filter,omitempty"`
+	// Query  *Query      `json:"query,omitempty"`
+	Bool *BoolObject `json:"bool,omitempty"`
+	// } `json:"filtered,omitempty"`
 }
 
 type Query struct {
@@ -15,19 +16,19 @@ type Filter struct {
 	Bool *BoolObject `json:"bool,omitempty"`
 }
 
-//省市县
+// 省市县
 type AreaCityDistrictMust struct {
 	*AreaCityDistrict `json:"terms,omitempty"`
 }
 
-//省市县
+// 省市县
 type AreaCityDistrict struct {
 	Area     []string `json:"area,omitempty"`
 	City     []string `json:"city,omitempty"`
 	District []string `json:"district,omitempty"`
 }
 
-//网站名称
+// 网站名称
 type SiteMust struct {
 	*Site `json:"terms,omitempty"`
 }
@@ -43,12 +44,12 @@ type DistrictObject struct {
 	} `json:"term,omitempty"`
 }
 
-//一级分类、二级分类
+// 一级分类、二级分类
 type ToptypeSubtypeMust struct {
 	*ToptypeSubtype `json:"terms,omitempty"`
 }
 
-//一级分类、二级分类
+// 一级分类、二级分类
 type ToptypeSubtype struct {
 	Toptype []string `json:"toptype,omitempty"`
 	Subtype []string `json:"subtype,omitempty"`
@@ -59,7 +60,7 @@ type ToptypeSubtype struct {
 //	*Scopeclass `json:"prefix,omitempty"`
 //}
 
-//公告类型
+// 公告类型
 type ScopeclassMust struct {
 	*Scopeclass `json:"terms,omitempty"`
 }
@@ -85,12 +86,12 @@ type BudgetOrBidamount struct {
 	Lt  float64 `json:"lt,omitempty"`
 }
 
-//时间范围
+// 时间范围
 type PublishtimeMust struct {
 	PublishtimeObject *PublishtimeObject `json:"range,omitempty"`
 }
 
-//时间范围
+// 时间范围
 type PublishtimeObject struct {
 	//查询时间
 	Publishtime  *Publishtime  `json:"publishtime,omitempty"`
@@ -98,7 +99,7 @@ type PublishtimeObject struct {
 	Publishtime2 *Publishtime2 `json:"publishtime2,omitempty"`
 }
 
-//查询时间
+// 查询时间
 type Publishtime struct {
 	Gte int `json:"gte"`
 	Lt  int `json:"lt"`
@@ -114,7 +115,7 @@ type ExpurasingtimeMust struct {
 	ExpurasingtimeObject *ExpurasingtimeObject `json:"range,omitempty"`
 }
 
-//时间范围
+// 时间范围
 type ExpurasingtimeObject struct {
 	//查询时间
 	Expurasingtime  *Expurasingtime  `json:"procurementlist.expurasingtime,omitempty"`
@@ -122,7 +123,7 @@ type ExpurasingtimeObject struct {
 	Expurasingtime2 *Expurasingtime2 `json:"procurementlist.expurasingtime2,omitempty"`
 }
 
-//查询时间
+// 查询时间
 type Expurasingtime struct {
 	Gte int `json:"gte"`
 	Lt  int `json:"lt"`
@@ -134,12 +135,12 @@ type Expurasingtime2 struct {
 	Lt int `json:"lt"`
 }
 
-//字段是或否存在
+// 字段是或否存在
 type ExistfieldsObjectMust struct {
 	ExistfieldsObject *ExistfieldsObject `json:"constant_score,omitempty"`
 }
 
-//字段存在 存在放MustNot，字符串为空忽略
+// 字段存在 存在放MustNot，字符串为空忽略
 type ExistfieldsObject struct {
 	Filter struct {
 		Missing struct {
@@ -153,7 +154,7 @@ type ShouldObj struct {
 	MultiMatch  *MultiMatch  `json:"multi_match,omitempty"`
 }
 
-//匹配方式
+// 匹配方式
 type MatchPhrase struct {
 	Title       string `json:"title,omitempty"`
 	Detail      string `json:"detail,omitempty"`
@@ -168,7 +169,7 @@ type MultiMatch struct {
 	Fields []string `json:"fields,omitempty"`
 }
 
-//匹配方式 采购单位、中标单位 正则匹配
+// 匹配方式 采购单位、中标单位 正则匹配
 type Regular struct {
 	Regexp1 *MultiMatch `json:"regexp1,omitempty"`
 	Regexp2 *MultiMatch `json:"regexp2,omitempty"`
@@ -203,13 +204,13 @@ type Winner struct {
 
 /********************************************/
 
-//排除词 must_not
+// 排除词 must_not
 type NotkeyObject struct {
 	MatchPhrase *MatchPhrase `json:"match_phrase,omitempty"`
 	MultiMatch  *MultiMatch  `json:"multi_match,omitempty"`
 }
 
-//采购单位行业 should
+// 采购单位行业 should
 type BuyerclassObject struct {
 	Terms struct {
 		Buyerclass []string `json:"buyerclass,omitempty"`
@@ -222,14 +223,14 @@ type TopscopeclassObject struct {
 	} `json:"terms,omitempty"`
 }
 
-//公告行业二级 should
+// 公告行业二级 should
 type SubscopeclassObject struct {
 	Terms struct {
 		Subscopeclass []string `json:"s_subscopeclass,omitempty"`
 	} `json:"terms,omitempty"`
 }
 
-//xf should
+// xf should
 type NewEsObject struct {
 	Bool struct {
 		Must    []interface{} `json:"must,omitempty"`

src/config.json

@@ -1,10 +1,10 @@
 {
   "port": "7001",
   "udpport": ":11117",
-  "mgodb": "192.168.3.207:27092",
+  "mgodb": "192.168.3.71:29099",
   "dbsize": 5,
   "dbname": "datatag_ent",
-  "mgodben": "192.168.3.207:27092",
+  "mgodben": "192.168.3.71:29099",
   "dbnameen": "enterprise",
   "mongoenc": "winner_enterprise",
   "subday": 15,
@@ -15,7 +15,7 @@
     "dbsize": 5
   },
   "preview_href": "http://192.168.20.214:5500/data-preview.html",
-  "elasticsearch": "http://192.168.3.206:9800",
+  "elasticsearch": "http://192.168.3.241:9205",
   "elasticsearch_index": "bidding",
   "elasticsearch_type": "bidding",
   "elasticPoolSize": 10,
@@ -30,26 +30,26 @@
     "memo": "生索引"
   },
   "buyerinfo": {
-    "addr": "192.168.3.207:27001,192.168.3.206:27002",
+    "addr": "192.168.3.71:29099",
     "size": 15,
     "db": "mixdata",
     "hospcoll": "buyer_rank",
     "buyerentcoll": "buyer_ent",
-    "username": "jyDevGroup",
-    "password": "jy@DevGroup",
+    "username": "",
+    "password": "",
     "replSet": ""
   },
   "enterprise": {
-    "addr": "192.168.3.207:27001,192.168.3.206:27002",
+    "addr": "192.168.3.71:29099",
     "size": 15,
     "db": "mixdata",
     "coll": "qyxy_std",
-    "username": "jyDevGroup",
-    "password": "jy@DevGroup",
+    "username": "",
+    "password": "",
     "replSet": ""
   },
   "save": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.71:29099",
     "size": 15,
     "db": "datatag_ent",
     "coll": "usermail_history",
@@ -64,10 +64,13 @@
     "size": 15
   },
   "es": {
-    "addr": "http://192.168.3.206:9800",
+    "addr": "http://192.168.3.241:9205",
     "index": "bidding",
     "itype": "bidding",
-    "pool": 15
+    "pool": 15,
+    "version": "v7",
+    "userName": "",
+    "password": ""
   },
   "mail": [
     {
@@ -213,33 +216,33 @@
   "need_projectid_appid": [
     "jyFApXQQIEAw5TTUZOMBpD"
   ],
-  "redis_addrs": "datag=192.168.3.206:1712,other=192.168.3.206:1712",
+  "redis_addrs": "datag=192.168.3.11:1712,other=192.168.3.11:1712",
   "jyPushMysql": {
     "username": "root",
     "password": "Topnet123",
-    "address": "192.168.3.11:3366",
+    "address": "192.168.3.149:3306",
     "dbName": "jianyu"
   },
   "jyMysql": {
     "username": "root",
     "password": "Topnet123",
-    "address": "192.168.3.11:3366",
+    "address": "192.168.3.149:3306",
     "dbName": "jianyu"
   },
   "pushMysql": {
     "username": "root",
     "password": "Topnet123",
-    "address": "192.168.3.11:3366",
+    "address": "192.168.3.149:3306",
     "dbName": "jianyu",
     "maxIdleConns": 2
   },
   "bidding_mgo": {
-    "addr": "192.168.3.207:27001,192.168.3.206:27002",
+    "addr": "192.168.3.71:29099",
     "coll": "bidding",
     "db": "qfw_data",
     "size": 15,
-    "username": "jyDevGroup",
-    "password": "jy@DevGroup",
+    "username": "",
+    "password": "",
     "replSet": ""
   },
   "file_downloadurl": "http://jy-datafile.oss-cn-beijing.aliyuncs.com/",
@@ -272,7 +275,7 @@
   "elasticsearch_interim_index": "bidding",
   "elasticsearch_total_index": "bidding",
   "site_mgo": {
-    "addr": "192.168.3.207:27092",
+    "addr": "192.168.3.71:29099",
     "size": 5,
     "db": "datatag_ent",
     "coll": "site"

src/history/task.go

@@ -1,6 +1,7 @@
 package history
 
 import (
+	"context"
 	"encoding/json"
 	"log"
 	mongoutil "qfw/mongodb"
@@ -9,15 +10,19 @@ import (
 	. "sqlmodel"
 	"strings"
 	"sync"
+	"time"
 	. "util"
 
 	"gopkg.in/mgo.v2/bson"
 
 	// "go.mongodb.org/mongo-driver/bson/primitive"
+	esv "es"
+
+	esV7 "github.com/olivere/elastic"
 	es "gopkg.in/olivere/elastic.v1"
 )
 
-//获取客户打标签规则
+// 获取客户打标签规则
 func (c *Customer) GetTagRules(tagRules []map[string]interface{}) {
 	log.Println("开始加载标签规则...")
 	defer qu.Catch()
@@ -53,7 +58,7 @@ func (c *Customer) GetTagRules(tagRules []map[string]interface{}) {
 	}
 }
 
-//获取部门信息
+// 获取部门信息
 func (c *Customer) GetDepartments(stype string, departments []map[string]interface{}) {
 	log.Println("开始获取部门信息...")
 	defer qu.Catch()
@@ -89,12 +94,24 @@ func (c *Customer) GetDepartments(stype string, departments []map[string]interfa
 	}
 }
 
-//获取数据
+// 获取数据
 func (c *Customer) GetData(stype string, dataSource int) {
 	log.Println("开始匹配数据...")
 	defer qu.Catch()
-	client := Es.GetEsConn()
-	defer Es.DestoryEsConn(client)
+	esConfig := Sysconfig["es"].(map[string]interface{})
+	esversion := qu.ObjToString(esConfig["version"])
+	if esversion == "v1" {
+		esCon := esv.VarEs.(*esv.EsV1)
+		c.EsConGetDataV1(stype, dataSource, esCon)
+	} else {
+		esCon := esv.VarEs.(*esv.EsV7)
+		c.EsConGetDataV7(stype, dataSource, esCon)
+	}
+}
+
+func (c *Customer) EsConGetDataV1(stype string, dataSource int, esCon *esv.EsV1) {
+	client := esCon.GetEsConn()
+	defer esCon.DestoryEsConn(client)
 	for _, dm := range c.Departments {
 		for _, sr := range dm.Rules {
 			ch := make(chan bool, 10)
@@ -400,7 +417,336 @@ func (c *Customer) GetData(stype string, dataSource int) {
 	}
 }
 
-//数据去重
+type MySource struct {
+	Querys string
+}
+
+func (m *MySource) Source() (interface{}, error) {
+	mp := make(map[string]interface{})
+	json.Unmarshal([]byte(m.Querys), &mp)
+	return mp["query"], nil
+}
+
+func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *esv.EsV7) {
+	client := esCon.GetEsConn()
+	defer esCon.DestoryEsConn(client)
+	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
+	for _, dm := range c.Departments {
+		for _, sr := range dm.Rules {
+			ch := make(chan bool, 10)
+			wg := &sync.WaitGroup{}
+			esIndex := Index
+			if dataSource == 2 {
+				esIndex = EsAllIndex
+			}
+			escount := Es.Count(esIndex, Itype, sr.EsQuery)
+			log.Println("index", esIndex, "type", Itype)
+			log.Println("查询总数:", escount, "规则ID:", sr.ID, "EsQuery:", sr.EsQuery)
+			if escount == 0 {
+				continue
+			}
+			//查询条件类型转换
+			// var q esV7.Query
+			// tmpQuery := esV7.BoolQuery{
+			// 	QueryStrings: sr.EsQuery,
+			// }
+			// q = tmpQuery
+			cc := &MySource{
+				Querys: sr.EsQuery,
+			}
+			//游标查询，index不支持别名，只能写索引库的名称
+			res, err := client.Scroll(esIndex).Query(cc).Size(200).Do(ctx) //查询一条获取游标
+			if err == nil {
+				numDocs := 0
+				scrollId := res.ScrollId
+				count := 1
+				for {
+					if scrollId == "" {
+						log.Println("ScrollId Is Error")
+						break
+					}
+					var searchResult *esV7.SearchResult
+					var err error
+					if count == 1 {
+						searchResult = res
+					} else {
+						searchResult, err = client.Scroll(esIndex).Size(200).ScrollId(scrollId).Do(ctx) //查询
+						if err != nil {
+							if err.Error() == "EOS" { //迭代完毕
+								log.Println("Es Search Data Over：", err)
+							} else {
+								log.Println("Es Search Data Error：", err)
+							}
+							break
+						}
+					}
+					log.Println("此次处理条数 ", len(searchResult.Hits.Hits))
+					for _, hit := range searchResult.Hits.Hits {
+						//开始处理数据
+						wg.Add(1)
+						ch <- true
+						go func(tmpHit *esV7.SearchHit) {
+							defer func() {
+								<-ch
+								wg.Done()
+							}()
+							tmp := make(map[string]interface{})
+							if json.Unmarshal(tmpHit.Source, &tmp) == nil {
+								if stype != "history" {
+									if !SkipData(tmp) {
+										qu.Debug("跳过该条数据，发布时间在入库时间7天之前，", qu.ObjToString(tmp["_id"]))
+										return
+									}
+								}
+								id := qu.ObjToString(tmp["id"])
+								//亚信
+								if CheckBidOpenAppidMap[c.AppId] {
+									if tmp["bidopentime"] != nil {
+										bidopentime := qu.Int64All(tmp["bidopentime"])
+										comeintime := qu.Int64All(tmp["comeintime"])
+										if bidopentime-comeintime <= 7*24*60*60 {
+											qu.Debug("跳过该条数据，开标时间-入库时间<=7天，", id)
+											return
+										}
+									}
+								}
+								//河南移动，过滤掉中国移动采购网招标数据
+								if CheckBidHrefRuleIdMap[dm.ID] {
+									if strings.Contains(qu.ObjToString(tmp["href"]), "b2b.10086.cn") {
+										qu.Debug("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
+										return
+									}
+								}
+								//
+								tmp["id"] = id //记录数据原有id
+								delete(tmp, "_id")
+								if sr.ExtFieldType == 2 {
+									findwinner := ""
+									s_winner := strings.Split(qu.ObjToString(tmp["s_winner"]), ",")
+									if len(s_winner) > 0 {
+										for i := 0; i < len(s_winner); i++ {
+											findwinners := strings.TrimSpace(s_winner[i])
+											if findwinners != "" {
+												for _, v := range Sysconfig["s_winner_filter"].([]interface{}) {
+													strings.ReplaceAll(findwinners, v.(string), "")
+												}
+												if findwinners != "" {
+													findwinner = findwinners
+													break
+												}
+											}
+										}
+									}
+									// findwinner := strings.TrimSpace(qu.ObjToString(tmp["winner"]))
+									if findwinner != "" {
+										finddata := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
+										if finddata != nil {
+											if legal_person := qu.ObjToString(finddata["legal_person"]); legal_person != "" {
+												tmp["legal_person"] = legal_person
+											}
+											if email := qu.ObjToString(finddata["company_email"]); email != "" {
+												tmp["company_email"] = email
+											}
+											if phone := qu.ObjToString(finddata["company_phone"]); phone != "" {
+												tmp["company_phone"] = phone
+											}
+										}
+									}
+								}
+								matchKey := map[string]bool{}     //记录所有匹配上的关键词
+								matchKeyType := map[string]bool{} //记录关键词对应的匹配方式
+								//先获取用到的所有字段值
+								fieldText := map[string]interface{}{}
+								for field, _ := range sr.Fields {
+									text := qu.ObjToString(tmp[field])
+									text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+									fieldText[field] = text
+								}
+								//清理词清理
+								for _, cwm := range sr.GCW.MatchType {
+									if text := qu.ObjToString(fieldText[cwm]); text != "" {
+										for _, gcw_reg := range sr.GCW.KeyReg {
+											text = gcw_reg.ReplaceAllString(text, "")
+										}
+										fieldText[cwm] = text
+									}
+								}
+								//精准筛选规则2022-10-19
+								if c.Exact == 1 && sr.ExactRule != "" {
+									nameArr := []string{}
+									data, ok := Mgo.Find("groups", map[string]interface{}{"ruleId": sr.ID}, nil, nil, false, -1, -1)
+									if ok && data != nil && len(*data) > 0 {
+										for _, v := range *data {
+											nameArr = append(nameArr, qu.ObjToString(v["name"]))
+										}
+									}
+									exactResult := exactMatchs(sr.ExactRule, qu.ObjToString(tmp["title"]), qu.ObjToString(tmp["detail"]), sr.Maths, nameArr)
+									qu.Debug("-------------------精准匹配", id, exactResult)
+									if !exactResult {
+										return
+									}
+								}
+								//
+								/*
+									因为要记录所有匹配上的关键词，所有优先匹配附加词，在匹配关键词
+								*/
+								//1.附加词匹配
+								IsMatch := false
+								// qu.Debug("sr.AW---", len(sr.AW))
+								for i, aw := range sr.AW {
+									// qu.Debug("-------------------------开始附加词匹配--------------------------")
+									IsMatchAddKey := RegMatch(fieldText, aw.MatchType, aw.KeyReg, nil, nil, false, true)
+									// qu.Debug(IsMatchAddKey, "------------------------------------------------------------")
+
+									//2.关键词匹配
+									if IsMatchAddKey {
+										kw := sr.KW[i]
+										// qu.Debug("-------------------------开始关键词匹配--------------------------")
+										IsMatchKey := RegMatch(fieldText, kw.MatchType, kw.KeyReg, matchKey, matchKeyType, true, false)
+										// qu.Debug(IsMatchKey, "------------------------------------------------------------")
+										if IsMatchKey {
+											IsMatch = true
+										}
+									}
+								}
+								if len(sr.AW) == 0 {
+									IsMatch = true
+								}
+								/*
+									到此已经匹配完数据
+								*/
+								qu.Debug("---------------------", id, IsMatch)
+								if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
+									// tmpMatchKey := MapDataToArr(matchKey)
+									tmpMatchKeyType := MapDataToArr(matchKeyType)
+									tmp["matchkey"] = GetMactchKeys(sr.Maths, tmp)
+									tmp["matchtype"] = strings.Join(tmpMatchKeyType, ",")
+									tmp["ruleid"] = sr.ID
+									tmp["rulename"] = sr.Name
+									tmpBuyerClass := qu.ObjToString(tmp["buyerclass"])
+
+									//开始打标签
+									//qu.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
+									if c.IsTagRule {
+										tagNameMap := map[string]bool{}
+										tagIdMap := map[string]bool{}
+										//qu.Debug("c.TagRules---", len(c.TagRules))
+										for _, tr := range c.TagRules {
+											if tr.DepartRuleIds[sr.ID] {
+												//先获取用到的所有字段值
+												for field, _ := range tr.Fields {
+													if fieldText[field] == nil { //补充fieldText
+														text := qu.ObjToString(tmp[field])
+														text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
+														fieldText[field] = text
+													}
+												}
+												matchKeyTag := map[string]bool{}     //记录所有标签里的匹配上的关键词
+												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
+
+												//qu.Debug("-------------------------开始排除词匹配--------------------------")
+												//qu.Debug("tr.NW---", len(tr.NW))
+												for j, tag_nw := range tr.NW { //排除词匹配
+													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
+													//qu.Debug(IsMatchNotKey, "------------------------------------------------------------")
+													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
+														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
+															tagname := tr.TagNames[j]
+															tagBuyerClass := tr.BuyerClass[j]
+															if tagBuyerClass != "" {
+																if strings.Contains(tagBuyerClass, tmpBuyerClass) {
+																	if tagname == "" {
+																		tempList := []string{}
+																		for k, _ := range matchKeyTag {
+																			tempList = append(tempList, k)
+																		}
+																		tagname = strings.Join(tempList, ",")
+																		log.Println("=====tagname为空取匹配词为标签名称", tagname)
+																	}
+																	//qu.Debug("tagname-----", tagname)
+																	tagNameMap[tagname] = true
+																	tagIdMap[tr.ID] = true
+																}
+															} else {
+																if tagname == "" {
+																	tempList := []string{}
+																	for k, _ := range matchKeyTag {
+																		tempList = append(tempList, k)
+																	}
+																	tagname = strings.Join(tempList, ",")
+																	log.Println("=====tagname为空取匹配词为标签名称", tagname)
+																}
+																//qu.Debug("tagname-----", tagname)
+																tagNameMap[tagname] = true
+																tagIdMap[tr.ID] = true
+															}
+														}
+													}
+												}
+											}
+										}
+										//tagname
+										tagNameArr := MapDataToArr(tagNameMap)
+										tagIdArr := MapDataToArr(tagIdMap)
+										if len(tagNameArr) > 0 {
+											tmp["tagname"] = strings.Join(tagNameArr, ",")
+											if DisPackageAppidMap[c.AppId] {
+												tmp["buyer_type"] = strings.Join(tagNameArr, ",")
+											}
+											if c.PushModel == 2 {
+												tmp["item"] = strings.Join(tagNameArr, ",")
+											}
+											tmp["tagid"] = strings.Join(tagIdArr, ",")
+										}
+									}
+									//item
+									switch c.PushModel {
+									case 0:
+										tmp["item"] = "数据"
+									case 1:
+										tmp["item"] = dm.Name
+									case 2:
+										//tmp["item"] = sr.Name
+									case 3:
+										tmp["item"] = dm.Name + "_" + sr.Name
+									case 4:
+										tmp["item"] = sr.Name
+									}
+									//appid
+									tmp["appid"] = c.AppId
+									//部门名称
+									tmp["departname"] = dm.Name
+									tmp["departid"] = dm.ID
+									//存储数据
+									dm.DataLock.Lock()
+									//qu.Debug("tmp---", tmp)
+									tmpMap := map[string]interface{}{id: tmp}
+									dm.DepartmentData[sr.ID] = append(dm.DepartmentData[sr.ID], tmpMap)
+									dm.DataLock.Unlock()
+								} else {
+									// qu.Debug("------------", id, IsMatch)
+								}
+							}
+						}(hit)
+						numDocs += 1
+						if numDocs%500 == 0 {
+							log.Println("Current：", numDocs)
+						}
+					}
+					scrollId = searchResult.ScrollId
+					count++
+				}
+				wg.Wait()
+				client.ClearScroll().ScrollId(scrollId).Do(ctx) //清理游标
+				log.Println("SearchRule ID", sr.ID, "Result Data Count：", numDocs)
+			} else {
+				log.Println("Customer:", c.Name, "Departmnet", dm.Name, "TagName", sr.Name, "Es Search Data Error,Tag ID:", sr.ID)
+			}
+		}
+	}
+}
+
+// 数据去重
 func (c *Customer) RemoveRepeatData() {
 	log.Println("开始数据去重...")
 	defer qu.Catch()
@@ -452,7 +798,7 @@ func (c *Customer) RemoveRepeatData() {
 	}
 }
 
-//组装保存数据
+// 组装保存数据
 func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter, dataTable, entId, i_contact int, xlsxData *XlsxData) {
 	log.Println("开始组装保存数据...")
 	defer qu.Catch()
@@ -525,7 +871,7 @@ func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter,
 	log.Println("数据保存完毕...		Save Number：", n)
 }
 
-//获取用户所有规则
+// 获取用户所有规则
 func (d *Department) GetSearchRules(cid, stype string, idRange bson.M, searchRules []map[string]interface{}) {
 	defer qu.Catch()
 	if len(searchRules) > 0 {
@@ -586,16 +932,16 @@ func (d *Department) GetSearchRules(cid, stype string, idRange bson.M, searchRul
 	}
 }
 
-//获取转换后的es语句
+// 获取转换后的es语句
 func (sr *SearchRule) GetEs(department, esquery string, tmpRange bson.M) {
 	defer qu.Catch()
 	query := map[string]*QueryObjecct{}
 	if json.Unmarshal([]byte(esquery), &query) == nil {
 		qb := query["query"]
-		filter := qb.Filtered.Filter
-		if filter != nil && filter.Bool != nil { //有filter
+		filter := qb.Bool
+		if filter != nil { //有filter
 			index := -1 //记录range的位置
-			for i, m := range filter.Bool.Must {
+			for i, m := range filter.Must {
 				mMap := m.(map[string]interface{})
 				if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
 					if esRange["publishtime"] != nil {
@@ -605,17 +951,17 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange bson.M) {
 				}
 			}
 			if index > -1 {
-				filter.Bool.Must[index] = tmpRange
+				filter.Must[index] = tmpRange
 			} else {
-				filter.Bool.Must = append(filter.Bool.Must, tmpRange)
+				filter.Must = append(filter.Must, tmpRange)
 			}
 		} else { //无filter则添加
 			bo := &BoolObject{}
 			bo.Must = append(bo.Must, tmpRange)
-			tmpFilter := &Filter{
-				Bool: bo,
-			}
-			qb.Filtered.Filter = tmpFilter
+			// tmpFilter := &Filter{
+			// 	Bool: bo,
+			// }
+			qb.Bool = bo
 		}
 		strquery, err := json.Marshal(query)
 		if err == nil {
@@ -628,7 +974,7 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange bson.M) {
 	}
 }
 
-//全局清理词处理
+// 全局清理词处理
 func (sr *SearchRule) GetClearWord(key, match string) {
 	defer qu.Catch()
 	//匹配方式
@@ -655,7 +1001,7 @@ func (sr *SearchRule) GetClearWord(key, match string) {
 	sr.GCW = cw
 }
 
-//关键词、附加词处理
+// 关键词、附加词处理
 func (sr *SearchRule) GetKeyAddWord(o_rules []map[string]interface{}) {
 	defer qu.Catch()
 	kw, aw, _, _, _ := GetNotkeyAndKeyAddWord(o_rules, sr.Fields, false)
@@ -664,7 +1010,7 @@ func (sr *SearchRule) GetKeyAddWord(o_rules []map[string]interface{}) {
 	//sr.Fields = fields
 }
 
-//排除词、关键词、附加词处理
+// 排除词、关键词、附加词处理
 func (tr *TagRule) GetKeyAddNotKeyWord(o_list []map[string]interface{}) {
 	defer qu.Catch()
 	kw, aw, nkw, tagnames, buyerclass := GetNotkeyAndKeyAddWord(o_list, tr.Fields, true)

src/history/util_history.go

@@ -864,7 +864,7 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, h
 		tmp["owner"] = qu.ObjToString(info["owner"])
 		tmp["total_investment"] = qu.ObjToString(info["total_investment"])
 		tmp["approvestatus"] = qu.ObjToString(info["approvestatus"])
-		tmp["approvetime"] = qu.Int64All(info["approvetime"])
+		tmp["approvetime"] = qu.ObjToString(info["approvetime"])
 		tmp["project_completedate"] = qu.Int64All(info["project_completedate"])
 	} else {
 		infos := MgoBidding.FindOne("bidding_back", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})

src/service/customer_service.go

@@ -233,7 +233,7 @@ func (c *Customer) DepartEdit() {
 	})
 }
 
-//部门启用、停用
+// 部门启用、停用
 func (c *Customer) SetupDepart() {
 	defer qu.Catch()
 	if c.Method() == "POST" {
@@ -305,7 +305,7 @@ func (c *Customer) DepartDel() {
 	})
 }
 
-//转企
+// 转企
 func (c *Customer) CuserToEuser() {
 	defer qu.Catch()
 
@@ -401,7 +401,7 @@ func GetCustomerAppId(customer string) string {
 	return appid
 }
 
-//同步
+// 同步
 func (c *Customer) SynEuser() {
 	defer qu.Catch()
 	_id := c.GetString("_id")
@@ -645,7 +645,7 @@ func filter(sql string) string {
 	return sql
 }
 
-//匹配包含detail的组
+// 匹配包含detail的组
 func checkDetail(sql string) (arrMap map[string]string) {
 	arrMap = map[string]string{}
 	res := reg_detail.FindAllStringSubmatch(sql, -1)
@@ -659,12 +659,12 @@ func checkDetail(sql string) (arrMap map[string]string) {
 	return
 }
 
-//校验是否有单字
+// 校验是否有单字
 func checkSingleWord(sql string) bool {
 	return len(reg_single.FindStringSubmatch(sql)) > 0
 }
 
-//计算每组字段的替换值
+// 计算每组字段的替换值
 func makeReplace(s_match string) (s_replace string) {
 	arr := []string{}
 	for _, v := range strings.Split(s_match, ",") {
@@ -1119,7 +1119,7 @@ func (c *Customer) CustomerHistory() {
 	c.Render("private/customer_history.html", &c.T)
 }
 
-//选择历史规则
+// 选择历史规则
 func (c *Customer) CustomerAllRule() {
 	customerId := c.GetString("c_id")
 	history_id := c.GetString("hid")
@@ -1185,7 +1185,7 @@ func (c *Customer) CustomerAllRule() {
 	}
 }
 
-//保存历史规则
+// 保存历史规则
 func (c *Customer) SaveHistoryRule() {
 	rules_id := c.GetString("rulesid")
 	history_id := c.GetString("hid")
@@ -1378,7 +1378,7 @@ func (c *Customer) HistoryRuleCreate() {
 	}
 }
 
-//私有标签关联数据
+// 私有标签关联数据
 func (c *Customer) HistoryTagAssociated() {
 	defer qu.Catch()
 	userid := c.GetString("userid")
@@ -1472,7 +1472,7 @@ func AddHistoryLogTag(ruleData map[string]interface{}, isUpdate bool, id string)
 
 }
 
-//历史任务删除私有标签
+// 历史任务删除私有标签
 func (c *Customer) HistoryTagDel() {
 	defer qu.Catch()
 	// userid := c.GetString("userid")

src/util/config.go

@@ -6,7 +6,9 @@ import (
 	m "mongodb"
 	"qfw/mongodb"
 	qu "qfw/util"
-	"qfw/util/elastic"
+
+	// "qfw/util/elastic"
+	elastic "es"
 	"qfw/util/mysql"
 	"qfw/util/redis"
 	"sort"
@@ -42,7 +44,7 @@ var (
 	HospColl        string
 	BuyerEntColl    string
 	EnpsColl        string
-	Es              *elastic.Elastic //es
+	Es              elastic.Es //es
 	Index           string
 	Itype           string
 	TaskTime        int64 //定时任务时间
@@ -254,7 +256,11 @@ func initEs() {
 	EsAllIndex = qu.ObjToString(Sysconfig["elasticsearch_all_index"])
 	TotalIndex = qu.ObjToString(Sysconfig["elasticsearch_total_index"])
 	InterimIndex = qu.ObjToString(Sysconfig["elasticsearch_interim_index"])
-	elastic.InitElasticSize(qu.ObjToString(Sysconfig["elasticsearch"]), qu.IntAllDef(qu.Int64All(Sysconfig["elasticPoolSize"]), 30))
+	// elastic.InitElasticSize(qu.ObjToString(Sysconfig["elasticsearch"]), qu.IntAllDef(qu.Int64All(Sysconfig["elasticPoolSize"]), 30))
+	es := Sysconfig["es"].(map[string]interface{})
+	Es = elastic.NewEs(qu.ObjToString(es["version"]), qu.ObjToString(es["addr"]), qu.IntAllDef(es["pool"], 15), qu.ObjToString(es["userName"]), qu.ObjToString(es["password"]))
+	Index = qu.ObjToString(es["index"])
+	Itype = qu.ObjToString(es["itype"])
 }
 
 func initTagField() {
@@ -349,15 +355,11 @@ func initdb() {
 	MgoSite.InitPool()
 
 	//es
-	es := Sysconfig["es"].(map[string]interface{})
-	Es = &elastic.Elastic{
-		S_esurl: qu.ObjToString(es["addr"]),
-		I_size:  qu.IntAllDef(es["pool"], 15),
-	}
-	Es.InitElasticSize()
+	// Es = &elastic.Elastic{
+	// 	S_esurl: qu.ObjToString(es["addr"]),
+	// 	I_size:  qu.IntAllDef(es["pool"], 15),
+	// }
 	redis.InitRedis(qu.ObjToString(Sysconfig["redis_addrs"]))
-	Index = qu.ObjToString(es["index"])
-	Itype = qu.ObjToString(es["itype"])
 	jyMysql := qu.ObjToMap(Sysconfig["jyPushMysql"])
 	JyMysql = &mysql.Mysql{
 		Address:  (*jyMysql)["address"].(string),
@@ -403,7 +405,7 @@ func (s ByPinyin) Less(i, j int) bool {
 	return true
 }
 
-//UTF82GBK : transform UTF8 rune into GBK byte array
+// UTF82GBK : transform UTF8 rune into GBK byte array
 func UTF82GBK(src string) ([]byte, error) {
 	GB18030 := simplifiedchinese.All[0]
 	return ioutil.ReadAll(transform.NewReader(bytes.NewReader([]byte(src)), GB18030.NewEncoder()))

src/util/utiltag.go

@@ -72,7 +72,7 @@ func UtilEsSaveData(sdataid, index string, datas *[]map[string]interface{}) erro
 	return nil
 }
 
-//客户规则
+// 客户规则
 func UtilEsFind1(tags map[string]interface{}, dataType string) (error, string, int64) {
 	defer util.Catch()
 	sdataid := util.ObjToString(tags["s_dataid"])
@@ -273,10 +273,10 @@ func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
 	if json.Unmarshal([]byte(esquery), &query) == nil {
 		qb := query["query"]
 		for i := 0; i <= 2; i++ {
-			filter := qb.Filtered.Filter
-			if filter != nil && filter.Bool != nil { //有filter
+			filter := qb.Bool
+			if filter != nil { //有filter
 				index := -1 //记录range的位置
-				for i, m := range filter.Bool.Must {
+				for i, m := range filter.Must {
 					mMap := m.(map[string]interface{})
 					if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
 						if esRange["publishtime"] != nil {
@@ -295,9 +295,9 @@ func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
 					},
 				}
 				if index > -1 {
-					filter.Bool.Must[index] = tmpRange_
+					filter.Must[index] = tmpRange_
 				} else {
-					filter.Bool.Must = append(filter.Bool.Must, tmpRange_)
+					filter.Must = append(filter.Must, tmpRange_)
 				}
 				tmpQuery := query
 				strQuery, err := json.Marshal(tmpQuery)
@@ -308,8 +308,8 @@ func FilterTimeSql(esquery string, startTime, endTime int64) (string, error) {
 					return esquery, err
 				}
 			} else {
-				qb.Filtered.Filter = &sql.Filter{}
-				qb.Filtered.Filter.Bool = &sql.BoolObject{}
+				// qb.Filter = &sql.Filter{}
+				qb.Bool = &sql.BoolObject{}
 				continue
 			}
 		}
@@ -367,10 +367,10 @@ func UtilEsFind2(tags map[string]interface{}, n int) (error, int64) {
 	if json.Unmarshal([]byte(esquery), &query) == nil {
 
 		qb := query["query"]
-		filter := qb.Filtered.Filter
-		if filter != nil && filter.Bool != nil { //有filter
+		filter := qb.Bool
+		if filter != nil { //有filter
 			index := -1 //记录range的位置
-			for i, m := range filter.Bool.Must {
+			for i, m := range filter.Must {
 				mMap := m.(map[string]interface{})
 				if esRange, ok := mMap["range"].(map[string]interface{}); ok && esRange != nil { //有range
 					if esRange["publishtime"] != nil {
@@ -391,9 +391,9 @@ func UtilEsFind2(tags map[string]interface{}, n int) (error, int64) {
 					},
 				}
 				if index > -1 {
-					filter.Bool.Must[index] = tmpRange_
+					filter.Must[index] = tmpRange_
 				} else {
-					filter.Bool.Must = append(filter.Bool.Must, tmpRange_)
+					filter.Must = append(filter.Must, tmpRange_)
 				}
 				tmpQuery := query
 				strQuery, err := json.Marshal(tmpQuery)
@@ -629,46 +629,6 @@ func Utiltags(tag map[string]interface{}) string {
 			Lt: tab.Iendtime,
 		}}})
 	}
-	/*if tab.Sbudget != "" {
-		if strings.Contains(tab.Sbudget, "大于") && strings.Contains(tab.Sbudget, "小于") {
-			arr := strings.Split(tab.Sbudget, ",")
-			limit := &sql.BudgetOrBidamount{
-				Gte: util.Float64All(strings.Replace(arr[0], "大于", "", -1)),
-				Lt:  util.Float64All(strings.Replace(arr[1], "小于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
-		} else if strings.Contains(tab.Sbudget, "大于") {
-			limit := &sql.BudgetOrBidamount{
-				Gte: util.Float64All(strings.Replace(tab.Sbudget, "大于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
-		} else if strings.Contains(tab.Sbudget, "小于") {
-			limit := &sql.BudgetOrBidamount{
-				Lt: util.Float64All(strings.Replace(tab.Sbudget, "小于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
-		}
-	}*/
-	/*if tab.Sbidamount != "" {
-		if strings.Contains(tab.Sbidamount, "大于") && strings.Contains(tab.Sbidamount, "小于") {
-			arr := strings.Split(tab.Sbidamount, ",")
-			limit := &sql.BudgetOrBidamount{
-				Gte: util.Float64All(strings.Replace(arr[0], "大于", "", -1)),
-				Lt:  util.Float64All(strings.Replace(arr[1], "小于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
-		} else if strings.Contains(tab.Sbidamount, "大于") {
-			limit := &sql.BudgetOrBidamount{
-				Gte: util.Float64All(strings.Replace(tab.Sbidamount, "大于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
-		} else if strings.Contains(tab.Sbidamount, "小于") {
-			limit := &sql.BudgetOrBidamount{
-				Lt: util.Float64All(strings.Replace(tab.Sbidamount, "小于", "", -1)),
-			}
-			ffBoolObject.Must = append(ffBoolObject.Must, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
-		}
-	}*/
 	if tab.Sglobalbuyerclass != "" {
 		if len(tab.Sglobalbuyerclass) > 0 {
 			buyerclass := sql.BuyerclassObject{
@@ -696,7 +656,19 @@ func Utiltags(tag map[string]interface{}) string {
 	if len(tab.Sexistfields) > 0 {
 		tmpsfields := strings.Split(tab.Sexistfields, ",")
 		for _, v := range tmpsfields {
-			fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
+			// fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
+			// 	ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
+			// 		Missing struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		} `json:"missing,omitempty"`
+			// 	}{
+			// 		Missing: struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		}{Field: v},
+			// 	},
+			// 	},
+			// })
+			ffBoolObject.Must = append(ffBoolObject.Must, sql.ExistfieldsObjectMust{
 				ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
 					Missing struct {
 						Field string `json:"field,omitempty"`
@@ -816,7 +788,19 @@ func Utiltags(tag map[string]interface{}) string {
 			bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.BidamountMust{BidamountObj: &sql.BidamountObj{Bidamount: limit}})
 		}
 		if tab.SbidamountFieldExist == 1 {
-			bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.ExistfieldsObjectMust{
+			// bidamountSql.Bool.Should = append(bidamountSql.Bool.Should, sql.ExistfieldsObjectMust{
+			// 	ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
+			// 		Missing struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		} `json:"missing,omitempty"`
+			// 	}{
+			// 		Missing: struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		}{Field: "bidamount"},
+			// 	},
+			// 	},
+			// })
+			fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
 				ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
 					Missing struct {
 						Field string `json:"field,omitempty"`
@@ -852,7 +836,19 @@ func Utiltags(tag map[string]interface{}) string {
 			budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.BudgetMust{BudgetObj: &sql.BudgetObj{Budget: limit}})
 		}
 		if tab.SbudgetFieldExist == 1 {
-			budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.ExistfieldsObjectMust{
+			// budgetSql.Bool.Should = append(budgetSql.Bool.Should, sql.ExistfieldsObjectMust{
+			// 	ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
+			// 		Missing struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		} `json:"missing,omitempty"`
+			// 	}{
+			// 		Missing: struct {
+			// 			Field string `json:"field,omitempty"`
+			// 		}{Field: "budget"},
+			// 	},
+			// 	},
+			// })
+			fqBoolObject.MustNot = append(fqBoolObject.MustNot, sql.ExistfieldsObjectMust{
 				ExistfieldsObject: &sql.ExistfieldsObject{Filter: struct {
 					Missing struct {
 						Field string `json:"field,omitempty"`
@@ -868,43 +864,45 @@ func Utiltags(tag map[string]interface{}) string {
 	}
 
 	if len(ffBoolObject.Must) > 0 || len(ffBoolObject.MustNot) > 0 || len(ffBoolObject.Should) > 0 {
-		QueryObjecct.Filtered.Filter = &sql.Filter{}
-		QueryObjecct.Filtered.Filter.Bool = &ffBoolObject
+		// QueryObjecct.Filter = &sql.Filter{}
+		QueryObjecct.Bool = &ffBoolObject
 	}
 	if len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0 {
-		QueryObjecct.Filtered.Query = &sql.Query{}
-		QueryObjecct.Filtered.Query.Bool = &fqBoolObject
+		QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, &fqBoolObject)
 		if len(torules.Should) > 0 {
-			QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, map[string]interface{}{
+			QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
 				"bool": torules,
 			})
 		}
 	} else if len(torules.Should) > 0 {
-		if QueryObjecct.Filtered.Query == nil {
-			QueryObjecct.Filtered.Query = &sql.Query{}
-			QueryObjecct.Filtered.Query.Bool = &fqBoolObject
+		if QueryObjecct.Bool == nil {
+			// QueryObjecct.Query = &sql.Query{}
+			QueryObjecct.Bool = &sql.BoolObject{}
 		}
-		QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, map[string]interface{}{
+		QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, map[string]interface{}{
 			"bool": torules,
 		})
 	}
 	if (len(budgetSql.Bool.Should) > 0 || len(bidamountSql.Bool.Should) > 0) && (len(fqBoolObject.Must) > 0 || len(fqBoolObject.MustNot) > 0 || len(fqBoolObject.Should) > 0) {
+		if QueryObjecct.Bool == nil {
+			QueryObjecct.Bool = &sql.BoolObject{}
+		}
 		if len(budgetSql.Bool.Should) > 0 {
-			QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, budgetSql)
+			QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
 		}
 		if len(bidamountSql.Bool.Should) > 0 {
-			QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, bidamountSql)
+			QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
 		}
 	} else {
-		QueryObjecct.Filtered.Query = &sql.Query{}
-		QueryObjecct.Filtered.Query.Bool = &fqBoolObject
+		if QueryObjecct.Bool == nil {
+			QueryObjecct.Bool = &sql.BoolObject{}
+		}
 		if len(budgetSql.Bool.Should) > 0 {
-			QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, budgetSql)
+			QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, budgetSql)
 		}
 		if len(bidamountSql.Bool.Should) > 0 {
-			QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, bidamountSql)
+			QueryObjecct.Bool.Must = append(QueryObjecct.Bool.Must, bidamountSql)
 		}
-		//QueryObjecct.Filtered.Query.Bool.Must = append(QueryObjecct.Filtered.Query.Bool.Must, budgetSql)
 	}
 	rdata := make(map[string]interface{})
 	rdata["query"] = QueryObjecct