xg

customerdata/src/config.json

@@ -0,0 +1,17 @@
+{
+	"mgodb": "192.168.3.207:27092",
+	"dbsize": 10,
+	"dbname": "datatag",
+	"tasktime": 1,
+	"save": {
+		"addr": "192.168.3.207:27092",
+		"size": 15,
+		"db": "mxs"
+	},
+	"es": {
+		"addr": "http://127.0.0.1:9800",
+		"index": "bidding_v1",
+		"itype": "bidding",
+		"pool": 15
+	}
+}

customerdata/src/historytask.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"log"
+)
+
+//qu "qfw/util"
+
+func HistoryTask(sid, eid, customer string) {
+	log.Println("Start HistoryTask...")
+}

customerdata/src/main.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"flag"
+	qu "qfw/util"
+	"qfw/util/elastic"
+	"util/mgodb"
+)
+
+var (
+	Sysconfig map[string]interface{}
+	MgoTag    *mgodb.MongodbSim //标签库连接
+	MgoSave   *mgodb.MongodbSim //数据保存库连接
+	Es        *elastic.Elastic  //es
+	Index     string
+	Itype     string
+	TaskTime  int64 //定时任务时间
+	//历史数据
+	SId          string
+	EId          string
+	CustomerName string
+)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	//mgo datatag
+	MgoTag = &mgodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(Sysconfig["mgodb"]),
+		DbName:      qu.ObjToString(Sysconfig["dbname"]),
+		Size:        qu.IntAllDef(Sysconfig["dbsize"], 15),
+	}
+	MgoTag.InitPool()
+	//mgo save
+	save := Sysconfig["save"].(map[string]interface{})
+	MgoSave = &mgodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(save["addr"]),
+		DbName:      qu.ObjToString(save["db"]),
+		Size:        qu.IntAllDef(save["size"], 15),
+	}
+	MgoSave.InitPool()
+	//es
+	es := Sysconfig["es"].(map[string]interface{})
+	Es = &elastic.Elastic{
+		S_esurl: qu.ObjToString(es["addr"]),
+		I_size:  qu.IntAllDef(es["pool"], 15),
+	}
+	Es.InitElasticSize()
+	Index = qu.ObjToString(es["index"])
+	Itype = qu.ObjToString(es["itype"])
+	//
+	TaskTime = qu.Int64All(Sysconfig["tasktime"])
+
+}
+func main() {
+	go TimeTask() //定时任务
+	flag.StringVar(&SId, "sid", "", "起始id")
+	flag.StringVar(&EId, "eid", "", "结束id")
+	flag.StringVar(&CustomerName, "customer", "", "客户名称")
+	flag.Parse()
+	if SId != "" && EId != "" && CustomerName != "" {
+		go HistoryTask(SId, EId, CustomerName)
+	}
+	ch := make(chan bool, 1)
+	<-ch
+}

customerdata/src/model/model.go

@@ -0,0 +1,146 @@
+package model
+
+type QueryObjecct struct {
+	Filtered struct {
+		Filter *Filter `json:"filter,omitempty"`
+		Query  *Query  `json:"query,omitempty"`
+	} `json:"filtered,omitempty"`
+}
+
+type Query struct {
+	Bool *BoolObject `json:"bool,omitempty"`
+}
+
+type Filter struct {
+	Bool *BoolObject `json:"bool,omitempty"`
+}
+
+//省市县
+type AreaCityDistrictMust struct {
+	*AreaCityDistrict `json:"terms,omitempty"`
+}
+
+//省市县
+type AreaCityDistrict struct {
+	Area     []string `json:"area,omitempty"`
+	City     []string `json:"city,omitempty"`
+	District []string `json:"district,omitempty"`
+}
+
+type DistrictObject struct {
+	Term struct {
+		City     string `json:"city,omitempty"`
+		District string `json:"district,omitempty"`
+	} `json:"term,omitempty"`
+}
+
+//一级分类、二级分类
+type ToptypeSubtypeMust struct {
+	*ToptypeSubtype `json:"terms,omitempty"`
+}
+
+//一级分类、二级分类
+type ToptypeSubtype struct {
+	Toptype []string `json:"toptype,omitempty"`
+	Subtype []string `json:"subtype,omitempty"`
+}
+
+//公告类型
+type ScopeclassMust struct {
+	*Scopeclass `json:"terms,omitempty"`
+}
+type Scopeclass struct {
+	Globaltopscopeclass []string `json:"s_topscopeclass,omitempty"`
+	Globalsubscopeclass []string `json:"s_subscopeclass,omitempty"`
+}
+
+type BudgetMust struct {
+	BudgetObj *BudgetObj `json:"range,omitempty"`
+}
+type BudgetObj struct {
+	Budget *BudgetOrBidamount `json:"budget,omitempty"`
+}
+type BidamountMust struct {
+	BidamountObj *BidamountObj `json:"range,omitempty"`
+}
+type BidamountObj struct {
+	Bidamount *BudgetOrBidamount `json:"bidamount,omitempty"`
+}
+type BudgetOrBidamount struct {
+	Gte float64 `json:"gte,omitempty"`
+	Lt  float64 `json:"lt,omitempty"`
+}
+
+//时间范围
+type PublishtimeMust struct {
+	PublishtimeObject *PublishtimeObject `json:"range,omitempty"`
+}
+
+//时间范围
+type PublishtimeObject struct {
+	//查询时间
+	Publishtime  *Publishtime  `json:"publishtime,omitempty"`
+	Publishtime1 *Publishtime1 `json:"publishtime1,omitempty"`
+	Publishtime2 *Publishtime2 `json:"publishtime2,omitempty"`
+}
+
+//查询时间
+type Publishtime struct {
+	Gte int `json:"gte"`
+	Lt  int `json:"lt"`
+}
+type Publishtime1 struct {
+	Gte int `json:"gte"`
+}
+type Publishtime2 struct {
+	Lt int `json:"lt"`
+}
+
+//字段是或否存在
+type ExistfieldsObjectMust struct {
+	ExistfieldsObject *ExistfieldsObject `json:"constant_score,omitempty"`
+}
+
+//字段存在 存在放MustNot，字符串为空忽略
+type ExistfieldsObject struct {
+	Filter struct {
+		Missing struct {
+			Field string `json:"field,omitempty"`
+		} `json:"missing,omitempty"`
+	} `json:"filter,omitempty"`
+}
+
+type ShouldObj struct {
+	MatchPhrase *MatchPhrase `json:"match_phrase,omitempty"`
+	MultiMatch  *MultiMatch  `json:"multi_match,omitempty"`
+}
+
+//匹配方式
+type MatchPhrase struct {
+	Title       string `json:"title,omitempty"`
+	Detail      string `json:"detail,omitempty"`
+	Purchasing  string `json:"purchasing,omitempty"`
+	Attachments string `json:"attachments,omitempty"`
+	Projectname string `json:"projectname,omitempty"`
+}
+
+type MultiMatch struct {
+	Query  string   `json:"query,omitempty"`
+	Type   string   `json:"type,omitempty"`
+	Fields []string `json:"fields,omitempty"`
+}
+
+//xf should
+type NewEsObject struct {
+	Bool struct {
+		Must    []interface{} `json:"must,omitempty"`
+		MustNot []interface{} `json:"must_not,omitempty"`
+		Should  []interface{} `json:"should,omitempty"`
+	} `json:"bool,omitempty"`
+}
+
+type BoolObject struct {
+	Must    []interface{} `json:"must,omitempty"`
+	MustNot []interface{} `json:"must_not,omitempty"`
+	Should  []interface{} `json:"should,omitempty"`
+}

customerdata/src/task.go

@@ -0,0 +1,188 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	. "model"
+	qu "qfw/util"
+
+	"sync"
+	"time"
+
+	"github.com/cron"
+	bson "gopkg.in/mgo.v2/bson"
+	es "gopkg.in/olivere/elastic.v1"
+)
+
+var CustomerArr []*Customer
+
+type Customer struct {
+	Name    string   //客户名称
+	AppId   string   //
+	EsQuery []string //客户所有标签的es语句
+	//Coll  string   //保存的表
+}
+
+func TimeTask() {
+	//InitCustomer()
+	c := cron.New()
+	cronstr := "0 */" + fmt.Sprint(TaskTime) + " * * * ?"
+	//cronstr := "0 0 */" + fmt.Sprint(TaskTime) + " * * ?"//每TaskTime小时执行一次
+	qu.Debug("cronstr:", cronstr)
+	c.AddFunc(cronstr, func() { Start() })
+	c.Start()
+}
+func Start() {
+	InitCustomer()
+	GetCustomerData()
+}
+
+//加载客户
+func InitCustomer() {
+	qu.Debug("Init Customer...")
+	tmpRange := GetIdRange() //获取id区间
+	qu.Debug(tmpRange)
+	return
+	customers, _ := MgoTag.Find("customer", map[string]interface{}{"i_use": 1}, nil, nil)
+	for _, c := range customers {
+		csr := &Customer{}
+		esArr := []string{}
+		customer := qu.ObjToString(c["s_customername"])
+		tags, _ := MgoTag.Find("tags", map[string]interface{}{"s_customer": customer}, nil, map[string]interface{}{"s_esquery": 1})
+		for _, t := range tags {
+			esquery := qu.ObjToString(t["s_esquery"])
+			query := map[string]*QueryObjecct{}
+			if json.Unmarshal([]byte(esquery), &query) == nil {
+				qb := query["query"]
+				filter := qb.Filtered.Filter
+				if filter != nil && filter.Bool != nil { //有filter
+					index := 0 //记录range的位置
+					for i, m := range filter.Bool.Must {
+						mMap := m.(map[string]interface{})
+						if mMap["range"] != nil { //有range
+							index = i
+							break
+						}
+					}
+					if index != 0 {
+						filter.Bool.Must[index] = tmpRange
+					} else {
+						filter.Bool.Must = append(filter.Bool.Must, tmpRange)
+					}
+				} else { //无filter则添加
+					bo := &BoolObject{}
+					bo.Must = append(bo.Must, tmpRange)
+					tmpFilter := &Filter{
+						Bool: bo,
+					}
+					qb.Filtered.Filter = tmpFilter
+				}
+				strquery, _ := json.Marshal(query)
+				qu.Debug("strquery---", string(strquery))
+				esArr = append(esArr, string(strquery))
+			}
+		}
+		if len(esArr) > 0 {
+			csr.EsQuery = esArr
+			CustomerArr = append(CustomerArr, csr)
+		}
+	}
+	qu.Debug(len(CustomerArr))
+}
+
+func GetCustomerData() {
+	for _, customer := range CustomerArr {
+		customer.GetData()
+	}
+	CustomerArr = []*Customer{} //置空
+}
+
+func (c *Customer) GetData() {
+	client := Es.GetEsConn()
+	defer Es.DestoryEsConn(client)
+	for _, esquery := range c.EsQuery {
+		ch := make(chan bool, 20)
+		wg := &sync.WaitGroup{}
+		escount := Es.Count(Index, Itype, esquery)
+		log.Println("查询总数:", escount, "查询条件:", esquery)
+		if escount == 0 {
+			return
+		}
+		//查询条件类型转换
+		var q es.Query
+		tmpQuery := es.BoolQuery{
+			QueryStrings: esquery,
+		}
+		q = tmpQuery
+
+		//游标查询，index不支持别名，只能写索引库的名称
+		res, err := client.Scroll(Index).Query(q).Size(1000).Do() //查询一条获取游标
+		if err == nil {
+			numDocs := 0
+			numTags := int64(0)
+			scrollId := res.ScrollId
+			for {
+				if scrollId == "" {
+					log.Println("ScrollId Is Error")
+					break
+				}
+				searchResult, err := client.Scroll(Index).Size(1000).ScrollId(scrollId).Do() //查询
+				if err != nil {
+					if err.Error() == "EOS" { //迭代完毕
+						log.Println("Es Search Data Over：", err)
+					} else {
+						log.Println("Es Search Data Error：", err)
+					}
+					break
+				}
+
+				for _, hit := range searchResult.Hits.Hits {
+					//开始处理数据
+					wg.Add(1)
+					ch <- true
+					go func(tmpHit *es.SearchHit) {
+						defer func() {
+							<-ch
+							wg.Done()
+						}()
+						tmp := make(map[string]interface{})
+						if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
+							numTags++
+						}
+					}(hit)
+					wg.Wait()
+					numDocs += 1
+					if numDocs%500 == 0 {
+						//log.Println("Current：", numDocs)
+					}
+				}
+				scrollId = searchResult.ScrollId
+			}
+			client.ClearScroll().ScrollId(scrollId).Do() //清理游标
+			log.Println("Result Data Count：", numDocs, "	Tags Data Count：", numTags)
+		}
+	}
+}
+
+//根据时间获取起始和终止ID范围
+func GetIdRange() bson.M {
+	now := time.Now()
+	end := now.Unix() - int64(60*now.Minute()) - int64(now.Second())
+	start := end - TaskTime*3600
+	endTime := time.Unix(end, 0)
+	startTime := time.Unix(start, 0)
+	eid := bson.NewObjectIdWithTime(endTime).Hex()
+	sid := bson.NewObjectIdWithTime(startTime).Hex()
+	// sid = "5cf14f800000000000000000"
+	// eid = "5ed3d4800000000000000000"
+	tmpRange := bson.M{
+		"range": bson.M{
+			"id": bson.M{
+				"gte": sid,
+				"lte": eid,
+			},
+		},
+	}
+	return tmpRange
+}

src/customerdata/src/config.json

@@ -1,10 +0,0 @@
-{
-	"mgodb": "192.168.3.207:27092",
-	"dbsize": 5,
-	"dbname": "mxs",
-	"save": {
-		"addr": "192.168.3.207:27092",
-		"size": 15,
-		"db": "mxs"
-	}
-}

src/customerdata/src/main.go

@@ -1,18 +0,0 @@
-package main
-
-import (
-	qu "qfw/util"
-)
-
-var (
-	Sysconfig map[string]interface{}
-	//MgoD      *u.MongodbSim
-)
-
-func init() {
-	qu.ReadConfig(&Sysconfig)
-}
-func main() {
-	ch := make(chan bool, 1)
-	<-ch
-}

src/customerdata/src/task.go

@@ -1,7 +0,0 @@
-package main
-
-func Init() {
-
-}
-
-func TimeTask() {}

src/tagservice/src/historytask.go → tagservice/src/historytask.go

@@ -129,69 +129,70 @@ func MgoHistoryData() {
 			}()
 			tmpTagNameMap := map[string][]string{} //记录标签
 			tmpPreTagMap := map[string][]string{}  //记录父标签
-
-			for _, ru := range MgoRulesH {
-				//全局排除词
-				IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
-				if IsMatchGNotKey { //全局排除词匹配成功
-					continue
-				} else {
-					//全局附加词
-					IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
-					if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
-						continue
-					}
-				}
-			L:
-				for _, r := range ru.Rule {
-					IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
-					if IsMatchNotKey { //排除词匹配成功,过滤当前rule
-						continue
-					}
-					//关键词匹配
-					for _, kwm := range r.KW.MatchType {
-						if text := qu.ObjToString(tmp[kwm]); text != "" {
-							text = ProcessData(text)
-							for i, kw_reg := range r.KW.KeyReg {
-								IsContinue := false
-								if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
-									if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-										IsContinue = true
-									} else if !r.KW.KeyWordMap[i] {
-										IsContinue = true
-									}
-								}
-								if IsContinue { //关键词匹配成功,匹配附加词
-									if len(r.AW.KeyReg) == 0 { //无附加词
-										//tmpTagNameMap[r.TagName] = true
-										RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-										break L
-									} else {
-										for _, awm := range r.AW.MatchType {
-											if text := qu.ObjToString(tmp[awm]); text != "" {
-												text = ProcessData(text)
-												for j, aw_reg := range r.AW.KeyReg {
-													if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
-														if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-															//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-															RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-															break L
-														} else if !r.AW.AddWordMap[j] {
-															//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-															RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-															break L
-														}
-													}
-												}
-											}
-										}
-									}
-								}
-							}
-						}
-					}
-				}
-			}
+			GetTags(MgoRulesH, tmp, tmpTagNameMap, tmpPreTagMap)
+			// for _, ru := range MgoRulesH {
+			// 	//全局排除词
+			// 	IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
+			// 	if IsMatchGNotKey { //全局排除词匹配成功
+			// 		continue
+			// 	} else {
+			// 		//全局附加词
+			// 		IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
+			// 		if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
+			// 			continue
+			// 		}
+			// 	}
+			// L:
+			// 	for _, r := range ru.Rule {
+			// 		//
+			// 		IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
+			// 		if IsMatchNotKey { //排除词匹配成功,过滤当前rule
+			// 			continue
+			// 		}
+			// 		//关键词匹配
+			// 		for _, kwm := range r.KW.MatchType {
+			// 			if text := qu.ObjToString(tmp[kwm]); text != "" {
+			// 				text = ProcessData(text)
+			// 				for i, kw_reg := range r.KW.KeyReg {
+			// 					IsContinue := false
+			// 					if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
+			// 						if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+			// 							IsContinue = true
+			// 						} else if !r.KW.KeyWordMap[i] {
+			// 							IsContinue = true
+			// 						}
+			// 					}
+			// 					if IsContinue { //关键词匹配成功,匹配附加词
+			// 						if len(r.AW.KeyReg) == 0 { //无附加词
+			// 							//tmpTagNameMap[r.TagName] = true
+			// 							RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+			// 							break L
+			// 						} else {
+			// 							for _, awm := range r.AW.MatchType {
+			// 								if text := qu.ObjToString(tmp[awm]); text != "" {
+			// 									text = ProcessData(text)
+			// 									for j, aw_reg := range r.AW.KeyReg {
+			// 										if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
+			// 											if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+			// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+			// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+			// 												break L
+			// 											} else if !r.AW.AddWordMap[j] {
+			// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+			// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+			// 												break L
+			// 											}
+			// 										}
+			// 									}
+			// 								}
+			// 							}
+			// 						}
+			// 					}
+			// 				}
+			// 			}
+			// 		}
+			// 	}
+			// }
 			addMap := map[string]interface{}{}
 			if len(tmpTagNameMap) > 0 { //有新标签或者历史标签
 				atomic.AddInt64(&n, +1) //n++ 计数
@@ -291,69 +292,70 @@ func EsHistoryData() {
 					if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
 						tmpTagNameMap := map[string][]string{} //记录标签
 						tmpPreTagMap := map[string][]string{}  //记录父标签
-						for _, ru := range EsRulesH {
-							//全局排除词
-							IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
-							if IsMatchGNotKey { //全局排除词匹配成功
-								continue
-							} else {
-								//全局附加词
-								IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
-								if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
-									continue
-								}
-							}
-						L:
-							for _, r := range ru.Rule {
-								//排除词
-								IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
-								if IsMatchNotKey { //排除词匹配成功,过滤当前rule
-									continue
-								}
-								//关键词匹配
-								for _, kwm := range r.KW.MatchType {
-									if text := qu.ObjToString(tmp[kwm]); text != "" {
-										text = ProcessData(text)
-										for i, kw_reg := range r.KW.KeyReg {
-											IsContinue := false
-											if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
-												if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-													IsContinue = true
-												} else if !r.KW.KeyWordMap[i] {
-													IsContinue = true
-												}
-											}
-											if IsContinue { //关键词匹配成功,匹配附加词
-												if len(r.AW.KeyReg) == 0 { //无附加词
-													//tmpTagNameMap[r.TagName] = true
-													RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-													break L
-												} else {
-													for _, awm := range r.AW.MatchType {
-														if text := qu.ObjToString(tmp[awm]); text != "" {
-															text = ProcessData(text)
-															for j, aw_reg := range r.AW.KeyReg {
-																if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
-																	if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-																		//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-																		RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-																		break L
-																	} else if !r.AW.AddWordMap[j] {
-																		//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-																		RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-																		break L
-																	}
-																}
-															}
-														}
-													}
-												}
-											}
-										}
-									}
-								}
-							}
-						}
+						GetTags(EsRulesH, tmp, tmpTagNameMap, tmpPreTagMap)
+						// for _, ru := range EsRulesH {
+						// 	//全局排除词
+						// 	IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
+						// 	if IsMatchGNotKey { //全局排除词匹配成功
+						// 		continue
+						// 	} else {
+						// 		//全局附加词
+						// 		IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
+						// 		if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
+						// 			continue
+						// 		}
+						// 	}
+						// L:
+						// 	for _, r := range ru.Rule {
+						// 		//排除词
+						// 		IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
+						// 		if IsMatchNotKey { //排除词匹配成功,过滤当前rule
+						// 			continue
+						// 		}
+						// 		//关键词匹配
+						// 		for _, kwm := range r.KW.MatchType {
+						// 			if text := qu.ObjToString(tmp[kwm]); text != "" {
+						// 				text = ProcessData(text)
+						// 				for i, kw_reg := range r.KW.KeyReg {
+						// 					IsContinue := false
+						// 					if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
+						// 						if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+						// 							IsContinue = true
+						// 						} else if !r.KW.KeyWordMap[i] {
+						// 							IsContinue = true
+						// 						}
+						// 					}
+						// 					if IsContinue { //关键词匹配成功,匹配附加词
+						// 						if len(r.AW.KeyReg) == 0 { //无附加词
+						// 							//tmpTagNameMap[r.TagName] = true
+						// 							RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 							break L
+						// 						} else {
+						// 							for _, awm := range r.AW.MatchType {
+						// 								if text := qu.ObjToString(tmp[awm]); text != "" {
+						// 									text = ProcessData(text)
+						// 									for j, aw_reg := range r.AW.KeyReg {
+						// 										if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
+						// 											if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+						// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+						// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 												break L
+						// 											} else if !r.AW.AddWordMap[j] {
+						// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+						// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 												break L
+						// 											}
+						// 										}
+						// 									}
+						// 								}
+						// 							}
+						// 						}
+						// 					}
+						// 				}
+						// 			}
+						// 		}
+						// 	}
+						// }
 						addMap := map[string]interface{}{}
 						if len(tmpTagNameMap) > 0 { //有新标签或者历史标签
 							atomic.AddInt64(&numTags, +1) //n++ 计数

src/tagservice/src/newtask.go → tagservice/src/newtask.go

@@ -200,74 +200,7 @@ func (t *Task) RunMgo() {
 			}()
 			tmpTagNameMap := map[string][]string{} //记录标签
 			tmpPreTagMap := map[string][]string{}  //记录父标签
-			for _, ru := range t.PRules {
-				//全局排除词
-				IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
-				if IsMatchGNotKey { //全局排除词匹配成功
-					continue
-				} else {
-					//全局附加词
-					IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
-					if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
-						continue
-					}
-				}
-			L:
-				for _, r := range ru.Rule {
-					//排除词
-					IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
-					if IsMatchNotKey { //排除词匹配成功
-						continue
-					}
-					//L:
-					//关键词匹配
-					for _, kwm := range r.KW.MatchType {
-						if text := qu.ObjToString(tmp[kwm]); text != "" {
-							text = ProcessData(text)
-							for i, kw_reg := range r.KW.KeyReg {
-								IsContinue := false
-								if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
-									if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-										IsContinue = true
-									} else if !r.KW.KeyWordMap[i] {
-										IsContinue = true
-									}
-								}
-								if IsContinue { //关键词匹配成功,匹配附加词
-									if len(r.AW.KeyReg) == 0 { //无附加词
-										//tmpTagNameMap[r.TagName] = true
-										RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-										break L
-									} else {
-										for _, awm := range r.AW.MatchType {
-											if text := qu.ObjToString(tmp[awm]); text != "" {
-												text = ProcessData(text)
-												for j, aw_reg := range r.AW.KeyReg {
-													if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
-														if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-															//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-															RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-															break L
-														} else if !r.AW.AddWordMap[j] {
-															//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-															RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-															break L
-														}
-													}
-													// if aw_reg.MatchString(text) { //附加词匹配成功
-													// 	tmpTagName[r.TagName] = true
-													// 	break L
-													// }
-												}
-											}
-										}
-									}
-								}
-							}
-						}
-					}
-				}
-			}
+			GetTags(t.PRules, tmp, tmpTagNameMap, tmpPreTagMap)
 			clearMap := map[string]interface{}{}
 			if t.IsClear { //清理
 				ClearData(clearMap, t.AllTagField, tmpTagNameMap)
@@ -413,106 +346,71 @@ func (t *Task) RunEs() {
 					if json.Unmarshal(*tmpHit.Source, &tmp) == nil {
 						tmpTagNameMap := map[string][]string{} //记录标签
 						tmpPreTagMap := map[string][]string{}  //记录父标签
-						for _, ru := range t.PRules {
-							//全局排除词
-							IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
-							if IsMatchGNotKey { //全局排除词匹配成功
-								continue
-							} else {
-								//全局附加词
-								IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
-								if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
-									continue
-								}
-							}
-						L:
-							for _, r := range ru.Rule {
-								// IsMatch := false
-								// L1:
-								// 	//排除词匹配
-								// 	for _, nwm := range r.NW.MatchType {
-								// 		if text := qu.ObjToString(tmp[nwm]); text != "" {
-								// 			text = ProcessData(text)
-								// 			//i=0时,nw_regArr任意一个匹配表示有排除词
-								// 			//i=1时,nw_regArr所有匹配表示有排除词
-								// 			for i, nw_regArr := range r.NW.KeyReg {
-								// 				andMatchNum := 0
-								// 				for j, nw_reg := range nw_regArr {
-								// 					if nw_indexArr := nw_reg.FindAllStringIndex(text, -1); len(nw_indexArr) > 0 { //排除词匹配成功
-								// 						if r.NW.NotWordMap[i][j] && CheckLetter(text, nw_reg, nw_indexArr) { //nw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-								// 							andMatchNum++
-								// 							if i == 0 {
-								// 								IsMatch = true
-								// 								break L1
-								// 							} else if i == 1 && len(nw_regArr) == andMatchNum {
-								// 								IsMatch = true
-								// 								break L1
-								// 							}
-								// 						} else if !r.NW.NotWordMap[i][j] {
-								// 							andMatchNum++
-								// 							if i == 0 {
-								// 								IsMatch = true
-								// 								break L1
-								// 							} else if i == 1 && len(nw_regArr) == andMatchNum {
-								// 								IsMatch = true
-								// 								break L1
-								// 							}
-								// 						}
-								// 					}
-								// 				}
-								// 			}
-								// 		}
-								// 	}
-								//排除词
-								IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
-								if IsMatchNotKey { //排除词匹配成功,过滤当前rule
-									continue
-								}
-								// L:
-								//关键词匹配
-								for _, kwm := range r.KW.MatchType {
-									if text := qu.ObjToString(tmp[kwm]); text != "" {
-										text = ProcessData(text)
-										for i, kw_reg := range r.KW.KeyReg {
-											IsContinue := false
-											if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
-												if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-													IsContinue = true
-												} else if !r.KW.KeyWordMap[i] {
-													IsContinue = true
-												}
-											}
-											if IsContinue { //关键词匹配成功,匹配附加词
-												if len(r.AW.KeyReg) == 0 { //无附加词
-													//tmpTagNameMap[r.TagName] = true
-													RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-													break L
-												} else {
-													for _, awm := range r.AW.MatchType {
-														if text := qu.ObjToString(tmp[awm]); text != "" {
-															text = ProcessData(text)
-															for j, aw_reg := range r.AW.KeyReg {
-																if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
-																	if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
-																		//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-																		RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-																		break L
-																	} else if !r.AW.AddWordMap[j] {
-																		//tmpTagNameMap[r.TagName] = true //附加词匹配成功
-																		RecordData(ru, tmpTagNameMap, tmpPreTagMap)
-																		break L
-																	}
-																}
-															}
-														}
-													}
-												}
-											}
-										}
-									}
-								}
-							}
-						}
+						GetTags(t.PRules, tmp, tmpTagNameMap, tmpPreTagMap)
+						// for _, ru := range t.PRules {
+						// 	//全局排除词
+						// 	IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
+						// 	if IsMatchGNotKey { //全局排除词匹配成功
+						// 		continue
+						// 	} else {
+						// 		//全局附加词
+						// 		IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
+						// 		if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
+						// 			continue
+						// 		}
+						// 	}
+						// L:
+						// 	for _, r := range ru.Rule {
+						// 		//排除词
+						// 		IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
+						// 		if IsMatchNotKey { //排除词匹配成功,过滤当前rule
+						// 			continue
+						// 		}
+						// 		// L:
+						// 		//关键词匹配
+						// 		for _, kwm := range r.KW.MatchType {
+						// 			if text := qu.ObjToString(tmp[kwm]); text != "" {
+						// 				text = ProcessData(text)
+						// 				for i, kw_reg := range r.KW.KeyReg {
+						// 					IsContinue := false
+						// 					if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
+						// 						if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+						// 							IsContinue = true
+						// 						} else if !r.KW.KeyWordMap[i] {
+						// 							IsContinue = true
+						// 						}
+						// 					}
+						// 					if IsContinue { //关键词匹配成功,匹配附加词
+						// 						if len(r.AW.KeyReg) == 0 { //无附加词
+						// 							//tmpTagNameMap[r.TagName] = true
+						// 							RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 							break L
+						// 						} else {
+						// 							for _, awm := range r.AW.MatchType {
+						// 								if text := qu.ObjToString(tmp[awm]); text != "" {
+						// 									text = ProcessData(text)
+						// 									for j, aw_reg := range r.AW.KeyReg {
+						// 										if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
+						// 											if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+						// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+						// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 												break L
+						// 											} else if !r.AW.AddWordMap[j] {
+						// 												//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+						// 												RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+						// 												break L
+						// 											}
+						// 										}
+						// 									}
+						// 								}
+						// 							}
+						// 						}
+						// 					}
+						// 				}
+						// 			}
+						// 		}
+						// 	}
+						// }
 						clearMap := map[string]interface{}{}
 						if t.IsClear { //清理
 							ClearData(clearMap, t.AllTagField, tmpTagNameMap)
@@ -932,6 +830,77 @@ func InitRules(tasktype string) (rules []*PRule) {
 	return
 }
 
+func GetTags(prules []*PRule, tmp map[string]interface{}, tmpTagNameMap, tmpPreTagMap map[string][]string) {
+	for _, ru := range prules {
+		//全局排除词
+		IsMatchGNotKey := RegMatch(tmp, ru.GNW.MatchType, ru.GNW.KeyReg, ru.GNW.NotWordMap)
+		if IsMatchGNotKey { //全局排除词匹配成功
+			continue
+		} else {
+			//全局附加词
+			IsMatchGAddKey := RegMatch(tmp, ru.GAW.MatchType, ru.GAW.KeyReg, ru.GAW.MatchWordMap)
+			if !IsMatchGAddKey && len(ru.GAW.MatchType) != 0 { //全局附加词没有匹配成功
+				continue
+			}
+		}
+	L:
+		for _, r := range ru.Rule {
+			//排除词
+			IsMatchNotKey := RegMatch(tmp, r.NW.MatchType, r.NW.KeyReg, r.NW.NotWordMap)
+			if IsMatchNotKey { //排除词匹配成功
+				continue
+			}
+			//L:
+			//关键词匹配
+			for _, kwm := range r.KW.MatchType {
+				if text := qu.ObjToString(tmp[kwm]); text != "" {
+					text = ProcessData(text)
+					for i, kw_reg := range r.KW.KeyReg {
+						IsContinue := false
+						if kw_indexArr := kw_reg.FindAllStringIndex(text, -1); len(kw_indexArr) > 0 { //关键词匹配成功
+							if r.KW.KeyWordMap[i] && CheckLetter(text, kw_reg, kw_indexArr) { //kw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+								IsContinue = true
+							} else if !r.KW.KeyWordMap[i] {
+								IsContinue = true
+							}
+						}
+						if IsContinue { //关键词匹配成功,匹配附加词
+							if len(r.AW.KeyReg) == 0 { //无附加词
+								//tmpTagNameMap[r.TagName] = true
+								RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+								break L
+							} else {
+								for _, awm := range r.AW.MatchType {
+									if text := qu.ObjToString(tmp[awm]); text != "" {
+										text = ProcessData(text)
+										for j, aw_reg := range r.AW.KeyReg {
+											if aw_indexArr := aw_reg.FindAllStringIndex(text, -1); len(aw_indexArr) > 0 { //关键词匹配成功
+												if r.AW.AddWordMap[j] && CheckLetter(text, aw_reg, aw_indexArr) { //aw_reg有字母，判断是否是包含关系(AAAIBBB or AI)
+													//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+													RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+													break L
+												} else if !r.AW.AddWordMap[j] {
+													//tmpTagNameMap[r.TagName] = true //附加词匹配成功
+													RecordData(ru, tmpTagNameMap, tmpPreTagMap)
+													break L
+												}
+											}
+											// if aw_reg.MatchString(text) { //附加词匹配成功
+											// 	tmpTagName[r.TagName] = true
+											// 	break L
+											// }
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
 //匹配
 func RegMatch(tmp map[string]interface{}, matchType []string, matchReg [][]*regexp.Regexp, matchMap []map[int]bool) bool {
 	defer qu.Catch()