|
|
@@ -5,21 +5,23 @@ import (
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"log"
|
|
|
+ qu "qfw/util"
|
|
|
"qfw/util/redis"
|
|
|
"regexp"
|
|
|
+ . "sqlmodel"
|
|
|
"strings"
|
|
|
"sync"
|
|
|
"time"
|
|
|
mgoutil "util/mgodb"
|
|
|
|
|
|
- "gopkg.in/util.Mgo.v2/bson"
|
|
|
+ "gopkg.in/mgo.v2/bson"
|
|
|
|
|
|
"github.com/cron"
|
|
|
|
|
|
esv "es"
|
|
|
|
|
|
- "app.yhyue.com/moapp/jybase/logger"
|
|
|
"github.com/antonmedv/expr"
|
|
|
+ "github.com/donnie4w/go-logger/logger"
|
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
|
|
|
|
esV7 "github.com/olivere/elastic"
|
|
|
@@ -39,7 +41,7 @@ func StartTask() {
|
|
|
|
|
|
// 加载客户
|
|
|
func GetCustomerData() {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
log.Println("Init Customer...")
|
|
|
idRange, idRange2, ok, endTime := GetIdRange() //获取id区间
|
|
|
if !ok {
|
|
|
@@ -50,11 +52,11 @@ func GetCustomerData() {
|
|
|
customers, _ := MgoTag.Find("euser", map[string]interface{}{"i_push": 1, "b_delete": false}, nil, nil)
|
|
|
for _, c := range customers {
|
|
|
customerId := mgoutil.BsonTOStringId(c["_id"])
|
|
|
- customer := common.ObjToString(c["s_name"]) //客户名称
|
|
|
- appId := common.ObjToString(c["s_appid"]) //appid
|
|
|
- extends := common.ObjToString(c["s_extends"]) //扩展信息
|
|
|
- pushModel := common.IntAll(c["i_pushmodel"]) //推送模式
|
|
|
- dataSave := common.ObjToString(c["s_dataSave"])
|
|
|
+ customer := qu.ObjToString(c["s_name"]) //客户名称
|
|
|
+ appId := qu.ObjToString(c["s_appid"]) //appid
|
|
|
+ extends := qu.ObjToString(c["s_extends"]) //扩展信息
|
|
|
+ pushModel := qu.IntAll(c["i_pushmodel"]) //推送模式
|
|
|
+ dataSave := qu.ObjToString(c["s_dataSave"])
|
|
|
log.Println("当前客户 ", customer)
|
|
|
cus := &Customer{}
|
|
|
cus.SaveDataMap = map[string]map[string]interface{}{}
|
|
|
@@ -87,37 +89,37 @@ func GetCustomerData() {
|
|
|
cus.GetTagRules() //获取客户打标签规则
|
|
|
cus.GetDepartments("") //获取客户信息
|
|
|
//PrintLog(cus) //打印查看初始化的信息
|
|
|
- common.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsTagRule2, cus.IsTagRule3, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.TagRules2), len(cus.TagRules3), len(cus.Departments))
|
|
|
+ qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsTagRule2, cus.IsTagRule3, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.TagRules2), len(cus.TagRules3), len(cus.Departments))
|
|
|
cus.GetData("") //获取数据
|
|
|
cus.RemoveRepeatData() //数据去重
|
|
|
cus.AssembelAndSaveData() //组装、保存数据
|
|
|
}
|
|
|
Sysconfig.LatestId = LatestId
|
|
|
Sysconfig.LatestTime = endTime
|
|
|
- common.WriteSysConfig(Sysconfig)
|
|
|
+ qu.WriteSysConfig(Sysconfig)
|
|
|
logger.Debug("定时任务结束-endId-Sysconfig.LatestTime ", endTime)
|
|
|
}
|
|
|
|
|
|
// 获取客户打标签规则
|
|
|
func (c *Customer) GetTagRules() {
|
|
|
log.Println("开始加载标签规则...")
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
tagRules, _ := MgoTag.Find("eusertagrule", map[string]interface{}{"s_userid": c.ID, "i_isuse": 1, "b_delete": false}, nil, nil)
|
|
|
if len(tagRules) > 0 {
|
|
|
for _, tr := range tagRules {
|
|
|
- tagType := common.ObjToString(tr["tagType"])
|
|
|
+ tagType := qu.ObjToString(tr["tagType"])
|
|
|
if tagType == "" || tagType == "1" {
|
|
|
c.IsTagRule = true //查到打标签规则，表示打标签
|
|
|
TR := &TagRule{}
|
|
|
TR.Fields = make(map[string]interface{})
|
|
|
TR.DepartRuleIds = make(map[string]bool)
|
|
|
id := mgoutil.BsonTOStringId(tr["_id"])
|
|
|
- name := common.ObjToString(tr["s_name"])
|
|
|
+ name := qu.ObjToString(tr["s_name"])
|
|
|
TR.ID = id
|
|
|
TR.Name = name
|
|
|
TR.CustomerId = c.ID
|
|
|
//部门规则id组
|
|
|
- if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
+ if departRuleIds := qu.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
for _, drid := range strings.Split(departRuleIds, ",") {
|
|
|
TR.DepartRuleIds[drid] = true
|
|
|
}
|
|
|
@@ -138,12 +140,12 @@ func (c *Customer) GetTagRules() {
|
|
|
TR.Fields = make(map[string]interface{})
|
|
|
TR.DepartRuleIds = make(map[string]bool)
|
|
|
id := mgoutil.BsonTOStringId(tr["_id"])
|
|
|
- name := common.ObjToString(tr["s_name"])
|
|
|
+ name := qu.ObjToString(tr["s_name"])
|
|
|
TR.ID = id
|
|
|
TR.Name = name
|
|
|
TR.CustomerId = c.ID
|
|
|
//部门规则id组
|
|
|
- if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
+ if departRuleIds := qu.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
for _, drid := range strings.Split(departRuleIds, ",") {
|
|
|
TR.DepartRuleIds[drid] = true
|
|
|
}
|
|
|
@@ -163,12 +165,12 @@ func (c *Customer) GetTagRules() {
|
|
|
TR.Fields = make(map[string]interface{})
|
|
|
TR.DepartRuleIds = make(map[string]bool)
|
|
|
id := mgoutil.BsonTOStringId(tr["_id"])
|
|
|
- name := common.ObjToString(tr["s_name"])
|
|
|
+ name := qu.ObjToString(tr["s_name"])
|
|
|
TR.ID = id
|
|
|
TR.Name = name
|
|
|
TR.CustomerId = c.ID
|
|
|
//部门规则id组
|
|
|
- if departRuleIds := common.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
+ if departRuleIds := qu.ObjToString(tr["o_departruleids"]); departRuleIds != "" {
|
|
|
for _, drid := range strings.Split(departRuleIds, ",") {
|
|
|
TR.DepartRuleIds[drid] = true
|
|
|
}
|
|
|
@@ -185,7 +187,7 @@ func (c *Customer) GetTagRules() {
|
|
|
// 获取部门信息
|
|
|
func (c *Customer) GetDepartments(stype string) {
|
|
|
log.Println("开始获取部门信息...")
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
departments, _ := MgoTag.Find("euserdepart", map[string]interface{}{"s_userid": c.ID, "i_isuse": 1, "b_delete": false}, nil, nil)
|
|
|
if len(departments) > 0 {
|
|
|
for _, ds := range departments {
|
|
|
@@ -194,13 +196,13 @@ func (c *Customer) GetDepartments(stype string) {
|
|
|
DM.DepartmentData = map[string][]map[string]interface{}{}
|
|
|
DM.SaveDataMap = map[string]map[string]interface{}{}
|
|
|
id := mgoutil.BsonTOStringId(ds["_id"])
|
|
|
- name := common.ObjToString(ds["s_name"])
|
|
|
+ name := qu.ObjToString(ds["s_name"])
|
|
|
DM.ID = id
|
|
|
DM.Name = name
|
|
|
DM.CustomerID = c.ID
|
|
|
DM.GetSearchRules(c.ID, stype, c.IdRange, c.IdRanges) //获取某个部门的所有规则
|
|
|
c.Departments = append(c.Departments, DM)
|
|
|
- //common.Debug("Departments---", DM.ID, DM.Name, DM.CustomerID, len(DM.Rules))
|
|
|
+ //qu.Debug("Departments---", DM.ID, DM.Name, DM.CustomerID, len(DM.Rules))
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -208,9 +210,9 @@ func (c *Customer) GetDepartments(stype string) {
|
|
|
// 获取数据
|
|
|
func (c *Customer) GetData(stype string) {
|
|
|
log.Println("开始匹配数据...")
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
esConfig := Sysconfig.Es
|
|
|
- esversion := common.ObjToString(esConfig["version"])
|
|
|
+ esversion := qu.ObjToString(esConfig["version"])
|
|
|
if esversion == "v1" {
|
|
|
} else {
|
|
|
esCon := esv.VarEs.(*esv.EsV7)
|
|
|
@@ -303,26 +305,26 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
if json.Unmarshal(hit.Source, &tmp) == nil {
|
|
|
if stype != "history" {
|
|
|
if !SkipData(tmp) {
|
|
|
- common.Debug("跳过该条数据，发布时间在入库时间7天之前，", common.ObjToString(tmp["_id"]))
|
|
|
+ qu.Debug("跳过该条数据，发布时间在入库时间7天之前，", qu.ObjToString(tmp["_id"]))
|
|
|
continue
|
|
|
}
|
|
|
}
|
|
|
- id := common.ObjToString(tmp["id"])
|
|
|
+ id := qu.ObjToString(tmp["id"])
|
|
|
//亚信
|
|
|
if CheckBidOpenAppidMap[c.AppId] {
|
|
|
if tmp["bidopentime"] != nil {
|
|
|
- bidopentime := common.Int64All(tmp["bidopentime"])
|
|
|
- comeintime := common.Int64All(tmp["comeintime"])
|
|
|
+ bidopentime := qu.Int64All(tmp["bidopentime"])
|
|
|
+ comeintime := qu.Int64All(tmp["comeintime"])
|
|
|
if bidopentime-comeintime <= 7*24*60*60 {
|
|
|
- common.Debug("跳过该条数据，开标时间-入库时间<=7天，", id)
|
|
|
+ qu.Debug("跳过该条数据，开标时间-入库时间<=7天，", id)
|
|
|
continue
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
//河南移动，过滤掉中国移动采购网招标数据
|
|
|
if CheckBidHrefRuleIdMap[dm.ID] {
|
|
|
- if strings.Contains(common.ObjToString(tmp["href"]), "b2b.10086.cn") {
|
|
|
- common.Debug("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
|
|
|
+ if strings.Contains(qu.ObjToString(tmp["href"]), "b2b.10086.cn") {
|
|
|
+ qu.Debug("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
|
|
|
continue
|
|
|
}
|
|
|
}
|
|
|
@@ -339,7 +341,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
delete(tmp, "_id")
|
|
|
if sr.ExtFieldType == 2 {
|
|
|
findwinner := ""
|
|
|
- s_winner := strings.Split(common.ObjToString(tmp["s_winner"]), ",")
|
|
|
+ s_winner := strings.Split(qu.ObjToString(tmp["s_winner"]), ",")
|
|
|
if len(s_winner) > 0 {
|
|
|
for i := 0; i < len(s_winner); i++ {
|
|
|
findwinners := strings.TrimSpace(s_winner[i])
|
|
|
@@ -354,27 +356,27 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- // findwinner := strings.TrimSpace(common.ObjToString(tmp["winner"]))
|
|
|
+ // findwinner := strings.TrimSpace(qu.ObjToString(tmp["winner"]))
|
|
|
if findwinner != "" {
|
|
|
finddata := MgoEnps.FindOne(EnpsColl, bson.M{"company_name": findwinner})
|
|
|
if finddata != nil {
|
|
|
- if legal_person := common.ObjToString(finddata["legal_person"]); legal_person != "" {
|
|
|
+ if legal_person := qu.ObjToString(finddata["legal_person"]); legal_person != "" {
|
|
|
tmp["legal_person"] = legal_person
|
|
|
}
|
|
|
- if email := common.ObjToString(finddata["company_email"]); email != "" {
|
|
|
+ if email := qu.ObjToString(finddata["company_email"]); email != "" {
|
|
|
tmp["company_email"] = email
|
|
|
}
|
|
|
- if phone := common.ObjToString(finddata["company_phone"]); phone != "" {
|
|
|
+ if phone := qu.ObjToString(finddata["company_phone"]); phone != "" {
|
|
|
tmp["company_phone"] = phone
|
|
|
}
|
|
|
//从最新年报中获取 中标单位联系电话、中标单位邮箱
|
|
|
// if annual_reports, ok := finddata["annual_reports"].(primitive.A); ok && len(annual_reports) > 0 {
|
|
|
// anreport := Sort_year_report(annual_reports)
|
|
|
// if len(anreport) > 0 {
|
|
|
- // if email := common.ObjToString(anreport["company_email"]); email != "" {
|
|
|
+ // if email := qu.ObjToString(anreport["company_email"]); email != "" {
|
|
|
// tmp["company_email"] = email
|
|
|
// }
|
|
|
- // if phone := common.ObjToString(anreport["company_phone"]); phone != "" {
|
|
|
+ // if phone := qu.ObjToString(anreport["company_phone"]); phone != "" {
|
|
|
// tmp["company_phone"] = phone
|
|
|
// }
|
|
|
// }
|
|
|
@@ -387,13 +389,13 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
//先获取用到的所有字段值
|
|
|
fieldText := map[string]interface{}{}
|
|
|
for field, _ := range sr.Fields {
|
|
|
- text := common.ObjToString(tmp[field])
|
|
|
+ text := qu.ObjToString(tmp[field])
|
|
|
text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
|
|
|
fieldText[field] = text
|
|
|
}
|
|
|
//清理词清理
|
|
|
for _, cwm := range sr.GCW.MatchType {
|
|
|
- if text := common.ObjToString(fieldText[cwm]); text != "" {
|
|
|
+ if text := qu.ObjToString(fieldText[cwm]); text != "" {
|
|
|
for _, gcw_reg := range sr.GCW.KeyReg {
|
|
|
text = gcw_reg.ReplaceAllString(text, "")
|
|
|
}
|
|
|
@@ -406,11 +408,11 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
data, _ := MgoTag.Find("groups", map[string]interface{}{"ruleId": sr.ID}, nil, nil)
|
|
|
if data != nil && len(data) > 0 {
|
|
|
for _, v := range data {
|
|
|
- nameArr = append(nameArr, common.ObjToString(v["name"]))
|
|
|
+ nameArr = append(nameArr, qu.ObjToString(v["name"]))
|
|
|
}
|
|
|
}
|
|
|
- exactResult := exactMatchs(sr.ExactRule, common.ObjToString(tmp["title"]), common.ObjToString(tmp["detail"]), sr.Maths, nameArr)
|
|
|
- common.Debug("-------------------精准匹配", id, exactResult)
|
|
|
+ exactResult := exactMatchs(sr.ExactRule, qu.ObjToString(tmp["title"]), qu.ObjToString(tmp["detail"]), sr.Maths, nameArr)
|
|
|
+ qu.Debug("-------------------精准匹配", id, exactResult)
|
|
|
if !exactResult {
|
|
|
continue
|
|
|
}
|
|
|
@@ -420,18 +422,18 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
*/
|
|
|
//1.附加词匹配
|
|
|
IsMatch := false
|
|
|
- //common.Debug("sr.AW---", len(sr.AW))
|
|
|
+ //qu.Debug("sr.AW---", len(sr.AW))
|
|
|
for i, aw := range sr.AW {
|
|
|
- //common.Debug("-------------------------开始附加词匹配--------------------------")
|
|
|
+ //qu.Debug("-------------------------开始附加词匹配--------------------------")
|
|
|
IsMatchAddKey := RegMatch(fieldText, aw.MatchType, aw.KeyReg, nil, nil, false, true)
|
|
|
- //common.Debug(IsMatchAddKey, "------------------------------------------------------------")
|
|
|
+ //qu.Debug(IsMatchAddKey, "------------------------------------------------------------")
|
|
|
|
|
|
//2.关键词匹配
|
|
|
if IsMatchAddKey {
|
|
|
kw := sr.KW[i]
|
|
|
- //common.Debug("-------------------------开始关键词匹配--------------------------")
|
|
|
+ //qu.Debug("-------------------------开始关键词匹配--------------------------")
|
|
|
IsMatchKey := RegMatch(fieldText, kw.MatchType, kw.KeyReg, matchKey, matchKeyType, true, false)
|
|
|
- //common.Debug(IsMatchKey, "------------------------------------------------------------")
|
|
|
+ //qu.Debug(IsMatchKey, "------------------------------------------------------------")
|
|
|
if IsMatchKey {
|
|
|
IsMatch = true
|
|
|
}
|
|
|
@@ -443,7 +445,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
/*
|
|
|
到此已经匹配完数据
|
|
|
*/
|
|
|
- common.Debug("---------------------", id, IsMatch, matchKey)
|
|
|
+ qu.Debug("---------------------", id, IsMatch, matchKey)
|
|
|
if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
|
|
|
// tmpMatchKey := MapDataToArr(matchKey)
|
|
|
tmpMatchKeyType := MapDataToArr(matchKeyType)
|
|
|
@@ -451,14 +453,14 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tmp["matchtype"] = strings.Join(tmpMatchKeyType, ",")
|
|
|
tmp["ruleid"] = sr.ID
|
|
|
tmp["rulename"] = sr.Name
|
|
|
- tmpBuyerClass := common.ObjToString(tmp["buyerclass"])
|
|
|
+ tmpBuyerClass := qu.ObjToString(tmp["buyerclass"])
|
|
|
|
|
|
//开始打标签
|
|
|
- //common.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
|
|
|
+ //qu.Debug("c.IsTagRule+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
|
|
|
if c.IsTagRule {
|
|
|
tagNameMap := map[string]bool{}
|
|
|
tagIdMap := map[string]bool{}
|
|
|
- //common.Debug("c.TagRules---", len(c.TagRules))
|
|
|
+ //qu.Debug("c.TagRules---", len(c.TagRules))
|
|
|
//log.Println(c.TagRules,"=========",)
|
|
|
for _, tr := range c.TagRules {
|
|
|
if tr.DepartRuleIds[sr.ID] {
|
|
|
@@ -466,18 +468,18 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
//先获取用到的所有字段值
|
|
|
for field, _ := range tr.Fields {
|
|
|
if fieldText[field] == nil { //补充fieldText
|
|
|
- text := common.ObjToString(tmp[field])
|
|
|
+ text := qu.ObjToString(tmp[field])
|
|
|
text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
|
|
|
fieldText[field] = text
|
|
|
}
|
|
|
}
|
|
|
- //common.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
- //common.Debug("tr.NW---", len(tr.NW))
|
|
|
+ //qu.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
+ //qu.Debug("tr.NW---", len(tr.NW))
|
|
|
matchKeyTag := map[string]bool{} //记录所有标签里的匹配上的关键词
|
|
|
matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
|
|
|
for j, tag_nw := range tr.NW { //排除词匹配
|
|
|
IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
|
|
|
- //common.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
+ //qu.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
|
|
|
// log.Println(j, tr.TagNames[j])
|
|
|
if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
|
|
|
@@ -493,7 +495,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -506,7 +508,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -532,7 +534,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
if c.IsTagRule2 {
|
|
|
tagNameMap := map[string]bool{}
|
|
|
tagIdMap := map[string]bool{}
|
|
|
- //common.Debug("c.TagRules---", len(c.TagRules))
|
|
|
+ //qu.Debug("c.TagRules---", len(c.TagRules))
|
|
|
//log.Println(c.TagRules,"=========",)
|
|
|
for _, tr := range c.TagRules2 {
|
|
|
if tr.DepartRuleIds[sr.ID] {
|
|
|
@@ -540,18 +542,18 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
//先获取用到的所有字段值
|
|
|
for field, _ := range tr.Fields {
|
|
|
if fieldText[field] == nil { //补充fieldText
|
|
|
- text := common.ObjToString(tmp[field])
|
|
|
+ text := qu.ObjToString(tmp[field])
|
|
|
text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
|
|
|
fieldText[field] = text
|
|
|
}
|
|
|
}
|
|
|
- //common.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
- //common.Debug("tr.NW---", len(tr.NW))
|
|
|
+ //qu.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
+ //qu.Debug("tr.NW---", len(tr.NW))
|
|
|
matchKeyTag := map[string]bool{} //记录所有标签里的匹配上的关键词
|
|
|
matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
|
|
|
for j, tag_nw := range tr.NW { //排除词匹配
|
|
|
IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
|
|
|
- //common.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
+ //qu.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
|
|
|
// log.Println(j, tr.TagNames[j])
|
|
|
if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
|
|
|
@@ -567,7 +569,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -580,7 +582,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -606,7 +608,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
if c.IsTagRule3 {
|
|
|
tagNameMap := map[string]bool{}
|
|
|
tagIdMap := map[string]bool{}
|
|
|
- //common.Debug("c.TagRules---", len(c.TagRules))
|
|
|
+ //qu.Debug("c.TagRules---", len(c.TagRules))
|
|
|
//log.Println(c.TagRules,"=========",)
|
|
|
for _, tr := range c.TagRules3 {
|
|
|
if tr.DepartRuleIds[sr.ID] {
|
|
|
@@ -614,18 +616,18 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
//先获取用到的所有字段值
|
|
|
for field, _ := range tr.Fields {
|
|
|
if fieldText[field] == nil { //补充fieldText
|
|
|
- text := common.ObjToString(tmp[field])
|
|
|
+ text := qu.ObjToString(tmp[field])
|
|
|
text = ProcessData(text) //处理文本(字母转大写，删除一些符号)
|
|
|
fieldText[field] = text
|
|
|
}
|
|
|
}
|
|
|
- //common.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
- //common.Debug("tr.NW---", len(tr.NW))
|
|
|
+ //qu.Debug("-------------------------开始排除词匹配--------------------------")
|
|
|
+ //qu.Debug("tr.NW---", len(tr.NW))
|
|
|
matchKeyTag := map[string]bool{} //记录所有标签里的匹配上的关键词
|
|
|
matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
|
|
|
for j, tag_nw := range tr.NW { //排除词匹配
|
|
|
IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
|
|
|
- //common.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
+ //qu.Debug(IsMatchNotKey, "------------------------------------------------------------")
|
|
|
if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
|
|
|
// log.Println(j, tr.TagNames[j])
|
|
|
if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
|
|
|
@@ -641,7 +643,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -654,7 +656,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tagname = strings.Join(tempList, ",")
|
|
|
log.Println("=====tagname为空取匹配词为标签名称", tagname)
|
|
|
}
|
|
|
- //common.Debug("tagname-----", tagname)
|
|
|
+ //qu.Debug("tagname-----", tagname)
|
|
|
tagNameMap[tagname] = true
|
|
|
tagIdMap[tr.ID] = true
|
|
|
}
|
|
|
@@ -697,12 +699,12 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
tmp["departid"] = dm.ID
|
|
|
//存储数据
|
|
|
dm.DataLock.Lock()
|
|
|
- //common.Debug("tmp---", tmp)
|
|
|
+ //qu.Debug("tmp---", tmp)
|
|
|
tmpMap := map[string]interface{}{id: tmp}
|
|
|
dm.DepartmentData[sr.ID] = append(dm.DepartmentData[sr.ID], tmpMap)
|
|
|
dm.DataLock.Unlock()
|
|
|
} else {
|
|
|
- common.Debug("------------", id, IsMatch)
|
|
|
+ qu.Debug("------------", id, IsMatch)
|
|
|
}
|
|
|
}
|
|
|
numDocs += 1
|
|
|
@@ -727,7 +729,7 @@ func (c *Customer) EsConGetDataV7(stype string, esCon *esv.EsV7) {
|
|
|
// 数据去重
|
|
|
func (c *Customer) RemoveRepeatData() {
|
|
|
log.Println("开始数据去重...")
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
for _, dm := range c.Departments {
|
|
|
for _, dataMapArr := range dm.DepartmentData { //一个部门的所有数据
|
|
|
for _, dataMap := range dataMapArr {
|
|
|
@@ -752,7 +754,7 @@ func (c *Customer) RemoveRepeatData() {
|
|
|
} else if c.PushModel == 4 { //规则模式不去重
|
|
|
//c.SaveDataArr = append(c.SaveDataArr, tmp)
|
|
|
if c.SaveDataArr[dataId] == nil {
|
|
|
- tmp["itemdist"] = map[string]interface{}{common.ObjToString(tmp["item"]): common.ObjToString(tmp["matchkey"])}
|
|
|
+ tmp["itemdist"] = map[string]interface{}{qu.ObjToString(tmp["item"]): qu.ObjToString(tmp["matchkey"])}
|
|
|
c.SaveDataArr[dataId] = tmp
|
|
|
} else { //数据重复
|
|
|
dm_history := c.SaveDataArr[dataId]
|
|
|
@@ -761,7 +763,7 @@ func (c *Customer) RemoveRepeatData() {
|
|
|
}
|
|
|
} else if c.PushModel == 1 {
|
|
|
if c.SaveDataMap[dataId] == nil {
|
|
|
- tmp["itemdist"] = map[string]interface{}{common.ObjToString(tmp["item"]): common.ObjToString(tmp["matchkey"])}
|
|
|
+ tmp["itemdist"] = map[string]interface{}{qu.ObjToString(tmp["item"]): qu.ObjToString(tmp["matchkey"])}
|
|
|
c.SaveDataMap[dataId] = tmp
|
|
|
} else { //数据重复
|
|
|
dm_history := c.SaveDataMap[dataId]
|
|
|
@@ -779,7 +781,7 @@ func (c *Customer) RemoveRepeatData() {
|
|
|
// 组装保存数据
|
|
|
func (c *Customer) AssembelAndSaveData() {
|
|
|
log.Println("开始组装保存数据...")
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
ch := make(chan bool, 10)
|
|
|
wg := &sync.WaitGroup{}
|
|
|
n := 0
|
|
|
@@ -853,7 +855,7 @@ func (c *Customer) AssembelAndSaveData() {
|
|
|
|
|
|
// 获取用户所有规则
|
|
|
func (d *Department) GetSearchRules(cid, stype string, idRange, idRanges bson.M) {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
searchRules, _ := MgoTag.Find("euserdepartrule", map[string]interface{}{"s_userid": cid, "s_departid": d.ID, "i_isuse": 1, "b_delete": false}, nil, nil)
|
|
|
if len(searchRules) > 0 {
|
|
|
for _, sr := range searchRules {
|
|
|
@@ -861,19 +863,19 @@ func (d *Department) GetSearchRules(cid, stype string, idRange, idRanges bson.M)
|
|
|
SR.Fields = make(map[string]interface{})
|
|
|
|
|
|
id := mgoutil.BsonTOStringId(sr["_id"])
|
|
|
- name := common.ObjToString(sr["s_name"])
|
|
|
+ name := qu.ObjToString(sr["s_name"])
|
|
|
SR.ID = id
|
|
|
SR.Name = name
|
|
|
SR.CustomerID = cid
|
|
|
SR.DepartmentID = d.ID
|
|
|
- SR.ExtFieldType = common.IntAll(sr["i_extfieldstype"])
|
|
|
+ SR.ExtFieldType = qu.IntAll(sr["i_extfieldstype"])
|
|
|
//SR.RuleData = &sync.Map{}
|
|
|
- esquery := common.ObjToString(sr["s_esquery"])
|
|
|
+ esquery := qu.ObjToString(sr["s_esquery"])
|
|
|
if IsNewSql != 0 {
|
|
|
- esquery = common.ObjToString(sr["s_esquery_search"])
|
|
|
+ esquery = qu.ObjToString(sr["s_esquery_search"])
|
|
|
}
|
|
|
- clearKey := common.ObjToString(sr["s_globalclearkey"])
|
|
|
- clearKeyMatch := common.ObjToString(sr["s_globalclearkeymatch"])
|
|
|
+ clearKey := qu.ObjToString(sr["s_globalclearkey"])
|
|
|
+ clearKeyMatch := qu.ObjToString(sr["s_globalclearkeymatch"])
|
|
|
|
|
|
//获取es
|
|
|
if stype == "history" {
|
|
|
@@ -887,9 +889,9 @@ func (d *Department) GetSearchRules(cid, stype string, idRange, idRanges bson.M)
|
|
|
for _, v := range o_rules {
|
|
|
orule, _ := v.(map[string]interface{})
|
|
|
SR.Maths = append(SR.Maths, map[string]string{
|
|
|
- "s_matchkey": common.ObjToString(orule["s_matchkey"]),
|
|
|
- "s_keymatch": common.ObjToString(orule["s_keymatch"]),
|
|
|
- "s_group": common.ObjToString(orule["s_group"]),
|
|
|
+ "s_matchkey": qu.ObjToString(orule["s_matchkey"]),
|
|
|
+ "s_keymatch": qu.ObjToString(orule["s_keymatch"]),
|
|
|
+ "s_group": qu.ObjToString(orule["s_group"]),
|
|
|
})
|
|
|
}
|
|
|
}
|
|
|
@@ -902,7 +904,7 @@ func (d *Department) GetSearchRules(cid, stype string, idRange, idRanges bson.M)
|
|
|
|
|
|
// 获取转换后的es语句
|
|
|
func (sr *SearchRule) GetEs(department, esquery string, tmpRange, tmpRanges bson.M) {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
query := map[string]*QueryObjecct{}
|
|
|
if json.Unmarshal([]byte(esquery), &query) == nil {
|
|
|
qb := query["query"]
|
|
|
@@ -950,11 +952,11 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange, tmpRanges bson
|
|
|
|
|
|
// 全局清理词处理
|
|
|
func (sr *SearchRule) GetClearWord(key, match string) {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
//匹配方式
|
|
|
cwmArr := []string{}
|
|
|
for _, mv := range strings.Split(match, ",") {
|
|
|
- if field := common.ObjToString(MatchType[mv]); field != "" {
|
|
|
+ if field := qu.ObjToString(MatchType[mv]); field != "" {
|
|
|
cwmArr = append(cwmArr, field)
|
|
|
sr.Fields[field] = true
|
|
|
}
|
|
|
@@ -977,7 +979,7 @@ func (sr *SearchRule) GetClearWord(key, match string) {
|
|
|
|
|
|
// 关键词、附加词处理
|
|
|
func (sr *SearchRule) GetKeyAddWord(o_rules primitive.A) {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
kw, aw, _, _, _ := GetNotkeyAndKeyAddWord(o_rules, sr.Fields, false)
|
|
|
sr.KW = kw
|
|
|
sr.AW = aw
|
|
|
@@ -986,7 +988,7 @@ func (sr *SearchRule) GetKeyAddWord(o_rules primitive.A) {
|
|
|
|
|
|
// 排除词、关键词、附加词处理
|
|
|
func (tr *TagRule) GetKeyAddNotKeyWord(o_list primitive.A) {
|
|
|
- defer common.Catch()
|
|
|
+ defer qu.Catch()
|
|
|
kw, aw, nkw, tagnames, buyerclass := GetNotkeyAndKeyAddWord(o_list, tr.Fields, true)
|
|
|
tr.NW = nkw
|
|
|
tr.KW = kw
|
|
|
@@ -1002,31 +1004,31 @@ func GetMactchKeys(match []map[string]string, data map[string]interface{}) strin
|
|
|
types := keys["s_keymatch"]
|
|
|
key := keys["s_matchkey"]
|
|
|
if strings.Contains(types, "1") {
|
|
|
- title := common.ObjToString(data["title"])
|
|
|
+ title := qu.ObjToString(data["title"])
|
|
|
keyWord = KeyWordToDatas(types, title, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "2") {
|
|
|
- detail := common.ObjToString(data["detail"])
|
|
|
+ detail := qu.ObjToString(data["detail"])
|
|
|
keyWord = KeyWordToDatas(types, detail, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "3") {
|
|
|
- purchasing := common.ObjToString(data["purchasing"])
|
|
|
+ purchasing := qu.ObjToString(data["purchasing"])
|
|
|
keyWord = KeyWordToDatas(types, purchasing, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "4") {
|
|
|
- filetext := common.ObjToString(data["filetext"])
|
|
|
+ filetext := qu.ObjToString(data["filetext"])
|
|
|
keyWord = KeyWordToDatas(types, filetext, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "5") {
|
|
|
- projectname := common.ObjToString(data["projectname"])
|
|
|
+ projectname := qu.ObjToString(data["projectname"])
|
|
|
keyWord = KeyWordToDatas(types, projectname, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "6") || strings.Contains(types, "8") {
|
|
|
- buyer := common.ObjToString(data["buyer"])
|
|
|
+ buyer := qu.ObjToString(data["buyer"])
|
|
|
keyWord = KeyWordToDatas(types, buyer, key, keyWord)
|
|
|
}
|
|
|
if strings.Contains(types, "7") || strings.Contains(types, "9") {
|
|
|
- winner := common.ObjToString(data["s_winner"])
|
|
|
+ winner := qu.ObjToString(data["s_winner"])
|
|
|
keyWord = KeyWordToDatas(types, winner, key, keyWord)
|
|
|
}
|
|
|
}
|
