铁塔

CMPlatform/history/historytask.go

@@ -11,8 +11,9 @@ import (
 	"sync"
 	"time"
 
+	"log"
+
 	"app.yhyue.com/moapp/jybase/common"
-	"app.yhyue.com/moapp/jybase/go-xweb/log"
 	"app.yhyue.com/moapp/jybase/go-xweb/xweb"
 	"app.yhyue.com/moapp/jybase/mongodb"
 	"github.com/antonmedv/expr"
@@ -43,10 +44,10 @@ var (
 )
 
 func (this *HistoryData) HistoryTask(history_id string) {
-	log.Debug("开始历史任务...")
+	log.Println("开始历史任务...")
 	go UpdateHistoryState(1, history_id, 0)
 	//加载一个客户
-	log.Debug("history_id", history_id)
+	log.Println("history_id", history_id)
 	//是否根据项目id去重
 	xlsxData := &XlsxData{}
 	customer, _ := util.Mgo.Find("historylog", map[string]interface{}{"_id": mongodb.StringTOBsonId(history_id)}, nil, nil, false, -1, -1)
@@ -102,9 +103,9 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		//
 		//if ProjectAppidMap[appId] {
 		if isFilter == 1 {
-			log.Debug("g根据项目id去重。。。")
+			log.Println("g根据项目id去重。。。")
 			start := time.Now().Unix()
-			log.Debug("加载projectId---开始")
+			log.Println("加载projectId---开始")
 			if dataTable == 0 {
 				InitProjectId(appId)
 			}
@@ -112,7 +113,7 @@ func (this *HistoryData) HistoryTask(history_id string) {
 				InitUsermailProjectId(appId)
 			}
 			end := time.Now().Unix()
-			log.Debug("加载projectId---结束,耗时", end-start, "秒")
+			log.Println("加载projectId---结束,耗时", end-start, "秒")
 		} else {
 			projectIdMap = sync.Map{}
 		}
@@ -151,7 +152,7 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		dep_rules := common.ObjArrToMapArr(dep_rule)
 		cus.GetTagRules(tag_rules, tag_rules2, tag_rules3) //获取客户打标签规则
 		cus.GetDepartments("history", dep_rules)           //获取客户信息
-		log.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsTagRule2, cus.IsTagRule3, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.TagRules2), len(cus.TagRules3), len(cus.Departments))
+		log.Println("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsTagRule2, cus.IsTagRule3, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.TagRules2), len(cus.TagRules3), len(cus.Departments))
 		cus.GetData("history", i_dataSource)                                                              //获取数据
 		cus.RemoveRepeatData()                                                                            //数据去重
 		cus.AssembelAndSaveData(history_id, isFilter, noticeFilter, dataTable, entId, iContact, xlsxData) //组装、保存数据
@@ -165,7 +166,7 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		if i_pushtype == 0 {
 			time.Sleep(3 * time.Second)
 			if len(xlsxData.xlsxArr) != xlsxData.xlsxCount {
-				log.Error("excel数据量错误")
+				log.Println("excel数据量错误")
 			}
 			GetXlsxs(xlsxData.xlsxArr, iContact, customer_name, email, history_id, isfile, isHenanMobile, isfilehref, appId)
 			UpdateHistoryState(2, history_id, xlsxData.xlsxCount)
@@ -178,7 +179,7 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		}
 		projectIdMap = sync.Map{}
 	} else {
-		log.Error("初始化客户信息失败")
+		log.Println("初始化客户信息失败")
 	}
 }
 
@@ -660,10 +661,10 @@ func (this *HistoryData) DataTest(id string) {
 func UpdateJyqyfwDatacount(appid string, newGetLen int) {
 	update_b := util.MgoCus.Update("user", bson.M{"appid": appid}, bson.M{"$inc": bson.M{"plan.current": -newGetLen}}, false, false)
 	if update_b {
-		log.Debug("历史数据导入mysql表扣量成功 appid：" + appid + "扣除数据量：" + fmt.Sprintln(newGetLen))
+		log.Println("历史数据导入mysql表扣量成功 appid：" + appid + "扣除数据量：" + fmt.Sprintln(newGetLen))
 		return
 	}
-	log.Debug("历史数据导入mysql表扣量失败 appid：" + appid + "应扣除数据量：" + fmt.Sprintln(newGetLen))
+	log.Println("历史数据导入mysql表扣量失败 appid：" + appid + "应扣除数据量：" + fmt.Sprintln(newGetLen))
 }
 
 func (this *HistoryData) HistoryDelete(hid string) {
@@ -674,10 +675,10 @@ func (this *HistoryData) HistoryDelete(hid string) {
 	}
 	ok := util.Mgo.UpdateById("historylog", mongodb.StringTOBsonId(hid), set)
 	if ok {
-		log.Debug("历史任务逻辑删除成功", hid)
+		log.Println("历史任务逻辑删除成功", hid)
 		delOk := util.Mgo.Del("usermail_history", bson.M{"historyId": hid})
 		if delOk {
-			log.Debug("历史任务下数据总量删除成功")
+			log.Println("历史任务下数据总量删除成功")
 		}
 	}
 }
@@ -874,7 +875,7 @@ func exactMatch(rule string, data []map[string]interface{}, nameArr []string) bo
 	//可以将编译后的表达式，存放在缓存中
 	program, err := expr.Compile(rule, expr.Env(realdata))
 	if err != nil {
-		log.Error("表达式错误 ", err)
+		log.Println("表达式错误 ", err)
 		return false
 	}
 
@@ -893,10 +894,10 @@ func exactMatch(rule string, data []map[string]interface{}, nameArr []string) bo
 			}
 		}
 	}
-	log.Debug("匹配结果 ", realdata)
+	log.Println("匹配结果 ", realdata)
 	output, err := expr.Run(program, realdata)
 	if err != nil {
-		log.Error("表达式执行错误 ", err)
+		log.Println("表达式执行错误 ", err)
 		return false
 	}
 	return output.(bool)
@@ -921,7 +922,7 @@ func exactMatchs(rule, title, detail string, match []map[string]string, nameArr
 	//可以将编译后的表达式，存放在缓存中
 	program, err := expr.Compile(rule, expr.Env(realdata))
 	if err != nil {
-		log.Error("表达式错误 ", err)
+		log.Println("表达式错误 ", err)
 		return false
 	}
 
@@ -944,10 +945,10 @@ func exactMatchs(rule, title, detail string, match []map[string]string, nameArr
 			}
 		}
 	}
-	log.Debug("匹配结果 ", realdata)
+	log.Println("匹配结果 ", realdata)
 	output, err := expr.Run(program, realdata)
 	if err != nil {
-		log.Error("表达式执行错误 ", err)
+		log.Println("表达式执行错误 ", err)
 		return false
 	}
 	return output.(bool)

CMPlatform/history/task.go

@@ -1,26 +1,27 @@
 package history
 
 import (
-	"app.yhyue.com/moapp/jybase/common"
-	elastic "app.yhyue.com/moapp/jybase/es"
-	"app.yhyue.com/moapp/jybase/go-xweb/log"
-	"app.yhyue.com/moapp/jybase/mongodb"
-	"app.yhyue.com/moapp/jybase/redis"
 	"cmplatform/util"
 	sql "cmplatform/util/sqlmodel"
 	"context"
 	"encoding/json"
-	esV7 "github.com/olivere/elastic/v7"
-	"go.mongodb.org/mongo-driver/bson"
+	"log"
 	"regexp"
 	"strings"
 	"sync"
 	"time"
+
+	"app.yhyue.com/moapp/jybase/common"
+	elastic "app.yhyue.com/moapp/jybase/es"
+	"app.yhyue.com/moapp/jybase/mongodb"
+	"app.yhyue.com/moapp/jybase/redis"
+	esV7 "github.com/olivere/elastic/v7"
+	"go.mongodb.org/mongo-driver/bson"
 )
 
 // 获取客户打标签规则
 func (c *Customer) GetTagRules(tagRules, tagRules2, tagRules3 []map[string]interface{}) {
-	log.Debug("开始加载标签规则...")
+	log.Println("开始加载标签规则...")
 	defer common.Catch()
 	if len(tagRules) > 0 {
 		c.IsTagRule = true //查到打标签规则，表示打标签
@@ -116,7 +117,7 @@ func (c *Customer) GetTagRules(tagRules, tagRules2, tagRules3 []map[string]inter
 
 // 获取部门信息
 func (c *Customer) GetDepartments(stype string, departments []map[string]interface{}) {
-	log.Debug("开始获取部门信息...")
+	log.Println("开始获取部门信息...")
 	defer common.Catch()
 	if len(departments) > 0 {
 		departMap := map[string]interface{}{}
@@ -152,7 +153,7 @@ func (c *Customer) GetDepartments(stype string, departments []map[string]interfa
 
 // 获取数据
 func (c *Customer) GetData(stype string, dataSource int) {
-	log.Debug("开始匹配数据...")
+	log.Println("开始匹配数据...")
 	defer common.Catch()
 	esConfig := util.Sysconfig["es"].(map[string]interface{})
 	esversion := common.ObjToString(esConfig["version"])
@@ -162,10 +163,10 @@ func (c *Customer) GetData(stype string, dataSource int) {
 		for {
 			listLen := redis.GetInt("session", "es_status")
 			if listLen == 0 {
-				log.Debug("es空闲！")
+				log.Println("es空闲！")
 				break
 			} else if listLen == 1 || listLen == 2 {
-				log.Debug("es繁忙，", listLen)
+				log.Println("es繁忙，", listLen)
 			}
 			time.Sleep(5 * time.Second)
 		}
@@ -177,7 +178,7 @@ func (c *Customer) GetData(stype string, dataSource int) {
 				redis.LPOP("datag", "jyqyfw_es_query")
 				break
 			} else {
-				log.Debug("企业级服务es进程数过多，", listLens)
+				log.Println("企业级服务es进程数过多，", listLens)
 			}
 			time.Sleep(5 * time.Second)
 		}
@@ -197,8 +198,8 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 				esIndex = util.EsAllIndex
 			}
 			escount := esCon.Count(esIndex, "", sr.EsQuery)
-			log.Debug("index", esIndex, "type")
-			log.Debug("查询总数:", escount, "规则ID:", sr.ID, "EsQuery:", sr.EsQuery)
+			log.Println("index", esIndex, "type")
+			log.Println("查询总数:", escount, "规则ID:", sr.ID, "EsQuery:", sr.EsQuery)
 			if escount == 0 {
 				continue
 			}
@@ -219,7 +220,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 				count := 1
 				for {
 					if scrollId == "" {
-						log.Debug("ScrollId Is Error")
+						log.Println("ScrollId Is Error")
 						break
 					}
 					var searchResult *esV7.SearchResult
@@ -230,14 +231,14 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 						searchResult, err = client.Scroll(esIndex).Size(200).ScrollId(scrollId).Do(ctx) //查询
 						if err != nil {
 							if err.Error() == "EOS" { //迭代完毕
-								log.Debug("Es Search Data Over：", err)
+								log.Println("Es Search Data Over：", err)
 							} else {
-								log.Debug("Es Search Data Error：", err)
+								log.Println("Es Search Data Error：", err)
 							}
 							break
 						}
 					}
-					log.Debug("此次处理条数 ", len(searchResult.Hits.Hits))
+					log.Println("此次处理条数 ", len(searchResult.Hits.Hits))
 					for _, hit := range searchResult.Hits.Hits {
 						//开始处理数据
 						wg.Add(1)
@@ -251,7 +252,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 							if json.Unmarshal(tmpHit.Source, &tmp) == nil {
 								if stype != "history" {
 									if !SkipData(tmp) {
-										log.Debug("跳过该条数据，发布时间在入库时间7天之前，", common.ObjToString(tmp["_id"]))
+										log.Println("跳过该条数据，发布时间在入库时间7天之前，", common.ObjToString(tmp["_id"]))
 										return
 									}
 								}
@@ -262,7 +263,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 										bidopentime := common.Int64All(tmp["bidopentime"])
 										comeintime := common.Int64All(tmp["comeintime"])
 										if bidopentime-comeintime <= 7*24*60*60 {
-											log.Debug("跳过该条数据，开标时间-入库时间<=7天，", id)
+											log.Println("跳过该条数据，开标时间-入库时间<=7天，", id)
 											return
 										}
 									}
@@ -270,7 +271,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 								//河南移动，过滤掉中国移动采购网招标数据
 								if CheckBidHrefRuleIdMap[dm.ID] {
 									if strings.Contains(common.ObjToString(tmp["href"]), "b2b.10086.cn") {
-										log.Debug("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
+										log.Println("跳过该条数据，公告原网址中包含 b2b.10086.cn，", id)
 										return
 									}
 								}
@@ -338,7 +339,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 										}
 									}
 									exactResult := exactMatchs(sr.ExactRule, common.ObjToString(tmp["title"]), common.ObjToString(tmp["detail"]), sr.Maths, nameArr)
-									log.Debug("-------------------精准匹配", id, exactResult)
+									log.Println("-------------------精准匹配", id, exactResult)
 									if !exactResult {
 										return
 									}
@@ -372,7 +373,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 								/*
 									到此已经匹配完数据
 								*/
-								log.Debug("---------------------", id, IsMatch)
+								log.Println("---------------------", id, IsMatch)
 								if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
 									// tmpMatchKey := MapDataToArr(matchKey)
 									tmpMatchKeyType := MapDataToArr(matchKeyType)
@@ -414,7 +415,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																			tempList = append(tempList, k)
 																		}
 																		tagname = strings.Join(tempList, ",")
-																		log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																		log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																	}
 																	tagNameMap[tagname] = true
 																	tagIdMap[tr.ID] = true
@@ -426,7 +427,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																		tempList = append(tempList, k)
 																	}
 																	tagname = strings.Join(tempList, ",")
-																	log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																	log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																}
 																tagNameMap[tagname] = true
 																tagIdMap[tr.ID] = true
@@ -467,7 +468,6 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 												matchKeyTypeTag := map[string]bool{} //记录标签里的关键词对应的匹配方式
 												//common.Debug("-------------------------开始排除词匹配--------------------------")
 												for j, tag_nw := range tr.NW { //排除词匹配
-													log.Debug("打标签2222tag_nw ", tag_nw, fieldText)
 													IsMatchNotKey := RegMatch(fieldText, tag_nw.MatchType, tag_nw.KeyReg, nil, nil, false, false)
 													if !IsMatchNotKey { //排除词未匹配，匹配附加词关键词
 														if RegMatch(fieldText, tr.AW[j].MatchType, tr.AW[j].KeyReg, nil, nil, false, true) && RegMatch(fieldText, tr.KW[j].MatchType, tr.KW[j].KeyReg, matchKeyTag, matchKeyTypeTag, true, false) {
@@ -481,7 +481,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																			tempList = append(tempList, k)
 																		}
 																		tagname = strings.Join(tempList, ",")
-																		log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																		log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																	}
 																	tagNameMap[tagname] = true
 																	tagIdMap[tr.ID] = true
@@ -493,7 +493,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																		tempList = append(tempList, k)
 																	}
 																	tagname = strings.Join(tempList, ",")
-																	log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																	log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																}
 																tagNameMap[tagname] = true
 																tagIdMap[tr.ID] = true
@@ -547,7 +547,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																			tempList = append(tempList, k)
 																		}
 																		tagname = strings.Join(tempList, ",")
-																		log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																		log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																	}
 																	tagNameMap[tagname] = true
 																	tagIdMap[tr.ID] = true
@@ -559,7 +559,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 																		tempList = append(tempList, k)
 																	}
 																	tagname = strings.Join(tempList, ",")
-																	log.Debug("=====tagname为空取匹配词为标签名称", tagname)
+																	log.Println("=====tagname为空取匹配词为标签名称", tagname)
 																}
 																tagNameMap[tagname] = true
 																tagIdMap[tr.ID] = true
@@ -614,7 +614,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 						}(hit)
 						numDocs += 1
 						if numDocs%500 == 0 {
-							log.Debug("Current：", numDocs)
+							log.Println("Current：", numDocs)
 						}
 					}
 					scrollId = searchResult.ScrollId
@@ -622,9 +622,9 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 				}
 				wg.Wait()
 				client.ClearScroll().ScrollId(scrollId).Do(ctx) //清理游标
-				log.Debug("SearchRule ID", sr.ID, "Result Data Count：", numDocs)
+				log.Println("SearchRule ID", sr.ID, "Result Data Count：", numDocs)
 			} else {
-				log.Debug("Customer:", c.Name, "Departmnet", dm.Name, "TagName", sr.Name, "Es Search Data Error,Tag ID:", sr.ID)
+				log.Println("Customer:", c.Name, "Departmnet", dm.Name, "TagName", sr.Name, "Es Search Data Error,Tag ID:", sr.ID)
 			}
 		}
 	}
@@ -632,7 +632,7 @@ func (c *Customer) EsConGetDataV7(stype string, dataSource int, esCon *elastic.E
 
 // 数据去重
 func (c *Customer) RemoveRepeatData() {
-	log.Debug("开始数据去重...")
+	log.Println("开始数据去重...")
 	defer common.Catch()
 	for _, dm := range c.Departments {
 		for _, dataMapArr := range dm.DepartmentData { //一个部门的所有数据
@@ -684,7 +684,7 @@ func (c *Customer) RemoveRepeatData() {
 
 // 组装保存数据
 func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter, dataTable, entId, i_contact int, xlsxData *XlsxData) {
-	log.Debug("开始组装保存数据...")
+	log.Println("开始组装保存数据...")
 	defer common.Catch()
 	ch := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
@@ -705,7 +705,7 @@ func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter,
 			}(tmp)
 			n++
 			if n%500 == 0 {
-				log.Debug("Current：", n)
+				log.Println("Current：", n)
 			}
 		}
 		wg.Wait()
@@ -726,7 +726,7 @@ func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter,
 					}(tmp)
 					n++
 					if n%500 == 0 {
-						log.Debug("Current：", n)
+						log.Println("Current：", n)
 					}
 				}
 			}
@@ -752,7 +752,7 @@ func (c *Customer) AssembelAndSaveData(historyId string, isFilter, noticeFilter,
 		}
 		wg.Wait()
 	}*/
-	log.Debug("数据保存完毕...		Save Number：", n)
+	log.Println("数据保存完毕...		Save Number：", n)
 }
 
 // 获取用户所有规则
@@ -851,10 +851,10 @@ func (sr *SearchRule) GetEs(department, esquery string, tmpRange bson.M) {
 		if err == nil {
 			sr.EsQuery = string(strquery)
 		} else {
-			log.Debug("Department:", department, "Es Error,Tag ID:", sr.ID)
+			log.Println("Department:", department, "Es Error,Tag ID:", sr.ID)
 		}
 	} else {
-		log.Debug("Department:", department, "Es Error,Tag ID:", sr.ID)
+		log.Println("Department:", department, "Es Error,Tag ID:", sr.ID)
 	}
 }
 

CMPlatform/main.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"app.yhyue.com/moapp/jybase/common"
-	"app.yhyue.com/moapp/jybase/go-xweb/xweb"
 	"cmplatform/client"
 	_ "cmplatform/filter"
 	"cmplatform/front"
@@ -12,6 +10,9 @@ import (
 	"cmplatform/util"
 	"log"
 	"time"
+
+	"app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/go-xweb/xweb"
 )
 
 func init() {
@@ -46,8 +47,8 @@ func init() {
 	xweb.AddAction(&service.Groups{})
 	xweb.RootApp().AppConfig.SessionTimeout = 24 * time.Hour
 	xweb.RootApp().Logger.SetOutputLevel(1)
-	//go history.SaveMgo()
-	//go history.SaveUsermailMgo()
+	go history.SaveMgo()
+	go history.SaveUsermailMgo()
 	//mails, _ := util.Sysconfig["mail"].([]interface{})
 	//if len(mails) > 0 {
 	//	history.Gmails = make([]*mail.GmailAuth, len(mails))