Merge branch 'dev1.1' of http://192.168.3.207:8080/dataservice/datatools into dev1.1

telemarketingEtl/config.yaml

@@ -60,9 +60,9 @@ regex:
   - "^/jyapp/big/page/client_portrayal:appClient"#app-客户监控
   - "^/swordfish/page_big_pc/analysis_filter:pcForecast"#pc-潜在项目预测
   - "^/jyapp/big/page/forecast_detail:appForecast"#app-潜在项目预测列表页
-  - "^/swordfish/frontPage/collection/sess/index:pcCollection"#pc-标讯收藏
-  - "^/jyapp/frontPage/collection/sess/index:appCollection"#app-标讯收藏
-  - "^/weixin/frontPage/collection/sess/index:wxCollection"#wx-标讯收藏
+ # - "^/swordfish/frontPage/collection/sess/index:pcCollection"#pc-标讯收藏
+ # - "^/jyapp/frontPage/collection/sess/index:appCollection"#app-标讯收藏
+ # - "^/weixin/frontPage/collection/sess/index:wxCollection"#wx-标讯收藏
   - "^/product/index:productIndex"#pc-会员介绍页面
 classOneHighSeas_A: 3  #一级公海A 3天内点击过剑鱼付费产品介绍页...
 classOneHighSeas_B: 30 #一级公海B 购买了超级订阅产品剩余使用时长不足30天的客户；

telemarketingEtl/entity/dwd_f_crm_open_sea.go

@@ -465,15 +465,15 @@ func ReturnOpenSea() {
 
 				args2 := []interface{}{id}
 				//获取跟进内容
-				sql1 := `select COUNT(1) FROM dwd_f_crm_trail_content WHERE clue_id =?;`
+				sql1 := `SELECT COUNT(1) FROM dwd_f_crm_trail_content WHERE clue_id =?;`
 				sql2 := `SELECT COUNT(1) FROM dwd_f_crm_trail_content WHERE clue_id =? `
 				if intime != "" {
-					sql2 += ` createime < ?`
+					sql2 += ` and createtime > ?`
 					args2 = append(args2, intime)
 				}
 				//保留未跟进线索
 				if c1, c2 := config.JianyuSubjectdb.CountBySql(sql1, id),
-					config.JianyuSubjectdb.CountBySql(sql2, args2...); c1 != 0 && c2 <= 0 {
+					config.JianyuSubjectdb.CountBySql(sql2, args2...); c1 != 0 && c2 > 0 {
 					log.Println("不满足线索过滤", id)
 					continue
 				}
@@ -589,13 +589,13 @@ func Recycle() {
 	tA := t.AddDate(0, 0, -handReturn)
 	recycleB := g.Cfg().MustGet(ctx, "recycleB").Int()
 	tB := t.AddDate(0, 0, -recycleB)
-	for sql, _ := range map[string]interface{}{
-		fmt.Sprintf(`select id from dwd_f_crm_open_sea where comeinsource=%v and comeintime > "%s" ;`, 2, tA.Format(date.Date_Full_Layout)): 1,
+	for sql, clue_level := range map[string]interface{}{
+		fmt.Sprintf(`select id from dwd_f_crm_open_sea where comeinsource=%v and comeintime > "%s" ;`, 2, tA.Format(date.Date_Full_Layout)): "A",
 		fmt.Sprintf(`SELECT a.uid,b.phone,c.createTime,a.id FROM dwd_f_crm_clue_info a 
 					LEFT JOIN dwd_f_userbase_contacts b ON b.baseinfo_id=a.uid
 					LEFT JOIN Call_Accounting.voice_record c ON c.CalledNo = b.phone
 					WHERE a.trailstatus=01 AND a.uid !="" AND b.phone IS NOT NULL AND c.State="dealing" AND c.createTime >"%s"
-					`, tB.Format(date.Date_Full_Layout)): 2,
+					`, tB.Format(date.Date_Full_Layout)): "B",
 	} {
 		func(sql string) {
 			config.JianyuSubjectdb.SelectByBath(500, func(l *[]map[string]interface{}) bool {
@@ -609,7 +609,7 @@ func Recycle() {
 					whs = append(whs, "?")
 				}
 				wh := strings.Join(whs, ",")
-				config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_open_sea SET level = 4 WHERE id in (`+wh+`)`, ids...)
+				config.JianyuSubjectdb.UpdateOrDeleteBySql(`UPDATE dwd_f_crm_open_sea SET level = 4,clue_level = "`+fmt.Sprint(clue_level)+`" WHERE id in (`+wh+`)`, ids...)
 				return true
 			}, sql)
 		}(sql)

telemarketingEtl/entity/dwd_f_userbase_event_info.go

@@ -184,7 +184,7 @@ func EventInfoAdd(start, end int64) {
 								}
 							}
 						}
-					} else if pcBuyPortraitReg.MatchString(url_) && refer != "/swordfish/page_big_pc/my_client" {
+					} else if pcBuyPortraitReg.MatchString(url_) && !strings.Contains(refer, "/swordfish/page_big_pc/my_client") {
 						eventtype = BUYPORTRAIT
 						na := PcBuyPortraitNameReg.FindStringSubmatch(url_)
 						if len(na) > 1 {
@@ -196,7 +196,7 @@ func EventInfoAdd(start, end int64) {
 						if len(na) > 1 {
 							name = util.Unescape(na[1])
 						}
-					} else if PcEntIntelligentceMonitorReg.MatchString(url_) {
+					} else if PcEntIntelligentceMonitorReg.MatchString(url_) && !strings.Contains(refer, "/swordfish/page_big_pc/free/ent_follow") {
 						eventtype = ENTPORTRAIT
 						na := PcEntPortraitNameReg.FindStringSubmatch(url_)
 						if len(na) > 1 {
@@ -214,7 +214,7 @@ func EventInfoAdd(start, end int64) {
 						}
 					} else if ProductIndexReg.MatchString(url_) {
 						eventtype = PRODUCTINDEX
-					} else if PcClientReg.MatchString(url_) && refer == "/swordfish/page_big_pc/my_client" {
+					} else if PcClientReg.MatchString(url_) && strings.Contains(refer, "/swordfish/page_big_pc/my_client") {
 						eventtype = CLIENT
 						name = PcClient(url_)
 					} else if AppClientReg.MatchString(url_) {
@@ -223,7 +223,7 @@ func EventInfoAdd(start, end int64) {
 					} else if PcProjectMonitorReg.MatchString(url_) {
 						eventtype = PROJECTPROGRESS
 						name = ProjectProgress(url_)
-					} else if PcEntIntelligentceMonitorReg.MatchString(url_) {
+					} else if PcEntIntelligentceMonitorReg.MatchString(url_) && strings.Contains(refer, "/swordfish/page_big_pc/free/ent_follow") {
 						eventtype = ENTFOLLOW
 						name = EntFollow(url_)
 					} else if PcForecastReg.MatchString(url_) || AppForecastReg.MatchString(url_) {
@@ -233,31 +233,15 @@ func EventInfoAdd(start, end int64) {
 						eventtype = ENTSEARCH
 					}
 
-					/*
-
-						CLIENT          = "客户监控"
-						PROJECTPROGRESS = "项目进度监控"
-						ENTFOLLOW       = "企业情报监控"
-						FORECAST        = "潜在项目预测"
-						ENTSEARCH       = "企业搜索"
-					*/
 					createtime := time.Unix(gconv.Int64(v["date"]), 0).Format(date.Date_Full_Layout)
 					values = append(values, userid, eventtype, name, url_, platform, createtime)
 					if n%index == 0 {
-						Pool <- true
-						Wait.Add(1)
-						go func(fields []string, values []interface{}) {
-							defer func() {
-								<-Pool
-								Wait.Done()
-							}()
-							if len(values) > 0 {
-								id1, _ := config.JianyuSubjectdb.InsertBatch(DWD_F_USERBASE_EVENT_INFO, fields, values) //id1:数量  id2:开始id索引
-								if id1 <= 0 {
-									log.Println(values, "失败")
-								}
+						if len(values) > 0 {
+							id1, _ := config.JianyuSubjectdb.InsertBatch(DWD_F_USERBASE_EVENT_INFO, fields, values) //id1:数量  id2:开始id索引
+							if id1 <= 0 {
+								log.Println(values, "失败")
 							}
-						}(fields, values)
+						}
 						values = []interface{}{}
 					}
 					if n%200 == 0 {
@@ -270,6 +254,7 @@ func EventInfoAdd(start, end int64) {
 					if id1 <= 0 {
 						log.Println(values, "失败~")
 					}
+					values = []interface{}{}
 				}
 			} else {
 				log.Println(err)
@@ -375,6 +360,7 @@ func EntFollow(urls string) string {
 
 	// 使用正则表达式提取加密戳
 	match := re.FindStringSubmatch(urls)
+	log.Println(match)
 	// 输出结果
 	if len(match) < 2 {
 		return ""
@@ -384,6 +370,7 @@ func EntFollow(urls string) string {
 	}
 	id, _ := url.QueryUnescape(match[1])
 	idstr := util.DecodeId(id)
+	log.Println(id, idstr)
 	entinfo, _ := config.MgoEnt.FindOneByField("qyxy_std", map[string]interface{}{"_id": idstr}, map[string]interface{}{
 		"company_name": 1, //公司名称
 	})