Merge branch 'dev_v1.5.37_wh' of jianyu/datatools into feature/v1.5.37

clueSync/config.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"app.yhyue.com/moapp/jybase/common"
+	"app.yhyue.com/moapp/jybase/mysql"
 	"context"
 	"fmt"
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -42,144 +43,36 @@ type (
 		LastUserBindingTime     string  `json:"lastUserBindingTime"`
 	}
 	DB struct {
-		CornExp1      int64  `json:"cornexp1"`
-		CornExp2      int64  `json:"cornexp2"`
-		CornExp2Start string `json:"cornexp2Strat"`
-		CornExp2End   string `json:"cornexp2End"`
-		CornExp3      string `json:"cornexp3"`
-		CornExp4      string `json:"cornexp4"`
-		CornExp5      int64  `json:"cornexp5"`
-		CornExp6      int64  `json:"cornexp6"`
-		CornExp7      string `json:"cornexp7"`
-		CornExp8      int64  `json:"cornexp8"`
-		CornExp9      string `json:"cornexp9"`
-		TiDb          struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"tiDb"`
-		TiDbData struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"tiDbData"`
-		TiDbPower struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"tiDbPower"`
-		WorderOrder struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"workOrder"`
-		BaseService struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"baseService"`
-		BiService struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"biService"`
-		ThirdParty struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"thirdParty"`
-		Jyactivities struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"jyactivities"`
-		DebrisProduct struct {
-			Host     string `json:"host"`
-			Port     int    `json:"port"`
-			Database string `json:"database"`
-			User     string `json:"user"`
-			Password string `json:"password"`
-		} `json:"debrisProduct"`
-		Mysql struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"mysql"`
-		DataAnalysis struct {
-			Host        string `json:"host"`
-			Port        int    `json:"port"`
-			Database    string `json:"database"`
-			User        string `json:"user"`
-			Password    string `json:"password"`
-			PollSize    int    `json:"poolsize"`
-			MaxIdle     int    `json:"maxidle"`
-			MaxLeftTime int    `json:"maxleft"`
-		} `json:"dataAnalysis"`
-		Mgo struct {
+		CornExp1      int64       `json:"cornexp1"`
+		CornExp2      int64       `json:"cornexp2"`
+		CornExp2Start string      `json:"cornexp2Strat"`
+		CornExp2End   string      `json:"cornexp2End"`
+		CornExp3      string      `json:"cornexp3"`
+		CornExp4      string      `json:"cornexp4"`
+		CornExp5      int64       `json:"cornexp5"`
+		CornExp6      int64       `json:"cornexp6"`
+		CornExp7      string      `json:"cornexp7"`
+		CornExp8      int64       `json:"cornexp8"`
+		CornExp9      string      `json:"cornexp9"`
+		TiDb          MysqlConfig `json:"tiDb"`
+		TiDbData      MysqlConfig `json:"tiDbData"`
+		TiDbPower     MysqlConfig `json:"tiDbPower"`
+		WorderOrder   MysqlConfig `json:"workOrder"`
+		BaseService   MysqlConfig `json:"baseService"`
+		BiService     MysqlConfig `json:"biService"`
+		ThirdParty    MysqlConfig `json:"thirdParty"`
+		Jyactivities  MysqlConfig `json:"jyactivities"`
+		DebrisProduct MysqlConfig `json:"debrisProduct"`
+		Mysql         MysqlConfig `json:"mysql"`
+		DataAnalysis  MysqlConfig `json:"dataAnalysis"`
+		Mgo           struct {
 			Address string `json:"address"`
 			DbName  string `json:"dbName"`
 			DbSize  int    `json:"dbSize"`
 		} `json:"mgo"`
-		MgoLog struct {
-			Address  string `json:"address"`
-			DbName   string `json:"dbName"`
-			DbSize   int    `json:"dbSize"`
-			User     string `json:"user"`
-			Password string `json:"password"`
-		} `json:"mgoLog"`
-		MgoQyxy struct {
-			Address  string `json:"address"`
-			DbName   string `json:"dbName"`
-			DbSize   int    `json:"dbSize"`
-			User     string `json:"user"`
-			Password string `json:"password"`
-		} `json:"mgoQyxy"`
-		Es struct {
+		MgoLog  MongoConfig `json:"mgoLog"`
+		MgoQyxy MongoConfig `json:"mgoQyxy"`
+		Es      struct {
 			Address  string `json:"address"`
 			DbSize   int    `json:"dbSize"`
 			Version  string `json:"version"`
@@ -220,6 +113,7 @@ type (
 		EntId      int          `json:"entId"`
 		Mail       GmailAuth    `json:"mail"` //邮箱配置
 		Clickhouse CHouseConfig `json:"clickhouse"`
+		Crm        MysqlConfig  `json:"crmTiDb"`
 	}
 	LableJson struct {
 		ConditionConfig []ConditionEntity `json:"conditionConfig"`
@@ -241,6 +135,23 @@ type (
 		MaxIdleConns int
 		MaxOpenConns int
 	}
+	MysqlConfig struct {
+		Host        string `json:"host"`
+		Port        int    `json:"port"`
+		Database    string `json:"database"`
+		User        string `json:"user"`
+		Password    string `json:"password"`
+		PollSize    int    `json:"poolsize"`
+		MaxIdle     int    `json:"maxidle"`
+		MaxLeftTime int    `json:"maxleft"`
+	}
+	MongoConfig struct {
+		Address  string `json:"address"`
+		DbName   string `json:"dbName"`
+		DbSize   int    `json:"dbSize"`
+		User     string `json:"user"`
+		Password string `json:"password"`
+	}
 )
 type GmailAuth struct {
 	SmtpHost string //邮箱服务器
@@ -339,3 +250,13 @@ func ConnectClickhouse(cHouseConfig *CHouseConfig) error {
 	}
 	return nil
 }
+func MysqlTidbInit(data MysqlConfig) *mysql.Mysql {
+	return &mysql.Mysql{
+		Address:      data.Host + ":" + fmt.Sprint(data.Port),
+		UserName:     data.User,
+		PassWord:     data.Password,
+		DBName:       data.Database,
+		MaxOpenConns: data.PollSize,
+		MaxIdleConns: data.MaxIdle,
+	}
+}

clueSync/db.json

@@ -25,7 +25,7 @@
   "tiDb1": {
     "host": "127.0.0.1",
     "port": 33063,
-    "database": "jianyu_subjectdb_test",
+    "database": "jianyu_subjectdb",
     "user": "datagroup",
     "password": "Dgrpdb#2024@36",
     "poolsize": 20,
@@ -119,6 +119,16 @@
     "maxidle": 40,
     "maxleft": 40
   },
+  "mysql1": {
+    "host": "127.0.0.1",
+    "port": 33063,
+    "database": "jianyu",
+    "user": "datagroup",
+    "password": "Dgrpdb#2024@36",
+    "poolsize": 20,
+    "maxidle": 40,
+    "maxleft": 40
+  },
   "mgo": {
     "address": "172.20.45.129:27002",
     "dbName": "qfw",
@@ -206,5 +216,15 @@
     "dbName": "pub_tags",
     "maxIdleConns": 5,
     "maxOpenConns": 30
+  },
+  "crmTiDb": {
+    "host": "172.20.45.129",
+    "port": 4000,
+    "database": "crm",
+    "user": "root",
+    "password": "=PDT49#80Z!RVv52_z",
+    "poolsize": 20,
+    "maxidle": 40,
+    "maxleft": 40
   }
 }

clueSync/everything.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"app.yhyue.com/moapp/jybase/redis"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"github.com/gogf/gf/util/gconv"
@@ -452,6 +453,8 @@ func bigCustomer() {
 					"interest":         interest,
 					"data_requirement": data_requirement,
 					"appoint_email":    sourceData["appoint_email"],
+					"companyType":      gconv.String(v["companyType"]),
+					"industry":         gconv.String(v["industry"]),
 				})
 			}
 			if gconv.Int64(v["createtime"]) > bigSaleTime {
@@ -597,11 +600,13 @@ func bigCustomer() {
 			if err != nil {
 				log.Println("xls error", err, dir)
 			} else {
-
 				status := mail.GSendMail_q("剑鱼标讯", email, "", "", fileName, detailName, dir, fileName+".xlsx", &Gmail)
+				//往营销里边插入数据
+				go MarketingSave(vb, email)
 				if status {
 					log.Println("send mail success", fileName, email)
 				}
+
 			}
 		}
 	}
@@ -1025,3 +1030,98 @@ func EmailSelect(batch, mailStr string) (string, string) {
 	TiDb.UpdateOrDeleteBySql(`update dwd_f_crm_clue_big_autodraw_record set count = count + 1 where name = ? and   type=?`, name, batch)
 	return minemail, name
 }
+
+// 进入营销处理
+func MarketingSave(data []map[string]interface{}, email string) {
+	personData := TiDb.FindOne("dwd_f_crm_clue_big_autodraw_record", map[string]interface{}{
+		"type":  "dk",
+		"email": email,
+	}, "", "")
+	if personData == nil || len(*personData) == 0 {
+		log.Println("大客用户获取失败")
+		return
+	}
+	positionId := gconv.Int64((*personData)["position_id"])
+	name := gconv.String((*personData)["name"])
+	userMapping := TiDb.FindOne("data_service.user_system", map[string]interface{}{"position_id": positionId, "status": 1}, "", "")
+	if userMapping == nil || len(*userMapping) == 0 {
+		log.Println("大客用户信息获取失败")
+		return
+	}
+	entId := gconv.Int64((*userMapping)["ent_id"])
+	entUserId := gconv.Int64((*userMapping)["ent_user_id"])
+	CrmService.ExecTx("", func(tx *sql.Tx) bool {
+		for _, v := range data {
+			remarkJson := map[string]interface{}{
+				"提交时间":   gconv.String(v["createTime"]),
+				"销售线索来源": gconv.String(v["source"]),
+				"具体来源":   gconv.String(v["interest"]),
+				"姓名":     common.If(gconv.String(v["username"]) == "", gconv.String(v["phone"]), gconv.String(v["username"])),
+				"手机号":    gconv.String(v["phone"]),
+				"公司名称":   gconv.String(v["company"]),
+				"数据需求":   gconv.String(v["data_requirement"]),
+				"邮箱":     gconv.String(v["email"]),
+				"公司类型":   gconv.String(v["companyType"]),
+				"职位":     gconv.String(v["job"]),
+				"行业":     gconv.String(v["industry"]),
+			}
+			//线索保存
+			filterStr := gconv.String(filter)
+			filterStr = strings.ReplaceAll(filterStr, "{", "")
+			filterStr = strings.ReplaceAll(filterStr, "}", "")
+			filterStr = strings.ReplaceAll(filterStr, ",", "\n")
+			filterStr = strings.ReplaceAll(filterStr, `"`, `  `)
+			clueId := CrmService.InsertByTx(tx, "sale_clue", map[string]interface{}{
+				"position_id":           positionId,
+				"ent_id":                entId,
+				"ent_user_id":           entUserId,
+				"employ_info_id":        0,
+				"name":                  common.If(gconv.String(v["company"]) == "", gconv.String(v["phone"]), gconv.String(v["company"])),
+				"source_BAK":            "",
+				"summary":               common.If(gconv.String(v["company"]) == "", gconv.String(v["phone"]), gconv.String(v["company"])),
+				"is_close":              0,
+				"close_reason_classify": nil,
+				"close_reason_desc":     nil,
+				"create_person":         name,
+				"create_time":           time.Now().Format(date.Date_Full_Layout),
+				"channel":               "剑鱼平台提交兴趣点",
+				"source":                gconv.String(v["source"]),
+			})
+			//任务保存
+			filter, _ := json.Marshal(remarkJson)
+			taskId := CrmService.InsertByTx(tx, "task", map[string]interface{}{
+				"ent_id":            entId,
+				"name":              common.If(gconv.String(v["username"]) == "", gconv.String(v["phone"]), gconv.String(v["username"])),
+				"source":            1,
+				"source_id":         clueId,
+				"create_way":        2,
+				"status":            2,
+				"create_time":       time.Now().Format(date.Date_Full_Layout),
+				"join_task_vehicle": 0,
+				"remark":            filterStr,
+			})
+			if taskId <= 0 {
+				return false
+			}
+			//任务划转处理
+			CrmService.InsertByTx(tx, "task_transfer", map[string]interface{}{
+				"task_id":        taskId,
+				"transfer_id":    positionId,
+				"responsible_id": positionId,
+				"is_transfer":    0,
+				"create_time":    time.Now().Format(date.Date_Full_Layout),
+			})
+			//任务团队
+			CrmService.InsertByTx(tx, "task_team", map[string]interface{}{
+				"task_id":     taskId,
+				"position_id": positionId,
+				"ent_user_id": entUserId,
+				"name":        name,
+				"role":        1,
+				"create_time": time.Now().Format(date.Date_Full_Layout),
+			})
+		}
+		return true
+	})
+	return
+}

clueSync/lable.json

@@ -120,6 +120,36 @@
         }
       ]
     },
+    {
+      "topName": "其他",
+      "subName": "90天未活跃重新登录",
+      "conditionArr": [
+        {
+          "code": "fullBaseUserId",
+          "fool": true
+        },
+        {
+          "code": "active_1_90",
+          "fool": false
+        },
+        {
+          "code": "active_1",
+          "fool": true
+        },
+        {
+          "code": "newUser_90",
+          "fool": false
+        },
+        {
+          "code": "telemarketing_follow_7_day",
+          "fool": false
+        },
+        {
+          "code": "telemarketing_listen_7_day",
+          "fool": false
+        }
+      ]
+    },
     {
       "topName": "公海中电销历史标记用户",
       "subName": "访问付费产品购买页的近期未建联用户",
@@ -332,4 +362,5 @@
       ]
     }
   ]
-}
+
+}

clueSync/main.go

@@ -9,7 +9,6 @@ import (
 	"app.yhyue.com/moapp/jybase/redis"
 	"bp.jydev.jianyu360.cn/BaseService/pushpkg/p"
 	"flag"
-	"fmt"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/robfig/cron"
 	"log"
@@ -21,6 +20,7 @@ var (
 	lableJson           = new(LableJson)
 	Mysql               *mysql.Mysql
 	TiDb                *mysql.Mysql
+	dataServiceMsql     *mysql.Mysql
 	ClickhouseConn      driver.Conn
 	TiDbData            *mysql.Mysql
 	TiDbPower           *mysql.Mysql
@@ -31,6 +31,7 @@ var (
 	Base                *mysql.Mysql
 	BiService           *mysql.Mysql
 	DataAnalysisService *mysql.Mysql
+	CrmService          *mysql.Mysql
 	Es                  elastic.Es
 	Mgo                 *mongodb.MongodbSim
 	MgoLog              *mongodb.MongodbSim
@@ -55,103 +56,42 @@ func main() {
 		User:     db.Mail.User,
 		Pwd:      db.Mail.Pwd,
 	}
-	TiDb = &mysql.Mysql{
-		Address:      db.TiDb.Host + ":" + fmt.Sprint(db.TiDb.Port),
-		UserName:     db.TiDb.User,
-		PassWord:     db.TiDb.Password,
-		DBName:       db.TiDb.Database,
-		MaxOpenConns: db.TiDb.PollSize,
-		MaxIdleConns: db.TiDb.MaxIdle,
-	}
+	TiDb = MysqlTidbInit(db.TiDb)
 	TiDb.Init()
-	TiDbData = &mysql.Mysql{
-		Address:      db.TiDbData.Host + ":" + fmt.Sprint(db.TiDbData.Port),
-		UserName:     db.TiDbData.User,
-		PassWord:     db.TiDbData.Password,
-		DBName:       db.TiDbData.Database,
-		MaxOpenConns: db.TiDbData.PollSize,
-		MaxIdleConns: db.TiDbData.MaxIdle,
-	}
+
+	TiDbData = MysqlTidbInit(db.TiDbData)
 	TiDbData.Init()
-	TiDbPower = &mysql.Mysql{
-		Address:      db.TiDbPower.Host + ":" + fmt.Sprint(db.TiDbPower.Port),
-		UserName:     db.TiDbPower.User,
-		PassWord:     db.TiDbPower.Password,
-		DBName:       db.TiDbPower.Database,
-		MaxOpenConns: db.TiDbPower.PollSize,
-		MaxIdleConns: db.TiDbPower.MaxIdle,
-	}
+
+	TiDbPower = MysqlTidbInit(db.TiDbPower)
 	TiDbPower.Init()
-	ThirdParty = &mysql.Mysql{
-		Address:      db.ThirdParty.Host + ":" + fmt.Sprint(db.ThirdParty.Port),
-		UserName:     db.ThirdParty.User,
-		PassWord:     db.ThirdParty.Password,
-		DBName:       db.ThirdParty.Database,
-		MaxOpenConns: db.ThirdParty.PollSize,
-		MaxIdleConns: db.ThirdParty.MaxIdle,
-	}
+
+	ThirdParty = MysqlTidbInit(db.ThirdParty)
 	ThirdParty.Init()
-	Jyactivities = &mysql.Mysql{
-		Address:      db.Jyactivities.Host + ":" + fmt.Sprint(db.Jyactivities.Port),
-		UserName:     db.Jyactivities.User,
-		PassWord:     db.Jyactivities.Password,
-		DBName:       db.Jyactivities.Database,
-		MaxOpenConns: db.Jyactivities.PollSize,
-		MaxIdleConns: db.Jyactivities.MaxIdle,
-	}
+
+	Jyactivities = MysqlTidbInit(db.Jyactivities)
 	Jyactivities.Init()
-	debrisProductMysql = &mysql.Mysql{
-		Address:  db.DebrisProduct.Host + ":" + fmt.Sprint(db.DebrisProduct.Port),
-		UserName: db.DebrisProduct.User,
-		PassWord: db.DebrisProduct.Password,
-		DBName:   db.DebrisProduct.Database,
-	}
+
+	debrisProductMysql = MysqlTidbInit(db.DebrisProduct)
 	debrisProductMysql.Init()
-	WorkOrder = &mysql.Mysql{
-		Address:      db.WorderOrder.Host + ":" + fmt.Sprint(db.WorderOrder.Port),
-		UserName:     db.WorderOrder.User,
-		PassWord:     db.WorderOrder.Password,
-		DBName:       db.WorderOrder.Database,
-		MaxOpenConns: db.WorderOrder.PollSize,
-		MaxIdleConns: db.WorderOrder.MaxIdle,
-	}
+
+	WorkOrder = MysqlTidbInit(db.WorderOrder)
 	WorkOrder.Init()
-	Mysql = &mysql.Mysql{
-		Address:      db.Mysql.Host + ":" + fmt.Sprint(db.Mysql.Port),
-		UserName:     db.Mysql.User,
-		PassWord:     db.Mysql.Password,
-		DBName:       db.Mysql.Database,
-		MaxOpenConns: db.Mysql.PollSize,
-		MaxIdleConns: db.Mysql.MaxIdle,
-	}
+
+	Mysql = MysqlTidbInit(db.Mysql)
 	Mysql.Init()
-	Base = &mysql.Mysql{
-		Address:      db.BaseService.Host + ":" + fmt.Sprint(db.BaseService.Port),
-		UserName:     db.BaseService.User,
-		PassWord:     db.BaseService.Password,
-		DBName:       db.BaseService.Database,
-		MaxOpenConns: db.BaseService.PollSize,
-		MaxIdleConns: db.BaseService.MaxIdle,
-	}
+
+	Base = MysqlTidbInit(db.BaseService)
 	Base.Init()
-	BiService = &mysql.Mysql{
-		Address:      db.BiService.Host + ":" + fmt.Sprint(db.BiService.Port),
-		UserName:     db.BiService.User,
-		PassWord:     db.BiService.Password,
-		DBName:       db.BiService.Database,
-		MaxOpenConns: db.BiService.PollSize,
-		MaxIdleConns: db.BiService.MaxIdle,
-	}
+
+	BiService = MysqlTidbInit(db.BiService)
 	BiService.Init()
-	DataAnalysisService = &mysql.Mysql{
-		Address:      db.DataAnalysis.Host + ":" + fmt.Sprint(db.DataAnalysis.Port),
-		UserName:     db.DataAnalysis.User,
-		PassWord:     db.DataAnalysis.Password,
-		DBName:       db.DataAnalysis.Database,
-		MaxOpenConns: db.DataAnalysis.PollSize,
-		MaxIdleConns: db.DataAnalysis.MaxIdle,
-	}
+
+	DataAnalysisService = MysqlTidbInit(db.DataAnalysis)
 	DataAnalysisService.Init()
+
+	CrmService = MysqlTidbInit(db.Crm)
+	CrmService.Init()
+
 	ConnectClickhouse(&db.Clickhouse)
 	Es = elastic.NewEs(db.Es.Version, db.Es.Address, db.Es.DbSize, db.Es.UserName, db.Es.Password)
 	redis.InitRedis(db.RedisServer)
@@ -161,6 +101,8 @@ func main() {
 	InitArea()
 	InitProduct(db.ProductArr)
 	InitUserChannel()
+	/*bigCustomer()
+	return*/
 	if *mode == 1 {
 		go ordersClue() //后台订单进线索
 		//一秒钟一次