feat: tj

updateData/config.json

@@ -1,11 +1,11 @@
 {
     "mongodb": {
-		"address": "192.168.3.206:27080",
+		"address": "192.168.3.206:27002",
 		"size": 5,
-		"dbName": "qfw",
+		"dbName": "mixdata",
 		"replSet": "",
-		"userName":"",
-		"password":""
+		"userName":"jyDevGroup",
+		"password":"jy@DevGroup"
 	},
     "tidb1": {
 	        "dbName": "base_service",
@@ -16,16 +16,11 @@
 			"maxIdleConns": 5
 	 },
     "tidb2": {
-	        "dbName": "base_service",
+	        "dbName": "zxl",
 	        "address": "192.168.3.14:4000",
 	        "userName": "root",
 	        "passWord": "=PDT49#80Z!RVv52_z",
 			"maxOpenConns": 5,
 			"maxIdleConns": 5
-	 },
-	"poolSize":20,
-	"testId":"5c3ecf30c9ebc245590dc5fa",
-	"timestamp":0,
-	"testEntId":0,
-	"testEntUserId":0
+	 }
 }

updateData/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"sync"
 
 	qu "app.yhyue.com/moapp/jybase/common"
 	"app.yhyue.com/moapp/jybase/mongodb"
@@ -67,14 +68,63 @@ func init() {
 
 func main() {
 	//
-	do1()
+	log.Println("电销 开始")
+	// do1()
+	log.Println("电销 结束")
 
 	//
-	do2()
+	log.Println("合力易捷 开始")
+	// do2()
+	log.Println("合力易捷 结束")
+
 }
 
 //电销线索刷库
 func do1() {
+	var (
+		pool = make(chan bool, 5)
+		wait = &sync.WaitGroup{}
+	)
+	i := 0
+	for {
+		count := TidbBi.CountBySql(`SELECT count(1) FROM dwd_f_crm_clue_info where company_nature is null`)
+		if count == 0 {
+			log.Println("find no data end")
+			return
+		}
+		TidbBi.SelectByBath(10, func(l *[]map[string]interface{}) bool {
+			for _, v := range *l {
+				pool <- true
+				wait.Add(1)
+				i++
+				go func(thisData map[string]interface{}) {
+					defer func() {
+						<-pool
+						wait.Done()
+					}()
+					id := gconv.Int64(thisData["id"])
+					query := map[string]interface{}{
+						"id": id,
+					}
+					cluename := gconv.String(thisData["cluename"])
+					update := getCompanyType(cluename)
+					ok := TidbBi.Update("dwd_f_crm_clue_info", query, update)
+					if !ok {
+						log.Println("crm clue info update err", query, update)
+					}
+				}(v)
+			}
+			if i%5000 == 0 {
+				log.Println(fmt.Sprintf("current --- %d ", i))
+			}
+			return true
+		}, `SELECT * FROM dwd_f_crm_clue_info where company_nature is null limit 500`)
+	}
+	wait.Wait()
+}
+
+//电销线索刷库 无并发
+func do1_bak() {
 	i := 0
 	for {
 		count := TidbBi.CountBySql(`SELECT count(1) FROM dwd_f_crm_clue_info where company_nature is null`)
@@ -103,11 +153,55 @@ func do1() {
 			return true
 		}, `SELECT * FROM dwd_f_crm_clue_info where company_nature is null limit 500`)
 	}
-
 }
 
 //合力易捷刷库
 func do2() {
+	i := 0
+	var (
+		pool = make(chan bool, 5)
+		wait = &sync.WaitGroup{}
+	)
+	for {
+		count := TidbCall.CountBySql(`SELECT count(1) FROM customer where company_nature is null`)
+		if count == 0 {
+			log.Println("find no data end")
+			return
+		}
+		TidbCall.SelectByBath(10, func(l *[]map[string]interface{}) bool {
+			for _, v := range *l {
+				pool <- true
+				wait.Add(1)
+				i++
+				go func(thisData map[string]interface{}) {
+					defer func() {
+						<-pool
+						wait.Done()
+					}()
+					id := gconv.String(thisData["unique_id"])
+					query := map[string]interface{}{
+						"unique_id": id,
+					}
+					company := gconv.String(thisData["company"])
+					update := getCompanyType(company)
+					ok := TidbCall.Update("customer", query, update)
+					if !ok {
+						log.Println("customer info update err", query, update)
+					}
+				}(v)
+			}
+
+			if i%5000 == 0 {
+				log.Println(fmt.Sprintf("current --- %d ", i))
+			}
+			return true
+		}, `SELECT * FROM customer where company_nature is null limit 1000`)
+	}
+	wait.Wait()
+}
+
+//合力易捷刷库 无并发
+func do2_bak() {
 	i := 0
 	for {
 		count := TidbCall.CountBySql(`SELECT count(1) FROM customer where company_nature is null`)
@@ -134,9 +228,8 @@ func do2() {
 				log.Println(fmt.Sprintf("current --- %d ", i))
 			}
 			return true
-		}, `SELECT * FROM customer where company_nature is null limit 500`)
+		}, `SELECT * FROM customer where company_nature is null limit 1000`)
 	}
-
 }
 
 //公司性质、公司核验