P416

biddingSync/config.go

@@ -0,0 +1,33 @@
+package main
+
+type (
+	Config struct {
+		CornExp string `json:"cornexp"`
+		Bidding struct {
+			Address  string `json:"address"`
+			DbName   string `json:"dbName"`
+			DbSize   int    `json:"dbSize"`
+			UserName string `json:"username"`
+			Password string `json:"password"`
+			ReplSet  string `json:"replSet"`
+		} `json:"bidding"`
+		Es struct {
+			Address  string `json:"address"`
+			DbSize   int    `json:"dbSize"`
+			Index    string `json:"index"`
+			IType    string `json:"iType"`
+			Version  string `json:"version"`
+			UserName string `json:"userName"`
+			Password string `json:"password"`
+		} `json:"es"`
+		Es2 struct {
+			Address  string `json:"address"`
+			DbSize   int    `json:"dbSize"`
+			Index    string `json:"index"`
+			IType    string `json:"iType"`
+			Version  string `json:"version"`
+			UserName string `json:"userName"`
+			Password string `json:"password"`
+		} `json:"es2"`
+	}
+)

biddingSync/config.json

@@ -0,0 +1,21 @@
+{
+	"cornexp": "0 */5 * * * ?",
+	"db": {
+		"address": "172.17.4.86:27080",
+		"dbName": "jyqyfw",
+		"dbSize": 5,
+		"colName": "usermail_xhs_project",
+		"temporaryColName": "usermail_xhs_project_tmp"
+	},
+	"es": {
+		"address": "http://172.17.4.184:19800",
+		"dbSize": 10,
+		"index": "projectset",
+		"iType": "projectset",
+		"version": "v7",
+		"userName": "",
+  		"password":
+	},
+	"lastTime": 1660438800,
+	"rule": "结果"
+}

biddingSync/main.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"log"
+	"strings"
+
+	elastic "es"
+	"mongodb"
+	common "qfw/util"
+
+	"github.com/tealeg/xlsx"
+)
+
+var (
+	Bidding *mongodb.MongodbSim
+	Es      elastic.Es
+	Es2     elastic.Es
+	cfg     = new(Config)
+	sMap    = map[string]string{}
+)
+
+func init() {
+	common.ReadConfig(&cfg)
+	Bidding = mongodb.NewMgoWithUser(cfg.Bidding.Address, cfg.Bidding.DbName, cfg.Bidding.UserName, cfg.Bidding.Password, cfg.Bidding.DbSize)
+	Es = elastic.NewEs(cfg.Es.Version, cfg.Es.Address, cfg.Es.DbSize, cfg.Es.UserName, cfg.Es.Password)
+	Es2 = elastic.NewEs(cfg.Es2.Version, cfg.Es2.Address, cfg.Es2.DbSize, cfg.Es2.UserName, cfg.Es2.Password)
+}
+
+func main() {
+	FindBidding()
+	FindBiddingBack()
+	sMaps()
+}
+
+func FindBidding() {
+	session := Bidding.GetMgoConn()
+	query := map[string]interface{}{}
+	defer Bidding.DestoryMongoConn(session)
+	count := 0
+	iter := session.DB(cfg.Bidding.DbName).C("bidding").Find(&query).Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		count++
+		id := mongodb.BsonIdToSId(thisData["_id"])
+		title := common.ObjToString(thisData["title"])
+		projectName := common.ObjToString(thisData["projectname"])
+		spidercode := common.ObjToString(thisData["spidercode"])
+		if strings.Contains(title, "..") {
+			if strings.Contains(projectName, "..") {
+				title = strings.ReplaceAll(title, ".", "")
+			} else {
+				title = projectName
+			}
+			sMap[spidercode] = "1"
+			ok := Bidding.UpdateById("bidding", id, map[string]interface{}{"$set": map[string]interface{}{"title": title}})
+			if ok {
+				updateData(id, title, spidercode)
+			} else {
+				log.Println("bidding修改失败！！", id)
+			}
+		}
+		thisData = map[string]interface{}{}
+	}
+}
+
+func FindBiddingBack() {
+	session := Bidding.GetMgoConn()
+	query := map[string]interface{}{}
+	defer Bidding.DestoryMongoConn(session)
+	count := 0
+	iter := session.DB(cfg.Bidding.DbName).C("bidding_back").Find(&query).Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		count++
+		id := mongodb.BsonIdToSId(thisData["_id"])
+		title := common.ObjToString(thisData["title"])
+		projectName := common.ObjToString(thisData["projectname"])
+		spidercode := common.ObjToString(thisData["spidercode"])
+		if strings.Contains(title, "..") {
+			if strings.Contains(projectName, "..") {
+				thisData["title"] = strings.ReplaceAll(title, ".", "")
+			} else {
+				thisData["title"] = projectName
+			}
+			sMap[spidercode] = "1"
+			ok := Bidding.UpdateById("bidding", id, map[string]interface{}{"$set": map[string]interface{}{"title": title}})
+			if ok {
+				updateData(id, title, spidercode)
+			} else {
+				log.Println("bidding修改失败！！", id)
+			}
+		}
+		thisData = map[string]interface{}{}
+	}
+}
+
+func updateData(id, title, spidercode string) {
+	data := Es.GetById("bidding", "bidding", id)
+	if data != nil && len(*data) > 0 {
+		dataMap := (*data)[0]
+		dataMap["title"] = title
+		ok := Es.UpdateNewDoc("bidding", "bidding", dataMap)
+		if ok {
+			log.Println("es老集群修改成功", id, spidercode)
+		} else {
+			log.Println("es老集群修改失败 ！！！", id, title)
+		}
+		oks := Es2.UpdateNewDoc("bidding", "bidding", dataMap)
+		if oks {
+			log.Println("es新集群修改成功", id, spidercode)
+		} else {
+			log.Println("es新集群修改失败 ！！！", id, title)
+		}
+	}
+}
+
+func sMaps() {
+	xf := xlsx.NewFile()
+	sh, _ := xf.AddSheet("详细数据")
+	for k, _ := range sMap {
+		row := sh.AddRow()
+		row.AddCell().SetString(k)
+	}
+	err := xf.Save("./爬虫代码统计.xlsx")
+	if err != nil {
+		log.Println("xls error")
+	}
+}

clueSync/config.json

@@ -20,8 +20,8 @@
 	"regTimes": 7,
 	"redisServer":"192.168.3.206:1712",
 	"tiDb": {
-		"host": "192.168.3.149",
-		"port": 4000,
+		"host": "127.0.0.1",
+		"port": 4001,
 		"database": "Jianyu_subjectdb",
 		"user": "root",
 		"password": "Tibi#20211222",
@@ -30,8 +30,8 @@
 		"maxleft": 40
 	},
 	"tiDbData": {
-		"host": "192.168.3.149",
-		"port": 4000,
+		"host": "127.0.0.1",
+		"port": 4001,
 		"database": "Call_Accounting",
 		"user": "root",
 		"password": "Tibi#20211222",

clueSync/hlyj.go

@@ -2,11 +2,12 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
+	// "encoding/json"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"app.yhyue.com/moapp/jybase/date"
@@ -31,13 +32,6 @@ func getToken() string {
 
 func hlyjSync() {
 	token := getToken()
-	updateData := map[string]interface{}{"accessToken": token, "accountId": "N000000029739", "id": "15656791606"}
-	update_data_byte, _ := json.Marshal(&updateData)
-	_, err := doPost("https://a1.7x24cc.com/cincc_serv/customer/getOne", update_data_byte)
-	if err != nil {
-		log.Println("调用接口失败", err)
-	}
-	return
 	count := 0
 	if token != "" {
 		TiDbData.SelectByBath(500, func(l *[]map[string]interface{}) bool {
@@ -45,6 +39,9 @@ func hlyjSync() {
 				count++
 				log.Println("第", count, "条")
 				unique_id := common.ObjToString(v["unique_id"])
+				if count%20000 == 0 {
+					token = getToken()
+				}
 				url := `https://a1.7x24cc.com/commonInte?flag=1008&account=N000000029739&accessToken=` + token + `&cusObj={"unique_id":"` + unique_id + `","belongTo":"直销组"}&dbType=0001`
 				// updateData := map[string]interface{}{"belongTo": "直销组", "unique_id": unique_id}
 				// update_data := map[string]interface{}{
@@ -92,6 +89,9 @@ func hlyjSync2() {
 				for _, v := range *l {
 					count++
 					log.Println("第", count, "条")
+					if count%20000 == 0 {
+						token = getToken()
+					}
 					unique_id := common.ObjToString(v["unique_id"])
 					empNo := common.ObjToString(v["empNo"])
 					belongTo := "市场部"
@@ -142,10 +142,20 @@ func hlyjSync3() {
 	xlFile := xlsx.NewFile()
 	sheet1, _ := xlFile.AddSheet("市场创建时间在前")
 	sheet2, _ := xlFile.AddSheet("电销创建时间在前")
-	TiDbData.SelectByBath(500, func(l *[]map[string]interface{}) bool {
-		for _, v := range *l {
-			count++
-			log.Println("第", count, "条")
+	l := TiDbData.SelectBySql(`SELECT phone,createTime from customer where belongTo = "市场部"`)
+	// TiDbData.SelectByBath(100, func(l *[]map[string]interface{}) bool {
+	wg := new(sync.WaitGroup)
+	ch := make(chan bool, 20)
+	for _, v := range *l {
+		count++
+		log.Println("第", count, "条")
+		wg.Add(1)
+		ch <- true
+		go func(v map[string]interface{}) {
+			defer func() {
+				wg.Done()
+				<-ch
+			}()
 			// unique_id := common.ObjToString(v["unique_id"])
 			// empNo := common.ObjToString(v["empNo"])
 			// belongTo := common.ObjToString(v["belongTo"])
@@ -158,28 +168,46 @@ func hlyjSync3() {
 				clueCreateTimes1, _ := time.ParseInLocation(date.Date_Full_Layout, clueCreateTime, time.Local)
 				createTimes := createTimes1.Unix()
 				clueCreateTimes := clueCreateTimes1.Unix()
-				if clueCreateTimes > createTimes {
+				if clueCreateTimes >= createTimes {
 					row := sheet1.AddRow()
+					cluename := common.ObjToString((*clueData)["cluename"])
+					trailstatus := common.ObjToString((*clueData)["trailstatus"])
+					seatNumber := common.ObjToString((*clueData)["seatNumber"])
+					row.AddCell().SetValue(cluename)
 					row.AddCell().SetValue(phone)
-					row.AddCell().SetValue(createTime)
+					row.AddCell().SetValue(trailstatus)
+					row.AddCell().SetValue(seatNumber)
 					row.AddCell().SetValue(clueCreateTime)
+					row.AddCell().SetValue(createTime)
 					log.Println("存入市场", phone)
 				} else {
+					company := common.ObjToString(v["company"])
+					status999 := common.ObjToString(v["status999"])
+					empNo := common.ObjToString(v["empNo"])
 					row := sheet2.AddRow()
+					row.AddCell().SetValue(company)
 					row.AddCell().SetValue(phone)
-					row.AddCell().SetValue(clueCreateTime)
+					row.AddCell().SetValue(status999)
+					row.AddCell().SetValue(empNo)
 					row.AddCell().SetValue(createTime)
+					row.AddCell().SetValue(clueCreateTime)
 					log.Println("存入电销", phone)
 				}
 			} else {
 				row := sheet1.AddRow()
+				row.AddCell().SetValue("")
 				row.AddCell().SetValue(phone)
+				row.AddCell().SetValue("")
+				row.AddCell().SetValue("")
+				row.AddCell().SetValue("")
 				row.AddCell().SetValue(createTime)
 				log.Println("无线索存入市场", phone)
 			}
-		}
-		return true
-	}, `SELECT phone,createTime from customer where belongTo = "市场部"`)
+		}(v)
+	}
+	wg.Wait()
+	// return true
+	// }, `SELECT phone,createTime from customer where belongTo = "市场部"`)
 	xlFile.Save("./线索统计.xlsx")
 }
 

clueSync/job.go

@@ -20,8 +20,8 @@ func orders() {
 	//一个小时未支付进入线索 A
 	log.Println("未支付订单定时任务开始")
 	lastOrderId := cfg.LastOrderId
-	selectTimeStart := time.Unix(time.Now().Unix()-7200, 0).Unix()
-	selectTimeEnd := time.Unix(time.Now().Unix()-3600, 0).Unix()
+	selectTimeStart := time.Unix(time.Now().Unix()-7200, 0).Format(date.Date_Full_Layout)
+	selectTimeEnd := time.Unix(time.Now().Unix()-3600, 0).Format(date.Date_Full_Layout)
 	sql := fmt.Sprintf(`select * from dataexport_order where create_time <= "%s" and create_time >= "%s" and id > %s`, selectTimeEnd, selectTimeStart, fmt.Sprint(lastOrderId))
 	data := Mysql.SelectBySql(sql)
 	if data != nil && *data != nil && len(*data) > 0 {

clueSync/kc.go

@@ -376,7 +376,7 @@ func ordersClue() {
 			salesperson := common.ObjToString(v["salesperson"])
 			user_phone := common.ObjToString(v["user_phone"])
 			userId, uId, cluename, seatNumber, positionId, trailstatus, clueId := common.ObjToString(v["user_id"]), "", "", "", int64(0), "", int64(0)
-			product_type_str2 := `"大会员","企业商机管理"`
+			product_type_str2 := `"大会员","企业商机管理","VIP订阅"`
 			product_type := common.ObjToString(v["product_type"])
 			if order_status == 1 && is_backstage_order == 1 && salesperson != "" && !strings.HasPrefix(user_phone, "9") && strings.Contains(product_type_str2, product_type) {
 				//销售部

clueSync/main.go

@@ -127,8 +127,8 @@ func main() {
 		e := cron.New()
 		e.AddFunc(cfg.CornExp5, func() {
 			tagAddSync()
-			kcSync()
 			ordersClue()
+			kcSync()
 			kcAuto()
 		})
 		e.Start()

clueSync/nohup.out

@@ -1911993,3 +1911993,215 @@ main.main()
 2023/09/26 18:53:33 第 111616 条
 2023/09/26 18:53:33 select createtime from dwd_f_crm_clue_info where phone=? limit 0,1 [13541005127]
 2023/09/26 18:53:33 无线索存入市场 13541005127
+[mysql] 2023/10/11 18:15:08 packets.go:37: read tcp 192.168.130.192:49442->192.168.3.149:4000: read: operation timed out
+[mysql] 2023/10/11 18:16:39 packets.go:73: read tcp 192.168.130.192:49419->192.168.3.149:4000: read: operation timed out
+[mysql] 2023/10/11 18:16:39 packets.go:428: busy buffer
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae880?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc0007efab0?, 0x1f4, 0xc000485de0, 0xc0007ef936?, {0x19d63ed?, 0xc000830ab0?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae880?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc0006734f0?, 0x1f4, 0xc00066fde0, 0xc0006730f6?, {0x19d63ed?, 0xc000777710?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae880?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc000825400?, 0x1f4, 0xc00055fde0, 0xc000825286?, {0x19d63ed?, 0xc0008862d0?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae880?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc0008a2450?, 0x1f4, 0xc00044dde0, 0xc0008a22d6?, {0x19d63ed?, 0xc000845440?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae880?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc00072ae80?, 0x1f4, 0xc000573de0, 0xc00072aa86?, {0x19d63ed?, 0xc000747f50?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+[mysql] 2023/10/30 12:29:04 packets.go:73: read tcp 127.0.0.1:51571->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/30 12:29:04 packets.go:428: busy buffer
+[mysql] 2023/10/30 20:19:17 packets.go:73: read tcp 127.0.0.1:55924->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/30 20:19:17 packets.go:428: busy buffer
+[mysql] 2023/10/30 21:04:53 packets.go:73: unexpected EOF
+[mysql] 2023/10/30 21:04:53 packets.go:428: busy buffer
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae8a0?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc000a141e0?, 0x1f4, 0xc0003f5de0, 0xc000a14066?, {0x19d640d?, 0xc0009db590?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+[mysql] 2023/10/31 10:35:40 packets.go:73: read tcp 127.0.0.1:62514->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/31 10:35:40 packets.go:428: busy buffer
+[mysql] 2023/10/31 13:30:43 packets.go:73: read tcp 127.0.0.1:52424->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/31 13:30:43 packets.go:428: busy buffer
+[mysql] 2023/10/31 18:25:00 packets.go:73: read tcp 127.0.0.1:64535->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/31 18:25:00 packets.go:428: busy buffer
+[mysql] 2023/10/31 19:35:58 packets.go:73: unexpected EOF
+[mysql] 2023/10/31 19:35:58 packets.go:428: busy buffer
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae8a0?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc00089c210?, 0x1f4, 0xc000541de0, 0xc00089c096?, {0x19d640d?, 0xc000811590?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+[mysql] 2023/10/31 19:37:32 packets.go:37: unexpected EOF
+[mysql] 2023/10/31 19:43:49 packets.go:73: unexpected EOF
+[mysql] 2023/10/31 19:43:49 packets.go:428: busy buffer
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae8a0?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc0005048c0?, 0x1f4, 0xc000685de0, 0xc00060df06?, {0x19d640d?, 0xc0003fd3e0?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+[mysql] 2023/10/31 21:28:38 packets.go:73: read tcp 127.0.0.1:54096->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/10/31 21:28:38 packets.go:428: busy buffer
+[mysql] 2023/11/01 10:44:07 packets.go:37: unexpected EOF
+[mysql] 2023/11/01 10:44:44 packets.go:73: unexpected EOF
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18ae8a0?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:59 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc00071c000?, 0x1f4, 0xc00034dde0, 0xc00065d566?, {0x19d640d?, 0xc000345590?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:36 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54
+[mysql] 2023/11/01 11:47:58 packets.go:73: read tcp 127.0.0.1:65124->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/01 11:47:58 packets.go:428: busy buffer
+[mysql] 2023/11/01 13:46:33 packets.go:73: read tcp 127.0.0.1:53079->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/01 13:46:33 packets.go:428: busy buffer
+[mysql] 2023/11/01 15:40:58 packets.go:123: closing bad idle connection: connection reset by peer
+[mysql] 2023/11/01 15:40:58 connection.go:173: driver: bad connection
+[mysql] 2023/11/03 14:26:09 packets.go:73: read tcp 127.0.0.1:64877->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/03 14:26:09 packets.go:428: busy buffer
+[mysql] 2023/11/03 15:03:35 packets.go:73: read tcp 127.0.0.1:56236->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/03 15:03:35 packets.go:428: busy buffer
+[mysql] 2023/11/03 16:29:04 packets.go:37: read tcp 127.0.0.1:61320->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/03 16:29:04 packets.go:428: busy buffer
+[mysql] 2023/11/09 11:02:49 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:51 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:52 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:52 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:54 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:54 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:56 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:56 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:57 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:02:59 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:00 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:00 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:00 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:01 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:01 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:02 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:03 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:03:04 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:09:34 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:12:17 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:19:55 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 11:52:05 packets.go:73: read tcp 127.0.0.1:54180->127.0.0.1:4001: read: connection reset by peer
+[mysql] 2023/11/09 11:52:05 packets.go:428: busy buffer
+[mysql] 2023/11/09 13:27:05 packets.go:37: unexpected EOF
+[mysql] 2023/11/09 13:27:05 packets.go:73: unexpected EOF
+[mysql] 2023/11/09 13:27:05 packets.go:428: busy buffer
+panic: interface conversion: interface {} is nil, not bool
+
+goroutine 1 [running]:
+main.hlyjSync.func1(0x18aefc0?)
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:60 +0x6da
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).Select(0xc000842a50?, 0x1f4, 0xc0001c7de0, 0xc000842696?, {0x19d6d99?, 0xc000687710?}, {0x0, 0x0, 0x0})
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:309 +0x462
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBathByTx(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:325
+app.yhyue.com/moapp/jybase/mysql.(*Mysql).SelectByBath(...)
+	/Users/xuzhiheng/go/pkg/mod/app.yhyue.com/moapp/jybase@v0.0.0-20230517064222-e0bdfc8ee0e8/mysql/mysql.go:322
+main.hlyjSync()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/hlyj.go:37 +0x96
+main.main()
+	/Users/xuzhiheng/jianyu/works/datatools/clueSync/main.go:213 +0xe54

esUpdate/config.go

@@ -0,0 +1,15 @@
+package main
+
+type (
+	Config struct {
+		Es struct {
+			Address  string `json:"address"`
+			DbSize   int    `json:"dbSize"`
+			Index    string `json:"index"`
+			IType    string `json:"iType"`
+			Version  string `json:"version"`
+			UserName string `json:"userName"`
+			Password string `json:"password"`
+		} `json:"es"`
+	}
+)

esUpdate/config.json

@@ -0,0 +1,11 @@
+{
+	"es": {
+		"address": "http://127.0.0.1:9802",
+		"dbSize": 10,
+		"index": "projectset",
+		"iType": "projectset",
+		"version": "v7",
+		"userName": "jybid",
+  		"password": "Top2023_JEB01i@31"
+	}
+}

esUpdate/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"log"
+	"net/url"
+
+	elastic "es"
+	common "qfw/util"
+)
+
+var (
+	Es  elastic.Es
+	cfg = new(Config)
+)
+
+func init() {
+	common.ReadConfig(&cfg)
+	Es = elastic.NewEs(cfg.Es.Version, cfg.Es.Address, cfg.Es.DbSize, cfg.Es.UserName, cfg.Es.Password)
+}
+
+func main() {
+	var dataType = 1 //1-修改 2-删除
+	if dataType == 1 {
+		updateData()
+	} else if dataType == 2 {
+		delData()
+	}
+}
+
+func updateData() {
+	data := Es.GetById("bidding", "bidding", "5c1bcdd1a5cb26b9b77912f6")
+	if data != nil && len(*data) > 0 {
+		dataMap := (*data)[0]
+		dataMap["title"] = `中国水泥厂有限公司孔山水泥用灰岩矿闭坑地质报告编制项目`
+		ok := Es.UpdateNewDoc("bidding", "bidding", dataMap)
+		if ok {
+			log.Println("修改成功")
+		}
+	}
+}
+
+func delData() {
+	a := []string{
+		"ABCY1xBdDwOLyo7Ent1c3IJIjMCCj10V3dgKw4gPy8eeFZwGAFUCRI%3D",
+		"ABCY1xBdDIvIyMsM391c3IJIjMCCj10V3dgK1gsPyEeVX1zZ1JUChs%3D",
+		"ABCY1xBdDIvIyMsM391c3IJIjMCCj10V3dgK1gsPyEeUX1zZ15UCiM%3D",
+		"ABCY1xBdDIvIyMsM391c3IJIjMCCj10V3dgK1gsPyEeWlVzYS9UCc8%3D",
+		"ABCY1xBdDIvIy04NH91c3IJIjMCCj10V3dgK1gsNj0gUXlzdjNUCi8%3D",
+		"ABCY1xBdSlYUTYsJFV1c3IJIjMCCj10V3dgK1gjJiEdfFNzfgVUCeg%3D",
+		"ABCY1xBdSlYLDksJFl1c3IJIjMCCj10V3dgK1gjKTogeFVzcRlUCe4%3D",
+		"ABCY1xBdSkFKDYvRGN1c3IJIjMCCj10V3dgK1gjKDoge3xzcQlUCdo%3D",
+		"ABCY1xBdSkFODk7JGd1c3IJIjMCCj10V3dgK1gjKDoNZFFwDlpUCXM%3D",
+		"ABCY1xBdTIvOD8vGXxlc08oMCcoSCd3XHx2KDgjKi4neFVzYidUCjI%3D",
+	}
+	for _, v := range a {
+		vv, _ := url.QueryUnescape(v)
+		vid := common.CommonDecodeArticle("content", vv)[0]
+		log.Println(vid)
+		// ok := Es.DelById("bidding", "bidding", vid)
+		// if ok {
+		// 	log.Println("删除成功")
+		// }
+	}
+}

item_xlsxssss copy/README.md

@@ -0,0 +1 @@
+#mongo数据转为自定义字段的excel

item_xlsxssss copy/config.json

@@ -0,0 +1,40 @@
+{
+	"mgoAddr": "192.168.3.167:27080",
+	"mgoDbName": "jyqyfw_historyData",
+	"mgoColl" : "20210204Ykjy",
+	"mgoSize": 20,
+	"dataType": 2,
+	"exportType": 1,
+	"termFind": {
+		
+	},
+	"keyWordFind": {
+		
+	},
+	"fields": {
+		"company_id": "企业标识", "company_name": "企业名称", "company_code": "注册号", "credit_no": "统一社会信用代码", 
+		"org_code": "组织机构代码", "tax_code": "纳税人识别号",
+		"establish_date": "成立日期", "legal_person": "法定代表人", "company_type": "企业类型", 
+		"company_status": "登记状态", "authority": "登记机关",
+		"issue_date": "核准日期",
+		"operation_startdate": "营业期限开始日期", "operation_enddate": "营业期限截止日期",
+		"capital": "注册资本（万元）",
+		"company_address": "企业地址", "business_scope": "经营范围",
+		"cancel_date": "注销时间", 
+		"cancel_reason": "注销原因", "revoke_date": "吊销时间",
+		"revoke_reason": "吊销原因", "use_flag": "使用标记",
+		"createtime": "入库时间", "updatetime": "最后更新时间",
+		"legal_person_type": "法人类型",
+		"real_capital": "实缴资本","en_name":"英文名称","company_area":"所属省份",
+		"company_city":"所属城市","company_district":"所属区县",
+		"company_phone":"企业电话","company_email":"企业邮箱"
+	},
+	"fieldSort":[
+		"company_id", "company_name", "company_code","credit_no", "org_code","tax_code",
+ "establish_date", "legal_person","company_type","company_status", "authority","issue_date",
+ "operation_startdate",  "operation_enddate","capital", 
+ "company_address", "business_scope", "cancel_date", "cancel_reason", "revoke_date",
+"revoke_reason","use_flag", "createtime", "updatetime", "legal_person_type",
+"real_capital","en_name","company_area","company_city","company_district","company_phone","company_email"
+	]
+}

item_xlsxssss copy/main.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"log"
+	"mongodb"
+	"os"
+	qu "qfw/util"
+	"regexp"
+	"strconv"
+
+	// "github.com/lauyoume/gopinyin"
+
+	"time"
+
+	"github.com/tealeg/xlsx"
+)
+
+type Config struct {
+	MgoAddr     string                            `json:"mgoAddr"`
+	MgoDbName   string                            `json:"mgoDbName"`
+	MgoColl     string                            `json:"mgoColl"`
+	MgoSize     int                               `json:"mgoSize"`
+	DataType    int                               `json:"dataType"`
+	ExportType  int                               `json:"exportType"`
+	TermFind    map[string]interface{}            `json:"termFind"`
+	KeyWordFind map[string]map[string]interface{} `json:"keyWordFind"`
+	Fields      map[string]string                 `json:"fields"`
+	FieldSort   []string                          `json:"fieldSort"`
+}
+
+var (
+	sysConfig  Config
+	Mgo        *mongodb.MongodbSim
+	SE         = qu.SimpleEncrypt{Key: "topJYBX2019"}
+	ClearHtml  = regexp.MustCompile("<[^>]*>")
+	ClearOther = regexp.MustCompile("[\n\r\\s\u3000\u2003\u00a0]")
+	MaxWorker  = os.Getenv("MAX_WORKERS")
+	MaxQueue   = os.Getenv("MAX_QUEUE")
+	// industry = []string{"烟草", "军队", "教育", "公安", "税务", "能源", "金融", "数字政府", "水利", "应急管理", "公路", "生态环境", "纪委", "自然资源", "运营商", "机场", "法院", "检察院", "铁路", "轨道交通"}
+)
+
+func init() {
+	qu.ReadConfig(&sysConfig)
+	log.Println("配置加载成功")
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.145.163:27083",
+		DbName:      "mixdata",
+		Size:        20,
+		UserName:    "JS3Z_Rbid_ProG",
+		Password:    "JS3Z@B5I3aR7Ch",
+	}
+	Mgo.InitPool()
+	log.Println("数据库加载成功")
+}
+
+func GetXlsxs(mMap []map[string]interface{}) {
+	xf := xlsx.NewFile()
+	sh, _ := xf.AddSheet("详细数据")
+	for _, v := range mMap {
+		row := sh.AddRow()
+		cell1 := row.AddCell()
+		cell2 := row.AddCell()
+		cell1.SetString(qu.ObjToString(v["company_name"]))
+		cell2.SetString(qu.ObjToString(v["credit_no"]))
+	}
+	//生文件
+	t := strconv.FormatInt(time.Now().Unix(), 10)
+	dir := "./xlsx/" + t + "/"
+	if b, _ := PathExists(dir); !b {
+		err1 := os.MkdirAll(dir, os.ModePerm)
+		if err1 != nil {
+			log.Println("mkdir err", dir)
+		}
+	}
+	fname := "_" + t + ".xlsx"
+	err := xf.Save(dir + fname)
+	if err != nil {
+		log.Println("xls error", fname)
+	}
+}
+
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func main() {
+	query := map[string]interface{}{}
+	count, session := 0, Mgo.GetMgoConn()
+	defer func() {
+		Mgo.DestoryMongoConn(session)
+	}()
+	iter := session.DB("mixdata").C("winner_enterprise").Find(&query).Select(map[string]interface{}{"company_name": 1, "credit_no": 1}).Sort("_id").Iter()
+	data := []map[string]interface{}{}
+	datas := []map[string]interface{}{}
+	datass := []map[string]interface{}{}
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		count++
+		log.Println("第", count, "条")
+		company_name := qu.ObjToString(thisData["company_name"])
+		credit_no := qu.ObjToString(thisData["credit_no"])
+		if company_name != "" && credit_no != "" {
+			a := map[string]interface{}{
+				"company_name": company_name,
+				"credit_no":    credit_no,
+			}
+			if count > 900000 {
+				if count > 1800000 {
+					datass = append(datass, a)
+				} else {
+					datas = append(datas, a)
+				}
+			} else {
+				data = append(data, a)
+			}
+		} else {
+			log.Println("信息为空", company_name, credit_no)
+		}
+	}
+	if len(data) > 0 {
+		log.Println("数据条数 ", len(data))
+		log.Println("数据条数 ", len(datas))
+		log.Println("数据条数 ", len(datass))
+		GetXlsxs(data)
+		GetXlsxs(datas)
+		GetXlsxs(datass)
+		log.Println("生成excel成功")
+	}
+}

item_xlsxssss copy/nohup.out

@@ -0,0 +1,3 @@
+2023/11/01 16:19:16 配置加载成功
+2023/11/01 16:19:16 数据库加载成功
+2023/11/01 16:19:46 mgo find err server selection error: server selection timeout, current topology: { Type: ReplicaSetNoPrimary, Servers: [{ Addr: 172.17.145.163:27083, Type: Unknown, State: Connected, Average RTT: 0, Last error: connection() : dial tcp 172.17.145.163:27083: i/o timeout }, { Addr: 172.17.4.187:27082, Type: Unknown, State: Connected, Average RTT: 0, Last error: connection() : dial tcp 172.17.4.187:27082: i/o timeout }, { Addr: 172.17.148.166:27080, Type: Unknown, State: Connected, Average RTT: 0, Last error: connection() : dial tcp 172.17.148.166:27080: i/o timeout }, { Addr: 172.17.145.163:27081, Type: Unknown, State: Connected, Average RTT: 0, Last error: connection() : dial tcp 172.17.145.163:27081: i/o timeout }, ] }

lt_etl/main.go

@@ -17,11 +17,11 @@ var (
 	cfg   = new(Config)
 )
 
-func init() {
-	common.ReadConfig(&cfg)
-	Mgo = mongodb.NewMgo(cfg.Db.Address, cfg.Db.DbName, cfg.Db.DbSize)
-	BzMgo = mongodb.NewMgo(cfg.Bz.Address, cfg.Bz.DbName, cfg.Bz.DbSize)
-}
+// func init() {
+// 	common.ReadConfig(&cfg)
+// 	Mgo = mongodb.NewMgo(cfg.Db.Address, cfg.Db.DbName, cfg.Db.DbSize)
+// 	BzMgo = mongodb.NewMgo(cfg.Bz.Address, cfg.Bz.DbName, cfg.Bz.DbSize)
+// }
 
 func runJob() {
 	log.Println("中国联通数据迁移任务开始------")
@@ -103,6 +103,12 @@ func runJob() {
 }
 
 func main() {
+	a := []string{}
+	a = append(a, "")
+	a = append(a, "")
+	b := strings.Join(a, ",")
+	log.Println(b)
+	return
 	runJob()
 	c := cron.New()
 	c.AddFunc(cfg.CornExp, func() {

lt_json/main.go

@@ -153,16 +153,29 @@ func main() {
 					}
 				}
 				bidtype := util.ObjToString(v["bidtype"])
+				toptype := util.ObjToString(v["toptype"])
+				subtype := util.ObjToString(v["subtype"])
 				if bidtype != "" {
 					bidtypeMap := map[string]string{
 						"招标": "0",
 						"邀标": "1",
-						"竞价": "2",
 						"竞谈": "2",
 						"单一": "3",
 						"询价": "4",
+						"竞价": "6",
 					}
-					v["bidtype"] = bidtypeMap["bidtype"]
+					v["bidtype"] = bidtypeMap[bidtype]
+				} else {
+					subtypeMap := map[string]string{
+						"招标": "0",
+						"邀标": "1",
+						"竞谈": "2",
+						"单一": "3",
+						"询价": "4",
+						"其它": "5",
+						"竞价": "6",
+					}
+					v["bidtype"] = subtypeMap[subtype]
 				}
 				bidway := util.ObjToString(v["bidway"])
 				if bidway == "电子投标" {
@@ -220,41 +233,55 @@ func main() {
 				v["buyerperson"] = strings.Split(buyerperson, ",")
 				v["buyertel"] = strings.Split(buyertel, ",")
 				winnerArr := strings.Split(s_winner, ",")
-				v["s_winner"] = winnerArr
 				v["winnerperson"] = strings.Split(winnerperson, ",")
 				v["winnertel"] = strings.Split(winnertel, ",")
 				v["agency"] = strings.Split(agency, ",")
 				v["agencyperson"] = strings.Split(agencyperson, ",")
 				v["agencytel"] = strings.Split(agencytel, ",")
 				if v["budget"] != nil {
-					v["budget"] = map[string]string{
-						"unit":   "",
-						"amount": strconv.FormatFloat(util.Float64All(v["budget"]), 'f', -1, 64),
+					v["budget"] = []map[string]string{
+						map[string]string{
+							"unit":   "",
+							"amount": strconv.FormatFloat(util.Float64All(v["budget"]), 'f', -1, 64),
+						},
 					}
 				} else {
-					v["budget"] = map[string]string{
-						"unit":   "",
-						"amount": "",
+					v["budget"] = []map[string]string{
+						map[string]string{
+							"unit":   "",
+							"amount": "",
+						},
 					}
 				}
 				if v["bidamount"] != nil {
-					v["bidamount"] = map[string]string{
-						"unit":   "",
-						"amount": strconv.FormatFloat(util.Float64All(v["bidamount"]), 'f', -1, 64),
+					v["bidamount"] = []map[string]string{
+						map[string]string{
+							"unit":   "",
+							"amount": strconv.FormatFloat(util.Float64All(v["bidamount"]), 'f', -1, 64),
+						},
 					}
 				} else {
-					v["bidamount"] = map[string]string{
-						"unit":   "",
-						"amount": "",
+					v["bidamount"] = []map[string]string{
+						map[string]string{
+							"unit":   "",
+							"amount": "",
+						},
 					}
 				}
-				if v["multipackage"] != nil {
-					v["multipackage"] = 0
+				multipackage := 0
+				isOptimization := 0
+				if v["multipackage"] == nil {
+					v["multipackage"] = "0"
 				} else {
 					v["multipackage"] = fmt.Sprint(v["multipackage"])
+					multipackage = util.IntAll(v["multipackage"])
+				}
+				if v["isOptimization"] == nil {
+					v["isOptimization"] = "0"
+				} else {
+					v["isOptimization"] = fmt.Sprint(v["isOptimization"])
+					isOptimization = util.IntAll(v["isOptimization"])
 				}
-				toptype := util.ObjToString(v["toptype"])
-				subtype := util.ObjToString(v["subtype"])
 				if subtype == "其它" {
 					if toptype == "预告" {
 						v["toptype"] = "1"
@@ -275,72 +302,252 @@ func main() {
 						v["subtype"] = s
 					}
 				}
+				//多包合并
+				if isOptimization == 1 && multipackage == 1 && v["package"] != nil {
+					wArr := []string{}
+					bArr := []map[string]string{}
+					if packages, ok := v["package"].(map[string]interface{}); ok {
+						for _, vv := range packages {
+							if packages1, oks := vv.(map[string]interface{}); oks {
+								if winner_all, wok := packages1["winner_all"].([]map[string]interface{}); wok {
+									for _, win := range winner_all {
+										winnerStr := util.ObjToString(win["winner"])
+										bidamountStr := strconv.FormatFloat(util.Float64All(win["bidamount"]), 'f', -1, 64)
+										wArr = append(wArr, winnerStr)
+										bArr = append(bArr, map[string]string{"unit": "", "amount": bidamountStr})
+									}
+								} else if winner_all, wok := packages1["winner_all"].([]interface{}); wok {
+									for _, win := range winner_all {
+										if win1, wok1 := win.(map[string]interface{}); wok1 {
+											winnerStr := util.ObjToString(win1["winner"])
+											bidamountStr := strconv.FormatFloat(util.Float64All(win1["bidamount"]), 'f', -1, 64)
+											wArr = append(wArr, winnerStr)
+											bArr = append(bArr, map[string]string{"unit": "", "amount": bidamountStr})
+										}
+									}
+								}
+							}
+						}
+						winnerArr = wArr
+						v["winner"] = wArr
+						v["bidamount"] = bArr
+					}
+				} else if multipackage == 1 && v["package"] != nil {
+					wArr := []string{}
+					bArr := []map[string]string{}
+					if packages, ok := v["package"].([]map[string]interface{}); ok {
+						for _, vv := range packages {
+							winnerStr := util.ObjToString(vv["winner"])
+							bidamountStr := strconv.FormatFloat(util.Float64All(vv["bidamount"]), 'f', -1, 64)
+							wArr = append(wArr, winnerStr)
+							bArr = append(bArr, map[string]string{"unit": "", "amount": bidamountStr})
+						}
+					} else if packages, ok := v["package"].([]interface{}); ok {
+						for _, vv := range packages {
+							if packages1, vvok := vv.(map[string]interface{}); vvok {
+								winnerStr := util.ObjToString(packages1["winner"])
+								bidamountStr := strconv.FormatFloat(util.Float64All(packages1["bidamount"]), 'f', -1, 64)
+								wArr = append(wArr, winnerStr)
+								bArr = append(bArr, map[string]string{"unit": "", "amount": bidamountStr})
+							}
+						}
+					}
+					winnerArr = wArr
+					v["winner"] = wArr
+					v["bidamount"] = bArr
+				} else if strings.Contains(s_winner, ",") {
+					winnerArr = []string{s_winner}
+				}
+				//
+				delete(v, "package")
 				tagNameArr := []string{}
 				mainCompanyArr := []string{}
+				v["s_winner"] = winnerArr
 				for _, vv := range winnerArr {
 					isOk := false
-					for _, yd := range strings.Split(ydkey, ",") {
-						if strings.Contains(yd, "&&") {
-							keys := strings.Split(yd, "&&")
-							key1, key2 := keys[0], keys[1]
-							if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
-								isOk = true
-								tagNameArr = append(tagNameArr, ydtag)
-								mainCompanyArr = append(mainCompanyArr, ydcompany)
+					if strings.Contains(vv, ",") {
+						v["multipackage"] = "2"
+						tagNameArr111 := []string{}
+						mainCompanyArr111 := []string{}
+						for _, vvv := range strings.Split(vv, ",") {
+							isOks := false
+							for _, yd := range strings.Split(ydkey, ",") {
+								if strings.Contains(yd, "&&") {
+									keys := strings.Split(yd, "&&")
+									key1, key2 := keys[0], keys[1]
+									if strings.Contains(vvv, key1) && strings.Contains(vvv, key2) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, ydtag)
+											mainCompanyArr111 = append(mainCompanyArr111, ydcompany)
+										}
+									}
+									if isOks {
+										break
+									}
+								} else {
+									if strings.Contains(vvv, yd) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, ydtag)
+											mainCompanyArr111 = append(mainCompanyArr111, ydcompany)
+										}
+									}
+									if isOks {
+										break
+									}
+								}
 							}
-						} else {
-							if strings.Contains(vv, yd) {
-								isOk = true
-								tagNameArr = append(tagNameArr, ydtag)
-								mainCompanyArr = append(mainCompanyArr, ydcompany)
+							for _, yd := range strings.Split(ltkey, ",") {
+								if strings.Contains(yd, "&&") {
+									keys := strings.Split(yd, "&&")
+									key1, key2 := keys[0], keys[1]
+									if strings.Contains(vvv, key1) && strings.Contains(vvv, key2) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, lttag)
+											mainCompanyArr111 = append(mainCompanyArr111, ltcompany)
+										}
+									}
+									if isOks {
+										break
+									}
+								} else {
+									if strings.Contains(vvv, yd) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, lttag)
+											mainCompanyArr111 = append(mainCompanyArr111, ltcompany)
+										}
+									}
+									if isOks {
+										break
+									}
+								}
 							}
-						}
-					}
-					for _, yd := range strings.Split(ltkey, ",") {
-						if strings.Contains(yd, "&&") {
-							keys := strings.Split(yd, "&&")
-							key1, key2 := keys[0], keys[1]
-							if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
-								if !isOk {
-									isOk = true
-									tagNameArr = append(tagNameArr, lttag)
-									mainCompanyArr = append(mainCompanyArr, ltcompany)
+							for _, yd := range strings.Split(dxkey, ",") {
+								if strings.Contains(yd, "&&") {
+									keys := strings.Split(yd, "&&")
+									key1, key2 := keys[0], keys[1]
+									if strings.Contains(vvv, key1) && strings.Contains(vvv, key2) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, dxtag)
+											mainCompanyArr111 = append(mainCompanyArr111, dxcompany)
+										}
+									}
+									if isOks {
+										break
+									}
+								} else {
+									if strings.Contains(vvv, yd) {
+										if !isOks {
+											isOks = true
+											tagNameArr111 = append(tagNameArr111, dxtag)
+											mainCompanyArr111 = append(mainCompanyArr111, dxcompany)
+										}
+									}
+									if isOks {
+										break
+									}
 								}
 							}
-						} else {
-							if strings.Contains(vv, yd) {
-								if !isOk {
-									isOk = true
-									tagNameArr = append(tagNameArr, lttag)
-									mainCompanyArr = append(mainCompanyArr, ltcompany)
+							if !isOks {
+								tagNameArr111 = append(tagNameArr111, " ")
+								mainCompanyArr111 = append(mainCompanyArr111, " ")
+							}
+							log.Println("isOks ", isOks)
+							log.Println("tagNameArr111 ", tagNameArr111)
+							log.Println("mainCompanyArr111 ", mainCompanyArr111)
+						}
+						tagNameArr = append(tagNameArr, strings.Join(tagNameArr111, ","))
+						mainCompanyArr = append(mainCompanyArr, strings.Join(mainCompanyArr111, ","))
+					} else {
+						for _, yd := range strings.Split(ydkey, ",") {
+							if strings.Contains(yd, "&&") {
+								keys := strings.Split(yd, "&&")
+								key1, key2 := keys[0], keys[1]
+								if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, ydtag)
+										mainCompanyArr = append(mainCompanyArr, ydcompany)
+									}
+								}
+								if isOk {
+									break
+								}
+							} else {
+								if strings.Contains(vv, yd) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, ydtag)
+										mainCompanyArr = append(mainCompanyArr, ydcompany)
+									}
+								}
+								if isOk {
+									break
 								}
 							}
 						}
-					}
-					for _, yd := range strings.Split(dxkey, ",") {
-						if strings.Contains(yd, "&&") {
-							keys := strings.Split(yd, "&&")
-							key1, key2 := keys[0], keys[1]
-							if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
-								if !isOk {
-									isOk = true
-									tagNameArr = append(tagNameArr, dxtag)
-									mainCompanyArr = append(mainCompanyArr, dxcompany)
+						for _, yd := range strings.Split(ltkey, ",") {
+							if strings.Contains(yd, "&&") {
+								keys := strings.Split(yd, "&&")
+								key1, key2 := keys[0], keys[1]
+								if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, lttag)
+										mainCompanyArr = append(mainCompanyArr, ltcompany)
+									}
+								}
+								if isOk {
+									break
+								}
+							} else {
+								if strings.Contains(vv, yd) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, lttag)
+										mainCompanyArr = append(mainCompanyArr, ltcompany)
+									}
+								}
+								if isOk {
+									break
 								}
 							}
-						} else {
-							if strings.Contains(vv, yd) {
-								if !isOk {
-									isOk = true
-									tagNameArr = append(tagNameArr, dxtag)
-									mainCompanyArr = append(mainCompanyArr, dxcompany)
+						}
+						for _, yd := range strings.Split(dxkey, ",") {
+							if strings.Contains(yd, "&&") {
+								keys := strings.Split(yd, "&&")
+								key1, key2 := keys[0], keys[1]
+								if strings.Contains(vv, key1) && strings.Contains(vv, key2) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, dxtag)
+										mainCompanyArr = append(mainCompanyArr, dxcompany)
+									}
+								}
+								if isOk {
+									break
+								}
+							} else {
+								if strings.Contains(vv, yd) {
+									if !isOk {
+										isOk = true
+										tagNameArr = append(tagNameArr, dxtag)
+										mainCompanyArr = append(mainCompanyArr, dxcompany)
+									}
+								}
+								if isOk {
+									break
 								}
 							}
 						}
-					}
-					if !isOk {
-						tagNameArr = append(tagNameArr, "")
-						mainCompanyArr = append(mainCompanyArr, "")
+						if !isOk {
+							tagNameArr = append(tagNameArr, "")
+							mainCompanyArr = append(mainCompanyArr, "")
+						}
 					}
 				}
 				v["tagname"] = tagNameArr

tmp_projectId/config.go

@@ -0,0 +1,33 @@
+package main
+
+type (
+	Config struct {
+		CornExp string `json:"cornexp"`
+		Db      struct {
+			Address          string `json:"address"`
+			DbName           string `json:"dbName"`
+			DbSize           int    `json:"dbSize"`
+			ColName          string `json:"colName"`
+			TemporaryColName string `json:"temporaryColName"`
+		} `json:"db"`
+		Bidding struct {
+			Address  string `json:"address"`
+			DbName   string `json:"dbName"`
+			DbSize   int    `json:"dbSize"`
+			UserName string `json:"username"`
+			Password string `json:"password"`
+			ReplSet  string `json:"replSet"`
+		} `json:"bidding"`
+		Es struct {
+			Address  string `json:"address"`
+			DbSize   int    `json:"dbSize"`
+			Index    string `json:"index"`
+			IType    string `json:"iType"`
+			Version  string `json:"version"`
+			UserName string `json:"userName"`
+			Password string `json:"password"`
+		} `json:"es"`
+		LastTime int64  `json:"lastTime"`
+		Rule     string `json:"rule"`
+	}
+)

tmp_projectId/config.json

@@ -0,0 +1,21 @@
+{
+	"cornexp": "0 */5 * * * ?",
+	"db": {
+		"address": "172.17.4.86:27080",
+		"dbName": "jyqyfw",
+		"dbSize": 5,
+		"colName": "usermail_xhs_project",
+		"temporaryColName": "usermail_xhs_project_tmp"
+	},
+	"es": {
+		"address": "http://172.17.4.184:19800",
+		"dbSize": 10,
+		"index": "projectset",
+		"iType": "projectset",
+		"version": "v7",
+		"userName": "",
+  		"password":
+	},
+	"lastTime": 1660438800,
+	"rule": "结果"
+}

tmp_projectId/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+
+	elastic "es"
+	"mongodb"
+	common "qfw/util"
+
+	"github.com/robfig/cron"
+)
+
+var (
+	Mgo   *mongodb.MongodbSim
+	Es    elastic.Es
+	cfg   = new(Config)
+	SEXhs = common.SimpleEncrypt{Key: "topJYBX2019"}
+	mode  = flag.Int("m", 1, "")
+)
+
+func init() {
+	common.ReadConfig(&cfg)
+	Mgo = mongodb.NewMgo(cfg.Db.Address, cfg.Db.DbName, cfg.Db.DbSize)
+	Es = elastic.NewEs(cfg.Es.Version, cfg.Es.Address, cfg.Es.DbSize, cfg.Es.UserName, cfg.Es.Password)
+}
+
+func FindData() {
+	session := Mgo.GetMgoConn()
+	query := map[string]interface{}{
+		"createtime": map[string]interface{}{
+			"$gte": time.Now().Unix() - 86400,
+		},
+		"$or": []map[string]interface{}{
+			map[string]interface{}{
+				"projectId": "",
+			},
+			map[string]interface{}{
+				"projectId": map[string]interface{}{
+					"$exists": 0,
+				},
+			},
+		}}
+	defer Mgo.DestoryMongoConn(session)
+	iter := session.DB(cfg.Db.DbName).C("usermail").Find(&query).Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		id := common.ObjToString(thisData["id"])
+		query := `{"query": {"bool": {"must": [{"term": {"ids": "%s"}}],"must_not": [],"should": []}}}`
+		querys := fmt.Sprintf(query, id)
+		projectId := ""
+		data := Es.Get("projectset", "projectset", querys)
+		if data != nil && *data != nil && len(*data) > 0 {
+			projectId = common.ObjToString((*data)[0]["_id"])
+			log.Println("projectId", projectId)
+		} else {
+			log.Println("ES未查到项目id", id)
+		}
+		//
+		if projectId != "" {
+			ok := Mgo.UpdateById("usermail", thisData["_id"], map[string]interface{}{"$set": map[string]interface{}{"projectId": projectId}})
+			if ok {
+				log.Println("项目id更新成功 ", id, projectId)
+			} else {
+				log.Println("项目id更新失败！ ", id, projectId)
+			}
+		}
+		thisData = map[string]interface{}{}
+	}
+}
+
+func main() {
+	flag.Parse()
+	if *mode == 1 {
+		FindHistory()
+	} else {
+		FindData()
+		c := cron.New()
+		c.AddFunc(cfg.CornExp, func() {
+			FindData()
+		})
+		c.Start()
+		select {}
+	}
+}
+
+func FindHistory() {
+	session := Mgo.GetMgoConn()
+	query := map[string]interface{}{
+		"$or": []map[string]interface{}{
+			map[string]interface{}{
+				"projectId": "",
+			},
+			map[string]interface{}{
+				"projectId": map[string]interface{}{
+					"$exists": 0,
+				},
+			},
+		}}
+	defer Mgo.DestoryMongoConn(session)
+	iter := session.DB(cfg.Db.DbName).C("usermail").Find(&query).Iter()
+	thisData := map[string]interface{}{}
+	for {
+		if !iter.Next(&thisData) {
+			break
+		}
+		id := common.ObjToString(thisData["id"])
+		query := `{"query": {"bool": {"must": [{"term": {"ids": "%s"}}],"must_not": [],"should": []}}}`
+		querys := fmt.Sprintf(query, id)
+		projectId := ""
+		data := Es.Get("projectset", "projectset", querys)
+		if data != nil && *data != nil && len(*data) > 0 {
+			projectId = common.ObjToString((*data)[0]["_id"])
+			log.Println("projectId", projectId)
+		} else {
+			log.Println("ES未查到项目id", id)
+		}
+		//
+		if projectId != "" {
+			ok := Mgo.UpdateById("usermail", thisData["_id"], map[string]interface{}{"$set": map[string]interface{}{"projectId": projectId}})
+			if ok {
+				log.Println("项目id更新成功 ", id, projectId)
+			} else {
+				log.Println("项目id更新失败！ ", id, projectId)
+			}
+		}
+		thisData = map[string]interface{}{}
+	}
+}