data_validation/tools/main.go

package main

import (
	"fmt"
	"github.com/tealeg/xlsx"
	"go.mongodb.org/mongo-driver/bson"
	mgo "mongodb"
	"os"
	qu "qfw/util"
	"strings"
	"sync"
	"time"
	"util"
)

var (
	Sysconfig  map[string]interface{}
	Mgo, MgoJy *mgo.MongodbSim
)

func init() {
	qu.ReadConfig(&Sysconfig)
	MgoJy = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(Sysconfig["jy_addr"]),
		Size:        qu.IntAll(Sysconfig["size"]),
		DbName:      qu.ObjToString(Sysconfig["jy_db"]),
	}
	MgoJy.InitPool()
	Mgo = &mgo.MongodbSim{
		MongodbAddr: qu.ObjToString(Sysconfig["addr"]),
		Size:        qu.IntAll(Sysconfig["size"]),
		DbName:      qu.ObjToString(Sysconfig["db"]),
	}
	Mgo.InitPool()
}

func ShowTable() {
	fmt.Println("================================")
	fmt.Println("小工具-数据处理")
	fmt.Println("1、查找数据，打上标识(isOptimization=1)")
	fmt.Println("2、导出数据到文档(判重)")
	fmt.Println("3、导入数据")
	fmt.Println("4、推送数据")
	fmt.Println("0、EXIT")
	fmt.Println("================================")
}

func main() {
	for {
		ShowTable()
		var flag int
		fmt.Print("请输入你的操作:")
		fmt.Scan(&flag)
		if flag == 0 {
			fmt.Println("退出成功")
			os.Exit(0)
		} else if flag == 1 {
			fmt.Println("请输入表名...")
			var coll string
			fmt.Scan(&coll)
			task1(coll)
		} else if flag == 2 {
			fmt.Println("请输入表名...")
			var coll string
			fmt.Scan(&coll)
			task2(coll)
		} else if flag == 3 {
			fmt.Println("请输入表名...")
			var coll, path string
			fmt.Scan(&coll)
			if coll != "" {
				fmt.Println("请输入文档地址...")
				fmt.Scan(&path)
				if path != "" {
					task3(coll, path)
				} else {
					qu.Debug("请输入文档地址...")
				}
			} else {
				qu.Debug("请输入表名...")
			}
		} else if flag == 4 {
			fmt.Println("请输入表名...")
			var coll string
			fmt.Scan(&coll)
			if coll != "" {
				task4(coll)
			}
		}
	}
}

var FieldsMap = map[string]string{
	"是否优选": "is_push",
	//"是否有效标识":    "is_effective",
	//"运营商中标标签":   "tagname",
	//"主体公司":      "tagname2",
	//"中标人":       "s_winner",
	//"招标人":       "buyer",
	//"中标金额":      "bidamount",
	//"是否为多标多包数据": "multipackage",
}

func task1(coll string) {
	if coll != "" {
		sess := Mgo.GetMgoConn()
		defer Mgo.DestoryMongoConn(sess)

		ch := make(chan bool, 2)
		wg := &sync.WaitGroup{}

		//q := bson.M{"isOptimization": 1}
		query := sess.DB(Mgo.DbName).C(coll).Find(nil).Select(nil).Iter()
		count := 0
		for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
			if count%500 == 0 {
				qu.Debug("current ---", count)
			}
			ch <- true
			wg.Add(1)
			go func(tmp map[string]interface{}) {
				defer func() {
					<-ch
					wg.Done()
				}()
				info := tmp["v_baseinfo"].(map[string]interface{})
				if qu.IntAll(info["publishtime"]) < 1717171200 {
					return
				}
				if (qu.ObjToString(info["subtype"]) == "中标" || qu.ObjToString(info["subtype"]) == "成交" ||
					qu.ObjToString(info["subtype"]) == "合同") && qu.ObjToString(info["rulename"]) != "运营商中标_20230915" &&
					(qu.Float64All(info["bidamount"]) <= 1000 || qu.Float64All(info["bidamount"]) >= 50000000 || qu.ObjToString(info["s_winner"]) == "") {
					//qu.Debug(tmp["_id"])
					if strings.Contains(qu.ObjToString(info["rulename"]), "运营商中标") {
						Mgo.UpdateById(coll, mgo.BsonIdToSId(tmp["_id"]), bson.M{"$set": bson.M{"isOptimization": 1}})
					}
				}
			}(tmp)
			tmp = make(map[string]interface{})
		}
		wg.Wait()
		qu.Debug(fmt.Sprintf("数据更新成功完成"))
	} else {
		qu.Debug("表明不能为空")
	}
	os.Exit(0)
}

func task2(coll string) {
	if coll != "" {
		sess := Mgo.GetMgoConn()
		defer Mgo.DestoryMongoConn(sess)

		file, err := xlsx.OpenFile("./taskexportdata.xlsx")
		if err != nil {
			qu.Debug("Load Excel Model Error")
			return
		}
		sheet := file.Sheets[0]

		q := bson.M{"isOptimization": 1}
		query := sess.DB(Mgo.DbName).C(coll).Find(q).Select(nil).Iter()
		count := 0
		for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
			baseinfo := tmp["v_baseinfo"].(map[string]interface{})
			if baseinfo["package"] != nil {
				pkg := baseinfo["package"].(map[string]interface{})
				for _, p := range pkg {
					row := sheet.AddRow()
					p1 := p.(map[string]interface{})
					winner := []string{}
					bidamount := float64(0)
					if p1["winner_all"] != nil {
						if all := p1["winner_all"].([]interface{}); all != nil {
							if len(all) > 0 {
								for _, a := range all {
									a1 := a.(map[string]interface{})
									if qu.ObjToString(a1["winner"]) != "" {
										winner = append(winner, qu.ObjToString(a1["winner"]))
									}
									bidamount = qu.Float64All(a1["bidamount"])
								}
							}
						}
					}
					row.AddCell().SetValue(qu.ObjToString(tmp["id"]))
					row.AddCell().SetValue(strings.Join(winner, ","))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["buyer"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["projectcode"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["projectname"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["title"]))
					row.AddCell().SetValue(bidamount)
					row.AddCell().SetValue(qu.ObjToString(baseinfo["area"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["city"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["tagname"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["tagname2"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["href"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["jybxhref"]))
					row.AddCell().SetValue(qu.ObjToString(baseinfo["is_effective"]))
				}
			} else {
				row := sheet.AddRow()
				row.AddCell().SetValue(qu.ObjToString(tmp["id"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["s_winner"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["buyer"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["projectcode"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["projectname"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["title"]))
				row.AddCell().SetValue(baseinfo["bidamount"])
				row.AddCell().SetValue(qu.ObjToString(baseinfo["area"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["city"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["tagname"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["tagname2"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["href"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["jybxhref"]))
				row.AddCell().SetValue(qu.ObjToString(baseinfo["is_effective"]))
			}
		}
		qu.Debug(fmt.Sprintf("推送数据成功，推送成功： %d条", count))
		fname := fmt.Sprintf("./导出数据%d.xlsx", time.Now().Unix())
		qu.Debug("File Name：", fname)
		err = file.Save(fname)
		if err != nil {
			qu.Debug("Save Excel" + fname + "Error")
			return
		}
	} else {
		qu.Debug("表明不能为空")
	}
	os.Exit(0)
}

func task3(coll, path string) {
	if coll != "" && path != "" {
		file, err := xlsx.OpenFile(path)
		if err != nil {
			panic(err)
		}
		sheet := file.Sheets[0]
		count := 0
		idcolnum := -1
		cellFieldName := map[int]string{}
		//lastid := ""
		//c1 := 0 // 多包第n条数据
		repeatMap := make(map[string]bool)
		for rn, row := range sheet.Rows {
			//update := make(map[string]interface{})
			if rn == 0 {
				for index, cell := range row.Cells {
					if cell.Value == "唯一标识" || cell.Value == "标讯编码（infoID）" { //id所在列
						idcolnum = index
					}
					if v := FieldsMap[cell.Value]; v != "" {
						cellFieldName[index] = v
					}
				}
				if idcolnum == -1 {
					break
				}
				continue
			} else {
				id := util.SE.DecodeString(row.Cells[idcolnum].String())
				if !repeatMap[id] {
					info, b := Mgo.FindOne(coll, bson.M{"id": id})
					if b && len(*info) > 0 && qu.IntAll((*info)["is_push"]) != 1 {
						qu.Debug("id----", id)
						MgoJy.Save("tmp_02", *info)
					}
					repeatMap[id] = true
				}
				//for i, f := range cellFieldName {
				//	if val := row.Cells[i].Value; val != "" {
				//		if f == "is_push" {
				//			update[f] = qu.IntAll(val)
				//		} else if f == "multipackage" {
				//			update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.IntAll(val)
				//		} else if f == "bidamount" {
				//			update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.Float64All(val)
				//		} else {
				//			update[fmt.Sprintf("v_baseinfo.%s", f)] = val
				//		}
				//	}
				//}
				//if qu.IntAll(update["v_baseinfo.multipackage"]) == 1 {
				//	if c1 == 0 {
				//		count++
				//	}
				//	if c1 > 0 && lastid == id {
				//		info, _ := Mgo.FindById("f_sourceinfo_chinaunicom_zb_data", id, bson.M{"v_baseinfo.tagname": 1, "v_baseinfo.tagname2": 1, "v_baseinfo.s_winner": 1,
				//			"v_baseinfo.bidamount": 1, "v_baseinfo.package": 1})
				//		if len(*info) > 0 {
				//			baseinfo := (*info)["v_baseinfo"].(map[string]interface{})
				//			if baseinfo["package"] != nil {
				//				packageM := baseinfo["package"].(map[string]interface{})
				//				m := make(map[string]interface{})
				//				if update["v_baseinfo.bidamount"] != nil {
				//					m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
				//				}
				//				if update["v_baseinfo.s_winner"] != nil {
				//					m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
				//				}
				//				packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
				//				update["v_baseinfo.package"] = packageM
				//			}
				//			if s := qu.ObjToString(baseinfo["tagname"]); s != "" {
				//				update["v_baseinfo.tagname"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname"])
				//			}
				//			if s := qu.ObjToString(baseinfo["tagname2"]); s != "" {
				//				update["v_baseinfo.tagname2"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname2"])
				//			}
				//			if s := qu.ObjToString(baseinfo["s_winner"]); s != "" {
				//				update["v_baseinfo.s_winner"] = s + "," + qu.ObjToString(update["v_baseinfo.s_winner"])
				//			}
				//			update["v_baseinfo.bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"]) + qu.Float64All(baseinfo["bidamount"])
				//			c1++
				//		}
				//	} else {
				//		c1 = 0
				//		packageM := make(map[string]interface{})
				//		m := make(map[string]interface{})
				//		if update["v_baseinfo.bidamount"] != nil {
				//			m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
				//		}
				//		if update["v_baseinfo.s_winner"] != nil {
				//			m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
				//		}
				//		packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
				//		update["v_baseinfo.package"] = packageM
				//		c1++
				//	}
				//} else {
				//	count++
				//	c1 = 0
				//}
				//lastid = id
				//Mgo.Update(coll, bson.M{"id": id}, bson.M{"$set": update}, false, false)
			}
		}
		qu.Debug("over----", len(repeatMap))
		qu.Debug(fmt.Sprintf("更新数据成功，更新： %d条", count))
	} else {
		qu.Debug(coll, path)
	}
	os.Exit(0)
}

func task4(coll string) {
	sess := MgoJy.GetMgoConn()
	defer MgoJy.DestoryMongoConn(sess)

	ch := make(chan bool, 5)
	wg := &sync.WaitGroup{}

	//var q bson.M
	count := 0
	// 联通中标数据 特殊处理
	//q = bson.M{"is_push": 1}
	query := sess.DB("wjh").C(coll).Find(nil).Select(nil).Iter()
	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			//binfo, b := Mgo.FindOne("usermail", bson.M{"id": qu.ObjToString(tmp["id"]), "appid": "jyGQ1XQQsEAwNeSENOFR9D"})
			//if b && len(*binfo) > 0 {
			//	qu.Debug(qu.ObjToString(tmp["id"]))
			//	return
			//}
			info := tmp["v_baseinfo"].(map[string]interface{})
			info["createtime"] = time.Now().Unix()
			if qu.ObjToString(info["id"]) == "" {
				info["id"] = tmp["id"]
			}
			info["isOptimization"] = 1
			info["ispanchong"] = 1
			Mgo.Save("usermail", info)
			//MgoJy.UpdateById(coll, tmp["_id"], bson.M{"$set": bson.M{"sendflag": true}})

		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	qu.Debug(fmt.Sprintf("推送数据成功，推送成功： %d条", count))
}