no message

isrepeat/main.go

@@ -6,81 +6,220 @@ import (
 	"github.com/tealeg/xlsx"
 	"go.mongodb.org/mongo-driver/bson"
 	mgo "mongodb"
+	"os"
 	qu "qfw/util"
+	"sync"
+	"time"
 	"util"
 )
 
 var (
-	path string
-
 	Mgo *mgo.MongodbSim
 )
 
 func init() {
 	Mgo = &mgo.MongodbSim{
-		MongodbAddr: "192.168.3.166:27082", // 127.0.0.1:27084
+		MongodbAddr: "127.0.0.1:27086", // 127.0.0.1:27084
 		Size:        5,
 		DbName:      "jyqykhfw",
 	}
 	Mgo.InitPool()
 }
 
+func ShowTable() {
+	fmt.Println("================================")
+	fmt.Println("小工具-数据处理")
+	fmt.Println("1、导入清洗数据")
+	fmt.Println("2、推送数据")
+	fmt.Println("3、数据判重")
+	fmt.Println("0、EXIT")
+	fmt.Println("================================")
+}
+
 func main() {
-	flag.StringVar(&path, "f", "", "文件路径")
-	flag.Parse()
+	for {
+		ShowTable()
+		var flag int
+		fmt.Print("请输入你的操作:")
+		fmt.Scan(&flag)
+		if flag == 0 {
+			fmt.Println("退出成功")
+			os.Exit(0)
+		} else if flag == 1 {
+			fmt.Println("请输入文档地址.")
+			var p string
+			fmt.Scan(&p)
+			if p != "" {
+				task1(p)
+			}
+		} else if flag == 2 {
+			fmt.Println("导入推送表数据.")
+			task2()
+		} else if flag == 3 {
+			fmt.Println("请输入文档地址，进行数据判重.")
+			var p string
+			fmt.Scan(&p)
+			if p != "" {
+				task3()
+			}
+		}
+	}
+}
+
+var FieldsMap = map[string]string{
+	"是否优选":      "is_push",
+	"运营商中标标签":   "tagname",
+	"主体公司":      "tagname2",
+	"中标人":       "s_winner",
+	"招标人":       "buyer",
+	"中标金额":      "bidamount",
+	"是否为多标多包数据": "multipackage",
+}
+
+func task1(path string) {
 	if path != "" {
 		file, err := xlsx.OpenFile(path)
 		if err != nil {
 			panic(err)
 		}
 		sheet := file.Sheets[0]
+		count := 0
+		idcolnum := -1
 		cellFieldName := map[int]string{}
+		lastid := ""
+		c1 := 0 // 多包第n条数据
 		for rn, row := range sheet.Rows {
+			update := make(map[string]interface{})
 			if rn == 0 {
 				for index, cell := range row.Cells {
-					//if cell.Value == "招标人" || cell.Value == "中标人" || cell.Value == "中标金额" {
-					//	qu.Debug(cell.Value, index)
-					//}
+					if cell.Value == "唯一标识" || cell.Value == "标讯编码（infoID）" { //id所在列
+						idcolnum = index
+					}
 					if v := FieldsMap[cell.Value]; v != "" {
 						cellFieldName[index] = v
 					}
 				}
+				if idcolnum == -1 {
+					break
+				}
+				continue
 			} else {
-				q := bson.M{}
+				id := row.Cells[idcolnum].String()
+				id = util.SE.DecodeString(id)
 				for i, f := range cellFieldName {
-					if val := row.Cells[i].Value; val != "" && (f == "s_winner" || f == "buyer") {
-						q[f] = val
-					}
-					if val := row.Cells[i].Value; val != "" && f == "bidamount" {
-						if qu.Float64All(val) != 0 {
-							q[f] = qu.Float64All(val)
+					if val := row.Cells[i].Value; val != "" {
+						if f == "is_push" {
+							update[f] = qu.IntAll(val)
+						} else if f == "multipackage" {
+							update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.IntAll(val)
+						} else if f == "bidamount" {
+							update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.Float64All(val)
+						} else {
+							update[fmt.Sprintf("v_baseinfo.%s", f)] = val
 						}
 					}
 				}
-				info, _ := Mgo.FindOne("zglt_history", q)
-				if len(*info) > 0 {
-					row.Cells[12].SetValue(-1)
+				if qu.IntAll(update["v_baseinfo.multipackage"]) == 1 {
+					if c1 == 0 {
+						count++
+					}
+					if c1 > 0 && lastid == id {
+						info, _ := Mgo.FindById("f_sourceinfo_chinaunicom_zb_data", id, bson.M{"v_baseinfo.tagname": 1, "v_baseinfo.tagname2": 1, "v_baseinfo.s_winner": 1,
+							"v_baseinfo.bidamount": 1, "v_baseinfo.package": 1})
+						if len(*info) > 0 {
+							baseinfo := (*info)["v_baseinfo"].(map[string]interface{})
+							if baseinfo["package"] != nil {
+								packageM := baseinfo["package"].(map[string]interface{})
+								m := make(map[string]interface{})
+								if update["v_baseinfo.bidamount"] != nil {
+									m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
+								}
+								if update["v_baseinfo.s_winner"] != nil {
+									m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
+								}
+								packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
+								update["v_baseinfo.package"] = packageM
+							}
+							if s := qu.ObjToString(baseinfo["tagname"]); s != "" {
+								update["v_baseinfo.tagname"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname"])
+							}
+							if s := qu.ObjToString(baseinfo["tagname2"]); s != "" {
+								update["v_baseinfo.tagname2"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname2"])
+							}
+							if s := qu.ObjToString(baseinfo["s_winner"]); s != "" {
+								update["v_baseinfo.s_winner"] = s + "," + qu.ObjToString(update["v_baseinfo.s_winner"])
+							}
+							update["v_baseinfo.bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"]) + qu.Float64All(baseinfo["bidamount"])
+							c1++
+						}
+					} else {
+						c1 = 0
+						packageM := make(map[string]interface{})
+						m := make(map[string]interface{})
+						if update["v_baseinfo.bidamount"] != nil {
+							m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
+						}
+						if update["v_baseinfo.s_winner"] != nil {
+							m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
+						}
+						packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
+						update["v_baseinfo.package"] = packageM
+						c1++
+					}
 				} else {
-					row.Cells[12].SetValue(1)
+					count++
+					c1 = 0
 				}
+				// 临时
+				update["tag"] = "临时"
+				lastid = id
+				qu.Debug(update)
+				Mgo.UpdateById("f_sourceinfo_chinaunicom_zb_data", id, bson.M{"$set": update})
 			}
 		}
-		err = file.Save(path)
-	} else {
-		flag.PrintDefaults()
+		qu.Debug(fmt.Sprintf("更新数据成功，更新： %d条", count))
+		os.Exit(0)
 	}
 }
 
-var FieldsMap = map[string]string{
-	"是否优选":    "is_push",
-	"运营商中标标签": "tagname",
-	"主体公司":    "tagname2",
-	"中标人":     "s_winner",
-	"招标人":     "buyer",
-	"中标金额":    "bidamount",
+func task2() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	ch := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+
+	q := bson.M{"tag": "临时"}
+	query := sess.DB(Mgo.DbName).C("f_sourceinfo_chinaunicom_zb_data").Find(q).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); query.Next(&tmp); count++ {
+		ch <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			info := tmp["v_baseinfo"].(map[string]interface{})
+			info["createtime"] = time.Now().Unix()
+			if qu.ObjToString(info["id"]) == "" {
+				info["id"] = tmp["id"]
+			}
+			info["isOptimization"] = 1
+			delete(info, "field_source")
+			delete(info, "regions_log")
+			Mgo.Save("tmp_usermail", info)
+
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	qu.Debug(fmt.Sprintf("推送数据成功，推送成功： %d条", count))
+	os.Exit(0)
 }
 
-func main1() {
+func task3() {
+	var path string
 	flag.StringVar(&path, "f", "", "文件路径")
 	flag.Parse()
 	if path != "" {
@@ -89,37 +228,35 @@ func main1() {
 			panic(err)
 		}
 		sheet := file.Sheets[0]
-		idcolnum := -1
 		cellFieldName := map[int]string{}
-		update := make(map[string]interface{})
 		for rn, row := range sheet.Rows {
 			if rn == 0 {
 				for index, cell := range row.Cells {
-					if cell.Value == "唯一标识" || cell.Value == "标讯编码（infoID）" { //id所在列
-						idcolnum = index
-					}
+					//if cell.Value == "招标人" || cell.Value == "中标人" || cell.Value == "中标金额" {
+					//	qu.Debug(cell.Value, index)
+					//}
 					if v := FieldsMap[cell.Value]; v != "" {
 						cellFieldName[index] = v
 					}
 				}
-				if idcolnum == -1 {
-					break
-				}
-				continue
 			} else {
-				id := row.Cells[idcolnum].String()
-				id = util.SE.DecodeString(id)
+				q := bson.M{}
 				for i, f := range cellFieldName {
-					if val := row.Cells[i].Value; val != "" {
-						if f == "is_push" {
-							update[f] = qu.IntAll(val)
-						} else {
-							update[fmt.Sprintf("v_baseinfo.%s", f)] = val
+					if val := row.Cells[i].Value; val != "" && (f == "s_winner" || f == "buyer") {
+						q[f] = val
+					}
+					if val := row.Cells[i].Value; val != "" && f == "bidamount" {
+						if qu.Float64All(val) != 0 {
+							q[f] = qu.Float64All(val)
 						}
 					}
 				}
-				qu.Debug(id, update)
-				Mgo.UpdateById("f_sourceinfo_chinaunicom_zb_data", id, bson.M{"$set": update})
+				info, _ := Mgo.FindOne("zglt_history", q)
+				if len(*info) > 0 {
+					row.Cells[12].SetValue(-1)
+				} else {
+					row.Cells[12].SetValue(1)
+				}
 			}
 		}
 		err = file.Save(path)

src/config.json

@@ -26,6 +26,7 @@
         "addr":"192.168.3.166:27082",
         "db": "qfw",
         "history": "usermail_history",
+        "pushcoll": "usermail",
         "user": "user",
         "size": 10
     },

src/front/group.go

@@ -336,7 +336,7 @@ func syncJyData(source, tid, pid string) int {
 				info["id"] = tmp["id"]
 			}
 			info["isOptimization"] = 1
-			util.MgoJy.Save("usermail", info)
+			util.MgoJy.Save(util.JYPushColl, info)
 			util.Mgo.UpdateById(source, tmp["_id"], bson.M{"$set": bson.M{"sendflag": true}})
 
 		}(tmp)
@@ -839,7 +839,7 @@ func (f *Front) GroupExportData() {
 		for _, l := range *list {
 			row := sheet.AddRow()
 			baseinfo := l["v_baseinfo"].(map[string]interface{})
-			row.AddCell().SetValue(qu.ObjToString(l["id"]))
+			row.AddCell().SetValue(util.SE.EncodeString(qu.ObjToString(l["id"])))
 			row.AddCell().SetValue(qu.ObjToString(baseinfo["s_winner"]))
 			row.AddCell().SetValue(qu.ObjToString(baseinfo["buyer"]))
 			row.AddCell().SetValue(qu.ObjToString(baseinfo["projectcode"]))
@@ -896,6 +896,8 @@ func (f *Front) GroupImportData() {
 	cellFieldName := map[int]string{}
 	update := make(map[string]interface{})
 	count := 0
+	lastid := ""
+	c1 := 0 // 多包第n条数据
 	for rn, row := range rows {
 		if rn == 0 {
 			for index, cell := range row.Cells {
@@ -912,17 +914,75 @@ func (f *Front) GroupImportData() {
 			continue
 		} else {
 			id := row.Cells[idcolnum].String()
+			id = util.SE.DecodeString(id)
 			for i, f := range cellFieldName {
 				if val := row.Cells[i].Value; val != "" {
 					if f == "is_push" {
 						update[f] = qu.IntAll(val)
+					} else if f == "multipackage" {
+						update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.IntAll(val)
+					} else if f == "bidamount" {
+						update[fmt.Sprintf("v_baseinfo.%s", f)] = qu.Float64All(val)
 					} else {
 						update[fmt.Sprintf("v_baseinfo.%s", f)] = val
 					}
 				}
 			}
+			if qu.IntAll(update["v_baseinfo.multipackage"]) == 1 {
+				if c1 == 0 {
+					count++
+				}
+				if c1 > 0 && lastid == id {
+					info, _ := util.Mgo.FindById(sourceinfo, id, bson.M{"v_baseinfo.tagname": 1, "v_baseinfo.tagname2": 1, "v_baseinfo.s_winner": 1,
+						"v_baseinfo.bidamount": 1, "v_baseinfo.package": 1})
+					if len(*info) > 0 {
+						baseinfo := (*info)["v_baseinfo"].(map[string]interface{})
+						if baseinfo["package"] != nil {
+							packageM := baseinfo["package"].(map[string]interface{})
+							m := make(map[string]interface{})
+							if update["v_baseinfo.bidamount"] != nil {
+								m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
+							}
+							if update["v_baseinfo.s_winner"] != nil {
+								m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
+							}
+							packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
+							update["v_baseinfo.package"] = packageM
+						}
+						if s := qu.ObjToString(baseinfo["tagname"]); s != "" {
+							update["v_baseinfo.tagname"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname"])
+						}
+						if s := qu.ObjToString(baseinfo["tagname2"]); s != "" {
+							update["v_baseinfo.tagname2"] = s + "," + qu.ObjToString(update["v_baseinfo.tagname2"])
+						}
+						if s := qu.ObjToString(baseinfo["s_winner"]); s != "" {
+							update["v_baseinfo.s_winner"] = s + "," + qu.ObjToString(update["v_baseinfo.s_winner"])
+						}
+						update["v_baseinfo.bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"]) + qu.Float64All(baseinfo["bidamount"])
+						c1++
+					}
+				} else {
+					c1 = 0
+					packageM := make(map[string]interface{})
+					m := make(map[string]interface{})
+					if update["v_baseinfo.bidamount"] != nil {
+						m["bidamount"] = qu.Float64All(update["v_baseinfo.bidamount"])
+					}
+					if update["v_baseinfo.s_winner"] != nil {
+						m["s_winner"] = qu.ObjToString(update["v_baseinfo.s_winner"])
+					}
+					packageM[fmt.Sprint(c1)] = map[string]interface{}{"winner_all": append([]interface{}{}, m)}
+					update["v_baseinfo.package"] = packageM
+					c1++
+				}
+			} else {
+				count++
+				c1 = 0
+			}
+			// 临时
+			update["tag"] = "临时"
+			lastid = id
 			util.Mgo.UpdateById(sourceinfo, id, bson.M{"$set": update})
-			count++
 		}
 	}
 	f.ServeJson(map[string]interface{}{"success": true, "msg": "数据导入成功", "count": count})

src/front/remark.go

@@ -1251,6 +1251,9 @@ func (f *Front) CheckSave() {
 		if len(baseUnsetResult) > 0 {
 			set["$unset"] = baseUnsetResult
 		}
+		qu.Debug(sourceInfo)
+		qu.Debug(infoId)
+		qu.Debug(set)
 		success = util.Mgo.UpdateById(sourceInfo, infoId, set)
 		//2、更新marked表
 		tmp, _ := util.Mgo.FindById(sourceInfo, infoId, map[string]interface{}{"v_baseinfo": 1, "v_taginfo": 1, "i_ckdata": 1})
@@ -1873,6 +1876,9 @@ func (f *Front) JyUserDataMark() {
 		if len(baseUnsetResult) > 0 {
 			set["$unset"] = baseUnsetResult
 		}
+		qu.Debug(sourceInfo)
+		qu.Debug(infoId)
+		qu.Debug(set)
 		success = util.Mgo.UpdateById(sourceInfo, infoId, set)
 		//2、更新marked表
 		tmp, _ := util.Mgo.FindById(sourceInfo, infoId, map[string]interface{}{"v_baseinfo": 1, "v_taginfo": 1, "i_ckdata": 1})
@@ -2030,6 +2036,9 @@ func (f *Front) UserDataMark() {
 		if len(baseUnsetResult) > 0 {
 			set["$unset"] = baseUnsetResult
 		}
+		qu.Debug(sourceInfo)
+		qu.Debug(infoId)
+		qu.Debug(set)
 		success = util.Mgo.UpdateById(sourceInfo, infoId, set)
 		//2、更新marked表
 		tmp, _ := util.Mgo.FindById(sourceInfo, infoId, map[string]interface{}{"v_baseinfo": 1, "v_taginfo": 1, "i_ckdata": 1})

src/main.go

@@ -35,6 +35,6 @@ func init() {
 }
 
 func main() {
-
 	xweb.Run(":" + qu.ObjToString(Sysconfig["port"]))
+
 }

src/util/config.go

@@ -29,6 +29,7 @@ var (
 	ProjectColl         string
 	MgoJy               *mongodb.MongodbSim //jy
 	JyHistory           string
+	JYPushColl          string
 	JyUser              string
 	CustomerFieldMap_EH map[string]string
 	CustomerFieldMap_HE map[string]string
@@ -121,6 +122,7 @@ func InitConfig() {
 	//jy
 	jy := Sysconfig["jy"].(map[string]interface{})
 	JyHistory = qu.ObjToString(jy["history"])
+	JYPushColl = qu.ObjToString(jy["pushcoll"])
 	JyUser = qu.ObjToString(jy["user"])
 	MgoJy = &mongodb.MongodbSim{
 		MongodbAddr: qu.ObjToString(jy["addr"]),