Merge branch 'dev-ent_v1.1' of ssh://192.168.3.207:10022/qmx/datatag into dev-ent_v1.1

customerdata/src/task.go

@@ -192,7 +192,7 @@ func (c *Customer) GetData(stype string) {
 						//开始处理数据
 						wg.Add(1)
 						ch <- true
-						func(tmpHit *es.SearchHit) {
+						go func(tmpHit *es.SearchHit) {
 							defer func() {
 								<-ch
 								wg.Done()

src/history/datamodel.go

@@ -1,6 +1,7 @@
 package history
 
 import (
+	"qfw/util"
 	"regexp"
 	"sync"
 
@@ -10,6 +11,8 @@ import (
 var MgoSaveCache = make(chan map[string]interface{}, 1000)
 var xlsxArr []map[string]interface{}
 var SP = make(chan bool, 5)
+var xlsxCount = 0
+var SE = util.SimpleEncrypt{Key: "topJYBX2019"}
 var FielsArr = map[string]interface{}{
 	"title":            "公告标题",
 	"content":          "公告内容",

src/history/historytask.go

@@ -34,7 +34,8 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		appId := qu.ObjToString(c["s_appid"])        //appid
 		extends := qu.ObjToString(c["s_extends"])    //扩展信息
 		pushModel := qu.IntAll(c["i_pushmodel"])     //推送模式
-		email := qu.ObjToString(c["sendMail"])
+		email := qu.ObjToString(c["sendMail"])       //推送邮箱
+		isDup := qu.IntAll(c["isDup"])               //是否去重
 
 		cus := &Customer{}
 		cus.SaveDataMap = map[string]map[string]interface{}{}
@@ -74,18 +75,22 @@ func (this *HistoryData) HistoryTask(history_id string) {
 		cus.GetTagRules(tag_rules)               //获取客户打标签规则
 		cus.GetDepartments("history", dep_rules) //获取客户信息
 		qu.Debug("customer:", cus.ID, cus.Name, cus.PushModel, cus.AppId, cus.IsTagRule, cus.IsSearchHosp, cus.IsSearchEnps, len(cus.TagRules), len(cus.Departments))
-		cus.GetData("history")                             //获取数据
-		cus.RemoveRepeatData()                             //数据去重
-		resultcount := cus.AssembelAndSaveData(history_id) //组装、保存数据
+		cus.GetData("history")                     //获取数据
+		cus.RemoveRepeatData()                     //数据去重
+		cus.AssembelAndSaveData(history_id, isDup) //组装、保存数据
 		//发邮件
-		log.Println("xlsxArr", len(xlsxArr))
-		if len(xlsxArr) == resultcount {
+		go func() {
+			time.Sleep(3 * time.Second)
+			log.Println("xlsxArr", len(xlsxArr))
+			log.Println("xlsxCount", xlsxCount)
+			if len(xlsxArr) != xlsxCount {
+				log.Println("excel数据量错误")
+			}
 			GetXlsxs(xlsxArr, customer_name, email, history_id)
-		} else {
-			log.Println("excel数据量错误")
-		}
-		xlsxArr = []map[string]interface{}{}
-		go UpdateHistoryState(2, history_id, resultcount)
+			go UpdateHistoryState(2, history_id, xlsxCount)
+			xlsxArr = []map[string]interface{}{}
+			xlsxCount = 0
+		}()
 	} else {
 		log.Println("初始化客户信息失败")
 	}
@@ -123,12 +128,14 @@ func UpdateHistoryState(state int, history_id string, count int) {
 }
 
 func (this *HistoryData) HistoryClone(hid string) {
+	user := this.GetSession("user").(map[string]interface{})
 	data, ok := Mgo.FindOne("historylog", map[string]interface{}{"_id": mongoutil.StringTOBsonId(hid)})
 	if ok && data != nil && *data != nil {
 		(*data)["state"] = 0
 		(*data)["result_count"] = nil
 		(*data)["createtime"] = time.Now().Unix()
 		(*data)["updatetime"] = time.Now().Unix()
+		(*data)["create_user"] = user["name"]
 		Mgo.Save("historylog", *data)
 	}
 }

src/history/task.go

@@ -140,7 +140,7 @@ func (c *Customer) GetData(stype string) {
 						//开始处理数据
 						wg.Add(1)
 						ch <- true
-						func(tmpHit *es.SearchHit) {
+						go func(tmpHit *es.SearchHit) {
 							defer func() {
 								<-ch
 								wg.Done()
@@ -175,7 +175,6 @@ func (c *Customer) GetData(stype string) {
 														tmp["company_phone"] = phone
 													}
 												}
-
 											}
 										}
 									}
@@ -223,7 +222,7 @@ func (c *Customer) GetData(stype string) {
 								/*
 									到此已经匹配完数据
 								*/
-								// qu.Debug("---------------------", id, IsMatch, matchKey)
+								qu.Debug("---------------------", id, IsMatch)
 								if IsMatch { //匹配成功，数据上新增规则id,matchKey,item并临时保存数据
 									tmpMatchKey := MapDataToArr(matchKey)
 									tmpMatchKeyType := MapDataToArr(matchKeyType)
@@ -358,28 +357,27 @@ func (c *Customer) RemoveRepeatData() {
 }
 
 //组装保存数据
-func (c *Customer) AssembelAndSaveData(historyId string) int {
+func (c *Customer) AssembelAndSaveData(historyId string, isDup int) {
 	log.Println("开始组装保存数据...")
 	defer qu.Catch()
 	ch := make(chan bool, 10)
 	wg := &sync.WaitGroup{}
+	var datalock = sync.RWMutex{}
 	n := 0
 	if c.PushModel == 0 && len(c.SaveDataMap) > 0 {
 		for _, tmp := range c.SaveDataMap {
 			wg.Add(1)
 			ch <- true
 			go func(data map[string]interface{}) {
+				datalock.Lock()
 				defer func() {
 					<-ch
+					datalock.Unlock()
 					wg.Done()
 				}()
-				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId)
-				if !ok {
-					n--
-				}
+				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId, isDup)
 			}(tmp)
 			n++
-			xlsxArr = append(xlsxArr, tmp)
 			if n%500 == 0 {
 				log.Println("Current：", n)
 			}
@@ -392,17 +390,15 @@ func (c *Customer) AssembelAndSaveData(historyId string) int {
 					wg.Add(1)
 					ch <- true
 					go func(data map[string]interface{}) {
+						datalock.Lock()
 						defer func() {
 							<-ch
+							datalock.Unlock()
 							wg.Done()
 						}()
-						ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId)
-						if !ok {
-							n--
-						}
+						AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId, isDup)
 					}(tmp)
 					n++
-					xlsxArr = append(xlsxArr, tmp)
 					if n%500 == 0 {
 						log.Println("Current：", n)
 					}
@@ -415,17 +411,15 @@ func (c *Customer) AssembelAndSaveData(historyId string) int {
 			wg.Add(1)
 			ch <- true
 			go func(data map[string]interface{}) {
+				datalock.Lock()
 				defer func() {
 					<-ch
+					datalock.Unlock()
 					wg.Done()
 				}()
-				ok := AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId)
-				if !ok {
-					n--
-				}
+				AssembelSave(data, c.IsSearchHosp, c.IsSearchEnps, historyId, c.AppId, isDup)
 			}(tmp)
 			n++
-			xlsxArr = append(xlsxArr, tmp)
 			if n%500 == 0 {
 				log.Println("Current：", n)
 			}
@@ -433,7 +427,6 @@ func (c *Customer) AssembelAndSaveData(historyId string) int {
 		wg.Wait()
 	}
 	log.Println("数据保存完毕...		Save Number：", n)
-	return n
 }
 
 //获取用户所有规则

src/history/util_history.go

@@ -447,7 +447,7 @@ func MergeField(str1, str2 string) string {
 }
 
 //补充信息并保存
-func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, historyId, appid string) bool {
+func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, historyId, appid string, isDup int) {
 	if IsSearchHosp { //医院信息
 		SearchHospInfo(tmp)
 	}
@@ -465,12 +465,12 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, h
 		tmp["bidopentime"] = qu.Int64All(bidopentime)
 	}
 	//
+	isOk := true
 	if ProjectAppidMap[appid] {
 		projectId := GetProjectId(id)
 		if projectId != "" {
 			projectIds := strings.Split(projectId, ",")
 			tmp["projectId"] = projectId
-			isOk := true
 			for _, v := range projectIds {
 				if _, ok := projectIdMap.Load(v); ok {
 					isOk = false
@@ -480,12 +480,23 @@ func AssembelSave(tmp map[string]interface{}, IsSearchHosp, IsSearchEnps bool, h
 			}
 			if !isOk {
 				MgoSave.Save(SaveCollProject, tmp)
-				return false
 			}
 		}
 	}
-	MgoSaveCache <- tmp
-	return true
+	if isDup == 1 {
+		ok := checkBidId(appid, id)
+		if !ok {
+			isOk = false
+			tmp["chongfu"] = id
+			MgoSave.Save(SaveCollbid, tmp)
+		}
+	}
+
+	if isOk {
+		MgoSaveCache <- tmp
+		xlsxArr = append(xlsxArr, tmp)
+		xlsxCount++
+	}
 }
 
 //查询第三方医院等级信息和招标客户的社会征信代码
@@ -582,11 +593,13 @@ func SaveMgo() {
 	log.Println("Mgo Save...")
 	arru := make([]map[string]interface{}, 500)
 	indexu := 0
+	counts := 0
 	for {
 		select {
 		case v := <-MgoSaveCache:
 			arru[indexu] = v
 			indexu++
+			counts++
 			if indexu == 500 {
 				SP <- true
 				go func(arru []map[string]interface{}) {
@@ -791,79 +804,6 @@ func SkipData(tmp map[string]interface{}) bool {
 	return false
 }
 
-//生成xlsx
-// func GetXlsxs(mMap []map[string]interface{}, fn, email string) {
-// 	//信息
-// 	var err error
-// 	//样式
-// 	fx := xlsx.NewFile()
-// 	style := xlsx.NewStyle()
-// 	border := *xlsx.NewBorder("thin", "thin", "thin", "thin")
-// 	style.Border = border
-// 	style.Font.Size = 14
-// 	style.Alignment.Vertical = "center"
-// 	style.ApplyBorder = true
-// 	//
-// 	styleHead := xlsx.NewStyle()
-// 	styleHead.Font.Bold = true
-// 	styleHead.Font.Size = 18
-// 	styleHead.Border = border
-// 	styleHead.ApplyBorder = true
-
-// 	sheet1, err := fx.AddSheet("Sheet")
-// 	if err != nil {
-// 		log.Println("Err", err.Error())
-// 	}
-// 	head := sheet1.AddRow()
-// 	fields := []string{"title", "href", "detail", "publishtime", "matchkey", "area", "city", "district", "buyer", "buyerperson", "buyertel", "s_winner", "legal_person", "company_phone", "bidamount", "rank", "purchasing", "company_address", "capital", "establish_date", "business_scope", "stock_name", "buyer_credit_no", "winner_credit_no", "id"}
-// 	// for r, _ := range FielsArr {
-// 	// 	fields = append(fields, r)
-// 	// }
-
-// 	for _, t := range fields {
-// 		if FielsArr[t] != nil {
-// 			cell := head.AddCell()
-// 			cell.SetValue(FielsArr[t])
-// 			cell.SetStyle(styleHead)
-// 		}
-// 	}
-
-// 	for _, v := range mMap {
-// 		row := sheet1.AddRow()
-// 		for _, t := range fields {
-// 			cell := row.AddCell()
-// 			cell.SetStyle(style)
-// 			if v[t] != nil {
-// 				cell.SetValue(v[t])
-// 			}
-// 		}
-// 	}
-
-// 	//生文件
-// 	t := strconv.FormatInt(time.Now().Unix(), 10)
-// 	dir := "./web/res/xlsx/" + t + "/"
-// 	if b, _ := PathExists(dir); !b {
-// 		err1 := os.MkdirAll(dir, os.ModePerm)
-// 		if err1 != nil {
-// 			log.Println("mkdir err", dir)
-// 		}
-// 	}
-// 	fname := t + ".xlsx"
-// 	err = fx.Save(dir + fname)
-// 	if err != nil {
-// 		log.Println("xls error", fname)
-// 	} else {
-// 		for i := 0; i < len(Gmails); i++ {
-// 			gmail := Gmails[i]
-// 			status := mail.GSendMail_q("剑鱼标讯", email, "", "", fn, "", dir+fname, fname, gmail)
-// 			if status {
-// 				log.Println("send mail success", fname, email)
-// 				break
-// 			}
-// 		}
-// 	}
-// }
-
 func GetXlsxs(mMap []map[string]interface{}, fn, email, id string) {
 	if id != "" {
 		query := bson.M{
@@ -953,9 +893,9 @@ func GetXlsxs(mMap []map[string]interface{}, fn, email, id string) {
 				xf.Sheets = xf.Sheets[1:2]
 			} else {
 				sh := xf.Sheets[2]
-				for i, v := range mMap {
+				for _, v := range mMap {
 					row := sh.AddRow()
-					row.AddCell().SetInt(i + 1)
+					// row.AddCell().SetInt(i + 1)
 					row.AddCell().SetValue(v["matchkey"])
 					row.AddCell().SetValue(v["area"])
 					row.AddCell().SetValue(v["city"])
@@ -969,26 +909,27 @@ func GetXlsxs(mMap []map[string]interface{}, fn, email, id string) {
 					}
 					row.AddCell().SetValue(v["href"])
 					if v["bidamount"] != nil {
-						row.AddCell().SetFloat(qu.Float64All(v["bidamount"]) / 10000)
+						row.AddCell().SetFloat(qu.Float64All(v["bidamount"]))
 					} else {
 						row.AddCell()
 					}
+					row.AddCell().SetValue(v["purchasing"])
 					row.AddCell().SetValue(v["buyer"])
 					row.AddCell().SetValue(v["buyerperson"])
 					row.AddCell().SetValue(v["buyertel"])
+					row.AddCell().SetValue(v["buyer_credit_no"])
+					row.AddCell().SetValue(v["rank"])
 					row.AddCell().SetValue(v["s_winner"])
 					row.AddCell().SetValue(v["legal_person"])
 					row.AddCell().SetValue(v["company_phone"])
+					row.AddCell().SetValue(v["winner_credit_no"])
 					row.AddCell().SetValue(v["company_address"])
-					row.AddCell().SetValue(v["rank"])
-					row.AddCell().SetValue(v["purchasing"])
 					row.AddCell().SetValue(v["capital"])
 					row.AddCell().SetValue(v["establish_date"])
 					row.AddCell().SetValue(v["business_scope"])
 					row.AddCell().SetValue(v["stock_name"])
-					row.AddCell().SetValue(v["buyer_credit_no"])
-					row.AddCell().SetValue(v["winner_credit_no"])
-					row.AddCell().SetValue(v["id"])
+					ids := SE.EncodeString(qu.ObjToString(v["id"]))
+					row.AddCell().SetValue(ids)
 				}
 				xf.Sheets = xf.Sheets[2:3]
 			}
@@ -1039,7 +980,6 @@ func GetProjectId(id string) string {
 	if data != nil && *data != nil && len(*data) > 0 {
 		for _, v := range *data {
 			projectIdArr = append(projectIdArr, qu.ObjToString(v["_id"]))
-
 		}
 		projectId = strings.Join(projectIdArr, ",")
 		log.Println("projectId", projectId)
@@ -1071,3 +1011,18 @@ func InitProjectId(appid string) {
 		thisData = map[string]interface{}{}
 	}
 }
+
+func checkBidId(appid, id string) bool {
+	isOk := true
+	userMailData, ok := Mgo.FindOne(SaveUserMail, bson.M{"appid": appid, "id": id})
+	if ok && userMailData != nil && *userMailData != nil {
+		log.Println("用户 ", appid, " 在 ", SaveUserMail, " 中找到重复公告id ", id)
+		isOk = false
+	}
+	userHistoryData, oks := Mgo.FindOne(SaveColl, bson.M{"appid": appid, "id": id})
+	if oks && userHistoryData != nil && *userHistoryData != nil {
+		log.Println("用户 ", appid, " 在 ", SaveColl, " 中找到重复公告id ", id)
+		isOk = false
+	}
+	return isOk
+}

src/util/config.go

@@ -36,6 +36,8 @@ var (
 	EId             string
 	CustomerName    string
 	SaveCollProject string
+	SaveCollbid     string
+	SaveUserMail    string
 )
 
 var (
@@ -187,7 +189,9 @@ func initdb() {
 	//mgo save
 	save := Sysconfig["save"].(map[string]interface{})
 	SaveColl = qu.ObjToString(save["coll"])
-	SaveCollProject = qu.ObjToString(save["coll"])
+	SaveCollProject = qu.ObjToString(save["coll_project"])
+	SaveCollbid = qu.ObjToString(save["coll_bid"])
+	SaveUserMail = qu.ObjToString(save["coll_usermail"])
 	MgoSave = &mgo.MongodbSim{
 		MongodbAddr: qu.ObjToString(save["addr"]),
 		DbName:      qu.ObjToString(save["db"]),