数据重复处理

jyBXSubscribe/rpc/model/push.go

@@ -360,8 +360,8 @@ func (s *subscribePush) getDatasFromMysql(spqp *SubPushQueryParam, starttime, en
 		logx.Info("searchSql", searchSql)
 		//查询总数
 		count = spqp.BaseServiceMysql.CountBySql(fmt.Sprintf("select count(a.id)" + searchSql))
-		findSql = "SELECT  aa.*,  bb.labelvalues as industry  FROM (select a.id,a.date,a.infoid,a.isvisit,a.matchkeys,a.type,b.isvalidfile as attachment_count,b.site," +
-			"b.title,b.area_code,b.buyerclass_code,b.subtype_code,b.publishtime,b.budget,b.bidamount,b.projectname,b.bidopentime "
+		findSql = "SELECT  aa.*, cc.name as  buyer, bb.labelvalues as industry  FROM (select DISTINCT a.id,a.date,a.infoid,a.isvisit,a.matchkeys,a.type,b.isvalidfile as attachment_count,b.site," +
+			"b.title,b.area_code,b.buyerclass_code,b.subtype_code,b.publishtime,b.budget,b.bidamount,b.projectname,b.bidopentime,b.buyer_id  "
 
 	} else {
 		if spqp.IsEnt {
@@ -384,11 +384,11 @@ func (s *subscribePush) getDatasFromMysql(spqp *SubPushQueryParam, starttime, en
 			if spqp.BuySubject == 1 {
 				//企业主体是企业的个人查询
 				if spqp.UserType == SubVipFlag {
-					userStr = fmt.Sprintf("  a.userid='%s' and ( (a.source=1 and  a.product=1 ) or a.source>1  ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
+					userStr = fmt.Sprintf("  a.userid='%s' and (  a.product=1 or a.source>1     ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
 				} else if spqp.UserType == MemberFlag {
-					userStr = fmt.Sprintf("  a.userid='%s' and ( (a.source=1 and  a.product=2 ) or a.source>1  ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
+					userStr = fmt.Sprintf("  a.userid='%s' and (  a.product=2  or a.source>1  ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
 				} else {
-					userStr = fmt.Sprintf("  a.userid='%s' and ( (a.source=1 and  a.product is null ) or a.source>1  ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
+					userStr = fmt.Sprintf("  a.userid='%s' and (  a.product is null  or a.source>1  ) ", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
 				}
 			} else {
 				if s.ModuleFlag == EntnicheFlag {
@@ -398,7 +398,7 @@ func (s *subscribePush) getDatasFromMysql(spqp *SubPushQueryParam, starttime, en
 					//个人订阅查询
 					userStr = fmt.Sprintf("  a.userid='%s'", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
 				}
-				// userStr = fmt.Sprintf("  a.userid='%s'", common.If(s.ModuleFlag == EntnicheFlag, spqp.EntUserId, common.InterfaceToStr(spqp.NewUserId)))
+
 			}
 		}
 
@@ -536,13 +536,13 @@ func (s *subscribePush) getDatasFromMysql(spqp *SubPushQueryParam, starttime, en
 			" order by a.id desc", aboutDbMsg[s.ModuleFlag].MysqlTable, Baseinfo, userStr, leftJoinStr, strings.Join(querys, " and "))
 		logx.Info("searchSql", searchSql)
 		//查询总数
-		logx.Info(fmt.Sprintf("select   count(a.id) " + searchSql))
-		count = spqp.BaseServiceMysql.CountBySql(fmt.Sprintf("select   count(a.id) " + searchSql))
+		logx.Info(fmt.Sprintf("select   count(DISTINCT a.id) " + searchSql))
+		count = spqp.BaseServiceMysql.CountBySql(fmt.Sprintf("select   count( DISTINCT a.id) " + searchSql))
 		start2 = time.Now().Unix()
 		logx.Info("count:", count, "---", s.ModuleFlag)
-		log.Println("查询数据总数用时", start2-start)
-		findSql = "SELECT  aa.*,  bb.labelvalues as industry  FROM (select a.id,a.date,a.infoid,a.isvisit,a.matchkeys,a.type,b.isvalidfile as attachment_count,b.site," +
-			"b.title,b.area_code,b.buyerclass_code,b.subtype_code,b.publishtime,b.budget,b.bidamount,b.projectname,b.bidopentime "
+		logx.Info("查询数据总数用时", start2-start)
+		findSql = "SELECT  aa.*,  bb.labelvalues as industry, cc.name as  buyer FROM (select a.id,a.date,a.infoid,a.isvisit,a.matchkeys,a.type,b.isvalidfile as attachment_count,b.site," +
+			"b.title,b.area_code,b.buyerclass_code,b.subtype_code,b.publishtime,b.budget,b.bidamount,b.projectname,b.bidopentime,b.buyer_id  "
 		if s.ModuleFlag == EntnicheFlag {
 			findSql += ", IF(a.source is NULL,1,a.source) as  source "
 		}
@@ -554,15 +554,15 @@ func (s *subscribePush) getDatasFromMysql(spqp *SubPushQueryParam, starttime, en
 	if isLimit {
 		findSql += fmt.Sprintf(" limit %d,%d", (spqp.PageNum-1)*size, size)
 	}
-	findSql += fmt.Sprintf(") aa LEFT JOIN %s bb ON ( aa.infoid = bb.infoid )  AND bb.labelcode = 2 ", BidTags)
+	findSql += fmt.Sprintf(") aa LEFT JOIN %s bb ON ( aa.infoid = bb.infoid )  AND bb.labelcode = 2  left join  global_common_data.dws_f_ent_baseinfo  cc on aa.buyer_id=cc.name_id ORDER BY aa.id desc ", BidTags)
 	logx.Info(spqp.NewUserId, "subscribePush query sql：", findSql)
 	list := spqp.BaseServiceMysql.SelectBySql(findSql)
 	start3 := time.Now().Unix()
-	log.Println("查询数据用时", start3-start2)
+	logx.Info("查询数据用时", start3-start2)
 	if list != nil && len(*list) > 0 {
 		pushCas, subscribeInfo := s.GetJyPushs(*list, codeMap)
 		start4 := time.Now().Unix()
-		log.Println("清洗数据耗时", start4-start3)
+		logx.Info("清洗数据耗时", start4-start3)
 		if !spqp.Export {
 			result = subscribeInfo
 		} else {
@@ -689,15 +689,15 @@ func (s *subscribePush) GetJyPushs(datas []map[string]interface{}, codeMap *code
 		buyerClassMap[v] = k
 	}
 	array := []map[string]interface{}{}
-	m := map[int64]string{}
+	m := map[int64]int64{}
 	for _, v := range datas {
 		id := v["id"].(int64)
-		if _, ok := m[id]; ok {
+		if m[id] != 0 {
 		} else {
+			m[id] = id
 			array = append(array, v)
 		}
 	}
-
 	for _, v := range array {
 		keys := []string{}
 		if matchkeys := common.ObjToString(v["matchkeys"]); matchkeys != "" {
@@ -731,6 +731,8 @@ func (s *subscribePush) GetJyPushs(datas []map[string]interface{}, codeMap *code
 		}
 		publishTimeStr := common.InterfaceToStr(v["publishtime"])
 		publishTime, _ := time.ParseInLocation("2006-01-02 15:04:05", publishTimeStr, time.Local)
+		bidopentimeStr := common.InterfaceToStr(v["bidopentime"])
+		bidopentime, _ := time.ParseInLocation("2006-01-02 15:04:05", bidopentimeStr, time.Local)
 		subscribeInfo = append(subscribeInfo, &bxsubscribe.SubscribeInfo{
 			XId:          encrypt.EncodeArticleId2ByCheck(common.ObjToString(v["infoid"])),
 			Title:        common.InterfaceToStr(v["title"]),
@@ -748,10 +750,11 @@ func (s *subscribePush) GetJyPushs(datas []map[string]interface{}, codeMap *code
 			BidAmount:    common.InterfaceToStr(v["bidamount"]),
 			ProjectName:  common.InterfaceToStr(v["projectname"]),
 			Winner:       common.InterfaceToStr(v["s_winner"]),
-			BidOpenTime:  common.Int64All(v["bidopentime"]),
+			BidOpenTime:  bidopentime.Unix(),
 			CaFileExists: common.IntAll(v["attachment_count"]) > 0,
 			Source:       common.Int64All(v["source"]),
 			Site:         common.InterfaceToStr(v["site"]),
+			Buyer:        common.InterfaceToStr(v["buyer"]),
 		})
 	}
 	return pushCas, subscribeInfo
@@ -990,7 +993,7 @@ func (s *subscribePush) DefaultDatas(spqp *SubPushQueryParam) (hasNextPage bool,
 		return false, 0, nil
 	}
 	t1 := time.Now()
-	log.Println("userId:", spqp.UserId, "newUserId:", spqp.NewUserId)
+	logx.Info("userId:", spqp.UserId, "newUserId:", spqp.NewUserId)
 	//用户信息
 	bsp := s.getUserInfo(spqp)
 	logx.Info("获取用户信息耗时：", time.Since(t1))
@@ -1068,7 +1071,7 @@ func (s *subscribePush) listManager(spqp *SubPushQueryParam, list []map[string]i
 		resultList[i].CaIndex = id
 		id++
 	}
-	log.Println("数据处理耗时：", time.Since(t2))
+	logx.Info("数据处理耗时：", time.Since(t2))
 	return
 }