提交

kdxf/src/main.go

@@ -7,29 +7,25 @@ import (
 	"qfw/mongodb"
 	qu "qfw/util"
 	"qfw/util/mysql"
-	"qfw/util/redis"
 	"strconv"
 	"strings"
 )
 
 var (
 	Sysconfig        map[string]interface{}
-	JyMysql          *mysql.Mysql
 	JyzhshiMysql     *mysql.Mysql
 	JypushzhshiMysql *mysql.Mysql
 	Mgo              *mongodb.MongodbSim
+
+)
+
+const (
+	ONLINETABLE ="20211008kdxfentdataexport"
+	ENDTIME = 1633017600
 )
 
 func init() {
 	qu.ReadConfig(&Sysconfig)
-	jyMysql := qu.ObjToMap(Sysconfig["mysql"])
-	JyMysql = &mysql.Mysql{
-		Address:  (*jyMysql)["address"].(string),
-		UserName: (*jyMysql)["username"].(string),
-		PassWord: (*jyMysql)["password"].(string),
-		DBName:   (*jyMysql)["dbName"].(string),
-	}
-	JyMysql.Init()
 	// 这个是正式库
 	jyzhshiMysql := qu.ObjToMap(Sysconfig["jyzhengshimysql"])
 	JyzhshiMysql = &mysql.Mysql{
@@ -55,10 +51,7 @@ func init() {
 		Size:        qu.IntAllDef(mogConfig["size"], 15),
 	}
 	Mgo.InitPool()
-
-	redis.InitRedis(qu.ObjToString(Sysconfig["redis_addrs"]))
 }
-//订阅数据推送统计  将结果粘到excel  分列
 func pushtj() {
 
 	//	 查剑鱼主库用户
@@ -72,9 +65,11 @@ func pushtj() {
 	}
 	idStr :=strings.Join(idStrList,",")
 	log.Println(idStr)
-	// todo  每次修改结束时间
-	pushtjSql:="SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <1625068800  GROUP BY userid "
-	//	 聚合推送库信息
+	endTime :=strconv.Itoa(ENDTIME)
+	log.Println(endTime)
+	pushtjSql:="SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <"+endTime +" GROUP BY userid "
+
+	log.Println(pushtjSql)//	 聚合推送库信息
 	tjRs := JypushzhshiMysql.SelectBySql(pushtjSql)
 	tjMap := map[int64]interface{}{}
 	for _,v := range *tjRs{
@@ -91,10 +86,10 @@ func pushtj() {
 			user["name"]="系统管理员"
 		}
 		user["count"] = count
-		if count !=0{
+		//if count !=0{
 
 		fmt.Println(user["name"],user["phone"],user["count"])
-		}
+		//}
 	}
 
 }
@@ -105,7 +100,7 @@ func onlineDataTj()  {
 	defer Mgo.DestoryMongoConn(sess)
 	var res []bson.M
 
-	err := sess.DB(Mgo.DbName).C("20210701kdxfentdataexport").Pipe([]bson.M{
+	err := sess.DB(Mgo.DbName).C(ONLINETABLE).Pipe([]bson.M{
 		//bson.M{"$match": bson.M{"createtime": bson.M{ "$lt":1619280000}}},
 		bson.M{"$group": bson.M{"_id": bson.M{"id": "$infoid"}, "count": bson.M{"$sum": 1}}},
 	}).All(&res)
@@ -114,10 +109,11 @@ func onlineDataTj()  {
 }
 
 func main() {
+	log.Println("开始")
 	// 线上线下去重统计
-	dataImport()
+	//dataImport()
 	// 线上去重统计
-	onlineDataTj()
+	//onlineDataTj()
 	// 推送数据统计
 	pushtj()
 }
@@ -141,22 +137,13 @@ func dataImport() {
 	}
 	query["appid"] = "jyHDhXQQIAAgdZQEBLERV2"
 	//线下导出数据内部去重
-	it := sess.DB("jyqyfw_historyData").C("20201126Kdxf").Find(map[string]interface{}{"createtime": map[string]interface{}{"$lt": 1625068800}}).Select(fields).Sort("_id").Iter()
+	// todo 每次修改结束时间
+	it := sess.DB("jyqyfw_historyData").C("20201126Kdxf").Find(map[string]interface{}{"createtime": map[string]interface{}{"$lt": ENDTIME}}).Select(fields).Sort("_id").Iter()
 	var filterCount = 0
 	var saveCount = 0
 	var count = 0
 	kdxfData := map[string]bool{}
 	for m := make(map[string]interface{}); it.Next(&m); {
-		//先进行redis去重
-		/*isExist, err := redis.Exists("other", "entexportdata_"+qu.ObjToString(m["id"])+"_"+fmt.Sprintln(entid))
-		if err != nil {
-			log.Println("企业订阅数据导出redis判重失败")
-			continue
-		} else if isExist {
-			filterCount++
-			log.Println("数据重复，id ", qu.ObjToString(m["id"]), "entid ", entid)
-			continue
-		}*/
 		if kdxfData[qu.ObjToString(m["id"])] {
 			continue
 		} else {
@@ -169,19 +156,12 @@ func dataImport() {
 	//线下数据跟线上导出数据去重
 	//todo 赵珑月 导过数据后改表名 表名会变   建索引
 	for k, _ := range kdxfData {
-		//log.Println(k)
-		cm := Mgo.Count("20210701kdxfentdataexport", map[string]interface{}{"infoid": qu.ObjToString(k)})
+		cm := Mgo.Count(ONLINETABLE, map[string]interface{}{"infoid": qu.ObjToString(k)})
 		if cm > 0 {
-			log.Println("20210701kdxfentdataexport，id ", qu.ObjToString(k), "entid ", entid)
+			log.Println(ONLINETABLE,"id ", qu.ObjToString(k), "entid ", entid)
 			filterCount++
 			continue
 		}
-		/*cy := JyMysql.Count("pushentniche_test", map[string]interface{}{"infoid": qu.ObjToString(k)})
-		if cy > 0 {
-			log.Println("pushentniche_test数据重复，id ", qu.ObjToString(k), "entid ", entid)
-			filterCount++
-			continue
-		}*/
 		saveCount++
 		log.Println("重复数据量：", filterCount, "线上线下去重后数据量：", saveCount)
 	}