doc:科大讯飞数据量统计步骤及sql整理

kdxf/doc/科大讯飞统计步骤.md

@@ -111,6 +111,7 @@ count:{$sum:1}
 
 
 科大讯飞-医疗/科大讯飞-卫健委
+将两张表数据合到同一张表里，执行sql进行统计
 
 按照公告id  医疗和卫健委，相同数据时间早的计数。
 ```

kdxf/src/config.json

@@ -24,5 +24,11 @@
     "password": "",
     "address": "127.0.0.1:3356",
     "dbName": "jianyu"
-  }
+  },
+  "from": "20211101Kdxf_jy",
+  "to": "20221125kdxf_xwh_jy_tj",
+  "cond": {
+ "createtime":{ "$lte":1669305600}
+  },
+  "kdxfentdataexport": "20221125kdxfentdataexport"
 }

kdxf/src/main.go

@@ -16,34 +16,33 @@ var (
 	JyzhshiMysql     *mysql.Mysql
 	JypushzhshiMysql *mysql.Mysql
 	Mgo              *mongodb.MongodbSim
-
 )
 
 const (
-	ONLINETABLE ="20211008kdxfentdataexport"
-	ENDTIME = 1633017600
+	ONLINETABLE = "20211008kdxfentdataexport"
+	ENDTIME     = 1633017600
 )
 
 func init() {
 	qu.ReadConfig(&Sysconfig)
-	// 这个是正式库
-	jyzhshiMysql := qu.ObjToMap(Sysconfig["jyzhengshimysql"])
-	JyzhshiMysql = &mysql.Mysql{
-		Address:  (*jyzhshiMysql)["address"].(string),
-		UserName: (*jyzhshiMysql)["username"].(string),
-		PassWord: (*jyzhshiMysql)["password"].(string),
-		DBName:   (*jyzhshiMysql)["dbName"].(string),
-	}
-	JyzhshiMysql.Init()
-	// 这个也是正式库
-	jypushzhshiMysql := qu.ObjToMap(Sysconfig["pushzhengshimysql"])
-	JypushzhshiMysql = &mysql.Mysql{
-		Address:  (*jypushzhshiMysql)["address"].(string),
-		UserName: (*jypushzhshiMysql)["username"].(string),
-		PassWord: (*jypushzhshiMysql)["password"].(string),
-		DBName:   (*jypushzhshiMysql)["dbName"].(string),
-	}
-	JypushzhshiMysql.Init()
+	//// 这个是正式库
+	//jyzhshiMysql := qu.ObjToMap(Sysconfig["jyzhengshimysql"])
+	//JyzhshiMysql = &mysql.Mysql{
+	//	Address:  (*jyzhshiMysql)["address"].(string),
+	//	UserName: (*jyzhshiMysql)["username"].(string),
+	//	PassWord: (*jyzhshiMysql)["password"].(string),
+	//	DBName:   (*jyzhshiMysql)["dbName"].(string),
+	//}
+	//JyzhshiMysql.Init()
+	//// 这个也是正式库
+	//jypushzhshiMysql := qu.ObjToMap(Sysconfig["pushzhengshimysql"])
+	//JypushzhshiMysql = &mysql.Mysql{
+	//	Address:  (*jypushzhshiMysql)["address"].(string),
+	//	UserName: (*jypushzhshiMysql)["username"].(string),
+	//	PassWord: (*jypushzhshiMysql)["password"].(string),
+	//	DBName:   (*jypushzhshiMysql)["dbName"].(string),
+	//}
+	//JypushzhshiMysql.Init()
 	mogConfig := Sysconfig["mgo"].(map[string]interface{})
 	Mgo = &mongodb.MongodbSim{
 		MongodbAddr: qu.ObjToString(mogConfig["addr"]),
@@ -52,6 +51,7 @@ func init() {
 	}
 	Mgo.InitPool()
 }
+
 func pushtj() {
 
 	//	 查剑鱼主库用户
@@ -61,40 +61,40 @@ func pushtj() {
 	idStrList := []string{}
 	for _, user := range *userRs {
 		//log.Println( user["id"],qu.ObjToString( user["id"]))
-		idStrList =append(idStrList,strconv.FormatInt(user["id"].(int64), 10))
+		idStrList = append(idStrList, strconv.FormatInt(user["id"].(int64), 10))
 	}
-	idStr :=strings.Join(idStrList,",")
+	idStr := strings.Join(idStrList, ",")
 	log.Println(idStr)
-	endTime :=strconv.Itoa(ENDTIME)
+	endTime := strconv.Itoa(ENDTIME)
 	log.Println(endTime)
-	pushtjSql:="SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <"+endTime +" GROUP BY userid "
+	pushtjSql := "SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <" + endTime + " GROUP BY userid "
 
-	log.Println(pushtjSql)//	 聚合推送库信息
+	log.Println(pushtjSql) //	 聚合推送库信息
 	tjRs := JypushzhshiMysql.SelectBySql(pushtjSql)
 	tjMap := map[int64]interface{}{}
-	for _,v := range *tjRs{
-		tjMap[v["userid"].(int64)]=v["count"]
+	for _, v := range *tjRs {
+		tjMap[v["userid"].(int64)] = v["count"]
 	}
 	log.Println(tjMap)
 
-	for _,user := range *userRs{
+	for _, user := range *userRs {
 		count := tjMap[user["id"].(int64)]
-		if count == nil{
-			count =0
+		if count == nil {
+			count = 0
 		}
-		if user["name"]=="我"{
-			user["name"]="系统管理员"
+		if user["name"] == "我" {
+			user["name"] = "系统管理员"
 		}
 		user["count"] = count
 		//if count !=0{
 
-		fmt.Println(user["name"],user["phone"],user["count"])
+		fmt.Println(user["name"], user["phone"], user["count"])
 		//}
 	}
 
 }
 
-func onlineDataTj()  {
+func onlineDataTj() {
 	//线上去重统计
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
@@ -115,59 +115,51 @@ func main() {
 	// 线上去重统计
 	//onlineDataTj()
 	// 推送数据统计
-	pushtj()
+	//pushtj()
+	//updateAppid() //  商机管理 补充appid 和 id 字段
+	//insertData()  //  导数据从一个集合到另一个集合  TODO 待测试
 }
 
-
-func dataImport() {
-	entid := qu.IntAll(Sysconfig["entid"])
-	query := map[string]interface{}{}
+// 刷库 线上数据补充消委会appid和id字段  平时可以不用程序执行，直接执行mongos语句就可以， 开代理时本地刷太慢 可以放到测试环境上跑比较快
+func updateAppid() {
+	fmt.Println("8877")
 	sess := Mgo.GetMgoConn()
-	defer Mgo.DestoryMongoConn(sess)
-	//fields := Sysconfig["Fields"].(map[string]int)
 	fields := map[string]int{
-		"_id":        1,
-		"area":       1,
-		"buyerclass": 1,
-		"city":       1,
-		"subtype":    1,
-		"toptype":    1,
-		"matchkey":   1,
-		"id":         1,
+		"_id":    1,
+		"infoid": 1,
 	}
-	query["appid"] = "jyHDhXQQIAAgdZQEBLERV2"
-	//线下导出数据内部去重
-	// todo 每次修改结束时间
-	it := sess.DB("jyqyfw_historyData").C("20201126Kdxf").Find(map[string]interface{}{"createtime": map[string]interface{}{"$lt": ENDTIME}}).Select(fields).Sort("_id").Iter()
-	var filterCount = 0
-	var saveCount = 0
-	var count = 0
-	kdxfData := map[string]bool{}
+	it := sess.DB("jyqyfw_historyData").C(qu.ObjToString(Sysconfig["kdxfentdataexport"])).Find(map[string]interface{}{"appid": map[string]interface{}{"$exists": false}}).Select(fields).Sort("_id").Iter()
+	i := 0
 	for m := make(map[string]interface{}); it.Next(&m); {
-		if kdxfData[qu.ObjToString(m["id"])] {
-			continue
-		} else {
-			kdxfData[qu.ObjToString(m["id"])] = true
-			count++
-		}
+		i++
+		fmt.Println(m["_id"])
+		Mgo.UpdateById(qu.ObjToString(Sysconfig["kdxfentdataexport"]), mongodb.BsonIdToSId(m["_id"]), map[string]interface{}{"$set": map[string]interface{}{
+
+			"id":    m["infoid"],
+			"appid": "jyHDhXQQIAAgdZQEBLERV2",
+		},
+		})
+		fmt.Println(i)
+		//break
 	}
-	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
-	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
-	//线下数据跟线上导出数据去重
-	//todo 赵珑月 导过数据后改表名 表名会变   建索引
-	for k, _ := range kdxfData {
-		cm := Mgo.Count(ONLINETABLE, map[string]interface{}{"infoid": qu.ObjToString(k)})
-		if cm > 0 {
-			log.Println(ONLINETABLE,"id ", qu.ObjToString(k), "entid ", entid)
-			filterCount++
-			continue
-		}
-		saveCount++
-		log.Println("重复数据量：", filterCount, "线上线下去重后数据量：", saveCount)
-	}
-
-	log.Println("数据处理完成。。。。")
-	log.Println("线上线下重复数据量：", filterCount, "线上线下去重后数据量：", saveCount, "线下数据内部去重后数据量：", count)
 }
 
+// 测试环境导数据用  平时用工具直接导不需要这个程序，开代理太慢数据导不动，可以放到测试服务器上跑   TODO 待测试
+func insertData() {
+	fmt.Println("8877")
+	sess := Mgo.GetMgoConn()
+	fmt.Println(Sysconfig)
+	it := sess.DB("jyqyfw_historyData").C(qu.ObjToString(Sysconfig["from"])).Find(Sysconfig["cond"]).Sort("_id").Iter()
+	i := 0
+	for m := make(map[string]interface{}); it.Next(&m); {
+		i++
+		fmt.Println(m["_id"])
+		fmt.Println()
+		rs := Mgo.SaveByOriID(qu.ObjToString(Sysconfig["to"]), m)
+		fmt.Println(rs)
+		fmt.Println(i)
+		break
+	}
+}
 
+//