科大讯飞统计程序

kdxf/readme.md

@@ -0,0 +1,11 @@
+# 科大讯飞数据导出统计程序
+
+1. 读线下导出数据,线下数据自己内部去重（根据id）
+2. 去重后的数据查询线上数据库（infoid），存在则不计入总数，不存在则统计。每次需要更换表名，建索引会快一些
+---
+- 202104247 移除查询条件中的appid,因为部分数据appid缺失
+
+- 20210602  查询条件为线下数据查询 20201126Kdxf  限制时间范围，之后这个可能还需要调整，根据龙玥那边提供的条件进行处理。
+  统计结果 
+![img.png](img.png)
+- 20210702 增加统计订阅数据推送数量

kdxf/src/config.json

@@ -0,0 +1,28 @@
+{
+  "mysql": {
+    "username": "root",
+    "password": "Topnet123",
+    "address": "192.168.3.11:3366",
+    "dbName": "jianyu"
+  },
+  "mgo": {
+    "addr": "192.168.3.167:27080",
+    "size": 15,
+    "db": "jyqyfw_historyData",
+    "coll": "20201126Kdxf"
+  },
+  "entid": 10522,
+  "redis_addrs": "other=192.168.3.128:1712",
+  "jyzhengshimysql": {
+    "username": "root",
+    "password": "",
+    "address": "127.0.0.1:3355",
+    "dbName": "jianyu"
+  },
+  "pushzhengshimysql": {
+    "username": "root",
+    "password": "",
+    "address": "127.0.0.1:3356",
+    "dbName": "jianyu"
+  }
+}

kdxf/src/main.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"fmt"
+	"gopkg.in/mgo.v2/bson"
+	"log"
+	"qfw/mongodb"
+	qu "qfw/util"
+	"qfw/util/mysql"
+	"qfw/util/redis"
+	"strconv"
+	"strings"
+)
+
+var (
+	Sysconfig        map[string]interface{}
+	JyMysql          *mysql.Mysql
+	JyzhshiMysql     *mysql.Mysql
+	JypushzhshiMysql *mysql.Mysql
+	Mgo              *mongodb.MongodbSim
+)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	jyMysql := qu.ObjToMap(Sysconfig["mysql"])
+	JyMysql = &mysql.Mysql{
+		Address:  (*jyMysql)["address"].(string),
+		UserName: (*jyMysql)["username"].(string),
+		PassWord: (*jyMysql)["password"].(string),
+		DBName:   (*jyMysql)["dbName"].(string),
+	}
+	JyMysql.Init()
+	// 这个是正式库
+	jyzhshiMysql := qu.ObjToMap(Sysconfig["jyzhengshimysql"])
+	JyzhshiMysql = &mysql.Mysql{
+		Address:  (*jyzhshiMysql)["address"].(string),
+		UserName: (*jyzhshiMysql)["username"].(string),
+		PassWord: (*jyzhshiMysql)["password"].(string),
+		DBName:   (*jyzhshiMysql)["dbName"].(string),
+	}
+	JyzhshiMysql.Init()
+	// 这个也是正式库
+	jypushzhshiMysql := qu.ObjToMap(Sysconfig["pushzhengshimysql"])
+	JypushzhshiMysql = &mysql.Mysql{
+		Address:  (*jypushzhshiMysql)["address"].(string),
+		UserName: (*jypushzhshiMysql)["username"].(string),
+		PassWord: (*jypushzhshiMysql)["password"].(string),
+		DBName:   (*jypushzhshiMysql)["dbName"].(string),
+	}
+	JypushzhshiMysql.Init()
+	mogConfig := Sysconfig["mgo"].(map[string]interface{})
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(mogConfig["addr"]),
+		DbName:      qu.ObjToString(mogConfig["db"]),
+		Size:        qu.IntAllDef(mogConfig["size"], 15),
+	}
+	Mgo.InitPool()
+
+	redis.InitRedis(qu.ObjToString(Sysconfig["redis_addrs"]))
+}
+//订阅数据推送统计  将结果粘到excel  分列
+func pushtj() {
+
+	//	 查剑鱼主库用户
+	userRs := JyzhshiMysql.SelectBySql("select id,`name`,phone from entniche_user WHERE ent_id=10511")
+	//idRs :=JyzhshiMysql.SelectBySql("select id from entniche_user WHERE ent_id=10511")
+	log.Println(userRs)
+	idStrList := []string{}
+	for _, user := range *userRs {
+		//log.Println( user["id"],qu.ObjToString( user["id"]))
+		idStrList =append(idStrList,strconv.FormatInt(user["id"].(int64), 10))
+	}
+	idStr :=strings.Join(idStrList,",")
+	log.Println(idStr)
+	// todo  每次修改结束时间
+	pushtjSql:="SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <1625068800  GROUP BY userid "
+	//	 聚合推送库信息
+	tjRs := JypushzhshiMysql.SelectBySql(pushtjSql)
+	tjMap := map[int64]interface{}{}
+	for _,v := range *tjRs{
+		tjMap[v["userid"].(int64)]=v["count"]
+	}
+	log.Println(tjMap)
+
+	for _,user := range *userRs{
+		count := tjMap[user["id"].(int64)]
+		if count == nil{
+			count =0
+		}
+		if user["name"]=="我"{
+			user["name"]="系统管理员"
+		}
+		user["count"] = count
+		if count !=0{
+
+		fmt.Println(user["name"],user["phone"],user["count"])
+		}
+	}
+
+}
+
+func onlineDataTj()  {
+	//线上去重统计
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	var res []bson.M
+
+	err := sess.DB(Mgo.DbName).C("20210701kdxfentdataexport").Pipe([]bson.M{
+		//bson.M{"$match": bson.M{"createtime": bson.M{ "$lt":1619280000}}},
+		bson.M{"$group": bson.M{"_id": bson.M{"id": "$infoid"}, "count": bson.M{"$sum": 1}}},
+	}).All(&res)
+	fmt.Println(err)
+	fmt.Println("线上数据去重后数据量:", len(res))
+}
+
+func main() {
+	// 线上线下去重统计
+	dataImport()
+	// 线上去重统计
+	onlineDataTj()
+	// 推送数据统计
+	pushtj()
+}
+
+
+func dataImport() {
+	entid := qu.IntAll(Sysconfig["entid"])
+	query := map[string]interface{}{}
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	//fields := Sysconfig["Fields"].(map[string]int)
+	fields := map[string]int{
+		"_id":        1,
+		"area":       1,
+		"buyerclass": 1,
+		"city":       1,
+		"subtype":    1,
+		"toptype":    1,
+		"matchkey":   1,
+		"id":         1,
+	}
+	query["appid"] = "jyHDhXQQIAAgdZQEBLERV2"
+	//线下导出数据内部去重
+	it := sess.DB("jyqyfw_historyData").C("20201126Kdxf").Find(map[string]interface{}{"createtime": map[string]interface{}{"$lt": 1625068800}}).Select(fields).Sort("_id").Iter()
+	var filterCount = 0
+	var saveCount = 0
+	var count = 0
+	kdxfData := map[string]bool{}
+	for m := make(map[string]interface{}); it.Next(&m); {
+		//先进行redis去重
+		/*isExist, err := redis.Exists("other", "entexportdata_"+qu.ObjToString(m["id"])+"_"+fmt.Sprintln(entid))
+		if err != nil {
+			log.Println("企业订阅数据导出redis判重失败")
+			continue
+		} else if isExist {
+			filterCount++
+			log.Println("数据重复，id ", qu.ObjToString(m["id"]), "entid ", entid)
+			continue
+		}*/
+		if kdxfData[qu.ObjToString(m["id"])] {
+			continue
+		} else {
+			kdxfData[qu.ObjToString(m["id"])] = true
+			count++
+		}
+	}
+	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
+	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
+	//线下数据跟线上导出数据去重
+	//todo 赵珑月 导过数据后改表名 表名会变   建索引
+	for k, _ := range kdxfData {
+		//log.Println(k)
+		cm := Mgo.Count("20210701kdxfentdataexport", map[string]interface{}{"infoid": qu.ObjToString(k)})
+		if cm > 0 {
+			log.Println("20210701kdxfentdataexport，id ", qu.ObjToString(k), "entid ", entid)
+			filterCount++
+			continue
+		}
+		/*cy := JyMysql.Count("pushentniche_test", map[string]interface{}{"infoid": qu.ObjToString(k)})
+		if cy > 0 {
+			log.Println("pushentniche_test数据重复，id ", qu.ObjToString(k), "entid ", entid)
+			filterCount++
+			continue
+		}*/
+		saveCount++
+		log.Println("重复数据量：", filterCount, "线上线下去重后数据量：", saveCount)
+	}
+
+	log.Println("数据处理完成。。。。")
+	log.Println("线上线下重复数据量：", filterCount, "线上线下去重后数据量：", saveCount, "线下数据内部去重后数据量：", count)
+}
+
+

kdxf/src/main.go_bak20210702

@@ -0,0 +1,285 @@
+package main
+
+import (
+	"fmt"
+	"gopkg.in/mgo.v2/bson"
+	"log"
+	"qfw/mongodb"
+	qu "qfw/util"
+	"qfw/util/mysql"
+	"qfw/util/redis"
+	"strconv"
+	"strings"
+)
+
+var (
+	Sysconfig        map[string]interface{}
+	JyMysql          *mysql.Mysql
+	JyzhshiMysql     *mysql.Mysql
+	JypushzhshiMysql *mysql.Mysql
+	Mgo              *mongodb.MongodbSim
+	BuryClassType    map[string]interface{}
+	ProvinceMap      map[string]interface{}
+)
+
+func init() {
+	qu.ReadConfig(&Sysconfig)
+	//jyMysql := qu.ObjToMap(Sysconfig["mysql"])
+	//JyMysql = &mysql.Mysql{
+	//	Address:  (*jyMysql)["address"].(string),
+	//	UserName: (*jyMysql)["username"].(string),
+	//	PassWord: (*jyMysql)["password"].(string),
+	//	DBName:   (*jyMysql)["dbName"].(string),
+	//}
+	//JyMysql.Init()
+	//jyzhshiMysql := qu.ObjToMap(Sysconfig["jyzhengshimysql"])
+	//JyzhshiMysql = &mysql.Mysql{
+	//	Address:  (*jyzhshiMysql)["address"].(string),
+	//	UserName: (*jyzhshiMysql)["username"].(string),
+	//	PassWord: (*jyzhshiMysql)["password"].(string),
+	//	DBName:   (*jyzhshiMysql)["dbName"].(string),
+	//}
+	//JyzhshiMysql.Init()
+	//jypushzhshiMysql := qu.ObjToMap(Sysconfig["pushzhengshimysql"])
+	//JypushzhshiMysql = &mysql.Mysql{
+	//	Address:  (*jypushzhshiMysql)["address"].(string),
+	//	UserName: (*jypushzhshiMysql)["username"].(string),
+	//	PassWord: (*jypushzhshiMysql)["password"].(string),
+	//	DBName:   (*jypushzhshiMysql)["dbName"].(string),
+	//}
+	//JypushzhshiMysql.Init()
+	mogConfig := Sysconfig["mgo"].(map[string]interface{})
+	Mgo = &mongodb.MongodbSim{
+		MongodbAddr: qu.ObjToString(mogConfig["addr"]),
+		DbName:      qu.ObjToString(mogConfig["db"]),
+		Size:        qu.IntAllDef(mogConfig["size"], 15),
+	}
+	Mgo.InitPool()
+
+	redis.InitRedis(qu.ObjToString(Sysconfig["redis_addrs"]))
+	//
+	//initBuryClassType()
+	//initProvince()
+}
+func pushtj() {
+
+	//	 查剑鱼主库用户
+	userRs := JyzhshiMysql.SelectBySql("select id,`name`,phone from entniche_user WHERE ent_id=10511")
+	//idRs :=JyzhshiMysql.SelectBySql("select id from entniche_user WHERE ent_id=10511")
+	log.Println(userRs)
+	idStrList := []string{}
+	for _, user := range *userRs {
+		//log.Println( user["id"],qu.ObjToString( user["id"]))
+		idStrList =append(idStrList,strconv.FormatInt(user["id"].(int64), 10))
+	}
+	idStr :=strings.Join(idStrList,",")
+	log.Println(idStr)
+	pushtjSql:="SELECT userid,COUNT(*) count FROM pushentniche WHERE userid in (" + idStr + ") and date <1625068800  GROUP BY userid "
+	//	 聚合推送库信息
+	tjRs := JypushzhshiMysql.SelectBySql(pushtjSql)
+	tjMap := map[int64]interface{}{}
+	for _,v := range *tjRs{
+		tjMap[v["userid"].(int64)]=v["count"]
+	}
+	log.Println(tjMap)
+
+	for _,user := range *userRs{
+		count := tjMap[user["id"].(int64)]
+		if count == nil{
+			count =0
+		}
+		user["count"] = count
+		//fmt.Println(user["name"])
+		//fmt.Println(user["phone"],)
+		//fmt.Println(user["count"])
+		//if count !=0{
+		fmt.Println(user["name"],user["phone"],user["count"])
+
+		//}
+	}
+
+}
+func main() {
+	//dtafilter()
+
+
+
+	// 线上线下去重统计
+	dataImport()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+
+	var res []bson.M
+	//线上去重统计
+	err := sess.DB(Mgo.DbName).C("20210701kdxfentdataexport").Pipe([]bson.M{
+		//bson.M{"$match": bson.M{"createtime": bson.M{ "$lt":1619280000}}},
+		bson.M{"$group": bson.M{"_id": bson.M{"id": "$infoid"}, "count": bson.M{"$sum": 1}}},
+	}).All(&res)
+	fmt.Println(err)
+	fmt.Println("线上数据去重后数据量:", len(res))
+	//pushtj()
+	// 推送数据统计
+}
+
+func dtafilter() {
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	query := map[string]interface{}{
+		"entid": 10511,
+	}
+	fields := map[string]int{
+		"_id":    1,
+		"infoid": 1,
+	}
+	it := sess.DB("jyqyfw_historyData").C("20201222Kdxf_entexportdata").Find(query).Select(fields).Sort("_id").Iter()
+	var count = 0
+	var ccc = 0
+	kdxfData := map[string]bool{}
+	for m := make(map[string]interface{}); it.Next(&m); {
+		ccc++
+		if kdxfData[qu.ObjToString(m["infoid"])] {
+			continue
+		} else {
+			kdxfData[qu.ObjToString(m["infoid"])] = true
+			count++
+		}
+	}
+	log.Println("加载20201222Kdxf_entexportdata数据完成", len(kdxfData), count, ccc)
+}
+
+func dataImport() {
+	entid := qu.IntAll(Sysconfig["entid"])
+	query := map[string]interface{}{}
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	//fields := Sysconfig["Fields"].(map[string]int)
+	fields := map[string]int{
+		"_id":        1,
+		"area":       1,
+		"buyerclass": 1,
+		"city":       1,
+		"subtype":    1,
+		"toptype":    1,
+		"matchkey":   1,
+		"id":         1,
+	}
+	query["appid"] = "jyHDhXQQIAAgdZQEBLERV2"
+	//线下导出数据内部去重
+	it := sess.DB("jyqyfw_historyData").C("20201126Kdxf").Find(map[string]interface{}{"createtime": map[string]interface{}{"$lt": 1625068800}}).Select(fields).Sort("_id").Iter()
+	var filterCount = 0
+	var saveCount = 0
+	var count = 0
+	kdxfData := map[string]bool{}
+	for m := make(map[string]interface{}); it.Next(&m); {
+		//先进行redis去重
+		/*isExist, err := redis.Exists("other", "entexportdata_"+qu.ObjToString(m["id"])+"_"+fmt.Sprintln(entid))
+		if err != nil {
+			log.Println("企业订阅数据导出redis判重失败")
+			continue
+		} else if isExist {
+			filterCount++
+			log.Println("数据重复，id ", qu.ObjToString(m["id"]), "entid ", entid)
+			continue
+		}*/
+		if kdxfData[qu.ObjToString(m["id"])] {
+			continue
+		} else {
+			kdxfData[qu.ObjToString(m["id"])] = true
+			count++
+		}
+	}
+	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
+	log.Println("加载20201126Kdxf完成", len(kdxfData), count)
+	//线下数据跟线上导出数据去重
+	//todo 赵珑月 导过数据后改表名 表名会变   建索引
+	for k, _ := range kdxfData {
+		//log.Println(k)
+		cm := Mgo.Count("20210701kdxfentdataexport", map[string]interface{}{"infoid": qu.ObjToString(k)})
+		if cm > 0 {
+			log.Println("20210701kdxfentdataexport，id ", qu.ObjToString(k), "entid ", entid)
+			filterCount++
+			continue
+		}
+		/*cy := JyMysql.Count("pushentniche_test", map[string]interface{}{"infoid": qu.ObjToString(k)})
+		if cy > 0 {
+			log.Println("pushentniche_test数据重复，id ", qu.ObjToString(k), "entid ", entid)
+			filterCount++
+			continue
+		}*/
+		saveCount++
+		log.Println("重复数据量：", filterCount, "线上线下去重后数据量：", saveCount)
+	}
+
+	log.Println("数据处理完成。。。。")
+	log.Println("线上线下重复数据量：", filterCount, "线上线下去重后数据量：", saveCount, "线下数据内部去重后数据量：", count)
+}
+
+func initBuryClassType() {
+	infoType := JyMysql.SelectBySql("SELECT id,name FROM infotype")
+	BuryClassType = make(map[string]interface{})
+	for _, val := range *infoType {
+		k := qu.ObjToString(val["name"])
+		BuryClassType[k] = val["id"]
+	}
+}
+
+func initProvince() {
+	proArr := JyMysql.SelectBySql("SELECT id,name FROM province")
+	ProvinceMap = make(map[string]interface{})
+	for _, val := range *proArr {
+		k := qu.ObjToString(val["name"])
+		ProvinceMap[k] = val["id"]
+	}
+}
+
+//历史数据存到mysql pushentniche推送表中
+func saveMysqlPush(tmp map[string]interface{}, entId int) bool {
+	saveMap := map[string]interface{}{}
+	saveMap["entid"] = entId
+	infoid := qu.ObjToString(tmp["id"])
+	saveMap["infoid"] = infoid
+	/*matchkeys := qu.ObjToString(tmp["matchkey"])
+	if matchkeys != "" {
+		matchkeysStr := ""
+		matchkeysArr := strings.Split(matchkeys, ",")
+		for i, val := range matchkeysArr {
+			if i < len(matchkeysArr)-1 {
+				matchkeysStr += val + " "
+			} else {
+				matchkeysStr += val
+			}
+		}
+		saveMap["matchkeys"] = matchkeysStr
+	}
+	area := qu.ObjToString(tmp["area"])
+	city := qu.ObjToString(tmp["city"])
+	buyerClass := qu.ObjToString(tmp["buyerclass"])
+	toptype := qu.ObjToString(tmp["toptype"])
+	subtype := qu.ObjToString(tmp["subtype"])
+	saveMap["deptid"] = 0
+	saveMap["userid"] = 0
+	saveMap["date"] = time.Now().Unix()
+	if area != "" {
+		saveMap["area"] = qu.IntAll(ProvinceMap[area])
+	}
+	if city != "" {
+		saveMap["city"] = qu.IntAll(ProvinceMap[city])
+	}
+	if buyerClass != "" {
+		saveMap["buyerclass"] = qu.IntAll(BuryClassType[buyerClass])
+	}
+	if toptype != "" {
+		saveMap["toptype"] = qu.IntAll(BuryClassType[toptype])
+	}
+	if subtype != "" {
+		saveMap["subtype"] = qu.IntAll(BuryClassType[subtype])
+	}*/
+	b := JyMysql.Insert("pushentniche_test", saveMap)
+	if b <= 0 {
+		log.Println("数据插入pushentniche_test表失败，id ", qu.ObjToString(tmp["id"]), "entid ", entId)
+		return false
+	} /*else {
+		redis.Put("other", "entexportdata_"+infoid+"_"+fmt.Sprintln(entId), 1, -1)
+	}*/
+	return true
+}

kdxf/src/util/mgodb/mgo.go

@@ -0,0 +1,357 @@
+package mongodb
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+var Mgo *MongodbSim
+
+type MgoSess struct {
+	Db     string
+	Coll   string
+	Query  interface{}
+	Sorts  []string
+	fields interface{}
+	limit  int64
+	skip   int64
+	M      *MongodbSim
+}
+
+type MgoIter struct {
+	Cursor *mongo.Cursor
+}
+
+func (mt *MgoIter) Next(result interface{}) bool {
+	if mt.Cursor != nil {
+		if mt.Cursor.Next(nil) {
+			err := mt.Cursor.Decode(result)
+			if err != nil {
+				log.Println("mgo cur err", err.Error())
+				mt.Cursor.Close(nil)
+				return false
+			}
+			return true
+		} else {
+			mt.Cursor.Close(nil)
+			return false
+		}
+	} else {
+		return false
+	}
+
+}
+
+func (ms *MgoSess) DB(name string) *MgoSess {
+	ms.Db = name
+	return ms
+}
+
+func (ms *MgoSess) C(name string) *MgoSess {
+	ms.Coll = name
+	return ms
+}
+
+func (ms *MgoSess) Find(q interface{}) *MgoSess {
+	ms.Query = q
+	return ms
+}
+
+func (ms *MgoSess) Select(fields interface{}) *MgoSess {
+	ms.fields = fields
+	return ms
+}
+
+func (ms *MgoSess) Limit(limit int64) *MgoSess {
+	ms.limit = limit
+	return ms
+}
+func (ms *MgoSess) Skip(skip int64) *MgoSess {
+	ms.skip = skip
+	return ms
+}
+
+func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
+	ms.Sorts = sorts
+	return ms
+}
+
+func (ms *MgoSess) Iter() *MgoIter {
+	it := &MgoIter{}
+	find := options.Find()
+	if ms.skip > 0 {
+		find.SetSkip(ms.skip)
+	}
+	if ms.limit > 0 {
+		find.SetLimit(ms.limit)
+	}
+	find.SetBatchSize(100)
+	if len(ms.Sorts) > 0 {
+		sort := bson.M{}
+		for _, k := range ms.Sorts {
+			switch k[:1] {
+			case "-":
+				sort[k[1:]] = -1
+			case "+":
+				sort[k[1:]] = 1
+			default:
+				sort[k] = 1
+			}
+		}
+		find.SetSort(sort)
+	}
+	if ms.fields != nil {
+		find.SetProjection(ms.fields)
+	}
+	cur, err := ms.M.C.Database(ms.Db).Collection(ms.Coll).Find(ms.M.Ctx, ms.Query, find)
+	if err != nil {
+		log.Println("mgo find err", err.Error())
+	} else {
+		it.Cursor = cur
+	}
+	return it
+}
+
+type MongodbSim struct {
+	MongodbAddr string
+	Size        int
+	//	MinSize     int
+	DbName   string
+	C        *mongo.Client
+	Ctx      context.Context
+	ShortCtx context.Context
+	pool     chan bool
+}
+
+func (m *MongodbSim) GetMgoConn() *MgoSess {
+	//m.Open()
+	ms := &MgoSess{}
+	ms.M = m
+	return ms
+}
+
+func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
+	//m.Close()
+	ms.M = nil
+	ms = nil
+}
+
+func (m *MongodbSim) InitPool() {
+	opts := options.Client()
+	opts.SetConnectTimeout(3 * time.Second)
+	opts.ApplyURI("mongodb://" + m.MongodbAddr)
+	opts.SetMaxPoolSize(uint64(m.Size))
+	m.pool = make(chan bool, m.Size)
+	opts.SetMaxConnIdleTime(2 * time.Hour)
+	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
+	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
+	client, err := mongo.Connect(m.ShortCtx, opts)
+	if err != nil {
+		log.Println("mgo init error:", err.Error())
+	} else {
+		m.C = client
+		log.Println("init success")
+	}
+}
+
+func (m *MongodbSim) Open() {
+	m.pool <- true
+}
+func (m *MongodbSim) Close() {
+	<-m.pool
+}
+
+//批量插入
+func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewUpdateOneModel()
+		write.SetFilter(d[0])
+		write.SetUpdate(d[1])
+		write.SetUpsert(true)
+		writes = append(writes, write)
+	}
+	r, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo upsert error:", e.Error())
+		return nil, false
+	}
+	//	else {
+	//		if r.UpsertedCount != int64(len(doc)) {
+	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
+	//		}
+	//		return true
+	//	}
+	return r.UpsertedIDs, true
+}
+
+//批量插入
+func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	for _, d := range doc {
+		write := mongo.NewInsertOneModel()
+		write.SetDocument(d)
+		writes = append(writes, write)
+	}
+	_, e := coll.BulkWrite(m.Ctx, writes)
+	if e != nil {
+		log.Println("mgo savebulk error:", e.Error())
+		return false
+	}
+	return true
+}
+
+//保存
+func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.InsertOne(m.Ctx, doc)
+	if err != nil {
+		return nil
+	}
+	return r.InsertedID
+}
+
+//更新by Id
+func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, doc)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+//更新Update
+func (m *MongodbSim) Update(c string, query map[string]interface{}, up map[string]interface{}) (map[int64]interface{}, bool) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	var writes []mongo.WriteModel
+	write := mongo.NewUpdateOneModel()
+	write.SetFilter(query)
+	write.SetUpdate(up)
+	write.SetUpsert(true)
+	writes = append(writes, write)
+	r, err := coll.BulkWrite(m.Ctx, writes)
+	if err != nil {
+		return nil, false
+	}
+	return r.UpsertedIDs, true
+}
+
+//删除by id
+func (m *MongodbSim) DeleteById(c, id string) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//通过条件删除
+func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	r, err := coll.DeleteMany(m.Ctx, query)
+	if err != nil {
+		return 0
+	}
+	return r.DeletedCount
+}
+
+//findbyid
+func (m *MongodbSim) FindById(c, id string, fields interface{}) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.FindOne()
+	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, op.SetProjection(fields))
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//findone
+func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	//op := options.FindOne()
+	//r := coll.FindOne(m.Ctx, query, op.SetProjection(fields))
+	r := coll.FindOne(m.Ctx, query)
+	v := map[string]interface{}{}
+	r.Decode(&v)
+	return v
+}
+
+//find
+func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+//findbylimit
+func (m *MongodbSim) FindByLimit(c string, query map[string]interface{}, sort, fields interface{}, start int64, limit int64) ([]map[string]interface{}, error) {
+	m.Open()
+	defer m.Close()
+	coll := m.C.Database(m.DbName).Collection(c)
+	op := options.Find()
+	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields), op.SetSkip(start), op.SetLimit(limit))
+	if err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	var results []map[string]interface{}
+	if err = r.All(m.Ctx, &results); err != nil {
+		log.Fatal(err)
+		return nil, err
+	}
+	return results, nil
+}
+
+//创建_id
+func NewObjectId() primitive.ObjectID {
+	return primitive.NewObjectID()
+}
+
+func StringTOBsonId(id string) primitive.ObjectID {
+	objectId, _ := primitive.ObjectIDFromHex(id)
+	return objectId
+}
+
+func BsonTOStringId(id interface{}) string {
+	return id.(primitive.ObjectID).Hex()
+}