备份~模板修复数据

data_template/src/datanaly.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"fmt"
+	qu "qfw/util"
+	"regexp"
+	"strings"
+)
+
+var (
+	fieldDict = map[string]interface{}{"agency":1,"buyer":1,"budget":1,"s_winner":1,"bidamount":1,"area":1,"city":1,"projectcode":1}
+	fieldArr = []string{"agency","buyer","budget","s_winner","bidamount","area","city","projectcode"}
+	temp_data = make(map[string]map[string]interface{},0)
+	field_marked = make(map[string]int,0) //字段标注量
+	field_right = make(map[string]int,0)  //字段正确量
+	field_false = make(map[string]int,0)  //字段错误量
+	remark_data = make(map[string]map[string]interface{},0)
+	gseReg *regexp.Regexp = regexp.MustCompile("^(err)")
+)
+//模板数据~加载
+func initPrepare(){
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{
+		"i_ckdata": map[string]interface{}{
+			"$gt":0,
+		},
+	},0
+	it := sess.DB(save_mgo.DbName).C("f_sourceinfo_extract_0525_moban").Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		tmpid := BsonTOStringId(tmp["_id"])
+		v_baseinfo := *qu.ObjToMap(tmp["v_baseinfo"])
+		v_taginfo := *qu.ObjToMap(tmp["v_taginfo"])
+		remark := qu.ObjToString(v_taginfo["reamrk"])
+		if gseReg.MatchString(remark) { //是否过滤异常标注信息
+			tmp = make(map[string]interface{})
+			continue
+
+			remark_arr := strings.Split(remark,"~")
+			d := map[string]interface{}{}
+			for _,v := range remark_arr{
+				if fieldDict[v]!=nil {
+					d[v] = v
+				}
+			}
+			remark_data[tmpid] = d
+		}
+		dict := map[string]interface{}{}
+		for _,key := range fieldArr {
+			if key=="budget" || key=="bidamount" {
+				dict[key] = qu.Float64All(v_baseinfo[key])
+			}else {
+				dict[key] = qu.ObjToString(v_baseinfo[key])
+			}
+		}
+		dict["subtype"] = v_baseinfo["subtype"]
+		dict["toptype"] = v_baseinfo["toptype"]
+		dict["is_ck"] = qu.IntAll(tmp["i_ckdata"])
+		temp_data[tmpid] = dict
+		tmp = make(map[string]interface{})
+	}
+
+	for _,v := range fieldArr {
+		field_right[v] = 0
+		field_false[v] = 0
+		field_marked[v] = 0
+	}
+
+}
+
+//数据字段准确率
+func dealWithDataFieldAccuracy(coll_name string){
+	//准备模板数据~
+	initPrepare()
+	//分析数据~逻辑
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,isok:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C(coll_name).Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);{
+		tmpid := BsonTOStringId(tmp["_id"])
+		data :=  temp_data[tmpid]
+		subtype := qu.ObjToString(tmp["subtype"])
+
+		if qu.IntAll(data["is_ck"]) > 0 { //存在标注数据
+			isok++
+			for _,key := range fieldArr {
+				is_continue:=!(subtype=="单一"||subtype==""||subtype=="中标"||subtype=="成交"||subtype=="合同") &&
+					(key=="bidamount" || key=="s_winner")
+				if is_continue {
+					continue
+				}
+				remark := remark_data[tmpid]
+				if remark[key] != nil {
+					field_false[key] = field_false[key]+1
+					field_marked[key] = field_marked[key]+1
+					continue
+				}
+				if key=="bidamount" || key=="budget"  {
+					if qu.Float64All(tmp[key]) == qu.Float64All(data[key]) {
+						field_right[key] = field_right[key]+1
+					}else {
+						field_false[key] = field_false[key]+1
+					}
+				}else {
+					if qu.ObjToString(tmp[key]) == qu.ObjToString(data[key]) {
+						field_right[key] = field_right[key]+1
+					}else {
+						field_false[key] = field_false[key]+1
+					}
+				}
+				field_marked[key] = field_marked[key]+1
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	arr0,arr1,arr2 :=[]int{},[]int{},[]int{}
+	rate := []string{}
+	for _,v := range fieldArr {
+		arr0 = append(arr0,field_marked[v])
+		arr1 = append(arr1,field_right[v])
+		arr2 = append(arr2,field_false[v])
+
+		rate = append(rate,fmt.Sprintf("%.4f",float64(field_right[v])/float64(field_marked[v])))
+	}
+	fmt.Println("字段标注名：",fieldArr)
+	fmt.Println("字段标注量：",arr0)
+	fmt.Println("字段正确量：",arr1)
+	fmt.Println("字段错误量：",arr2)
+	fmt.Println("字段准确率：",rate)
+}

data_template/src/expdata.go

@@ -1,192 +1,129 @@
 package main
-
+//导出源数据等~
 import (
 	log "github.com/donnie4w/go-logger/logger"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	qu "qfw/util"
 )
 
 
-//导出数据
-func expExampleExtractData() {
+//秘钥~base64
+func encodeSEUrl(uid string) string {
+	var SE = qu.SimpleEncrypt{Key: "topJYBX2019"}
+	str := SE.EncodeString(uid)
+	return str
+}
+
+//模板新增标注数据
+func increaseTemplateData(from string ,to string){
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
-	q ,total, isok := map[string]interface{}{},0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_unit_bidding").Find(&q).Iter()
+	q,total:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C(from).Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%1000==0 {
-			log.Debug("cur index ",total,isok)
+			log.Debug("curent index ",total)
 		}
 		tmpid := BsonTOStringId(tmp["_id"])
+		uid := encodeSEUrl(tmpid)
 
-		data_new := save_mgo.FindById("zktest_extract_new",tmpid)
-		save_mgo.Save("zktest_unit_ext_new",data_new)
-
-		data_old := save_mgo.FindById("zktest_extract_old",tmpid)
-		save_mgo.Save("zktest_unit_ext_old",data_old)
-
+		save_mgo.UpdateById(to,tmpid, map[string]interface{}{
+			"$set":map[string]interface{}{
+				"uid":uid,
+			},
+		})
 		tmp = make(map[string]interface{})
 	}
-	log.Debug("is over ",total,isok)
+	log.Debug("is over ",total)
 }
-//从源数据处-导出单元测试样例
-func expExampleBiddingData()  {
+
+//导出数据-bidding相关
+func exportFromMongoToBiddingData(from string,to string){
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
-	q ,total, isok := map[string]interface{}{},0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_extract_bidding").Find(&q).Iter()
+	q,total:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C(from).Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%1000==0 {
-			log.Debug("cur index ",total,isok)
+			log.Debug("curent index ",total)
 		}
 		tmpid := BsonTOStringId(tmp["_id"])
-		if total%7==0 {
-			isok++
-			save_mgo.Save("zktest_unit_bidding",tmp)
-			data := save_mgo.FindById("zktest_extract_marked",tmpid)
-			save_mgo.Save("zktest_unit_marked",data)
+		data := save_mgo.FindById("bidding",tmpid)
+		if data!=nil && len(data)>2 {
+			save_mgo.Save(to,data)
+		}else {
+			data = save_mgo.FindById("bidding_back",tmpid)
+			if data!=nil && len(data)>2 {
+				save_mgo.Save(to,data)
+			}
 		}
 		tmp = make(map[string]interface{})
 	}
-	log.Debug("is over ",total,isok)
+	log.Debug("is over ",total)
 }
-//同步marked数据
-func synchronizationMarkedata(){
+
+//导出数据-extract相关
+func exportFromMongoToExtractData(from string,to string){
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
-	q ,total, isok := map[string]interface{}{},0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_extract_old").Find(&q).Iter()
+	q,total,isok:=map[string]interface{}{},0,0
+	it := sess.DB(save_mgo.DbName).C(from).Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%1000==0 {
-			log.Debug("cur index ",total,isok)
+			log.Debug("curent index ",total,isok)
 		}
 		tmpid := BsonTOStringId(tmp["_id"])
-
-		data := save_mgo.FindById("marked",tmpid)
+		data := save_mgo.FindById("result_20220218",tmpid)
 		if data!=nil && len(data)>2 {
-			update := data
-			update["id"] = tmpid
-			save_mgo.Save("zktest_marked",update)
-		} else {
-			data = save_mgo.FindOne("marked", map[string]interface{}{
-				"id":tmpid,
-			})
+			isok++
+			save_mgo.Save(to,data)
+		}else {
+			data = save_mgo.FindById("result_20220219",tmpid)
 			if data!=nil && len(data)>2 {
-				update := data
-				update["id"] = tmpid
-				update["_id"] = tmp["_id"]
-				save_mgo.Save("zktest_marked",update)
-			}else {
 				isok++
+				save_mgo.Save(to,data)
 			}
 		}
-
 		tmp = make(map[string]interface{})
 	}
 	log.Debug("is over ",total,isok)
 }
-//导出bidding源数据
-func expSourceBidding() {
-	log.Debug("导出源数据...")
+
+
+//导出
+func exportFenLeipecialData(){
+	//导出招标非单一数据
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
-	q ,total, isok := map[string]interface{}{},0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_marked_id").Find(&q).Iter()
+	q,total:=map[string]interface{}{
+		"_id":map[string]interface{}{
+			"$lt":StringTOBsonId("62bdc8800000000000000000"),
+		},
+	},0
+	isok :=0
+	it := sess.DB(save_mgo.DbName).C("bidding").Find(&q).Sort("-_id").Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		if total%1000==0 {
-			log.Debug("cur index ",total,isok)
+		if total%10000==0 {
+			log.Debug("curent index ",total,isok)
 		}
-		tmpid := qu.ObjToString(tmp["id"])
-		data := save_mgo.FindById("bidding",tmpid)
-		if data!=nil && len(data)>2 {
+		//tmpid := BsonTOStringId(tmp["_id"])
+		toptype:=qu.ObjToString(tmp["toptype"])
+		subtype:=qu.ObjToString(tmp["subtype"])
+
+		if toptype=="招标" && subtype!="单一" {
 			isok++
-			save_mgo.Save("zktest_marked_data",data)
-		}else {
-			data_back := save_mgo.FindById("bidding_back",tmpid)
-			if data_back!=nil && len(data_back)>2 {
-				isok++
-				save_mgo.Save("zktest_marked_data",data_back)
-			}else {
-				log.Debug("使用id字段查不到数据",tmp["_id"])
-				tmpid = BsonTOStringId(tmp["_id"])
-				data_other := save_mgo.FindById("bidding",tmpid)
-				if data_other!=nil && len(data_other)>2 {
-					isok++
-					save_mgo.Save("zktest_marked_data",data_other)
-				}else {
-					data_other_back := save_mgo.FindById("bidding_back",tmpid)
-					if data_other_back!=nil && len(data_other_back)>2 {
-						isok++
-						save_mgo.Save("zktest_marked_data",data_other_back)
-					}else {
-						log.Debug("两个id均查不到",tmp["_id"])
-					}
-				}
-			}
+			save_mgo.Save("zk_fenlei_data",tmp)
+		}
+		if isok>=100000 {
+			return
 		}
-
 		tmp = make(map[string]interface{})
 	}
 	log.Debug("is over ",total,isok)
 
-}
 
+}
 
 
 
 
 
-//导出错误金额-中标金额,预算金额
-func expdataJinEr(){
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
-	q ,total, isok := map[string]interface{}{},0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_extract_new").Find(&q).Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		if total%1000==0 {
-			log.Debug("cur index ",total,isok)
-		}
-		wrong_key_item := []string{}
-		if res, ok := tmp["wrong_key_item"].(primitive.A); ok {
-			wrong_key_item = qu.ObjArrToStringArr(res)
-		}else {
-			if res, ok := tmp["wrong_key_item"].([]interface{}); ok {
-				wrong_key_item = qu.ObjArrToStringArr(res)
-			}
-		}
-		isWrong  := false
-		for _,v:= range wrong_key_item {
-			if qu.ObjToString(v)=="bidamount" {
-				isWrong = true
-				break
-			}
-		}
-		if isWrong {
-			isok++
-			money := qu.Float64All(tmp["bidamount"])
-			tmpid := BsonTOStringId(tmp["_id"])
-			data := save_mgo.FindById("zktest_extract_marked",tmpid)
-			save_dict := map[string]interface{}{}
-			save_dict["_id"] = tmp["_id"]
-			save_dict["money"] = tmp["bidamount"]
-			save_dict["new_money"] = data["bidamount"]
-			save_dict["href"] = tmp["href"]
-			save_dict["jytest_href"] = tmp["jytest_href"]
-			mark := ""
-			if money == 0.0 {
-				mark = "空"
-			}else if money > 0.0 && money <=1000.0 {
-				mark = "过小"
-			}else if money > 1000.0 && money <=100000000.0 {
-				mark = "正常"
-			}else if money > 100000000.0 {
-				mark = "过大"
-			}
-			save_dict["mark"] = mark
-			save_mgo.Save("zktest_zktest_bidamount",save_dict)
-		}
-
-		tmp = make(map[string]interface{})
-	}
-	log.Debug("is over ",total,isok)
-}

data_template/src/main.go

@@ -1,40 +1,108 @@
 package main
 
+import "time"
+
 var (
-	save_mgo        	*MongodbSim
+	save_mgo,qy_mgo,spi_mgo		*MongodbSim
+	isLocal	bool
 )
+//初始化mgo
+func initMgo() {
+	if isLocal {
+		save_mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:27017",
+			DbName:      "zhengkun",
+			Size:        10,
+			UserName: "",
+			Password: "",
+		}
+		save_mgo.InitPool()
 
-func init()  {
-	save_mgo = &MongodbSim{
-		MongodbAddr: "127.0.0.1:27017",
-		DbName:      "zhengkun",
-		Size:        10,
-		UserName: "",
-		Password: "",
+		qy_mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:27017",
+			DbName:      "mixdata",
+			Size:        10,
+			UserName: "",
+			Password: "",
+		}
+		qy_mgo.InitPool()
+
+		spi_mgo = &MongodbSim{
+			MongodbAddr: "127.0.0.1:27017",
+			DbName:      "zhengkun",
+			Size:        10,
+			UserName: "",
+			Password: "",
+		}
+		spi_mgo.InitPool()
+	}else {
+		save_mgo = &MongodbSim{
+			MongodbAddr: "172.17.4.85:27080",
+			DbName:      "qfw",
+			Size:        10,
+			UserName: "",
+			Password: "",
+		}
+		save_mgo.InitPool()
+
+		//save_mgo = &MongodbSim{
+		//	MongodbAddr: "172.17.4.87:27080",
+		//	DbName:      "hospital",
+		//	Size:        10,
+		//	UserName: "",
+		//	Password: "",
+		//}
+		//save_mgo.InitPool()
+
+		qy_mgo = &MongodbSim{
+			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+			DbName:      "mixdata",
+			Size:        10,
+			UserName: "zhengkun",
+			Password: "zk@123123",
+		}
+		qy_mgo.InitPool()
+
+		spi_mgo = &MongodbSim{
+			MongodbAddr: "172.17.4.181:27001",
+			DbName:      "mixdata",
+			Size:        10,
+			UserName: "",
+			Password: "",
+		}
+		spi_mgo.InitPool()
 	}
-	save_mgo.InitPool()
-
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
-	//	DbName:      "qfw",
-	//	Size:        10,
-	//	UserName: "zhengkun",
-	//	Password: "zk@123123",
-	//}
-	//save_mgo.InitPool()
 }
 
-func main()  {
-	//根据标注数据id导出bidding源数据
-	//expSourceBidding()//导出源数据
-	//synchronizationMarkedata() //同步marked数据
-	//expExampleBiddingData() //导出单元测试数据
-	//expExampleExtractData()//导出-测试数据的抽取数据
+
+func init()  {
+	//isLocal = true //本地
+	initMgo()
+}
+func main(){
+
+	repairfullrepeatdata()
+	time.Sleep(999*time.Hour)
+}
+
 
 
 
 
-	expdataJinEr()//导出错误金额数据
 
+
+func test()  {
+	//dealWithDataFieldAccuracy("a_marked_bidding_old")
+	//increaseTemplateData("zktest_upload_data","zktest_upload_data")
+
+	//var port, path string
+	//flag.StringVar(&port, "port", "8088", "端口")
+	//flag.StringVar(&path, "path", "/", "路径")
+	//flag.Parse()
+	//http.Handle("/", http.FileServer(http.Dir(path)))
+	//log.Debug("port:", port)
+	//http.ListenAndServe(":"+port, nil)
 }
 
+
+

data_template/src/temporary.go

@@ -0,0 +1,375 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	log "github.com/donnie4w/go-logger/logger"
+	"io"
+	"os"
+	qu "qfw/util"
+	"regexp"
+	"strings"
+	"sync"
+)
+var numberlock 		sync.Mutex
+
+var strReg *regexp.Regexp = regexp.MustCompile(".*(镇|乡|县|区|市|省|旗|州|师)")
+var startReg  *regexp.Regexp = regexp.MustCompile("^[委）) ][^员]")
+var startReg_1  *regexp.Regexp = regexp.MustCompile("^[委）) ]")
+var endReg  *regexp.Regexp = regexp.MustCompile("(医院)$")
+
+//修复全量判重数据
+func repairfullrepeatdata() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(save_mgo.DbName).C("zktes_full_repeat").Find(&q).Sort("_id").Iter()
+	pool := make(chan bool, 1)
+	wg := &sync.WaitGroup{}
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total %10000 == 0 {
+			log.Debug("cur index ", total,tmp["_id"])
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+
+			tmpid := BsonTOStringId(tmp["_id"])
+			repeat_id :=qu.ObjToString(tmp["repeat_id"])
+			reason := "全量~"+qu.ObjToString(tmp["reason"])
+			update := map[string]interface{}{
+				"repeat_id":repeat_id,
+				"reason":reason,
+				"repeat":1,
+			}
+			save_mgo.UpdateById("result_20220218",tmpid,map[string]interface{}{
+				"$set": update,
+			})
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Debug("is over ",total)
+}
+
+
+
+
+//计算数据分布情况
+func calculatedatadisposition() {
+
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(save_mgo.DbName).C("special_enterprise").Find(&q).Sort("_id").Select(map[string]interface{}{
+		"company_name":1,
+		"authority":1,
+	}).Iter()
+	pool := make(chan bool, 8)
+	wg := &sync.WaitGroup{}
+	total := 0
+	dict := map[string]int{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total %100000 == 0 {
+			log.Debug("cur index ", total,len(dict))
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			company_name := qu.ObjToString(tmp["company_name"])
+			new_str := strReg.ReplaceAllString(company_name,"")
+			if strings.Contains(new_str,"委员会办公室") {
+				new_str = "委员会办公室"
+			}else {
+				if strings.Contains(new_str,"办公室") {
+					new_str = "办公室"
+				}
+			}
+			if startReg.MatchString(new_str) {
+				new_str = startReg_1.ReplaceAllString(new_str,"")
+			}
+
+
+			if endReg.MatchString(new_str) {
+				new_str = "医院"
+			}
+
+			numberlock.Lock()
+			num := dict[new_str]
+			dict[new_str] = num+1
+			numberlock.Unlock()
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+	log.Debug("is over ",total,len(dict))
+
+	rankArr :=  []int{}
+	nameArr := []string{}
+	for k,v := range dict {
+		if v<1000 {
+			continue
+		}
+		nameArr = append(nameArr,k)
+		rankArr = append(rankArr,qu.IntAll(v))
+	}
+	rankArr,nameArr = dealWithSort(rankArr,nameArr)
+	log.Debug(len(rankArr),len(nameArr))
+	//列取的数据
+	for k,v := range rankArr {
+		//if k>=50 {
+		//	break
+		//}
+		fmt.Println(v,"~",nameArr[k])
+	}
+}
+
+
+//处理字典排序
+func dealWithSort(slice []int , name []string) ([]int,[]string) {
+	for n := 0; n <= len(slice); n++ {
+		for i := 1; i < len(slice)-n; i++ {
+			if slice[i] > slice[i-1] {
+				slice[i], slice[i-1] = slice[i-1], slice[i]
+				name[i], name[i-1] = name[i-1], name[i]
+			}
+		}
+	}
+	return slice,name
+}
+
+
+//分析可修复buyer数量
+func statisticalbuyererr() {
+	coll_arr := []string{"special_enterprise","special_foundation","special_gov_unit","special_law_office","special_social_organ","special_trade_union"}
+	q := map[string]interface{}{}
+	pool := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	sess := qy_mgo.GetMgoConn()
+	defer qy_mgo.DestoryMongoConn(sess)
+	it := sess.DB(qy_mgo.DbName).C("buyer_err").Find(&q).Sort("_id").Select(map[string]interface{}{
+		"name":1,
+	}).Iter()
+	total,isok,isok1,isok2 := 0,0,0,0
+	iserr :=0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total %10000 == 0 {
+			log.Debug("cur index ", total, isok ,isok1, isok2)
+		}
+		if iserr > 30000 {
+			break
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			name := qu.ObjToString(tmp["name"])
+			query := map[string]interface{}{
+				"company_name":name,
+			}
+			if name!="" {
+				//for _,v := range coll_arr{
+				//	//位置181~对比特殊企业
+				//	data_yx := spi_mgo.FindOne(v, query)
+				//	if data_yx!=nil && len(data_yx)>2 {
+				//		numberlock.Lock()
+				//		isok2++
+				//		numberlock.Unlock()
+				//		break
+				//	}
+				//}
+
+				isExists:= false
+				data_yy := save_mgo.FindOne("data_info", map[string]interface{}{
+					"name":name,
+				})
+				if data_yy!=nil && len(data_yy)>2 {
+					numberlock.Lock()
+					isok++
+					isExists = true
+					numberlock.Unlock()
+				} else {
+					//对比企业
+					data_qy := qy_mgo.FindOne("qyxy_std", query)
+					if data_qy!=nil && len(data_qy)>2 {
+						numberlock.Lock()
+						isok1++
+						isExists = true
+						numberlock.Unlock()
+					}else {
+						for _,v := range coll_arr{
+							//位置181~对比特殊企业
+							data_yx := spi_mgo.FindOne(v, query)
+							if data_yx!=nil && len(data_yx)>2 {
+								numberlock.Lock()
+								isok2++
+								isExists = true
+								numberlock.Unlock()
+								break
+							}
+						}
+					}
+				}
+
+
+				if !isExists {
+					numberlock.Lock()
+					iserr++
+					if iserr%100==0  {
+						fmt.Println(name)
+					}
+					numberlock.Unlock()
+				}
+
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+
+	log.Debug("最终修复数据~",total,"~",isok,"~",isok1,"~",isok2)
+
+}
+
+
+
+//分析医械数据~已存在情况
+func statisticalyxdata() {
+	coll_arr := []string{"special_enterprise","special_foundation","special_gov_unit","special_law_office","special_social_organ","special_trade_union"}
+	q := map[string]interface{}{}
+	pool := make(chan bool, 5)
+	wg := &sync.WaitGroup{}
+	//循环遍历数据
+	for _,v := range coll_arr {
+		sess := save_mgo.GetMgoConn()
+		defer save_mgo.DestoryMongoConn(sess)
+		it := sess.DB(save_mgo.DbName).C(v).Find(&q).Sort("_id").Select(map[string]interface{}{
+			"company_name":1,
+		}).Iter()
+		total,isok := 0,0
+		for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+			if total %100000 == 0 {
+				log.Debug("cur index ",total,isok)
+			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				company_name := qu.ObjToString(tmp["company_name"])
+				query := map[string]interface{}{
+					"company_name":company_name,
+				}
+				if company_name!="" {
+					data := qy_mgo.FindOne("qyxy_std", query)
+					if data!=nil && len(data)>2 {
+						numberlock.Lock()
+						isok++
+						numberlock.Unlock()
+					}
+				}
+			}(tmp)
+			tmp = make(map[string]interface{})
+		}
+		wg.Wait()
+		log.Debug(v,"~",total,"~",isok)
+	}
+}
+
+
+
+
+//临时导入mgo数据,源txt
+func importYLQXMgoData(path string) {
+	var startReg = regexp.MustCompile("^(.{0,10}[(])")
+	var endReg = regexp.MustCompile("[)]$")
+	var spaceReg = regexp.MustCompile(",[ ]{2,}")
+	fileHanle,err := os.OpenFile(path, os.O_RDONLY, 0666)
+	if err != nil {
+		log.Debug("读取文件~异常")
+		return
+	}
+	defer fileHanle.Close()
+	reader := bufio.NewReader(fileHanle)
+	var results []string
+	pre_str := ""
+	for  {
+		line, _, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		str := string(line)
+		if str=="GO" || str=="VALUES" {
+			continue
+		}
+		if len(results)==0 && strings.Contains(str,"INSERT"){
+			str = strings.ReplaceAll(str,"INSERT INTO","")
+			arr := strings.Split(str,"(")
+			s_1 := strings.ReplaceAll(arr[0]," ","")
+			s_2 := strings.ReplaceAll(arr[1]," ","")
+			s_2 = strings.ReplaceAll(s_2,")","")
+			results = append(results, s_1)
+			results = append(results, s_2)
+		}else {
+			if strings.Contains(str,"INSERT") {
+				continue
+			}
+			if strings.Contains(str,"VALUES") {
+				if pre_str!="" { //处理字符串
+					pre_str = startReg.ReplaceAllString(pre_str, "")
+					pre_str = endReg.ReplaceAllString(pre_str, "")
+					pre_str = spaceReg.ReplaceAllString(pre_str, ",")
+					pre_str = strings.ReplaceAll(pre_str, "\n", ",")
+					results = append(results, pre_str)
+				}
+				pre_str = str
+			}else {
+				pre_str += str
+			}
+		}
+
+	}
+
+	iserr := 0
+	coll := ""
+	keyArr := []string{}
+	for k,v := range results{
+		if k==0 {
+			coll = v
+		}else if k==1 {
+			keyArr = strings.Split(v,",")
+		}else {
+			arr :=  strings.Split(v,", ")
+			if len(arr)!= len(keyArr) {
+				iserr++
+			}else {
+				dict := make(map[string]interface{},0)
+				for k,v := range keyArr {
+					str := arr[k]
+					str = strings.ReplaceAll(str,"'","")
+					dict[v] = str
+				}
+				save_mgo.Save(coll,dict)
+			}
+		}
+	}
+
+
+	log.Debug("最终解析：",coll,"~",len(results),"~",iserr)
+
+}
+

listen_data/src/main.go

@@ -18,7 +18,7 @@ var (
 	save_mgo,site_mgo,python_mgo        	*MongodbSim            //mongodb操作对象
 	save_c_name,site_c_name,python_c_name,xlsx_name,site_unique_name	string
 	prepareData								[]map[string]interface{}
-	startRun,startSend								string
+	startRun,startSend						string
 )
 func initMgo()  {
 	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
@@ -116,34 +116,34 @@ func main()  {
 	//}
 	//save_mgo.InitPool()
 
-	save_mgo = &MongodbSim{
-		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
-		DbName:      "qfw",
-		Size:        10,
-		UserName: "zhengkun",
-		Password: "zk@123123",
-	}
-	save_mgo.InitPool()
-
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "192.168.3.166:27082",
-	//	DbName:      "zhaolongyue",
+	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+	//	DbName:      "qfw",
 	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
+	//	UserName: "zhengkun",
+	//	Password: "zk@123123",
 	//}
 	//save_mgo.InitPool()
 
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "127.0.0.1:27017",
-	//	DbName:      "zhengkun",
+	//	MongodbAddr: "192.168.3.166:27082",
+	//	DbName:      "zhaolongyue",
 	//	Size:        10,
 	//	UserName: "",
 	//	Password: "",
 	//}
 	//save_mgo.InitPool()
 
-	decodeJyUrl()
+	save_mgo = &MongodbSim{
+		MongodbAddr: "127.0.0.1:27017",
+		DbName:      "zhengkun",
+		Size:        10,
+		UserName: "",
+		Password: "",
+	}
+	save_mgo.InitPool()
+
+	decodeJyUrl("ABCY1wAfikOOyY7RGBlc3UjIzAoAS1mcWBwKB4FKiENdFFzfytUCU0")
 
 	return
 

listen_data/src/zkmethod.go

@@ -94,8 +94,8 @@ func encodeSEUrl(){
 }
 
 //解密
-func decodeJyUrl()  {
-	test := "ABCY1wAfikFIywvEmd5cwcOCzIoIDRgYWR0KB4nIC4ee2Bzfw1UCLw%3D"
+func decodeJyUrl(str string)  {
+	test := str+"%3D"
 	var Decode  = qu.CommonDecodeArticle("content", test)
 	log.Debug(Decode[0])
 	return
@@ -107,6 +107,114 @@ func encodeJyUrl()  {
 	log.Debug(Encode)
 }
 
+//更新剑鱼链接
+func updateJyhref() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q ,total:= map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C("zktest_exists_addr").Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("cur index ",total)
+		}
+		tmpid := BsonTOStringId(tmp["_id"])
+		jytest_href := fmt.Sprintf(Url, qu.CommonEncodeArticle("content", tmpid))
+		save_mgo.UpdateById("zktest_exists_addr",tmpid, map[string]interface{}{
+			"$set":map[string]interface{}{
+				"jytest_href":jytest_href,
+			},
+		})
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ...",total)
+}
+
+//bidding数据 导出空城市 site 区分数量
+func dealWithAreaCityData() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q ,total:= map[string]interface{}{
+		"comeintime":map[string]interface{}{
+			"$gte":1654012800,
+			"$lt":1655568000,
+		},
+	},0
+	dict_1 := map[string]int{}
+	dict_2 := map[string]int{}
+	dict_3 := map[string]int{}
+	dict_4 := map[string]int{}
+	isok1,isok2,isok3,isok4 := 0,0,0,0
+	it := sess.DB(save_mgo.DbName).C("bidding").Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("cur index ",total,isok1,isok2,isok3,isok4)
+		}
+		if isok1==500 && isok2==500 && isok3==500 && isok4==500 {
+			break
+		}
+		area := qu.ObjToString(tmp["area"])
+		city := qu.ObjToString(tmp["city"])
+		detail := qu.ObjToString(tmp["detail"])
+		site := qu.ObjToString(tmp["site"])
+		if strings.Contains(detail,"地址"){
+			if  city==""{
+				if area!="全国" {
+					if qu.IntAll(dict_1[site]) < 3  && isok1 < 500{
+						dict_1[site] = qu.IntAll(dict_1[site])+1
+						isok1++
+						save_mgo.Save("zktest_exists_addr",tmp)
+					}
+				}else {
+					if qu.IntAll(dict_2[site]) < 3 && isok2 < 500{
+						dict_2[site] = qu.IntAll(dict_2[site])+1
+						isok2++
+						save_mgo.Save("zktest_exists_addr",tmp)
+					}
+				}
+			}
+		}else {
+			if  city==""{
+				if area!="全国" {
+					if qu.IntAll(dict_3[site]) < 3 && isok3 < 500 {
+						dict_3[site] = qu.IntAll(dict_3[site])+1
+						isok3++
+						save_mgo.Save("zktest_unexists_addr",tmp)
+					}
+				}else {
+					if qu.IntAll(dict_4[site]) < 3 && isok4 < 500{
+						dict_4[site] = qu.IntAll(dict_4[site])+1
+						isok4++
+						save_mgo.Save("zktest_unexists_addr",tmp)
+					}
+				}
+			}
+		}
+
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ...",total,isok1,isok2,isok3,isok4)
+}
+
+
+
+//大金额数据
+func dealWithMuchMoneyData() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q ,total:= map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C("22222").Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("cur index ",total)
+		}
+		save_mgo.Save("zkzkzk", map[string]interface{}{
+			"_id":StringTOBsonId(qu.ObjToString(tmp["_id"])),
+		})
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ...",total)
+}
+
 
 
 //导出错误中标单位数据

process_medical/src/medical_bidding.go

@@ -381,7 +381,7 @@ func exportNewPurBidData()  {
 //导出指定-迈瑞相关数据-已弃用
 func exportMaiRuiInfoId(){
 	query := "SELECT product_name FROM f_product WHERE company_name LIKE CONCAT('%',?,'%')"
-	proinfo := MysqlTool.SelectBySql(query, "迈瑞")
+	proinfo := MysqlTool.Insert(query, "迈瑞")
 	log.Debug(len(*proinfo))
 	proNameArr :=[]string{}
 	proNameDict := map[string]string{}