监听-修改-主要

data_monitoring/listen_lan/src/main.go

@@ -46,7 +46,6 @@ func init() {
 
 func main() {
 	go taskTime()
-
 	time.Sleep(99999 * time.Hour)
 
 }
@@ -113,8 +112,6 @@ func dealWithOtherData()  {
 		row.AddCell().SetString(qu.ObjToString(before_st_data["date"]))
 	}
 
-
-
 	data_1 := *qu.ObjToMap(spider_data["data"])
 	data_2 := *qu.ObjToMap(before_st_data["data"])
 

data_monitoring/listen_online/src/config.json

@@ -1,19 +1,19 @@
 {
   "save_mgodb": {
-    "addr": "172.17.4.87:27080",
-    "db": "editor",
+    "addr": "192.168.3.207:27092",
+    "db": "zhengkun",
     "coll": "monitor_other",
     "pool": 5
   },
   "qy_mgodb": {
-    "addr": "172.17.4.87:27080",
-    "db": "spider",
+    "addr": "192.168.3.207:27092",
+    "db": "zhengkun",
     "coll": "baidu_enterprise",
     "pool": 5
   },
   "st_mgodb": {
-    "addr": "172.17.4.85:27080",
-    "db": "qfw",
+    "addr": "192.168.3.207:27092",
+    "db": "zhengkun",
     "coll": "result_20210109",
     "pool": 5
   },
@@ -21,8 +21,19 @@
   "es_type_st": "bidding",
   "save_other_name": "monitor_other",
   "save_site_name" : "monitor_site",
+  "unique_site_name": "z_site_unique",
   "jkmail": {
     "to": "zhengkun@topnet.net.cn",
     "api": "http://172.17.145.179:19281/_send/_mail"
-  }
+  },
+  "smtpMail": {
+    "from": "zhengkun@topnet.net.cn",
+    "to": "zhengkun@topnet.net.cn",
+    "cc": "zhengkun@topnet.net.cn",
+    "smtpHost": "smtp.qq.com",
+    "smtpPort": "465",
+    "user":     "920032221@qq.com",
+    "pwd":      "xomkphsjsamybdbj"
+  },
+  "xlsx_name" : "site_qy.xlsx"
 }

data_monitoring/listen_online/src/errmail.go

@@ -5,10 +5,14 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"qfw/util/mail"
+	"os"
+	qu "qfw/util"
 )
 var tomail string
 var api string
-
+var from,to,cc, smtpHost,user,pwd string
+var smtpPort int
 //api模式 二选一皆可
 func sendErrMailApi(title,body string)  {
 	jkmail, _ := sysconfig["jkmail"].(map[string]interface{})
@@ -25,4 +29,35 @@ func sendErrMailApi(title,body string)  {
 	}else {
 		log.Println("邮件发送失败：", err)
 	}
+}
+
+func sendErrMailSmtp(title,body string) {
+
+
+	smtpMail, _ := sysconfig["smtpMail"].(map[string]interface{})
+
+	if smtpMail != nil {
+		from, _ = smtpMail["from"].(string)
+		to, _ = smtpMail["to"].(string)
+		cc, _ = smtpMail["cc"].(string)
+		smtpHost, _ = smtpMail["smtpHost"].(string)
+		smtpPort= qu.IntAll(smtpMail["smtpPort"])
+		user, _ = smtpMail["user"].(string)
+		pwd, _ = smtpMail["pwd"].(string)
+	}
+	f, _ := os.Open(xlsx_name)
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		fmt.Println("err:",err)
+		return
+	}
+
+
+	ok := mail.GSendMail_B(user, from, cc, from, title, body, f.Name(), b, &mail.GmailAuth{
+		SmtpHost: smtpHost,
+		SmtpPort: smtpPort,
+		User:     user,
+		Pwd:      pwd,
+	})
+	fmt.Println(ok)
 }

data_monitoring/listen_online/src/main.go

@@ -3,7 +3,9 @@ package main
 import (
 	"fmt"
 	"github.com/cron"
+	"github.com/tealeg/xlsx"
 	"log"
+	"os"
 	qu "qfw/util"
 	"time"
 )
@@ -15,7 +17,7 @@ var (
 	qy_c_name,st_c_name						string
 	es_index_st,es_type_st					string
 	save_other_name,save_site_name			string
-	xlsx_name								string
+	xlsx_name,unique_site_name				string
 )
 
 func initMgo()  {
@@ -53,6 +55,9 @@ func initMgo()  {
 	es_type_st = qu.ObjToString(sysconfig["es_type_st"])
 	save_other_name = qu.ObjToString(sysconfig["save_other_name"])
 	save_site_name = qu.ObjToString(sysconfig["save_site_name"])
+	unique_site_name = qu.ObjToString(sysconfig["unique_site_name"])
+	xlsx_name = qu.ObjToString(sysconfig["xlsx_name"])
+
 }
 
 func init() {
@@ -63,54 +68,78 @@ func init() {
 
 func main() {
 	go taskTime()
-
+	dealWithUniqueData()
 	time.Sleep(99999 * time.Hour)
-
 }
 
 func taskTime()  {
 
 	log.Println("部署定时任务")
 	c := cron.New()
-	//站点相关全量-es-mongo
-	c.AddFunc("0 30 8 ? * *", func() { dealWithSTData() })
-	//企业变更-mongo
-	c.AddFunc("0 50 8 ? * *", func() { dealWithOtherData() })
+	//企业变更-mongo-site-es
+	c.AddFunc("0 50 8 ? * *", func() { dealWithUniqueData() })
 	c.Start()
 }
 
-func dealWithOtherData()  {
+func dealWithUniqueData()  {
 
 	log.Println("开始统计相关数据...")
 
-	start := int(time.Now().Unix())
-	qy_date,qy_num := dealWithQYData()
-	log.Println(qy_date,qy_num)
-
-	comeintime :=qu.Int64All(time.Now().Unix())
-	log.Println(comeintime)
-	save_mgo.Save(save_other_name, map[string]interface{}{
-		"name":"企业",
-		"num":qu.IntAll(qy_num),
-		"comeintime":comeintime,
-		"date":qy_date,
-	})
-	//获取前两天数据
-	now:=time.Now() //当前天
-	cur_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
-	before_time := cur_time-86400*2
-	before_date :=qu.FormatDateByInt64(&before_time, qu.DATEFORMAT)
+	qy_data := dealWithQYData()
+	st_data :=dealWithSTData()
+
+	log.Println("处理数据...发送邮件...")
+
+	os.Remove(xlsx_name)
+	f :=xlsx.NewFile()
+	sheet, _ := f.AddSheet("统计")
 	//企业数据
-	before_qy_data :=save_mgo.FindOne(save_other_name, map[string]interface{}{
-		"name":"企业",
-		"date":before_date,
-	})
-
-	body :=fmt.Sprintf("日期:%s-数量:%s",qy_date,qy_num)
-	if len(before_qy_data)>0 {
-		body =body+fmt.Sprintf("~~日期:%s-数量:%d",before_date,before_qy_data["num"])
+	row := sheet.AddRow()
+	row.AddCell().Value = "企业变更/日期"
+	row.AddCell().Value = "数量"
+	for k,v :=range qy_data {
+		row = sheet.AddRow()
+		row.AddCell().SetString(k)
+		row.AddCell().SetString(fmt.Sprintf("%d",qu.IntAll(v)))
+	}
+	row = sheet.AddRow()
+	row = sheet.AddRow()
+
+
+	if len(st_data)>0 { //有数据
+		row = sheet.AddRow()
+		row.AddCell().Value = "主要站点/日期"
+		row.AddCell().Value = "名称"
+		row.AddCell().Value = "数量"
+		row.AddCell().Value = "总计"
+
+
+		mgodata := *qu.ObjToMap(st_data["mgodata"])
+		date := qu.ObjToString(st_data["date"])
+		total := qu.IntAll(st_data["mgonum"])
+		isT := false
+		for key,_ := range uniqueData {
+			row = sheet.AddRow()
+			if !isT {
+				row.AddCell().SetString(date)
+				row.AddCell().SetString(key)
+				row.AddCell().SetString(fmt.Sprintf("%d",qu.IntAll(mgodata[key])))
+				row.AddCell().SetString(fmt.Sprintf("%d",qu.IntAll(total)))
+				isT = true
+			}else {
+				row.AddCell().Value = ""
+				row.AddCell().SetString(key)
+				row.AddCell().SetString(fmt.Sprintf("%d",qu.IntAll(mgodata[key])))
+				row.AddCell().Value = ""
+			}
+		}
+	}
+	err := f.Save(xlsx_name)
+	if err != nil {
+		log.Println("保存xlsx失败:", err)
+	}else {
+		log.Println("保存xlsx成功:", err)
 	}
 
-	sendErrMailApi("企业-变更",body)
-	log.Println("定时处理完毕...",int(time.Now().Unix())-start,"秒")
+	sendErrMailSmtp("企业变更-主要站点","附件")
 }

data_monitoring/listen_online/src/priselisten.go

@@ -1,31 +1,30 @@
 package main
 
 import (
-	"fmt"
+	"log"
 	qu "qfw/util"
 	"time"
 )
 
 
-func dealWithQYData()(string,string)  {
+func dealWithQYData()map[string]interface{} {
 
 
 	defer qu.Catch()
-
+	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
 	lt_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
 	gte_time := lt_time-86400
-
 	q := map[string]interface{}{
 		"down_time": map[string]interface{}{
 			"$gte":  gte_time,
 			"$lt": lt_time,
 		},
 	}
-
 	sess := qy_mgo.GetMgoConn()
 	defer qy_mgo.DestoryMongoConn(sess)
 
+	data := make(map[string]interface{},0)
 	//细节才需要遍历
 	it := sess.DB(qy_mgo.DbName).C(qy_c_name).Find(&q).Select(map[string]interface{}{
 		"site":1,
@@ -34,7 +33,31 @@ func dealWithQYData()(string,string)  {
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		tmp = make(map[string]interface{})
 	}
-	return qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT),fmt.Sprintf("%d",total)
+
+	qy_date := qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT)
+	data[qy_date] = total
+	comeintime :=qu.Int64All(time.Now().Unix())
+	save_mgo.Save(save_other_name, map[string]interface{}{
+		"name":"企业",
+		"num":qu.IntAll(total),
+		"comeintime":comeintime,
+		"date":qy_date,
+	})
+
+	//获取前两天数据
+	before_time := lt_time-86400*2
+	before_date :=qu.FormatDateByInt64(&before_time, qu.DATEFORMAT)
+	//企业数据
+	before_qy_data :=save_mgo.FindOne(save_other_name, map[string]interface{}{
+		"name":"企业",
+		"date":before_date,
+	})
+	if len(before_qy_data)>1 {
+		data[before_date] =qu.IntAll(before_qy_data["num"])
+	}
+	log.Println("企业变更处理完毕...",int(time.Now().Unix())-start,"秒")
+
+	return data
 }
 
 

data_monitoring/listen_online/src/sitelisten.go

@@ -10,12 +10,42 @@ import (
 	es_elastic "qfw/common/src/gopkg.in/olivere/elastic.v1"
 )
 
+var uniqueData  map[string]interface{}   //指定站点
 
-func dealWithSTData()  {
+func initUniqueSite()  {
+
+	uniqueData = make(map[string]interface{})
+
+	query := map[string]interface{}{}
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	it := sess.DB(save_mgo.DbName).C(unique_site_name).Find(&query).Iter()
+	total:=0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		site:=qu.ObjToString(tmp["site"])
+		uniqueData[site] = 1
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("is unique_site over",len(uniqueData))
+
+}
+
+func dealWithSTData() map[string]interface{} {
 
 	log.Println("开始统计全量站点数据...")
 	defer qu.Catch()
 
+
+
+	initUniqueSite() //准备独特站点
+
+	if len(uniqueData)<1 {
+		log.Println("独特站点-准备异常")
+		sendErrMailApi("独特站点-线上-准备异常","")
+		return map[string]interface{}{}
+	}
+
 	start := int(time.Now().Unix())
 	now:=time.Now() //当前天
 	lt_time:= time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
@@ -27,25 +57,22 @@ func dealWithSTData()  {
 
 	comeintime:=qu.Int64All(time.Now().Unix())
 	date := qu.FormatDateByInt64(&gte_time, qu.DATEFORMAT)
-	save_mgo.Save(save_site_name, map[string]interface{}{
+	save_data :=  map[string]interface{}{
 		"comeintime":comeintime,
 		"date":date,
-		"name":"全量站点",
+		"name":"全量主要站点",
 		"mgonum":mgonum,
 		"esnum":esnum,
 		"mgodata":mgodata["detail"],
 		"esdata":esdata["detail"],
-	})
+	}
+	save_mgo.Save(save_site_name, save_data)
 
 	log.Println("站点-定时处理完毕...","用时：",int(time.Now().Unix())-start,"秒")
-}
-
-
-
-
-
 
+	return save_data
 
+}
 
 
 
@@ -83,7 +110,7 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 	if cursor.Results.Hits == nil {
 		log.Println("expected results.Hits != nil; got nil")
 	}
-	//log.Println("es total ：",cursor.TotalHits(),"处理分析中......")
+	log.Println("es total ：",cursor.TotalHits(),"处理分析中......")
 	for {
 		searchResult, err := cursor.Next()
 		if err != nil {
@@ -100,13 +127,16 @@ func toCalculateEsData(gte_time string,lt_time string) map[string]interface{} {
 				log.Println("json Unmarshal error")
 				continue
 			}
-			total++
+
 			site := qu.ObjToString(tmp["site"])	//统计站点
-			if dict[site] == nil {
-				dict[site] = 1
-			}else {
-				num := qu.IntAll(dict[site])+1
-				dict[site] = num
+			if uniqueData[site]!=nil { //统计有效站点
+				total++
+				if dict[site] == nil {
+					dict[site] = 1
+				}else {
+					num := qu.IntAll(dict[site])+1
+					dict[site] = num
+				}
 			}
 		}
 	}
@@ -147,16 +177,18 @@ func toCalculateMgoData(gte_time int64,lt_time int64) map[string]interface{} {
 		"site":1,
 	}).Iter()
 	total:=0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+	for tmp := make(map[string]interface{}); it.Next(&tmp);  {
 		//统计站点
 		site := qu.ObjToString(tmp["site"])
-		if dict[site] == nil {
-			dict[site] = 1
-		}else {
-			num := qu.IntAll(dict[site])+1
-			dict[site] = num
+		if uniqueData[site]!=nil { //统计有效站点
+			total++
+			if dict[site] == nil {
+				dict[site] = 1
+			}else {
+				num := qu.IntAll(dict[site])+1
+				dict[site] = num
+			}
 		}
-
 		tmp = make(map[string]interface{})
 	}
 	return map[string]interface{}{

data_monitoring/words_vaild/src/main.go

@@ -1,81 +0,0 @@
-package main
-
-import (
-	"log"
-	qu "qfw/util"
-	"qfw/util/elastic"
-	"strings"
-	"unicode/utf8"
-)
-
-func init()  {
-	elastic.InitElasticSize("http://192.168.3.11:9800",10)
-}
-
-func main()  {
-
-	defer qu.Catch()
-	log.Println("处理 ... 指定企业名称 ...")
-	/*
-	云南和合泰商贸有限公司
-	安徽省微乡华艺环境工程有限公司
-	*/
-	new_name,b :=dealWithScoreRules("安徽省微乡华艺环境工程有限公司")
-	if b {
-		log.Println("最终",new_name)
-	}
-
-}
-
-
-func dealWithScoreRules(name string) (string,bool) {
-	new_name,isok :="",false
-	query:= `{"query":{"bool":{"must":[{"query_string":{"default_field":"azktest.name_2","query":"`+name+`"}}],"must_not":[],"should":[]}},"from":0,"size":1,"sort":[],"facets":{}}`
-	//默认取最高分-分析多个分-遍历器查询
-	data := *elastic.Get("azktest","azktest",query)
-	if len(data)>0 && data != nil {
-		new_name = qu.ObjToString(data[0]["name"])
-	}
-	if new_name!="" { //分析hit比例
-		total,hit := dealWithWordsRules(name,new_name)
-		if float64(hit)/float64(total)>=0.8 {
-			isok = true
-		}
-	}
-	return new_name,isok
-}
-
-//击中数量以及比例
-func dealWithWordsRules(info_name string ,source_name string) (int,int){
-	total,hit :=0,0
-	nameArr,_ := calculateWordCount(info_name)
-	_,total = calculateWordCount(source_name)
-	for _,v1 := range nameArr {
-		if strings.Contains(source_name,v1) {
-			hit++
-		}
-	}
-	return total,hit
-}
-
-//分词结果
-func calculateWordCount(name string) ([]string,int) {
-
-	arr ,space:= make([]string,0),2
-	total := utf8.RuneCountInString(name)-(space-1)
-	if name == "" || total<=0  {
-		return arr,0
-	}
-	nameRune := []rune(name)
-	for i:=0;i<total ;i++  {
-		new_str := string(nameRune[i:space+i])
-		arr = append(arr,new_str)
-	}
-	return arr,len(arr)
-}
-
-
-
-
-
-

data_monitoring/words_vaild/src1/main.go

@@ -1,304 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/tealeg/xlsx"
-	"log"
-	"os"
-	qu "qfw/util"
-	"qfw/util/elastic"
-	"strings"
-	"sync"
-	"unicode/utf8"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-
-)
-var (
-	sysconfig			map[string]interface{} //配置文件
-	save_mgo        	*MongodbSim
-)
-
-func init()  {
-	save_mgo = &MongodbSim{
-		MongodbAddr: "192.168.3.207:27092",
-		DbName:      "zhengkun",
-		Size:        5,
-	}
-	save_mgo.InitPool()
-
-	elastic.InitElasticSize("http://192.168.3.11:9800",20)
-}
-
-func dealWithDataXlsx()  {
-
-	q := map[string]interface{}{}
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
-	it := sess.DB(save_mgo.DbName).C("zk_test_words").Find(&q).Iter()
-	total:=0
-	saveArr := make([]map[string]string,0)
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total % 10000 == 0 {
-			log.Println("current index",total,tmp["_id"])
-		}
-
-		if total % 30 ==0 {
-			name:=qu.ObjToString(tmp["name"])
-			dict := make(map[string]string)
-			dict["name"] = name
-			for i:=0; i<5;i++ {
-				value,total,hit :="","",""
-				key := "word_"+fmt.Sprintf("%d",i)
-				if tmp[key]!=nil {
-
-					if arr,ok := tmp[key].(primitive.A);ok {
-						dataArr :=qu.ObjArrToMapArr(arr)
-						value =qu.ObjToString(dataArr[0]["name"])
-						if i!=0 {
-							total = fmt.Sprintf("%d",dataArr[0]["all_words"])
-							hit = fmt.Sprintf("%d",dataArr[0]["hit_words"])
-						}
-					}
-
-				}
-				key1,key2:="total"+fmt.Sprintf("%d",i),"hit"+fmt.Sprintf("%d",i)
-				dict[key] = value
-				dict[key1] = total
-				dict[key2] = hit
-
-			}
-			saveArr= append(saveArr,dict)
-		}
-		tmp = make(map[string]interface{})
-	}
-
-
-	os.Remove("words.xlsx")	//写excle
-	f :=xlsx.NewFile()
-
-
-	for i:=0; i<5;i++ {
-		key := "word_"+fmt.Sprintf("%d",i)
-		sheet, _ := f.AddSheet("统计"+key)
-		row := sheet.AddRow()
-		row.AddCell().Value = "name"
-		row.AddCell().Value = key
-		if i!=0 {
-			row.AddCell().Value = "total"
-			row.AddCell().Value = "hit"
-		}
-		key1,key2:="total"+fmt.Sprintf("%d",i),"hit"+fmt.Sprintf("%d",i)
-
-		for _,tmp := range saveArr {
-			row = sheet.AddRow()
-			row.AddCell().SetString(tmp["name"])
-			row.AddCell().SetString(tmp[key])
-			row.AddCell().SetString(fmt.Sprintf("%s",tmp[key1]))
-			row.AddCell().SetString(fmt.Sprintf("%s",tmp[key2]))
-		}
-	}
-
-	err := f.Save("words.xlsx")
-	if err != nil {
-		log.Println("保存xlsx失败:", err)
-	}else {
-		log.Println("保存xlsx成功:", err)
-	}
-}
-
-func main()  {
-
-	//导出xlsx
-	dealWithDataXlsx()
-	return
-
-
-
-
-	defer qu.Catch()
-	log.Println("处理 ... 指定企业名称 ...")
-
-	//分析错误数据
-	//
-	q := map[string]interface{}{}
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
-	//细节才需要遍历
-	it := sess.DB(save_mgo.DbName).C("zk_company_test").Find(&q).Iter()
-	total:=0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total % 10000 == 0 {
-			log.Println("current index",total,tmp["_id"])
-		}
-
-		name:=qu.ObjToString(tmp["name"])
-		save_dict := make(map[string]interface{},0)
-		for i:=0; i<5;i++ {
-			key := "word_"+fmt.Sprintf("%d",i)
-			dataArr :=dealWithScoreRules(name,i)
-			if dataArr ==nil || len(dataArr)<1 {
-				//无数据
-			}else {
-				save_dict[key] = dealWithWordsRules(name,dataArr,i)
-			}
-
-		}
-
-		if len(save_dict)>0 {
-			save_dict["name"]  = name
-			save_mgo.Save("zk_test_words",save_dict)
-		}
-
-		tmp = make(map[string]interface{})
-	}
-
-}
-
-//分数维度
-func dealWithScoreRules(name string,space int) []map[string]interface{} {
-	key := ""
-	if space>0&&space<5{
-		key = fmt.Sprintf("%d",space)
-	}
-	query:= `{"query":{"bool":{"must":[{"query_string":{"default_field":"azktest.name_`+key+`","query":"`+name+`"}}],"must_not":[],"should":[]}},"from":0,"size":3,"sort":[],"facets":{}}`
-
-	if key=="" {
-		query = `{"query":{"bool":{"must":[{"query_string":{"default_field":"azktest.name","query":"`+name+`"}}],"must_not":[],"should":[]}},"from":0,"size":3,"sort":[],"facets":{}}`
-	}
-	client := elastic.GetEsConn()
-	defer elastic.DestoryEsConn(client)
-	searchResult, err := client.Search().Index("azktest").Type("azktest").Source(query).Do()
-	if err != nil {
-		log.Println("从ES查询出错", err.Error())
-		return nil
-	}
-	resNum := len(searchResult.Hits.Hits)
-	res := make([]map[string]interface{}, resNum)
-	if searchResult.Hits != nil {
-		if resNum < 5000 {
-			for i, hit := range searchResult.Hits.Hits {
-				data := make(map[string]interface{},0)
-				json.Unmarshal(*hit.Source, &data)
-				res[i] = map[string]interface{}{
-					"name":data["name"],
-					"score":*hit.Score,
-				}
-			}
-		} else {
-			log.Println("查询结果太多，查询到：", resNum, "条")
-		}
-
-	}
-	return res
-}
-
-
-
-//击中数量以及比例
-func dealWithWordsRules(name string ,source []map[string]interface{},space int) []map[string]interface{} {
-
-	nameArr,_ := calculateWordCount(name,space)
-	newArr := make([]map[string]interface{},0)
-	for _,v := range source {
-		total,hit :=0,0
-		source_name :=qu.ObjToString(v["name"])
-		_,total = calculateWordCount(source_name,space)
-		for _,v1 := range nameArr {
-			if strings.Contains(source_name,v1) {
-				hit++
-			}
-		}
-
-
-		if space==0 {
-			newArr = append(newArr, map[string]interface{}{
-				"name":source_name,
-				"score":qu.Float64All(v["score"]),
-			})
-		}else {
-			newArr = append(newArr, map[string]interface{}{
-				"name":source_name,
-				"score":qu.Float64All(v["score"]),
-				"all_words" : total,
-				"hit_words" : hit,
-			})
-		}
-	}
-	return newArr
-}
-
-//分词结果
-func calculateWordCount(name string,space int) ([]string,int) {
-	arr := make([]string,0)
-	total := utf8.RuneCountInString(name)-(space-1)
-	if name == "" || space<=0 || total<=0  {
-		return arr,0
-	}
-	nameRune := []rune(name)
-	for i:=0;i<total ;i++  {
-		new_str := string(nameRune[i:space+i])
-		arr = append(arr,new_str)
-	}
-	return arr,len(arr)
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-func readyDataEs()  {
-
-	q := map[string]interface{}{}
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
-	//多线程升索引
-	pool_es := make(chan bool, 10)
-	wg_es := &sync.WaitGroup{}
-	//细节才需要遍历
-	it := sess.DB(save_mgo.DbName).C("zk_company_name").Find(&q).Iter()
-	total:=0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total % 10000 == 0 {
-			log.Println("current index",total,tmp["_id"])
-		}
-		savetmp := make(map[string]interface{}, 0)
-		savetmp["_id"] = tmp["_id"]
-		savetmp["name"] = qu.ObjToString(tmp["company_name"])
-		savetmp["name_1"] = qu.ObjToString(tmp["company_name"])
-		savetmp["name_2"] = qu.ObjToString(tmp["company_name"])
-		savetmp["name_3"] = qu.ObjToString(tmp["company_name"])
-		savetmp["name_4"] = qu.ObjToString(tmp["company_name"])
-		pool_es <- true
-		wg_es.Add(1)
-		go func(savetmp map[string]interface{}) {
-			defer func() {
-				<-pool_es
-				wg_es.Done()
-			}()
-			elastic.Save("azktest","azktest", savetmp)
-		}(savetmp)
-		tmp = make(map[string]interface{})
-	}
-	wg_es.Wait()
-
-
-	log.Println("is over",total)
-}
-

data_monitoring/words_vaild/src1/mgo.go

@@ -1,328 +0,0 @@
-package main
-
-import (
-	"context"
-	"log"
-	"time"
-
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
-)
-
-type MgoSess struct {
-	Db     string
-	Coll   string
-	Query  interface{}
-	Sorts  []string
-	fields interface{}
-	limit  int64
-	skip   int64
-	M      *MongodbSim
-}
-
-type MgoIter struct {
-	Cursor *mongo.Cursor
-}
-
-func (mt *MgoIter) Next(result interface{}) bool {
-	if mt.Cursor != nil {
-		if mt.Cursor.Next(nil) {
-			err := mt.Cursor.Decode(result)
-			if err != nil {
-				log.Println("mgo cur err", err.Error())
-				mt.Cursor.Close(nil)
-				return false
-			}
-			return true
-		} else {
-			mt.Cursor.Close(nil)
-			return false
-		}
-	} else {
-		return false
-	}
-
-}
-
-func (ms *MgoSess) DB(name string) *MgoSess {
-	ms.Db = name
-	return ms
-}
-
-func (ms *MgoSess) C(name string) *MgoSess {
-	ms.Coll = name
-	return ms
-}
-
-func (ms *MgoSess) Find(q interface{}) *MgoSess {
-	ms.Query = q
-	return ms
-}
-
-func (ms *MgoSess) Select(fields interface{}) *MgoSess {
-	ms.fields = fields
-	return ms
-}
-
-func (ms *MgoSess) Limit(limit int64) *MgoSess {
-	ms.limit = limit
-	return ms
-}
-func (ms *MgoSess) Skip(skip int64) *MgoSess {
-	ms.skip = skip
-	return ms
-}
-
-func (ms *MgoSess) Sort(sorts ...string) *MgoSess {
-	ms.Sorts = sorts
-	return ms
-}
-
-func (ms *MgoSess) Iter() *MgoIter {
-	it := &MgoIter{}
-	find := options.Find()
-	if ms.skip > 0 {
-		find.SetSkip(ms.skip)
-	}
-	if ms.limit > 0 {
-		find.SetLimit(ms.limit)
-	}
-	find.SetBatchSize(100)
-	if len(ms.Sorts) > 0 {
-		sort := bson.M{}
-		for _, k := range ms.Sorts {
-			switch k[:1] {
-			case "-":
-				sort[k[1:]] = -1
-			case "+":
-				sort[k[1:]] = 1
-			default:
-				sort[k] = 1
-			}
-		}
-		find.SetSort(sort)
-	}
-	if ms.fields != nil {
-		find.SetProjection(ms.fields)
-	}
-	cur, err := ms.M.C.Database(ms.Db).Collection(ms.Coll).Find(ms.M.Ctx, ms.Query, find)
-	if err != nil {
-		log.Println("mgo find err", err.Error())
-	} else {
-		it.Cursor = cur
-	}
-	return it
-}
-
-type MongodbSim struct {
-	MongodbAddr string
-	Size        int
-	//	MinSize     int
-	DbName   string
-	C        *mongo.Client
-	Ctx      context.Context
-	ShortCtx context.Context
-	pool     chan bool
-	UserName string
-	Password string
-}
-
-func (m *MongodbSim) GetMgoConn() *MgoSess {
-	//m.Open()
-	ms := &MgoSess{}
-	ms.M = m
-	return ms
-}
-
-func (m *MongodbSim) DestoryMongoConn(ms *MgoSess) {
-	//m.Close()
-	ms.M = nil
-	ms = nil
-}
-
-func (m *MongodbSim) InitPool() {
-	opts := options.Client()
-	opts.SetConnectTimeout(3 * time.Second)
-	opts.ApplyURI("mongodb://" + m.MongodbAddr)
-	opts.SetMaxPoolSize(uint64(m.Size))
-	m.pool = make(chan bool, m.Size)
-
-	if m.UserName !="" && m.Password !="" {
-		cre := options.Credential{
-			Username:m.UserName,
-			Password:m.Password,
-		}
-		opts.SetAuth(cre)
-	}
-
-
-
-	opts.SetMaxConnIdleTime(2 * time.Hour)
-	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)
-	m.ShortCtx, _ = context.WithTimeout(context.Background(), 1*time.Minute)
-	client, err := mongo.Connect(m.ShortCtx, opts)
-	if err != nil {
-		log.Println("mgo init error:", err.Error())
-	} else {
-		m.C = client
-		log.Println("init success")
-	}
-}
-
-func (m *MongodbSim) Open() {
-	m.pool <- true
-}
-func (m *MongodbSim) Close() {
-	<-m.pool
-}
-
-//批量插入
-func (m *MongodbSim) UpSertBulk(c string, doc ...[]map[string]interface{}) (map[int64]interface{}, bool) {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	var writes []mongo.WriteModel
-	for _, d := range doc {
-		write := mongo.NewUpdateOneModel()
-		write.SetFilter(d[0])
-		write.SetUpdate(d[1])
-		write.SetUpsert(true)
-		writes = append(writes, write)
-	}
-	r, e := coll.BulkWrite(m.Ctx, writes)
-	if e != nil {
-		log.Println("mgo upsert error:", e.Error())
-		return nil, false
-	}
-	//	else {
-	//		if r.UpsertedCount != int64(len(doc)) {
-	//			log.Println("mgo upsert uncomplete:uc/dc", r.UpsertedCount, len(doc))
-	//		}
-	//		return true
-	//	}
-	return r.UpsertedIDs, true
-}
-
-//批量插入
-func (m *MongodbSim) SaveBulk(c string, doc ...map[string]interface{}) bool {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	var writes []mongo.WriteModel
-	for _, d := range doc {
-		write := mongo.NewInsertOneModel()
-		write.SetDocument(d)
-		writes = append(writes, write)
-	}
-	_, e := coll.BulkWrite(m.Ctx, writes)
-	if e != nil {
-		log.Println("mgo savebulk error:", e.Error())
-		return false
-	}
-	return true
-}
-
-//保存
-func (m *MongodbSim) Save(c string, doc map[string]interface{}) interface{} {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	r, err := coll.InsertOne(m.Ctx, doc)
-	if err != nil {
-		return nil
-	}
-	return r.InsertedID
-}
-
-//更新by Id
-func (m *MongodbSim) UpdateById(c, id string, doc map[string]interface{}) bool {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	_, err := coll.UpdateOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)}, doc)
-	if err != nil {
-		return false
-	}
-	return true
-}
-
-//删除by id
-func (m *MongodbSim) DeleteById(c, id string) int64 {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	r, err := coll.DeleteOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
-	if err != nil {
-		return 0
-	}
-	return r.DeletedCount
-}
-
-//通过条件删除
-func (m *MongodbSim) Delete(c string, query map[string]interface{}) int64 {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	r, err := coll.DeleteMany(m.Ctx, query)
-	if err != nil {
-		return 0
-	}
-	return r.DeletedCount
-}
-
-//findbyid
-func (m *MongodbSim) FindById(c, id string) map[string]interface{} {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	r := coll.FindOne(m.Ctx, map[string]interface{}{"_id": StringTOBsonId(id)})
-	v := map[string]interface{}{}
-	r.Decode(&v)
-	return v
-}
-
-//findone
-func (m *MongodbSim) FindOne(c string, query map[string]interface{}) map[string]interface{} {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	r := coll.FindOne(m.Ctx, query)
-	v := map[string]interface{}{}
-	r.Decode(&v)
-	return v
-}
-
-//find
-func (m *MongodbSim) Find(c string, query map[string]interface{}, sort, fields interface{}) ([]map[string]interface{}, error) {
-	m.Open()
-	defer m.Close()
-	coll := m.C.Database(m.DbName).Collection(c)
-	op := options.Find()
-	r, err := coll.Find(m.Ctx, query, op.SetSort(sort), op.SetProjection(fields))
-	if err != nil {
-		log.Fatal(err)
-		return nil, err
-	}
-	var results []map[string]interface{}
-	if err = r.All(m.Ctx, &results); err != nil {
-		log.Fatal(err)
-		return nil, err
-	}
-	return results, nil
-}
-
-//创建_id
-func NewObjectId() primitive.ObjectID {
-	return primitive.NewObjectID()
-}
-
-func StringTOBsonId(id string) primitive.ObjectID {
-	objectId, _ := primitive.ObjectIDFromHex(id)
-	return objectId
-}
-
-func BsonTOStringId(id interface{}) string {
-	return id.(primitive.ObjectID).Hex()
-}