修改-分数

udpdataclear/udpSensitiveWords/main.go

@@ -5,12 +5,14 @@ import (
 )
 
 func init() {
-
+	//return
 	util.InitC()
 }
 func main() {
 
-	//go util.AddTaskSensitiveWordsData() //增量
+	//util.TemporaryTestNewData()
+	//return
+	go util.AddTaskSensitiveWordsData() //增量
 
 	// 主函数中添加
 	util.ExtractUdp() //udp通知抽取

udpdataclear/udpSensitiveWords/util/udpdata.go

@@ -105,7 +105,7 @@ func cheakname(name string) (up bool, log, rname string) {
 		return true, cheaklog, name
 	}
 
-	rname, isok, datas := dealWithNameScoreRules(name)
+	rname, isok, _ ,datas := dealWithNameScoreRules(name)
 	if len(datas) > 0 {
 		for _, v := range datas {
 			filter.AddWord(v["name"].(string))
@@ -280,12 +280,17 @@ func AddTaskSensitiveWordsData() {
 					}
 
 					//存mgo
-					con.Database("mixdata").Collection("unique_qyxy").InsertOne(nil, bson.M{
+					new_tmp ,err:= con.Database("mixdata").Collection("unique_qyxy").InsertOne(nil, bson.M{
 						"qy_name": company_name,
 					})
+					if err==nil {
+						dealWithEsData(company_name, BsonTOStringId(new_tmp.InsertedID))
+					}
 					//存敏感词
+
+
 					//存es=判断+新增
-					dealWithEsData(company_name, BsonTOStringId(tmp["_id"]))
+
 				}
 			}
 		}
@@ -331,6 +336,7 @@ func dealWithEsData(name string, tmpid string) {
 func TemporaryTest()  {
 	log.Println("测试......导出数据")
 
+
 	QfwMgo85 = &MongodbSim{
 		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
 		Size:        5,
@@ -362,6 +368,10 @@ func TemporaryTest()  {
 			log.Println("current index",total,isOK)
 		}
 
+		if total<91400 {
+			tmp = make(map[string]interface{})
+			continue
+		}
 		name:=ObjToString(tmp["name"])
 		tmpid := BsonTOStringId(tmp["_id"])
 		pool_es <- true
@@ -373,7 +383,7 @@ func TemporaryTest()  {
 				wg_es.Done()
 			}()
 			//start := int(time.Now().Unix())
-			new_name,b,_ :=dealWithNameScoreRules(name)
+			new_name,b,score,_ :=dealWithNameScoreRules(name)
 			//log.Println("耗时：",int(time.Now().Unix())-start,"秒",b,name,new_name,tmpid)
 			if new_name!="" && b {
 				isOK++
@@ -381,6 +391,7 @@ func TemporaryTest()  {
 					"$set": map[string]interface{}{
 						"is_word": 1,
 						"name_word" : new_name,
+						"score":score,
 					},
 				})
 			}else {
@@ -388,6 +399,7 @@ func TemporaryTest()  {
 					"$set": map[string]interface{}{
 						"is_word": -1,
 						"name_word" : new_name,
+						"score":score,
 					},
 				})
 			}
@@ -401,6 +413,44 @@ func TemporaryTest()  {
 	log.Println("is over",total,isOK)
 }
 
+func TemporaryTestNewData()  {
+	log.Println("测试......导出新数据")
+	QfwMgo85 = &MongodbSim{
+		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
+		Size:        5,
+		DbName:      "mixdata",
+		UserName:    "fengweiqiang",
+		PassWord:    "fwq@123123",
+	}
+	QfwMgo85.InitPool()
+
+	q := map[string]interface{}{
+		"is_word":map[string]interface{}{
+			"$exists":1,
+		},
+	}
+	sess := QfwMgo85.GetMgoConn()
+	defer QfwMgo85.DestoryMongoConn(sess)
+	it := sess.DB(QfwMgo85.DbName).C("winner_err_new").Find(&q).Select(map[string]interface{}{
+		"name":1,
+		"name_word":1,
+		"is_word":1,
+	}).Iter()
+	total:=0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total % 100 == 0 {
+			log.Println("current index",total)
+		}
+		savedict := tmp
+		QfwMgo85.Save("winner_err_new_word",savedict)
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Println("is over",total)
+
+}
+
 
 
 

udpdataclear/udpSensitiveWords/util/words.go

@@ -8,11 +8,11 @@ import (
 	"unicode/utf8"
 )
 
-func dealWithNameScoreRules(name string) (string, bool, []map[string]interface{}) {
+func dealWithNameScoreRules(name string) (string, bool, float64 , []map[string]interface{}) {
 	new_name, new_score, isok := "", float64(0), false
 	old_name := escapeNew(name)
 	if old_name == "" {
-		return "", false, nil
+		return "", false, new_score ,nil
 	}
 
 	query_name := old_name
@@ -27,7 +27,7 @@ func dealWithNameScoreRules(name string) (string, bool, []map[string]interface{}
 	searchResult, err := Client_Es.Search().Index(es_index).Type(es_type).Source(tmp).Do()
 	if err != nil {
 		log.Println("ES查询出错", name, old_name,err)
-		return "", false, nil
+		return "", false,new_score, nil
 	}
 	if searchResult.Hits!= nil{
 		resNum := len(searchResult.Hits.Hits)
@@ -65,13 +65,13 @@ func dealWithNameScoreRules(name string) (string, bool, []map[string]interface{}
 						if strings.Contains(str1,str2)||strings.Contains(str2,str1) {
 
 						}else {
-							return new_name, false, res
+							return new_name, false,new_score, res
 						}
 					}
 					str1,str2 = startWordReg_2.FindString(name),startWordReg_2.FindString(new_name)
 					if str1!="" && str2!=""{
 						if str1 != str2 {
-							return new_name, false, res
+							return new_name, false,new_score, res
 						}
 					}
 					isok = true
@@ -89,7 +89,7 @@ func dealWithNameScoreRules(name string) (string, bool, []map[string]interface{}
 					str1 = strings.ReplaceAll(str1,"工程","")
 					str2 = strings.ReplaceAll(str2,"工程","")
 					if str1==str2 {
-						return new_name, true, res
+						return new_name, true,new_score, res
 					}
 
 				}else {
@@ -97,9 +97,9 @@ func dealWithNameScoreRules(name string) (string, bool, []map[string]interface{}
 				}
 			}
 		}
-		return new_name, isok, res
+		return new_name, isok,new_score, res
 	}
-	return new_name,isok,nil
+	return new_name,isok,new_score,nil
 }
 
 //击中数量以及比例