标准分

udpdataclear/udpSensitiveWords/main.go

@@ -5,13 +5,11 @@ import (
 )
 
 func init() {
-	//return
+
 	util.InitC()
 }
 func main() {
 
-	//util.TemporaryTestNewData()
-	//return
 	go util.AddTaskSensitiveWordsData() //增量
 
 	// 主函数中添加

udpdataclear/udpSensitiveWords/util/udpdata.go

@@ -8,10 +8,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"gopkg.in/olivere/elastic.v1"
 	"log"
 	"net"
-	"net/http"
 	"regexp"
 	"strings"
 	"sync"
@@ -336,121 +334,15 @@ func dealWithEsData(name string, tmpid string) {
 
 }
 
-func TemporaryTest() {
-	log.Println("测试......导出数据")
 
 
-	QfwMgo85 = &MongodbSim{
-		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
-		Size:        5,
-		DbName:      "mixdata",
-		UserName:    "fengweiqiang",
-		PassWord:    "fwq@123123",
-	}
-	QfwMgo85.InitPool()
-
-	Client_Es, _ = elastic.NewClient(http.DefaultClient, "http://ela.spdata.jianyu360.com")
-
-	es_type, es_index = "unique_qy", "unique_qy"
-
-	q := map[string]interface{}{
-		"check_history": map[string]interface{}{
-			"$exists": 0,
-		},
-	}
-	sess := QfwMgo85.GetMgoConn()
-	defer QfwMgo85.DestoryMongoConn(sess)
-	//多线程升索引
-	pool_es := make(chan bool, 3)
-	wg_es := &sync.WaitGroup{}
-	it := sess.DB(QfwMgo85.DbName).C("winner_err_new").Find(&q).Iter()
-	total, isOK := 0, 0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%100 == 0 {
-			log.Println("current index", total, isOK)
-		}
-		name := ObjToString(tmp["name"])
-		tmpid := BsonTOStringId(tmp["_id"])
-		pool_es <- true
-		wg_es.Add(1)
-
-		go func(name string, tmpid string) {
-			defer func() {
-				<-pool_es
-				wg_es.Done()
-			}()
-			//start := int(time.Now().Unix())
-			new_name,b,score,_ :=dealWithNameScoreRules(name)
-			//log.Println("耗时：",int(time.Now().Unix())-start,"秒",b,name,new_name,tmpid)
-			if new_name != "" && b {
-				isOK++
-				QfwMgo85.UpdateById("winner_err_new", tmpid, map[string]interface{}{
-					"$set": map[string]interface{}{
-						"is_word": 1,
-						"name_word" : new_name,
-						"score":score,
-
-					},
-				})
-			} else {
-				QfwMgo85.UpdateById("winner_err_new", tmpid, map[string]interface{}{
-					"$set": map[string]interface{}{
-
-						"is_word": -1,
-						"name_word" : new_name,
-						"score":score,
-
-					},
-				})
-			}
 
-		}(name, tmpid)
-		tmp = make(map[string]interface{})
-	}
 
-	wg_es.Wait()
 
-	log.Println("is over", total, isOK)
-}
 
 
-func TemporaryTestNewData()  {
-	log.Println("测试......导出新数据")
-	QfwMgo85 = &MongodbSim{
-		MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
-		Size:        5,
-		DbName:      "mixdata",
-		UserName:    "fengweiqiang",
-		PassWord:    "fwq@123123",
-	}
-	QfwMgo85.InitPool()
 
-	q := map[string]interface{}{
-		"is_word":map[string]interface{}{
-			"$exists":1,
-		},
-	}
-	sess := QfwMgo85.GetMgoConn()
-	defer QfwMgo85.DestoryMongoConn(sess)
-	it := sess.DB(QfwMgo85.DbName).C("winner_err_new").Find(&q).Select(map[string]interface{}{
-		"name":1,
-		"name_word":1,
-		"is_word":1,
-	}).Iter()
-	total:=0
-	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total % 100 == 0 {
-			log.Println("current index",total)
-		}
-		savedict := tmp
-		QfwMgo85.Save("winner_err_new_word",savedict)
-
-		tmp = make(map[string]interface{})
-	}
 
-	log.Println("is over",total)
-
-}
 
 
 

udpdataclear/udpSensitiveWords/util/words.go

@@ -15,12 +15,18 @@ func dealWithNameScoreRules(name string) (string, bool, float64 , []map[string]i
 		return "", false, new_score ,nil
 	}
 
+	//标准分：
+	standard_score := float64(4.0)
 	query_name := old_name
 	endstr := endWordReg.FindString(query_name)
 	if endstr !="" {
+		standard_score = 3.0
 		query_name = strings.ReplaceAll(query_name,endstr,"")
 	}
 
+
+
+
 	query := `{"query":{"bool":{"must":[{"query_string":{"default_field":"unique_qy.name_word","query":"` + query_name + `"}}],"must_not":[],"should":[]}},"from":"0","size":"200"}`
 	tmp := make(map[string]interface{})
 	json.Unmarshal([]byte(query), &tmp)
@@ -58,7 +64,7 @@ func dealWithNameScoreRules(name string) (string, bool, float64 , []map[string]i
 			if proportion >= 1.0 {
 				isok = true
 			} else {
-				if float64(hit)/float64(total) >= 0.8 && new_score > 4.0 {
+				if float64(hit)/float64(total) >= 0.8 && new_score > standard_score {
 
 					str1,str2:=startWordReg_1.FindString(name),startWordReg_1.FindString(new_name)
 					if str1!="" && str2!="" {
@@ -75,7 +81,7 @@ func dealWithNameScoreRules(name string) (string, bool, float64 , []map[string]i
 						}
 					}
 					isok = true
-				}else if new_score > 4.0 {
+				}else if new_score > standard_score {
 					str1,str2:=name,new_name
 					str1 = strings.ReplaceAll(str1,"责任","")
 					str2 = strings.ReplaceAll(str2,"责任","")