save - es - 准备

udpdataclear/udpSensitiveWords/grpc_server/data.go

@@ -10,6 +10,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"encoding/json"
 )
 
 var reg_alias = regexp.MustCompile("(税务局|工商行政管理局|文化广播电视新闻出版局|外国专家局|" +
@@ -44,7 +45,7 @@ var con_strReg *regexp.Regexp = regexp.MustCompile("(\\?|？|%|代码标识|删
 var uncon_strReg *regexp.Regexp = regexp.MustCompile("(园|政府|集团|公司|有限|合伙|企|院|学|局|处)")
 
 
-
+//部署-历史-敏感词库
 func initSensitiveWordsData()  {
 	log.Println("初始化敏感词-源数据...")
 	gteid, err := primitive.ObjectIDFromHex(YamlConfig.TaskGteId)
@@ -116,27 +117,47 @@ func addTaskSensitiveWordsData()  {
 						strings.Contains(ObjToString(tmp["company_type_old"]),"个体") {
 						continue
 					}
-
 					//存mgo
 					con.Database("mixdata").Collection("unique_qyxy").InsertOne(nil, bson.M{
 						"qy_name": company_name,
 					})
 					//存敏感词
-					Filter.AddWord(tmp["qy_name"].(string))
+					Filter.AddWord(company_name)
 					//存es=判断+新增
-
-
-
-
-
+					dealWithEsData(company_name,BsonTOStringId(tmp["_id"]))
 				}
 			}
 		}
 		log.Println("tick ok", cronData)
 	}
 }
+//处理是否新增es
+func dealWithEsData(name string,tmpid string)  {
+	query:= `{"query":{"bool":{"must":[{"term":{"`+es_index+`.name":"`+name+`"}}],"must_not":[],"should":[]}},"from":0,"size":10,"sort":[],"facets":{}}`
+	tmp := make(map[string]interface{})
+	json.Unmarshal([]byte(query),&tmp)
+	searchResult, err := Client_Es.Search().Index(es_index).Type(es_type).Source(tmp).Do()
+	if err != nil {
+		log.Println("从ES查询出错", err.Error())
+	}else {
+		data := make(map[string]interface{},0)
+		if searchResult.Hits != nil {
+			for _, hit := range searchResult.Hits.Hits {
+				json.Unmarshal(*hit.Source, &data)
+			}
+		}
+		if len(data)==0{
+			log.Println("无数据-新增")
+			_, err := Client_Es.Index().Index(es_index).Type(es_type).Id(tmpid).BodyJson(map[string]interface{}{
+				"":"",
+			}).Do()
+			if err != nil {
+				log.Println("新增失败：",name,tmpid)
+			}
+		}
+	}
 
-
+}
 
 
 

udpdataclear/udpSensitiveWords/grpc_server/main.go

@@ -57,13 +57,13 @@ func init() {
 
 
 func main() {
-
+	
 	//淡赌跑断
 
 	if YamlConfig.IsAddTask==0{
 		initSensitiveWordsData() //初始化敏感词数据
 	}else {
-		go addTaskSensitiveWordsData() //增量
+		go addTaskSensitiveWordsData() //增量-改配置文件
 	}
 
 
@@ -77,6 +77,7 @@ func main() {
 	if err := s.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %v", err)
 	}
+
 }