敏感词sensitive过滤

src/jy/extract/exportask.go

@@ -59,8 +59,8 @@ func extractAndExport(v string, t map[string]interface{}) {
 	limit := qu.IntAll(t["limit"])
 	list, _ := e.TaskInfo.FDB.Find(e.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 	for _, v := range *list {
-		if v["check_sensitive2"] != nil { //去除含敏感词数据
-			if v["check_sensitive2"].(string) != "" {
+		if v["sensitive"] != nil { //去除含敏感词数据
+			if v["sensitive"].(string) != "" {
 				continue
 			}
 		}

src/jy/extract/extract.go

@@ -26,7 +26,7 @@ var (
 	TaskList  map[string]*ExtractTask                //任务列表
 	saveLimit = 200                                  //抽取日志批量保存
 	PageSize  = 5000                                 //查询分页
-	Fields    = `{"title":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"check_sensitive2":1}`
+	Fields    = `{"title":1,"detail":1,"contenthtml":1,"site":1,"spidercode":1,"toptype":1,"subtype":1,"area":1,"city":1,"comeintime":1,"publishtime":1,"sensitive":1}`
 )
 
 //启动测试抽取
@@ -148,8 +148,8 @@ func RunExtractTask(taskId string) {
 		log.Printf("page=%d,query=%v", i+1, query)
 		list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 		for _, v := range *list {
-			if v["check_sensitive2"] != nil { //去除含敏感词数据
-				if v["check_sensitive2"].(string) != "" {
+			if v["sensitive"] != nil { //去除含敏感词数据
+				if v["sensitive"].(string) != "" {
 					continue
 				}
 			}

src/jy/extract/extractudp.go

@@ -136,8 +136,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl, query) > 0 {
 				list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 				for _, v := range *list {
-					if v["check_sensitive2"] != nil { //去除含敏感词数据
-						if v["check_sensitive2"].(string) != "" {
+					if v["sensitive"] != nil { //去除含敏感词数据
+						if v["sensitive"].(string) != "" {
 							continue
 						}
 					}
@@ -157,8 +157,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 			if ext.TaskInfo.FDB.Count(ext.TaskInfo.FromColl+"_back", queryback) > 0 {
 				list2, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl+"_back", queryback, nil, Fields, false, 0, limit)
 				for _, v := range *list2 {
-					if v["check_sensitive2"] != nil { //去除含敏感词数据
-						if v["check_sensitive2"].(string) != "" {
+					if v["sensitive"] != nil { //去除含敏感词数据
+						if v["sensitive"].(string) != "" {
 							continue
 						}
 					}
@@ -192,8 +192,8 @@ func ExtractByUdp(sid, eid string, instanceId ...string) {
 			log.Printf("page=%d,query=%v", i+1, query)
 			list, _ := ext.TaskInfo.FDB.Find(ext.TaskInfo.FromColl, query, nil, Fields, false, 0, limit)
 			for _, v := range *list {
-				if v["check_sensitive2"] != nil { //去除含敏感词数据
-					if v["check_sensitive2"].(string) != "" {
+				if v["sensitive"] != nil { //去除含敏感词数据
+					if v["sensitive"].(string) != "" {
 						continue
 					}
 				}