更新采购单位增量；添加特殊企业验证

createEsIndex/buyertask.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	util "app.yhyue.com/data_processing/common_utils"
 	"app.yhyue.com/data_processing/common_utils/log"
+	"context"
 	"esindex/config"
 	"fmt"
 	"go.uber.org/zap"
@@ -10,22 +12,46 @@ import (
 
 //buyerOnce  处理增量数据
 func buyerOnce() {
+	if len(specialNames) < 1 {
+		initSpecialNames()
+	}
+
 	rowsPerPage := 1000
-	currentPage := 1
-	total := 0
+	now := time.Now()
+	//tarTime := time.Date(now.Year(), now.Month(), now.Day()-7, 00, 00, 00, 00, time.Local)
+
+	curTime := now.Format("2006-01-02")
+
+	finalId := 0
+	lastSql := fmt.Sprintf(`
+	 SELECT
+              t.id
+
+          FROM
+              dws_f_ent_tags AS t
+              LEFT JOIN dws_f_ent_baseinfo AS b ON b.name_id = t.name_id
+
+          WHERE  t.createtime >= '%v' OR t.updatetime >= '%v'
+
+          ORDER BY t.id DESC  LIMIT 1
+`, curTime, curTime)
+
+	lastInfo := Mysql.SelectBySql(lastSql)
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+
+	log.Info("buyerOnce", zap.Int("finalId", finalId))
+	lastid, total := 0, 0
+	realCount := 0
+
 	for {
-		log.Info("buyerOnce", zap.Int("currentPage", currentPage))
-		arrEs := make([]map[string]interface{}, 0)
-		offset := (currentPage - 1) * rowsPerPage
-		//year, month, day := 2022, time.October, 01
-		//now := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
-		now := time.Now()
-		curTime := now.Format("2006-01-02")
 		query := fmt.Sprintf(`
-           SELECT
+ SELECT
                b.name, 
                t.id,
                t.name_id, 
+			   b.company_id,
                t.createtime,
                t.updatetime,
                c.area, 
@@ -38,151 +64,219 @@ func buyerOnce() {
                LEFT JOIN dws_f_ent_baseinfo AS b ON b.name_id = t.name_id
                LEFT JOIN code_area AS c ON b.city_code = c.code 
 
-			WHERE  t.createtime > '%v' OR t.updatetime > '%v'
-           ORDER BY t.createtime ASC
-           LIMIT %d, %d;
-       `, curTime, curTime, offset, rowsPerPage)
+			WHERE  t.createtime >= '%v' OR t.updatetime >= '%v'
+           ORDER BY t.id ASC
+          LIMIT %d;
+      `, curTime, curTime, rowsPerPage)
+
+		ctx := context.Background()
+		rows, err := Mysql.DB.QueryContext(ctx, query)
+		if err != nil {
+			log.Info("dealData", zap.Any("QueryContext err", err))
+		}
 
-		result := MysqlB.SelectBySql(query)
-		if result == nil {
+		if finalId == lastid {
+			log.Info("buyerOnce over", zap.Any("total", total), zap.Any("lastid", lastid))
 			break
 		}
 
-		if len(*result) > 0 {
-			for _, re := range *result {
-				tmp := make(map[string]interface{}, 0)
-				tmp["name"] = re["name"]
-				tmp["buyer_name"] = re["name"]
-				tmp["id"] = re["name_id"]
-				tmp["province"] = re["area"]
-				tmp["city"] = re["city"]
-				tmp["buyerclass"] = re["buyerclass"]
-
-				if re["createtime"] != nil {
-					if createtime, ok := re["createtime"].(time.Time); ok {
-						tmp["createtime"] = createtime.Unix()
-						if re["updatetime"] != nil {
-							if updatetime, ok := re["updatetime"].(time.Time); ok {
-								tmp["updatetime"] = updatetime.Unix()
+		columns, err := rows.Columns()
+		if err != nil {
+			log.Info("buyerOnce", zap.Any("rows.Columns", err))
+		}
+
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			//MySQL 更新
+			update := map[string]interface{}{}
+
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Info("buyerOnce", zap.Any("rows.Scan", err))
+				break
+			}
+
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+
+			total++
+			if total%100 == 0 {
+				log.Info("buyerOnce", zap.Int("current total", total))
+			}
+			lastid = util.IntAll(ret["id"])
+
+			name := util.ObjToString(ret["name"])
+			//company_id ！= ‘’ 或者在凭安 特殊企业或者在爬虫采购单位里
+			if util.ObjToString(ret["company_id"]) != "" || specialNames[name] {
+				realCount++
+				update["status"] = 1
+				data := make(map[string]interface{}, 0)
+				data["name"] = name
+				data["name_id"] = ret["name_id"]
+				data["id"] = ret["name_id"]
+				data["buyer_name"] = name
+				data["province"] = ret["area"]
+				data["city"] = ret["city"]
+				data["buyerclass"] = ret["buyerclass"]
+				if ret["createtime"] != nil {
+					if createtime, ok := ret["createtime"].(time.Time); ok {
+						data["createtime"] = createtime.Unix()
+						if ret["updatetime"] != nil {
+							if updatetime, ok := ret["updatetime"].(time.Time); ok {
+								data["updatetime"] = updatetime.Unix()
 							}
 						} else {
-							tmp["updatetime"] = createtime.Unix()
+							data["updatetime"] = createtime.Unix()
 						}
 					}
 				}
 
-				sql := fmt.Sprintf(`select * from dws_f_ent_contact where name_id = '%v'`, re["name_id"])
-
-				counts := MysqlB.SelectBySql(sql)
-				if len(*counts) > 0 {
-					tmp["is_contact"] = true
+				sql := fmt.Sprintf(`select count(id) from dws_f_ent_contact where name_id = '%v'`, ret["name_id"])
+				counts := Mysql.CountBySql(sql)
+				if counts > 0 {
+					data["is_contact"] = true
 				} else {
-					tmp["is_contact"] = false
+					data["is_contact"] = false
 				}
-				arrEs = append(arrEs, tmp)
-			}
-			total = total + len(arrEs)
 
-			err := Es.InsertOrUpdate(config.Conf.DB.Es.IndexBuyer, arrEs)
-			if err != nil {
-				log.Info("buyerOnce", zap.Any("InsertOrUpdate err", err))
+				//更新MySQL
+				where := map[string]interface{}{
+					"name_id": ret["name_id"],
+				}
+				if len(update) > 0 {
+					Mysql.Update("dws_f_ent_tags", where, update)
+				}
+
+				//生索引
+				arrEs := make([]map[string]interface{}, 0) //最终生索引数据
+				arrEs = append(arrEs, data)
+				err = Es.InsertOrUpdate(config.Conf.DB.Es.IndexBuyer, arrEs)
+				if err != nil {
+					log.Info("buyerOnce", zap.Any("InsertOrUpdate err", err))
+				}
+				log.Info("buyerOnce", zap.Any("ret ", ret))
 			}
+
 		}
 
-		if len(arrEs) < rowsPerPage {
-			break
+		rows.Close()
+
+		if err := rows.Err(); err != nil {
+			log.Info("buyerOnce", zap.Any("err", err))
 		}
-		// 继续查询下一页
-		currentPage++
+
 	}
 
-	log.Info("buyerOnce", zap.Int("结束，总数是：", total))
+	log.Info("buyerOnce", zap.Int("结束，总数是：", total), zap.Int("realCount", realCount))
 }
 
-//buyerall 全量数据
-func buyerall() {
-	rowsPerPage := 5000
-	currentPage := 1
-	total := 0
-
-	for {
-		log.Info("buyerall", zap.Int("currentPage", currentPage))
-		arrEs := make([]map[string]interface{}, 0)
-		offset := (currentPage - 1) * rowsPerPage
-		query := fmt.Sprintf(`
-            SELECT
-               b.name, 
-               t.id,
-               t.name_id, 
-               t.createtime,
-               t.updatetime,
-               c.area, 
-               c.city, 
-               class.name AS buyerclass
-              
-           FROM 
-               dws_f_ent_tags AS t 
-               LEFT JOIN code_buyerclass AS class ON class.code = t.labelvalues
-               LEFT JOIN dws_f_ent_baseinfo AS b ON b.name_id = t.name_id
-               LEFT JOIN code_area AS c ON b.city_code = c.code
+//dealSpecialNames 拿到凭安 特企数据 以及爬虫采购单位数据
+func initSpecialNames() {
+	sess := MgoS.GetMgoConn()
+	defer MgoS.DestoryMongoConn(sess)
 
-           ORDER BY t.createtime ASC
-			
-           LIMIT %d, %d;
-       `, offset, rowsPerPage)
+	query := sess.DB(MgoS.DbName).C("special_enterprise").Find(nil).Iter()
+	count := 0
+	log.Info("dealSpecialNames", zap.String("special_enterprise", "special_enterprise"))
 
-		result := MysqlB.SelectBySql(query)
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
+	}
 
-		if result == nil {
-			break
+	//2.
+	time.Sleep(time.Second)
+	log.Info("dealSpecialNames", zap.String("special_foundation", "special_foundation"))
+	query2 := sess.DB("mixdata").C("special_foundation").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query2.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
 		}
+	}
 
-		if len(*result) > 0 {
-			for _, re := range *result {
-				tmp := make(map[string]interface{}, 0)
-				tmp["name"] = re["name"]
-				tmp["buyer_name"] = re["name"]
-				tmp["id"] = re["name_id"]
-				tmp["_id"] = re["name_id"]
-				tmp["province"] = re["area"]
-				tmp["city"] = re["city"]
-				tmp["buyerclass"] = re["buyerclass"]
-				if re["createtime"] != nil {
-					if createtime, ok := re["createtime"].(time.Time); ok {
-						tmp["createtime"] = createtime.Unix()
-						if re["updatetime"] != nil {
-							if updatetime, ok := re["updatetime"].(time.Time); ok {
-								tmp["updatetime"] = updatetime.Unix()
-							}
-						} else {
-							tmp["updatetime"] = createtime.Unix()
-						}
-					}
-				}
+	//3.
+	time.Sleep(time.Second)
+	log.Info("dealSpecialNames", zap.String("special_gov_unit", "special_gov_unit"))
+	query3 := sess.DB("mixdata").C("special_gov_unit").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query3.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
+	}
 
-				sql := fmt.Sprintf(`select id from dws_f_ent_contact where name_id = '%v'`, re["name_id"])
+	//4.
+	time.Sleep(time.Second)
+	log.Info("dealSpecialNames", zap.String("special_hongkong_company", "special_hongkong_company"))
+	query4 := sess.DB("mixdata").C("special_hongkong_company").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query4.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
+	}
 
-				counts := MysqlB.SelectBySql(sql)
-				if len(*counts) > 0 {
-					tmp["is_contact"] = true
-				} else {
-					tmp["is_contact"] = false
-				}
-				arrEs = append(arrEs, tmp)
-			}
-			total = total + len(arrEs)
-			//保存es
-			Es.BulkSave(config.Conf.DB.Es.IndexBuyer, arrEs)
+	//5.
+	time.Sleep(time.Second)
+	log.Info("dealSpecialNames", zap.String("special_hongkong_company_history", "special_hongkong_company_history"))
+	query5 := sess.DB("mixdata").C("special_hongkong_company_history").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query5.Next(tmp); count++ {
+		if tmp["use_name"] != nil && tmp["use_name"] != "" {
+			specialNames[tmp["use_name"].(string)] = true
 		}
+	}
+	//6.
+	time.Sleep(time.Second)
+	log.Info("dealSpecialNames", zap.String("special_law_office", "special_law_office"))
+	query6 := sess.DB("mixdata").C("special_law_office").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query6.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
+	}
 
-		// 如果本次查询返回的数据不足每页请求的数量，说明已经查询到最后一页
-		if len(*result) < rowsPerPage {
-			break
+	//7.
+
+	log.Info("dealSpecialNames", zap.String("special_social_organ", "special_social_organ"))
+	query7 := sess.DB("mixdata").C("special_social_organ").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query7.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
 		}
+	}
 
-		// 继续查询下一页
-		currentPage++
+	log.Info("dealSpecialNames", zap.String("special_trade_union", "special_trade_union"))
+	//8.
+	query8 := sess.DB("mixdata").C("special_trade_union").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query8.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
+	}
+	//9. 爬虫采集的采购单位
+	query9 := sess.DB("mixdata").C("special_buyer_name").Find(nil).Iter()
+	count = 0
+	for tmp := make(map[string]interface{}); query9.Next(tmp); count++ {
+		if tmp["company_name"] != nil && tmp["company_name"] != "" {
+			specialNames[tmp["company_name"].(string)] = true
+		}
 	}
 
-	log.Info("buyerall", zap.Int("结束，总数是：", total))
 }

createEsIndex/common.toml

@@ -4,7 +4,7 @@
     jyport = 11118
 
 [db]
-[db.mongoB]
+[db.mongoB] ## bidding标讯数据
 #    addr = "127.0.0.1:27017"
     addr = "192.168.3.206:27002"    ## 测试环境
 #    dbname = "wcc"
@@ -14,7 +14,7 @@
     user = "root"
     password = "root"
 
-[db.mongoP]
+[db.mongoP] ## projectset 项目信息
     addr = "192.168.3.206:27002"
     dbname = "qfw_data"
     coll = "projectset"
@@ -30,7 +30,16 @@
     user = "root"
     password = "root"
 
-[db.mysqlB]  ## buyer 采购单位
+[db.mongoS] ##  181 特殊企业，采购单位验证
+    addr = "127.0.0.1:27001"
+#    addr = "172.17.4.181:27001"
+    dbname = "mixdata"
+    coll = ""
+    size = 15
+    user = ""
+    password = ""
+
+[db.mysqlB]  ## buyer 采购单位 MySQL配置
     addr = "192.168.3.14:4000"      ## 测试环境
     dbname = "global_common_data"
     username = "root"

createEsIndex/config/conf.go

@@ -63,6 +63,7 @@ type db struct {
 	MongoB mgo
 	MongoP mgo
 	MongoQ mgo
+	MongoS mgo
 	Es     es
 	Oss    oss
 	MysqlB mysql

createEsIndex/es_test.go

@@ -151,7 +151,7 @@ func TestBuyer(t *testing.T) {
             LIMIT %d, %d;
         `, offset, rowsPerPage)
 
-		result := MysqlB.SelectBySql(query)
+		result := Mysql.SelectBySql(query)
 
 		if len(*result) > 0 {
 			for _, re := range *result {
@@ -172,3 +172,25 @@ func TestBuyer(t *testing.T) {
 	fmt.Println("over --------")
 	fmt.Println("total --------", total)
 }
+
+func TestIsHanStart(t *testing.T) {
+	name := "\\u001c 陈巴尔虎旗天顺矿业有限责任公司"
+	fmt.Println("aa", IsCompanyName(name))
+
+	//fmt.Println("uni", IsUnicodeStart(name))
+	fmt.Println("name", getCompanyName(name))
+	name = "RT农业发展（乌鲁木齐）有限责任公司"
+	//fmt.Println("uni", IsUnicodeStart(name))
+
+	fmt.Println("bb", IsCompanyName(name))
+	fmt.Println("name", getCompanyName(name))
+	name = "(宁波)综命能源服务有限公司"
+	fmt.Println("name", getCompanyName(name))
+	//fmt.Println("uni", IsUnicodeStart(name))
+	fmt.Println("cc", IsCompanyName(name))
+
+	a := 15
+	b := 2
+	fmt.Println((a / b) + 1)
+
+}

createEsIndex/init.go

@@ -76,15 +76,26 @@ func InitMgo() {
 	}
 	MgoQ.InitPool()
 	log.Info("InitMgo", zap.Any("MgoQ duration", time.Since(now).Seconds()))
-	MysqlB = &mysqldb.Mysql{
+	Mysql = &mysqldb.Mysql{
 		Address:  config.Conf.DB.MysqlB.Addr,
 		DBName:   config.Conf.DB.MysqlB.Dbname,
 		UserName: config.Conf.DB.MysqlB.Username,
 		PassWord: config.Conf.DB.MysqlB.Password,
 	}
-	MysqlB.Init()
+	Mysql.Init()
 
 	log.Info("InitMgo", zap.Any("MysqlB duration", time.Since(now).Seconds()))
+
+	MgoS = &mongodb.MongodbSim{
+		MongodbAddr: config.Conf.DB.MongoS.Addr,
+		DbName:      config.Conf.DB.MongoS.Dbname,
+		Size:        config.Conf.DB.MongoS.Size,
+		UserName:    config.Conf.DB.MongoS.User,
+		Password:    config.Conf.DB.MongoS.Password,
+	}
+	MgoS.InitPool()
+
+	log.Info("InitMgo", zap.Any("MgoS duration", time.Since(now).Seconds()))
 }
 
 func InitEs() {

createEsIndex/main.go

@@ -21,10 +21,11 @@ import (
 )
 
 var (
-	MgoB   *mongodb.MongodbSim
-	MgoP   *mongodb.MongodbSim
-	MgoQ   *mongodb.MongodbSim
-	MysqlB *mysqldb.Mysql
+	MgoB  *mongodb.MongodbSim
+	MgoP  *mongodb.MongodbSim
+	MgoQ  *mongodb.MongodbSim
+	MgoS  *mongodb.MongodbSim
+	Mysql *mysqldb.Mysql
 
 	Es, Es1 *elastic.Elastic
 
@@ -48,7 +49,7 @@ var (
 	//detailLength = 50000 // es保存detail长度
 	fileLength = 200000 // es保存附件文本长度,大于20万 时做一个日志记录
 	//pscopeLength = 32766 // projectscope长度
-
+	specialNames = make(map[string]bool, 0) //存储181 凭安提供的特企,爬虫采购单位
 )
 
 func init() {
@@ -180,13 +181,13 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 					}()
 					biddingDelByExtracttype(data, mapInfo)
 				}()
-			case "buyer_all": //buyer 全量数据
+			case "buyer_once": // 采购单位昨天增量数据
 				pool <- true
 				go func() {
 					defer func() {
 						<-pool
 					}()
-					buyerall()
+					buyerOnce()
 				}()
 			default:
 				pool <- true

createEsIndex/utils.go

@@ -1,9 +1,17 @@
 package main
 
 import (
+	"app.yhyue.com/data_processing/common_utils/log"
 	"app.yhyue.com/data_processing/common_utils/mongodb"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.uber.org/zap"
 	"math"
+	"regexp"
 	"sort"
+	"strings"
+	"unicode"
+	"unicode/utf8"
 )
 
 //convertToMongoID convertToMongoID
@@ -91,3 +99,261 @@ func chargeType(ftype, etype string) bool {
 
 	return false
 }
+
+//IsHanStart 判断字符串是否以汉字开头或者以字母开头
+func IsHanStart(s string) bool {
+	if s == "" {
+		return false
+	}
+
+	return unicode.Is(unicode.Scripts["Han"], []rune(s)[0]) || unicode.IsLetter([]rune(s)[0])
+}
+
+// 判断字符是否为汉字
+func IsHan(c rune) bool {
+	return unicode.Is(unicode.Scripts["Han"], c)
+}
+
+// 判断公司名称是否以圆括号开头且括号内汉字开头
+func IsBracketStartWithHan(s string) bool {
+	if len(s) == 0 || s[0] != '(' {
+		return false
+	}
+
+	// 索引 i 和 j 分别是左右圆括号的位置，如果找不到右圆括号则返回 false
+	i, j := 0, 0
+	for j = i + 1; j < len(s); j++ {
+		if s[j] == ')' {
+			break
+		}
+	}
+	if j >= len(s) {
+		return false
+	}
+
+	// 检查圆括号内是否以汉字或字母开头
+	bracketContent := s[i+1 : j]
+	if len(bracketContent) == 0 || (!unicode.IsLetter(rune(bracketContent[0])) && !IsHan([]rune(bracketContent)[0])) {
+		return false
+	}
+
+	return true
+}
+
+//IsCompanyName 判断字符串是否以汉字开头、以括号开头并且括号里面是汉字、以"公司"结尾，其中一个条件符合即返回true，否则返回false
+func IsCompanyName(s string) bool {
+	r := []rune(s)
+	//if len(r) >= 6 && (string(r[len(r)-6:]) == "有限公司" || string(r[len(r)-6:]) == "股份有限公司") {
+	//	return (IsHanStart(s) || IsBracketStartWithHan(s))
+	//} else if len(r) >= 2 && string(r[len(r)-2:]) == "公司" {
+	//	return (IsHanStart(s) || IsBracketStartWithHan(s))
+	//}
+	if len(r) > 2 {
+		return (IsHanStart(s) || IsBracketStartWithHan(s))
+	}
+	return false
+}
+
+//GetChineseCharacters 提取字符串中的汉字
+func GetChineseCharacters(s string) string {
+	re := regexp.MustCompile(`[\p{Han}]+`)
+	return re.FindString(s)
+}
+
+func getCompanyName(name string) string {
+
+	if IsCompanyName(name) {
+		return name
+	}
+
+	return GetChineseCharacters(name)
+
+}
+
+func IsUnicodeStart(s string) bool {
+	if len(s) == 0 {
+		return false
+	}
+	_, size := utf8.DecodeRuneInString(s)
+	return size > 0
+}
+
+//RemoveDuplicateSuffix 去除字符串末尾的重复字词
+func RemoveDuplicateSuffix(str string, suffix string) string {
+	// 构建正则表达式：^(.*?)(重复的结尾词)+$
+	re := regexp.MustCompile(fmt.Sprintf(`^(.*?)(%s)+$`, suffix))
+	matches := re.FindStringSubmatch(str)
+	if len(matches) == 3 {
+		return matches[1] + matches[2]
+	}
+	return str
+}
+
+func findName(name string) []map[string]interface{} {
+	filter := bson.M{"name": name, "status": 1}
+	info, _ := MgoB.Find("wcc_buyer", filter, nil, nil, false, -1, -1)
+
+	return *info
+}
+
+func findNameID(id string) []map[string]interface{} {
+	filter := bson.M{"name_id": id, "status": 1}
+	info, _ := MgoB.Find("wcc_buyer", filter, nil, nil, false, -1, -1)
+
+	return *info
+}
+
+//isStringRepeating 判断字符串内字符完全重复，例如：山东大学山东大学
+func isStringRepeating(str string) bool {
+	for i := 0; i < len(str); i++ {
+		for j := i + 1; j < len(str); j++ {
+			if str[i] != str[j] {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+//IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}
+
+//checkName 检查名称，不符合的返回false
+func checkName(name string) (res bool) {
+	res = true
+
+	faleNames := []string{"管理有限公司", "有限公司", "技术公司", "制水公司", "工程有限公司", "耐材公司", "检测公司", "某公司", "现对公司", "集装箱码头分公司", "质检分公司", "司公司", "供电分公司", "建设公司", "水城分公司", "工信公司", "棉纺公司", "安装公司", "管公司", "测试公司", "四公司", "电力分公司",
+		"原平分公司", "兰州分公司", "基础设施公司", "陆丰分公司", "郴州分公司", "大渡口分公司", "汽车四分公司", "州大学", "开有限公司", "中小学幼儿园", "上级公司下级公司", "楼及幼儿园", "冷询有限公司", "市第一医院", "发展有阶公司", "楼及号地幼儿园", "第一幼儿园", "楼急救中心", "住宅楼及局", "号国际钻井公司", "管理委员会",
+		"区地块幼儿园", "省人民政府", "具有国家铁路局", "住宅楼及幼儿园", "丿再生资源技术有限公司", "为保证医院", "网及灾备中心", "地块幼儿园", "求石油昆仑燃气限公司", "栋楼及幼儿园", "号固井压裂公司", "号吐哈钻井公司", "股为限公司", "住宅楼及社区服务中心", "井与分公司", "场监督管局", "为了保证本次政府",
+		"人民政府东环街遂办事处", "龙湖区分公司", "澄海侨史馆", "中国移动科", "中国移动学院", "中国移动分公司"}
+	for _, v := range faleNames {
+		if name == v {
+			res = false
+			break
+		}
+	}
+
+	if len([]rune(name)) < 6 {
+		if strings.HasSuffix(name, "公司") {
+			res = false
+		} else if strings.HasSuffix(name, "办事处") {
+			res = false
+		} else if strings.HasSuffix(name, "分公司") {
+			res = false
+		} else if strings.HasSuffix(name, "管委会") {
+			res = false
+		} else if strings.HasSuffix(name, "支行") {
+			res = false
+		} else if strings.HasSuffix(name, "网络部") {
+			res = false
+		} else if strings.HasSuffix(name, "部队") {
+			res = false
+		} else if strings.HasSuffix(name, "医科大学") {
+			res = false
+		}
+	}
+
+	if len([]rune(name)) < 7 && strings.HasSuffix(name, "中医药大学") {
+		res = false
+	}
+
+	if !TDName(name) {
+		res = false
+	}
+
+	return
+}
+
+//TDName 根据开头、结尾、排除词，筛选名称，筛选不符合的数据
+func TDName(name string) (res bool) {
+	res = true
+
+	if isStringRepeating(name) { //重复数据,山东大学山东大学
+		res = false
+		log.Info("TDName", zap.String("isStringRepeating", name))
+		return res
+	}
+
+	countWords := []string{"教育局", "学校", "大学", "中学", "审计局", "郑州局", "中医院", "部队", "有限公司"}
+	//教育局出现二次
+	for _, c := range countWords {
+		if strings.Count(name, c) > 1 {
+			res = false
+			log.Info("TDName", zap.String("countWords", c), zap.String("name", name))
+			return res
+		}
+	}
+
+	//公司出现三次，错误
+	if strings.Count(name, "公司") > 2 {
+		res = false
+		log.Info("TDName", zap.String("公司", "公司出现三次"))
+		return res
+	}
+
+	//公司出现 二次，但是不是分公司
+	if strings.Count(name, "公司") > 1 {
+		if !strings.Contains(name, "分公司") && !strings.Contains(name, "子公司") && !strings.Contains(name, "市公司") {
+			res = false
+			log.Info("TDName", zap.String("公司", "公司出现二次，但没有分公司, 子公司"))
+			return res
+		}
+	}
+
+	//开头的单词
+	preWords := []string{"取消", "转发", "省", "会议室", "某", "某单位", "某公司", "某学校", "测试", "某大学", "某医院", "拟建", "机械", "县", "市", "区", "巡察", "人民", "对", "楼", "家长", "春季", "于", "度", "初级中学", "州", "总部", "一号", "含", "育场", "选择", "第包", "医院",
+		"共赢资产", "从", "实验中学", "公共", "乘用车", "部与", "一直", "一体化", "现对", "现为", "之", "份", "半坡", "下属", "程", "信息", "带", "工务", "技术", "于", "号", "辖属", "造", "计财", "实验", "原", "器械", "是", "师", "雨山区", "云城区", "阜涣", "公联", "批",
+		"总承包", "姓市", "秋季学期", "哈巴河", "保险", "开发区", "尘设资", "团", "物业", "农村", "部", "东省", "于", "章", "董", "分局", "再", "微", "明", "建", "公司", "审计", "满足", "中学", "届", "大学", "按摩", "中心", "附属", "总", "教育局", "中学", "小学", "学院",
+		"中医院", "人医院", "学校", "输", "十六", "有限", "节能", "二公司", "分公司", "子公司", "公司", "审计", "河河", "地公司", "六公司", "三公司", "十六公司", "节能",
+		"股份", "运输公司", "堇", "家纺", "港区"}
+	// 开头
+	for _, word := range preWords {
+		if strings.HasPrefix(name, word) {
+			res = false
+			log.Info("TDName", zap.String("preWords", word), zap.String("name", name))
+			return res
+		}
+	}
+
+	//结尾单词
+	sufWords := []string{"市", "某单位", "某公司", "某学校", "测试", "某大学", "某医院", "地产鲨", "项目组", "研究", "公国司", "物资", "评估与医院", "年室", "招标", "设有", "复限公司", "服务", "有限责任", "罩套", "有阴公", "人队", "报告", "厕所", "污水处", "检查", "镇中心",
+		"显微镜", "段", "现场局", "租赁", "镇", "市中心", "勐阿糖厂", "展厅", "等医院", "广场室", "水利顷", "市室", "交界处", "自助银行", "若源局", "布局", "联合体", "气室", "活动场", "包装公司", "投资", "有限", "等", "设备", "急件", "有限公司有限公司", "公司公司", "分公司分公司",
+		"大学大学", "学院学院", "大学场", "学校室", "部队部队", "内部", "部分支公司"}
+
+	//结尾
+	for _, word := range sufWords {
+		if strings.HasSuffix(name, word) {
+			res = false
+			log.Info("TDName", zap.String("sufWords", word), zap.String("name", name))
+			return res
+		}
+	}
+
+	//包含的单词
+	falseWords := []string{"丿", "艹", "[ ", "【", "?", "亻", "#", "~", "^", "亻", "*", "$", "、", "/", "*", "<>", "[ ", "【", "?", "某", "病历系统", "不予", "薯公", "标段", "电压互感器", "我",
+		"省道", "询比", "学校和幼儿园", "住宅", "询价", "办公区", "项目", "中旎髓细兴", "办么", "测试", "项目部", "飞地园", "购", "鉴定", "实验室", "车务段", "可测性设计", "手术室", "第二期", "退", "催促",
+		"协议中", "影响", "工作", "动力厂", "干扰源", "指定", "达到", "控制器", "竞价", "人哥", "络络络", "路段", "示范区", "一般用", "采供", "修缮", "招标单位", "联系方式", "沟通", "配线架", "及配建",
+		"任务", "分包", "其他", "行为", "中标人", "教室", "安置", "暂行", "建设单位", "选聘", "合同", "转发", "勿救", "既有", "装置", "任务", "开展", "依据", "指定", "试剂", "景观", "包件", "法人", "合格",
+		"短管", "投标文件", "核实", "测汞仪", "黑名单", "制作类", "汇编", "收购", "拿", "丿", "艹", "关于", "要求", "举办", "显示所", "右阴公司", "发射", "加装", "复印纸", "终止", "生产处", "提供", "新建",
+		"该项目", "安装", "造价", "改造", "样品", "课室", "水毁道路", "闲置资产", "#", "南校园", "扩建", "道路维修", "等所", "维稳中心", "异议", "音乐厅", "慰问", "介入", "改建", "即", "建设", "某单位",
+		"医院医院", "中国电建中国电建", "建设建设", "建设规模", "满足", "郑州郑州", "学校学校", "&", ";", "；", "就医院", "酒医院", "至医院", "对", "所需", "提升", "提高", "学期", "学年", "规模", "必须",
+		"建成", "各公司", "公司与公司", "公司的公司", "公司办公司", "养护"}
+
+	// 包含
+	for _, word := range falseWords {
+		if strings.Contains(name, word) {
+			res = false
+			log.Info("TDName", zap.String("falseWords", word), zap.String("name", name))
+			return res
+		}
+	}
+
+	return
+}