添加竞品 网站 过滤

bidding_listen/config.go

@@ -1,11 +1,12 @@
 package main
 
 type GlobalConf struct {
-	Mongob MgoConf
-	Cron   CronConf
-	Mongop MgoConf
-	Mysql  MysqlConf
-	Log    Log
+	Mongob      MgoConf
+	Cron        CronConf
+	Mongop      MgoConf
+	Mysql       MysqlConf
+	Log         Log
+	Mongospider MgoConf
 }
 
 type MgoConf struct {

bidding_listen/config.toml

@@ -15,6 +15,12 @@
     username = ""
     password = ""
 
+[mongospider] ## 竞品网站数据库
+    host = "172.17.4.87:27080"
+    db = "spider"
+    coll = "spider_compete"
+    username = ""
+    password = ""
 
 
 [cron] ## 定时任务

bidding_listen/init.go

@@ -44,6 +44,7 @@ func init() {
 	InitMgo()
 	//InitEs()
 	InitMysql()
+	initSpider()
 	//
 
 }
@@ -119,5 +120,32 @@ func InitMgo() {
 	}
 	MgoP.InitPool()
 
+	MgoSpider = &mongodb.MongodbSim{
+		MongodbAddr: GF.Mongospider.Host,
+		DbName:      GF.Mongospider.DB,
+		Size:        10,
+		UserName:    GF.Mongospider.Username,
+		Password:    GF.Mongospider.Password,
+	}
+	MgoSpider.InitPool()
+
 	log.Info("InitMgo", zap.Any("duration", time.Since(now).Seconds()))
 }
+
+//initSpider 初始化 竞品站点
+func initSpider() {
+	fmt.Println("开始初始化 -- 竞品站点")
+	sess := MgoSpider.GetMgoConn()
+	defer MgoSpider.DestoryMongoConn(sess)
+	query := sess.DB(GF.Mongospider.DB).C(GF.Mongospider.Coll).Find(nil).Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if tmp["site"] != nil {
+			site := tmp["site"].(string)
+			SpiderSites = append(SpiderSites, site)
+		}
+	}
+	log.Info("initSpider", zap.Int("初始化竞品站点完毕,竞品网站数量为：", len(SpiderSites)))
+
+}

bidding_listen/main.go

@@ -6,7 +6,6 @@ import (
 	"app.yhyue.com/data_processing/common_utils/mongodb"
 	"app.yhyue.com/data_processing/common_utils/mysqldb"
 	"fmt"
-	es "github.com/olivere/elastic/v7"
 	"github.com/robfig/cron/v3"
 	"go.uber.org/zap"
 	"sync"
@@ -14,10 +13,12 @@ import (
 )
 
 var (
-	Mgo      *mongodb.MongodbSim
-	MgoP     *mongodb.MongodbSim
-	Mysql    *mysqldb.Mysql
-	EsClient *es.Client
+	Mgo       *mongodb.MongodbSim
+	MgoP      *mongodb.MongodbSim
+	MgoSpider *mongodb.MongodbSim
+	Mysql     *mysqldb.Mysql
+	//EsClient    *es.Client
+	SpiderSites = make([]string, 0) //排除，竞品站点
 )
 
 func main() {
@@ -110,6 +111,11 @@ func saveBidding(tmp map[string]interface{}) {
 	insert["bidding_id"] = id
 	insert["spidercode"] = tmp["spidercode"]
 	insert["site"] = tmp["site"]
+	site := tmp["site"].(string)
+	//竞品网站数据，直接过滤
+	if IsInStringArray(site, SpiderSites) {
+		return
+	}
 
 	if tmp["comeintime"] != nil {
 		time, _ := convertToTime(tmp["comeintime"])

bidding_listen/utils.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"sort"
 	"time"
 )
 
@@ -15,3 +16,13 @@ func convertToTime(value interface{}) (time.Time, error) {
 		return time.Time{}, fmt.Errorf("unsupported type: %T", v)
 	}
 }
+
+//IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}