es更新

src/config.json

@@ -28,11 +28,21 @@
 		"username": "",
 		"password": ""
 	},
+	"esb": {
+		"addr": "http://192.168.3.128:9800",
+		"pool": 2,
+		"index": "bidding",
+		"type": "bidding",
+		"username": "",
+		"password": ""
+	},
 	"es": {
 		"addr": "http://192.168.3.128:9800",
 		"pool": 2,
 		"index": "bidding",
-		"type": "bidding"
+		"type": "bidding",
+		"username": "",
+		"password": ""
 	},
 	"msgservers": {
 		"comm": {

src/timetask/random.go

@@ -2,6 +2,7 @@ package timetask
 
 import (
 	"encoding/json"
+	elc "gopkg.in/olivere/elastic/v7"
 	qu "qfw/util"
 	"regexp"
 	"strings"
@@ -318,8 +319,9 @@ func GetSpiderWarnData() {
 				if info == "Detail File Err" { //正文是链接的，进行链接判重
 					hrefDetail := httpReg.FindString(qu.ObjToString(data["detail"]))
 					if hrefDetail != "" {
-						esQuery := `{"query": {"bool": {"must": [{"term": {"href": "` + hrefDetail + `"}}]}}}`
-						if util.Es.Count(util.EsIndex, util.EsType, esQuery) >= 1 {
+						//esQuery := `{"query": {"bool": {"must": [{"term": {"href": "` + hrefDetail + `"}}]}}}`
+						esQuery := elc.NewBoolQuery().Must(elc.NewTermQuery("href", hrefDetail))
+						if util.Es.Count(util.EsIndex, util.EsType, esQuery) > 0 {
 							return
 						}
 					}

src/timetask/report.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/tealeg/xlsx"
+	elc "gopkg.in/olivere/elastic/v7"
 	"net/http"
 	qu "qfw/util"
 	gm "qfw/util/mail"
@@ -285,8 +286,9 @@ func SpiderWeeklyReport() {
 	qu.Debug("mgo采集量增长率：", AllMgoNumIncDecRatio)
 
 	//本周es索引量
-	esQuery := `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}}]}}}`
-	ThisWeekEsNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
+	//esQuery := `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}}]}}}`
+	esQuery := elc.NewBoolQuery().Must(elc.NewRangeQuery("comeintime").Gte(sTime).Lt(eTime))
+	ThisWeekEsNum := util.EsB.Count(util.EsBIndex, util.EsBType, esQuery)
 	qu.Debug("上周，本周es索引总量：", LastWeekEsNum, ThisWeekEsNum)
 	//es索引量增长率
 	AllEsNumIncDecRatio := float64(0)
@@ -310,8 +312,9 @@ func SpiderWeeklyReport() {
 	//竞品es本周索引量(元博网（采购与招标网）、中国招标与采购网)
 	ThisWeekCompeteEsNum := 0
 	for _, competeSite := range []string{Compete_ZGZBYCGW, Compete_YBW} {
-		esQuery = `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}},{"term": {"site": "` + competeSite + `"}}]}}}`
-		tmpNum := util.Es.Count(util.EsIndex, util.EsType, esQuery)
+		//esQuery = `{"query": {"bool": {"must": [{"range": {"comeintime": {"gte": "` + fmt.Sprint(sTime) + `","lt": "` + fmt.Sprint(eTime) + `"}}},{"term": {"site": "` + competeSite + `"}}]}}}`
+		esQuery = elc.NewBoolQuery().Must(elc.NewRangeQuery("comeintime").Gte(sTime).Lt(eTime)).Must(elc.NewTermQuery("site", competeSite))
+		tmpNum := util.EsB.Count(util.EsBIndex, util.EsBType, esQuery)
 		qu.Debug(competeSite, "es统计量：", tmpNum)
 		ThisWeekCompeteEsNum += int(tmpNum)
 	}

src/util/config.go

@@ -3,7 +3,7 @@ package util
 import (
 	mgo "mongodb"
 	qu "qfw/util"
-	es "qfw/util/elastic"
+	es "qfw/util/elastic.v7"
 	gm "qfw/util/mail"
 	"regexp"
 )
@@ -14,7 +14,10 @@ var (
 	MgoS                       *mgo.MongodbSim //spider
 	MgoPy                      *mgo.MongodbSim //py_spider
 	MgoB                       *mgo.MongodbSim //bidding
-	Es                         *es.Elastic     //
+	EsB                        *es.Elastic     //正式环境
+	EsBIndex                   string          //
+	EsBType                    string          //
+	Es                         *es.Elastic     //三个月全量
 	EsIndex                    string          //
 	EsType                     string          //
 	CommServers                map[string]interface{}
@@ -84,10 +87,23 @@ func InitEs() {
 	EsIndex = qu.ObjToString(esMap["index"])
 	EsType = qu.ObjToString(esMap["type"])
 	Es = &es.Elastic{
-		S_esurl: qu.ObjToString(esMap["addr"]),
-		I_size:  qu.IntAll(esMap["pool"]),
+		S_esurl:  qu.ObjToString(esMap["addr"]),
+		I_size:   qu.IntAll(esMap["pool"]),
+		Username: qu.ObjToString(esMap["username"]),
+		Password: qu.ObjToString(esMap["password"]),
 	}
 	Es.InitElasticSize()
+
+	esbMap := Config["esb"].(map[string]interface{})
+	EsBIndex = qu.ObjToString(esbMap["index"])
+	EsBType = qu.ObjToString(esbMap["type"])
+	EsB = &es.Elastic{
+		S_esurl:  qu.ObjToString(esbMap["addr"]),
+		I_size:   qu.IntAll(esbMap["pool"]),
+		Username: qu.ObjToString(esbMap["username"]),
+		Password: qu.ObjToString(esbMap["password"]),
+	}
+	EsB.InitElasticSize()
 }
 
 func InitOther() {