优化附件长度限制，配置文件限制

createEsIndex/bidding_es.go

@@ -136,7 +136,7 @@ func biddingTask(mapInfo map[string]interface{}) {
 	wg.Wait()
 	log.Info("biddingTask over", zap.Int("count", c1), zap.Int("index", index))
 
-	//发送udp，
+	////发送udp，附件补采 才需要
 	//data := map[string]interface{}{
 	//	"stype": "update",
 	//	"gtid":  mongodb.StringTOBsonId(mapInfo["gtid"].(string)),
@@ -152,8 +152,11 @@ func biddingTask(mapInfo map[string]interface{}) {
 	//	log.Info("biddingTask ", zap.Any("WriteUdp err", err), zap.Any("target", target))
 	//}
 	//
-	//log.Info("biddingTask ", zap.Any("target", target))
-	// 重采平台需要
+	//log.Info("biddingTask ", zap.Any("target", target), zap.Any("data", data))
+	//
+
+	//
+	//重采平台需要
 	//mapInfo["stype"] = ""
 	//datas, _ := json.Marshal(mapInfo)
 	//var next = &net.UDPAddr{
@@ -607,7 +610,12 @@ func getFileText(tmp map[string]interface{}) (filetext string) {
 					if resultMap, ok := result.(map[string]interface{}); resultMap != nil && ok {
 						if attach_url := util.ObjToString(resultMap["attach_url"]); attach_url != "" {
 							bs := oss.OssGetObject(attach_url, mongodb.BsonIdToSId(tmp["_id"])) //oss读数据
-							if utf8.RuneCountInString(filetext+bs) < 550000 {
+							//附件总长度限制550000，其中最后一个文件长度限制50000
+							size := config.Conf.DB.Oss.Filesize
+							if size <= 0 {
+								size = 500000
+							}
+							if utf8.RuneCountInString(filetext+bs) < 50000+size {
 								filetext += bs + "\n"
 							} else {
 								if len(bs) > 50000 {
@@ -615,10 +623,9 @@ func getFileText(tmp map[string]interface{}) (filetext string) {
 								} else {
 									filetext += bs
 								}
-
 							}
 							//附件总长度限制550000
-							if utf8.RuneCountInString(filetext) >= 550000 {
+							if utf8.RuneCountInString(filetext) >= 50000+size {
 								return
 							}
 

createEsIndex/common.toml

@@ -52,6 +52,7 @@
     accesskey = "LTAI4G5x9aoZx8dDamQ7vfZi"
     accesssecret = "Bk98FsbPYXcJe72n1bG3Ssf73acuNh"
     bucketname = "topjy"
+    filesize = 500000  ## 单位字节，附件总字节长度限制；超过就不再读取
 [db.es]
     addr = "http://192.168.3.149:9200"      ## 正常bidding 链接
     addrp = "http://192.168.3.149:9200"   ## 采集使用的单机版地址

createEsIndex/config/conf.go

@@ -82,6 +82,7 @@ type oss struct {
 	AccessKey    string
 	AccessSecret string
 	BucketName   string
+	Filesize     int
 }
 type mgo struct {
 	Addr     string