更新修复 nseo_id 覆盖bug

qyxy_inc_data_new/main.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"compress/gzip"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"go.uber.org/zap"
 	"io"
@@ -14,7 +15,9 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"os"
+	"runtime"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -26,10 +29,8 @@ var (
 
 	CurrentColl string
 
-	collCount int
-
+	//collCount int
 	saveLog   = make(map[string]interface{})
-	saveArr   [][]map[string]interface{}
 	UdpClient udp.UdpClient
 
 	changeAddr   *net.UDPAddr
@@ -99,19 +100,14 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 
 func task(path string) {
 	files, _ := ioutil.ReadDir(path)
-
 	jyUpdatetime = time.Now().Unix() //数据更新时间，更新quxy_change 使用
-	//拿到company_change 插入之前的ID
-	//dats, _ := MongoTool.Find("company_change", nil, `{"_id":-1}`, nil, true, -1, 1)
-	//ds := *dats
-	//lastID := ds[0]["_id"]
 
 	//annual_report_base/20221122/split.json.gz
 	for _, f := range files {
 		if f.IsDir() {
 			start := time.Now()
 			CurrentColl = f.Name()                                    //annual_report_base
-			collCount = 0                                             // 当前表的数据数量
+			collCount := 0                                            // 当前表的数据数量
 			log.Info("task", zap.String("collection name", f.Name())) //annual_report_base
 			//util.Debug("collection name：---", f.Name())
 			if !strings.HasSuffix(path, "/") {
@@ -121,17 +117,22 @@ func task(path string) {
 			subFiles, _ := ioutil.ReadDir(subPath)
 
 			for _, s := range subFiles {
-				util.Debug(s.Name())
+				log.Info("task ", zap.String("当前文件：", s.Name()))
 				if s.IsDir() {
-					taskinfo(subPath + s.Name()) //annual_report_base/20221122
+					collCount = taskinfo(subPath+s.Name(), collCount) //annual_report_base/20221122
+					//// 增加WaitGroup计数
+					//go func(dirPath string, collCount int) {
+					//	collCount = taskinfo(dirPath, collCount)
+					//}(subPath+s.Name(), collCount)
+					//taskinfo(subPath + s.Name()) //annual_report_base/20221122
 				}
 			}
 			// 判断最后的数据不足500条时 执行
-			if len(saveArr) > 0 {
-				tmps := saveArr
-				MongoTool.UpSertBulk(CurrentColl, tmps...)
-				saveArr = [][]map[string]interface{}{}
-			}
+			//if len(saveArr) > 0 {
+			//	tmps := saveArr
+			//	MongoTool.UpSertBulk(CurrentColl, tmps...)
+			//	saveArr = [][]map[string]interface{}{}
+			//}
 
 			duration := time.Since(start)
 			result := map[string]interface{}{
@@ -160,32 +161,47 @@ func task(path string) {
 	//SendMail(sendMsg)
 }
 
-//taskinfo 读取压缩包文件
-func taskinfo(path string) {
+// taskinfo 读取压缩包文件
+func taskinfo(path string, collCount int) int {
 	count := 0 //读取的数量
 	file := path + "/split.json.gz"
-	log.Info("taskinfo", zap.Any("current date", file))
+	log.Info("taskinfo", zap.Any("current file", file))
+
+	// 检查文件是否存在
+	fileInfo, err := os.Stat(file)
+	if err != nil {
+		log.Error("Error opening file:", zap.Error(err))
+		return collCount
+	}
+
+	// 检查文件大小是否为0
+	if fileInfo.Size() == 0 {
+		log.Warn(file, zap.Error(errors.New("文件大小为0")))
+		return collCount
+	}
+
 	// 打开本地gz格式压缩包
 	fr, err := os.Open(file)
 	if err != nil {
-		panic(err)
+		log.Info("taskinfo", zap.Any("err", err))
 	} else {
-		println("open file success!")
+		fmt.Println("open file success!", file)
 	}
 	// defer: 在函数退出时,执行关闭文件
 	defer fr.Close()
 	// 创建gzip文件读取对象
 	gr, err := gzip.NewReader(fr)
 	if err != nil {
-		panic(err)
+		log.Info("taskinfo", zap.Any("err", err))
 	}
 	// defer: 在函数退出时,执行关闭gzip对象
 	defer gr.Close()
 
 	bfRd := bufio.NewReader(gr)
+	wg := sync.WaitGroup{}
+	ch := make(chan bool, 5)
 	for {
 		line, err := bfRd.ReadBytes('\n')
-		count = hookfn(line, count)
 		if err != nil {
 			if err == io.EOF {
 				log.Info("taskinfo", zap.String("EOF", "read gzip data finish! "))
@@ -193,48 +209,78 @@ func taskinfo(path string) {
 			} else {
 				log.Info("taskinfo", zap.Any("[read gzip data err]:", err))
 			}
-		}
-		if count%5000 == 0 {
-			log.Info("taskinfo", zap.Any("current exec", fmt.Sprintf("%s-%d", file, count)))
-			//util.Debug("current exc---", file, count)
+		} else {
+			count++
+			if count%5000 == 0 {
+				printMemoryUsage()
+				log.Info("taskinfo", zap.Int("current count:"+file, count))
+			}
+
+			ch <- true
+			wg.Add(1)
+			go func(line []byte) {
+				defer func() {
+					<-ch
+					wg.Done()
+				}()
+				hookfn(line)
+			}(line)
+
 		}
 	}
+
+	wg.Wait()
+	collCount += count
+	return collCount
 }
 
-//hookfn 处理数据，500条处理一次
-func hookfn(line []byte, count int) int {
+// hookfn 处理数据，500条处理一次
+func hookfn(line []byte) {
 	tmp := make(map[string]interface{})
 	err := json.Unmarshal(line, &tmp)
 	if err != nil {
 		log.Info("hookfn", zap.Any("Unmarshal err", err))
-		//util.Debug("err---", err)
-	}
-	count++
-	collCount++
-	tmp["_id"] = util.IntAll(tmp["id"])
-	tmp["id"] = fmt.Sprintf("%d", util.IntAll(tmp["id"]))
-	if CurrentColl == "company_change" {
-		tmp["jy_updatetime"] = jyUpdatetime
 	}
 
-	saveInfo := []map[string]interface{}{
-		{"_id": tmp["_id"]},
-		{"$set": tmp},
+	if len(tmp) == 0 {
+		return
 	}
 
-	saveArr = append(saveArr, saveInfo)
-	//500 条处理一次，打印一次记录
-	if len(saveArr) > 500 {
-		tmps := saveArr
-		MongoTool.UpSertBulk(CurrentColl, tmps...)
-		saveArr = [][]map[string]interface{}{}
+	if util.IntAll(tmp["id"]) == 0 {
+		MongoTool.Save("wcc"+CurrentColl, tmp)
+	} else {
+		tmp["_id"] = util.IntAll(tmp["id"])
+		tmp["id"] = fmt.Sprintf("%d", util.IntAll(tmp["id"]))
+		if CurrentColl == "company_change" {
+			tmp["jy_updatetime"] = jyUpdatetime
+		}
+
+		saveInfo := []map[string]interface{}{map[string]interface{}{"_id": tmp["_id"]}, map[string]interface{}{"$set": tmp}}
+		tmpArr := [][]map[string]interface{}{saveInfo}
+		MongoTool.UpSertBulk(CurrentColl, tmpArr...)
 	}
-	return count
+
 }
 
-//SendUdpMsg 通知处理企业新增数据
+// SendUdpMsg 通知处理企业新增数据
 func SendUdpMsg(data map[string]interface{}, target *net.UDPAddr) {
 	bytes, _ := json.Marshal(data)
 	UdpClient.WriteUdp(bytes, udp.OP_TYPE_DATA, target)
 	log.Info("SendUdpMsg", zap.Any("data", data), zap.Any("target", target))
 }
+
+func printMemoryUsage() {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+
+	// 将字节转换为兆字节（MB）
+	allocatedMB := float64(memStats.Alloc) / 1024 / 1024
+	totalAllocatedMB := float64(memStats.TotalAlloc) / 1024 / 1024
+	heapAllocMB := float64(memStats.HeapAlloc) / 1024 / 1024
+
+	log.Info("printMemoryUsage", zap.Any("当前程序已分配的内存大小", allocatedMB))
+	log.Info("printMemoryUsage", zap.Any("程序自启动以来总共分配的内存大小", totalAllocatedMB))
+	log.Info("printMemoryUsage", zap.Any("堆上当前已分配但尚未释放的内存", heapAllocMB))
+	log.Info("printMemoryUsage", zap.Any("堆上分配的对象数", memStats.HeapObjects))
+
+}

qyxy_std_new/config.toml

@@ -11,5 +11,5 @@
     esport  =18890
     path = "" ## 路径默认为空，使用udp 传递的参数
     targetip = "127.0.0.1"
-    autoid = 333342012 ## 上次结束的 id
-    seoid = 31464188955 ## 上次结束的 seo_id 数字
+    autoid = 338653106 ## 上次结束的 id
+    seoid = 35728211634 ## 上次结束的 seo_id 数字

qyxy_std_new/main.go

@@ -13,6 +13,7 @@ import (
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/udp"
 	"net"
 	"os"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -47,7 +48,7 @@ func main() {
 		IP:   net.ParseIP(GF.Env.Targetip),
 	}
 	log.Info("main", zap.Any("qyxyEsAddr", qyxyEsAddr))
-	startTime = time.Now().Unix()
+
 	UdpClient.Listen(processUdpMsg)
 	log.Info("main", zap.String("Udp服务监听本地端口", localPort))
 
@@ -83,6 +84,7 @@ func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
 				// 开始执行
 				log.Info("processUdpMsg", zap.String("readPath", readPath))
 				if readPath != "" {
+					startTime = time.Now().Unix()
 					go dealPath(readPath)
 				}
 			}
@@ -97,7 +99,7 @@ func dealPath(path string) {
 	if !strings.HasSuffix(path, "/") {
 		path = path + "/" ///Users/wangchengcheng/Desktop/jianyu/upload/20221119
 	}
-	var wg sync.WaitGroup
+	//var wg sync.WaitGroup
 	//std 程序只需要关注6个表
 	for _, c := range CollArr {
 		subPath := path + c + "/"
@@ -107,10 +109,11 @@ func dealPath(path string) {
 			log.Info("dealPath", zap.Any("os.Stat err", subPath))
 			continue
 		}
-		wg.Add(1)
-		go dealSubPath(c, subPath, &wg)
+		//wg.Add(1)
+		dealSubPath(c, subPath)
+		//go dealSubPath(c, subPath, &wg)
 	}
-	wg.Wait()
+	//wg.Wait()
 
 	//更新 nseo_id
 	updateStd()
@@ -137,25 +140,26 @@ func dealPath(path string) {
 	SendUdpMsg(data, qyxyEsAddr)
 }
 
-//dealSubPath 处理最里面层级数据；Users/wangchengcheng/Desktop/jianyu/upload/20221119/company_base/20221224
-//c 当前表；
-func dealSubPath(c, subPath string, wg *sync.WaitGroup) {
-	defer wg.Done()
+// dealSubPath 处理最里面层级数据；Users/wangchengcheng/Desktop/jianyu/upload/20221119/company_base/20221224
+// c 当前表；
+func dealSubPath(c, subPath string) {
+	//defer wg.Done()
 	log.Info("dealSubPath", zap.String("开始处理path:", subPath))
 	start := time.Now()
-	var fileWg sync.WaitGroup
+	//var fileWg sync.WaitGroup
 	var linesMap sync.Map
 
 	subFiles, _ := ioutil.ReadDir(subPath)
 	for _, s := range subFiles { //七天的文件夹名
 		if s.IsDir() {
-			fileWg.Add(1)
+			//fileWg.Add(1)
+			dealinfo(c, subPath+s.Name(), &linesMap)
 			///Users/wangchengcheng/Desktop/jianyu/upload/20221119/company_base/20221224
-			go dealinfo(c, subPath+s.Name(), &fileWg, &linesMap)
+			//go dealinfo(c, subPath+s.Name(), &fileWg, &linesMap)
 		}
 	}
 
-	fileWg.Wait()
+	//fileWg.Wait()
 
 	cCount, _ := linesMap.Load(c)
 
@@ -168,30 +172,40 @@ func dealSubPath(c, subPath string, wg *sync.WaitGroup) {
 	savelog.Store(c, result)
 }
 
-func dealinfo(c, path string, wg *sync.WaitGroup, linesMap *sync.Map) {
-	defer wg.Done()
+func dealinfo(c, path string, linesMap *sync.Map) {
+	//defer wg.Done()
 	count := 0
 	file := path + "/split.json.gz"
 	log.Info("dealinfo", zap.Any("current date", file))
-	_, err := os.Stat(file)
+	fileInfo, err := os.Stat(file)
+
+	if fileInfo.Size() == 0 {
+		return
+	}
+
 	if err == nil {
 		// 打开本地gz格式压缩包
 		fr, err := os.Open(file)
 		if err != nil {
-			panic(err)
+			log.Info("dealinfo", zap.Error(err))
+			return
 		} else {
-			println("open file success!")
+			fmt.Println("open file success!", file)
 		}
 		// defer: 在函数退出时,执行关闭文件
 		defer fr.Close()
 		// 创建gzip文件读取对象
 		gr, err := gzip.NewReader(fr)
 		if err != nil {
-			panic(err)
+			log.Info("reader "+file, zap.Error(err))
+			return
 		}
 		// defer: 在函数退出时,执行关闭gzip对象
 		defer gr.Close()
 
+		wg := sync.WaitGroup{}
+		ch := make(chan bool, 3)
+
 		bfRd := bufio.NewReader(gr)
 		for {
 			line, err := bfRd.ReadBytes('\n')
@@ -206,18 +220,28 @@ func dealinfo(c, path string, wg *sync.WaitGroup, linesMap *sync.Map) {
 			}
 			if len(line) > 0 {
 				count++
-				hookfn(c, line)
 				linesMap.Store(c, count)
+				ch <- true
+				wg.Add(1)
+				go func(c string, line []byte) {
+					defer func() {
+						<-ch
+						wg.Done()
+					}()
+					hookfn(c, line)
+				}(c, line)
 			}
 
-			if count%1000 == 0 {
+			if count%5000 == 0 {
+				printMemoryUsage()
 				log.Info("dealinfo", zap.Any("current exc---", fmt.Sprintf("%s-%d", file, count)))
 			}
 		}
+		wg.Wait()
 	}
 }
 
-//hookfn 处理拿到的每行数据
+// hookfn 处理拿到的每行数据
 func hookfn(c string, line []byte) {
 	tmp := make(map[string]interface{})
 	err := json.Unmarshal(line, &tmp)
@@ -253,7 +277,7 @@ func hookfn(c string, line []byte) {
 
 }
 
-//dealCompanyBase company_base数据表
+// dealCompanyBase company_base数据表
 func dealCompanyBase(data map[string]interface{}) {
 	update := make(map[string]interface{})
 	save := make(map[string]interface{})
@@ -416,7 +440,7 @@ func dealCompanyBase(data map[string]interface{}) {
 
 }
 
-//dealCompanyEmployee company_employee
+// dealCompanyEmployee company_employee
 func dealCompanyEmployee(data map[string]interface{}) {
 	save := make(map[string]interface{})
 	save["_id"] = data["company_id"]
@@ -480,7 +504,7 @@ func dealCompanyEmployee(data map[string]interface{}) {
 	//}
 }
 
-//dealCompanyPartner
+// dealCompanyPartner
 func dealCompanyPartner(data map[string]interface{}) {
 	save := make(map[string]interface{})
 	save["_id"] = data["company_id"]
@@ -560,7 +584,7 @@ func dealCompanyPartner(data map[string]interface{}) {
 	//}
 }
 
-//dealAnnualReportBase annual_report_base
+// dealAnnualReportBase annual_report_base
 func dealAnnualReportBase(data map[string]interface{}) {
 	save := make(map[string]interface{})
 	save["_id"] = data["company_id"]
@@ -666,7 +690,7 @@ func dealAnnualReportBase(data map[string]interface{}) {
 	//}
 }
 
-//dealHistoryName company_history_name
+// dealHistoryName company_history_name
 func dealHistoryName(data map[string]interface{}) {
 	save := make(map[string]interface{})
 	save["_id"] = data["company_id"]
@@ -704,7 +728,7 @@ func dealHistoryName(data map[string]interface{}) {
 
 }
 
-//dealAnnualReportWebsite annual_report_website
+// dealAnnualReportWebsite annual_report_website
 func dealAnnualReportWebsite(data map[string]interface{}) {
 	save := make(map[string]interface{})
 	save["_id"] = data["company_id"]
@@ -737,3 +761,19 @@ func dealAnnualReportWebsite(data map[string]interface{}) {
 	//}
 
 }
+
+func printMemoryUsage() {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+
+	// 将字节转换为兆字节（MB）
+	allocatedMB := float64(memStats.Alloc) / 1024 / 1024
+	totalAllocatedMB := float64(memStats.TotalAlloc) / 1024 / 1024
+	heapAllocMB := float64(memStats.HeapAlloc) / 1024 / 1024
+
+	log.Info("printMemoryUsage", zap.Any("当前程序已分配的内存大小", allocatedMB))
+	log.Info("printMemoryUsage", zap.Any("程序自启动以来总共分配的内存大小", totalAllocatedMB))
+	log.Info("printMemoryUsage", zap.Any("堆上当前已分配但尚未释放的内存", heapAllocMB))
+	log.Info("printMemoryUsage", zap.Any("堆上分配的对象数", memStats.HeapObjects))
+
+}

qyxy_std_new/task.go

@@ -3,12 +3,12 @@ package main
 import (
 	"context"
 	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 
 	//"github.com/cron"
 	"github.com/wangbin/jiebago"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"log"
 	"math/rand"
 	"regexp"
@@ -21,12 +21,13 @@ var (
 	AreaFiled = []string{"credit_no", "company_code"}
 	WordsArr  = []string{"研发", "研制", "开发", "生产", "制造", "制作", "加工", "种植"}
 
-	seg       jiebago.Segmenter
-	regPre, _ = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
-	seed      = 188
+	seg        jiebago.Segmenter
+	regPre, _  = regexp.Compile(`^(.+[省|市|区|县|州])?(.+)`)
+	seed       = 188
+	startSeoId int64
 )
 
-//	企业基本信息
+// 企业基本信息
 var company_base = []string{"company_name", "company_code", "credit_no", "org_code", "legal_person", "company_status",
 	"authority", "establish_date", "issue_date", "operation_startdate", "operation_enddate", "capital", "company_type",
 	"company_status", "company_address", "business_scope", "cancel_date", "cancel_reason", "revoke_date", "revoke_reason",
@@ -82,12 +83,12 @@ func DealMemberNo(memberNo string) (no int) {
 	return no
 }
 
-//updateStd 根据qyxy_std 表，更新 nseo_id
+// updateStd 根据qyxy_std 表，更新 nseo_id
 func updateStd() {
 	id := int64(GF.Env.Autoid) //起始 autoid
 	//生成seoid
 	//31395235740
-	startSeoId := int64(GF.Env.Seoid) //起始 nseo_id
+	startSeoId = int64(GF.Env.Seoid) //起始 nseo_id
 
 	rand.Seed(time.Now().UnixNano())
 
@@ -110,7 +111,12 @@ func updateStd() {
 		count++
 		if cur != nil {
 			cur.Decode(&tmp)
-			id = util.Int64All(tmp["autoid"])
+			id := util.Int64All(tmp["autoid"])
+			//存在就不处理
+			if _, ok := tmp["nseo_id"]; ok {
+				continue
+			}
+
 			ed, _ := tmp["establish_date"].(string)
 			pre := ""
 			if len(ed) == 10 {
@@ -127,16 +133,15 @@ func updateStd() {
 			update := make(map[string]interface{})
 			update["$set"] = bson.M{
 				"nseo_id": nseo_id,
-				"autoid":  id,
 			}
 			where := map[string]interface{}{
 				"_id": tmp["_id"],
 			}
 
-			go MongoTool.Update("qyxy_std", where, update, true, false)
+			MongoTool.Update("qyxy_std", where, update, true, false)
 
 			if count%10000 == 0 {
-				log.Println("current,id", count, id, startSeoId, nseo_id)
+				log.Println("current,autoid,startSeoId,nseo_id", count, id, startSeoId, nseo_id)
 			}
 
 			tmp = make(map[string]interface{})
@@ -147,6 +152,6 @@ func updateStd() {
 
 	}
 
-	log.Println("over ----;autoid: ", id, "seo_id:", startSeoId)
-	time.Sleep(20 * time.Minute)
+	log.Println("over ----;", "seo_id:", startSeoId)
+
 }

qyxy_std_new/util.go

@@ -46,7 +46,7 @@ var currencyItem = map[string]string{
 	"澳元":  "澳币",
 }
 
-//获取币种
+// 获取币种
 func GetCurrency(text string) (currency string) {
 	if text == "" {
 		return
@@ -61,7 +61,7 @@ func GetCurrency(text string) (currency string) {
 	return
 }
 
-//金额转换
+// 金额转换
 func ObjToMoney(text string) float64 {
 	isfindUnit := true
 	ret := capitalMoney(text)
@@ -157,7 +157,7 @@ func capitalMoney(text string) float64 {
 	return (ret + decimals) * suffixUnit
 }
 
-//数字金额转换
+// 数字金额转换
 func numMoney(text string) (moneyFloat float64, flag bool) {
 	//tmp := fmt.Sprintf("%f", data[0])
 	repUnit := float64(1)
@@ -249,14 +249,14 @@ func numMoney(text string) (moneyFloat float64, flag bool) {
 	return
 }
 
-//清理所有空白符
+// 清理所有空白符
 func CutAllSpace(text string) string {
 	tmp := cutAllSpace.ReplaceAllString(text, "")
 	tmp = replaceSymbol(tmp, spaces)
 	return tmp
 }
 
-//符号替换
+// 符号替换
 func replaceString(con string, ret, rep []string) string {
 	for k, v := range ret {
 		if len(rep) > k {
@@ -266,7 +266,7 @@ func replaceString(con string, ret, rep []string) string {
 	return con
 }
 
-//过滤符号
+// 过滤符号
 func replaceSymbol(con string, rep []string) string {
 	for _, v := range rep {
 		con = strings.Replace(con, v, "", -1)
@@ -291,7 +291,7 @@ func IsInSlice(slice []string, s string) bool {
 	return isIn
 }
 
-//SendUdpMsg 通知处理企业新增数据
+// SendUdpMsg 通知处理企业新增数据
 func SendUdpMsg(data map[string]interface{}, target *net.UDPAddr) {
 	bytes, _ := json.Marshal(data)
 	UdpClient.WriteUdp(bytes, udp.OP_TYPE_DATA, target)

update_seo/update_std/go.mod

@@ -1,4 +1,4 @@
-module update_seo
+module update_std
 
 go 1.18
 
@@ -15,13 +15,10 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
-	github.com/olivere/elastic/v7 v7.0.32 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/afero v1.10.0 // indirect
@@ -33,9 +30,6 @@ require (
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.mongodb.org/mongo-driver v1.10.1 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
@@ -44,6 +38,5 @@ require (
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

update_seo/update_std/go.sum

@@ -37,7 +37,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
@@ -45,7 +44,6 @@ github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejn
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -66,7 +64,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -146,7 +143,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -161,14 +157,12 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
-github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -237,14 +231,8 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
 go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
-go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
-go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
-go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
 go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -555,7 +543,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=