update

environment/config.toml

@@ -30,14 +30,7 @@
 
 
 [env]
-#    file = "./data.xlsx" ## 读取文件
-#    sheet= "Sheet1" ## 默认读取数据在Sheet1
-#    savecoll = "wcc_1101"
-#
-#    bidfields =["title", "area", "city", "district", "toptype", "subtype", "detail", "publishtime", "href",
-#        "projectname", "projectcode", "projectscope", "budget", "bidamount", "buyer", "buyerperson", "buyertel", "buyeraddr", "agency",
-#        "s_winner", "winnerperson", "winnertel", "legal_person", "company_phone", "company_email"]
-#
-#    njfields = ["title","detail","toptype","subtype","projectname","owner","approvedept","area","city","district","project_stage_code"]
+    file = "./data.xlsx" ## 读取文件
+    sheet= "Sheet1" ## 默认读取数据在Sheet1
 
 

environment/main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"fmt"
 	"github.com/gin-gonic/gin"
+	"github.com/xuri/excelize/v2"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
@@ -33,6 +35,45 @@ func main() {
 	log.Println("数据处理完毕!")
 }
 
+// updateXlsx 更新xlsx 文件标签
+func updateXlsx() {
+	filePath := GF.Env.File
+	if filePath == "" {
+		log.Println("文件路径为空")
+		return
+	}
+	// 1. 读取 Excel（获取 A 列数据）
+	f, err := excelize.OpenFile(filePath)
+	if err != nil {
+		log.Fatal("❌ 无法打开 Excel 文件:", err)
+	}
+	defer f.Close()
+
+	//读取央企
+	rows, err := f.GetRows(GF.Env.Sheet)
+	if err != nil {
+		log.Fatal("❌ 无法读取 Sheet1:", err)
+	}
+	for i := 1; i < len(rows); i++ {
+		//project_name := rows[i][0]
+		name := rows[i][1]
+		res1 := getCompanyType(name)
+		res2 := getMarketType(name)
+		res3 := getTop(name)
+
+		if res1 != "" {
+			f.SetCellValue(GF.Env.Sheet, fmt.Sprintf("D%v", i+1), res1)
+		}
+		if res2 != "" {
+			f.SetCellValue(GF.Env.Sheet, fmt.Sprintf("E%v", i+1), res2)
+		}
+		if res3 != "" {
+			f.SetCellValue(GF.Env.Sheet, fmt.Sprintf("F%v", i+1), res3)
+		}
+	}
+	f.Save()
+}
+
 // updateBidding 更新企业类型信息等
 func updateBidding() {
 	sess := MgoB.GetMgoConn()

environment/readme.txt

@@ -18,5 +18,6 @@
 
 -----------------------      -----------------------   -----------------------   -----------------------
 
-1、标记企业类型：央企、国企、央企下属、事业单位、民企;
-2、上市类型：A股、新三板、非上市
+1、标记企业类型：央企、国企、央企下属、事业单位、民企; --company_type
+2、上市类型：A股、新三板、非上市  -- market_type
+3、公司最上级：公司的最上层投资公司 -- top_name

getEs/bidding.go

@@ -454,7 +454,7 @@ func getDataFromFile() {
 	}
 
 	// 2. 读取 Excel
-	f, err := excelize.OpenFile("202502项目3_人工验证版.xlsx")
+	f, err := excelize.OpenFile("投资入库项目_202504.xlsx")
 	if err != nil {
 		log.Fatal("❌ 无法打开 Excel 文件:", err)
 	}
@@ -478,16 +478,19 @@ func getDataFromFile() {
 		}
 
 		//已经有数据，跳过
-		if len(row) > 7 {
+		if len(row) > 6 {
 			continue
 		}
-		projectName := row[3] // B 列数据
+		projectName := row[1]
 		if projectName == "" {
 			continue
 		}
 
 		buyer2 := row[2] //单位明细
-		log.Println("projectName", projectName)
+		if i%100 == 0 {
+			log.Println("projectName", projectName)
+		}
+
 		// 查询 ES，获取最相似的前 6 个完整文档
 		results, err := searchES(client, projectName, buyer2)
 		if err != nil {
@@ -519,9 +522,9 @@ func searchES(client *elastic.Client, projectName, buyer2 string) ([]map[string]
 	query := elastic.NewBoolQuery().
 		Must(
 			//elastic.NewMatchQuery("projectname.pname", projectName), // 模糊匹配 projectname
-			elastic.NewMatchQuery("title", projectName), // 模糊匹配 projectname
-			//elastic.NewMatchQuery("detail", projectName), // 模糊匹配 projectname
-			elastic.NewTermQuery("area", "安徽"),                       // 过滤区域
+			//elastic.NewMatchQuery("title", projectName), // 模糊匹配 projectname
+			elastic.NewMatchQuery("detail", projectName), // 模糊匹配 projectname
+			//elastic.NewTermQuery("area", "安徽"),                       // 过滤区域
 			elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一"), // 过滤 subtype
 			//elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向"), // 过滤 subtype
 			//elastic.NewTermsQuery("toptype", "拟建"), // 过滤 subtype
@@ -555,7 +558,7 @@ func searchES(client *elastic.Client, projectName, buyer2 string) ([]map[string]
 
 		bidamount := util.Float64All(doc["bidamount"])
 		if !ok {
-			log.Printf("⚠️ 缺少 projectname 字段，跳过：%v", doc)
+			log.Printf("⚠️ 缺少 projectname 字段，跳过：%v", doc["projectname"])
 			continue
 		}
 
@@ -631,7 +634,7 @@ func searchES(client *elastic.Client, projectName, buyer2 string) ([]map[string]
 		projectNameValue, ok := doc["projectname"].(string)
 		bidamount := util.Float64All(doc["bidamount"])
 		if !ok {
-			log.Printf("⚠️ 缺少 projectname 字段，跳过：%v", doc)
+			log.Printf("⚠️ 缺少 projectname 字段，跳过：%v", doc["projectname"])
 			continue
 		}
 
@@ -680,11 +683,11 @@ func searchES(client *elastic.Client, projectName, buyer2 string) ([]map[string]
 		}
 	}
 
-	for _, vv := range results {
-		id := util.ObjToString(vv["id"])
-		bidData, _ := MgoB.FindById("bidding", id, nil)
-		MgoB.SaveByOriID("wcc_biddind_20250326", bidData)
-	}
+	//for _, vv := range results {
+	//	id := util.ObjToString(vv["id"])
+	//	bidData, _ := MgoB.FindById("bidding", id, nil)
+	//	MgoB.SaveByOriID("wcc_biddind_20250326", bidData)
+	//}
 	return results, nil
 }
 

getEs/main.go

@@ -20,25 +20,29 @@ var (
 	MgoB *mongodb.MongodbSim
 )
 
-//func InitMgo() {
-//	MgoB = &mongodb.MongodbSim{
-//		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
-//		//MongodbAddr: "127.0.0.1:27083",
-//		Size:     10,
-//		DbName:   "qfw",
-//		UserName: "SJZY_RWbid_ES",
-//		Password: "SJZY@B4i4D5e6S",
-//		//Direct:      true,
-//	}
-//	MgoB.InitPool()
-//}
+func InitMgo() {
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "qfw",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+}
 
 func main() {
-	//InitMgo()
-	//getDataFromFile()
+	//dealXlsx()
+	//getQyxyNation()
+	//getQyxyNation()
+
+	InitMgo()
+	getDataFromFile()
 	//updateXlsxDa()
 
-	//return
+	return
 	/**
 	getProjectData  click 是一起使用的，统计获取中标企业信息
 	*/

getEs/qyxy_nation_type.go

@@ -0,0 +1,598 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"github.com/xuri/excelize/v2"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"log"
+	"math"
+)
+
+type Company struct {
+	CompanyName   string  `json:"company_name"`
+	Capital       float64 `json:"capital"`
+	CompanyType   string  `json:"company_type"`
+	CompanyStatus string  `json:"company_status"`
+}
+
+var (
+	indexName = "qyxy"
+)
+
+func getQyxyNation() {
+	url := "http://172.17.4.184:19908"
+	//url := "http://127.0.0.1:19908"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+
+	if err != nil {
+		log.Fatalf("Elasticsearch 连接失败: %v", err)
+	}
+
+	// 1. 创建 Excel 文件
+	f := excelize.NewFile()
+
+	// 2. 定义分组字段
+	groupFields := []string{"national_sub"}
+	sheetNames := []string{"按二级分类"}
+
+	for i, field := range groupFields {
+		log.Printf("查询并导出：%s\n", field)
+		err := exportGroupToSheet2(client, f, field, sheetNames[i])
+		if err != nil {
+			log.Fatalf("导出失败（%s）: %v", field, err)
+		}
+	}
+
+	// 删除默认 Sheet
+	f.DeleteSheet("Sheet1")
+
+	// 保存文件
+	if err := f.SaveAs("top_companies_by_industry_levels.xlsx"); err != nil {
+		log.Fatalf("保存失败: %v", err)
+	}
+	log.Println("✅ 所有分组已导出完成: top_companies_by_industry_levels.xlsx")
+}
+
+func exportGroupToSheet(client *elastic.Client, f *excelize.File, groupField, sheetName string) error {
+	ctx := context.Background()
+	// 查询条件
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("company_status", "存续")).
+		MustNot(elastic.NewTermQuery("company_type", "个体工商户"))
+
+	// 字段筛选器
+	sourceCtx := elastic.NewFetchSourceContext(true).Include(
+		"company_name", "capital", "company_type", "company_status", groupField,
+	)
+
+	// 聚合构建
+	topHitsAgg := elastic.NewTopHitsAggregation().
+		Size(5).
+		Sort("capital", false).
+		FetchSourceContext(sourceCtx)
+
+	termsAgg := elastic.NewTermsAggregation().Field(groupField).Size(500).
+		SubAggregation("top_capital_companies", topHitsAgg)
+
+	// 查询执行
+	res, err := client.Search().
+		Index(indexName).
+		Query(query).
+		Size(0).
+		Aggregation("by_group", termsAgg).
+		Do(ctx)
+	if err != nil {
+		return fmt.Errorf("Elasticsearch 查询失败: %v", err)
+	}
+
+	// 创建工作表
+	index, _ := f.NewSheet(sheetName)
+	headers := []string{groupField, "company_name", "capital", "company_type", "company_status"}
+	for i, h := range headers {
+		cell, _ := excelize.CoordinatesToCellName(i+1, 1)
+		f.SetCellValue(sheetName, cell, h)
+	}
+
+	// 解析结果并写入
+	agg, ok := res.Aggregations.Terms("by_group")
+	if !ok {
+		return fmt.Errorf("未找到分组聚合结果")
+	}
+
+	row := 2
+	for _, bucket := range agg.Buckets {
+		topHits, ok := bucket.TopHits("top_capital_companies")
+		if !ok {
+			continue
+		}
+		for _, hit := range topHits.Hits.Hits {
+			var c Company
+			if err := json.Unmarshal(hit.Source, &c); err != nil {
+				continue
+			}
+			data := []interface{}{bucket.Key, c.CompanyName, c.Capital, c.CompanyType, c.CompanyStatus}
+			for col, val := range data {
+				cell, _ := excelize.CoordinatesToCellName(col+1, row)
+				f.SetCellValue(sheetName, cell, val)
+			}
+			row++
+		}
+	}
+
+	// 激活第一个 Sheet
+	if sheetName == "按一级分类" {
+		f.SetActiveSheet(index)
+	}
+
+	return nil
+}
+
+func exportGroupToSheet3(client *elastic.Client, f *excelize.File, groupField, sheetName string) error {
+	ctx := context.Background()
+	// 查询条件
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("company_status", "存续")).
+		MustNot(elastic.NewTermQuery("company_type", "个体工商户"))
+
+	// 第一阶段：获取分组文档总数
+	countAgg := elastic.NewTermsAggregation().Field(groupField).Size(500)
+	countRes, err := client.Search().
+		Index(indexName).
+		Query(query).
+		Size(0).
+		Aggregation("by_group", countAgg).
+		Do(ctx)
+	if err != nil {
+		return fmt.Errorf("获取分组总数失败: %v", err)
+	}
+
+	// 解析分组桶
+	agg, ok := countRes.Aggregations.Terms("by_group")
+	if !ok {
+		return fmt.Errorf("未找到分组聚合结果")
+	}
+
+	// 创建工作表
+	//index, _ := f.NewSheet(sheetName)
+	headers := []string{groupField, "company_name", "capital", "company_type", "company_status"}
+	for i, h := range headers {
+		cell, _ := excelize.CoordinatesToCellName(i+1, 1)
+		f.SetCellValue(sheetName, cell, h)
+	}
+
+	// 第二阶段：分批查询每个分组的5%数据
+	//row := 2
+	for _, bucket := range agg.Buckets {
+		log.Println(bucket)
+		// 计算5%的数据量，至少取1条
+		//size := int(math.Max(1, float64(bucket.DocCount)*0.05))
+
+	}
+
+	return nil
+}
+
+func exportGroupToSheet21(client *elastic.Client, f *excelize.File, groupField, sheetName string) error {
+	ctx := context.Background()
+	// 查询条件
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("company_status", "存续")).
+		MustNot(elastic.NewTermQuery("company_type", "个体工商户"))
+
+	// 第一步：获取分组及每个分组的文档总数
+	countAgg := elastic.NewTermsAggregation().Field(groupField).Size(500)
+	countRes, err := client.Search().
+		Index(indexName).
+		Query(query).
+		Size(0).
+		Aggregation("by_group", countAgg).
+		Do(ctx)
+	if err != nil {
+		return fmt.Errorf("获取分组总数失败: %v", err)
+	}
+
+	agg, ok := countRes.Aggregations.Terms("by_group")
+	if !ok {
+		return fmt.Errorf("未找到分组聚合结果")
+	}
+
+	// 创建工作表
+	index, _ := f.NewSheet(sheetName)
+	headers := []string{groupField, "company_name", "capital", "company_type", "company_status"}
+	for i, h := range headers {
+		cell, _ := excelize.CoordinatesToCellName(i+1, 1)
+		f.SetCellValue(sheetName, cell, h)
+	}
+
+	row := 2
+
+	// 第二步：对每个分组进行查询并提取前 5% 的数据
+	sourceCtx := elastic.NewFetchSourceContext(true).Include(
+		"company_name", "capital", "company_type", "company_status", groupField,
+	)
+
+	for _, bucket := range agg.Buckets {
+		groupVal := bucket.Key
+		docCount := bucket.DocCount
+		size := int(math.Max(1, float64(docCount)*0.05))
+
+		groupStr := fmt.Sprintf("%v", groupVal)
+
+		query := elastic.NewBoolQuery().Filter(
+			elastic.NewTermQuery(groupField, groupStr),
+		)
+
+		src, _ := query.Source()
+		b, _ := json.MarshalIndent(src, "", "  ")
+		fmt.Println("Query body:\n", string(b))
+		//sourceCtx := elastic.NewFetchSourceContext(true).Include("company_name", "capital") // 确保不为 nil
+
+		res, err := client.Search().
+			Index(indexName).
+			Query(query).
+			Sort("capital", false).
+			Size(size).
+			FetchSourceContext(sourceCtx).
+			Do(ctx)
+
+		if err != nil {
+			fmt.Printf("查询失败 group=%v, err=%v, type=%T\n", groupVal, err, err)
+			continue
+		}
+
+		if res == nil {
+			fmt.Println("查询结果为空（res=nil）")
+			continue
+		}
+
+		fmt.Printf("查询成功: 命中 %d 条\n", res.Hits.TotalHits.Value)
+
+		subAgg, ok := res.Aggregations.Terms("by_group")
+		if !ok || len(subAgg.Buckets) == 0 {
+			continue
+		}
+
+		topHits, ok := subAgg.Buckets[0].TopHits("top_capital_companies")
+		if !ok {
+			continue
+		}
+
+		for _, hit := range topHits.Hits.Hits {
+			var c Company
+			if err := json.Unmarshal(hit.Source, &c); err != nil {
+				continue
+			}
+			data := []interface{}{groupVal, c.CompanyName, c.Capital, c.CompanyType, c.CompanyStatus}
+			for col, val := range data {
+				cell, _ := excelize.CoordinatesToCellName(col+1, row)
+				f.SetCellValue(sheetName, cell, val)
+			}
+			row++
+		}
+	}
+
+	// 设置激活工作表
+	if sheetName == "按一级分类" {
+		f.SetActiveSheet(index)
+	}
+
+	return nil
+}
+
+func exportGroupToSheet22(client *elastic.Client, f *excelize.File, groupField, sheetName string) error {
+	ctx := context.Background()
+
+	// 查询条件：状态为存续，非个体工商户
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("company_status", "存续")).
+		MustNot(elastic.NewTermQuery("company_type", "个体工商户"))
+
+	// 聚合：按 groupField 分组
+	countAgg := elastic.NewTermsAggregation().Field(groupField).Size(500)
+	countRes, err := client.Search().
+		Index(indexName).
+		Query(query).
+		Size(0).
+		Aggregation("by_group", countAgg).
+		Do(ctx)
+
+	if err != nil {
+		return fmt.Errorf("获取分组总数失败: %v", err)
+	}
+
+	agg, ok := countRes.Aggregations.Terms("by_group")
+	if !ok {
+		return fmt.Errorf("未找到分组聚合结果")
+	}
+
+	// 创建 Sheet 和标题
+	index, _ := f.NewSheet(sheetName)
+	headers := []string{groupField, "company_name", "capital", "company_type", "company_status"}
+	for i, h := range headers {
+		cell, _ := excelize.CoordinatesToCellName(i+1, 1)
+		f.SetCellValue(sheetName, cell, h)
+	}
+	row := 2
+
+	// 遍历每个分组，查询 top 5% 公司
+	for _, bucket := range agg.Buckets {
+		groupVal := bucket.Key
+		docCount := bucket.DocCount
+		if docCount == 0 {
+			continue
+		}
+		size := int(math.Max(1, float64(docCount)*0.05))
+		groupStr := fmt.Sprintf("%v", groupVal)
+
+		// 构造查询（加 exists 过滤，避免排序错误）
+		subQuery := elastic.NewBoolQuery().
+			Must(elastic.NewTermQuery(groupField, groupStr)).
+			Must(elastic.NewExistsQuery("capital")) // 避免排序失败
+
+		sourceCtx := elastic.NewFetchSourceContext(true).Include(
+			"company_name", "capital", "company_type", "company_status", groupField,
+		)
+
+		// 查询
+		scroll := client.Scroll(indexName).
+			Query(subQuery).
+			SortWithInfo(elastic.SortInfo{
+				Field:     "capital",
+				Ascending: false,
+				Missing:   "_last",
+			}).
+			Size(500). // 每批 500 条
+			FetchSourceContext(sourceCtx)
+
+		defer scroll.Clear(ctx) // 自动释放
+		var fetched int
+		for fetched < size {
+			res, err := scroll.Do(ctx)
+			if err == io.EOF {
+				break // 没有更多数据
+			}
+			if err != nil {
+				fmt.Printf("scroll 查询失败 group=%v, err=%v\n", groupVal, err)
+				break
+			}
+
+			for _, hit := range res.Hits.Hits {
+				if fetched >= size {
+					break
+				}
+
+				var c map[string]interface{}
+				if err := json.Unmarshal(hit.Source, &c); err != nil {
+					continue
+				}
+
+				data := []interface{}{
+					groupStr,
+					c["company_name"],
+					c["capital"],
+					c["company_type"],
+					c["company_status"],
+				}
+
+				for col, val := range data {
+					cell, _ := excelize.CoordinatesToCellName(col+1, row)
+					f.SetCellValue(sheetName, cell, val)
+				}
+				row++
+				fetched++
+			}
+		}
+
+		// 清理 scroll 上下文
+		_ = scroll.Clear(ctx)
+
+		if err != nil {
+			fmt.Printf("查询失败 group=%v, err=%v, type=%T\n", groupVal, err, err)
+			if elasticErr, ok := err.(*elastic.Error); ok && elasticErr.Details != nil {
+				fmt.Printf("详细错误: type=%v, reason=%v\n", elasticErr.Details.Type, elasticErr.Details.Reason)
+				for i, cause := range elasticErr.Details.RootCause {
+					fmt.Printf("RootCause[%d]: %v (%v)\n", i, cause.Reason, cause.Type)
+				}
+			}
+			continue
+		}
+
+	}
+
+	// 设置激活表
+	if sheetName == "按一级分类" {
+		f.SetActiveSheet(index)
+	}
+
+	return nil
+}
+
+func exportGroupToSheet2(client *elastic.Client, f *excelize.File, groupField, sheetName string) error {
+	const maxRowsPerSheet = 1048576
+	ctx := context.Background()
+
+	// 查询条件：状态为存续，非个体工商户
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("company_status", "存续")).
+		MustNot(elastic.NewTermQuery("company_type", "个体工商户"))
+
+	// 聚合：按 groupField 分组
+	countAgg := elastic.NewTermsAggregation().Field(groupField).Size(500)
+	countRes, err := client.Search().
+		Index(indexName).
+		Query(query).
+		Size(0).
+		Aggregation("by_group", countAgg).
+		Do(ctx)
+
+	if err != nil {
+		return fmt.Errorf("获取分组总数失败: %v", err)
+	}
+
+	agg, ok := countRes.Aggregations.Terms("by_group")
+	if !ok {
+		return fmt.Errorf("未找到分组聚合结果")
+	}
+
+	headers := []string{groupField, "company_name", "capital", "company_type", "company_status"}
+
+	sheetIndex := 0
+	currSheet := sheetName
+	f.NewSheet(currSheet)
+	writeHeaders(f, currSheet, headers)
+	row := 2
+
+	for _, bucket := range agg.Buckets {
+		groupVal := bucket.Key
+		docCount := bucket.DocCount
+		if docCount == 0 {
+			continue
+		}
+		size := int(math.Max(1, float64(docCount)*0.05))
+		groupStr := fmt.Sprintf("%v", groupVal)
+
+		subQuery := elastic.NewBoolQuery().
+			Must(elastic.NewTermQuery(groupField, groupStr)).
+			Must(elastic.NewExistsQuery("capital"))
+
+		sourceCtx := elastic.NewFetchSourceContext(true).Include(
+			"company_name", "capital", "company_type", "company_status", groupField,
+		)
+
+		scroll := client.Scroll(indexName).
+			Query(subQuery).
+			SortWithInfo(elastic.SortInfo{
+				Field:     "capital",
+				Ascending: false,
+				Missing:   "_last",
+			}).
+			Size(500).
+			FetchSourceContext(sourceCtx)
+
+		defer scroll.Clear(ctx)
+
+		var fetched int
+		for fetched < size {
+			res, err := scroll.Do(ctx)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Printf("scroll 查询失败 group=%v, err=%v\n", groupVal, err)
+				break
+			}
+
+			for _, hit := range res.Hits.Hits {
+				if fetched >= size {
+					break
+				}
+
+				// 超出最大行数，创建新 sheet
+				if row > maxRowsPerSheet {
+					sheetIndex++
+					currSheet = fmt.Sprintf("%s_%d", sheetName, sheetIndex)
+					f.NewSheet(currSheet)
+					writeHeaders(f, currSheet, headers)
+					row = 2
+				}
+
+				var c map[string]interface{}
+				if err := json.Unmarshal(hit.Source, &c); err != nil {
+					continue
+				}
+
+				if util.ObjToString(c["company_status"]) != "存续" {
+					continue
+				}
+
+				data := []interface{}{
+					groupStr,
+					c["company_name"],
+					c["capital"],
+					c["company_type"],
+					c["company_status"],
+				}
+
+				for col, val := range data {
+					cell, _ := excelize.CoordinatesToCellName(col+1, row)
+					f.SetCellValue(currSheet, cell, val)
+				}
+				row++
+				fetched++
+			}
+		}
+	}
+
+	// 设置激活表
+	if sheetName == "按一级分类" {
+		if index, err := f.GetSheetIndex(sheetName); err == nil {
+			f.SetActiveSheet(index)
+		}
+	}
+	return nil
+}
+
+func writeHeaders(f *excelize.File, sheetName string, headers []string) {
+	for i, h := range headers {
+		cell, _ := excelize.CoordinatesToCellName(i+1, 1)
+		f.SetCellValue(sheetName, cell, h)
+	}
+}
+
+func dealXlsx() {
+	log.Println("开始处理数据")
+	// 打开原始文件
+	f, err := excelize.OpenFile("top_companies_by_industry_levels.xlsx")
+	if err != nil {
+		panic(err)
+	}
+
+	// 获取第一个工作表
+	sheetName := f.GetSheetName(3)
+	rows, err := f.GetRows(sheetName)
+	if err != nil {
+		panic(err)
+	}
+
+	// 创建新文件用于保存筛选后的数据
+	newFile := excelize.NewFile()
+	newSheet := newFile.GetSheetName(0)
+
+	// 遍历原始数据并筛选“存续”状态
+	rowIndex := 0
+	for i, row := range rows {
+		if rowIndex == 0 {
+			// 写入表头
+			_ = newFile.SetSheetRow(newSheet, fmt.Sprintf("A%d", rowIndex+1), &row)
+			rowIndex++
+			continue
+		}
+		if i%10000 == 0 {
+			log.Println("iiiiiii", row[1])
+		}
+
+		// 判断公司状态列（假设在第 5 列，即下标 4）
+		if len(row) >= 5 && row[4] == "存续" {
+			_ = newFile.SetSheetRow(newSheet, fmt.Sprintf("A%d", rowIndex+1), &row)
+			rowIndex++
+		}
+	}
+
+	// 保存新文件
+	if err := newFile.SaveAs("筛选后-存续公司3.xlsx"); err != nil {
+		panic(err)
+	}
+
+	log.Println("筛选完成，结果保存为 筛选后-存续公司3.xlsx")
+}

project_chuan/config.go

@@ -0,0 +1,85 @@
+package main
+
+import "time"
+
+type GlobalConf struct {
+	MongoP   MgoConf
+	MongoB   MgoConf
+	Mongo181 MgoConf
+	Env      EnvConf
+	Mysql    MysqlConf
+	Log      Log
+	Es       EsConf
+}
+
+type MgoConf struct {
+	Host     string
+	DB       string
+	Coll     string // 查询表
+	Username string
+	Password string
+	Size     int
+	Direct   bool
+}
+
+type Log struct {
+	LogPath    string
+	MaxSize    int
+	Compress   bool
+	MaxAge     int
+	MaxBackups int
+	LogLevel   string
+	Format     string
+}
+
+type EnvConf struct {
+	File      string
+	Sheet     string
+	Savecoll  string
+	BidFields []string
+	NjFields  []string
+}
+
+type MysqlConf struct {
+	Address  string
+	Dbname   string
+	Dbname2  string
+	Dbname3  string
+	Username string
+	Password string
+	Table    string
+	Table2   string
+	Table3   string
+	Table4   string
+	Table5   string
+	Table6   string
+	Table7   string
+	Stime    string
+	Etime    string
+	Test     bool
+}
+
+type EsConf struct {
+	URL      string
+	Username string
+	Password string
+	Index    string
+}
+
+type DwdFnzjFollowRecord struct {
+	ID               uint64     `gorm:"column:id;primaryKey;autoIncrement;not null;comment:自增主键" json:"id"`
+	ProposedID       string     `gorm:"column:proposed_id;type:varchar(32);not null;comment:拟在建项目ID" json:"proposed_id"`
+	InfoID           string     `gorm:"column:infoid;type:varchar(32);comment:标讯id" json:"infoid"`
+	Title            string     `gorm:"column:title;type:varchar(500);comment:标题" json:"title"`
+	FollowNum        string     `gorm:"column:follow_num;type:varchar(15);comment:跟进数量" json:"follow_num"`
+	ProjectStageCode string     `gorm:"column:project_stage_code;type:varchar(100);comment:进展阶段" json:"project_stage_code"`
+	JYBXHref         string     `gorm:"column:jybxhref;type:varchar(500);comment:剑鱼标讯链接" json:"jybxhref"`
+	ProjectScale     string     `gorm:"column:project_scale;type:text;comment:建设规模及建设内容" json:"project_scale"`
+	PublishTime      *time.Time `gorm:"column:publishtime;type:datetime;comment:跟进时间（发布时间）" json:"publishtime"`
+	CreateTime       time.Time  `gorm:"column:createtime;type:datetime;not null;comment:创建时间" json:"createtime"`
+}
+
+// TableName sets the insert table name for this struct type
+func (DwdFnzjFollowRecord) TableName() string {
+	return "dwd_f_nzj_follw_record"
+}

project_chuan/config.toml

@@ -0,0 +1,61 @@
+[mongob]  ## bidding 数据
+    host = "127.0.0.1:27083"
+    #host = "172.31.31.202:27081,172.20.45.128:27080",
+    db = "qfw"
+    coll = "wcc_bidding_0507"
+    username = "SJZY_RWbid_ES"
+    password = "SJZY@B4i4D5e6S"
+    size = 10
+    direct = true
+
+[mongop]  ## 85 项目信息数据库
+    host = "127.0.0.1:27080"
+    #host = "172.17.4.85:27080"
+    db = "qfw"
+    username = ""
+    password = ""
+    size = 10
+    direct = true
+
+
+[mysql]
+    address = "127.0.0.1:4001" ## 本地
+    #    address = "172.17.162.27:14000" ## 线上
+    dbname=   "Jianyu_subjectdb" ##
+    username = "datascbi"
+    password= "Da#Bi20221111SC"
+    etime = ""  ## 数据截止时间，"2024-11-02"
+    test = false
+
+
+[es]
+#    url = "http://172.17.4.184:19908" ## 线上地址
+    url = "http://127.0.0.1:19908"
+    username = "jybid"
+    password = "Top2023_JEB01i@31"
+
+
+[env]
+    file = "./data.xlsx" ## 读取文件
+    sheet= "Sheet1" ## 默认读取数据在Sheet1
+
+
+
+## 日志
+# 日志
+[log]
+# 日志路径，为空将输出控制台
+logpath = ""
+#    logpath = "logs/log.out"
+# log size (M)
+maxsize = 10
+# compress log
+compress = true
+# log save  time (day)
+maxage =  7
+# save total log file total
+maxbackups = 10
+# log level
+loglevel  = "debug"
+# text or json output
+format = "json"

project_chuan/go.mod

@@ -0,0 +1,51 @@
+module project_chuan
+
+go 1.23.7
+
+require (
+	github.com/olivere/elastic/v7 v7.0.32
+	github.com/spf13/viper v1.20.1
+	go.uber.org/zap v1.22.0
+	gorm.io/driver/mysql v1.5.7
+	gorm.io/gorm v1.26.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

project_chuan/go.sum

@@ -0,0 +1,252 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
+github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
+github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
+github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
+github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
+github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
+gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

project_chuan/init.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"os"
+	"time"
+)
+
+var (
+	MgoB            *mongodb.MongodbSim //
+	MgoP            *mongodb.MongodbSim //
+	GF              GlobalConf
+	JianyuSubjectDB *gorm.DB
+)
+
+func init() {
+	err := InitConfig()
+	if err != nil {
+		fmt.Println("InitConfig err", err)
+		os.Exit(1)
+	}
+
+	InitLog()
+	InitMgo()
+	//InitMysql()
+	//
+
+}
+
+func InitConfig() (err error) {
+	viper.SetConfigFile("config.toml") // 指定配置文件路径
+	viper.SetConfigName("config")      // 配置文件名称(无扩展名)
+	viper.SetConfigType("toml")        // 如果配置文件的名称中没有扩展名，则需要配置此项
+
+	viper.AddConfigPath("./")
+	viper.AddConfigPath("./conf/")  // 还可以在工作目录中查找配置
+	viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置
+	err = viper.ReadInConfig()      // 查找并读取配置文件
+	if err != nil {                 // 处理读取配置文件的错误
+		return
+	}
+
+	err = viper.Unmarshal(&GF)
+
+	return err
+
+}
+
+func InitMgo() {
+	//181 凭安库
+	//Mgo181 = &mongodb.MongodbSim{
+	//	MongodbAddr: GF.Mongo181.Host,
+	//	Size:        10,
+	//	DbName:      GF.Mongo181.DB,
+	//	UserName:    GF.Mongo181.Username,
+	//	Password:    GF.Mongo181.Password,
+	//	Direct:      GF.Mongo181.Direct,
+	//}
+	//Mgo181.InitPool()
+	MgoP = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoP.Host,
+		Size:        10,
+		DbName:      GF.MongoP.DB,
+		UserName:    GF.MongoP.Username,
+		Password:    GF.MongoP.Password,
+		Direct:      GF.MongoP.Direct,
+	}
+
+	MgoP.InitPool()
+
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoB.Host,
+		Size:        10,
+		DbName:      GF.MongoB.DB,
+		UserName:    GF.MongoB.Username,
+		Password:    GF.MongoB.Password,
+		Direct:      GF.MongoB.Direct,
+	}
+
+	MgoB.InitPool()
+}
+
+func InitLog() {
+	now := time.Now()
+	err := log.InitLog(
+		log.Path(GF.Log.LogPath),
+		log.Level(GF.Log.LogLevel),
+		log.Compress(GF.Log.Compress),
+		log.MaxSize(GF.Log.MaxSize),
+		log.MaxBackups(GF.Log.MaxBackups),
+		log.MaxAge(GF.Log.MaxAge),
+		log.Format(GF.Log.Format),
+	)
+	if err != nil {
+		fmt.Printf("InitLog failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	log.Info("InitLog", zap.Any("duration", time.Since(now).Seconds()))
+}
+
+func InitMysql() {
+	now := time.Now()
+	username := GF.Mysql.Username
+	password := GF.Mysql.Password
+	host := GF.Mysql.Address // 本地
+	//host := "172.17.162.27:14000" //线上
+
+	var err error
+	dsn3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, GF.Mysql.Dbname)
+	JianyuSubjectDB, err = gorm.Open(mysql.Open(dsn3), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Error), //不打印日志
+	})
+	if err != nil {
+		log.Info("InitMysql", zap.String("初始化MySQL", "Jianyu_subjectdb 数据库失败"), zap.Error(err))
+		return
+	} else {
+		log.Info("InitMysql", zap.String("初始化MySQL", "Jianyu_subjectdb 数据库成功"))
+	}
+
+	log.Info("InitMysql", zap.Any("duration", time.Since(now).Seconds()))
+
+}

project_chuan/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"go.uber.org/zap"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+)
+
+func main() {
+	dealProposed22Concurrent()
+	//dealProposed()
+	log.Info("main", zap.String("aaaa", "数据处理完毕"))
+}

project_chuan/project.go

@@ -0,0 +1,566 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"go.uber.org/zap"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"strings"
+	"sync"
+)
+
+// dealProposed22Concurrent 多协程处理
+func dealProposed22Concurrent() {
+	// 1. 初始化 ES 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(GF.Es.URL),
+		elastic.SetBasicAuth(GF.Es.Username, GF.Es.Password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatal("创建 Elasticsearch 客户端失败", zap.Error(err))
+	}
+
+	// 2. 初始化 MongoDB 连接
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+
+	coll := sess.DB("qfw").C("projectset_proposed")
+	query := map[string]interface{}{
+		"firsttime": map[string]interface{}{
+			"$gte": 1735660800,
+		},
+	}
+	iter := coll.Find(query).Select(nil).Iter()
+
+	// 3. 并发控制
+	const maxWorkers = 20
+	taskCh := make(chan map[string]interface{}, 2000)
+	var wg sync.WaitGroup
+
+	// 4. 启动 worker 处理任务
+	for i := 0; i < maxWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for doc := range taskCh {
+				processOneProposed(doc, client)
+			}
+		}()
+	}
+
+	// 5. 逐条读取数据并派发任务
+	log.Info("111111", zap.String("222222", "开始处理数据"))
+	count := 0
+	for doc := make(map[string]interface{}); iter.Next(doc); {
+		count++
+		if count%1000 == 0 {
+			log.Info("dealProposed", zap.Int("current", count), zap.Any("projectname", doc["projectname"]))
+		}
+		taskCh <- cloneMap(doc) // 防止 map 重用
+	}
+	close(taskCh)
+	wg.Wait()
+}
+
+func processOneProposed(tmp map[string]interface{}, client *elastic.Client) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warn("panic in processOneProposed", zap.Any("recover", r))
+		}
+	}()
+
+	proposedID := mongodb.BsonIdToSId(tmp["_id"])
+	projectName := util.ObjToString(tmp["projectname"])
+	buyer := util.ObjToString(tmp["owner"])
+
+	results, err := searchES22(client, projectName, buyer, 75, 10)
+	if err != nil {
+		log.Warn("searchES22 error", zap.Error(err))
+		return
+	}
+
+	projectIds := []string{}
+	biddingIds := []string{}
+	biddings := []map[string]interface{}{}
+
+	for _, re := range results {
+		biddingID := util.ObjToString(re["id"])
+		biddingIds = append(biddingIds, biddingID)
+		biddings = append(biddings, map[string]interface{}{
+			"id":          re["id"],
+			"title":       re["title"],
+			"projectname": re["projectname"],
+			"score":       re["score"],
+			"toptype":     re["toptype"],
+			"subtype":     re["subtype"],
+		})
+	}
+
+	for _, bid := range biddingIds {
+		where2 := map[string]interface{}{"ids": bid}
+		projectset, _ := MgoP.FindOne("projectset_20230904", where2)
+		if projectset != nil && len(*projectset) > 0 {
+			projectIds = append(projectIds, mongodb.BsonIdToSId((*projectset)["_id"]))
+		}
+	}
+
+	insert := map[string]interface{}{
+		"proposed_id":  proposedID,
+		"bidding_ids":  removeDuplicates(biddingIds),
+		"project_ids":  removeDuplicates(projectIds),
+		"biddings":     biddings,
+		"project_name": tmp["projectname"],
+	}
+
+	MgoP.InsertOrUpdate("qfw", "wcc_dealProposed22", insert)
+}
+
+func cloneMap(src map[string]interface{}) map[string]interface{} {
+	dst := make(map[string]interface{}, len(src))
+	for k, v := range src {
+		dst[k] = v
+	}
+	return dst
+}
+
+func dealProposed22() {
+	url := GF.Es.URL
+	//url := "http://127.0.0.1:19908"
+	username := GF.Es.Username
+	password := GF.Es.Password
+	//index := "bidding" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Info("创建 Elasticsearch 客户端失败", zap.Error(err))
+	}
+
+	//
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+
+	log.Info("dealProposed", zap.Any("开始处理：拟建数据表", "projectset_proposed"))
+	where := map[string]interface{}{
+		"firsttime": map[string]interface{}{
+			"$gte": 1735660800,
+		},
+	}
+	queryMgo := sess.DB("qfw").C("projectset_proposed").Find(&where).Select(nil).Iter()
+	count := 0
+	for tmp := make(map[string]interface{}); queryMgo.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Info("dealProposed", zap.Any("current", count), zap.Any("projectname", tmp["projectname"]))
+		}
+
+		proposed_id := mongodb.BsonIdToSId(tmp["_id"])
+		project_name := util.ObjToString(tmp["projectname"])
+		buyer := util.ObjToString(tmp["owner"])
+		results, err := searchES22(client, project_name, buyer, 60, 10)
+		if err != nil {
+			log.Info("searchES22", zap.Error(err))
+		}
+
+		projectIds := make([]string, 0) //拟建对应的在建项目ID
+		biddingIds := make([]string, 0) //拟建项目对在建项目中的标讯ids
+		biddings := make([]map[string]interface{}, 0)
+
+		for _, re := range results {
+			bidding_id := util.ObjToString(re["id"])
+			biddingIds = append(biddingIds, bidding_id)
+			bidding := map[string]interface{}{
+				"id":          re["id"],
+				"title":       re["title"],
+				"projectname": re["projectname"],
+				"score":       re["score"],
+				"toptype":     re["toptype"],
+				"subtype":     re["subtype"],
+			}
+			biddings = append(biddings, bidding)
+		}
+
+		for _, bid := range biddingIds {
+			where2 := map[string]interface{}{
+				"ids": bid,
+			}
+			projectset, _ := MgoP.FindOne("projectset_20230904", where2)
+			if projectset != nil && len((*projectset)) > 0 {
+				projectIds = append(projectIds, mongodb.BsonIdToSId((*projectset)["_id"]))
+			}
+		}
+
+		insert := map[string]interface{}{
+			"proposed_id":  proposed_id,
+			"bidding_ids":  removeDuplicates(biddingIds),
+			"project_ids":  removeDuplicates(projectIds),
+			"biddings":     biddings,
+			"project_name": tmp["projectname"],
+		}
+
+		MgoP.InsertOrUpdate("qfw", "wcc_dealProposed22", insert)
+	}
+}
+
+// dealProposed 处理拟建数据表
+func dealProposed() {
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+
+	log.Info("dealProposed", zap.Any("开始处理：拟建数据表", "projectset_proposed"))
+	where := map[string]interface{}{
+		"firsttime": map[string]interface{}{
+			"$gte": 1735660800,
+		},
+	}
+	queryMgo := sess.DB("qfw").C("projectset_proposed").Find(&where).Select(nil).Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); queryMgo.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Info("dealProposed", zap.Any("current", count), zap.Any("projectname", tmp["projectname"]))
+		}
+
+		proposed_id := mongodb.BsonIdToSId(tmp["_id"])
+
+		insert := make(map[string]interface{})
+		insert["proposed_id"] = proposed_id
+		var nzj_follw_records = make([]DwdFnzjFollowRecord, 0)
+		err := JianyuSubjectDB.Where("proposed_id = ? ", proposed_id).Find(&nzj_follw_records).Error
+		if err != nil {
+			log.Info("dealProposed", zap.Error(err))
+		}
+		//拟建标讯，没有找到对应的在建项目数据
+		if len(nzj_follw_records) == 0 {
+			insert["has_bidding"] = false
+			MgoP.InsertOrUpdate("qfw", "wcc_ok_project_proposed", insert)
+			continue
+		}
+
+		projectIds := make([]string, 0) //拟建对应的在建项目ID
+		biddingIds := make([]string, 0) //拟建项目对在建项目中的标讯ids
+		for _, v := range nzj_follw_records {
+			biddingIds = append(biddingIds, v.InfoID)
+		}
+
+		biddingIds = removeDuplicates(biddingIds)
+		insert["bidding_ids"] = biddingIds
+
+		for _, bid := range biddingIds {
+			where2 := map[string]interface{}{
+				"ids": bid,
+			}
+			projectset, _ := MgoP.FindOne("projectset_20230904", where2)
+			if projectset != nil && len((*projectset)) > 0 {
+				projectIds = append(projectIds, mongodb.BsonIdToSId((*projectset)["_id"]))
+			}
+		}
+
+		if len(projectIds) > 0 {
+			insert["project_ids"] = projectIds
+		} else {
+			insert["has_project"] = false
+		}
+
+		MgoP.InsertOrUpdate("qfw", "wcc_ok_project_proposed", insert)
+	}
+
+	log.Info("dealProposed", zap.Any("数据处理完毕：拟建数据表", "projectset_proposed"))
+}
+
+// removeDuplicates 去除重复字符串
+func removeDuplicates(arr []string) []string {
+	uniqueMap := make(map[string]bool)
+	var result []string
+	for _, str := range arr {
+		if !uniqueMap[str] {
+			uniqueMap[str] = true
+			result = append(result, str)
+		}
+	}
+	return result
+}
+
+func searchES22(client *elastic.Client, projectName, buyer2 string, scoreThreshold float64, maxResults int) ([]map[string]interface{}, error) {
+	fieldsToTry := []string{"projectname.pname", "title", "detail"}
+	filtersToTry := [][]elastic.Query{
+		{elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一")},
+		{elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向")},
+		{elastic.NewTermsQuery("toptype", "拟建")},
+	}
+
+	var allResults []*elastic.SearchHit
+	seenIDs := make(map[string]bool)
+
+	for _, field := range fieldsToTry {
+		for _, filter := range filtersToTry {
+			// 构建查询
+			query := elastic.NewBoolQuery().
+				Must(elastic.NewMatchQuery(field, projectName)).
+				Filter(filter...)
+
+			// 执行查询
+			searchResult, err := client.Search().
+				Index("bidding").
+				Query(query).
+				Size(70). // 多取一些，后面做筛选和去重
+				Do(context.Background())
+			if err != nil {
+				return nil, err
+			}
+
+			for _, hit := range searchResult.Hits.Hits {
+				if !seenIDs[hit.Id] {
+					allResults = append(allResults, hit)
+					seenIDs[hit.Id] = true
+				}
+			}
+
+			if len(allResults) >= maxResults {
+				break
+			}
+		}
+		if len(allResults) >= maxResults {
+			break
+		}
+	}
+
+	var results []map[string]interface{}
+	seenProjectNames := make(map[string]bool)
+	seenProjectCodes := make(map[string]bool)
+	bidamountMap := make(map[float64]bool)
+
+	for _, hit := range allResults {
+		var doc map[string]interface{}
+		if err := json.Unmarshal(hit.Source, &doc); err != nil {
+			log.Info("解析文档失败", zap.Error(err))
+			continue
+		}
+
+		projectNameValue := util.ObjToString(doc["projectname"])
+		if projectNameValue == "" {
+			continue
+		}
+
+		projectCode := util.ObjToString(doc["projectcode"])
+		if seenProjectCodes[projectCode] {
+			continue
+		}
+		seenProjectCodes[projectCode] = true
+
+		bidamount := util.Float64All(doc["bidamount"])
+		if bidamountMap[bidamount] {
+			continue
+		}
+		bidamountMap[bidamount] = true
+
+		// 相似度筛选
+		score := *hit.Score
+		doc["score"] = score //相似度
+		if score < scoreThreshold {
+			continue
+		}
+
+		//id := util.ObjToString(doc["id"])
+		//doc["jyhref"] = GetJyURLByID(id)
+
+		//if site := util.ObjToString(doc["site"]); site == "中华人民共和国自然资源部" {
+		//	doc["title"] = "土地出让" + "-" + util.ObjToString(doc["title"])
+		//}
+
+		// enrich: total_investment
+		//if bidData, _ := MgoB.FindById("bidding", id, nil); bidData != nil {
+		//	if util.Float64All((*bidData)["total_investment"]) > 0 {
+		//		doc["total_investment"] = (*bidData)["total_investment"]
+		//	}
+		//}
+
+		doc["score"] = score
+		detail := util.ObjToString(doc["detail"])
+
+		// 字段中必须包含 projectName
+		if buyer2 != "" {
+			if !strings.Contains(detail, projectName) && !strings.Contains(detail, buyer2) {
+				continue
+			}
+		}
+
+		if seenProjectNames[projectNameValue] {
+			continue
+		}
+		seenProjectNames[projectNameValue] = true
+
+		results = append(results, doc)
+
+		if len(results) >= maxResults {
+			break
+		}
+	}
+
+	return results, nil
+}
+
+func searchES(client *elastic.Client, projectName, buyer2 string) ([]map[string]interface{}, error) {
+	query := elastic.NewBoolQuery().
+		Must(
+			//elastic.NewMatchQuery("projectname.pname", projectName), // 模糊匹配 projectname
+			//elastic.NewMatchQuery("title", projectName), // 模糊匹配 projectname
+			elastic.NewMatchQuery("detail", projectName), // 模糊匹配 projectname
+			//elastic.NewTermQuery("area", "安徽"),                       // 过滤区域
+			elastic.NewTermsQuery("subtype", "中标", "成交", "合同", "单一"), // 过滤 subtype
+			//elastic.NewTermsQuery("toptype", "招标", "预告", "采购意向"), // 过滤 subtype
+			//elastic.NewTermsQuery("toptype", "拟建"), // 过滤 subtype
+		)
+
+	searchResult, err := client.Search().
+		Index("bidding").
+		Query(query).
+		Size(70). // 先取 12 条，确保足够数据
+		Do(context.Background())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// 结果集
+	var results []map[string]interface{}
+	seenProjectNames := make(map[string]bool) // 用于去重
+	seenProjectCode := make(map[string]bool)  // 用于去重
+	bidamountMap := make(map[float64]bool)
+	for _, hit := range searchResult.Hits.Hits {
+		var doc map[string]interface{}
+		err := json.Unmarshal(hit.Source, &doc)
+		if err != nil {
+			log.Info("解析文档失败", zap.Error(err))
+			continue
+		}
+
+		// 获取 `projectname`，防止 key 不存在时的错误
+		projectNameValue, ok := doc["projectname"].(string)
+
+		bidamount := util.Float64All(doc["bidamount"])
+		if !ok {
+			log.Info("⚠️ 缺少 projectname 字段，跳过：", zap.Any("projectname", doc["projectname"]))
+			continue
+		}
+
+		projectCodeValue := util.ObjToString(doc["projectcode"])
+
+		if seenProjectCode[projectCodeValue] {
+			continue
+		}
+		if projectCodeValue != "" {
+			seenProjectCode[projectCodeValue] = true
+		}
+
+		// **处理额外字段**
+		id := util.ObjToString(doc["id"])
+		bidData, _ := MgoB.FindById("bidding", id, nil)
+		if util.Float64All((*bidData)["total_investment"]) > 0 {
+			doc["total_investment"] = (*bidData)["total_investment"]
+		}
+		doc["jyhref"] = GetJyURLByID(id)
+		score := *hit.Score
+
+		site := util.ObjToString(doc["site"])
+		if site == "中华人民共和国自然资源部" {
+			doc["title"] = "土地出让" + "-" + util.ObjToString(doc["title"])
+		}
+		doc["score"] = score //相似度
+
+		detail := util.ObjToString(doc["detail"])
+		if !strings.Contains(detail, projectName) {
+			continue
+		}
+
+		// **去重逻辑**：如果 `projectname` 已经出现过，则跳过
+		if seenProjectNames[projectNameValue] {
+			continue
+		}
+
+		if bidamountMap[bidamount] {
+			continue
+		}
+		// **记录该 `projectname`，避免重复**
+		seenProjectNames[projectNameValue] = true
+		bidamountMap[bidamount] = true
+
+		// **加入结果集**
+		results = append(results, doc)
+
+		// **如果已经找到 6 条不同 `projectname`，就跳出循环**
+		if len(results) >= 10 {
+			break
+		}
+	}
+
+	//2、判断正文包含采购单位
+	for _, hit := range searchResult.Hits.Hits {
+		var doc map[string]interface{}
+		err := json.Unmarshal(hit.Source, &doc)
+		if err != nil {
+			log.Info("解析文档失败：", zap.Error(err))
+			continue
+		}
+
+		// 获取 `projectname`，防止 key 不存在时的错误
+		projectNameValue, ok := doc["projectname"].(string)
+		bidamount := util.Float64All(doc["bidamount"])
+		if !ok {
+			log.Info("⚠️ 缺少 projectname 字段，跳过：", zap.Any("projectname", doc["projectname"]))
+			continue
+		}
+
+		// **处理额外字段**
+		id := util.ObjToString(doc["id"])
+		doc["jyhref"] = GetJyURLByID(id)
+		score := *hit.Score
+		doc["score"] = score //相似度
+		site := util.ObjToString(doc["site"])
+		if site == "中华人民共和国自然资源部" {
+			doc["title"] = "土地出让" + "-" + util.ObjToString(doc["title"])
+		}
+
+		//判断正文包含采购单位
+		detail := util.ObjToString(doc["detail"])
+		if !strings.Contains(detail, buyer2) {
+			continue
+		}
+		// **去重逻辑**：如果 `projectname` 已经出现过，则跳过
+		if seenProjectNames[projectNameValue] {
+			continue
+		}
+		if bidamountMap[bidamount] {
+			continue
+		}
+
+		// **记录该 `projectname`，避免重复**
+		seenProjectNames[projectNameValue] = true
+		bidamountMap[bidamount] = true
+
+		// **加入结果集**
+		results = append(results, doc)
+
+		// **如果已经找到 6 条不同 `projectname`，就跳出循环**
+		if len(results) >= 10 {
+			break
+		}
+	}
+
+	return results, nil
+}
+
+// GetJyURLByID 获取剑鱼地址
+func GetJyURLByID(id string) string {
+	var Url = "https://www.jianyu360.com/article/content/%s.html"
+	url := fmt.Sprintf(Url, util.CommonEncodeArticle("content", id))
+
+	return url
+}

project_chuan/readme.txt

@@ -0,0 +1,31 @@
+处理项目的所有串联关系
+项目拟建-在建-标讯  的所有对应关系
+
+拟建数据--> 对应的在建项目数据--> 对应的标讯数据
+
+
+数据范围：2025年之后的数据
+
+
+处理逻辑：
+1、遍历拟建数据表 projectset_proposed,然后拿 拟建项目ID去 表 projectset_comb 查询，
+projectset_comb 的 主键ID 是 projectset_proposed的主键ID，projectset_comb.ids 是
+数组，里面的pid 对应的是在建项目 projectset_20230904 的主键ID。
+ 需要记录一下拟建和在建的关联关系，一个拟建对应多个在建项目；还有对应的标讯项目ID
+
+拟建ID，项目ids，标讯ids
+
+没有匹配上在建项目的，在建信息直接为空
+
+2、遍历项目数据，获取项目信息，然后排除掉已经关联拟建的数据，逆向反推出 拟建数据，写入数据表
+
+
+---------------------
+
+结果数据表：wcc_ok_project_proposed
+
+has_project:代表，是否找到了对应的在建项目，没有找到是false，找到了就没有该字段
+proposed_id: 拟建项目对应的主键ID
+bidding_ids:对应的标讯信息id，是个数组
+
+

qyxy_stand_tags/es.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"io"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"log"
+)
+
+func updateEs() {
+	url := "http://172.17.4.184:19908"
+	//url := "http://127.0.0.1:19908"
+	username := "jybid"
+	password := "Top2023_JEB01i@31"
+	index := "qyxy" //索引名称
+	// 创建 Elasticsearch 客户端
+	client, err := elastic.NewClient(
+		elastic.SetURL(url),
+		elastic.SetBasicAuth(username, password),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+	// 构建查询条件
+	//query := elastic.NewBoolQuery().
+	//	MustNot(elastic.NewTermQuery("company_type", "个体工商户")) // 排除 company_type 为 "个体工商户"
+	////Filter(elastic.NewTermQuery("company_area", "广东"))      // 过滤 company_area 为 "广东"
+
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewMatchQuery("company_status", "存续")).
+		MustNot(elastic.NewMatchQuery("company_type", "个体工商户"))
+
+	ctx := context.Background()
+	//开始滚动搜索
+	scrollID := ""
+	scroll := "10m"
+	searchSource := elastic.NewSearchSource().
+		Query(query).
+		Size(10000).
+		//Sort("_doc", true) //升序排序
+		Sort("_doc", false) //降序排序
+
+	searchService := client.Scroll(index).
+		Size(10000).
+		Scroll(scroll).
+		SearchSource(searchSource)
+
+	res, err := searchService.Do(ctx)
+
+	if err != nil {
+		if err == io.EOF {
+			fmt.Println("没有数据")
+		} else {
+			panic(err)
+		}
+
+	}
+	//defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	fmt.Println("总数是：", res.TotalHits())
+	num := 0
+	total := 0
+	for len(res.Hits.Hits) > 0 {
+		for _, hit := range res.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+
+			companyID := util.ObjToString(doc["id"])
+			where := map[string]interface{}{
+				"company_id": companyID,
+			}
+
+			industry, _ := Mgo181.FindOne("company_industry", where)
+			if industry != nil && len(*industry) > 0 {
+				num++
+				update := make(map[string]interface{}, 0)
+				if util.ObjToString((*industry)["industry_l1_name"]) != "" {
+					update["national_top"] = (*industry)["industry_l1_name"]
+				}
+				if util.ObjToString((*industry)["industry_l2_name"]) != "" {
+					update["national_sub"] = (*industry)["industry_l2_name"]
+				}
+
+				if util.ObjToString((*industry)["industry_l3_name"]) != "" {
+					update["national_subsub"] = (*industry)["industry_l3_name"]
+				}
+				if len(update) > 0 {
+					if num%10000 == 0 {
+						log.Println("cccccc", num, companyID)
+					}
+					//where := map[string]interface{}{
+					//	"_id": companyID,
+					//}
+					//MgoT.Update("qyxy_std", where, map[string]interface{}{"$set": updateMgo}, true, false)
+					//MgoQy.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
+					updatePool <- []map[string]interface{}{
+						{"_id": companyID},
+						{"$set": update},
+					}
+					//
+					// 更新es
+					updateEsPool <- []map[string]interface{}{
+						{"_id": companyID},
+						update,
+					}
+				}
+			}
+		}
+
+		total = total + len(res.Hits.Hits)
+		scrollID = res.ScrollId
+		res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+		log.Println("current count:", total)
+		if err != nil {
+			if err == io.EOF {
+				// 滚动到最后一批数据，退出循环
+				break
+			}
+			log.Println("滚动搜索失败：", err, res)
+			break // 处理错误时退出循环
+		}
+	}
+	// 在循环外调用 ClearScroll
+	_, err = client.ClearScroll().ScrollId(scrollID).Do(ctx)
+	if err != nil {
+		log.Printf("清理滚动搜索失败：%s", err)
+	}
+
+	fmt.Println("结束~~~~~~~~~~~~~~~")
+
+}

qyxy_stand_tags/go.mod

@@ -0,0 +1,28 @@
+module qyxy_stand_tags
+
+go 1.23.7
+
+require jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

qyxy_stand_tags/go.sum

@@ -0,0 +1,198 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

qyxy_stand_tags/init.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	es "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+)
+
+func InitEs() {
+	Es = &es.Elastic{
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   10,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+}
+
+func InitMgo() {
+	MgoQy = &mongodb.MongodbSim{
+		MongodbAddr: "172.31.31.202:27081,172.20.45.128:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		Size:     10,
+		DbName:   "mixdata",
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoQy.InitPool()
+
+	Mgo181 = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.4.181:27001",
+		//MongodbAddr: "127.0.0.1:27001",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "",
+		Password: "",
+		//Direct:      true,
+	}
+	Mgo181.InitPool()
+
+}

qyxy_stand_tags/main.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	es "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"time"
+)
+
+var (
+	MgoQy        *mongodb.MongodbSim
+	Mgo181       *mongodb.MongodbSim
+	Es           *es.Elastic
+	updatePool   = make(chan []map[string]interface{}, 20000)
+	updateEsPool = make(chan []map[string]interface{}, 20000)
+	updateEsSp   = make(chan bool, 10) //保存协程
+)
+
+func main() {
+	InitEs()
+	InitMgo()
+	go updateEsMethod()
+	go updateMethod()
+
+	updateEs()
+
+	select {}
+
+}
+
+// updateAllQyxy 处理更新 企业库 存量数据标签
+func updateAllQyxy() {
+	log.Println("开始处理数据  updateAllQyxy")
+	sess := Mgo181.GetMgoConn()
+	defer Mgo181.DestoryMongoConn(sess)
+	where11 := map[string]interface{}{
+		"_id": map[string]interface{}{
+			"$gt": 9039986,
+		},
+	}
+	queryMgo := sess.DB("mixdata").C("company_industry").Find(&where11).Select(nil).Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); queryMgo.Next(tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count, tmp["company_id"])
+		}
+
+		companyID := util.ObjToString(tmp["company_id"])
+		update := make(map[string]interface{}, 0)
+		if util.ObjToString(tmp["industry_l1_name"]) != "" {
+			update["national_top"] = tmp["industry_l1_name"]
+		}
+		if util.ObjToString(tmp["industry_l2_name"]) != "" {
+			update["national_sub"] = tmp["industry_l2_name"]
+		}
+
+		if util.ObjToString(tmp["industry_l3_name"]) != "" {
+			update["national_subsub"] = tmp["industry_l3_name"]
+		}
+
+		if len(update) > 0 {
+			//where := map[string]interface{}{
+			//	"_id": companyID,
+			//}
+			//MgoT.Update("qyxy_std", where, map[string]interface{}{"$set": updateMgo}, true, false)
+			//MgoQy.Update("qyxy_std", where, map[string]interface{}{"$set": update}, true, false)
+			updatePool <- []map[string]interface{}{
+				{"_id": companyID},
+				{"$set": update},
+			}
+			//
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": companyID},
+				update,
+			}
+		}
+
+	}
+
+	log.Println("数据迭代完毕")
+}
+
+// updateEsMethod 更新es
+func updateEsMethod() {
+	arru := make([][]map[string]interface{}, 500)
+	indexu := 0
+	for {
+		select {
+		case v := <-updateEsPool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 500 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("qyxy", arru...)
+				}(arru)
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		case <-time.After(100 * time.Millisecond):
+			if indexu > 0 {
+				updateEsSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateEsSp
+					}()
+					Es.UpdateBulk("qyxy", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		}
+	}
+}
+
+// updateMethod 更新MongoDB
+func updateMethod() {
+	updateSp := make(chan bool, 10)
+	arru := make([][]map[string]interface{}, 500)
+	indexu := 0
+	for {
+		select {
+		case v := <-updatePool:
+			arru[indexu] = v
+			indexu++
+			if indexu == 500 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoQy.UpdateBulk("qyxy_std", arru...)
+
+				}(arru)
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		case <-time.After(100 * time.Millisecond):
+			if indexu > 0 {
+				updateSp <- true
+				go func(arru [][]map[string]interface{}) {
+					defer func() {
+						<-updateSp
+					}()
+					MgoQy.UpdateBulk("qyxy_std", arru...)
+				}(arru[:indexu])
+				arru = make([][]map[string]interface{}, 500)
+				indexu = 0
+			}
+		}
+	}
+}

qyxy_stand_tags/readme.txt

@@ -0,0 +1,8 @@
+结合凭安数据，给企业库(qyxy)添加 国标行业分类字段，主要参考 company_industry
+
+qyxy es 索引添加三个字段，对应国标一级、二级和三级分类
+                        national_top  national_sub   national_subsub
+
+
+
+