update

data_tmp/bidding.go

@@ -1,5 +1,212 @@
 package main
 
-func main() {
+import (
+	"fmt"
+	es7 "github.com/olivere/elastic/v7"
+	"go.mongodb.org/mongo-driver/bson"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+)
+
+// getBidding 根据biddingID获取相关数据;深信服需求
+func getBidding() {
+	//es
+	Es := &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+
+	//mongodb 163
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	Mgo2 := &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "mixdata",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	Mgo2.InitPool()
+
+	defer util.Catch()
+	sess := Mgo.GetMgoConn()
+	defer Mgo.DestoryMongoConn(sess)
+	it := sess.DB("qfw").C("20240626_sxf_id").Find(nil).Select(nil).Sort("_id").Iter()
+
+	fmt.Println("taskRun 开始")
+	count := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
+			log.Println("current:", count)
+		}
+		biddingID := util.ObjToString(tmp["id"])
+		err, biddingData := Es.GetById("bidding", biddingID)
+		if err != nil {
+			log.Println("err", biddingID, "获取数据失败")
+			continue
+		}
+
+		whereBuyer := map[string]interface{}{
+			"name": biddingData["buyer"],
+		}
+		// 医院等级
+		hospData, _ := Mgo2.FindOne("buyer_rank", whereBuyer)
+		if hospData != nil && len(*hospData) > 0 {
+			if rank := util.ObjToString((*hospData)["rank"]); rank != "" {
+				biddingData["rank"] = (*hospData)["rank"]
+			}
+		}
+		// 中标单位
+		if s_winner := util.ObjToString(tmp["s_winner"]); s_winner != "" {
+			winner := strings.Split(s_winner, ",")[0] //取第一个中标单位
+			data, _ := Mgo2.FindOne("qyxy_std", bson.M{"company_name": winner})
+			if data != nil && len(*data) > 0 {
+				//注册省份
+				if company_area := util.ObjToString((*data)["company_area"]); company_area != "" {
+					biddingData["company_area"] = company_area
+				}
+				//注册城市
+				if company_city := util.ObjToString((*data)["company_city"]); company_city != "" {
+					biddingData["company_city"] = company_city
+				}
+				//注册区县
+				if company_district := util.ObjToString((*data)["company_district"]); company_district != "" {
+					biddingData["company_district"] = company_district
+				}
+				//中标单位联系人
+				if legal_person := util.ObjToString((*data)["legal_person"]); legal_person != "" {
+					biddingData["legal_person"] = legal_person
+				}
+				//中标单位的注册地址
+				if company_address := util.ObjToString((*data)["company_address"]); company_address != "" {
+					biddingData["company_address"] = company_address
+				}
+				//注册资金"capital" : "324 万","capital" : 124.8,
+				if (*data)["capital"] != nil {
+					biddingData["capital"] = util.Float64All((*data)["capital"])
+				}
+				//注册时间"establish_date" : ISODate("1949-10-01T00:00:00.000+0000")
+				if establish_date := util.ObjToString((*data)["establish_date"]); establish_date != "" {
+					biddingData["establish_date"] = establish_date
+				}
+				//经营范围
+				if business_scope := util.ObjToString((*data)["business_scope"]); business_scope != "" {
+					biddingData["business_scope"] = business_scope
+				}
+				//中标单位的社会征信代码
+				if credit_no := util.ObjToString((*data)["credit_no"]); credit_no != "" {
+					biddingData["winner_credit_no"] = credit_no
+				}
+				//股东名单
+				if stock_name := util.ObjToString((*data)["stock_name"]); stock_name != "" {
+					biddingData["stock_name"] = stock_name
+				}
+				if email := util.ObjToString((*data)["company_email"]); email != "" {
+					biddingData["company_email"] = email
+				}
+				if phone := util.ObjToString((*data)["company_phone"]); phone != "" {
+					biddingData["company_phone"] = phone
+				}
+			}
+		}
+		if agency := util.ObjToString(tmp["agency"]); agency != "" {
+			data, _ := Mgo2.FindOne("qyxy_std", bson.M{"company_name": agency})
+			if data != nil && len(*data) > 0 {
+				//招标代理机构的社会征信代码
+				if credit_no := util.ObjToString((*data)["credit_no"]); credit_no != "" {
+					biddingData["agency_credit_no"] = credit_no
+				}
+			}
+		}
+
+		// 构建 BoolQuery
+		boolQuery := es7.NewBoolQuery().
+			Must(es7.NewTermQuery("ids", biddingID)).
+			MustNot().
+			Should()
+		data := Es.Get("projectset", boolQuery)
+		if data != nil && *data != nil && len(*data) > 0 {
+			biddingData["projectID"] = (*data)[0]["id"]
+		}
+
+		biddingData["jybxhref"] = GetJyURLByID(biddingID)
+
+		buyerEntData, _ := Mgo2.FindOne("qyxy_std", bson.M{"company_name": biddingData["buyer"]})
+		if buyerEntData != nil && len(*buyerEntData) > 0 {
+			if credit_no := util.ObjToString((*buyerEntData)["credit_no"]); credit_no != "" {
+				biddingData["buyer_credit_no"] = credit_no
+			}
+		}
+		//
+		info, _ := Mgo.FindById("bidding", biddingID, nil)
+		if info == nil || len(*info) == 0 {
+			info, _ = Mgo.FindById("bidding_back", biddingID, nil)
+		}
+		if info != nil && len(*info) > 0 {
+			if (*info)["projectinfo"] != nil {
+				projectInfo := util.ObjToMap((*info)["projectinfo"])
+				if projectInfo != nil && len(*projectInfo) > 0 {
+					if (*projectInfo)["attachments"] != nil {
+						filesArr := []map[string]interface{}{}
+						fileUrl := util.ObjToString("http://jy-datafile.oss-cn-beijing.aliyuncs.com/")
+						if attachments, oks := (*projectInfo)["attachments"].(map[string]interface{}); oks {
+							for _, v := range attachments {
+								if atta, ok := v.(map[string]interface{}); ok {
+									if atta["fid"] != nil && util.ObjToString(atta["fid"]) != "" {
+										fileMap := map[string]interface{}{
+											"filename": atta["filename"],
+											"url":      fileUrl + util.ObjToString(atta["fid"]),
+										}
+										filesArr = append(filesArr, fileMap)
+									} else {
+										if atta["org_url"] != nil {
+											fileMap := map[string]interface{}{
+												"filename": atta["filename"],
+												"url":      util.ObjToString(atta["org_url"]),
+											}
+											filesArr = append(filesArr, fileMap)
+										}
+									}
+								}
+							}
+						}
+						//log.Debug("查询附件结果：" + id + ", 附件数量：" + fmt.Sprint(len(filesArr)))
+						if len(filesArr) > 0 {
+							biddingData["filehref"] = filesArr
+						}
+					}
+				}
+			}
+		}
+
+		Mgo.Save("wcc_0626_sxf", biddingData)
+	}
+
+}
+
+// GetJyURLByID 获取剑鱼地址
+func GetJyURLByID(id string) string {
+	var Url = "https://www.jianyu360.com/article/content/%s.html"
+	url := fmt.Sprintf(Url, util.CommonEncodeArticle("content", id))
+
+	return url
 
 }

data_tmp/data_test.go

@@ -1,6 +1,10 @@
 package main
 
 import (
+	"fmt"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	es "jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
 	"testing"
 )
@@ -12,3 +16,90 @@ func TestGetAreaInfo(t *testing.T) {
 	res := GetAreaInfo(data)
 	log.Println(res)
 }
+
+// TestProject 更新测试环境项目字段
+func TestProject(t *testing.T) {
+	// 测试环境
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "192.168.3.206:27002",
+		Size:        10,
+		DbName:      "qfw_data",
+		UserName:    "root",
+		Password:    "root",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+	//mongodb 163
+	//Mgo := &mongodb.MongodbSim{
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27083",
+	//	DbName:      "qfw",
+	//	Size:        10,
+	//	UserName:    "SJZY_RWbid_ES",
+	//	Password:    "SJZY@B4i4D5e6S",
+	//	Direct:      true,
+	//}
+	//Mgo.InitPool()
+
+	// 项目数据
+	MgoP := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.85:27080",
+		MongodbAddr: "127.0.0.1:27080",
+		Size:        10,
+		DbName:      "qfw",
+		Direct:      true,
+	}
+	MgoP.InitPool()
+
+	//测试环境
+	Es := &es.Elastic{
+		S_esurl:  "http://192.168.3.149:9201",
+		I_size:   5,
+		Username: "",
+		Password: "",
+	}
+	Es.InitElasticSize()
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+
+	it := sess.DB("qfw_data").C("projectset").Find(nil).Select(nil).Iter()
+	count := 0
+	log.Println("开始")
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%5000 == 0 {
+			log.Println("current:", count, tmp["projectname"])
+		}
+
+		projectID := mongodb.BsonIdToSId(tmp["_id"])
+		project, _ := MgoP.FindById("projectset_20230904", projectID, nil)
+		if project != nil && len(*project) > 0 {
+			if tag_top, ok := (*project)["tag_topinformation"]; ok {
+				update := map[string]interface{}{
+					"tag_topinformation": tag_top,
+				}
+				Es.UpdateDocument("projectset", projectID, update)
+			}
+		}
+
+	}
+
+}
+
+func TestTimeStr(T *testing.T) {
+	// Test cases
+	fmt.Println(getTTMethod("2024"))        // Expected: 20240000
+	fmt.Println(getTTMethod("202409"))      // Expected: 20240900
+	fmt.Println(getTTMethod("20240929"))    // Expected: 20240929
+	fmt.Println(getTTMethod("2024/4/11"))   // Expected: 20240411
+	fmt.Println(getTTMethod("2024-01-01"))  // Expected: 20240101
+	fmt.Println(getTTMethod("2024.02.11"))  // Expected: 20240211
+	fmt.Println(getTTMethod("2024年"))       // Expected: 20240000
+	fmt.Println(getTTMethod("2024年09月"))    // Expected: 20240900
+	fmt.Println(getTTMethod("2024年09月29日")) // Expected: 20240929
+	fmt.Println(getTTMethod("2024/9/9"))    // Expected: 20240909
+	fmt.Println(getTTMethod("2024-9-9"))    // Expected: 20240909
+	fmt.Println(getTTMethod("2024.9.9"))    // Expected: 20240909
+	fmt.Println(getTTMethod("2024年9月9日"))   // Expected: 20240909
+}

data_tmp/go.mod

@@ -13,9 +13,12 @@ require (
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect

data_tmp/go.sum

@@ -51,6 +51,7 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
@@ -59,12 +60,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

data_tmp/main.go

@@ -15,7 +15,9 @@ func main() {
 	//getWinnerArea()
 	//winnerCity()
 	//test1()
-	statisticWinner()
+	//statisticWinner()
+	getBidding()
+	log.Println("处理结束")
 }
 
 // getWinnerArea  获取中标单位所在区域

data_tmp/test.go

@@ -1,12 +1 @@
 package main
-
-import "log"
-
-func test1() {
-	data := map[string]interface{}{
-		//"detail": "北京鑫台华科技有限公司",
-		"buyer": "北京鑫台华科技有限公司",
-	}
-	res := GetAreaInfo(data)
-	log.Println(res)
-}

data_tmp/tools.go

@@ -6,7 +6,9 @@ import (
 	"io"
 	"net/http"
 	"reflect"
+	"regexp"
 	"sort"
+	"strconv"
 	"time"
 )
 
@@ -71,3 +73,58 @@ func GetAreaInfo(data map[string]interface{}) map[string]interface{} {
 	}
 	return info
 }
+
+// getTTMethod 时间格式转换
+func getTTMethod(str string) int64 {
+	TimeV1 := regexp.MustCompile("^(\\d{4})[年.]?$")
+	TimeV2 := regexp.MustCompile("^(\\d{4})[年./-]?(\\d{1,2})[月./-]?$")
+	TimeV3 := regexp.MustCompile("^(\\d{4})[年./-]?(\\d{1,2})[月./-]?(\\d{1,2})[日]?$")
+
+	// Handle "YYYY" format
+	if TimeV1.MatchString(str) {
+		arr := TimeV1.FindStringSubmatch(str)
+		st := arr[1] + "0000"
+		parseInt, err := strconv.ParseInt(st, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	}
+
+	// Handle "YYYYMM" or "YYYY/MM" or "YYYY-MM" or "YYYY.MM" format
+	if TimeV2.MatchString(str) {
+		arr := TimeV2.FindStringSubmatch(str)
+		year := arr[1]
+		month := arr[2]
+		if len(month) == 1 {
+			month = "0" + month
+		}
+		str2 := year + month + "00"
+		parseInt, err := strconv.ParseInt(str2, 10, 64)
+		if err == nil {
+			return parseInt
+		}
+	}
+
+	// Handle "YYYYMMDD" or "YYYY/MM/DD" or "YYYY-MM-DD" or "YYYY.MM.DD" format
+	if TimeV3.MatchString(str) {
+		match := TimeV3.FindStringSubmatch(str)
+		if len(match) >= 4 {
+			year := match[1]
+			month := match[2]
+			day := match[3]
+			if len(month) == 1 {
+				month = "0" + month
+			}
+			if len(day) == 1 {
+				day = "0" + day
+			}
+			dateStr := year + month + day
+			parseInt, err := strconv.ParseInt(dateStr, 10, 64)
+			if err == nil {
+				return parseInt
+			}
+		}
+	}
+
+	return 0
+}

modify_bidamount/bid_test.go

@@ -1,30 +1,29 @@
 package main
 
 import (
-	"fmt"
 	"testing"
 )
 
 func TestProcessBids(t *testing.T) {
-	slice := []interface{}{
-		map[string]interface{}{"infoid": "1", "bidamount": "1000"},
-		map[string]interface{}{"infoid": "2", "bidamount": 1000000000.0},
-		map[string]interface{}{"infoid": "3", "bidamount": "5000000"},
-		map[string]interface{}{"infoid": "4"},
-	}
+	//slice := []interface{}{
+	//	map[string]interface{}{"infoid": "1", "bidamount": "1000"},
+	//	map[string]interface{}{"infoid": "2", "bidamount": 1000000000.0},
+	//	map[string]interface{}{"infoid": "3", "bidamount": "5000000"},
+	//	map[string]interface{}{"infoid": "4"},
+	//}
+	//
+	//tmp := map[string]interface{}{
+	//	"id":     "11111",
+	//	"name":   "testaaa",
+	//	"list":   slice,
+	//	"budget": 1000,
+	//}
 
-	tmp := map[string]interface{}{
-		"id":     "11111",
-		"name":   "testaaa",
-		"list":   slice,
-		"budget": 1000,
-	}
-
-	if list, ok := tmp["list"].([]interface{}); ok {
-		newList, _, _, _, _, logs := ProcessBids(list, "11111", 1000)
-		fmt.Println("logs", logs)
-		fmt.Println("oldlist", tmp["list"])
-		fmt.Println("newlist", newList)
-	}
+	//if list, ok := tmp["list"].([]interface{}); ok {
+	//	//newList, _, _, _, _, logs := ProcessBids(list, "11111", 1000)
+	//	fmt.Println("logs", logs)
+	//	fmt.Println("oldlist", tmp["list"])
+	//	fmt.Println("newlist", newList)
+	//}
 
 }

test/company.go

@@ -1,5 +1,46 @@
 package main
 
-func main() {
+import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+)
 
+// getCompany  获取公司信息
+func getCompany() {
+	//本地
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "127.0.0.1:27017",
+		DbName:      "wcc",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		//Direct:      true,
+	}
+	Mgo.InitPool()
+
+	// 181 凭安库
+	Mgo2 := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.4.181:27001",
+		MongodbAddr: "127.0.0.1:27001",
+		DbName:      "mixdata",
+		Size:        10,
+		UserName:    "",
+		Password:    "",
+		Direct:      true,
+	}
+	Mgo2.InitPool()
+
+	//1.获取凭安特企，河南数据
+	where := map[string]interface{}{
+		"province_short": "HEN",
+	}
+
+	defer util.Catch()
+	sess := Mgo2.GetMgoConn()
+	defer Mgo2.DestoryMongoConn(sess)
+	count := 0
+	it := sess.DB("mixdata").C("special_gov_unit").Find(where).Select(nil).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+
+	}
 }

test/go.mod

@@ -1,3 +1,28 @@
 module test
 
 go 1.20
+
+require jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

test/go.sum

@@ -0,0 +1,198 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

test/main.go

@@ -1,14 +1,21 @@
 package main
 
 import (
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"sort"
 	"strings"
 )
 
 func main() {
-	name := "宣威市西泽乡社会保障服务中心"
-	rs := ruleBuyer(name)
-	log.Println("rs", rs)
+	//name := "宣威市西泽乡社会保障服务中心"
+	//rs := ruleBuyer(name)
+	//log.Println("rs", rs)
+
+	projectTopInformation() //处理项目情报
+	log.Println("处理完毕")
 }
 
 // ruleBuyer 判断采购单位正确,名称错误返回true，名称正确返回false
@@ -72,3 +79,163 @@ func ruleBuyer(input string) (res bool) {
 	return res
 
 }
+
+// projectTopInformation 处理项目表情报一级分类
+func projectTopInformation() {
+	//firsttime 大于21年12月的数据，依赖ids 中的ID查找标讯ID，
+	//然后吧 tag_topinformation 去重求和 更新项目的 tag_topinformation 字段
+	where := map[string]interface{}{
+		"pici": map[string]interface{}{
+			"$gte": 1638288000, //2021-12-01
+		},
+	}
+
+	//测试环境
+	Es := &elastic.Elastic{
+		S_esurl:  "http://192.168.3.149:9201",
+		I_size:   5,
+		Username: "",
+		Password: "",
+	}
+	Es.InitElasticSize()
+
+	//Es := &elastic.Elastic{
+	//	//S_esurl: "http://127.0.0.1:19908",
+	//	S_esurl:  "http://172.17.4.184:19908",
+	//	I_size:   5,
+	//	Username: "jybid",
+	//	Password: "Top2023_JEB01i@31",
+	//}
+	//Es.InitElasticSize()
+
+	//MgoB := &mongodb.MongodbSim{
+	//	MongodbAddr: "172.17.189.140:27080",
+	//	//MongodbAddr: "127.0.0.1:27083",
+	//	DbName:   "qfw",
+	//	Size:     10,
+	//	UserName: "SJZY_RWbid_ES",
+	//	Password: "SJZY@B4i4D5e6S",
+	//	//Direct:      true,
+	//}
+	//MgoB.InitPool()
+
+	////85 抽取库
+	//Mgo := &mongodb.MongodbSim{
+	//	//MongodbAddr: "127.0.0.1:27080",
+	//	MongodbAddr: "172.17.4.85:27080",
+	//	DbName:      "qfw",
+	//	Size:        10,
+	//	//Direct: true,
+	//}
+	//Mgo.InitPool()
+	// 测试环境
+	MgoB := &mongodb.MongodbSim{
+		MongodbAddr: "192.168.3.206:27002",
+		Size:        10,
+		DbName:      "qfw_data",
+		UserName:    "root",
+		Password:    "root",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+
+	defer util.Catch()
+	sess := MgoB.GetMgoConn()
+	defer MgoB.DestoryMongoConn(sess)
+	count := 0
+	it := sess.DB("qfw_data").C("projectset").Find(where).Select(nil).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%5000 == 0 {
+			log.Println("current:", count, tmp["_id"])
+		}
+		if tmp["ids"] == nil {
+			continue
+		}
+
+		if ids, ok := tmp["ids"].([]interface{}); ok {
+			var tags = make([]string, 0)
+			for _, id := range ids {
+				biddingData, _ := MgoB.FindById("bidding", util.ObjToString(id), nil)
+				topinformation := (*biddingData)["tag_topinformation"]
+				if topinformation == nil {
+					continue
+				} else {
+					if tagInfos, ok2 := topinformation.([]interface{}); ok2 {
+						for _, v := range tagInfos {
+							if !IsInStringArray(util.ObjToString(v), tags) {
+								tags = append(tags, util.ObjToString(v))
+							}
+						}
+					}
+				}
+			}
+
+			if len(tags) > 0 {
+				oldTags := make([]string, 0)
+				if tmp["tag_topinformation"] != nil {
+					if oldtop := tmp["tag_topinformation"].([]interface{}); ok {
+						for _, v := range oldtop {
+							oldTags = append(oldTags, util.ObjToString(v))
+						}
+					}
+				}
+
+				if SlicesEqual(tags, oldTags) {
+					continue
+				}
+
+				//pid := mongodb.BsonIdToSId(tmp["_id"])
+				//update := map[string]interface{}{
+				//	"tag_topinformation": tags,
+				//}
+				//MgoB.UpdateById("projectset", pid, map[string]interface{}{"$set": update})
+				//err := Es.UpdateDocument("projectset", pid, update)
+				//if err != nil && err.Error() != "Document not updated: noop" {
+				//	log.Println("es update err", err)
+				//}
+			}
+		}
+	}
+}
+
+// IsInStringArray 判断数组中是否存在字符串
+func IsInStringArray(str string, arr []string) bool {
+	// 先对字符串数组进行排序
+	sort.Strings(arr)
+	// 使用二分查找算法查找字符串
+	pos := sort.SearchStrings(arr, str)
+	// 如果找到了则返回 true，否则返回 false
+	return pos < len(arr) && arr[pos] == str
+}
+
+// SlicesEqual checks if two string slices contain the same elements, regardless of order
+func SlicesEqual(slice1, slice2 []string) bool {
+	// First check if their lengths are the same
+	if len(slice1) != len(slice2) {
+		return false
+	}
+
+	// Create a map to count occurrences of each element in slice1
+	countMap := make(map[string]int)
+	for _, v := range slice1 {
+		countMap[v]++
+	}
+
+	// Subtract the count for each element in slice2
+	for _, v := range slice2 {
+		countMap[v]--
+		// If any count goes below zero, the slices are not equal
+		if countMap[v] < 0 {
+			return false
+		}
+	}
+
+	// If all counts are zero, the slices are equal
+	for _, count := range countMap {
+		if count != 0 {
+			return false
+		}
+	}
+
+	return true
+}

test/test_test.go

@@ -10,16 +10,11 @@ import (
 
 func TestA(T *testing.T) {
 	a := []int{1, 2, 3}
-	b := a[:2]
-	b = append(b, 4)
-	fmt.Println(a, b) //[1 2 4],[1 2 4]
-
-	b = append(b, 5)
-	fmt.Println(a, b) //[1 2 4],[1 2 4 5]
-
-	b[0] = 10
-	fmt.Println(a, b) //[1 2 4],[10 2 4 5]
-
+	b := make([]int, 0)
+	for _, v := range a {
+		b = append(b, v)
+	}
+	log.Println(b)
 }
 
 func TestB(T *testing.T) {

updateBidding/main.go

@@ -1,11 +1,8 @@
 package main
 
 import (
-	"context"
 	"fmt"
 	"github.com/wcc4869/common_utils/log"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo/options"
 	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
@@ -33,13 +30,13 @@ var (
 func main() {
 	//mongodb 163
 	//Mgo = &mongodb.MongodbSim{
-	//	MongodbAddr: "172.17.189.140:27080",
-	//	//MongodbAddr: "127.0.0.1:27083",
-	//	DbName:   "qfw",
-	//	Size:     10,
-	//	UserName: "SJZY_RWbid_ES",
-	//	Password: "SJZY@B4i4D5e6S",
-	//	//Direct:      true,
+	//	//MongodbAddr: "172.17.189.140:27080",
+	//	MongodbAddr: "127.0.0.1:27083",
+	//	DbName:      "qfw",
+	//	Size:        10,
+	//	UserName:    "SJZY_RWbid_ES",
+	//	Password:    "SJZY@B4i4D5e6S",
+	//	Direct:      true,
 	//}
 	//Mgo.InitPool()
 
@@ -54,41 +51,41 @@ func main() {
 	//MgoR.InitPool()
 
 	////测试环境MongoDB
-	//MgoT = &mongodb.MongodbSim{
-	//	//MongodbAddr: "172.17.189.140:27080",
-	//	MongodbAddr: "192.168.3.206:27002",
-	//	DbName:      "qfw_data",
-	//	Size:        10,
-	//	UserName:    "root",
-	//	Password:    "root",
-	//	//Direct:      true,
-	//}
-	//MgoT.InitPool()
+	MgoT = &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "192.168.3.206:27002",
+		DbName:      "qfw_data",
+		Size:        10,
+		UserName:    "root",
+		Password:    "root",
+		//Direct:      true,
+	}
+	MgoT.InitPool()
 
 	// 测试环境es
-	//Es = &elastic.Elastic{
-	//	S_esurl: "http://192.168.3.149:9201",
-	//	//S_esurl:  "http://172.17.4.184:19805",
-	//	I_size:   5,
-	//	Username: "",
-	//	Password: "",
-	//}
-	//Es.InitElasticSize()
-
-	//es
 	Es = &elastic.Elastic{
-		//S_esurl: "http://127.0.0.1:19908",
-		S_esurl:  "http://172.17.4.184:19908",
+		S_esurl: "http://192.168.3.149:9201",
+		//S_esurl:  "http://172.17.4.184:19805",
 		I_size:   5,
-		Username: "jybid",
-		Password: "Top2023_JEB01i@31",
+		Username: "",
+		Password: "",
 	}
 	Es.InitElasticSize()
 
+	//es
+	//Es = &elastic.Elastic{
+	//	S_esurl: "http://127.0.0.1:19908",
+	//	//S_esurl:  "http://172.17.4.184:19908",
+	//	I_size:   5,
+	//	Username: "jybid",
+	//	Password: "Top2023_JEB01i@31",
+	//}
+	//Es.InitElasticSize()
+
 	// es 新集群
 	//EsNew = &elastic.Elastic{
-	//	//S_esurl: "http://127.0.0.1:19905",
-	//	S_esurl:  "http://172.17.4.184:19905",
+	//	S_esurl: "http://127.0.0.1:19905",
+	//	//S_esurl:  "http://172.17.4.184:19905",
 	//	I_size:   5,
 	//	Username: "jybid",
 	//	Password: "Top2023_JEB01i@31",
@@ -96,13 +93,13 @@ func main() {
 	//EsNew.InitElasticSize()
 
 	//go updateMethod()   //更新mongodb
-	//go updateEsMethod() //更新es
-	go updateProjectEsMethod()
+	go updateEsMethod() //更新es
+	//go updateProjectEsMethod()
 	//taskRunProject()
 	//taskRunBidding()
-	//dealDataTest()// 测试环境数据处理
+	dealBiddingTest() // 测试环境数据处理
 
-	updateProject()
+	//updateProject()
 	fmt.Println("over")
 	c := make(chan bool, 1)
 	<-c
@@ -114,9 +111,6 @@ func taskRunBidding() {
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
 
-	//pool := make(chan bool, 2) //处理协程
-	//wg := &sync.WaitGroup{}
-
 	//查询条件
 	//q := map[string]interface{}{
 	//	//"_id": map[string]interface{}{
@@ -135,8 +129,8 @@ func taskRunBidding() {
 	//	"toptype": map[string]interface{}{"$exists": 0},
 	//}
 
-	selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
-	it := sess.DB("qfw").C("zktest_0520_id").Find(nil).Select(selected).Sort("_id").Iter()
+	//selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
+	it := sess.DB("qfw").C("zktest_bidding_0619_compare").Find(nil).Select(nil).Sort("_id").Iter()
 
 	fmt.Println("taskRun 开始")
 	count := 0
@@ -146,9 +140,9 @@ func taskRunBidding() {
 			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
 		}
 
-		//1.更新MongoDB
 		update := map[string]interface{}{}
 
+		// 1.更新省市区
 		//if area, ok := tmp["area"]; ok && area != nil {
 		//	update["area"] = area
 		//} else {
@@ -167,151 +161,37 @@ func taskRunBidding() {
 		//	update["district"] = ""
 		//}
 
-		//========//
-		//toptype := ""
-		//if toptype, ok := tmp["toptype"]; ok && toptype != nil {
-		//	update["toptype"] = toptype
-		//} else {
-		//	update["toptype"] = ""
-		//}
-		////
-		//if subtype, ok := tmp["subtype"]; ok && subtype != nil {
-		//	if util.ObjToString(tmp["toptype"]) == "结果" && util.ObjToString(tmp["subtype"]) == "招标" {
-		//		update["subtype"] = ""
-		//	}
-		//} else {
-		//	update["subtype"] = ""
-		//}
-		//update["toptype"] = "其它"
-		//update["subtype"] = "其它"
-		//if len(update) > 0 {
-		//	//更新MongoDB
-		//	updatePool <- []map[string]interface{}{
-		//		//{"_id": tmp["id"]},
-		//		{"_id": tmp["_id"]},
-		//		{"$set": update},
-		//	}
-		//	//====//
-		//	//biddingID := util.ObjToString(tmp["id"])
-		//	//biddingID := mongodb.BsonIdToSId(tmp["_id"])
-		//	//Mgo.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
-		//	////2.es 项目 更新字段
-		//	//Es.UpdateDocument("bidding", biddingID, update)
-		//	//EsNew.UpdateDocument("bidding", biddingID, update)
-		//	//if err != nil {
-		//	//	log.Info("bidding es update err", err, biddingID)
-		//	//}
-		//}
-
-		//2.es 更新字段
-		//esUpdate := make(map[string]interface{})
-		//if subtitle_projectname, ok := tmp["subtitle_projectname"]; ok && subtitle_projectname != nil {
-		//	esUpdate["subtitle_projectname"] = subtitle_projectname
-		//}
-
-		biddingID := util.ObjToString(tmp["id"])
-		bidamount := util.Float64All(tmp["bidamount"])
-		_, ok := tmp["bidamount"]
-		if ok && bidamount > 0 {
-			if biddingID != "" {
-				update["bidamount"] = bidamount
-			}
+		// 2.更新中标单位，中标金额
+		if winner, ok := tmp["winner"]; ok && winner != nil {
+			update["winner"] = winner
+		} else {
+			update["winner"] = ""
+		}
+		if s_winner, ok := tmp["s_winner"]; ok && s_winner != nil {
+			update["s_winner"] = s_winner
+		} else {
+			update["s_winner"] = ""
 		}
+		if bidamount, ok := tmp["bidamount"]; ok && bidamount != nil {
+			update["bidamount"] = bidamount
+		} else {
+			update["bidamount"] = nil
+		}
+
+		//biddingID := util.ObjToString(tmp["id"])
+		biddingID := mongodb.BsonIdToSId(tmp["_id"])
 
 		if len(update) > 0 {
-			// 更新es
-			updateEsPool <- []map[string]interface{}{
-				{"_id": biddingID},
-				//{"_id": mongodb.BsonIdToSId(tmp["_id"])},
-				update,
+			//Mgo.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
+			//2.es 项目 更新字段
+			err := Es.UpdateDocument("bidding", biddingID, update)
+			err = EsNew.UpdateDocument("bidding", biddingID, update)
+			if err != nil && err.Error() != "Document not updated: noop" {
+				log.Info("bidding es update err", err, biddingID)
 			}
 		}
 
-		//if len(update) > 0 {
-		//	id := mongodb.BsonIdToSId(tmp["_id"])
-		//	//id := mongodb.BsonIdToSId(tmp["_id"])
-		//	err := Es.UpdateDocument("projectset", id, esUpdate)
-		//	if err != nil {
-		//		if strings.Contains(err.Error(), "Document not updated:") {
-		//			continue
-		//		} else {
-		//			fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
-		//		}
-		//	}
-		//}
-
-		//if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
-		//	tmp = make(map[string]interface{})
-		//	continue
-		//}
-		//// 针对存量数据，重复数据不进索引
-		//if util.IntAll(tmp["extracttype"]) == -1 {
-		//	continue
-		//}
-		//
-		////针对产权数据，暂时不入es 索引库
-		//if util.IntAll(tmp["infoformat"]) == 3 {
-		//	continue
-		//}
-		//只有 紧急直接零星采购公告 栏目的数据，需要改成 结果-成交
-		//channel := util.ObjToString(tmp["channel"])
-		//if channel != "紧急直接零星采购公告" {
-		//	continue
-		//}
-
-		//realNum++
-		//pool <- true
-		//wg.Add(1)
-		//go func(tmp map[string]interface{}) {
-		//	defer func() {
-		//		<-pool
-		//		wg.Done()
-		//	}()
-
-		////2.es 更新字段
-		//esUpdate := make(map[string]interface{})
-		//if autoid, ok := tmp["autoid"]; ok && autoid != nil {
-		//	esUpdate["autoid"] = autoid
-		//}
-		//
-		//if len(esUpdate) > 0 {
-		//	err := Es.UpdateDocument("bidding", mongodb.BsonIdToSId(tmp["_id"]), esUpdate)
-		//	if err != nil {
-		//		if strings.Contains(err.Error(), "Document not updated:") {
-		//			return
-		//		} else {
-		//			fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
-		//		}
-		//	}
-		//}
-
-		//if err != nil {
-		//	fmt.Println(err, mongodb.BsonIdToSId(tmp["_id"]))
-		//}
-		//if tag_set, ok := tmp["tag_set"]; ok && tag_set != nil {
-		//	esUpdate["tag_set"] = tag_set
-		//}
-		//
-		//if tag_topinformation, ok := tmp["tag_topinformation"]; ok && tag_topinformation != nil {
-		//	esUpdate["tag_topinformation"] = tag_topinformation
-		//}
-		//
-		//if tag_subinformation, ok := tmp["tag_subinformation"]; ok && tag_subinformation != nil {
-		//	esUpdate["tag_subinformation"] = tag_subinformation
-		//}
-
-		//if len(esUpdate) > 0 {
-		//	// 更新es
-		//	updateEsPool <- []map[string]interface{}{
-		//		{"_id": mongodb.BsonIdToSId(tmp["_id"])},
-		//		esUpdate,
-		//	}
-		//}
-
-		//}(tmp)
-		//tmp = make(map[string]interface{})
 	}
-	//wg.Wait()
 
 	log.Info("Run Over...Count:", log.Int("count", count))
 
@@ -537,120 +417,49 @@ func dealResult() {
 	log.Info("Run Over...Count:", log.Int("count", count))
 }
 
-// dealDataTest 处理测试环境数据
-func dealDataTest() {
+// dealBiddingTest 处理测试环境数据
+func dealBiddingTest() {
 	defer util.Catch()
 	sess := MgoT.GetMgoConn()
 	defer MgoT.DestoryMongoConn(sess)
 
-	where := map[string]interface{}{
-		"_id": map[string]interface{}{
-			"$gte": mongodb.StringTOBsonId("635051528aea8786d196e24a"),
-			"$lte": mongodb.StringTOBsonId("639356ca8aea8786d1995c4b"),
-		},
-	}
-
-	//where := map[string]interface{}{
-	//	"_id": mongodb.StringTOBsonId("639356ca8aea8786d1995c4b"),
-	//}
+	it := sess.DB("qfw_data").C("bidding").Find(nil).Select(nil).Iter()
 
-	ctx := context.Background()
-	coll := sess.M.C.Database("qfw_data").Collection("bidding")
-	find := options.Find().SetBatchSize(1000).SetSort(bson.D{bson.E{"_id", -1}}).SetProjection(bson.M{"_id": 1, "title": 1, "subtype": 1})
-	cur, err := coll.Find(ctx, where, find)
-	if err != nil {
-		fmt.Println(err)
-	}
-
-	///////
-	selected := map[string]interface{}{"signaturedate": 1, "contractperiod": 1, "expiredate": 1}
-	//it := sess.DB("qfw_data").C("bidding").Find(where).Select(nil).Iter()
+	fmt.Println("taskRun 开始")
 	count := 0
-	realNum := 0
-	for tmp := make(map[string]interface{}); cur.Next(ctx); count++ {
-		if cur != nil {
-			cur.Decode(&tmp)
-		}
-		if count%1000 == 0 {
+	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
+		if count%10000 == 0 {
 			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
 		}
-		idStr := mongodb.BsonIdToSId(tmp["_id"])
 
-		data, _ := Mgo.FindById("zktest_quanliang_0210_fbs", idStr, selected)
-
-		if len(*data) == 0 {
-			continue
-		}
-		update := make(map[string]interface{})
-		if (*data)["signaturedate"] != nil {
-			update["signaturedate"] = (*data)["signaturedate"]
-		}
-		if (*data)["contractperiod"] != nil {
-			update["contractperiod"] = (*data)["contractperiod"]
-		}
-		if (*data)["expiredate"] != nil {
-			update["expiredate"] = (*data)["expiredate"]
+		update := map[string]interface{}{}
+		// 2.更新中标单位，中标金额
+		if tag_topinformation, ok := tmp["tag_topinformation"]; ok && tag_topinformation != nil {
+			update["tag_topinformation"] = tag_topinformation
 		}
 
-		if len(update) == 0 {
-			continue
+		if property_form, ok := tmp["property_form"]; ok && property_form != nil {
+			update["property_form"] = property_form
 		}
 
-		fmt.Println(idStr)
-		MgoT.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
-		//bidding 表
-		//if idStr > "5a862e7040d2d9bbe88e3b1f" {
-		//	bidding, _ := Mgo.FindById("bidding", idStr, map[string]interface{}{"extracttype": 1})
-		//	data := *bidding
-		//	Mgo.UpdateById("bidding", idStr, map[string]interface{}{"$set": update})
-		//
-		//	// 针对存量数据，重复数据不进索引
-		//	if util.IntAll(data["extracttype"]) == -1 {
-		//		tmp = make(map[string]interface{})
-		//		continue
-		//	}
-		//
-		//} else {
-		//	//bidding_back
-		//	bidding, _ := Mgo.FindById("bidding_back", idStr, map[string]interface{}{"extracttype": 1})
-		//	data := *bidding
-		//	Mgo.UpdateById("bidding_back", idStr, map[string]interface{}{"$set": update})
-		//	// 针对存量数据，重复数据不进索引
-		//	if util.IntAll(data["extracttype"]) == -1 {
-		//		tmp = make(map[string]interface{})
-		//		continue
-		//	}
-		//}
-
-		realNum++
+		biddingID := mongodb.BsonIdToSId(tmp["_id"])
 
-		//2.es 更新字段
-		esUpdate := update
-		esUpdate["id"] = idStr
-		if len(esUpdate) > 0 {
-			fmt.Println(idStr)
+		if len(update) > 0 {
+			//Mgo.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
+			//2.es 项目 更新字段
+			//err := Es.UpdateDocument("bidding", biddingID, update)
+			//if err != nil && err.Error() != "Document not updated: noop" {
+			//	log.Info("bidding es update err", err, biddingID)
+			//}
 			// 更新es
 			updateEsPool <- []map[string]interface{}{
-				{"_id": mongodb.BsonIdToSId(tmp["_id"])},
-				esUpdate,
+				{"_id": biddingID},
+				update,
 			}
 		}
-
-		//err := Es.UpdateDocument("bidding", idStr, update)
-		//if err != nil {
-		//	log.Error("es update", err)
-		//}
-		//
-		//err = EsNew.UpdateDocument("bidding", idStr, update)
-		//if err != nil {
-		//	log.Error("esNew update", err)
-		//}
-
-		tmp = make(map[string]interface{})
 	}
 
-	log.Info("Run Over...Count:", log.Int("count", count), log.Int("realNum", realNum))
-
+	log.Info("Run Over...Count:", log.Int("count", count))
 }
 
 // updateMethod 更新MongoDB
@@ -705,7 +514,7 @@ func updateEsMethod() {
 						<-updateEsSp
 					}()
 					Es.UpdateBulk("bidding", arru...)
-					EsNew.UpdateBulk("bidding", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -718,7 +527,7 @@ func updateEsMethod() {
 						<-updateEsSp
 					}()
 					Es.UpdateBulk("bidding", arru...)
-					EsNew.UpdateBulk("bidding", arru...)
+					//EsNew.UpdateBulk("bidding", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

xlsx/main.go

@@ -32,7 +32,7 @@ func main() {
 	//fmt.Println(11111)
 	//HighBidding()
 	//exportQy()
-	exportPhone() // 导出联系电话
+	exportPhone2() // 导出联系电话
 	//clickhouseData()
 	//getName()
 	log.Println("over")
@@ -259,6 +259,116 @@ func exportPhone() {
 	f.Save()
 }
 
+// exportPhone 根据企业名单，导出联系人电话,只要抽取到的数据，联系人，联系电话 只要一个
+func exportPhone2() {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	//
+
+	username := "datascbi"
+	password := "Da#Bi20221111SC"
+	host := "127.0.0.1:4001"
+	//host := "172.17.4.242:4000"
+	database := "global_common_data"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Println("Failed to connect to database:", err)
+		return
+	}
+
+	f, err := excelize.OpenFile("./河南省物业企业名单.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	type BaseInfo struct {
+		Name   string `json:"name"`
+		NameId string `json:"name_id"`
+	}
+	type Contact struct {
+		ContactName string `json:"contact_name"`
+		ContactTel  string `json:"contact_tel"`
+	}
+	for i := 1; i < len(rows); i++ {
+		name := rows[i][1]
+		if !strings.Contains(name, "公司") {
+			continue
+		}
+		log.Println(i, "----", name)
+		//contactsMap := make([]string, 0)
+
+		//stds, _ := Mgo.FindOne("qyxy_std", map[string]interface{}{"company_name": name})
+		//var reportsMap = make([]string, 0)
+		//if reports, ok := (*stds)["annual_reports"]; ok {
+		//	if rs, ok := reports.([]interface{}); ok {
+		//		for _, v := range rs {
+		//			if da, ok := v.(map[string]interface{}); ok {
+		//				if util.ObjToString(da["operator_name"]) != "" && util.ObjToString(da["company_phone"]) != "" {
+		//					tm := util.ObjToString(da["operator_name"]) + "_" + util.ObjToString(da["company_phone"])
+		//					if !IsInStringArray(tm, reportsMap) {
+		//						reportsMap = append(reportsMap, tm)
+		//					}
+		//				} else if util.ObjToString(da["company_phone"]) != "" {
+		//					if !IsInStringArray(util.ObjToString(da["company_phone"]), reportsMap) {
+		//						reportsMap = append(reportsMap, util.ObjToString(da["company_phone"]))
+		//					}
+		//				}
+		//			}
+		//
+		//		}
+		//	}
+		//}
+		//if len(reportsMap) > 0 {
+		//	res := strings.Join(reportsMap, ",")
+		//	f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), res)
+		//}
+
+		baseinfo := BaseInfo{}
+		db.Table("dws_f_ent_baseinfo").Select("name", "name_id").Where("name = ? ", name).Scan(&baseinfo)
+		if baseinfo.NameId != "" {
+			contacts := []Contact{}
+			db.Table("dws_f_ent_contact").Select("contact_name", "contact_tel").Where("name_id = ? ", baseinfo.NameId).Scan(&contacts)
+			if len(contacts) > 0 {
+				for _, v := range contacts {
+					if strings.Contains(v.ContactTel, ">") {
+						continue
+					}
+					if v.ContactName != "" && v.ContactTel != "" {
+						f.SetCellValue("Sheet1", fmt.Sprintf("D%v", i+1), v.ContactName)
+						f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), v.ContactTel)
+						break
+					}
+				}
+			}
+		}
+
+	}
+
+	f.Save()
+}
+
 func test() {
 	//Mgo := &mongodb.MongodbSim{
 	//	//MongodbAddr: "172.17.189.140:27080",

xlsx/xlsx_test.go

@@ -94,6 +94,47 @@ func TestA(T *testing.T) {
 	fmt.Println("over")
 }
 
+func TestA2(T *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		MongodbAddr: "127.0.0.1:27017",
+		Size:        10,
+		DbName:      "wcc",
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./拼数据.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet3")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		log.Print(rows[i][2])
+		insert := map[string]interface{}{
+			"company_name": rows[i][2],
+			"organizer":    rows[i][1],
+		}
+
+		err = Mgo.InsertOrUpdate("wcc", "wcc_henan_gov2", insert)
+		if err != nil {
+			log.Println("err", err)
+		}
+	}
+
+	fmt.Println("over")
+}
+
 func TestB(T *testing.T) {
 	Mgo := &mongodb.MongodbSim{
 		//MongodbAddr: "172.17.189.140:27080",