update

bidding_test/tools.go

@@ -167,3 +167,25 @@ func GetJyURLByID(id string) string {
 
 	return url
 }
+
+// GetIdByURL 解密url，获取bidding ID
+func GetIdByURL(url string) string {
+	if strings.Contains(url, "work-bench") {
+		return ""
+	}
+	if strings.Contains(url, "/article/content") {
+		urls := strings.Split(url, "content/")
+		res := strings.Split(urls[1], ".html")
+		ids := util.CommonDecodeArticle("content", res[0])
+		return ids[0]
+	}
+	if strings.HasSuffix(url, "appid") {
+		urls := strings.Split(url, "entservice/")
+		res := strings.Split(urls[1], ".html")
+		se := util.SimpleEncrypt{Key: "entservice"}
+		id := se.DecodeString(res[0])
+		return id
+	}
+
+	return ""
+}

china_area/area_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"log"
+	"net/http"
+	"testing"
+)
+
+func TestQuery(T *testing.T) {
+	url := "https://bulletin.cebpubservice.com/biddingBulletin/2024-05-30/12948222.html"
+	res, err := http.Get(url)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println(doc.Html())
+}

china_area/go.mod

@@ -0,0 +1,20 @@
+module china_area
+
+go 1.20
+
+require (
+	github.com/PuerkitoBio/goquery v1.9.2
+	github.com/xuri/excelize/v2 v2.8.1
+)
+
+require (
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
+	github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect
+	github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+)

china_area/go.sum

@@ -0,0 +1,62 @@
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
+github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
+github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
+github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0=
+github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
+github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ=
+github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE=
+github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4=
+github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

china_area/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"china_area/utils"
+	"sync"
+)
+
+func main() {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		utils.DoGetInfo()
+	}()
+	go func() {
+		defer wg.Done()
+		utils.GetStaticInfo()
+	}()
+	wg.Wait()
+}

china_area/readme.md

@@ -0,0 +1,17 @@
+# 获取行政区划数据
+
+## 数据来源网址
+
+国家统计局：`https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/`
+
+国家民政部：`http://xzqh.mca.gov.cn/map`
+
+-----------------------------------------------
+
+
+## 数据文件
+
+- 民政部导出数据：20240530_民政部统计数据.xlsx
+- 统计局导出数据：20240530_统计局统计数据.xlsx
+
+- 更新时间：2024.5.30

china_area/utils/getInfo.go

@@ -0,0 +1,135 @@
+package utils
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+)
+
+// DoGetInfo 获取民政部 行政区划
+func DoGetInfo() {
+	var pInfos = []ProvinceInfo{}
+	//暂不包含台湾省的数据
+	provinces := []string{
+		"北京市（京）:110000",
+		"天津市（津）:120000",
+		"河北省（冀）:130000",
+		"河北省（冀）:130000",
+		"山西省（晋）:140000",
+		"内蒙古自治区（内蒙古）:150000",
+		"辽宁省（辽）:210000",
+		"吉林省（吉）:220000",
+		"黑龙江省（黑）:230000",
+		"上海市（沪）:310000",
+		"江苏省（苏）:320000",
+		"浙江省（浙）:330000",
+		"安徽省（皖）:340000",
+		"福建省（闽）:350000",
+		"江西省（赣）:360000",
+		"山东省（鲁）:370000",
+		"河南省（豫）:410000",
+		"湖北省（鄂）:420000",
+		"湖南省（湘）:430000",
+		"广东省（粤）:440000",
+		"广西壮族自治区（桂）:450000",
+		"海南省（琼）:460000",
+		"重庆市（渝）:500000",
+		"四川省（川、蜀）:510000",
+		"贵州省（黔、贵）:520000",
+		"云南省（滇、云）:530000",
+		"西藏自治区（藏）:540000",
+		"陕西省（陕、秦）:610000",
+		"甘肃省（甘、陇）:620000",
+		"青海省（青）:630000",
+		"宁夏回族自治区（宁）:640000",
+		"新疆维吾尔自治区（新）:650000",
+		"香港特别行政区（港）:810000",
+		"澳门特别行政区（澳）:820000",
+	}
+	//provinces := strings.Split("北京市（京）:110000,天津市（津）:120000,天津市（津）:120000,上海市（沪）:310000,江苏省（苏）:320000,浙江省（浙）:330000,安徽省（皖）:340000,福建省（闽）:350000,江西省（赣）:360000,山东省（鲁）:370000,河南省（豫）:410000,湖北省（鄂）:420000,湖南省（湘）:430000,广东省（粤）:440000,广西壮族自治区（桂）:450000,海南省（琼）:460000,重庆市（渝）:500000,四川省（川、蜀）:510000,贵州省（黔、贵）:520000,云南省（滇、云）:530000,西藏自治区（藏）:540000,陕西省（陕、秦）:610000,甘肃省（甘、陇）:620000,青海省（青）:630000,宁夏回族自治区（宁）:640000,新疆维吾尔自治区（新）:650000,香港特别行政区（港）:810000,澳门特别行政区（澳）:820000", ",")
+
+	//1.遍历省份,获取每个省份下的地级市
+	for index, province := range provinces {
+		provinceInfo := strings.Split(province, ":")
+		var pInfo = ProvinceInfo{}
+		pInfo.Name = provinceInfo[0]
+		pInfo.Code = provinceInfo[1]
+
+		fmt.Printf("进度: %d/%d 正在查询省份%s...\r\n", (index + 1), len(provinces), province)
+		citys, err := getRegionInfo(provinceInfo[0], "")
+		if err != nil {
+			continue
+		}
+		//2. 遍历地级市，获取每个地级市下的区县
+		//地级市编码
+		getCitys(citys, provinceInfo[0], &pInfo)
+		pInfos = append(pInfos, pInfo)
+		time.Sleep(time.Second * 1)
+	}
+
+	//3. 输出json,csv
+	//writeJsonFile(pInfos)
+	writeCsvFile(pInfos)
+	log.Println("查询完成，已输出json、csv文件到：", GetExeDir())
+}
+
+// 获取某个省份下所有城市
+func getCitys(citys []map[string]interface{}, province string, pInfo *ProvinceInfo) {
+	for _, city := range citys {
+		cName := city["diji"].(string)
+
+		cCode := city["quHuaDaiMa"].(string)
+
+		var cInfo = CityInfo{}
+		cInfo.Name = cName
+		cInfo.Code = cCode
+
+		areas, err := getRegionInfo(province, cName)
+		if err != nil {
+			continue
+		}
+		getAreas(areas, &cInfo)
+		pInfo.CityInfo = append(pInfo.CityInfo, cInfo)
+
+	}
+}
+
+// 获取某个城市下所有的区县
+func getAreas(areas []map[string]interface{}, cInfo *CityInfo) {
+	for _, area := range areas {
+		aName := area["xianji"].(string)
+		aCode := area["quHuaDaiMa"].(string)
+
+		var aInfo = AreaInfo{}
+		aInfo.Name = aName
+		aInfo.Code = aCode
+		cInfo.AreaInfo = append(cInfo.AreaInfo, aInfo)
+
+	}
+}
+func getRegionInfo(province string, city string) (jsonArr []map[string]interface{}, err error) {
+	url := "http://xzqh.mca.gov.cn/selectJson"
+	pData := ""
+	//var myMap map[string] string
+	// var myMap = make(map[string]string)
+	if province != "" {
+		pData = "shengji=" + province
+	}
+	if city != "" {
+		pData += "&diji=" + city
+	}
+	var headers = make(map[string]string)
+	headers["Content-Type"] = "application/x-www-form-urlencoded; charset=utf-8"
+	resp, err := Execute(url, "POST", ([]byte)(pData), headers)
+	if err != nil {
+		log.Println("Execute err>>", err.Error())
+		return nil, err
+	}
+
+	json.Unmarshal(resp.([]byte), &jsonArr)
+
+	//log.Println(jsonArr)
+	return jsonArr, nil
+}

china_area/utils/regionInfo.go

@@ -0,0 +1,17 @@
+package utils
+
+type ProvinceInfo struct {
+	Name      string
+	Code      string
+	CityInfo  []CityInfo
+	StandName string
+}
+type CityInfo struct {
+	Name     string
+	Code     string
+	AreaInfo []AreaInfo
+}
+type AreaInfo struct {
+	Name string
+	Code string
+}

china_area/utils/stastic.go

@@ -0,0 +1,235 @@
+package utils
+
+import (
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"github.com/xuri/excelize/v2"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+type ProvinceData struct {
+	Name   string     `json:"name"`   //北京市，天津市，河北省
+	Href   string     `json:"href"`   //65.html;
+	Code   string     `json:"code"`   //65
+	Cities []CityData `json:"cities"` // 城市数据
+}
+
+// CityData 城市信息
+type CityData struct {
+	Name  string     `json:"name"`
+	Code  string     `json:"code"` //320800000000
+	Href  string     `json:"href"`
+	Areas []AreaData `json:"areas"`
+}
+
+type AreaData struct {
+	Name string `json:"name"`
+	Code string `json:"code"` //
+	Href string `json:"href"`
+}
+
+type ResData struct {
+	Level        int    `json:"level"`
+	Province     string `json:"province"`
+	ProvinceCode string `json:"province_code"`
+	City         string `json:"city"`
+	CityCode     string `json:"city_code"`
+	Area         string `json:"area"`
+	AreaCode     string `json:"area_code"`
+}
+
+var baseUrl = "https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/"
+
+// GetStaticInfo 获取国家统计局数据
+func GetStaticInfo() {
+	res, err := http.Get(baseUrl)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ProvinceDatas := make([]ProvinceData, 0)
+
+	doc.Find("tr[class=provincetr] td").Each(func(i int, selection *goquery.Selection) {
+		//fmt.Println(selection.Html())
+		//fmt.Println("省份", selection.Text())
+		//fmt.Println("a", selection.Find("a").Text())
+		href, exists := selection.Find("a").Attr("href")
+		//href, exists := selection.Attr("href")
+		if exists {
+			//fmt.Println("省份href", href)
+		}
+		p := ProvinceData{
+			Name: selection.Text(),
+			Code: strings.Replace(href, ".html", "", -1),
+			Href: href,
+		}
+		ProvinceDatas = append(ProvinceDatas, p)
+	})
+	// 获取城市数据
+	getCityData(ProvinceDatas)
+
+	var resD = make([]ResData, 0)
+	for _, v := range ProvinceDatas {
+		data := ResData{Level: 1, Province: v.Name, ProvinceCode: v.Code}
+		resD = append(resD, data)
+		for _, city := range v.Cities {
+			cityData := ResData{Level: 2, Province: v.Name, ProvinceCode: v.Code, City: city.Name, CityCode: city.Code}
+			resD = append(resD, cityData)
+			for _, area := range city.Areas {
+				areaData := ResData{Level: 3, Province: v.Name, ProvinceCode: v.Code, City: city.Name, CityCode: city.Code, Area: area.Name, AreaCode: area.Code}
+				resD = append(resD, areaData)
+			}
+		}
+	}
+
+	//导出数据文件
+	file := time.Now().Format("20060102") + "_统计局统计数据.xlsx"
+	//var xlsx *excelize.File
+	currentPwd, _ := os.Getwd()
+	exportFile := fmt.Sprintf("%s/%s", currentPwd, file)
+	sheet := "2023"
+	xlsx := excelize.NewFile(excelize.Options{ShortDatePattern: "yyyy/m/dd"})
+	xlsx.NewSheet(sheet)
+	xlsx.DeleteSheet("Sheet1")
+	line := 0
+	subtitles := []interface{}{"层级", "省份", "省份代码", "城市", "城市代码", "区县", "区县代码"}
+	line++
+	//设置第一行title
+	_ = xlsx.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &subtitles)
+	fmt.Println("导出数据总数：-------", len(resD))
+
+	for k, _ := range resD {
+		line++
+		val := []interface{}{
+			resD[k].Level, resD[k].Province, resD[k].ProvinceCode, resD[k].City, resD[k].CityCode, resD[k].Area, resD[k].AreaCode,
+		}
+		xlsx.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &val)
+	}
+
+	xlsx.Path = exportFile
+	xlsx.Save()
+
+	fmt.Println("数据导出结束")
+}
+
+// getCityData 获取城市
+func getCityData(pds []ProvinceData) {
+	for i := range pds {
+		v := &pds[i]
+		fmt.Println("省份", v.Name, "获取城市数据")
+		//
+		req, err := http.NewRequest("GET", baseUrl+v.Href, nil)
+		if err != nil {
+			log.Fatal(err)
+		}
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/2.0")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer resp.Body.Close()
+
+		//获取城市信息
+		//cityUrl := baseUrl + v.Href
+		//res, err := http.Get(cityUrl)
+		//if err != nil {
+		//	log.Fatal(err)
+		//}
+		//defer res.Body.Close()
+		//if res.StatusCode != 200 {
+		//	log.Println(cityUrl, "请求失败")
+		//}
+		cityDoc, err := goquery.NewDocumentFromReader(resp.Body)
+		if err != nil {
+			log.Fatal(err)
+		}
+		//
+		cities := make([]CityData, 0)
+		cityDoc.Find("tr[class=citytr]").Each(func(i int, selection *goquery.Selection) {
+			//fmt.Println(selection.Html())
+			//fmt.Println("省份", selection.Text())
+			//fmt.Println("first", selection.Find("a").First().Text())
+			//fmt.Println("last", selection.Find("a").Last().Text())
+			city := CityData{
+				Name: selection.Find("a").Last().Text(),
+				Code: selection.Find("a").First().Text(),
+			}
+			href, exists := selection.Find("a").First().Attr("href")
+			if exists {
+				//fmt.Println("省份href", href)
+				city.Href = href
+			}
+			cities = append(cities, city)
+		})
+		// 获取市区数据
+		getArea(cities)
+		//fmt.Println(cities)
+		v.Cities = cities
+		time.Sleep(time.Second)
+	}
+}
+
+// getArea 获取辖区信息
+func getArea(cities []CityData) {
+	for i := range cities {
+		v := &cities[i]
+		fmt.Println("城市", v.Name, "获取区县信息")
+		//获取城市信息
+		cityUrl := baseUrl + v.Href
+		res, err := http.Get(cityUrl)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != 200 {
+			log.Println(cityUrl, "请求失败")
+		}
+		areaDoc, err := goquery.NewDocumentFromReader(res.Body)
+		if err != nil {
+			log.Fatal(err)
+		}
+		//
+		areas := make([]AreaData, 0)
+		areaDoc.Find("tr[class=countytr]").Each(func(i int, selection *goquery.Selection) {
+			//fmt.Println(selection.Html())
+			area := AreaData{
+				Name: selection.Find("a").Last().Text(),
+				Code: selection.Find("a").First().Text(),
+			}
+
+			href, exists := selection.Find("a").First().Attr("href")
+			if exists {
+				//fmt.Println("省份href", href)
+				area.Href = href
+			}
+
+			if area.Name != "" {
+				areas = append(areas, area)
+			}
+		})
+		v.Areas = areas
+		time.Sleep(time.Microsecond * 100)
+	}
+}
+
+// getStreet 获取街道数据
+func getStreet(areas []AreaData) {
+	for i := range areas {
+		v := &areas[i]
+		fmt.Println("区域", v.Name, "获取街道信息")
+	}
+}

china_area/utils/userHelper.go

@@ -0,0 +1,112 @@
+package utils
+
+import (
+	"bytes"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+)
+
+var myClient *http.Client
+
+func Execute(url, method string, pData []byte, headers map[string]string) (result interface{}, err error) {
+	request, err := http.NewRequest(method, url, bytes.NewBuffer(pData))
+	if err != nil {
+		return "NewRequest err>>", err
+	}
+	if len(headers) > 0 {
+		for key, value := range headers {
+			request.Header.Add(key, value)
+		}
+	}
+	if myClient == nil {
+		myClient = &http.Client{}
+	}
+	response, err := myClient.Do(request)
+	if err != nil {
+		return "client.Do err>>", err
+	}
+	defer response.Body.Close()
+
+	//fmt.Println("response code>>", response.StatusCode)
+	buff := &bytes.Buffer{}
+	_, err = buff.ReadFrom(response.Body)
+	if err != nil {
+		return "readData err>>", err
+	}
+	return buff.Bytes(), nil
+
+}
+
+// 获取当前执行文件所在目录
+func GetExeDir() (path string) {
+	//fPath, _ := os.Executable()
+	//return filepath.Dir(fPath)
+
+	fPath, _ := os.Getwd()
+	return fPath
+
+}
+
+// 输出文件到根目录 json
+func writeJsonFile(pInfos []ProvinceInfo) {
+
+	//写json
+	buff, _ := json.Marshal(pInfos)
+	fPath := fmt.Sprintf("%s\\json_%s.json", GetExeDir(), time.Now().Format("20060102150405"))
+	os.WriteFile(fPath, buff, os.ModePerm)
+
+}
+
+// 输出文件到根目录   csv
+func writeCsvFile(pInfos []ProvinceInfo) {
+
+	// Create a new CSV file
+	fPath := fmt.Sprintf("%s_民政部统计数据.csv", time.Now().Format("20060102"))
+	file, err := os.Create(fPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	// Create a CSV writer
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+
+	// Write the CSV header
+	header := []string{"type", "Province Name", "Province Code", "City Name", "City Code", "Area Name", "Area Code"}
+	if err := writer.Write(header); err != nil {
+		log.Fatal(err)
+	}
+
+	// Write data to the CSV file
+	for _, pInfo := range pInfos {
+		provinceName := pInfo.Name
+		provinceCode := pInfo.Code
+		//province
+		record := []string{"省份", provinceName, provinceCode}
+		writer.Write(record)
+		for _, cInfo := range pInfo.CityInfo {
+			cityName := cInfo.Name
+			cityCode := cInfo.Code
+
+			record = []string{"城市", provinceName, provinceCode, cityName, cityCode}
+			writer.Write(record)
+			for _, aInfo := range cInfo.AreaInfo {
+				areaName := aInfo.Name
+				areaCode := aInfo.Code
+
+				record = []string{"区县", provinceName, provinceCode, cityName, cityCode, areaName, areaCode}
+				if err := writer.Write(record); err != nil {
+					log.Fatal(err)
+				}
+			}
+		}
+	}
+
+	log.Printf("CSV file saved to: %s\n", fPath)
+}

company_invest/allData.go

@@ -491,3 +491,67 @@ func updateInfoId() {
 	log.Println("迭代结束")
 
 }
+
+// dealErrData 处理 b_name=郑州仲裁委员会办公室 的错误数据。由于没有郑州市，导致拼接错误
+func dealErrData() {
+	username := GF.Clickhouse.Username
+	password := GF.Clickhouse.Password
+	host := GF.Clickhouse.Host
+
+	encodedPassword := url.QueryEscape(password)
+	dn := fmt.Sprintf("clickhouse://%s:%s@%s/information?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host)
+
+	db, err := gorm.Open(clickhouse.Open(dn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		log.Fatal("打开数据库失败：", err)
+	} else {
+		log.Println("连接数据库成功", db.Name())
+	}
+
+	batchSize := 10
+	page := 1
+
+	for {
+		var entMapCodes []EntMapCode
+		if err := db.Raw("SELECT * FROM ent_map_code where b_name = '郑州仲裁委员会办公室' ORDER BY create_time LIMIT ? OFFSET ? ", batchSize, (page-1)*batchSize).Scan(&entMapCodes).Error; err != nil {
+			if err == gorm.ErrRecordNotFound {
+				return
+			}
+			log.Fatal("failed to fetch data:", err)
+		}
+
+		for _, entMapCode := range entMapCodes {
+			am := entMapCode.AName
+			if am == "郑州市人民政府" {
+				continue
+			}
+			bm := entMapCode.BName
+			newBM := strings.ReplaceAll(am, "人民政府", "仲裁委员会办公室")
+			newCode := entMapCode
+			newCode.BName = newBM
+			db.Where(&EntMapCode{AName: am, BName: bm}).Delete(&entMapCode)
+			log.Println(am, newBM)
+			BInfo := EntInfo{}
+			db.Model(&EntInfo{}).Where("company_name = ? ", newBM).Select("company_name", "id").First(&BInfo)
+			if BInfo.ID != "" {
+				newCode.BId = BInfo.ID
+			} else {
+				newCode.BId = ""
+			}
+			db.Create(newCode)
+		}
+
+		// 如果查询结果为空，则退出循环
+		if len(entMapCodes) < batchSize {
+			break
+		}
+
+		page++
+		// 释放查询结果的内存
+		entMapCodes = nil
+	}
+
+	log.Println("over")
+}

company_invest/main.go

@@ -104,7 +104,8 @@ func main() {
 	//increBidding()
 	//increInvest()
 	//return
-
+	dealErrData()
+	return
 	local, _ := time.LoadLocation("Asia/Shanghai")
 	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
 	//1.处理投资关系，增量数据，凭安每周推送数据;每周三定时执行

modify_bidamount/bid_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestProcessBids(t *testing.T) {
+	slice := []interface{}{
+		map[string]interface{}{"infoid": "1", "bidamount": "1000"},
+		map[string]interface{}{"infoid": "2", "bidamount": 1000000000.0},
+		map[string]interface{}{"infoid": "3", "bidamount": "5000000"},
+		map[string]interface{}{"infoid": "4"},
+	}
+
+	tmp := map[string]interface{}{
+		"id":     "11111",
+		"name":   "testaaa",
+		"list":   slice,
+		"budget": 1000,
+	}
+
+	if list, ok := tmp["list"].([]interface{}); ok {
+		newList, _, _, _, _, logs := ProcessBids(list, "11111", 1000)
+		fmt.Println("logs", logs)
+		fmt.Println("oldlist", tmp["list"])
+		fmt.Println("newlist", newList)
+	}
+
+}

modify_bidamount/go.mod

@@ -0,0 +1,31 @@
+module modify_bidamount
+
+go 1.20
+
+require (
+	github.com/robfig/cron/v3 v3.0.1
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/olivere/elastic/v7 v7.0.32 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+)

modify_bidamount/go.sum

@@ -0,0 +1,200 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

modify_bidamount/main.go

@@ -0,0 +1,231 @@
+package main
+
+import (
+	"github.com/robfig/cron/v3"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/elastic"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"time"
+)
+
+var (
+	MgoP       *mongodb.MongodbSim
+	MgoB       *mongodb.MongodbSim
+	Es         *elastic.Elastic
+	EsNew      *elastic.Elastic
+	logProject = "projectset_amount_logs" //记录项目表更新记录
+	logBidding = "bidding_amount_logs"    // 记录bidding表更新记录
+)
+
+func Init() {
+	//mongodb
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
+	}
+	MgoB.InitPool()
+	//
+	//85
+	MgoP = &mongodb.MongodbSim{
+		//MongodbAddr: "127.0.0.1:27080",
+		MongodbAddr: "172.17.4.85:27080",
+		DbName:      "qfw",
+		Size:        10,
+		//Direct: true,
+	}
+	MgoP.InitPool()
+
+	//
+	Es = &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	Es.InitElasticSize()
+
+	// es 新集群
+	EsNew = &elastic.Elastic{
+		//S_esurl: "http://127.0.0.1:19905",
+		S_esurl:  "http://172.17.4.184:19905",
+		I_size:   5,
+		Username: "jybid",
+		Password: "Top2023_JEB01i@31",
+	}
+	EsNew.InitElasticSize()
+
+}
+
+func main() {
+	Init()
+
+	spec := "0 00 01 * * *" // 每天01点执行;秒分时日月星期
+
+	local, _ := time.LoadLocation("Asia/Shanghai")
+	c := cron.New(cron.WithLocation(local), cron.WithSeconds())
+	eid, err := c.AddFunc(spec, dealIncData)
+	if err != nil {
+		log.Println("AddFunc err", err, eid)
+	}
+
+	c.Start()
+	defer c.Stop()
+	//
+	select {}
+}
+
+// dealIncData 处理增量数据
+func dealIncData() {
+	log.Println("开始处理增量数据")
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+	now := time.Now()
+	yesterday := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location())
+	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+
+	q := map[string]interface{}{
+		"pici": map[string]interface{}{
+			"$gte": yesterday.Unix(),
+			"$lt":  today.Unix(),
+		},
+		//"ids": "6653e39c66cf0db42a619be7",
+	}
+	log.Println("q", q)
+	query := sess.DB("qfw").C("projectset_20230904").Find(q).Select(nil).Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count, tmp["projectname"])
+		}
+
+		budget := util.Float64All(tmp["budget"])       //预算
+		bidamount := util.Float64All(tmp["bidamount"]) //中标金额
+
+		bidstatus := util.ObjToString(tmp["bidstatus"])
+		if bidstatus == "中标" || bidstatus == "成交" || bidstatus == "合同" || bidstatus == "单一" {
+			projectId := mongodb.BsonIdToSId(tmp["_id"])
+			oldTmp := tmp //备份原来的数据
+			if list, ok := tmp["list"].([]interface{}); ok {
+				if budget == 0 {
+					//通过list 字段，获取其中预算金额
+					budget = getBudget(list)
+				}
+				newList, _, _, minBid, _, logs := ProcessBids(list, projectId, budget, bidamount)
+				if len(newList) > 0 && minBid > 0 {
+					//1.备份数据
+					oldTmp["logs"] = logs
+					MgoP.SaveByOriID(logProject, oldTmp)
+					//2.更新项目MongoDB数据
+					update := make(map[string]interface{})
+					if _, ok := tmp["bidamount"]; ok {
+						update["bidamount"] = minBid
+					}
+					if _, ok := tmp["sortprice"]; ok {
+						update["sortprice"] = minBid
+					}
+
+					update["list"] = newList
+					MgoP.UpdateById("projectset_20230904", projectId, map[string]interface{}{"$set": update})
+					//3.更新项目es
+					Es.UpdateDocument("projectset", projectId, update)
+					//4.更新对应的bidding数据
+					if len(logs) > 0 {
+						for _, log := range logs {
+							if log.InfoID != "" { //对应bidding  ID
+								updateB := map[string]interface{}{
+									"bidamount": log.UpdatedBid,
+								}
+								MgoB.UpdateById("bidding", log.InfoID, map[string]interface{}{"$set": updateB})
+								Es.UpdateDocument("bidding", log.InfoID, updateB)
+								EsNew.UpdateDocument("bidding", log.InfoID, updateB)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	log.Println("理增量数据处理完毕")
+}
+
+// dealAllData 处理存量数据
+func dealAllData() {
+	log.Println("开始处理存量数据")
+	sess := MgoP.GetMgoConn()
+	defer MgoP.DestoryMongoConn(sess)
+
+	now := time.Now()
+	yesterday := time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, now.Location())
+
+	q := map[string]interface{}{
+		"pici": map[string]interface{}{
+			"$lt": yesterday.Unix(),
+		},
+	}
+	log.Println("q", q)
+	query := sess.DB("qfw").C("projectset_20230904").Find(q).Select(nil).Iter()
+	count := 0
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%1000 == 0 {
+			log.Println("current:", count, tmp["projectname"])
+		}
+
+		budget := util.Float64All(tmp["budget"])       //预算
+		bidamount := util.Float64All(tmp["bidamount"]) //中标金额
+		bidstatus := util.ObjToString(tmp["bidstatus"])
+		if bidstatus == "中标" || bidstatus == "成交" || bidstatus == "合同" || bidstatus == "单一" {
+			projectId := mongodb.BsonIdToSId(tmp["_id"])
+			oldTmp := tmp //备份原来的数据
+			if list, ok := tmp["list"].([]interface{}); ok {
+				if budget == 0 {
+					//通过list 字段，获取其中预算金额
+					budget = getBudget(list)
+				}
+				newList, _, _, minBid, _, logs := ProcessBids(list, projectId, budget, bidamount)
+				if len(newList) > 0 && minBid > 0 {
+					//1.备份数据
+					oldTmp["logs"] = logs
+					MgoP.SaveByOriID(logProject, oldTmp)
+					//2.更新项目MongoDB数据
+					update := make(map[string]interface{})
+					if _, ok := tmp["bidamount"]; ok {
+						update["bidamount"] = minBid
+					}
+					if _, ok := tmp["sortprice"]; ok {
+						update["sortprice"] = minBid
+					}
+
+					update["list"] = newList
+					MgoP.UpdateById("projectset_20230904", projectId, map[string]interface{}{"$set": update})
+					//3.更新项目es
+					Es.UpdateDocument("projectset", projectId, update)
+					//4.更新对应的bidding数据
+					if len(logs) > 0 {
+						for _, log := range logs {
+							if log.InfoID != "" { //对应bidding  ID
+								updateB := map[string]interface{}{
+									"bidamount": log.UpdatedBid,
+								}
+								MgoB.UpdateById("bidding", log.InfoID, map[string]interface{}{"$set": updateB})
+								Es.UpdateDocument("bidding", log.InfoID, updateB)
+								EsNew.UpdateDocument("bidding", log.InfoID, updateB)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	log.Println("理增量数据处理完毕")
+}

modify_bidamount/readme.txt

@@ -0,0 +1,42 @@
+
+需求地址：https://jykb.jydev.jianyu360.com/task/67775
+目的：
+    修复项目中，不同阶段，金额 数字不相等数据；本次只修复数值相差比较大，是10000倍的数据；就以小的金额为准, 更新大的数值为小值；
+
+更新范围：
+      项目表中，所有项目状态(bidstatus) 等于单一、中标、成交、合同的数据。
+      每条更新的bidding数据和项目数据，做好备份；原始数据 存入 bidding_amount_logs,projectset_amount_logs 数据表
+      projectset_amount_logs 在原来项目信息基础上，添加一个changes 字段，是个数组，记录最外层修改的字段 bidamount,
+      记录old，new 值；记录 list 变化，[infoid,old_amount,new_amount]
+
+
+更新逻辑：
+         budget 预算，bidamount_max list 里面，中标金额最大值,bidamount_min 最小值
+         如果外围没有budget 字段，就取list 里面的最大值 budget
+
+         bidamount_max 大于 10亿
+         1.bidamount_max 大于 budget 一千倍，并且 bidamount_max 是 bidamount_min 一万倍
+         2.bidamount_max 大于budget  一万倍
+
+更新字段：
+    项目表：
+        1.外层 bidamount 中标金额
+        2.list 里大的或者小的金额 bidamount;没有这个字段无需处理
+
+
+    标讯表：
+        项目表中 ids 中的标讯数据，有 bidamount 字段，并且 等于最小或者最大值，才更新
+
+更新范围：
+        MongoDB：
+            85 项目表
+            184 bidding表
+        es
+            bidding表和projectset 表
+-----------------------------------------------------------------------------
+更新方式：
+        增量数据：定时任务，每天处理，根据pici 字段处理增量数据
+        存量数据：脚本一次性处理
+
+
+

modify_bidamount/utils.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"encoding/json"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+)
+
+type LogEntry struct {
+	ProjectID   string
+	InfoID      string
+	OriginalBid float64
+	UpdatedBid  float64
+}
+
+// ProcessBids 处理中标金额
+func ProcessBids(slice []interface{}, pid string, budget float64, bidamount float64) ([]interface{}, float64, string, float64, string, []LogEntry) {
+	var maxBid, minBid float64
+	var maxInfoID, minInfoID string
+	found := false
+	var logs []LogEntry
+	//newList := make([]interface{}, 0)
+	// 深拷贝传入的slice
+	copySlice := deepCopySlice(slice)
+	// Find the max and min bidamounts
+	for _, item := range copySlice {
+		if m, ok := item.(map[string]interface{}); ok {
+			if bid, ok := m["bidamount"]; ok {
+				bidFloat := util.Float64All(bid)
+				if !found {
+					maxBid = bidFloat
+					minBid = bidFloat
+					maxInfoID = m["infoid"].(string)
+					minInfoID = m["infoid"].(string)
+					found = true
+				} else {
+					if bidFloat > maxBid {
+						maxBid = bidFloat
+						maxInfoID = m["infoid"].(string)
+					}
+					if bidFloat < minBid {
+						minBid = bidFloat
+						minInfoID = m["infoid"].(string)
+					}
+				}
+			}
+		}
+	}
+
+	// 最大金额是最小金额的 1万倍时，才处理数据
+	if !found || minBid == 0 || maxBid < 1000000000 || budget == 0 || bidamount == 0 || maxBid == minBid || maxBid <= bidamount {
+		return []interface{}{}, 0, "", 0, "", logs
+	}
+
+	// 处理数据，把过大的金额数值，更新为小的
+	if maxBid/budget >= 10000 || (maxBid/minBid == 10000 && maxBid/budget > 1000) {
+		for _, item := range copySlice {
+			if m, ok := item.(map[string]interface{}); ok {
+				if bid, ok := m["bidamount"]; ok && util.Float64All(bid) == maxBid {
+					logs = append(logs, LogEntry{
+						InfoID:      m["infoid"].(string),
+						ProjectID:   pid,
+						OriginalBid: maxBid,
+						UpdatedBid:  minBid,
+					})
+					m["bidamount"] = minBid
+				}
+			}
+		}
+		return copySlice, maxBid, maxInfoID, minBid, minInfoID, logs
+	}
+
+	//if maxBid < 1000000000 {
+	//	// Update the max bid with min bid value
+	//	for _, item := range slice {
+	//		if m, ok := item.(map[string]interface{}); ok {
+	//			if bid, ok := m["bidamount"]; ok && convertToFloat(bid) == maxBid {
+	//				logs = append(logs, LogEntry{
+	//					InfoID:      m["infoid"].(string),
+	//					OriginalBid: maxBid,
+	//					UpdatedBid:  minBid,
+	//				})
+	//				m["bidamount"] = minBid
+	//			}
+	//		}
+	//	}
+	//} else {
+	//	// Update the min bid with max bid value
+	//	for _, item := range slice {
+	//		if m, ok := item.(map[string]interface{}); ok {
+	//			if bid, ok := m["bidamount"]; ok && convertToFloat(bid) == minBid {
+	//				logs = append(logs, LogEntry{
+	//					InfoID:      m["infoid"].(string),
+	//					OriginalBid: minBid,
+	//					UpdatedBid:  maxBid,
+	//				})
+	//				m["bidamount"] = maxBid
+	//			}
+	//		}
+	//	}
+	//}
+	return []interface{}{}, 0, "", 0, "", logs
+}
+
+// getBudget 获取预算,循环数据，获取最大值
+func getBudget(list []interface{}) float64 {
+	budget := float64(0)
+	for _, item := range list {
+		if m, ok := item.(map[string]interface{}); ok {
+			if bid, ok := m["budget"]; ok {
+				bidFloat := util.Float64All(bid)
+				if bidFloat > budget {
+					budget = bidFloat
+				}
+			}
+		}
+	}
+
+	return budget
+}
+
+func deepCopySlice(src []interface{}) []interface{} {
+	var dst []interface{}
+	// 使用JSON序列化和反序列化实现深拷贝
+	bytes, err := json.Marshal(src)
+	if err != nil {
+		panic(err)
+	}
+	err = json.Unmarshal(bytes, &dst)
+	if err != nil {
+		panic(err)
+	}
+	return dst
+}

operator/config.go

@@ -12,20 +12,14 @@ type MgoConf struct {
 }
 
 type GlobalConf struct {
-	MongoPA    MgoConf
-	MongoB     MgoConf
-	Clickhouse CkConf
-	Env        EnvConf
-}
-
-type CkConf struct {
-	Host     string
-	Username string
-	Password string
+	MongoF MgoConf
+	MongoT MgoConf
+	MongoB MgoConf
+	Env    EnvConf
 }
 
 type EnvConf struct {
-	Spec1 string
-	Spec2 string
-	Spec3 string
+	Appid      string
+	Createtime int64
+	Fields     []string
 }

operator/config.toml

@@ -0,0 +1,39 @@
+[mongoB] ## 高质量库标讯的数据
+#    host = "172.17.189.140:27080"
+    host = "127.0.0.1:27083"
+    db = "qfw_high"
+    coll = "bidding"  ## 数据表名
+    username = "SJZY_RWbid_ES"
+    password = "SJZY@B4i4D5e6S"
+    size = 15
+    direct = true
+
+
+[mongoF] ## 172.17.4.86
+    host = "127.0.0.1:27082"
+    db = "jyqyfw"  ## 数据库
+    coll = "usermail"  ## 数据表名
+    username = ""
+    password = ""
+    size = 15
+    direct = true
+
+
+[mongoT] ## 166 服务器
+    host = "192.168.3.166:27082"
+    db = "zsdx_yyszb-2024Zsdx_yyszb" ## 数据库名
+    coll = "usermail_bidding"  ## 数据表名
+    username = ""
+    password = ""
+    size = 15
+    direct = true
+
+[env]
+    appid = "jyPRZXQgoJCANdSUBBAiNr"  ## 筛选 appid 客户数据
+    createtime = 1715158328  ## 获取86 表中，创建时间 大于 这个值的数据
+    ## 需要替换高质量表字段;省份、城市、项目名称、项目编号、预算、中标金额、采购单位、中标单位
+    fields =["area", "city", "projectname", "projectcode", "budget", "bidamount", "buyer", "s_winner"]
+
+
+
+

operator/go.mod

@@ -0,0 +1,46 @@
+module operator
+
+go 1.20
+
+require (
+	github.com/spf13/viper v1.18.2
+	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.1 // indirect
+	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	go.mongodb.org/mongo-driver v1.10.1 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.16.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

operator/go.sum

@@ -0,0 +1,241 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/aws/aws-sdk-go v1.43.21/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/dchest/captcha v1.0.0 h1:vw+bm/qMFvTgcjQlYVTuQBJkarm5R0YSsDKhm1HZI2o=
+github.com/dchest/captcha v1.0.0/go.mod h1:7zoElIawLp7GUMLcj54K9kbw+jEyvz2K0FDdRRYhvWo=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
+github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
+github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4=
+go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.5.0/go.mod h1:Jm/m+rNp/z0eqJc74H7LPwQ3G87qkU/AnnAydAjSAHk=
+go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3 h1:mTokQIoOu/oZ2oCSAPayIFfnglIHP0qbOw1Ez6biKDo=
+jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240412074219-927f3f682cb3/go.mod h1:1Rp0ioZBhikjXHYYXmnzL6RNfvTDM/2XvRB+vuPLurI=

operator/main.go

@@ -1 +1,137 @@
-package operator
+package main
+
+import (
+	"github.com/spf13/viper"
+	//"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+)
+
+var (
+	GF    GlobalConf
+	MgoFm *mongodb.MongodbSim // 84
+	MgoTo *mongodb.MongodbSim // 166
+	MgoB  *mongodb.MongodbSim //bidding 标讯数据库
+
+)
+
+func InitConfig() (err error) {
+	viper.SetConfigFile("config.toml") // 指定配置文件路径
+	viper.SetConfigName("config")      // 配置文件名称(无扩展名)
+	viper.SetConfigType("toml")        // 如果配置文件的名称中没有扩展名，则需要配置此项
+
+	viper.AddConfigPath("./")
+	viper.AddConfigPath("./conf/")  // 还可以在工作目录中查找配置
+	viper.AddConfigPath("../conf/") // 还可以在工作目录中查找配置
+	err = viper.ReadInConfig()      // 查找并读取配置文件
+	if err != nil {                 // 处理读取配置文件的错误
+		return
+	}
+
+	err = viper.Unmarshal(&GF)
+
+	return err
+
+}
+
+//func InitLog() {
+//	err := log.InitLog(
+//		//log.Path("./logs/log.out"),
+//		log.Path(""),
+//		log.Level("info"),
+//		log.Compress(true),
+//		log.MaxSize(10),
+//		log.MaxBackups(10),
+//		log.MaxAge(7),
+//		log.Format("json"),
+//	)
+//	if err != nil {
+//		fmt.Printf("InitLog failed: %v\n", err)
+//	}
+//}
+
+func InitMgo() {
+	MgoB = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoB.Host,
+		DbName:      GF.MongoB.DB,
+		Size:        GF.MongoB.Size,
+		UserName:    GF.MongoB.Username,
+		Password:    GF.MongoB.Password,
+		Direct:      GF.MongoB.Direct,
+	}
+	MgoB.InitPool()
+
+	MgoFm = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoF.Host,
+		DbName:      GF.MongoF.DB,
+		Size:        GF.MongoF.Size,
+		UserName:    GF.MongoF.Username,
+		Password:    GF.MongoF.Password,
+		Direct:      GF.MongoF.Direct,
+	}
+	MgoFm.InitPool()
+
+	MgoTo = &mongodb.MongodbSim{
+		MongodbAddr: GF.MongoT.Host,
+		DbName:      GF.MongoT.DB,
+		Size:        GF.MongoT.Size,
+		UserName:    GF.MongoT.Username,
+		Password:    GF.MongoT.Password,
+		Direct:      GF.MongoT.Direct,
+	}
+	MgoTo.InitPool()
+
+}
+
+func main() {
+	InitConfig()
+	//InitLog()
+	InitMgo()
+
+	//f := GF
+	//fmt.Println(f)
+	dealData()
+	log.Println("数据处理完毕")
+}
+
+func dealData() {
+	sess := MgoFm.GetMgoConn()
+	defer MgoFm.DestoryMongoConn(sess)
+
+	where := map[string]interface{}{}
+	if GF.Env.Appid != "" {
+		where["appid"] = GF.Env.Appid
+	}
+
+	if GF.Env.Createtime > 0 {
+		where["createtime"] = map[string]interface{}{
+			"$gt": GF.Env.Createtime,
+		}
+	}
+
+	query := sess.DB(GF.MongoF.DB).C(GF.MongoF.Coll).Find(where).Select(nil).Iter()
+	count := 0
+
+	//fields := []string{"area", "city", "projectname", "projectcode", "budget", "bidamount", "buyer", "s_winner"}
+
+	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
+		if count%100 == 0 {
+			log.Println("current ---", count)
+		}
+
+		biddingId := mongodb.BsonIdToSId(tmp["_id"])
+		biddingH, _ := MgoB.FindById("bidding", biddingId, nil)
+		if biddingH != nil && len(*biddingH) > 0 {
+			tmp["is_high"] = true
+			for _, field := range GF.Env.Fields {
+				if val, ok := (*biddingH)[field]; ok && val != nil {
+					tmp[field] = val
+				}
+			}
+			MgoTo.SaveByOriID(GF.MongoT.Coll, tmp)
+		} else {
+			tmp["is_high"] = false
+			MgoTo.SaveByOriID(GF.MongoT.Coll, tmp)
+		}
+	}
+}

operator/readme.txt

@@ -0,0 +1,15 @@
+运营商专版产品数据输出流程
+
+2、设置自动移库(例如：86-usermail→166-zsdx_yyszb-2024Zsdx_yyszb)；--未实现
+(备注：支持不同客户配置，不同客户可以指定不同表。)
+86→166需根据创建时间(createtime)
+3、移库后的目标库与高质量库进行比对；--未实现
+数据唯一标识比对后，字段取值：省份、城市、项目名称、项目编号、预算、中标金额、采购单位、中标单位
+备注：高质量对比后，数据上打个标识,用于区分是否是高质量数据
+
+
+============================
+
+1.配置 appid,createtime ;根据条件获取172.17.4.86 数据，
+2.根据ID判断高质量库数据是否存在，存在就用高质量库数据覆盖 省份、城市、项目名称、项目编号、预算、中标金额、采购单位、中标单位;写入166对应库表
+3.不存在高质量库，直接迁移到166对应库表

test/test_test.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"log"
+	"net/http"
 	"testing"
 )
 
@@ -34,3 +37,50 @@ func TestB(T *testing.T) {
 		}
 	}
 }
+
+// TestGetArea 获取行政区划 城市区划代码
+func TestGetArea(T *testing.T) {
+	//url := "http://xzqh.mca.gov.cn/getInfo?code=100000&type=2" //省份列表
+	//url := "http://xzqh.mca.gov.cn/getInfo?code=100000&type=2" //省份列表
+	//url := "http://xzqh.mca.gov.cn/data/120000_Point.geojson" //
+	url := "http://xzqh.mca.gov.cn/data/quanguo_Point.geojson" //
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Printf("Error  %v", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("Error getting watcher execution status. Status code: %d", resp.StatusCode)
+		return
+	}
+
+	var result map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		log.Printf("Error decoding watcher execution result: %v", err)
+		return
+	}
+
+	//if len(result) > 0 {
+	//	for k, _ := range result {
+	//		log.Println(k)
+	//		urlD := fmt.Sprintf("http://xzqh.mca.gov.cn/data/%s_Point.geojson", "810000")
+	//		resp, err := http.Get(urlD)
+	//		if err != nil {
+	//			log.Printf("Error  %v", err)
+	//			return
+	//		}
+	//		defer resp.Body.Close()
+	//
+	//		var resultD map[string]interface{}
+	//		err = json.NewDecoder(resp.Body).Decode(&resultD)
+	//		if err != nil {
+	//			log.Printf("Error decoding watcher execution result: %v", err)
+	//		}
+	//
+	//	}
+	//}
+	//log.Println(result)
+}

updateBidding/main.go

@@ -33,13 +33,13 @@ var (
 func main() {
 	//mongodb
 	Mgo = &mongodb.MongodbSim{
-		//MongodbAddr: "172.17.189.140:27080",
-		MongodbAddr: "127.0.0.1:27083",
-		DbName:      "qfw",
-		Size:        10,
-		UserName:    "SJZY_RWbid_ES",
-		Password:    "SJZY@B4i4D5e6S",
-		Direct:      true,
+		MongodbAddr: "172.17.189.140:27080",
+		//MongodbAddr: "127.0.0.1:27083",
+		DbName:   "qfw",
+		Size:     10,
+		UserName: "SJZY_RWbid_ES",
+		Password: "SJZY@B4i4D5e6S",
+		//Direct:      true,
 	}
 	Mgo.InitPool()
 
@@ -77,8 +77,8 @@ func main() {
 
 	//es
 	Es = &elastic.Elastic{
-		S_esurl: "http://127.0.0.1:19908",
-		//S_esurl:  "http://172.17.4.184:19908",
+		//S_esurl: "http://127.0.0.1:19908",
+		S_esurl:  "http://172.17.4.184:19908",
 		I_size:   5,
 		Username: "jybid",
 		Password: "Top2023_JEB01i@31",
@@ -87,21 +87,21 @@ func main() {
 
 	// es 新集群
 	EsNew = &elastic.Elastic{
-		S_esurl: "http://127.0.0.1:19905",
-		//S_esurl:  "http://172.17.4.184:19905",
+		//S_esurl: "http://127.0.0.1:19905",
+		S_esurl:  "http://172.17.4.184:19905",
 		I_size:   5,
 		Username: "jybid",
 		Password: "Top2023_JEB01i@31",
 	}
 	EsNew.InitElasticSize()
 
-	go updateMethod()   //更新mongodb
+	//go updateMethod()   //更新mongodb
 	go updateEsMethod() //更新es
 
 	//taskRunProject()
 	taskRunBidding()
 	//dealDataTest()// 测试环境数据处理
-	fmt.Println(111)
+	fmt.Println("over")
 	c := make(chan bool, 1)
 	<-c
 }
@@ -117,29 +117,30 @@ func taskRunBidding() {
 
 	//查询条件
 	//q := map[string]interface{}{
-	//	"_id": map[string]interface{}{
-	//		//"$gt":  mongodb.StringTOBsonId("5a862f0640d2d9bbe88e3cea"),
-	//		//"$lte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
-	//
-	//		//"$gte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
-	//		"$lte": mongodb.StringTOBsonId("661e347d66cf0db42aa1a52f"),
-	//	},
+	//	//"_id": map[string]interface{}{
+	//	//	//"$gt":  mongodb.StringTOBsonId("5a862f0640d2d9bbe88e3cea"),
+	//	//	//"$lte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
+	//	//
+	//	//	//"$gte": mongodb.StringTOBsonId("65d73ba366cf0db42aca6e2f"),
+	//	//	"$lte": mongodb.StringTOBsonId("661e347d66cf0db42aa1a52f"),
+	//	//},
 	//	//"comeintime": map[string]interface{}{
 	//	//	"$gt": 1669824000,
 	//	//	//"$lte": 1669864950,
 	//	//	"$lte": 1702265941,
 	//	//},
 	//	//"site": "国家能源e购",
+	//	"toptype": map[string]interface{}{"$exists": 0},
 	//}
 
 	selected := map[string]interface{}{"contenthtml": 0, "detail": 0}
-	it := sess.DB("qfw").C("wcc_subtype_err_0429").Find(nil).Select(selected).Sort("_id").Iter()
+	it := sess.DB("qfw").C("zktest_0520_id").Find(nil).Select(selected).Sort("_id").Iter()
 
 	fmt.Println("taskRun 开始")
 	count := 0
 	//realNum := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
-		if count%10000 == 0 {
+		if count%1000 == 0 {
 			log.Info("current", log.Int("count", count), log.Any("_id", tmp["_id"]))
 		}
 
@@ -166,36 +167,39 @@ func taskRunBidding() {
 
 		//========//
 		//toptype := ""
-		if toptype, ok := tmp["toptype"]; ok && toptype != nil {
-			update["toptype"] = toptype
-		} else {
-			update["toptype"] = ""
-		}
-		//
-		if subtype, ok := tmp["subtype"]; ok && subtype != nil {
-			if util.ObjToString(tmp["toptype"]) == "结果" && util.ObjToString(tmp["subtype"]) == "招标" {
-				update["subtype"] = ""
-			}
-		} else {
-			update["subtype"] = ""
-		}
-		if len(update) > 0 {
-			//更新MongoDB
-			//updatePool <- []map[string]interface{}{
-			//	{"_id": tmp["id"]},
-			//	//{"_id": tmp["_id"]},
-			//	{"$set": update},
-			//}
-			//====//
-			biddingID := util.ObjToString(tmp["id"])
-			Mgo.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
-			//2.es 项目 更新字段
-			err := Es.UpdateDocument("bidding", biddingID, update)
-			err = EsNew.UpdateDocument("bidding", biddingID, update)
-			if err != nil {
-				log.Info("bidding es update err", err, biddingID)
-			}
-		}
+		//if toptype, ok := tmp["toptype"]; ok && toptype != nil {
+		//	update["toptype"] = toptype
+		//} else {
+		//	update["toptype"] = ""
+		//}
+		////
+		//if subtype, ok := tmp["subtype"]; ok && subtype != nil {
+		//	if util.ObjToString(tmp["toptype"]) == "结果" && util.ObjToString(tmp["subtype"]) == "招标" {
+		//		update["subtype"] = ""
+		//	}
+		//} else {
+		//	update["subtype"] = ""
+		//}
+		//update["toptype"] = "其它"
+		//update["subtype"] = "其它"
+		//if len(update) > 0 {
+		//	//更新MongoDB
+		//	updatePool <- []map[string]interface{}{
+		//		//{"_id": tmp["id"]},
+		//		{"_id": tmp["_id"]},
+		//		{"$set": update},
+		//	}
+		//	//====//
+		//	//biddingID := util.ObjToString(tmp["id"])
+		//	//biddingID := mongodb.BsonIdToSId(tmp["_id"])
+		//	//Mgo.UpdateById("bidding", biddingID, map[string]interface{}{"$set": update})
+		//	////2.es 项目 更新字段
+		//	//Es.UpdateDocument("bidding", biddingID, update)
+		//	//EsNew.UpdateDocument("bidding", biddingID, update)
+		//	//if err != nil {
+		//	//	log.Info("bidding es update err", err, biddingID)
+		//	//}
+		//}
 
 		//2.es 更新字段
 		//esUpdate := make(map[string]interface{})
@@ -203,14 +207,23 @@ func taskRunBidding() {
 		//	esUpdate["subtitle_projectname"] = subtitle_projectname
 		//}
 
-		//if len(update) > 0 {
-		//	// 更新es
-		//	updateEsPool <- []map[string]interface{}{
-		//		{"_id": tmp["id"]},
-		//		//{"_id": mongodb.BsonIdToSId(tmp["_id"])},
-		//		update,
-		//	}
-		//}
+		biddingID := util.ObjToString(tmp["id"])
+		bidamount := util.Float64All(tmp["bidamount"])
+		_, ok := tmp["bidamount"]
+		if ok && bidamount > 0 {
+			if biddingID != "" {
+				update["bidamount"] = bidamount
+			}
+		}
+
+		if len(update) > 0 {
+			// 更新es
+			updateEsPool <- []map[string]interface{}{
+				{"_id": biddingID},
+				//{"_id": mongodb.BsonIdToSId(tmp["_id"])},
+				update,
+			}
+		}
 
 		//if len(update) > 0 {
 		//	id := mongodb.BsonIdToSId(tmp["_id"])
@@ -689,8 +702,8 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("projectset", arru...)
-					//EsNew.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
+					EsNew.UpdateBulk("bidding", arru...)
 				}(arru)
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0
@@ -702,8 +715,8 @@ func updateEsMethod() {
 					defer func() {
 						<-updateEsSp
 					}()
-					Es.UpdateBulk("projectset", arru...)
-					//EsNew.UpdateBulk("bidding", arru...)
+					Es.UpdateBulk("bidding", arru...)
+					EsNew.UpdateBulk("bidding", arru...)
 				}(arru[:indexu])
 				arru = make([][]map[string]interface{}, 200)
 				indexu = 0

xlsx/clickhouse.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xuri/excelize/v2"
+	"gorm.io/driver/clickhouse"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"log"
+	"net/url"
+	"sync"
+)
+
+// EntMapCode 映射关系表
+type EntMapCode struct {
+	AId         string `json:"a_id"`
+	BId         string `json:"b_id"`
+	AName       string `json:"a_name"`
+	BName       string `json:"b_name"`
+	Code        string `json:"code"`         //关系属性代码
+	InvestRatio string `json:"invest_ratio"` //投资比例
+	InvestPrice string `json:"invest_price"` //投资金额
+	CreateTime  int64  `json:"create_time"`
+	UpdateTime  int64  `json:"update_time"`
+}
+
+func (EntMapCode) TableName() string {
+	return "ent_map_code"
+}
+
+// clickhouseData 获取clickhouse 数据
+func clickhouseData() {
+	host := "cc-2ze9tv451wov14w9e.clickhouse.ads.aliyuncs.com:9000"
+	username := "biservice"
+	password := "Bi_top95215#"
+
+	encodedPassword := url.QueryEscape(password)
+	dn := fmt.Sprintf("clickhouse://%s:%s@%s/information?dial_timeout=10s&read_timeout=20s", username, encodedPassword, host)
+	db, err := gorm.Open(clickhouse.Open(dn), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		log.Fatal("链接数据库失败")
+	} else {
+		log.Println("链接成功")
+	}
+
+	f, err := excelize.OpenFile("./河南单温.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("单位")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	var wg sync.WaitGroup
+	sem := make(chan bool, 15) // 控制同时最多只有10个协程
+	for i := 1; i < len(rows); i++ {
+		if len(rows[i]) > 1 && rows[i][1] != "" {
+			continue
+		}
+
+		wg.Add(1)
+		sem <- true
+		go func(i int) {
+			defer wg.Done()
+			company := rows[i][0]
+			log.Println(i, "----", company)
+			exist := EntMapCode{}
+			db.Where(&EntMapCode{BName: company}).First(&exist)
+			if exist.AName != "" {
+				f.SetCellValue("单位", fmt.Sprintf("%s%d", "B", i+1), exist.AName)
+			}
+
+			if exist.Code == "0101" {
+				f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "管辖关系")
+			} else if exist.Code == "0102" {
+				f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "直属关系")
+			} else if exist.Code == "0103" {
+				f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "组成关系")
+			}
+
+			<-sem // 释放一个位置，允许其他协程开始执行
+		}(i)
+	}
+
+	wg.Wait() // 等待所有协程结束
+
+	f.Save()
+	log.Println("结束")
+	//for i := 1; i < len(rows); i++ {
+	//	company := rows[i][0]
+	//	log.Println("aaa", company)
+	//
+	//	exist := EntMapCode{}
+	//	db.Where(&EntMapCode{BName: company}).First(&exist)
+	//	if exist.AName != "" {
+	//		f.SetCellValue("单位", fmt.Sprintf("%s%d", "B", i+1), exist.AName)
+	//	}
+	//
+	//	if exist.Code == "0101" {
+	//		f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "管辖关系")
+	//	} else if exist.Code == "0102" {
+	//		f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "直属关系")
+	//	} else if exist.Code == "0103" {
+	//		f.SetCellValue("单位", fmt.Sprintf("%s%d", "C", i+1), "组成关系")
+	//	}
+	//}
+	//
+	//f.Save()
+	//log.Println("结束")
+
+}

xlsx/exportBidding.go

@@ -134,3 +134,25 @@ func GetJyURLByID(id string) string {
 
 	return url
 }
+
+// GetIdByURL 解密url，获取bidding ID
+func GetIdByURL(url string) string {
+	if strings.Contains(url, "work-bench") {
+		return ""
+	}
+	if strings.Contains(url, "/article/content") {
+		urls := strings.Split(url, "content/")
+		res := strings.Split(urls[1], ".html")
+		ids := util.CommonDecodeArticle("content", res[0])
+		return ids[0]
+	}
+	if strings.HasSuffix(url, "appid") {
+		urls := strings.Split(url, "entservice/")
+		res := strings.Split(urls[1], ".html")
+		se := util.SimpleEncrypt{Key: "entservice"}
+		id := se.DecodeString(res[0])
+		return id
+	}
+
+	return ""
+}

xlsx/go.mod

@@ -3,36 +3,56 @@ module xlsx
 go 1.20
 
 require (
+	github.com/olivere/elastic/v7 v7.0.32
 	github.com/tealeg/xlsx v1.0.5
 	github.com/xuri/excelize/v2 v2.8.0
 	go.mongodb.org/mongo-driver v1.13.1
+	gorm.io/driver/clickhouse v0.6.0
+	gorm.io/driver/mysql v1.5.4
+	gorm.io/gorm v1.25.7
 	jygit.jydev.jianyu360.cn/data_processing/common_utils v0.0.0-20240108061147-857f0a039c16
 )
 
 require (
+	github.com/ClickHouse/ch-go v0.58.2 // indirect
+	github.com/ClickHouse/clickhouse-go/v2 v2.15.0 // indirect
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/dchest/captcha v1.0.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.6.1 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/paulmach/orb v0.10.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/xuri/efp v0.0.0-20230802181842-ad255f2331ca // indirect
 	github.com/xuri/nfp v0.0.0-20230819163627-dc951e3ffe1a // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	gorm.io/driver/mysql v1.5.4 // indirect
-	gorm.io/gorm v1.25.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

xlsx/main.go

@@ -31,7 +31,9 @@ func main() {
 
 	//fmt.Println(11111)
 	//HighBidding()
-	test()
+	//exportQy()
+	//exportPhone()
+	clickhouseData()
 	log.Println("over")
 }
 
@@ -167,7 +169,7 @@ func exportPhone() {
 		return
 	}
 
-	f, err := excelize.OpenFile("./物业公司清单.xlsx")
+	f, err := excelize.OpenFile("./四川中标客户导出.xlsx")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -192,7 +194,8 @@ func exportPhone() {
 		ContactTel  string `json:"contact_tel"`
 	}
 	for i := 1; i < len(rows); i++ {
-		name := rows[i][1]
+		name := rows[i][0]
+		log.Println(i, "----", name)
 		stds, _ := Mgo.FindOne("qyxy_std", map[string]interface{}{"company_name": name})
 		var reportsMap = make([]string, 0)
 		contactsMap := make([]string, 0)

xlsx/project.go

@@ -63,7 +63,7 @@ func getProject() {
 		//bidding, _ := Mgo.FindById("bidding", biddingID, nil)
 		//href := (*bidding)["href"]
 		//f.SetCellValue("Sheet1", fmt.Sprintf("E%v", i+1), href)
-		//f.SetCellValue("Sheet1", fmt.Sprintf("F%v", i+1), GetJyURLByID(biddingID))
+		f.SetCellValue("Sheet1", fmt.Sprintf("F%v", i+1), GetJyURLByID(biddingID))
 
 	}
 

xlsx/qyxy.go

@@ -0,0 +1,253 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic/v7"
+	"github.com/xuri/excelize/v2"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
+	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
+	"log"
+	"strings"
+)
+
+// exportQy  导出企业相关信息，匹配表格中的，企业所在地、企业联系人、职位、联系方式、2023年的中标项目数以及总金额
+func exportQy() {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "mixdata",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	username := "datascbi"
+	password := "Da#Bi20221111SC"
+	host := "127.0.0.1:4001"
+	//host := "172.17.4.242:4000"
+	database := "global_common_data"
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", username, password, host, database)
+	// 连接到数据库
+	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
+	if err != nil {
+		log.Println("Failed to connect to database:", err)
+		return
+	}
+
+	f, err := excelize.OpenFile("./2023年数字中国参展商名录.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("Sheet1")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	type BaseInfo struct {
+		Name   string `json:"name"`
+		NameId string `json:"name_id"`
+	}
+	type Contact struct {
+		ContactName string `json:"contact_name"`
+		ContactTel  string `json:"contact_tel"`
+	}
+
+	//
+	url2 := "http://127.0.0.1:19905"
+	username2 := "jybid"
+	password2 := "Top2023_JEB01i@31"
+	// 创建 Elasticsearch 客户端
+	client2, err := elastic.NewClient(
+		elastic.SetURL(url2),
+		elastic.SetBasicAuth(username2, password2),
+		elastic.SetSniff(false),
+	)
+	if err != nil {
+		log.Fatalf("创建 Elasticsearch 客户端失败：%s", err)
+	}
+
+	subtitles := []string{"企业名称", "企业省份", "企业城市", "标讯-联系方式", "凭安-联系方式", "2023中标项目数", "2023中标总金额"}
+
+	results := make([][]interface{}, 0)
+	for i := 1; i < len(rows); i++ {
+		name := rows[i][0]
+		//1.标讯抽取的联系方式
+		contactsMap := make([]string, 0) //联系人信息
+		baseinfo := BaseInfo{}
+		log.Println(name)
+		db.Table("dws_f_ent_baseinfo").Select("name", "name_id").Where("name = ? ", name).Order("createtime desc").Limit(10).Scan(&baseinfo)
+		if baseinfo.NameId != "" {
+			contacts := []Contact{}
+			db.Table("dws_f_ent_contact").Select("contact_name", "contact_tel").Where("name_id = ? ", baseinfo.NameId).Scan(&contacts)
+			if len(contacts) > 0 {
+				for _, v := range contacts {
+					if strings.Contains(v.ContactTel, ">") {
+						continue
+					}
+					if v.ContactName != "" && v.ContactTel != "" {
+						s := v.ContactName + "_" + v.ContactTel
+						if !IsInStringArray(s, contactsMap) {
+							contactsMap = append(contactsMap, s)
+						}
+					} else {
+						if !IsInStringArray(v.ContactTel, contactsMap) {
+							contactsMap = append(contactsMap, v.ContactTel)
+						}
+					}
+				}
+			}
+		}
+		//2.凭安联系方式
+		stds, _ := Mgo.FindOne("qyxy_std", map[string]interface{}{"company_name": name})
+		var reportsMap = make([]string, 0)
+		if reports, ok := (*stds)["annual_reports"]; ok {
+			if rs, ok := reports.([]interface{}); ok {
+				for _, v := range rs {
+					if da, ok := v.(map[string]interface{}); ok {
+						if util.ObjToString(da["operator_name"]) != "" && util.ObjToString(da["company_phone"]) != "" {
+							tm := util.ObjToString(da["operator_name"]) + "_" + util.ObjToString(da["company_phone"])
+							if !IsInStringArray(tm, reportsMap) {
+								reportsMap = append(reportsMap, tm)
+							}
+						} else if util.ObjToString(da["company_phone"]) != "" {
+							if !IsInStringArray(util.ObjToString(da["company_phone"]), reportsMap) {
+								reportsMap = append(reportsMap, util.ObjToString(da["company_phone"]))
+							}
+						}
+					}
+				}
+			}
+		}
+
+		//查询企业中标数据
+		count, amount := getAmountData(client2, "bidding", name)
+		results = append(results, []interface{}{name, (*stds)["company_area"], (*stds)["company_city"], strings.Join(contactsMap, ","), strings.Join(reportsMap, ","), count, amount})
+	}
+
+	sheet := "联系方式"
+	f.NewSheet(sheet)
+	line := 0
+	line++
+	//设置第一行title
+	_ = f.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &subtitles)
+
+	for k, _ := range results {
+		line++
+		err = f.SetSheetRow(sheet, fmt.Sprintf("%s%d", "A", line), &results[k])
+		if err != nil {
+			log.Println(err)
+		}
+	}
+	f.Save()
+}
+
+// getAmountData  根据企业名称，获取2023年中标数量，以及中标总金额
+func getAmountData(client *elastic.Client, index string, companyName string) (int64, float32) {
+	query := elastic.NewBoolQuery().
+		Must(elastic.NewTermQuery("s_winner", companyName)).
+		Must(elastic.NewRangeQuery("publishtime").Gte(1672502400).Lt(1704038400)) //2023-01-01 -- 2024-01-01
+	//ctx := context.Background()
+
+	//
+	searchResult, err := client.Search().Size(10000).
+		Index(index).
+		Query(query).
+		Do(context.Background())
+	if err != nil {
+		log.Fatalf("Error executing search: %s", err)
+	}
+
+	totalAmount := float32(0)
+	if len(searchResult.Hits.Hits) > 0 {
+		for _, hit := range searchResult.Hits.Hits {
+			var doc map[string]interface{}
+			err := json.Unmarshal(hit.Source, &doc)
+			if err != nil {
+				log.Printf("解析文档失败：%s", err)
+				continue
+			}
+
+			bidamount := float32(util.Float64All(doc["bidamount"]))
+			totalAmount += bidamount
+		}
+
+	}
+	return searchResult.TotalHits(), totalAmount
+	//////////
+
+	//开始滚动搜索
+	//scrollID := ""
+	//scroll := "10m"
+	//searchSource := elastic.NewSearchSource().
+	//	Query(query).
+	//	Size(10000).
+	//	Sort("_doc", true) //升序排序
+	////Sort("_doc", false) //降序排序
+	//
+	//searchService := client.Scroll(index).
+	//	Size(10000).
+	//	Scroll(scroll).
+	//	SearchSource(searchSource)
+	//
+	//res, err := searchService.Do(ctx)
+	//if err != nil {
+	//	if err == io.EOF {
+	//		fmt.Println("没有数据")
+	//	} else {
+	//		panic(err)
+	//	}
+	//
+	//}
+	////defer client.ClearScroll().ScrollId(scrollID).Do(ctx) // 在退出时清理资源
+	//fmt.Println(companyName, "总数是：", res.TotalHits())
+	//total := 0
+	//totalAmount = float32(0)
+	//for len(res.Hits.Hits) > 0 {
+	//	for _, hit := range res.Hits.Hits {
+	//		var doc map[string]interface{}
+	//		err := json.Unmarshal(hit.Source, &doc)
+	//		if err != nil {
+	//			log.Printf("解析文档失败：%s", err)
+	//			continue
+	//		}
+	//
+	//		bidamount := float32(util.Float64All(doc["bidamount"]))
+	//		totalAmount += bidamount
+	//	}
+	//
+	//	total = total + len(res.Hits.Hits)
+	//	scrollID = res.ScrollId
+	//	res, err = client.Scroll().ScrollId(scrollID).Scroll(scroll).Do(ctx)
+	//	log.Println("current count:", total)
+	//	if err != nil {
+	//		if err == io.EOF {
+	//			// 滚动到最后一批数据，退出循环
+	//			break
+	//		}
+	//		log.Println("滚动搜索失败：", err, res)
+	//		break // 处理错误时退出循环
+	//	}
+	//}
+	//// 在循环外调用 ClearScroll
+	//_, err = client.ClearScroll().ScrollId(scrollID).Do(ctx)
+	//if err != nil {
+	//	log.Printf("清理滚动搜索失败：%s", err)
+	//}
+	//
+	//fmt.Println(companyName, "结束~~~~~~~~~~~~~~~")
+	//
+	//return res.TotalHits(), totalAmount
+}

xlsx/xlsx_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/xuri/excelize/v2"
 	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
 	"log"
+	"strings"
 	"testing"
 )
 
@@ -105,7 +106,7 @@ func TestB(T *testing.T) {
 	}
 	Mgo.InitPool()
 
-	f, err := excelize.OpenFile("./fanglei.11111.xlsx")
+	f, err := excelize.OpenFile("./剑鱼标讯_数据导出.xlsx")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -122,9 +123,16 @@ func TestB(T *testing.T) {
 		return
 	}
 	for i := 1; i < len(rows); i++ {
-		id := rows[i][0]
+		url := rows[i][2]
+		log.Println(rows[i][1])
+		id := GetIdByURL(url)
 		re, _ := Mgo.FindById("bidding", id, nil)
-		Mgo.SaveByOriID("wcc_0327_bidding", re)
+		(*re)["wcc_id"] = i
+		(*re)["toptype_old"] = (*re)["toptype"]
+		(*re)["subtype_old"] = (*re)["subtype"]
+		delete(*re, "toptype")
+		delete(*re, "subtype")
+		Mgo.SaveByOriID("wcc_bidding_0518", re)
 	}
 
 	log.Println("over")
@@ -147,7 +155,7 @@ func TestAAA(t *testing.T) {
 	sess := Mgo.GetMgoConn()
 	defer Mgo.DestoryMongoConn(sess)
 
-	f, err := excelize.OpenFile("./四川专项债.xlsx")
+	f, err := excelize.OpenFile("./河南智慧城市_专项债.xlsx")
 	if err != nil {
 		fmt.Println(err)
 		return
@@ -245,3 +253,121 @@ func TestAAA(t *testing.T) {
 	//}
 
 }
+
+func TestGetIdByURL(t *testing.T) {
+	url := "https://www.jianyu360.cn/article/content/AHnY1xBfikoAjYsNGd4cE8JIzAvFj1jcXNlKDhROC4eWlFwA15UCZQ%3D.html"
+	fmt.Println(GetIdByURL(url))
+}
+
+// TestArea 中国行政区划数据-民政部
+func TestArea(t *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./20240530_民政部统计数据.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("2023")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		data := map[string]interface{}{}
+		if rows[i][0] == "省份" {
+			data["level"] = 1
+		} else if rows[i][0] == "城市" {
+			data["level"] = 2
+		} else {
+			data["level"] = 3
+		}
+		log.Println(rows[i][1])
+		parts := strings.Split(rows[i][1], "（")
+		data["province"] = parts[0]
+		data["province_code"] = rows[i][2]
+		if len(rows[i]) > 3 {
+			data["city"] = rows[i][3]
+		}
+		if len(rows[i]) > 4 {
+			data["city_code"] = rows[i][4]
+		}
+		if len(rows[i]) > 5 {
+			data["area"] = rows[i][5]
+		}
+		if len(rows[i]) > 6 {
+			data["area_code"] = rows[i][6]
+		}
+
+		Mgo.Save("wcc_area_minzhengbu", data)
+	}
+}
+
+// TestArea2 中国行政区划数据-统计局
+func TestArea2(t *testing.T) {
+	Mgo := &mongodb.MongodbSim{
+		//MongodbAddr: "172.17.189.140:27080",
+		MongodbAddr: "127.0.0.1:27083",
+		Size:        10,
+		DbName:      "qfw",
+		UserName:    "SJZY_RWbid_ES",
+		Password:    "SJZY@B4i4D5e6S",
+		Direct:      true,
+	}
+	Mgo.InitPool()
+
+	f, err := excelize.OpenFile("./20240530_统计局统计数据.xlsx")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	rows, err := f.GetRows("2023")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	for i := 1; i < len(rows); i++ {
+		data := map[string]interface{}{}
+		data["level"] = rows[i][0]
+		log.Println(rows[i][1], rows[3])
+		data["province"] = rows[i][1]
+		data["province_code"] = rows[i][2]
+		if len(rows[i]) > 3 {
+			data["city"] = rows[i][3]
+		}
+		if len(rows[i]) > 4 {
+			data["city_code"] = rows[i][4]
+		}
+		if len(rows[i]) > 5 {
+			data["area"] = rows[i][5]
+		}
+		if len(rows[i]) > 6 {
+			data["area_code"] = rows[i][6]
+		}
+
+		Mgo.Save("wcc_area_tongjiju", data)
+	}
+}