jyps/export_bidding/xiamen.go

package main

import (
	"go.mongodb.org/mongo-driver/bson"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"log"
	"strings"
	"sync"
	"time"
	"unicode/utf8"
)

// dealXmData 处理中标企业注册地为厦门的数据;陈伟铭 需求
/**
中标方企业注册地为厦门的企业数量一共是多少，可以分别看下22年度，23年度的两个数据总量
*/
func dealXmData() {
	Mgo := &mongodb.MongodbSim{
		MongodbAddr: "172.17.189.140:27080",
		//MongodbAddr: "127.0.0.1:27083",
		DbName:   "qfw",
		Size:     10,
		UserName: "SJZY_RWbid_ES",
		Password: "SJZY@B4i4D5e6S",
		//Direct:      true,
	}
	Mgo.InitPool()

	MgoC := &mongodb.MongodbSim{
		MongodbAddr: "172.17.189.140:27080",
		//MongodbAddr: "127.0.0.1:27083",
		DbName:   "mixdata",
		Size:     10,
		UserName: "SJZY_RWbid_ES",
		Password: "SJZY@B4i4D5e6S",
		//Direct:      true,
	}
	MgoC.InitPool()

	defer util.Catch()
	sess := Mgo.GetMgoConn()
	defer Mgo.DestoryMongoConn(sess)

	pool := make(chan bool, 10) //处理协程
	wg := &sync.WaitGroup{}

	// 设置查询条件
	filter := bson.D{
		{"publishtime", bson.M{"$gte": 1640966400, "$lt": 1704038400}},
		{"subtype", bson.M{"$in": []string{"中标", "单一", "成交", "合同"}}},
	}

	selected := map[string]interface{}{
		"contenthtml":      0, // 0表示不返回该字段
		"attach_text":      0, // 0表示不返回该字段
		"detail":           0, // 0表示不返回该字段
		"purchasingsource": 0, // 0表示不返回该字段
		"jsondata":         0, // 0表示不返回该字段
		"package":          0, // 0表示不返回该字段
	}

	it := sess.DB("qfw").C("bidding").Find(&filter).Select(&selected).Iter()
	count := 0
	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
		if count%10000 == 0 {
			log.Println("current :", count)
		}

		pool <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-pool
				wg.Done()
			}()

			if sensitive := util.ObjToString(tmp["sensitive"]); sensitive == "测试" { //bidding中有敏感词，不生索引
				tmp = make(map[string]interface{})
				return
			}
			// 针对存量数据，重复数据不进索引
			if util.IntAll(tmp["extracttype"]) == -1 {
				return
			}

			swinner := util.ObjToString(tmp["s_winner"])
			if swinner == "" {
				return
			}

			if utf8.RuneCountInString(swinner) < 4 {
				return
			}

			publishtime := util.Int64All(tmp["publishtime"])
			projectName := util.ObjToString(tmp["projectname"])

			if strings.Contains(swinner, ",") {
				winners := strings.Split(swinner, ",")
				for _, v := range winners {
					if utf8.RuneCountInString(v) < 4 {
						continue
					} else {
						da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": v})
						if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" {
							continue
						}
						city := util.ObjToString((*da)["company_city"])
						if city != "厦门市" {
							continue
						}

						insert := map[string]interface{}{
							"winner":         v,
							"credit_no":      (*da)["credit_no"],
							"bidding_id":     mongodb.BsonIdToSId(tmp["_id"]),
							"projectname":    projectName,
							"company_type":   (*da)["company_type"],
							"company_status": (*da)["company_status"],
							"company_area":   (*da)["company_area"],
							"company_city":   (*da)["company_city"],
							"year":           time.Unix(publishtime, 0).Year(),
						}
						Mgo.Save("wcc_xiamen_winner", insert)
					}
				}

			} else {
				da, _ := MgoC.FindOne("qyxy_std", map[string]interface{}{"company_name": swinner})
				if da == nil || (*da)["credit_no"] == nil || util.ObjToString((*da)["credit_no"]) == "" {
					return
				}

				city := util.ObjToString((*da)["company_city"])
				if city != "厦门市" {
					return
				}

				insert := map[string]interface{}{
					"winner":         swinner,
					"credit_no":      (*da)["credit_no"],
					"bidding_id":     mongodb.BsonIdToSId(tmp["_id"]),
					"projectname":    projectName,
					"company_type":   (*da)["company_type"],
					"company_status": (*da)["company_status"],
					"company_area":   (*da)["company_area"],
					"company_city":   (*da)["company_city"],
					"year":           time.Unix(publishtime, 0).Year(),
				}
				Mgo.Save("wcc_xiamen_winner", insert)
			}

		}(tmp)
		tmp = make(map[string]interface{})

	}

	wg.Wait()
	log.Println("结束")
}

// CountBidamount 统计厦门中标单位，中标金额总数
func CountBidamount() {
	Mgo := &mongodb.MongodbSim{
		MongodbAddr: "172.17.189.140:27080",
		//MongodbAddr: "127.0.0.1:27083",
		DbName:   "qfw",
		Size:     10,
		UserName: "SJZY_RWbid_ES",
		Password: "SJZY@B4i4D5e6S",
		//Direct:      true,
	}
	Mgo.InitPool()
	defer util.Catch()
	sess := Mgo.GetMgoConn()
	defer Mgo.DestoryMongoConn(sess)

	var BidMap = make(map[string]bool)
	it := sess.DB("qfw").C("wcc_xiamen_winner").Find(nil).Select(nil).Iter()
	count := 0

	var total = float64(0)
	for tmp := make(map[string]interface{}); it.Next(&tmp); count++ {
		biddingID := util.ObjToString(tmp["bidding_id"])
		if BidMap[biddingID] {
			continue
		}
		data, _ := Mgo.FindById("bidding", biddingID, nil)
		BidMap[biddingID] = true
		bid := util.Float64All((*data)["bidamount"])

		total += bid
		tmp = make(map[string]interface{})
	}

	log.Println("total", total)
}