jy-data-extract/qyxy/qyxy_redis/main.go

package main

import (
	"encoding/json"
	"flag"
	"mongodb"
	"qfw/util"
	"qfw/util/redis"
	"sort"
	"strings"
	"sync"
)

var (
	MongoTool *mongodb.MongodbSim

	PnameLength      int
	PurchasingLength int
)

func init() {
	MongoTool = &mongodb.MongodbSim{
		MongodbAddr: "172.17.4.85:27080", // 172.17.4.85:27080
		Size:        10,
		DbName:      "qfw",
	}
	MongoTool.InitPool()
	//MongoTool = &mongodb.MongodbSim{
	//	MongodbAddr: "172.17.4.187:27082,172.17.145.163:27083",
	//	Size:        10,
	//	DbName:      "mixdata",
	//	UserName:    "SJZY_RWMIX_Other",
	//	Password:    "SJZY@M34I6x7D9ata",
	//}
	//MongoTool.InitPool()

	//redis.InitRedis1("qyxy_id=172.17.4.189:8379", 2)				// 企业company_id（中标单位）
	redis.InitRedis1("qyxy=172.17.4.189:8379", 3) //	剑鱼企业搜索搜索

	PnameLength = 500
	PurchasingLength = 500
}

var name string

func main1() {
	flag.StringVar(&name, "n", "", "企业名称")
	flag.Parse()
	if name == "" {
		flag.PrintDefaults()
		return
	}
	text := redis.GetStr("qyxy_id", name)
	//maps := make(map[string]interface{})
	//err1 := json.Unmarshal([]byte(text), &maps)
	//if err1 != nil {
	//	util.Debug("-----map解析异常")
	//}
	util.Debug("company_id=", text)
}

func main() {
	sess := MongoTool.GetMgoConn()
	defer MongoTool.DestoryMongoConn(sess)

	ch := make(chan bool, 3)
	wg := &sync.WaitGroup{}

	//q := map[string]interface{}{
	//	"_id": map[string]interface{}{
	//		"$gte": mongodb.StringTOBsonId("5c2a3d000000000000000000"),
	//		"$lte": mongodb.StringTOBsonId("5e0a1f000000000000000000"),
	//	},
	//}
	field := map[string]interface{}{"agency": 1, "buyer": 1, "s_winner": 1, "projectname": 1, "purchasing": 1, "area": 1}
	//field := map[string]interface{}{"use_flag": 1, "company_type": 1, "company_name": 1}
	query := sess.DB("qfw").C("projectset_20200918").Find(nil).Select(field).Iter()
	count := 0
	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
		if count%5000 == 0 {
			util.Debug("current ---", count)
		}
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			//redis.PutCKV("qyxy_id", util.ObjToString(tmp["company_name"]), util.ObjToString(tmp["_id"]))
			taskinfo1(tmp)
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	util.Debug("over ---", count)
}

func taskinfo1(tmp map[string]interface{}) {
	if util.ObjToString(tmp["s_winner"]) != "" {
		winners := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
		for _, v := range winners {
			winnermap := make(map[string]interface{})
			if b, err := redis.Exists("qyxy", v); err == nil && b {
				text := redis.GetStr("qyxy", v)
				err1 := json.Unmarshal([]byte(text), &winnermap)
				if err1 != nil {
					util.Debug(v, "winner-----map解析异常")
				}
				// bid_projectname
				pname := util.ObjToString(tmp["projectname"])
				if pname != "" {
					if winnermap["bid_projectname"] != nil {
						arr := winnermap["bid_projectname"].([]interface{})
						pnameArr := util.ObjArrToStringArr(arr)
						if BinarySearch(pnameArr, util.ObjToString(tmp["projectname"])) == -1 {
							pnameArr = append(pnameArr, util.ObjToString(tmp["projectname"]))
						}
						if len(pnameArr) > PnameLength {
							pnameArr1 := pnameArr[len(pnameArr)-PnameLength:]
							winnermap["bid_projectname"] = pnameArr1
						} else {
							winnermap["bid_projectname"] = pnameArr
						}
					} else {
						pname := []string{util.ObjToString(tmp["projectname"])}
						winnermap["bid_projectname"] = pname
					}
				}
				// bid_purchasing
				if util.ObjToString(tmp["purchasing"]) != "" {
					if winnermap["bid_purchasing"] != nil {
						arr := winnermap["bid_purchasing"].([]interface{})
						purs := util.ObjArrToStringArr(arr)
						if BinarySearch(purs, util.ObjToString(tmp["purchasing"])) == -1 {
							purs = append(purs, util.ObjToString(tmp["purchasing"]))
						}
						if len(purs) > PurchasingLength {
							purs1 := purs[len(purs)-PurchasingLength:]
							winnermap["bid_purchasing"] = purs1
						} else {
							winnermap["bid_purchasing"] = purs
						}
					} else {
						purs := []string{util.ObjToString(tmp["purchasing"])}
						winnermap["bid_purchasing"] = purs
					}
				}
				// bid_area
				if winnermap["bid_area"] != nil {
					arr := winnermap["bid_area"].([]interface{})
					areas := util.ObjArrToStringArr(arr)
					if BinarySearch(areas, util.ObjToString(tmp["area"])) == -1 {
						areas = append(areas, util.ObjToString(tmp["area"]))
					}
					winnermap["bid_area"] = areas
				} else {
					areas := []string{util.ObjToString(tmp["area"])}
					winnermap["bid_area"] = areas
				}
				// bid_unittype
				if winnermap["bid_unittype"] != nil {
					arr := winnermap["bid_unittype"].([]interface{})
					types := util.ObjArrToStringArr(arr)
					if BinarySearch(types, "中标单位") == -1 {
						types = append(types, "中标单位")
					}
					winnermap["bid_unittype"] = types
				} else {
					types := []string{"中标单位"}
					winnermap["bid_unittype"] = types
				}
			} else {
				// bid_projectname
				if tmp["projectname"] != nil && tmp["projectname"] != "" {
					winnermap["bid_projectname"] = []string{util.ObjToString(tmp["projectname"])}
				}
				// bid_purchasing
				if tmp["purchasing"] != nil && tmp["purchasing"] != "" {
					winnermap["bid_purchasing"] = strings.Split(util.ObjToString(tmp["purchasing"]), ",")
				}
				// bid_area
				winnermap["bid_area"] = []string{util.ObjToString(tmp["area"])}
				// bid_unittype
				winnermap["bid_unittype"] = []string{"中标单位"}
			}
			if len(winnermap) > 0 {
				data, _ := json.Marshal(winnermap)
				redis.Put("qyxy", v, string(data), 0)

			}
		}
	}
	if buyer := util.ObjToString(tmp["buyer"]); buyer != "" {
		buyermap := make(map[string]interface{})
		if b, err := redis.Exists("qyxy", buyer); err == nil && b {
			text := redis.GetStr("qyxy", buyer)
			err1 := json.Unmarshal([]byte(text), &buyermap)
			if err1 != nil {
				util.Debug(buyer, "buyer-----map解析异常")
			}
			// bid_unittype
			if buyermap["bid_unittype"] != nil {
				arr := buyermap["bid_unittype"].([]interface{})
				types := util.ObjArrToStringArr(arr)
				if BinarySearch(types, "采购单位") == -1 {
					types = append(types, "采购单位")
				}
				buyermap["bid_unittype"] = types
			} else {
				types := []string{"采购单位"}
				buyermap["bid_unittype"] = types
			}
		} else {
			buyermap["bid_unittype"] = []string{"采购单位"}
		}
		if len(buyermap) > 0 {
			data, _ := json.Marshal(buyermap)
			redis.Put("qyxy", buyer, string(data), 0)
		}
	}
	if agency := util.ObjToString(tmp["agency"]); agency != "" {
		agencymap := make(map[string]interface{})
		if b, err := redis.Exists("qyxy", agency); err == nil && b {
			text := redis.GetStr("qyxy", agency)
			err1 := json.Unmarshal([]byte(text), &agencymap)
			if err1 != nil {
				util.Debug(agency, "agency----map解析异常")
			}
			// bid_unittype
			if agencymap["bid_unittype"] != nil {
				arr := agencymap["bid_unittype"].([]interface{})
				types := util.ObjArrToStringArr(arr)
				if BinarySearch(types, "代理机构") == -1 {
					types = append(types, "代理机构")
				}
				agencymap["bid_unittype"] = types
			} else {
				types := []string{"代理机构"}
				agencymap["bid_unittype"] = types
			}
		} else {
			agencymap["bid_unittype"] = []string{"代理机构"}
		}
		if len(agencymap) > 0 {
			data, _ := json.Marshal(agencymap)
			redis.Put("qyxy", agency, string(data), 0)
		}
	}
}

func BinarySearch(s []string, k string) int {
	sort.Strings(s)
	lo, hi := 0, len(s)-1
	for lo <= hi {
		m := (lo + hi) >> 1
		if s[m] < k {
			lo = m + 1
		} else if s[m] > k {
			hi = m - 1
		} else {
			return m
		}
	}
	return -1
}