jy-data-extract/qyxy/src_supplement/main.go

package main

import (
	"encoding/json"
	"mongodb"
	"qfw/util"
	"qfw/util/redis"
	"sort"
	"strings"
	"sync"
)

var (
	MongoTool		*mongodb.MongodbSim
)

func init() {
	MongoTool = &mongodb.MongodbSim{
		MongodbAddr: "172.17.4.85:27080",
		Size:        10,
		DbName:      "qfw",
	}
	MongoTool.InitPool()
	redis.InitRedis("qyxy_winner=127.0.0.1:6379")
}

func main() {
	sess := MongoTool.GetMgoConn()
	defer MongoTool.DestoryMongoConn(sess)

	ch := make(chan bool, 10)
	wg := &sync.WaitGroup{}

	//q := map[string]interface{}{
	//	"_id": map[string]interface{}{
	//		"$gte": mongodb.StringTOBsonId("5c2a3d000000000000000000"),
	//		"$lte": mongodb.StringTOBsonId("5e0a1f000000000000000000"),
	//	},
	//}
	field := map[string]interface{}{"agency": 1, "buyer": 1, "s_winner": 1, "projectname": 1, "purchasing": 1,
		"area": 1}
	query := sess.DB("qfw").C("project").Find(nil).Select(field).Iter()
	count := 0
	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
		if count%2000 == 0 {
			util.Debug("current ---", count)
		}
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			taskinfo(tmp)
		}(tmp)
		tmp = make(map[string]interface{})
	}
	wg.Wait()
	util.Debug("over ---", count)
}

func taskinfo(tmp map[string]interface{}) {
	winners := strings.Split(util.ObjToString(tmp["s_winner"]), ",")
	for _, v := range winners{
		winnermap := make(map[string]interface{})
		if b, err := redis.Exists("qyxy_winner", v); err == nil && b {
			text := redis.GetStr("qyxy_winner", v)
			err1 := json.Unmarshal([]byte(text), &winnermap)
			if err1 != nil {
				util.Debug(v, "winner-----map解析异常")
			}
			// bid_projectname
			pname := util.ObjToString(tmp["projectname"])
			if pname != "" {
				if winnermap["bid_projectname"] != nil {
					arr := winnermap["bid_projectname"].([]interface{})
					pnameArr := util.ObjArrToStringArr(arr)
					if BinarySearch(pnameArr, util.ObjToString(tmp["projectname"])) == -1 {
						pnameArr = append(pnameArr, util.ObjToString(tmp["projectname"]))
					}
					winnermap["bid_projectname"] = pnameArr
				}else {
					pname := []string{util.ObjToString(tmp["projectname"])}
					winnermap["bid_projectname"] = pname
				}
			}
			// bid_purchasing
			if util.ObjToString(tmp["purchasing"]) != "" {
				if winnermap["bid_purchasing"] != nil {
					arr := winnermap["bid_purchasing"].([]interface{})
					purs := util.ObjArrToStringArr(arr)
					if BinarySearch(purs, util.ObjToString(tmp["purchasing"])) == -1 {
						purs = append(purs, util.ObjToString(tmp["purchasing"]))
					}
					winnermap["bid_purchasing"] = purs
				}else {
					purs := []string{util.ObjToString(tmp["purchasing"])}
					winnermap["bid_purchasing"] = purs
				}
			}
			// bid_area
			if winnermap["bid_area"] != nil {
				arr := winnermap["bid_area"].([]interface{})
				areas := util.ObjArrToStringArr(arr)
				if BinarySearch(areas, util.ObjToString(tmp["area"])) == -1 {
					areas = append(areas, util.ObjToString(tmp["area"]))
				}
				winnermap["bid_area"] = areas
			}else {
				areas := []string{util.ObjToString(tmp["area"])}
				winnermap["bid_area"] = areas
			}
			// bid_unittype
			if winnermap["bid_unittype"] != nil {
				arr := winnermap["bid_unittype"].([]interface{})
				types := util.ObjArrToStringArr(arr)
				if BinarySearch(types, "2") == -1 {
					types = append(types, "2")
				}
				winnermap["bid_unittype"] = types
			}else {
				types := []string{"2"}
				winnermap["bid_unittype"] = types
			}
		}else {
			// bid_projectname
			if tmp["projectname"] != nil && tmp["projectname"] != "" {
				winnermap["bid_projectname"] = []string{util.ObjToString(tmp["projectname"])}
			}
			// bid_purchasing
			if tmp["purchasing"] != nil && tmp["purchasing"] != "" {
				winnermap["bid_purchasing"] = strings.Split(util.ObjToString(tmp["purchasing"]), ",")
			}
			// bid_area
			winnermap["bid_area"] = []string{util.ObjToString(tmp["area"])}
			// bid_unittype
			winnermap["bid_unittype"] = []string{"2"}
		}
		if len(winnermap) > 0 {
			data , _ := json.Marshal(winnermap)
			redis.Put("qyxy_winner", v, string(data), 0)
		}
	}
	if buyer := util.ObjToString(tmp["buyer"]); buyer != "" {
		buyermap := make(map[string]interface{})
		if b, err := redis.Exists("qyxy_winner", buyer); err == nil && b {
			text := redis.GetStr("qyxy_winner", buyer)
			err1 := json.Unmarshal([]byte(text), &buyermap)
			if err1 != nil {
				util.Debug(buyer, "buyer-----map解析异常")
			}
			// bid_unittype
			if buyermap["bid_unittype"] != nil {
				arr := buyermap["bid_unittype"].([]interface{})
				types := util.ObjArrToStringArr(arr)
				if BinarySearch(types, "2") == -1 {
					types = append(types, "2")
				}
				buyermap["bid_unittype"] = types
			}else {
				types := []string{"1"}
				buyermap["bid_unittype"] = types
			}
		}else {
			buyermap["bid_unittype"] = []string{"1"}
		}
		if len(buyermap) > 0 {
			data , _ := json.Marshal(buyermap)
			redis.Put("qyxy_winner", buyer, string(data), 0)
		}
	}
	if agency := util.ObjToString(tmp["agency"]); agency != "" {
		agencymap := make(map[string]interface{})
		if b, err := redis.Exists("qyxy_winner", agency); err == nil && b {
			text := redis.GetStr("qyxy_winner", agency)
			err1 := json.Unmarshal([]byte(text), &agencymap)
			if err1 != nil {
				util.Debug(agency, "agency----map解析异常")
			}
			// bid_unittype
			if agencymap["bid_unittype"] != nil {
				arr := agencymap["bid_unittype"].([]interface{})
				types := util.ObjArrToStringArr(arr)
				if BinarySearch(types, "3") == -1 {
					types = append(types, "3")
				}
				agencymap["bid_unittype"] = types
			}else {
				types := []string{"3"}
				agencymap["bid_unittype"] = types
			}
		}else {
			agencymap["bid_unittype"] = []string{"3"}
		}
		if len(agencymap) > 0 {
			data , _ := json.Marshal(agencymap)
			redis.Put("qyxy_winner", agency, string(data), 0)
		}
	}
}

func BinarySearch(s []string, k string) int {
	sort.Strings(s)
	lo, hi := 0, len(s)-1
	for lo <= hi {
		m := (lo + hi) >> 1
		if s[m] < k {
			lo = m + 1
		} else if s[m] > k {
			hi = m - 1
		} else {
			return m
		}
	}
	return -1
}