jy-data-extract/udpcreateindex/src/projectindex.go

package main

import (
	"strconv"
	"strings"

	//"fmt"
	"log"
	"qfw/util"
	elastic "qfw/util/elastic"

	mgov "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func projectTask(data []byte, project, mapInfo map[string]interface{}) {
	defer util.Catch()
	q, _ := mapInfo["query"].(map[string]interface{})
	if q == nil {
		q = map[string]interface{}{
			"_id": bson.M{
				"$gt":  util.StringTOBsonId(mapInfo["gtid"].(string)),
				"$lte": util.StringTOBsonId(mapInfo["lteid"].(string)),
			},
		}
	}
	var session *mgov.Session
	if project["addr"] != nil {
		session = project2db.GetMgoConn(3600)
		defer project2db.DestoryMongoConn(session)
	} else {
		session = extractmgo.GetMgoConn(3600)
		defer extractmgo.DestoryMongoConn(session)
	}
	c, _ := project["collect"].(string)
	db, _ := project["db"].(string)
	index, _ := project["index"].(string)
	itype, _ := project["type"].(string)
	count, _ := session.DB(db).C(c).Find(&q).Count()
	savepool := make(chan bool, 10)

	log.Println(db, c, "查询语句:", q, "同步总数:", count, "elastic库:", index)
	query := session.DB(db).C(c).Find(q).Iter()
	arr := make([]map[string]interface{}, savesizei)
	var n int
	i := 0
	for tmp := make(map[string]interface{}); query.Next(tmp); i = i + 1 {
		pp := map[string]map[string]interface{}{}
		if packages, ok := tmp["package"].(map[string]interface{}); ok {
			for _, pks := range packages {
				if pk, ok := pks.([]interface{}); ok {
					for _, v := range pk {
						if p, ok := v.(map[string]interface{}); ok {
							winner := util.ObjToString(p["winner"])
							bidamount := util.Float64All((p["bidamount"]))
							if len(winner) > 4 && bidamount > 0 {
								p := map[string]interface{}{
									"winner":    winner,
									"bidamount": bidamount,
								}
								pp[winner] = p
							}
						}
					}
				}
			}
		} else {
			winner := util.ObjToString(tmp["winner"])
			bidamount := util.Float64All(tmp["bidamount"])
			if len(winner) > 4 && bidamount > 0 {
				p := map[string]interface{}{
					"winner":    winner,
					"bidamount": bidamount,
				}
				pp[winner] = p
			}
		}

		pk1 := []map[string]interface{}{}
		for _, v := range pp {
			pk1 = append(pk1, v)
		}
		if len(pk1) > 0 {
			tmp["package1"] = pk1
		}
		budget := util.Float64All(tmp["budget"])
		bidamount := util.Float64All(tmp["bidamount"])
		if float64(budget) > 0 && float64(bidamount) > 0 {
			rate := float64(1) - float64(bidamount)/float64(budget)
			f, _ := strconv.ParseFloat(strconv.FormatFloat(rate, 'f', 2, 64), 64)
			tmp["project_rate"] = f
		}
		if topscopeclass, ok := tmp["topscopeclass"].([]interface{}); ok {
			tc := []string{}
			for _, v := range topscopeclass {
				str := util.ObjToString(v)
				str = strings.ReplaceAll(str, "t", "")
				str = strings.ReplaceAll(str, "d", "")
				str = strings.ReplaceAll(str, "p", "")
				tc = append(tc, str)
			}
			tmp["topscopeclass"] = tc
		}
		//不生索引字段
		delete(tmp, "package")
		delete(tmp, "winnerorder")
		delete(tmp, "infofield")
		delete(tmp, "budgettag")
		delete(tmp, "bidamounttag")
		list := tmp["list"].([]interface{})
		for _, m := range list {
			tmpM := m.(map[string]interface{})
			if bidamount, ok := tmpM["bidamount"].(string); ok && len(bidamount) > 0 { //bidamount为string类型，转成float
				tmpB := util.Float64All(tmpM["bidamount"])
				tmpM["bidamount"] = tmpB
			}
			//projectscope截断
			listProjectscopeRune := []rune(util.ObjToString(tmpM["projectscope"]))
			if len(listProjectscopeRune) > 1000 {
				tmpM["projectscope"] = string(listProjectscopeRune[:1000])
			}
		}
		//projectscope截断
		projectscopeRune := []rune(util.ObjToString(tmp["projectscope"]))
		if len(projectscopeRune) > 1000 {
			tmp["projectscope"] = string(projectscopeRune[:1000])
		}
		//		if s_budget := fmt.Sprint(tmp["budget"]); s_budget == "" || s_budget == "<nil>" || s_budget == "null" {
		//			tmp["budget"] = nil
		//		}
		//		if s_bidamount := fmt.Sprint(tmp["bidamount"]); s_bidamount == "" || s_bidamount == "<nil>" || s_bidamount == "null" {
		//			tmp["bidamount"] = nil
		//		}
		//go IS.Add("project")
		arr[i] = tmp
		n++
		if i == savesizei-1 {
			savepool <- true
			tmps := arr
			go func(tmpn *[]map[string]interface{}) {
				defer func() {
					<-savepool
				}()
				elastic.BulkSave(index, itype, tmpn, true)
			}(&tmps)
			i = 0
			arr = make([]map[string]interface{}, savesizei)
		}
		if n%savesizei == 0 {
			log.Println("当前:", n)
		}
		tmp = make(map[string]interface{})
	}

	if i > 0 {
		elastic.BulkSave(index, itype, &arr, true)
	}
	log.Println(mapInfo, "create project index...over", n)
}