datatools/biddingSync/main.go

package main

import (
	"log"
	"strings"

	elastic "es"
	"mongodb"
	common "qfw/util"

	"github.com/tealeg/xlsx"
)

var (
	Bidding *mongodb.MongodbSim
	Es      elastic.Es
	Es2     elastic.Es
	cfg     = new(Config)
	sMap    = map[string]string{}
)

func init() {
	common.ReadConfig(&cfg)
	Bidding = mongodb.NewMgoWithUser(cfg.Bidding.Address, cfg.Bidding.DbName, cfg.Bidding.UserName, cfg.Bidding.Password, cfg.Bidding.DbSize)
	Es = elastic.NewEs(cfg.Es.Version, cfg.Es.Address, cfg.Es.DbSize, cfg.Es.UserName, cfg.Es.Password)
	Es2 = elastic.NewEs(cfg.Es2.Version, cfg.Es2.Address, cfg.Es2.DbSize, cfg.Es2.UserName, cfg.Es2.Password)
}

func main() {
	// go FindBiddingBack()
	// sMaps()
	FindBidding()
}

func FindBidding() {
	session := Bidding.GetMgoConn()
	query := map[string]interface{}{}
	defer Bidding.DestoryMongoConn(session)
	count := 0
	iter := session.DB(cfg.Bidding.DbName).C("bidding").Find(&query).Sort("-_id").Iter()
	thisData := map[string]interface{}{}
	for {
		if !iter.Next(&thisData) {
			break
		}
		count++
		id := mongodb.BsonIdToSId(thisData["_id"])
		title := common.ObjToString(thisData["title"])
		projectName := common.ObjToString(thisData["projectname"])
		spidercode := common.ObjToString(thisData["spidercode"])
		if strings.Contains(title, "..") {
			if strings.Contains(projectName, "..") {
				title = strings.ReplaceAll(title, ".", "")
			} else {
				title = projectName
			}
			sMap[spidercode] = "1"
			ok := Bidding.UpdateById("bidding", id, map[string]interface{}{"$set": map[string]interface{}{"title": title}})
			if ok {
				updateData(id, title, spidercode)
			} else {
				log.Println("bidding修改失败！！", id)
			}
		}
		thisData = map[string]interface{}{}
	}
}

func FindBiddingBack() {
	session := Bidding.GetMgoConn()
	query := map[string]interface{}{}
	defer Bidding.DestoryMongoConn(session)
	count := 0
	iter := session.DB(cfg.Bidding.DbName).C("bidding_back").Find(&query).Iter()
	thisData := map[string]interface{}{}
	for {
		if !iter.Next(&thisData) {
			break
		}
		count++
		id := mongodb.BsonIdToSId(thisData["_id"])
		title := common.ObjToString(thisData["title"])
		projectName := common.ObjToString(thisData["projectname"])
		spidercode := common.ObjToString(thisData["spidercode"])
		if strings.Contains(title, "..") {
			if strings.Contains(projectName, "..") {
				thisData["title"] = strings.ReplaceAll(title, ".", "")
			} else {
				thisData["title"] = projectName
			}
			sMap[spidercode] = "1"
			ok := Bidding.UpdateById("bidding", id, map[string]interface{}{"$set": map[string]interface{}{"title": title}})
			if ok {
				updateData(id, title, spidercode)
			} else {
				log.Println("bidding修改失败！！", id)
			}
		}
		thisData = map[string]interface{}{}
	}
}

func updateData(id, title, spidercode string) {
	data := Es.GetById("bidding", "bidding", id)
	if data != nil && len(*data) > 0 {
		dataMap := (*data)[0]
		dataMap["title"] = title
		ok := Es.UpdateNewDoc("bidding", "bidding", dataMap)
		if ok {
			log.Println("es老集群修改成功", id, spidercode)
		} else {
			log.Println("es老集群修改失败 ！！！", id, title)
		}
		oks := Es2.UpdateNewDoc("bidding", "bidding", dataMap)
		if oks {
			log.Println("es新集群修改成功", id, spidercode)
		} else {
			log.Println("es新集群修改失败 ！！！", id, title)
		}
	}
}

func sMaps() {
	xf := xlsx.NewFile()
	sh, _ := xf.AddSheet("详细数据")
	for k, _ := range sMap {
		row := sh.AddRow()
		row.AddCell().SetString(k)
	}
	err := xf.Save("./爬虫代码统计.xlsx")
	if err != nil {
		log.Println("xls error")
	}
}