jy-data-extract/standardata/src/historyrepair.go

// historyrepair 处理多线程重复数据问题
package main

import (
	"dbutil/mongo"
	"dbutil/redis"
	"log"
	qu "qfw/util"

	"go.mongodb.org/mongo-driver/bson"
)

func historyrepair(db, coll, datatype string, dbnum int) {
	sess := MongoTo.GetMgoConn()
	defer MongoTo.Close()
	field := ""
	if datatype == "winner" {
		field = "company_name"
	} else if datatype == "buyer" {
		field = "buyer_name"
	} else if datatype == "agency" {
		field = "agency_name"
	}
	it := sess.DB(db).C(coll).Find(bson.M{}).Select(bson.M{field: 1}).Iter()
	index := 0
	delnum := 0
	for tmp := map[string]interface{}{}; it.Next(&tmp); index++ {
		name := qu.ObjToString(tmp[field])
		id := mongo.BsonTOStringId(tmp["_id"])
		str, _ := redis.GetRedisStr(datatype, dbnum, name)
		if str != "" {
			MongoTo.DeleteById(coll, id)
			delnum++
		} else {
			redis.PutRedis(datatype, dbnum, name, id, -1)
		}
		tmp = map[string]interface{}{}
		if index%100 == 0 {
			log.Println(index, delnum)
		}
	}
	log.Println(index, delnum)
}

func historyrepairErr(db, coll, datatype string, dbnum int) {
	sess := MongoTo.GetMgoConn()
	defer MongoTo.Close()
	it := sess.DB(db).C(coll).Find(bson.M{}).Select(bson.M{"name": 1}).Iter()
	index := 0
	delnum := 0
	for tmp := map[string]interface{}{}; it.Next(&tmp); index++ {
		name := qu.ObjToString(tmp["name"])
		id := mongo.BsonTOStringId(tmp["_id"])
		str, _ := redis.GetRedisStr(datatype, dbnum, name)
		if str != "" {
			MongoTo.DeleteById(coll, id)
			delnum++
		} else {
			redis.PutRedis(datatype, dbnum, name, id, -1)
		}
		tmp = map[string]interface{}{}
		if index%100 == 0 {
			log.Println(index, delnum)
		}
	}
	log.Println(index, delnum)
}