data_field_dispose/data_mgo_to_tidb/mending/mend.go

package mending

import (
	"bufio"
	"data_tidb/bidding"
	u "data_tidb/util"
	"fmt"
	"go.uber.org/zap"
	"io"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/log"
	"os"
	"strings"
)

// 修复id...
func MendingInfos() {
	log.Debug("开始...")
	arr := getMendInfos()
	arr = []string{"61c3cc040bb57633ab730673", "62280023f32ba517371dced4", "6325b0e067a6b0a2860fb55a"}
	for k, v := range arr {
		if k%100 == 0 {
			log.Debug(fmt.Sprintf("cur index %d ~ %s", k, v))
		}
		deleteInfoFromId(v)
		increaseInfoFromId(v)
	}
	log.Debug("is over ... ")
}

// 根据id删除异常数据...
func deleteInfoFromId(info_id string) {
	query := map[string]interface{}{"s_info_id": info_id}
	//1、标讯基础模型
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_baseinfo, query, info_id)
	//2、标讯正文
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_detail, query, info_id)
	//3、标讯扩展
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_expand_baseinfo, query, info_id)
	//4、附件信息
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_file_baseinfo, query, info_id)
	//5、附件文本
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_file_text, query, info_id)
	//6、采购意向
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_intention_baseinfo, query, info_id)
	//7、分包信息
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_package_baseinfo, query, info_id)
	//8、分包投标人
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_package_bidder_baseinfo, query, info_id)
	//9、分包物品
	u.DeleteGlobalMysqlData(u.T_dwd_f_bid_package_goods_baseinfo, query, info_id)
}

// 根据id新增信息...
func increaseInfoFromId(info_id string) {
	if data := u.MongoB.FindById("bidding", info_id); data != nil {
		if len(data) > 0 {
			bidding.TaskIncreaseInfo(data)
		} else {
			log.Debug("未查询到数据...", zap.String("bidding", info_id))
		}
	}
}

func getMendInfos() []string {
	arrs := []string{}
	temp := map[string]string{}
	r, err := os.Open("res/mend.txt")
	if err != nil {
		log.Error(err.Error())
	}
	defer r.Close()
	rd := bufio.NewReader(r)
	total := 0
	for {
		total++
		if total%10000 == 0 {
			log.Debug(fmt.Sprintf("cur index %d", total))
		}
		line, err := rd.ReadString('\n') //以'\n'为结束符读入一行
		if err != nil || io.EOF == err {
			break
		}
		if line == "\n" {
			continue
		}
		line = strings.ReplaceAll(line, "\n", "")
		line = strings.ReplaceAll(line, "", "")

		if temp[line] == "" {
			temp[line] = line
			arrs = append(arrs, line)
		}
	}
	log.Debug(fmt.Sprintf("总计：%d ~ %d", total, len(arrs)))
	return arrs
}