package main import ( "fmt" log "github.com/donnie4w/go-logger/logger" "go.mongodb.org/mongo-driver/bson/primitive" qu "qfw/util" "strings" "time" "unicode/utf8" ) var proNameDict = map[string]string{} //生成两个 mysql 表 func saveBiddingPurMysql() { log.Debug("分析数据-保存两个mysql表...") sess := save_mgo.GetMgoConn() defer save_mgo.DestoryMongoConn(sess) q, total, isok := map[string]interface{}{}, 0, 0 it := sess.DB(save_mgo.DbName).C(o_bidding_coll).Find(&q).Iter() for tmp := make(map[string]interface{}); it.Next(&tmp); total++ { if total%1000 == 0 { log.Debug("curent index ", total, isok) } tmpid := BsonTOStringId(tmp["_id"]) repeat := qu.IntAll(tmp["repeat"]) subtype := qu.ObjToString(tmp["subtype"]) if repeat != 1 { if subtype == "单一" || subtype == "中标" || subtype == "成交" || subtype == "合同" { isok++ dealWithBidData(tmp, tmpid) dealWithPurData(tmp, tmpid) } } tmp = make(map[string]interface{}) } log.Debug("is over ", total, isok) } //构建-招投标数据-无异常 func dealWithBidData(tmp map[string]interface{}, tmpid string) { data := map[string]interface{}{} data["info_id"] = tmp["info_id"] data["projectname"] = tmp["projectname"] publishtime := qu.Int64All(tmp["publishtime"]) new_publishtime := "" //转日期 if publishtime > 0 { new_publishtime = time.Unix(publishtime, 0).Format(timeLayout) data["publishtime"] = new_publishtime } data["buyer_name"] = qu.ObjToString(tmp["buyer"]) data["buyer_depart"] = "" if tmp["budget"] != nil && qu.Float64All(tmp["budget"]) < 1000000000.0 { data["budget"] = qu.Float64All(tmp["budget"]) } if tmp["bidamount"] != nil && qu.Float64All(tmp["bidamount"]) < 1000000000.0 { data["bidamount"] = qu.Float64All(tmp["bidamount"]) } data["agency_name"] = qu.ObjToString(tmp["agency"]) data["winner_name"] = qu.ObjToString(tmp["s_winner"]) data["jy_href"] = qu.ObjToString(tmp["jyhref"]) updatetime := time.Unix(time.Now().Unix(), 0).Format(timeLayout) data["updatetime"] = updatetime // info_type buyer_classcode area_code industry_code info_type_key := qu.ObjToString(tmp["toptype"]) + "_" + qu.ObjToString(tmp["subtype"]) data["info_type"] = bidclass_dict[info_type_key] buyerclass := qu.ObjToString(tmp["buyerclass"]) data["buyer_classcode"] = buyerclass_dict[buyerclass] area_key := qu.ObjToString(tmp["area"]) + "_" + qu.ObjToString(tmp["city"]) + "_" + qu.ObjToString(tmp["district"]) data["area_code"] = area_dict[area_key] subscopeclass := []string{} if res, ok := tmp["subscopeclass"].(primitive.A); ok { subscopeclass = qu.ObjArrToStringArr(res) } else { if res, ok := tmp["subscopeclass"].([]interface{}); ok { subscopeclass = qu.ObjArrToStringArr(res) } } industry_code, industry_code_other := "", "" for _, v := range subscopeclass { if strings.Contains(v, "医疗卫生") { industry_code = industry_dict[v] } else { if industry_code_other != "" { industry_code_other = industry_code_other + "," + industry_dict[v] } else { industry_code_other = industry_dict[v] } } } data["industry_code"] = industry_code data["industry_code_other"] = industry_code_other insertMysqlData("f_bidding", data, tmpid) } //构建-标的物信息-关联企业信息 func dealWithPurData(tmp map[string]interface{}, tmpid string) { p_list := []map[string]interface{}{} if yl_purchasinglist, ok := tmp["yl_purchasinglist"].(primitive.A); ok { p_list = qu.ObjArrToMapArr(yl_purchasinglist) } else { if yl_purchasinglist, ok := tmp["yl_purchasinglist"].([]interface{}); ok { p_list = qu.ObjArrToMapArr(yl_purchasinglist) } } info_id := qu.ObjToString(tmp["info_id"]) s_winner := qu.ObjToString(tmp["s_winner"]) for _, v := range p_list { data := map[string]interface{}{} product_name := qu.ObjToString(v["product_name"]) brandname := qu.ObjToString(v["brandname"]) data["info_id"] = info_id data["bid_name"] = qu.ObjToString(v["itemname"]) data["product_name"] = product_name data["product_brand"] = brandname data["product_specs"] = "" product_model := qu.ObjToString(v["model"]) if utf8.RuneCountInString(product_model) > 100 { product_model = "" } data["product_model"] = product_model if v["unitprice"] != nil { data["product_price"] = qu.Float64All(v["unitprice"]) } if v["number"] != nil { data["product_num"] = qu.Float64All(v["number"]) } if v["totalprice"] != nil { data["product_totalprice"] = qu.Float64All(v["totalprice"]) } if v["unitname"] != nil { data["product_unit"] = qu.ObjToString(v["unitname"]) } //涉及-bid_company_name bid_company_id bid_company_ysname -关联f_company if s_winner != "" { if strings.Contains(s_winner, ",") { data["bid_company_id"] = "" data["bid_company_name"] = "" data["bid_company_ysname"] = s_winner } else { //查询f_company... query := map[string]interface{}{"company_name": s_winner} find_data := MysqlTool.FindOne("f_company", query, "company_id", "") if find_data == nil { data["bid_company_id"] = "" data["bid_company_name"] = s_winner data["bid_company_ysname"] = "" } else { data["bid_company_id"] = qu.ObjToString((*find_data)["company_id"]) data["bid_company_name"] = s_winner data["bid_company_ysname"] = "" } } } else { data["bid_company_id"] = "" data["bid_company_name"] = "" data["bid_company_ysname"] = "" } //涉及-make_company_id make_company_name make_company_ysname -关联f_product query := "SELECT company_id,company_name FROM f_product WHERE product_name = ? and (business_type = ? or business_type = ?)" proinfo := MysqlTool.SelectBySql(query, product_name, "1", "3") if len(*proinfo) == 1 { //查询到多条-同产品信息 info := (*proinfo)[0] data["make_company_id"] = qu.ObjToString(info["company_id"]) data["make_company_name"] = qu.ObjToString(info["company_name"]) data["make_company_ysname"] = "" } else if len(*proinfo) > 1 { //如果有多个企业-生产信息 isUse, c_name, c_id := matchedManufacturer(brandname, *proinfo) if isUse { data["make_company_id"] = c_id data["make_company_name"] = c_name data["make_company_ysname"] = "" } else { data["make_company_id"] = "" data["make_company_name"] = "" data["make_company_ysname"] = c_name } } else { data["make_company_id"] = "" data["make_company_name"] = "" data["make_company_ysname"] = "" } //product_code分类 product_class_1 := qu.ObjToString(v["product_class_1"]) product_class_2 := qu.ObjToString(v["product_class_2"]) product_class_3 := qu.ObjToString(v["product_class_3"]) key_4 := "四级_" + product_name if proclass_name_dict[key_4] != "" { data["product_code"] = proclass_name_dict[key_4] } else { pro_key := fmt.Sprintf("%s_%s_%s", product_class_1, product_class_2, product_class_3) data["product_code"] = proclass_name_dict[pro_key] } insertMysqlData("f_bidproduct", data, tmpid) } } //根据品牌字段匹配-有效生产企业-过多10条 func matchedManufacturer(brandName string, infoArr []map[string]interface{}) (bool, string, string) { c_name := "" namedict := map[string]string{} for i := 0; i < len(infoArr); i++ { info := infoArr[i] company_name := qu.ObjToString(info["company_name"]) company_id := qu.ObjToString(info["company_id"]) if brandName != "" { //品牌匹配 if strings. Contains(company_name, brandName) { return true, company_name, company_id } } if len(namedict) < 10 { if c_name == "" { c_name = company_name namedict[company_name] = company_id } else { if namedict[company_name] == "" { c_name += "," + company_name namedict[company_name] = company_id } } } } if len(namedict) == 1 { return true, c_name, namedict[c_name] } return false, c_name, "" } //准备工作-导出源数据很重要... 163 func exportUsefulPurBidData() { sess := qy_mgo.GetMgoConn() defer qy_mgo.DestoryMongoConn(sess) //构建标准产品数据 dataArr, _ := save_mgo.Find("zktest_mairui", nil, nil, nil) for _, v := range dataArr { name := qu.ObjToString(v["name"]) proNameDict[name] = name } log.Debug("迈瑞产品...", len(proNameDict)) q, total := map[string]interface{}{ "publishtime": map[string]interface{}{ "$gte": 1546272000, }, }, 0 isok := 0 it := sess.DB("qfw").C(s_bidding_coll).Find(&q).Sort("_id").Iter() for tmp := make(map[string]interface{}); it.Next(&tmp); total++ { if total%10000 == 0 { log.Debug("curent index ", total, tmp["_id"], "~", isok) } p_list := []map[string]interface{}{} new_data := map[string]interface{}{} if yl_purchasinglist, ok := tmp["yl_purchasinglist"].(primitive.A); ok { p_list = qu.ObjArrToMapArr(yl_purchasinglist) } else { if yl_purchasinglist, ok := tmp["yl_purchasinglist"].([]interface{}); ok { p_list = qu.ObjArrToMapArr(yl_purchasinglist) } } //临时-匹配是不是含有-有效迈瑞-产品 if len(p_list) > 0 { new_p_list := matchUsefulMrBidProduct(p_list) if len(new_p_list) > 0 { isok++ s_winner := qu.ObjToString(tmp["s_winner"]) new_data["yl_purchasinglist"] = p_list tmpid := BsonTOStringId(tmp["_id"]) new_data["info_id"] = tmpid new_data["_id"] = tmp["_id"] new_data["subtype"] = qu.ObjToString(tmp["subtype"]) new_data["toptype"] = qu.ObjToString(tmp["toptype"]) new_data["projectname"] = qu.ObjToString(tmp["projectname"]) new_data["publishtime"] = qu.IntAll(tmp["publishtime"]) new_data["buyer"] = qu.ObjToString(tmp["buyer"]) new_data["buyerclass"] = qu.ObjToString(tmp["buyerclass"]) if tmp["budget"] != nil { new_data["budget"] = tmp["budget"] } if tmp["bidamount"] != nil { new_data["bidamount"] = tmp["bidamount"] } new_data["s_winner"] = s_winner new_data["agency"] = qu.ObjToString(tmp["agency"]) new_data["area"] = qu.ObjToString(tmp["area"]) new_data["city"] = qu.ObjToString(tmp["city"]) new_data["district"] = qu.ObjToString(tmp["district"]) new_data["jyhref"] = fmt.Sprintf(Url, qu.CommonEncodeArticle("content", tmpid)) if tmp["subscopeclass"] != nil { new_data["subscopeclass"] = tmp["subscopeclass"] } save_mgo.Save(o_bidding_coll, new_data) save_mgo.Save("zktest_mysql_bidding_repeat", tmp) //业务判重需要- } } tmp = make(map[string]interface{}) } log.Debug("is bidding over ", total, isok) } func matchUsefulMrBidProduct(p_list []map[string]interface{}) (new_p_list []map[string]interface{}) { for _, v := range p_list { product_name := qu.ObjToString(v["product_name"]) if proNameDict[product_name] != "" { new_p_list = append(new_p_list, v) } } return new_p_list } //根据目标数据,去掉重复 - 重新导出数据 func exportNewPurBidData() { sess := save_mgo.GetMgoConn() defer save_mgo.DestoryMongoConn(sess) q, total := map[string]interface{}{}, 0 isok := 0 it := sess.DB(save_mgo.DbName).C("zktest_mysql_bidding_repeat").Find(&q).Sort("_id").Select(map[string]interface{}{ "repeat": 1, }).Iter() for tmp := make(map[string]interface{}); it.Next(&tmp); total++ { if total%1000 == 0 { log.Debug("curent index ", total, tmp["_id"], "~", isok) } tmpid := BsonTOStringId(tmp["_id"]) repeat := tmp["repeat"].(bool) if repeat { isok++ save_mgo.UpdateById(o_bidding_coll, tmpid, map[string]interface{}{ "$set": map[string]interface{}{ "repeat": 1, }, }) } tmp = make(map[string]interface{}) } log.Debug("is over ", total, isok) } //导出指定-迈瑞相关数据-已弃用 func exportMaiRuiInfoId() { query := "SELECT product_name FROM f_product WHERE company_name LIKE CONCAT('%',?,'%')" proinfo := MysqlTool.SelectBySql(query, "迈瑞") log.Debug(len(*proinfo)) proNameArr := []string{} proNameDict := map[string]string{} for _, v := range *proinfo { product_name := qu.ObjToString(v["product_name"]) if proNameDict[product_name] == "" { proNameDict[product_name] = product_name proNameArr = append(proNameArr, product_name) } } //812个产品名称 - - bid_infoDict := map[string]string{} for k, v := range proNameArr { if k%100 == 0 { log.Debug("当前:", len(bid_infoDict)) } q := "SELECT info_id FROM f_bidproduct WHERE product_name=?" infoArr := MysqlTool.SelectBySql(q, v) for _, info := range *infoArr { info_id := qu.ObjToString(info["info_id"]) if bid_infoDict[info_id] == "" { bid_infoDict[info_id] = info_id save_mgo.Save("a_mairui", map[string]interface{}{ "info_id": info_id, }) } } } }