package main import ( "encoding/json" "fmt" "github.com/garyburd/redigo/redis" "go.mongodb.org/mongo-driver/bson/primitive" "gopkg.in/mgo.v2/bson" "log" mu "mfw/util" "net" "sort" "strings" "time" ) //定时任务 //1.存异常表 //2.合并原始库新增 func TimedTask() { //time.Sleep(time.Hour*70) t2 := time.NewTimer(time.Second * 5) for range t2.C { Fcconn := FClient.GetMgoConn() defer FClient.DestoryMongoConn(Fcconn) tmpLast := map[string]interface{}{} if iter := Fcconn.DB(Config["mgodb_extract_kf"]).C("winner_new").Find(bson.M{}).Sort("-_id").Limit(1).Iter(); iter != nil { if !iter.Next(&tmpLast) { //临时表无数据 log.Println("临时表无数据:") t2.Reset(time.Minute * 5) continue } else { log.Println("临时表有数据:", tmpLast) fconn := FClient.GetMgoConn() defer FClient.DestoryMongoConn(fconn) cursor := fconn.DB(Config["mgodb_extract_kf"]).C("winner_new").Find(bson.M{ "_id": bson.M{ "$lte": tmpLast["_id"], }, }).Sort("_id").Iter() if cursor == nil { log.Println("查询失败") t2.Reset(time.Second * 5) continue } //遍历临时表数据,匹配不到原始库存入异常表 tmp := make(map[string]interface{}) for cursor.Next(&tmp) { tmpId := tmp["_id"].(primitive.ObjectID).Hex() //再重新查找redis,存在发udp处理,不存在走新增合并 rdb := RedisPool.Get() if _, err := redis.String(rdb.Do("GET", tmp["winner"])); err == nil { //{"gtid":"57d7ad2f61a0721f152d2ad5","lteid":"5e20968d85a9271abf0ad6c2","stype":""} //redis存在发送udp进行处理 by, _ := json.Marshal(map[string]interface{}{ "gtid": tmpId, "lteid": tmpId, "stype": "", }) if e := udpclient.WriteUdp(by, mu.OP_TYPE_DATA, &net.UDPAddr{ IP: net.ParseIP("127.0.0.1"), Port: Updport, }); e != nil { log.Println(e) } //存在的话删除tmp mongo表 FClient.DbName = Config["mgodb_extract_kf"] if DeletedCount := FClient.DeleteById("winner_new", tmpId); DeletedCount == 0 { log.Println("删除临时表err:", DeletedCount) } if err := rdb.Close(); err != nil { log.Println(err) } continue } else { if err = rdb.Close(); err != nil { log.Println(err) } } //查询redis不存在新增 FClient.DbName = Config["mgodb_enterprise"] resulttmp := FClient.FindOne(Config["mgodb_enterprise_c"], bson.M{"company_name": tmp["winner"]}) if resulttmp != nil { //log.Println(r) //匹配不到原始库,存入异常表删除临时表 FClient.DbName = Config["mgodb_extract_kf"] if saveid := FClient.Save("winner_err", tmp); saveid == nil { log.Println("存入异常表错误", tmp) } FClient.DbName = Config["mgodb_extract_kf"] if deleteNum := FClient.DeleteById("winner_new", tmpId); deleteNum == 0 { log.Println("删除临时表错误", deleteNum) } continue } else { //log.Println(123) //匹配到原始库,新增 resulttmp if resulttmp["credit_no"] != nil { if credit_no, ok := resulttmp["credit_no"].(string); ok && strings.TrimSpace(credit_no) != "" && len(strings.TrimSpace(credit_no)) > 8 { dataNo := strings.TrimSpace(credit_no)[2:8] if Addrs[dataNo] != nil { if v, ok := Addrs[dataNo].(map[string]interface{}); ok { if resulttmp["province"] == nil || resulttmp["province"] == "" { resulttmp["province"] = v["province"] } resulttmp["city"] = v["city"] resulttmp["district"] = v["district"] } } } } contacts := make([]map[string]interface{}, 0) contact := make(map[string]interface{}, 0) if resulttmp["legal_person"] != nil { contact["contact_person"] = resulttmp["legal_person"] //联系人 } else { contact["contact_person"] = "" //联系人 } contact["contact_type"] = "法定代表人" //法定代表人 //log.Println(1) if resulttmp["annual_reports"] != nil { bytes, err := json.Marshal(resulttmp["annual_reports"]) if err != nil { log.Println("annual_reports err:", err) } phonetmp := make([]map[string]interface{}, 0) err = json.Unmarshal(bytes, &phonetmp) if err != nil { log.Println("Unmarshal err:", err) } for _, vv := range phonetmp { if vv["company_phone"] != nil { if vv["company_phone"] == "" { continue } else { contact["phone"] = vv["company_phone"] //联系电话 break } } else { contact["phone"] = "" //联系电话 } } } //log.Println(k, contact["phone"], resulttmp["_id"]) //time.Sleep(10 * time.Second) if contact["phone"] == nil { contact["phone"] = "" //联系电话 } contact["topscopeclass"] = "企业公示" //项目类型 contact["updatetime"] = time.Now().Unix() //更新时间 contacts = append(contacts, contact) resulttmp["contact"] = contacts savetmp := make(map[string]interface{}, 0) for _, sk := range Fields { if sk == "establish_date" { if resulttmp[sk] != nil { savetmp[sk] = resulttmp[sk].(primitive.DateTime).Time().UTC().Unix() continue } } else if sk == "capital" { //log.Println(sk, resulttmp[sk]) savetmp[sk] = ObjToMoney([]interface{}{resulttmp[sk], ""})[0] continue } else if sk == "partners" { //log.Println(sk, resulttmp[sk], ) //fmt.Println(reflect.TypeOf(resulttmp[sk])) if resulttmp[sk] != nil { if ppms, ok := resulttmp[sk].(primitive.A); ok { for i, _ := range ppms { if ppms[i].(map[string]interface{})["stock_type"] != nil { ppms[i].(map[string]interface{})["stock_type"] = "企业公示" } delete(ppms[i].(map[string]interface{}), "identify_type") } savetmp[sk] = ppms } } else { savetmp[sk] = []interface{}{} } continue } else if sk == "_id" { savetmp["tmp"+sk] = resulttmp[sk] continue } else if sk == "area_code" { //行政区划代码 savetmp[sk] = fmt.Sprint(resulttmp[sk]) continue } else if sk == "report_websites" { //网址 if resulttmp["report_websites"] == nil { savetmp["website"] = "" } else { report_websitesArr := []string{} if ppms, ok := resulttmp[sk].(primitive.A); ok { for _, v := range ppms { if vvv, ok := v.(map[string]interface{}); ok { if rv, ok := vvv["website_url"].(string); ok { report_websitesArr = append(report_websitesArr, rv) } } } } sort.Strings(report_websitesArr) savetmp["website"] = strings.Join(report_websitesArr, ";") } continue } else if sk == "wechat_accounts" { savetmp[sk] = []interface{}{} continue } if resulttmp[sk] == nil && sk != "history_name" && sk != "wechat_accounts" && sk != "establish_date" && sk != "capital" && sk != "partners" && sk != "contact" && sk != "report_websites" { savetmp[sk] = "" } else { savetmp[sk] = resulttmp[sk] } } //tmps = append(tmps, savetmp) savetmp["updatatime"] = time.Now().Unix() //保存mongo FClient.DbName = Config["mgodb_extract_kf"] saveid := FClient.Save(Config["mgo_qyk_c"], savetmp) if saveid != nil { //保存redis rc := RedisPool.Get() var _id string if v, ok := saveid.(primitive.ObjectID); ok { _id = v.Hex() } if _, err := rc.Do("SET", savetmp["company_name"], _id); err != nil { log.Println("save redis err:", tmp["_id"], savetmp["_id"], savetmp["company_name"], err) if err := rc.Close(); err != nil { log.Println(err) } } else { //保存es delete(savetmp, "_id") if err := rc.Close(); err != nil { log.Println(err) } //esConn := elastic.GetEsConn() //defer elastic.DestoryEsConn(esConn) if _, err := EsConn.Index().Index(Config["elasticsearch_index"]).Type(Config["elasticsearch_type"]).Id(_id).BodyJson(savetmp).Refresh(true).Do(); err != nil { log.Println("save es err :", tmp["_id"], savetmp["_id"], err) } else { //删除临时表 FClient.DbName = Config["mgodb_extract_kf"] if deleteNum := FClient.DeleteById("winner_new", tmpId); deleteNum == 0 { log.Println("删除临时表失败", deleteNum) } } } } else { log.Println("save mongo err:", saveid, tmp["_id"]) } } } } } t2.Reset(time.Minute) } }