package main import ( "data_credible/config" "fmt" "go.mongodb.org/mongo-driver/bson" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb" "strings" ) func task1(tmp map[string]interface{}) { id := util.ObjToString(tmp["id"]) save := make(map[string]interface{}) if id == "" { id = mongodb.BsonIdToSId(tmp["_id"]) } save["infoid"] = id save["data_tag"] = 2 site := util.ObjToString(tmp["site"]) save["site"] = site if SiteMap[site] != nil { s1 := strings.Split(util.ObjToString(SiteMap[site]), "-") if s1[0] != "" { save["site_toptype"] = s1[0] } if s1[1] != "" { save["site_subtype"] = s1[1] } } else { //log.Error("siteMap not find", zap.String("site", site)) } if CodeMap[util.ObjToString(tmp["spidercode"])] != nil { save["collection_platform"] = CodeMap[util.ObjToString(tmp["spidercode"])] } else { //log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(tmp["spidercode"]))) } save["href"] = tmp["href"] if tmp["jybxhref"] != nil { save["jyhref"] = tmp["jybxhref"] } else { save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id)) } for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} { if tmp[s] != nil { save[s] = tmp[s] } } p := util.Int64All(tmp["publishtime"]) save["publishtime"] = p save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout) if tmp["updatetime"] != nil { u := util.Int64All(tmp["updatetime"]) save["updatetime"] = u save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout) } else { u := util.Int64All(tmp["createtime"]) save["updatetime"] = u save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout) } if util.ObjToString(tmp["purchasing"]) != "" { save["is_purchasing"] = 1 } else { save["is_purchasing"] = 0 } tagM := make(map[string]interface{}) // 标注字段 check := make(map[string]interface{}) tag := 0 // 标注字段 tagC := 0 tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0}) blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1}) for _, f := range config.Conf.Serve.Fields { if tmp[fmt.Sprintf("ck_%s", f)] != nil { tag++ tagM[f] = util.IntAll(tmp[fmt.Sprintf("ck_%s", f)]) } if tmp[f] == nil { continue } m := make(map[string]interface{}) if f == "package" { if tagM[f] != nil { tagC++ m["clean_flag"] = 1 m["is_mark"] = 2 execObj(f, save, tmp, *tlog) } else { execObj1(f, save, tmp, *tlog) m["clean_flag"] = 0 } } else if f == "purchasinglist" || f == "procurementlist" { if tagM[f] != nil { tagC++ m["clean_flag"] = 1 m["is_mark"] = 2 execObj(f, save, tmp, *blog) } else { execObj1(f, save, tmp, *blog) m["clean_flag"] = 0 } } else { if tagM[f] != nil { m["clean_flag"] = 1 t := util.IntAll(tagM[f]) if t == 1 { m["is_mark"] = t } else { m["is_mark"] = 2 } if t != 1 { tagC++ if (*tlog)[f] != nil { m["pre_value"] = (*tlog)[f] } m["aft_value"] = tmp[f] } else { m["pre_value"] = tmp[f] } } else { m["pre_value"] = tmp[f] m["clean_flag"] = 0 } } check[f] = m } save["need_field_num"] = tag save["v_fieldcheck"] = check save["clean_field_num"] = tagC if len(save) > 0 { savePool <- save } } func taskInfo(tmp map[string]interface{}) { if info, ok := tmp["v_baseinfo"].(map[string]interface{}); ok { id := util.ObjToString(info["id"]) if id == "" { id = util.ObjToString(tmp["id"]) } save := make(map[string]interface{}) save["infoid"] = id save["data_tag"] = 1 site := util.ObjToString(info["site"]) save["site"] = site if SiteMap[site] != nil { s1 := strings.Split(util.ObjToString(SiteMap[site]), "-") if s1[0] != "" { save["site_toptype"] = s1[0] } if s1[1] != "" { save["site_subtype"] = s1[1] } } else { //log.Error("siteMap not find", zap.String("site", site)) } if CodeMap[util.ObjToString(info["spidercode"])] != nil { save["collection_platform"] = CodeMap[util.ObjToString(info["spidercode"])] } else { //log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(info["spidercode"]))) } save["href"] = info["href"] if tmp["jybxhref"] != nil { save["jyhref"] = tmp["jybxhref"] } else { save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id)) } for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} { if info[s] != nil { save[s] = info[s] } } p := util.Int64All(info["publishtime"]) save["publishtime"] = p save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout) if tmp["i_updatetime"] != nil { u := util.Int64All(tmp["i_updatetime"]) save["updatetime"] = u save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout) } else { u := util.Int64All(tmp["i_createtime"]) save["updatetime"] = u save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout) } if util.ObjToString(info["purchasing"]) != "" { save["is_purchasing"] = 1 } else { save["is_purchasing"] = 0 } tag := tmp["v_taginfo"].(map[string]interface{}) save["need_field_num"] = len(tag) check := make(map[string]interface{}) tagC := 0 tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0}) blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1, "toptype": 1, "subtype": 1}) save["toptype"] = (*blog)["toptype"] save["subtype"] = (*blog)["subtype"] for _, f := range config.Conf.Serve.Fields { if info[f] == nil { continue } m := make(map[string]interface{}) if f == "package" { if tag[f] != nil { tagC++ m["clean_flag"] = 1 m["is_mark"] = 2 execObj(f, save, info, *tlog) } else { execObj1(f, save, info, *tlog) m["clean_flag"] = 0 } } else if f == "purchasinglist" || f == "procurementlist" { if tag[f] != nil { tagC++ m["clean_flag"] = 1 m["is_mark"] = 2 execObj(f, save, info, *blog) } else { execObj1(f, save, tmp, *blog) m["clean_flag"] = 0 } } else { if tag[f] != nil { m["clean_flag"] = 1 t := util.IntAll(tag[f]) m["is_mark"] = t if t != 1 { tagC++ if (*tlog)[f] != nil { m["pre_value"] = (*tlog)[f] } m["aft_value"] = info[f] } else { m["pre_value"] = info[f] } } else { m["pre_value"] = info[f] m["clean_flag"] = 0 } } check[f] = m } save["v_fieldcheck"] = check save["clean_field_num"] = tagC if len(save) > 0 { savePool <- save } } } func execObj(f string, save, info, tinfo map[string]interface{}) { if info[f] == nil { return } m := make(map[string]interface{}) if f == "package" { obj := info[f].(map[string]interface{}) m["aft_num"] = len(obj) m["modify_num"] = len(obj) m["aft_value"] = obj if preObj, ok := tinfo[f].(map[string]interface{}); ok { m["pre_num"] = len(preObj) m["pre_value"] = preObj } else { m["pre_num"] = 0 } if save["multipackage"] == nil { if package1 := m["pre_value"]; package1 != nil { packageM, _ := package1.(map[string]interface{}) for _, p := range packageM { pm, _ := p.(map[string]interface{}) if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 || util.Float64All(pm["bidamount"]) > 0 { save["multipackage"] = 1 break } } } else { save["multipackage"] = 0 } } } else { obj := info[f].([]interface{}) m["aft_num"] = len(obj) m["modify_num"] = len(obj) m["aft_value"] = obj if preObj, ok := tinfo[f].([]interface{}); ok { m["pre_num"] = len(preObj) m["pre_value"] = preObj } else { m["pre_num"] = 0 } } save[f] = m } func execObj1(f string, save, info, tinfo map[string]interface{}) { if info[f] == nil { return } m := make(map[string]interface{}) if f == "package" { if preObj, ok := tinfo[f].(map[string]interface{}); ok { m["pre_num"] = len(preObj) m["pre_value"] = preObj } else { m["pre_num"] = 0 } } else { if preObj, ok := tinfo[f].([]interface{}); ok { m["pre_num"] = len(preObj) m["pre_value"] = preObj } else { m["pre_num"] = 0 } } save[f] = m }