fieldproject/data_credible/task.go

package main

import (
	"data_credible/config"
	"fmt"
	"go.mongodb.org/mongo-driver/bson"
	util "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"strings"
)

func task1(tmp map[string]interface{}) {
	id := util.ObjToString(tmp["id"])
	save := make(map[string]interface{})
	if id == "" {
		id = mongodb.BsonIdToSId(tmp["_id"])
	}
	save["infoid"] = id
	save["data_tag"] = 2
	site := util.ObjToString(tmp["site"])
	save["site"] = site
	if SiteMap[site] != nil {
		s1 := strings.Split(util.ObjToString(SiteMap[site]), "-")
		if s1[0] != "" {
			save["site_toptype"] = s1[0]
		}
		if s1[1] != "" {
			save["site_subtype"] = s1[1]
		}
	} else {
		//log.Error("siteMap not find", zap.String("site", site))
	}
	if CodeMap[util.ObjToString(tmp["spidercode"])] != nil {
		save["collection_platform"] = CodeMap[util.ObjToString(tmp["spidercode"])]
	} else {
		//log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(tmp["spidercode"])))
	}
	save["href"] = tmp["href"]
	if tmp["jybxhref"] != nil {
		save["jyhref"] = tmp["jybxhref"]
	} else {
		save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id))
	}
	for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} {
		if tmp[s] != nil {
			save[s] = tmp[s]
		}
	}

	p := util.Int64All(tmp["publishtime"])
	save["publishtime"] = p
	save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout)
	if tmp["updatetime"] != nil {
		u := util.Int64All(tmp["updatetime"])
		save["updatetime"] = u
		save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
	} else {
		u := util.Int64All(tmp["createtime"])
		save["updatetime"] = u
		save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
	}
	if util.ObjToString(tmp["purchasing"]) != "" {
		save["is_purchasing"] = 1
	} else {
		save["is_purchasing"] = 0
	}

	tagM := make(map[string]interface{}) // 标注字段
	check := make(map[string]interface{})
	tag := 0 // 标注字段
	tagC := 0
	tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0})
	blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1})
	for _, f := range config.Conf.Serve.Fields {
		if tmp[fmt.Sprintf("ck_%s", f)] != nil {
			tag++
			tagM[f] = util.IntAll(tmp[fmt.Sprintf("ck_%s", f)])
		}
		if tmp[f] == nil {
			continue
		}
		m := make(map[string]interface{})
		if f == "package" {
			if tagM[f] != nil {
				tagC++
				m["clean_flag"] = 1
				m["is_mark"] = 2
				execObj(f, save, tmp, *tlog)
			} else {
				execObj1(f, save, tmp, *tlog)
				m["clean_flag"] = 0
			}
		} else if f == "purchasinglist" || f == "procurementlist" {
			if tagM[f] != nil {
				tagC++
				m["clean_flag"] = 1
				m["is_mark"] = 2
				execObj(f, save, tmp, *blog)
			} else {
				execObj1(f, save, tmp, *blog)
				m["clean_flag"] = 0
			}
		} else {
			if tagM[f] != nil {
				m["clean_flag"] = 1
				t := util.IntAll(tagM[f])
				if t == 1 {
					m["is_mark"] = t
				} else {
					m["is_mark"] = 2
				}
				if t != 1 {
					tagC++
					if (*tlog)[f] != nil {
						m["pre_value"] = (*tlog)[f]
					}
					m["aft_value"] = tmp[f]
				} else {
					m["pre_value"] = tmp[f]
				}
			} else {
				m["pre_value"] = tmp[f]
				m["clean_flag"] = 0
			}
		}
		check[f] = m
	}
	save["need_field_num"] = tag
	save["v_fieldcheck"] = check
	save["clean_field_num"] = tagC
	if len(save) > 0 {
		savePool <- save
	}
}

func taskInfo(tmp map[string]interface{}) {
	if info, ok := tmp["v_baseinfo"].(map[string]interface{}); ok {
		id := util.ObjToString(info["id"])
		if id == "" {
			id = util.ObjToString(tmp["id"])
		}
		save := make(map[string]interface{})
		save["infoid"] = id
		save["data_tag"] = 1
		site := util.ObjToString(info["site"])
		save["site"] = site
		if SiteMap[site] != nil {
			s1 := strings.Split(util.ObjToString(SiteMap[site]), "-")
			if s1[0] != "" {
				save["site_toptype"] = s1[0]
			}
			if s1[1] != "" {
				save["site_subtype"] = s1[1]
			}
		} else {
			//log.Error("siteMap not find", zap.String("site", site))
		}
		if CodeMap[util.ObjToString(info["spidercode"])] != nil {
			save["collection_platform"] = CodeMap[util.ObjToString(info["spidercode"])]
		} else {
			//log.Error("codeMap not find", zap.String("spidercode", util.ObjToString(info["spidercode"])))
		}
		save["href"] = info["href"]
		if tmp["jybxhref"] != nil {
			save["jyhref"] = tmp["jybxhref"]
		} else {
			save["jyhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", id))
		}
		for _, s := range []string{"spidercode", "buyerclass", "toptype", "subtype", "s_topscopeclass", "s_subscopeclass", "area", "city", "multipackage"} {
			if info[s] != nil {
				save[s] = info[s]
			}
		}
		p := util.Int64All(info["publishtime"])
		save["publishtime"] = p
		save["s_publishtime"] = util.FormatDateByInt64(&p, util.Date_Short_Layout)
		if tmp["i_updatetime"] != nil {
			u := util.Int64All(tmp["i_updatetime"])
			save["updatetime"] = u
			save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
		} else {
			u := util.Int64All(tmp["i_createtime"])
			save["updatetime"] = u
			save["s_updatetime"] = util.FormatDateByInt64(&u, util.Date_Short_Layout)
		}
		if util.ObjToString(info["purchasing"]) != "" {
			save["is_purchasing"] = 1
		} else {
			save["is_purchasing"] = 0
		}
		tag := tmp["v_taginfo"].(map[string]interface{})
		save["need_field_num"] = len(tag)
		check := make(map[string]interface{})
		tagC := 0
		tlog, _ := MongoE.FindById(config.Conf.DB.MongoE.Coll, id, bson.M{"field_source": 0, "kvtext": 0})
		blog, _ := Mongo.FindById("bidding", id, bson.M{"purchasinglist": 1, "procurementlist": 1, "toptype": 1, "subtype": 1})
		save["toptype"] = (*blog)["toptype"]
		save["subtype"] = (*blog)["subtype"]
		for _, f := range config.Conf.Serve.Fields {
			if info[f] == nil {
				continue
			}
			m := make(map[string]interface{})
			if f == "package" {
				if tag[f] != nil {
					tagC++
					m["clean_flag"] = 1
					m["is_mark"] = 2
					execObj(f, save, info, *tlog)
				} else {
					execObj1(f, save, info, *tlog)
					m["clean_flag"] = 0
				}
			} else if f == "purchasinglist" || f == "procurementlist" {
				if tag[f] != nil {
					tagC++
					m["clean_flag"] = 1
					m["is_mark"] = 2
					execObj(f, save, info, *blog)
				} else {
					execObj1(f, save, tmp, *blog)
					m["clean_flag"] = 0
				}
			} else {
				if tag[f] != nil {
					m["clean_flag"] = 1
					t := util.IntAll(tag[f])
					m["is_mark"] = t
					if t != 1 {
						tagC++
						if (*tlog)[f] != nil {
							m["pre_value"] = (*tlog)[f]
						}
						m["aft_value"] = info[f]
					} else {
						m["pre_value"] = info[f]
					}
				} else {
					m["pre_value"] = info[f]
					m["clean_flag"] = 0
				}
			}
			check[f] = m
		}
		save["v_fieldcheck"] = check
		save["clean_field_num"] = tagC
		if len(save) > 0 {
			savePool <- save
		}
	}
}

func execObj(f string, save, info, tinfo map[string]interface{}) {
	if info[f] == nil {
		return
	}
	m := make(map[string]interface{})
	if f == "package" {
		obj := info[f].(map[string]interface{})
		m["aft_num"] = len(obj)
		m["modify_num"] = len(obj)
		m["aft_value"] = obj
		if preObj, ok := tinfo[f].(map[string]interface{}); ok {
			m["pre_num"] = len(preObj)
			m["pre_value"] = preObj
		} else {
			m["pre_num"] = 0
		}

		if save["multipackage"] == nil {
			if package1 := m["pre_value"]; package1 != nil {
				packageM, _ := package1.(map[string]interface{})
				for _, p := range packageM {
					pm, _ := p.(map[string]interface{})
					if util.ObjToString(pm["winner"]) != "" || util.Float64All(pm["budget"]) > 0 ||
						util.Float64All(pm["bidamount"]) > 0 {
						save["multipackage"] = 1
						break
					}
				}
			} else {
				save["multipackage"] = 0
			}
		}
	} else {
		obj := info[f].([]interface{})
		m["aft_num"] = len(obj)
		m["modify_num"] = len(obj)
		m["aft_value"] = obj
		if preObj, ok := tinfo[f].([]interface{}); ok {
			m["pre_num"] = len(preObj)
			m["pre_value"] = preObj
		} else {
			m["pre_num"] = 0
		}
	}
	save[f] = m
}

func execObj1(f string, save, info, tinfo map[string]interface{}) {
	if info[f] == nil {
		return
	}
	m := make(map[string]interface{})
	if f == "package" {
		if preObj, ok := tinfo[f].(map[string]interface{}); ok {
			m["pre_num"] = len(preObj)
			m["pre_value"] = preObj
		} else {
			m["pre_num"] = 0
		}
	} else {
		if preObj, ok := tinfo[f].([]interface{}); ok {
			m["pre_num"] = len(preObj)
			m["pre_value"] = preObj
		} else {
			m["pre_num"] = 0
		}
	}
	save[f] = m
}