|
@@ -0,0 +1,457 @@
|
|
|
+package main
|
|
|
+
|
|
|
+import (
|
|
|
+ elastic "app.yhyue.com/moapp/jybase/es"
|
|
|
+ "fmt"
|
|
|
+ "go.mongodb.org/mongo-driver/bson/primitive"
|
|
|
+ u "jygit.jydev.jianyu360.cn/data_processing/common_utils"
|
|
|
+ "jygit.jydev.jianyu360.cn/data_processing/common_utils/redis"
|
|
|
+ "log"
|
|
|
+ "reflect"
|
|
|
+ "sort"
|
|
|
+ "strconv"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+func InitEs() {
|
|
|
+ Es = elastic.NewEs("v7", conf.Config.Es.Addr, conf.Config.Es.Size, conf.Config.Es.User, conf.Config.Es.Password)
|
|
|
+}
|
|
|
+
|
|
|
+// @Description subscopeclass、topscopeclass、package
|
|
|
+// 20230523 多包处理 subpackage = 1
|
|
|
+// @Author J 2022/6/7 5:54 PM
|
|
|
+func fieldFun(data map[string]interface{}) {
|
|
|
+ // 附件重采,数据同步时不更新判重标识(正常抽取判重)
|
|
|
+ if u.IntAll(data["repeat"]) == 1 {
|
|
|
+ data["extracttype"] = -1
|
|
|
+ //update["repeat_id"] = compare["repeat_id"]
|
|
|
+ } else {
|
|
|
+ data["extracttype"] = 1
|
|
|
+ }
|
|
|
+ //未抽取、判重处理
|
|
|
+ if u.IntAll(data["dataging"]) == 1 { //修改未抽取的bidding数据的dataging
|
|
|
+ data["dataging"] = 0 //
|
|
|
+ }
|
|
|
+ if subscopeclass, ok := data["subscopeclass"].(primitive.A); ok && len(subscopeclass) > 0 { //subscopeclass
|
|
|
+ m1 := map[string]bool{}
|
|
|
+ var newclass []string
|
|
|
+ for _, sc := range subscopeclass {
|
|
|
+ sclass, _ := sc.(string)
|
|
|
+ if !m1[sclass] {
|
|
|
+ m1[sclass] = true
|
|
|
+ newclass = append(newclass, sclass)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //data["subscopeclass"] = newclass
|
|
|
+ data["s_subscopeclass"] = strings.Join(newclass, ",")
|
|
|
+ }
|
|
|
+ if topscopeclass, ok := data["topscopeclass"].(primitive.A); ok && len(topscopeclass) > 0 { //topscopeclass
|
|
|
+ m2 := map[string]bool{}
|
|
|
+ var newclass []string
|
|
|
+ for _, tc := range topscopeclass {
|
|
|
+ tclass, _ := tc.(string)
|
|
|
+ tclass = regLetter.ReplaceAllString(tclass, "") // 去除字母
|
|
|
+ if !m2[tclass] {
|
|
|
+ m2[tclass] = true
|
|
|
+ newclass = append(newclass, tclass)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //data["topscopeclass"] = topscopeclass
|
|
|
+ data["s_topscopeclass"] = strings.Join(newclass, ",")
|
|
|
+ }
|
|
|
+ //package
|
|
|
+ if package1 := data["package"]; package1 != nil {
|
|
|
+ packageM, _ := package1.(map[string]interface{})
|
|
|
+ //data["package"] = packageM
|
|
|
+ for _, p := range packageM {
|
|
|
+ pm, _ := p.(map[string]interface{})
|
|
|
+ if u.ObjToString(pm["winner"]) != "" || u.Float64All(pm["budget"]) > 0 || u.Float64All(pm["bidamount"]) > 0 {
|
|
|
+ data["multipackage"] = 1
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ data["multipackage"] = 0
|
|
|
+ }
|
|
|
+
|
|
|
+ // subpackage
|
|
|
+ if data["package"] != nil && data["s_winner"] != nil && data["bidamount"] != nil {
|
|
|
+ if pg, ok := data["package"].(map[string]interface{}); ok && len(pg) > 1 {
|
|
|
+ var bmt []float64
|
|
|
+ var swn []string
|
|
|
+ for _, p := range pg {
|
|
|
+ p1 := p.(map[string]interface{})
|
|
|
+ if p1["bidamount"] != nil {
|
|
|
+ bmt = append(bmt, u.Float64All(p1["bidamount"]))
|
|
|
+ }
|
|
|
+ if w := u.ObjToString(p1["winner"]); w != "" {
|
|
|
+ swn = append(swn, w)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(bmt) > 1 && len(swn) > 1 {
|
|
|
+ sn := strings.Split(u.ObjToString(data["s_winner"]), ",")
|
|
|
+ sort.Strings(sn)
|
|
|
+ sort.Strings(swn)
|
|
|
+ swn1 := u.ObjArrToStringArr(Duplicate(swn)) // 去重
|
|
|
+ if strings.Join(swn1, ",") == strings.Join(sn, ",") {
|
|
|
+ bidamount := 0.0
|
|
|
+ for _, f := range bmt {
|
|
|
+ bidamount += f
|
|
|
+ }
|
|
|
+ if bidamount == u.Float64All(data["bidamount"]) {
|
|
|
+ data["subpackage"] = 1
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// Duplicate
|
|
|
+// @Description 去重
|
|
|
+// @Author J 2023/5/24 09:53
|
|
|
+func Duplicate(a interface{}) (ret []interface{}) {
|
|
|
+ va := reflect.ValueOf(a)
|
|
|
+ for i := 0; i < va.Len(); i++ {
|
|
|
+ if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ ret = append(ret, va.Index(i).Interface())
|
|
|
+ }
|
|
|
+ return ret
|
|
|
+}
|
|
|
+
|
|
|
+// @Description entidlist
|
|
|
+// @Author J 2022/6/7 2:36 PM
|
|
|
+func companyFun(s_winner string) (cid []string) {
|
|
|
+ sWinnerarr := strings.Split(s_winner, ",")
|
|
|
+ for _, w := range sWinnerarr {
|
|
|
+ if w != "" {
|
|
|
+ id := redis.GetStr("qyxy_id", w)
|
|
|
+ if id == "" {
|
|
|
+ ents, _ := MgoQ.Find(conf.Config.MongodbQ.Coll, map[string]interface{}{"company_name": w}, map[string]interface{}{"updatetime": -1}, map[string]interface{}{"company_name": 1}, false, -1, -1)
|
|
|
+ if len(*ents) > 0 {
|
|
|
+ id = u.ObjToString((*ents)[0]["_id"])
|
|
|
+ redis.PutCKV("qyxy_id", w, id)
|
|
|
+ } else {
|
|
|
+ ent, _ := MgoP.FindOne(conf.Config.MongodbP.Coll, map[string]interface{}{"history_name": w})
|
|
|
+ if len(*ent) > 0 {
|
|
|
+ id = u.ObjToString((*ent)["company_id"])
|
|
|
+ redis.PutCKV("qyxy_id", w, id)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if id == "" {
|
|
|
+ id = "-"
|
|
|
+ }
|
|
|
+ cid = append(cid, id)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return cid
|
|
|
+}
|
|
|
+
|
|
|
+// @Description update 修改bidding表,extractM修改抽取表
|
|
|
+// @Author J 2022/6/10 10:29 AM
|
|
|
+func typeFunc(data map[string]interface{}) {
|
|
|
+ if jyData, ok := data["jyfb_data"].(map[string]interface{}); ok {
|
|
|
+ if t := u.ObjToString(jyData["type"]); t != "" {
|
|
|
+ switch t {
|
|
|
+ //case "采购信息":
|
|
|
+ case "招标公告":
|
|
|
+ if u.ObjToString(data["toptype"]) != "招标" {
|
|
|
+ data["toptype"] = "招标"
|
|
|
+ delete(data, "subtype")
|
|
|
+ }
|
|
|
+ case "采购意向":
|
|
|
+ if u.ObjToString(data["toptype"]) != "采购意向" {
|
|
|
+ data["toptype"] = "采购意向"
|
|
|
+ data["subtype"] = "采购意向"
|
|
|
+ }
|
|
|
+ case "招标预告":
|
|
|
+ if u.ObjToString(data["toptype"]) != "预告" {
|
|
|
+ data["toptype"] = "预告"
|
|
|
+ delete(data, "subtype")
|
|
|
+ }
|
|
|
+ case "招标结果":
|
|
|
+ if u.ObjToString(data["toptype"]) != "结果" {
|
|
|
+ data["toptype"] = "结果"
|
|
|
+ delete(data, "subtype")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// @Description 附件有效字段(isValidFile)
|
|
|
+// @Author J 2022/7/8 14:41
|
|
|
+func validFile(tmp map[string]interface{}) int {
|
|
|
+ isContinue := false
|
|
|
+ if pinfo, o := tmp["projectinfo"].(map[string]interface{}); o {
|
|
|
+ if atts, o1 := pinfo["attachments"].(map[string]interface{}); o1 {
|
|
|
+ for _, att := range atts {
|
|
|
+ if att == nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if reflect.TypeOf(att).String() == "string" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ att1 := att.(map[string]interface{})
|
|
|
+ if fid := u.ObjToString(att1["fid"]); fid != "" {
|
|
|
+ isContinue = true
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if isContinue {
|
|
|
+ if attachTxt, o := tmp["attach_text"].(map[string]interface{}); o {
|
|
|
+ if len(attachTxt) > 0 {
|
|
|
+ for _, at := range attachTxt {
|
|
|
+ if at1, ok := at.(map[string]interface{}); ok && len(at1) > 0 {
|
|
|
+ for k, _ := range at1 {
|
|
|
+ if reflect.TypeOf(at1[k]).String() == "string" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ at2 := at1[k].(map[string]interface{})
|
|
|
+ s := strings.ToLower(u.ObjToString(at2["file_name"]))
|
|
|
+ //和王江含确认,此处判断老版本存在问题,正确为排除jpg、jpeg、png、pdf类型(2024-03-12)
|
|
|
+ //if !strings.Contains(s, "jpg") || !strings.Contains(s, "jpeg") != strings.Contains(s, "png") || strings.Contains(s, "pdf") {
|
|
|
+ if !filterFileType.MatchString(s) {
|
|
|
+ if strings.Contains(s, "swf") || strings.Contains(s, "html") {
|
|
|
+ return -1
|
|
|
+ } else if AnalysisFile(OssGetObject(u.ObjToString(at2["attach_url"]))) {
|
|
|
+ return 1
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ break
|
|
|
+ } else {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ flag := false
|
|
|
+ for _, att := range atts {
|
|
|
+ if att == nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if reflect.TypeOf(att).String() == "string" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ att1 := att.(map[string]interface{})
|
|
|
+ if fid := u.ObjToString(att1["fid"]); fid != "" {
|
|
|
+ ftype := strings.ToLower(u.ObjToString(tmp["ftype"]))
|
|
|
+ if ftype != "swf" && ftype != "html" && OssObjExists("jy-datafile", fid) {
|
|
|
+ return 1
|
|
|
+ } else {
|
|
|
+ flag = true
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if flag {
|
|
|
+ return -1
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return 0
|
|
|
+}
|
|
|
+
|
|
|
+var DateTimeSelect = []string{"bidopentime", "bidendtime", "signaturedate", "comeintime"}
|
|
|
+
|
|
|
+// @Description 发布时间处理
|
|
|
+// @Author J 2023/5/23 14:32
|
|
|
+func methodPb(tmp map[string]interface{}) {
|
|
|
+ if tmp["ext_publishtime"] != nil {
|
|
|
+ if newPb := u.Int64All(tmp["ext_publishtime"]); newPb < time.Now().Unix() && newPb > 1420041600 {
|
|
|
+ tmp["publishtime"] = newPb
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, d := range DateTimeSelect {
|
|
|
+ if tmp[d] != nil && u.Int64All(tmp[d]) < time.Now().Unix() {
|
|
|
+ tmp["publishtime"] = u.Int64All(tmp[d])
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// @Description 获取情报标签
|
|
|
+// @Author 徐志恒 2024/2/21 09:53
|
|
|
+func getTagSet(data map[string]interface{}) map[string]map[string]interface{} {
|
|
|
+ tagSet := map[string]map[string]interface{}{}
|
|
|
+ wuye := map[string]interface{}{}
|
|
|
+ buyer := u.ObjToString(data["buyer"])
|
|
|
+ publishtime := u.Int64All(data["publishtime"])
|
|
|
+ bidamount := u.Float64All(data["bidamount"])
|
|
|
+ wuye["isfirsthand"] = 62
|
|
|
+ if buyer != "" {
|
|
|
+ sql := `{
|
|
|
+ "query": {
|
|
|
+ "bool": {
|
|
|
+ "must": [
|
|
|
+ {
|
|
|
+ "term": {
|
|
|
+ "buyer": "` + buyer + `"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "term": {
|
|
|
+ "tag_topinformation": "情报_物业"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "term": {
|
|
|
+ "subtype": "合同"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "range": {
|
|
|
+ "publishtime": {
|
|
|
+ "lte": ` + fmt.Sprint(publishtime) + `
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "sort": {
|
|
|
+ "publishtime": "asc"
|
|
|
+ },
|
|
|
+ "_source": [
|
|
|
+ "s_winner"
|
|
|
+ ],
|
|
|
+ "size": 10000
|
|
|
+ }`
|
|
|
+ data := Es.Get("bidding", "bidding", sql)
|
|
|
+ if data != nil && len(*data) > 0 {
|
|
|
+ count := 0
|
|
|
+ first := u.ObjToString((*data)[0]["s_winner"])
|
|
|
+ for k, v := range *data {
|
|
|
+ winner := u.ObjToString(v["s_winner"])
|
|
|
+ if k > 0 && first != winner {
|
|
|
+ first = winner
|
|
|
+ count++
|
|
|
+ }
|
|
|
+ }
|
|
|
+ changehand := fmt.Sprintf("%.2f", float64(count)/float64(len(*data)))
|
|
|
+ changehands, _ := strconv.ParseFloat(changehand, 64)
|
|
|
+ wuye["changehand"] = changehands
|
|
|
+ log.Println("changehands", count, len(*data), changehands)
|
|
|
+ if changehands > 0.3 {
|
|
|
+ wuye["changehandindex"] = 61
|
|
|
+ }
|
|
|
+ if len(*data) > 1 {
|
|
|
+ wuye["isfirsthand"] = 0
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if data["projectinfo"] != nil {
|
|
|
+ projectInfo := u.ObjToMap(data["projectinfo"])
|
|
|
+ if projectInfo != nil && len(*projectInfo) > 0 {
|
|
|
+ if (*projectInfo)["attachments"] != nil {
|
|
|
+ wuye["isfile"] = 63
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ wuye["period"] = getperiod(data)
|
|
|
+ wuye["scale"] = getBidamountRange(bidamount)
|
|
|
+ if data["property_form"] != nil {
|
|
|
+ property_form := u.ObjArrToStringArr(data["property_form"].([]interface{}))
|
|
|
+ wuye["property_form"] = getpropertyform(property_form)
|
|
|
+ }
|
|
|
+ tagSet["wuye"] = wuye
|
|
|
+ return tagSet
|
|
|
+}
|
|
|
+
|
|
|
+func getBidamountRange(value float64) int {
|
|
|
+ switch {
|
|
|
+ case value < 500000:
|
|
|
+ return 1
|
|
|
+ case value >= 500000 && value < 1000000:
|
|
|
+ return 2
|
|
|
+ case value >= 1000000 && value < 2000000:
|
|
|
+ return 3
|
|
|
+ case value >= 2000000 && value < 5000000:
|
|
|
+ return 4
|
|
|
+ default:
|
|
|
+ return 5
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func getpropertyform(value []string) string {
|
|
|
+ arr := []string{}
|
|
|
+ categories := map[string]string{
|
|
|
+ "住宅": "21",
|
|
|
+ "政府办公楼": "22",
|
|
|
+ "学校": "23",
|
|
|
+ "医院": "24",
|
|
|
+ "产业园区": "25",
|
|
|
+ "旅游景区": "26",
|
|
|
+ "交通运输": "27",
|
|
|
+ "商务办公楼": "28",
|
|
|
+ "酒店": "29",
|
|
|
+ }
|
|
|
+ for _, v := range value {
|
|
|
+ if categories[v] != "" {
|
|
|
+ arr = append(arr, categories[v])
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return strings.Join(arr, ",")
|
|
|
+}
|
|
|
+
|
|
|
+func getperiod(data map[string]interface{}) int {
|
|
|
+ res := 16
|
|
|
+ signaturedate := u.Int64All(data["signaturedate"]) //合同签订日期
|
|
|
+ expiredate := u.Int64All(data["expiredate"]) //合同截止日期
|
|
|
+ // contractperiod := util.ObjToString(data["contractperiod"]) //合同期限
|
|
|
+ project_duration := u.IntAll(data["project_duration"]) //工期时长
|
|
|
+ project_timeunit := u.ObjToString(data["project_timeunit"]) //工期单位
|
|
|
+ result := float64(0)
|
|
|
+ if expiredate > 0 && signaturedate > 0 {
|
|
|
+ result = calculateYearDifference(signaturedate, expiredate)
|
|
|
+ } else if project_duration > 0 && project_timeunit != "" {
|
|
|
+ if strings.Contains(project_timeunit, "年") {
|
|
|
+ if project_duration == 1 {
|
|
|
+ res = 12
|
|
|
+ } else if project_duration == 2 {
|
|
|
+ res = 13
|
|
|
+ } else if project_duration == 3 {
|
|
|
+ res = 14
|
|
|
+ } else if project_duration == 5 {
|
|
|
+ res = 15
|
|
|
+ }
|
|
|
+ return res
|
|
|
+ } else if strings.Contains(project_timeunit, "月") {
|
|
|
+ result = float64(project_duration) / 12
|
|
|
+ } else if strings.Contains(project_timeunit, "周") {
|
|
|
+ result = float64(project_duration) * 7 / 365
|
|
|
+ } else if strings.Contains(project_timeunit, "日") || strings.Contains(project_timeunit, "天") {
|
|
|
+ result = float64(project_duration) / 365
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if result == 0 {
|
|
|
+ res = 16
|
|
|
+ } else if result < 1 {
|
|
|
+ res = 11
|
|
|
+ } else if result >= 1 && result < 2 {
|
|
|
+ res = 12
|
|
|
+ } else if result >= 2 && result < 3 {
|
|
|
+ res = 13
|
|
|
+ } else if result >= 3 && result < 4 {
|
|
|
+ res = 14
|
|
|
+ } else if result >= 5 {
|
|
|
+ res = 15
|
|
|
+ }
|
|
|
+ return res
|
|
|
+}
|
|
|
+
|
|
|
+func calculateYearDifference(startTime int64, endTime int64) float64 {
|
|
|
+ start := time.Unix(startTime, 0)
|
|
|
+ end := time.Unix(endTime, 0)
|
|
|
+ duration := end.Sub(start)
|
|
|
+ years := duration.Hours() / 24 / 365
|
|
|
+ return years
|
|
|
+}
|