锐捷数据 导出-

data_export_yx/src/cgyx.go

@@ -1,15 +1,81 @@
 package main
 
 import (
+	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
+	"github.com/tealeg/xlsx"
 	"go.mongodb.org/mongo-driver/bson/primitive"
+	"os"
 	qu "qfw/util"
-	"regexp"
 	"sync"
+	"time"
 )
 
-var P_NameReg *regexp.Regexp = regexp.MustCompile(`(项目)`)
+func exportExcle(){
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total := map[string]interface{}{},0
+
+	os.Remove("rj_data.xlsx")
+	f :=xlsx.NewFile()
+	sheet, _ := f.AddSheet("标的物项目信息")
+	row := sheet.AddRow()
+	row.AddCell().Value = "信息匹配词"
+	row.AddCell().Value = "大区"
+	row.AddCell().Value = "省份（锐捷划分）"
+	row.AddCell().Value = "省份"
+	row.AddCell().Value = "城市"
+	row.AddCell().Value = "行业"
+	row.AddCell().Value = "项目名称"
+	row.AddCell().Value = "项目金额（万元）"
+	row.AddCell().Value = "公告地址"
+	row.AddCell().Value = "剑鱼标讯地址"
+	row.AddCell().Value = "采购意向名称"
+	row.AddCell().Value = "采购内容"
+	row.AddCell().Value = "采购单位"
+	row.AddCell().Value = "采购时间"
+	row.AddCell().Value = "附件下载地址"
+	row.AddCell().Value = "采购单位联系人"
+	row.AddCell().Value = "采购单位联系电话"
+	row.AddCell().Value = "发布时间"
+
+
+	it := sess.DB(save_mgo.DbName).C(export_coll).Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++ {
+		if total%1000 == 0 {
+			log.Debug("curent index ", total, tmp["_id"])
+		}
+		row = sheet.AddRow()
+		row.AddCell().Value = qu.ObjToString(tmp["matchkey"])
+		row.AddCell().Value = qu.ObjToString(tmp["region"])
+		row.AddCell().Value = qu.ObjToString(tmp["region_pro"])
+		row.AddCell().Value = qu.ObjToString(tmp["area"])
+		row.AddCell().Value = qu.ObjToString(tmp["city"])
+		row.AddCell().Value = qu.ObjToString(tmp["buyerclass"])
+		row.AddCell().Value = qu.ObjToString(tmp["p_name"])
+		row.AddCell().Value = qu.ObjToString(tmp["p_price"])
+		row.AddCell().Value = qu.ObjToString(tmp["href"])
+		row.AddCell().Value = qu.ObjToString(tmp["jybxhref"])
+		row.AddCell().Value = qu.ObjToString(tmp["title"])
+		row.AddCell().Value = qu.ObjToString(tmp["p_scope"])
+		row.AddCell().Value = qu.ObjToString(tmp["buyer"])
+		row.AddCell().Value = qu.ObjToString(tmp["p_time"])
+		row.AddCell().Value = qu.ObjToString(tmp["file_url"])
+		row.AddCell().Value = qu.ObjToString(tmp["buyerperson"])
+		row.AddCell().Value = qu.ObjToString(tmp["buyertel"])
+		row.AddCell().Value = qu.ObjToString(tmp["publishtime"])
+
+		tmp = make(map[string]interface{})
+	}
+
+	err := f.Save("rj_data.xlsx")
+	if err != nil {
+		log.Debug("保存xlsx失败:", err)
+	}else {
+		log.Debug("保存xlsx成功:", err)
+	}
 
+}
 
 func dealWithCgyxData()  {
 
@@ -46,20 +112,70 @@ func dealWithCgyxData()  {
 					p_list = qu.ObjArrToMapArr(purchasinglist)
 				}
 			}
-			//清理正文-标签
-			//detail := cleandetail(qu.ObjToString(tmp["detail"]))
-			//tmp["detail"] = detail
 
-			if len(p_list)>0 {
-				new_list := comparisonList(p_list,tmp)
-				if len(new_list)>0 {
-					isok+=len(new_list)
+			f_list := []map[string]interface{}{}
+			if filehref, ok := tmp["filehref"].(primitive.A); ok {
+				f_list = qu.ObjArrToMapArr(filehref)
+			}else {
+				if filehref, ok := tmp["filehref"].([]interface{}); ok {
+					f_list = qu.ObjArrToMapArr(filehref)
 				}
+			}
 
+			//其他字段需要
+			file_url := ""
+			if len(f_list)>0 {
+				file_url = qu.ObjToString(f_list[0]["url"])
+			}
+			area := qu.ObjToString(tmp["area"])
+			city := qu.ObjToString(tmp["city"])
+			publishtime := qu.Int64All(tmp["publishtime"])
+			p_time_str := time.Unix(publishtime, 0).Format(timeLayout_1)
 
 
+			region := ""
+			region_pro := ""
+
+			if city_map[city]!=nil {
+				region = city_map[city]["region"]
+				region_pro = city_map[city]["region_pro"]
+			}else {
+				if area_map[area]!=nil {
+					region = area_map[area]["region"]
+					region_pro = area_map[area]["region_pro"]
+				}
+			}
 
 
+
+
+			if len(p_list)>0 {
+				new_list := comparisonList(p_list,tmp)
+				if len(new_list)>0 {
+					isok+=len(new_list) //计数使用
+					for _,list := range new_list {
+						data := map[string]interface{}{}
+						data["matchkey"] = qu.ObjToString(tmp["matchkey"])
+						data["region"] = region
+						data["region_pro"] = region_pro
+						data["area"] = area
+						data["city"] = city
+						data["buyerclass"] = qu.ObjToString(tmp["buyerclass"])
+						data["p_name"] = qu.ObjToString(list["p_name"])
+						data["p_price"] = qu.ObjToString(list["p_price"])
+						data["href"] = qu.ObjToString(tmp["href"])
+						data["jybxhref"] = qu.ObjToString(tmp["jybxhref"])
+						data["title"] = qu.ObjToString(tmp["title"])
+						data["p_scope"] = qu.ObjToString(list["p_scope"])
+						data["buyer"] = qu.ObjToString(tmp["buyer"])
+						data["p_time"] = qu.ObjToString(list["p_time"])
+						data["file_url"] = file_url
+						data["buyerperson"] = qu.ObjToString(tmp["buyerperson"])
+						data["buyertel"] = qu.ObjToString(tmp["buyertel"])
+						data["publishtime"] = p_time_str
+						save_mgo.Save(export_coll,data)
+					}
+				}
 			}
 		}(tmp)
 		tmp = make(map[string]interface{})
@@ -74,51 +190,51 @@ func dealWithCgyxData()  {
 func comparisonList(p_list []map[string]interface{} ,tmp map[string]interface{}) []map[string]interface{} {
 	new_list := []map[string]interface{}{}
 	for _,list := range p_list {
-		b := isValidList(list,tmp)
+		b,data := isValidList(list,tmp)
 		if b {
-			new_list = append(new_list,list)
+			new_list = append(new_list,data)
 		}
 	}
 	return new_list
 }
 
 
-func isValidList(list map[string]interface{},tmp map[string]interface{}) bool {
+func isValidList(list map[string]interface{},tmp map[string]interface{}) (bool,map[string]interface{}) {
 	publishtime := qu.Int64All(tmp["publishtime"])
-
-	field_dataArr :=[]string{}
-	field_dataArr = append(field_dataArr,qu.ObjToString(tmp["title"]))
-	field_dataArr = append(field_dataArr,qu.ObjToString(tmp["projectname"]))
-	field_dataArr = append(field_dataArr,qu.ObjToString(list["itemname"]))
-	field_dataArr = append(field_dataArr,qu.ObjToString(list["projectname"]))
-
 	expurasingtime := qu.ObjToString(list["expurasingtime"])
 	totalprice := qu.Float64All(list["totalprice"])
 	if expurasingtime=="" {
-		return false
+		return false,nil
 	}
 	//时间格式转换
 	exp_time := cleanStrToTimestamp(expurasingtime,publishtime)
 	if exp_time>0 && exp_time<end_time && exp_time>=start_time{
 	}else {
-		return false
+		return false,nil
 	}
 	//时间格式转换
 	if totalprice<300000.0 {
-		return false
+		return false,nil
 	}
 
 	//项目名称-过滤词相关  tmp 任意可选
-	for _,v :=range words_arr {
-		projectname:=""
-		keywords := qu.ObjToString(v["keywords"])
-		outwords := qu.ObjToString(v["outwords"])
-		k_reg := regexp.MustCompile(keywords)
-		o_reg := regexp.MustCompile(outwords)
-		if k_reg.MatchString(projectname) && !o_reg.MatchString(projectname) {
-			return true
-		}
-	}
-
-	return false
+	//for _,v :=range words_arr {
+	//	projectname:=""
+	//	keywords := qu.ObjToString(v["keywords"])
+	//	outwords := qu.ObjToString(v["outwords"])
+	//	k_reg := regexp.MustCompile(keywords)
+	//	o_reg := regexp.MustCompile(outwords)
+	//	if k_reg.MatchString(projectname) && !o_reg.MatchString(projectname) {
+	//		return true
+	//	}
+	//}
+
+	data := map[string]interface{}{}
+	exp_time_str := time.Unix(exp_time, 0).Format(timeLayout)
+	data["p_name"]= qu.ObjToString(list["projectname"])
+	data["p_price"]= fmt.Sprintf("%.2f",totalprice/10000.0)
+	data["p_scope"]= qu.ObjToString(list["projectscope"])
+	data["p_time"] = exp_time_str
+
+	return true,data
 }

data_export_yx/src/main.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"github.com/tealeg/xlsx"
 	"log"
 	qu "qfw/util"
+	"strings"
 	"time"
 )
 
@@ -12,10 +14,12 @@ var (
 	save_coll,export_coll		string
 	start_time,end_time			int64
 	words_arr					[]map[string]interface{}
+	timeLayout = "2006-01"
+	timeLayout_1 = "2006-01-02"
+	city_map					map[string]map[string]string
+	area_map					map[string]map[string]string
 )
 func initMgo()  {
-
-
 	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
 	save_coll = qu.ObjToString(saveconf["coll"])
 	save_mgo = &MongodbSim{
@@ -32,11 +36,61 @@ func initMgo()  {
 	words_arr = qu.ObjArrToMapArr(sysconfig["words_arr"].([]interface{}))
 	log.Println(sysconfig)
 }
+
+func initRJQY() {
+	city_map = map[string]map[string]string{}
+	area_map = map[string]map[string]string{}
+	ff,_ := xlsx.OpenFile("rjqy.xlsx")
+	for _, sheet := range ff.Sheets {
+		for _, row := range sheet.Rows {
+			var strArr []string
+			for _, cell := range row.Cells {
+				strArr = append(strArr, cell.String())
+			}
+			//匹配特殊市
+			str_0 := strArr[0]
+			str_1 := strArr[1]
+			str_2 := strArr[2]
+			if str_2!="" {
+				arr := strings.Split(str_2,"、")
+				if strings.Contains(str_2,"市") {
+					for _,v := range arr {
+						city_map[v] = map[string]string{
+							"region":str_0,
+							"region_pro":str_1,
+						}
+					}
+					area_map[str_1] = map[string]string{
+						"region":str_0,
+						"region_pro":str_1,
+					}
+				}else {
+					for _,v := range arr {
+						area_map[v] = map[string]string{
+							"region":str_0,
+							"region_pro":str_1,
+						}
+					}
+				}
+			}else {
+				area_map[str_1] = map[string]string{
+					"region":str_0,
+					"region_pro":str_1,
+				}
+			}
+		}
+	}
+}
+
 func init()  {
 	qu.ReadConfig(&sysconfig)
 	initMgo()
+	initRJQY()
 }
 func main()  {
 	log.Println("测试...")
-	dealWithCgyxData()
+	//dealWithCgyxData()
+
+	exportExcle()
+
 }

listen_data/src/main.go

@@ -108,16 +108,25 @@ func main()  {
 	//}
 	//save_mgo.InitPool()
 
+	//save_mgo = &MongodbSim{
+	//	MongodbAddr: "172.17.4.85:27080",
+	//	DbName:      "qfw",
+	//	Size:        10,
+	//	UserName: "",
+	//	Password: "",
+	//}
+	//save_mgo.InitPool()
+
+
 	save_mgo = &MongodbSim{
-		MongodbAddr: "172.17.4.85:27080",
+		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 		DbName:      "qfw",
 		Size:        10,
-		UserName: "",
-		Password: "",
+		UserName: "zhengkun",
+		Password: "zk@123123",
 	}
 	save_mgo.InitPool()
 
-
 	/*
 		repairYiXieChaData() 		//医械查
 		dealWithClassData()			//AB 匹配
@@ -125,9 +134,6 @@ func main()  {
 		repairEqClassData()			//同分类-补充-0123
 	*/
 
-
-	repairCopyNextData()
-
 	return
 
 

listen_data/src/zkmethod.go

@@ -54,6 +54,36 @@ func encodeJyUrl()  {
 }
 
 
+func aaaaaa(){
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total := map[string]interface{}{},0
+
+	it := sess.DB("qfw").C("zktest_cgyx_data").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%1000==0 {
+			log.Debug("curent index ",total,tmp["_id"])
+		}
+		tttid := qu.ObjToString(tmp["id"])
+		data := save_mgo.FindById("bidding",tttid)
+
+		if data==nil ||len(data)<2 {
+			log.Debug("异常",tttid)
+		}else {
+			updateid := BsonTOStringId(tmp["_id"])
+			save_mgo.UpdateById("zktest_cgyx_data",updateid, map[string]interface{}{
+				"$set": map[string]interface{}{
+					"purchasinglist":data["purchasinglist"],
+				},
+			})
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is set bdw over ",total)
+}
+
+
 //copy数据 2022-02-19之后
 func repairCopyNextData() {
 	//暂时截止 2-23日 621508800000000000000000