备份 标讯数据

data_export_bank/src/config.json

@@ -2,19 +2,19 @@
   "bid_mgodb": {
     "addr": "127.0.0.1:27017",
     "db": "zhengkun",
-    "coll": "extract_test",
+    "coll": "zktest_export_bank",
     "pool": 10
   },
   "ext_mgodb": {
     "addr": "127.0.0.1:27017",
     "db": "zhengkun",
-    "coll": "extract_test",
+    "coll": "zktest_export_bank_ext",
     "pool": 10
   },
   "qy_mgodb": {
     "addr": "127.0.0.1:27017",
-    "db": "zhengkun",
-    "coll": "extract_test",
+    "db": "mixdata",
+    "coll": "qyxy_std",
     "pool": 10
   },
   "bid_tagText": [
@@ -58,8 +58,8 @@
       "招标人",
       "招标人联系人",
       "联系电话",
-      "中标公告全文",
-      "中标公告链接"
+      "中标公告链接",
+      "中标公告全文"
     ],
     [
       "序号",

data_export_bank/src/export_bidata.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
 	"github.com/tealeg/xlsx"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"os"
 	qu "qfw/util"
 	"time"
@@ -56,13 +57,6 @@ func initBidata()  {
 
 
 	log.Debug("测试保存数据...")
-
-	f_1.Save("招标信息.xlsx")
-	f_2.Save("中标信息.xlsx")
-	f_3.Save("招标项目.xlsx")
-	f_4.Save("开标公示.xlsx")
-	f_5.Save("评标公示.xlsx")
-
 }
 
 func exportBidata() {
@@ -124,9 +118,15 @@ func exportBidata() {
 		tmp = make(map[string]interface{})
 	}
 
-	log.Debug("is over bid ~",total,num_1,num_2,num_3,num_4)
+	log.Debug("is over bid ~",total,"~",num_1,num_2,num_3,num_4)
 
 
+	if total>0 {
+		f_1.Save("招标信息.xlsx")
+		f_2.Save("中标信息.xlsx")
+		f_3.Save("招标项目.xlsx")
+		f_4.Save("开标公示.xlsx")
+	}
 }
 
 //导出抽取数据-评标相关
@@ -151,27 +151,48 @@ func exportExtdata() {
 			continue
 		}
 		spidercode := qu.ObjToString(tmp["spidercode"])
-		if pb_spicode[spidercode]!=""||tmp["winnerorder"]!=nil{
+		winnerorder := []map[string]interface{}{}
+		if w_order, ok := tmp["winnerorder"].(primitive.A); ok {
+			winnerorder = qu.ObjArrToMapArr(w_order)
+		}else {
+			if w_order, ok := tmp["winnerorder"].([]interface{}); ok {
+				winnerorder = qu.ObjArrToMapArr(w_order)
+			}
+		}
+		if pb_spicode[spidercode]!=""||len(winnerorder)>0{
 			num_5++
-			data_5 := dealWithPingBiao(tmp,num_5)
+			winners:=""
+			if len(winnerorder)>0 {
+				for _,v := range winnerorder {
+					entname := qu.ObjToString(v["entname"])
+					if entname!=""{
+						if winners=="" {
+							winners = entname
+						}else {
+							winners = winners+","+entname
+						}
+					}
+				}
+			}
+			if winners=="" {
+				winners = qu.ObjToString(tmp["s_winner"])
+			}
+			data_5 := dealWithPingBiao(tmp,num_5,winners)
 			row_5:= sheet_5.AddRow()
 			for _,v := range data_5{
 				row_5.AddCell().Value = v
 			}
 		}
 
-
-
-
-
-
 		tmp = make(map[string]interface{})
 	}
 
 
 	log.Debug("is over ext ~",total,num_5)
 
-
+	if total>0 {
+		f_5.Save("评标公示.xlsx")
+	}
 }
 
 
@@ -194,32 +215,47 @@ func dealWithZhaoBiao(tmp map[string]interface{},index int)[]string {
 		dataArr = append(dataArr,"")
 	}
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
-
 	publishtime := qu.Int64All(tmp["publishtime"])
-	dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
+	if publishtime>0 {
+		dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
+	}else {
+		dataArr = append(dataArr,"")
+	}
+
 	dataArr = append(dataArr,"")//公告结束日期
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
+	buyer := qu.ObjToString(tmp["buyer"])
+	dataArr = append(dataArr,buyer)
 	price := qu.Float64All(tmp["budget"])
-	price_str := ""
+	price_str,price_unit := "",""
 	if price>0.0 {
 		price_str = fmt.Sprintf("%.2f",price)
+		price_unit = "元"
 	}
 	dataArr = append(dataArr,price_str)
-	dataArr = append(dataArr,"元")
+	dataArr = append(dataArr,price_unit)
+
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyeraddr"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyerperson"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyertel"]))
 
-	dataArr = append(dataArr,"")//邮箱
-	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
+
+	if buyer!="" {
+		qy_data := qy_mgo.FindOne(qy_coll, map[string]interface{}{
+			"company_name":buyer,
+		})
+		dataArr = append(dataArr,qu.ObjToString(qy_data["company_email"]))//邮箱
+	}else {
+		dataArr = append(dataArr,"")//邮箱
+	}
+
 	dataArr = append(dataArr,qu.ObjToString(tmp["href"]))
+	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
 
 	return dataArr
 }
 
 //中标信息
 func dealWithZhongBiao(tmp map[string]interface{},index int)[]string {
-
 	dataArr := []string{}
 	spidercode := qu.ObjToString(tmp["spidercode"])
 	dataArr = append(dataArr,fmt.Sprintf("%d",index))
@@ -236,35 +272,47 @@ func dealWithZhongBiao(tmp map[string]interface{},index int)[]string {
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
 
 	publishtime := qu.Int64All(tmp["publishtime"])
-	dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
-	dataArr = append(dataArr,"")//公告结束日期
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
-	price := qu.Float64All(tmp["budget"])
-	price_str := ""
+	if publishtime>0 {
+		dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
+	}else {
+		dataArr = append(dataArr,"")
+	}
+	dataArr = append(dataArr,"")//招标公告发布日期
+	bidopentime := qu.Int64All(tmp["bidopentime"])
+	if bidopentime>0 {
+		dataArr = append(dataArr,time.Unix(bidopentime, 0).Format(timeLayout))
+	}else {
+		dataArr = append(dataArr,"")
+	}
+
+	dataArr = append(dataArr,qu.ObjToString(tmp["s_winner"]))
+	price := qu.Float64All(tmp["bidamount"])
+	price_str,price_unit := "",""
 	if price>0.0 {
 		price_str = fmt.Sprintf("%.2f",price)
+		price_unit = "元"
 	}
 	dataArr = append(dataArr,price_str)
-	dataArr = append(dataArr,"元")
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyeraddr"]))
+	dataArr = append(dataArr,price_unit)
+
+	dataArr = append(dataArr,"")//评标委员会名单
+	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyerperson"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["buyertel"]))
-
-	dataArr = append(dataArr,"")//邮箱
-	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["href"]))
+	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
 
 	return dataArr
 }
 
 //项目信息
 func dealWithXiangMu(tmp map[string]interface{},index int)[]string {
-
 	dataArr := []string{}
 	spidercode := qu.ObjToString(tmp["spidercode"])
 	dataArr = append(dataArr,fmt.Sprintf("%d",index))
 	dataArr = append(dataArr,qu.ObjToString(tmp["site"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["projectname"]))
+	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["area"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["city"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["district"]))
@@ -273,38 +321,37 @@ func dealWithXiangMu(tmp map[string]interface{},index int)[]string {
 	}else {
 		dataArr = append(dataArr,"")
 	}
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
-
-	publishtime := qu.Int64All(tmp["publishtime"])
-	dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
-	dataArr = append(dataArr,"")//公告结束日期
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
-	price := qu.Float64All(tmp["budget"])
-	price_str := ""
-	if price>0.0 {
-		price_str = fmt.Sprintf("%.2f",price)
+	dataArr = append(dataArr,"")//招标项目建立时间
+	agency := qu.ObjToString(tmp["agency"])
+	if agency!="" {
+		qy_data := qy_mgo.FindOne(qy_coll, map[string]interface{}{
+			"company_name":agency,
+		})
+		dataArr = append(dataArr,qu.ObjToString(qy_data["credit_no"]))//招标代理机构代码
+	}else {
+		dataArr = append(dataArr)//招标代理机构代码
 	}
-	dataArr = append(dataArr,price_str)
-	dataArr = append(dataArr,"元")
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyeraddr"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerperson"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyertel"]))
-
-	dataArr = append(dataArr,"")//邮箱
-	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["href"]))
+	dataArr = append(dataArr,agency)
+	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
+	//组织~~~部门名称
+	dataArr = append(dataArr,"")
+	dataArr = append(dataArr,"")
+	dataArr = append(dataArr,"")
+	dataArr = append(dataArr,"")
+	dataArr = append(dataArr,"")
+	dataArr = append(dataArr,qu.ObjToString(tmp["projectscope"]))
 
 	return dataArr
 }
 
 //开标信息
 func dealWithKaiBiao(tmp map[string]interface{},index int)[]string {
-
 	dataArr := []string{}
 	spidercode := qu.ObjToString(tmp["spidercode"])
 	dataArr = append(dataArr,fmt.Sprintf("%d",index))
 	dataArr = append(dataArr,qu.ObjToString(tmp["site"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["projectname"]))
+	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["area"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["city"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["district"]))
@@ -313,38 +360,23 @@ func dealWithKaiBiao(tmp map[string]interface{},index int)[]string {
 	}else {
 		dataArr = append(dataArr,"")
 	}
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
-
-	publishtime := qu.Int64All(tmp["publishtime"])
-	dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
-	dataArr = append(dataArr,"")//公告结束日期
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
-	price := qu.Float64All(tmp["budget"])
-	price_str := ""
-	if price>0.0 {
-		price_str = fmt.Sprintf("%.2f",price)
+	bidopentime := qu.Int64All(tmp["bidopentime"])
+	if bidopentime>0 {
+		dataArr = append(dataArr,time.Unix(bidopentime, 0).Format(timeLayout))
+	}else {
+		dataArr = append(dataArr,"")
 	}
-	dataArr = append(dataArr,price_str)
-	dataArr = append(dataArr,"元")
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyeraddr"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerperson"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyertel"]))
-
-	dataArr = append(dataArr,"")//邮箱
-	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["href"]))
-
 	return dataArr
 }
 
 //评标信息
-func dealWithPingBiao(tmp map[string]interface{},index int)[]string {
-
+func dealWithPingBiao(tmp map[string]interface{},index int,winners string)[]string {
 	dataArr := []string{}
 	spidercode := qu.ObjToString(tmp["spidercode"])
 	dataArr = append(dataArr,fmt.Sprintf("%d",index))
 	dataArr = append(dataArr,qu.ObjToString(tmp["site"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["projectname"]))
+	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["area"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["city"]))
 	dataArr = append(dataArr,qu.ObjToString(tmp["district"]))
@@ -353,27 +385,14 @@ func dealWithPingBiao(tmp map[string]interface{},index int)[]string {
 	}else {
 		dataArr = append(dataArr,"")
 	}
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerclass"]))
-
 	publishtime := qu.Int64All(tmp["publishtime"])
-	dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
-	dataArr = append(dataArr,"")//公告结束日期
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyer"]))
-	price := qu.Float64All(tmp["budget"])
-	price_str := ""
-	if price>0.0 {
-		price_str = fmt.Sprintf("%.2f",price)
+	if publishtime>0 {
+		dataArr = append(dataArr,time.Unix(publishtime, 0).Format(timeLayout))
+	}else {
+		dataArr = append(dataArr,"")
 	}
-	dataArr = append(dataArr,price_str)
-	dataArr = append(dataArr,"元")
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyeraddr"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyerperson"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["buyertel"]))
-
-	dataArr = append(dataArr,"")//邮箱
-	dataArr = append(dataArr,qu.ObjToString(tmp["detail"]))
-	dataArr = append(dataArr,qu.ObjToString(tmp["href"]))
-
+	dataArr = append(dataArr,"")//公式结束时间
+	dataArr = append(dataArr,winners)
 	return dataArr
 }
 

data_export_bank/src/export_qydata.go

@@ -61,10 +61,10 @@ func initQydata()  {
 		row_11.AddCell().Value = v
 	}
 
-	f_6.Save("公司资质.xlsx")
-	f_7.Save("注册人员.xlsx")
-	f_8.Save("黑名单.xlsx")
-	f_9.Save("不良名单.xlsx")
-	f_10.Save("诚信惩戒.xlsx")
-	f_11.Save("围标串标.xlsx")
+	//f_6.Save("公司资质.xlsx")
+	//f_7.Save("注册人员.xlsx")
+	//f_8.Save("黑名单.xlsx")
+	//f_9.Save("不良名单.xlsx")
+	//f_10.Save("诚信惩戒.xlsx")
+	//f_11.Save("围标串标.xlsx")
 }

data_export_bank/src/main.go

@@ -35,13 +35,12 @@ func initMgo()  {
 
 	qyconf := sysconfig["qy_mgodb"].(map[string]interface{})
 	qy_coll = qu.ObjToString(qyconf["coll"])
-	bid_mgo = &MongodbSim{
+	qy_mgo = &MongodbSim{
 		MongodbAddr: qyconf["addr"].(string),
 		DbName:      qyconf["db"].(string),
 		Size:        qu.IntAllDef(qyconf["pool"], 5),
 	}
-	bid_mgo.InitPool()
-
+	qy_mgo.InitPool()
 }
 
 func initConfig()  {
@@ -99,7 +98,7 @@ func main()  {
 	log.Println("测试...")
 
 	exportBidata() //招标-中标-项目-开标
-	exportExtdata() //评标
+	//exportExtdata() //评标
 
 
 

listen_data/src/main.go

@@ -101,7 +101,7 @@ func main()  {
 
 	save_mgo = &MongodbSim{
 		MongodbAddr: "127.0.0.1:27017",
-		DbName:      "zhengkun",
+		DbName:      "mixdata",
 		Size:        10,
 		UserName: "",
 		Password: "",
@@ -109,24 +109,25 @@ func main()  {
 	save_mgo.InitPool()
 
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.85:27080",
+	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 	//	DbName:      "qfw",
 	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
+	//	UserName: "zhengkun",
+	//	Password: "zk@123123",
 	//}
 	//save_mgo.InitPool()
 
 
 	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+	//	MongodbAddr: "172.17.4.85:27080",
 	//	DbName:      "qfw",
 	//	Size:        10,
-	//	UserName: "zhengkun",
-	//	Password: "zk@123123",
+	//	UserName: "",
+	//	Password: "",
 	//}
 	//save_mgo.InitPool()
 
+
 	/*
 		repairYiXieChaData() 		//医械查
 		dealWithClassData()			//AB 匹配
@@ -135,10 +136,6 @@ func main()  {
 	*/
 
 
-
-
-
-	decodeJyUrl()
 	return
 
 

listen_data/src/zkmethod.go

@@ -52,27 +52,84 @@ func encodeJyUrl()  {
 	var Encode  = fmt.Sprintf(Url, qu.CommonEncodeArticle("content", "5fd2f37bf0f9d716c171c021"))
 	log.Debug(Encode)
 }
+
+//修复抽取数据
+func addCityXinXingExt() {
+	/*
+		"code" : "650109004002",
+	    "pcode" : "650109004",
+	    "province" : "新疆",
+	    "city" : "乌鲁木齐市",
+	    "district" : "米东区",
+	    "town" : "古牧地东路街道",
+	    "village" : "北苑社区居委会"
+	*/
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q ,total := map[string]interface{}{},0
+	it := sess.DB("mixdata").C("111222").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		province := qu.ObjToString(tmp["province"])
+		city := qu.ObjToString(tmp["city"])
+		district := qu.ObjToString(tmp["district"])
+		town := qu.ObjToString(tmp["town"])
+		village := qu.ObjToString(tmp["village"])
+
+		city_code := fmt.Sprintf("%d",qu.IntAll(tmp["city_code"]))
+		district_code := fmt.Sprintf("%d",qu.IntAll(tmp["district_code"]))
+		town_code := fmt.Sprintf("%d",qu.IntAll(tmp["town_code"]))
+		village_code := fmt.Sprintf("%d",qu.IntAll(tmp["village_code"]))
+
+		data := map[string]interface{}{}
+		data["province"] = province
+		data["city"] = city
+		data["district"] = district
+
+		if village !=""{ //五级
+			data["town"] = town
+			data["village"] = village
+			data["code"] = village_code
+			data["pcode"] = town_code
+		}else {
+			if town!="" {//四级
+				data["town"] = town
+				data["code"] = town_code
+				data["pcode"] = district_code
+			}else {
+				if city!="" { //三级
+					data["code"] = district_code
+					data["pcode"] = city_code
+				}else {
+					log.Debug("异常")
+				}
+			}
+		}
+
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is over ",total)
+}
+
+
 //copy数据 2022-02-19之后
 func repairCopyNextData() {
-	//621508800000000000000000 2-23 之后的数据
-	log.Debug("copy 之后的数据")
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
-			"$gt": StringTOBsonId("121508800000000000000000"),
-			"$lte": StringTOBsonId("921c1c8032770a446e8b2732"),
+			"$lte": StringTOBsonId("621508800000000000000000"),
 		},
 	}
 	log.Debug("查询条件~",q)
-	total:= 0
-	//适当增加线程数
-	repair_pool := make(chan bool, 5)
+	total,isok:= 0,0
+	repair_pool := make(chan bool, 10)
 	repair_wg := &sync.WaitGroup{}
-	it := sess.DB("qfw").C("result_20210109").Find(&q).Sort("_id").Iter()
+	it := sess.DB("qfw").C("result_20220218").Find(&q).Sort("_id").Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%10000==0 {
-			log.Debug("curent index ",total,tmp["_id"])
+			log.Debug("curent index ",total,isok,tmp["_id"])
 		}
 		repair_pool <- true
 		repair_wg.Add(1)
@@ -81,14 +138,36 @@ func repairCopyNextData() {
 				<-repair_pool
 				repair_wg.Done()
 			}()
-			save_mgo.Save("result_20220219",tmp)
-
+			tmpid := BsonTOStringId(tmp["_id"])
+			update_dict := map[string]interface{}{}
+			data := save_mgo.FindById("result_20210108",tmpid)
+			if data!=nil && len(data)>2 {
+				if data["buyerclass"]!=nil {
+					update_dict["buyerclass"] = data["buyerclass"]
+				}
+				if data["subscopeclass"]!=nil {
+					update_dict["subscopeclass"] = data["subscopeclass"]
+				}
+				if data["topscopeclass"]!=nil {
+					update_dict["topscopeclass"] = data["topscopeclass"]
+				}
 
+				if len(update_dict)>0 {
+					isok++
+					save_mgo.UpdateById("result_20220218",tmpid, map[string]interface{}{
+						"$set": update_dict,
+					})
+				}else {
+					//log.Debug("未分类~~",tmpid)
+				}
+			}else {
+				//log.Debug("未关联~~",tmpid)
+			}
 		}(tmp)
 		tmp = make(map[string]interface{})
 	}
 	repair_wg.Wait()
-	log.Debug("is copy over ",total)
+	log.Debug("is copy over ",total,isok)
 
 }