备份

process_ medical/src/main.go

@@ -12,6 +12,13 @@ var (
 
 var timeReg = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})$`)
 var timeReg1 = regexp.MustCompile(`^(\d{4}-\d{1,2}-\d{1,2})$`)
+var Url = "https://www.jianyu360.com/article/content/%s.html"
+var bidclass_dict = map[string]string{}
+var buyerclass_dict = map[string]string{}
+var industryclass_dict = map[string]string{}
+var area_dict = map[string]string{}
+var proclass_dict = map[string]string{}
+
 
 func init()  {
 	//qy_mgo = &MongodbSim{
@@ -55,7 +62,7 @@ func init()  {
 	//	Password: "",
 	//}
 	//save_mgo.InitPool()
-	//
+
 	//MysqlTool = &Mysql{
 	//	Address: "192.168.3.109:4000",
 	//	UserName: "jianyu",
@@ -74,6 +81,34 @@ func init()  {
 	//
 
 
+	//save_mgo = &MongodbSim{
+	//	MongodbAddr: "127.0.0.1:27017",
+	//	DbName:      "mixdata",
+	//	Size:        10,
+	//	UserName: "",
+	//	Password: "",
+	//}
+	//save_mgo.InitPool()
+
+
+
+
+
+
+}
+
+
+
+
+func main()  {
+	MysqlTool = &Mysql{
+		Address: "192.168.3.109:4000",
+		UserName: "jianyu",
+		PassWord: "top@123",
+		DBName: "jy_lycp",
+	}
+	MysqlTool.Init()
+
 	save_mgo = &MongodbSim{
 		MongodbAddr: "127.0.0.1:27017",
 		DbName:      "mixdata",
@@ -82,18 +117,44 @@ func init()  {
 		Password: "",
 	}
 	save_mgo.InitPool()
-}
 
 
+	initMySqlMgoData() //加载数据-本地
 
 
-func main()  {
 	
 
+}
+
+
+
+
+
 
 
-}
 
 
+//导出源数据-程序
+func exportSource() {
+	save_mgo = &MongodbSim{
+		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+		DbName:      "qfw",
+		Size:        10,
+		UserName: "zhengkun",
+		Password: "zk@123123",
+	}
+	save_mgo.InitPool()
 
+	spi_mgo = &MongodbSim{
+		MongodbAddr: "172.17.4.87:27080",
+		DbName:      "py_spider",
+		Size:        10,
+		UserName: "",
+		Password: "",
+	}
+	spi_mgo.InitPool()
+
+
+	exportUsefulPurchasing()
+}
 

process_ medical/src/medical_bidding.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"fmt"
+	log "github.com/donnie4w/go-logger/logger"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	qu "qfw/util"
+	"strings"
+)
+
+
+
+
+
+
+
+
+
+func exportUsefulPurchasing() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{},0
+	isok:=0
+	it := sess.DB(save_mgo.DbName).C("ylhydata").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("curent index ",total,tmp["_id"],"~",isok)
+		}
+		p_list := []map[string]interface{}{}
+		new_data := map[string]interface{}{}
+		if yl_purchasinglist, ok := tmp["yl_purchasinglist"].(primitive.A); ok {
+			p_list = qu.ObjArrToMapArr(yl_purchasinglist)
+		}else {
+			if yl_purchasinglist, ok := tmp["yl_purchasinglist"].([]interface{}); ok {
+				p_list = qu.ObjArrToMapArr(yl_purchasinglist)
+			}
+		}
+		if len(p_list)>0 {
+			isok++
+			s_winner := qu.ObjToString(tmp["s_winner"])
+			new_data["yl_purchasinglist"] = p_list
+			tmpid := BsonTOStringId(tmp["_id"])
+			new_data["_id"] = tmp["_id"]
+			new_data["info_id"] = tmpid
+			new_data["subtype"] = tmp["subtype"]
+			new_data["toptype"] = tmp["toptype"]
+			new_data["projectname"] =tmp["projectname"]
+			new_data["publishtime"] =tmp["publishtime"]
+			new_data["buyer"] =tmp["buyer"]
+			new_data["buyerclass"] =tmp["buyerclass"]
+			new_data["budget"] =tmp["budget"]
+			new_data["bidamount"] = tmp["bidamount"]
+			new_data["s_winner"] =s_winner
+			new_data["agency"] =tmp["agency"]
+			new_data["area"] =tmp["area"]
+			new_data["city"] =tmp["city"]
+			new_data["district"] =tmp["district"]
+			new_data["jyhref"] = fmt.Sprintf(Url, qu.CommonEncodeArticle("content", BsonTOStringId(tmp["_id"])))
+			new_data["subscopeclass"] = tmp["subscopeclass"]
+			company_id:=""
+			//根据s_winner-关联company_id
+			if s_winner != "" && !strings.Contains(s_winner,",") {
+				spi_data := spi_mgo.FindOne("zktest_mysql_company_info", map[string]interface{}{
+					"company_name":s_winner,
+				})
+				if spi_data!=nil && len(spi_data)>2 {
+					company_id = qu.ObjToString(spi_data["company_id"])
+				}
+			}
+			new_data["company_id"] = company_id
+
+			spi_mgo.Save("zktest_mysql_source",new_data)
+		}
+
+
+
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is over ",total,isok)
+}
+
+func getUsefulPurchasingData(arr []map[string]interface{}) []map[string]interface{} {
+	new_arr :=[]map[string]interface{}{}
+	for _,v:=range arr {
+		if v["product_class_1"]!=nil {
+			new_arr = append(new_arr,v)
+		}
+	}
+	return new_arr
+}

process_ medical/src/medical_company.go

@@ -78,7 +78,7 @@ func dealWithCompanyData()  {
 			isok++
 			data := updateData(v,type_str)
 			if len(data)>2 && data!=nil {
-				save_mgo.Save("111111111111111",data)
+				save_mgo.Save("zktest_mysql_company_info",data)
 			}
 		}
 		tmp = make(map[string]interface{})

process_ medical/src/medical_hospital.go

@@ -0,0 +1,9 @@
+package main
+
+
+func hospitalSaveMysql() {
+
+
+
+
+}

process_ medical/src/medical_prepare.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	log "github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
+	"unicode/utf8"
+)
+
+
+func initMySqlMgoData() {
+	//招标分类
+	init_bidclass()
+	init_buyerclass()
+	init_industryclass()
+	init_area()
+	init_proclass()
+
+	log.Debug("init mysql 完毕",len(bidclass_dict),len(buyerclass_dict),len(industryclass_dict))
+	log.Debug("init mgo 完毕",len(area_dict),len(proclass_dict))
+
+}
+
+
+
+func init_bidclass() {
+	arr:= *MysqlTool.Find("v_bidclass",nil,"","",-1,-1)
+	pre_str :=""
+	for _,v := range arr {
+		level := qu.ObjToString(v["level"])
+		name := qu.ObjToString(v["name"])
+		code := qu.ObjToString(v["code"])
+		if level=="1" {
+			pre_str = name
+			if name=="采购意向" {
+				key := pre_str+"_"
+				bidclass_dict[key] = code
+			}
+		}else {
+			key := pre_str+"_"+name
+			bidclass_dict[key] = code
+		}
+	}
+}
+func init_buyerclass() {
+	//采购单位分类 v_buyerclass
+	arr:= *MysqlTool.Find("v_buyerclass",nil,"","",-1,-1)
+	for _,v := range arr {
+		code :=qu.ObjToString(v["code"])
+		name:= qu.ObjToString(v["name"])
+		buyerclass_dict[name]  = code
+	}
+}
+func init_industryclass() {
+	//行业代码表 v_industryclass
+	arr := *MysqlTool.Find("v_industryclass",nil,"","",-1,-1)
+	pre_str :=""
+	for _,v := range arr {
+		level := qu.ObjToString(v["level"])
+		name := qu.ObjToString(v["name"])
+		code := qu.ObjToString(v["code"])
+		if level=="1" {
+			pre_str = name
+		}else {
+			key := pre_str+"_"+name
+			industryclass_dict[key] = code
+		}
+	}
+}
+func init_area() {
+
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C("zktest_mysql_area").Find(&q).Iter()
+	dict := map[string]string{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		name := qu.ObjToString(tmp["name"])
+		code := qu.ObjToString(tmp["code"])
+		dict[code] = name
+		tmp = make(map[string]interface{})
+	}
+
+	//最多六位
+	for k,v := range dict {
+		key := ""
+		if utf8.RuneCountInString(k)==2 {
+			key = v+"_"+"_"
+		} else if utf8.RuneCountInString(k)==4 {
+			pro_code := k[:2]
+			pro_name := dict[pro_code]
+			key = pro_name+"_"+v+"_"
+		}else if utf8.RuneCountInString(k)==6 {
+			pro_code := k[:2]
+			pro_name := dict[pro_code]
+			city_code := k[:4]
+			city_name := dict[city_code]
+			key = pro_name+"_"+city_name+"_"+v
+		}else {
+
+		}
+		if key!="" {
+			area_dict[key] = k
+		}
+	}
+
+
+
+}
+func init_proclass() {
+
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C("zktest_mysql_ylqxclass").Find(&q).Iter()
+	dict := map[string]string{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		name := qu.ObjToString(tmp["name"])
+		code := qu.ObjToString(tmp["code"])
+		dict[code] = name
+		tmp = make(map[string]interface{})
+	}
+	//最多六位
+	for k,v := range dict {
+		key := ""
+		if utf8.RuneCountInString(k)==2 {
+			key = v+"_"+"_"
+		} else if utf8.RuneCountInString(k)==4 {
+			code_1 := k[:2]
+			name_1 := dict[code_1]
+			key = name_1+"_"+v+"_"
+		}else if utf8.RuneCountInString(k)==6 {
+			code_1 := k[:2]
+			name_1 := dict[code_1]
+			code_2 := k[:4]
+			name_2 := dict[code_2]
+			key = name_1+"_"+name_2+"_"+v
+		}else if utf8.RuneCountInString(k)==8 {
+			key = "四级"+"_"+v
+		}
+		if key!="" {
+			proclass_dict[key] = k
+		}
+	}
+}

process_ medical/src/medical_product.go

@@ -4,6 +4,7 @@ import (
 	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
 	"strings"
+	"unicode/utf8"
 )
 var class_all_data = map[string]*ClassAll{}
 var class_data = map[string]string{} //记录四级分类
@@ -47,13 +48,31 @@ func productSaveMysql()  {
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
 	q,total:=map[string]interface{}{},0
-	it := sess.DB(save_mgo.DbName).C("zktesy_mysql_product_info").Find(&q).Iter()
+	it := sess.DB(save_mgo.DbName).C("zktest_mysql_product_info").Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%10000==0 {
 			log.Debug("curent index ",total)
 		}
-		delete(tmp,"_id")
-		MysqlTool.Insert("f_product",tmp)
+		//main_part
+		//changes
+
+		company_id := qu.ObjToString(tmp["company_id"])
+		if company_id!="" {
+			delete(tmp,"_id")
+			main_part := qu.ObjToString(tmp["main_part"])
+			changes := qu.ObjToString(tmp["changes"])
+
+			if utf8.RuneCountInString(main_part)>2000 {
+				tmp["main_part"] = ""
+				log.Debug("main_part~",company_id)
+			}
+			if utf8.RuneCountInString(changes)>2000{
+				tmp["changes"] = ""
+				log.Debug("changes~",company_id)
+			}
+
+			MysqlTool.Insert("f_product",tmp)
+		}
 		tmp = make(map[string]interface{})
 	}
 

process_ medical/src/medical_purchasing.go

@@ -0,0 +1,5 @@
+package main
+
+
+
+