备份-正式-V表

process_ medical/src/config.json

@@ -0,0 +1,36 @@
+{
+  "save_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5
+  },
+  "spi_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5
+  },
+  "qy_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5,
+    "username":"",
+    "password":""
+  },
+  "mysql": {
+    "addr": "192.168.3.109:4000",
+    "db": "jy_lycp",
+    "username": "jianyu",
+    "password": "top@123"
+  },
+  "data_source": {
+    "s_xlsx_name": "table.xlsx",
+    "s_area_coll": "address_jy_2021",
+    "s_product_coll": "nmpa_product",
+    "s_company_coll": "nmpa_company",
+    "s_qybase_coll": "company_base"
+  },
+  "data_output": {
+    "o_area_coll":"zktest_mysql_area",
+    "o_class_coll":"zktest_mysql_ylqxclass"
+  }
+}

process_ medical/src/init_config.go

@@ -0,0 +1,64 @@
+package main
+
+import qu "qfw/util"
+
+func initConfig()  {
+	initMgo()
+
+	souceconf:= sysconfig["data_source"].(map[string]interface{})
+	s_xlsx_name = qu.ObjToString(souceconf["s_xlsx_name"])
+	s_area_coll = qu.ObjToString(souceconf["s_area_coll"])
+	s_product_coll = qu.ObjToString(souceconf["s_product_coll"])
+	s_company_coll = qu.ObjToString(souceconf["s_company_coll"])
+	s_qybase_coll = qu.ObjToString(souceconf["s_qybase_coll"])
+
+	outputconf:= sysconfig["data_output"].(map[string]interface{})
+	o_area_coll = qu.ObjToString(outputconf["o_area_coll"])
+	o_class_coll = qu.ObjToString(outputconf["o_class_coll"])
+
+
+
+
+
+}
+
+
+
+//初始化Mgo
+func initMgo() {
+	spiconf := sysconfig["spi_mgodb"].(map[string]interface{})
+	spi_mgo = &MongodbSim{
+		MongodbAddr: spiconf["addr"].(string),
+		DbName:      spiconf["db"].(string),
+		Size:        qu.IntAllDef(spiconf["pool"], 5),
+	}
+	spi_mgo.InitPool()
+
+	qyconf := sysconfig["qy_mgodb"].(map[string]interface{})
+	qy_mgo = &MongodbSim{
+		MongodbAddr: qyconf["addr"].(string),
+		DbName:      qyconf["db"].(string),
+		Size:        qu.IntAllDef(qyconf["pool"], 5),
+		UserName: 	qyconf["username"].(string),
+		Password: 	qyconf["password"].(string),
+	}
+	qy_mgo.InitPool()
+
+	saveconf := sysconfig["save_mgodb"].(map[string]interface{})
+	save_mgo = &MongodbSim{
+		MongodbAddr: saveconf["addr"].(string),
+		DbName:      saveconf["db"].(string),
+		Size:        qu.IntAllDef(saveconf["pool"], 5),
+	}
+	save_mgo.InitPool()
+
+
+	sqlconf := sysconfig["mysql"].(map[string]interface{})
+	MysqlTool = &Mysql{
+		Address: sqlconf["addr"].(string),
+		UserName: sqlconf["username"].(string),
+		PassWord: sqlconf["password"].(string),
+		DBName: sqlconf["db"].(string),
+	}
+	//MysqlTool.Init()
+}

process_ medical/src/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	qu "qfw/util"
 	"regexp"
 )
 
@@ -8,6 +9,8 @@ var (
 	sysconfig    		map[string]interface{} //配置文件
 	save_mgo,qy_mgo,spi_mgo     *MongodbSim            //mongodb操作对象
     MysqlTool			*Mysql
+	s_xlsx_name,s_area_coll,s_product_coll,s_company_coll,s_qybase_coll			string
+	o_area_coll,o_class_coll	string
 )
 
 var timeReg = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})$`)
@@ -21,141 +24,25 @@ var proclass_dict = map[string]string{}
 var timeLayout = "2006-01-02 15:04:05"
 
 func init()  {
-	//qy_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
-	//	DbName:      "mixdata",
-	//	Size:        10,
-	//	UserName: "zhengkun",
-	//	Password: "zk@123123",
-	//}
-	//qy_mgo.InitPool()
-	//
-	//spi_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.181:27001",
-	//	DbName:      "mixdata",
-	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
-	//}
-	//spi_mgo.InitPool()
-	//
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "172.17.4.87:27080",
-	//	DbName:      "py_spider",
-	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
-	//}
-	//save_mgo.InitPool()
-
-
-
-
-
-
-
-	//save_mgo = &MongodbSim{
-	//	MongodbAddr: "192.168.3.207:27092",
-	//	DbName:      "zhengkun",
-	//	Size:        10,
-	//	UserName: "",
-	//	Password: "",
-	//}
-	//save_mgo.InitPool()
-
-	MysqlTool = &Mysql{
-		Address: "192.168.3.109:4000",
-		UserName: "jianyu",
-		PassWord: "top@123",
-		DBName: "jy_lycp",
-	}
-	MysqlTool.Init()
-
-	//MysqlTool = &Mysql{
-	//	Address: "127.0.0.1:3306",
-	//	UserName: "root",
-	//	PassWord: "123456",
-	//	DBName: "zhengkun",
-	//}
-	//MysqlTool.Init()
-	//
-
-
-	save_mgo = &MongodbSim{
-		MongodbAddr: "127.0.0.1:27017",
-		DbName:      "mixdata",
-		Size:        10,
-		UserName: "",
-		Password: "",
-	}
-	save_mgo.InitPool()
-
-
-
-
-
-
+	qu.ReadConfig(&sysconfig)
+	initConfig()
 }
 
 
-
-
 func main()  {
-	//exportSource() //导出源数据
-	//dealWithSource()  //处理源数据-bidding
-
 
-	hospitalSaveMysql() //处理医院数据
 
 
+	//v-表
+	//exportAreaCitysMgo()
+	//exportMedicalClassMgo()
 }
 
 
-//处理源数据
 func dealWithSource() {
-	MysqlTool = &Mysql{
-		Address: "192.168.3.109:4000",
-		UserName: "jianyu",
-		PassWord: "top@123",
-		DBName: "jy_lycp",
-	}
-	MysqlTool.Init()
-
-	save_mgo = &MongodbSim{
-		MongodbAddr: "192.168.3.207:27092",
-		DbName:      "zhengkun",
-		Size:        10,
-		UserName: "",
-		Password: "",
-	}
-	save_mgo.InitPool()
-
-	initMySqlMgoData() //加载数据-本地
-
 	dealWithBiddingSource()
 }
-
-//导出源数据-程序
 func exportSource() {
-	save_mgo = &MongodbSim{
-		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
-		DbName:      "qfw",
-		Size:        10,
-		UserName: "zhengkun",
-		Password: "zk@123123",
-	}
-	save_mgo.InitPool()
-
-	spi_mgo = &MongodbSim{
-		MongodbAddr: "172.17.4.87:27080",
-		DbName:      "py_spider",
-		Size:        10,
-		UserName: "",
-		Password: "",
-	}
-	spi_mgo.InitPool()
-
-
 	exportUsefulPurchasing()
 }
 

process_ medical/src/mark

@@ -0,0 +1,80 @@
+{
+  "save_mgodb": {
+    "addr": "172.17.4.87:27080",
+    "db": "py_spider",
+    "pool": 5
+  },
+  "spi_mgodb": {
+    "addr": "172.17.4.181:27001",
+    "db": "mixdata",
+    "pool": 5
+  },
+  "qy_mgodb": {
+    "addr": "172.17.145.163:27083,172.17.4.187:27082",
+    "db": "mixdata",
+    "pool": 5,
+    "username":"zhengkun",
+    "password":"zk@123123"
+  },
+  "mysql": {
+    "addr": "192.168.3.109:4000",
+    "db": "jy_lycp",
+    "username": "jianyu",
+    "password": "top@123"
+  },
+  "data_source": {
+    "s_xlsx_name": "table.xlsx",
+    "s_area_coll": "address_jy_2021",
+    "s_product_coll": "nmpa_product",
+    "s_company_coll": "nmpa_company",
+    "s_qybase_coll": "company_base"
+  },
+  "data_output": {
+    "o_area_coll":"zktest_mysql_area",
+    "o_class_coll":"zktest_mysql_ylqxclass"
+  }
+}
+
+
+
+
+
+
+
+//测试
+{
+  "save_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5
+  },
+  "spi_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5
+  },
+  "qy_mgodb": {
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
+    "pool": 5,
+    "username":"",
+    "password":""
+  },
+  "mysql": {
+    "addr": "192.168.3.109:4000",
+    "db": "jy_lycp",
+    "username": "jianyu",
+    "password": "top@123"
+  },
+  "data_source": {
+    "s_xlsx_name": "table.xlsx",
+    "s_area_coll": "address_jy_2021",
+    "s_product_coll": "nmpa_product",
+    "s_company_coll": "nmpa_company",
+    "s_qybase_coll": "company_base"
+  },
+  "data_output": {
+    "o_area_coll":"zktest_mysql_area",
+    "o_class_coll":"zktest_mysql_ylqxclass"
+  }
+}

process_ medical/src/medical_vdata.go

@@ -10,16 +10,17 @@ import (
 	qu "qfw/util"
 	"regexp"
 	"strings"
+	"time"
 )
 
 
 
-
-func v_ylqxclass() {
+//保存mysql
+func saveYlqxclassMysql() {
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
 	q,total:=map[string]interface{}{},0
-	it := sess.DB(save_mgo.DbName).C("zktest_mysql_ylqxclass").Sort("_id").Find(&q).Iter()
+	it := sess.DB(save_mgo.DbName).C(o_class_coll).Sort("_id").Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%1000==0 {
 			log.Debug("curent index ",total)
@@ -30,11 +31,11 @@ func v_ylqxclass() {
 	}
 	log.Debug("is over ",total)
 }
-func v_area() {
+func saveAreaMysql() {
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
 	q,total:=map[string]interface{}{},0
-	it := sess.DB(save_mgo.DbName).C("mysql_area").Sort("_id").Find(&q).Iter()
+	it := sess.DB(save_mgo.DbName).C(o_area_coll).Sort("_id").Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
 		if total%1000==0 {
 			log.Debug("curent index ",total)
@@ -47,38 +48,38 @@ func v_area() {
 }
 
 
+
+
+
+
 //城市划分-mgo
-func citysClassification() {
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
+func exportAreaCitysMgo() {
+	log.Debug("开始处理-区域划分")
+	sess := qy_mgo.GetMgoConn()
+	defer qy_mgo.DestoryMongoConn(sess)
 	q,total:=map[string]interface{}{
 		"town":map[string]interface{}{
 			"$exists":0,
 		},
 	},0
-	//tagDict := map[string]map[string]interface{}{}
-	it := sess.DB(save_mgo.DbName).C("address_jy_2021").Find(&q).Iter()
+	it := sess.DB(qy_mgo.DbName).C(s_area_coll).Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		if total%1000==0 {
-			log.Debug("curent index ",total)
-		}
 		province:=qu.ObjToString(tmp["province"])
 		city:=qu.ObjToString(tmp["city"])
 		district:=qu.ObjToString(tmp["district"])
 		province_code := fmt.Sprintf("%d",tmp["province_code"])
 		city_code:= fmt.Sprintf("%d",tmp["city_code"])
 		district_code:=fmt.Sprintf("%d",tmp["district_code"])
-		//code pcode level name
 		data := map[string]interface{}{}
 		if district =="" {
-			if city=="" { //一级
+			if city=="" {
 				data = map[string]interface{}{
 					"name":province,
 					"code":province_code,
 					"pcode":"",
 					"level":"1",
 				}
-			}else { //二级
+			}else {
 				data = map[string]interface{}{
 					"name":city,
 					"code":city_code,
@@ -87,7 +88,6 @@ func citysClassification() {
 				}
 			}
 		}else {
-			//三级
 			data = map[string]interface{}{
 				"name":district,
 				"code":district_code,
@@ -95,16 +95,16 @@ func citysClassification() {
 				"level":"3",
 			}
 		}
-		save_mgo.Save("mysql_area",data)
+		save_mgo.Save(o_area_coll,data)
 		tmp = make(map[string]interface{})
 	}
-	log.Debug("is over ",total)
+	log.Debug("is area over ",total)
 }
 
-
 //医疗分类-mgo
-func medicalClassification() {
-	ff,_ := xlsx.OpenFile("table.xlsx")
+func exportMedicalClassMgo() {
+	log.Debug("开始处理-医疗分类")
+	ff,_ := xlsx.OpenFile(s_xlsx_name)
 	spaceReg := regexp.MustCompile(`([ \n]+)`)
 	proReg := regexp.MustCompile(`(^[0-9][0-9])`)
 	for index, sheet := range ff.Sheets {
@@ -120,7 +120,7 @@ func medicalClassification() {
 			"level":"1",
 			"name":class,
 		}
-		save_mgo.Save("mysql_ylqxclass",dict_1)
+		save_mgo.Save(o_class_coll,dict_1)
 
 		for k, row := range sheet.Rows {
 			if k==0 {
@@ -145,7 +145,7 @@ func medicalClassification() {
 					"level":"2",
 					"name":new_class_2,
 				}
-				save_mgo.Save("mysql_ylqxclass",dict_2)
+				save_mgo.Save(o_class_coll,dict_2)
 				code_3 := code_2+proReg.FindString(class_3)
 				dict_3 := map[string]interface{}{
 					"code":code_3,
@@ -153,7 +153,7 @@ func medicalClassification() {
 					"level":"3",
 					"name":new_class_3,
 				}
-				save_mgo.Save("mysql_ylqxclass",dict_3)
+				save_mgo.Save(o_class_coll,dict_3)
 				e_arr := strings.Split(class_example,"、")
 				pre_str = proReg.FindString(class_3)//重置一下
 				num_sort = 0 //重置一下
@@ -166,7 +166,7 @@ func medicalClassification() {
 						"level":"4",
 						"name":v,
 					}
-					save_mgo.Save("mysql_ylqxclass",dict_4)
+					save_mgo.Save(o_class_coll,dict_4)
 				}
 
 			}else { //3-4
@@ -182,7 +182,7 @@ func medicalClassification() {
 						"level":"3",
 						"name":new_class_3,
 					}
-					save_mgo.Save("mysql_ylqxclass",dict_3)
+					save_mgo.Save(o_class_coll,dict_3)
 				}
 				e_arr := strings.Split(class_example,"、")
 				for _,v := range e_arr {
@@ -194,55 +194,114 @@ func medicalClassification() {
 						"level":"4",
 						"name":v,
 					}
-					save_mgo.Save("mysql_ylqxclass",dict_4)
+					save_mgo.Save(o_class_coll,dict_4)
 				}
-
 			}
 		}
 	}
-}
-func dealSortStr(index int) string {
-	code :=""
-	if index<10{
-		code = fmt.Sprintf("0%d",index)
-	}else {
-		code = fmt.Sprintf("%d",index)
-	}
-	return code
-}
-
-
-
 
 
+	log.Debug("is table class over")
+	time.Sleep(2*time.Second)
+	log.Debug("修复-分类数据")
+	repairYlqxClass1()
+	repairYlqxClass2()
+}
 
-
-
-
-func testYLQXClassRepair() {
-
+//删除重复冗余选项
+func repairYlqxClass1() {
 	data:= map[string]string{}
-
 	sess := save_mgo.GetMgoConn()
 	defer save_mgo.DestoryMongoConn(sess)
 	q,total:=map[string]interface{}{},0
-	ok1,ok2:=0,0
-	it := sess.DB(save_mgo.DbName).C("zktest_mysql_ylqxclass").Find(&q).Iter()
+	it := sess.DB(save_mgo.DbName).C(o_class_coll).Find(&q).Iter()
 	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		if total%10000==0 {
-			log.Debug("curent index ",total,ok1,ok2)
-		}
 		tmpid:=BsonTOStringId(tmp["_id"])
 		key:= qu.ObjToString(tmp["name"])+"~"+qu.ObjToString(tmp["level"])+"~"+qu.ObjToString(tmp["pcode"])
-
 		if data[key]=="" {
-			ok1++
 			data[key] = tmpid
 		}else {
-			ok2++
-			save_mgo.DeleteById("zktest_mysql_ylqxclass",tmpid)
+			save_mgo.DeleteById(o_class_coll,tmpid)
 		}
 		tmp = make(map[string]interface{})
 	}
-	log.Debug("is over ",total,ok1,ok2)
+	log.Debug("is repair1 over ")
+}
+
+//补充新的医疗器械
+func repairYlqxClass2() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{
+		"product_class":"体外诊断试剂",
+	},0
+	it := sess.DB(save_mgo.DbName).C(s_product_coll).Find(&q).Select(map[string]interface{}{
+		"product_class_1":1,
+		"product_class_2":1,
+	}).Iter()
+	data := map[string]map[string]interface{}{}
+	rankArr := []string{}
+	infoData := map[string][]string{}
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("curent index ",total)
+		}
+		product_class_1 :=qu.ObjToString(tmp["product_class_1"])
+		product_class_2 :=qu.ObjToString(tmp["product_class_2"])
+		if product_class_1!="" {
+			if data[product_class_1]==nil {
+				if product_class_2!=""{
+					data[product_class_1] = map[string]interface{}{
+						product_class_2:1,
+					}
+					rankArr = append(rankArr,product_class_1)
+					infoData[product_class_1] = []string{product_class_2}
+				}else {
+					data[product_class_1] = map[string]interface{}{}
+					rankArr = append(rankArr,product_class_1)
+					infoData[product_class_1] = []string{}
+				}
+
+			}else {
+				pro := data[product_class_1]
+				infoArr := infoData[product_class_1]
+				if product_class_2!="" && pro[product_class_2]==nil {
+					pro[product_class_2] = 1
+					data[product_class_1] = pro
+					infoArr = append(infoArr,product_class_2)
+					infoData[product_class_1] = infoArr
+				}
+			}
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ",total,len(rankArr),len(data),len(infoData))
+
+	class := "体外诊断试剂"
+	code_1 := dealSortStr(23)
+	dict_1 := map[string]interface{}{"code":code_1,"pcode":"","level":"1","name":class}
+	save_mgo.Save(o_class_coll,dict_1)
+	for k,v :=range rankArr {
+		code_2 := code_1+dealSortStr(k+1)
+		dict_2 := map[string]interface{}{"code":code_2,"pcode":code_1,"level":"2","name":v}
+		save_mgo.Save(o_class_coll,dict_2)
+		infoArr := infoData[v]
+		for k1,v1 := range infoArr{
+			code_3 := code_2+dealSortStr(k1+1)
+			dict_3 := map[string]interface{}{"code":code_3,"pcode":code_2,"level":"3","name":v1}
+			save_mgo.Save(o_class_coll,dict_3)
+		}
+	}
+
+	log.Debug("is repair2 over ")
+}
+
+func dealSortStr(index int) string {
+	code :=""
+	if index<10{
+		code = fmt.Sprintf("0%d",index)
+	}else {
+		code = fmt.Sprintf("%d",index)
+	}
+	return code
 }