医疗器械-数据清洗

listen_data/src/main.go

@@ -99,15 +99,25 @@ func main()  {
 	//}
 	//save_mgo.InitPool()
 
+	//save_mgo = &MongodbSim{
+	//	MongodbAddr: "127.0.0.1:27017",
+	//	DbName:      "zhengkun",
+	//	Size:        10,
+	//	UserName: "",
+	//	Password: "",
+	//}
+	//save_mgo.InitPool()
+
 	save_mgo = &MongodbSim{
-		MongodbAddr: "127.0.0.1:27017",
-		DbName:      "zhengkun",
+		MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
+		DbName:      "qfw",
 		Size:        10,
-		UserName: "",
-		Password: "",
+		UserName: "zhengkun",
+		Password: "zk@123123",
 	}
 	save_mgo.InitPool()
 
+
 	/*
 		repairYiXieChaData() 		//医械查
 		dealWithClassData()			//AB 匹配
@@ -117,7 +127,7 @@ func main()  {
 	*/
 
 
-	dealWithBingTuanData()
+	decodeJyUrl()
 
 	return
 

listen_data/src/zkmethod.go

@@ -40,7 +40,7 @@ var MysqlTool	*Mysql
 
 //解密
 func decodeJyUrl()  {
-	test := "ABCY1wFYj1YIDY7I2d2cHU8CjACHjJjXHRzKCg7KyEgVXlzfwlUCbM%3D"
+	test := "AbLY1wEcT0FOyo4NHNxZ3IkCCQCIDFjcWhwKS8wOT0wWmpzfVJUCds%3D"
 	var Decode  = qu.CommonDecodeArticle("content", test)
 	log.Debug(Decode[0])
 	return
@@ -54,7 +54,48 @@ func encodeJyUrl()  {
 }
 
 
+func tongjiYLQX() {
+	//ylhydata
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{},0
+	ok1,ok2,ok3:=0,0,0
+	it := sess.DB("qfw").C("ylhydata").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%10000==0 {
+			log.Debug("curent index ",total,"~",ok1,ok2,ok3)
+		}
+		p_list := []map[string]interface{}{}
+		yl_list := []map[string]interface{}{}
+		if purchasinglist, ok := tmp["purchasinglist"].(primitive.A); ok {
+			p_list = qu.ObjArrToMapArr(purchasinglist)
+		}else {
+			if purchasinglist, ok := tmp["purchasinglist"].([]interface{}); ok {
+				p_list = qu.ObjArrToMapArr(purchasinglist)
+			}
+		}
+		if yl_purchasinglist, ok := tmp["yl_purchasinglist"].(primitive.A); ok {
+			yl_list = qu.ObjArrToMapArr(yl_purchasinglist)
+		}else {
+			if yl_purchasinglist, ok := tmp["yl_purchasinglist"].([]interface{}); ok {
+				yl_list = qu.ObjArrToMapArr(yl_purchasinglist)
+			}
+		}
+
+		if len(yl_list)>0 {
+			ok1++
+		}
+		ok2+=len(p_list)
+		ok3+=len(yl_list)
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is bidding over ",total,ok1,ok2,ok3)
+
+}
 
+
+//处理新疆兵团数据
 func dealWithBingTuanData() {
 	f, err := os.Open("1.txt")
 	if err != nil {
@@ -112,12 +153,6 @@ func dealWithBingTuanData() {
 	log.Debug("总计",total,"行")
 }
 
-
-
-
-
-
-
 //导出相关数据
 func exporToptypeRealData(){
 	os.Remove("类别字段统计.xlsx")

process_medical/src/config.json

@@ -1,17 +1,17 @@
 {
   "save_mgodb": {
-    "addr": "192.168.3.207:27092",
-    "db": "zhengkun",
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
     "pool": 5
   },
   "spi_mgodb": {
-    "addr": "192.168.3.207:27092",
-    "db": "zhengkun",
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
     "pool": 5
   },
   "qy_mgodb": {
-    "addr": "192.168.3.207:27092",
-    "db": "zhengkun",
+    "addr": "127.0.0.1:27017",
+    "db": "mixdata",
     "pool": 5,
     "username":"",
     "password":""

process_medical/src/main.go

@@ -42,7 +42,8 @@ func main()  {
 	//saveBiddingPurMysql()
 
 
-
+	//exportAreaMysql()
+	exportAreaMysql()
 }
 
 

process_medical/src/medical_bidding.go

@@ -222,7 +222,8 @@ func matchedManufacturer(brandName string,infoArr []map[string]interface{}) (boo
 		company_name := qu.ObjToString(info["company_name"])
 		company_id := qu.ObjToString(info["company_id"])
 		if brandName!="" { //品牌匹配
-			if strings.Contains(company_name,brandName) {
+			if strings.
+				Contains(company_name,brandName) {
 				return true,company_name,company_id
 			}
 		}

process_medical/src/medical_vdata_area.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"fmt"
+	log "github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
+)
+
+func saveAreaMysql() {
+	sess := save_mgo.GetMgoConn()
+	defer save_mgo.DestoryMongoConn(sess)
+	q,total:=map[string]interface{}{},0
+	it := sess.DB(save_mgo.DbName).C(o_area_coll).Sort("_id").Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
+		if total%1000==0 {
+			log.Debug("curent index ",total)
+		}
+		tmpid := BsonTOStringId(tmp["_id"])
+		delete(tmp,"_id")
+		code := qu.IntAll(tmp["code"])
+		tmp["code"] = code
+		insertMysqlData("v_area",tmp,tmpid)
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("is over ",total)
+}
+
+func exportAreaMysql() {
+	sess := qy_mgo.GetMgoConn()
+	defer qy_mgo.DestoryMongoConn(sess)
+	q, total := map[string]interface{}{
+		"town": map[string]interface{}{
+			"$exists": 0,
+		},
+	}, 0
+	//二级map
+	infoMap := map[string]string{}
+	codeMap := map[string]string{}
+	it := sess.DB(qy_mgo.DbName).C(s_area_coll).Find(&q).Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		province := qu.ObjToString(tmp["province"])
+		city := qu.ObjToString(tmp["city"])
+		district := qu.ObjToString(tmp["district"])
+		province_code := fmt.Sprintf("%d0000", tmp["province_code"])
+		city_code := fmt.Sprintf("%d00", tmp["city_code"])
+		district_code := fmt.Sprintf("%d", tmp["district_code"])
+		data := map[string]interface{}{}
+		if district == "" {
+			if city == "" {
+				data = map[string]interface{}{
+					"name":  province,
+					"code":  province_code,
+					"level": "1",
+					"pcode": "",
+				}
+				save_mgo.Save(o_area_coll,data)
+				if codeMap[province_code] =="" {
+					codeMap[province_code] = province_code
+				}else {
+					log.Debug(data)
+				}
+			} else {
+				if city == "北京市" {
+					city_code = "110100"
+				}
+				if city == "天津市" {
+					city_code = "120100"
+				}
+				if city == "重庆市" {
+					city_code = "500100"
+				}
+				if city == "上海市" {
+					city_code = "310100"
+				}
+				data = map[string]interface{}{
+					"name":  city,
+					"code":  city_code,
+					"level": "2",
+					"pcode": province_code,
+				}
+				infoMap[city_code] = city_code
+				save_mgo.Save(o_area_coll,data)
+
+				if codeMap[city_code] =="" {
+					codeMap[city_code] = city_code
+				}else {
+					log.Debug(data)
+				}
+
+
+
+				if city == "重庆市" {
+					city_code = "500200"
+					data = map[string]interface{}{
+						"name":  city,
+						"code":  city_code,
+						"level": "2",
+						"pcode": province_code,
+					}
+					infoMap[city_code] = city_code
+					save_mgo.Save(o_area_coll,data)
+
+					if codeMap[city_code] =="" {
+						codeMap[city_code] = city_code
+					}else {
+						log.Debug(data)
+					}
+				}
+			}
+		} else {
+
+			if infoMap[city_code] == "" {
+				data = map[string]interface{}{
+					"name":  district,
+					"code":  district_code,
+					"level": "2",
+					"pcode": province_code,
+				}
+				infoMap[district_code] = district_code
+				save_mgo.Save(o_area_coll,data)
+
+				if codeMap[district_code] =="" {
+					codeMap[district_code] = district_code
+				}else {
+					log.Debug(data)
+				}
+			} else {
+				//确保一下-是否重复三级城市
+				data = map[string]interface{}{
+					"name":  district,
+					"code":  district_code,
+					"level": "3",
+					"pcode": city_code,
+				}
+				if codeMap[district_code] =="" {
+					save_mgo.Save(o_area_coll,data)
+					codeMap[district_code] = district_code
+				}else {
+					log.Debug(data)
+				}
+
+			}
+			tmp = make(map[string]interface{})
+		}
+	}
+	log.Debug("is area over ", total)
+}

process_medical/src/medical_vdata.go → process_medical/src/medical_vdata_class.go

@@ -1,8 +1,5 @@
 package main
 
-
-//v表处理
-
 import (
 	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
@@ -14,8 +11,6 @@ import (
 	"unicode/utf8"
 )
 
-
-
 //保存mysql
 func saveYlqxclassMysql() {
 	sess := save_mgo.GetMgoConn()
@@ -33,132 +28,6 @@ func saveYlqxclassMysql() {
 	}
 	log.Debug("is over ",total)
 }
-func saveAreaMysql() {
-	sess := save_mgo.GetMgoConn()
-	defer save_mgo.DestoryMongoConn(sess)
-	q,total:=map[string]interface{}{},0
-	it := sess.DB(save_mgo.DbName).C(o_area_coll).Sort("_id").Find(&q).Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		if total%1000==0 {
-			log.Debug("curent index ",total)
-		}
-		tmpid := BsonTOStringId(tmp["_id"])
-		delete(tmp,"_id")
-		code := qu.IntAll(tmp["code"])
-		tmp["code"] = code
-		insertMysqlData("v_area_copy",tmp,tmpid)
-		tmp = make(map[string]interface{})
-	}
-	log.Debug("is over ",total)
-}
-
-
-
-
-
-
-//城市划分-mgo
-func exportAreaCitysMysql() {
-	log.Debug("开始处理-区域划分")
-
-	xzqh_data := map[string]map[string]string{}
-	//准备数据
-	ff,_ := xlsx.OpenFile(s_area_xlsx_name)
-	for _, sheet := range ff.Sheets {
-		for k, row := range sheet.Rows {
-			if k==0 {
-				continue
-			}
-			var arr []string
-			for k1, cell := range row.Cells {
-				if k1==3||k1==4 {
-					num ,_:= cell.Float()
-					arr = append(arr, fmt.Sprintf("%.6f",num))
-				}else {
-					arr = append(arr, cell.String())
-				}
-			}
-			code := arr[0]
-			xzqh_data[code] = map[string]string{
-				"region":arr[1],
-				"longitude":arr[3],
-				"latitude":arr[4],
-			}
-		}
-		log.Debug(len(xzqh_data))
-	}
-
-	sess := qy_mgo.GetMgoConn()
-	defer qy_mgo.DestoryMongoConn(sess)
-	q,total:=map[string]interface{}{
-		"town":map[string]interface{}{
-			"$exists":0,
-		},
-	},0
-	it := sess.DB(qy_mgo.DbName).C(s_area_coll).Find(&q).Iter()
-	for tmp := make(map[string]interface{}); it.Next(&tmp);total++{
-		province:=qu.ObjToString(tmp["province"])
-		city:=qu.ObjToString(tmp["city"])
-		district:=qu.ObjToString(tmp["district"])
-		province_code := fmt.Sprintf("%d0000",tmp["province_code"])
-		city_code:= fmt.Sprintf("%d00",tmp["city_code"])
-		district_code:=fmt.Sprintf("%d",tmp["district_code"])
-		info := map[string]string{}
-		data := map[string]interface{}{}
-		if district =="" {
-			if city=="" {
-				info = xzqh_data[province_code]
-				data = map[string]interface{}{
-					"name":province,
-					"code":province_code,
-					"level":"1",
-					"region":qu.ObjToString(info["region"]),
-					"longitude":qu.ObjToString(info["longitude"]),
-					"latitude":qu.ObjToString(info["latitude"]),
-				}
-			}else {
-				info = xzqh_data[city_code]
-				data = map[string]interface{}{
-					"name":city,
-					"code":city_code,
-					"level":"2",
-					"p_name":province,
-					"p_code":province_code,
-					"region":qu.ObjToString(info["region"]),
-					"longitude":qu.ObjToString(info["longitude"]),
-					"latitude":qu.ObjToString(info["latitude"]),
-					"pcode":province_code,
-				}
-			}
-		}else {
-			info = xzqh_data[district_code]
-			data = map[string]interface{}{
-				"name":district,
-				"code":district_code,
-				"level":"3",
-				"p_name":province,
-				"p_code":province_code,
-				"c_name":city,
-				"c_code":city_code,
-				"region":qu.ObjToString(info["region"]),
-				"longitude":qu.ObjToString(info["longitude"]),
-				"latitude":qu.ObjToString(info["latitude"]),
-				"pcode":city_code,
-			}
-		}
-		save_mgo.Save(o_area_coll,data)
-
-		tmp = make(map[string]interface{})
-	}
-	log.Debug("is area over ",total)
-
-
-}
-
-
-
-
-
 
 
 //医疗分类-mgo
@@ -268,7 +137,6 @@ func exportMedicalClassMysql() {
 	repairYlqxClass1()
 	repairYlqxClass2()
 }
-
 //删除重复冗余选项
 func repairYlqxClass1() {
 	data:= map[string]string{}
@@ -288,7 +156,6 @@ func repairYlqxClass1() {
 	}
 	log.Debug("is repair1 over ")
 }
-
 //补充新的医疗器械
 func repairYlqxClass2() {
 	sess := save_mgo.GetMgoConn()
@@ -356,7 +223,6 @@ func repairYlqxClass2() {
 
 	log.Debug("is repair2 over ")
 }
-
 //2246位编号
 func convertSerialNumber(index int,position int) string {
 	code :=""
@@ -372,4 +238,4 @@ func convertSerialNumber(index int,position int) string {
 		code = fmt.Sprintf("%d",index)
 	}
 	return code
-}
+}