Merge branch 'master' of http://192.168.3.207:8080/data_processing/fieldproject

README.md

@@ -23,5 +23,11 @@
 
 ### fieldproject_medical(医疗数据处理模块) 
 + data_preparation (基础信息~数据准备~)
-  + 1、企业产品  2、医院整合  3、产品整合  4、修复代码表等
+  + 1、企业产品信息  
+  + 2、医院相关信息  
+  + 3、产品相关整合  
+  + 4、代码表
 
++ data_service (服务)
+  + 1、标的物相关信息  
+  + 2、相关服务

fieldproject_medical/data_preparation/src/class/initdata.go

@@ -13,11 +13,12 @@ const (
 	V_Institution_Alias      = "institution_alias"
 	V_Institution_Departtags = "institution_departtags"
 	V_Institution_Typetags   = "institution_typetags"
-	V_Code_Level             = "code_level"
-	V_Code_Department        = "code_department"
-	V_Code_Productclass      = "code_productclass"
-	V_Code_Type              = "code_type "
-	V_Code_Sdleveltypeequip  = "code_sdleveltypeequip"
+
+	V_Code_Level            = "code_level"
+	V_Code_Department       = "code_department"
+	V_Code_Productclass     = "code_productclass"
+	V_Code_Type             = "code_type "
+	V_Code_Sdleveltypeequip = "code_sdleveltypeequip"
 
 	V_Code_Bidfield      = "code_bidfield"
 	V_Code_Bidscope      = "code_bidscope"
@@ -29,15 +30,17 @@ const (
 
 	S_Product_Coll = "zktest_mysql_product_info"
 	S_Company_Coll = "zktest_mysql_company_info"
-	S_Region_Coll  = "zktest_mysql_area_new"
 )
 
 var (
 	Save_Mgo, Spi_Mgo                 *MongodbSim
 	MysqlMedicalTool, MysqlGlobalTool *Mysql
-	Config                            map[string]interface{}
+	TypeConfig, DepartConfig          map[string]interface{}
 	//类型规则
 	Medical_RuleType = []map[string]interface{}{}
+	//科室规则
+	Medical_RuleDepart = []map[string]interface{}{}
+
 	//涉及的代码表
 	Medical_Type_Datas    = map[string]string{}
 	Medical_Level_Datas   = map[string]string{}
@@ -49,22 +52,21 @@ var (
 	AreaCode     = map[string]string{}
 	CityCode     = map[string]string{}
 	DistrictCode = map[string]string{}
-	isLocal      bool
+	IsLocal      bool
 	TimeLayout   = "2006-01-02 15:04:05"
 	TimeTmeplate = "2006-01-02"
 )
 
 func InitClass() {
-	isLocal = true //本地
 	initMgo()
-	initLocalData()
 	initMysql()
+	initLocalData() //医院相关使用
 	initVCode()
 }
 
 //初始化mgo
 func initMgo() {
-	if isLocal {
+	if IsLocal {
 		Save_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
 			DbName:      "zhengkun",
@@ -74,15 +76,6 @@ func initMgo() {
 		}
 		Save_Mgo.InitPool()
 
-		//Save_Mgo = &MongodbSim{
-		//	MongodbAddr: "192.168.3.207:27092",
-		//	DbName:      "zhengkun",
-		//	Size:        10,
-		//	UserName:    "",
-		//	Password:    "",
-		//}
-		//Save_Mgo.InitPool()
-
 		Spi_Mgo = &MongodbSim{
 			MongodbAddr: "127.0.0.1:27017",
 			DbName:      "zhengkun",
@@ -113,18 +106,25 @@ func initMgo() {
 }
 
 func initMysql() {
+	username, password := "root", "=PDT49#80Z!RVv52_z"
+	address := "192.168.3.217:4000"
+	if !IsLocal {
+		username = "zhengkun"
+		password = "Zk#20220824"
+		address = "172.17.4.242:4000"
+	}
 	MysqlMedicalTool = &Mysql{
-		Address:  "192.168.3.217:4000",
-		UserName: "root",
-		PassWord: "=PDT49#80Z!RVv52_z",
-		DBName:   "medical_fileld_data",
+		Address:  address,
+		UserName: username,
+		PassWord: password,
+		DBName:   "medical_field_data",
 	}
 	MysqlMedicalTool.Init()
 
 	MysqlGlobalTool = &Mysql{
-		Address:  "192.168.3.217:4000",
-		UserName: "root",
-		PassWord: "=PDT49#80Z!RVv52_z",
+		Address:  address,
+		UserName: username,
+		PassWord: password,
 		DBName:   "global_common_data",
 	}
 	MysqlGlobalTool.Init()
@@ -140,6 +140,7 @@ func initVCode() {
 
 func initLocalData() {
 	initRuleTypeData()
+	initRuleDepartData()
 	initYiXueData()
 }
 

fieldproject_medical/data_preparation/src/class/initlocal.go

@@ -8,16 +8,28 @@ import (
 
 //加载类型规则
 func initRuleTypeData() {
-	qu.ReadConfig("./rule_type.json", &Config)
-	if arr, ok := Config["rule"].([]interface{}); ok {
+	qu.ReadConfig("./rule_type.json", &TypeConfig)
+	if arr, ok := TypeConfig["rule"].([]interface{}); ok {
 		Medical_RuleType = qu.ObjArrToMapArr(arr)
 	}
-	if arr, ok := Config["rule"].(primitive.A); ok {
+	if arr, ok := TypeConfig["rule"].(primitive.A); ok {
 		Medical_RuleType = qu.ObjArrToMapArr(arr)
 	}
 	log.Debug("类型规则~", len(Medical_RuleType))
 }
 
+//加载科室规则
+func initRuleDepartData() {
+	qu.ReadConfig("./rule_depart.json", &DepartConfig)
+	if arr, ok := DepartConfig["rule"].([]interface{}); ok {
+		Medical_RuleDepart = qu.ObjArrToMapArr(arr)
+	}
+	if arr, ok := DepartConfig["rule"].(primitive.A); ok {
+		Medical_RuleDepart = qu.ObjArrToMapArr(arr)
+	}
+	log.Debug("科室规则~", len(Medical_RuleDepart))
+}
+
 //加载医学百科数据
 func initYiXueData() {
 	sess := Save_Mgo.GetMgoConn()

fieldproject_medical/data_preparation/src/company/company.go

@@ -14,6 +14,7 @@ var companylock sync.Mutex
 func RunCompanyProductInfo() {
 	log.Debug("处理~企业~产品信息~")
 	dealWithCompanyProductInfo()
+	time.Sleep(10 * time.Second)
 }
 
 //处理企业产品信息

fieldproject_medical/data_preparation/src/hospital/hospital.go

@@ -1,7 +1,9 @@
 package hospital
 
 import (
+	"class"
 	log "github.com/donnie4w/go-logger/logger"
+	"gopkg.in/mgo.v2"
 	"time"
 )
 
@@ -9,20 +11,34 @@ var (
 	save_hospital_coll    = "zktest_hospital_info"
 	merge_hospital_coll_1 = "zktest_hospital_info_merge"
 	merge_hospital_coll_2 = "zktest_hospital_info_merge_new"
+	merge_hospital_coll_3 = "zktest_hospital_info_merge_over"
+
 	history_hospital_coll = "f_hospital_codes"
 )
 
 func RunHospital() {
 	RunBuildHospitalInfo()
-	RunCleanHospitalInfo()
-	RunResetHospitalInfo(save_hospital_coll)
+	RunCleanHospitalNameInfo(save_hospital_coll)
+	RunResetHospitalInfo(save_hospital_coll, "name")
+	//增加索引
+	createMgoIndex(save_hospital_coll, []string{"repeat_id", "name"})
 	RunMergeHospitalInfo(save_hospital_coll, merge_hospital_coll_1)
-}
-func RunHospitalOnline() {
+	//增加索引
+	createMgoIndex(merge_hospital_coll_1, []string{"repeat_id", "name"})
 	RunRepairHospitalInfo()
-	RunResetHospitalInfo(merge_hospital_coll_1)
+	RunCleanHospitalSpecTypeInfo(merge_hospital_coll_1)
+	RunResetHospitalInfo(merge_hospital_coll_1, "name")
 	RunMergeHospitalInfo(merge_hospital_coll_1, merge_hospital_coll_2)
-	//彻底结束~是否删除~中间关联表？
+
+	//根据最终的company_id 重置判重标记
+	RunResetHospitalInfo(merge_hospital_coll_2, "company_id")
+	//增加索引
+	createMgoIndex(merge_hospital_coll_2, []string{"repeat_id", "name"})
+	RunMergeHospitalInfo(merge_hospital_coll_2, merge_hospital_coll_3)
+
+	//修改mgo~特殊情况下~数据信息~
+	RepairMgoFalseInfoData()
+
 }
 
 //开始执行医院数据
@@ -35,15 +51,20 @@ func RunBuildHospitalInfo() {
 	time.Sleep(10 * time.Second)
 }
 
-//开始执行清洗医院数据
-func RunCleanHospitalInfo() {
-	//清洗~医院信息~名称
-	cleanHospitalInfoData()
+//开始执行清洗医院名称数据
+func RunCleanHospitalNameInfo(coll_name string) {
+	cleanHospitalNameInfoData(coll_name)
+	time.Sleep(10 * time.Second)
+}
+
+//开始执行清洗医院特殊类型数据
+func RunCleanHospitalSpecTypeInfo(coll_name string) {
+	cleanHospitalSpecTypeInfoData(coll_name)
 	time.Sleep(10 * time.Second)
 }
 
-func RunResetHospitalInfo(source_coll string) {
-	resetRepeatHospital(source_coll)
+func RunResetHospitalInfo(source_coll string, key string) {
+	resetRepeatHospital(source_coll, key)
 	time.Sleep(10 * time.Second)
 }
 
@@ -59,4 +80,30 @@ func RunRepairHospitalInfo() {
 	time.Sleep(10 * time.Second)
 	compareHospital()
 	time.Sleep(10 * time.Second)
+	updateMarkIdHospital()
+	time.Sleep(10 * time.Second)
+}
+
+//创建mgo索引
+func createMgoIndex(table string, indexArr []string) {
+	addr, dbname := "127.0.0.1:27017", "zhengkun"
+	if !class.IsLocal {
+		addr = "172.17.4.87:27080"
+		dbname = "py_theme"
+	}
+	mongoDBDialInfo := &mgo.DialInfo{
+		Addrs:    []string{addr},
+		Timeout:  30 * time.Second,
+		Database: dbname,
+	}
+	session, err := mgo.DialWithInfo(mongoDBDialInfo)
+	if err != nil {
+		log.Debug("CreateSession failed:%\n", err)
+	}
+	coll := session.DB(dbname).C(table)
+	for _, v := range indexArr {
+		err = coll.EnsureIndexKey(v)
+		log.Debug("创建索引~", v, "~", table, err)
+	}
+
 }

fieldproject_medical/data_preparation/src/hospital/hospital_clean.go

@@ -17,13 +17,13 @@ var suffixReg_4 = regexp.MustCompile("[(（]原([:：])?(.*)[)）]$")
 var suffixReg_5 = regexp.MustCompile("(院|区|部)[)）]$")
 
 //整合医院数据
-func cleanHospitalInfoData() {
-	log.Debug("清洗医院信息~~开始~~")
+func cleanHospitalSpecTypeInfoData(coll_name string) {
+	log.Debug("清洗医院特殊信息~~开始~~")
 	data_hospitals = map[string]string{}
 	sess := class.Save_Mgo.GetMgoConn()
 	defer class.Save_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
-	it := sess.DB(class.Save_Mgo.DbName).C(save_hospital_coll).Find(&q).Sort("_id").Iter()
+	it := sess.DB(class.Save_Mgo.DbName).C(coll_name).Find(&q).Sort("_id").Iter()
 	total := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%5000 == 0 {
@@ -32,24 +32,53 @@ func cleanHospitalInfoData() {
 		//更新表
 		update := map[string]interface{}{}
 		tmpid := class.BsonTOStringId(tmp["_id"])
-		//清洗当前别名
-		cleanHospitalAlias(tmp, &update)
-		//清洗医院名称~涉及别名
-		cleanHospitalName(tmp, &update)
 		//清洗等级,性质
 		cleanHospitalLevelBuss(tmp, &update)
 		//清洗类型
 		cleanHospitalType(tmp, &update)
 
 		if len(update) > 0 && tmpid != "" {
-			class.Save_Mgo.UpdateById(save_hospital_coll, tmpid, map[string]interface{}{
+			class.Save_Mgo.UpdateById(coll_name, tmpid, map[string]interface{}{
+				"$set": update,
+			})
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("清洗医院特殊信息~~over~~", total)
+
+}
+
+//整合医院数据
+func cleanHospitalNameInfoData(coll_name string) {
+	log.Debug("清洗医院名称信息~~开始~~")
+	data_hospitals = map[string]string{}
+	sess := class.Save_Mgo.GetMgoConn()
+	defer class.Save_Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(class.Save_Mgo.DbName).C(coll_name).Find(&q).Sort("_id").Iter()
+	total := 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%5000 == 0 {
+			log.Debug("cur index ", total)
+		}
+		//更新表
+		update := map[string]interface{}{}
+		tmpid := class.BsonTOStringId(tmp["_id"])
+		//清洗当前别名
+		cleanHospitalAlias(tmp, &update)
+		//清洗医院名称~涉及别名
+		cleanHospitalName(tmp, &update)
+
+		if len(update) > 0 && tmpid != "" {
+			class.Save_Mgo.UpdateById(coll_name, tmpid, map[string]interface{}{
 				"$set": update,
 			})
 		}
 		tmp = make(map[string]interface{})
 	}
 
-	log.Debug("清洗医院信息~~over~~", total)
+	log.Debug("清洗医院名称信息~~over~~", total)
 
 }
 

fieldproject_medical/data_preparation/src/hospital/hospital_extra.go

@@ -20,13 +20,13 @@ func compareHospital() {
 	it := sess.DB(class.Save_Mgo.DbName).C(merge_hospital_coll_1).Find(&q).Sort("_id").Iter()
 	total, isok := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
+		if total%5000 == 0 {
 			log.Debug("cur index ", total, isok)
 		}
 		name := qu.ObjToString(tmp["name"])
 		alias := qu.ObjToString(tmp["alias"])
 		if name != "" {
-			dataArr, _ := class.Spi_Mgo.Find(history_hospital_coll, map[string]interface{}{
+			dataArr, _ := class.Save_Mgo.Find(history_hospital_coll, map[string]interface{}{
 				"search_name": name,
 			}, nil, map[string]interface{}{
 				"hospital_name": 1,

fieldproject_medical/data_preparation/src/hospital/hospital_merge.go

@@ -27,7 +27,7 @@ func mergeRepeatHospital(source_coll string, output_coll string) {
 	it := sess.DB(class.Save_Mgo.DbName).C(source_coll).Find(&q).Sort("_id").Iter()
 	total, isok := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
+		if total%5000 == 0 {
 			log.Debug("cur index ", total, "~", isok)
 		}
 		dataArr, _ := class.Save_Mgo.Find(source_coll, map[string]interface{}{

fieldproject_medical/data_preparation/src/hospital/hospital_pingan.go

@@ -0,0 +1,83 @@
+package hospital
+
+import (
+	"class"
+	log "github.com/donnie4w/go-logger/logger"
+	qu "qfw/util"
+	"time"
+)
+
+var province_map = map[string]string{
+	"BJ": "北京", "TJ": "天津", "SH": "上海", "CQ": "重庆", "HB": "河北", "SX": "山西", "NMG": "内蒙古", "LN": "辽宁", "JL": "吉林",
+	"HLJ": "黑龙江", "JS": "江苏", "ZJ": "浙江", "AH": "安徽", "FJ": "福建", "JX": "江西", "SD": "山东", "HEN": "河南", "HUB": "湖北",
+	"HUN": "湖南", "GD": "广东", "GX": "广西", "HAIN": "海南", "SC": "四川", "GZ": "贵州", "YN": "云南", "XZ": "西藏", "SAX": "陕西",
+	"GS": "甘肃", "QH": "青海", "NX": "宁夏", "XJ": "新疆",
+}
+
+//补充凭安数据~ special_enterprise_use
+func SupplementPingAnMedicalData() {
+	log.Debug("开始~凭安补充~数据~~~")
+	sess := class.Save_Mgo.GetMgoConn()
+	defer class.Save_Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(class.Save_Mgo.DbName).C("special_enterprise_use").Find(&q).Sort("_id").Iter()
+	total, isok := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%1000 == 0 {
+			log.Debug("cur index ", total, isok)
+		}
+		company_id := qu.ObjToString(tmp["company_id"])
+		if company_id != "" {
+			save_data := map[string]interface{}{
+				"company_name": tmp["company_name"],
+				"company_id":   tmp["company_id"],
+			}
+			data := class.MysqlMedicalTool.FindOne(class.V_Institution_Baseinfo, map[string]interface{}{
+				"company_id": company_id,
+			}, "company_id", "")
+			if data == nil { //可以补充
+				isok++
+				//构建数据信息
+				dealWithPingAnInfo(tmp)
+				save_data["repeat"] = 0
+			} else {
+				save_data["repeat"] = 1
+			}
+			class.Save_Mgo.Save("zktest_hospital_supplement_data", save_data)
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("凭安补充医疗机构~~over~~ ", total, isok)
+}
+
+func dealWithPingAnInfo(tmp map[string]interface{}) {
+	data_info := map[string]interface{}{}
+	hospital_name := qu.ObjToString(tmp["company_name"])
+	data_info["company_id"] = qu.ObjToString(tmp["company_id"])
+	data_info["mi_name"] = hospital_name
+	data_info["address"] = qu.ObjToString(tmp["company_address"])
+	data_info["introduce"] = qu.ObjToString(tmp["business_scope"])
+
+	type_1, type_2 := confrimFromFieldHospitalRule(hospital_name, hospital_name, false)
+	new_type := type_1 + "~" + type_2
+	new_level, new_type, new_bustype, new_sdequipment := codificationOfInfo("其它", new_type, "公立")
+	data_info["level_code"] = new_level
+	data_info["mi_type_code"] = new_type
+	data_info["business_type"] = new_bustype
+	data_info["sdequipment_code"] = new_sdequipment
+
+	province_short := qu.ObjToString(tmp["province_short"])
+	area := province_map[province_short]
+	//地域代码
+	area_code := regionallyCode(area, "", "")
+	data_info["area_code"] = area_code
+
+	data_info["sourceweb"] = "凭安补充"
+	data_info["pcompany_id"] = ""
+	data_info["mark_id"] = 4
+	data_info["comeintime"] = time.Unix(time.Now().Unix(), 0).Format(class.TimeLayout)
+	data_info["updatetime"] = time.Unix(time.Now().Unix(), 0).Format(class.TimeLayout)
+	//插入sql
+	class.InsertMedicalMysqlData(class.V_Institution_Baseinfo, data_info, qu.ObjToString(tmp["company_id"]))
+
+}

fieldproject_medical/data_preparation/src/hospital/hospital_repair.go

@@ -3,7 +3,10 @@ package hospital
 import (
 	"class"
 	log "github.com/donnie4w/go-logger/logger"
+	"github.com/uuid"
 	qu "qfw/util"
+	"regexp"
+	"strings"
 	"sync"
 )
 
@@ -18,7 +21,7 @@ func repairHospital() {
 	wg := &sync.WaitGroup{}
 	total := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%1000 == 0 {
+		if total%5000 == 0 {
 			log.Debug("cur index ", total)
 		}
 		pool <- true
@@ -63,3 +66,68 @@ func repairHospital() {
 	log.Debug("补充信息信息~~over~~ ", total)
 
 }
+
+//补充自生id未校验数据
+func updateMarkIdHospital() {
+	log.Debug("开始更新机构~自生id~~ ")
+	sess := class.Save_Mgo.GetMgoConn()
+	defer class.Save_Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	it := sess.DB(class.Save_Mgo.DbName).C(merge_hospital_coll_1).Find(&q).Sort("_id").Iter()
+	total, isok := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total, isok)
+		}
+		tmpid := class.BsonTOStringId(tmp["_id"])
+		company_id := qu.ObjToString(tmp["company_id"])
+		mark_id := qu.IntAll(tmp["mark_id"])
+		if mark_id == 0 && company_id == "" {
+			isok++
+			c_id := uuid.New().String()
+			c_id = strings.ReplaceAll(c_id, "-", "")
+			class.Save_Mgo.UpdateById(merge_hospital_coll_1, tmpid, map[string]interface{}{
+				"$set": map[string]interface{}{
+					"company_id": c_id,
+					"mark_id":    3,
+				},
+			})
+		}
+		tmp = make(map[string]interface{})
+	}
+	log.Debug("更新机构~自生id~~over~~ ", total, isok)
+}
+
+//根据最后的规则~修复特殊~数据
+func RepairMgoFalseInfoData() {
+	log.Debug("最后执行信息的~特殊清洗~~~")
+	repairFalseReg := regexp.MustCompile("[()（）]")
+	sess := class.Save_Mgo.GetMgoConn()
+	defer class.Save_Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"type": "综合医院~",
+	}
+	it := sess.DB(class.Save_Mgo.DbName).C(merge_hospital_coll_3).Find(&q).Sort("_id").Select(map[string]interface{}{
+		"name": 1,
+	}).Iter()
+	total, isok := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%5000 == 0 {
+			log.Debug("cur index ", total, "~", isok)
+		}
+		tmpid := class.BsonTOStringId(tmp["_id"])
+		name := qu.ObjToString(tmp["name"])
+		if strings.Contains(name, "卫生院") &&
+			!repairFalseReg.MatchString(name) {
+			isok++
+			class.Save_Mgo.UpdateById(merge_hospital_coll_3, tmpid, map[string]interface{}{
+				"$set": map[string]interface{}{
+					"type": "乡镇卫生院~",
+				},
+			})
+		}
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is spec repair over ", total, "~", isok)
+}

fieldproject_medical/data_preparation/src/hospital/hospital_reset.go

@@ -7,25 +7,29 @@ import (
 )
 
 //重置重复标记~
-func resetRepeatHospital(coll_name string) {
+func resetRepeatHospital(coll_name string, repeat_name string) {
 	log.Debug("开始重置~~重复标记~~")
 	data_hospitals = map[string]string{}
 	sess := class.Save_Mgo.GetMgoConn()
 	defer class.Save_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
 	it := sess.DB(class.Save_Mgo.DbName).C(coll_name).Find(&q).Sort("_id").Select(map[string]interface{}{
-		"name": 1,
+		"name":       1,
+		"company_id": 1,
 	}).Iter()
 	total, isok := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%5000 == 0 {
 			log.Debug("cur index ", total, "~", isok)
 		}
-		name := qu.ObjToString(tmp["name"])
+		key := qu.ObjToString(tmp[repeat_name])
 		tmpid := class.BsonTOStringId(tmp["_id"])
+		if key == "" {
+			log.Debug("异常~", tmpid, "~", key)
+		}
 		update := map[string]interface{}{}
-		if data_hospitals[name] == "" {
-			data_hospitals[name] = tmpid
+		if data_hospitals[key] == "" {
+			data_hospitals[key] = tmpid
 			update = map[string]interface{}{
 				"repeat":    0,
 				"repeat_id": "",
@@ -34,9 +38,10 @@ func resetRepeatHospital(coll_name string) {
 			isok++
 			update = map[string]interface{}{
 				"repeat":    1,
-				"repeat_id": data_hospitals[name],
+				"repeat_id": data_hospitals[key],
 			}
 		}
+
 		class.Save_Mgo.UpdateById(coll_name, tmpid, map[string]interface{}{
 			"$set": update,
 		})

fieldproject_medical/data_preparation/src/hospital/hospital_sql.go

@@ -18,19 +18,14 @@ func ExportHospitalInfoToMysql() {
 	sess := class.Save_Mgo.GetMgoConn()
 	defer class.Save_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
-	it := sess.DB(class.Save_Mgo.DbName).C(merge_hospital_coll_2).Find(&q).Sort("_id").Iter()
-	pool := make(chan bool, 1)
+	it := sess.DB(class.Save_Mgo.DbName).C(merge_hospital_coll_3).Find(&q).Sort("_id").Iter()
+	pool := make(chan bool, 5)
 	wg := &sync.WaitGroup{}
-	total, isok := 0, 0
+	total := 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%1000 == 0 {
-			log.Debug("cur index ", total, "~", isok)
+			log.Debug("cur index ", total, "~")
 		}
-		if qu.IntAll(tmp["mark_id"]) == 0 {
-			tmp = make(map[string]interface{})
-			continue
-		}
-		isok++
 		pool <- true
 		wg.Add(1)
 		go func(tmp map[string]interface{}) {
@@ -50,7 +45,7 @@ func ExportHospitalInfoToMysql() {
 	}
 	wg.Wait()
 
-	log.Debug("is cur over ", total, "~", isok)
+	log.Debug("is cur over ", total)
 
 }
 
@@ -149,7 +144,6 @@ func createDepartsInfo(tmp map[string]interface{}, level string) {
 		}
 		data := map[string]interface{}{}
 		data["company_id"] = qu.ObjToString(tmp["company_id"])
-
 		data["departname_code"] = depart_code
 		data["departname_class1"] = qu.ObjToString(v["departclass1"])
 		data["departname_class2"] = qu.ObjToString(v["departclass2"])

fieldproject_medical/data_preparation/src/hospital/hospital_ways.go

@@ -17,6 +17,7 @@ var med_lev_Reg1 = regexp.MustCompile("^([一二三])([甲乙丙特])$")
 var med_lev_Reg2 = regexp.MustCompile("^([一二三])(级)(医院)?$")
 var dataLock sync.Mutex
 var addressReg = regexp.MustCompile("(.*市).*")
+var departSpecReg1 = regexp.MustCompile("(小儿眼科与斜弱视学科|眼表疾病学科|眼科|眼眶及眼肿瘤学科|青光眼与视神经疾病学科|视光学科|白内障与晶状体疾病学科|玻璃体视网膜疾病学科|角膜科)")
 
 //查询企业id
 func inquirBaseInfoid(name string) string {
@@ -38,7 +39,7 @@ func inquirBaseInfoid(name string) string {
 
 //补充~地域区划
 func supplementRegionally(area *string, city *string, district *string, name string, address string) {
-	return
+
 	//查询其他表~
 	if *area == "" || *area == "全国" {
 		*area = "全国"
@@ -69,6 +70,9 @@ func supplementRegionally(area *string, city *string, district *string, name str
 		return
 	}
 
+	if class.IsLocal { //测试数据~不请求 api
+		return
+	}
 	//地址截取至~XXX市
 	address = addressReg.ReplaceAllString(address, "${1}")
 	//地址字段抽取~请求接口
@@ -196,28 +200,132 @@ func mergeDepartData(dataArr []map[string]interface{}) []map[string]interface{}
 //规则计算科室对应代码等等
 func confirmDepartNameCode(class1 string, class2 string) (string, string) {
 	new_name := ""
-	new_code := "" //默认其它~
-	//一系列对比~
-	departs := class.Medical_Departs_Datas[class1]
-	if departs != nil { //一级科室能匹配上
+	new_code := ""
+	if class1 == "" {
+		return "其它", class.Medical_Departs_Datas["其它"]["departs_code"]
+	}
+	pre_b, pre_name, pre_code := confirmDepartPreRule(class1, class2)
+	if pre_b {
+		return pre_name, pre_code
+	}
+
+	if class.Medical_Departs_Datas[class1] != nil {
 		new_name = class1
-		new_code = departs["departs_code"]
-		if departs[class2] != "" { //二级科室能匹配上
-			new_name = class1 + "~" + class2
-			new_code = departs[class2]
-		} else {
-			if departs["其它"] != "" {
-				new_name = new_name + "~" + "其它"
-				new_code = departs["其它"]
+		new_code = class.Medical_Departs_Datas[class1]["departs_code"]
+	} else {
+		spec_data := map[string][]map[string]interface{}{}
+	F:
+		for _, v := range class.Medical_RuleDepart {
+			for k, v1 := range v {
+				rule := *qu.ObjToMap(v1)
+				exclude := qu.ObjToString(rule["exclude"])
+				first := qu.ObjToString(rule["first"])
+				second := convertInterfaceMap(rule["second"])
+				if first != "" && regexp.MustCompile(first).MatchString(class1) {
+					new_name = k
+					new_code = class.Medical_Departs_Datas[new_name]["departs_code"]
+					spec_data = map[string][]map[string]interface{}{}
+					break F
+				}
+				if exclude != "" && class2 != "" && regexp.MustCompile(exclude).MatchString(class1) {
+					spec_data[k] = second
+				}
 			}
 		}
-	} else {
+		if len(spec_data) > 0 {
+			for k, v := range spec_data {
+				if class.Medical_Departs_Datas[k][class2] != "" {
+					new_name = k + "~" + class2
+					new_code = class.Medical_Departs_Datas[k][class2]
+					return new_name, new_code
+				}
+				for _, v1 := range v {
+					for k2, v2 := range v1 {
+						rule := qu.ObjToString(v2)
+						if rule != "" && regexp.MustCompile(rule).MatchString(class2) {
+							new_name = k + "~" + k2
+							new_code = class.Medical_Departs_Datas[k][k2]
+							return new_name, new_code
+						}
+					}
+				}
+			}
+			return "其它", class.Medical_Departs_Datas["其它"]["departs_code"]
+		}
+	}
+
+	//已找到一级需要~关联二级
+	if new_name != "" && new_code != "" {
+		departs := class.Medical_Departs_Datas[new_name]
+		if class2 != "" {
+			if departs[class2] != "" {
+				new_name += "~" + class2
+				new_code = departs[class2]
+				return new_name, new_code
+			} else {
+				for _, v := range class.Medical_RuleDepart {
+					for k, v1 := range v {
+						if k == new_name {
+							data := *qu.ObjToMap(v1)
+							second := convertInterfaceMap(data["second"])
+							for _, v2 := range second {
+								for k3, v3 := range v2 {
+									rule := qu.ObjToString(v3)
+									if rule != "" && regexp.MustCompile(rule).MatchString(class2) {
+										new_name += "~" + k3
+										new_code = class.Medical_Departs_Datas[k][k3]
+										return new_name, new_code
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		//未找到合适的二级数据~
+		if departs["其它"] != "" {
+			new_name += "~" + "其它"
+			new_code = departs["其它"]
+		}
+	}
+	if new_name == "" || new_code == "" {
 		new_name = "其它"
 		new_code = class.Medical_Departs_Datas[new_name]["departs_code"]
 	}
 	return new_name, new_code
 }
 
+//确认科室~前置规则~
+func confirmDepartPreRule(class1 string, class2 string) (bool, string, string) {
+	//前置规则比对~
+	if class1 == "骨科" {
+		return true, "外科" + "~" + "骨科", class.Medical_Departs_Datas["外科"]["骨科"]
+	}
+	if class1 == "五官" && departSpecReg1.MatchString(class2) {
+		return true, "眼科", class.Medical_Departs_Datas["眼科"]["departs_code"]
+	}
+	if (class1 == "传染病科" && strings.Contains(class2, "结核病")) ||
+		((class1 == "传染病科" || class1 == "其他科室") && strings.Contains(class2, "地方病科")) {
+		return true, "结核病科", class.Medical_Departs_Datas["结核病科"]["departs_code"]
+	}
+	if class1 == "其他科室" {
+		if strings.Contains(class2, "临终关怀") {
+			return true, "临终关怀科", class.Medical_Departs_Datas["临终关怀科"]["departs_code"]
+		}
+		if strings.Contains(class2, "特种医学科") {
+			return true, "特种医学与军事医学科", class.Medical_Departs_Datas["特种医学与军事医学科"]["departs_code"]
+		}
+		if strings.Contains(class2, "重症医学科") {
+			return true, "重症医学科", class.Medical_Departs_Datas["重症医学科"]["departs_code"]
+		}
+		if strings.Contains(class2, "中西医结合") {
+			return true, "中西医结合科", class.Medical_Departs_Datas["中西医结合科"]["departs_code"]
+		}
+	}
+	return false, "", ""
+}
+
 //类型规则
 func cleanHospitalType(tmp map[string]interface{}, update *map[string]interface{}) {
 	med_type := qu.ObjToString(tmp["type"])

fieldproject_medical/data_preparation/src/main.go

@@ -9,16 +9,18 @@ import (
 )
 
 func init() {
+	//class.IsLocal = true
 	class.InitClass()
 }
 func main() {
 	log.Debug("run main ... ")
 	//处理医院
 	//hospital.RunHospital()
-	//暂时需要线上补充~数据
-	//hospital.RunHospitalOnline()
-	//导入信息~医疗关联sql表
 	//hospital.ExportHospitalInfoToMysql()
+	//hospital.SupplementPingAnMedicalData()
+
+	//代码表构建
+	//vcode.RunVCodeData()
 
 	//产品基本信息
 	//product.RunMedicalProductInfo()
@@ -26,10 +28,7 @@ func main() {
 	//企业生产经营产品~信息
 	//company.RunCompanyProductInfo()
 
-	//代码表构建
-	//vcode.RunVCodeData()
-
-	time.Sleep(999 * time.Hour)
+	time.Sleep(99999 * time.Hour)
 }
 
 //暂不使用~

fieldproject_medical/data_preparation/src/product/product.go

@@ -15,6 +15,7 @@ var productlock sync.Mutex
 func RunMedicalProductInfo() {
 	log.Debug("开始构建~产品信息~")
 	dealWithMedicalProductInfo()
+	time.Sleep(10 * time.Second)
 }
 
 //处理产品信息~ medical_equipment_productinfo

fieldproject_medical/data_preparation/src/repair/repairclass.go

@@ -5,32 +5,90 @@ import (
 	"fmt"
 	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
+	"regexp"
+	"strings"
 )
 
 var (
 	NameLevelPcode = map[string]map[string]string{}
+	CodeClassData  = map[string]map[string]string{}
+	NameClassData  = map[string]string{}
 )
 
-//修复全量表数 新版本分类目录
-func RpairProductAllCodeName() {
-	//构建分类目录
-	datas := *class.MysqlMedicalTool.Find(class.V_Code_Productclass, nil, "", "", -1, -1)
-	for _, v := range datas {
-		name := qu.ObjToString(v["name"])
-		level := fmt.Sprintf("%d", v["level"])
-		pcode := qu.ObjToString(v["pcode"])
-		code := qu.ObjToString(v["code"])
+var repairNameReg_1 = regexp.MustCompile("[()（）]")
 
-		key := name + "~" + level + "~" + pcode
-		if NameLevelPcode[key] == nil {
-			NameLevelPcode[key] = map[string]string{
-				"name": name,
-				"code": code,
+//修复异常类型数据~测试版~是否需要？
+func RepairUnTypeHospitalData() {
+	/*
+		医疗机构名称包含“卫生院”，type是“综合医院”，且医疗机构名称不包含括号。这批数据的类型更新为乡镇卫生院
+	*/
+	datas := class.MysqlMedicalTool.Find("institution_baseinfo", map[string]interface{}{
+		"mi_type_code": "01",
+	}, "mi_name,id,mi_type_code", "", -1, -1)
+	if datas != nil {
+		log.Debug("数量~", len(*datas))
+		ok := 0
+		for _, v := range *datas {
+			name := qu.ObjToString(v["mi_name"])
+			if strings.Contains(name, "卫生院") &&
+				!repairNameReg_1.MatchString(name) {
+				ok++
 			}
 		}
+		log.Debug("待修改数量~", ok)
 	}
-	log.Debug("分类目录准备~", len(NameLevelPcode))
+}
 
+//修复未分类产品信息
+func RepairUnClassProduct() {
+	initRepairVCodeData()
+	sess := class.Save_Mgo.GetMgoConn()
+	defer class.Save_Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{
+		"product_code": "",
+	}
+	it := sess.DB(class.Save_Mgo.DbName).C(class.S_Product_Coll).Find(&q).Select(map[string]interface{}{
+		"product_name": 1,
+		"make_country": 1,
+	}).Sort("_id").Iter()
+	total, isok := 0, 0
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total, "~", isok)
+		}
+		product_name := qu.ObjToString(tmp["product_name"])
+		make_country := qu.ObjToString(tmp["make_country"])
+		code := NameClassData[product_name]
+		tmpid := class.BsonTOStringId(tmp["_id"])
+		if code != "" {
+			isok++
+			update := CodeClassData[code]
+			update["product_code"] = code
+			class.Save_Mgo.UpdateById(class.S_Product_Coll, tmpid, map[string]interface{}{
+				"$set": update,
+			})
+		} else {
+			//是否进口
+			if make_country == "2" {
+				isok++
+				update := map[string]interface{}{}
+				update["product_class1"] = "进口器械"
+				update["product_code"] = "99"
+				class.Save_Mgo.UpdateById(class.S_Product_Coll, tmpid, map[string]interface{}{
+					"$set": update,
+				})
+			}
+		}
+
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is over ", total, "~", isok)
+}
+
+//修复全量表数 新版本分类目录
+func RepairProductAllCodeName() {
+	initRepairVCodeData()
 	sess := class.Save_Mgo.GetMgoConn()
 	defer class.Save_Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{}
@@ -106,7 +164,6 @@ func RpairProductAllCodeName() {
 
 	log.Debug("is over ", total, "~", is_err)
 }
-
 func acquireMedicalClassData(name string, level string, pcode string) (string, string) {
 	new_name, new_code := "", ""
 	key := name + "~" + level + "~" + pcode
@@ -117,3 +174,39 @@ func acquireMedicalClassData(name string, level string, pcode string) (string, s
 	}
 	return new_name, new_code
 }
+func initRepairVCodeData() {
+	//构建分类目录
+	datas := *class.MysqlMedicalTool.Find(class.V_Code_Productclass, nil, "", "", -1, -1)
+	for _, v := range datas {
+		name := qu.ObjToString(v["name"])
+		level := fmt.Sprintf("%d", v["level"])
+		pcode := qu.ObjToString(v["pcode"])
+		code := qu.ObjToString(v["code"])
+		key := name + "~" + level + "~" + pcode
+		if NameLevelPcode[key] == nil {
+			NameLevelPcode[key] = map[string]string{
+				"name": name,
+				"code": code,
+			}
+		}
+
+		dict := CodeClassData[code]
+		if pcode != "" {
+			dict = CodeClassData[pcode]
+			if dict == nil {
+				log.Debug("顺序错误~")
+			}
+		}
+		new_dict := map[string]string{}
+		for k, v := range dict {
+			new_dict[k] = v
+		}
+		new_key := "product_class" + level
+		new_dict[new_key] = name
+		CodeClassData[code] = new_dict
+
+		NameClassData[name] = code
+
+	}
+	log.Debug("分类目录准备~", len(NameLevelPcode), len(CodeClassData), len(NameClassData))
+}

fieldproject_medical/data_preparation/src/rule_depart.json

@@ -0,0 +1,212 @@
+{
+    "rule": [        {
+            "妇产科": {
+                "exclude": "^(其他科室)$",
+                "first": "^(妇产科学)$",
+                "second": [
+                    {"计划生育科": "(计划生育|计划生育服务部|计划生育服务指导中心|计划生育服务中心|计划生育技术服务部|计划生育技术服务中心|计划生育科|计划生育门诊|计划生育与不孕不育科|计划生育中心|计划生育综合科)"},
+                    {"优生学科": "(婚检优生科|遗传、优生科|遗传优生科|遗传优生与妇保|遗传优生咨询|遗传与优生门诊|优生科|优生遗传、同位素科|优生遗传科|优生遗传咨询专病门诊|优生优育科|优生优育门诊|优生优育遗传中心|优生优育指导部)"},
+                    {"生殖健康与不孕症科": "(不孕不育科|不孕症门诊|生殖健康门诊|生殖健康与不孕症科|遗传生殖医学中心|生殖中心|生殖不孕|生殖医学中心|17楼生殖中心|北城生殖医学科|不孕不育生殖科|不孕不育生殖医学中心|不孕不育与生殖健康科|不孕不育与生殖男科|大学城生殖医学科|大学城五楼生殖医学科|福强生殖健康科|福强生殖医学中心|福强生殖综合门诊|福强生殖综合特诊|福强生殖综合诊|福强院区生殖健康科|河西生殖中心门诊|湖北省生殖保健中心|焦作市生殖医学中心|临床生殖医学中心|生殖保健|生殖保健科|生殖保健门诊|生殖保健中心|生殖不孕|生殖不孕科|生殖不孕中心|生殖健康|生殖健康科|生殖健康科福强院|生殖健康科治疗室|生殖健康门诊|生殖健康男科|生殖健康男性门诊|生殖健康与不孕|生殖健康与不孕不育|生殖健康与不孕不育科|生殖健康与不孕不育门诊|生殖健康与不孕不育症科|生殖健康与不孕科|生殖健康与不孕门诊|生殖健康与不孕症|生殖健康中心|生殖健康助孕中心|生殖健康专家|生殖健康专科|生殖健康咨询专科|生殖科|生殖门诊|生殖与不孕不育科|生殖与不孕门诊|生殖与不孕研究所|生殖与不孕专家)"}
+               ]
+            }
+        },   {
+            "儿科": {
+                "exclude": "^(儿科学)$",
+                "first": "",
+                "second": [
+					{"新生儿科":"(新生儿)"},
+					{"小儿传染病科":"(结核科)"},
+					{"小儿消化科":"(消化科|消化内科|消化营养科)"},
+					{"小儿呼吸科":"(呼吸哮喘|呼吸|呼吸科)"},
+					{"小儿心脏病科":"(儿童心脏科|儿童心脏中心|小儿心脏科|小儿心脏中心|心脏小儿科|心儿科|心内科)"},
+					{"小儿肾病科":"(肾病专科|肾病科|肾病内科^消化|肾内科|肾脏科|肾脏内科|肾脏病)"},
+					{"小儿血液病科":"(血液病专科|血液专科|血液科|血液内科|小儿血液)"},
+					{"小儿神经病学科":"(儿科(神经专业组)|儿科神经内分泌专科|儿神经科|儿童神经科|神经内科|小儿神经病科|小儿神经内科|小儿神经消化科|小儿神经专科)"},
+					{"小儿内分泌科":"(儿科内分泌|儿科神经内分泌专科|儿童内分泌遗传代谢科|内分泌科|小儿内分泌遗传科|小儿内分泌专科|小儿血液内分泌科)"},
+					{"小儿免疫科":"(儿科风湿免疫专科|儿科肾脏风湿免疫科|儿童变态反应（过敏）与免疫科|儿童风湿免疫过敏科|儿童肾脏风湿免疫科|小儿风湿免疫科|小儿肾脏风湿免疫科|风湿科)"}
+               ]
+            }
+        },  {
+            "小儿外科": {
+                "exclude": "^(儿科学)$",
+                "first": "",
+                "second": [
+					{"小儿普通外科":"(儿童基础外科|儿童外科|儿外科|普通外科二科（儿外科）|普通小儿外科|普外儿外科|小儿普外科|小儿外科|小儿外科、普外科|新生儿外科)"},
+					{"小儿骨科":"(矫形及肢体重建外科/小儿骨科|小儿骨外科)"},
+					{"小儿泌尿外科":"(儿童泌尿外科|泌尿外科|泌尿小儿外科|泌尿与小儿外科|女性泌尿与小儿外科|小儿泌尿外科)"},
+					{"小儿胸心外科":"(儿童心胸外科|儿外胸外科|小儿心胸外科|小儿胸外科|胸心小儿外科)"},
+					{"小儿神经外科":"(儿童神经外科|小儿神经外科)"}
+
+               ]
+            }
+        },   {
+            "儿童保健科": {
+                "exclude": "^(儿科学)$",
+                "first": "",
+                "second": [
+					{"儿童生长发育科":"(儿童生长发育科|儿童生长发育专科|儿童发育行为科|儿童行为发育中心|发育儿科|发育行为儿科|发育行为儿童保健科|神经发育科|语言发育科)"},
+					{"儿童营养科":"(小儿消化营养科|小儿营养保健科)"},
+					{"儿童心理卫生科":"(儿少心理科|儿童心理科|儿童心理门诊|儿童心理指导部|青少年儿童心理中心|小儿精神心理科)"},
+					{"儿童康复科":"(儿科康复|儿科康复中心|儿童保健康复科|儿童保健康复中心|儿童孤独症康复治疗中心|儿童康复保健科|儿童康复科|儿童康复医学科|儿童康复治疗部|儿童康复中心|儿童脑病康复中心|儿童神经康复科|儿童医学康复中心|小儿康复科|小儿脑病康复中心|小儿神经康复科|小儿神经康复中心)"}
+               ]
+            }
+        },  
+        {
+            "耳鼻咽喉科": {
+                "exclude": "^(耳鼻喉头颈外科|五官)$",
+                "first": "",
+                "second": [
+                    {"耳科": "(耳病|耳疾|耳科|耳聋|耳鸣|耳内科|耳专科)"},
+                    {"鼻科": "(鼻炎|鼾病|鼾症)"},
+                    {"咽喉科": "(喉疾病|甲状腺病)"}
+               ]
+            }
+        },
+        {
+            "口腔科": {
+                "exclude": "^(耳鼻喉头颈外科|五官)$",
+                "first": "^(口腔)$",
+                "second": [
+                    {"口腔内科": "(口腔内科)"},
+                    {"口腔颌面外科": "(颌面)"},
+                    {"正畸科": "(矫形|正畸)"},
+                    {"口腔修复科": "(口腔修复|口腔科)"},
+                    {"口腔预防保健科": "(口腔预防)"}
+                ]
+            }
+        },
+        {
+            "皮肤科": {
+                "exclude": "",
+                "first": "^(皮肤性病科)$",
+                "second": [
+                    {"皮肤病科": "(皮肤科)"},
+                    {"性传播疾病科": "(艾滋病专科)"}
+                ]
+            }
+        },
+        {
+            "医疗美容科": {
+                "exclude": "",
+                "first": "^(皮肤美容|整形美容科)$",
+                "second": []
+            }
+        },
+        {
+            "精神科": {
+                "exclude": "^(其他科室)$",
+                "first": "^(精神|精神心理科)$",
+                "second": [
+					{"精神病科":"(精神病科)"},
+					{"精神卫生科":"(精神卫生)"},
+					{"药物依赖科":"(药物依赖|酒药依赖|药物和酒精依赖)"},
+					{"精神康复科":"(精神康复科)"},
+					{"临床心理科":"(临床心理|精神（心理）临床康复中心|临床.{1}科)"},
+					{"司法精神科":"(司法精神科)"}
+                ]
+            }
+        },
+        {
+            "传染科": {
+                "exclude": "^(传染|传染病科)$",
+                "first": "",
+                "second": [
+					{"肠道传染病科":"(肠道传染病|肠道传染病科)"},
+					{"肝炎科":"(肝病|肝炎|肝科|肝一科)"}
+                ]
+            }
+        },
+        {
+            "肿瘤科": {
+                "exclude": "",
+                "first": "^(肿瘤|肿瘤专科护理)$",
+                "second": [
+                ]
+            }
+        },
+        {
+            "急诊医学科": {
+                "exclude": "",
+                "first": "^(急诊科)$",
+                "second": [
+                ]
+            }
+        },
+        {
+            "康复医学科": {
+                "exclude": "",
+                "first": "^(康复)$",
+                "second": [
+                ]
+            }
+        },
+        {
+            "职业病科": {
+                "exclude": "^(其他科室)$",
+                "first": "",
+                "second": [
+					{"职业中毒科":"(职业病与中毒科|职业病与中毒医学科|职业中毒|中毒职业病)"},
+					{"尘肺科":"(尘肺)"}
+                ]
+            }
+        }, 
+        {
+            "麻醉科": {
+                "exclude": "",
+                "first": "^(麻醉医学科)$",
+                "second": [
+                ]
+            }
+        },
+        {
+            "医学影像科": {
+                "exclude": "^(其他科室|其它|核医学科)$",
+                "first": "",
+                "second": [
+					{"X线诊断科":"(X线科|X线诊断科)"},
+					{"CT诊断科":"(CT诊断)"},
+					{"磁共振成像诊断科":"(磁共振（MR）室|磁共振成像诊断科|磁共振检查|磁共振检查科|磁共振科|磁共振普通|磁共振室|磁共振影像科|磁共振诊断室|磁共振中心|放射科（磁共振室）|核磁共振科|核磁共振室|磁共振影像科|磁共振中心)"},
+					{"核医学科":"(核医学科|核医学普通|核医学实验检查|核医学室|核医学中心|核医学门诊|核医学实验室|核医学研究室|核医学专科)"},
+					{"超声诊断科":"(超声诊断)"},
+					{"神经肌肉电图科":"(神经肌肉检查室)"},
+					{"介入放射学科":"(介入放射)"},
+					{"放射治疗科":"(放射治疗科|放射治疗中心)"}
+                ]
+            }
+        },
+        {
+			"中医科": {
+                "exclude": "",
+                "first": "^(中医)$",
+                "second": [
+					{"内科":"(内.{1}科|内科)"},
+					{"外科":"(外科|外.{1}科)"},
+					{"妇产科":"(妇产|妇儿|妇科)"},
+					{"儿科":"(儿科|儿内科)"},
+					{"皮肤科":"(皮肤)"},
+					{"耳鼻咽喉科":"(耳、鼻、喉科|耳鼻喉|耳鼻喉科|耳鼻喉头颈外科|耳鼻喉咽科|耳鼻咽喉科|耳鼻咽喉头颈科|耳鼻咽喉头颈外科|耳鼻咽喉-头颈外科)"},
+					{"骨伤科":"(骨伤|骨科|骨外科)"},
+					{"肛肠科":"(肛肠)"},
+					{"老年病科":"(老年病|老年科|老年干部科|老年医学)"},
+					{"针灸科":"(针灸)"},
+					{"推拿科":"(推拿)"},
+					{"康复医学":"(康复)"},
+					{"急诊科":"(急诊)"}
+
+                ]
+            }
+        },
+        {
+			"民族医学科": {
+                "exclude": "^(其他科室)$",
+                "first": "",
+                "second": [
+					{"藏医学科":"(藏医、回医科|藏医科|藏医全科)"},
+					{"蒙医学科":"(布病蒙医科|蒙医儿科|蒙医耳鼻喉科|蒙医肺病科|蒙医风湿病科|蒙医妇科|蒙医骨伤科|蒙医康复科|蒙医科|蒙医老年病科|蒙医疗术科|蒙医脑病科|蒙医内分泌科|蒙医内科|蒙医脾胃病科|蒙医贴敷疗法科|蒙医五疗科|蒙医心血管内科|蒙医心脏病科|蒙医学科|蒙医血液病科|蒙医血液疑难病科|蒙医针灸五疗科|蒙医针推科|蒙医整骨科|蒙医肿瘤科|蒙医紫癜科)"},
+					{"彝医学科":"(彝医)"}
+                ]
+            }
+        }
+    ]
+
+}

fieldproject_medical/data_service/src/bidding/bidding.go

@@ -5,32 +5,43 @@ import (
 	log "github.com/donnie4w/go-logger/logger"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	qu "qfw/util"
+	"regexp"
 	"strings"
 	"sync"
 	"unicode/utf8"
 	ul "util"
 )
 
-var fields = map[string]interface{}{"toptype": 1, "subtype": 1, "subscopeclass": 1, "extracttype": 1, "purchasinglist": 1}
-
+var BidFields = map[string]interface{}{"toptype": 1, "subtype": 1, "s_topscopeclass": 1, "s_subscopeclass": 1, "buyerclass": 1, "buyer": 1, "extracttype": 1, "purchasinglist": 1}
 var datalock, numlock sync.Mutex
 
-func RunPurchasingInfo(gtid string, lteid string) {
-	log.Debug("开始处理标讯信息~~~", gtid, "~", lteid)
+var A_FieldReg *regexp.Regexp = regexp.MustCompile("(医疗卫生_设备|医疗卫生_耗材)")
+var B_FieldReg *regexp.Regexp = regexp.MustCompile("^(医疗|卫健委)$")
+
+func RunPurchasingInfo(gtid string, lteid string, obj_id bool) {
+	log.Debug("开始处理标讯信息~~~", ul.S_Bidding_Coll, "~", gtid, "~", lteid)
 	sess := ul.Mgo.GetMgoConn()
 	defer ul.Mgo.DestoryMongoConn(sess)
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
-			"$gt":  class.StringTOBsonId(gtid),
-			"$lte": class.StringTOBsonId(lteid),
+			"$gt":  gtid,
+			"$lte": lteid,
 		},
 	}
-	it := sess.DB(ul.Mgo.DbName).C(ul.S_Bidding_Coll).Find(&q).Sort("_id").Select(fields).Iter()
+	if obj_id {
+		q = map[string]interface{}{
+			"_id": map[string]interface{}{
+				"$gt":  class.StringTOBsonId(gtid),
+				"$lte": class.StringTOBsonId(lteid),
+			},
+		}
+	}
+	it := sess.DB(ul.Mgo.DbName).C(ul.S_Bidding_Coll).Find(&q).Sort("_id").Select(BidFields).Iter()
 	pool := make(chan bool, 8)
 	wg := &sync.WaitGroup{}
 	total, isok := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
-		if total%100 == 0 {
+		if total%1000 == 0 {
 			log.Debug("cur index ", total, "~", isok)
 		}
 		if qu.IntAll(tmp["extracttype"]) != 1 {
@@ -44,11 +55,21 @@ func RunPurchasingInfo(gtid string, lteid string) {
 				<-pool
 				wg.Done()
 			}()
-			//构建标的物基本信息
-			b := createBaseInfo(tmp)
+			infoid := ""
+			if obj_id {
+				infoid = class.BsonTOStringId(tmp["_id"])
+			} else {
+				infoid = qu.ObjToString(tmp["_id"])
+			}
+			b := createBaseInfo(tmp, infoid) //构建信息
 			if b {
 				numlock.Lock()
 				isok++
+				ul.Mgo.UpdateById(ul.U_Bidding_Coll, class.BsonTOStringId(tmp["_id"]), map[string]interface{}{
+					"$set": map[string]interface{}{
+						"bid_field": "0101",
+					},
+				})
 				numlock.Unlock()
 			}
 		}(tmp)
@@ -59,55 +80,61 @@ func RunPurchasingInfo(gtid string, lteid string) {
 	log.Debug("is over ", total, "~", isok)
 }
 
-//构建标的物基本信息
-func createBaseInfo(tmp map[string]interface{}) bool {
+//构建标的物基本信息~模式一~规则型
+func createBaseInfo(tmp map[string]interface{}, infoid string) bool {
 	p_list := IsMarkInterfaceMap(tmp["purchasinglist"])
-	sub_list := IsMarkInterfaceArr(tmp["subscopeclass"])
-	infoid := class.BsonTOStringId(tmp["_id"])
+	new_plist := []map[string]interface{}{}
 	bid_topsubtype_code := confrimTopSubCode(qu.ObjToString(tmp["toptype"]), qu.ObjToString(tmp["subtype"]))
-	b, industry_code, new_plist := checkWhetherValidInfo(p_list, sub_list)
-	if b {
+	industry_code := confrimIndustryCode(qu.ObjToString(tmp["s_topscopeclass"]), qu.ObjToString(tmp["s_subscopeclass"]))
+	isField := IsMedicalFieldInfoRuleFirst(tmp)
+	if isField {
+		_, new_plist = createNewPurchasingInfo(p_list, infoid)
+	} else {
+		if IsMedicalFieldInfoRuleSecond(tmp) {
+			isField, new_plist = createNewPurchasingInfo(p_list, infoid)
+		}
+	}
+	if isField { //最终根据是否为领域数据灌入数据
 		//标的物基本信息~记录标签
-		//insertBaseInfo(new_plist, infoid, bid_topsubtype_code)
+		insertBaseInfo(new_plist, infoid, bid_topsubtype_code)
 		//招标信息领域标签
-		//insertFiledTag(infoid)
+		insertFiledTag(infoid)
 		//招标信息行业标签
-		//insertIndustryTag(industry_code, infoid)
+		insertIndustryTag(industry_code, infoid)
 	}
-	if industry_code != "" && len(new_plist) > 0 && bid_topsubtype_code != "" && infoid != "" {
 
-	}
-	return b
-}
+	//临时处理
+	//isField, new_plist := createNewPurchasingInfo(p_list, infoid)
+	//if isField { //最终根据是否为领域数据灌入数据
+	//	//标的物基本信息~记录标签
+	//	insertBaseInfo(new_plist, infoid, bid_topsubtype_code)
+	//	//招标信息领域标签
+	//	insertFiledTag(infoid)
+	//	//招标信息行业标签
+	//	insertIndustryTag(industry_code, infoid)
+	//}
 
-//return 是否有效~行业代码~新标的物信息
-func checkWhetherValidInfo(p_list []map[string]interface{}, sub_list []string) (bool, string, []map[string]interface{}) {
-	is_exists := false
-	industry_code, b := IsIndustryInfo(sub_list)
-	new_plist := []map[string]interface{}{}
-	if len(p_list) > 0 && b {
-		is_exists, new_plist = createNewPurchasingInfo(p_list)
-	}
-	return is_exists, industry_code, new_plist
+	//阻止注释~报错
+	//if bid_topsubtype_code == "" || industry_code == "" || len(new_plist) == 0 {}
+
+	return isField
 }
 
 //返回新的~标的物信息（分类+整合）
-func createNewPurchasingInfo(p_list []map[string]interface{}) (bool, []map[string]interface{}) {
-	is_exists := false
+func createNewPurchasingInfo(p_list []map[string]interface{}, infoid string) (bool, []map[string]interface{}) {
 	new_plist := []map[string]interface{}{}
+	isExists := false
 	for _, v := range p_list {
 		//去重处理~名称~品牌~型号~暂无重复
-
 		data := map[string]interface{}{}
 		itemname := qu.ObjToString(v["itemname"])
-		if itemname == "" {
+		if itemname == "" || utf8.RuneCountInString(itemname) > 30 {
 			continue
 		}
 		//根据标的物名字~打上具体的分类数据~
-		is_yl, yl_code := confrimYlClassCode(itemname)
-
-		if !is_exists && is_yl {
-			is_exists = true //证明此条信息有效果
+		yl_code := confrimTargetMedicalClass(itemname, infoid)
+		if yl_code != "" {
+			isExists = true
 		}
 		data["medical_equipment_code"] = yl_code
 		data["itemname"] = itemname
@@ -118,7 +145,6 @@ func createNewPurchasingInfo(p_list []map[string]interface{}) (bool, []map[strin
 		}
 		data["model"] = model
 		data["specs"] = qu.ObjToString(v["specs"])
-
 		if v["unitname"] != nil {
 			data["unit"] = qu.ObjToString(v["unitname"])
 		}
@@ -140,37 +166,88 @@ func createNewPurchasingInfo(p_list []map[string]interface{}) (bool, []map[strin
 		}
 		new_plist = append(new_plist, data)
 	}
-	return is_exists, new_plist
+	return isExists, new_plist
 }
 
 //是否为医疗行业数据
-func IsIndustryInfo(sub_list []string) (string, bool) {
+func IsMedicalFieldInfoRuleFirst(tmp map[string]interface{}) bool {
+	top_class := qu.ObjToString(tmp["s_topscopeclass"])
+	sub_class := qu.ObjToString(tmp["s_subscopeclass"])
+	buyer_class := qu.ObjToString(tmp["buyerclass"])
+	buyer := qu.ObjToString(tmp["buyer"])
+	if isFieldInfoMethodFirst(top_class, sub_class, buyer_class, buyer) {
+		return true
+	}
+	return false
+}
+
+//是否为医疗行业数据
+func IsMedicalFieldInfoRuleSecond(tmp map[string]interface{}) bool {
+	top_class := qu.ObjToString(tmp["s_topscopeclass"])
+	sub_class := qu.ObjToString(tmp["s_subscopeclass"])
+	buyer_class := qu.ObjToString(tmp["buyerclass"])
+	buyer := qu.ObjToString(tmp["buyer"])
+	if isFieldInfoMethodSecond(top_class, sub_class, buyer_class, buyer) {
+		return true
+	}
+	return false
+}
+
+//第一组规则条件是否校验通过
+func isFieldInfoMethodFirst(top_class string, sub_class string, buyer_class string, buyer string) bool {
+	is_A, is_B := false, false
+	if A_FieldReg.MatchString(sub_class) || (sub_class == "" && strings.Contains(top_class, "医疗卫生")) {
+		is_A = true
+	}
+	if !is_A {
+		return false
+	}
+	if B_FieldReg.MatchString(buyer_class) || (buyer_class == "制造业" && strings.Contains(buyer, "医疗器械") && strings.Contains(buyer, "公司")) {
+		is_B = true
+	}
+	return is_A && is_B
+}
+
+//第二组规则条件是否校验通过
+func isFieldInfoMethodSecond(top_class string, sub_class string, buyer_class string, buyer string) bool {
+	is_A, is_B := false, false
+	if A_FieldReg.MatchString(sub_class) || (sub_class == "" && strings.Contains(top_class, "医疗卫生")) {
+		is_A = true
+	}
+	if !is_A {
+		return false
+	}
+	if !(B_FieldReg.MatchString(buyer_class) || (buyer_class == "制造业" && strings.Contains(buyer, "医疗器械") && strings.Contains(buyer, "公司"))) {
+		is_B = true
+	}
+	return is_A && is_B
+}
+
+//确认行业code
+func confrimIndustryCode(top_class string, sub_class string) string {
 	code := ""
-	b := false
-	for _, v := range sub_list {
+	top_arr := strings.Split(top_class, ",")
+	for _, v := range top_arr {
+		vc := ul.Bid_Industry[v]["code"]
+		if code != "" {
+			code += ","
+		}
+		code += vc
+	}
+	sub_arr := strings.Split(sub_class, ",")
+	for _, v := range sub_arr {
 		arr := strings.Split(v, "_")
 		if len(arr) == 2 {
 			industry_1 := arr[0]
 			industry_2 := arr[1]
-			if industry_1 == "医疗卫生" && (industry_2 == "设备" || industry_2 == "耗材") {
-				b = true
-				code = ul.Bid_Industry[industry_1][industry_2]
-				break
+			vc := ul.Bid_Industry[industry_1][industry_2]
+			if code != "" {
+				code += ","
 			}
+			code += vc
 		}
 	}
-	return code, b
-}
-
-//根据标的物名称~打医疗分类
-func confrimYlClassCode(name string) (bool, string) {
-	is_b := false
-	yl_code := ""
-	yl_code = ConfrimTargetMedicalClass(name)
-	if utf8.RuneCountInString(yl_code) > 0 {
-		is_b = true
-	}
-	return is_b, yl_code
+	return code
 }
 
 //根据标的物名称~打医疗分类

fieldproject_medical/data_service/src/bidding/bidsql.go

@@ -25,7 +25,6 @@ func insertBaseInfo(p_list []map[string]interface{}, infoid string, bid_topsubty
 				ul.InsertGlobalMysqlData(ul.V_Bid_Purchasing_Field_Record, info_tag, infoid)
 			}
 		}
-
 	}
 }
 

fieldproject_medical/data_service/src/bidding/clean.go

@@ -9,7 +9,7 @@ import (
 var arcReg = regexp.MustCompile("[(（].*[)）]")
 
 //异常词
-var abnormalReg = regexp.MustCompile("^((医疗|普通|医用)设备|[一二三四五六七八九A-Za-z1-9]包)$")
+var abnormalReg = regexp.MustCompile("^((医疗|普通|医用)设备|[一二三四五六七八九A-Za-z1-9]包|其他[:：])$")
 
 var dataLock sync.Mutex
 

fieldproject_medical/data_service/src/bidding/purchasing.go

@@ -9,11 +9,11 @@ import (
 )
 
 //计算相似度分类
-func ConfrimTargetMedicalClass(name string) string {
+func confrimTargetMedicalClass(name string, infoid string) string {
 	//清洗~名称
 	name = cleanItemName(name)
-	if utf8.RuneCountInString(name) <= 2 {
-		return "" //过短数据不进行分类
+	if utf8.RuneCountInString(name) <= 2 || utf8.RuneCountInString(name) >= 30 {
+		return "" //过长短数据不进行分类
 	}
 	//完全匹配校验
 	b, med_code := completeMatching(name)
@@ -32,16 +32,15 @@ func ConfrimTargetMedicalClass(name string) string {
 			}
 		}
 	}
-	return calculateSimilarityScore(indexDocs, itemArr)
+	if ul.IsLocal {
+		return calculateSimilarityScoreTest(indexDocs, itemArr, infoid)
+	}
+	return calculateSimilarityScore(indexDocs, itemArr, infoid)
 }
 
 //计算相似度得分
-func calculateSimilarityScore(indexDocs map[int][]string, itemArr []string) string {
+func calculateSimilarityScore(indexDocs map[int][]string, itemArr []string, infoid string) string {
 	scoreDocs := map[int]float64{}
-	//临时记录~
-	scoreDocs_1 := map[int]float64{}
-	scoreDocs_2 := map[int]float64{}
-
 	itemName := strings.Join(itemArr, "")
 	for k, v := range indexDocs {
 		v_str := strings.Join(v, "")
@@ -51,34 +50,17 @@ func calculateSimilarityScore(indexDocs map[int][]string, itemArr []string) stri
 		dice_score := strsim.Compare(v_str, itemName, strsim.DiceCoefficient())
 		//优化空间~高分选取阈值~低分过滤阈值~综合阈值
 		finally_score := (base_score + dice_score) / 2
-		if finally_score > 0.55 && dice_score > 0.0 {
+		if finally_score >= 0.55 && dice_score > 0.0 {
 			scoreDocs[k] = qu.FloatFormat(finally_score, 2)
-			//临时记录一下分数
-			scoreDocs_1[k] = qu.FloatFormat(base_score, 2)
-			scoreDocs_2[k] = qu.FloatFormat(dice_score, 2)
 		}
 	}
 	if len(scoreDocs) == 0 {
 		return ""
 	}
 	//取出最高有效分~数据
-	index, score := getMaxScore(scoreDocs)
+	index, _ := getMaxScore(scoreDocs)
 	match_str := strings.Join(ul.NgrmDocIndex[index], "")
 	med_code := ul.ProductDocText[match_str]
-	//临时~测试保存数据
-	catalog := ul.CodeCatalog[med_code]
-	ul.Mgo.Save("zzzzzz", map[string]interface{}{
-		"name":       strings.Join(itemArr, ""),
-		"match_name": match_str,
-		"score":      score,
-		"score_1":    scoreDocs_1[index],
-		"score_2":    scoreDocs_2[index],
-		"code":       med_code,
-		"class_1":    catalog["class_1"],
-		"class_2":    catalog["class_2"],
-		"class_3":    catalog["class_3"],
-		"class_4":    catalog["class_4"],
-	})
 	return med_code
 }
 
@@ -219,3 +201,51 @@ func completeMatching(name string) (bool, string) {
 	}
 	return is_b, med_code
 }
+
+//测试版~导出数据
+func calculateSimilarityScoreTest(indexDocs map[int][]string, itemArr []string, infoid string) string {
+	scoreDocs := map[int]float64{}
+	scoreDocs_1 := map[int]float64{}
+	scoreDocs_2 := map[int]float64{}
+	itemName := strings.Join(itemArr, "")
+	for k, v := range indexDocs {
+		v_str := strings.Join(v, "")
+		//基础分计算
+		base_score := confrimBaseScore(v, itemArr)
+		//近义词计算
+		dice_score := strsim.Compare(v_str, itemName, strsim.DiceCoefficient())
+		//优化空间~高分选取阈值~低分过滤阈值~综合阈值
+		finally_score := (base_score + dice_score) / 2
+		if finally_score >= 0.55 && dice_score > 0.0 {
+			scoreDocs[k] = qu.FloatFormat(finally_score, 2)
+			//临时记录一下分数
+			scoreDocs_1[k] = qu.FloatFormat(base_score, 2)
+			scoreDocs_2[k] = qu.FloatFormat(dice_score, 2)
+
+		}
+	}
+	if len(scoreDocs) == 0 {
+		return ""
+	}
+	//取出最高有效分~数据
+	index, score := getMaxScore(scoreDocs)
+	match_str := strings.Join(ul.NgrmDocIndex[index], "")
+	med_code := ul.ProductDocText[match_str]
+
+	//测试使用~存数据
+	catalog := ul.CodeCatalog[med_code]
+	ul.Mgo.Save("zzzzzz_query", map[string]interface{}{
+		"infoid":     infoid,
+		"name":       strings.Join(itemArr, ""),
+		"match_name": match_str,
+		"score":      score,
+		"score_1":    scoreDocs_1[index],
+		"score_2":    scoreDocs_2[index],
+		"code":       med_code,
+		"class_1":    catalog["class_1"],
+		"class_2":    catalog["class_2"],
+		"class_3":    catalog["class_3"],
+		"class_4":    catalog["class_4"],
+	})
+	return med_code
+}

fieldproject_medical/data_service/src/export/export.go

@@ -5,42 +5,172 @@ import (
 	"class"
 	log "github.com/donnie4w/go-logger/logger"
 	qu "qfw/util"
+	"sync"
 	ul "util"
 )
 
-//导出测试数据
-func ExportBiddingTestData() {
+var n_lock sync.Mutex
+
+//导出规则二测试的数据
+func ExportRuleSecondTestData(coll_name string) {
+	log.Debug("开始处理标讯信息~~~")
 	sess := ul.Mgo.GetMgoConn()
 	defer ul.Mgo.DestoryMongoConn(sess)
-	//临时取~测试数据~8月1日
 	q := map[string]interface{}{
 		"_id": map[string]interface{}{
-			"$gte": class.StringTOBsonId("62e6a7000000000000000000"),
+			"$gt":  class.StringTOBsonId("130e34000000000000000000"),
+			"$lte": class.StringTOBsonId("630e34000000000000000000"),
 		},
 	}
-	total, isok := 0, 0
-	it := sess.DB(ul.Mgo.DbName).C("bidding").Find(&q).Sort("_id").Iter()
+	it := sess.DB(ul.Mgo.DbName).C(ul.S_Bidding_Coll).Find(&q).Sort("_id").Select(bidding.BidFields).Iter()
+	pool := make(chan bool, 24)
+	wg := &sync.WaitGroup{}
+	total, isok1, isok2 := 0, 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
 		if total%10000 == 0 {
-			log.Debug("cur index ", total, "~", isok)
-		}
-		if isok > 5000 {
-			break
+			log.Debug("cur index ", total, "~", isok1, "~", isok2)
 		}
 		if qu.IntAll(tmp["extracttype"]) != 1 {
 			tmp = make(map[string]interface{})
 			continue
 		}
-		//是否为医疗行业数据
-		p_list := bidding.IsMarkInterfaceMap(tmp["purchasinglist"])
-		sub_list := bidding.IsMarkInterfaceArr(tmp["subscopeclass"])
-		_, b := bidding.IsIndustryInfo(sub_list)
-		if b && len(p_list) > 0 {
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			p_list := bidding.IsMarkInterfaceMap(tmp["purchasinglist"])
+			if bidding.IsMedicalFieldInfoRuleSecond(tmp) && len(p_list) > 0 {
+				n_lock.Lock()
+				isok1++
+				n_lock.Unlock()
+				ul.Mgo.Save(coll_name, tmp)
+
+				//tmpid := class.BsonTOStringId(tmp["_id"])
+				//data := ul.MysqlGlobalTool.FindOne(ul.V_Bid_Fieldtags, map[string]interface{}{
+				//	"infoid": tmpid,
+				//}, "infoid", "")
+				//if data == nil {
+				//	n_lock.Lock()
+				//	isok2++
+				//	n_lock.Unlock()
+				//} else {
+				//	log.Debug("存在~", tmpid)
+				//}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+	wg.Wait()
+
+	log.Debug("is over ", total, "~", isok1, "~", isok2)
+}
+
+//更新修复导出的数据
+func UpdateRepairExportTestData(coll_name string) {
+	sess := ul.Mgo.GetMgoConn()
+	defer ul.Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	total, isok := 0, 0
+	it := sess.DB(ul.Mgo.DbName).C(coll_name).Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%10000 == 0 {
+			log.Debug("cur index ", total, "~", isok)
+		}
+		tmpid := class.BsonTOStringId(tmp["_id"])
+		s := ul.MysqlGlobalTool.FindOne(ul.V_Bid_Fieldtags, map[string]interface{}{
+			"infoid": tmpid,
+		}, "infoid", "")
+		if s != nil { //此条信息已存在
 			isok++
-			ul.Mgo.Save("zzzzzzzzz", tmp)
+			ul.Mgo.DeleteById(coll_name, tmpid)
 		}
 		tmp = make(map[string]interface{})
 	}
+	log.Debug("is over ", total, "~", isok)
+}
+
+//临时短暂修复地域数据~非领域
+func ShortRepairCityData() {
+	log.Debug("临时短暂修复地域数据~非领域~~~")
+	sess := ul.Mgo.GetMgoConn()
+	defer ul.Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	total, isok := 0, 0
+	it := sess.DB(ul.Mgo.DbName).C("zzzzz_uncity_new").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%100 == 0 {
+			log.Debug("cur index ", total, "~", isok)
+		}
+		tmpid := ul.BsonTOStringId(tmp["_id"])
+		area := qu.ObjToString(tmp["area"])
+		city := qu.ObjToString(tmp["city"])
+		district := qu.ObjToString(tmp["district"])
+		update := map[string]interface{}{
+			"_id":      tmp["_id"],
+			"area":     area,
+			"city":     city,
+			"district": district,
+		}
+		if city != "" {
+			data := ul.Mgo.FindById("zzzzz_uncity", tmpid)
+			d_city := qu.ObjToString(data["city"])
+			d_district := qu.ObjToString(data["district"])
+
+			if (d_city == "" && city != "") || d_district == "" && district != "" {
+				isok++
+				ul.Mgo.Save("zktest_uncity_repair", update)
+			}
+		}
+
+		tmp = make(map[string]interface{})
+	}
 
 	log.Debug("is over ", total, "~", isok)
 }
+
+//临时短暂删除重复数据
+func ShortDeleteRepeatData() {
+	log.Debug("短暂计算待删除重复数据~~~")
+	sess := ul.Mgo.GetMgoConn()
+	defer ul.Mgo.DestoryMongoConn(sess)
+	q := map[string]interface{}{}
+	pool := make(chan bool, 6)
+	wg := &sync.WaitGroup{}
+	total, isok1, isok2 := 0, 0, 0
+	it := sess.DB(ul.Mgo.DbName).C("bidding_field_0101").Find(&q).Sort("_id").Iter()
+	for tmp := make(map[string]interface{}); it.Next(&tmp); total++ {
+		if total%5000 == 0 {
+			log.Debug("cur index ", total, "~", isok1, "~", isok2)
+		}
+		pool <- true
+		wg.Add(1)
+		go func(tmp map[string]interface{}) {
+			defer func() {
+				<-pool
+				wg.Done()
+			}()
+			tmpid := ul.BsonTOStringId(tmp["_id"])
+			data := ul.Mgo.FindById("bidding_field_0901", tmpid)
+			if data != nil && len(data) > 2 {
+				isok1++
+				ul.Mgo.DeleteById("bidding_field_0101", tmpid)
+				log.Debug(tmpid)
+			} else {
+				s := ul.MysqlGlobalTool.FindOne(ul.V_Bid_Fieldtags, map[string]interface{}{
+					"infoid": tmpid,
+				}, "infoid", "")
+				if s != nil {
+					isok2++
+					ul.Mgo.DeleteById("bidding_field_0101", tmpid)
+					log.Debug(tmpid)
+				}
+			}
+		}(tmp)
+		tmp = make(map[string]interface{})
+	}
+
+	log.Debug("is over ", total, "~", isok1, "~", isok2)
+}

fieldproject_medical/data_service/src/main.go

@@ -2,23 +2,23 @@ package main
 
 import (
 	"bidding"
-	log "github.com/donnie4w/go-logger/logger"
-	"net/http"
-	"service"
-	"time"
+	"udp"
 	ul "util"
 )
 
 func init() {
+	udp.InitUdpMsg()
 	ul.InitClass()
-	service.InitService()
 }
 
 func main() {
-	log.Debug("run main ... ")
-	bidding.RunPurchasingInfo("100000000000000000000000", "900000000000000000000000")
-	return
-	//测试相似度计算
-	http.ListenAndServe(":9991", nil)
-	time.Sleep(999 * time.Hour)
+	//Test()
+	lock := make(chan bool)
+	<-lock
+}
+
+func Test() {
+	gtid := "630e34000000000000000000"
+	lteid := "6314cb800000000000000000"
+	bidding.RunPurchasingInfo(gtid, lteid, true)
 }

fieldproject_medical/data_service/src/mark

@@ -1,23 +1,135 @@
+8月31日 ~ id
+630e34000000000000000000
+9月05日 ~ id
+6314cb800000000000000000
+
+
+
+
+
+
+
+
+
 for k, v := range Medical_Class_Code {
-		key := ""
-		if utf8.RuneCountInString(k) == 2 {
-			key = v + "_" + "_"
-		} else if utf8.RuneCountInString(k) == 4 {
-			code_1 := k[:2]
-			name_1 := Medical_Class_Code[code_1]
-			key = name_1 + "_" + v + "_"
-		} else if utf8.RuneCountInString(k) == 7 {
-			code_1 := k[:2]
-			name_1 := Medical_Class_Code[code_1]
-			code_2 := k[:4]
-			name_2 := Medical_Class_Code[code_2]
-			key = name_1 + "_" + name_2 + "_" + v
-		} else if utf8.RuneCountInString(k) == 11 {
-			key = "四级" + "_" + v
+	key := ""
+    if utf8.RuneCountInString(k) == 2 {
+    	key = v + "_" + "_"
+    } else if utf8.RuneCountInString(k) == 4 {
+    	code_1 := k[:2]
+    	name_1 := Medical_Class_Code[code_1]
+    	key = name_1 + "_" + v + "_"
+    } else if utf8.RuneCountInString(k) == 7 {
+  	    code_1 := k[:2]
+   		name_1 := Medical_Class_Code[code_1]
+   		code_2 := k[:4]
+   		name_2 := Medical_Class_Code[code_2]
+    	key = name_1 + "_" + name_2 + "_" + v
+    } else if utf8.RuneCountInString(k) == 11 {
+    	key = "四级" + "_" + v
+    }
+    if key != "" {
+    	if Medical_Class_Name[key] == "" {
+    		Medical_Class_Name[key] = k
+    	}
+    }
+}
+
+
+/*  3000万一段
+    130e34000000000000000000
+    5dd35291a5cb26b9b787710b
+
+    5dd35291a5cb26b9b787710b
+    5ff7d667f0f9d716c18babff
+
+    5ff7d667f0f9d716c18babff
+    616cf96744ff2888b4a0c71b
+
+    616cf96744ff2888b4a0c71b
+    630e34000000000000000000
+*/
+
+
+/*
+    5a865ba4ddb36226a576530f  //少一条
+	5fbe3b91f0f9d716c16205eb
+
+    5fbe3b91f0f9d716c16205eb
+    630e33f9555d34cf90e0c283
+*/
+
+
+/*
+    5a868de3ddb36226a5764a75
+	5fc49168f0f9d716c165a22c
+
+    5fc49168f0f9d716c165a22c
+    630e30ad555d34cf90e0aed1
+*/
+
+
+
+
+func taskMedicalData() {
+	pool := make(chan bool, 10) //控制线程数
+	wg := &sync.WaitGroup{}
+
+	finalId := 0
+	lastInfo := MysqlM.SelectBySql(fmt.Sprintf("SELECT * FROM %s where mark_id = 1 ORDER BY id DESC LIMIT 1", "institution_baseinfo"))
+	if len(*lastInfo) > 0 {
+		finalId = util.IntAll((*lastInfo)[0]["id"])
+	}
+	log.Info("查询最后id---", zap.Int("finally id: ", finalId))
+	lastid, count := 0, 0
+	for {
+		log.Info("重新查询，lastid---", zap.Int("lastid: ", lastid))
+		q := fmt.Sprintf("SELECT id, company_id FROM %s WHERE id > %d AND mark_id = 1 ORDER BY id ASC limit 1000000", "institution_baseinfo", lastid)
+		rows, err := MysqlM.DB.Query(q)
+		if err != nil {
+			log.Error("mysql query err ", zap.Error(err))
 		}
-		if key != "" {
-			if Medical_Class_Name[key] == "" {
-				Medical_Class_Name[key] = k
+		columns, err := rows.Columns()
+		if finalId == lastid {
+			log.Info("----finish-----", zap.Int("count: ", count))
+			break
+		}
+		for rows.Next() {
+			scanArgs := make([]interface{}, len(columns))
+			values := make([]interface{}, len(columns))
+			ret := make(map[string]interface{})
+			for k := range values {
+				scanArgs[k] = &values[k]
+			}
+			err = rows.Scan(scanArgs...)
+			if err != nil {
+				log.Error("mysql scan err ", zap.Error(err))
+				break
+			}
+			for i, col := range values {
+				if v, ok := col.([]uint8); ok {
+					ret[columns[i]] = string(v)
+				} else {
+					ret[columns[i]] = col
+				}
+			}
+			lastid = util.IntAll(ret["id"])
+			count++
+			if count%2000 == 0 {
+				log.Info("current----", zap.Int("count: ", count), zap.Int("lastid: ", lastid))
 			}
+			pool <- true
+			wg.Add(1)
+			go func(tmp map[string]interface{}) {
+				defer func() {
+					<-pool
+					wg.Done()
+				}()
+				taskB(util.ObjToString(tmp["company_id"]))
+			}(ret)
+			ret = make(map[string]interface{})
 		}
-	}
+		_ = rows.Close()
+		wg.Wait()
+	}
+}

fieldproject_medical/data_service/src/service/service.go

@@ -11,11 +11,11 @@ var datalock sync.Mutex
 
 func InitService() {
 
-	http.HandleFunc("/getbid/tags", func(w http.ResponseWriter, r *http.Request) {
+	http.HandleFunc("/getbid/fieldtags", func(w http.ResponseWriter, r *http.Request) {
 		gtid := r.FormValue("gtid")
 		lteid := r.FormValue("lteid")
-		bidding.RunPurchasingInfo(gtid, lteid)
-		res, _ := json.Marshal("xxx ok")
+		bidding.RunPurchasingInfo(gtid, lteid, true)
+		res, _ := json.Marshal("ok")
 		w.Write(res)
 	})
 

fieldproject_medical/data_service/src/udp/udp.go

@@ -0,0 +1,47 @@
+package udp
+
+import (
+	"bidding"
+	"encoding/json"
+	log "github.com/donnie4w/go-logger/logger"
+	mu "mfw/util"
+	"net"
+	qu "qfw/util"
+)
+
+var (
+	udpclient mu.UdpClient
+)
+
+func InitUdpMsg() {
+	port := ":5550"
+	udpclient = mu.UdpClient{Local: ":5550", BufSize: 1024}
+	udpclient.Listen(processUdpMsg)
+	log.Debug("监听~", port)
+}
+
+//udp接收
+func processUdpMsg(act byte, data []byte, ra *net.UDPAddr) {
+	switch act {
+	case mu.OP_TYPE_DATA:
+		var mapInfo map[string]interface{}
+		err := json.Unmarshal(data, &mapInfo)
+		if err != nil {
+			udpclient.WriteUdp([]byte("err:"+err.Error()), mu.OP_NOOP, ra)
+		} else if mapInfo != nil {
+			sid, eid := qu.ObjToString(mapInfo["gtid"]), qu.ObjToString(mapInfo["lteid"])
+			if sid == "" || eid == "" {
+				log.Debug("异常~", sid, "~", eid)
+			} else {
+				key := sid + "-" + eid + "-" + qu.ObjToString(mapInfo["stype"])
+				udpclient.WriteUdp([]byte(key), mu.OP_NOOP, ra)
+				bidding.RunPurchasingInfo(sid, eid, true)
+			}
+		}
+	case mu.OP_NOOP: //下个节点回应
+		ok := string(data)
+		if ok != "" {
+			log.Debug("ok:", ok)
+		}
+	}
+}

fieldproject_medical/data_service/src/util/initcfg.go

@@ -16,7 +16,9 @@ const (
 	V_Code_Productclass  = "code_productclass"
 	V_Product_Baseinfo   = "product_baseinfo"
 
-	S_Bidding_Coll = "zktest_mysql_bidding_test"
+	S_Bidding_Coll = "bidding"
+
+	U_Bidding_Coll = "bidding"
 )
 
 var (
@@ -35,11 +37,10 @@ var (
 	//编号~对应具体分类详细
 	CodeCatalog = map[string]map[string]string{}
 
-	isLocal bool
+	IsLocal bool
 )
 
 func InitClass() {
-	isLocal = true //本地
 	initMgo()
 	initMysql()
 	initVCode()
@@ -47,20 +48,19 @@ func InitClass() {
 
 //初始化mgo
 func initMgo() {
-	if isLocal {
+	if IsLocal {
 		//Mgo = &MongodbSim{
-		//	MongodbAddr: "127.0.0.1:27017",
-		//	DbName:      "zhengkun",
+		//	MongodbAddr: "192.168.3.207:27001",
+		//	DbName:      "qfw_data",
 		//	Size:        10,
-		//	UserName:    "",
-		//	Password:    "",
+		//	UserName:    "root",
+		//	Password:    "root",
 		//}
 		//Mgo.InitPool()
-
 		Mgo = &MongodbSim{
-			MongodbAddr: "192.168.3.207:27092",
+			MongodbAddr: "127.0.0.1:27017",
 			DbName:      "zhengkun",
-			Size:        10,
+			Size:        30,
 			UserName:    "",
 			Password:    "",
 		}
@@ -69,7 +69,7 @@ func initMgo() {
 		Mgo = &MongodbSim{
 			MongodbAddr: "172.17.145.163:27083,172.17.4.187:27082",
 			DbName:      "qfw",
-			Size:        10,
+			Size:        30,
 			UserName:    "zhengkun",
 			Password:    "zk@123123",
 		}
@@ -78,29 +78,34 @@ func initMgo() {
 }
 
 func initMysql() {
+	username, password := "root", "=PDT49#80Z!RVv52_z"
+	address := "192.168.3.217:4000"
+	if !IsLocal {
+		username = "zhengkun"
+		password = "Zk#20220824"
+		address = "172.17.4.242:4000"
+	}
 	MysqlMedicalTool = &Mysql{
-		Address:  "192.168.3.217:4000",
-		UserName: "root",
-		PassWord: "=PDT49#80Z!RVv52_z",
-		DBName:   "medical_fileld_data",
+		Address:  address,
+		UserName: username,
+		PassWord: password,
+		DBName:   "medical_field_data",
 	}
 	MysqlMedicalTool.Init()
 
 	MysqlGlobalTool = &Mysql{
-		Address:  "192.168.3.217:4000",
-		UserName: "root",
-		PassWord: "=PDT49#80Z!RVv52_z",
+		Address:  address,
+		UserName: username,
+		PassWord: password,
 		DBName:   "global_common_data",
 	}
 	MysqlGlobalTool.Init()
-
 }
 
 //加载代码表~
 func initVCode() {
 	initBidInfoClass()     //招标信息分类
 	initBidIndustryClass() //行业分类
-
 	GSE.LoadDict("./dictionary.txt")
 	initMedicalLevelClass() //标准分类表
 	initMedicalProduct()    //分类产品

fieldproject_medical/data_service/src/util/initmed.go

@@ -32,6 +32,9 @@ func loadNgrmProductData(datasArr []map[string]interface{}) {
 	for k, data := range datasArr {
 		ngrm_index := cur_index + k
 		name := qu.ObjToString(data["product_name"])
+		if name == "其他" {
+			continue
+		}
 		code := qu.ObjToString(data["medical_equipment_code"])
 		arr := GSE.Cut(name, true)
 		NgrmDocIndex[ngrm_index] = arr
@@ -59,6 +62,9 @@ func loadNgrmProductData(datasArr []map[string]interface{}) {
 func loadNgrmCatalogData(datasArr []map[string]interface{}) {
 	for ngrm_index, data := range datasArr {
 		name := qu.ObjToString(data["name"])
+		if name == "其他" {
+			continue
+		}
 		code := qu.ObjToString(data["code"])
 		arr := GSE.Cut(name, true)
 		NgrmDocIndex[ngrm_index] = arr