调整~ 优化 ~ 减少tidb 查询次数 ， 本地构建关联数据

fieldproject_medical/data_preparation/src/relation_add/relation_add.go

@@ -8,28 +8,14 @@ import (
 )
 
 var (
-	up_project_ids       []string
-	up_project_infos     map[string][]map[string]interface{}
 	up_winner_ids        []string
 	up_winner_infos      map[string][]map[string]interface{}
-	up_p_w_infos         map[string][]map[string]interface{}
-	up_bw_infos          map[string]map[string]string
-	up_bw_index          []string
+	up_project_ids       []string
+	up_project_infos     map[string][]map[string]interface{}
 	up_med_level         map[string]string
-	up_equipment_departs map[string][]string
+	un_equipment_departs map[string][]string
 )
 
-func initResetVar() {
-	up_project_ids = []string{}
-	up_project_infos = map[string][]map[string]interface{}{}
-	up_winner_ids = []string{}
-	up_winner_infos = map[string][]map[string]interface{}{}
-	up_p_w_infos = map[string][]map[string]interface{}{}
-	up_bw_infos, up_bw_index = map[string]map[string]string{}, []string{}
-	up_med_level = map[string]string{}
-	up_equipment_departs = map[string][]string{}
-}
-
 //科室设备对应代码
 func initEquipmentDepartCode() {
 	datas := *class.MysqlMedicalDevTool.Find("dwd_f_yl_sddepart_equip_proclass", nil,
@@ -37,12 +23,20 @@ func initEquipmentDepartCode() {
 	for _, v := range datas {
 		sddepartname_code := qu.ObjToString(v["sddepartname_code"])
 		equipment := qu.ObjToString(v["equipment"])
-		codes := up_equipment_departs[equipment]
+		codes := un_equipment_departs[equipment]
 		codes = append(codes, sddepartname_code)
-		up_equipment_departs[equipment] = codes
+		un_equipment_departs[equipment] = codes
 
 	}
-	log.Debug("设备对应的科室代码~", len(up_equipment_departs))
+	log.Debug("设备对应的科室代码~", len(un_equipment_departs))
+}
+func initResetVar() {
+	up_winner_ids = []string{}
+	up_winner_infos = map[string][]map[string]interface{}{}
+	up_project_ids = []string{}
+	up_project_infos = map[string][]map[string]interface{}{}
+	up_med_level = map[string]string{}
+	un_equipment_departs = map[string][]string{}
 }
 
 //增量统计更新关系表
@@ -50,21 +44,18 @@ func RelationIncreaseInfo() {
 	initResetVar()
 	initEquipmentDepartCode()
 	//取项目相关信息
-	findUpdateProjectIds()
-	findUpdateProjectInfos()
-	log.Debug("项目标识~", len(up_project_ids), "~项目信息~", len(up_project_infos))
-	//取中标相关信息
-	findUpdateWinnerIds()
-	findUpdateWinnerInfos()
-	log.Debug("中标标识~", len(up_winner_ids), "~中标信息~", len(up_winner_infos))
-	log.Debug("中标标识对应项目信息~", len(up_p_w_infos), "~等级信息~", len(up_med_level))
-
-	//取唯一性数据
-	findBuyerAndWinnerUniqueness()
+	project_ids := findUpdateProjectIds()
+	project_infos := findUpdateProjectInfos(project_ids)
+	findUpdateWinnerIds(project_ids)
+	p_win_infos := findUpdateWinnerInfos()
+	log.Debug("关联中标标识~", len(up_winner_ids), "~信息~", len(up_winner_infos))
+	up_bw_index, up_bw_infos := findBuyerAndWinnerUniqueness(project_ids, project_infos, p_win_infos)
 	log.Debug("唯一性构建完毕~", len(up_bw_index))
+	findUpdateAllProjectInfos()
+	log.Debug("关联项目标识~", len(up_project_ids), "~信息~", len(up_project_infos))
 
 	//更新操作buyer_winner_relation
-	updateBuyerWinnerInfoRelation()
+	updateBuyerWinnerInfoRelation(up_bw_index, up_bw_infos)
 	//更新操作winner_relation
 	updateWinnerInfoRelation()
 
@@ -73,7 +64,8 @@ func RelationIncreaseInfo() {
 }
 
 //找项目标识
-func findUpdateProjectIds() {
+func findUpdateProjectIds() []string {
+	up_ids := []string{}
 	query := map[string]interface{}{
 		"updatetime": map[string]string{
 			"gte": "2022-11-08 15:02:28",
@@ -94,14 +86,16 @@ func findUpdateProjectIds() {
 		projectid := qu.ObjToString(v["projectid"])
 		if unique_id[projectid] == "" && projectid != "" {
 			unique_id[projectid] = projectid
-			up_project_ids = append(up_project_ids, projectid)
+			up_ids = append(up_ids, projectid)
 		}
 	}
+	return up_ids
 }
 
 //找项目信息
-func findUpdateProjectInfos() {
-	for _, p_id := range up_project_ids {
+func findUpdateProjectInfos(up_ids []string) map[string][]map[string]interface{} {
+	up_infos := map[string][]map[string]interface{}{}
+	for _, p_id := range up_ids {
 		datas := *class.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_baseinfo", map[string]interface{}{
 			"projectid": p_id,
 		}, "buyer,buyer_id,mi_level_code,jgtime", "id", -1, -1)
@@ -115,11 +109,11 @@ func findUpdateProjectInfos() {
 				continue
 			}
 			infoArr := []map[string]interface{}{}
-			if up_project_infos[p_id] != nil {
-				infoArr = up_project_infos[p_id]
+			if up_infos[p_id] != nil {
+				infoArr = up_infos[p_id]
 			}
 			infoArr = append(infoArr, v)
-			up_project_infos[p_id] = infoArr
+			up_infos[p_id] = infoArr
 			buyer_id := qu.ObjToString(v["buyer_id"])
 			mi_level_code := qu.ObjToString(v["mi_level_code"])
 			if buyer_id != "" {
@@ -127,11 +121,12 @@ func findUpdateProjectInfos() {
 			}
 		}
 	}
+	return up_infos
 }
 
 //找中标标识
-func findUpdateWinnerIds() {
-	for _, p_id := range up_project_ids {
+func findUpdateWinnerIds(up_ids []string) {
+	for _, p_id := range up_ids {
 		//找所有项目下的中标单位
 		datas := *class.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_win_baseinfo ", map[string]interface{}{
 			"projectid": p_id,
@@ -156,7 +151,8 @@ func findUpdateWinnerIds() {
 }
 
 //找中标标识,中标信息,对应项目信息
-func findUpdateWinnerInfos() {
+func findUpdateWinnerInfos() map[string][]map[string]interface{} {
+	p_win_infos := map[string][]map[string]interface{}{}
 	for _, w_id := range up_winner_ids {
 		//中标标识~对应所有的中标单位
 		datas := *class.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_win_baseinfo ", map[string]interface{}{
@@ -184,26 +180,31 @@ func findUpdateWinnerInfos() {
 				up_winner_infos[winner_id] = w_infos
 
 				//项目标识~对应的中标信息
-				p_infos := []map[string]interface{}{}
-				if up_p_w_infos[projectid] != nil {
-					p_infos = up_p_w_infos[projectid]
+				if projectid != "" {
+					p_infos := []map[string]interface{}{}
+					if p_win_infos[projectid] != nil {
+						p_infos = p_win_infos[projectid]
+					}
+					p_infos = append(p_infos, v)
+					p_win_infos[projectid] = p_infos
 				}
-				p_infos = append(p_infos, v)
-				up_p_w_infos[projectid] = p_infos
 			}
 		}
 	}
+	return p_win_infos
 }
 
 //找唯一性信息
-func findBuyerAndWinnerUniqueness() {
-	for _, v := range up_project_ids {
-		infos := up_project_infos[v]
+func findBuyerAndWinnerUniqueness(up_ids []string, up_infos map[string][]map[string]interface{}, p_win_infos map[string][]map[string]interface{}) ([]string, map[string]map[string]string) {
+	up_bw_index := []string{}
+	up_bw_infos := map[string]map[string]string{}
+	for _, v := range up_ids {
+		infos := up_infos[v]
 		buyer, buyer_id, b := findBuyerInfo(infos)
 		if !b {
 			continue
 		}
-		winners := up_p_w_infos[v]
+		winners := p_win_infos[v]
 		for _, win_value := range winners {
 			winner := qu.ObjToString(win_value["winner"])
 			winner_id := qu.ObjToString(win_value["winner_id"])
@@ -223,6 +224,7 @@ func findBuyerAndWinnerUniqueness() {
 			}
 		}
 	}
+	return up_bw_index, up_bw_infos
 }
 
 //找采购单位
@@ -241,3 +243,36 @@ func findBuyerInfo(infos []map[string]interface{}) (string, string, bool) {
 	}
 	return buyer, buyer_id, b
 }
+
+//找所有相关联~项目的信息
+func findUpdateAllProjectInfos() {
+	project_unique := map[string]string{}
+	for _, winner_id := range up_winner_ids {
+		win_infos := up_winner_infos[winner_id]
+		for _, v := range win_infos {
+			projectid := qu.ObjToString(v["projectid"])
+			if project_unique[projectid] == "" && projectid != "" {
+				up_project_ids = append(up_project_ids, projectid)
+				project_unique[projectid] = projectid
+			}
+		}
+	}
+	for _, p_id := range up_project_ids {
+		p_infos := *class.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_baseinfo", map[string]interface{}{
+			"projectid": p_id,
+		}, "buyer,buyer_id,projectid,itemname,sdequipment_code,mi_level_code,jgtime", "id", -1, -1)
+		new_p_infos := []map[string]interface{}{}
+		for _, p_v := range p_infos {
+			jgtime := qu.ObjToString(p_v["jgtime"])
+			if jgtime == "" {
+				continue
+			}
+			t, _ := time.ParseInLocation(class.TimeLayout, jgtime, time.Local)
+			if t.Unix() < int64(1514736000) {
+				continue
+			}
+			new_p_infos = append(new_p_infos, p_v)
+		}
+		up_project_infos[p_id] = new_p_infos
+	}
+}

fieldproject_medical/data_preparation/src/relation_add/relation_method.go

@@ -36,20 +36,10 @@ func updateLastTimeAndCooperatNum(buyer_id string, winner_id string) (string, in
 	cooperat_unique := map[string]string{}
 	if len(project_ids) > 0 {
 		for k, p_id := range project_ids {
-			p_infos := *cl.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_baseinfo", map[string]interface{}{
-				"projectid": p_id,
-			}, "buyer_id,jgtime", "id", -1, -1)
+			p_infos := up_project_infos[p_id]
 			if len(p_infos) > 0 {
 				new_buyer_id := ""
 				for _, p_v := range p_infos {
-					jgtime := qu.ObjToString(p_v["jgtime"])
-					if jgtime == "" {
-						continue
-					}
-					t, _ := time.ParseInLocation(cl.TimeLayout, jgtime, time.Local)
-					if t.Unix() < int64(1514736000) {
-						continue
-					}
 					if qu.ObjToString(p_v["buyer_id"]) != "" {
 						new_buyer_id = qu.ObjToString(p_v["buyer_id"])
 						break
@@ -77,7 +67,7 @@ func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string
 	for _, v := range win_infos {
 		is_winner := qu.IntAll(v["is_winner"])
 		projectid := qu.ObjToString(v["projectid"])
-		if is_winner == 1 {
+		if is_winner == 1 && projectid != "" {
 			project_ids = append(project_ids, projectid)
 		}
 	}
@@ -87,19 +77,9 @@ func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string
 	p_codes_unique := map[string]string{}
 	if len(project_ids) > 0 {
 		for _, p_id := range project_ids {
-			p_infos := *cl.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_baseinfo", map[string]interface{}{
-				"projectid": p_id,
-			}, "buyer_id,itemname,sdequipment_code,jgtime", "id", -1, -1)
+			p_infos := up_project_infos[p_id]
 			if len(p_infos) > 0 {
 				for _, p_v := range p_infos {
-					jgtime := qu.ObjToString(p_v["jgtime"])
-					if jgtime == "" {
-						continue
-					}
-					t, _ := time.ParseInLocation(cl.TimeLayout, jgtime, time.Local)
-					if t.Unix() < int64(1514736000) {
-						continue
-					}
 					new_buyer_id := qu.ObjToString(p_v["buyer_id"])
 					if new_buyer_id == buyer_id && new_buyer_id != "" {
 						itemname := qu.ObjToString(p_v["itemname"])
@@ -125,7 +105,7 @@ func updatePurchasingAllInfos(winner_id string, buyer_id string) (string, string
 	p_depart_unique := map[string]string{}
 	p_depart_codes := []string{}
 	for _, v := range p_names {
-		codes := up_equipment_departs[v]
+		codes := un_equipment_departs[v]
 		for _, v1 := range codes {
 			if v1 != "" && p_depart_unique[v1] == "" {
 				p_depart_unique[v1] = v1

fieldproject_medical/data_preparation/src/relation_add/relation_update.go

@@ -8,12 +8,12 @@ import (
 )
 
 //更新buyer_winner 关系数据
-func updateBuyerWinnerInfoRelation() {
-	for k, v := range up_bw_index {
-		if k%1000 == 0 {
+func updateBuyerWinnerInfoRelation(bw_index []string, bw_infos map[string]map[string]string) {
+	for k, v := range bw_index {
+		if k%10 == 0 {
 			log.Debug("cur index ", k)
 		}
-		tmp := up_bw_infos[v]
+		tmp := bw_infos[v]
 		buyer := qu.ObjToString(tmp["buyer"])
 		buyer_id := qu.ObjToString(tmp["buyer_id"])
 		winner := qu.ObjToString(tmp["winner"])
@@ -54,13 +54,13 @@ func updateBuyerWinnerInfoRelation() {
 		}
 
 	}
-	log.Debug("is b_w update over ", len(up_bw_index))
+	log.Debug("is b_w update over ", len(bw_index))
 }
 
 //更新winner 关系数据
 func updateWinnerInfoRelation() {
 	for k, w_id := range up_winner_ids {
-		if k%1000 == 0 {
+		if k%10 == 0 {
 			log.Debug("cur index ", k)
 		}
 		old_infos := up_winner_infos[w_id]
@@ -105,20 +105,10 @@ func updateWinnerInfoRelation() {
 					}
 					//取项目信息
 					projectid := qu.ObjToString(info["projectid"])
-					p_infos := *cl.MysqlMedicalDevTool.Find("dwd_f_yl_purchasing_baseinfo", map[string]interface{}{
-						"projectid": projectid,
-					}, "buyer_id,jgtime", "id", -1, -1)
+					p_infos := up_project_infos[projectid]
 					if len(p_infos) > 0 {
 						buyer_id := ""
 						for _, p_info := range p_infos {
-							jgtime := qu.ObjToString(info["jgtime"])
-							if jgtime == "" {
-								continue
-							}
-							t, _ := time.ParseInLocation(cl.TimeLayout, jgtime, time.Local)
-							if t.Unix() < int64(1514736000) {
-								continue
-							}
 							if qu.ObjToString(p_info["buyer_id"]) != "" {
 								buyer_id = qu.ObjToString(p_info["buyer_id"])
 								break