xiugai

service/clue.go

@@ -45,90 +45,107 @@ func ClueImportTtSync(this *biservice.ClueImportReq) (string, int) {
 		result, status, counts := "导入成功", 1, 0
 		data := BiService.Find("customer_data_temp", map[string]interface{}{"import_pc": this.Pcbh}, "", "", -1, -1)
 		if data != nil && len(*data) > 0 {
+			wg := new(sync.WaitGroup)
+			ch := make(chan bool, 20)
 			for _, v := range *data {
-				v["isCompetitors"] = "否"
-				details := ""
-				id := common.ObjToString(v["id"])
-				href := common.ObjToString(v["href"])
-
-				for _, vv := range strings.Split(competitors, ",") {
-					if strings.Contains(href, vv) {
-						v["isCompetitors"] = "是"
-					}
-				}
-				customer_data := BiService.FindOne("customer_data_ttzl", map[string]interface{}{"id": id}, "", "")
-				if customer_data != nil {
-					log.Println("数据重复", id)
-					continue
-				}
-				delete(v, "import_pc")
-				dataId := BiService.Insert("customer_data_ttzl", v)
-				if dataId > 0 {
-					//
-					biddingData, ok := Bidding.FindOne("bidding", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
-					if ok && biddingData != nil && len(*biddingData) > 0 {
-						details = common.ObjToString((*biddingData)["detail"])
-					}
-					//
-					BiService.Insert("customer_data_ttzl_gl", map[string]interface{}{"msg_id": dataId, "info_id": id, "details": details, "isHistory": "否"})
-					delete(v, "isCompetitors")
-					delete(v, "msg_id")
-					v["_id"] = v["id"]
-					delete(v, "id")
-					v["details"] = cleanHTML(details)
-					if common.ObjToString(v["publishtime"]) != "" {
-						v["publishtime2"] = common.ObjToString(v["publishtime"])
-						publishtime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["publishtime"]), time.Local)
-						v["publishtime"] = publishtime.Unix()
-					} else {
-						delete(v, "publishtime")
-					}
-					// if common.ObjToString(v["docstarttime"]) != "" {
-					// 	docstarttime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["docstarttime"]), time.Local)
-					// 	v["docstarttime"] = docstarttime.Unix()
-					// } else {
-					// 	delete(v, "docstarttime")
-					// }
-					// if common.ObjToString(v["docendtime"]) != "" {
-					// 	docendtime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["docendtime"]), time.Local)
-					// 	v["docendtime"] = docendtime.Unix()
-					// } else {
-					// 	delete(v, "docendtime")
-					// }
-					// if common.ObjToString(v["bidstarttime"]) != "" {
-					// 	bidstarttime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["bidstarttime"]), time.Local)
-					// 	v["bidstarttime"] = bidstarttime.Unix()
-					// } else {
-					// 	delete(v, "bidstarttime")
-					// }
-					// if common.ObjToString(v["bidendtime"]) != "" {
-					// 	bidendtime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["bidendtime"]), time.Local)
-					// 	v["bidendtime"] = bidendtime.Unix()
-					// } else {
-					// 	delete(v, "bidendtime")
-					// }
-					// if common.ObjToString(v["bidopentime"]) != "" {
-					// 	bidopentime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["bidopentime"]), time.Local)
-					// 	v["bidopentime"] = bidopentime.Unix()
-					// } else {
-					// 	delete(v, "bidopentime")
-					// }
-					for k, vv := range v {
-						if vv == nil {
-							delete(v, k)
+				wg.Add(1)
+				ch <- true
+				go func(v map[string]interface{}) {
+					defer func() {
+						wg.Done()
+						<-ch
+					}()
+					v["isCompetitors"] = "否"
+					details := ""
+					id := common.ObjToString(v["id"])
+					href := common.ObjToString(v["href"])
+					for _, vv := range strings.Split(competitors, ",") {
+						if strings.Contains(href, vv) {
+							v["isCompetitors"] = "是"
 						}
 					}
-					oks := Es.Save("ttbid", "ttbid", v)
-					if oks {
-						counts++
-						log.Println("es存入成功", id)
+					delete(v, "import_pc")
+					delete(v, "projectId")
+					customer_data := BiService.FindOne("customer_data_ttzl", map[string]interface{}{"id": id}, "", "")
+					if customer_data != nil {
+						log.Println("数据重复", id)
+						msg_id := common.Int64All((*customer_data)["msg_id"])
+						bok := BiService.Update("customer_data_ttzl", map[string]interface{}{"msg_id": msg_id}, v)
+						if bok {
+							//
+							biddingData, ok := Bidding.FindOne("bidding", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
+							if ok && biddingData != nil && len(*biddingData) > 0 {
+								details = common.ObjToString((*biddingData)["detail"])
+							}
+							//
+							delete(v, "isCompetitors")
+							delete(v, "msg_id")
+							v["_id"] = v["id"]
+							delete(v, "id")
+							v["details"] = cleanHTML(details)
+							if common.ObjToString(v["publishtime"]) != "" {
+								v["publishtime2"] = common.ObjToString(v["publishtime"])
+								publishtime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["publishtime"]), time.Local)
+								v["publishtime"] = publishtime.Unix()
+							} else {
+								delete(v, "publishtime")
+							}
+							for k, vv := range v {
+								if vv == nil {
+									delete(v, k)
+								}
+							}
+							oks := Es.UpdateNewDoc("ttbid", "ttbid", v)
+							if oks {
+								counts++
+								log.Println("es更新成功", id)
+							} else {
+								log.Println("es更新失败！！", id)
+							}
+						} else {
+							log.Println("tidb更新失败！！", id)
+						}
 					} else {
-						log.Println("es存入失败！！", id)
+						dataId := BiService.Insert("customer_data_ttzl", v)
+						if dataId > 0 {
+							//
+							biddingData, ok := Bidding.FindOne("bidding", map[string]interface{}{"_id": mongodb.StringTOBsonId(id)})
+							if ok && biddingData != nil && len(*biddingData) > 0 {
+								details = common.ObjToString((*biddingData)["detail"])
+							}
+							//
+							BiService.Insert("customer_data_ttzl_gl", map[string]interface{}{"msg_id": dataId, "info_id": id, "details": details, "isHistory": "否"})
+							delete(v, "isCompetitors")
+							delete(v, "msg_id")
+							v["_id"] = v["id"]
+							delete(v, "id")
+							v["details"] = cleanHTML(details)
+							if common.ObjToString(v["publishtime"]) != "" {
+								v["publishtime2"] = common.ObjToString(v["publishtime"])
+								publishtime, _ := time.ParseInLocation(date.Date_Full_Layout, common.ObjToString(v["publishtime"]), time.Local)
+								v["publishtime"] = publishtime.Unix()
+							} else {
+								delete(v, "publishtime")
+							}
+							for k, vv := range v {
+								if vv == nil {
+									delete(v, k)
+								}
+							}
+							oks := Es.Save("ttbid", "ttbid", v)
+							if oks {
+								counts++
+								log.Println("es存入成功", id)
+							} else {
+								log.Println("es存入失败！！", id)
+							}
+						} else {
+							log.Println("tidb存入失败！！", id)
+						}
 					}
-				} else {
-					log.Println("tidb存入失败！！", id)
-				}
+				}(v)
 			}
+			wg.Wait()
 		}
 		BiService.Update("customer_data_import_record", map[string]interface{}{"import_pc": this.Pcbh}, map[string]interface{}{
 			"import_end_time": time.Now().Format(date.Date_Full_Layout),