项目

tmp_project/main.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"regexp"
+	"sync"
 	"time"
 
 	"context"
@@ -16,11 +18,12 @@ import (
 )
 
 var (
-	Mgo     *mongodb.MongodbSim
-	Bidding *mongodb.MongodbSim
-	Es      elastic.Es
-	cfg     = new(Config)
-	SEXhs   = common.SimpleEncrypt{Key: "topJYBX2019"}
+	Mgo       *mongodb.MongodbSim
+	Bidding   *mongodb.MongodbSim
+	Es        elastic.Es
+	cfg       = new(Config)
+	SEXhs     = common.SimpleEncrypt{Key: "topJYBX2019"}
+	ClearHtml = regexp.MustCompile("<[^>]*>")
 )
 
 func init() {
@@ -144,13 +147,14 @@ func FindData(data map[string]map[string]interface{}) int {
 	defer func() {
 		Mgo.DestoryMongoConn(session)
 	}()
+	wg := &sync.WaitGroup{}
+	ch := make(chan bool, 10)
 	iter := session.DB(cfg.Db.DbName).C("usermail").Find(&query).Sort("_id").Iter()
 	thisData := map[string]interface{}{}
 	for {
 		if !iter.Next(&thisData) {
 			break
 		}
-		//
 		id := mongodb.BsonIdToSId(thisData["_id"])
 		info_id := mongodb.BsonIdToSId(thisData["id"])
 		appid := mongodb.BsonIdToSId(thisData["appid"])
@@ -158,53 +162,65 @@ func FindData(data map[string]map[string]interface{}) int {
 		if projectId == "" {
 			querystr := `{"query": {"bool": {"must": [{"term": {"projectset.ids": "%s"}}],"must_not": [],"should": []}}}`
 			querystrs := fmt.Sprintf(querystr, id)
-			data := Es.Get("projectset", "projectset", querystrs)
-			if data != nil && *data != nil && len(*data) > 0 {
-				projectId = common.ObjToString((*data)[0]["_id"])
+			datas := Es.Get("projectset", "projectset", querystrs)
+			if datas != nil && *datas != nil && len(*datas) > 0 {
+				projectId = common.ObjToString((*datas)[0]["_id"])
 				Mgo.UpdateById("usermail", id, map[string]interface{}{"$set": map[string]interface{}{"projectId": projectId}})
 			}
 		}
 		if data[projectId] != nil {
 			projectData := data[projectId]
-			if projectDataInList, ok := projectData["list"].([]interface{}); ok {
-				for _, v := range projectDataInList {
-					if v_map, oks := v.(map[string]interface{}); oks {
-						infoid := common.ObjToString(v_map["infoid"])
-						topType := common.ObjToString(v_map["toptype"])
-						if topType == cfg.Rule {
-							log.Println("匹配到项目结果---", id, "-", projectId)
-							count++
-							esData := Es.GetByIdField("bidding", "bidding", infoid, "")
-							if esData != nil && len(*esData) > 0 {
-								(*esData)["projectId"] = projectId
-								(*esData)["sourceId"] = info_id
-								(*esData)["id"] = infoid
-								(*esData)["appid"] = appid
-								(*esData)["createtime"] = time.Now().Unix()
-								(*esData)["details"] = getDetails(infoid)
-								if Mgo.Count(cfg.Db.ColName, map[string]interface{}{"id": infoid}) < 1 {
-									mgoId := Mgo.Save(cfg.Db.ColName, *esData)
-									if mgoId != "" {
-										// delok := Mgo.Del(cfg.Db.TemporaryColName, map[string]interface{}{"_id": thisData["_id"]})
-										// if delok {
-										// 	log.Println("新华三定时数据删除成功---", id, "-", projectId, "-", mgoId)
-										// } else {
-										// 	log.Println("新华三定时数据删除失败！！！", id, "-", projectId, "-", mgoId)
-										// }
-										log.Println("保存到项目接口成功---", id, "-", projectId, "-", mgoId)
-									} else {
-										log.Println("保存到项目接口失败！！！", id, "-", projectId)
+			wg.Add(1)
+			ch <- true
+			go func(thisData, projectData map[string]interface{}) {
+				defer func() {
+					<-ch
+					wg.Done()
+				}()
+				if projectDataInList, ok := projectData["list"].([]interface{}); ok {
+					for _, v := range projectDataInList {
+						if v_map, oks := v.(map[string]interface{}); oks {
+							infoid := common.ObjToString(v_map["infoid"])
+							topType := common.ObjToString(v_map["toptype"])
+							if topType == cfg.Rule {
+								log.Println("匹配到项目结果---", id, "-", projectId)
+								count++
+								// esData := Es.GetByIdField("bidding", "bidding", infoid, "")
+								esData, _ := Bidding.FindOne("bidding", map[string]interface{}{"_id": mongodb.StringTOBsonId(infoid)})
+								if esData != nil && len(*esData) > 0 {
+									(*esData)["projectId"] = projectId
+									(*esData)["sourceId"] = info_id
+									(*esData)["id"] = infoid
+									(*esData)["appid"] = appid
+									(*esData)["createtime"] = time.Now().Unix()
+									details := common.ObjToString((*esData)["detail"])
+									(*esData)["details"] = details
+									(*esData)["detail"] = ClearHtml.ReplaceAllString(details, "")
+									if Mgo.Count(cfg.Db.ColName, map[string]interface{}{"id": infoid}) < 1 {
+										mgoId := Mgo.Save(cfg.Db.ColName, *esData)
+										if mgoId != "" {
+											// delok := Mgo.Del(cfg.Db.TemporaryColName, map[string]interface{}{"_id": thisData["_id"]})
+											// if delok {
+											// 	log.Println("新华三定时数据删除成功---", id, "-", projectId, "-", mgoId)
+											// } else {
+											// 	log.Println("新华三定时数据删除失败！！！", id, "-", projectId, "-", mgoId)
+											// }
+											log.Println("保存到项目接口成功---", id, "-", projectId, "-", mgoId)
+										} else {
+											log.Println("保存到项目接口失败！！！", id, "-", projectId)
+										}
 									}
 								}
 							}
 						}
 					}
 				}
-			}
+			}(thisData, projectData)
 		}
 		//
 		thisData = map[string]interface{}{}
 	}
+	wg.Wait()
 	return count
 }