|
|
@@ -3,9 +3,13 @@ package main
|
|
|
import (
|
|
|
util "app.yhyue.com/data_processing/common_utils"
|
|
|
"app.yhyue.com/data_processing/common_utils/log"
|
|
|
+ "app.yhyue.com/data_processing/common_utils/mongodb"
|
|
|
+ "app.yhyue.com/data_processing/common_utils/redis"
|
|
|
"encoding/json"
|
|
|
"fmt"
|
|
|
"github.com/go-ego/gse"
|
|
|
+ "go.mongodb.org/mongo-driver/bson"
|
|
|
+ "go.uber.org/zap"
|
|
|
"io/ioutil"
|
|
|
"net/http"
|
|
|
"net/url"
|
|
|
@@ -35,8 +39,8 @@ var (
|
|
|
)
|
|
|
|
|
|
func initSeg() {
|
|
|
- //_ = seg.LoadDict("./t_1.txt")
|
|
|
- _ = seg.LoadDict()
|
|
|
+ _ = seg.LoadDict("./t_1.txt")
|
|
|
+ //_ = seg.LoadDict()
|
|
|
seg.AddToken("渼陂", 3, "")
|
|
|
seg.LoadStop("./stopwords.txt")
|
|
|
|
|
|
@@ -66,7 +70,7 @@ func taskC() {
|
|
|
"district": 1,
|
|
|
}
|
|
|
|
|
|
- query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.Serve.ProColl).Find(nil).Select(f).Iter()
|
|
|
+ query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
|
|
|
count := 0
|
|
|
for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
|
|
|
if count%2000 == 0 {
|
|
|
@@ -89,23 +93,26 @@ func taskC() {
|
|
|
var eArr []map[string]interface{}
|
|
|
n1, n2 := 0, 0
|
|
|
// approvecode、approvenumber
|
|
|
- q := Method2(util.ObjToString(tmp["approvecode"]), util.ObjToString(tmp["approvenumber"]))
|
|
|
- if q != "" {
|
|
|
- binfo := Es.Get("projectset", q)
|
|
|
- if binfo != nil && len(*binfo) > 0 {
|
|
|
- for _, m := range *binfo {
|
|
|
- n1 = len(*binfo)
|
|
|
- mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 1})
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
+ //q := Method2(util.ObjToString(tmp["approvecode"]), util.ObjToString(tmp["approvenumber"]))
|
|
|
+ //if q != "" {
|
|
|
+ // binfo := Es.Get("projectset", q)
|
|
|
+ // if binfo != nil && len(*binfo) > 0 {
|
|
|
+ // for _, m := range *binfo {
|
|
|
+ // n1 = len(*binfo)
|
|
|
+ // mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 1})
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
|
|
|
wds, q := Method1(util.ObjToString(tmp["projectname"]))
|
|
|
if q != "" {
|
|
|
- binfo := Es.Get("projectset", q)
|
|
|
+ binfo := Es.Get("projectset_v1", q)
|
|
|
if binfo != nil && len(*binfo) > 0 {
|
|
|
n2 = len(*binfo)
|
|
|
for _, m := range *binfo {
|
|
|
+ if b, _ := redis.Exists(config.Conf.DB.Redis.Pcode, util.ObjToString(m["_id"])); b {
|
|
|
+ continue
|
|
|
+ }
|
|
|
if util.ObjToString(m["bidstatus"]) == "拟建" {
|
|
|
eArr = append(eArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "bidstatus": m["bidstatus"]})
|
|
|
continue
|
|
|
@@ -114,7 +121,10 @@ func taskC() {
|
|
|
eArr = append(eArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "bidstatus": m["bidstatus"]})
|
|
|
continue
|
|
|
}
|
|
|
+
|
|
|
+ redis.PutCKV(config.Conf.DB.Redis.Pcode, util.ObjToString(m["_id"]), mongodb.BsonIdToSId(tmp["_id"]))
|
|
|
mArr = append(mArr, map[string]interface{}{"pid": util.ObjToString(m["_id"]), "projectname": m["projectname"], "source": 2})
|
|
|
+
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -128,6 +138,7 @@ func taskC() {
|
|
|
save["esearch"] = q
|
|
|
save["size_1"] = n1
|
|
|
save["size_2"] = n2
|
|
|
+ save["createtime"] = time.Now().Unix()
|
|
|
savePpPool <- save
|
|
|
//}
|
|
|
}(tmp)
|
|
|
@@ -379,7 +390,7 @@ func SavePpMethod() {
|
|
|
defer func() {
|
|
|
<-savePpSp
|
|
|
}()
|
|
|
- MgoPro.SaveBulk("projectset_comb", arru...)
|
|
|
+ MgoPro.SaveBulk(config.Conf.DB.MongoP.CombColl, arru...)
|
|
|
}(arru)
|
|
|
arru = make([]map[string]interface{}, saveSize)
|
|
|
indexu = 0
|
|
|
@@ -391,7 +402,375 @@ func SavePpMethod() {
|
|
|
defer func() {
|
|
|
<-savePpSp
|
|
|
}()
|
|
|
- MgoPro.SaveBulk("projectset_comb", arru...)
|
|
|
+ MgoPro.SaveBulk(config.Conf.DB.MongoP.CombColl, arru...)
|
|
|
+ }(arru[:indexu])
|
|
|
+ arru = make([]map[string]interface{}, saveSize)
|
|
|
+ indexu = 0
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+var StageCode []TagMatching
|
|
|
+
|
|
|
+func initStage() {
|
|
|
+ info, _ := MgoBid.Find(config.Conf.Serve.TagRule, bson.M{"label_name": "project_stage"}, `{"_id": 1}`, nil, false, -1, -1)
|
|
|
+ for _, m := range *info {
|
|
|
+ tag := TagMatching{}
|
|
|
+ tag.tagName = util.ObjToString(m["label_name"])
|
|
|
+ tag.tagCode = util.ObjToString(m["code"])
|
|
|
+ // 关键词
|
|
|
+ tag.matchField = []string{"title", "project"}
|
|
|
+ if v := util.ObjToString(m["keyword"]); v != "" {
|
|
|
+ tag.matchKey = util.ObjToString(m["keyword"])
|
|
|
+ tag.matchKeyReg = GetRegex(util.ObjToString(m["keyword"]))
|
|
|
+ }
|
|
|
+ // 附件词
|
|
|
+ if f := util.ObjToString(m["match_fjword"]); f != "" {
|
|
|
+ tag.addField = strings.Split(f, ",")
|
|
|
+ for _, s := range tag.addField {
|
|
|
+ SelectF[s] = 1
|
|
|
+ }
|
|
|
+ if v := util.ObjToString(m["fjword"]); v != "" {
|
|
|
+ tag.addKey = util.ObjToString(m["fjword"])
|
|
|
+ tag.addKeyReg = GetRegex(util.ObjToString(m["fjword"]))
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 排除词
|
|
|
+ if f := util.ObjToString(m["match_pcword"]); f != "" {
|
|
|
+ tag.excludeField = strings.Split(f, ",")
|
|
|
+ for _, s := range tag.excludeField {
|
|
|
+ SelectF[s] = 1
|
|
|
+ }
|
|
|
+ if v := util.ObjToString(m["pcword"]); v != "" {
|
|
|
+ tag.excludeKey = util.ObjToString(m["pcword"])
|
|
|
+ tag.excludeKeyReg = GetRegex(util.ObjToString(m["pcword"]))
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 清理词
|
|
|
+ if v := util.ObjToString(m["qlword"]); v != "" {
|
|
|
+ tag.clearKey = strings.Split(util.ObjToString(m["qlword"]), ",")
|
|
|
+ }
|
|
|
+ StageCode = append(StageCode, tag)
|
|
|
+ }
|
|
|
+ log.Info("initStage", zap.Int("StageCode", len(StageCode)))
|
|
|
+}
|
|
|
+
|
|
|
+func taskD() {
|
|
|
+ sess := MgoPro.GetMgoConn()
|
|
|
+ defer MgoPro.DestoryMongoConn(sess)
|
|
|
+
|
|
|
+ ch := make(chan bool, config.Conf.Serve.Thread)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+
|
|
|
+ //q := bson.M{"_id": mongodb.StringTOBsonId("60a2995b8a2adb30a57172ec")}
|
|
|
+ query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.CombColl).Find(nil).Iter()
|
|
|
+ count := 0
|
|
|
+ for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
|
|
|
+ if count%2000 == 0 {
|
|
|
+ log.Info(fmt.Sprintf("current --- %d", count))
|
|
|
+ }
|
|
|
+
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+
|
|
|
+ if ids, ok := tmp["ids"].([]interface{}); ok {
|
|
|
+ //id := mongodb.BsonIdToSId(tmp["_id"])
|
|
|
+
|
|
|
+ for _, p := range ids {
|
|
|
+ p1 := p.(map[string]interface{})
|
|
|
+ info, _ := MgoPro.FindById(config.Conf.DB.MongoP.ProjectColl, util.ObjToString(p1["pid"]), nil)
|
|
|
+ if list, ok1 := (*info)["list"].([]interface{}); ok1 {
|
|
|
+ for _, l := range list {
|
|
|
+ l1 := l.(map[string]interface{})
|
|
|
+ m := make(map[string]interface{})
|
|
|
+ m["project_stage_code"] = tagFunc(l1)
|
|
|
+ m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
|
|
|
+ m["title"] = util.ObjToString(l1["title"])
|
|
|
+ if t := util.Int64All(l1["publishtime"]); t > 0 {
|
|
|
+ m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
|
|
|
+ }
|
|
|
+ m["infoid"] = util.ObjToString(l1["infoid"])
|
|
|
+ m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
|
|
|
+ m["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ MgoPro.Save("projectset_comb_temp1", m)
|
|
|
+ //MysqlTool.Insert("dwd_f_nzj_follw_record", m)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
|
|
|
+ // s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
|
|
|
+ // if s <= 0 {
|
|
|
+ // saveEnt := make(map[string]interface{})
|
|
|
+ // saveEnt["proposed_id"] = id
|
|
|
+ // saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ // saveEnt["name"] = buyer
|
|
|
+ // if eid := redis.GetStr("ent_id", buyer); eid != "" {
|
|
|
+ // arr := strings.Split(eid, "_")
|
|
|
+ // saveEnt["name_id"] = arr[0]
|
|
|
+ // if len(arr) == 2 {
|
|
|
+ // saveEnt["area_code"] = arr[1]
|
|
|
+ // } else if len(arr) == 3 {
|
|
|
+ // saveEnt["city_code"] = arr[2]
|
|
|
+ // }
|
|
|
+ // info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
|
|
|
+ // if info != nil && len(*info) > 0 {
|
|
|
+ // saveEnt["address"] = (*info)[0]["address"]
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // saveEnt["identity_type"] = 2
|
|
|
+ // saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
|
|
|
+ // saveEntPool1 <- saveEnt
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ //if winner := util.ObjToString((*info)["buyer"]); winner != "" {
|
|
|
+ // for _, w := range strings.Split(winner, ",") {
|
|
|
+ // s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
|
|
|
+ // if s <= 0 {
|
|
|
+ // saveEnt := make(map[string]interface{})
|
|
|
+ // saveEnt["proposed_id"] = id
|
|
|
+ // saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ // saveEnt["name"] = w
|
|
|
+ // if eid := redis.GetStr("ent_id", w); eid != "" {
|
|
|
+ // arr := strings.Split(eid, "_")
|
|
|
+ // saveEnt["name_id"] = arr[0]
|
|
|
+ // if len(arr) == 2 {
|
|
|
+ // saveEnt["area_code"] = arr[1]
|
|
|
+ // } else if len(arr) == 3 {
|
|
|
+ // saveEnt["city_code"] = arr[2]
|
|
|
+ // }
|
|
|
+ // info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
|
|
|
+ // if info != nil && len(*info) > 0 {
|
|
|
+ // saveEnt["address"] = (*info)[0]["address"]
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // saveEnt["identity_type"] = 3
|
|
|
+ // saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
|
|
|
+ // saveEntPool1 <- saveEnt
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ }
|
|
|
+
|
|
|
+ //size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
|
|
|
+ //info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
|
|
|
+ //MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
|
|
|
+ }
|
|
|
+
|
|
|
+ }(tmp)
|
|
|
+ tmp = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ log.Info(fmt.Sprintf("over --- %d", count))
|
|
|
+}
|
|
|
+
|
|
|
+// @Description 施工准备(06)、施工(07)、设计(05)
|
|
|
+// @Author J 2023/4/21 14:45
|
|
|
+func tagFunc(info map[string]interface{}) string {
|
|
|
+ tag := taskFuc1(info)
|
|
|
+ if tag["project_stage"] != "" {
|
|
|
+ return util.ObjToString(tag["project_stage"])
|
|
|
+ }
|
|
|
+ if util.ObjToString(info["toptype"]) == "招标" || util.ObjToString(info["toptype"]) == "预告" {
|
|
|
+ return "06"
|
|
|
+ }
|
|
|
+ return "00"
|
|
|
+}
|
|
|
+
|
|
|
+// @Description 在建项目增量
|
|
|
+// @Author J 2023/4/24 13:58
|
|
|
+func taskAA() {
|
|
|
+ sess := MgoPro.GetMgoConn()
|
|
|
+ defer MgoPro.DestoryMongoConn(sess)
|
|
|
+
|
|
|
+ ch := make(chan bool, config.Conf.Serve.Thread)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+
|
|
|
+ q := bson.M{"pici": bson.M{"$gte": config.Conf.Serve.Pici}}
|
|
|
+ query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.ProjectColl).Find(q).Iter()
|
|
|
+ count := 0
|
|
|
+ for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
|
|
|
+ if count%2000 == 0 {
|
|
|
+ log.Info(fmt.Sprintf("current --- %d", count))
|
|
|
+ }
|
|
|
+ if pc := util.Int64All(tmp["pici"]); pc > config.Conf.Serve.Pici {
|
|
|
+ config.Conf.Serve.Pici = pc
|
|
|
+ }
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+ id := mongodb.BsonIdToSId(tmp["_id"])
|
|
|
+ if str := redis.GetStr(config.Conf.DB.Redis.Pcode, id); str != "" {
|
|
|
+ strs := strings.Split(str, "-")
|
|
|
+ if len(tmp["list"].([]interface{})) != util.IntAll(strs[1]) {
|
|
|
+ for _, info := range tmp["list"].([]interface{}) {
|
|
|
+ info1 := info.(map[string]interface{})
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ }(tmp)
|
|
|
+ tmp = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ log.Info(fmt.Sprintf("over --- %d, pici ---%d", count, config.Conf.Serve.Pici))
|
|
|
+}
|
|
|
+
|
|
|
+// @Description 拟建项目增量
|
|
|
+// @Author J 2023/4/24 13:59
|
|
|
+func taskBB() {
|
|
|
+ sess := MgoPro.GetMgoConn()
|
|
|
+ defer MgoPro.DestoryMongoConn(sess)
|
|
|
+
|
|
|
+ ch := make(chan bool, config.Conf.Serve.Thread)
|
|
|
+ wg := &sync.WaitGroup{}
|
|
|
+
|
|
|
+ //q := bson.M{"_id": mongodb.StringTOBsonId("60a2995b8a2adb30a57172ec")}
|
|
|
+ query := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.CombColl).Find(nil).Iter()
|
|
|
+ count := 0
|
|
|
+ for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
|
|
|
+ if count%2000 == 0 {
|
|
|
+ log.Info(fmt.Sprintf("current --- %d", count))
|
|
|
+ }
|
|
|
+
|
|
|
+ ch <- true
|
|
|
+ wg.Add(1)
|
|
|
+ go func(tmp map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-ch
|
|
|
+ wg.Done()
|
|
|
+ }()
|
|
|
+
|
|
|
+ if ids, ok := tmp["ids"].([]interface{}); ok {
|
|
|
+ //id := mongodb.BsonIdToSId(tmp["_id"])
|
|
|
+
|
|
|
+ for _, p := range ids {
|
|
|
+ p1 := p.(map[string]interface{})
|
|
|
+ info, _ := MgoPro.FindById(config.Conf.DB.MongoP.ProjectColl, util.ObjToString(p1["pid"]), nil)
|
|
|
+ if list, ok1 := (*info)["list"].([]interface{}); ok1 {
|
|
|
+ for _, l := range list {
|
|
|
+ l1 := l.(map[string]interface{})
|
|
|
+ m := make(map[string]interface{})
|
|
|
+ m["project_stage_code"] = tagFunc(l1)
|
|
|
+ m["proposed_id"] = mongodb.BsonIdToSId(tmp["_id"])
|
|
|
+ m["title"] = util.ObjToString(l1["title"])
|
|
|
+ if t := util.Int64All(l1["publishtime"]); t > 0 {
|
|
|
+ m["publishtime"] = util.FormatDateByInt64(&t, util.Date_Full_Layout)
|
|
|
+ }
|
|
|
+ m["infoid"] = util.ObjToString(l1["infoid"])
|
|
|
+ m["jybxhref"] = fmt.Sprintf(config.Conf.Serve.JyHref, util.CommonEncodeArticle("content", util.ObjToString(l1["infoid"])))
|
|
|
+ m["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ MgoPro.Save("projectset_comb_temp1", m)
|
|
|
+ //MysqlTool.Insert("dwd_f_nzj_follw_record", m)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //if buyer := util.ObjToString((*info)["buyer"]); buyer != "" {
|
|
|
+ // s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": buyer})
|
|
|
+ // if s <= 0 {
|
|
|
+ // saveEnt := make(map[string]interface{})
|
|
|
+ // saveEnt["proposed_id"] = id
|
|
|
+ // saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ // saveEnt["name"] = buyer
|
|
|
+ // if eid := redis.GetStr("ent_id", buyer); eid != "" {
|
|
|
+ // arr := strings.Split(eid, "_")
|
|
|
+ // saveEnt["name_id"] = arr[0]
|
|
|
+ // if len(arr) == 2 {
|
|
|
+ // saveEnt["area_code"] = arr[1]
|
|
|
+ // } else if len(arr) == 3 {
|
|
|
+ // saveEnt["city_code"] = arr[2]
|
|
|
+ // }
|
|
|
+ // info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
|
|
|
+ // if info != nil && len(*info) > 0 {
|
|
|
+ // saveEnt["address"] = (*info)[0]["address"]
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // saveEnt["identity_type"] = 2
|
|
|
+ // saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
|
|
|
+ // saveEntPool1 <- saveEnt
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ //if winner := util.ObjToString((*info)["buyer"]); winner != "" {
|
|
|
+ // for _, w := range strings.Split(winner, ",") {
|
|
|
+ // s := MysqlTool.Count("dwd_f_nzj_ent", bson.M{"proposed_id": id, "name": w})
|
|
|
+ // if s <= 0 {
|
|
|
+ // saveEnt := make(map[string]interface{})
|
|
|
+ // saveEnt["proposed_id"] = id
|
|
|
+ // saveEnt["createtime"] = time.Now().Format(util.Date_Full_Layout)
|
|
|
+ // saveEnt["name"] = w
|
|
|
+ // if eid := redis.GetStr("ent_id", w); eid != "" {
|
|
|
+ // arr := strings.Split(eid, "_")
|
|
|
+ // saveEnt["name_id"] = arr[0]
|
|
|
+ // if len(arr) == 2 {
|
|
|
+ // saveEnt["area_code"] = arr[1]
|
|
|
+ // } else if len(arr) == 3 {
|
|
|
+ // saveEnt["city_code"] = arr[2]
|
|
|
+ // }
|
|
|
+ // info := MysqlTool1.Find("dws_f_ent_baseinfo", bson.M{"name_id": arr[0]}, "address", "", -1, -1)
|
|
|
+ // if info != nil && len(*info) > 0 {
|
|
|
+ // saveEnt["address"] = (*info)[0]["address"]
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // saveEnt["identity_type"] = 3
|
|
|
+ // saveEnt["area_code"] = AreaCode[util.ObjToString(tmp["area"])]
|
|
|
+ // saveEntPool1 <- saveEnt
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ }
|
|
|
+
|
|
|
+ //size := MysqlTool.Count("dwd_f_nzj_follw_record", bson.M{"proposed_id": id})
|
|
|
+ //info := MysqlTool.FindOne("dwd_f_nzj_follw_record", bson.M{"proposed_id": id}, "project_stage_code", "publishtime desc")
|
|
|
+ //MysqlTool.Update("dwd_f_nzj_baseinfo", bson.M{"proposed_id": id}, bson.M{"follow_num": size, "project_stage_code": (*info)["project_stage_code"], "updatetime": time.Now().Format(util.Date_Full_Layout)})
|
|
|
+ }
|
|
|
+
|
|
|
+ }(tmp)
|
|
|
+ tmp = make(map[string]interface{})
|
|
|
+ }
|
|
|
+ wg.Wait()
|
|
|
+ log.Info(fmt.Sprintf("over --- %d", count))
|
|
|
+}
|
|
|
+
|
|
|
+var saveEntPool1 = make(chan map[string]interface{}, 5000)
|
|
|
+var saveEntSp1 = make(chan bool, 1)
|
|
|
+
|
|
|
+func SaveEntFunc1(table string, arr []string) {
|
|
|
+ arru := make([]map[string]interface{}, saveSize)
|
|
|
+ indexu := 0
|
|
|
+ for {
|
|
|
+ select {
|
|
|
+ case v := <-saveEntPool1:
|
|
|
+ arru[indexu] = v
|
|
|
+ indexu++
|
|
|
+ if indexu == saveSize {
|
|
|
+ saveEntSp1 <- true
|
|
|
+ go func(arru []map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-saveEntSp1
|
|
|
+ }()
|
|
|
+ MysqlTool.InsertBulk(table, arr, arru...)
|
|
|
+ }(arru)
|
|
|
+ arru = make([]map[string]interface{}, saveSize)
|
|
|
+ indexu = 0
|
|
|
+ }
|
|
|
+ case <-time.After(1000 * time.Millisecond):
|
|
|
+ if indexu > 0 {
|
|
|
+ saveEntSp1 <- true
|
|
|
+ go func(arru []map[string]interface{}) {
|
|
|
+ defer func() {
|
|
|
+ <-saveEntSp1
|
|
|
+ }()
|
|
|
+ MysqlTool.InsertBulk(table, arr, arru...)
|
|
|
}(arru[:indexu])
|
|
|
arru = make([]map[string]interface{}, saveSize)
|
|
|
indexu = 0
|
