|
@@ -41,69 +41,54 @@ func (m *FMatchUser) LoadIncDatas() bool {
|
|
|
unix := time.Now().Unix()
|
|
|
query := fmt.Sprintf(projectQuery, TaskConfig.Pici, unix, projectPagingQuery)
|
|
|
logger.Info("开始加载项目。。。", query)
|
|
|
- count := int(elastic.Count(Es_Projectset, Es_Projectset, fmt.Sprintf(projectQuery, TaskConfig.Pici, unix, "")))
|
|
|
- logger.Info("需要加载项目", count, "条")
|
|
|
- if count > Config.MaxLoadProjectCount {
|
|
|
- count = Config.MaxLoadProjectCount
|
|
|
- logger.Info("需要加载的项目数超过限制,只加载", Config.MaxLoadProjectCount, "条")
|
|
|
- }
|
|
|
- if count == 0 {
|
|
|
- return false
|
|
|
- }
|
|
|
var length int64
|
|
|
- for t := 1; t <= 5; t++ {
|
|
|
- length = 0
|
|
|
- m.Projects = &sync.Map{}
|
|
|
- elastic.VarEs.(*elastic.EsV7).Scroll(Es_Projectset, "5m", query, func(fv map[string]interface{}) bool {
|
|
|
- if int(length) > count {
|
|
|
- return false
|
|
|
- }
|
|
|
- length++
|
|
|
- projectName := util.ObjToString(fv["projectname"])
|
|
|
- projectCode := util.ObjToString(fv["projectcode"])
|
|
|
- if projectName == "" && projectCode == "" {
|
|
|
- return true
|
|
|
- }
|
|
|
- follow := &Follow{
|
|
|
- ProjectName: projectName,
|
|
|
- ProjectCode: projectCode,
|
|
|
- }
|
|
|
- infos := CSortList{}
|
|
|
- list, _ := fv["list"].([]interface{})
|
|
|
- for _, pv := range list {
|
|
|
- pvMap, _ := pv.(map[string]interface{})
|
|
|
- s_subscopeclass := pvMap["s_subscopeclass"]
|
|
|
- if s_subscopeclass == nil {
|
|
|
- s_subscopeclass = fv["s_subscopeclass"]
|
|
|
- }
|
|
|
- infos = append(infos, map[string]interface{}{
|
|
|
- "_id": pvMap["infoid"],
|
|
|
- "area": pvMap["area"],
|
|
|
- "publishtime": pvMap["publishtime"],
|
|
|
- "s_subscopeclass": s_subscopeclass,
|
|
|
- "subtype": pvMap["subtype"],
|
|
|
- "title": pvMap["title"],
|
|
|
- "toptype": pvMap["toptype"],
|
|
|
- "projectname": pvMap["projectname"],
|
|
|
- "projectcode": pvMap["projectcode"],
|
|
|
- "href": pvMap["href"],
|
|
|
- "infoformat": pvMap["infoformat"],
|
|
|
- "budget": pvMap["budget"],
|
|
|
- "bidamount": pvMap["bidamount"],
|
|
|
- })
|
|
|
- }
|
|
|
- follow.Infos = infos
|
|
|
- for _, pv := range list {
|
|
|
- pvMap, _ := pv.(map[string]interface{})
|
|
|
- m.Projects.Store(util.ObjToString(pvMap["infoid"]), follow)
|
|
|
- }
|
|
|
+ m.Projects = &sync.Map{}
|
|
|
+ elastic.VarEs.(*elastic.EsV7).Scroll(Es_Projectset, "5m", query, func(fv map[string]interface{}) bool {
|
|
|
+ if int(length) > Config.MaxLoadProjectCount {
|
|
|
+ logger.Info("需要加载的项目数超过限制,只加载", Config.MaxLoadProjectCount, "条")
|
|
|
+ return false
|
|
|
+ }
|
|
|
+ length++
|
|
|
+ projectName := util.ObjToString(fv["projectname"])
|
|
|
+ projectCode := util.ObjToString(fv["projectcode"])
|
|
|
+ if projectName == "" && projectCode == "" {
|
|
|
return true
|
|
|
- })
|
|
|
- if int(length) >= count {
|
|
|
- break
|
|
|
}
|
|
|
- logger.Info("加载项目", "第", t, "次加载数据完成,数据总数", length, count, ",由于数据量不够,重新加载")
|
|
|
- }
|
|
|
+ follow := &Follow{
|
|
|
+ ProjectName: projectName,
|
|
|
+ ProjectCode: projectCode,
|
|
|
+ }
|
|
|
+ infos := CSortList{}
|
|
|
+ list, _ := fv["list"].([]interface{})
|
|
|
+ for _, pv := range list {
|
|
|
+ pvMap, _ := pv.(map[string]interface{})
|
|
|
+ s_subscopeclass := pvMap["s_subscopeclass"]
|
|
|
+ if s_subscopeclass == nil {
|
|
|
+ s_subscopeclass = fv["s_subscopeclass"]
|
|
|
+ }
|
|
|
+ infos = append(infos, map[string]interface{}{
|
|
|
+ "_id": pvMap["infoid"],
|
|
|
+ "area": pvMap["area"],
|
|
|
+ "publishtime": pvMap["publishtime"],
|
|
|
+ "s_subscopeclass": s_subscopeclass,
|
|
|
+ "subtype": pvMap["subtype"],
|
|
|
+ "title": pvMap["title"],
|
|
|
+ "toptype": pvMap["toptype"],
|
|
|
+ "projectname": pvMap["projectname"],
|
|
|
+ "projectcode": pvMap["projectcode"],
|
|
|
+ "href": pvMap["href"],
|
|
|
+ "infoformat": pvMap["infoformat"],
|
|
|
+ "budget": pvMap["budget"],
|
|
|
+ "bidamount": pvMap["bidamount"],
|
|
|
+ })
|
|
|
+ }
|
|
|
+ follow.Infos = infos
|
|
|
+ for _, pv := range list {
|
|
|
+ pvMap, _ := pv.(map[string]interface{})
|
|
|
+ m.Projects.Store(util.ObjToString(pvMap["infoid"]), follow)
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
logger.Info("加载项目结束。。。", length)
|
|
|
if length == 0 {
|
|
|
logger.Info("没有新的项目数据")
|