fieldproject/proposed_project/load_data.go

package main

import (
	"app.yhyue.com/data_processing/common_utils/log"
	"encoding/json"
	"fmt"
	"go.uber.org/zap"
	"proposed_project/config"
	"sync"
	"time"
)

func loadProject() {
	sess := MgoPro.GetMgoConn()
	defer MgoPro.DestoryMongoConn(sess)
	f := map[string]interface{}{"list": 0}
	it := sess.DB(MgoPro.DbName).C(config.Conf.DB.MongoP.ProposedColl).Find(nil).Select(f).Iter()
	count := 0

	ch := make(chan bool, 3)
	wg := &sync.WaitGroup{}
	for result := make(map[string]interface{}); it.Next(result); count++ {
		if count%40000 == 0 {
			log.Info(fmt.Sprintf("iter --- %d", count))
		}
		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()
			bys, _ := json.Marshal(tmp)
			var p *Project
			_ = json.Unmarshal(bys, &p)
			AllPidMapLock.Lock()
			AllPidMap[p.Id] = &ID{P: p}
			AllPidMapLock.Unlock()
			AllCodeMapLock.Lock()
			AllCodeMap[p.ApproveCode] = p.Id
			AllCodeMapLock.Unlock()

			AllPnMapLock.Lock()
			res := AllPnMap[p.Id]
			if res != nil {
				res.Id[p.Id] = p.ProjectName
			} else {
				res = &Pname{
					Id: map[string]string{p.Id: p.ProjectName},
				}
			}
			AllPnMapLock.Unlock()
		}(result)
		result = make(map[string]interface{})
	}
	wg.Wait()
	time.Sleep(2 * time.Second)
	log.Info("loadProject", zap.Int("count", count), zap.Int("pcode", len(AllCodeMap)))
}