jy-data-extract/fullproject/src_dev2/load_data.go

package main

import (
	"time"
	//"encoding/json"
	"log"
)

//初始加载数据，默认加载最近6个月的数据

func loadData(projectColl string, month int, bCacheRedis bool) {
	sess := MongoTool.GetMgoConn()
	defer MongoTool.DestoryMongoConn(sess)

	q := map[string]interface{}{}
	it := sess.DB(MongoTool.DbName).C(projectColl).Find(&q).Iter()
	AllIdsMapLock.Lock()
	tmp := &ProjectInfo{}
	n := 0
	for it.Next(tmp) {
		n++
		if n%1000 == 0 {
			log.Println("current", n, "\n", tmp.Id, tmp)
			time.Sleep(2 * time.Second)
		}
		for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
			if v != "" {
				k := mapPn[v]
				if k == nil {
					k = &Key{Arr: []string{}}
					mapPn[v] = k
				}
				k.Arr = append(k.Arr, tmp.Id.Hex())
			}
		}
		for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
			if v != "" {
				k := mapPc[v]
				if k == nil {
					k = &Key{Arr: []string{}}
					mapPc[v] = k
				}
				k.Arr = append(k.Arr, tmp.Id.Hex())
			}
		}
		if tmp.Buyer != "" {
			k := mapPb[tmp.Buyer]
			if k == nil {
				k = &Key{Arr: []string{}}
				mapPb[tmp.Buyer] = k
			}
			k.Arr = append(k.Arr, tmp.Id.Hex())
		}

		AllIdsMap2[tmp.Id.Hex()] = &ID{Id: tmp.Id.Hex(), lastTime: tmp.LastTime}

		if bCacheRedis {
			//存入redis

		}

	}
	AllIdsMapLock.Unlock()
	log.Println("load over")
}