jy-data-extract/fullproject/src_back/load_data.go

package main

import (
	//"encoding/json"
	"log"
)

//初始加载数据，默认加载最近6个月的数据

func (p *ProjectTask) loadData(starttime int64) {
	log.Println("load start..", starttime)
	p.findLock.Lock()
	defer p.findLock.Unlock()
	p.AllIdsMapLock.Lock()
	defer p.AllIdsMapLock.Unlock()
	sess := MongoTool.GetMgoConn()
	defer MongoTool.DestoryMongoConn(sess)
	q := map[string]interface{}{
		"lasttime": map[string]interface{}{"$gt": starttime},
	}
	it := sess.DB(MongoTool.DbName).C(p.coll).Find(&q).Iter()
	tmp := &ProjectInfo{}
	n := 0
	for it.Next(tmp) {
		n++
		if n%1000 == 0 {
			log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb), tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
		}
		for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) {
			if v != "" {
				v = pcReplace.ReplaceAllString(v, "")
				if v != "" {
					k := p.mapPn[v]
					if k == nil {
						k = &Key{Arr: []string{tmp.Id.Hex()}}
						p.mapPn[v] = k
					} else {
						k.Arr = append(k.Arr, tmp.Id.Hex())
					}
				}
			}
		}
		for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) {
			if v != "" {
				v = pcReplace.ReplaceAllString(v, "")
				if v != "" {
					k := p.mapPc[v]
					if k == nil {
						k = &Key{Arr: []string{tmp.Id.Hex()}}
						p.mapPc[v] = k
					} else {
						k.Arr = append(k.Arr, tmp.Id.Hex())
					}
				}
			}
		}
		if tmp.Buyer != "" && len([]rune(tmp.Buyer)) > 2 {
			k := p.mapPb[tmp.Buyer]
			if k == nil {
				k = &Key{Arr: []string{tmp.Id.Hex()}}
				p.mapPb[tmp.Buyer] = k
			} else {
				k.Arr = append(k.Arr, tmp.Id.Hex())
			}
		}
		p.AllIdsMap[tmp.Id.Hex()] = &ID{Id: tmp.Id.Hex(), P: tmp}
		tmp = &ProjectInfo{}
	}

	log.Println("load over..", n)
}