package main import ( "encoding/json" "log" "medical_project/config" "time" ) func loadData(starttime int64) { log.Println("load project start..", starttime) findLock.Lock() defer findLock.Unlock() AllIdsMapLock.Lock() defer AllIdsMapLock.Unlock() sess := MongoPro.GetMgoConn() defer MongoPro.DestoryMongoConn(sess) loadOver := make(chan bool) q := map[string]interface{}{ "lasttime": map[string]interface{}{"$gte": starttime}, } //field := map[string]interface{}{"list": 0} it := sess.DB(config.Conf.DB.MongoP.Dbname).C(config.Conf.DB.MongoP.Coll).Find(&q).Select(nil).Iter() n := 0 pool := make(chan *Project, 1000) go func() { for { select { case tmp := <-pool: n++ if n%10000 == 0 { log.Println("current", n, "\n", tmp.Id, len(mapPn), len(mapPc), len(mapPb)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer) } if tmp != nil { id := tmp.Id.Hex() for _, v := range append([]string{tmp.ProjectName}, tmp.MPN...) { if v != "" { //v = pcReplace.ReplaceAllString(v, "") if v != "" { k := mapPn[v] if k == nil { k = &Key{Arr: []string{id}} mapPn[v] = k } else { k.Arr = append(k.Arr, id) } } } } for _, v := range append([]string{tmp.ProjectCode}, tmp.MPC...) { if v != "" { //v = pcReplace.ReplaceAllString(v, "") if v != "" { k := mapPc[v] if k == nil { k = &Key{Arr: []string{id}} mapPc[v] = k } else { k.Arr = append(k.Arr, id) } } } } if tmp.Buyer != "" && len([]rune(tmp.Buyer)) > 2 { k := mapPb[tmp.Buyer] if k == nil { k = &Key{Arr: []string{id}} mapPb[tmp.Buyer] = k } else { k.Arr = append(k.Arr, id) } } AllIdsMap[id] = &ID{Id: id, P: tmp} } case <-loadOver: return } } }() for { result := make(map[string]interface{}) if it.Next(&result) { go func(res map[string]interface{}) { bys, _ := json.Marshal(result) var tmp *Project _ = json.Unmarshal(bys, &tmp) saveFiled(result, tmp) pool <- tmp }(result) } else { break } } time.Sleep(2 * time.Second) loadOver <- true log.Println("load project over..", n) } func saveFiled(res map[string]interface{}, tmp *Project) { tmpMap := make(map[string]InfoField) infoMap := res["infofield"].(map[string]interface{}) for _, v := range infoMap { var field InfoField b, _ := json.Marshal(v) _ = json.Unmarshal(b, &field) tmpMap[tmp.Id.Hex()] = field } tmp.InfoFiled = tmpMap }