修改

fullproject/src_v1/load_data.go

@@ -30,7 +30,7 @@ func (p *ProjectTask) loadData(starttime int64) {
 			case tmp := <-pool:
 				n++
 				if n%10000 == 0 {
-					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
+					log.Println("current", n, "\n", tmp.Id, len(p.mapPn), len(p.mapPc), len(p.mapPb), len(p.mapHref)) //, tmp.ProjectName, tmp.MPN, tmp.ProjectCode, tmp.MPC, tmp.Buyer)
 				}
 				if tmp != nil {
 					id := tmp.Id.Hex()
@@ -152,7 +152,7 @@ func saveFiled(p *ProjectTask, res map[string]interface{}, tmp *ProjectInfo) {
 
 	tmpMap := make(map[string]InfoField)
 	infoMap := res["infofield"].(map[string]interface{})
-	for _, v := range infoMap{
+	for _, v := range infoMap {
 		var field InfoField
 		b, _ := json.Marshal(v)
 		_ = json.Unmarshal(b, &field)

udpfilterdup/src/datamap.go

@@ -71,7 +71,11 @@ func NewDatamap(days int, lastid string) *datamap {
 	//初始化加载数据
 	sess := mgo.GetMgoConn()
 	defer mgo.DestoryMongoConn(sess)
-	it := sess.DB(mgo.DbName).C(extract).Find(mongodb.ObjToMQ(`{"_id":{"$lte":"`+lastid+`"}}`, true)).Sort("-_id").Iter()
+	query := map[string]interface{}{"_id": map[string]interface{}{
+		"$lte": StringTOBsonId(lastid),
+	}}
+	log.Println("query", query)
+	it := sess.DB(mgo.DbName).C(extract).Find(query).Sort("-_id").Iter()
 	now1 := int64(0)
 	n, continuSum := 0, 0
 	for tmp := make(map[string]interface{}); it.Next(&tmp); n++ {

udpfilterdup/src/mgo.go

@@ -144,7 +144,7 @@ func (m *MongodbSim) InitPool() {
 	opts := options.Client()
 	opts.SetConnectTimeout(3 * time.Second)
 	opts.ApplyURI("mongodb://" + m.MongodbAddr)
-	opts.SetMaxPoolSize(uint16(m.Size))
+	opts.SetMaxPoolSize(uint64(m.Size))
 	m.pool = make(chan bool, m.Size)
 	opts.SetMaxConnIdleTime(2 * time.Hour)
 	m.Ctx, _ = context.WithTimeout(context.Background(), 99999*time.Hour)