jyps/cpData/main.go

package main

import (
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mongodb"
	"log"
	"sync"
)

func main() {
	scp()
}

func scp() {
	Mgo := &mongodb.MongodbSim{
		MongodbAddr: "192.168.3.206:27002",
		DbName:      "qfw_data",
		Size:        10,
		UserName:    "root",
		Password:    "root",
	}
	Mgo.InitPool()

	sess := Mgo.GetMgoConn()
	defer Mgo.DestoryMongoConn(sess)

	//start, _ := time.Parse("2006-01-02 15:04:05", "2023-09-06 21:00:00")
	//end, _ := time.Parse("2006-01-02 15:04:05", "2023-09-07 09:00:00")
	//////
	//q := map[string]interface{}{
	//	"comeintime": map[string]interface{}{
	//		"$gte": start.Unix(),
	//		"$lte": end.Unix(),
	//	},
	//	//"modifyinfo.toptype": map[string]interface{}{
	//	//	"$exists": 1,
	//	//},
	//	"toptype": "招标",
	//	//"subtype": "其它",
	//
	//}

	query := sess.DB("qfw_data").C("projectset_proposed").Find(nil).Select(map[string]interface{}{"nav_column": 0}).Iter()
	//query := sess.DB("wcc").C("wcc_huahui_data_result_0829_200").Find(q).Select(nil).Iter()
	//c, _ := sess.DB("qfw").C("bidding").Find(q).Select(nil).Count()
	//fmt.Println("ccccc=>", c)
	//query := sess.DB("mixdata").C("bidding_wcc_nijian").Find(nil).Iter()
	count := 0
	////
	ch := make(chan bool, 10)
	wg := &sync.WaitGroup{}

	for tmp := make(map[string]interface{}); query.Next(tmp); count++ {
		if count%1000 == 0 {
			log.Println("current --- ", count)
		}

		ch <- true
		wg.Add(1)
		go func(tmp map[string]interface{}) {
			defer func() {
				<-ch
				wg.Done()
			}()

			Mgo.SaveByOriID("wcc_projectset_proposed", tmp)

		}(tmp)
		tmp = make(map[string]interface{})

	}

	wg.Wait()
	log.Println("dealTmp over ", count)

}