package main import ( "app.yhyue.com/moapp/jybase/encrypt" . "app.yhyue.com/moapp/jybase/mongodb" . "dataIdentify/db" . "dataIdentify/service" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" "log" "net/rpc" "strings" "sync" "testing" ) const ( maxSize = 100 ) // { // $unset: { // "bid_commonwealth" : "", // "quote_mode" : "", // "model_dataidentify" : "" // } // } func TestAddField(t *testing.T) { pool := make(chan bool, 5) wait := &sync.WaitGroup{} sess := Mgo_Main.GetMgoConn() defer Mgo_Main.DestoryMongoConn(sess) it := sess.DB("qfw_data").C("wcj_bidding").Find(map[string]interface{}{}).Select(nil).Sort("-_id").Iter() index := 0 for tm := make(map[string]interface{}); it.Next(tm); { index++ if index%100 == 0 { log.Println("index", index) } pool <- true wait.Add(1) go func(m map[string]interface{}) { defer func() { <-pool wait.Done() }() href := "https://www.jianyu360.com/nologin/content/" + encrypt.CommonEncodeArticle("content", BsonIdToSId(m["_id"])) + ".html" Mgo_Main.Update("wcj_bidding", map[string]interface{}{ "_id": m["_id"], }, map[string]interface{}{"$set": map[string]interface{}{ "href": href, }}, false, false) }(tm) tm = make(map[string]interface{}) } wait.Wait() log.Println("over...", index) } // 示例测试 func TestRule(t *testing.T) { Start("5b0e1b8da5cb26b9b7b8c9ac") } func TestAll(t *testing.T) { pool := make(chan bool, g.Config().MustGet(gctx.New(), "poolSize").Int()) wait := &sync.WaitGroup{} sess := Mgo_Main.GetMgoConn() defer Mgo_Main.DestoryMongoConn(sess) it := sess.DB("qfw_data").C("wcj_bidding").Find(map[string]interface{}{ "_id": StringTOBsonId("67c123333309c0998b619793"), }).Select(SelectField).Sort("-_id").Iter() index := 0 for tm := make(map[string]interface{}); it.Next(tm); { index++ if index%100 == 0 { log.Println("index", index) } pool <- true wait.Add(1) go func(m map[string]interface{}) { defer func() { <-pool wait.Done() }() _id := BsonIdToSId(m["_id"]) flag, quoteMode, bidCommonwealth, other := Pretreatment(_id, m) if !flag { return } set := map[string]interface{}{} if quoteMode != "" { set["quote_mode"] = quoteMode } if bidCommonwealth != -1 { set["bid_commonwealth"] = bidCommonwealth } if set != nil { set["model_dataidentify"] = other } if len(set) > 0 { Mgo_Main.UpdateById("wcj_bidding", m["_id"], map[string]interface{}{"$set": set}) } }(tm) tm = make(map[string]interface{}) } wait.Wait() log.Println("over...", index) } func TestExtract(t *testing.T) { log.Println("start...") sess := Mgo_Main.GetMgoConn() defer Mgo_Main.DestoryMongoConn(sess) it := sess.DB("qfw_data").C("bidding_hasdetail").Find(map[string]interface{}{ "_id": map[string]interface{}{ "$lt": StringTOBsonId("68639706408dd61d6e5d05cd"), }, }).Select(map[string]interface{}{ "detail": 1, "_id": 1, "subtype": 1, "toptype": 1, "s_winner": 1, "bidamount": 1, }).Sort("-_id").Iter() types := "邀标、竞谈、单一、招标、中标、成交、合同" all := map[string]int{} for _, v := range strings.Split(types, "、") { all[v] = 0 } var isOver = func() bool { for _, v := range all { if v < maxSize { return false } } return true } index := 0 for m := make(map[string]interface{}); it.Next(m); { index++ if index%500 == 0 { log.Println("index", index) } subtype, _ := m["subtype"].(string) if v, ok := all[subtype]; !ok || v >= maxSize { continue } _id := Mgo_Main.SaveByOriID("wcj_bidding", m) log.Println("save", _id) all[subtype]++ if isOver() { break } } log.Println("over...", index) } func TestRpc(t *testing.T) { conn, err := rpc.DialHTTP("tcp", "127.0.0.1:8811") if err != nil { log.Println(err) return } defer conn.Close() var reply map[string]string err = conn.Call("DataIdentify.Execute", "5b0dfeb4a5cb26b9b79c1330", &reply) if err != nil { log.Println(err) } log.Println(reply) }