dataIdentify/main_test.go

package main

import (
	"app.yhyue.com/moapp/jybase/encrypt"
	. "app.yhyue.com/moapp/jybase/mongodb"
	. "dataIdentify/db"
	. "dataIdentify/service"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/os/gctx"
	"log"
	"net/rpc"
	"strings"
	"sync"
	"testing"
)

const (
	maxSize = 100
)

//	{
//		$unset: {
//		"bid_commonwealth" : "",
//		"quote_mode" : "",
//		"model_dataidentify" : ""
//		}
//	}
func TestAddField(t *testing.T) {
	pool := make(chan bool, 5)
	wait := &sync.WaitGroup{}
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	it := sess.DB("qfw_data").C("wcj_bidding").Find(map[string]interface{}{}).Select(nil).Sort("-_id").Iter()
	index := 0
	for tm := make(map[string]interface{}); it.Next(tm); {
		index++
		if index%100 == 0 {
			log.Println("index", index)
		}
		pool <- true
		wait.Add(1)
		go func(m map[string]interface{}) {
			defer func() {
				<-pool
				wait.Done()
			}()
			href := "https://www.jianyu360.com/nologin/content/" + encrypt.CommonEncodeArticle("content", BsonIdToSId(m["_id"])) + ".html"
			Mgo_Main.Update("wcj_bidding", map[string]interface{}{
				"_id": m["_id"],
			}, map[string]interface{}{"$set": map[string]interface{}{
				"href": href,
			}}, false, false)
		}(tm)
		tm = make(map[string]interface{})
	}
	wait.Wait()
	log.Println("over...", index)
}

// 示例测试
func TestRule(t *testing.T) {
	Start("5b0e1b8da5cb26b9b7b8c9ac")
}
func TestAll(t *testing.T) {
	pool := make(chan bool, g.Config().MustGet(gctx.New(), "poolSize").Int())
	wait := &sync.WaitGroup{}
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	it := sess.DB("qfw_data").C("wcj_bidding").Find(map[string]interface{}{
		"_id": StringTOBsonId("67c123333309c0998b619793"),
	}).Select(SelectField).Sort("-_id").Iter()
	index := 0
	for tm := make(map[string]interface{}); it.Next(tm); {
		index++
		if index%100 == 0 {
			log.Println("index", index)
		}
		pool <- true
		wait.Add(1)
		go func(m map[string]interface{}) {
			defer func() {
				<-pool
				wait.Done()
			}()
			_id := BsonIdToSId(m["_id"])
			flag, quoteMode, bidCommonwealth, other := Pretreatment(_id, m)
			if !flag {
				return
			}
			set := map[string]interface{}{}
			if quoteMode != "" {
				set["quote_mode"] = quoteMode
			}
			if bidCommonwealth != -1 {
				set["bid_commonwealth"] = bidCommonwealth
			}
			if set != nil {
				set["model_dataidentify"] = other
			}
			if len(set) > 0 {
				Mgo_Main.UpdateById("wcj_bidding", m["_id"], map[string]interface{}{"$set": set})
			}
		}(tm)
		tm = make(map[string]interface{})
	}
	wait.Wait()
	log.Println("over...", index)
}
func TestExtract(t *testing.T) {
	log.Println("start...")
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	it := sess.DB("qfw_data").C("bidding_hasdetail").Find(map[string]interface{}{
		"_id": map[string]interface{}{
			"$lt": StringTOBsonId("68639706408dd61d6e5d05cd"),
		},
	}).Select(map[string]interface{}{
		"detail":    1,
		"_id":       1,
		"subtype":   1,
		"toptype":   1,
		"s_winner":  1,
		"bidamount": 1,
	}).Sort("-_id").Iter()
	types := "邀标、竞谈、单一、招标、中标、成交、合同"
	all := map[string]int{}
	for _, v := range strings.Split(types, "、") {
		all[v] = 0
	}
	var isOver = func() bool {
		for _, v := range all {
			if v < maxSize {
				return false
			}
		}
		return true
	}
	index := 0
	for m := make(map[string]interface{}); it.Next(m); {
		index++
		if index%500 == 0 {
			log.Println("index", index)
		}
		subtype, _ := m["subtype"].(string)
		if v, ok := all[subtype]; !ok || v >= maxSize {
			continue
		}
		_id := Mgo_Main.SaveByOriID("wcj_bidding", m)
		log.Println("save", _id)
		all[subtype]++
		if isOver() {
			break
		}
	}
	log.Println("over...", index)
}
func TestRpc(t *testing.T) {
	conn, err := rpc.DialHTTP("tcp", "127.0.0.1:8811")
	if err != nil {
		log.Println(err)
		return
	}
	defer conn.Close()
	var reply map[string]string
	err = conn.Call("DataIdentify.Execute", "5b0dfeb4a5cb26b9b79c1330", &reply)
	if err != nil {
		log.Println(err)
	}
	log.Println(reply)
}