dataIdentify/extract.go

package main

import (
	"app.yhyue.com/moapp/jybase/encrypt"
	. "app.yhyue.com/moapp/jybase/mongodb"
	. "dataIdentify/db"
	. "dataIdentify/service"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/os/gctx"
	"github.com/gogf/gf/v2/util/gconv"
	"github.com/gogf/gf/v2/util/grand"
	"log"
	"strings"
	"time"
)

func main() {
	var maxSize = 200
	zblhtSize := 0
	log.Println("start...")
	sess := Mgo_Main.GetMgoConn()
	defer Mgo_Main.DestoryMongoConn(sess)
	SelectField["publishtime"] = 1
	SelectField["href"] = 1
	SelectField["s_winner"] = 1
	it := sess.DB(Mgo_Main.DbName).C(g.Config().MustGet(gctx.New(), "mongodb.main.collection").String()).Find(map[string]interface{}{
		"_id": map[string]interface{}{
			"$lt": StringTOBsonId("686ce21dd5d8e4081f8f2d98"),
		},
		//"_id":         StringTOBsonId("6763aa5555a3d7e571cda133"),
		"extracttype": 1,
	}).Select(SelectField).Sort("-_id").Iter()
	types := "中标、成交、合同"
	all := map[string]int{}
	for _, v := range strings.Split(types, "、") {
		all[v] = 0
	}
	var isOver = func() bool {
		for _, v := range all {
			if v < maxSize {
				return false
			}
		}
		return true
	}
	index := 0
	for m := make(map[string]interface{}); it.Next(m); {
		index++
		if index%500 == 0 {
			log.Println("index", index)
		}
		subtype, _ := m["subtype"].(string)
		if _, ok := all[subtype]; !ok {
			continue
		}
		if !isOver() {
			publishtime := gconv.Int(m["publishtime"])
			if publishtime%grand.N(1, 1000) != 0 {
				continue
			}
			if all[subtype] >= maxSize {
				continue
			}
		}
		_id := BsonIdToSId(m["_id"])
		href := "https://www.jianyu360.com/nologin/content/" + encrypt.CommonEncodeArticle("content", _id) + ".html"
		m["jybxhref"] = href
		var flag bool
		var quoteMode string
		var bidCommonwealth int
		if isOver() {
			flag, _, _, bidCommonwealth, _ = Pretreatment(_id, m, 2)
			if bidCommonwealth == 1 {
				flag, quoteMode, _, _, _ = Pretreatment(_id, m, 1)
				if quoteMode == "" || quoteMode == QuoteMode_Other {
					continue
				}
			}
		} else {
			flag, quoteMode, _, bidCommonwealth, _ = Pretreatment(_id, m, 0)
			if quoteMode == "" || quoteMode == QuoteMode_Other {
				continue
			}
		}
		if !flag {
			return
		}
		m["quote_mode"] = quoteMode
		if bidCommonwealth == 1 {
			zblhtSize++
		}
		m["bid_commonwealth"] = bidCommonwealth
		delete(m, "detail")
		if !isOver() || bidCommonwealth == 1 {
			newId := Mgo_Main.SaveByOriID("wcj_bidding_"+time.Now().Format("20060102"), m)
			log.Println("save", newId)
		}
		all[subtype]++
		if isOver() && zblhtSize == maxSize {
			break
		}
	}
	log.Println("over...", index)
}