dataflow/savebidding/keyword.go

package main

import (
	"encoding/json"
	u "jygit.jydev.jianyu360.cn/data_processing/common_utils"
	"jygit.jydev.jianyu360.cn/data_processing/common_utils/mfw"
	"log"
	"regexp"
	"strings"
	"time"
)

var (
	reg      = regexp.MustCompile("^[0-9a-zA-Z-.]+$")
	regNo    = regexp.MustCompile("^[0-9]*$")
	regSpace = regexp.MustCompile("(?ism)(<style.*?>.*?</style>)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|(　+)|(" + string(rune(160)) + "+)")
	regRow   = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+")
	regDh    = regexp.MustCompile("[,]+")
	regNewDb = regexp.MustCompile("([:,、：，。.;])[,]")
	DesLen   = 120
	keypool  = make(chan bool, 1)
	client   *mfw.Client
)

func InitKeywordClient() {
	cf := &mfw.ClientConfig{
		ClientName:      "剑鱼抽关键词",
		EventHandler:    func(p *mfw.Packet) {},
		MsgServerAddr:   conf.Config.KeywordAddr,
		CanHandleEvents: []int{},
		OnConnectSuccess: func() {
			log.Println("剑鱼关键词 client")
		},
		ReadBufferSize:  10,
		WriteBufferSize: 10,
	}
	client, _ = mfw.NewClient(cf)
}

func DealInfo(bid, data map[string]interface{}) {
	defer u.Catch()
	if bid["keywords"] != nil && bid["description"] != nil {
		return
	} else {
		data["keywords"] = ""
		data["description"] = ""
	}
	title := u.ObjToString(bid["title"])
	var m [][]string
	select {
	case <-func() <-chan bool {
		ch := make(chan bool, 1)
		go func(chan bool) {
			select {
			case keypool <- true:
				defer func() {
					<-keypool
				}()
				ret, _ := client.Call("", mfw.UUID(8), 4010, mfw.SENDTO_TYPE_RAND_RECIVER, title, 1)
				json.Unmarshal(ret, &m)
			case <-time.After(10 * time.Millisecond):
			}
			ch <- true
		}(ch)
		return ch
	}():
	case <-time.After(40 * time.Millisecond):
	}
	arr := []string{}
	keyword := []string{}
	keywordnew := []string{}
	for _, tmp := range m {
		if reg.MatchString(tmp[0]) {
			arr = append(arr, tmp[0])
		} else {
			if len(arr) > 0 {
				str := strings.Join(arr, "")
				keyword = append(keyword, str)
				arr = []string{}
			}
			if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) {
				keyword = append(keyword, tmp[0])
			}
		}
	}
	for _, v := range keyword {
		v = regNo.ReplaceAllString(v, "")
		if len(v) > 0 {
			keywordnew = append(keywordnew, v)
		}
	}
	keywords := strings.Join(keywordnew, ",")
	data["keywords"] = keywords
	content := ""
	if bid["detail_bak"] != nil {
		content = u.ObjToString(bid["detail_bak"])
	} else {
		content = u.ObjToString(bid["detail"])
	}
	//内容替换
	content = strings.Replace(content, " ", "", -1)
	content = regSpace.ReplaceAllString(content, "")
	content = regRow.ReplaceAllString(content, ",")
	content = regDh.ReplaceAllString(content, ",")
	content = regNewDb.ReplaceAllString(content, "$1")
	if strings.HasPrefix(content, ",") {
		content = content[1:]
	}
	tc := []rune(content)
	ltc := len(tc)
	description := content
	if ltc > DesLen {
		description = string(tc[:DesLen])
	}
	data["description"] = description
	//保存到数据库
	return
}