package main import ( "encoding/json" "field_sync_ai/config" "log" "regexp" "strings" "time" util "jygit.jydev.jianyu360.cn/data_processing/common_utils" "jygit.jydev.jianyu360.cn/data_processing/common_utils/mfw" ) var ( client *mfw.Client reg = regexp.MustCompile("^[0-9a-zA-Z-.]+$") regNo = regexp.MustCompile("^[0-9]*$") regSpace = regexp.MustCompile("(?ism)(.*?)|([.#]?\\w{1,20}\\{.*?\\})|(<.*?>)|(\\\\t)+|\\t|( +)|( +)|(" + string(rune(160)) + "+)") regRow = regexp.MustCompile("(?i)<(tr|div|p)[^>]*?>|(\\n)+") regDh = regexp.MustCompile("[,]+") regNewDb = regexp.MustCompile("([:,、:,。.;])[,]") DesLen = 120 ) func inits() { cf := &mfw.ClientConfig{ ClientName: "剑鱼抽关键词", EventHandler: func(p *mfw.Packet) {}, MsgServerAddr: config.Conf.Serve.MsgAddr, CanHandleEvents: []int{}, OnConnectSuccess: func() { log.Println("剑鱼关键词 client") }, ReadBufferSize: 10, WriteBufferSize: 10, } client, _ = mfw.NewClient(cf) } var keypool = make(chan bool, 1) func DealInfo(obj, update *map[string]interface{}) { defer util.Catch() if (*obj)["keywords"] != nil && (*obj)["description"] != nil { return } else { (*update)["keywords"] = "" (*update)["description"] = "" } title := util.ObjToString((*obj)["title"]) var m [][]string select { case <-func() <-chan bool { ch := make(chan bool, 1) go func(chan bool) { select { case keypool <- true: defer func() { <-keypool }() ret, _ := client.Call("", mfw.UUID(8), 4010, mfw.SENDTO_TYPE_RAND_RECIVER, title, 1) json.Unmarshal(ret, &m) case <-time.After(10 * time.Millisecond): } ch <- true }(ch) return ch }(): case <-time.After(40 * time.Millisecond): } arr := []string{} keyword := []string{} keywordnew := []string{} for _, tmp := range m { if reg.MatchString(tmp[0]) { arr = append(arr, tmp[0]) } else { if len(arr) > 0 { str := strings.Join(arr, "") keyword = append(keyword, str) arr = []string{} } if len(tmp[0]) > 3 && (strings.HasPrefix(tmp[1], "n") || tmp[1] == "v" || tmp[1] == "vn" || strings.HasPrefix(tmp[1], "g")) { keyword = append(keyword, tmp[0]) } } } for _, v := range keyword { v = regNo.ReplaceAllString(v, "") if len(v) > 0 { keywordnew = append(keywordnew, v) } } keywords := strings.Join(keywordnew, ",") (*update)["keywords"] = keywords content := "" if (*obj)["detail_bak"] != nil { content = util.ObjToString((*obj)["detail_bak"]) } else { content = util.ObjToString((*obj)["detail"]) } //内容替换 content = strings.Replace(content, " ", "", -1) content = regSpace.ReplaceAllString(content, "") content = regRow.ReplaceAllString(content, ",") content = regDh.ReplaceAllString(content, ",") content = regNewDb.ReplaceAllString(content, "$1") if strings.HasPrefix(content, ",") { content = content[1:] } tc := []rune(content) ltc := len(tc) description := content if ltc > DesLen { description = string(tc[:DesLen]) } (*update)["description"] = description //保存到数据库 return }